* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
* *
* Implementation of character decoding as described in ISO 10303-21 table 2 and *
* table 4 *
* *
********************************************************************************/
#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING
#include "character_decoder.h"
#include "exception.h"
#include "file_reader.h"
#include "logger.h"
#include <codecvt>
#include <iomanip>
#include <sstream>
#include <string>
#include <algorithm>
#include <boost/locale.hpp>
#define FIRST_SOLIDUS (1 << 1)
#define PAGE (1 << 2)
#define ALPHABET (1 << 3)
#define SECOND_SOLIDUS (1 << 4)
#define ALPHABET_DEFINITION (1 << 5)
#define APOSTROPHE (1 << 6)
#define ARBITRARY (1 << 7)
#define EXTENDED2 (1 << 8)
#define EXTENDED4 (1 << 9)
#define HEX(N) (1 << (9 + (N)))
#define THIRD_SOLIDUS (1 << 18)
#define ENDEXTENDED_X (1 << 19)
#define ENDEXTENDED_0 (1 << 20)
#define IGNORED_DIRECTIVE (1 << 22)
#define ENCOUNTERED_HEX (1 << 23)
#define EXPECTS_ALPHABET(S) ((S) & FIRST_SOLIDUS)
#define EXPECTS_PAGE(S) ((S) & FIRST_SOLIDUS)
#define EXPECTS_ARBITRARY(S) ((S) & FIRST_SOLIDUS)
#define EXPECTS_N_OR_F(S) ((S) & FIRST_SOLIDUS && !((S) & ARBITRARY))
#define EXPECTS_ARBITRARY2(S) ((S) & ARBITRARY && !((S) & SECOND_SOLIDUS))
#define EXPECTS_ALPHABET_DEFINITION(S) ((S) & FIRST_SOLIDUS && (S) & ALPHABET)
#define EXPECTS_SOLIDUS(S) ((S) & ALPHABET_DEFINITION || (S) & PAGE || (S) & ARBITRARY || (S) & EXTENDED2 || (S) & EXTENDED4 || (S) & ENDEXTENDED_0 || (S) & IGNORED_DIRECTIVE || ((S) & EXTENDED4 && (S) & HEX(8)) || ((S) & EXTENDED2 && (S) & HEX(4)))
#define EXPECTS_CHARACTER(S) ((S) & PAGE && (S) & SECOND_SOLIDUS)
#define EXPECTS_HEX(S) ((S) & HEX(1) || (S) & HEX(3) || (S) & HEX(5) || (S) & HEX(6) || (S) & HEX(7) || ((S) & ARBITRARY && (S) & SECOND_SOLIDUS) || ((S) & EXTENDED2 && (S) & HEX(2)) || ((S) & EXTENDED4 && (S) & HEX(4)))
#define EXPECTS_ENDEXTENDED_X(S) ((S) & THIRD_SOLIDUS)
#define EXPECTS_ENDEXTENDED_0(S) ((S) & ENDEXTENDED_X)
#define IS_VALID_ALPHABET_DEFINITION(C) ((C) >= 0x41 && (C) <= 0x49)
#define IS_LOWERCASE_HEX(C) (((C) >= 0x61 && (C) <= 0x66))
#define IS_HEXADECIMAL(C) (((C) >= 0x30 && (C) <= 0x39) || ((C) >= 0x41 && (C) <= 0x46) || IS_LOWERCASE_HEX(C))
#define HEX_TO_INT(C) (((C) >= 0x30 && (C) <= 0x39) ? (C)-0x30 : ((C) >= 0x41 && (C) <= 0x46) ? ((C)-0x41 + 10) \
: ((C)-0x61 + 10))
#define CLEAR_HEX(C) ((C) &= ~(HEX(1) | HEX(2) | HEX(3) | HEX(4) | HEX(5) | HEX(6) | HEX(7) | HEX(8)))
using namespace ifcopenshell;
namespace SWAR {
constexpr uint32_t ONES32 = 0x01010101u;
constexpr uint32_t HIGHS32 = 0x80808080u;
constexpr uint64_t ONES64 = 0x0101010101010101ull;
constexpr uint64_t HIGHS64 = 0x8080808080808080ull;
constexpr uint32_t splat32(unsigned char c) {
return ONES32 * c;
}
constexpr uint64_t splat64(unsigned char c) {
return ONES64 * c;
}
inline uint32_t has_zero_byte(uint32_t x) {
return (x - ONES32) & ~x & HIGHS32;
}
inline uint64_t has_zero_byte(uint64_t x) {
return (x - ONES64) & ~x & HIGHS64;
}
inline uint32_t eq_mask(uint32_t x, uint32_t c) {
return has_zero_byte(x ^ c);
}
inline uint64_t eq_mask(uint64_t x, uint64_t c) {
return has_zero_byte(x ^ c);
}
namespace chars {
constexpr uint32_t apostrophe32 = splat32('\'');
constexpr uint32_t solidus32 = splat32('\\');
constexpr uint64_t apostrophe64 = splat64('\'');
constexpr uint64_t solidus64 = splat64('\\');
}
inline uint32_t has_special_char(uint32_t x) {
return has_zero_byte(x) |
eq_mask(x, chars::apostrophe32) |
eq_mask(x, chars::solidus32) |
(x & HIGHS32);
}
inline uint64_t has_special_char(uint64_t x) {
return has_zero_byte(x) |
eq_mask(x, chars::apostrophe64) |
eq_mask(x, chars::solidus64) |
(x & HIGHS64);
}
inline void append_ascii(std::u32string& builder, const char* bytes, size_t count) {
builder.reserve(builder.size() + count);
for (size_t i = 0; i < count; ++i) {
builder.push_back(static_cast<unsigned char>(bytes[i]));
}
}
}
template <typename Reader>
character_decoder<Reader>::character_decoder(Reader* stream, logger& logger)
: logger_(logger) {
stream_ = stream;
codepage_ = 0;
builder_.reserve(1024);
}
template <typename Reader>
character_decoder<Reader>::~character_decoder() {
}
namespace {
template <typename Reader, bool Decode = true>
std::string read_string(std::u32string& builder_, Reader& stream_, logger& logger_, typename ifcopenshell::character_decoder<Reader>::ConversionMode mode, char substitution_character) {
unsigned int parse_state = 0;
builder_.clear();
char current_char;
int codepage = 1;
unsigned int hex = 0;
unsigned int hex_count = 0;
while (!stream_.eof()) {
if (parse_state == 0U) {
if (stream_.remaining() >= 8) {
uint64_t x = stream_.peek_u64();
if (SWAR::has_special_char(x) == 0) {
if constexpr (Decode) {
SWAR::append_ascii(builder_, reinterpret_cast<const char*>(&x), 8);
}
stream_.increment(8);
continue;
}
}
if (stream_.remaining() >= 4) {
uint32_t x = stream_.peek_u32();
if (SWAR::has_special_char(x) == 0) {
if constexpr (Decode) {
SWAR::append_ascii(builder_, reinterpret_cast<const char*>(&x), 4);
}
stream_.increment(4);
continue;
}
}
}
current_char = stream_.peek();
if (current_char == 0) {
break;
}
if (EXPECTS_CHARACTER(parse_state)) {
if constexpr (Decode) {
builder_.push_back(ifcopenshell::convert_codepage(codepage, current_char + 0x80));
}
parse_state = 0;
} else if (current_char == '\'' && (parse_state == 0U)) {
parse_state = APOSTROPHE;
} else if (current_char == '\\' && (parse_state == 0U)) {
parse_state = FIRST_SOLIDUS;
} else if (current_char == '\\' && EXPECTS_SOLIDUS(parse_state)) {
if (((parse_state & ALPHABET_DEFINITION) != 0U) ||
((parse_state & IGNORED_DIRECTIVE) != 0U) ||
((parse_state & ENDEXTENDED_0) != 0U)) {
parse_state = hex = hex_count = 0;
} else if ((parse_state & ENCOUNTERED_HEX) != 0U) {
parse_state += THIRD_SOLIDUS;
parse_state -= ENCOUNTERED_HEX;
} else {
parse_state += SECOND_SOLIDUS;
}
} else if (current_char == 'X' && EXPECTS_ENDEXTENDED_X(parse_state)) {
parse_state += ENDEXTENDED_X;
} else if (current_char == '0' && EXPECTS_ENDEXTENDED_0(parse_state)) {
parse_state += ENDEXTENDED_0;
} else if (current_char == 'X' && EXPECTS_ARBITRARY(parse_state)) {
parse_state += ARBITRARY;
} else if (current_char == '2' && EXPECTS_ARBITRARY2(parse_state)) {
parse_state += EXTENDED2;
} else if (current_char == '4' && EXPECTS_ARBITRARY2(parse_state)) {
parse_state += EXTENDED2 + EXTENDED4;
} else if (current_char == 'P' && EXPECTS_ALPHABET(parse_state)) {
parse_state += ALPHABET;
} else if ((current_char == 'N' || current_char == 'F') && EXPECTS_N_OR_F(parse_state)) {
parse_state += IGNORED_DIRECTIVE;
} else if (IS_VALID_ALPHABET_DEFINITION(current_char) && EXPECTS_ALPHABET_DEFINITION(parse_state)) {
codepage = current_char - 0x40;
parse_state += ALPHABET_DEFINITION;
} else if (current_char == 'S' && EXPECTS_PAGE(parse_state)) {
parse_state += PAGE;
} else if (IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state)) {
if (IS_LOWERCASE_HEX(current_char)) {
logger_.warning("SYN", 2, "Lowercase hexadecimal character '" + std::string(1, current_char) +
"' found at offset " + std::to_string(stream_.tell()) +
". It is recommended to use uppercase for hexadecimal.");
}
hex <<= 4;
parse_state += HEX((++hex_count));
hex += HEX_TO_INT(current_char);
if ((hex_count == 2 && ((parse_state & EXTENDED2) == 0U)) ||
(hex_count == 4 && ((parse_state & EXTENDED4) == 0U)) ||
(hex_count == 8)) {
if constexpr (Decode) {
builder_.push_back(hex);
}
if (hex_count == 2) {
parse_state = 0;
} else {
CLEAR_HEX(parse_state);
parse_state |= ENCOUNTERED_HEX;
}
hex = hex_count = 0;
}
} else if ((parse_state != 0U) && !(
(current_char == '\\' && parse_state == FIRST_SOLIDUS) ||
(current_char == '\'' && parse_state == APOSTROPHE))) {
if (parse_state == APOSTROPHE && current_char != '\'') {
break;
}
throw invalid_token_exception(stream_.tell(), current_char);
} else {
parse_state = hex = hex_count = 0;
if constexpr (Decode) {
builder_.push_back(current_char);
}
}
stream_.increment();
}
if constexpr (!Decode) {
return std::string();
}
if (mode == ifcopenshell::character_decoder<Reader>::UTF8) {
if (builder_.empty()) {
static std::string empty;
return empty;
}
auto iter = std::max_element(builder_.begin(), builder_.end());
if (*iter <= 0x7e) {
std::string result(builder_.begin(), builder_.end());
return result;
}
return ifcopenshell::convert_utf8(builder_);
}
if (mode == ifcopenshell::character_decoder<Reader>::SUBSTITUTE) {
std::string result;
result.reserve(builder_.size());
std::transform(builder_.begin(), builder_.end(), std::back_inserter(result), [&substitution_character](std::u32string::value_type character) {
if (character >= 0x20 && character <= 0x7e) {
return (char)character;
}
return substitution_character;
});
return result;
}
if (mode == ifcopenshell::character_decoder<Reader>::ESCAPE) {
std::stringstream stream;
stream << std::hex << std::setw(4) << std::setfill('0');
std::for_each(builder_.begin(), builder_.end(), [&stream](std::u32string::value_type character) {
if (character >= 0x20 && character <= 0x7e) {
stream.put((char)character);
} else {
stream << "\\u" << static_cast<uint32_t>(character);
}
});
return stream.str();
}
throw ifcopenshell::exception("Invalid conversion mode");
}
}
template <typename Reader>
character_decoder<Reader>::operator std::string() {
return read_string(builder_, *stream_, logger_, mode, substitution_character);
}
template <typename Reader>
void character_decoder<Reader>::skip() {
read_string<Reader, false>(builder_, *stream_, logger_, mode, substitution_character);
}
template <typename Reader>
std::string character_decoder<Reader>::get(size_t& ptr) {
auto local_stream = *stream_;
local_stream.seek(ptr);
auto s = read_string(builder_, local_stream, logger_, mode, substitution_character);
ptr = local_stream.tell();
return s;
}
template class IFC_PARSE_API ifcopenshell::character_decoder<file_reader<full_buffer_impl>>;
template class IFC_PARSE_API ifcopenshell::character_decoder<file_reader<paged_file_impl>>;
template class IFC_PARSE_API ifcopenshell::character_decoder<file_reader<pushed_sequential_impl>>;
#ifdef USE_MMAP
template class IFC_PARSE_API ifcopenshell::character_decoder<file_reader<mmap_impl>>;
#endif
character_encoder::character_encoder(const std::string& input)
: str_(ifcopenshell::convert_utf8(input)) {}
character_encoder::operator std::string() {
std::ostringstream oss;
oss.put('\'');
const int num_bytes = (str_.empty() || (*std::max_element(str_.begin(), str_.end()) > 0xffff)) ? 4 : 2;
const std::string num_bytes_str = std::string(1, (char) (num_bytes + 0x30));
bool in_extended = false;
for (auto it = str_.begin(); it != str_.end(); ++it) {
auto character = *it;
const bool within_spf_range = character >= 0x20 && character <= 0x7e;
if (in_extended && within_spf_range) {
oss << "\\X0\\";
} else if (!in_extended && !within_spf_range) {
oss << "\\X" << num_bytes_str << "\\";
}
if (within_spf_range) {
oss.put((char)character);
if (character == '\\' || character == '\'') {
oss.put((char)character);
}
} else {
oss << std::hex << std::setw(num_bytes * 2) << std::uppercase << std::setfill('0') << (int)character;
}
in_extended = !within_spf_range;
}
if (in_extended) {
oss << "\\X0\\";
}
oss.put('\'');
return oss.str();
}
Generate with:
import codecs, string
codecs.register_error('zero', lambda e: (chr(0),e.start + 1))
def _():
for i in range(1,17):
try:
chrs = bytes(range(256)).decode('iso-8859-%d' % i, errors='zero')
except: chrs = "\x00" * 256
yield str(list(map(ord, chrs))).translate(string.maketrans("[]", "{}"))
"{%s}" % ",".join(_())
*/
std::u32string::value_type ifcopenshell::convert_codepage(int codepage, int character) {
if (codepage < 1 || codepage > 16 || codepage == 12) {
throw ifcopenshell::exception("Invalid codepage");
}
if (character < 0 || character > 255) {
throw ifcopenshell::exception("Invalid character ordinal");
}
static std::array<std::array<std::u32string::value_type, 256>, 16> codepage_data = {{{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 260, 728, 321, 164, 317, 346, 167, 168, 352, 350, 356, 377, 173, 381, 379, 176, 261, 731, 322, 180, 318, 347, 711, 184, 353, 351, 357, 378, 733, 382, 380, 340, 193, 194, 258, 196, 313, 262, 199, 268, 201, 280, 203, 282, 205, 206, 270, 272, 323, 327, 211, 212, 336, 214, 215, 344, 366, 218, 368, 220, 221, 354, 223, 341, 225, 226, 259, 228, 314, 263, 231, 269, 233, 281, 235, 283, 237, 238, 271, 273, 324, 328, 243, 244, 337, 246, 247, 345, 367, 250, 369, 252, 253, 355, 729}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 294, 728, 163, 164, 0, 292, 167, 168, 304, 350, 286, 308, 173, 0, 379, 176, 295, 178, 179, 180, 181, 293, 183, 184, 305, 351, 287, 309, 189, 0, 380, 192, 193, 194, 0, 196, 266, 264, 199, 200, 201, 202, 203, 204, 205, 206, 207, 0, 209, 210, 211, 212, 288, 214, 215, 284, 217, 218, 219, 220, 364, 348, 223, 224, 225, 226, 0, 228, 267, 265, 231, 232, 233, 234, 235, 236, 237, 238, 239, 0, 241, 242, 243, 244, 289, 246, 247, 285, 249, 250, 251, 252, 365, 349, 729}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 260, 312, 342, 164, 296, 315, 167, 168, 352, 274, 290, 358, 173, 381, 175, 176, 261, 731, 343, 180, 297, 316, 711, 184, 353, 275, 291, 359, 330, 382, 331, 256, 193, 194, 195, 196, 197, 198, 302, 268, 201, 280, 203, 278, 205, 206, 298, 272, 325, 332, 310, 212, 213, 214, 215, 216, 370, 218, 219, 220, 360, 362, 223, 257, 225, 226, 227, 228, 229, 230, 303, 269, 233, 281, 235, 279, 237, 238, 299, 273, 326, 333, 311, 244, 245, 246, 247, 248, 371, 250, 251, 252, 361, 363, 729}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 173, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 8470, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 167, 1118, 1119}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 1548, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 0, 0, 0, 1567, 0, 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 0, 0, 0, 0, 0, 1600, 1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 8216, 8217, 163, 8364, 8367, 166, 167, 168, 169, 890, 171, 172, 173, 0, 8213, 176, 177, 178, 179, 900, 901, 902, 183, 904, 905, 906, 187, 908, 189, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 0, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 0}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 0, 162, 163, 164, 165, 166, 167, 168, 169, 215, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 247, 187, 188, 189, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8215, 1488, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1496, 1497, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514, 0, 0, 8206, 8207, 0}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 286, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 304, 350, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 287, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 305, 351, 255}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 260, 274, 290, 298, 296, 310, 167, 315, 272, 352, 358, 381, 173, 362, 330, 176, 261, 275, 291, 299, 297, 311, 183, 316, 273, 353, 359, 382, 8213, 363, 331, 256, 193, 194, 195, 196, 197, 198, 302, 268, 201, 280, 203, 278, 205, 206, 207, 208, 325, 332, 211, 212, 213, 214, 360, 216, 370, 218, 219, 220, 221, 222, 223, 257, 225, 226, 227, 228, 229, 230, 303, 269, 233, 281, 235, 279, 237, 238, 239, 240, 326, 333, 243, 244, 245, 246, 361, 248, 371, 250, 251, 252, 253, 254, 312}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 3585, 3586, 3587, 3588, 3589, 3590, 3591, 3592, 3593, 3594, 3595, 3596, 3597, 3598, 3599, 3600, 3601, 3602, 3603, 3604, 3605, 3606, 3607, 3608, 3609, 3610, 3611, 3612, 3613, 3614, 3615, 3616, 3617, 3618, 3619, 3620, 3621, 3622, 3623, 3624, 3625, 3626, 3627, 3628, 3629, 3630, 3631, 3632, 3633, 3634, 3635, 3636, 3637, 3638, 3639, 3640, 3641, 3642, 0, 0, 0, 0, 3647, 3648, 3649, 3650, 3651, 3652, 3653, 3654, 3655, 3656, 3657, 3658, 3659, 3660, 3661, 3662, 3663, 3664, 3665, 3666, 3667, 3668, 3669, 3670, 3671, 3672, 3673, 3674, 3675, 0, 0, 0, 0}},
{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 8221, 162, 163, 164, 8222, 166, 167, 216, 169, 342, 171, 172, 173, 174, 198, 176, 177, 178, 179, 8220, 181, 182, 183, 248, 185, 343, 187, 188, 189, 190, 230, 260, 302, 256, 262, 196, 197, 280, 274, 268, 201, 377, 278, 290, 310, 298, 315, 352, 323, 325, 211, 332, 213, 214, 215, 370, 321, 346, 362, 220, 379, 381, 223, 261, 303, 257, 263, 228, 229, 281, 275, 269, 233, 378, 279, 291, 311, 299, 316, 353, 324, 326, 243, 333, 245, 246, 247, 371, 322, 347, 363, 252, 380, 382, 8217}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 7682, 7683, 163, 266, 267, 7690, 167, 7808, 169, 7810, 7691, 7922, 173, 174, 376, 7710, 7711, 288, 289, 7744, 7745, 182, 7766, 7809, 7767, 7811, 7776, 7923, 7812, 7813, 7777, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 372, 209, 210, 211, 212, 213, 214, 7786, 216, 217, 218, 219, 220, 221, 374, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 373, 241, 242, 243, 244, 245, 246, 7787, 248, 249, 250, 251, 252, 253, 375, 255}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 8364, 165, 352, 167, 353, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 381, 181, 182, 183, 382, 185, 186, 187, 338, 339, 376, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255}},
{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 260, 261, 321, 8364, 8222, 352, 167, 353, 169, 536, 171, 377, 173, 378, 379, 176, 177, 268, 322, 381, 8221, 182, 183, 382, 269, 537, 187, 338, 339, 376, 380, 192, 193, 194, 258, 196, 262, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 272, 323, 210, 211, 212, 336, 214, 346, 368, 217, 218, 219, 220, 280, 538, 223, 224, 225, 226, 259, 228, 263, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 273, 324, 242, 243, 244, 337, 246, 347, 369, 249, 250, 251, 252, 281, 539, 255}}}};
auto result = codepage_data[codepage - 1][character];
if (result == 0) {
throw ifcopenshell::exception("Character not defined");
}
return result;
}
std::string ifcopenshell::convert_utf8(const std::u32string& string) {
return boost::locale::conv::utf_to_utf<char>(string);
}
std::u32string ifcopenshell::convert_utf8(const std::string& s) {
bool is_ascii = true;
for (char c : s) {
if (static_cast<unsigned char>(c) >= 128) {
is_ascii = false;
break;
}
}
if (is_ascii) {
return std::u32string(s.begin(), s.end());
}
return boost::locale::conv::utf_to_utf<char32_t>(s);
}