OpenTTD Source: src/core/utf8.cpp Source File

OpenTTD Source 15.3

utf8.cpp
Go to the documentation of this file.
1/*
2 * This file is part of OpenTTD.
3 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <https://www.gnu.org/licenses/old-licenses/gpl-2.0>.
6 */
7
12#include "../stdafx.h"
13#include "utf8.hpp"
14#include "../safeguards.h"
15
21[[nodiscard]] std::pair<char[4], size_t> EncodeUtf8(char32_t c)
22{
23 std::pair<char[4], size_t> result{};
24 auto &[buf, len] = result;
25 if (c < 0x80) {
26 buf[len++] = c;
27 } else if (c < 0x800) {
28 buf[len++] = 0xC0 + GB(c, 6, 5);
29 buf[len++] = 0x80 + GB(c, 0, 6);
30 } else if (c < 0x10000) {
31 buf[len++] = 0xE0 + GB(c, 12, 4);
32 buf[len++] = 0x80 + GB(c, 6, 6);
33 buf[len++] = 0x80 + GB(c, 0, 6);
34 } else if (c < 0x110000) {
35 buf[len++] = 0xF0 + GB(c, 18, 3);
36 buf[len++] = 0x80 + GB(c, 12, 6);
37 buf[len++] = 0x80 + GB(c, 6, 6);
38 buf[len++] = 0x80 + GB(c, 0, 6);
39 }
40 return result;
41}
42
48[[nodiscard]] std::pair<size_t, char32_t> DecodeUtf8(std::string_view buf)
49{
50 if (buf.size() >= 1 && !HasBit(buf[0], 7)) {
51 /* Single byte character: 0xxxxxxx */
52 char32_t c = buf[0];
53 return {1, c};
54 } else if (buf.size() >= 2 && GB(buf[0], 5, 3) == 6) {
55 if (IsUtf8Part(buf[1])) {
56 /* Double byte character: 110xxxxx 10xxxxxx */
57 char32_t c = GB(buf[0], 0, 5) << 6 | GB(buf[1], 0, 6);
58 if (c >= 0x80) return {2, c};
59 }
60 } else if (buf.size() >= 3 && GB(buf[0], 4, 4) == 14) {
61 if (IsUtf8Part(buf[1]) && IsUtf8Part(buf[2])) {
62 /* Triple byte character: 1110xxxx 10xxxxxx 10xxxxxx */
63 char32_t c = GB(buf[0], 0, 4) << 12 | GB(buf[1], 0, 6) << 6 | GB(buf[2], 0, 6);
64 if (c >= 0x800) return {3, c};
65 }
66 } else if (buf.size() >= 4 && GB(buf[0], 3, 5) == 30) {
67 if (IsUtf8Part(buf[1]) && IsUtf8Part(buf[2]) && IsUtf8Part(buf[3])) {
68 /* 4 byte character: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
69 char32_t c = GB(buf[0], 0, 3) << 18 | GB(buf[1], 0, 6) << 12 | GB(buf[2], 0, 6) << 6 | GB(buf[3], 0, 6);
70 if (c >= 0x10000 && c <= 0x10FFFF) return {4, c};
71 }
72 }
73 return {};
74}
75
83 Utf8View::iterator Utf8View::GetIterAtByte(size_t offset) const
84{
85 assert(offset <= this->src.size());
86 if (offset >= this->src.size()) return this->end();
87
88 /* Sanitize iterator to point to the start of a codepoint */
89 auto it = iterator(this->src, offset + 1);
90 --it;
91 return it;
92}
GB
static constexpr uint GB(const T x, const uint8_t s, const uint8_t n)
Fetch n bits from x, started at bit s.
Definition bitmath_func.hpp:32
HasBit
constexpr bool HasBit(const T x, const uint8_t y)
Checks if a bit in a value is set.
Definition bitmath_func.hpp:103
Utf8View::iterator
Bidirectional input iterator over codepoints.
Definition utf8.hpp:43
Utf8View::GetIterAtByte
iterator GetIterAtByte(size_t offset) const
Create iterator pointing at codepoint, which occupies the byte position "offset".
Definition utf8.cpp:83
DecodeUtf8
std::pair< size_t, char32_t > DecodeUtf8(std::string_view buf)
Decode a character from UTF-8.
Definition utf8.cpp:48
EncodeUtf8
std::pair< char[4], size_t > EncodeUtf8(char32_t c)
Encode a character to UTF-8.
Definition utf8.cpp:21
utf8.hpp
Handling of UTF-8 encoded data.