OpenTTD Source: src/string.cpp Source File

OpenTTD Source 14.1

string.cpp
Go to the documentation of this file.
1 /*
2 * This file is part of OpenTTD.
3 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
6 */
7
10 #include "stdafx.h"
11 #include "debug.h"
12 #include "core/alloc_func.hpp"
13 #include "core/math_func.hpp"
14 #include "error_func.h"
15 #include "string_func.h"
16 #include "string_base.h"
17
18 #include "table/control_codes.h"
19
20 #include <sstream>
21 #include <iomanip>
22
23 #ifdef _MSC_VER
24 # define strncasecmp strnicmp
25 #endif
26
27 #ifdef _WIN32
28 # include "os/windows/win32.h"
29 #endif
30
31 #ifdef WITH_UNISCRIBE
32 # include "os/windows/string_uniscribe.h"
33 #endif
34
35 #ifdef WITH_ICU_I18N
36 /* Required by StrNaturalCompare. */
37 # include <unicode/ustring.h>
38 # include "language.h"
39 # include "gfx_func.h"
40 #endif /* WITH_ICU_I18N */
41
42 #if defined(WITH_COCOA)
43 # include "os/macosx/string_osx.h"
44 #endif
45
46 #include "safeguards.h"
47
48
65 char *strecpy(char *dst, const char *src, const char *last)
66 {
67 assert(dst <= last);
68 while (dst != last && *src != '\0') {
69 *dst++ = *src++;
70 }
71 *dst = '\0';
72
73 if (dst == last && *src != '\0') {
74 #if defined(STRGEN) || defined(SETTINGSGEN)
75 FatalError("String too long for destination buffer");
76 #else /* STRGEN || SETTINGSGEN */
77 Debug(misc, 0, "String too long for destination buffer");
78 #endif /* STRGEN || SETTINGSGEN */
79 }
80 return dst;
81 }
82
88 std::string FormatArrayAsHex(std::span<const byte> data)
89 {
90 std::string str;
91 str.reserve(data.size() * 2 + 1);
92
93 for (auto b : data) {
94 fmt::format_to(std::back_inserter(str), "{:02X}", b);
95 }
96
97 return str;
98 }
99
100
113 template <class T>
114 static void StrMakeValid(T &dst, const char *str, const char *last, StringValidationSettings settings)
115 {
116 /* Assume the ABSOLUTE WORST to be in str as it comes from the outside. */
117
118 while (str <= last && *str != '\0') {
119 size_t len = Utf8EncodedCharLen(*str);
120 char32_t c;
121 /* If the first byte does not look like the first byte of an encoded
122 * character, i.e. encoded length is 0, then this byte is definitely bad
123 * and it should be skipped.
124 * When the first byte looks like the first byte of an encoded character,
125 * then the remaining bytes in the string are checked whether the whole
126 * encoded character can be there. If that is not the case, this byte is
127 * skipped.
128 * Finally we attempt to decode the encoded character, which does certain
129 * extra validations to see whether the correct number of bytes were used
130 * to encode the character. If that is not the case, the byte is probably
131 * invalid and it is skipped. We could emit a question mark, but then the
132 * logic below cannot just copy bytes, it would need to re-encode the
133 * decoded characters as the length in bytes may have changed.
134 *
135 * The goals here is to get as much valid Utf8 encoded characters from the
136 * source string to the destination string.
137 *
138 * Note: a multi-byte encoded termination ('\0') will trigger the encoded
139 * char length and the decoded length to differ, so it will be ignored as
140 * invalid character data. If it were to reach the termination, then we
141 * would also reach the "last" byte of the string and a normal '\0'
142 * termination will be placed after it.
143 */
144 if (len == 0 || str + len > last + 1 || len != Utf8Decode(&c, str)) {
145 /* Maybe the next byte is still a valid character? */
146 str++;
147 continue;
148 }
149
150 if ((IsPrintable(c) && (c < SCC_SPRITE_START || c > SCC_SPRITE_END)) || ((settings & SVS_ALLOW_CONTROL_CODE) != 0 && c == SCC_ENCODED)) {
151 /* Copy the character back. Even if dst is current the same as str
152 * (i.e. no characters have been changed) this is quicker than
153 * moving the pointers ahead by len */
154 do {
155 *dst++ = *str++;
156 } while (--len != 0);
157 } else if ((settings & SVS_ALLOW_NEWLINE) != 0 && c == '\n') {
158 *dst++ = *str++;
159 } else {
160 if ((settings & SVS_ALLOW_NEWLINE) != 0 && c == '\r' && str[1] == '\n') {
161 str += len;
162 continue;
163 }
164 str += len;
165 if ((settings & SVS_REPLACE_TAB_CR_NL_WITH_SPACE) != 0 && (c == '\r' || c == '\n' || c == '\t')) {
166 /* Replace the tab, carriage return or newline with a space. */
167 *dst++ = ' ';
168 } else if ((settings & SVS_REPLACE_WITH_QUESTION_MARK) != 0) {
169 /* Replace the undesirable character with a question mark */
170 *dst++ = '?';
171 }
172 }
173 }
174
175 /* String termination, if needed, is left to the caller of this function. */
176 }
177
185 void StrMakeValidInPlace(char *str, const char *last, StringValidationSettings settings)
186 {
187 char *dst = str;
188 StrMakeValid(dst, str, last, settings);
189 *dst = '\0';
190 }
191
199 void StrMakeValidInPlace(char *str, StringValidationSettings settings)
200 {
201 /* We know it is '\0' terminated. */
202 StrMakeValidInPlace(str, str + strlen(str), settings);
203 }
204
212 std::string StrMakeValid(std::string_view str, StringValidationSettings settings)
213 {
214 if (str.empty()) return {};
215
216 auto buf = str.data();
217 auto last = buf + str.size() - 1;
218
219 std::ostringstream dst;
220 std::ostreambuf_iterator<char> dst_iter(dst);
221 StrMakeValid(dst_iter, buf, last, settings);
222
223 return dst.str();
224 }
225
233 bool StrValid(const char *str, const char *last)
234 {
235 /* Assume the ABSOLUTE WORST to be in str as it comes from the outside. */
236
237 while (str <= last && *str != '\0') {
238 size_t len = Utf8EncodedCharLen(*str);
239 /* Encoded length is 0 if the character isn't known.
240 * The length check is needed to prevent Utf8Decode to read
241 * over the terminating '\0' if that happens to be placed
242 * within the encoding of an UTF8 character. */
243 if (len == 0 || str + len > last) return false;
244
245 char32_t c;
246 len = Utf8Decode(&c, str);
247 if (!IsPrintable(c) || (c >= SCC_SPRITE_START && c <= SCC_SPRITE_END)) {
248 return false;
249 }
250
251 str += len;
252 }
253
254 return *str == '\0';
255 }
256
263 static void StrLeftTrimInPlace(std::string &str)
264 {
265 size_t pos = str.find_first_not_of(' ');
266 str.erase(0, pos);
267 }
268
275 static void StrRightTrimInPlace(std::string &str)
276 {
277 size_t pos = str.find_last_not_of(' ');
278 if (pos != std::string::npos) str.erase(pos + 1);
279 }
280
288 void StrTrimInPlace(std::string &str)
289 {
290 StrLeftTrimInPlace(str);
291 StrRightTrimInPlace(str);
292 }
293
300 bool StrStartsWithIgnoreCase(std::string_view str, const std::string_view prefix)
301 {
302 if (str.size() < prefix.size()) return false;
303 return StrEqualsIgnoreCase(str.substr(0, prefix.size()), prefix);
304 }
305
307 struct CaseInsensitiveCharTraits : public std::char_traits<char> {
308 static bool eq(char c1, char c2) { return toupper(c1) == toupper(c2); }
309 static bool ne(char c1, char c2) { return toupper(c1) != toupper(c2); }
310 static bool lt(char c1, char c2) { return toupper(c1) < toupper(c2); }
311
312 static int compare(const char *s1, const char *s2, size_t n)
313 {
314 while (n-- != 0) {
315 if (toupper(*s1) < toupper(*s2)) return -1;
316 if (toupper(*s1) > toupper(*s2)) return 1;
317 ++s1; ++s2;
318 }
319 return 0;
320 }
321
322 static const char *find(const char *s, size_t n, char a)
323 {
324 for (; n > 0; --n, ++s) {
325 if (toupper(*s) == toupper(a)) return s;
326 }
327 return nullptr;
328 }
329 };
330
332 typedef std::basic_string_view<char, CaseInsensitiveCharTraits> CaseInsensitiveStringView;
333
340 bool StrEndsWithIgnoreCase(std::string_view str, const std::string_view suffix)
341 {
342 if (str.size() < suffix.size()) return false;
343 return StrEqualsIgnoreCase(str.substr(str.size() - suffix.size()), suffix);
344 }
345
353 int StrCompareIgnoreCase(const std::string_view str1, const std::string_view str2)
354 {
355 CaseInsensitiveStringView ci_str1{ str1.data(), str1.size() };
356 CaseInsensitiveStringView ci_str2{ str2.data(), str2.size() };
357 return ci_str1.compare(ci_str2);
358 }
359
366 bool StrEqualsIgnoreCase(const std::string_view str1, const std::string_view str2)
367 {
368 if (str1.size() != str2.size()) return false;
369 return StrCompareIgnoreCase(str1, str2) == 0;
370 }
371
378 size_t Utf8StringLength(const char *s)
379 {
380 size_t len = 0;
381 const char *t = s;
382 while (Utf8Consume(&t) != 0) len++;
383 return len;
384 }
385
392 size_t Utf8StringLength(const std::string &str)
393 {
394 return Utf8StringLength(str.c_str());
395 }
396
397 bool strtolower(std::string &str, std::string::size_type offs)
398 {
399 bool changed = false;
400 for (auto ch = str.begin() + offs; ch != str.end(); ++ch) {
401 auto new_ch = static_cast<char>(tolower(static_cast<unsigned char>(*ch)));
402 changed |= new_ch != *ch;
403 *ch = new_ch;
404 }
405 return changed;
406 }
407
415 bool IsValidChar(char32_t key, CharSetFilter afilter)
416 {
417 switch (afilter) {
418 case CS_ALPHANUMERAL: return IsPrintable(key);
419 case CS_NUMERAL: return (key >= '0' && key <= '9');
420 case CS_NUMERAL_SPACE: return (key >= '0' && key <= '9') || key == ' ';
421 case CS_NUMERAL_SIGNED: return (key >= '0' && key <= '9') || key == '-';
422 case CS_ALPHA: return IsPrintable(key) && !(key >= '0' && key <= '9');
423 case CS_HEXADECIMAL: return (key >= '0' && key <= '9') || (key >= 'a' && key <= 'f') || (key >= 'A' && key <= 'F');
424 default: NOT_REACHED();
425 }
426 }
427
428
429 /* UTF-8 handling routines */
430
431
438 size_t Utf8Decode(char32_t *c, const char *s)
439 {
440 assert(c != nullptr);
441
442 if (!HasBit(s[0], 7)) {
443 /* Single byte character: 0xxxxxxx */
444 *c = s[0];
445 return 1;
446 } else if (GB(s[0], 5, 3) == 6) {
447 if (IsUtf8Part(s[1])) {
448 /* Double byte character: 110xxxxx 10xxxxxx */
449 *c = GB(s[0], 0, 5) << 6 | GB(s[1], 0, 6);
450 if (*c >= 0x80) return 2;
451 }
452 } else if (GB(s[0], 4, 4) == 14) {
453 if (IsUtf8Part(s[1]) && IsUtf8Part(s[2])) {
454 /* Triple byte character: 1110xxxx 10xxxxxx 10xxxxxx */
455 *c = GB(s[0], 0, 4) << 12 | GB(s[1], 0, 6) << 6 | GB(s[2], 0, 6);
456 if (*c >= 0x800) return 3;
457 }
458 } else if (GB(s[0], 3, 5) == 30) {
459 if (IsUtf8Part(s[1]) && IsUtf8Part(s[2]) && IsUtf8Part(s[3])) {
460 /* 4 byte character: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
461 *c = GB(s[0], 0, 3) << 18 | GB(s[1], 0, 6) << 12 | GB(s[2], 0, 6) << 6 | GB(s[3], 0, 6);
462 if (*c >= 0x10000 && *c <= 0x10FFFF) return 4;
463 }
464 }
465
466 *c = '?';
467 return 1;
468 }
469
470
478 template <class T>
479 inline size_t Utf8Encode(T buf, char32_t c)
480 {
481 if (c < 0x80) {
482 *buf = c;
483 return 1;
484 } else if (c < 0x800) {
485 *buf++ = 0xC0 + GB(c, 6, 5);
486 *buf = 0x80 + GB(c, 0, 6);
487 return 2;
488 } else if (c < 0x10000) {
489 *buf++ = 0xE0 + GB(c, 12, 4);
490 *buf++ = 0x80 + GB(c, 6, 6);
491 *buf = 0x80 + GB(c, 0, 6);
492 return 3;
493 } else if (c < 0x110000) {
494 *buf++ = 0xF0 + GB(c, 18, 3);
495 *buf++ = 0x80 + GB(c, 12, 6);
496 *buf++ = 0x80 + GB(c, 6, 6);
497 *buf = 0x80 + GB(c, 0, 6);
498 return 4;
499 }
500
501 *buf = '?';
502 return 1;
503 }
504
505 size_t Utf8Encode(char *buf, char32_t c)
506 {
507 return Utf8Encode<char *>(buf, c);
508 }
509
510 size_t Utf8Encode(std::ostreambuf_iterator<char> &buf, char32_t c)
511 {
512 return Utf8Encode<std::ostreambuf_iterator<char> &>(buf, c);
513 }
514
515 size_t Utf8Encode(std::back_insert_iterator<std::string> &buf, char32_t c)
516 {
517 return Utf8Encode<std::back_insert_iterator<std::string> &>(buf, c);
518 }
519
527 size_t Utf8TrimString(char *s, size_t maxlen)
528 {
529 size_t length = 0;
530
531 for (const char *ptr = strchr(s, '\0'); *s != '\0';) {
532 size_t len = Utf8EncodedCharLen(*s);
533 /* Silently ignore invalid UTF8 sequences, our only concern trimming */
534 if (len == 0) len = 1;
535
536 /* Take care when a hard cutoff was made for the string and
537 * the last UTF8 sequence is invalid */
538 if (length + len >= maxlen || (s + len > ptr)) break;
539 s += len;
540 length += len;
541 }
542
543 *s = '\0';
544 return length;
545 }
546
547 #ifdef DEFINE_STRCASESTR
548 char *strcasestr(const char *haystack, const char *needle)
549 {
550 size_t hay_len = strlen(haystack);
551 size_t needle_len = strlen(needle);
552 while (hay_len >= needle_len) {
553 if (strncasecmp(haystack, needle, needle_len) == 0) return const_cast<char *>(haystack);
554
555 haystack++;
556 hay_len--;
557 }
558
559 return nullptr;
560 }
561 #endif /* DEFINE_STRCASESTR */
562
571 static std::string_view SkipGarbage(std::string_view str)
572 {
573 while (!str.empty() && (str[0] < '0' || IsInsideMM(str[0], ';', '@' + 1) || IsInsideMM(str[0], '[', '`' + 1) || IsInsideMM(str[0], '{', '~' + 1))) str.remove_prefix(1);
574 return str;
575 }
576
585 int StrNaturalCompare(std::string_view s1, std::string_view s2, bool ignore_garbage_at_front)
586 {
587 if (ignore_garbage_at_front) {
588 s1 = SkipGarbage(s1);
589 s2 = SkipGarbage(s2);
590 }
591
592 #ifdef WITH_ICU_I18N
593 if (_current_collator) {
594 UErrorCode status = U_ZERO_ERROR;
595 int result = _current_collator->compareUTF8(icu::StringPiece(s1.data(), s1.size()), icu::StringPiece(s2.data(), s2.size()), status);
596 if (U_SUCCESS(status)) return result;
597 }
598 #endif /* WITH_ICU_I18N */
599
600 #if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
601 int res = OTTDStringCompare(s1, s2);
602 if (res != 0) return res - 2; // Convert to normal C return values.
603 #endif
604
605 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
606 int res = MacOSStringCompare(s1, s2);
607 if (res != 0) return res - 2; // Convert to normal C return values.
608 #endif
609
610 /* Do a normal comparison if ICU is missing or if we cannot create a collator. */
611 return StrCompareIgnoreCase(s1, s2);
612 }
613
614 #ifdef WITH_ICU_I18N
615
616 #include <unicode/stsearch.h>
617
626 static int ICUStringContains(const std::string_view str, const std::string_view value, bool case_insensitive)
627 {
628 if (_current_collator) {
629 std::unique_ptr<icu::RuleBasedCollator> coll(dynamic_cast<icu::RuleBasedCollator *>(_current_collator->clone()));
630 if (coll) {
631 UErrorCode status = U_ZERO_ERROR;
632 coll->setStrength(case_insensitive ? icu::Collator::SECONDARY : icu::Collator::TERTIARY);
633 coll->setAttribute(UCOL_NUMERIC_COLLATION, UCOL_OFF, status);
634
635 auto u_str = icu::UnicodeString::fromUTF8(icu::StringPiece(str.data(), str.size()));
636 auto u_value = icu::UnicodeString::fromUTF8(icu::StringPiece(value.data(), value.size()));
637 icu::StringSearch u_searcher(u_value, u_str, coll.get(), nullptr, status);
638 if (U_SUCCESS(status)) {
639 auto pos = u_searcher.first(status);
640 if (U_SUCCESS(status)) return pos != USEARCH_DONE ? 1 : 0;
641 }
642 }
643 }
644
645 return -1;
646 }
647 #endif /* WITH_ICU_I18N */
648
656 [[nodiscard]] bool StrNaturalContains(const std::string_view str, const std::string_view value)
657 {
658 #ifdef WITH_ICU_I18N
659 int res_u = ICUStringContains(str, value, false);
660 if (res_u >= 0) return res_u > 0;
661 #endif /* WITH_ICU_I18N */
662
663 #if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
664 int res = Win32StringContains(str, value, false);
665 if (res >= 0) return res > 0;
666 #endif
667
668 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
669 int res = MacOSStringContains(str, value, false);
670 if (res >= 0) return res > 0;
671 #endif
672
673 return str.find(value) != std::string_view::npos;
674 }
675
683 [[nodiscard]] bool StrNaturalContainsIgnoreCase(const std::string_view str, const std::string_view value)
684 {
685 #ifdef WITH_ICU_I18N
686 int res_u = ICUStringContains(str, value, true);
687 if (res_u >= 0) return res_u > 0;
688 #endif /* WITH_ICU_I18N */
689
690 #if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
691 int res = Win32StringContains(str, value, true);
692 if (res >= 0) return res > 0;
693 #endif
694
695 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
696 int res = MacOSStringContains(str, value, true);
697 if (res >= 0) return res > 0;
698 #endif
699
700 CaseInsensitiveStringView ci_str{ str.data(), str.size() };
701 CaseInsensitiveStringView ci_value{ value.data(), value.size() };
702 return ci_str.find(ci_value) != CaseInsensitiveStringView::npos;
703 }
704
711 static int ConvertHexNibbleToByte(char c)
712 {
713 if (c >= '0' && c <= '9') return c - '0';
714 if (c >= 'A' && c <= 'F') return c + 10 - 'A';
715 if (c >= 'a' && c <= 'f') return c + 10 - 'a';
716 return -1;
717 }
718
730 bool ConvertHexToBytes(std::string_view hex, std::span<uint8_t> bytes)
731 {
732 if (bytes.size() != hex.size() / 2) {
733 return false;
734 }
735
736 /* Hex-string lengths are always divisible by 2. */
737 if (hex.size() % 2 != 0) {
738 return false;
739 }
740
741 for (size_t i = 0; i < hex.size() / 2; i++) {
742 auto hi = ConvertHexNibbleToByte(hex[i * 2]);
743 auto lo = ConvertHexNibbleToByte(hex[i * 2 + 1]);
744
745 if (hi < 0 || lo < 0) {
746 return false;
747 }
748
749 bytes[i] = (hi << 4) | lo;
750 }
751
752 return true;
753 }
754
755 #ifdef WITH_UNISCRIBE
756
757 /* static */ std::unique_ptr<StringIterator> StringIterator::Create()
758 {
759 return std::make_unique<UniscribeStringIterator>();
760 }
761
762 #elif defined(WITH_ICU_I18N)
763
764 #include <unicode/utext.h>
765 #include <unicode/brkiter.h>
766
768 class IcuStringIterator : public StringIterator
769 {
770 icu::BreakIterator *char_itr;
771 icu::BreakIterator *word_itr;
772
773 std::vector<UChar> utf16_str;
774 std::vector<size_t> utf16_to_utf8;
775
776 public:
777 IcuStringIterator() : char_itr(nullptr), word_itr(nullptr)
778 {
779 UErrorCode status = U_ZERO_ERROR;
780 this->char_itr = icu::BreakIterator::createCharacterInstance(icu::Locale(_current_language != nullptr ? _current_language->isocode : "en"), status);
781 this->word_itr = icu::BreakIterator::createWordInstance(icu::Locale(_current_language != nullptr ? _current_language->isocode : "en"), status);
782
783 this->utf16_str.push_back('\0');
784 this->utf16_to_utf8.push_back(0);
785 }
786
787 ~IcuStringIterator() override
788 {
789 delete this->char_itr;
790 delete this->word_itr;
791 }
792
793 void SetString(const char *s) override
794 {
795 const char *string_base = s;
796
797 /* Unfortunately current ICU versions only provide rudimentary support
798 * for word break iterators (especially for CJK languages) in combination
799 * with UTF-8 input. As a work around we have to convert the input to
800 * UTF-16 and create a mapping back to UTF-8 character indices. */
801 this->utf16_str.clear();
802 this->utf16_to_utf8.clear();
803
804 while (*s != '\0') {
805 size_t idx = s - string_base;
806
807 char32_t c = Utf8Consume(&s);
808 if (c < 0x10000) {
809 this->utf16_str.push_back((UChar)c);
810 } else {
811 /* Make a surrogate pair. */
812 this->utf16_str.push_back((UChar)(0xD800 + ((c - 0x10000) >> 10)));
813 this->utf16_str.push_back((UChar)(0xDC00 + ((c - 0x10000) & 0x3FF)));
814 this->utf16_to_utf8.push_back(idx);
815 }
816 this->utf16_to_utf8.push_back(idx);
817 }
818 this->utf16_str.push_back('\0');
819 this->utf16_to_utf8.push_back(s - string_base);
820
821 UText text = UTEXT_INITIALIZER;
822 UErrorCode status = U_ZERO_ERROR;
823 utext_openUChars(&text, this->utf16_str.data(), this->utf16_str.size() - 1, &status);
824 this->char_itr->setText(&text, status);
825 this->word_itr->setText(&text, status);
826 this->char_itr->first();
827 this->word_itr->first();
828 }
829
830 size_t SetCurPosition(size_t pos) override
831 {
832 /* Convert incoming position to an UTF-16 string index. */
833 uint utf16_pos = 0;
834 for (uint i = 0; i < this->utf16_to_utf8.size(); i++) {
835 if (this->utf16_to_utf8[i] == pos) {
836 utf16_pos = i;
837 break;
838 }
839 }
840
841 /* isBoundary has the documented side-effect of setting the current
842 * position to the first valid boundary equal to or greater than
843 * the passed value. */
844 this->char_itr->isBoundary(utf16_pos);
845 return this->utf16_to_utf8[this->char_itr->current()];
846 }
847
848 size_t Next(IterType what) override
849 {
850 int32_t pos;
851 switch (what) {
852 case ITER_CHARACTER:
853 pos = this->char_itr->next();
854 break;
855
856 case ITER_WORD:
857 pos = this->word_itr->following(this->char_itr->current());
858 /* The ICU word iterator considers both the start and the end of a word a valid
859 * break point, but we only want word starts. Move to the next location in
860 * case the new position points to whitespace. */
861 while (pos != icu::BreakIterator::DONE &&
862 IsWhitespace(Utf16DecodeChar((const uint16_t *)&this->utf16_str[pos]))) {
863 int32_t new_pos = this->word_itr->next();
864 /* Don't set it to DONE if it was valid before. Otherwise we'll return END
865 * even though the iterator wasn't at the end of the string before. */
866 if (new_pos == icu::BreakIterator::DONE) break;
867 pos = new_pos;
868 }
869
870 this->char_itr->isBoundary(pos);
871 break;
872
873 default:
874 NOT_REACHED();
875 }
876
877 return pos == icu::BreakIterator::DONE ? END : this->utf16_to_utf8[pos];
878 }
879
880 size_t Prev(IterType what) override
881 {
882 int32_t pos;
883 switch (what) {
884 case ITER_CHARACTER:
885 pos = this->char_itr->previous();
886 break;
887
888 case ITER_WORD:
889 pos = this->word_itr->preceding(this->char_itr->current());
890 /* The ICU word iterator considers both the start and the end of a word a valid
891 * break point, but we only want word starts. Move to the previous location in
892 * case the new position points to whitespace. */
893 while (pos != icu::BreakIterator::DONE &&
894 IsWhitespace(Utf16DecodeChar((const uint16_t *)&this->utf16_str[pos]))) {
895 int32_t new_pos = this->word_itr->previous();
896 /* Don't set it to DONE if it was valid before. Otherwise we'll return END
897 * even though the iterator wasn't at the start of the string before. */
898 if (new_pos == icu::BreakIterator::DONE) break;
899 pos = new_pos;
900 }
901
902 this->char_itr->isBoundary(pos);
903 break;
904
905 default:
906 NOT_REACHED();
907 }
908
909 return pos == icu::BreakIterator::DONE ? END : this->utf16_to_utf8[pos];
910 }
911 };
912
913 /* static */ std::unique_ptr<StringIterator> StringIterator::Create()
914 {
915 return std::make_unique<IcuStringIterator>();
916 }
917
918 #else
919
921 class DefaultStringIterator : public StringIterator
922 {
923 const char *string;
924 size_t len;
925 size_t cur_pos;
926
927 public:
928 DefaultStringIterator() : string(nullptr), len(0), cur_pos(0)
929 {
930 }
931
932 void SetString(const char *s) override
933 {
934 this->string = s;
935 this->len = strlen(s);
936 this->cur_pos = 0;
937 }
938
939 size_t SetCurPosition(size_t pos) override
940 {
941 assert(this->string != nullptr && pos <= this->len);
942 /* Sanitize in case we get a position inside an UTF-8 sequence. */
943 while (pos > 0 && IsUtf8Part(this->string[pos])) pos--;
944 return this->cur_pos = pos;
945 }
946
947 size_t Next(IterType what) override
948 {
949 assert(this->string != nullptr);
950
951 /* Already at the end? */
952 if (this->cur_pos >= this->len) return END;
953
954 switch (what) {
955 case ITER_CHARACTER: {
956 char32_t c;
957 this->cur_pos += Utf8Decode(&c, this->string + this->cur_pos);
958 return this->cur_pos;
959 }
960
961 case ITER_WORD: {
962 char32_t c;
963 /* Consume current word. */
964 size_t offs = Utf8Decode(&c, this->string + this->cur_pos);
965 while (this->cur_pos < this->len && !IsWhitespace(c)) {
966 this->cur_pos += offs;
967 offs = Utf8Decode(&c, this->string + this->cur_pos);
968 }
969 /* Consume whitespace to the next word. */
970 while (this->cur_pos < this->len && IsWhitespace(c)) {
971 this->cur_pos += offs;
972 offs = Utf8Decode(&c, this->string + this->cur_pos);
973 }
974
975 return this->cur_pos;
976 }
977
978 default:
979 NOT_REACHED();
980 }
981
982 return END;
983 }
984
985 size_t Prev(IterType what) override
986 {
987 assert(this->string != nullptr);
988
989 /* Already at the beginning? */
990 if (this->cur_pos == 0) return END;
991
992 switch (what) {
993 case ITER_CHARACTER:
994 return this->cur_pos = Utf8PrevChar(this->string + this->cur_pos) - this->string;
995
996 case ITER_WORD: {
997 const char *s = this->string + this->cur_pos;
998 char32_t c;
999 /* Consume preceding whitespace. */
1000 do {
1001 s = Utf8PrevChar(s);
1002 Utf8Decode(&c, s);
1003 } while (s > this->string && IsWhitespace(c));
1004 /* Consume preceding word. */
1005 while (s > this->string && !IsWhitespace(c)) {
1006 s = Utf8PrevChar(s);
1007 Utf8Decode(&c, s);
1008 }
1009 /* Move caret back to the beginning of the word. */
1010 if (IsWhitespace(c)) Utf8Consume(&s);
1011
1012 return this->cur_pos = s - this->string;
1013 }
1014
1015 default:
1016 NOT_REACHED();
1017 }
1018
1019 return END;
1020 }
1021 };
1022
1023 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
1024 /* static */ std::unique_ptr<StringIterator> StringIterator::Create()
1025 {
1026 std::unique_ptr<StringIterator> i = OSXStringIterator::Create();
1027 if (i != nullptr) return i;
1028
1029 return std::make_unique<DefaultStringIterator>();
1030 }
1031 #else
1032 /* static */ std::unique_ptr<StringIterator> StringIterator::Create()
1033 {
1034 return std::make_unique<DefaultStringIterator>();
1035 }
1036 #endif /* defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN) */
1037
1038 #endif
StrStartsWithIgnoreCase
bool StrStartsWithIgnoreCase(std::string_view str, const std::string_view prefix)
Check whether the given string starts with the given prefix, ignoring case.
Definition: string.cpp:300
FormatArrayAsHex
std::string FormatArrayAsHex(std::span< const byte > data)
Format a byte array into a continuous hex string.
Definition: string.cpp:88
StringIterator::Prev
virtual size_t Prev(IterType what=ITER_CHARACTER)=0
Move the cursor back by one iteration unit.
IcuStringIterator::utf16_to_utf8
std::vector< size_t > utf16_to_utf8
Mapping from UTF-16 code point position to index in the UTF-8 source string.
Definition: string.cpp:774
CaseInsensitiveStringView
std::basic_string_view< char, CaseInsensitiveCharTraits > CaseInsensitiveStringView
Case insensitive string view.
Definition: string.cpp:332
SVS_ALLOW_NEWLINE
@ SVS_ALLOW_NEWLINE
Allow newlines; replaces '\r ' with ' ' during processing.
Definition: string_type.h:47
StringIterator::IterType
IterType
Type of the iterator.
Definition: string_base.h:17
IsInsideMM
constexpr bool IsInsideMM(const T x, const size_t min, const size_t max) noexcept
Checks if a value is in an interval.
Definition: math_func.hpp:268
win32.h
Win32StringContains
int Win32StringContains(const std::string_view str, const std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
Definition: win32.cpp:614
StrRightTrimInPlace
static void StrRightTrimInPlace(std::string &str)
Trim the spaces from the end of given string in place, i.e.
Definition: string.cpp:275
StringIterator::END
static const size_t END
Sentinel to indicate end-of-iteration.
Definition: string_base.h:23
math_func.hpp
GB
constexpr static debug_inline uint GB(const T x, const uint8_t s, const uint8_t n)
Fetch n bits from x, started at bit s.
Definition: bitmath_func.hpp:32
CaseInsensitiveCharTraits
Case insensitive implementation of the standard character type traits.
Definition: string.cpp:307
Utf8PrevChar
char * Utf8PrevChar(char *s)
Retrieve the previous UNICODE character in an UTF-8 encoded string.
Definition: string_func.h:148
StrMakeValid
static void StrMakeValid(T &dst, const char *str, const char *last, StringValidationSettings settings)
Copies the valid (UTF-8) characters from str up to last to the dst.
Definition: string.cpp:114
Utf16DecodeChar
char32_t Utf16DecodeChar(const uint16_t *c)
Decode an UTF-16 character.
Definition: string_func.h:201
IcuStringIterator::word_itr
icu::BreakIterator * word_itr
ICU iterator for words.
Definition: string.cpp:771
_current_collator
std::unique_ptr< icu::Collator > _current_collator
Collator for the language currently in use.
Definition: strings.cpp:59
StringIterator::Next
virtual size_t Next(IterType what=ITER_CHARACTER)=0
Advance the cursor by one iteration unit.
CS_ALPHA
@ CS_ALPHA
Only alphabetic values.
Definition: string_type.h:29
StrNaturalCompare
int StrNaturalCompare(std::string_view s1, std::string_view s2, bool ignore_garbage_at_front)
Compares two strings using case insensitive natural sort.
Definition: string.cpp:585
StrEndsWithIgnoreCase
bool StrEndsWithIgnoreCase(std::string_view str, const std::string_view suffix)
Check whether the given string ends with the given suffix, ignoring case.
Definition: string.cpp:340
ConvertHexToBytes
bool ConvertHexToBytes(std::string_view hex, std::span< uint8_t > bytes)
Convert a hex-string to a byte-array, while validating it was actually hex.
Definition: string.cpp:730
StringIterator::SetString
virtual void SetString(const char *s)=0
Set a new iteration string.
IcuStringIterator::char_itr
icu::BreakIterator * char_itr
ICU iterator for characters.
Definition: string.cpp:770
StringIterator::ITER_CHARACTER
@ ITER_CHARACTER
Iterate over characters (or more exactly grapheme clusters).
Definition: string_base.h:18
Debug
#define Debug(category, level, format_string,...)
Ouptut a line of debugging information.
Definition: debug.h:37
control_codes.h
IcuStringIterator::Next
size_t Next(IterType what) override
Advance the cursor by one iteration unit.
Definition: string.cpp:848
string_osx.h
gfx_func.h
Utf8StringLength
size_t Utf8StringLength(const char *s)
Get the length of an UTF-8 encoded string in number of characters and thus not the number of bytes th...
Definition: string.cpp:378
StrValid
bool StrValid(const char *str, const char *last)
Checks whether the given string is valid, i.e.
Definition: string.cpp:233
SVS_ALLOW_CONTROL_CODE
@ SVS_ALLOW_CONTROL_CODE
Allow the special control codes.
Definition: string_type.h:48
error_func.h
StringIterator
Class for iterating over different kind of parts of a string.
Definition: string_base.h:14
IcuStringIterator::SetCurPosition
size_t SetCurPosition(size_t pos) override
Change the current string cursor.
Definition: string.cpp:830
StringIterator::SetCurPosition
virtual size_t SetCurPosition(size_t pos)=0
Change the current string cursor.
StrMakeValidInPlace
void StrMakeValidInPlace(char *str, const char *last, StringValidationSettings settings)
Scans the string for invalid characters and replaces then with a question mark '?' (if not ignored).
Definition: string.cpp:185
StrTrimInPlace
void StrTrimInPlace(std::string &str)
Trim the spaces from given string in place, i.e.
Definition: string.cpp:288
ConvertHexNibbleToByte
static int ConvertHexNibbleToByte(char c)
Convert a single hex-nibble to a byte.
Definition: string.cpp:711
MacOSStringContains
int MacOSStringContains(const std::string_view str, const std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
Definition: string_osx.cpp:352
MacOSStringCompare
int MacOSStringCompare(std::string_view s1, std::string_view s2)
Compares two strings using case insensitive natural sort.
Definition: string_osx.cpp:328
_current_language
const LanguageMetadata * _current_language
The currently loaded language.
Definition: strings.cpp:54
safeguards.h
IcuStringIterator::utf16_str
std::vector< UChar > utf16_str
UTF-16 copy of the string.
Definition: string.cpp:773
settings
fluid_settings_t * settings
FluidSynth settings handle.
Definition: fluidsynth.cpp:21
StringIterator::Create
static std::unique_ptr< StringIterator > Create()
Create a new iterator instance.
Definition: string.cpp:913
StringIterator::ITER_WORD
@ ITER_WORD
Iterate over words.
Definition: string_base.h:19
language.h
stdafx.h
CS_ALPHANUMERAL
@ CS_ALPHANUMERAL
Both numeric and alphabetic and spaces and stuff.
Definition: string_type.h:25
LanguagePackHeader::isocode
char isocode[16]
the ISO code for the language (not country code)
Definition: language.h:31
StringValidationSettings
StringValidationSettings
Settings for the string validation.
Definition: string_type.h:44
SkipGarbage
static std::string_view SkipGarbage(std::string_view str)
Skip some of the 'garbage' in the string that we don't want to use to sort on.
Definition: string.cpp:571
string_func.h
Utf8EncodedCharLen
int8_t Utf8EncodedCharLen(char c)
Return the length of an UTF-8 encoded value based on a single char.
Definition: string_func.h:123
IsValidChar
bool IsValidChar(char32_t key, CharSetFilter afilter)
Only allow certain keys.
Definition: string.cpp:415
ICUStringContains
static int ICUStringContains(const std::string_view str, const std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
Definition: string.cpp:626
alloc_func.hpp
IcuStringIterator::SetString
void SetString(const char *s) override
Set a new iteration string.
Definition: string.cpp:793
StrLeftTrimInPlace
static void StrLeftTrimInPlace(std::string &str)
Trim the spaces from the begin of given string in place, i.e.
Definition: string.cpp:263
IcuStringIterator
String iterator using ICU as a backend.
Definition: string.cpp:768
StrEqualsIgnoreCase
bool StrEqualsIgnoreCase(const std::string_view str1, const std::string_view str2)
Compares two string( view)s for equality, while ignoring the case of the characters.
Definition: string.cpp:366
IsWhitespace
bool IsWhitespace(char32_t c)
Check whether UNICODE character is whitespace or not, i.e.
Definition: string_func.h:248
CS_NUMERAL_SIGNED
@ CS_NUMERAL_SIGNED
Only numbers and '-' for negative values.
Definition: string_type.h:28
Utf8TrimString
size_t Utf8TrimString(char *s, size_t maxlen)
Properly terminate an UTF8 string to some maximum length.
Definition: string.cpp:527
CS_NUMERAL_SPACE
@ CS_NUMERAL_SPACE
Only numbers and spaces.
Definition: string_type.h:27
Utf8Encode
size_t Utf8Encode(T buf, char32_t c)
Encode a unicode character and place it in the buffer.
Definition: string.cpp:479
SVS_REPLACE_TAB_CR_NL_WITH_SPACE
@ SVS_REPLACE_TAB_CR_NL_WITH_SPACE
Replace tabs ('\t'), carriage returns ('\r') and newlines (' ') with spaces.
Definition: string_type.h:54
Utf8Decode
size_t Utf8Decode(char32_t *c, const char *s)
Decode and consume the next UTF-8 encoded character.
Definition: string.cpp:438
strecpy
char * strecpy(char *dst, const char *src, const char *last)
Copies characters from one buffer to another.
Definition: string.cpp:65
CS_HEXADECIMAL
@ CS_HEXADECIMAL
Only hexadecimal characters.
Definition: string_type.h:30
SVS_REPLACE_WITH_QUESTION_MARK
@ SVS_REPLACE_WITH_QUESTION_MARK
Replace the unknown/bad bits with question marks.
Definition: string_type.h:46
StrNaturalContains
bool StrNaturalContains(const std::string_view str, const std::string_view value)
Checks if a string is contained in another string with a locale-aware comparison that is case sensiti...
Definition: string.cpp:656
StrNaturalContainsIgnoreCase
bool StrNaturalContainsIgnoreCase(const std::string_view str, const std::string_view value)
Checks if a string is contained in another string with a locale-aware comparison that is case insensi...
Definition: string.cpp:683
CS_NUMERAL
@ CS_NUMERAL
Only numeric ones.
Definition: string_type.h:26
CharSetFilter
CharSetFilter
Valid filter types for IsValidChar.
Definition: string_type.h:24
StrCompareIgnoreCase
int StrCompareIgnoreCase(const std::string_view str1, const std::string_view str2)
Compares two string( view)s, while ignoring the case of the characters.
Definition: string.cpp:353
debug.h
string_uniscribe.h
IcuStringIterator::Prev
size_t Prev(IterType what) override
Move the cursor back by one iteration unit.
Definition: string.cpp:880
HasBit
constexpr debug_inline bool HasBit(const T x, const uint8_t y)
Checks if a bit in a value is set.
Definition: bitmath_func.hpp:103