OpenTTD Source: src/strgen/strgen_base.cpp Source File

OpenTTD Source 15.3

strgen_base.cpp
Go to the documentation of this file.
1/*
2 * This file is part of OpenTTD.
3 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <https://www.gnu.org/licenses/old-licenses/gpl-2.0>.
6 */
7
10#include "../stdafx.h"
11#include "../core/endian_func.hpp"
12#include "../core/math_func.hpp"
13#include "../error_func.h"
14#include "../string_func.h"
15#include "../core/string_builder.hpp"
16#include "../table/control_codes.h"
17
18#include "strgen.h"
19
20#include "../table/strgen_tables.h"
21
22#include "../safeguards.h"
23
24 StrgenState _strgen;
25 static bool _translated;
26static std::string_view _cur_ident;
27static ParsedCommandStruct _cur_pcs;
28static size_t _cur_argidx;
29
30 struct ParsedCommandString {
31 const CmdStruct *cmd = nullptr;
32 std::string param;
33 std::optional<size_t> argno;
34 std::optional<uint8_t> casei;
35};
36static ParsedCommandString ParseCommandString(StringConsumer &consumer);
37static size_t TranslateArgumentIdx(size_t arg, size_t offset = 0);
38
44Case::Case(uint8_t caseidx, std::string_view string) :
45 caseidx(caseidx), string(string)
46{
47}
48
56LangString::LangString(std::string_view name, std::string_view english, size_t index, size_t line) :
57 name(name), english(english), index(index), line(line)
58{
59}
60
62 void LangString::FreeTranslation()
63{
64 this->translated.clear();
65 this->translated_cases.clear();
66}
67
72StringData::StringData(size_t tabs) : tabs(tabs), max_strings(tabs * TAB_SIZE)
73{
74 this->strings.resize(max_strings);
75 this->next_string_id = 0;
76}
77
79 void StringData::FreeTranslation()
80{
81 for (size_t i = 0; i < this->max_strings; i++) {
82 LangString *ls = this->strings[i].get();
83 if (ls != nullptr) ls->FreeTranslation();
84 }
85}
86
92 void StringData::Add(std::shared_ptr<LangString> ls)
93{
94 this->name_to_string[ls->name] = ls;
95 this->strings[ls->index] = std::move(ls);
96}
97
103 LangString *StringData::Find(std::string_view s)
104{
105 auto it = this->name_to_string.find(s);
106 if (it == this->name_to_string.end()) return nullptr;
107
108 return it->second.get();
109}
110
117 static uint32_t VersionHashStr(uint32_t hash, std::string_view s)
118{
119 for (auto c : s) {
120 hash = std::rotl(hash, 3) ^ c;
121 hash = (hash & 1 ? hash >> 1 ^ 0xDEADBEEF : hash >> 1);
122 }
123 return hash;
124}
125
130uint32_t StringData::Version() const
131{
132 uint32_t hash = 0;
133
134 for (size_t i = 0; i < this->max_strings; i++) {
135 const LangString *ls = this->strings[i].get();
136
137 if (ls != nullptr) {
138 hash ^= i * 0x717239;
139 hash = (hash & 1 ? hash >> 1 ^ 0xDEADBEEF : hash >> 1);
140 hash = VersionHashStr(hash, ls->name);
141
142 StringConsumer consumer(ls->english);
143 ParsedCommandString cs;
144 while ((cs = ParseCommandString(consumer)).cmd != nullptr) {
145 if (cs.cmd->flags.Test(CmdFlag::DontCount)) continue;
146
147 hash ^= (cs.cmd - _cmd_structs) * 0x1234567;
148 hash = (hash & 1 ? hash >> 1 ^ 0xF00BAA4 : hash >> 1);
149 }
150 }
151 }
152
153 return hash;
154}
155
160 size_t StringData::CountInUse(size_t tab) const
161{
162 size_t count = TAB_SIZE;
163 while (count > 0 && this->strings[(tab * TAB_SIZE) + count - 1] == nullptr) --count;
164 return count;
165}
166
167void EmitSingleChar(StringBuilder &builder, std::string_view param, char32_t value)
168{
169 if (!param.empty()) StrgenWarning("Ignoring trailing letters in command");
170 builder.PutUtf8(value);
171}
172
173/* The plural specifier looks like
174 * {NUM} {PLURAL <ARG#> passenger passengers} then it picks either passenger/passengers depending on the count in NUM */
175static std::pair<std::optional<size_t>, std::optional<size_t>> ParseRelNum(StringConsumer &consumer)
176{
177 consumer.SkipUntilCharNotIn(StringConsumer::WHITESPACE_NO_NEWLINE);
178 std::optional<size_t> v = consumer.TryReadIntegerBase<size_t>(10);
179 std::optional<size_t> offset;
180 if (v.has_value() && consumer.ReadCharIf(':')) {
181 /* Take the Nth within */
182 offset = consumer.TryReadIntegerBase<size_t>(10);
183 if (!offset.has_value()) StrgenFatal("Expected number for substring parameter");
184 }
185 return {v, offset};
186}
187
188/* Parse out the next word, or nullptr */
189std::optional<std::string_view> ParseWord(StringConsumer &consumer)
190{
191 consumer.SkipUntilCharNotIn(StringConsumer::WHITESPACE_NO_NEWLINE);
192 if (!consumer.AnyBytesLeft()) return {};
193
194 if (consumer.ReadCharIf('"')) {
195 /* parse until next " or NUL */
196 auto result = consumer.ReadUntilChar('"', StringConsumer::KEEP_SEPARATOR);
197 if (!consumer.ReadCharIf('"')) StrgenFatal("Unterminated quotes");
198 return result;
199 } else {
200 /* proceed until whitespace or NUL */
201 return consumer.ReadUntilCharIn(StringConsumer::WHITESPACE_NO_NEWLINE);
202 }
203}
204
205/* This is encoded like
206 * CommandByte <ARG#> <NUM> {Length of each string} {each string} */
207static void EmitWordList(StringBuilder &builder, const std::vector<std::string> &words)
208{
209 builder.PutUint8(static_cast<uint8_t>(words.size()));
210 for (size_t i = 0; i < words.size(); i++) {
211 size_t len = words[i].size();
212 if (len > UINT8_MAX) StrgenFatal("WordList {}/{} string '{}' too long, max bytes {}", i + 1, words.size(), words[i], UINT8_MAX);
213 builder.PutUint8(static_cast<uint8_t>(len));
214 }
215 for (size_t i = 0; i < words.size(); i++) {
216 builder.Put(words[i]);
217 }
218}
219
220void EmitPlural(StringBuilder &builder, std::string_view param, char32_t)
221{
222 StringConsumer consumer(param);
223
224 /* Parse out the number, if one exists. Otherwise default to prev arg. */
225 auto [argidx, offset] = ParseRelNum(consumer);
226 if (!argidx.has_value()) {
227 if (_cur_argidx == 0) StrgenFatal("Plural choice needs positional reference");
228 argidx = _cur_argidx - 1;
229 }
230
231 const CmdStruct *cmd = _cur_pcs.consuming_commands[*argidx];
232 if (!offset.has_value()) {
233 /* Use default offset */
234 if (cmd == nullptr || !cmd->default_plural_offset.has_value()) {
235 StrgenFatal("Command '{}' has no (default) plural position", cmd == nullptr ? "<empty>" : cmd->cmd);
236 }
237 offset = cmd->default_plural_offset;
238 }
239
240 /* Parse each string */
241 std::vector<std::string> words;
242 for (;;) {
243 auto word = ParseWord(consumer);
244 if (!word.has_value()) break;
245 words.emplace_back(*word);
246 }
247
248 if (words.empty()) {
249 StrgenFatal("{}: No plural words", _cur_ident);
250 }
251
252 size_t expected = _plural_forms[_strgen.lang.plural_form].plural_count;
253 if (expected != words.size()) {
254 if (_translated) {
255 StrgenFatal("{}: Invalid number of plural forms. Expecting {}, found {}.", _cur_ident,
256 expected, words.size());
257 } else {
258 if (_strgen.show_warnings) StrgenWarning("'{}' is untranslated. Tweaking english string to allow compilation for plural forms", _cur_ident);
259 if (words.size() > expected) {
260 words.resize(expected);
261 } else {
262 while (words.size() < expected) {
263 words.push_back(words.back());
264 }
265 }
266 }
267 }
268
269 builder.PutUtf8(SCC_PLURAL_LIST);
270 builder.PutUint8(_strgen.lang.plural_form);
271 builder.PutUint8(static_cast<uint8_t>(TranslateArgumentIdx(*argidx, *offset)));
272 EmitWordList(builder, words);
273}
274
275void EmitGender(StringBuilder &builder, std::string_view param, char32_t)
276{
277 StringConsumer consumer(param);
278 if (consumer.ReadCharIf('=')) {
279 /* This is a {G=DER} command */
280 auto gender = consumer.Read(StringConsumer::npos);
281 auto nw = _strgen.lang.GetGenderIndex(gender);
282 if (nw >= MAX_NUM_GENDERS) StrgenFatal("G argument '{}' invalid", gender);
283
284 /* now nw contains the gender index */
285 builder.PutUtf8(SCC_GENDER_INDEX);
286 builder.PutUint8(nw);
287 } else {
288 /* This is a {G 0 foo bar two} command.
289 * If no relative number exists, default to +0 */
290 auto [argidx, offset] = ParseRelNum(consumer);
291 if (!argidx.has_value()) argidx = _cur_argidx;
292 if (!offset.has_value()) offset = 0;
293
294 const CmdStruct *cmd = _cur_pcs.consuming_commands[*argidx];
295 if (cmd == nullptr || !cmd->flags.Test(CmdFlag::Gender)) {
296 StrgenFatal("Command '{}' can't have a gender", cmd == nullptr ? "<empty>" : cmd->cmd);
297 }
298
299 std::vector<std::string> words;
300 for (;;) {
301 auto word = ParseWord(consumer);
302 if (!word.has_value()) break;
303 words.emplace_back(*word);
304 }
305 if (words.size() != _strgen.lang.num_genders) StrgenFatal("Bad # of arguments for gender command");
306
307 assert(IsInsideBS(cmd->value, SCC_CONTROL_START, UINT8_MAX));
308 builder.PutUtf8(SCC_GENDER_LIST);
309 builder.PutUint8(static_cast<uint8_t>(TranslateArgumentIdx(*argidx, *offset)));
310 EmitWordList(builder, words);
311 }
312}
313
314static const CmdStruct *FindCmd(std::string_view s)
315{
316 auto it = std::ranges::find(_cmd_structs, s, &CmdStruct::cmd);
317 if (it != std::end(_cmd_structs)) return &*it;
318 return nullptr;
319}
320
321static uint8_t ResolveCaseName(std::string_view str)
322{
323 uint8_t case_idx = _strgen.lang.GetCaseIndex(str);
324 if (case_idx >= MAX_NUM_CASES) StrgenFatal("Invalid case-name '{}'", str);
325 return case_idx + 1;
326}
327
328/* returns cmd == nullptr on eof */
329static ParsedCommandString ParseCommandString(StringConsumer &consumer)
330{
331 ParsedCommandString result;
332
333 /* Scan to the next command, exit if there's no next command. */
334 consumer.SkipUntilChar('{', StringConsumer::KEEP_SEPARATOR);
335 if (!consumer.ReadCharIf('{')) return {};
336
337 if (auto argno = consumer.TryReadIntegerBase<uint32_t>(10); argno.has_value()) {
338 result.argno = argno;
339 if (!consumer.ReadCharIf(':')) StrgenFatal("missing arg #");
340 }
341
342 /* parse command name */
343 auto command = consumer.ReadUntilCharIn("} =.");
344 result.cmd = FindCmd(command);
345 if (result.cmd == nullptr) {
346 StrgenError("Undefined command '{}'", command);
347 return {};
348 }
349
350 /* parse case */
351 if (consumer.ReadCharIf('.')) {
352 if (!result.cmd->flags.Test(CmdFlag::Case)) {
353 StrgenFatal("Command '{}' can't have a case", result.cmd->cmd);
354 }
355
356 auto casep = consumer.ReadUntilCharIn("} ");
357 result.casei = ResolveCaseName(casep);
358 }
359
360 /* parse params */
361 result.param = consumer.ReadUntilChar('}', StringConsumer::KEEP_SEPARATOR);
362
363 if (!consumer.ReadCharIf('}')) {
364 StrgenError("Missing }} from command '{}'", result.cmd->cmd);
365 return {};
366 }
367
368 return result;
369}
370
378StringReader::StringReader(StringData &data, const std::string &file, bool master, bool translation) :
379 data(data), file(file), master(master), translation(translation)
380{
381}
382
383 ParsedCommandStruct ExtractCommandString(std::string_view s, bool)
384{
385 ParsedCommandStruct p;
386 StringConsumer consumer(s);
387
388 size_t argidx = 0;
389 for (;;) {
390 /* read until next command from a. */
391 auto cs = ParseCommandString(consumer);
392
393 if (cs.cmd == nullptr) break;
394
395 /* Sanity checking */
396 if (cs.argno.has_value() && cs.cmd->consumes == 0) StrgenFatal("Non consumer param can't have a paramindex");
397
398 if (cs.cmd->consumes > 0) {
399 if (cs.argno.has_value()) argidx = *cs.argno;
400 if (argidx >= p.consuming_commands.max_size()) StrgenFatal("invalid param idx {}", argidx);
401 if (p.consuming_commands[argidx] != nullptr && p.consuming_commands[argidx] != cs.cmd) StrgenFatal("duplicate param idx {}", argidx);
402
403 p.consuming_commands[argidx++] = cs.cmd;
404 } else if (!cs.cmd->flags.Test(CmdFlag::DontCount)) { // Ignore some of them
405 p.non_consuming_commands.emplace_back(cs.cmd, std::move(cs.param));
406 }
407 }
408
409 return p;
410}
411
412const CmdStruct *TranslateCmdForCompare(const CmdStruct *a)
413{
414 if (a == nullptr) return nullptr;
415
416 if (a->cmd == "STRING1" ||
417 a->cmd == "STRING2" ||
418 a->cmd == "STRING3" ||
419 a->cmd == "STRING4" ||
420 a->cmd == "STRING5" ||
421 a->cmd == "STRING6" ||
422 a->cmd == "STRING7" ||
423 a->cmd == "RAW_STRING") {
424 return FindCmd("STRING");
425 }
426
427 return a;
428}
429
430static bool CheckCommandsMatch(std::string_view a, std::string_view b, std::string_view name)
431{
432 /* If we're not translating, i.e. we're compiling the base language,
433 * it is pointless to do all these checks as it'll always be correct.
434 * After all, all checks are based on the base language.
435 */
436 if (!_strgen.translation) return true;
437
438 bool result = true;
439
440 ParsedCommandStruct templ = ExtractCommandString(b, true);
441 ParsedCommandStruct lang = ExtractCommandString(a, true);
442
443 /* For each string in templ, see if we find it in lang */
444 if (templ.non_consuming_commands.max_size() != lang.non_consuming_commands.max_size()) {
445 StrgenWarning("{}: template string and language string have a different # of commands", name);
446 result = false;
447 }
448
449 for (auto &templ_nc : templ.non_consuming_commands) {
450 /* see if we find it in lang, and zero it out */
451 auto it = std::ranges::find(lang.non_consuming_commands, templ_nc);
452 if (it != std::end(lang.non_consuming_commands)) {
453 /* it was found in both. zero it out from lang so we don't find it again */
454 it->cmd = nullptr;
455 } else {
456 StrgenWarning("{}: command '{}' exists in template file but not in language file", name, templ_nc.cmd->cmd);
457 result = false;
458 }
459 }
460
461 /* if we reach here, all non consumer commands match up.
462 * Check if the non consumer commands match up also. */
463 for (size_t i = 0; i < templ.consuming_commands.max_size(); i++) {
464 if (TranslateCmdForCompare(templ.consuming_commands[i]) != lang.consuming_commands[i]) {
465 StrgenWarning("{}: Param idx #{} '{}' doesn't match with template command '{}'", name, i,
466 lang.consuming_commands[i] == nullptr ? "<empty>" : TranslateCmdForCompare(lang.consuming_commands[i])->cmd,
467 templ.consuming_commands[i] == nullptr ? "<empty>" : templ.consuming_commands[i]->cmd);
468 result = false;
469 }
470 }
471
472 return result;
473}
474
475void StringReader::HandleString(std::string_view src)
476{
477 /* Ignore blank lines */
478 if (src.empty()) return;
479
480 StringConsumer consumer(src);
481 if (consumer.ReadCharIf('#')) {
482 if (consumer.ReadCharIf('#') && !consumer.ReadCharIf('#')) this->HandlePragma(consumer.Read(StringConsumer::npos), _strgen.lang);
483 return; // ignore comments
484 }
485
486 /* Read string name */
487 std::string_view str_name = StrTrimView(consumer.ReadUntilChar(':', StringConsumer::KEEP_SEPARATOR), StringConsumer::WHITESPACE_NO_NEWLINE);
488 if (!consumer.ReadCharIf(':')) {
489 StrgenError("Line has no ':' delimiter");
490 return;
491 }
492
493 /* Read string case */
494 std::optional<std::string_view> casep;
495 if (auto index = str_name.find("."); index != std::string_view::npos) {
496 casep = str_name.substr(index + 1);
497 str_name = str_name.substr(0, index);
498 }
499
500 /* Read string data */
501 std::string_view value = consumer.Read(StringConsumer::npos);
502
503 /* Check string is valid UTF-8 */
504 for (StringConsumer validation_consumer(value); validation_consumer.AnyBytesLeft(); ) {
505 auto c = validation_consumer.TryReadUtf8();
506 if (!c.has_value()) StrgenFatal("Invalid UTF-8 sequence in '{}'", value);
507 if (*c <= 0x001F || // ASCII control character range
508 *c == 0x200B || // Zero width space
509 (*c >= 0xE000 && *c <= 0xF8FF) || // Private range
510 (*c >= 0xFFF0 && *c <= 0xFFFF)) { // Specials range
511 StrgenFatal("Unwanted UTF-8 character U+{:04X} in sequence '{}'", static_cast<uint32_t>(*c), value);
512 }
513 }
514
515 /* Check if this string already exists.. */
516 LangString *ent = this->data.Find(str_name);
517
518 if (this->master) {
519 if (casep.has_value()) {
520 StrgenError("Cases in the base translation are not supported.");
521 return;
522 }
523
524 if (ent != nullptr) {
525 StrgenError("String name '{}' is used multiple times", str_name);
526 return;
527 }
528
529 if (this->data.strings[this->data.next_string_id] != nullptr) {
530 StrgenError("String ID 0x{:X} for '{}' already in use by '{}'", this->data.next_string_id, str_name, this->data.strings[this->data.next_string_id]->name);
531 return;
532 }
533
534 /* Allocate a new LangString */
535 this->data.Add(std::make_unique<LangString>(str_name, value, this->data.next_string_id++, _strgen.cur_line));
536 } else {
537 if (ent == nullptr) {
538 StrgenWarning("String name '{}' does not exist in master file", str_name);
539 return;
540 }
541
542 if (!ent->translated.empty() && !casep.has_value()) {
543 StrgenError("String name '{}' is used multiple times", str_name);
544 return;
545 }
546
547 /* make sure that the commands match */
548 if (!CheckCommandsMatch(value, ent->english, str_name)) return;
549
550 if (casep.has_value()) {
551 ent->translated_cases.emplace_back(ResolveCaseName(*casep), value);
552 } else {
553 ent->translated = value;
554 /* If the string was translated, use the line from the
555 * translated language so errors in the translated file
556 * are properly referenced to. */
557 ent->line = _strgen.cur_line;
558 }
559 }
560}
561
562 void StringReader::HandlePragma(std::string_view str, LanguagePackHeader &lang)
563{
564 StringConsumer consumer(str);
565 auto name = consumer.ReadUntilChar(' ', StringConsumer::SKIP_ALL_SEPARATORS);
566 if (name == "plural") {
567 lang.plural_form = consumer.ReadIntegerBase<uint32_t>(10);
568 if (lang.plural_form >= lengthof(_plural_forms)) {
569 StrgenFatal("Invalid pluralform {}", lang.plural_form);
570 }
571 } else {
572 StrgenFatal("unknown pragma '{}'", name);
573 }
574}
575
576 void StringReader::ParseFile()
577{
578 _strgen.warnings = _strgen.errors = 0;
579
580 _strgen.translation = this->translation;
581 _strgen.file = this->file;
582
583 /* For each new file we parse, reset the genders, and language codes. */
584 _strgen.lang = {};
585
586 _strgen.cur_line = 1;
587 while (this->data.next_string_id < this->data.max_strings) {
588 std::optional<std::string> line = this->ReadLine();
589 if (!line.has_value()) return;
590
591 this->HandleString(StrTrimView(line.value(), StringConsumer::WHITESPACE_OR_NEWLINE));
592 _strgen.cur_line++;
593 }
594
595 if (this->data.next_string_id == this->data.max_strings) {
596 StrgenError("Too many strings, maximum allowed is {}", this->data.max_strings);
597 }
598}
599
604 void HeaderWriter::WriteHeader(const StringData &data)
605{
606 size_t last = 0;
607 for (size_t i = 0; i < data.max_strings; i++) {
608 if (data.strings[i] != nullptr) {
609 this->WriteStringID(data.strings[i]->name, i);
610 last = i;
611 }
612 }
613
614 this->WriteStringID("STR_LAST_STRINGID", last);
615}
616
617static size_t TranslateArgumentIdx(size_t argidx, size_t offset)
618{
619 if (argidx >= _cur_pcs.consuming_commands.max_size()) {
620 StrgenFatal("invalid argidx {}", argidx);
621 }
622 const CmdStruct *cs = _cur_pcs.consuming_commands[argidx];
623 if (cs != nullptr && cs->consumes <= offset) {
624 StrgenFatal("invalid argidx offset {}:{}", argidx, offset);
625 }
626
627 if (_cur_pcs.consuming_commands[argidx] == nullptr) {
628 StrgenFatal("no command for this argidx {}", argidx);
629 }
630
631 size_t sum = 0;
632 for (size_t i = 0; i < argidx; i++) {
633 cs = _cur_pcs.consuming_commands[i];
634
635 if (cs == nullptr && sum > i) continue;
636
637 sum += (cs != nullptr) ? cs->consumes : 1;
638 }
639
640 return sum + offset;
641}
642
643static void PutArgidxCommand(StringBuilder &builder)
644{
645 builder.PutUtf8(SCC_ARG_INDEX);
646 builder.PutUint8(static_cast<uint8_t>(TranslateArgumentIdx(_cur_argidx)));
647}
648
649static std::string PutCommandString(std::string_view str)
650{
651 std::string result;
652 StringBuilder builder(result);
653 StringConsumer consumer(str);
654 _cur_argidx = 0;
655
656 for (;;) {
657 /* Process characters as they are until we encounter a { */
658 builder.Put(consumer.ReadUntilChar('{', StringConsumer::KEEP_SEPARATOR));
659 if (!consumer.AnyBytesLeft()) break;
660
661 auto cs = ParseCommandString(consumer);
662 auto *cmd = cs.cmd;
663 if (cmd == nullptr) break;
664
665 if (cs.casei.has_value()) {
666 builder.PutUtf8(SCC_SET_CASE); // {SET_CASE}
667 builder.PutUint8(*cs.casei);
668 }
669
670 /* For params that consume values, we need to handle the argindex properly */
671 if (cmd->consumes > 0) {
672 /* Check if we need to output a move-param command */
673 if (cs.argno.has_value() && *cs.argno != _cur_argidx) {
674 _cur_argidx = *cs.argno;
675 PutArgidxCommand(builder);
676 }
677
678 /* Output the one from the master string... it's always accurate. */
679 cmd = _cur_pcs.consuming_commands[_cur_argidx++];
680 if (cmd == nullptr) {
681 StrgenFatal("{}: No argument exists at position {}", _cur_ident, _cur_argidx - 1);
682 }
683 }
684
685 cmd->proc(builder, cs.param, cmd->value);
686 }
687 return result;
688}
689
694 void LanguageWriter::WriteLength(size_t length)
695{
696 char buffer[2];
697 size_t offs = 0;
698 if (length >= 0x4000) {
699 StrgenFatal("string too long");
700 }
701
702 if (length >= 0xC0) {
703 buffer[offs++] = static_cast<char>(static_cast<uint8_t>((length >> 8) | 0xC0));
704 }
705 buffer[offs++] = static_cast<char>(static_cast<uint8_t>(length & 0xFF));
706 this->Write({buffer, offs});
707}
708
713 void LanguageWriter::WriteLang(const StringData &data)
714{
715 std::vector<size_t> in_use;
716 for (size_t tab = 0; tab < data.tabs; tab++) {
717 size_t n = data.CountInUse(tab);
718
719 in_use.push_back(n);
720 _strgen.lang.offsets[tab] = TO_LE16(static_cast<uint16_t>(n));
721
722 for (size_t j = 0; j != in_use[tab]; j++) {
723 const LangString *ls = data.strings[(tab * TAB_SIZE) + j].get();
724 if (ls != nullptr && ls->translated.empty()) _strgen.lang.missing++;
725 }
726 }
727
728 _strgen.lang.ident = TO_LE32(LanguagePackHeader::IDENT);
729 _strgen.lang.version = TO_LE32(data.Version());
730 _strgen.lang.missing = TO_LE16(_strgen.lang.missing);
731 _strgen.lang.winlangid = TO_LE16(_strgen.lang.winlangid);
732
733 this->WriteHeader(&_strgen.lang);
734
735 for (size_t tab = 0; tab < data.tabs; tab++) {
736 for (size_t j = 0; j != in_use[tab]; j++) {
737 const LangString *ls = data.strings[(tab * TAB_SIZE) + j].get();
738
739 /* For undefined strings, just set that it's an empty string */
740 if (ls == nullptr) {
741 this->WriteLength(0);
742 continue;
743 }
744
745 std::string output;
746 StringBuilder builder(output);
747 _cur_ident = ls->name;
748 _strgen.cur_line = ls->line;
749
750 /* Produce a message if a string doesn't have a translation. */
751 if (ls->translated.empty()) {
752 if (_strgen.show_warnings) {
753 StrgenWarning("'{}' is untranslated", ls->name);
754 }
755 if (_strgen.annotate_todos) {
756 builder.Put("<TODO> ");
757 }
758 }
759
760 /* Extract the strings and stuff from the english command string */
761 _cur_pcs = ExtractCommandString(ls->english, false);
762
763 _translated = !ls->translated_cases.empty() || !ls->translated.empty();
764 const std::string &cmdp = _translated ? ls->translated : ls->english;
765
766 if (!ls->translated_cases.empty()) {
767 /* Need to output a case-switch.
768 * It has this format
769 * <0x9E> <NUM CASES> <CASE1> <LEN1> <STRING1> <CASE2> <LEN2> <STRING2> <CASE3> <LEN3> <STRING3> <LENDEFAULT> <STRINGDEFAULT>
770 * Each LEN is printed using 2 bytes in little endian order. */
771 builder.PutUtf8(SCC_SWITCH_CASE);
772 builder.PutUint8(static_cast<uint8_t>(ls->translated_cases.size()));
773
774 /* Write each case */
775 for (const Case &c : ls->translated_cases) {
776 auto case_str = PutCommandString(c.string);
777 builder.PutUint8(c.caseidx);
778 builder.PutUint16LE(static_cast<uint16_t>(case_str.size()));
779 builder.Put(case_str);
780 }
781 }
782
783 std::string def_str;
784 if (!cmdp.empty()) def_str = PutCommandString(cmdp);
785 if (!ls->translated_cases.empty()) {
786 builder.PutUint16LE(static_cast<uint16_t>(def_str.size()));
787 }
788 builder.Put(def_str);
789
790 this->WriteLength(output.size());
791 this->Write(output);
792 }
793 }
794}
BaseBitSet::Test
constexpr bool Test(Tvalue_type value) const
Test if the value-th bit is set.
Definition base_bitset_type.hpp:141
BaseStringBuilder::PutUtf8
void PutUtf8(char32_t c)
Append UTF.8 char.
Definition string_builder.cpp:113
BaseStringBuilder::PutUint16LE
void PutUint16LE(uint16_t value)
Append binary uint16 using little endian.
Definition string_builder.cpp:37
BaseStringBuilder::Put
void Put(std::string_view str)
Append string.
Definition string_builder.hpp:34
BaseStringBuilder::PutUint8
void PutUint8(uint8_t value)
Append binary uint8.
Definition string_builder.cpp:18
StringBuilder
Compose data into a growing std::string.
Definition string_builder.hpp:65
StringConsumer
Parse data from a string / buffer.
Definition string_consumer.hpp:27
StringConsumer::ReadCharIf
bool ReadCharIf(char c)
Check whether the next 8-bit char matches 'c', and skip it.
Definition string_consumer.hpp:478
StringConsumer::TryReadIntegerBase
std::optional< T > TryReadIntegerBase(int base, bool clamp=false)
Try to read and parse an integer in number 'base', and then advance the reader.
Definition string_consumer.hpp:874
StringConsumer::ReadUntilChar
std::string_view ReadUntilChar(char c, SeparatorUsage sep)
Read data until the first occurrence of 8-bit char 'c', and advance reader.
Definition string_consumer.hpp:765
StringConsumer::SkipUntilChar
void SkipUntilChar(char c, SeparatorUsage sep)
Skip data until the first occurrence of 8-bit char 'c'.
Definition string_consumer.hpp:774
StringConsumer::SKIP_ALL_SEPARATORS
@ SKIP_ALL_SEPARATORS
Read and discard all consecutive separators, do not include any in the result.
Definition string_consumer.hpp:697
StringConsumer::KEEP_SEPARATOR
@ KEEP_SEPARATOR
Keep the separator in the data as next value to be read.
Definition string_consumer.hpp:695
StringConsumer::AnyBytesLeft
bool AnyBytesLeft() const noexcept
Check whether any bytes left to read.
Definition string_consumer.hpp:70
StringConsumer::WHITESPACE_OR_NEWLINE
static const std::string_view WHITESPACE_OR_NEWLINE
ASCII whitespace characters, including new-line.
Definition string_consumer.hpp:45
StringConsumer::WHITESPACE_NO_NEWLINE
static const std::string_view WHITESPACE_NO_NEWLINE
ASCII whitespace characters, excluding new-line.
Definition string_consumer.hpp:40
StringConsumer::SkipUntilCharNotIn
void SkipUntilCharNotIn(std::string_view chars)
Skip 8-bit chars, while they are in 'chars', until they are not.
Definition string_consumer.hpp:683
StringConsumer::ReadUntilCharIn
std::string_view ReadUntilCharIn(std::string_view chars)
Read 8-bit chars, while they are not in 'chars', until they are; and advance reader.
Definition string_consumer.hpp:648
StringConsumer::ReadIntegerBase
T ReadIntegerBase(int base, T def=0, bool clamp=false)
Read and parse an integer in number 'base', and advance the reader.
Definition string_consumer.hpp:891
StringConsumer::Read
std::string_view Read(size_type len)
Read the next 'len' bytes, and advance reader.
Definition string_consumer.hpp:530
StringConsumer::npos
static constexpr size_type npos
Special value for "end of data".
Definition string_consumer.hpp:34
MAX_NUM_GENDERS
static const uint8_t MAX_NUM_GENDERS
Maximum number of supported genders.
Definition language.h:20
MAX_NUM_CASES
static const uint8_t MAX_NUM_CASES
Maximum number of supported cases.
Definition language.h:21
IsInsideBS
constexpr bool IsInsideBS(const T x, const size_t base, const size_t size)
Checks if a value is between a window started at some base point.
Definition math_func.hpp:252
lengthof
#define lengthof(array)
Return the length of an fixed size array.
Definition stdafx.h:271
strgen.h
Structures related to strgen.
_translated
static bool _translated
Whether the current language is not the master language.
Definition strgen_base.cpp:25
VersionHashStr
static uint32_t VersionHashStr(uint32_t hash, std::string_view s)
Create a compound hash.
Definition strgen_base.cpp:117
_plural_forms
static const PluralForm _plural_forms[]
All plural forms used.
Definition strgen_tables.h:174
CmdFlag::Gender
@ Gender
These commands support genders.
CmdFlag::Case
@ Case
These commands support cases.
CmdFlag::DontCount
@ DontCount
These commands aren't counted for comparison.
TAB_SIZE
static const uint TAB_SIZE
Number of strings per StringTab.
Definition strings_type.h:52
Case
Container for the different cases of a string.
Definition strgen.h:22
Case::Case
Case(uint8_t caseidx, std::string_view string)
Create a new case.
Definition strgen_base.cpp:44
CmdStruct
Definition strgen_tables.h:22
HeaderWriter::WriteStringID
virtual void WriteStringID(const std::string &name, size_t stringid)=0
Write the string ID.
HeaderWriter::WriteHeader
void WriteHeader(const StringData &data)
Write the header information.
Definition strgen_base.cpp:604
LangString
Information about a single string.
Definition strgen.h:30
LangString::line
size_t line
Line of string in source-file.
Definition strgen.h:35
LangString::english
std::string english
English text.
Definition strgen.h:32
LangString::translated_cases
std::vector< Case > translated_cases
Cases of the translation.
Definition strgen.h:36
LangString::translated
std::string translated
Translated text.
Definition strgen.h:33
LangString::FreeTranslation
void FreeTranslation()
Free all data related to the translation.
Definition strgen_base.cpp:62
LangString::name
std::string name
Name of the string.
Definition strgen.h:31
LangString::LangString
LangString(std::string_view name, std::string_view english, size_t index, size_t line)
Create a new string.
Definition strgen_base.cpp:56
LanguagePackHeader
Header of a language file.
Definition language.h:24
LanguagePackHeader::GetCaseIndex
uint8_t GetCaseIndex(std::string_view case_str) const
Get the index for the given case.
Definition language.h:79
LanguagePackHeader::plural_form
uint8_t plural_form
plural form index
Definition language.h:41
LanguagePackHeader::version
uint32_t version
32-bits of auto generated version info which is basically a hash of strings.h
Definition language.h:28
LanguagePackHeader::offsets
uint16_t offsets[TEXT_TAB_END]
the offsets
Definition language.h:32
LanguagePackHeader::winlangid
uint16_t winlangid
Windows language ID: Windows cannot and will not convert isocodes to something it can use to determin...
Definition language.h:51
LanguagePackHeader::num_genders
uint8_t num_genders
the number of genders of this language
Definition language.h:53
LanguagePackHeader::missing
uint16_t missing
number of missing strings.
Definition language.h:40
LanguagePackHeader::ident
uint32_t ident
32-bits identifier
Definition language.h:27
LanguagePackHeader::IDENT
static const uint32_t IDENT
Identifier for OpenTTD language files, big endian for "LANG".
Definition language.h:25
LanguagePackHeader::GetGenderIndex
uint8_t GetGenderIndex(std::string_view gender_str) const
Get the index for the given gender.
Definition language.h:68
LanguageWriter::WriteHeader
virtual void WriteHeader(const LanguagePackHeader *header)=0
Write the header metadata.
LanguageWriter::WriteLength
virtual void WriteLength(size_t length)
Write the length as a simple gamma.
Definition strgen_base.cpp:694
LanguageWriter::Write
virtual void Write(std::string_view buffer)=0
Write a number of bytes.
LanguageWriter::WriteLang
virtual void WriteLang(const StringData &data)
Actually write the language.
Definition strgen_base.cpp:713
ParsedCommandString
Definition strgen_base.cpp:30
ParsedCommandStruct
Definition strgen.h:144
PluralForm::plural_count
size_t plural_count
The number of plural forms.
Definition strgen_tables.h:165
StrgenState
Global state shared between strgen.cpp, game_text.cpp and strgen_base.cpp.
Definition strgen.h:161
StrgenState::file
std::string file
The filename of the input, so we can refer to it in errors/warnings.
Definition strgen.h:162
StrgenState::translation
bool translation
Is the current file actually a translation or not.
Definition strgen.h:168
StrgenState::lang
LanguagePackHeader lang
Header information about a language.
Definition strgen.h:169
StrgenState::cur_line
size_t cur_line
The current line we're parsing in the input file.
Definition strgen.h:163
StringData
Information about the currently known strings.
Definition strgen.h:43
StringData::tabs
size_t tabs
The number of 'tabs' of strings.
Definition strgen.h:46
StringData::Add
void Add(std::shared_ptr< LangString > ls)
Add a newly created LangString.
Definition strgen_base.cpp:92
StringData::max_strings
size_t max_strings
The maximum number of strings.
Definition strgen.h:47
StringData::next_string_id
size_t next_string_id
The next string ID to allocate.
Definition strgen.h:48
StringData::FreeTranslation
void FreeTranslation()
Free all data related to the translation.
Definition strgen_base.cpp:79
StringData::Find
LangString * Find(std::string_view s)
Find a LangString based on the string name.
Definition strgen_base.cpp:103
StringData::StringData
StringData(size_t tabs)
Create a new string data container.
Definition strgen_base.cpp:72
StringData::name_to_string
std::unordered_map< std::string, std::shared_ptr< LangString >, StringHash, std::equal_to<> > name_to_string
Lookup table for the strings.
Definition strgen.h:45
StringData::strings
std::vector< std::shared_ptr< LangString > > strings
List of all known strings.
Definition strgen.h:44
StringData::Version
uint32_t Version() const
Make a hash of the file to get a unique "version number".
Definition strgen_base.cpp:130
StringData::CountInUse
size_t CountInUse(size_t tab) const
Count the number of tab elements that are in use.
Definition strgen_base.cpp:160
StringReader::file
const std::string file
The file we are reading.
Definition strgen.h:61
StringReader::StringReader
StringReader(StringData &data, const std::string &file, bool master, bool translation)
Prepare reading.
Definition strgen_base.cpp:378
StringReader::data
StringData & data
The data to fill during reading.
Definition strgen.h:60
StringReader::ParseFile
virtual void ParseFile()
Start parsing the file.
Definition strgen_base.cpp:576
StringReader::translation
bool translation
Are we reading a translation, implies !master. However, the base translation will have this false.
Definition strgen.h:63
StringReader::HandlePragma
virtual void HandlePragma(std::string_view str, LanguagePackHeader &lang)
Handle the pragma of the file.
Definition strgen_base.cpp:562
StringReader::ReadLine
virtual std::optional< std::string > ReadLine()=0
Read a single line from the source of strings.
StringReader::master
bool master
Are we reading the master file?
Definition strgen.h:62