GenerateUnicodeLocale.cpp 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473
  1. /*
  2. * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <AK/AllOf.h>
  7. #include <AK/CharacterTypes.h>
  8. #include <AK/Format.h>
  9. #include <AK/HashMap.h>
  10. #include <AK/JsonObject.h>
  11. #include <AK/JsonParser.h>
  12. #include <AK/JsonValue.h>
  13. #include <AK/LexicalPath.h>
  14. #include <AK/QuickSort.h>
  15. #include <AK/SourceGenerator.h>
  16. #include <AK/String.h>
  17. #include <AK/StringBuilder.h>
  18. #include <LibCore/ArgsParser.h>
  19. #include <LibCore/DirIterator.h>
  20. #include <LibCore/File.h>
  21. struct Locale {
  22. String language;
  23. Optional<String> territory;
  24. Optional<String> variant;
  25. HashMap<String, String> languages;
  26. HashMap<String, String> territories;
  27. HashMap<String, String> scripts;
  28. };
  29. struct UnicodeLocaleData {
  30. HashMap<String, Locale> locales;
  31. Vector<String> languages;
  32. Vector<String> territories;
  33. Vector<String> scripts;
  34. Vector<String> variants;
  35. };
  36. static void write_to_file_if_different(Core::File& file, StringView contents)
  37. {
  38. auto const current_contents = file.read_all();
  39. if (StringView { current_contents.bytes() } == contents)
  40. return;
  41. VERIFY(file.seek(0));
  42. VERIFY(file.truncate(0));
  43. VERIFY(file.write(contents));
  44. }
  45. static void parse_identity(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  46. {
  47. LexicalPath languages_path(move(locale_path)); // Note: Every JSON file defines identity data, so we can use any of them.
  48. languages_path = languages_path.append("languages.json"sv);
  49. VERIFY(Core::File::exists(languages_path.string()));
  50. auto languages_file_or_error = Core::File::open(languages_path.string(), Core::OpenMode::ReadOnly);
  51. VERIFY(!languages_file_or_error.is_error());
  52. auto languages = JsonParser(languages_file_or_error.value()->read_all()).parse();
  53. VERIFY(languages.has_value());
  54. auto const& main_object = languages->as_object().get("main"sv);
  55. auto const& locale_object = main_object.as_object().get(languages_path.parent().basename());
  56. auto const& identity_object = locale_object.as_object().get("identity"sv);
  57. auto const& language_string = identity_object.as_object().get("language"sv);
  58. auto const& territory_string = identity_object.as_object().get("territory"sv);
  59. auto const& variant_string = identity_object.as_object().get("variant"sv);
  60. locale.language = language_string.as_string();
  61. if (!locale_data.languages.contains_slow(locale.language))
  62. locale_data.languages.append(locale.language);
  63. if (territory_string.is_string()) {
  64. locale.territory = territory_string.as_string();
  65. if (!locale_data.territories.contains_slow(*locale.territory))
  66. locale_data.territories.append(*locale.territory);
  67. }
  68. if (variant_string.is_string()) {
  69. locale.variant = variant_string.as_string();
  70. if (!locale_data.variants.contains_slow(*locale.variant))
  71. locale_data.variants.append(*locale.variant);
  72. }
  73. }
  74. static void parse_locale_languages(String locale_path, Locale& locale)
  75. {
  76. LexicalPath languages_path(move(locale_path));
  77. languages_path = languages_path.append("languages.json"sv);
  78. VERIFY(Core::File::exists(languages_path.string()));
  79. auto languages_file_or_error = Core::File::open(languages_path.string(), Core::OpenMode::ReadOnly);
  80. VERIFY(!languages_file_or_error.is_error());
  81. auto languages = JsonParser(languages_file_or_error.value()->read_all()).parse();
  82. VERIFY(languages.has_value());
  83. auto const& main_object = languages->as_object().get("main"sv);
  84. auto const& locale_object = main_object.as_object().get(languages_path.parent().basename());
  85. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  86. auto const& languages_object = locale_display_names_object.as_object().get("languages"sv);
  87. languages_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  88. locale.languages.set(key, value.as_string());
  89. });
  90. }
  91. static void parse_locale_territories(String locale_path, Locale& locale)
  92. {
  93. LexicalPath territories_path(move(locale_path));
  94. territories_path = territories_path.append("territories.json"sv);
  95. VERIFY(Core::File::exists(territories_path.string()));
  96. auto territories_file_or_error = Core::File::open(territories_path.string(), Core::OpenMode::ReadOnly);
  97. VERIFY(!territories_file_or_error.is_error());
  98. auto territories = JsonParser(territories_file_or_error.value()->read_all()).parse();
  99. VERIFY(territories.has_value());
  100. auto const& main_object = territories->as_object().get("main"sv);
  101. auto const& locale_object = main_object.as_object().get(territories_path.parent().basename());
  102. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  103. auto const& territories_object = locale_display_names_object.as_object().get("territories"sv);
  104. territories_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  105. locale.territories.set(key, value.as_string());
  106. });
  107. }
  108. static void parse_locale_scripts(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  109. {
  110. LexicalPath scripts_path(move(locale_path));
  111. scripts_path = scripts_path.append("scripts.json"sv);
  112. VERIFY(Core::File::exists(scripts_path.string()));
  113. auto scripts_file_or_error = Core::File::open(scripts_path.string(), Core::OpenMode::ReadOnly);
  114. VERIFY(!scripts_file_or_error.is_error());
  115. auto scripts = JsonParser(scripts_file_or_error.value()->read_all()).parse();
  116. VERIFY(scripts.has_value());
  117. auto const& main_object = scripts->as_object().get("main"sv);
  118. auto const& locale_object = main_object.as_object().get(scripts_path.parent().basename());
  119. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  120. auto const& scripts_object = locale_display_names_object.as_object().get("scripts"sv);
  121. scripts_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  122. locale.scripts.set(key, value.as_string());
  123. if (!locale_data.scripts.contains_slow(key))
  124. locale_data.scripts.append(key);
  125. });
  126. }
  127. static void parse_all_locales(String locale_names_path, UnicodeLocaleData& locale_data)
  128. {
  129. LexicalPath locale_names(move(locale_names_path));
  130. locale_names = locale_names.append("main"sv);
  131. VERIFY(Core::File::is_directory(locale_names.string()));
  132. Core::DirIterator iterator(locale_names.string(), Core::DirIterator::SkipParentAndBaseDir);
  133. if (iterator.has_error()) {
  134. warnln("{}: {}", locale_names.string(), iterator.error_string());
  135. VERIFY_NOT_REACHED();
  136. }
  137. while (iterator.has_next()) {
  138. auto locale_path = iterator.next_full_path();
  139. VERIFY(Core::File::is_directory(locale_path));
  140. auto& locale = locale_data.locales.ensure(LexicalPath::basename(locale_path));
  141. parse_identity(locale_path, locale_data, locale);
  142. parse_locale_languages(locale_path, locale);
  143. parse_locale_territories(locale_path, locale);
  144. parse_locale_scripts(locale_path, locale_data, locale);
  145. }
  146. }
  147. static String format_identifier(StringView owner, String identifier)
  148. {
  149. identifier.replace("-"sv, "_"sv, true);
  150. if (all_of(identifier, is_ascii_digit))
  151. return String::formatted("{}_{}", owner[0], identifier);
  152. return identifier.to_titlecase();
  153. }
  154. static void generate_unicode_locale_header(Core::File& file, UnicodeLocaleData& locale_data)
  155. {
  156. StringBuilder builder;
  157. SourceGenerator generator { builder };
  158. auto generate_enum = [&](StringView name, StringView default_, Vector<String>& values) {
  159. quick_sort(values);
  160. generator.set("name", name);
  161. generator.set("underlying", ((values.size() + !default_.is_empty()) < 256) ? "u8"sv : "u16"sv);
  162. generator.append(R"~~~(
  163. enum class @name@ : @underlying@ {)~~~");
  164. if (!default_.is_empty()) {
  165. generator.set("default", default_);
  166. generator.append(R"~~~(
  167. @default@,)~~~");
  168. }
  169. for (auto const& value : values) {
  170. generator.set("value", format_identifier(name, value));
  171. generator.append(R"~~~(
  172. @value@,)~~~");
  173. }
  174. generator.append(R"~~~(
  175. };
  176. )~~~");
  177. };
  178. generator.append(R"~~~(
  179. #pragma once
  180. #include <AK/Optional.h>
  181. #include <AK/StringView.h>
  182. #include <AK/Types.h>
  183. #include <LibUnicode/Forward.h>
  184. namespace Unicode {
  185. )~~~");
  186. auto locales = locale_data.locales.keys();
  187. generate_enum("Locale"sv, "None"sv, locales);
  188. generate_enum("Language"sv, {}, locale_data.languages);
  189. generate_enum("Territory"sv, {}, locale_data.territories);
  190. generate_enum("ScriptTag"sv, {}, locale_data.scripts);
  191. generate_enum("Variant"sv, {}, locale_data.variants);
  192. generator.append(R"~~~(
  193. namespace Detail {
  194. Optional<Locale> locale_from_string(StringView const& locale);
  195. Optional<StringView> get_locale_language_mapping(StringView locale, StringView language);
  196. Optional<Language> language_from_string(StringView const& language);
  197. Optional<StringView> get_locale_territory_mapping(StringView locale, StringView territory);
  198. Optional<Territory> territory_from_string(StringView const& territory);
  199. Optional<StringView> get_locale_script_tag_mapping(StringView locale, StringView script_tag);
  200. Optional<ScriptTag> script_tag_from_string(StringView const& script_tag);
  201. }
  202. }
  203. )~~~");
  204. write_to_file_if_different(file, generator.as_string_view());
  205. }
  206. static void generate_unicode_locale_implementation(Core::File& file, UnicodeLocaleData& locale_data)
  207. {
  208. StringBuilder builder;
  209. SourceGenerator generator { builder };
  210. generator.set("locales_size"sv, String::number(locale_data.locales.size()));
  211. generator.set("territories_size", String::number(locale_data.territories.size()));
  212. generator.append(R"~~~(
  213. #include <AK/Array.h>
  214. #include <AK/HashMap.h>
  215. #include <AK/Span.h>
  216. #include <LibUnicode/UnicodeLocale.h>
  217. namespace Unicode {
  218. )~~~");
  219. auto format_mapping_name = [](StringView format, StringView name) {
  220. auto mapping_name = name.to_lowercase_string();
  221. mapping_name.replace("-"sv, "_"sv, true);
  222. return String::formatted(format, mapping_name);
  223. };
  224. auto append_mapping_list = [&](String name, auto const& keys, auto const& mappings) {
  225. generator.set("name", name);
  226. generator.set("size", String::number(keys.size()));
  227. generator.append(R"~~~(
  228. static constexpr Array<StringView, @size@> @name@ { {
  229. )~~~");
  230. constexpr size_t max_values_per_row = 10;
  231. size_t values_in_current_row = 0;
  232. for (auto const& key : keys) {
  233. if (values_in_current_row++ > 0)
  234. generator.append(" ");
  235. if (auto it = mappings.find(key); it != mappings.end())
  236. generator.set("mapping"sv, String::formatted("\"{}\"sv", it->value));
  237. else
  238. generator.set("mapping"sv, "{}"sv);
  239. generator.append("@mapping@,");
  240. if (values_in_current_row == max_values_per_row) {
  241. values_in_current_row = 0;
  242. generator.append("\n ");
  243. }
  244. }
  245. generator.append(R"~~~(
  246. } };
  247. )~~~");
  248. };
  249. auto append_mapping = [&](StringView name, StringView format, auto const& keys, auto get_mapping_callback) {
  250. Vector<String> mapping_names;
  251. for (auto const& locale : locale_data.locales) {
  252. auto mapping_name = format_mapping_name(format, locale.key);
  253. append_mapping_list(mapping_name, keys, get_mapping_callback(locale.value));
  254. mapping_names.append(move(mapping_name));
  255. }
  256. quick_sort(mapping_names);
  257. generator.set("name", name);
  258. generator.set("size", String::number(locale_data.locales.size()));
  259. generator.append(R"~~~(
  260. static constexpr Array<Span<StringView const>, @size@> @name@ { {
  261. )~~~");
  262. constexpr size_t max_values_per_row = 10;
  263. size_t values_in_current_row = 0;
  264. for (auto& mapping_name : mapping_names) {
  265. if (values_in_current_row++ > 0)
  266. generator.append(" ");
  267. generator.set("name", move(mapping_name));
  268. generator.append("@name@.span(),");
  269. if (values_in_current_row == max_values_per_row) {
  270. values_in_current_row = 0;
  271. generator.append("\n ");
  272. }
  273. }
  274. generator.append(R"~~~(
  275. } };
  276. )~~~");
  277. };
  278. append_mapping("s_languages"sv, "s_languages_{}", locale_data.languages, [](auto const& value) { return value.languages; });
  279. append_mapping("s_territories"sv, "s_territories_{}", locale_data.territories, [](auto const& value) { return value.territories; });
  280. append_mapping("s_scripts"sv, "s_scripts_{}", locale_data.scripts, [](auto const& value) { return value.scripts; });
  281. generator.append(R"~~~(
  282. namespace Detail {
  283. )~~~");
  284. auto append_mapping_search = [&](StringView enum_title, StringView enum_snake, StringView collection_name) {
  285. generator.set("enum_title", enum_title);
  286. generator.set("enum_snake", enum_snake);
  287. generator.set("collection_name", collection_name);
  288. generator.append(R"~~~(
  289. Optional<StringView> get_locale_@enum_snake@_mapping(StringView locale, StringView @enum_snake@)
  290. {
  291. auto locale_value = locale_from_string(locale);
  292. if (!locale_value.has_value())
  293. return {};
  294. auto @enum_snake@_value = @enum_snake@_from_string(@enum_snake@);
  295. if (!@enum_snake@_value.has_value())
  296. return {};
  297. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  298. auto @enum_snake@_index = to_underlying(*@enum_snake@_value);
  299. auto const& mappings = @collection_name@.at(locale_index);
  300. auto @enum_snake@_mapping = mappings.at(@enum_snake@_index);
  301. if (@enum_snake@_mapping.is_empty())
  302. return {};
  303. return @enum_snake@_mapping;
  304. }
  305. )~~~");
  306. };
  307. auto append_from_string = [&](StringView enum_title, StringView enum_snake, Vector<String> const& values) {
  308. generator.set("enum_title", enum_title);
  309. generator.set("enum_snake", enum_snake);
  310. generator.append(R"~~~(
  311. Optional<@enum_title@> @enum_snake@_from_string(StringView const& @enum_snake@)
  312. {
  313. static HashMap<StringView, @enum_title@> @enum_snake@_values { {)~~~");
  314. for (auto const& value : values) {
  315. generator.set("key"sv, value);
  316. generator.set("value"sv, format_identifier(enum_title, value));
  317. generator.append(R"~~~(
  318. { "@key@"sv, @enum_title@::@value@ },)~~~");
  319. }
  320. generator.append(R"~~~(
  321. } };
  322. if (auto value = @enum_snake@_values.get(@enum_snake@); value.has_value())
  323. return value.value();
  324. return {};
  325. }
  326. )~~~");
  327. };
  328. append_from_string("Locale"sv, "locale"sv, locale_data.locales.keys());
  329. append_mapping_search("Language"sv, "language"sv, "s_languages"sv);
  330. append_from_string("Language"sv, "language"sv, locale_data.languages);
  331. append_mapping_search("Territory"sv, "territory"sv, "s_territories"sv);
  332. append_from_string("Territory"sv, "territory"sv, locale_data.territories);
  333. append_mapping_search("ScriptTag"sv, "script_tag"sv, "s_scripts"sv);
  334. append_from_string("ScriptTag"sv, "script_tag"sv, locale_data.scripts);
  335. generator.append(R"~~~(
  336. }
  337. }
  338. )~~~");
  339. write_to_file_if_different(file, generator.as_string_view());
  340. }
  341. int main(int argc, char** argv)
  342. {
  343. char const* generated_header_path = nullptr;
  344. char const* generated_implementation_path = nullptr;
  345. char const* locale_names_path = nullptr;
  346. Core::ArgsParser args_parser;
  347. args_parser.add_option(generated_header_path, "Path to the Unicode locale header file to generate", "generated-header-path", 'h', "generated-header-path");
  348. args_parser.add_option(generated_implementation_path, "Path to the Unicode locale implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  349. args_parser.add_option(locale_names_path, "Path to cldr-localenames directory", "locale-names-path", 'l', "locale-names-path");
  350. args_parser.parse(argc, argv);
  351. auto open_file = [&](StringView path, StringView flags, Core::OpenMode mode = Core::OpenMode::ReadOnly) {
  352. if (path.is_empty()) {
  353. warnln("{} is required", flags);
  354. args_parser.print_usage(stderr, argv[0]);
  355. exit(1);
  356. }
  357. auto file_or_error = Core::File::open(path, mode);
  358. if (file_or_error.is_error()) {
  359. warnln("Failed to open {}: {}", path, file_or_error.release_error());
  360. exit(1);
  361. }
  362. return file_or_error.release_value();
  363. };
  364. auto generated_header_file = open_file(generated_header_path, "-h/--generated-header-path", Core::OpenMode::ReadWrite);
  365. auto generated_implementation_file = open_file(generated_implementation_path, "-c/--generated-implementation-path", Core::OpenMode::ReadWrite);
  366. UnicodeLocaleData locale_data;
  367. parse_all_locales(locale_names_path, locale_data);
  368. generate_unicode_locale_header(generated_header_file, locale_data);
  369. generate_unicode_locale_implementation(generated_implementation_file, locale_data);
  370. return 0;
  371. }