GenerateUnicodeDateTimeFormat.cpp 79 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999
  1. /*
  2. * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AllOf.h>
  8. #include <AK/CharacterTypes.h>
  9. #include <AK/Format.h>
  10. #include <AK/GenericLexer.h>
  11. #include <AK/HashFunctions.h>
  12. #include <AK/HashMap.h>
  13. #include <AK/JsonObject.h>
  14. #include <AK/JsonParser.h>
  15. #include <AK/JsonValue.h>
  16. #include <AK/LexicalPath.h>
  17. #include <AK/SourceGenerator.h>
  18. #include <AK/String.h>
  19. #include <AK/StringBuilder.h>
  20. #include <AK/Traits.h>
  21. #include <AK/Utf8View.h>
  22. #include <LibCore/ArgsParser.h>
  23. #include <LibCore/DirIterator.h>
  24. #include <LibCore/File.h>
  25. #include <LibUnicode/DateTimeFormat.h>
  26. using StringIndexType = u16;
  27. constexpr auto s_string_index_type = "u16"sv;
  28. using CalendarPatternIndexType = u16;
  29. constexpr auto s_calendar_pattern_index_type = "u16"sv;
  30. using CalendarRangePatternIndexType = u16;
  31. constexpr auto s_calendar_range_pattern_index_type = "u16"sv;
  32. using CalendarFormatIndexType = u8;
  33. constexpr auto s_calendar_format_index_type = "u8"sv;
  34. using SymbolListIndexType = u16;
  35. constexpr auto s_symbol_list_index_type = "u16"sv;
  36. using CalendarSymbolsIndexType = u16;
  37. constexpr auto s_calendar_symbols_index_type = "u16"sv;
  38. struct CalendarPattern : public Unicode::CalendarPattern {
  39. bool contains_only_date_fields() const
  40. {
  41. return !day_period.has_value() && !hour.has_value() && !minute.has_value() && !second.has_value() && !fractional_second_digits.has_value() && !time_zone_name.has_value();
  42. }
  43. bool contains_only_time_fields() const
  44. {
  45. return !weekday.has_value() && !era.has_value() && !year.has_value() && !month.has_value() && !day.has_value();
  46. }
  47. unsigned hash() const
  48. {
  49. auto hash = pair_int_hash(pattern_index, pattern12_index);
  50. hash = pair_int_hash(hash, skeleton_index);
  51. auto hash_field = [&](auto const& field) {
  52. if (field.has_value())
  53. hash = pair_int_hash(hash, static_cast<u8>(*field));
  54. else
  55. hash = pair_int_hash(hash, -1);
  56. };
  57. hash_field(era);
  58. hash_field(year);
  59. hash_field(month);
  60. hash_field(weekday);
  61. hash_field(day);
  62. hash_field(day_period);
  63. hash_field(hour);
  64. hash_field(minute);
  65. hash_field(second);
  66. hash_field(fractional_second_digits);
  67. hash_field(time_zone_name);
  68. return hash;
  69. }
  70. bool operator==(CalendarPattern const& other) const
  71. {
  72. return (skeleton_index == other.skeleton_index)
  73. && (pattern_index == other.pattern_index)
  74. && (pattern12_index == other.pattern12_index)
  75. && (era == other.era)
  76. && (year == other.year)
  77. && (month == other.month)
  78. && (weekday == other.weekday)
  79. && (day == other.day)
  80. && (day_period == other.day_period)
  81. && (hour == other.hour)
  82. && (minute == other.minute)
  83. && (second == other.second)
  84. && (fractional_second_digits == other.fractional_second_digits)
  85. && (time_zone_name == other.time_zone_name);
  86. }
  87. StringIndexType skeleton_index { 0 };
  88. StringIndexType pattern_index { 0 };
  89. StringIndexType pattern12_index { 0 };
  90. };
  91. template<>
  92. struct AK::Formatter<CalendarPattern> : Formatter<FormatString> {
  93. ErrorOr<void> format(FormatBuilder& builder, CalendarPattern const& pattern)
  94. {
  95. auto field_to_i8 = [](auto const& field) -> i8 {
  96. if (!field.has_value())
  97. return -1;
  98. return static_cast<i8>(*field);
  99. };
  100. return Formatter<FormatString>::format(builder,
  101. "{{ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {} }}",
  102. pattern.skeleton_index,
  103. pattern.pattern_index,
  104. pattern.pattern12_index,
  105. field_to_i8(pattern.era),
  106. field_to_i8(pattern.year),
  107. field_to_i8(pattern.month),
  108. field_to_i8(pattern.weekday),
  109. field_to_i8(pattern.day),
  110. field_to_i8(pattern.day_period),
  111. field_to_i8(pattern.hour),
  112. field_to_i8(pattern.minute),
  113. field_to_i8(pattern.second),
  114. field_to_i8(pattern.fractional_second_digits),
  115. field_to_i8(pattern.time_zone_name));
  116. }
  117. };
  118. template<>
  119. struct AK::Traits<CalendarPattern> : public GenericTraits<CalendarPattern> {
  120. static unsigned hash(CalendarPattern const& c) { return c.hash(); }
  121. };
  122. struct CalendarRangePattern : public CalendarPattern {
  123. unsigned hash() const
  124. {
  125. auto hash = CalendarPattern::hash();
  126. if (field.has_value())
  127. hash = pair_int_hash(hash, static_cast<u8>(*field));
  128. hash = pair_int_hash(hash, start_range);
  129. hash = pair_int_hash(hash, separator);
  130. hash = pair_int_hash(hash, end_range);
  131. return hash;
  132. }
  133. bool operator==(CalendarRangePattern const& other) const
  134. {
  135. if (!CalendarPattern::operator==(other))
  136. return false;
  137. return (field == other.field)
  138. && (start_range == other.start_range)
  139. && (separator == other.separator)
  140. && (end_range == other.end_range);
  141. }
  142. Optional<Unicode::CalendarRangePattern::Field> field {};
  143. StringIndexType start_range { 0 };
  144. StringIndexType separator { 0 };
  145. StringIndexType end_range { 0 };
  146. };
  147. template<>
  148. struct AK::Formatter<CalendarRangePattern> : Formatter<FormatString> {
  149. ErrorOr<void> format(FormatBuilder& builder, CalendarRangePattern const& pattern)
  150. {
  151. auto field_to_i8 = [](auto const& field) -> i8 {
  152. if (!field.has_value())
  153. return -1;
  154. return static_cast<i8>(*field);
  155. };
  156. return Formatter<FormatString>::format(builder,
  157. "{{ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {} }}",
  158. pattern.skeleton_index,
  159. field_to_i8(pattern.field),
  160. pattern.start_range,
  161. pattern.separator,
  162. pattern.end_range,
  163. field_to_i8(pattern.era),
  164. field_to_i8(pattern.year),
  165. field_to_i8(pattern.month),
  166. field_to_i8(pattern.weekday),
  167. field_to_i8(pattern.day),
  168. field_to_i8(pattern.day_period),
  169. field_to_i8(pattern.hour),
  170. field_to_i8(pattern.minute),
  171. field_to_i8(pattern.second),
  172. field_to_i8(pattern.fractional_second_digits),
  173. field_to_i8(pattern.time_zone_name));
  174. }
  175. };
  176. template<>
  177. struct AK::Traits<CalendarRangePattern> : public GenericTraits<CalendarRangePattern> {
  178. static unsigned hash(CalendarRangePattern const& c) { return c.hash(); }
  179. };
  180. struct CalendarFormat {
  181. unsigned hash() const
  182. {
  183. auto hash = pair_int_hash(full_format, long_format);
  184. hash = pair_int_hash(hash, medium_format);
  185. hash = pair_int_hash(hash, short_format);
  186. return hash;
  187. }
  188. bool operator==(CalendarFormat const& other) const
  189. {
  190. return (full_format == other.full_format)
  191. && (long_format == other.long_format)
  192. && (medium_format == other.medium_format)
  193. && (short_format == other.short_format);
  194. }
  195. CalendarPatternIndexType full_format { 0 };
  196. CalendarPatternIndexType long_format { 0 };
  197. CalendarPatternIndexType medium_format { 0 };
  198. CalendarPatternIndexType short_format { 0 };
  199. };
  200. template<>
  201. struct AK::Formatter<CalendarFormat> : Formatter<FormatString> {
  202. ErrorOr<void> format(FormatBuilder& builder, CalendarFormat const& pattern)
  203. {
  204. return Formatter<FormatString>::format(builder,
  205. "{{ {}, {}, {}, {} }}",
  206. pattern.full_format,
  207. pattern.long_format,
  208. pattern.medium_format,
  209. pattern.short_format);
  210. }
  211. };
  212. template<>
  213. struct AK::Traits<CalendarFormat> : public GenericTraits<CalendarFormat> {
  214. static unsigned hash(CalendarFormat const& c) { return c.hash(); }
  215. };
  216. using SymbolList = Vector<StringIndexType>;
  217. struct CalendarSymbols {
  218. unsigned hash() const
  219. {
  220. auto hash = pair_int_hash(narrow_symbols, short_symbols);
  221. hash = pair_int_hash(hash, long_symbols);
  222. return hash;
  223. }
  224. bool operator==(CalendarSymbols const& other) const
  225. {
  226. return (narrow_symbols == other.narrow_symbols)
  227. && (short_symbols == other.short_symbols)
  228. && (long_symbols == other.long_symbols);
  229. }
  230. SymbolListIndexType narrow_symbols { 0 };
  231. SymbolListIndexType short_symbols { 0 };
  232. SymbolListIndexType long_symbols { 0 };
  233. };
  234. template<>
  235. struct AK::Formatter<CalendarSymbols> : Formatter<FormatString> {
  236. ErrorOr<void> format(FormatBuilder& builder, CalendarSymbols const& symbols)
  237. {
  238. return Formatter<FormatString>::format(builder,
  239. "{{ {}, {}, {} }}",
  240. symbols.narrow_symbols,
  241. symbols.short_symbols,
  242. symbols.long_symbols);
  243. }
  244. };
  245. template<>
  246. struct AK::Traits<CalendarSymbols> : public GenericTraits<CalendarSymbols> {
  247. static unsigned hash(CalendarSymbols const& c) { return c.hash(); }
  248. };
  249. struct Calendar {
  250. StringIndexType calendar { 0 };
  251. CalendarFormatIndexType date_formats { 0 };
  252. CalendarFormatIndexType time_formats { 0 };
  253. CalendarFormatIndexType date_time_formats { 0 };
  254. Vector<CalendarPatternIndexType> available_formats {};
  255. CalendarRangePatternIndexType default_range_format { 0 };
  256. Vector<CalendarRangePatternIndexType> range_formats {};
  257. Vector<CalendarRangePatternIndexType> range12_formats {};
  258. Vector<CalendarSymbolsIndexType> symbols {};
  259. };
  260. struct TimeZone {
  261. StringIndexType time_zone { 0 };
  262. StringIndexType long_name { 0 };
  263. StringIndexType short_name { 0 };
  264. };
  265. struct DayPeriod {
  266. Unicode::DayPeriod day_period {};
  267. u8 begin { 0 };
  268. u8 end { 0 };
  269. };
  270. struct Locale {
  271. HashMap<String, Calendar> calendars;
  272. HashMap<String, TimeZone> time_zones;
  273. Vector<DayPeriod> day_periods;
  274. };
  275. struct UnicodeLocaleData {
  276. UniqueStringStorage<StringIndexType> unique_strings;
  277. UniqueStorage<CalendarPattern, CalendarPatternIndexType> unique_patterns;
  278. UniqueStorage<CalendarRangePattern, CalendarRangePatternIndexType> unique_range_patterns;
  279. UniqueStorage<CalendarFormat, CalendarFormatIndexType> unique_formats;
  280. UniqueStorage<SymbolList, SymbolListIndexType> unique_symbol_lists;
  281. UniqueStorage<CalendarSymbols, CalendarSymbolsIndexType> unique_calendar_symbols;
  282. HashMap<String, Locale> locales;
  283. HashMap<String, Vector<Unicode::HourCycle>> hour_cycles;
  284. Vector<String> hour_cycle_regions;
  285. HashMap<String, StringIndexType> meta_zones;
  286. Vector<String> time_zones;
  287. Vector<String> calendars;
  288. Vector<Alias> calendar_aliases {
  289. // FIXME: Aliases should come from BCP47. See: https://unicode-org.atlassian.net/browse/CLDR-15158
  290. { "gregorian"sv, "gregory"sv },
  291. };
  292. };
  293. static Optional<Unicode::DayPeriod> day_period_from_string(StringView day_period)
  294. {
  295. if (day_period == "am"sv)
  296. return Unicode::DayPeriod::AM;
  297. if (day_period == "pm"sv)
  298. return Unicode::DayPeriod::PM;
  299. if (day_period == "morning1"sv)
  300. return Unicode::DayPeriod::Morning1;
  301. if (day_period == "morning2"sv)
  302. return Unicode::DayPeriod::Morning2;
  303. if (day_period == "afternoon1"sv)
  304. return Unicode::DayPeriod::Afternoon1;
  305. if (day_period == "afternoon2"sv)
  306. return Unicode::DayPeriod::Afternoon2;
  307. if (day_period == "evening1"sv)
  308. return Unicode::DayPeriod::Evening1;
  309. if (day_period == "evening2"sv)
  310. return Unicode::DayPeriod::Evening2;
  311. if (day_period == "night1"sv)
  312. return Unicode::DayPeriod::Night1;
  313. if (day_period == "night2"sv)
  314. return Unicode::DayPeriod::Night2;
  315. return {};
  316. };
  317. static ErrorOr<void> parse_hour_cycles(String core_path, UnicodeLocaleData& locale_data)
  318. {
  319. // https://unicode.org/reports/tr35/tr35-dates.html#Time_Data
  320. LexicalPath time_data_path(move(core_path));
  321. time_data_path = time_data_path.append("supplemental"sv);
  322. time_data_path = time_data_path.append("timeData.json"sv);
  323. auto time_data_file = TRY(Core::File::open(time_data_path.string(), Core::OpenMode::ReadOnly));
  324. auto time_data = TRY(JsonValue::from_string(time_data_file->read_all()));
  325. auto const& supplemental_object = time_data.as_object().get("supplemental"sv);
  326. auto const& time_data_object = supplemental_object.as_object().get("timeData"sv);
  327. auto parse_hour_cycle = [](StringView hour_cycle) -> Optional<Unicode::HourCycle> {
  328. if (hour_cycle == "h"sv)
  329. return Unicode::HourCycle::H12;
  330. if (hour_cycle == "H"sv)
  331. return Unicode::HourCycle::H23;
  332. if (hour_cycle == "K"sv)
  333. return Unicode::HourCycle::H11;
  334. if (hour_cycle == "k"sv)
  335. return Unicode::HourCycle::H24;
  336. return {};
  337. };
  338. time_data_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  339. auto allowed_hour_cycles_string = value.as_object().get("_allowed"sv).as_string();
  340. auto allowed_hour_cycles = allowed_hour_cycles_string.split_view(' ');
  341. Vector<Unicode::HourCycle> hour_cycles;
  342. for (auto allowed_hour_cycle : allowed_hour_cycles) {
  343. if (auto hour_cycle = parse_hour_cycle(allowed_hour_cycle); hour_cycle.has_value())
  344. hour_cycles.append(*hour_cycle);
  345. }
  346. locale_data.hour_cycles.set(key, move(hour_cycles));
  347. if (!locale_data.hour_cycle_regions.contains_slow(key))
  348. locale_data.hour_cycle_regions.append(key);
  349. });
  350. return {};
  351. };
  352. static ErrorOr<void> parse_meta_zones(String core_path, UnicodeLocaleData& locale_data)
  353. {
  354. // https://unicode.org/reports/tr35/tr35-dates.html#Metazones
  355. LexicalPath meta_zone_path(move(core_path));
  356. meta_zone_path = meta_zone_path.append("supplemental"sv);
  357. meta_zone_path = meta_zone_path.append("metaZones.json"sv);
  358. auto meta_zone_file = TRY(Core::File::open(meta_zone_path.string(), Core::OpenMode::ReadOnly));
  359. auto meta_zone = TRY(JsonValue::from_string(meta_zone_file->read_all()));
  360. auto const& supplemental_object = meta_zone.as_object().get("supplemental"sv);
  361. auto const& meta_zone_object = supplemental_object.as_object().get("metaZones"sv);
  362. auto const& meta_zone_array = meta_zone_object.as_object().get("metazones"sv);
  363. meta_zone_array.as_array().for_each([&](JsonValue const& value) {
  364. auto const& mapping = value.as_object().get("mapZone"sv);
  365. auto const& meta_zone = mapping.as_object().get("_other"sv);
  366. auto const& golden_zone = mapping.as_object().get("_type"sv);
  367. auto golden_zone_index = locale_data.unique_strings.ensure(golden_zone.as_string());
  368. locale_data.meta_zones.set(meta_zone.as_string(), golden_zone_index);
  369. });
  370. // UTC does not appear in metaZones.json. Define it for convenience so other parsers don't need to check for its existence.
  371. locale_data.meta_zones.set("UTC"sv, locale_data.unique_strings.ensure("UTC"sv));
  372. return {};
  373. };
  374. static constexpr auto is_char(char ch)
  375. {
  376. return [ch](auto c) { return c == ch; };
  377. }
  378. // For patterns that are 12-hour aware, we need to generate two patterns: one with the day period
  379. // (e.g. {ampm}) in the pattern, and one without the day period. We need to take care to remove
  380. // extra spaces around the day period. Some example expected removals:
  381. //
  382. // "{hour}:{minute} {ampm}" becomes "{hour}:{minute}" (remove the space before {ampm})
  383. // "{ampm} {hour}" becomes "{hour}" (remove the space after {ampm})
  384. // "{hour}:{minute} {ampm} {timeZoneName}" becomes "{hour}:{minute} {timeZoneName}" (remove one of the spaces around {ampm})
  385. static String remove_period_from_pattern(String pattern)
  386. {
  387. for (auto remove : AK::Array { "({ampm})"sv, "{ampm}"sv, "({dayPeriod})"sv, "{dayPeriod}"sv }) {
  388. auto index = pattern.find(remove);
  389. if (!index.has_value())
  390. continue;
  391. constexpr u32 space = ' ';
  392. constexpr u32 open = '{';
  393. constexpr u32 close = '}';
  394. Utf8View utf8_pattern { pattern };
  395. Optional<u32> before_removal;
  396. Optional<u32> after_removal;
  397. for (auto it = utf8_pattern.begin(); utf8_pattern.byte_offset_of(it) < *index; ++it)
  398. before_removal = *it;
  399. if (auto it = utf8_pattern.iterator_at_byte_offset(*index + remove.length()); it != utf8_pattern.end())
  400. after_removal = *it;
  401. if ((before_removal == space) && (after_removal != open)) {
  402. pattern = String::formatted("{}{}",
  403. pattern.substring_view(0, *index - 1),
  404. pattern.substring_view(*index + remove.length()));
  405. } else if ((after_removal == space) && (before_removal != close)) {
  406. pattern = String::formatted("{}{}",
  407. pattern.substring_view(0, *index),
  408. pattern.substring_view(*index + remove.length() + 1));
  409. } else {
  410. pattern = String::formatted("{}{}",
  411. pattern.substring_view(0, *index),
  412. pattern.substring_view(*index + remove.length()));
  413. }
  414. }
  415. return pattern;
  416. }
  417. static Optional<CalendarPattern> parse_date_time_pattern_raw(String pattern, String skeleton, UnicodeLocaleData& locale_data)
  418. {
  419. // https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table
  420. using Unicode::CalendarPatternStyle;
  421. CalendarPattern format {};
  422. if (!skeleton.is_empty())
  423. format.skeleton_index = locale_data.unique_strings.ensure(move(skeleton));
  424. GenericLexer lexer { pattern };
  425. StringBuilder builder;
  426. bool hour12 { false };
  427. while (!lexer.is_eof()) {
  428. // Literal strings enclosed by quotes are to be appended to the pattern as-is without further
  429. // processing (this just avoids conflicts with the patterns below).
  430. if (lexer.next_is(is_quote)) {
  431. builder.append(lexer.consume_quoted_string());
  432. continue;
  433. }
  434. auto starting_char = lexer.peek();
  435. auto segment = lexer.consume_while([&](char ch) { return ch == starting_char; });
  436. // Era
  437. if (all_of(segment, is_char('G'))) {
  438. builder.append("{era}");
  439. if (segment.length() <= 3)
  440. format.era = CalendarPatternStyle::Short;
  441. else if (segment.length() == 4)
  442. format.era = CalendarPatternStyle::Long;
  443. else
  444. format.era = CalendarPatternStyle::Narrow;
  445. }
  446. // Year
  447. else if (all_of(segment, is_any_of("yYuUr"sv))) {
  448. builder.append("{year}");
  449. if (segment.length() == 2)
  450. format.year = CalendarPatternStyle::TwoDigit;
  451. else
  452. format.year = CalendarPatternStyle::Numeric;
  453. }
  454. // Quarter
  455. else if (all_of(segment, is_any_of("qQ"sv))) {
  456. // Intl.DateTimeFormat does not support quarter formatting, so drop these patterns.
  457. return {};
  458. }
  459. // Month
  460. else if (all_of(segment, is_any_of("ML"sv))) {
  461. builder.append("{month}");
  462. if (segment.length() == 1)
  463. format.month = CalendarPatternStyle::Numeric;
  464. else if (segment.length() == 2)
  465. format.month = CalendarPatternStyle::TwoDigit;
  466. else if (segment.length() == 3)
  467. format.month = CalendarPatternStyle::Short;
  468. else if (segment.length() == 4)
  469. format.month = CalendarPatternStyle::Long;
  470. else if (segment.length() == 5)
  471. format.month = CalendarPatternStyle::Narrow;
  472. } else if (all_of(segment, is_char('l'))) {
  473. // Using 'l' for month formatting is deprecated by TR-35, ensure it is not used.
  474. return {};
  475. }
  476. // Week
  477. else if (all_of(segment, is_any_of("wW"sv))) {
  478. // Intl.DateTimeFormat does not support week formatting, so drop these patterns.
  479. return {};
  480. }
  481. // Day
  482. else if (all_of(segment, is_char('d'))) {
  483. builder.append("{day}");
  484. if (segment.length() == 1)
  485. format.day = CalendarPatternStyle::Numeric;
  486. else
  487. format.day = CalendarPatternStyle::TwoDigit;
  488. } else if (all_of(segment, is_any_of("DFg"sv))) {
  489. builder.append("{day}");
  490. format.day = CalendarPatternStyle::Numeric;
  491. }
  492. // Weekday
  493. else if (all_of(segment, is_char('E'))) {
  494. builder.append("{weekday}");
  495. if (segment.length() == 4)
  496. format.weekday = CalendarPatternStyle::Long;
  497. else if (segment.length() == 5)
  498. format.weekday = CalendarPatternStyle::Narrow;
  499. else
  500. format.weekday = CalendarPatternStyle::Short;
  501. } else if (all_of(segment, is_any_of("ec"sv))) {
  502. builder.append("{weekday}");
  503. // TR-35 defines "e", "c", and "cc" as as numeric, and "ee" as 2-digit, but those
  504. // pattern styles are not supported by Intl.DateTimeFormat.
  505. if (segment.length() <= 2)
  506. return {};
  507. if (segment.length() == 4)
  508. format.weekday = CalendarPatternStyle::Long;
  509. else if (segment.length() == 5)
  510. format.weekday = CalendarPatternStyle::Narrow;
  511. else
  512. format.weekday = CalendarPatternStyle::Short;
  513. }
  514. // Period
  515. else if (all_of(segment, is_any_of("ab"sv))) {
  516. builder.append("{ampm}");
  517. hour12 = true;
  518. } else if (all_of(segment, is_char('B'))) {
  519. builder.append("{dayPeriod}");
  520. hour12 = true;
  521. if (segment.length() == 4)
  522. format.day_period = CalendarPatternStyle::Long;
  523. else if (segment.length() == 5)
  524. format.day_period = CalendarPatternStyle::Narrow;
  525. else
  526. format.day_period = CalendarPatternStyle::Short;
  527. }
  528. // Hour
  529. else if (all_of(segment, is_any_of("hHKk"sv))) {
  530. builder.append("{hour}");
  531. if ((segment[0] == 'h') || (segment[0] == 'K'))
  532. hour12 = true;
  533. if (segment.length() == 1)
  534. format.hour = CalendarPatternStyle::Numeric;
  535. else
  536. format.hour = CalendarPatternStyle::TwoDigit;
  537. } else if (all_of(segment, is_any_of("jJC"sv))) {
  538. // TR-35 indicates these should not be used.
  539. return {};
  540. }
  541. // Minute
  542. else if (all_of(segment, is_char('m'))) {
  543. builder.append("{minute}");
  544. if (segment.length() == 1)
  545. format.minute = CalendarPatternStyle::Numeric;
  546. else
  547. format.minute = CalendarPatternStyle::TwoDigit;
  548. }
  549. // Second
  550. else if (all_of(segment, is_char('s'))) {
  551. builder.append("{second}");
  552. if (segment.length() == 1)
  553. format.second = CalendarPatternStyle::Numeric;
  554. else
  555. format.second = CalendarPatternStyle::TwoDigit;
  556. } else if (all_of(segment, is_char('S'))) {
  557. builder.append("{fractionalSecondDigits}");
  558. VERIFY(segment.length() <= 3);
  559. format.fractional_second_digits = static_cast<u8>(segment.length());
  560. } else if (all_of(segment, is_char('A'))) {
  561. // Intl.DateTimeFormat does not support millisecond formatting, so drop these patterns.
  562. return {};
  563. }
  564. // Zone
  565. else if (all_of(segment, is_any_of("zZOvVXx"))) {
  566. builder.append("{timeZoneName}");
  567. if (segment.length() < 4)
  568. format.time_zone_name = CalendarPatternStyle::Short;
  569. else
  570. format.time_zone_name = CalendarPatternStyle::Long;
  571. }
  572. // Non-patterns
  573. else {
  574. builder.append(segment);
  575. }
  576. }
  577. pattern = builder.build();
  578. if (hour12) {
  579. format.pattern = remove_period_from_pattern(pattern);
  580. format.pattern12 = move(pattern);
  581. } else {
  582. format.pattern = move(pattern);
  583. }
  584. return format;
  585. }
  586. static Optional<CalendarPatternIndexType> parse_date_time_pattern(String pattern, String skeleton, UnicodeLocaleData& locale_data)
  587. {
  588. auto format = parse_date_time_pattern_raw(move(pattern), move(skeleton), locale_data);
  589. if (!format.has_value())
  590. return {};
  591. format->pattern_index = locale_data.unique_strings.ensure(move(format->pattern));
  592. if (format->pattern12.has_value())
  593. format->pattern12_index = locale_data.unique_strings.ensure(format->pattern12.release_value());
  594. return locale_data.unique_patterns.ensure(format.release_value());
  595. }
  596. template<typename... Chars>
  597. static constexpr bool char_is_one_of(char ch, Chars&&... chars)
  598. {
  599. return ((ch == chars) || ...);
  600. }
  601. static void parse_interval_patterns(Calendar& calendar, JsonObject const& interval_formats_object, UnicodeLocaleData& locale_data)
  602. {
  603. // https://unicode.org/reports/tr35/tr35-dates.html#intervalFormats
  604. auto name_of_field = [&](char field) {
  605. if (char_is_one_of(field, 'G'))
  606. return Unicode::CalendarRangePattern::Field::Era;
  607. if (char_is_one_of(field, 'y', 'Y', 'u', 'U', 'r'))
  608. return Unicode::CalendarRangePattern::Field::Year;
  609. if (char_is_one_of(field, 'M', 'L'))
  610. return Unicode::CalendarRangePattern::Field::Month;
  611. if (char_is_one_of(field, 'd', 'D', 'F', 'g'))
  612. return Unicode::CalendarRangePattern::Field::Day;
  613. if (char_is_one_of(field, 'a', 'b'))
  614. return Unicode::CalendarRangePattern::Field::AmPm;
  615. if (char_is_one_of(field, 'B'))
  616. return Unicode::CalendarRangePattern::Field::DayPeriod;
  617. if (char_is_one_of(field, 'h', 'H', 'K', 'k'))
  618. return Unicode::CalendarRangePattern::Field::Hour;
  619. if (char_is_one_of(field, 'm'))
  620. return Unicode::CalendarRangePattern::Field::Minute;
  621. if (char_is_one_of(field, 's'))
  622. return Unicode::CalendarRangePattern::Field::Second;
  623. if (char_is_one_of(field, 'S'))
  624. return Unicode::CalendarRangePattern::Field::FractionalSecondDigits;
  625. VERIFY_NOT_REACHED();
  626. };
  627. auto split_default_range_pattern = [&](auto skeleton, auto const& pattern) {
  628. auto start_range_end = pattern.find('}').value() + 1;
  629. auto end_range_begin = pattern.find_last('{').value();
  630. auto start_range = pattern.substring_view(0, start_range_end);
  631. auto separator = pattern.substring_view(start_range_end, end_range_begin - start_range_end);
  632. auto end_range = pattern.substring_view(end_range_begin);
  633. CalendarRangePattern format {};
  634. format.skeleton_index = locale_data.unique_strings.ensure(skeleton);
  635. format.start_range = locale_data.unique_strings.ensure(start_range);
  636. format.separator = locale_data.unique_strings.ensure(separator);
  637. format.end_range = locale_data.unique_strings.ensure(end_range);
  638. return format;
  639. };
  640. auto split_range_pattern = [&](auto skeleton, auto field, auto const& pattern, auto const& parsed_fields) {
  641. HashMap<StringView, size_t> partitions;
  642. StringView last_partition;
  643. auto begin_index = pattern.find('{');
  644. size_t end_index = 0;
  645. while (begin_index.has_value()) {
  646. end_index = pattern.find('}', *begin_index).value();
  647. auto partition = pattern.substring_view(*begin_index, end_index - *begin_index);
  648. if (partitions.contains(partition))
  649. break;
  650. partitions.set(partition, *begin_index);
  651. last_partition = partition;
  652. begin_index = pattern.find('{', end_index + 1);
  653. }
  654. VERIFY(begin_index.has_value() && !last_partition.is_empty());
  655. auto start_range_end = partitions.get(last_partition).value() + last_partition.length() + 1;
  656. auto start_range = pattern.substring_view(0, start_range_end);
  657. auto separator = pattern.substring_view(start_range_end, *begin_index - start_range_end);
  658. auto end_range = pattern.substring_view(*begin_index);
  659. CalendarRangePattern format {};
  660. format.skeleton_index = locale_data.unique_strings.ensure(skeleton);
  661. format.field = field;
  662. format.start_range = locale_data.unique_strings.ensure(start_range);
  663. format.separator = locale_data.unique_strings.ensure(separator);
  664. format.end_range = locale_data.unique_strings.ensure(end_range);
  665. format.for_each_calendar_field_zipped_with(parsed_fields, [](auto& format_field, auto const& parsed_field, auto) {
  666. format_field = parsed_field;
  667. });
  668. return format;
  669. };
  670. interval_formats_object.for_each_member([&](auto const& skeleton, auto const& value) {
  671. if (skeleton == "intervalFormatFallback"sv) {
  672. auto range_format = split_default_range_pattern(skeleton, value.as_string());
  673. calendar.default_range_format = locale_data.unique_range_patterns.ensure(move(range_format));
  674. return;
  675. }
  676. value.as_object().for_each_member([&](auto const& field, auto const& pattern) {
  677. if (field.ends_with("alt-variant"sv))
  678. return;
  679. VERIFY(field.length() == 1);
  680. auto name = name_of_field(field[0]);
  681. auto format = parse_date_time_pattern_raw(pattern.as_string(), skeleton, locale_data).release_value();
  682. auto range_format = split_range_pattern(skeleton, name, format.pattern, format);
  683. calendar.range_formats.append(locale_data.unique_range_patterns.ensure(move(range_format)));
  684. if (format.pattern12.has_value()) {
  685. auto range12_pattern = split_range_pattern(skeleton, name, *format.pattern12, format);
  686. calendar.range12_formats.append(locale_data.unique_range_patterns.ensure(move(range12_pattern)));
  687. } else {
  688. calendar.range12_formats.append(calendar.range_formats.last());
  689. }
  690. });
  691. });
  692. }
  693. static void generate_missing_patterns(Calendar& calendar, Vector<CalendarPattern> date_formats, Vector<CalendarPattern> time_formats, UnicodeLocaleData& locale_data)
  694. {
  695. // https://unicode.org/reports/tr35/tr35-dates.html#Missing_Skeleton_Fields
  696. auto replace_pattern = [&](auto format, auto time_format, auto date_format) {
  697. auto pattern = locale_data.unique_strings.get(format);
  698. auto time_pattern = locale_data.unique_strings.get(time_format);
  699. auto date_pattern = locale_data.unique_strings.get(date_format);
  700. auto new_pattern = pattern.replace("{0}", time_pattern).replace("{1}", date_pattern);
  701. return locale_data.unique_strings.ensure(move(new_pattern));
  702. };
  703. auto inject_fractional_second_digits = [&](auto format) {
  704. auto pattern = locale_data.unique_strings.get(format);
  705. auto new_pattern = pattern.replace("{second}"sv, "{second}{decimal}{fractionalSecondDigits}"sv);
  706. return locale_data.unique_strings.ensure(move(new_pattern));
  707. };
  708. auto append_if_unique = [&](auto format) {
  709. auto format_index = locale_data.unique_patterns.ensure(move(format));
  710. if (!calendar.available_formats.contains_slow(format_index))
  711. calendar.available_formats.append(format_index);
  712. };
  713. Vector<CalendarPattern> time_formats_with_fractional_second_digits;
  714. for (auto const& format : date_formats)
  715. append_if_unique(format);
  716. for (auto const& format : time_formats) {
  717. append_if_unique(format);
  718. if (format.second.has_value() && !format.fractional_second_digits.has_value()) {
  719. auto new_format = format;
  720. new_format.fractional_second_digits = 2;
  721. new_format.pattern_index = inject_fractional_second_digits(new_format.pattern_index);
  722. if (new_format.pattern12_index != 0)
  723. new_format.pattern12_index = inject_fractional_second_digits(new_format.pattern12_index);
  724. time_formats_with_fractional_second_digits.append(new_format);
  725. append_if_unique(move(new_format));
  726. }
  727. }
  728. time_formats.extend(move(time_formats_with_fractional_second_digits));
  729. for (auto const& date_format : date_formats) {
  730. auto const& date_time_formats = locale_data.unique_formats.get(calendar.date_time_formats);
  731. CalendarPatternIndexType date_time_format_index = 0;
  732. if (date_format.month == Unicode::CalendarPatternStyle::Long) {
  733. if (date_format.weekday.has_value())
  734. date_time_format_index = date_time_formats.full_format;
  735. else
  736. date_time_format_index = date_time_formats.long_format;
  737. } else if (date_format.month == Unicode::CalendarPatternStyle::Short) {
  738. date_time_format_index = date_time_formats.medium_format;
  739. } else {
  740. date_time_format_index = date_time_formats.short_format;
  741. }
  742. for (auto const& time_format : time_formats) {
  743. auto format = locale_data.unique_patterns.get(date_time_format_index);
  744. if (time_format.pattern12_index != 0)
  745. format.pattern12_index = replace_pattern(format.pattern_index, time_format.pattern12_index, date_format.pattern_index);
  746. format.pattern_index = replace_pattern(format.pattern_index, time_format.pattern_index, date_format.pattern_index);
  747. format.for_each_calendar_field_zipped_with(date_format, [](auto& field, auto const& date_field, auto) {
  748. if (date_field.has_value())
  749. field = date_field;
  750. });
  751. format.for_each_calendar_field_zipped_with(time_format, [](auto& field, auto const& time_field, auto) {
  752. if (time_field.has_value())
  753. field = time_field;
  754. });
  755. append_if_unique(move(format));
  756. }
  757. }
  758. }
  759. static void parse_calendar_symbols(Calendar& calendar, JsonObject const& calendar_object, UnicodeLocaleData& locale_data)
  760. {
  761. auto create_symbol_lists = [](size_t size) {
  762. SymbolList narrow_symbol_list;
  763. SymbolList short_symbol_list;
  764. SymbolList long_symbol_list;
  765. narrow_symbol_list.resize(size);
  766. short_symbol_list.resize(size);
  767. long_symbol_list.resize(size);
  768. return Array<SymbolList, 3> { {
  769. move(narrow_symbol_list),
  770. move(short_symbol_list),
  771. move(long_symbol_list),
  772. } };
  773. };
  774. auto store_symbol_lists = [&](auto symbol, auto symbol_lists) {
  775. auto symbol_index = to_underlying(symbol);
  776. if (symbol_index >= calendar.symbols.size())
  777. calendar.symbols.resize(symbol_index + 1);
  778. CalendarSymbols symbols {};
  779. symbols.narrow_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[0]));
  780. symbols.short_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[1]));
  781. symbols.long_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[2]));
  782. auto calendar_symbols_index = locale_data.unique_calendar_symbols.ensure(move(symbols));
  783. calendar.symbols[symbol_index] = calendar_symbols_index;
  784. };
  785. auto parse_era_symbols = [&](auto const& symbols_object) {
  786. auto const& narrow_symbols = symbols_object.get("eraNarrow"sv).as_object();
  787. auto const& short_symbols = symbols_object.get("eraAbbr"sv).as_object();
  788. auto const& long_symbols = symbols_object.get("eraNames"sv).as_object();
  789. auto symbol_lists = create_symbol_lists(2);
  790. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  791. if (auto key_index = key.to_uint(); key_index.has_value())
  792. symbols[*key_index] = locale_data.unique_strings.ensure(move(symbol));
  793. };
  794. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  795. append_symbol(symbol_lists[0], key, value.as_string());
  796. });
  797. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  798. append_symbol(symbol_lists[1], key, value.as_string());
  799. });
  800. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  801. append_symbol(symbol_lists[2], key, value.as_string());
  802. });
  803. store_symbol_lists(Unicode::CalendarSymbol::Era, move(symbol_lists));
  804. };
  805. auto parse_month_symbols = [&](auto const& symbols_object) {
  806. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  807. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  808. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  809. auto symbol_lists = create_symbol_lists(12);
  810. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  811. auto key_index = key.to_uint().value() - 1;
  812. symbols[key_index] = locale_data.unique_strings.ensure(move(symbol));
  813. };
  814. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  815. append_symbol(symbol_lists[0], key, value.as_string());
  816. });
  817. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  818. append_symbol(symbol_lists[1], key, value.as_string());
  819. });
  820. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  821. append_symbol(symbol_lists[2], key, value.as_string());
  822. });
  823. store_symbol_lists(Unicode::CalendarSymbol::Month, move(symbol_lists));
  824. };
  825. auto parse_weekday_symbols = [&](auto const& symbols_object) {
  826. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  827. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  828. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  829. auto symbol_lists = create_symbol_lists(7);
  830. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  831. if (key == "sun"sv)
  832. symbols[to_underlying(Unicode::Weekday::Sunday)] = locale_data.unique_strings.ensure(move(symbol));
  833. else if (key == "mon"sv)
  834. symbols[to_underlying(Unicode::Weekday::Monday)] = locale_data.unique_strings.ensure(move(symbol));
  835. else if (key == "tue"sv)
  836. symbols[to_underlying(Unicode::Weekday::Tuesday)] = locale_data.unique_strings.ensure(move(symbol));
  837. else if (key == "wed"sv)
  838. symbols[to_underlying(Unicode::Weekday::Wednesday)] = locale_data.unique_strings.ensure(move(symbol));
  839. else if (key == "thu"sv)
  840. symbols[to_underlying(Unicode::Weekday::Thursday)] = locale_data.unique_strings.ensure(move(symbol));
  841. else if (key == "fri"sv)
  842. symbols[to_underlying(Unicode::Weekday::Friday)] = locale_data.unique_strings.ensure(move(symbol));
  843. else if (key == "sat"sv)
  844. symbols[to_underlying(Unicode::Weekday::Saturday)] = locale_data.unique_strings.ensure(move(symbol));
  845. };
  846. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  847. append_symbol(symbol_lists[0], key, value.as_string());
  848. });
  849. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  850. append_symbol(symbol_lists[1], key, value.as_string());
  851. });
  852. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  853. append_symbol(symbol_lists[2], key, value.as_string());
  854. });
  855. store_symbol_lists(Unicode::CalendarSymbol::Weekday, move(symbol_lists));
  856. };
  857. auto parse_day_period_symbols = [&](auto const& symbols_object) {
  858. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  859. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  860. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  861. auto symbol_lists = create_symbol_lists(10);
  862. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  863. if (auto day_period = day_period_from_string(key); day_period.has_value())
  864. symbols[to_underlying(*day_period)] = locale_data.unique_strings.ensure(move(symbol));
  865. };
  866. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  867. append_symbol(symbol_lists[0], key, value.as_string());
  868. });
  869. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  870. append_symbol(symbol_lists[1], key, value.as_string());
  871. });
  872. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  873. append_symbol(symbol_lists[2], key, value.as_string());
  874. });
  875. store_symbol_lists(Unicode::CalendarSymbol::DayPeriod, move(symbol_lists));
  876. };
  877. parse_era_symbols(calendar_object.get("eras"sv).as_object());
  878. parse_month_symbols(calendar_object.get("months"sv).as_object().get("format"sv).as_object());
  879. parse_weekday_symbols(calendar_object.get("days"sv).as_object().get("format"sv).as_object());
  880. parse_day_period_symbols(calendar_object.get("dayPeriods"sv).as_object().get("format"sv).as_object());
  881. }
  882. static ErrorOr<void> parse_calendars(String locale_calendars_path, UnicodeLocaleData& locale_data, Locale& locale)
  883. {
  884. LexicalPath calendars_path(move(locale_calendars_path));
  885. if (!calendars_path.basename().starts_with("ca-"sv))
  886. return {};
  887. auto calendars_file = TRY(Core::File::open(calendars_path.string(), Core::OpenMode::ReadOnly));
  888. auto calendars = TRY(JsonValue::from_string(calendars_file->read_all()));
  889. auto const& main_object = calendars.as_object().get("main"sv);
  890. auto const& locale_object = main_object.as_object().get(calendars_path.parent().basename());
  891. auto const& dates_object = locale_object.as_object().get("dates"sv);
  892. auto const& calendars_object = dates_object.as_object().get("calendars"sv);
  893. auto ensure_calendar = [&](auto const& calendar) -> Calendar& {
  894. return locale.calendars.ensure(calendar, [&]() {
  895. auto calendar_index = locale_data.unique_strings.ensure(calendar);
  896. return Calendar { .calendar = calendar_index };
  897. });
  898. };
  899. auto parse_patterns = [&](auto const& patterns_object, auto const& skeletons_object, Vector<CalendarPattern>* patterns) {
  900. auto parse_pattern = [&](auto name) {
  901. auto format = patterns_object.get(name);
  902. auto skeleton = skeletons_object.get(name);
  903. auto format_index = parse_date_time_pattern(format.as_string(), skeleton.as_string_or(String::empty()), locale_data).value();
  904. if (patterns)
  905. patterns->append(locale_data.unique_patterns.get(format_index));
  906. return format_index;
  907. };
  908. CalendarFormat formats {};
  909. formats.full_format = parse_pattern("full"sv);
  910. formats.long_format = parse_pattern("long"sv);
  911. formats.medium_format = parse_pattern("medium"sv);
  912. formats.short_format = parse_pattern("short"sv);
  913. return locale_data.unique_formats.ensure(move(formats));
  914. };
  915. calendars_object.as_object().for_each_member([&](auto const& calendar_name, JsonValue const& value) {
  916. // The generic calendar is not a supported Unicode calendar key, so skip it:
  917. // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Locale/calendar#unicode_calendar_keys
  918. if (calendar_name == "generic"sv)
  919. return;
  920. auto& calendar = ensure_calendar(calendar_name);
  921. if (!locale_data.calendars.contains_slow(calendar_name))
  922. locale_data.calendars.append(calendar_name);
  923. Vector<CalendarPattern> date_formats;
  924. Vector<CalendarPattern> time_formats;
  925. auto const& date_formats_object = value.as_object().get("dateFormats"sv);
  926. auto const& date_skeletons_object = value.as_object().get("dateSkeletons"sv);
  927. calendar.date_formats = parse_patterns(date_formats_object.as_object(), date_skeletons_object.as_object(), &date_formats);
  928. auto const& time_formats_object = value.as_object().get("timeFormats"sv);
  929. auto const& time_skeletons_object = value.as_object().get("timeSkeletons"sv);
  930. calendar.time_formats = parse_patterns(time_formats_object.as_object(), time_skeletons_object.as_object(), &time_formats);
  931. auto const& date_time_formats_object = value.as_object().get("dateTimeFormats"sv);
  932. calendar.date_time_formats = parse_patterns(date_time_formats_object.as_object(), JsonObject {}, nullptr);
  933. auto const& available_formats = date_time_formats_object.as_object().get("availableFormats"sv);
  934. available_formats.as_object().for_each_member([&](auto const& skeleton, JsonValue const& pattern) {
  935. auto pattern_index = parse_date_time_pattern(pattern.as_string(), skeleton, locale_data);
  936. if (!pattern_index.has_value())
  937. return;
  938. auto const& format = locale_data.unique_patterns.get(*pattern_index);
  939. if (format.contains_only_date_fields())
  940. date_formats.append(format);
  941. else if (format.contains_only_time_fields())
  942. time_formats.append(format);
  943. if (!calendar.available_formats.contains_slow(*pattern_index))
  944. calendar.available_formats.append(*pattern_index);
  945. });
  946. auto const& interval_formats_object = date_time_formats_object.as_object().get("intervalFormats"sv);
  947. parse_interval_patterns(calendar, interval_formats_object.as_object(), locale_data);
  948. generate_missing_patterns(calendar, move(date_formats), move(time_formats), locale_data);
  949. parse_calendar_symbols(calendar, value.as_object(), locale_data);
  950. });
  951. return {};
  952. }
  953. static ErrorOr<void> parse_time_zone_names(String locale_time_zone_names_path, UnicodeLocaleData& locale_data, Locale& locale)
  954. {
  955. LexicalPath time_zone_names_path(move(locale_time_zone_names_path));
  956. time_zone_names_path = time_zone_names_path.append("timeZoneNames.json"sv);
  957. auto time_zone_names_file = TRY(Core::File::open(time_zone_names_path.string(), Core::OpenMode::ReadOnly));
  958. auto time_zone_names = TRY(JsonValue::from_string(time_zone_names_file->read_all()));
  959. auto const& main_object = time_zone_names.as_object().get("main"sv);
  960. auto const& locale_object = main_object.as_object().get(time_zone_names_path.parent().basename());
  961. auto const& dates_object = locale_object.as_object().get("dates"sv);
  962. auto const& time_zone_names_object = dates_object.as_object().get("timeZoneNames"sv);
  963. auto const& meta_zone_object = time_zone_names_object.as_object().get("metazone"sv);
  964. if (meta_zone_object.is_null())
  965. return {};
  966. auto parse_name = [&](StringView type, JsonObject const& meta_zone_object) -> Optional<StringIndexType> {
  967. auto const& names = meta_zone_object.get(type);
  968. if (!names.is_object())
  969. return {};
  970. auto const& daylight = names.as_object().get("daylight"sv);
  971. if (daylight.is_string())
  972. return locale_data.unique_strings.ensure(daylight.as_string());
  973. auto const& standard = names.as_object().get("standard"sv);
  974. if (standard.is_string())
  975. return locale_data.unique_strings.ensure(standard.as_string());
  976. return {};
  977. };
  978. auto parse_time_zone = [&](StringView meta_zone, JsonObject const& meta_zone_object) {
  979. auto golden_zone = locale_data.meta_zones.get(meta_zone).value();
  980. TimeZone time_zone { .time_zone = golden_zone };
  981. if (auto long_name = parse_name("long"sv, meta_zone_object); long_name.has_value())
  982. time_zone.long_name = long_name.value();
  983. if (auto short_name = parse_name("short"sv, meta_zone_object); short_name.has_value())
  984. time_zone.short_name = short_name.value();
  985. auto const& time_zone_name = locale_data.unique_strings.get(golden_zone);
  986. if (!locale_data.time_zones.contains_slow(time_zone_name))
  987. locale_data.time_zones.append(time_zone_name);
  988. locale.time_zones.set(time_zone_name, move(time_zone));
  989. };
  990. meta_zone_object.as_object().for_each_member([&](auto const& meta_zone, JsonValue const& value) {
  991. parse_time_zone(meta_zone, value.as_object());
  992. });
  993. // The long and short names for UTC are not under the "timeZoneNames/metazone" object, but are under "timeZoneNames/zone/Etc".
  994. auto const& zone_object = time_zone_names_object.as_object().get("zone"sv);
  995. auto const& etc_object = zone_object.as_object().get("Etc"sv);
  996. auto const& utc_object = etc_object.as_object().get("UTC"sv);
  997. parse_time_zone("UTC"sv, utc_object.as_object());
  998. return {};
  999. }
  1000. static ErrorOr<void> parse_day_periods(String core_path, UnicodeLocaleData& locale_data)
  1001. {
  1002. // https://unicode.org/reports/tr35/tr35-dates.html#Day_Period_Rule_Sets
  1003. LexicalPath day_periods_path(move(core_path));
  1004. day_periods_path = day_periods_path.append("supplemental"sv);
  1005. day_periods_path = day_periods_path.append("dayPeriods.json"sv);
  1006. auto day_periods_file = TRY(Core::File::open(day_periods_path.string(), Core::OpenMode::ReadOnly));
  1007. auto day_periods = TRY(JsonValue::from_string(day_periods_file->read_all()));
  1008. auto const& supplemental_object = day_periods.as_object().get("supplemental"sv);
  1009. auto const& day_periods_object = supplemental_object.as_object().get("dayPeriodRuleSet"sv);
  1010. auto parse_hour = [](auto const& time) {
  1011. auto hour_end_index = time.find(':').value();
  1012. // The times are in the form "hours:minutes", but we only need the hour segment.
  1013. // TR-35 explicitly states that minutes other than :00 must not be used.
  1014. VERIFY(time.substring_view(hour_end_index) == ":00"sv);
  1015. auto hour = time.substring_view(0, hour_end_index);
  1016. return hour.template to_uint<u8>().value();
  1017. };
  1018. auto parse_day_period = [&](auto const& symbol, auto const& ranges) -> Optional<DayPeriod> {
  1019. auto day_period = day_period_from_string(symbol);
  1020. if (!day_period.has_value())
  1021. return {};
  1022. auto begin = parse_hour(ranges.get("_from"sv).as_string());
  1023. auto end = parse_hour(ranges.get("_before"sv).as_string());
  1024. return DayPeriod { *day_period, begin, end };
  1025. };
  1026. day_periods_object.as_object().for_each_member([&](auto const& language, JsonValue const& value) {
  1027. auto locale = locale_data.locales.find(language);
  1028. if (locale == locale_data.locales.end())
  1029. return;
  1030. value.as_object().for_each_member([&](auto const& symbol, JsonValue const& ranges) {
  1031. if (auto day_period = parse_day_period(symbol, ranges.as_object()); day_period.has_value())
  1032. locale->value.day_periods.append(day_period.release_value());
  1033. });
  1034. });
  1035. return {};
  1036. };
  1037. static ErrorOr<void> parse_all_locales(String core_path, String dates_path, UnicodeLocaleData& locale_data)
  1038. {
  1039. TRY(parse_hour_cycles(core_path, locale_data));
  1040. TRY(parse_meta_zones(core_path, locale_data));
  1041. auto dates_iterator = TRY(path_to_dir_iterator(move(dates_path)));
  1042. auto remove_variants_from_path = [&](String path) -> ErrorOr<String> {
  1043. auto parsed_locale = TRY(CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, LexicalPath::basename(path)));
  1044. StringBuilder builder;
  1045. builder.append(locale_data.unique_strings.get(parsed_locale.language));
  1046. if (auto script = locale_data.unique_strings.get(parsed_locale.script); !script.is_empty())
  1047. builder.appendff("-{}", script);
  1048. if (auto region = locale_data.unique_strings.get(parsed_locale.region); !region.is_empty())
  1049. builder.appendff("-{}", region);
  1050. return builder.build();
  1051. };
  1052. while (dates_iterator.has_next()) {
  1053. auto dates_path = TRY(next_path_from_dir_iterator(dates_iterator));
  1054. auto calendars_iterator = TRY(path_to_dir_iterator(dates_path, {}));
  1055. auto language = TRY(remove_variants_from_path(dates_path));
  1056. auto& locale = locale_data.locales.ensure(language);
  1057. while (calendars_iterator.has_next()) {
  1058. auto calendars_path = TRY(next_path_from_dir_iterator(calendars_iterator));
  1059. TRY(parse_calendars(move(calendars_path), locale_data, locale));
  1060. }
  1061. TRY(parse_time_zone_names(move(dates_path), locale_data, locale));
  1062. }
  1063. TRY(parse_day_periods(move(core_path), locale_data));
  1064. return {};
  1065. }
  1066. static String format_identifier(StringView owner, String identifier)
  1067. {
  1068. identifier = identifier.replace("-"sv, "_"sv, true);
  1069. identifier = identifier.replace("/"sv, "_"sv, true);
  1070. if (all_of(identifier, is_ascii_digit))
  1071. return String::formatted("{}_{}", owner[0], identifier);
  1072. if (is_ascii_lower_alpha(identifier[0]))
  1073. return String::formatted("{:c}{}", to_ascii_uppercase(identifier[0]), identifier.substring_view(1));
  1074. return identifier;
  1075. }
  1076. static void generate_unicode_locale_header(Core::File& file, UnicodeLocaleData& locale_data)
  1077. {
  1078. StringBuilder builder;
  1079. SourceGenerator generator { builder };
  1080. generator.append(R"~~~(
  1081. #pragma once
  1082. #include <AK/Optional.h>
  1083. #include <AK/StringView.h>
  1084. #include <LibUnicode/Forward.h>
  1085. namespace Unicode {
  1086. )~~~");
  1087. generate_enum(generator, format_identifier, "Calendar"sv, {}, locale_data.calendars, locale_data.calendar_aliases);
  1088. generate_enum(generator, format_identifier, "HourCycleRegion"sv, {}, locale_data.hour_cycle_regions);
  1089. generate_enum(generator, format_identifier, "TimeZone"sv, {}, locale_data.time_zones);
  1090. generator.append(R"~~~(
  1091. namespace Detail {
  1092. Optional<Calendar> calendar_from_string(StringView calendar);
  1093. Optional<HourCycleRegion> hour_cycle_region_from_string(StringView hour_cycle_region);
  1094. Vector<Unicode::HourCycle> get_regional_hour_cycles(StringView region);
  1095. Optional<Unicode::CalendarFormat> get_calendar_date_format(StringView locale, StringView calendar);
  1096. Optional<Unicode::CalendarFormat> get_calendar_time_format(StringView locale, StringView calendar);
  1097. Optional<Unicode::CalendarFormat> get_calendar_date_time_format(StringView locale, StringView calendar);
  1098. Vector<Unicode::CalendarPattern> get_calendar_available_formats(StringView locale, StringView calendar);
  1099. Optional<Unicode::CalendarRangePattern> get_calendar_default_range_format(StringView locale, StringView calendar);
  1100. Vector<Unicode::CalendarRangePattern> get_calendar_range_formats(StringView locale, StringView calendar, StringView skeleton);
  1101. Vector<Unicode::CalendarRangePattern> get_calendar_range12_formats(StringView locale, StringView calendar, StringView skeleton);
  1102. Optional<StringView> get_calendar_era_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Era value);
  1103. Optional<StringView> get_calendar_month_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Month value);
  1104. Optional<StringView> get_calendar_weekday_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Weekday value);
  1105. Optional<StringView> get_calendar_day_period_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::DayPeriod value);
  1106. Optional<StringView> get_calendar_day_period_symbol_for_hour(StringView locale, StringView calendar, CalendarPatternStyle style, u8 hour);
  1107. Optional<TimeZone> time_zone_from_string(StringView time_zone);
  1108. Optional<StringView> get_time_zone_name(StringView locale, StringView time_zone, CalendarPatternStyle style);
  1109. }
  1110. }
  1111. )~~~");
  1112. VERIFY(file.write(generator.as_string_view()));
  1113. }
  1114. static void generate_unicode_locale_implementation(Core::File& file, UnicodeLocaleData& locale_data)
  1115. {
  1116. StringBuilder builder;
  1117. SourceGenerator generator { builder };
  1118. generator.set("string_index_type"sv, s_string_index_type);
  1119. generator.set("calendar_pattern_index_type"sv, s_calendar_pattern_index_type);
  1120. generator.set("calendar_range_pattern_index_type"sv, s_calendar_range_pattern_index_type);
  1121. generator.set("calendar_format_index_type"sv, s_calendar_format_index_type);
  1122. generator.set("symbol_list_index_type"sv, s_symbol_list_index_type);
  1123. generator.set("calendar_symbols_index_type"sv, s_calendar_symbols_index_type);
  1124. generator.append(R"~~~(
  1125. #include <AK/Array.h>
  1126. #include <AK/BinarySearch.h>
  1127. #include <LibUnicode/DateTimeFormat.h>
  1128. #include <LibUnicode/Locale.h>
  1129. #include <LibUnicode/UnicodeDateTimeFormat.h>
  1130. namespace Unicode::Detail {
  1131. )~~~");
  1132. locale_data.unique_strings.generate(generator);
  1133. generator.append(R"~~~(
  1134. template <typename SourceType, typename TargetType>
  1135. void convert_calendar_fields(SourceType const& source, TargetType& target)
  1136. {
  1137. if (source.era != -1)
  1138. target.era = static_cast<Unicode::CalendarPatternStyle>(source.era);
  1139. if (source.year != -1)
  1140. target.year = static_cast<Unicode::CalendarPatternStyle>(source.year);
  1141. if (source.month != -1)
  1142. target.month = static_cast<Unicode::CalendarPatternStyle>(source.month);
  1143. if (source.weekday != -1)
  1144. target.weekday = static_cast<Unicode::CalendarPatternStyle>(source.weekday);
  1145. if (source.day != -1)
  1146. target.day = static_cast<Unicode::CalendarPatternStyle>(source.day);
  1147. if (source.day_period != -1)
  1148. target.day_period = static_cast<Unicode::CalendarPatternStyle>(source.day_period);
  1149. if (source.hour != -1)
  1150. target.hour = static_cast<Unicode::CalendarPatternStyle>(source.hour);
  1151. if (source.minute != -1)
  1152. target.minute = static_cast<Unicode::CalendarPatternStyle>(source.minute);
  1153. if (source.second != -1)
  1154. target.second = static_cast<Unicode::CalendarPatternStyle>(source.second);
  1155. if (source.fractional_second_digits != -1)
  1156. target.fractional_second_digits = static_cast<u8>(source.fractional_second_digits);
  1157. if (source.time_zone_name != -1)
  1158. target.time_zone_name = static_cast<Unicode::CalendarPatternStyle>(source.time_zone_name);
  1159. }
  1160. struct CalendarPattern {
  1161. Unicode::CalendarPattern to_unicode_calendar_pattern() const {
  1162. Unicode::CalendarPattern calendar_pattern {};
  1163. calendar_pattern.skeleton = s_string_list[skeleton];
  1164. calendar_pattern.pattern = s_string_list[pattern];
  1165. if (pattern12 != 0)
  1166. calendar_pattern.pattern12 = s_string_list[pattern12];
  1167. convert_calendar_fields(*this, calendar_pattern);
  1168. return calendar_pattern;
  1169. }
  1170. @string_index_type@ skeleton { 0 };
  1171. @string_index_type@ pattern { 0 };
  1172. @string_index_type@ pattern12 { 0 };
  1173. i8 era { -1 };
  1174. i8 year { -1 };
  1175. i8 month { -1 };
  1176. i8 weekday { -1 };
  1177. i8 day { -1 };
  1178. i8 day_period { -1 };
  1179. i8 hour { -1 };
  1180. i8 minute { -1 };
  1181. i8 second { -1 };
  1182. i8 fractional_second_digits { -1 };
  1183. i8 time_zone_name { -1 };
  1184. };
  1185. struct CalendarRangePattern {
  1186. Unicode::CalendarRangePattern to_unicode_calendar_range_pattern() const {
  1187. Unicode::CalendarRangePattern calendar_range_pattern {};
  1188. if (field != -1)
  1189. calendar_range_pattern.field = static_cast<Unicode::CalendarRangePattern::Field>(field);
  1190. calendar_range_pattern.start_range = s_string_list[start_range];
  1191. calendar_range_pattern.separator = s_string_list[separator];
  1192. calendar_range_pattern.end_range = s_string_list[end_range];
  1193. convert_calendar_fields(*this, calendar_range_pattern);
  1194. return calendar_range_pattern;
  1195. }
  1196. @string_index_type@ skeleton { 0 };
  1197. i8 field { -1 };
  1198. @string_index_type@ start_range { 0 };
  1199. @string_index_type@ separator { 0 };
  1200. @string_index_type@ end_range { 0 };
  1201. i8 era { -1 };
  1202. i8 year { -1 };
  1203. i8 month { -1 };
  1204. i8 weekday { -1 };
  1205. i8 day { -1 };
  1206. i8 day_period { -1 };
  1207. i8 hour { -1 };
  1208. i8 minute { -1 };
  1209. i8 second { -1 };
  1210. i8 fractional_second_digits { -1 };
  1211. i8 time_zone_name { -1 };
  1212. };
  1213. )~~~");
  1214. locale_data.unique_patterns.generate(generator, "CalendarPattern"sv, "s_calendar_patterns"sv, 10);
  1215. locale_data.unique_range_patterns.generate(generator, "CalendarRangePattern"sv, "s_calendar_range_patterns"sv, 10);
  1216. generator.append(R"~~~(
  1217. struct CalendarFormat {
  1218. Unicode::CalendarFormat to_unicode_calendar_format() const {
  1219. Unicode::CalendarFormat calendar_format {};
  1220. calendar_format.full_format = s_calendar_patterns[full_format].to_unicode_calendar_pattern();
  1221. calendar_format.long_format = s_calendar_patterns[long_format].to_unicode_calendar_pattern();
  1222. calendar_format.medium_format = s_calendar_patterns[medium_format].to_unicode_calendar_pattern();
  1223. calendar_format.short_format = s_calendar_patterns[short_format].to_unicode_calendar_pattern();
  1224. return calendar_format;
  1225. }
  1226. @calendar_pattern_index_type@ full_format { 0 };
  1227. @calendar_pattern_index_type@ long_format { 0 };
  1228. @calendar_pattern_index_type@ medium_format { 0 };
  1229. @calendar_pattern_index_type@ short_format { 0 };
  1230. };
  1231. struct CalendarSymbols {
  1232. @symbol_list_index_type@ narrow_symbols { 0 };
  1233. @symbol_list_index_type@ short_symbols { 0 };
  1234. @symbol_list_index_type@ long_symbols { 0 };
  1235. };
  1236. struct CalendarData {
  1237. @string_index_type@ calendar { 0 };
  1238. @calendar_format_index_type@ date_formats {};
  1239. @calendar_format_index_type@ time_formats {};
  1240. @calendar_format_index_type@ date_time_formats {};
  1241. Span<@calendar_pattern_index_type@ const> available_formats {};
  1242. @calendar_range_pattern_index_type@ default_range_format { 0 };
  1243. Span<@calendar_range_pattern_index_type@ const> range_formats {};
  1244. Span<@calendar_range_pattern_index_type@ const> range12_formats {};
  1245. Array<@calendar_symbols_index_type@, 4> symbols {};
  1246. };
  1247. struct TimeZoneData {
  1248. @string_index_type@ time_zone { 0 };
  1249. @string_index_type@ long_name { 0 };
  1250. @string_index_type@ short_name { 0 };
  1251. };
  1252. struct DayPeriodData {
  1253. u8 day_period { 0 };
  1254. u8 begin { 0 };
  1255. u8 end { 0 };
  1256. };
  1257. )~~~");
  1258. locale_data.unique_formats.generate(generator, "CalendarFormat"sv, "s_calendar_formats"sv, 10);
  1259. locale_data.unique_symbol_lists.generate(generator, s_string_index_type, "s_symbol_lists"sv);
  1260. locale_data.unique_calendar_symbols.generate(generator, "CalendarSymbols"sv, "s_calendar_symbols"sv, 10);
  1261. auto append_pattern_list = [&](auto name, auto type, auto const& formats) {
  1262. generator.set("name", move(name));
  1263. generator.set("type", move(type));
  1264. generator.set("size", String::number(formats.size()));
  1265. generator.append(R"~~~(
  1266. static constexpr Array<@type@, @size@> @name@ { {)~~~");
  1267. bool first = true;
  1268. for (auto format : formats) {
  1269. generator.append(first ? " " : ", ");
  1270. generator.append(String::number(format));
  1271. first = false;
  1272. }
  1273. generator.append(" } };");
  1274. };
  1275. auto append_calendars = [&](String name, auto const& calendars) {
  1276. auto format_name = [&](StringView calendar_key, StringView type) {
  1277. return String::formatted("{}_{}_{}", name, calendar_key, type);
  1278. };
  1279. for (auto const& calendar_key : locale_data.calendars) {
  1280. auto const& calendar = calendars.find(calendar_key)->value;
  1281. append_pattern_list(format_name(calendar_key, "formats"sv), s_calendar_pattern_index_type, calendar.available_formats);
  1282. append_pattern_list(format_name(calendar_key, "range_formats"sv), s_calendar_range_pattern_index_type, calendar.range_formats);
  1283. append_pattern_list(format_name(calendar_key, "range12_formats"sv), s_calendar_range_pattern_index_type, calendar.range12_formats);
  1284. generator.append("\n");
  1285. }
  1286. generator.set("name", name);
  1287. generator.set("size", String::number(calendars.size()));
  1288. generator.append(R"~~~(
  1289. static constexpr Array<CalendarData, @size@> @name@ { {)~~~");
  1290. for (auto const& calendar_key : locale_data.calendars) {
  1291. auto const& calendar = calendars.find(calendar_key)->value;
  1292. generator.set("calendar"sv, String::number(calendar.calendar));
  1293. generator.set("date_formats"sv, String::number(calendar.date_formats));
  1294. generator.set("time_formats"sv, String::number(calendar.time_formats));
  1295. generator.set("date_time_formats"sv, String::number(calendar.date_time_formats));
  1296. generator.set("formats", format_name(calendar_key, "formats"sv));
  1297. generator.set("default_range_format", String::number(calendar.default_range_format));
  1298. generator.set("range_formats", format_name(calendar_key, "range_formats"sv));
  1299. generator.set("range12_formats", format_name(calendar_key, "range12_formats"sv));
  1300. generator.append(R"~~~(
  1301. { @calendar@, )~~~");
  1302. generator.append("@date_formats@, @time_formats@, @date_time_formats@, @formats@.span(), ");
  1303. generator.append("@default_range_format@, @range_formats@.span(), @range12_formats@.span(), {");
  1304. bool first = true;
  1305. for (auto symbols : calendar.symbols) {
  1306. generator.append(first ? " " : ", ");
  1307. generator.append(String::number(symbols));
  1308. first = false;
  1309. }
  1310. generator.append(" } },");
  1311. }
  1312. generator.append(R"~~~(
  1313. } };
  1314. )~~~");
  1315. };
  1316. auto append_time_zones = [&](String name, auto const& time_zones) {
  1317. generator.set("name", name);
  1318. generator.set("size", String::number(locale_data.time_zones.size()));
  1319. generator.append(R"~~~(
  1320. static constexpr Array<TimeZoneData, @size@> @name@ { {)~~~");
  1321. constexpr size_t max_values_per_row = 20;
  1322. size_t values_in_current_row = 0;
  1323. for (auto const& time_zone_key : locale_data.time_zones) {
  1324. auto time_zone = time_zones.find(time_zone_key);
  1325. if (values_in_current_row++ > 0)
  1326. generator.append(" ");
  1327. if (time_zone == time_zones.end()) {
  1328. generator.append("{},");
  1329. } else {
  1330. generator.set("time_zone", String::number(time_zone->value.time_zone));
  1331. generator.set("long_name", String::number(time_zone->value.long_name));
  1332. generator.set("short_name", String::number(time_zone->value.short_name));
  1333. generator.append("{ @time_zone@, @long_name@, @short_name@ },");
  1334. }
  1335. if (values_in_current_row == max_values_per_row) {
  1336. values_in_current_row = 0;
  1337. generator.append("\n ");
  1338. }
  1339. }
  1340. generator.append(R"~~~(
  1341. } };
  1342. )~~~");
  1343. };
  1344. auto append_day_periods = [&](String name, auto const& day_periods) {
  1345. generator.set("name", move(name));
  1346. generator.set("size", String::number(day_periods.size()));
  1347. generator.append(R"~~~(
  1348. static constexpr Array<DayPeriodData, @size@> @name@ { {)~~~");
  1349. for (auto const& day_period : day_periods) {
  1350. generator.set("day_period", String::number(static_cast<u8>(day_period.day_period)));
  1351. generator.set("begin", String::number(day_period.begin));
  1352. generator.set("end", String::number(day_period.end));
  1353. generator.append(" { @day_period@, @begin@, @end@ },");
  1354. }
  1355. generator.append(" } };");
  1356. };
  1357. auto append_hour_cycles = [&](String name, auto const& hour_cycle_region) {
  1358. auto const& hour_cycles = locale_data.hour_cycles.find(hour_cycle_region)->value;
  1359. generator.set("name", name);
  1360. generator.set("size", String::number(hour_cycles.size()));
  1361. generator.append(R"~~~(
  1362. static constexpr Array<u8, @size@> @name@ { { )~~~");
  1363. for (auto hour_cycle : hour_cycles) {
  1364. generator.set("hour_cycle", String::number(static_cast<u8>(hour_cycle)));
  1365. generator.append("@hour_cycle@, ");
  1366. }
  1367. generator.append(" } };");
  1368. };
  1369. generate_mapping(generator, locale_data.locales, "CalendarData"sv, "s_calendars"sv, "s_calendars_{}", [&](auto const& name, auto const& value) { append_calendars(name, value.calendars); });
  1370. generate_mapping(generator, locale_data.locales, "TimeZoneData"sv, "s_time_zones"sv, "s_time_zones_{}", [&](auto const& name, auto const& value) { append_time_zones(name, value.time_zones); });
  1371. generate_mapping(generator, locale_data.locales, "DayPeriodData"sv, "s_day_periods"sv, "s_day_periods_{}", [&](auto const& name, auto const& value) { append_day_periods(name, value.day_periods); });
  1372. generate_mapping(generator, locale_data.hour_cycle_regions, "u8"sv, "s_hour_cycles"sv, "s_hour_cycles_{}", [&](auto const& name, auto const& value) { append_hour_cycles(name, value); });
  1373. auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& values, Vector<Alias> const& aliases = {}) {
  1374. HashValueMap<String> hashes;
  1375. hashes.ensure_capacity(values.size());
  1376. for (auto const& value : values)
  1377. hashes.set(value.hash(), format_identifier(enum_title, value));
  1378. for (auto const& alias : aliases)
  1379. hashes.set(alias.alias.hash(), format_identifier(enum_title, alias.alias));
  1380. generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes));
  1381. };
  1382. append_from_string("Calendar"sv, "calendar"sv, locale_data.calendars, locale_data.calendar_aliases);
  1383. append_from_string("HourCycleRegion"sv, "hour_cycle_region"sv, locale_data.hour_cycle_regions);
  1384. append_from_string("TimeZone"sv, "time_zone"sv, locale_data.time_zones);
  1385. generator.append(R"~~~(
  1386. Vector<Unicode::HourCycle> get_regional_hour_cycles(StringView region)
  1387. {
  1388. auto region_value = hour_cycle_region_from_string(region);
  1389. if (!region_value.has_value())
  1390. return {};
  1391. auto region_index = to_underlying(*region_value);
  1392. auto const& regional_hour_cycles = s_hour_cycles.at(region_index);
  1393. Vector<Unicode::HourCycle> hour_cycles;
  1394. hour_cycles.ensure_capacity(regional_hour_cycles.size());
  1395. for (auto hour_cycle : regional_hour_cycles)
  1396. hour_cycles.unchecked_append(static_cast<Unicode::HourCycle>(hour_cycle));
  1397. return hour_cycles;
  1398. }
  1399. static CalendarData const* find_calendar_data(StringView locale, StringView calendar)
  1400. {
  1401. auto locale_value = locale_from_string(locale);
  1402. if (!locale_value.has_value())
  1403. return nullptr;
  1404. auto calendar_value = calendar_from_string(calendar);
  1405. if (!calendar_value.has_value())
  1406. return nullptr;
  1407. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1408. auto calendar_index = to_underlying(*calendar_value);
  1409. auto const& calendars = s_calendars.at(locale_index);
  1410. return &calendars[calendar_index];
  1411. }
  1412. Optional<Unicode::CalendarFormat> get_calendar_date_format(StringView locale, StringView calendar)
  1413. {
  1414. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1415. auto const& formats = s_calendar_formats.at(data->date_formats);
  1416. return formats.to_unicode_calendar_format();
  1417. }
  1418. return {};
  1419. }
  1420. Optional<Unicode::CalendarFormat> get_calendar_time_format(StringView locale, StringView calendar)
  1421. {
  1422. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1423. auto const& formats = s_calendar_formats.at(data->time_formats);
  1424. return formats.to_unicode_calendar_format();
  1425. }
  1426. return {};
  1427. }
  1428. Optional<Unicode::CalendarFormat> get_calendar_date_time_format(StringView locale, StringView calendar)
  1429. {
  1430. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1431. auto const& formats = s_calendar_formats.at(data->date_time_formats);
  1432. return formats.to_unicode_calendar_format();
  1433. }
  1434. return {};
  1435. }
  1436. Vector<Unicode::CalendarPattern> get_calendar_available_formats(StringView locale, StringView calendar)
  1437. {
  1438. Vector<Unicode::CalendarPattern> result {};
  1439. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1440. result.ensure_capacity(data->available_formats.size());
  1441. for (auto const& format : data->available_formats)
  1442. result.unchecked_append(s_calendar_patterns[format].to_unicode_calendar_pattern());
  1443. }
  1444. return result;
  1445. }
  1446. Optional<Unicode::CalendarRangePattern> get_calendar_default_range_format(StringView locale, StringView calendar)
  1447. {
  1448. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1449. auto const& pattern = s_calendar_range_patterns[data->default_range_format];
  1450. return pattern.to_unicode_calendar_range_pattern();
  1451. }
  1452. return {};
  1453. }
  1454. Vector<Unicode::CalendarRangePattern> get_calendar_range_formats(StringView locale, StringView calendar, StringView skeleton) {
  1455. Vector<Unicode::CalendarRangePattern> result {};
  1456. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1457. for (auto format : data->range_formats) {
  1458. auto const& pattern = s_calendar_range_patterns[format];
  1459. if (skeleton == s_string_list[pattern.skeleton])
  1460. result.append(pattern.to_unicode_calendar_range_pattern());
  1461. }
  1462. }
  1463. return result;
  1464. }
  1465. Vector<Unicode::CalendarRangePattern> get_calendar_range12_formats(StringView locale, StringView calendar, StringView skeleton) {
  1466. Vector<Unicode::CalendarRangePattern> result {};
  1467. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1468. for (auto format : data->range12_formats) {
  1469. auto const& pattern = s_calendar_range_patterns[format];
  1470. if (skeleton == s_string_list[pattern.skeleton])
  1471. result.append(pattern.to_unicode_calendar_range_pattern());
  1472. }
  1473. }
  1474. return result;
  1475. }
  1476. static Span<@string_index_type@ const> find_calendar_symbols(StringView locale, StringView calendar, CalendarSymbol symbol, CalendarPatternStyle style)
  1477. {
  1478. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1479. auto symbol_index = to_underlying(symbol);
  1480. auto calendar_symbols_index = data->symbols.at(symbol_index);
  1481. auto const& symbols = s_calendar_symbols.at(calendar_symbols_index);
  1482. @symbol_list_index_type@ symbol_list_index = 0;
  1483. switch (style) {
  1484. case CalendarPatternStyle::Narrow:
  1485. symbol_list_index = symbols.narrow_symbols;
  1486. break;
  1487. case CalendarPatternStyle::Short:
  1488. symbol_list_index = symbols.short_symbols;
  1489. break;
  1490. case CalendarPatternStyle::Long:
  1491. symbol_list_index = symbols.long_symbols;
  1492. break;
  1493. default:
  1494. VERIFY_NOT_REACHED();
  1495. }
  1496. return s_symbol_lists.at(symbol_list_index);
  1497. }
  1498. return {};
  1499. }
  1500. Optional<StringView> get_calendar_era_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Era value)
  1501. {
  1502. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Era, style);
  1503. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1504. return s_string_list[symbols.at(value_index)];
  1505. return {};
  1506. }
  1507. Optional<StringView> get_calendar_month_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Month value)
  1508. {
  1509. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Month, style);
  1510. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1511. return s_string_list[symbols.at(value_index)];
  1512. return {};
  1513. }
  1514. Optional<StringView> get_calendar_weekday_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Weekday value)
  1515. {
  1516. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Weekday, style);
  1517. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1518. return s_string_list[symbols.at(value_index)];
  1519. return {};
  1520. }
  1521. Optional<StringView> get_calendar_day_period_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::DayPeriod value)
  1522. {
  1523. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::DayPeriod, style);
  1524. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1525. return s_string_list[symbols.at(value_index)];
  1526. return {};
  1527. }
  1528. Optional<StringView> get_calendar_day_period_symbol_for_hour(StringView locale, StringView calendar, CalendarPatternStyle style, u8 hour)
  1529. {
  1530. auto locale_value = locale_from_string(locale);
  1531. if (!locale_value.has_value())
  1532. return {};
  1533. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1534. auto day_periods = s_day_periods[locale_index];
  1535. for (auto const& day_period : day_periods) {
  1536. if ((day_period.begin <= hour) && (hour < day_period.end)) {
  1537. auto period = static_cast<Unicode::DayPeriod>(day_period.day_period);
  1538. return Detail::get_calendar_day_period_symbol(locale, calendar, style, period);
  1539. }
  1540. }
  1541. // Fallback to fixed periods if the locale does not have flexible day periods.
  1542. // TR-35 states that the meaning of AM and PM does not change with locale.
  1543. if (hour < 12)
  1544. return Detail::get_calendar_day_period_symbol(locale, calendar, style, Unicode::DayPeriod::AM);
  1545. return Detail::get_calendar_day_period_symbol(locale, calendar, style, Unicode::DayPeriod::PM);
  1546. }
  1547. static TimeZoneData const* find_time_zone_data(StringView locale, StringView time_zone)
  1548. {
  1549. auto locale_value = locale_from_string(locale);
  1550. if (!locale_value.has_value())
  1551. return nullptr;
  1552. auto time_zone_value = time_zone_from_string(time_zone);
  1553. if (!time_zone_value.has_value())
  1554. return nullptr;
  1555. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1556. auto time_zone_index = to_underlying(*time_zone_value);
  1557. auto const& time_zones = s_time_zones.at(locale_index);
  1558. return &time_zones[time_zone_index];
  1559. }
  1560. Optional<StringView> get_time_zone_name(StringView locale, StringView time_zone, CalendarPatternStyle style)
  1561. {
  1562. if (auto const* data = find_time_zone_data(locale, time_zone); data != nullptr) {
  1563. @string_index_type@ time_zone_index = 0;
  1564. switch (style) {
  1565. case CalendarPatternStyle::Long:
  1566. time_zone_index = data->long_name;
  1567. break;
  1568. case CalendarPatternStyle::Short:
  1569. time_zone_index = data->short_name;
  1570. break;
  1571. default:
  1572. VERIFY_NOT_REACHED();
  1573. }
  1574. if (time_zone_index != 0)
  1575. return s_string_list[time_zone_index];
  1576. }
  1577. return {};
  1578. }
  1579. }
  1580. )~~~");
  1581. VERIFY(file.write(generator.as_string_view()));
  1582. }
  1583. ErrorOr<int> serenity_main(Main::Arguments arguments)
  1584. {
  1585. StringView generated_header_path;
  1586. StringView generated_implementation_path;
  1587. StringView core_path;
  1588. StringView dates_path;
  1589. Core::ArgsParser args_parser;
  1590. args_parser.add_option(generated_header_path, "Path to the Unicode locale header file to generate", "generated-header-path", 'h', "generated-header-path");
  1591. args_parser.add_option(generated_implementation_path, "Path to the Unicode locale implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  1592. args_parser.add_option(core_path, "Path to cldr-core directory", "core-path", 'r', "core-path");
  1593. args_parser.add_option(dates_path, "Path to cldr-dates directory", "dates-path", 'd', "dates-path");
  1594. args_parser.parse(arguments);
  1595. auto open_file = [&](StringView path) -> ErrorOr<NonnullRefPtr<Core::File>> {
  1596. if (path.is_empty()) {
  1597. args_parser.print_usage(stderr, arguments.argv[0]);
  1598. return Error::from_string_literal("Must provide all command line options"sv);
  1599. }
  1600. return Core::File::open(path, Core::OpenMode::ReadWrite);
  1601. };
  1602. auto generated_header_file = TRY(open_file(generated_header_path));
  1603. auto generated_implementation_file = TRY(open_file(generated_implementation_path));
  1604. UnicodeLocaleData locale_data;
  1605. TRY(parse_all_locales(core_path, dates_path, locale_data));
  1606. generate_unicode_locale_header(generated_header_file, locale_data);
  1607. generate_unicode_locale_implementation(generated_implementation_file, locale_data);
  1608. return 0;
  1609. }