GenerateUnicodeDateTimeFormat.cpp 81 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045
  1. /*
  2. * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AllOf.h>
  8. #include <AK/CharacterTypes.h>
  9. #include <AK/Format.h>
  10. #include <AK/GenericLexer.h>
  11. #include <AK/HashFunctions.h>
  12. #include <AK/HashMap.h>
  13. #include <AK/JsonObject.h>
  14. #include <AK/JsonParser.h>
  15. #include <AK/JsonValue.h>
  16. #include <AK/LexicalPath.h>
  17. #include <AK/SourceGenerator.h>
  18. #include <AK/String.h>
  19. #include <AK/StringBuilder.h>
  20. #include <AK/Traits.h>
  21. #include <AK/Utf8View.h>
  22. #include <LibCore/ArgsParser.h>
  23. #include <LibCore/DirIterator.h>
  24. #include <LibCore/File.h>
  25. #include <LibUnicode/DateTimeFormat.h>
  26. using StringIndexType = u16;
  27. constexpr auto s_string_index_type = "u16"sv;
  28. using CalendarPatternIndexType = u16;
  29. constexpr auto s_calendar_pattern_index_type = "u16"sv;
  30. using CalendarPatternListIndexType = u8;
  31. constexpr auto s_calendar_pattern_list_index_type = "u8"sv;
  32. using CalendarRangePatternIndexType = u16;
  33. constexpr auto s_calendar_range_pattern_index_type = "u16"sv;
  34. using CalendarRangePatternListIndexType = u8;
  35. constexpr auto s_calendar_range_pattern_list_index_type = "u8"sv;
  36. using CalendarFormatIndexType = u8;
  37. constexpr auto s_calendar_format_index_type = "u8"sv;
  38. using SymbolListIndexType = u16;
  39. constexpr auto s_symbol_list_index_type = "u16"sv;
  40. using CalendarSymbolsIndexType = u16;
  41. constexpr auto s_calendar_symbols_index_type = "u16"sv;
  42. using CalendarSymbolsListIndexType = u8;
  43. constexpr auto s_calendar_symbols_list_index_type = "u8"sv;
  44. using CalendarIndexType = u8;
  45. constexpr auto s_calendar_index_type = "u8"sv;
  46. struct CalendarPattern : public Unicode::CalendarPattern {
  47. bool contains_only_date_fields() const
  48. {
  49. return !day_period.has_value() && !hour.has_value() && !minute.has_value() && !second.has_value() && !fractional_second_digits.has_value() && !time_zone_name.has_value();
  50. }
  51. bool contains_only_time_fields() const
  52. {
  53. return !weekday.has_value() && !era.has_value() && !year.has_value() && !month.has_value() && !day.has_value();
  54. }
  55. unsigned hash() const
  56. {
  57. auto hash = pair_int_hash(pattern_index, pattern12_index);
  58. hash = pair_int_hash(hash, skeleton_index);
  59. auto hash_field = [&](auto const& field) {
  60. if (field.has_value())
  61. hash = pair_int_hash(hash, static_cast<u8>(*field));
  62. else
  63. hash = pair_int_hash(hash, -1);
  64. };
  65. hash_field(era);
  66. hash_field(year);
  67. hash_field(month);
  68. hash_field(weekday);
  69. hash_field(day);
  70. hash_field(day_period);
  71. hash_field(hour);
  72. hash_field(minute);
  73. hash_field(second);
  74. hash_field(fractional_second_digits);
  75. hash_field(time_zone_name);
  76. return hash;
  77. }
  78. bool operator==(CalendarPattern const& other) const
  79. {
  80. return (skeleton_index == other.skeleton_index)
  81. && (pattern_index == other.pattern_index)
  82. && (pattern12_index == other.pattern12_index)
  83. && (era == other.era)
  84. && (year == other.year)
  85. && (month == other.month)
  86. && (weekday == other.weekday)
  87. && (day == other.day)
  88. && (day_period == other.day_period)
  89. && (hour == other.hour)
  90. && (minute == other.minute)
  91. && (second == other.second)
  92. && (fractional_second_digits == other.fractional_second_digits)
  93. && (time_zone_name == other.time_zone_name);
  94. }
  95. StringIndexType skeleton_index { 0 };
  96. StringIndexType pattern_index { 0 };
  97. StringIndexType pattern12_index { 0 };
  98. };
  99. template<>
  100. struct AK::Formatter<CalendarPattern> : Formatter<FormatString> {
  101. ErrorOr<void> format(FormatBuilder& builder, CalendarPattern const& pattern)
  102. {
  103. auto field_to_i8 = [](auto const& field) -> i8 {
  104. if (!field.has_value())
  105. return -1;
  106. return static_cast<i8>(*field);
  107. };
  108. return Formatter<FormatString>::format(builder,
  109. "{{ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {} }}",
  110. pattern.skeleton_index,
  111. pattern.pattern_index,
  112. pattern.pattern12_index,
  113. field_to_i8(pattern.era),
  114. field_to_i8(pattern.year),
  115. field_to_i8(pattern.month),
  116. field_to_i8(pattern.weekday),
  117. field_to_i8(pattern.day),
  118. field_to_i8(pattern.day_period),
  119. field_to_i8(pattern.hour),
  120. field_to_i8(pattern.minute),
  121. field_to_i8(pattern.second),
  122. field_to_i8(pattern.fractional_second_digits),
  123. field_to_i8(pattern.time_zone_name));
  124. }
  125. };
  126. template<>
  127. struct AK::Traits<CalendarPattern> : public GenericTraits<CalendarPattern> {
  128. static unsigned hash(CalendarPattern const& c) { return c.hash(); }
  129. };
  130. struct CalendarRangePattern : public CalendarPattern {
  131. unsigned hash() const
  132. {
  133. auto hash = CalendarPattern::hash();
  134. if (field.has_value())
  135. hash = pair_int_hash(hash, static_cast<u8>(*field));
  136. hash = pair_int_hash(hash, start_range);
  137. hash = pair_int_hash(hash, separator);
  138. hash = pair_int_hash(hash, end_range);
  139. return hash;
  140. }
  141. bool operator==(CalendarRangePattern const& other) const
  142. {
  143. if (!CalendarPattern::operator==(other))
  144. return false;
  145. return (field == other.field)
  146. && (start_range == other.start_range)
  147. && (separator == other.separator)
  148. && (end_range == other.end_range);
  149. }
  150. Optional<Unicode::CalendarRangePattern::Field> field {};
  151. StringIndexType start_range { 0 };
  152. StringIndexType separator { 0 };
  153. StringIndexType end_range { 0 };
  154. };
  155. template<>
  156. struct AK::Formatter<CalendarRangePattern> : Formatter<FormatString> {
  157. ErrorOr<void> format(FormatBuilder& builder, CalendarRangePattern const& pattern)
  158. {
  159. auto field_to_i8 = [](auto const& field) -> i8 {
  160. if (!field.has_value())
  161. return -1;
  162. return static_cast<i8>(*field);
  163. };
  164. return Formatter<FormatString>::format(builder,
  165. "{{ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {} }}",
  166. pattern.skeleton_index,
  167. field_to_i8(pattern.field),
  168. pattern.start_range,
  169. pattern.separator,
  170. pattern.end_range,
  171. field_to_i8(pattern.era),
  172. field_to_i8(pattern.year),
  173. field_to_i8(pattern.month),
  174. field_to_i8(pattern.weekday),
  175. field_to_i8(pattern.day),
  176. field_to_i8(pattern.day_period),
  177. field_to_i8(pattern.hour),
  178. field_to_i8(pattern.minute),
  179. field_to_i8(pattern.second),
  180. field_to_i8(pattern.fractional_second_digits),
  181. field_to_i8(pattern.time_zone_name));
  182. }
  183. };
  184. template<>
  185. struct AK::Traits<CalendarRangePattern> : public GenericTraits<CalendarRangePattern> {
  186. static unsigned hash(CalendarRangePattern const& c) { return c.hash(); }
  187. };
  188. struct CalendarFormat {
  189. unsigned hash() const
  190. {
  191. auto hash = pair_int_hash(full_format, long_format);
  192. hash = pair_int_hash(hash, medium_format);
  193. hash = pair_int_hash(hash, short_format);
  194. return hash;
  195. }
  196. bool operator==(CalendarFormat const& other) const
  197. {
  198. return (full_format == other.full_format)
  199. && (long_format == other.long_format)
  200. && (medium_format == other.medium_format)
  201. && (short_format == other.short_format);
  202. }
  203. CalendarPatternIndexType full_format { 0 };
  204. CalendarPatternIndexType long_format { 0 };
  205. CalendarPatternIndexType medium_format { 0 };
  206. CalendarPatternIndexType short_format { 0 };
  207. };
  208. template<>
  209. struct AK::Formatter<CalendarFormat> : Formatter<FormatString> {
  210. ErrorOr<void> format(FormatBuilder& builder, CalendarFormat const& pattern)
  211. {
  212. return Formatter<FormatString>::format(builder,
  213. "{{ {}, {}, {}, {} }}",
  214. pattern.full_format,
  215. pattern.long_format,
  216. pattern.medium_format,
  217. pattern.short_format);
  218. }
  219. };
  220. template<>
  221. struct AK::Traits<CalendarFormat> : public GenericTraits<CalendarFormat> {
  222. static unsigned hash(CalendarFormat const& c) { return c.hash(); }
  223. };
  224. using SymbolList = Vector<StringIndexType>;
  225. struct CalendarSymbols {
  226. unsigned hash() const
  227. {
  228. auto hash = pair_int_hash(narrow_symbols, short_symbols);
  229. hash = pair_int_hash(hash, long_symbols);
  230. return hash;
  231. }
  232. bool operator==(CalendarSymbols const& other) const
  233. {
  234. return (narrow_symbols == other.narrow_symbols)
  235. && (short_symbols == other.short_symbols)
  236. && (long_symbols == other.long_symbols);
  237. }
  238. SymbolListIndexType narrow_symbols { 0 };
  239. SymbolListIndexType short_symbols { 0 };
  240. SymbolListIndexType long_symbols { 0 };
  241. };
  242. template<>
  243. struct AK::Formatter<CalendarSymbols> : Formatter<FormatString> {
  244. ErrorOr<void> format(FormatBuilder& builder, CalendarSymbols const& symbols)
  245. {
  246. return Formatter<FormatString>::format(builder,
  247. "{{ {}, {}, {} }}",
  248. symbols.narrow_symbols,
  249. symbols.short_symbols,
  250. symbols.long_symbols);
  251. }
  252. };
  253. template<>
  254. struct AK::Traits<CalendarSymbols> : public GenericTraits<CalendarSymbols> {
  255. static unsigned hash(CalendarSymbols const& c) { return c.hash(); }
  256. };
  257. using CalendarPatternList = Vector<CalendarPatternIndexType>;
  258. using CalendarRangePatternList = Vector<CalendarRangePatternIndexType>;
  259. using CalendarSymbolsList = Vector<CalendarSymbolsIndexType>;
  260. struct Calendar {
  261. unsigned hash() const
  262. {
  263. auto hash = int_hash(calendar);
  264. hash = pair_int_hash(hash, date_formats);
  265. hash = pair_int_hash(hash, time_formats);
  266. hash = pair_int_hash(hash, date_time_formats);
  267. hash = pair_int_hash(hash, available_formats);
  268. hash = pair_int_hash(hash, default_range_format);
  269. hash = pair_int_hash(hash, range_formats);
  270. hash = pair_int_hash(hash, range12_formats);
  271. hash = pair_int_hash(hash, symbols);
  272. return hash;
  273. }
  274. bool operator==(Calendar const& other) const
  275. {
  276. return (calendar == other.calendar)
  277. && (date_formats == other.date_formats)
  278. && (time_formats == other.time_formats)
  279. && (date_time_formats == other.date_time_formats)
  280. && (available_formats == other.available_formats)
  281. && (default_range_format == other.default_range_format)
  282. && (range_formats == other.range_formats)
  283. && (range12_formats == other.range12_formats)
  284. && (symbols == other.symbols);
  285. }
  286. StringIndexType calendar { 0 };
  287. CalendarFormatIndexType date_formats { 0 };
  288. CalendarFormatIndexType time_formats { 0 };
  289. CalendarFormatIndexType date_time_formats { 0 };
  290. CalendarPatternListIndexType available_formats { 0 };
  291. CalendarRangePatternIndexType default_range_format { 0 };
  292. CalendarRangePatternListIndexType range_formats { 0 };
  293. CalendarRangePatternListIndexType range12_formats { 0 };
  294. CalendarSymbolsListIndexType symbols { 0 };
  295. };
  296. template<>
  297. struct AK::Formatter<Calendar> : Formatter<FormatString> {
  298. ErrorOr<void> format(FormatBuilder& builder, Calendar const& calendar)
  299. {
  300. return Formatter<FormatString>::format(builder,
  301. "{{ {}, {}, {}, {}, {}, {}, {}, {}, {} }}",
  302. calendar.calendar,
  303. calendar.date_formats,
  304. calendar.time_formats,
  305. calendar.date_time_formats,
  306. calendar.available_formats,
  307. calendar.default_range_format,
  308. calendar.range_formats,
  309. calendar.range12_formats,
  310. calendar.symbols);
  311. }
  312. };
  313. template<>
  314. struct AK::Traits<Calendar> : public GenericTraits<Calendar> {
  315. static unsigned hash(Calendar const& c) { return c.hash(); }
  316. };
  317. struct TimeZone {
  318. StringIndexType time_zone { 0 };
  319. StringIndexType long_name { 0 };
  320. StringIndexType short_name { 0 };
  321. };
  322. struct DayPeriod {
  323. Unicode::DayPeriod day_period {};
  324. u8 begin { 0 };
  325. u8 end { 0 };
  326. };
  327. struct Locale {
  328. HashMap<String, CalendarIndexType> calendars;
  329. HashMap<String, TimeZone> time_zones;
  330. Vector<DayPeriod> day_periods;
  331. };
  332. struct UnicodeLocaleData {
  333. UniqueStringStorage<StringIndexType> unique_strings;
  334. UniqueStorage<CalendarPattern, CalendarPatternIndexType> unique_patterns;
  335. UniqueStorage<CalendarPatternList, CalendarPatternListIndexType> unique_pattern_lists;
  336. UniqueStorage<CalendarRangePattern, CalendarRangePatternIndexType> unique_range_patterns;
  337. UniqueStorage<CalendarRangePatternList, CalendarRangePatternListIndexType> unique_range_pattern_lists;
  338. UniqueStorage<CalendarFormat, CalendarFormatIndexType> unique_formats;
  339. UniqueStorage<SymbolList, SymbolListIndexType> unique_symbol_lists;
  340. UniqueStorage<CalendarSymbols, CalendarSymbolsIndexType> unique_calendar_symbols;
  341. UniqueStorage<CalendarSymbolsList, CalendarSymbolsListIndexType> unique_calendar_symbols_lists;
  342. UniqueStorage<Calendar, CalendarIndexType> unique_calendars;
  343. HashMap<String, Locale> locales;
  344. HashMap<String, Vector<Unicode::HourCycle>> hour_cycles;
  345. Vector<String> hour_cycle_regions;
  346. HashMap<String, StringIndexType> meta_zones;
  347. Vector<String> time_zones;
  348. Vector<String> calendars;
  349. Vector<Alias> calendar_aliases {
  350. // FIXME: Aliases should come from BCP47. See: https://unicode-org.atlassian.net/browse/CLDR-15158
  351. { "gregorian"sv, "gregory"sv },
  352. };
  353. };
  354. static Optional<Unicode::DayPeriod> day_period_from_string(StringView day_period)
  355. {
  356. if (day_period == "am"sv)
  357. return Unicode::DayPeriod::AM;
  358. if (day_period == "pm"sv)
  359. return Unicode::DayPeriod::PM;
  360. if (day_period == "morning1"sv)
  361. return Unicode::DayPeriod::Morning1;
  362. if (day_period == "morning2"sv)
  363. return Unicode::DayPeriod::Morning2;
  364. if (day_period == "afternoon1"sv)
  365. return Unicode::DayPeriod::Afternoon1;
  366. if (day_period == "afternoon2"sv)
  367. return Unicode::DayPeriod::Afternoon2;
  368. if (day_period == "evening1"sv)
  369. return Unicode::DayPeriod::Evening1;
  370. if (day_period == "evening2"sv)
  371. return Unicode::DayPeriod::Evening2;
  372. if (day_period == "night1"sv)
  373. return Unicode::DayPeriod::Night1;
  374. if (day_period == "night2"sv)
  375. return Unicode::DayPeriod::Night2;
  376. return {};
  377. };
  378. static ErrorOr<void> parse_hour_cycles(String core_path, UnicodeLocaleData& locale_data)
  379. {
  380. // https://unicode.org/reports/tr35/tr35-dates.html#Time_Data
  381. LexicalPath time_data_path(move(core_path));
  382. time_data_path = time_data_path.append("supplemental"sv);
  383. time_data_path = time_data_path.append("timeData.json"sv);
  384. auto time_data_file = TRY(Core::File::open(time_data_path.string(), Core::OpenMode::ReadOnly));
  385. auto time_data = TRY(JsonValue::from_string(time_data_file->read_all()));
  386. auto const& supplemental_object = time_data.as_object().get("supplemental"sv);
  387. auto const& time_data_object = supplemental_object.as_object().get("timeData"sv);
  388. auto parse_hour_cycle = [](StringView hour_cycle) -> Optional<Unicode::HourCycle> {
  389. if (hour_cycle == "h"sv)
  390. return Unicode::HourCycle::H12;
  391. if (hour_cycle == "H"sv)
  392. return Unicode::HourCycle::H23;
  393. if (hour_cycle == "K"sv)
  394. return Unicode::HourCycle::H11;
  395. if (hour_cycle == "k"sv)
  396. return Unicode::HourCycle::H24;
  397. return {};
  398. };
  399. time_data_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  400. auto allowed_hour_cycles_string = value.as_object().get("_allowed"sv).as_string();
  401. auto allowed_hour_cycles = allowed_hour_cycles_string.split_view(' ');
  402. Vector<Unicode::HourCycle> hour_cycles;
  403. for (auto allowed_hour_cycle : allowed_hour_cycles) {
  404. if (auto hour_cycle = parse_hour_cycle(allowed_hour_cycle); hour_cycle.has_value())
  405. hour_cycles.append(*hour_cycle);
  406. }
  407. locale_data.hour_cycles.set(key, move(hour_cycles));
  408. if (!locale_data.hour_cycle_regions.contains_slow(key))
  409. locale_data.hour_cycle_regions.append(key);
  410. });
  411. return {};
  412. };
  413. static ErrorOr<void> parse_meta_zones(String core_path, UnicodeLocaleData& locale_data)
  414. {
  415. // https://unicode.org/reports/tr35/tr35-dates.html#Metazones
  416. LexicalPath meta_zone_path(move(core_path));
  417. meta_zone_path = meta_zone_path.append("supplemental"sv);
  418. meta_zone_path = meta_zone_path.append("metaZones.json"sv);
  419. auto meta_zone_file = TRY(Core::File::open(meta_zone_path.string(), Core::OpenMode::ReadOnly));
  420. auto meta_zone = TRY(JsonValue::from_string(meta_zone_file->read_all()));
  421. auto const& supplemental_object = meta_zone.as_object().get("supplemental"sv);
  422. auto const& meta_zone_object = supplemental_object.as_object().get("metaZones"sv);
  423. auto const& meta_zone_array = meta_zone_object.as_object().get("metazones"sv);
  424. meta_zone_array.as_array().for_each([&](JsonValue const& value) {
  425. auto const& mapping = value.as_object().get("mapZone"sv);
  426. auto const& meta_zone = mapping.as_object().get("_other"sv);
  427. auto const& golden_zone = mapping.as_object().get("_type"sv);
  428. auto golden_zone_index = locale_data.unique_strings.ensure(golden_zone.as_string());
  429. locale_data.meta_zones.set(meta_zone.as_string(), golden_zone_index);
  430. });
  431. // UTC does not appear in metaZones.json. Define it for convenience so other parsers don't need to check for its existence.
  432. locale_data.meta_zones.set("UTC"sv, locale_data.unique_strings.ensure("UTC"sv));
  433. return {};
  434. };
  435. static constexpr auto is_char(char ch)
  436. {
  437. return [ch](auto c) { return c == ch; };
  438. }
  439. // For patterns that are 12-hour aware, we need to generate two patterns: one with the day period
  440. // (e.g. {ampm}) in the pattern, and one without the day period. We need to take care to remove
  441. // extra spaces around the day period. Some example expected removals:
  442. //
  443. // "{hour}:{minute} {ampm}" becomes "{hour}:{minute}" (remove the space before {ampm})
  444. // "{ampm} {hour}" becomes "{hour}" (remove the space after {ampm})
  445. // "{hour}:{minute} {ampm} {timeZoneName}" becomes "{hour}:{minute} {timeZoneName}" (remove one of the spaces around {ampm})
  446. static String remove_period_from_pattern(String pattern)
  447. {
  448. for (auto remove : AK::Array { "({ampm})"sv, "{ampm}"sv, "({dayPeriod})"sv, "{dayPeriod}"sv }) {
  449. auto index = pattern.find(remove);
  450. if (!index.has_value())
  451. continue;
  452. constexpr u32 space = ' ';
  453. constexpr u32 open = '{';
  454. constexpr u32 close = '}';
  455. Utf8View utf8_pattern { pattern };
  456. Optional<u32> before_removal;
  457. Optional<u32> after_removal;
  458. for (auto it = utf8_pattern.begin(); utf8_pattern.byte_offset_of(it) < *index; ++it)
  459. before_removal = *it;
  460. if (auto it = utf8_pattern.iterator_at_byte_offset(*index + remove.length()); it != utf8_pattern.end())
  461. after_removal = *it;
  462. if ((before_removal == space) && (after_removal != open)) {
  463. pattern = String::formatted("{}{}",
  464. pattern.substring_view(0, *index - 1),
  465. pattern.substring_view(*index + remove.length()));
  466. } else if ((after_removal == space) && (before_removal != close)) {
  467. pattern = String::formatted("{}{}",
  468. pattern.substring_view(0, *index),
  469. pattern.substring_view(*index + remove.length() + 1));
  470. } else {
  471. pattern = String::formatted("{}{}",
  472. pattern.substring_view(0, *index),
  473. pattern.substring_view(*index + remove.length()));
  474. }
  475. }
  476. return pattern;
  477. }
  478. static Optional<CalendarPattern> parse_date_time_pattern_raw(String pattern, String skeleton, UnicodeLocaleData& locale_data)
  479. {
  480. // https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table
  481. using Unicode::CalendarPatternStyle;
  482. CalendarPattern format {};
  483. if (!skeleton.is_empty())
  484. format.skeleton_index = locale_data.unique_strings.ensure(move(skeleton));
  485. GenericLexer lexer { pattern };
  486. StringBuilder builder;
  487. bool hour12 { false };
  488. while (!lexer.is_eof()) {
  489. // Literal strings enclosed by quotes are to be appended to the pattern as-is without further
  490. // processing (this just avoids conflicts with the patterns below).
  491. if (lexer.next_is(is_quote)) {
  492. builder.append(lexer.consume_quoted_string());
  493. continue;
  494. }
  495. auto starting_char = lexer.peek();
  496. auto segment = lexer.consume_while([&](char ch) { return ch == starting_char; });
  497. // Era
  498. if (all_of(segment, is_char('G'))) {
  499. builder.append("{era}");
  500. if (segment.length() <= 3)
  501. format.era = CalendarPatternStyle::Short;
  502. else if (segment.length() == 4)
  503. format.era = CalendarPatternStyle::Long;
  504. else
  505. format.era = CalendarPatternStyle::Narrow;
  506. }
  507. // Year
  508. else if (all_of(segment, is_any_of("yYuUr"sv))) {
  509. builder.append("{year}");
  510. if (segment.length() == 2)
  511. format.year = CalendarPatternStyle::TwoDigit;
  512. else
  513. format.year = CalendarPatternStyle::Numeric;
  514. }
  515. // Quarter
  516. else if (all_of(segment, is_any_of("qQ"sv))) {
  517. // Intl.DateTimeFormat does not support quarter formatting, so drop these patterns.
  518. return {};
  519. }
  520. // Month
  521. else if (all_of(segment, is_any_of("ML"sv))) {
  522. builder.append("{month}");
  523. if (segment.length() == 1)
  524. format.month = CalendarPatternStyle::Numeric;
  525. else if (segment.length() == 2)
  526. format.month = CalendarPatternStyle::TwoDigit;
  527. else if (segment.length() == 3)
  528. format.month = CalendarPatternStyle::Short;
  529. else if (segment.length() == 4)
  530. format.month = CalendarPatternStyle::Long;
  531. else if (segment.length() == 5)
  532. format.month = CalendarPatternStyle::Narrow;
  533. } else if (all_of(segment, is_char('l'))) {
  534. // Using 'l' for month formatting is deprecated by TR-35, ensure it is not used.
  535. return {};
  536. }
  537. // Week
  538. else if (all_of(segment, is_any_of("wW"sv))) {
  539. // Intl.DateTimeFormat does not support week formatting, so drop these patterns.
  540. return {};
  541. }
  542. // Day
  543. else if (all_of(segment, is_char('d'))) {
  544. builder.append("{day}");
  545. if (segment.length() == 1)
  546. format.day = CalendarPatternStyle::Numeric;
  547. else
  548. format.day = CalendarPatternStyle::TwoDigit;
  549. } else if (all_of(segment, is_any_of("DFg"sv))) {
  550. builder.append("{day}");
  551. format.day = CalendarPatternStyle::Numeric;
  552. }
  553. // Weekday
  554. else if (all_of(segment, is_char('E'))) {
  555. builder.append("{weekday}");
  556. if (segment.length() == 4)
  557. format.weekday = CalendarPatternStyle::Long;
  558. else if (segment.length() == 5)
  559. format.weekday = CalendarPatternStyle::Narrow;
  560. else
  561. format.weekday = CalendarPatternStyle::Short;
  562. } else if (all_of(segment, is_any_of("ec"sv))) {
  563. builder.append("{weekday}");
  564. // TR-35 defines "e", "c", and "cc" as as numeric, and "ee" as 2-digit, but those
  565. // pattern styles are not supported by Intl.DateTimeFormat.
  566. if (segment.length() <= 2)
  567. return {};
  568. if (segment.length() == 4)
  569. format.weekday = CalendarPatternStyle::Long;
  570. else if (segment.length() == 5)
  571. format.weekday = CalendarPatternStyle::Narrow;
  572. else
  573. format.weekday = CalendarPatternStyle::Short;
  574. }
  575. // Period
  576. else if (all_of(segment, is_any_of("ab"sv))) {
  577. builder.append("{ampm}");
  578. hour12 = true;
  579. } else if (all_of(segment, is_char('B'))) {
  580. builder.append("{dayPeriod}");
  581. hour12 = true;
  582. if (segment.length() == 4)
  583. format.day_period = CalendarPatternStyle::Long;
  584. else if (segment.length() == 5)
  585. format.day_period = CalendarPatternStyle::Narrow;
  586. else
  587. format.day_period = CalendarPatternStyle::Short;
  588. }
  589. // Hour
  590. else if (all_of(segment, is_any_of("hHKk"sv))) {
  591. builder.append("{hour}");
  592. if ((segment[0] == 'h') || (segment[0] == 'K'))
  593. hour12 = true;
  594. if (segment.length() == 1)
  595. format.hour = CalendarPatternStyle::Numeric;
  596. else
  597. format.hour = CalendarPatternStyle::TwoDigit;
  598. } else if (all_of(segment, is_any_of("jJC"sv))) {
  599. // TR-35 indicates these should not be used.
  600. return {};
  601. }
  602. // Minute
  603. else if (all_of(segment, is_char('m'))) {
  604. builder.append("{minute}");
  605. if (segment.length() == 1)
  606. format.minute = CalendarPatternStyle::Numeric;
  607. else
  608. format.minute = CalendarPatternStyle::TwoDigit;
  609. }
  610. // Second
  611. else if (all_of(segment, is_char('s'))) {
  612. builder.append("{second}");
  613. if (segment.length() == 1)
  614. format.second = CalendarPatternStyle::Numeric;
  615. else
  616. format.second = CalendarPatternStyle::TwoDigit;
  617. } else if (all_of(segment, is_char('S'))) {
  618. builder.append("{fractionalSecondDigits}");
  619. VERIFY(segment.length() <= 3);
  620. format.fractional_second_digits = static_cast<u8>(segment.length());
  621. } else if (all_of(segment, is_char('A'))) {
  622. // Intl.DateTimeFormat does not support millisecond formatting, so drop these patterns.
  623. return {};
  624. }
  625. // Zone
  626. else if (all_of(segment, is_any_of("zZOvVXx"))) {
  627. builder.append("{timeZoneName}");
  628. if (segment.length() < 4)
  629. format.time_zone_name = CalendarPatternStyle::Short;
  630. else
  631. format.time_zone_name = CalendarPatternStyle::Long;
  632. }
  633. // Non-patterns
  634. else {
  635. builder.append(segment);
  636. }
  637. }
  638. pattern = builder.build();
  639. if (hour12) {
  640. format.pattern = remove_period_from_pattern(pattern);
  641. format.pattern12 = move(pattern);
  642. } else {
  643. format.pattern = move(pattern);
  644. }
  645. return format;
  646. }
  647. static Optional<CalendarPatternIndexType> parse_date_time_pattern(String pattern, String skeleton, UnicodeLocaleData& locale_data)
  648. {
  649. auto format = parse_date_time_pattern_raw(move(pattern), move(skeleton), locale_data);
  650. if (!format.has_value())
  651. return {};
  652. format->pattern_index = locale_data.unique_strings.ensure(move(format->pattern));
  653. if (format->pattern12.has_value())
  654. format->pattern12_index = locale_data.unique_strings.ensure(format->pattern12.release_value());
  655. return locale_data.unique_patterns.ensure(format.release_value());
  656. }
  657. template<typename... Chars>
  658. static constexpr bool char_is_one_of(char ch, Chars&&... chars)
  659. {
  660. return ((ch == chars) || ...);
  661. }
  662. static void parse_interval_patterns(Calendar& calendar, JsonObject const& interval_formats_object, UnicodeLocaleData& locale_data)
  663. {
  664. // https://unicode.org/reports/tr35/tr35-dates.html#intervalFormats
  665. CalendarRangePatternList range_formats {};
  666. CalendarRangePatternList range12_formats {};
  667. auto name_of_field = [&](char field) {
  668. if (char_is_one_of(field, 'G'))
  669. return Unicode::CalendarRangePattern::Field::Era;
  670. if (char_is_one_of(field, 'y', 'Y', 'u', 'U', 'r'))
  671. return Unicode::CalendarRangePattern::Field::Year;
  672. if (char_is_one_of(field, 'M', 'L'))
  673. return Unicode::CalendarRangePattern::Field::Month;
  674. if (char_is_one_of(field, 'd', 'D', 'F', 'g'))
  675. return Unicode::CalendarRangePattern::Field::Day;
  676. if (char_is_one_of(field, 'a', 'b'))
  677. return Unicode::CalendarRangePattern::Field::AmPm;
  678. if (char_is_one_of(field, 'B'))
  679. return Unicode::CalendarRangePattern::Field::DayPeriod;
  680. if (char_is_one_of(field, 'h', 'H', 'K', 'k'))
  681. return Unicode::CalendarRangePattern::Field::Hour;
  682. if (char_is_one_of(field, 'm'))
  683. return Unicode::CalendarRangePattern::Field::Minute;
  684. if (char_is_one_of(field, 's'))
  685. return Unicode::CalendarRangePattern::Field::Second;
  686. if (char_is_one_of(field, 'S'))
  687. return Unicode::CalendarRangePattern::Field::FractionalSecondDigits;
  688. VERIFY_NOT_REACHED();
  689. };
  690. auto split_default_range_pattern = [&](auto skeleton, auto const& pattern) {
  691. auto start_range_end = pattern.find('}').value() + 1;
  692. auto end_range_begin = pattern.find_last('{').value();
  693. auto start_range = pattern.substring_view(0, start_range_end);
  694. auto separator = pattern.substring_view(start_range_end, end_range_begin - start_range_end);
  695. auto end_range = pattern.substring_view(end_range_begin);
  696. CalendarRangePattern format {};
  697. format.skeleton_index = locale_data.unique_strings.ensure(skeleton);
  698. format.start_range = locale_data.unique_strings.ensure(start_range);
  699. format.separator = locale_data.unique_strings.ensure(separator);
  700. format.end_range = locale_data.unique_strings.ensure(end_range);
  701. return format;
  702. };
  703. auto split_range_pattern = [&](auto skeleton, auto field, auto const& pattern, auto const& parsed_fields) {
  704. HashMap<StringView, size_t> partitions;
  705. StringView last_partition;
  706. auto begin_index = pattern.find('{');
  707. size_t end_index = 0;
  708. while (begin_index.has_value()) {
  709. end_index = pattern.find('}', *begin_index).value();
  710. auto partition = pattern.substring_view(*begin_index, end_index - *begin_index);
  711. if (partitions.contains(partition))
  712. break;
  713. partitions.set(partition, *begin_index);
  714. last_partition = partition;
  715. begin_index = pattern.find('{', end_index + 1);
  716. }
  717. VERIFY(begin_index.has_value() && !last_partition.is_empty());
  718. auto start_range_end = partitions.get(last_partition).value() + last_partition.length() + 1;
  719. auto start_range = pattern.substring_view(0, start_range_end);
  720. auto separator = pattern.substring_view(start_range_end, *begin_index - start_range_end);
  721. auto end_range = pattern.substring_view(*begin_index);
  722. CalendarRangePattern format {};
  723. format.skeleton_index = locale_data.unique_strings.ensure(skeleton);
  724. format.field = field;
  725. format.start_range = locale_data.unique_strings.ensure(start_range);
  726. format.separator = locale_data.unique_strings.ensure(separator);
  727. format.end_range = locale_data.unique_strings.ensure(end_range);
  728. format.for_each_calendar_field_zipped_with(parsed_fields, [](auto& format_field, auto const& parsed_field, auto) {
  729. format_field = parsed_field;
  730. });
  731. return format;
  732. };
  733. interval_formats_object.for_each_member([&](auto const& skeleton, auto const& value) {
  734. if (skeleton == "intervalFormatFallback"sv) {
  735. auto range_format = split_default_range_pattern(skeleton, value.as_string());
  736. calendar.default_range_format = locale_data.unique_range_patterns.ensure(move(range_format));
  737. return;
  738. }
  739. value.as_object().for_each_member([&](auto const& field, auto const& pattern) {
  740. if (field.ends_with("alt-variant"sv))
  741. return;
  742. VERIFY(field.length() == 1);
  743. auto name = name_of_field(field[0]);
  744. auto format = parse_date_time_pattern_raw(pattern.as_string(), skeleton, locale_data).release_value();
  745. auto range_format = split_range_pattern(skeleton, name, format.pattern, format);
  746. range_formats.append(locale_data.unique_range_patterns.ensure(move(range_format)));
  747. if (format.pattern12.has_value()) {
  748. auto range12_pattern = split_range_pattern(skeleton, name, *format.pattern12, format);
  749. range12_formats.append(locale_data.unique_range_patterns.ensure(move(range12_pattern)));
  750. } else {
  751. range12_formats.append(range_formats.last());
  752. }
  753. });
  754. });
  755. calendar.range_formats = locale_data.unique_range_pattern_lists.ensure(move(range_formats));
  756. calendar.range12_formats = locale_data.unique_range_pattern_lists.ensure(move(range12_formats));
  757. }
  758. static void generate_missing_patterns(Calendar& calendar, CalendarPatternList& formats, Vector<CalendarPattern> date_formats, Vector<CalendarPattern> time_formats, UnicodeLocaleData& locale_data)
  759. {
  760. // https://unicode.org/reports/tr35/tr35-dates.html#Missing_Skeleton_Fields
  761. auto replace_pattern = [&](auto format, auto time_format, auto date_format) {
  762. auto pattern = locale_data.unique_strings.get(format);
  763. auto time_pattern = locale_data.unique_strings.get(time_format);
  764. auto date_pattern = locale_data.unique_strings.get(date_format);
  765. auto new_pattern = pattern.replace("{0}", time_pattern).replace("{1}", date_pattern);
  766. return locale_data.unique_strings.ensure(move(new_pattern));
  767. };
  768. auto inject_fractional_second_digits = [&](auto format) {
  769. auto pattern = locale_data.unique_strings.get(format);
  770. auto new_pattern = pattern.replace("{second}"sv, "{second}{decimal}{fractionalSecondDigits}"sv);
  771. return locale_data.unique_strings.ensure(move(new_pattern));
  772. };
  773. auto append_if_unique = [&](auto format) {
  774. auto format_index = locale_data.unique_patterns.ensure(move(format));
  775. if (!formats.contains_slow(format_index))
  776. formats.append(format_index);
  777. };
  778. Vector<CalendarPattern> time_formats_with_fractional_second_digits;
  779. for (auto const& format : date_formats)
  780. append_if_unique(format);
  781. for (auto const& format : time_formats) {
  782. append_if_unique(format);
  783. if (format.second.has_value() && !format.fractional_second_digits.has_value()) {
  784. auto new_format = format;
  785. new_format.fractional_second_digits = 2;
  786. new_format.pattern_index = inject_fractional_second_digits(new_format.pattern_index);
  787. if (new_format.pattern12_index != 0)
  788. new_format.pattern12_index = inject_fractional_second_digits(new_format.pattern12_index);
  789. time_formats_with_fractional_second_digits.append(new_format);
  790. append_if_unique(move(new_format));
  791. }
  792. }
  793. time_formats.extend(move(time_formats_with_fractional_second_digits));
  794. for (auto const& date_format : date_formats) {
  795. auto const& date_time_formats = locale_data.unique_formats.get(calendar.date_time_formats);
  796. CalendarPatternIndexType date_time_format_index = 0;
  797. if (date_format.month == Unicode::CalendarPatternStyle::Long) {
  798. if (date_format.weekday.has_value())
  799. date_time_format_index = date_time_formats.full_format;
  800. else
  801. date_time_format_index = date_time_formats.long_format;
  802. } else if (date_format.month == Unicode::CalendarPatternStyle::Short) {
  803. date_time_format_index = date_time_formats.medium_format;
  804. } else {
  805. date_time_format_index = date_time_formats.short_format;
  806. }
  807. for (auto const& time_format : time_formats) {
  808. auto format = locale_data.unique_patterns.get(date_time_format_index);
  809. if (time_format.pattern12_index != 0)
  810. format.pattern12_index = replace_pattern(format.pattern_index, time_format.pattern12_index, date_format.pattern_index);
  811. format.pattern_index = replace_pattern(format.pattern_index, time_format.pattern_index, date_format.pattern_index);
  812. format.for_each_calendar_field_zipped_with(date_format, [](auto& field, auto const& date_field, auto) {
  813. if (date_field.has_value())
  814. field = date_field;
  815. });
  816. format.for_each_calendar_field_zipped_with(time_format, [](auto& field, auto const& time_field, auto) {
  817. if (time_field.has_value())
  818. field = time_field;
  819. });
  820. append_if_unique(move(format));
  821. }
  822. }
  823. }
  824. static void parse_calendar_symbols(Calendar& calendar, JsonObject const& calendar_object, UnicodeLocaleData& locale_data)
  825. {
  826. auto create_symbol_lists = [](size_t size) {
  827. SymbolList narrow_symbol_list;
  828. SymbolList short_symbol_list;
  829. SymbolList long_symbol_list;
  830. narrow_symbol_list.resize(size);
  831. short_symbol_list.resize(size);
  832. long_symbol_list.resize(size);
  833. return Array<SymbolList, 3> { {
  834. move(narrow_symbol_list),
  835. move(short_symbol_list),
  836. move(long_symbol_list),
  837. } };
  838. };
  839. CalendarSymbolsList symbols_list {};
  840. auto store_symbol_lists = [&](auto symbol, auto symbol_lists) {
  841. auto symbol_index = to_underlying(symbol);
  842. if (symbol_index >= symbols_list.size())
  843. symbols_list.resize(symbol_index + 1);
  844. CalendarSymbols symbols {};
  845. symbols.narrow_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[0]));
  846. symbols.short_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[1]));
  847. symbols.long_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[2]));
  848. auto calendar_symbols_index = locale_data.unique_calendar_symbols.ensure(move(symbols));
  849. symbols_list[symbol_index] = calendar_symbols_index;
  850. };
  851. auto parse_era_symbols = [&](auto const& symbols_object) {
  852. auto const& narrow_symbols = symbols_object.get("eraNarrow"sv).as_object();
  853. auto const& short_symbols = symbols_object.get("eraAbbr"sv).as_object();
  854. auto const& long_symbols = symbols_object.get("eraNames"sv).as_object();
  855. auto symbol_lists = create_symbol_lists(2);
  856. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  857. if (auto key_index = key.to_uint(); key_index.has_value())
  858. symbols[*key_index] = locale_data.unique_strings.ensure(move(symbol));
  859. };
  860. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  861. append_symbol(symbol_lists[0], key, value.as_string());
  862. });
  863. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  864. append_symbol(symbol_lists[1], key, value.as_string());
  865. });
  866. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  867. append_symbol(symbol_lists[2], key, value.as_string());
  868. });
  869. store_symbol_lists(Unicode::CalendarSymbol::Era, move(symbol_lists));
  870. };
  871. auto parse_month_symbols = [&](auto const& symbols_object) {
  872. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  873. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  874. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  875. auto symbol_lists = create_symbol_lists(12);
  876. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  877. auto key_index = key.to_uint().value() - 1;
  878. symbols[key_index] = locale_data.unique_strings.ensure(move(symbol));
  879. };
  880. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  881. append_symbol(symbol_lists[0], key, value.as_string());
  882. });
  883. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  884. append_symbol(symbol_lists[1], key, value.as_string());
  885. });
  886. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  887. append_symbol(symbol_lists[2], key, value.as_string());
  888. });
  889. store_symbol_lists(Unicode::CalendarSymbol::Month, move(symbol_lists));
  890. };
  891. auto parse_weekday_symbols = [&](auto const& symbols_object) {
  892. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  893. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  894. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  895. auto symbol_lists = create_symbol_lists(7);
  896. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  897. if (key == "sun"sv)
  898. symbols[to_underlying(Unicode::Weekday::Sunday)] = locale_data.unique_strings.ensure(move(symbol));
  899. else if (key == "mon"sv)
  900. symbols[to_underlying(Unicode::Weekday::Monday)] = locale_data.unique_strings.ensure(move(symbol));
  901. else if (key == "tue"sv)
  902. symbols[to_underlying(Unicode::Weekday::Tuesday)] = locale_data.unique_strings.ensure(move(symbol));
  903. else if (key == "wed"sv)
  904. symbols[to_underlying(Unicode::Weekday::Wednesday)] = locale_data.unique_strings.ensure(move(symbol));
  905. else if (key == "thu"sv)
  906. symbols[to_underlying(Unicode::Weekday::Thursday)] = locale_data.unique_strings.ensure(move(symbol));
  907. else if (key == "fri"sv)
  908. symbols[to_underlying(Unicode::Weekday::Friday)] = locale_data.unique_strings.ensure(move(symbol));
  909. else if (key == "sat"sv)
  910. symbols[to_underlying(Unicode::Weekday::Saturday)] = locale_data.unique_strings.ensure(move(symbol));
  911. };
  912. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  913. append_symbol(symbol_lists[0], key, value.as_string());
  914. });
  915. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  916. append_symbol(symbol_lists[1], key, value.as_string());
  917. });
  918. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  919. append_symbol(symbol_lists[2], key, value.as_string());
  920. });
  921. store_symbol_lists(Unicode::CalendarSymbol::Weekday, move(symbol_lists));
  922. };
  923. auto parse_day_period_symbols = [&](auto const& symbols_object) {
  924. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  925. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  926. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  927. auto symbol_lists = create_symbol_lists(10);
  928. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  929. if (auto day_period = day_period_from_string(key); day_period.has_value())
  930. symbols[to_underlying(*day_period)] = locale_data.unique_strings.ensure(move(symbol));
  931. };
  932. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  933. append_symbol(symbol_lists[0], key, value.as_string());
  934. });
  935. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  936. append_symbol(symbol_lists[1], key, value.as_string());
  937. });
  938. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  939. append_symbol(symbol_lists[2], key, value.as_string());
  940. });
  941. store_symbol_lists(Unicode::CalendarSymbol::DayPeriod, move(symbol_lists));
  942. };
  943. parse_era_symbols(calendar_object.get("eras"sv).as_object());
  944. parse_month_symbols(calendar_object.get("months"sv).as_object().get("format"sv).as_object());
  945. parse_weekday_symbols(calendar_object.get("days"sv).as_object().get("format"sv).as_object());
  946. parse_day_period_symbols(calendar_object.get("dayPeriods"sv).as_object().get("format"sv).as_object());
  947. calendar.symbols = locale_data.unique_calendar_symbols_lists.ensure(move(symbols_list));
  948. }
  949. static ErrorOr<void> parse_calendars(String locale_calendars_path, UnicodeLocaleData& locale_data, Locale& locale)
  950. {
  951. LexicalPath calendars_path(move(locale_calendars_path));
  952. if (!calendars_path.basename().starts_with("ca-"sv))
  953. return {};
  954. auto calendars_file = TRY(Core::File::open(calendars_path.string(), Core::OpenMode::ReadOnly));
  955. auto calendars = TRY(JsonValue::from_string(calendars_file->read_all()));
  956. auto const& main_object = calendars.as_object().get("main"sv);
  957. auto const& locale_object = main_object.as_object().get(calendars_path.parent().basename());
  958. auto const& dates_object = locale_object.as_object().get("dates"sv);
  959. auto const& calendars_object = dates_object.as_object().get("calendars"sv);
  960. auto create_calendar = [&](auto const& calendar) {
  961. auto calendar_index = locale_data.unique_strings.ensure(calendar);
  962. return Calendar { .calendar = calendar_index };
  963. };
  964. auto parse_patterns = [&](auto const& patterns_object, auto const& skeletons_object, Vector<CalendarPattern>* patterns) {
  965. auto parse_pattern = [&](auto name) {
  966. auto format = patterns_object.get(name);
  967. auto skeleton = skeletons_object.get(name);
  968. auto format_index = parse_date_time_pattern(format.as_string(), skeleton.as_string_or(String::empty()), locale_data).value();
  969. if (patterns)
  970. patterns->append(locale_data.unique_patterns.get(format_index));
  971. return format_index;
  972. };
  973. CalendarFormat formats {};
  974. formats.full_format = parse_pattern("full"sv);
  975. formats.long_format = parse_pattern("long"sv);
  976. formats.medium_format = parse_pattern("medium"sv);
  977. formats.short_format = parse_pattern("short"sv);
  978. return locale_data.unique_formats.ensure(move(formats));
  979. };
  980. calendars_object.as_object().for_each_member([&](auto const& calendar_name, JsonValue const& value) {
  981. // The generic calendar is not a supported Unicode calendar key, so skip it:
  982. // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Locale/calendar#unicode_calendar_keys
  983. if (calendar_name == "generic"sv)
  984. return;
  985. auto calendar = create_calendar(calendar_name);
  986. CalendarPatternList available_formats {};
  987. if (!locale_data.calendars.contains_slow(calendar_name))
  988. locale_data.calendars.append(calendar_name);
  989. Vector<CalendarPattern> date_formats;
  990. Vector<CalendarPattern> time_formats;
  991. auto const& date_formats_object = value.as_object().get("dateFormats"sv);
  992. auto const& date_skeletons_object = value.as_object().get("dateSkeletons"sv);
  993. calendar.date_formats = parse_patterns(date_formats_object.as_object(), date_skeletons_object.as_object(), &date_formats);
  994. auto const& time_formats_object = value.as_object().get("timeFormats"sv);
  995. auto const& time_skeletons_object = value.as_object().get("timeSkeletons"sv);
  996. calendar.time_formats = parse_patterns(time_formats_object.as_object(), time_skeletons_object.as_object(), &time_formats);
  997. auto const& date_time_formats_object = value.as_object().get("dateTimeFormats"sv);
  998. calendar.date_time_formats = parse_patterns(date_time_formats_object.as_object(), JsonObject {}, nullptr);
  999. auto const& available_formats_object = date_time_formats_object.as_object().get("availableFormats"sv);
  1000. available_formats_object.as_object().for_each_member([&](auto const& skeleton, JsonValue const& pattern) {
  1001. auto pattern_index = parse_date_time_pattern(pattern.as_string(), skeleton, locale_data);
  1002. if (!pattern_index.has_value())
  1003. return;
  1004. auto const& format = locale_data.unique_patterns.get(*pattern_index);
  1005. if (format.contains_only_date_fields())
  1006. date_formats.append(format);
  1007. else if (format.contains_only_time_fields())
  1008. time_formats.append(format);
  1009. if (!available_formats.contains_slow(*pattern_index))
  1010. available_formats.append(*pattern_index);
  1011. });
  1012. auto const& interval_formats_object = date_time_formats_object.as_object().get("intervalFormats"sv);
  1013. parse_interval_patterns(calendar, interval_formats_object.as_object(), locale_data);
  1014. generate_missing_patterns(calendar, available_formats, move(date_formats), move(time_formats), locale_data);
  1015. parse_calendar_symbols(calendar, value.as_object(), locale_data);
  1016. calendar.available_formats = locale_data.unique_pattern_lists.ensure(move(available_formats));
  1017. locale.calendars.set(calendar_name, locale_data.unique_calendars.ensure(move(calendar)));
  1018. });
  1019. return {};
  1020. }
  1021. static ErrorOr<void> parse_time_zone_names(String locale_time_zone_names_path, UnicodeLocaleData& locale_data, Locale& locale)
  1022. {
  1023. LexicalPath time_zone_names_path(move(locale_time_zone_names_path));
  1024. time_zone_names_path = time_zone_names_path.append("timeZoneNames.json"sv);
  1025. auto time_zone_names_file = TRY(Core::File::open(time_zone_names_path.string(), Core::OpenMode::ReadOnly));
  1026. auto time_zone_names = TRY(JsonValue::from_string(time_zone_names_file->read_all()));
  1027. auto const& main_object = time_zone_names.as_object().get("main"sv);
  1028. auto const& locale_object = main_object.as_object().get(time_zone_names_path.parent().basename());
  1029. auto const& dates_object = locale_object.as_object().get("dates"sv);
  1030. auto const& time_zone_names_object = dates_object.as_object().get("timeZoneNames"sv);
  1031. auto const& meta_zone_object = time_zone_names_object.as_object().get("metazone"sv);
  1032. if (meta_zone_object.is_null())
  1033. return {};
  1034. auto parse_name = [&](StringView type, JsonObject const& meta_zone_object) -> Optional<StringIndexType> {
  1035. auto const& names = meta_zone_object.get(type);
  1036. if (!names.is_object())
  1037. return {};
  1038. auto const& daylight = names.as_object().get("daylight"sv);
  1039. if (daylight.is_string())
  1040. return locale_data.unique_strings.ensure(daylight.as_string());
  1041. auto const& standard = names.as_object().get("standard"sv);
  1042. if (standard.is_string())
  1043. return locale_data.unique_strings.ensure(standard.as_string());
  1044. return {};
  1045. };
  1046. auto parse_time_zone = [&](StringView meta_zone, JsonObject const& meta_zone_object) {
  1047. auto golden_zone = locale_data.meta_zones.get(meta_zone).value();
  1048. TimeZone time_zone { .time_zone = golden_zone };
  1049. if (auto long_name = parse_name("long"sv, meta_zone_object); long_name.has_value())
  1050. time_zone.long_name = long_name.value();
  1051. if (auto short_name = parse_name("short"sv, meta_zone_object); short_name.has_value())
  1052. time_zone.short_name = short_name.value();
  1053. auto const& time_zone_name = locale_data.unique_strings.get(golden_zone);
  1054. if (!locale_data.time_zones.contains_slow(time_zone_name))
  1055. locale_data.time_zones.append(time_zone_name);
  1056. locale.time_zones.set(time_zone_name, move(time_zone));
  1057. };
  1058. meta_zone_object.as_object().for_each_member([&](auto const& meta_zone, JsonValue const& value) {
  1059. parse_time_zone(meta_zone, value.as_object());
  1060. });
  1061. // The long and short names for UTC are not under the "timeZoneNames/metazone" object, but are under "timeZoneNames/zone/Etc".
  1062. auto const& zone_object = time_zone_names_object.as_object().get("zone"sv);
  1063. auto const& etc_object = zone_object.as_object().get("Etc"sv);
  1064. auto const& utc_object = etc_object.as_object().get("UTC"sv);
  1065. parse_time_zone("UTC"sv, utc_object.as_object());
  1066. return {};
  1067. }
  1068. static ErrorOr<void> parse_day_periods(String core_path, UnicodeLocaleData& locale_data)
  1069. {
  1070. // https://unicode.org/reports/tr35/tr35-dates.html#Day_Period_Rule_Sets
  1071. LexicalPath day_periods_path(move(core_path));
  1072. day_periods_path = day_periods_path.append("supplemental"sv);
  1073. day_periods_path = day_periods_path.append("dayPeriods.json"sv);
  1074. auto day_periods_file = TRY(Core::File::open(day_periods_path.string(), Core::OpenMode::ReadOnly));
  1075. auto day_periods = TRY(JsonValue::from_string(day_periods_file->read_all()));
  1076. auto const& supplemental_object = day_periods.as_object().get("supplemental"sv);
  1077. auto const& day_periods_object = supplemental_object.as_object().get("dayPeriodRuleSet"sv);
  1078. auto parse_hour = [](auto const& time) {
  1079. auto hour_end_index = time.find(':').value();
  1080. // The times are in the form "hours:minutes", but we only need the hour segment.
  1081. // TR-35 explicitly states that minutes other than :00 must not be used.
  1082. VERIFY(time.substring_view(hour_end_index) == ":00"sv);
  1083. auto hour = time.substring_view(0, hour_end_index);
  1084. return hour.template to_uint<u8>().value();
  1085. };
  1086. auto parse_day_period = [&](auto const& symbol, auto const& ranges) -> Optional<DayPeriod> {
  1087. auto day_period = day_period_from_string(symbol);
  1088. if (!day_period.has_value())
  1089. return {};
  1090. auto begin = parse_hour(ranges.get("_from"sv).as_string());
  1091. auto end = parse_hour(ranges.get("_before"sv).as_string());
  1092. return DayPeriod { *day_period, begin, end };
  1093. };
  1094. day_periods_object.as_object().for_each_member([&](auto const& language, JsonValue const& value) {
  1095. auto locale = locale_data.locales.find(language);
  1096. if (locale == locale_data.locales.end())
  1097. return;
  1098. value.as_object().for_each_member([&](auto const& symbol, JsonValue const& ranges) {
  1099. if (auto day_period = parse_day_period(symbol, ranges.as_object()); day_period.has_value())
  1100. locale->value.day_periods.append(day_period.release_value());
  1101. });
  1102. });
  1103. return {};
  1104. };
  1105. static ErrorOr<void> parse_all_locales(String core_path, String dates_path, UnicodeLocaleData& locale_data)
  1106. {
  1107. TRY(parse_hour_cycles(core_path, locale_data));
  1108. TRY(parse_meta_zones(core_path, locale_data));
  1109. auto dates_iterator = TRY(path_to_dir_iterator(move(dates_path)));
  1110. auto remove_variants_from_path = [&](String path) -> ErrorOr<String> {
  1111. auto parsed_locale = TRY(CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, LexicalPath::basename(path)));
  1112. StringBuilder builder;
  1113. builder.append(locale_data.unique_strings.get(parsed_locale.language));
  1114. if (auto script = locale_data.unique_strings.get(parsed_locale.script); !script.is_empty())
  1115. builder.appendff("-{}", script);
  1116. if (auto region = locale_data.unique_strings.get(parsed_locale.region); !region.is_empty())
  1117. builder.appendff("-{}", region);
  1118. return builder.build();
  1119. };
  1120. while (dates_iterator.has_next()) {
  1121. auto dates_path = TRY(next_path_from_dir_iterator(dates_iterator));
  1122. auto calendars_iterator = TRY(path_to_dir_iterator(dates_path, {}));
  1123. auto language = TRY(remove_variants_from_path(dates_path));
  1124. auto& locale = locale_data.locales.ensure(language);
  1125. while (calendars_iterator.has_next()) {
  1126. auto calendars_path = TRY(next_path_from_dir_iterator(calendars_iterator));
  1127. TRY(parse_calendars(move(calendars_path), locale_data, locale));
  1128. }
  1129. TRY(parse_time_zone_names(move(dates_path), locale_data, locale));
  1130. }
  1131. TRY(parse_day_periods(move(core_path), locale_data));
  1132. return {};
  1133. }
  1134. static String format_identifier(StringView owner, String identifier)
  1135. {
  1136. identifier = identifier.replace("-"sv, "_"sv, true);
  1137. identifier = identifier.replace("/"sv, "_"sv, true);
  1138. if (all_of(identifier, is_ascii_digit))
  1139. return String::formatted("{}_{}", owner[0], identifier);
  1140. if (is_ascii_lower_alpha(identifier[0]))
  1141. return String::formatted("{:c}{}", to_ascii_uppercase(identifier[0]), identifier.substring_view(1));
  1142. return identifier;
  1143. }
  1144. static void generate_unicode_locale_header(Core::File& file, UnicodeLocaleData& locale_data)
  1145. {
  1146. StringBuilder builder;
  1147. SourceGenerator generator { builder };
  1148. generator.append(R"~~~(
  1149. #pragma once
  1150. #include <AK/Optional.h>
  1151. #include <AK/StringView.h>
  1152. #include <LibUnicode/Forward.h>
  1153. namespace Unicode {
  1154. )~~~");
  1155. generate_enum(generator, format_identifier, "Calendar"sv, {}, locale_data.calendars, locale_data.calendar_aliases);
  1156. generate_enum(generator, format_identifier, "HourCycleRegion"sv, {}, locale_data.hour_cycle_regions);
  1157. generate_enum(generator, format_identifier, "TimeZone"sv, {}, locale_data.time_zones);
  1158. generator.append(R"~~~(
  1159. namespace Detail {
  1160. Optional<Calendar> calendar_from_string(StringView calendar);
  1161. Optional<HourCycleRegion> hour_cycle_region_from_string(StringView hour_cycle_region);
  1162. Vector<Unicode::HourCycle> get_regional_hour_cycles(StringView region);
  1163. Optional<Unicode::CalendarFormat> get_calendar_date_format(StringView locale, StringView calendar);
  1164. Optional<Unicode::CalendarFormat> get_calendar_time_format(StringView locale, StringView calendar);
  1165. Optional<Unicode::CalendarFormat> get_calendar_date_time_format(StringView locale, StringView calendar);
  1166. Vector<Unicode::CalendarPattern> get_calendar_available_formats(StringView locale, StringView calendar);
  1167. Optional<Unicode::CalendarRangePattern> get_calendar_default_range_format(StringView locale, StringView calendar);
  1168. Vector<Unicode::CalendarRangePattern> get_calendar_range_formats(StringView locale, StringView calendar, StringView skeleton);
  1169. Vector<Unicode::CalendarRangePattern> get_calendar_range12_formats(StringView locale, StringView calendar, StringView skeleton);
  1170. Optional<StringView> get_calendar_era_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Era value);
  1171. Optional<StringView> get_calendar_month_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Month value);
  1172. Optional<StringView> get_calendar_weekday_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Weekday value);
  1173. Optional<StringView> get_calendar_day_period_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::DayPeriod value);
  1174. Optional<StringView> get_calendar_day_period_symbol_for_hour(StringView locale, StringView calendar, CalendarPatternStyle style, u8 hour);
  1175. Optional<TimeZone> time_zone_from_string(StringView time_zone);
  1176. Optional<StringView> get_time_zone_name(StringView locale, StringView time_zone, CalendarPatternStyle style);
  1177. }
  1178. }
  1179. )~~~");
  1180. VERIFY(file.write(generator.as_string_view()));
  1181. }
  1182. static void generate_unicode_locale_implementation(Core::File& file, UnicodeLocaleData& locale_data)
  1183. {
  1184. StringBuilder builder;
  1185. SourceGenerator generator { builder };
  1186. generator.set("string_index_type"sv, s_string_index_type);
  1187. generator.set("calendar_pattern_index_type"sv, s_calendar_pattern_index_type);
  1188. generator.set("calendar_pattern_list_index_type"sv, s_calendar_pattern_list_index_type);
  1189. generator.set("calendar_range_pattern_index_type"sv, s_calendar_range_pattern_index_type);
  1190. generator.set("calendar_range_pattern_list_index_type"sv, s_calendar_range_pattern_list_index_type);
  1191. generator.set("calendar_format_index_type"sv, s_calendar_format_index_type);
  1192. generator.set("symbol_list_index_type"sv, s_symbol_list_index_type);
  1193. generator.set("calendar_symbols_index_type"sv, s_calendar_symbols_index_type);
  1194. generator.set("calendar_symbols_list_index_type"sv, s_calendar_symbols_list_index_type);
  1195. generator.set("calendar_index_type"sv, s_calendar_index_type);
  1196. generator.append(R"~~~(
  1197. #include <AK/Array.h>
  1198. #include <AK/BinarySearch.h>
  1199. #include <LibUnicode/DateTimeFormat.h>
  1200. #include <LibUnicode/Locale.h>
  1201. #include <LibUnicode/UnicodeDateTimeFormat.h>
  1202. namespace Unicode::Detail {
  1203. )~~~");
  1204. locale_data.unique_strings.generate(generator);
  1205. generator.append(R"~~~(
  1206. template <typename SourceType, typename TargetType>
  1207. void convert_calendar_fields(SourceType const& source, TargetType& target)
  1208. {
  1209. if (source.era != -1)
  1210. target.era = static_cast<Unicode::CalendarPatternStyle>(source.era);
  1211. if (source.year != -1)
  1212. target.year = static_cast<Unicode::CalendarPatternStyle>(source.year);
  1213. if (source.month != -1)
  1214. target.month = static_cast<Unicode::CalendarPatternStyle>(source.month);
  1215. if (source.weekday != -1)
  1216. target.weekday = static_cast<Unicode::CalendarPatternStyle>(source.weekday);
  1217. if (source.day != -1)
  1218. target.day = static_cast<Unicode::CalendarPatternStyle>(source.day);
  1219. if (source.day_period != -1)
  1220. target.day_period = static_cast<Unicode::CalendarPatternStyle>(source.day_period);
  1221. if (source.hour != -1)
  1222. target.hour = static_cast<Unicode::CalendarPatternStyle>(source.hour);
  1223. if (source.minute != -1)
  1224. target.minute = static_cast<Unicode::CalendarPatternStyle>(source.minute);
  1225. if (source.second != -1)
  1226. target.second = static_cast<Unicode::CalendarPatternStyle>(source.second);
  1227. if (source.fractional_second_digits != -1)
  1228. target.fractional_second_digits = static_cast<u8>(source.fractional_second_digits);
  1229. if (source.time_zone_name != -1)
  1230. target.time_zone_name = static_cast<Unicode::CalendarPatternStyle>(source.time_zone_name);
  1231. }
  1232. struct CalendarPattern {
  1233. Unicode::CalendarPattern to_unicode_calendar_pattern() const {
  1234. Unicode::CalendarPattern calendar_pattern {};
  1235. calendar_pattern.skeleton = s_string_list[skeleton];
  1236. calendar_pattern.pattern = s_string_list[pattern];
  1237. if (pattern12 != 0)
  1238. calendar_pattern.pattern12 = s_string_list[pattern12];
  1239. convert_calendar_fields(*this, calendar_pattern);
  1240. return calendar_pattern;
  1241. }
  1242. @string_index_type@ skeleton { 0 };
  1243. @string_index_type@ pattern { 0 };
  1244. @string_index_type@ pattern12 { 0 };
  1245. i8 era { -1 };
  1246. i8 year { -1 };
  1247. i8 month { -1 };
  1248. i8 weekday { -1 };
  1249. i8 day { -1 };
  1250. i8 day_period { -1 };
  1251. i8 hour { -1 };
  1252. i8 minute { -1 };
  1253. i8 second { -1 };
  1254. i8 fractional_second_digits { -1 };
  1255. i8 time_zone_name { -1 };
  1256. };
  1257. struct CalendarRangePattern {
  1258. Unicode::CalendarRangePattern to_unicode_calendar_range_pattern() const {
  1259. Unicode::CalendarRangePattern calendar_range_pattern {};
  1260. if (field != -1)
  1261. calendar_range_pattern.field = static_cast<Unicode::CalendarRangePattern::Field>(field);
  1262. calendar_range_pattern.start_range = s_string_list[start_range];
  1263. calendar_range_pattern.separator = s_string_list[separator];
  1264. calendar_range_pattern.end_range = s_string_list[end_range];
  1265. convert_calendar_fields(*this, calendar_range_pattern);
  1266. return calendar_range_pattern;
  1267. }
  1268. @string_index_type@ skeleton { 0 };
  1269. i8 field { -1 };
  1270. @string_index_type@ start_range { 0 };
  1271. @string_index_type@ separator { 0 };
  1272. @string_index_type@ end_range { 0 };
  1273. i8 era { -1 };
  1274. i8 year { -1 };
  1275. i8 month { -1 };
  1276. i8 weekday { -1 };
  1277. i8 day { -1 };
  1278. i8 day_period { -1 };
  1279. i8 hour { -1 };
  1280. i8 minute { -1 };
  1281. i8 second { -1 };
  1282. i8 fractional_second_digits { -1 };
  1283. i8 time_zone_name { -1 };
  1284. };
  1285. )~~~");
  1286. locale_data.unique_patterns.generate(generator, "CalendarPattern"sv, "s_calendar_patterns"sv, 10);
  1287. locale_data.unique_pattern_lists.generate(generator, s_calendar_pattern_index_type, "s_calendar_pattern_lists"sv);
  1288. locale_data.unique_range_patterns.generate(generator, "CalendarRangePattern"sv, "s_calendar_range_patterns"sv, 10);
  1289. locale_data.unique_range_pattern_lists.generate(generator, s_calendar_range_pattern_index_type, "s_calendar_range_pattern_lists"sv);
  1290. generator.append(R"~~~(
  1291. struct CalendarFormat {
  1292. Unicode::CalendarFormat to_unicode_calendar_format() const {
  1293. Unicode::CalendarFormat calendar_format {};
  1294. calendar_format.full_format = s_calendar_patterns[full_format].to_unicode_calendar_pattern();
  1295. calendar_format.long_format = s_calendar_patterns[long_format].to_unicode_calendar_pattern();
  1296. calendar_format.medium_format = s_calendar_patterns[medium_format].to_unicode_calendar_pattern();
  1297. calendar_format.short_format = s_calendar_patterns[short_format].to_unicode_calendar_pattern();
  1298. return calendar_format;
  1299. }
  1300. @calendar_pattern_index_type@ full_format { 0 };
  1301. @calendar_pattern_index_type@ long_format { 0 };
  1302. @calendar_pattern_index_type@ medium_format { 0 };
  1303. @calendar_pattern_index_type@ short_format { 0 };
  1304. };
  1305. struct CalendarSymbols {
  1306. @symbol_list_index_type@ narrow_symbols { 0 };
  1307. @symbol_list_index_type@ short_symbols { 0 };
  1308. @symbol_list_index_type@ long_symbols { 0 };
  1309. };
  1310. struct CalendarData {
  1311. @string_index_type@ calendar { 0 };
  1312. @calendar_format_index_type@ date_formats { 0 };
  1313. @calendar_format_index_type@ time_formats { 0 };
  1314. @calendar_format_index_type@ date_time_formats { 0 };
  1315. @calendar_pattern_list_index_type@ available_formats { 0 };
  1316. @calendar_range_pattern_index_type@ default_range_format { 0 };
  1317. @calendar_range_pattern_list_index_type@ range_formats { 0 };
  1318. @calendar_range_pattern_list_index_type@ range12_formats { 0 };
  1319. @calendar_symbols_list_index_type@ symbols { 0 };
  1320. };
  1321. struct TimeZoneData {
  1322. @string_index_type@ time_zone { 0 };
  1323. @string_index_type@ long_name { 0 };
  1324. @string_index_type@ short_name { 0 };
  1325. };
  1326. struct DayPeriodData {
  1327. u8 day_period { 0 };
  1328. u8 begin { 0 };
  1329. u8 end { 0 };
  1330. };
  1331. )~~~");
  1332. locale_data.unique_formats.generate(generator, "CalendarFormat"sv, "s_calendar_formats"sv, 10);
  1333. locale_data.unique_symbol_lists.generate(generator, s_string_index_type, "s_symbol_lists"sv);
  1334. locale_data.unique_calendar_symbols.generate(generator, "CalendarSymbols"sv, "s_calendar_symbols"sv, 10);
  1335. locale_data.unique_calendar_symbols_lists.generate(generator, s_calendar_symbols_index_type, "s_calendar_symbol_lists"sv);
  1336. locale_data.unique_calendars.generate(generator, "CalendarData"sv, "s_calendars"sv, 10);
  1337. auto append_calendars = [&](String name, auto const& calendars) {
  1338. generator.set("name", name);
  1339. generator.set("size", String::number(calendars.size()));
  1340. generator.append(R"~~~(
  1341. static constexpr Array<@calendar_index_type@, @size@> @name@ { {)~~~");
  1342. bool first = true;
  1343. for (auto const& calendar_key : locale_data.calendars) {
  1344. auto calendar = calendars.find(calendar_key)->value;
  1345. generator.append(first ? " " : ", ");
  1346. generator.append(String::number(calendar));
  1347. first = false;
  1348. }
  1349. generator.append(" } };");
  1350. };
  1351. auto append_time_zones = [&](String name, auto const& time_zones) {
  1352. generator.set("name", name);
  1353. generator.set("size", String::number(locale_data.time_zones.size()));
  1354. generator.append(R"~~~(
  1355. static constexpr Array<TimeZoneData, @size@> @name@ { {)~~~");
  1356. constexpr size_t max_values_per_row = 20;
  1357. size_t values_in_current_row = 0;
  1358. for (auto const& time_zone_key : locale_data.time_zones) {
  1359. auto time_zone = time_zones.find(time_zone_key);
  1360. if (values_in_current_row++ > 0)
  1361. generator.append(" ");
  1362. if (time_zone == time_zones.end()) {
  1363. generator.append("{},");
  1364. } else {
  1365. generator.set("time_zone", String::number(time_zone->value.time_zone));
  1366. generator.set("long_name", String::number(time_zone->value.long_name));
  1367. generator.set("short_name", String::number(time_zone->value.short_name));
  1368. generator.append("{ @time_zone@, @long_name@, @short_name@ },");
  1369. }
  1370. if (values_in_current_row == max_values_per_row) {
  1371. values_in_current_row = 0;
  1372. generator.append("\n ");
  1373. }
  1374. }
  1375. generator.append(R"~~~(
  1376. } };
  1377. )~~~");
  1378. };
  1379. auto append_day_periods = [&](String name, auto const& day_periods) {
  1380. generator.set("name", move(name));
  1381. generator.set("size", String::number(day_periods.size()));
  1382. generator.append(R"~~~(
  1383. static constexpr Array<DayPeriodData, @size@> @name@ { {)~~~");
  1384. for (auto const& day_period : day_periods) {
  1385. generator.set("day_period", String::number(static_cast<u8>(day_period.day_period)));
  1386. generator.set("begin", String::number(day_period.begin));
  1387. generator.set("end", String::number(day_period.end));
  1388. generator.append(" { @day_period@, @begin@, @end@ },");
  1389. }
  1390. generator.append(" } };");
  1391. };
  1392. auto append_hour_cycles = [&](String name, auto const& hour_cycle_region) {
  1393. auto const& hour_cycles = locale_data.hour_cycles.find(hour_cycle_region)->value;
  1394. generator.set("name", name);
  1395. generator.set("size", String::number(hour_cycles.size()));
  1396. generator.append(R"~~~(
  1397. static constexpr Array<u8, @size@> @name@ { { )~~~");
  1398. for (auto hour_cycle : hour_cycles) {
  1399. generator.set("hour_cycle", String::number(static_cast<u8>(hour_cycle)));
  1400. generator.append("@hour_cycle@, ");
  1401. }
  1402. generator.append(" } };");
  1403. };
  1404. generate_mapping(generator, locale_data.locales, s_calendar_index_type, "s_locale_calendars"sv, "s_calendars_{}", [&](auto const& name, auto const& value) { append_calendars(name, value.calendars); });
  1405. generate_mapping(generator, locale_data.locales, "TimeZoneData"sv, "s_time_zones"sv, "s_time_zones_{}", [&](auto const& name, auto const& value) { append_time_zones(name, value.time_zones); });
  1406. generate_mapping(generator, locale_data.locales, "DayPeriodData"sv, "s_day_periods"sv, "s_day_periods_{}", [&](auto const& name, auto const& value) { append_day_periods(name, value.day_periods); });
  1407. generate_mapping(generator, locale_data.hour_cycle_regions, "u8"sv, "s_hour_cycles"sv, "s_hour_cycles_{}", [&](auto const& name, auto const& value) { append_hour_cycles(name, value); });
  1408. auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& values, Vector<Alias> const& aliases = {}) {
  1409. HashValueMap<String> hashes;
  1410. hashes.ensure_capacity(values.size());
  1411. for (auto const& value : values)
  1412. hashes.set(value.hash(), format_identifier(enum_title, value));
  1413. for (auto const& alias : aliases)
  1414. hashes.set(alias.alias.hash(), format_identifier(enum_title, alias.alias));
  1415. generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes));
  1416. };
  1417. append_from_string("Calendar"sv, "calendar"sv, locale_data.calendars, locale_data.calendar_aliases);
  1418. append_from_string("HourCycleRegion"sv, "hour_cycle_region"sv, locale_data.hour_cycle_regions);
  1419. append_from_string("TimeZone"sv, "time_zone"sv, locale_data.time_zones);
  1420. generator.append(R"~~~(
  1421. Vector<Unicode::HourCycle> get_regional_hour_cycles(StringView region)
  1422. {
  1423. auto region_value = hour_cycle_region_from_string(region);
  1424. if (!region_value.has_value())
  1425. return {};
  1426. auto region_index = to_underlying(*region_value);
  1427. auto const& regional_hour_cycles = s_hour_cycles.at(region_index);
  1428. Vector<Unicode::HourCycle> hour_cycles;
  1429. hour_cycles.ensure_capacity(regional_hour_cycles.size());
  1430. for (auto hour_cycle : regional_hour_cycles)
  1431. hour_cycles.unchecked_append(static_cast<Unicode::HourCycle>(hour_cycle));
  1432. return hour_cycles;
  1433. }
  1434. static CalendarData const* find_calendar_data(StringView locale, StringView calendar)
  1435. {
  1436. auto locale_value = locale_from_string(locale);
  1437. if (!locale_value.has_value())
  1438. return nullptr;
  1439. auto calendar_value = calendar_from_string(calendar);
  1440. if (!calendar_value.has_value())
  1441. return nullptr;
  1442. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1443. size_t calendar_index = to_underlying(*calendar_value);
  1444. auto const& calendar_indices = s_locale_calendars.at(locale_index);
  1445. calendar_index = calendar_indices[calendar_index];
  1446. return &s_calendars[calendar_index];
  1447. }
  1448. Optional<Unicode::CalendarFormat> get_calendar_date_format(StringView locale, StringView calendar)
  1449. {
  1450. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1451. auto const& formats = s_calendar_formats.at(data->date_formats);
  1452. return formats.to_unicode_calendar_format();
  1453. }
  1454. return {};
  1455. }
  1456. Optional<Unicode::CalendarFormat> get_calendar_time_format(StringView locale, StringView calendar)
  1457. {
  1458. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1459. auto const& formats = s_calendar_formats.at(data->time_formats);
  1460. return formats.to_unicode_calendar_format();
  1461. }
  1462. return {};
  1463. }
  1464. Optional<Unicode::CalendarFormat> get_calendar_date_time_format(StringView locale, StringView calendar)
  1465. {
  1466. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1467. auto const& formats = s_calendar_formats.at(data->date_time_formats);
  1468. return formats.to_unicode_calendar_format();
  1469. }
  1470. return {};
  1471. }
  1472. Vector<Unicode::CalendarPattern> get_calendar_available_formats(StringView locale, StringView calendar)
  1473. {
  1474. Vector<Unicode::CalendarPattern> result {};
  1475. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1476. auto const& available_formats = s_calendar_pattern_lists.at(data->available_formats);
  1477. result.ensure_capacity(available_formats.size());
  1478. for (auto const& format : available_formats)
  1479. result.unchecked_append(s_calendar_patterns[format].to_unicode_calendar_pattern());
  1480. }
  1481. return result;
  1482. }
  1483. Optional<Unicode::CalendarRangePattern> get_calendar_default_range_format(StringView locale, StringView calendar)
  1484. {
  1485. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1486. auto const& pattern = s_calendar_range_patterns[data->default_range_format];
  1487. return pattern.to_unicode_calendar_range_pattern();
  1488. }
  1489. return {};
  1490. }
  1491. Vector<Unicode::CalendarRangePattern> get_calendar_range_formats(StringView locale, StringView calendar, StringView skeleton) {
  1492. Vector<Unicode::CalendarRangePattern> result {};
  1493. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1494. auto const& range_formats = s_calendar_range_pattern_lists.at(data->range_formats);
  1495. for (auto format : range_formats) {
  1496. auto const& pattern = s_calendar_range_patterns[format];
  1497. if (skeleton == s_string_list[pattern.skeleton])
  1498. result.append(pattern.to_unicode_calendar_range_pattern());
  1499. }
  1500. }
  1501. return result;
  1502. }
  1503. Vector<Unicode::CalendarRangePattern> get_calendar_range12_formats(StringView locale, StringView calendar, StringView skeleton) {
  1504. Vector<Unicode::CalendarRangePattern> result {};
  1505. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1506. auto const& range12_formats = s_calendar_range_pattern_lists.at(data->range12_formats);
  1507. for (auto format : range12_formats) {
  1508. auto const& pattern = s_calendar_range_patterns[format];
  1509. if (skeleton == s_string_list[pattern.skeleton])
  1510. result.append(pattern.to_unicode_calendar_range_pattern());
  1511. }
  1512. }
  1513. return result;
  1514. }
  1515. static Span<@string_index_type@ const> find_calendar_symbols(StringView locale, StringView calendar, CalendarSymbol symbol, CalendarPatternStyle style)
  1516. {
  1517. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1518. auto const& symbols_list = s_calendar_symbol_lists[data->symbols];
  1519. auto symbol_index = to_underlying(symbol);
  1520. auto calendar_symbols_index = symbols_list.at(symbol_index);
  1521. auto const& symbols = s_calendar_symbols.at(calendar_symbols_index);
  1522. @symbol_list_index_type@ symbol_list_index = 0;
  1523. switch (style) {
  1524. case CalendarPatternStyle::Narrow:
  1525. symbol_list_index = symbols.narrow_symbols;
  1526. break;
  1527. case CalendarPatternStyle::Short:
  1528. symbol_list_index = symbols.short_symbols;
  1529. break;
  1530. case CalendarPatternStyle::Long:
  1531. symbol_list_index = symbols.long_symbols;
  1532. break;
  1533. default:
  1534. VERIFY_NOT_REACHED();
  1535. }
  1536. return s_symbol_lists.at(symbol_list_index);
  1537. }
  1538. return {};
  1539. }
  1540. Optional<StringView> get_calendar_era_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Era value)
  1541. {
  1542. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Era, style);
  1543. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1544. return s_string_list[symbols.at(value_index)];
  1545. return {};
  1546. }
  1547. Optional<StringView> get_calendar_month_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Month value)
  1548. {
  1549. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Month, style);
  1550. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1551. return s_string_list[symbols.at(value_index)];
  1552. return {};
  1553. }
  1554. Optional<StringView> get_calendar_weekday_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Weekday value)
  1555. {
  1556. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Weekday, style);
  1557. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1558. return s_string_list[symbols.at(value_index)];
  1559. return {};
  1560. }
  1561. Optional<StringView> get_calendar_day_period_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::DayPeriod value)
  1562. {
  1563. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::DayPeriod, style);
  1564. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1565. return s_string_list[symbols.at(value_index)];
  1566. return {};
  1567. }
  1568. Optional<StringView> get_calendar_day_period_symbol_for_hour(StringView locale, StringView calendar, CalendarPatternStyle style, u8 hour)
  1569. {
  1570. auto locale_value = locale_from_string(locale);
  1571. if (!locale_value.has_value())
  1572. return {};
  1573. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1574. auto day_periods = s_day_periods[locale_index];
  1575. for (auto const& day_period : day_periods) {
  1576. if ((day_period.begin <= hour) && (hour < day_period.end)) {
  1577. auto period = static_cast<Unicode::DayPeriod>(day_period.day_period);
  1578. return Detail::get_calendar_day_period_symbol(locale, calendar, style, period);
  1579. }
  1580. }
  1581. // Fallback to fixed periods if the locale does not have flexible day periods.
  1582. // TR-35 states that the meaning of AM and PM does not change with locale.
  1583. if (hour < 12)
  1584. return Detail::get_calendar_day_period_symbol(locale, calendar, style, Unicode::DayPeriod::AM);
  1585. return Detail::get_calendar_day_period_symbol(locale, calendar, style, Unicode::DayPeriod::PM);
  1586. }
  1587. static TimeZoneData const* find_time_zone_data(StringView locale, StringView time_zone)
  1588. {
  1589. auto locale_value = locale_from_string(locale);
  1590. if (!locale_value.has_value())
  1591. return nullptr;
  1592. auto time_zone_value = time_zone_from_string(time_zone);
  1593. if (!time_zone_value.has_value())
  1594. return nullptr;
  1595. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1596. auto time_zone_index = to_underlying(*time_zone_value);
  1597. auto const& time_zones = s_time_zones.at(locale_index);
  1598. return &time_zones[time_zone_index];
  1599. }
  1600. Optional<StringView> get_time_zone_name(StringView locale, StringView time_zone, CalendarPatternStyle style)
  1601. {
  1602. if (auto const* data = find_time_zone_data(locale, time_zone); data != nullptr) {
  1603. @string_index_type@ time_zone_index = 0;
  1604. switch (style) {
  1605. case CalendarPatternStyle::Long:
  1606. time_zone_index = data->long_name;
  1607. break;
  1608. case CalendarPatternStyle::Short:
  1609. time_zone_index = data->short_name;
  1610. break;
  1611. default:
  1612. VERIFY_NOT_REACHED();
  1613. }
  1614. if (time_zone_index != 0)
  1615. return s_string_list[time_zone_index];
  1616. }
  1617. return {};
  1618. }
  1619. }
  1620. )~~~");
  1621. VERIFY(file.write(generator.as_string_view()));
  1622. }
  1623. ErrorOr<int> serenity_main(Main::Arguments arguments)
  1624. {
  1625. StringView generated_header_path;
  1626. StringView generated_implementation_path;
  1627. StringView core_path;
  1628. StringView dates_path;
  1629. Core::ArgsParser args_parser;
  1630. args_parser.add_option(generated_header_path, "Path to the Unicode locale header file to generate", "generated-header-path", 'h', "generated-header-path");
  1631. args_parser.add_option(generated_implementation_path, "Path to the Unicode locale implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  1632. args_parser.add_option(core_path, "Path to cldr-core directory", "core-path", 'r', "core-path");
  1633. args_parser.add_option(dates_path, "Path to cldr-dates directory", "dates-path", 'd', "dates-path");
  1634. args_parser.parse(arguments);
  1635. auto open_file = [&](StringView path) -> ErrorOr<NonnullRefPtr<Core::File>> {
  1636. if (path.is_empty()) {
  1637. args_parser.print_usage(stderr, arguments.argv[0]);
  1638. return Error::from_string_literal("Must provide all command line options"sv);
  1639. }
  1640. return Core::File::open(path, Core::OpenMode::ReadWrite);
  1641. };
  1642. auto generated_header_file = TRY(open_file(generated_header_path));
  1643. auto generated_implementation_file = TRY(open_file(generated_implementation_path));
  1644. UnicodeLocaleData locale_data;
  1645. TRY(parse_all_locales(core_path, dates_path, locale_data));
  1646. generate_unicode_locale_header(generated_header_file, locale_data);
  1647. generate_unicode_locale_implementation(generated_implementation_file, locale_data);
  1648. return 0;
  1649. }