Meta: Add LibUnicode and LibLocale to gn build

A lot of code gen happening here. These generators are kind of
awkward to work with, and the fact that the CLDR data download
extracts over 8,000 files makes it hard to fit into the explicit
patterns GN expects of us.

Meta/gn/secondary/BUILD.gn

@@ -4,7 +4,9 @@ group("default") {
   deps = [
     "//Meta/Lagom/Tools/CodeGenerators/IPCCompiler",
     "//Tests",
+    "//Userland/Libraries/LibLocale",
     "//Userland/Libraries/LibTimeZone",
+    "//Userland/Libraries/LibUnicode",
   ]
   testonly = true
 }

Meta/gn/secondary/Userland/Libraries/LibLocale/BUILD.gn

@@ -0,0 +1,191 @@
+import("//Meta/gn/build/compiled_action.gni")
+import("//Meta/gn/build/download_cache.gni")
+import("//Meta/gn/build/download_file.gni")
+import("//Meta/gn/build/extract_archive_contents.gni")
+import("//Userland/Libraries/LibUnicode/enable_unicode_download.gni")
+
+locale_cache = cache_path + "CLDR/"
+
+if (enable_unicode_database_download) {
+  download_file("locale_database_download") {
+    version = "43.1.0"
+    url = "https://github.com/unicode-org/cldr-json/releases/download/" +
+          version + "/cldr-" + version + "-json-modern.zip"
+    cache = locale_cache
+    output = "cldr.zip"
+    version_file = "version.txt"
+  }
+
+  extract_archive_contents("locale_database_files") {
+    deps = [ ":locale_database_download" ]
+    archive = get_target_outputs(":locale_database_download")
+    directory = locale_cache
+    paths = [
+      "cldr-bcp47",
+      "cldr-core",
+      "cldr-dates-modern",
+      "cldr-localenames-modern",
+      "cldr-misc-modern",
+      "cldr-numbers-modern",
+      "cldr-units-modern",
+    ]
+  }
+
+  compiled_action("generate_datetime_format_sources") {
+    tool =
+        "//Meta/Lagom/Tools/CodeGenerators/LibLocale:GenerateDateTimeFormatData"
+    deps = [ ":locale_database_files" ]
+    outputs = [
+      "$target_gen_dir/DateTimeFormatData.h",
+      "$target_gen_dir/DateTimeFormatData.cpp",
+    ]
+    args = [
+      "-h",
+      rebase_path(outputs[0], root_build_dir),
+      "-c",
+      rebase_path(outputs[1], root_build_dir),
+      "-r",
+      rebase_path(locale_cache + "cldr-core", root_build_dir),
+      "-d",
+      rebase_path(locale_cache + "cldr-dates-modern", root_build_dir),
+    ]
+  }
+
+  compiled_action("generate_locale_sources") {
+    tool = "//Meta/Lagom/Tools/CodeGenerators/LibLocale:GenerateLocaleData"
+    deps = [ ":locale_database_files" ]
+    outputs = [
+      "$target_gen_dir/LocaleData.h",
+      "$target_gen_dir/LocaleData.cpp",
+    ]
+    args = [
+      "-h",
+      rebase_path(outputs[0], root_build_dir),
+      "-c",
+      rebase_path(outputs[1], root_build_dir),
+      "-b",
+      rebase_path(locale_cache + "cldr-bcp47", root_build_dir),
+      "-r",
+      rebase_path(locale_cache + "cldr-core", root_build_dir),
+      "-l",
+      rebase_path(locale_cache + "cldr-localenames-modern", root_build_dir),
+      "-m",
+      rebase_path(locale_cache + "cldr-misc-modern", root_build_dir),
+      "-n",
+      rebase_path(locale_cache + "cldr-numbers-modern", root_build_dir),
+      "-d",
+      rebase_path(locale_cache + "cldr-dates-modern", root_build_dir),
+    ]
+  }
+
+  compiled_action("generate_number_format_sources") {
+    tool =
+        "//Meta/Lagom/Tools/CodeGenerators/LibLocale:GenerateNumberFormatData"
+    deps = [ ":locale_database_files" ]
+    outputs = [
+      "$target_gen_dir/NumberFormatData.h",
+      "$target_gen_dir/NumberFormatData.cpp",
+    ]
+    args = [
+      "-h",
+      rebase_path(outputs[0], root_build_dir),
+      "-c",
+      rebase_path(outputs[1], root_build_dir),
+      "-r",
+      rebase_path(locale_cache + "cldr-core", root_build_dir),
+      "-n",
+      rebase_path(locale_cache + "cldr-numbers-modern", root_build_dir),
+      "-u",
+      rebase_path(locale_cache + "cldr-units-modern", root_build_dir),
+    ]
+  }
+
+  compiled_action("generate_plural_rules_sources") {
+    tool = "//Meta/Lagom/Tools/CodeGenerators/LibLocale:GeneratePluralRulesData"
+    deps = [ ":locale_database_files" ]
+    outputs = [
+      "$target_gen_dir/PluralRulesData.h",
+      "$target_gen_dir/PluralRulesData.cpp",
+    ]
+    args = [
+      "-h",
+      rebase_path(outputs[0], root_build_dir),
+      "-c",
+      rebase_path(outputs[1], root_build_dir),
+      "-r",
+      rebase_path(locale_cache + "cldr-core", root_build_dir),
+      "-l",
+      rebase_path(locale_cache + "cldr-localenames-modern", root_build_dir),
+    ]
+  }
+
+  compiled_action("generate_relative_time_format_sources") {
+    tool = "//Meta/Lagom/Tools/CodeGenerators/LibLocale:GenerateRelativeTimeFormatData"
+    deps = [ ":locale_database_files" ]
+    outputs = [
+      "$target_gen_dir/RelativeTimeFormatData.h",
+      "$target_gen_dir/RelativeTimeFormatData.cpp",
+    ]
+    args = [
+      "-h",
+      rebase_path(outputs[0], root_build_dir),
+      "-c",
+      rebase_path(outputs[1], root_build_dir),
+      "-d",
+      rebase_path(locale_cache + "cldr-dates-modern", root_build_dir),
+    ]
+  }
+
+  # FIXME: Add optimization for serenity to use these as a shlib
+  source_set("locale_data_sources") {
+    include_dirs = [
+      "//Userland/Libraries",
+      "$target_gen_dir/..",
+    ]
+    cflags_cc = [
+      "-g0",
+      "-Os",
+      "-Wno-parentheses-equality",
+    ]
+    deps = [
+      ":generate_datetime_format_sources",
+      ":generate_locale_sources",
+      ":generate_number_format_sources",
+      ":generate_plural_rules_sources",
+      ":generate_relative_time_format_sources",
+      "//AK",
+    ]
+    public_deps = [ "//Userland/Libraries/LibTimeZone" ]
+    sources = get_target_outputs(":generate_datetime_format_sources")
+    sources += get_target_outputs(":generate_locale_sources")
+    sources += get_target_outputs(":generate_number_format_sources")
+    sources += get_target_outputs(":generate_plural_rules_sources")
+    sources += get_target_outputs(":generate_relative_time_format_sources")
+  }
+}
+
+source_set("LibLocale") {
+  output_name = "locale"
+  include_dirs = [
+    "//Userland/Libraries",
+    "$target_gen_dir/..",
+  ]
+  sources = [
+    "DateTimeFormat.cpp",
+    "Locale.cpp",
+    "NumberFormat.cpp",
+    "PluralRules.cpp",
+    "RelativeTimeFormat.cpp",
+  ]
+  deps = [
+    "//AK",
+    "//Userland/Libraries/LibCore",
+    "//Userland/Libraries/LibUnicode",
+  ]
+  if (enable_unicode_database_download) {
+    deps += [ ":locale_data_sources" ]
+    defines = [ "ENABLE_UNICODE_DATA=1" ]
+  } else {
+    defines = [ "ENABLE_UNICODE_DATA=0" ]
+  }
+}

Meta/gn/secondary/Userland/Libraries/LibUnicode/BUILD.gn

@@ -0,0 +1,163 @@
+import("//Meta/gn/build/compiled_action.gni")
+import("//Meta/gn/build/download_cache.gni")
+import("//Meta/gn/build/download_file.gni")
+import("//Meta/gn/build/extract_archive_contents.gni")
+import("//Userland/Libraries/LibUnicode/enable_unicode_download.gni")
+
+unicode_cache = cache_path + "UCD/"
+emoji_cache = cache_path + "EMOJI/"
+
+if (enable_unicode_database_download) {
+  download_file("unicode_database_download") {
+    version = "15.0.0"
+    url = "https://www.unicode.org/Public/" + version + "/ucd/UCD.zip"
+    cache = unicode_cache
+    output = "UCD.zip"
+    version_file = "version.txt"
+  }
+  download_file("emoji_test_download") {
+    version = "15.0"
+    url = "https://www.unicode.org/Public/emoji/" + version + "/emoji-test.txt"
+    cache = emoji_cache
+    output = "emoji-test.txt"
+    version_file = "emoji-test-version.txt"
+  }
+
+  extract_archive_contents("unicode_database_files") {
+    deps = [ ":unicode_database_download" ]
+    archive = get_target_outputs(":unicode_database_download")
+    directory = unicode_cache
+    files = [
+      "UnicodeData.txt",
+      "SpecialCasing.txt",
+      "CaseFolding.txt",
+      "extracted/DerivedGeneralCategory.txt",
+      "PropList.txt",
+      "DerivedCoreProperties.txt",
+      "extracted/DerivedBinaryProperties.txt",
+      "PropertyAliases.txt",
+      "PropertyValueAliases.txt",
+      "Scripts.txt",
+      "ScriptExtensions.txt",
+      "Blocks.txt",
+      "emoji/emoji-data.txt",
+      "NameAliases.txt",
+      "DerivedNormalizationProps.txt",
+      "auxiliary/GraphemeBreakProperty.txt",
+      "auxiliary/WordBreakProperty.txt",
+      "auxiliary/SentenceBreakProperty.txt",
+    ]
+  }
+
+  compiled_action("generate_unicode_sources") {
+    tool = "//Meta/Lagom/Tools/CodeGenerators/LibUnicode:GenerateUnicodeData"
+    deps = [ ":unicode_database_files" ]
+    inputs = get_target_outputs(":unicode_database_files")
+    outputs = [
+      "$target_gen_dir/UnicodeData.h",
+      "$target_gen_dir/UnicodeData.cpp",
+    ]
+
+    args = [
+      "-h",
+      rebase_path(outputs[0], root_build_dir),
+      "-c",
+      rebase_path(outputs[1], root_build_dir),
+      "-u",
+      rebase_path(inputs[0], root_build_dir),
+      "-s",
+      rebase_path(inputs[1], root_build_dir),
+      "-o",
+      rebase_path(inputs[2], root_build_dir),
+      "-g",
+      rebase_path(inputs[3], root_build_dir),
+      "-p",
+      rebase_path(inputs[4], root_build_dir),
+      "-d",
+      rebase_path(inputs[5], root_build_dir),
+      "-b",
+      rebase_path(inputs[6], root_build_dir),
+      "-a",
+      rebase_path(inputs[7], root_build_dir),
+      "-v",
+      rebase_path(inputs[8], root_build_dir),
+      "-r",
+      rebase_path(inputs[9], root_build_dir),
+      "-x",
+      rebase_path(inputs[10], root_build_dir),
+      "-k",
+      rebase_path(inputs[11], root_build_dir),
+      "-e",
+      rebase_path(inputs[12], root_build_dir),
+      "-m",
+      rebase_path(inputs[13], root_build_dir),
+      "-n",
+      rebase_path(inputs[14], root_build_dir),
+      "-f",
+      rebase_path(inputs[15], root_build_dir),
+      "-w",
+      rebase_path(inputs[16], root_build_dir),
+      "-i",
+      rebase_path(inputs[17], root_build_dir),
+    ]
+  }
+
+  compiled_action("generate_emoji_sources") {
+    tool = "//Meta/Lagom/Tools/CodeGenerators/LibUnicode:GenerateEmojiData"
+    deps = [ ":emoji_test_download" ]
+    inputs = get_target_outputs(":emoji_test_download")
+    outputs = [
+      "$target_gen_dir/EmojiData.h",
+      "$target_gen_dir/EmojiData.cpp",
+    ]
+    args = [
+      "-h",
+      rebase_path(outputs[0], root_build_dir),
+      "-c",
+      rebase_path(outputs[1], root_build_dir),
+      "-e",
+      rebase_path(inputs[0], root_build_dir),
+      "-s",
+      rebase_path("//Base/home/anon/Documents/emoji-serenity.txt",
+                  root_build_dir),
+      "-r",
+      rebase_path("//Base/res/emoji", root_build_dir),
+    ]
+
+    # FIXME: How to add file/directory dependencies on
+    #        "//Base/home/anon/Documents/emoji-serenity.txt"
+    #        and "//Base/res/emoji"?
+  }
+}
+
+source_set("LibUnicode") {
+  output_name = "unicode"
+  include_dirs = [
+    "//Userland/Libraries",
+    "$target_gen_dir/..",
+  ]
+  sources = [
+    "CharacterTypes.cpp",
+    "CurrencyCode.cpp",
+    "Emoji.cpp",
+    "Normalize.cpp",
+    "Segmentation.cpp",
+    "String.cpp",
+    "UnicodeUtils.cpp",
+  ]
+  deps = [
+    "//AK",
+    "//Userland/Libraries/LibCore",
+  ]
+  if (enable_unicode_database_download) {
+    deps += [
+      ":generate_emoji_sources",
+      ":generate_unicode_sources",
+    ]
+    sources += get_target_outputs(":generate_unicode_sources")
+    sources += get_target_outputs(":generate_emoji_sources")
+    defines = [ "ENABLE_UNICODE_DATA=1" ]
+  } else {
+    defines = [ "ENABLE_UNICODE_DATA=0" ]
+  }
+}

Meta/gn/secondary/Userland/Libraries/LibUnicode/enable_unicode_download.gni

@@ -0,0 +1,7 @@
+declare_args() {
+  # If true, download CLDR and UCD files from unicode.org or the unicode
+  # consortium's github and use it in LibUnicode and LibLocale.
+  # Data will be downloaded to $cache_path/CLDR, $cache_path/UCD
+  # and $cache_path/EMOJI
+  enable_unicode_database_download = true
+}