Meta: Add file download and archive extraction tools to gn build

Use them to download and extract the TZDB files

Meta/gn/build/download_cache.gni

@@ -0,0 +1,4 @@
+declare_args() {
+  # Location of shared cache of downloaded files
+  cache_path = "$root_gen_dir/Cache/"
+}

Meta/gn/build/download_file.gni

@@ -0,0 +1,80 @@
+#
+# This file introduces a template for calling download_file.py
+#
+# download_file behaves like CMake's file(DOWNLOAD) with the addtion
+# of version checking the file against a build system defined version.
+#
+# Parameters:
+#   url (required) [string]
+#
+#   output (required) [string]
+#
+#   version (required) [string]
+#       Version of the file for caching purposes
+#
+#   version_file (reqiured) [string]
+#       Filename to write the version to in the filesystem
+#
+#   cache [String]
+#       Directory to clear on version mismatch
+#
+#
+# Example use:
+#
+#    download_file("my_tarball") {
+#        url = "http://example.com/xyz.tar.gz"
+#        output = "$root_gen_dir/MyModule/xyz.tar.gz"
+#        version = "1.2.3"
+#        version_file = "$root_gen_dir/MyModule/xyz_version.txt"
+#    }
+#
+
+template("download_file") {
+  assert(defined(invoker.url), "must set 'url' in $target_name")
+  assert(defined(invoker.output), "must set 'output' in $target_name")
+  assert(defined(invoker.version), "must set 'version' in $target_name")
+  assert(defined(invoker.version_file),
+         "must set 'version_file' in $target_name")
+
+  action(target_name) {
+    script = "//Meta/gn/build/download_file.py"
+
+    sources = []
+    if (defined(invoker.cache)) {
+      outputs = [
+        invoker.cache + "/" + invoker.output,
+        invoker.cache + "/" + invoker.version_file,
+      ]
+    } else {
+      outputs = [
+        invoker.output,
+        invoker.version_file,
+      ]
+    }
+    args = [
+      "-o",
+      rebase_path(outputs[0], root_build_dir),
+      "-f",
+      rebase_path(outputs[1], root_build_dir),
+      "-v",
+      invoker.version,
+      invoker.url,
+    ]
+    if (defined(invoker.cache)) {
+      args += [
+        "-c",
+        rebase_path(invoker.cache, root_build_dir),
+      ]
+    }
+
+    forward_variables_from(invoker,
+                           [
+                             "configs",
+                             "deps",
+                             "public_configs",
+                             "public_deps",
+                             "testonly",
+                             "visibility",
+                           ])
+  }
+}

Meta/gn/build/download_file.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+r"""Downloads a file as a build artifact.
+
+The file is downloaded to the specified directory.
+
+It's intended to be used for files that are cached between runs.
+
+"""
+
+import argparse
+import os
+import pathlib
+import shutil
+import sys
+import tempfile
+import urllib.request
+
+
+def main():
+    parser = argparse.ArgumentParser(
+                 epilog=__doc__,
+                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('url', help='input url')
+    parser.add_argument('-o', '--output', required=True,
+                        help='output file')
+    parser.add_argument('-v', '--version', required=True,
+                        help='version of file to detect mismatches and redownload')
+    parser.add_argument('-f', '--version-file', required=True,
+                        help='filesystem location to cache version')
+    parser.add_argument('-c', "--cache-path", required=False,
+                        help='path for cached files to clear on version mismatch')
+    args = parser.parse_args()
+
+    version_from_file = ''
+    version_file = pathlib.Path(args.version_file)
+    if version_file.exists():
+        with version_file.open('r') as f:
+            version_from_file = f.readline().strip()
+
+    if version_from_file == args.version:
+        return 0
+
+    # Fresh build or version mismatch, delete old cache
+    if (args.cache_path):
+        cache_path = pathlib.Path(args.cache_path)
+        shutil.rmtree(cache_path, ignore_errors=True)
+        cache_path.mkdir(parents=True)
+
+    print(f"Downloading version {args.version} of {args.output}...", end='')
+
+    with urllib.request.urlopen(args.url) as f:
+        try:
+            with tempfile.NamedTemporaryFile(delete=False,
+                                             dir=pathlib.Path(args.output).parent) as out:
+                out.write(f.read())
+                os.rename(out.name, args.output)
+        except IOError:
+            os.unlink(out.name)
+
+    print("done")
+
+    with open(version_file, 'w') as f:
+        f.write(args.version)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

Meta/gn/build/extract_archive_contents.gni

@@ -0,0 +1,76 @@
+#
+# This file introduces templates for calling extract_archive_contents.py
+#
+# extract_archive_contents.py behaves like CMake's file(ARCHIVE_EXTRACT)
+#
+# Parameters:
+#   archive (required) [string]
+#
+#   files (required) [list of strings]
+#       Relative paths to the root of the archive of files to extract
+#
+#   directory (required) [string]
+#       Output directory root for all the files
+#
+#   paths (optional) [list of strings]
+#       Relative paths to the root of the archive of directories to extract
+#
+# Example use:
+#
+#    extract_archive_contents("my_files") {
+#        archive = "$root_gen_dir/MyModule/xyz.tar.gz"
+#        directory = "$root_gen_dir/MyModule"
+#        files = [
+#           "file_one.txt",
+#           "file_two"
+#        ]
+#        paths = [ "some_dir" ]
+#    }
+#
+
+template("extract_archive_contents") {
+  assert(defined(invoker.archive), "must set 'archive' in $target_name")
+  assert(defined(invoker.files) || defined(invoker.paths),
+         "must set 'files' and/or 'paths' in $target_name")
+  assert(defined(invoker.directory), "must set 'directory' in $target_name")
+
+  action(target_name) {
+    script = "//Meta/gn/build/extract_archive_contents.py"
+
+    paths = []
+    if (defined(invoker.paths)) {
+      foreach(path, invoker.paths) {
+        paths += [ path + "/" ]
+      }
+    }
+    files = []
+    if (defined(invoker.files)) {
+      files = invoker.files
+    }
+    stamp_file = invoker.directory + "$target_name.stamp"
+
+    sources = invoker.archive
+    outputs = []
+    args = [
+             "-d",
+             rebase_path(invoker.directory, root_build_dir),
+             "-s",
+             rebase_path(stamp_file, root_build_dir),
+             rebase_path(sources[0], root_build_dir),
+           ] + files + paths
+    foreach(file, files) {
+      outputs += [ invoker.directory + file ]
+    }
+    outputs += [ stamp_file ]
+
+    forward_variables_from(invoker,
+                           [
+                             "configs",
+                             "deps",
+                             "public_configs",
+                             "public_deps",
+                             "testonly",
+                             "visibility",
+                           ])
+  }
+}

Meta/gn/build/extract_archive_contents.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+r"""Extracts files from an archive for use in the build
+
+It's intended to be used for files that are cached between runs.
+
+"""
+
+import argparse
+import pathlib
+import tarfile
+import zipfile
+import sys
+
+
+def extract_member(file, destination, path):
+    """
+    Extract a single file from a ZipFile or TarFile
+
+    :param ZipFile|TarFile file: Archive object to extract from
+    :param Path destination: Location to write the file
+    :param str path: Filename to extract from archive.
+    """
+    destination_path = destination / path
+    if destination_path.exists():
+        return
+    destination_path.parent.mkdir(parents=True, exist_ok=True)
+    if isinstance(file, tarfile.TarFile):
+        with file.extractfile(path) as member:
+            destination_path.write_text(member.read().decode('utf-8'))
+    else:
+        assert isinstance(file, zipfile.ZipFile)
+        with file.open(path) as member:
+            destination_path.write_text(member.read().decode('utf-8'))
+
+
+def extract_directory(file, destination, path):
+    """
+    Extract a directory from a ZipFile or TarFile
+
+    :param ZipFile|TarFile file: Archive object to extract from
+    :param Path destination: Location to write the files
+    :param str path: Directory name to extract from archive.
+    """
+    destination_path = destination / path
+    if destination_path.exists():
+        return
+    destination_path.mkdir(parents=True, exist_ok=True)
+    if not isinstance(file, zipfile.ZipFile):
+        raise NotImplementedError
+    # FIXME: This loops over the entire archive len(args.paths) times. Decrease complexity
+    for entry in file.namelist():
+        if entry.startswith(path):
+            entry_destination = destination / entry
+            if entry.endswith('/'):
+                entry_destination.mkdir(exist_ok=True)
+                continue
+            with file.open(entry) as member:
+                entry_destination.write_text(member.read().decode('utf-8'))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+                 epilog=__doc__,
+                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('archive', help='input archive')
+    parser.add_argument('paths', nargs='*', help='paths to extract from the archive')
+    parser.add_argument('-s', "--stamp", required=False,
+                        help='stamp file name to create after operation is done')
+    parser.add_argument('-d', "--destination", required=True,
+                        help='directory to write the extracted file to')
+    args = parser.parse_args()
+
+    archive = pathlib.Path(args.archive)
+    destination = pathlib.Path(args.destination)
+
+    def extract_paths(file, paths):
+        for path in paths:
+            if path.endswith('/'):
+                extract_directory(file, destination, path)
+            else:
+                extract_member(file, destination, path)
+
+    if tarfile.is_tarfile(archive):
+        with tarfile.open(archive) as f:
+            extract_paths(f, args.paths)
+    elif zipfile.is_zipfile(archive):
+        with zipfile.ZipFile(archive) as f:
+            extract_paths(f, args.paths)
+    else:
+        print(f"Unknown file type for {archive}, unable to extract {args.path}")
+        return 1
+
+    if args.stamp:
+        pathlib.Path(args.stamp).touch()
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

Meta/gn/secondary/BUILD.gn

@@ -4,6 +4,7 @@ group("default") {
   deps = [
     "//Meta/Lagom/Tools/CodeGenerators/IPCCompiler",
     "//Tests",
+    "//Userland/Libraries/LibTimeZone",
   ]
   testonly = true
 }

Meta/gn/secondary/Userland/Libraries/LibTimeZone/BUILD.gn

@@ -1,7 +1,44 @@
+import("//Meta/gn/build/compiled_action.gni")
+import("//Meta/gn/build/download_cache.gni")
+import("//Meta/gn/build/download_file.gni")
+import("//Meta/gn/build/extract_archive_contents.gni")
+
 declare_args() {
   # If true, Download tzdata from data.iana.org and use it in LibTimeZone
   # Data will be downloaded to $cache_path/TZDB
-  enable_timezone_database_download = false
+  enable_timezone_database_download = true
+}
+
+tzdb_cache = cache_path + "TZDB/"
+
+if (enable_timezone_database_download) {
+  download_file("timezone_database_download") {
+    version = "2023c"
+    url =
+        "https://data.iana.org/time-zones/releases/tzdata" + version + ".tar.gz"
+    cache = tzdb_cache
+    output = "tzdb.tar.gz"
+    version_file = "version.txt"
+  }
+  extract_archive_contents("timezone_database_files") {
+    deps = [ ":timezone_database_download" ]
+    archive = get_target_outputs(":timezone_database_download")
+    directory = tzdb_cache
+
+    # NOSORT
+    files = [
+      "zone1970.tab",
+      "africa",
+      "antarctica",
+      "asia",
+      "australasia",
+      "backward",
+      "etcetera",
+      "europe",
+      "northamerica",
+      "southamerica",
+    ]
+  }
 }
 
 source_set("LibTimeZone") {
@@ -17,6 +54,6 @@ source_set("LibTimeZone") {
     "//Userland/Libraries/LibCore",
   ]
   if (enable_timezone_database_download) {
-    deps += [ ":timezone_data" ]
+    deps += [ ":timezone_database_files" ]
   }
 }