pkg/chrootarchive: fix FreeBSD build

For unix targets, `goInChroot()` is only implemented for `Linux`,
hence FreeBSD build fails.

This change

- Adds FreeBSD-specific chrooted tar/untar implementation
- Fixes statUnix() to accomodate to FreeBSD devminor/devmajor
- quirk. See also: https://github.com/containerd/containerd/pull/5991

Signed-off-by: Artem Khramov <akhramov@pm.me>
Co-authored-by: Cory Snider <corhere@gmail.com>

pkg/archive/archive_unix.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"syscall"
 
@@ -43,6 +44,20 @@ func chmodTarEntry(perm os.FileMode) os.FileMode {
 // statUnix populates hdr from system-dependent fields of fi without performing
 // any OS lookups.
 func statUnix(fi os.FileInfo, hdr *tar.Header) error {
+	// Devmajor and Devminor are only needed for special devices.
+
+	// In FreeBSD, RDev for regular files is -1 (unless overridden by FS):
+	// https://cgit.freebsd.org/src/tree/sys/kern/vfs_default.c?h=stable/13#n1531
+	// (NODEV is -1: https://cgit.freebsd.org/src/tree/sys/sys/param.h?h=stable/13#n241).
+
+	// ZFS in particular does not override the default:
+	// https://cgit.freebsd.org/src/tree/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c?h=stable/13#n2027
+
+	// Since `Stat_t.Rdev` is uint64, the cast turns -1 into (2^64 - 1).
+	// Such large values cannot be encoded in a tar header.
+	if runtime.GOOS == "freebsd" && hdr.Typeflag != tar.TypeBlock && hdr.Typeflag != tar.TypeChar {
+		return nil
+	}
 	s, ok := fi.Sys().(*syscall.Stat_t)
 	if !ok {
 		return nil

pkg/chrootarchive/archive_freebsd.go

@@ -0,0 +1,225 @@
+package chrootarchive // import "github.com/docker/docker/pkg/chrootarchive"
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"syscall"
+
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	packCmd        = "freebsd-pack-in-chroot"
+	unpackCmd      = "freebsd-unpack-in-chroot"
+	unpackLayerCmd = "freebsd-unpack-layer-in-chroot"
+)
+
+func init() {
+	reexec.Register(packCmd, reexecMain(packInChroot))
+	reexec.Register(unpackCmd, reexecMain(unpackInChroot))
+	reexec.Register(unpackLayerCmd, reexecMain(unpackLayerInChroot))
+}
+
+func reexecMain(f func(options archive.TarOptions, args ...string) error) func() {
+	return func() {
+		if len(os.Args) < 2 {
+			fmt.Fprintln(os.Stderr, "root parameter is required")
+			os.Exit(1)
+		}
+
+		options, err := recvOptions()
+		root := os.Args[1]
+
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+
+		if err := syscall.Chroot(root); err != nil {
+			fmt.Fprintln(
+				os.Stderr,
+				os.PathError{Op: "chroot", Path: root, Err: err},
+			)
+			os.Exit(2)
+		}
+
+		if err := f(*options, os.Args[2:]...); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(3)
+		}
+	}
+}
+
+func doUnpack(decompressedArchive io.Reader, relDest, root string, options *archive.TarOptions) error {
+	optionsR, optionsW, err := os.Pipe()
+	if err != nil {
+		return err
+	}
+	defer optionsW.Close()
+	defer optionsR.Close()
+
+	stderr := bytes.NewBuffer(nil)
+
+	cmd := reexec.Command(unpackCmd, root, relDest)
+	cmd.Stdin = decompressedArchive
+	cmd.Stderr = stderr
+	cmd.ExtraFiles = []*os.File{
+		optionsR,
+	}
+
+	if err = cmd.Start(); err != nil {
+		return errors.Wrap(err, "re-exec error")
+	}
+
+	if err = json.NewEncoder(optionsW).Encode(options); err != nil {
+		return errors.Wrap(err, "tar options encoding failed")
+	}
+
+	if err = cmd.Wait(); err != nil {
+		return errors.Wrap(err, stderr.String())
+	}
+
+	return nil
+}
+
+func doPack(relSrc, root string, options *archive.TarOptions) (io.ReadCloser, error) {
+	optionsR, optionsW, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+	defer optionsW.Close()
+	defer optionsR.Close()
+
+	stderr := bytes.NewBuffer(nil)
+	cmd := reexec.Command(packCmd, root, relSrc)
+	cmd.ExtraFiles = []*os.File{
+		optionsR,
+	}
+	cmd.Stderr = stderr
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	r, w := io.Pipe()
+
+	if err = cmd.Start(); err != nil {
+		return nil, errors.Wrap(err, "re-exec error")
+	}
+
+	go func() {
+		_, _ = io.Copy(w, stdout)
+		// Cleanup once stdout pipe is closed.
+		if err = cmd.Wait(); err != nil {
+			r.CloseWithError(errors.Wrap(err, stderr.String()))
+		} else {
+			r.Close()
+		}
+	}()
+
+	if err = json.NewEncoder(optionsW).Encode(options); err != nil {
+		return nil, errors.Wrap(err, "tar options encoding failed")
+	}
+
+	return r, nil
+}
+
+func doUnpackLayer(root string, layer io.Reader, options *archive.TarOptions) (int64, error) {
+	var result int64
+	optionsR, optionsW, err := os.Pipe()
+	if err != nil {
+		return 0, err
+	}
+	defer optionsW.Close()
+	defer optionsR.Close()
+	buffer := bytes.NewBuffer(nil)
+
+	cmd := reexec.Command(unpackLayerCmd, root)
+	cmd.Stdin = layer
+	cmd.Stdout = buffer
+	cmd.Stderr = buffer
+	cmd.ExtraFiles = []*os.File{
+		optionsR,
+	}
+
+	if err = cmd.Start(); err != nil {
+		return 0, errors.Wrap(err, "re-exec error")
+	}
+
+	if err = json.NewEncoder(optionsW).Encode(options); err != nil {
+		return 0, errors.Wrap(err, "tar options encoding failed")
+	}
+
+	if err = cmd.Wait(); err != nil {
+		return 0, errors.Wrap(err, buffer.String())
+	}
+
+	if err = json.NewDecoder(buffer).Decode(&result); err != nil {
+		return 0, errors.Wrap(err, "json decoding error")
+	}
+
+	return result, nil
+}
+
+func unpackInChroot(options archive.TarOptions, args ...string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("destination parameter is required")
+	}
+
+	relDest := args[0]
+
+	return archive.Unpack(os.Stdin, relDest, &options)
+}
+
+func packInChroot(options archive.TarOptions, args ...string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("source parameter is required")
+	}
+
+	relSrc := args[0]
+
+	tb, err := archive.NewTarballer(relSrc, &options)
+
+	if err != nil {
+		return err
+	}
+
+	go tb.Do()
+
+	_, err = io.Copy(os.Stdout, tb.Reader())
+
+	return err
+}
+
+func unpackLayerInChroot(options archive.TarOptions, _args ...string) error {
+	// We need to be able to set any perms
+	_ = unix.Umask(0)
+
+	size, err := archive.UnpackLayer("/", os.Stdin, &options)
+	if err != nil {
+		return err
+	}
+
+	return json.NewEncoder(os.Stdout).Encode(size)
+}
+
+func recvOptions() (*archive.TarOptions, error) {
+	var options archive.TarOptions
+	optionsPipe := os.NewFile(3, "tar-options")
+	if optionsPipe == nil {
+		return nil, fmt.Errorf("could not read tar options from the pipe")
+	}
+	defer optionsPipe.Close()
+	err := json.NewDecoder(optionsPipe).Decode(&options)
+	if err != nil {
+		return &options, err
+	}
+
+	return &options, nil
+}

pkg/chrootarchive/archive_freebsd_test.go

@@ -0,0 +1,14 @@
+package chrootarchive // import "github.com/docker/docker/pkg/chrootarchive"
+
+import (
+	"testing"
+
+	"github.com/docker/docker/pkg/reexec"
+)
+
+func TestMain(m *testing.M) {
+	if reexec.Init() {
+		return
+	}
+	m.Run()
+}

pkg/chrootarchive/archive_linux.go

@@ -0,0 +1,54 @@
+package chrootarchive // import "github.com/docker/docker/pkg/chrootarchive"
+
+import (
+	"io"
+
+	"github.com/docker/docker/pkg/archive"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+func doUnpack(decompressedArchive io.Reader, relDest, root string, options *archive.TarOptions) error {
+	done := make(chan error)
+	err := goInChroot(root, func() { done <- archive.Unpack(decompressedArchive, relDest, options) })
+	if err != nil {
+		return err
+	}
+	return <-done
+}
+
+func doPack(relSrc, root string, options *archive.TarOptions) (io.ReadCloser, error) {
+	tb, err := archive.NewTarballer(relSrc, options)
+	if err != nil {
+		return nil, errors.Wrap(err, "error processing tar file")
+	}
+	err = goInChroot(root, tb.Do)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not chroot")
+	}
+	return tb.Reader(), nil
+}
+
+func doUnpackLayer(root string, layer io.Reader, options *archive.TarOptions) (int64, error) {
+	type result struct {
+		layerSize int64
+		err       error
+	}
+	done := make(chan result)
+
+	err := goInChroot(root, func() {
+		// We need to be able to set any perms
+		_ = unix.Umask(0)
+
+		size, err := archive.UnpackLayer("/", layer, options)
+		done <- result{layerSize: size, err: err}
+	})
+
+	if err != nil {
+		return 0, err
+	}
+
+	res := <-done
+
+	return res.layerSize, res.err
+}

pkg/chrootarchive/archive_unix.go

@@ -26,12 +26,7 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T
 		return err
 	}
 
-	done := make(chan error)
-	err = goInChroot(root, func() { done <- archive.Unpack(decompressedArchive, relDest, options) })
-	if err != nil {
-		return err
-	}
-	return <-done
+	return doUnpack(decompressedArchive, relDest, root, options)
 }
 
 func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) {
@@ -45,15 +40,7 @@ func invokePack(srcPath string, options *archive.TarOptions, root string) (io.Re
 		relSrc += "/"
 	}
 
-	tb, err := archive.NewTarballer(relSrc, options)
-	if err != nil {
-		return nil, errors.Wrap(err, "error processing tar file")
-	}
-	err = goInChroot(root, tb.Do)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not chroot")
-	}
-	return tb.Reader(), nil
+	return doPack(relSrc, root, options)
 }
 
 // resolvePathInChroot returns the equivalent to path inside a chroot rooted at root.

pkg/chrootarchive/diff_unix.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/containerd/containerd/pkg/userns"
 	"github.com/docker/docker/pkg/archive"
-	"golang.org/x/sys/unix"
 )
 
 // applyLayerHandler parses a diff in the standard layer format from `layer`, and
@@ -34,23 +33,5 @@ func applyLayerHandler(dest string, layer io.Reader, options *archive.TarOptions
 	if options.ExcludePatterns == nil {
 		options.ExcludePatterns = []string{}
 	}
-
-	type result struct {
-		layerSize int64
-		err       error
-	}
-
-	done := make(chan result)
-	err = goInChroot(dest, func() {
-		// We need to be able to set any perms
-		_ = unix.Umask(0)
-
-		size, err := archive.UnpackLayer("/", layer, options)
-		done <- result{layerSize: size, err: err}
-	})
-	if err != nil {
-		return 0, err
-	}
-	res := <-done
-	return res.layerSize, res.err
+	return doUnpackLayer(dest, layer, options)
 }