Implement tail for docker logs

Fixes #4330
Docker-DCO-1.1-Signed-off-by: Alexandr Morozov <lk4d4math@gmail.com> (github: LK4D4)

api/client/commands.go

@@ -1693,6 +1693,7 @@ func (cli *DockerCli) CmdLogs(args ...string) error {
 		cmd    = cli.Subcmd("logs", "CONTAINER", "Fetch the logs of a container")
 		follow = cmd.Bool([]string{"f", "-follow"}, false, "Follow log output")
 		times  = cmd.Bool([]string{"t", "-timestamps"}, false, "Show timestamps")
+		tail   = cmd.String([]string{"-tail"}, "all", "Output the specified number of lines at the end of logs(all logs by default)")
 	)
 
 	if err := cmd.Parse(args); err != nil {
@@ -1726,6 +1727,7 @@ func (cli *DockerCli) CmdLogs(args ...string) error {
 	if *follow {
 		v.Set("follow", "1")
 	}
+	v.Set("tail", *tail)
 
 	return cli.streamHelper("GET", "/containers/"+name+"/logs?"+v.Encode(), env.GetSubEnv("Config").GetBool("Tty"), nil, cli.out, cli.err, nil)
 }

api/server/server.go

@@ -378,6 +378,7 @@ func getContainersLogs(eng *engine.Engine, version version.Version, w http.Respo
 		return err
 	}
 	logsJob.Setenv("follow", r.Form.Get("follow"))
+	logsJob.Setenv("tail", r.Form.Get("tail"))
 	logsJob.Setenv("stdout", r.Form.Get("stdout"))
 	logsJob.Setenv("stderr", r.Form.Get("stderr"))
 	logsJob.Setenv("timestamps", r.Form.Get("timestamps"))

docs/sources/reference/api/docker_remote_api_v1.13.md

@@ -306,7 +306,7 @@ Get stdout and stderr logs from the container ``id``
 
     **Example request**:
 
-       GET /containers/4fa6e0f0c678/logs?stderr=1&stdout=1&timestamps=1&follow=1 HTTP/1.1
+       GET /containers/4fa6e0f0c678/logs?stderr=1&stdout=1&timestamps=1&follow=1&tail=10 HTTP/1.1
 
     **Example response**:
 
@@ -319,14 +319,12 @@ Get stdout and stderr logs from the container ``id``
 
      
 
-    -   **follow** – 1/True/true or 0/False/false, return stream.
-        Default false
-    -   **stdout** – 1/True/true or 0/False/false, if logs=true, return
-        stdout log. Default false
-    -   **stderr** – 1/True/true or 0/False/false, if logs=true, return
-        stderr log. Default false
-    -   **timestamps** – 1/True/true or 0/False/false, if logs=true, print
-        timestamps for every log line. Default false
+    -   **follow** – 1/True/true or 0/False/false, return stream. Default false
+    -   **stdout** – 1/True/true or 0/False/false, show stdout log. Default false
+    -   **stderr** – 1/True/true or 0/False/false, show stderr log. Default false
+    -   **timestamps** – 1/True/true or 0/False/false, print timestamps for
+        every log line. Default false
+    -   **tail** – Output specified number of lines at the end of logs: `all` or `<number>`. Default all
 
     Status Codes:
 

docs/sources/reference/commandline/cli.md

@@ -738,13 +738,15 @@ specify this by adding the server name.
 
       -f, --follow=false        Follow log output
       -t, --timestamps=false    Show timestamps
+      --tail="all"              Output the specified number of lines at the end of logs (all logs by default)
 
-The `docker logs` command batch-retrieves all logs
-present at the time of execution.
+The `docker logs` command batch-retrieves logs present at the time of execution.
 
-The ``docker logs --follow`` command will first return all logs from the
-beginning and then continue streaming new output from the container's `STDOUT`
-and `STDERR`.
+The `docker logs --follow` command will continue streaming the new output from
+the container's `STDOUT` and `STDERR`.
+
+Passing a negative number or a non-integer to --tail is invalid and the
+value is set to all in that case. This behavior may change in the future.
 
 ## port
 

integration-cli/docker_cli_logs_test.go

@@ -169,3 +169,47 @@ func TestLogsStderrInStdout(t *testing.T) {
 
 	logDone("logs - stderr in stdout (with pseudo-tty)")
 }
+
+func TestLogsTail(t *testing.T) {
+	testLen := 100
+	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("for i in $(seq 1 %d); do echo =; done;", testLen))
+
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, fmt.Sprintf("run failed with errors: %v", err))
+
+	cleanedContainerID := stripTrailingCharacters(out)
+	exec.Command(dockerBinary, "wait", cleanedContainerID).Run()
+
+	logsCmd := exec.Command(dockerBinary, "logs", "--tail", "5", cleanedContainerID)
+	out, _, _, err = runCommandWithStdoutStderr(logsCmd)
+	errorOut(err, t, fmt.Sprintf("failed to log container: %v %v", out, err))
+
+	lines := strings.Split(out, "\n")
+
+	if len(lines) != 6 {
+		t.Fatalf("Expected log %d lines, received %d\n", 6, len(lines))
+	}
+
+	logsCmd = exec.Command(dockerBinary, "logs", "--tail", "all", cleanedContainerID)
+	out, _, _, err = runCommandWithStdoutStderr(logsCmd)
+	errorOut(err, t, fmt.Sprintf("failed to log container: %v %v", out, err))
+
+	lines = strings.Split(out, "\n")
+
+	if len(lines) != testLen+1 {
+		t.Fatalf("Expected log %d lines, received %d\n", testLen+1, len(lines))
+	}
+
+	logsCmd = exec.Command(dockerBinary, "logs", "--tail", "random", cleanedContainerID)
+	out, _, _, err = runCommandWithStdoutStderr(logsCmd)
+	errorOut(err, t, fmt.Sprintf("failed to log container: %v %v", out, err))
+
+	lines = strings.Split(out, "\n")
+
+	if len(lines) != testLen+1 {
+		t.Fatalf("Expected log %d lines, received %d\n", testLen+1, len(lines))
+	}
+
+	deleteContainer(cleanedContainerID)
+	logDone("logs - logs tail")
+}

pkg/tailfile/tailfile.go

@@ -0,0 +1,61 @@
+package tailfile
+
+import (
+	"bytes"
+	"errors"
+	"os"
+)
+
+const blockSize = 1024
+
+var eol = []byte("\n")
+var ErrNonPositiveLinesNumber = errors.New("Lines number must be positive")
+
+//TailFile returns last n lines of file f
+func TailFile(f *os.File, n int) ([][]byte, error) {
+	if n <= 0 {
+		return nil, ErrNonPositiveLinesNumber
+	}
+	size, err := f.Seek(0, os.SEEK_END)
+	if err != nil {
+		return nil, err
+	}
+	block := -1
+	var data []byte
+	var cnt int
+	for {
+		var b []byte
+		step := int64(block * blockSize)
+		left := size + step // how many bytes to beginning
+		if left < 0 {
+			if _, err := f.Seek(0, os.SEEK_SET); err != nil {
+				return nil, err
+			}
+			b = make([]byte, blockSize+left)
+			if _, err := f.Read(b); err != nil {
+				return nil, err
+			}
+			data = append(b, data...)
+			break
+		} else {
+			b = make([]byte, blockSize)
+			if _, err := f.Seek(step, os.SEEK_END); err != nil {
+				return nil, err
+			}
+			if _, err := f.Read(b); err != nil {
+				return nil, err
+			}
+			data = append(b, data...)
+		}
+		cnt += bytes.Count(b, eol)
+		if cnt > n {
+			break
+		}
+		block--
+	}
+	lines := bytes.Split(data, eol)
+	if n < len(lines) {
+		return lines[len(lines)-n-1 : len(lines)-1], nil
+	}
+	return lines[:len(lines)-1], nil
+}

pkg/tailfile/tailfile_test.go

@@ -0,0 +1,148 @@
+package tailfile
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestTailFile(t *testing.T) {
+	f, err := ioutil.TempFile("", "tail-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	defer os.RemoveAll(f.Name())
+	testFile := []byte(`first line
+second line
+third line
+fourth line
+fifth line
+next first line
+next second line
+next third line
+next fourth line
+next fifth line
+last first line
+next first line
+next second line
+next third line
+next fourth line
+next fifth line
+next first line
+next second line
+next third line
+next fourth line
+next fifth line
+last second line
+last third line
+last fourth line
+last fifth line
+truncated line`)
+	if _, err := f.Write(testFile); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
+		t.Fatal(err)
+	}
+	expected := []string{"last fourth line", "last fifth line"}
+	res, err := TailFile(f, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i, l := range res {
+		t.Logf("%s", l)
+		if expected[i] != string(l) {
+			t.Fatalf("Expected line %s, got %s", expected[i], l)
+		}
+	}
+}
+
+func TestTailFileManyLines(t *testing.T) {
+	f, err := ioutil.TempFile("", "tail-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	defer os.RemoveAll(f.Name())
+	testFile := []byte(`first line
+second line
+truncated line`)
+	if _, err := f.Write(testFile); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
+		t.Fatal(err)
+	}
+	expected := []string{"first line", "second line"}
+	res, err := TailFile(f, 10000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i, l := range res {
+		t.Logf("%s", l)
+		if expected[i] != string(l) {
+			t.Fatalf("Expected line %s, got %s", expected[i], l)
+		}
+	}
+}
+
+func TestTailEmptyFile(t *testing.T) {
+	f, err := ioutil.TempFile("", "tail-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	defer os.RemoveAll(f.Name())
+	res, err := TailFile(f, 10000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(res) != 0 {
+		t.Fatal("Must be empty slice from empty file")
+	}
+}
+
+func TestTailNegativeN(t *testing.T) {
+	f, err := ioutil.TempFile("", "tail-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	defer os.RemoveAll(f.Name())
+	testFile := []byte(`first line
+second line
+truncated line`)
+	if _, err := f.Write(testFile); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := TailFile(f, -1); err != ErrNonPositiveLinesNumber {
+		t.Fatalf("Expected ErrNonPositiveLinesNumber, got %s", err)
+	}
+	if _, err := TailFile(f, 0); err != ErrNonPositiveLinesNumber {
+		t.Fatalf("Expected ErrNonPositiveLinesNumber, got %s", err)
+	}
+}
+
+func BenchmarkTail(b *testing.B) {
+	f, err := ioutil.TempFile("", "tail-test")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer f.Close()
+	defer os.RemoveAll(f.Name())
+	for i := 0; i < 10000; i++ {
+		if _, err := f.Write([]byte("tailfile pretty interesting line\n")); err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := TailFile(f, 1000); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

server/server.go

@@ -22,6 +22,7 @@
 package server
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -52,6 +53,7 @@ import (
 	"github.com/dotcloud/docker/image"
 	"github.com/dotcloud/docker/pkg/graphdb"
 	"github.com/dotcloud/docker/pkg/signal"
+	"github.com/dotcloud/docker/pkg/tailfile"
 	"github.com/dotcloud/docker/registry"
 	"github.com/dotcloud/docker/runconfig"
 	"github.com/dotcloud/docker/utils"
@@ -2153,8 +2155,10 @@ func (srv *Server) ContainerLogs(job *engine.Job) engine.Status {
 		name   = job.Args[0]
 		stdout = job.GetenvBool("stdout")
 		stderr = job.GetenvBool("stderr")
+		tail   = job.Getenv("tail")
 		follow = job.GetenvBool("follow")
 		times  = job.GetenvBool("timestamps")
+		lines  = -1
 		format string
 	)
 	if !(stdout || stderr) {
@@ -2163,6 +2167,9 @@ func (srv *Server) ContainerLogs(job *engine.Job) engine.Status {
 	if times {
 		format = time.StampMilli
 	}
+	if tail == "" {
+		tail = "all"
+	}
 	container := srv.daemon.Get(name)
 	if container == nil {
 		return job.Errorf("No such container: %s", name)
@@ -2190,25 +2197,47 @@ func (srv *Server) ContainerLogs(job *engine.Job) engine.Status {
 	} else if err != nil {
 		utils.Errorf("Error reading logs (json): %s", err)
 	} else {
-		dec := json.NewDecoder(cLog)
-		for {
-			l := &utils.JSONLog{}
-
-			if err := dec.Decode(l); err == io.EOF {
-				break
-			} else if err != nil {
-				utils.Errorf("Error streaming logs: %s", err)
-				break
-			}
-			logLine := l.Log
-			if times {
-				logLine = fmt.Sprintf("[%s] %s", l.Created.Format(format), logLine)
+		if tail != "all" {
+			var err error
+			lines, err = strconv.Atoi(tail)
+			if err != nil {
+				utils.Errorf("Failed to parse tail %s, error: %v, show all logs", err)
+				lines = -1
 			}
-			if l.Stream == "stdout" && stdout {
-				fmt.Fprintf(job.Stdout, "%s", logLine)
+		}
+		if lines != 0 {
+			if lines > 0 {
+				f := cLog.(*os.File)
+				ls, err := tailfile.TailFile(f, lines)
+				if err != nil {
+					return job.Error(err)
+				}
+				tmp := bytes.NewBuffer([]byte{})
+				for _, l := range ls {
+					fmt.Fprintf(tmp, "%s\n", l)
+				}
+				cLog = tmp
 			}
-			if l.Stream == "stderr" && stderr {
-				fmt.Fprintf(job.Stderr, "%s", logLine)
+			dec := json.NewDecoder(cLog)
+			for {
+				l := &utils.JSONLog{}
+
+				if err := dec.Decode(l); err == io.EOF {
+					break
+				} else if err != nil {
+					utils.Errorf("Error streaming logs: %s", err)
+					break
+				}
+				logLine := l.Log
+				if times {
+					logLine = fmt.Sprintf("[%s] %s", l.Created.Format(format), logLine)
+				}
+				if l.Stream == "stdout" && stdout {
+					fmt.Fprintf(job.Stdout, "%s", logLine)
+				}
+				if l.Stream == "stderr" && stderr {
+					fmt.Fprintf(job.Stderr, "%s", logLine)
+				}
 			}
 		}
 	}