daemon: always dump stack to file

Dumping to log is unusable in 90% of cases and inspecting file is much
more convenient.

Signed-off-by: Alexander Morozov <lk4d4@docker.com>
This commit is contained in:
Alexander Morozov 2016-11-01 07:32:55 -07:00
parent c600cfa93c
commit e5d36586ac
3 changed files with 33 additions and 33 deletions

View file

@ -7,15 +7,22 @@ import (
"os/signal"
"syscall"
psignal "github.com/docker/docker/pkg/signal"
stackdump "github.com/docker/docker/pkg/signal"
"github.com/Sirupsen/logrus"
)
func setupDumpStackTrap(_ string) {
func setupDumpStackTrap(root string) {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGUSR1)
go func() {
for range c {
psignal.DumpStacks("")
path, err := stackdump.DumpStacks(root)
if err != nil {
logrus.WithError(err).Error("failed to write goroutines dump")
continue
}
logrus.Infof("goroutine stacks written to %s", path)
}
}()
}

View file

@ -35,7 +35,12 @@ func setupDumpStackTrap(root string) {
logrus.Debugf("Stackdump - waiting signal at %s", ev)
for {
syscall.WaitForSingleObject(h, syscall.INFINITE)
signal.DumpStacks(root)
path, err := signal.DumpStacks(root)
if err != nil {
logrus.WithError(err).Error("failed to write goroutines dump")
continue
}
logrus.Infof("goroutine stacks written to %s", path)
}
}()
}

View file

@ -1,6 +1,7 @@
package signal
import (
"fmt"
"os"
gosignal "os/signal"
"path/filepath"
@ -10,6 +11,7 @@ import (
"time"
"github.com/Sirupsen/logrus"
"github.com/pkg/errors"
)
// Trap sets up a simplified signal "trap", appropriate for common
@ -64,8 +66,11 @@ func Trap(cleanup func()) {
}()
}
// DumpStacks dumps the runtime stack.
func DumpStacks(root string) {
const stacksLogNameTemplate = "goroutine-stacks-%s.log"
// DumpStacks appends the runtime stack into file in dir and returns full path
// to that file.
func DumpStacks(dir string) (string, error) {
var (
buf []byte
stackSize int
@ -77,32 +82,15 @@ func DumpStacks(root string) {
bufferLen *= 2
}
buf = buf[:stackSize]
// Note that if the daemon is started with a less-verbose log-level than "info" (the default), the goroutine
// traces won't show up in the log.
if root == "" {
logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
} else {
// Dumps the stacks to a file in the root directory of the daemon
// On Windows, this overcomes two issues - one being that if the stack is too big, it doesn't
// get written to the event log when the Windows daemon is running as a service.
// Second, using logrus, the tabs and new-lines end up getting written as literal
// \t and \n's, meaning you need to use something like notepad++ to convert the
// output into something readable using 'type' from a command line or notepad/notepad++ etc.
path := filepath.Join(root, "goroutine-stacks.log")
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
path := filepath.Join(dir, fmt.Sprintf(stacksLogNameTemplate, time.Now().Format(time.RFC3339)))
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
logrus.Warnf("Could not open %s to write the goroutine stacks: %v", path, err)
return
return "", errors.Wrap(err, "failed to open file to write the goroutine stacks")
}
defer f.Close()
f.WriteString("=== BEGIN goroutine stack dump ===\n")
f.WriteString(time.Now().String() + "\n")
if _, err := f.Write(buf); err != nil {
logrus.Warnf("Could not write goroutine stacks to %s: %v", path, err)
return
return "", errors.Wrap(err, "failed to write goroutine stacks")
}
f.WriteString("=== END goroutine stack dump ===\n")
f.Sync()
logrus.Infof("goroutine stacks written to %s", path)
}
return path, nil
}