daemon: always dump stack to file
Dumping to log is unusable in 90% of cases and inspecting file is much more convenient. Signed-off-by: Alexander Morozov <lk4d4@docker.com>
This commit is contained in:
parent
c600cfa93c
commit
e5d36586ac
3 changed files with 33 additions and 33 deletions
|
@ -7,15 +7,22 @@ import (
|
|||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
psignal "github.com/docker/docker/pkg/signal"
|
||||
stackdump "github.com/docker/docker/pkg/signal"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
)
|
||||
|
||||
func setupDumpStackTrap(_ string) {
|
||||
func setupDumpStackTrap(root string) {
|
||||
c := make(chan os.Signal, 1)
|
||||
signal.Notify(c, syscall.SIGUSR1)
|
||||
go func() {
|
||||
for range c {
|
||||
psignal.DumpStacks("")
|
||||
path, err := stackdump.DumpStacks(root)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("failed to write goroutines dump")
|
||||
continue
|
||||
}
|
||||
logrus.Infof("goroutine stacks written to %s", path)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
|
|
@ -35,7 +35,12 @@ func setupDumpStackTrap(root string) {
|
|||
logrus.Debugf("Stackdump - waiting signal at %s", ev)
|
||||
for {
|
||||
syscall.WaitForSingleObject(h, syscall.INFINITE)
|
||||
signal.DumpStacks(root)
|
||||
path, err := signal.DumpStacks(root)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("failed to write goroutines dump")
|
||||
continue
|
||||
}
|
||||
logrus.Infof("goroutine stacks written to %s", path)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package signal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
gosignal "os/signal"
|
||||
"path/filepath"
|
||||
|
@ -10,6 +11,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Trap sets up a simplified signal "trap", appropriate for common
|
||||
|
@ -64,8 +66,11 @@ func Trap(cleanup func()) {
|
|||
}()
|
||||
}
|
||||
|
||||
// DumpStacks dumps the runtime stack.
|
||||
func DumpStacks(root string) {
|
||||
const stacksLogNameTemplate = "goroutine-stacks-%s.log"
|
||||
|
||||
// DumpStacks appends the runtime stack into file in dir and returns full path
|
||||
// to that file.
|
||||
func DumpStacks(dir string) (string, error) {
|
||||
var (
|
||||
buf []byte
|
||||
stackSize int
|
||||
|
@ -77,32 +82,15 @@ func DumpStacks(root string) {
|
|||
bufferLen *= 2
|
||||
}
|
||||
buf = buf[:stackSize]
|
||||
// Note that if the daemon is started with a less-verbose log-level than "info" (the default), the goroutine
|
||||
// traces won't show up in the log.
|
||||
if root == "" {
|
||||
logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
|
||||
} else {
|
||||
// Dumps the stacks to a file in the root directory of the daemon
|
||||
// On Windows, this overcomes two issues - one being that if the stack is too big, it doesn't
|
||||
// get written to the event log when the Windows daemon is running as a service.
|
||||
// Second, using logrus, the tabs and new-lines end up getting written as literal
|
||||
// \t and \n's, meaning you need to use something like notepad++ to convert the
|
||||
// output into something readable using 'type' from a command line or notepad/notepad++ etc.
|
||||
path := filepath.Join(root, "goroutine-stacks.log")
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
|
||||
path := filepath.Join(dir, fmt.Sprintf(stacksLogNameTemplate, time.Now().Format(time.RFC3339)))
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
logrus.Warnf("Could not open %s to write the goroutine stacks: %v", path, err)
|
||||
return
|
||||
return "", errors.Wrap(err, "failed to open file to write the goroutine stacks")
|
||||
}
|
||||
defer f.Close()
|
||||
f.WriteString("=== BEGIN goroutine stack dump ===\n")
|
||||
f.WriteString(time.Now().String() + "\n")
|
||||
if _, err := f.Write(buf); err != nil {
|
||||
logrus.Warnf("Could not write goroutine stacks to %s: %v", path, err)
|
||||
return
|
||||
return "", errors.Wrap(err, "failed to write goroutine stacks")
|
||||
}
|
||||
f.WriteString("=== END goroutine stack dump ===\n")
|
||||
f.Sync()
|
||||
logrus.Infof("goroutine stacks written to %s", path)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue