| // Copyright 2020 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| package errutil |
| |
| import ( |
| "bufio" |
| "bytes" |
| "context" |
| "errors" |
| "fmt" |
| "io" |
| "net" |
| "os" |
| "runtime/pprof" |
| "strings" |
| "time" |
| |
| "go.fuchsia.dev/fuchsia/tools/lib/logger" |
| ) |
| |
| // HandleError prints backtraces on the device for certain errors. |
| func HandleError(ctx context.Context, serialSocketPath string, err error) { |
| if e := printBacktraces(ctx, serialSocketPath, err); e != nil { |
| logger.Errorf(ctx, "failed to dump backtraces: %v", e) |
| } |
| |
| // Construct the command line arguments that produced this failure. |
| var b bytes.Buffer |
| for i, arg := range os.Args { |
| if i != 0 { |
| b.WriteString(" ") |
| } |
| |
| b.WriteString(fmt.Sprintf("%q", arg)) |
| } |
| |
| logger.Infof(ctx, "This command can be reproduced by running: %s", b.String()) |
| } |
| |
| func printBacktraces(ctx context.Context, serialSocketPath string, err error) error { |
| if shouldPrintThreads(err) { |
| return nil |
| } |
| |
| printGoBacktrace(ctx) |
| |
| // We can't print the backtrace if we aren't connected to serial. |
| if serialSocketPath == "" { |
| logger.Warningf(ctx, "not to configured to serial, cannot run serial commands") |
| return nil |
| } |
| |
| serial, err := net.Dial("unix", serialSocketPath) |
| if err != nil { |
| return fmt.Errorf("failed to connect to serial socket: %v", err) |
| } |
| defer serial.Close() |
| |
| // Discard any of the output from the serial port. |
| go io.Copy(io.Discard, serial) |
| |
| if err := printSyslogs(ctx, serial); err != nil { |
| return err |
| } |
| |
| if err := printNetstackGoroutines(ctx, serial); err != nil { |
| return err |
| } |
| |
| if err := printNetstackInspect(ctx, serial); err != nil { |
| return err |
| } |
| |
| if err := printDeviceProcessBacktraces(ctx, serial); err != nil { |
| return err |
| } |
| |
| // dump the traces again so we can tell which trace might be hung. |
| if err := printDeviceProcessBacktraces(ctx, serial); err != nil { |
| return err |
| } |
| |
| return nil |
| } |
| |
| func printSyslogs(ctx context.Context, serial net.Conn) error { |
| logger.Infof(ctx, "printing system logs to serial") |
| |
| // Print syslog. This will also emit a newline before |
| // the command to get a fresh line, and print markers before and after |
| // the backtrace to make it easier to distinguish in the logs. |
| cmd := fmt.Sprintf("\n(echo '%s --- system logs ---' && /bin/log_listener --dump_logs yes && echo '------------------------------------') &\n", time.Now().Format(time.RFC3339Nano)) |
| |
| err := run(ctx, serial, cmd) |
| if err != nil { |
| return err |
| } |
| |
| // We don't know how long it'll take to dump all threads, so sleep for |
| // a minute to give `threads` a chance to complete. |
| time.Sleep(60 * time.Second) |
| |
| logger.Infof(ctx, "done waiting for backtraces to be printed") |
| |
| return nil |
| } |
| |
| func printGoBacktrace(ctx context.Context) { |
| logger.Infof(ctx, "printing go backtrace") |
| |
| var buf bytes.Buffer |
| pprof.Lookup("goroutine").WriteTo(&buf, 1) |
| |
| s := bufio.NewScanner(&buf) |
| for s.Scan() { |
| logger.Infof(ctx, "%s", s.Text()) |
| } |
| } |
| |
| func printDeviceProcessBacktraces(ctx context.Context, serial net.Conn) error { |
| logger.Infof(ctx, "printing all process backtraces to serial") |
| |
| // Print all process backtraces. This will also emit a newline before |
| // the command to get a fresh line, and print markers before and after |
| // the backtrace to make it easier to distinguish in the logs. |
| cmd := fmt.Sprintf("\n(echo '%s --- all process backtrace ---' && threads --all-processes && echo '------------------------------------') &\n", time.Now().Format(time.RFC3339Nano)) |
| err := run(ctx, serial, cmd) |
| if err != nil { |
| return err |
| } |
| |
| // We don't know how long it'll take to dump all threads, so sleep for |
| // a minute to give `threads` a chance to complete. |
| time.Sleep(60 * time.Second) |
| |
| logger.Infof(ctx, "done waiting for backtraces to be printed") |
| |
| return nil |
| } |
| |
| func printNetstackGoroutines(ctx context.Context, serial net.Conn) error { |
| logger.Infof(ctx, "printing netstack goroutine stack traces to serial") |
| |
| // Print out the netstack goroutines while only using shell-builtin commands to avoid hitting the package resolver. |
| const shellFmtStr = `(export PATH=; |
| echo '%s --- netstack goroutine traces ---'; |
| export P="$(component explore /core/network/netstack -c \"cat out/debug/goroutines\")" && |
| test -e "$P" && while IFS='' read line; do echo "$line"; done < "$P"; |
| echo '------------------------------------') & |
| ` |
| cmd := fmt.Sprintf(shellFmtStr, time.Now().Format(time.RFC3339Nano)) |
| |
| err := run(ctx, serial, cmd) |
| if err != nil { |
| return err |
| } |
| |
| time.Sleep(60 * time.Second) |
| |
| logger.Infof(ctx, "done waiting for netstack goroutine to be printed") |
| |
| return nil |
| } |
| |
| func printNetstackInspect(ctx context.Context, serial net.Conn) error { |
| logger.Infof(ctx, "printing netstack inspect to serial") |
| |
| // Print iquery. This will also emit a newline before |
| // the command to get a fresh line, and print markers before and after |
| // the backtrace to make it easier to distinguish in the logs. |
| cmd := fmt.Sprintf("\n(echo '%s --- iquery netstack logs ---' && /boot/bin/iquery show --manifest netstack.cm && echo '------------------------------------') &\n", time.Now().Format(time.RFC3339Nano)) |
| |
| err := run(ctx, serial, cmd) |
| if err != nil { |
| return err |
| } |
| |
| // We don't know how long it'll take to dump all threads, so sleep for |
| // a minute to give `inspect` a chance to complete. |
| time.Sleep(60 * time.Second) |
| |
| logger.Infof(ctx, "done waiting for inspect to be printed") |
| |
| return nil |
| } |
| |
| func run(ctx context.Context, serial net.Conn, cmd string) error { |
| logger.Infof(ctx, "executing %q", cmd) |
| |
| // Add a newline before executing the command to get a fresh line. Also |
| // run it in the background in case `threads` hangs. |
| if _, err := serial.Write([]byte(cmd)); err != nil { |
| return fmt.Errorf("failed to execute %q: %w", cmd, err) |
| } |
| |
| return nil |
| } |
| |
| func shouldPrintThreads(err error) bool { |
| return errors.Is(err, context.DeadlineExceeded) || |
| strings.Contains(err.Error(), "context deadline exceeded") || |
| strings.Contains(err.Error(), "use of closed network connection") || |
| strings.Contains(err.Error(), "device failed to initialize") |
| } |