Merge pull request #3132 from SiYue-ZO/fix/goroutine-recover

fix: add panic recovery to core-path goroutines

pkg/channels/manager.go

@@ -13,6 +13,7 @@ import (
 	"math"
 	"net"
 	"net/http"
+	"runtime/debug"
 	"sort"
 	"strings"
 	"sync"
@@ -1279,6 +1280,16 @@ func (m *Manager) StartAll(ctx context.Context) error {
 			for _, listener := range m.httpListeners {
 				ln := listener
 				go func() {
+					defer func() {
+						if r := recover(); r != nil {
+							logger.ErrorCF("channels", "HTTP server goroutine panic recovered",
+								map[string]any{
+									"addr":  ln.Addr().String(),
+									"panic": fmt.Sprintf("%v", r),
+									"stack": string(debug.Stack()),
+								})
+						}
+					}()
 					logger.InfoCF("channels", "Shared HTTP server listening", map[string]any{
 						"addr": ln.Addr().String(),
 					})
@@ -1292,6 +1303,16 @@ func (m *Manager) StartAll(ctx context.Context) error {
 			}
 		} else {
 			go func() {
+				defer func() {
+					if r := recover(); r != nil {
+						logger.ErrorCF("channels", "HTTP server goroutine panic recovered",
+							map[string]any{
+								"addr":  m.httpServer.Addr,
+								"panic": fmt.Sprintf("%v", r),
+								"stack": string(debug.Stack()),
+							})
+					}
+				}()
 				logger.InfoCF("channels", "Shared HTTP server listening", map[string]any{
 					"addr": m.httpServer.Addr,
 				})
@@ -1943,6 +1964,15 @@ func (m *Manager) Reload(ctx context.Context, cfg *config.Config) error {
 	// Commit hashes only on full success.
 	m.channelHashes = list
 	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				logger.ErrorCF("channels", "channel registration goroutine panic recovered",
+					map[string]any{
+						"panic": fmt.Sprintf("%v", r),
+						"stack": string(debug.Stack()),
+					})
+			}
+		}()
 		for _, f := range deferFuncs {
 			f()
 		}

pkg/events/subscription.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"log"
+	"runtime/debug"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -227,6 +228,11 @@ func (s *eventSubscription) dispatch(ctx context.Context, evt Event) {
 		s.wg.Add(1)
 		go func() {
 			defer s.wg.Done()
+			defer func() {
+				if r := recover(); r != nil {
+					log.Printf("events: subscriber %q goroutine panic recovered: %v\n%s", s.name, r, debug.Stack())
+				}
+			}()
 			s.handle(ctx, evt)
 		}()
 	case Keyed:
@@ -253,6 +259,15 @@ func (s *eventSubscription) handle(ctx context.Context, evt Event) {
 
 	done := make(chan handlerResult, 1)
 	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Printf(
+					"events: subscriber %q timeout-handler goroutine panic recovered: %v\n%s",
+					s.name, r, debug.Stack(),
+				)
+				done <- handlerResult{panicked: true}
+			}
+		}()
 		done <- s.invokeHandler(ctx, evt)
 	}()
 
@@ -302,6 +317,14 @@ func (s *eventSubscription) watchContext(ctx context.Context) {
 	}
 
 	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Printf(
+					"events: subscriber %q watchContext goroutine panic recovered: %v\n%s",
+					s.name, r, debug.Stack(),
+				)
+			}
+		}()
 		select {
 		case <-ctx.Done():
 			_ = s.Close()

pkg/tools/shell.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"runtime/debug"
 	"strings"
 	"sync"
 	"time"
@@ -417,6 +418,16 @@ func (t *ExecTool) runSync(ctx context.Context, command, cwd string) *ToolResult
 
 	done := make(chan error, 1)
 	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				logger.ErrorCF("shell", "cmd.Wait goroutine panic recovered",
+					map[string]any{
+						"panic": fmt.Sprintf("%v", r),
+						"stack": string(debug.Stack()),
+					})
+				done <- fmt.Errorf("panic in cmd.Wait: %v", r)
+			}
+		}()
 		done <- cmd.Wait()
 	}()
 
@@ -573,6 +584,18 @@ func (t *ExecTool) runBackground(ctx context.Context, command, cwd string, ptyEn
 	// so we need cmd.Wait() in a separate goroutine to detect process exit.
 	if session.PTY && session.ptyMaster != nil {
 		go func() {
+			defer func() {
+				if r := recover(); r != nil {
+					logger.ErrorCF("shell", "PTY cmd.Wait goroutine panic recovered",
+						map[string]any{
+							"panic": fmt.Sprintf("%v", r),
+							"stack": string(debug.Stack()),
+						})
+					session.mu.Lock()
+					session.Status = "error"
+					session.mu.Unlock()
+				}
+			}()
 			cmd.Wait() // Wait for process to exit
 			session.mu.Lock()
 			if cmd.ProcessState != nil {
@@ -583,6 +606,15 @@ func (t *ExecTool) runBackground(ctx context.Context, command, cwd string, ptyEn
 		}()
 
 		go func() {
+			defer func() {
+				if r := recover(); r != nil {
+					logger.ErrorCF("shell", "PTY read goroutine panic recovered",
+						map[string]any{
+							"panic": fmt.Sprintf("%v", r),
+							"stack": string(debug.Stack()),
+						})
+				}
+			}()
 			buf := make([]byte, 4096)
 			for {
 				n, err := session.ptyMaster.Read(buf)
@@ -613,6 +645,15 @@ func (t *ExecTool) runBackground(ctx context.Context, command, cwd string, ptyEn
 		// When Read() returns EOF (pipe closed), we break.
 		// When process exits, OS closes pipe write end → Read() returns EOF → we exit.
 		go func() {
+			defer func() {
+				if r := recover(); r != nil {
+					logger.ErrorCF("shell", "pipe read goroutine panic recovered",
+						map[string]any{
+							"panic": fmt.Sprintf("%v", r),
+							"stack": string(debug.Stack()),
+						})
+				}
+			}()
 			buf := make([]byte, 4096)
 
 			// Read stdout

pkg/tools/toolloop.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"runtime/debug"
 	"sync"
 
 	"github.com/sipeed/picoclaw/pkg/logger"
@@ -165,6 +166,17 @@ func RunToolLoop(
 			wg.Add(1)
 			go func(idx int, tc providers.ToolCall) {
 				defer wg.Done()
+				defer func() {
+					if r := recover(); r != nil {
+						logger.ErrorCF("toolloop", "tool execution goroutine panic recovered",
+							map[string]any{
+								"tool":  tc.Name,
+								"panic": fmt.Sprintf("%v", r),
+								"stack": string(debug.Stack()),
+							})
+						results[idx].result = ErrorResult(fmt.Sprintf("internal panic in tool %s", tc.Name))
+					}
+				}()
 
 				argsJSON, _ := json.Marshal(tc.Arguments)
 				argsPreview := utils.Truncate(string(argsJSON), 200)