package pico import ( "context" "encoding/base64" "encoding/json" "fmt" "mime" "net/http" "net/url" "os" "path/filepath" "strings" "sync" "sync/atomic" "time" "github.com/google/uuid" "github.com/gorilla/websocket" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/channels" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/identity" "github.com/sipeed/picoclaw/pkg/logger" "github.com/sipeed/picoclaw/pkg/utils" ) // picoConn represents a single WebSocket connection. type picoConn struct { id string conn *websocket.Conn sessionID string writeMu sync.Mutex closed atomic.Bool cancel context.CancelFunc // cancels per-connection goroutines (e.g. pingLoop) } var allowedInlineImageMIMETypes = map[string]struct{}{ "image/jpeg": {}, "image/png": {}, "image/gif": {}, "image/webp": {}, "image/bmp": {}, } func outboundMessageIsThought(msg bus.OutboundMessage) bool { if len(msg.Context.Raw) == 0 { return false } return strings.EqualFold(strings.TrimSpace(msg.Context.Raw["message_kind"]), MessageKindThought) } func outboundMessageIsToolFeedback(msg bus.OutboundMessage) bool { if len(msg.Context.Raw) == 0 { return false } return strings.EqualFold(strings.TrimSpace(msg.Context.Raw["message_kind"]), "tool_feedback") } func outboundMessageIsToolCalls(msg bus.OutboundMessage) bool { if len(msg.Context.Raw) == 0 { return false } return strings.EqualFold(strings.TrimSpace(msg.Context.Raw["message_kind"]), MessageKindToolCalls) } func outboundMessageFinalizesTrackedToolFeedback(msg bus.OutboundMessage) bool { return !outboundMessageIsToolFeedback(msg) && !outboundMessageIsThought(msg) && !outboundMessageIsToolCalls(msg) } // writeJSON sends a JSON message to the connection with write locking. func (pc *picoConn) writeJSON(v any) error { if pc.closed.Load() { return fmt.Errorf("connection closed") } pc.writeMu.Lock() defer pc.writeMu.Unlock() return pc.conn.WriteJSON(v) } // close closes the connection. func (pc *picoConn) close() { if pc.closed.CompareAndSwap(false, true) { if pc.cancel != nil { pc.cancel() } pc.conn.Close() } } // PicoChannel implements the native Pico Protocol WebSocket channel. // It serves as the reference implementation for all optional capability interfaces. type PicoChannel struct { *channels.BaseChannel bc *config.Channel config *config.PicoSettings upgrader websocket.Upgrader connections map[string]*picoConn // connID -> *picoConn sessionConnections map[string]map[string]*picoConn // sessionID -> connID -> *picoConn connsMu sync.RWMutex ctx context.Context cancel context.CancelFunc progress *channels.ToolFeedbackAnimator deleteMessageFn func(context.Context, string, string) error // broadcastFn lets tests intercept outbound broadcasts. nil → broadcastToSession. broadcastFn func(chatID string, msg PicoMessage) error } // NewPicoChannel creates a new Pico Protocol channel. func NewPicoChannel( bc *config.Channel, cfg *config.PicoSettings, messageBus *bus.MessageBus, ) (*PicoChannel, error) { if cfg.Token.String() == "" { return nil, fmt.Errorf("pico token is required") } base := channels.NewBaseChannel("pico", cfg, messageBus, bc.AllowFrom) allowOrigins := cfg.AllowOrigins checkOrigin := func(r *http.Request) bool { if len(allowOrigins) == 0 { return true // allow all if not configured } origin := r.Header.Get("Origin") for _, allowed := range allowOrigins { if allowed == "*" || allowed == origin { return true } } return false } ch := &PicoChannel{ BaseChannel: base, bc: bc, config: cfg, upgrader: websocket.Upgrader{ CheckOrigin: checkOrigin, ReadBufferSize: 1024, WriteBufferSize: 1024, }, connections: make(map[string]*picoConn), sessionConnections: make(map[string]map[string]*picoConn), } ch.progress = channels.NewToolFeedbackAnimator(ch.EditMessage) ch.deleteMessageFn = ch.DeleteMessage return ch, nil } // createAndAddConnection checks MaxConnections and registers a connection atomically. func (c *PicoChannel) createAndAddConnection(conn *websocket.Conn, sessionID string, maxConns int) (*picoConn, error) { c.connsMu.Lock() defer c.connsMu.Unlock() if len(c.connections) >= maxConns { return nil, channels.ErrTemporary } var connID string for { connID = uuid.New().String() if _, exists := c.connections[connID]; !exists { break } } pc := &picoConn{ id: connID, conn: conn, sessionID: sessionID, } c.connections[pc.id] = pc bySession, ok := c.sessionConnections[pc.sessionID] if !ok { bySession = make(map[string]*picoConn) c.sessionConnections[pc.sessionID] = bySession } bySession[pc.id] = pc return pc, nil } // removeConnection deletes a connection from indexes and returns it when found. func (c *PicoChannel) removeConnection(connID string) *picoConn { c.connsMu.Lock() defer c.connsMu.Unlock() pc, ok := c.connections[connID] if !ok { return nil } delete(c.connections, connID) if bySession, ok := c.sessionConnections[pc.sessionID]; ok { delete(bySession, connID) if len(bySession) == 0 { delete(c.sessionConnections, pc.sessionID) } } return pc } // takeAllConnections snapshots and clears all connection indexes. func (c *PicoChannel) takeAllConnections() []*picoConn { c.connsMu.Lock() defer c.connsMu.Unlock() all := make([]*picoConn, 0, len(c.connections)) for _, pc := range c.connections { all = append(all, pc) } clear(c.connections) clear(c.sessionConnections) return all } // sessionConnectionsSnapshot returns all active connections for a session. func (c *PicoChannel) sessionConnectionsSnapshot(sessionID string) []*picoConn { c.connsMu.RLock() defer c.connsMu.RUnlock() bySession, ok := c.sessionConnections[sessionID] if !ok || len(bySession) == 0 { return nil } conns := make([]*picoConn, 0, len(bySession)) for _, pc := range bySession { conns = append(conns, pc) } return conns } // currentConnCount returns a lock-protected snapshot of active connection count. func (c *PicoChannel) currentConnCount() int { c.connsMu.RLock() defer c.connsMu.RUnlock() return len(c.connections) } // Start implements Channel. func (c *PicoChannel) Start(ctx context.Context) error { logger.InfoC("pico", "Starting Pico Protocol channel") c.ctx, c.cancel = context.WithCancel(ctx) c.SetRunning(true) logger.InfoC("pico", "Pico Protocol channel started") return nil } // Stop implements Channel. func (c *PicoChannel) Stop(ctx context.Context) error { logger.InfoC("pico", "Stopping Pico Protocol channel") c.SetRunning(false) // Close all connections for _, pc := range c.takeAllConnections() { pc.close() } if c.cancel != nil { c.cancel() } if c.progress != nil { c.progress.StopAll() } logger.InfoC("pico", "Pico Protocol channel stopped") return nil } // WebhookPath implements channels.WebhookHandler. func (c *PicoChannel) WebhookPath() string { return "/pico/" } // ServeHTTP implements http.Handler for the shared HTTP server. func (c *PicoChannel) ServeHTTP(w http.ResponseWriter, r *http.Request) { path := strings.TrimPrefix(r.URL.Path, "/pico") switch path { case "/ws", "/ws/": c.handleWebSocket(w, r) default: if strings.HasPrefix(path, "/media/") { c.handleMediaDownload(w, r) return } http.NotFound(w, r) } } // Send implements Channel — sends a message to the appropriate WebSocket connection. func (c *PicoChannel) Send(ctx context.Context, msg bus.OutboundMessage) ([]string, error) { if !c.IsRunning() { return nil, channels.ErrNotRunning } isThought := outboundMessageIsThought(msg) isToolFeedback := outboundMessageIsToolFeedback(msg) isToolCalls := outboundMessageIsToolCalls(msg) if isToolFeedback { if msgID, handled, err := c.progress.Update(ctx, msg.ChatID, msg.Content); handled { if err != nil { return nil, err } return []string{msgID}, nil } } trackedMsgID, hasTrackedMsg := c.currentToolFeedbackMessage(msg.ChatID) if outboundMessageFinalizesTrackedToolFeedback(msg) { if msgIDs, handled := c.FinalizeToolFeedbackMessage(ctx, msg); handled { return msgIDs, nil } } content := msg.Content if isToolFeedback { content = channels.InitialAnimatedToolFeedbackContent(msg.Content) } msgID := uuid.New().String() payload := map[string]any{ PayloadKeyContent: content, "message_id": msgID, } if modelName := strings.TrimSpace(msg.Context.Raw[PayloadKeyModelName]); modelName != "" { payload[PayloadKeyModelName] = modelName } switch { case isThought: payload[PayloadKeyKind] = MessageKindThought // This field is kept solely for compatibility with legacy pico clients that // do not yet support the newer "kind" field. // DO NOT use it for any purpose other than legacy client compatibility. payload[PayloadKeyThought] = true case isToolCalls: payload[PayloadKeyKind] = MessageKindToolCalls if toolCalls, ok := picoToolCallsPayload(msg); ok { payload[PayloadKeyToolCalls] = toolCalls } } setContextUsagePayload(payload, msg.ContextUsage) outMsg := newMessage(TypeMessageCreate, payload) if err := c.broadcastToSession(msg.ChatID, outMsg); err != nil { return nil, err } if isToolFeedback { c.RecordToolFeedbackMessage(msg.ChatID, msgID, msg.Content) } else if hasTrackedMsg && outboundMessageFinalizesTrackedToolFeedback(msg) { c.dismissTrackedToolFeedbackMessage(ctx, msg.ChatID, trackedMsgID) } return []string{msgID}, nil } // EditMessage implements channels.MessageEditor. func (c *PicoChannel) EditMessage(ctx context.Context, chatID string, messageID string, content string) error { return c.editMessage(ctx, chatID, messageID, content, nil) } func (c *PicoChannel) EditMessageWithPayload( ctx context.Context, chatID string, messageID string, payload map[string]any, ) error { return c.editMessagePayload(ctx, chatID, messageID, payload, nil) } // DeleteMessage implements channels.MessageDeleter. func (c *PicoChannel) DeleteMessage(ctx context.Context, chatID string, messageID string) error { outMsg := newMessage(TypeMessageDelete, map[string]any{ "message_id": messageID, }) return c.broadcastToSession(chatID, outMsg) } func (c *PicoChannel) currentToolFeedbackMessage(chatID string) (string, bool) { if c.progress == nil { return "", false } return c.progress.Current(chatID) } func (c *PicoChannel) takeToolFeedbackMessage(chatID string) (string, string, bool) { if c.progress == nil { return "", "", false } return c.progress.Take(chatID) } func (c *PicoChannel) RecordToolFeedbackMessage(chatID, messageID, content string) { if c.progress == nil { return } c.progress.Record(chatID, messageID, content) } func (c *PicoChannel) ClearToolFeedbackMessage(chatID string) { if c.progress == nil { return } c.progress.Clear(chatID) } func (c *PicoChannel) DismissToolFeedbackMessage(ctx context.Context, chatID string) { msgID, ok := c.currentToolFeedbackMessage(chatID) if !ok { return } c.dismissTrackedToolFeedbackMessage(ctx, chatID, msgID) } func (c *PicoChannel) dismissTrackedToolFeedbackMessage(ctx context.Context, chatID, messageID string) { if strings.TrimSpace(chatID) == "" || strings.TrimSpace(messageID) == "" { return } c.ClearToolFeedbackMessage(chatID) deleteFn := c.deleteMessageFn if deleteFn == nil { deleteFn = c.DeleteMessage } _ = deleteFn(ctx, chatID, messageID) } func (c *PicoChannel) finalizeTrackedToolFeedbackMessage( ctx context.Context, chatID string, content string, editFn func(context.Context, string, string, map[string]any, *bus.ContextUsage) error, payload map[string]any, contextUsage *bus.ContextUsage, ) ([]string, bool) { msgID, baseContent, ok := c.takeToolFeedbackMessage(chatID) if !ok || editFn == nil { return nil, false } if payload == nil { payload = map[string]any{ PayloadKeyContent: content, } } if _, ok := payload[PayloadKeyContent]; !ok { payload[PayloadKeyContent] = content } if err := editFn(ctx, chatID, msgID, payload, contextUsage); err != nil { c.RecordToolFeedbackMessage(chatID, msgID, baseContent) return nil, false } return []string{msgID}, true } func (c *PicoChannel) FinalizeToolFeedbackMessage(ctx context.Context, msg bus.OutboundMessage) ([]string, bool) { if !outboundMessageFinalizesTrackedToolFeedback(msg) { return nil, false } payload := map[string]any{ PayloadKeyContent: msg.Content, } if modelName := strings.TrimSpace(msg.Context.Raw[PayloadKeyModelName]); modelName != "" { payload[PayloadKeyModelName] = modelName } return c.finalizeTrackedToolFeedbackMessage( ctx, msg.ChatID, msg.Content, c.editMessagePayload, payload, msg.ContextUsage, ) } // StartTyping implements channels.TypingCapable. func (c *PicoChannel) StartTyping(ctx context.Context, chatID string) (func(), error) { startMsg := newMessage(TypeTypingStart, nil) if err := c.broadcastToSession(chatID, startMsg); err != nil { return func() {}, err } return func() { stopMsg := newMessage(TypeTypingStop, nil) c.broadcastToSession(chatID, stopMsg) }, nil } // SendPlaceholder implements channels.PlaceholderCapable. // It sends a placeholder message via the Pico Protocol that will later be // edited to the actual response via EditMessage (channels.MessageEditor). func (c *PicoChannel) SendPlaceholder(ctx context.Context, chatID string) (string, error) { if !c.bc.Placeholder.Enabled { return "", nil } text := c.bc.Placeholder.GetRandomText() msgID := uuid.New().String() outMsg := newMessage(TypeMessageCreate, map[string]any{ PayloadKeyContent: text, PayloadKeyPlaceholder: true, "message_id": msgID, }) if err := c.broadcastToSession(chatID, outMsg); err != nil { return "", err } return msgID, nil } // BeginStream implements channels.StreamingCapable for Pico WebUI. func (c *PicoChannel) BeginStream(ctx context.Context, chatID string) (channels.Streamer, error) { if c == nil || c.config == nil || !c.config.Streaming.Enabled { return nil, fmt.Errorf("streaming disabled in config") } if !c.IsRunning() { return nil, channels.ErrNotRunning } streamCfg := c.config.Streaming.WithDefaults(0, 1) return &picoStreamer{ channel: c, chatID: chatID, throttleInterval: time.Duration(streamCfg.ThrottleSeconds) * time.Second, minGrowth: streamCfg.MinGrowthChars, }, nil } type picoStreamer struct { channel *PicoChannel chatID string modelName string turnInputTokens int turnOutputTokens int messageID string reasoningID string throttleInterval time.Duration minGrowth int lastLen int lastAt time.Time lastContent string reasoningLastLen int reasoningLastAt time.Time reasoningContent string mu sync.Mutex } func (s *picoStreamer) SetModelName(modelName string) { if s == nil { return } s.mu.Lock() defer s.mu.Unlock() s.modelName = strings.TrimSpace(modelName) } // SetTurnUsage records the real per-turn LLM token usage to emit on finalize. func (s *picoStreamer) SetTurnUsage(inputTokens, outputTokens int) { if s == nil { return } s.mu.Lock() defer s.mu.Unlock() s.turnInputTokens = inputTokens s.turnOutputTokens = outputTokens } func (s *picoStreamer) Update(ctx context.Context, content string) error { s.mu.Lock() defer s.mu.Unlock() return s.updateLocked(ctx, content, false, nil) } func (s *picoStreamer) Finalize(ctx context.Context, content string) error { return s.FinalizeWithContext(ctx, content, nil) } func (s *picoStreamer) FinalizeWithContext(ctx context.Context, content string, contextUsage *bus.ContextUsage) error { s.mu.Lock() defer s.mu.Unlock() return s.updateLocked(ctx, content, true, contextUsage) } func (s *picoStreamer) UpdateReasoning(ctx context.Context, content string) error { s.mu.Lock() defer s.mu.Unlock() return s.updateReasoningLocked(ctx, content, false) } func (s *picoStreamer) FinalizeReasoning(ctx context.Context, content string) error { s.mu.Lock() defer s.mu.Unlock() return s.updateReasoningLocked(ctx, content, true) } func (s *picoStreamer) Cancel(ctx context.Context) { if s == nil { return } s.mu.Lock() defer s.mu.Unlock() if s.channel == nil || s.messageID == "" { if s.channel != nil && s.reasoningID != "" { _ = s.channel.DeleteMessage(ctx, s.chatID, s.reasoningID) s.reasoningID = "" } return } _ = s.channel.DeleteMessage(ctx, s.chatID, s.messageID) s.messageID = "" if s.reasoningID != "" { _ = s.channel.DeleteMessage(ctx, s.chatID, s.reasoningID) s.reasoningID = "" } } func (s *picoStreamer) updateLocked( ctx context.Context, content string, force bool, contextUsage *bus.ContextUsage, ) error { if s == nil || s.channel == nil { return fmt.Errorf("streamer is not initialized") } if strings.TrimSpace(content) == "" && s.messageID == "" { return nil } now := time.Now() contentLen := len([]rune(content)) if s.messageID != "" && !force { growth := contentLen - s.lastLen if now.Sub(s.lastAt) < s.throttleInterval || growth < s.minGrowth { return nil } } return s.sendLocked(ctx, content, contextUsage) } func (s *picoStreamer) updateReasoningLocked(ctx context.Context, content string, force bool) error { if s == nil || s.channel == nil { return fmt.Errorf("streamer is not initialized") } if strings.TrimSpace(content) == "" && s.reasoningID == "" { return nil } now := time.Now() contentLen := len([]rune(content)) if s.reasoningID != "" && !force { growth := contentLen - s.reasoningLastLen if now.Sub(s.reasoningLastAt) < s.throttleInterval || growth < s.minGrowth { return nil } } return s.sendReasoningLocked(ctx, content) } func (s *picoStreamer) sendLocked(ctx context.Context, content string, contextUsage *bus.ContextUsage) error { now := time.Now() contentLen := len([]rune(content)) if s.messageID == "" { s.messageID = uuid.New().String() payload := map[string]any{ PayloadKeyContent: content, "message_id": s.messageID, } if s.modelName != "" { payload[PayloadKeyModelName] = s.modelName } setContextUsagePayload(payload, contextUsage) setTurnUsagePayload(payload, s.turnInputTokens, s.turnOutputTokens) outMsg := newMessage(TypeMessageCreate, payload) if err := s.channel.broadcast(s.chatID, outMsg); err != nil { return err } } else if content != s.lastContent || contextUsage != nil { payload := map[string]any{ PayloadKeyContent: content, "message_id": s.messageID, } if s.modelName != "" { payload[PayloadKeyModelName] = s.modelName } setTurnUsagePayload(payload, s.turnInputTokens, s.turnOutputTokens) if err := s.channel.editMessagePayload(ctx, s.chatID, s.messageID, payload, contextUsage); err != nil { return err } } s.lastContent = content s.lastLen = contentLen s.lastAt = now return nil } func (s *picoStreamer) sendReasoningLocked(ctx context.Context, content string) error { now := time.Now() contentLen := len([]rune(content)) if s.reasoningID == "" { s.reasoningID = uuid.New().String() payload := map[string]any{ PayloadKeyContent: content, "message_id": s.reasoningID, PayloadKeyKind: MessageKindThought, PayloadKeyThought: true, } if s.modelName != "" { payload[PayloadKeyModelName] = s.modelName } outMsg := newMessage(TypeMessageCreate, payload) if err := s.channel.broadcastToSession(s.chatID, outMsg); err != nil { return err } } else if content != s.reasoningContent { payload := map[string]any{ PayloadKeyContent: content, "message_id": s.reasoningID, PayloadKeyKind: MessageKindThought, PayloadKeyThought: true, } if s.modelName != "" { payload[PayloadKeyModelName] = s.modelName } outMsg := newMessage(TypeMessageUpdate, payload) if err := s.channel.broadcastToSession(s.chatID, outMsg); err != nil { return err } } s.reasoningContent = content s.reasoningLastLen = contentLen s.reasoningLastAt = now return nil } // SendMedia implements channels.MediaSender for the Pico web UI. // Media is delivered as a normal assistant message carrying structured // attachments plus an authenticated same-origin download URL. func (c *PicoChannel) SendMedia(ctx context.Context, msg bus.OutboundMediaMessage) ([]string, error) { if !c.IsRunning() { return nil, channels.ErrNotRunning } trackedMsgID, hasTrackedMsg := c.currentToolFeedbackMessage(msg.ChatID) store := c.GetMediaStore() if store == nil { return nil, fmt.Errorf("no media store available: %w", channels.ErrSendFailed) } attachments := make([]map[string]any, 0, len(msg.Parts)) caption := "" for _, part := range msg.Parts { localPath, meta, err := store.ResolveWithMeta(part.Ref) if err != nil { logger.ErrorCF("pico", "Failed to resolve media ref", map[string]any{ "ref": part.Ref, "error": err.Error(), }) continue } filename := strings.TrimSpace(part.Filename) if filename == "" { filename = strings.TrimSpace(meta.Filename) } if filename == "" { filename = filepath.Base(localPath) } contentType := strings.TrimSpace(part.ContentType) if contentType == "" { contentType = strings.TrimSpace(meta.ContentType) } if contentType == "" { contentType = "application/octet-stream" } attachmentType := strings.TrimSpace(part.Type) if attachmentType == "" { attachmentType = picoInferAttachmentType(filename, contentType) } attachmentURL, err := picoDownloadURLForRef(part.Ref) if err != nil { logger.ErrorCF("pico", "Failed to build media download URL", map[string]any{ "ref": part.Ref, "error": err.Error(), }) continue } attachments = append(attachments, map[string]any{ "type": attachmentType, "url": attachmentURL, "filename": filename, "content_type": contentType, }) if caption == "" && strings.TrimSpace(part.Caption) != "" { caption = strings.TrimSpace(part.Caption) } } if len(attachments) == 0 { return nil, fmt.Errorf("no deliverable media parts: %w", channels.ErrSendFailed) } msgID := uuid.New().String() outMsg := newMessage(TypeMessageCreate, map[string]any{ PayloadKeyContent: caption, "attachments": attachments, "message_id": msgID, }) if modelName := strings.TrimSpace(msg.Context.Raw[PayloadKeyModelName]); modelName != "" { outMsg.Payload[PayloadKeyModelName] = modelName } if err := c.broadcastToSession(msg.ChatID, outMsg); err != nil { return nil, err } if hasTrackedMsg { c.dismissTrackedToolFeedbackMessage(ctx, msg.ChatID, trackedMsgID) } return []string{msgID}, nil } func picoDownloadURLForRef(ref string) (string, error) { refID, err := picoMediaRefID(ref) if err != nil { return "", err } return "/pico/media/" + url.PathEscape(refID), nil } func picoMediaRefID(ref string) (string, error) { refID := strings.TrimSpace(strings.TrimPrefix(ref, "media://")) if refID == "" || strings.Contains(refID, "/") { return "", fmt.Errorf("invalid media ref %q", ref) } return refID, nil } func picoInferAttachmentType(filename, contentType string) string { contentType = strings.ToLower(strings.TrimSpace(contentType)) filename = strings.ToLower(strings.TrimSpace(filename)) switch { case strings.HasPrefix(contentType, "image/"): return "image" case strings.HasPrefix(contentType, "audio/"): return "audio" case strings.HasPrefix(contentType, "video/"): return "video" } switch ext := filepath.Ext(filename); ext { case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg": return "image" case ".mp3", ".wav", ".ogg", ".m4a", ".flac", ".aac", ".wma", ".opus": return "audio" case ".mp4", ".avi", ".mov", ".webm", ".mkv": return "video" default: return "file" } } func picoAllowsInlineDisplay(filename, contentType string) bool { contentType = strings.ToLower(strings.TrimSpace(contentType)) filename = strings.ToLower(strings.TrimSpace(filename)) if strings.Contains(contentType, "svg") || filepath.Ext(filename) == ".svg" { return false } return picoInferAttachmentType(filename, contentType) == "image" } func (c *PicoChannel) handleMediaDownload(w http.ResponseWriter, r *http.Request) { if !c.IsRunning() { http.Error(w, "channel not running", http.StatusServiceUnavailable) return } if !c.authenticate(r) { http.Error(w, "unauthorized", http.StatusUnauthorized) return } refID := strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(r.URL.Path, "/pico/media/"), "/")) if refID == "" { http.NotFound(w, r) return } store := c.GetMediaStore() if store == nil { http.Error(w, "media store unavailable", http.StatusServiceUnavailable) return } localPath, meta, err := store.ResolveWithMeta("media://" + refID) if err != nil { http.NotFound(w, r) return } file, err := os.Open(localPath) if err != nil { http.Error(w, "failed to open media", http.StatusInternalServerError) return } defer file.Close() info, err := file.Stat() if err != nil { http.Error(w, "failed to stat media", http.StatusInternalServerError) return } filename := strings.TrimSpace(meta.Filename) if filename == "" { filename = filepath.Base(localPath) } contentType := strings.TrimSpace(meta.ContentType) if contentType == "" { contentType = "application/octet-stream" } dispositionType := "attachment" if picoAllowsInlineDisplay(filename, contentType) { dispositionType = "inline" } if cd := mime.FormatMediaType(dispositionType, map[string]string{"filename": filename}); cd != "" { w.Header().Set("Content-Disposition", cd) } w.Header().Set("Content-Type", contentType) http.ServeContent(w, r, filename, info.ModTime(), file) } // broadcast routes through broadcastFn when set (tests), else broadcastToSession. func (c *PicoChannel) broadcast(chatID string, msg PicoMessage) error { if c.broadcastFn != nil { return c.broadcastFn(chatID, msg) } return c.broadcastToSession(chatID, msg) } // broadcastToSession sends a message to all connections with a matching session. func (c *PicoChannel) broadcastToSession(chatID string, msg PicoMessage) error { // chatID format: "pico:" sessionID := strings.TrimPrefix(chatID, "pico:") msg.SessionID = sessionID var sent bool for _, pc := range c.sessionConnectionsSnapshot(sessionID) { if err := pc.writeJSON(msg); err != nil { logger.DebugCF("pico", "Write to connection failed", map[string]any{ "conn_id": pc.id, "error": err.Error(), }) } else { sent = true } } if !sent { return fmt.Errorf("no active connections for session %s: %w", sessionID, channels.ErrSendFailed) } return nil } // handleWebSocket upgrades the HTTP connection and manages the WebSocket lifecycle. func (c *PicoChannel) handleWebSocket(w http.ResponseWriter, r *http.Request) { if !c.IsRunning() { http.Error(w, "channel not running", http.StatusServiceUnavailable) return } // Authenticate if !c.authenticate(r) { http.Error(w, "unauthorized", http.StatusUnauthorized) return } // Check connection limit maxConns := c.config.MaxConnections if maxConns <= 0 { maxConns = 100 } if c.currentConnCount() >= maxConns { http.Error(w, "too many connections", http.StatusServiceUnavailable) return } // Echo the matched subprotocol back so the browser accepts the upgrade. var responseHeader http.Header if proto := c.matchedSubprotocol(r); proto != "" { responseHeader = http.Header{"Sec-WebSocket-Protocol": {proto}} } conn, err := c.upgrader.Upgrade(w, r, responseHeader) if err != nil { logger.ErrorCF("pico", "WebSocket upgrade failed", map[string]any{ "error": err.Error(), }) return } // Determine session ID from query param or generate one sessionID := r.URL.Query().Get("session_id") if sessionID == "" { sessionID = uuid.New().String() } pc, err := c.createAndAddConnection(conn, sessionID, maxConns) if err != nil { _ = conn.WriteControl( websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseTryAgainLater, "too many connections"), time.Now().Add(2*time.Second), ) _ = conn.Close() return } logger.InfoCF("pico", "WebSocket client connected", map[string]any{ "conn_id": pc.id, "session_id": sessionID, }) go c.readLoop(pc) } // authenticate checks the request for a valid token: // 1. Authorization: Bearer header // 2. Sec-WebSocket-Protocol "token." (for browsers that can't set headers) // 3. Query parameter "token" (only when AllowTokenQuery is on) func (c *PicoChannel) authenticate(r *http.Request) bool { token := c.config.Token.String() if token == "" { return false } // Check Authorization header auth := r.Header.Get("Authorization") if after, ok := strings.CutPrefix(auth, "Bearer "); ok { if after == token { return true } } // Check Sec-WebSocket-Protocol subprotocol ("token.") if c.matchedSubprotocol(r) != "" { return true } // Check query parameter only when explicitly allowed if c.config.AllowTokenQuery { if r.URL.Query().Get("token") == token { return true } } return false } // matchedSubprotocol returns the "token." subprotocol that matches // the configured token, or "" if none do. func (c *PicoChannel) matchedSubprotocol(r *http.Request) string { token := c.config.Token.String() for _, proto := range websocket.Subprotocols(r) { if after, ok := strings.CutPrefix(proto, "token."); ok && after == token { return proto } } return "" } // readLoop reads messages from a WebSocket connection. func (c *PicoChannel) readLoop(pc *picoConn) { defer func() { pc.close() if removed := c.removeConnection(pc.id); removed != nil { logger.InfoCF("pico", "WebSocket client disconnected", map[string]any{ "conn_id": removed.id, "session_id": removed.sessionID, }) } }() readTimeout := time.Duration(c.config.ReadTimeout) * time.Second if readTimeout <= 0 { readTimeout = 60 * time.Second } _ = pc.conn.SetReadDeadline(time.Now().Add(readTimeout)) pc.conn.SetPongHandler(func(appData string) error { _ = pc.conn.SetReadDeadline(time.Now().Add(readTimeout)) return nil }) // Start ping ticker pingInterval := time.Duration(c.config.PingInterval) * time.Second if pingInterval <= 0 { pingInterval = 30 * time.Second } go c.pingLoop(pc, pingInterval) for { select { case <-c.ctx.Done(): return default: } _, rawMsg, err := pc.conn.ReadMessage() if err != nil { if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) { logger.DebugCF("pico", "WebSocket read error", map[string]any{ "conn_id": pc.id, "error": err.Error(), }) } return } _ = pc.conn.SetReadDeadline(time.Now().Add(readTimeout)) var msg PicoMessage if err := json.Unmarshal(rawMsg, &msg); err != nil { errMsg := newError("invalid_message", "failed to parse message") pc.writeJSON(errMsg) continue } c.handleMessage(pc, msg) } } // pingLoop sends periodic ping frames to keep the connection alive. func (c *PicoChannel) pingLoop(pc *picoConn, interval time.Duration) { ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-c.ctx.Done(): return case <-ticker.C: if pc.closed.Load() { return } pc.writeMu.Lock() err := pc.conn.WriteMessage(websocket.PingMessage, nil) pc.writeMu.Unlock() if err != nil { return } } } } // handleMessage processes an inbound Pico Protocol message. func (c *PicoChannel) handleMessage(pc *picoConn, msg PicoMessage) { switch msg.Type { case TypePing: pong := newMessage(TypePong, nil) pong.ID = msg.ID pc.writeJSON(pong) case TypeMessageSend: c.handleMessageSend(pc, msg) case TypeMediaSend: c.handleMessageSend(pc, msg) default: errMsg := newError("unknown_type", fmt.Sprintf("unknown message type: %s", msg.Type)) pc.writeJSON(errMsg) } } // handleMessageSend processes an inbound message.send from a client. func (c *PicoChannel) handleMessageSend(pc *picoConn, msg PicoMessage) { content, _ := msg.Payload["content"].(string) media, err := parseInlineImageMedia(msg.Payload) if err != nil { errMsg := newErrorWithPayload("invalid_media", err.Error(), map[string]any{ "request_id": msg.ID, }) pc.writeJSON(errMsg) return } if strings.TrimSpace(content) == "" && len(media) == 0 { errMsg := newErrorWithPayload("empty_content", "message content is empty", map[string]any{ "request_id": msg.ID, }) pc.writeJSON(errMsg) return } sessionID := msg.SessionID if sessionID == "" { sessionID = pc.sessionID } chatID := "pico:" + sessionID senderID := "pico-user" metadata := map[string]string{ "platform": "pico", "session_id": sessionID, "conn_id": pc.id, } logger.DebugCF("pico", "Received message", map[string]any{ "session_id": sessionID, "preview": truncate(content, 50), "media": len(media), }) sender := bus.SenderInfo{ Platform: "pico", PlatformID: senderID, CanonicalID: identity.BuildCanonicalID("pico", senderID), } if !c.IsAllowedSender(sender) { return } inboundCtx := bus.InboundContext{ Channel: "pico", ChatID: chatID, ChatType: "direct", SenderID: senderID, MessageID: msg.ID, Raw: metadata, } c.HandleInboundContext(c.ctx, chatID, content, media, inboundCtx, sender) } // truncate truncates a string to maxLen runes. func truncate(s string, maxLen int) string { runes := []rune(s) if len(runes) <= maxLen { return s } return string(runes[:maxLen]) + "..." } func parseInlineImageMedia(payload map[string]any) ([]string, error) { if len(payload) == 0 { return nil, nil } media, err := parseInlineImageValues(payload["media"]) if err != nil { return nil, err } attachments, err := parseInlineImageAttachments(payload["attachments"]) if err != nil { return nil, err } media = append(media, attachments...) return media, nil } func parseInlineImageValues(raw any) ([]string, error) { if raw == nil { return nil, nil } switch values := raw.(type) { case []any: media := make([]string, 0, len(values)) for i, item := range values { value, err := inlineImageValue(item) if err != nil { return nil, fmt.Errorf("media[%d]: %w", i, err) } if err := validateInlineImageDataURL(value); err != nil { return nil, fmt.Errorf("media[%d]: %w", i, err) } media = append(media, value) } return media, nil case []string: media := make([]string, 0, len(values)) for i, value := range values { value = strings.TrimSpace(value) if err := validateInlineImageDataURL(value); err != nil { return nil, fmt.Errorf("media[%d]: %w", i, err) } media = append(media, value) } return media, nil case string: value := strings.TrimSpace(values) if err := validateInlineImageDataURL(value); err != nil { return nil, err } return []string{value}, nil default: return nil, fmt.Errorf("media must be a string or array of strings") } } func parseInlineImageAttachments(raw any) ([]string, error) { if raw == nil { return nil, nil } values, ok := raw.([]any) if !ok { return nil, fmt.Errorf("attachments must be an array") } media := make([]string, 0, len(values)) for i, item := range values { attachment, ok := item.(map[string]any) if !ok { return nil, fmt.Errorf("attachments[%d]: attachment must be an object", i) } attachmentType, _ := attachment["type"].(string) attachmentType = strings.ToLower(strings.TrimSpace(attachmentType)) if attachmentType != "" && attachmentType != "image" { continue } value, err := inlineImageValue(attachment) if err != nil { if attachmentType == "image" { return nil, fmt.Errorf("attachments[%d]: %w", i, err) } continue } if !strings.HasPrefix(value, "data:") { continue } if err := validateInlineImageDataURL(value); err != nil { return nil, fmt.Errorf("attachments[%d]: %w", i, err) } media = append(media, value) } return media, nil } func inlineImageValue(item any) (string, error) { switch value := item.(type) { case string: value = strings.TrimSpace(value) if value == "" { return "", fmt.Errorf("image payload is empty") } return value, nil case map[string]any: for _, key := range []string{"url", "data_url"} { if raw, ok := value[key].(string); ok && strings.TrimSpace(raw) != "" { return strings.TrimSpace(raw), nil } } return "", fmt.Errorf("image payload must include url or data_url") default: return "", fmt.Errorf("image payload must be a string or object") } } func validateInlineImageDataURL(mediaURL string) error { if mediaURL == "" { return fmt.Errorf("image payload is empty") } if !strings.HasPrefix(mediaURL, "data:image/") { return fmt.Errorf("only inline image data URLs are supported") } header, data, found := strings.Cut(mediaURL, ",") if !found || strings.TrimSpace(data) == "" { return fmt.Errorf("image data URL is malformed") } if !strings.Contains(header, ";base64") { return fmt.Errorf("image data URL must be base64 encoded") } mimeType, _, _ := strings.Cut(strings.TrimPrefix(header, "data:"), ";") if _, ok := allowedInlineImageMIMETypes[mimeType]; !ok { return fmt.Errorf("unsupported image format: %s", mimeType) } data = strings.TrimSpace(data) if base64.StdEncoding.DecodedLen(len(data)) > config.DefaultMaxMediaSize { return fmt.Errorf("image exceeds %d byte limit", config.DefaultMaxMediaSize) } if _, err := base64.StdEncoding.DecodeString(data); err != nil { return fmt.Errorf("invalid base64 image data") } return nil } // setContextUsagePayload adds context window usage stats to a pico payload. func setContextUsagePayload(payload map[string]any, u *bus.ContextUsage) { if u == nil { return } payload["context_usage"] = map[string]any{ "used_tokens": u.UsedTokens, "total_tokens": u.TotalTokens, "history_tokens": u.HistoryTokens, "compress_at_tokens": u.CompressAtTokens, "summarize_at_tokens": u.SummarizeAtTokens, "used_percent": u.UsedPercent, } } // setTurnUsagePayload attaches real per-turn LLM token usage to the payload. // Input and output are kept separate (billed at different rates); total is a // convenience sum. Omitted entirely when both counts are zero. func setTurnUsagePayload(payload map[string]any, inputTokens, outputTokens int) { if inputTokens <= 0 && outputTokens <= 0 { return } payload[PayloadKeyUsage] = map[string]any{ "input_tokens": inputTokens, "output_tokens": outputTokens, "total_tokens": inputTokens + outputTokens, } } func picoToolCallsPayload(msg bus.OutboundMessage) ([]utils.VisibleToolCall, bool) { raw := strings.TrimSpace(msg.Context.Raw[PayloadKeyToolCalls]) if raw == "" { return nil, false } var toolCalls []utils.VisibleToolCall if err := json.Unmarshal([]byte(raw), &toolCalls); err != nil || len(toolCalls) == 0 { return nil, false } return toolCalls, true } func (c *PicoChannel) editMessage( ctx context.Context, chatID string, messageID string, content string, contextUsage *bus.ContextUsage, ) error { return c.editMessagePayload(ctx, chatID, messageID, map[string]any{ PayloadKeyContent: content, }, contextUsage) } func (c *PicoChannel) editMessagePayload( ctx context.Context, chatID string, messageID string, payload map[string]any, contextUsage *bus.ContextUsage, ) error { if payload == nil { payload = map[string]any{} } normalized := make(map[string]any, len(payload)+1) for key, value := range payload { normalized[key] = value } if _, ok := normalized[PayloadKeyContent]; !ok { normalized[PayloadKeyContent] = "" } normalized["message_id"] = messageID setContextUsagePayload(normalized, contextUsage) outMsg := newMessage(TypeMessageUpdate, normalized) return c.broadcastToSession(chatID, outMsg) }