pch/picoclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #3117 from not-the-author/codex/fix-vision-routing-and-onboard-embed

Browse source 
fix(agent): route media turns to image models
This commit is contained in:
Mauro 2026-06-13 18:03:40 +02:00 committed by GitHub
commit 15196b9cc9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 1222 additions and 146 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cmd/picoclaw/internal/onboard/command.go
View file

@ -1,14 +1,12 @@
package onboard
import (
"embed"
"github.com/spf13/cobra"
picoclaw "github.com/sipeed/picoclaw"
)
//go:generate go run ../../../../scripts/copydir.go ../../../../workspace ./workspace
//go:embed workspace
var embeddedFiles embed.FS
var embeddedFiles = picoclaw.OnboardWorkspace
func NewOnboardCommand() *cobra.Command {
var encrypt bool

12
onboard_workspace_embed.go Normal file
View file

@ -0,0 +1,12 @@
package picoclaw
import "embed"
// OnboardWorkspace embeds the default onboarding workspace template.
//
// Keeping this embed at the module root lets us source files directly from the
// tracked `workspace/` tree, instead of relying on a generated copy inside
// `cmd/...` that may be absent in clean checkouts and CI lint runs.
//
//go:embed workspace
var OnboardWorkspace embed.FS

2
pkg/agent/agent_init.go
View file

@ -296,7 +296,7 @@ func registerSharedTools(
// This keeps subagent vision support working even when the optimized
// sub-turn spawner path is unavailable.
subagentManager.SetMediaResolver(func(msgs []providers.Message) []providers.Message {
return resolveMediaRefs(msgs, al.mediaStore, cfg.Agents.Defaults.GetMaxMediaSize())
return resolveMediaRefs(msgs, al.mediaStore, cfg.Agents.Defaults.GetMaxMediaSize(), 0)
})
// Set the spawner that links into AgentLoop's turnState

45
pkg/agent/agent_media.go
View file

@ -31,6 +31,21 @@ var (
filePlaceholderRegex = regexp.MustCompile(`\[file(:\s+[^\]]*)?\]`)
)
func normalizeCurrentTurnStart(messages []providers.Message, currentTurnStart int) int {
if currentTurnStart < 0 {
return 0
}
if currentTurnStart > len(messages) {
return len(messages)
}
return currentTurnStart
}
func currentTurnMessages(messages []providers.Message, currentTurnStart int) []providers.Message {
currentTurnStart = normalizeCurrentTurnStart(messages, currentTurnStart)
return messages[currentTurnStart:]
}
// resolveMediaRefs resolves media:// refs in messages.
// For user messages: images get path tags only ([image:/path]) so the LLM
// can decide whether to view them via load_image or operate on the file.
@ -38,12 +53,20 @@ var (
// user message only after the contiguous tool-message block ends, so we don't
// break the tool-results-must-immediately-follow-assistant constraint that
// LLM APIs enforce.
// Only tool messages from the current turn may emit the synthetic user
// follow-up; historical tool results stay as plain path-tagged history.
// Non-image files always get path tags regardless of role.
// Returns a new slice; original messages are not mutated.
func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxSize int) []providers.Message {
func resolveMediaRefs(
messages []providers.Message,
store media.MediaStore,
maxSize int,
currentTurnStart int,
) []providers.Message {
if store == nil {
return messages
}
currentTurnStart = normalizeCurrentTurnStart(messages, currentTurnStart)
result := make([]providers.Message, 0, len(messages))
var pendingToolImages []string
@ -52,22 +75,14 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS
// When leaving a tool-message block, flush any accumulated images
// as a synthetic user message.
if m.Role != "tool" && len(pendingToolImages) > 0 {
result = append(result, providers.Message{
Role: "user",
Content: "[Loaded image from tool result above]",
Media: pendingToolImages,
})
result = append(result, toolImageFollowUpPromptMessage(pendingToolImages))
pendingToolImages = nil
}
if len(m.Media) == 0 {
result = append(result, m)
if idx == len(messages)-1 && len(pendingToolImages) > 0 {
result = append(result, providers.Message{
Role: "user",
Content: "[Loaded image from tool result above]",
Media: pendingToolImages,
})
result = append(result, toolImageFollowUpPromptMessage(pendingToolImages))
pendingToolImages = nil
}
continue
@ -104,7 +119,7 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS
mime := detectMIME(localPath, meta)
pathTags = append(pathTags, buildPathTag(mime, localPath))
if m.Role == "tool" && strings.HasPrefix(mime, "image/") {
if m.Role == "tool" && idx >= currentTurnStart && strings.HasPrefix(mime, "image/") {
dataURL := encodeImageToDataURL(localPath, mime, info, maxSize)
if dataURL != "" {
pendingToolImages = append(pendingToolImages, dataURL)
@ -120,11 +135,7 @@ func resolveMediaRefs(messages []providers.Message, store media.MediaStore, maxS
// If this is the last message and we have pending images, flush them.
if idx == len(messages)-1 && len(pendingToolImages) > 0 {
result = append(result, providers.Message{
Role: "user",
Content: "[Loaded image from tool result above]",
Media: pendingToolImages,
})
result = append(result, toolImageFollowUpPromptMessage(pendingToolImages))
pendingToolImages = nil
}
}

946
pkg/agent/agent_test.go
View file

File diff suppressed because it is too large Load diff

12
pkg/agent/instance.go
View file

@ -43,6 +43,7 @@ type AgentInstance struct {
SkillsFilter []string
MCPServerAllowlist map[string]struct{}
Candidates []providers.FallbackCandidate
ImageCandidates []providers.FallbackCandidate
// Router is non-nil when model routing is configured and the light model
// was successfully resolved. It scores each incoming message and decides
@ -198,9 +199,19 @@ func NewAgentInstance(
// Resolve fallback candidates
candidates := resolveModelCandidates(cfg, defaults.Provider, model, fallbacks)
imageCandidates := resolveModelCandidates(
cfg,
defaults.Provider,
defaults.ImageModel,
defaults.ImageModelFallbacks,
)
candidateProviders := make(map[string]providers.LLMProvider)
populateCandidateProvidersFromNames(cfg, workspace, fallbacks, candidateProviders)
if strings.TrimSpace(defaults.ImageModel) != "" {
imageNames := append([]string{defaults.ImageModel}, defaults.ImageModelFallbacks...)
populateCandidateProvidersFromNames(cfg, workspace, imageNames, candidateProviders)
}
// Model routing setup: pre-resolve light model candidates at creation time
// to avoid repeated model_list lookups on every incoming message.
@ -265,6 +276,7 @@ func NewAgentInstance(
SkillsFilter: skillsFilter,
MCPServerAllowlist: agentMCPServerAllowlist,
Candidates: candidates,
ImageCandidates: imageCandidates,
Router: router,
LightCandidates: lightCandidates,
LightProvider: lightProvider,

127
pkg/agent/llm_media.go
View file

@ -1,11 +1,16 @@
package agent
import (
"fmt"
"regexp"
"strings"
"github.com/sipeed/picoclaw/pkg/logger"
"github.com/sipeed/picoclaw/pkg/providers"
)
var resolvedImagePathTagRegex = regexp.MustCompile(`\[image:[^\s\]][^\]]*\]`)
func messagesContainMedia(messages []providers.Message) bool {
for _, msg := range messages {
for _, ref := range msg.Media {
@ -65,3 +70,125 @@ func isVisionUnsupportedError(err error) bool {
return false
}
func visionUnsupportedModelError(modelName string, imageModelConfigured bool) error {
modelName = strings.TrimSpace(modelName)
if imageModelConfigured {
if modelName != "" {
return fmt.Errorf(
"selected vision model %q does not support image input; update agents.defaults.image_model to a multimodal model",
modelName,
)
}
return fmt.Errorf(
"selected vision model does not support image input; update agents.defaults.image_model to a multimodal model",
)
}
if modelName != "" {
return fmt.Errorf(
"active model %q does not support image input; configure agents.defaults.image_model with a multimodal model",
modelName,
)
}
return fmt.Errorf(
"the active model does not support image input; configure agents.defaults.image_model with a multimodal model",
)
}
func sameCandidateSet(a, b []providers.FallbackCandidate) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i].StableKey() != b[i].StableKey() {
return false
}
}
return true
}
func messagesContainCurrentTurnMediaTurn(messages []providers.Message) bool {
for _, msg := range messages {
if len(msg.Media) > 0 {
return true
}
if resolvedImagePathTagRegex.MatchString(msg.Content) {
return true
}
}
return false
}
func (p *Pipeline) routeMediaTurn(ts *turnState, exec *turnExecution) error {
if p == nil || ts == nil || ts.agent == nil || exec == nil ||
!messagesContainCurrentTurnMediaTurn(currentTurnMessages(exec.callMessages, exec.currentTurnStart)) {
return nil
}
var targetCandidates []providers.FallbackCandidate
var targetModelName string
var routeReason string
switch {
case len(ts.agent.ImageCandidates) > 0:
targetCandidates = append([]providers.FallbackCandidate(nil), ts.agent.ImageCandidates...)
targetModelName = strings.TrimSpace(p.Cfg.Agents.Defaults.ImageModel)
routeReason = "configured_image_model"
case exec.usedLight && len(ts.agent.Candidates) > 0:
targetCandidates = append([]providers.FallbackCandidate(nil), ts.agent.Candidates...)
targetModelName = strings.TrimSpace(ts.agent.Model)
routeReason = "bypass_light_model_for_media"
default:
return nil
}
if len(targetCandidates) == 0 {
return nil
}
targetModel := resolvedCandidateModel(targetCandidates, targetModelName)
targetProvider := exec.activeProvider
firstCandidate := targetCandidates[0]
if provider, err := providerForFallbackCandidate(
ts.agent,
ts.agent.Provider,
targetCandidates,
firstCandidate.Provider,
firstCandidate.Model,
); err != nil {
return err
} else if provider != nil {
targetProvider = provider
}
resolvedModelName := resolvedCandidateModelName(targetCandidates, targetModelName)
if sameCandidateSet(exec.activeCandidates, targetCandidates) &&
exec.activeModel == targetModel &&
exec.llmModelName == resolvedModelName {
return nil
}
exec.activeCandidates = targetCandidates
exec.activeModel = targetModel
exec.activeProvider = targetProvider
exec.activeModelConfig = resolveActiveModelConfig(
p.Cfg,
ts.agent.Workspace,
targetCandidates,
targetModel,
p.Cfg.Agents.Defaults.Provider,
)
exec.llmModelName = resolvedModelName
exec.usedLight = false
logger.InfoCF("agent", "Media turn routing selected model", map[string]any{
"agent_id": ts.agent.ID,
"reason": routeReason,
"model": exec.activeModel,
"model_name": exec.llmModelName,
"candidates": len(exec.activeCandidates),
"messages_count": len(exec.callMessages),
})
return nil
}

20
pkg/agent/pipeline_execute.go
View file

@ -295,21 +295,16 @@ toolLoop:
contentForLLM = al.cfg.FilterSensitiveData(contentForLLM)
}
toolResultMsg := providers.Message{
Role: "tool",
Content: contentForLLM,
ToolCallID: tc.ID,
}
var toolResultMedia []string
if len(hookResult.Media) > 0 && !hookResult.ResponseHandled {
hookResult.ArtifactTags = buildArtifactTags(al.mediaStore, hookResult.Media)
contentForLLM = hookResult.ContentForLLM()
if al.cfg.Tools.IsFilterSensitiveDataEnabled() {
contentForLLM = al.cfg.FilterSensitiveData(contentForLLM)
}
toolResultMsg.Content = contentForLLM
toolResultMsg.Media = append(toolResultMsg.Media, hookResult.Media...)
toolResultMedia = append(toolResultMedia, hookResult.Media...)
}
toolResultMsg := toolResultPromptMessage(contentForLLM, tc.ID, toolResultMedia)
al.emitEvent(
runtimeevents.KindAgentToolExecEnd,
@ -695,14 +690,11 @@ toolLoop:
contentForLLM = al.cfg.FilterSensitiveData(contentForLLM)
}
toolResultMsg := providers.Message{
Role: "tool",
Content: contentForLLM,
ToolCallID: toolCallID,
}
var toolResultMedia []string
if len(toolResult.Media) > 0 && !toolResult.ResponseHandled {
toolResultMsg.Media = append(toolResultMsg.Media, toolResult.Media...)
toolResultMedia = append(toolResultMedia, toolResult.Media...)
}
toolResultMsg := toolResultPromptMessage(contentForLLM, toolCallID, toolResultMedia)
al.emitEvent(
runtimeevents.KindAgentToolExecEnd,
ts.eventMeta("runTurn", "turn.tool.end"),

80
pkg/agent/pipeline_llm.go
View file

@ -31,7 +31,7 @@ func (p *Pipeline) CallLLM(
// PreLLM: resolve media refs (except on iteration 1 where user media is already resolved)
if iteration > 1 {
exec.messages = resolveMediaRefs(exec.messages, p.MediaStore, maxMediaSize)
exec.messages = resolveMediaRefs(exec.messages, p.MediaStore, maxMediaSize, exec.currentTurnStart)
}
// PreLLM: graceful terminal handling
@ -64,6 +64,9 @@ func (p *Pipeline) CallLLM(
exec.providerToolDefs = nil
ts.markGracefulTerminalUsed()
}
if err := p.routeMediaTurn(ts, exec); err != nil {
return ControlBreak, err
}
exec.llmOpts = map[string]any{
"max_tokens": ts.agent.MaxTokens,
@ -170,11 +173,10 @@ func (p *Pipeline) CallLLM(
return response, streamErr
}
if len(exec.activeCandidates) > 1 && p.Fallback != nil {
fbResult, fbErr := p.Fallback.ExecuteCandidate(
providerCtx,
exec.activeCandidates,
func(ctx context.Context, candidate providers.FallbackCandidate) (*providers.LLMResponse, error) {
runCandidate := func(
ctx context.Context,
candidate providers.FallbackCandidate,
) (*providers.LLMResponse, error) {
candidateProvider, err := providerForFallbackCandidate(
ts.agent,
exec.activeProvider,
@ -198,8 +200,35 @@ func (p *Pipeline) CallLLM(
applyThinkingOption(callOpts, candidateProvider, candidateThinking, true, ts.agent.ID)
exec.suppressReasoning = shouldSuppressReasoningFor(candidateThinking)
return candidateProvider.Chat(ctx, messagesForCall, toolDefsForCall, candidate.Model, callOpts)
}
if len(exec.activeCandidates) > 1 && p.Fallback != nil {
var (
fbResult *providers.FallbackResult
fbErr error
)
if hasMediaRefs(messagesForCall) {
fbResult, fbErr = p.Fallback.ExecuteImage(
providerCtx,
exec.activeCandidates,
func(ctx context.Context, provider, model string) (*providers.LLMResponse, error) {
candidate := providers.FallbackCandidate{Provider: provider, Model: model}
for _, configured := range exec.activeCandidates {
if configured.Provider == provider && configured.Model == model {
candidate = configured
break
}
}
return runCandidate(ctx, candidate)
},
)
} else {
fbResult, fbErr = p.Fallback.ExecuteCandidate(
providerCtx,
exec.activeCandidates,
runCandidate,
)
}
if fbErr != nil {
return nil, fbErr
}
@ -250,33 +279,11 @@ func (p *Pipeline) CallLLM(
break
}
// Retry without media if vision is unsupported
if hasMediaRefs(exec.callMessages) && isVisionUnsupportedError(err) && retry < maxRetries {
al.emitEvent(
runtimeevents.KindAgentLLMRetry,
ts.eventMeta("runTurn", "turn.llm.retry"),
LLMRetryPayload{
Attempt: retry + 1,
MaxRetries: maxRetries,
Reason: "vision_unsupported",
Error: err.Error(),
Backoff: 0,
},
if hasMediaRefs(exec.callMessages) && isVisionUnsupportedError(err) {
return ControlBreak, visionUnsupportedModelError(
exec.llmModelName,
len(ts.agent.ImageCandidates) > 0,
)
logger.WarnCF("agent", "Vision unsupported, retrying without media", map[string]any{
"error": err.Error(),
"retry": retry,
})
exec.callMessages = stripMessageMedia(exec.callMessages)
if !ts.opts.NoHistory {
exec.history = stripMessageMedia(exec.history)
ts.agent.Sessions.SetHistory(ts.sessionKey, exec.history)
for i := range ts.persistedMessages {
ts.persistedMessages[i].Media = nil
}
ts.refreshRestorePointFromSession(ts.agent)
}
continue
}
errMsg := strings.ToLower(err.Error())
@ -381,7 +388,13 @@ func (p *Pipeline) CallLLM(
fullHistory := append(append([]providers.Message(nil), trimmedHistory...), protectedTurnTail...)
rebuildPromptReq := promptBuildRequestForTurn(ts, fullHistory, exec.summary, "", nil, p.Cfg)
rebuildPromptReq.ActiveSkills = append([]string(nil), contextualSkills...)
return ts.agent.ContextBuilder.BuildMessagesFromPrompt(rebuildPromptReq)
rebuilt := ts.agent.ContextBuilder.BuildMessagesFromPrompt(rebuildPromptReq)
return resolveMediaRefs(
rebuilt,
p.MediaStore,
maxMediaSize,
len(rebuilt)-len(protectedTurnTail),
)
}
originalHistoryCount := len(exec.history)
var fit bool
@ -401,6 +414,7 @@ func (p *Pipeline) CallLLM(
)
exec.history = append(trimmedStableHistory, protectedTurnTail...)
exec.messages = buildMessages(trimmedStableHistory)
exec.currentTurnStart = len(exec.messages) - len(protectedTurnTail)
if exec.gracefulTerminal {
msgs := append([]providers.Message(nil), exec.messages...)
exec.callMessages = append(msgs, ts.interruptHintMessage())

13
pkg/agent/pipeline_setup.go
View file

@ -39,8 +39,12 @@ func (p *Pipeline) SetupTurn(ctx context.Context, ts *turnState) (*turnExecution
initialPromptReq := promptBuildRequestForTurn(ts, history, summary, ts.userMessage, ts.media, cfg)
initialPromptReq.ActiveSkills = append([]string(nil), contextualSkills...)
messages := ts.agent.ContextBuilder.BuildMessagesFromPrompt(initialPromptReq)
currentTurnStart := len(messages)
if strings.TrimSpace(ts.userMessage) != "" || len(ts.media) > 0 {
currentTurnStart = len(messages) - 1
}
messages = resolveMediaRefs(messages, p.MediaStore, maxMediaSize)
messages = resolveMediaRefs(messages, p.MediaStore, maxMediaSize, currentTurnStart)
if !ts.opts.NoHistory {
toolDefs := filterToolsByTurnProfile(ts.agent.Tools.ToProviderDefs(), ts.profile)
@ -81,7 +85,11 @@ func (p *Pipeline) SetupTurn(ctx context.Context, ts *turnState) (*turnExecution
)
rebuildPromptReq.ActiveSkills = append([]string(nil), contextualSkills...)
rebuilt := ts.agent.ContextBuilder.BuildMessagesFromPrompt(rebuildPromptReq)
return resolveMediaRefs(rebuilt, p.MediaStore, maxMediaSize)
rebuiltCurrentTurnStart := len(rebuilt)
if strings.TrimSpace(ts.userMessage) != "" || len(ts.media) > 0 {
rebuiltCurrentTurnStart = len(rebuilt) - 1
}
return resolveMediaRefs(rebuilt, p.MediaStore, maxMediaSize, rebuiltCurrentTurnStart)
},
ts.agent.ContextWindow,
toolDefs,
@ -137,6 +145,7 @@ func (p *Pipeline) SetupTurn(ctx context.Context, ts *turnState) (*turnExecution
summary,
messages,
)
exec.currentTurnStart = currentTurnStart
exec.activeCandidates = activeCandidates
exec.activeModel = activeModel
exec.activeModelConfig = resolveActiveModelConfig(

2
pkg/agent/prompt.go
View file

@ -37,6 +37,7 @@ const (
PromptSlotMessage PromptSlot = "message"
PromptSlotSteering PromptSlot = "steering"
PromptSlotSubTurn PromptSlot = "subturn"
PromptSlotToolResult PromptSlot = "tool_result"
PromptSlotInterrupt PromptSlot = "interrupt"
PromptSlotOutput PromptSlot = "output"
)
@ -60,6 +61,7 @@ const (
PromptSourceUserMessage PromptSourceID = "turn:user_message"
PromptSourceSteering PromptSourceID = "turn:steering"
PromptSourceSubTurnResult PromptSourceID = "turn:subturn_result"
PromptSourceToolResult PromptSourceID = "turn:tool_result"
PromptSourceInterrupt PromptSourceID = "turn:interrupt"
)

23
pkg/agent/prompt_turn.go
View file

@ -194,6 +194,29 @@ func userPromptMessage(content string, media []string) providers.Message {
return promptMessageWithMetadata(msg, PromptLayerTurn, PromptSlotMessage, PromptSourceUserMessage)
}
func toolResultPromptMessage(content, toolCallID string, media []string) providers.Message {
msg := providers.Message{
Role: "tool",
Content: content,
ToolCallID: toolCallID,
}
if len(media) > 0 {
msg.Media = append([]string(nil), media...)
}
return promptMessageWithMetadata(msg, PromptLayerTurn, PromptSlotToolResult, PromptSourceToolResult)
}
func toolImageFollowUpPromptMessage(media []string) providers.Message {
msg := providers.Message{
Role: "user",
Content: "[Loaded image from tool result above]",
}
if len(media) > 0 {
msg.Media = append([]string(nil), media...)
}
return promptMessageWithMetadata(msg, PromptLayerTurn, PromptSlotToolResult, PromptSourceToolResult)
}
func steeringPromptMessage(msg providers.Message) providers.Message {
return promptMessageWithDefaultMetadata(msg, PromptLayerTurn, PromptSlotSteering, PromptSourceSteering)
}

8
pkg/agent/turn_coord.go
View file

@ -150,7 +150,7 @@ func (al *AgentLoop) runTurn(ctx context.Context, ts *turnState, pipeline *Pipel
// Inject pending steering messages
if len(pendingMessages) > 0 {
resolvedPending := resolveMediaRefs(pendingMessages, al.mediaStore, maxMediaSize)
resolvedPending := resolveMediaRefs(pendingMessages, al.mediaStore, maxMediaSize, 0)
totalContentLen := 0
for i, pm := range pendingMessages {
messages = append(messages, resolvedPending[i])
@ -431,7 +431,11 @@ func (al *AgentLoop) askSideQuestion(
messages := agent.ContextBuilder.BuildMessagesFromPrompt(promptReq)
maxMediaSize := al.GetConfig().Agents.Defaults.GetMaxMediaSize()
messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize)
currentTurnStart := len(messages)
if strings.TrimSpace(question) != "" || len(media) > 0 {
currentTurnStart = len(messages) - 1
}
messages = resolveMediaRefs(messages, al.mediaStore, maxMediaSize, currentTurnStart)
activeCandidates, activeModel, usedLight := al.selectCandidates(agent, question, messages)
selectedModelName := sideQuestionModelName(agent, usedLight)

2
pkg/agent/turn_state.go
View file

@ -118,6 +118,7 @@ type turnExecution struct {
pendingMessages []providers.Message // steering/SubTurn messages awaiting injection
history []providers.Message // from ContextManager.Assemble
summary string
currentTurnStart int
// Turn output
finalContent string
@ -168,6 +169,7 @@ func newTurnExecution(
summary: summary,
messages: messages,
pendingMessages: append([]providers.Message(nil), opts.InitialSteeringMessages...),
currentTurnStart: len(messages),
iteration: 0,
phase: LLMPhaseSetup,
}