diff --git a/pkg/channels/onebot/onebot.go b/pkg/channels/onebot/onebot.go index 76320b6d..392c34b7 100644 --- a/pkg/channels/onebot/onebot.go +++ b/pkg/channels/onebot/onebot.go @@ -24,6 +24,7 @@ import ( type OneBotChannel struct { *channels.BaseChannel config *config.OneBotSettings + downloadFn func(urlStr, filename string) string conn *websocket.Conn ctx context.Context cancel context.CancelFunc @@ -795,9 +796,7 @@ func (c *OneBotChannel) parseMessageSegments( } else if n, ok := data["name"].(string); ok && n != "" { filename = n } - localPath := utils.DownloadFile(url, filename, utils.DownloadOptions{ - LoggerPrefix: "onebot", - }) + localPath := c.downloadInboundFile(url, filename) if localPath != "" { mediaRefs = append(mediaRefs, storeFile(localPath, filename)) textParts = append(textParts, fmt.Sprintf("[%s]", segType)) @@ -809,9 +808,7 @@ func (c *OneBotChannel) parseMessageSegments( if data != nil { url, _ := data["url"].(string) if url != "" { - localPath := utils.DownloadFile(url, "voice.amr", utils.DownloadOptions{ - LoggerPrefix: "onebot", - }) + localPath := c.downloadInboundFile(url, "voice.amr") if localPath != "" { textParts = append(textParts, "[voice]") mediaRefs = append(mediaRefs, storeFile(localPath, "voice.amr")) @@ -847,6 +844,16 @@ func (c *OneBotChannel) parseMessageSegments( } } +func (c *OneBotChannel) downloadInboundFile(urlStr, filename string) string { + if c.downloadFn != nil { + return c.downloadFn(urlStr, filename) + } + return utils.DownloadFile(urlStr, filename, utils.DownloadOptions{ + LoggerPrefix: "onebot", + BlockPrivateTargets: true, + }) +} + func (c *OneBotChannel) handleRawEvent(raw *oneBotRawEvent) { switch raw.PostType { case "message": diff --git a/pkg/channels/onebot/onebot_test.go b/pkg/channels/onebot/onebot_test.go new file mode 100644 index 00000000..2b57879a --- /dev/null +++ b/pkg/channels/onebot/onebot_test.go @@ -0,0 +1,126 @@ +package onebot + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sipeed/picoclaw/pkg/media" +) + +func TestParseMessageSegments_BlocksLoopbackInboundMediaURL(t *testing.T) { + ch := &OneBotChannel{} + store := media.NewFileMediaStore() + + raw := json.RawMessage(`[ + {"type":"text","data":{"text":"see attachment"}}, + {"type":"image","data":{"url":"http://127.0.0.1:8080/evil.png","file":"evil.png"}} + ]`) + + result := ch.parseMessageSegments(raw, 0, store, "onebot:test:msg1") + + if got := result.Text; got != "see attachment" { + t.Fatalf("Text = %q, want %q", got, "see attachment") + } + if len(result.Media) != 0 { + t.Fatalf("Media count = %d, want 0", len(result.Media)) + } +} + +func TestParseMessageSegments_BlocksInboundMediaRedirectToLoopback(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("secret")) + })) + defer target.Close() + + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got, want := r.URL.String(), "http://example.com/evil.png"; got != want { + t.Fatalf("proxy request URL = %q, want %q", got, want) + } + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer proxy.Close() + + t.Setenv("HTTP_PROXY", proxy.URL) + t.Setenv("http_proxy", proxy.URL) + t.Setenv("HTTPS_PROXY", "") + t.Setenv("https_proxy", "") + t.Setenv("ALL_PROXY", "") + t.Setenv("all_proxy", "") + t.Setenv("NO_PROXY", "") + t.Setenv("no_proxy", "") + + ch := &OneBotChannel{} + store := media.NewFileMediaStore() + + raw := json.RawMessage(`[ + {"type":"text","data":{"text":"see attachment"}}, + {"type":"image","data":{"url":"http://example.com/evil.png","file":"evil.png"}} + ]`) + + result := ch.parseMessageSegments(raw, 0, store, "onebot:test:msg-redirect") + + if got := result.Text; got != "see attachment" { + t.Fatalf("Text = %q, want %q", got, "see attachment") + } + if len(result.Media) != 0 { + t.Fatalf("Media count = %d, want 0", len(result.Media)) + } +} + +func TestParseMessageSegments_StoresDownloadedMediaRef(t *testing.T) { + tmpDir := t.TempDir() + localPath := filepath.Join(tmpDir, "image.png") + if err := os.WriteFile(localPath, []byte("fake-image"), 0o600); err != nil { + t.Fatalf("os.WriteFile() error = %v", err) + } + + ch := &OneBotChannel{ + downloadFn: func(urlStr, filename string) string { + if urlStr != "https://cdn.example.com/image.png" { + t.Fatalf("download url = %q, want %q", urlStr, "https://cdn.example.com/image.png") + } + if filename != "image.png" { + t.Fatalf("download filename = %q, want %q", filename, "image.png") + } + return localPath + }, + } + store := media.NewFileMediaStore() + + raw := json.RawMessage(`[ + {"type":"text","data":{"text":"see attachment"}}, + {"type":"image","data":{"url":"https://cdn.example.com/image.png","file":"image.png"}} + ]`) + + result := ch.parseMessageSegments(raw, 0, store, "onebot:test:msg2") + + if got := result.Text; got != "see attachment[image]" { + t.Fatalf("Text = %q, want %q", got, "see attachment[image]") + } + if len(result.Media) != 1 { + t.Fatalf("Media count = %d, want 1", len(result.Media)) + } + if !strings.HasPrefix(result.Media[0], "media://") { + t.Fatalf("media ref = %q, want media:// prefix", result.Media[0]) + } + + resolvedPath, meta, err := store.ResolveWithMeta(result.Media[0]) + if err != nil { + t.Fatalf("ResolveWithMeta() error = %v", err) + } + if resolvedPath != localPath { + t.Fatalf("resolved path = %q, want %q", resolvedPath, localPath) + } + if meta.Source != "onebot" { + t.Fatalf("meta.Source = %q, want %q", meta.Source, "onebot") + } + if meta.Filename != "image.png" { + t.Fatalf("meta.Filename = %q, want %q", meta.Filename, "image.png") + } +} diff --git a/pkg/channels/telegram/telegram_test.go b/pkg/channels/telegram/telegram_test.go index ddecad28..19b181cf 100644 --- a/pkg/channels/telegram/telegram_test.go +++ b/pkg/channels/telegram/telegram_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "strconv" @@ -213,6 +215,33 @@ func TestSendMedia_ImageFallbacksToDocumentOnInvalidDimensions(t *testing.T) { assert.Equal(t, "caption", constructor.calls[1].Parameters["caption"]) } +func TestDownloadFileWithInfo_AllowsLocalConfiguredBaseURL(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got, want := r.URL.Path, "/file/bot"+testToken+"/photos/image"; got != want { + t.Fatalf("request path = %q, want %q", got, want) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("telegram-local-bot-api")) + })) + defer server.Close() + + ch, err := NewTelegramChannel( + &config.Channel{Type: config.ChannelTelegram, Enabled: true}, + &config.TelegramSettings{ + Token: *config.NewSecureString(testToken), + BaseURL: server.URL, + }, + nil, + ) + require.NoError(t, err) + + path := ch.downloadFileWithInfo(&telego.File{FilePath: "photos/image"}, "") + if path == "" { + t.Fatal("expected local base_url download to succeed") + } + defer os.Remove(path) +} + func TestSendMedia_ImageNonDimensionErrorDoesNotFallback(t *testing.T) { constructor := &multipartRecordingConstructor{} caller := &stubCaller{ diff --git a/pkg/tools/integration/web.go b/pkg/tools/integration/web.go index 4e1bb0fe..f013c89b 100644 --- a/pkg/tools/integration/web.go +++ b/pkg/tools/integration/web.go @@ -1996,16 +1996,9 @@ type WebFetchTool struct { client *http.Client format string fetchLimitBytes int64 - whitelist *privateHostWhitelist + whitelist *utils.PrivateHostWhitelist } -type privateHostWhitelist struct { - exact map[string]struct{} - cidrs []*net.IPNet -} - -type webFetchAllowedFirstHopHostKey struct{} - func NewWebFetchTool(maxChars int, format string, fetchLimitBytes int64) (*WebFetchTool, error) { // createHTTPClient cannot fail with an empty proxy string. return NewWebFetchToolWithConfig(maxChars, "", format, fetchLimitBytes, nil) @@ -2035,31 +2028,22 @@ func NewWebFetchToolWithConfig( if maxChars <= 0 { maxChars = defaultMaxChars } - whitelist, err := newPrivateHostWhitelist(privateHostWhitelist) + whitelist, err := utils.NewPrivateHostWhitelist(privateHostWhitelist) if err != nil { return nil, fmt.Errorf("failed to parse web fetch private host whitelist: %w", err) } - client, err := utils.CreateHTTPClient(proxy, fetchTimeout) + client, err := utils.CreateSafeHTTPClient(utils.SafeHTTPClientOptions{ + ProxyURL: proxy, + Timeout: fetchTimeout, + PrivateHostWhitelist: privateHostWhitelist, + AllowPrivateHosts: func() bool { + return allowPrivateWebFetchHosts.Load() + }, + MaxRedirects: maxRedirects, + }) if err != nil { return nil, fmt.Errorf("failed to create HTTP client for web fetch: %w", err) } - if transport, ok := client.Transport.(*http.Transport); ok { - dialer := &net.Dialer{ - Timeout: 15 * time.Second, - KeepAlive: 30 * time.Second, - } - transport.DialContext = newSafeDialContext(dialer, whitelist) - } - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - if len(via) >= maxRedirects { - return fmt.Errorf("stopped after %d redirects", maxRedirects) - } - if isObviousPrivateHost(req.URL.Hostname(), whitelist) { - return fmt.Errorf("redirect target is private or local network host") - } - allowConfiguredProxyFirstHop(req, client.Transport) - return nil - } if fetchLimitBytes <= 0 { fetchLimitBytes = 10 * 1024 * 1024 // Security Fallback } @@ -2121,7 +2105,9 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe // Lightweight pre-flight: block obvious localhost/literal-IP without DNS resolution. // The real SSRF guard is newSafeDialContext at connect time. hostname := parsedURL.Hostname() - if isObviousPrivateHost(hostname, t.whitelist) { + if utils.IsObviousPrivateHost(hostname, t.whitelist, func() bool { + return allowPrivateWebFetchHosts.Load() + }) { return ErrorResult("fetching private or local network hosts is not allowed") } @@ -2137,7 +2123,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe if reqErr != nil { return nil, nil, fmt.Errorf("failed to create request: %w", reqErr) } - allowConfiguredProxyFirstHop(req, t.client.Transport) + utils.AllowConfiguredProxyFirstHop(req, t.client.Transport) req.Header.Set("User-Agent", ua) resp, doErr := t.client.Do(req) if doErr != nil { @@ -2325,247 +2311,29 @@ func (t *WebFetchTool) extractText(htmlContent string) string { return strings.Join(cleanLines, "\n") } -// newSafeDialContext re-resolves DNS at connect time to mitigate DNS rebinding (TOCTOU) -// where a hostname resolves to a public IP during pre-flight but a private IP at connect time. func newSafeDialContext( dialer *net.Dialer, - whitelist *privateHostWhitelist, + whitelist *utils.PrivateHostWhitelist, ) func(context.Context, string, string) (net.Conn, error) { - return func(ctx context.Context, network, address string) (net.Conn, error) { - if allowPrivateWebFetchHosts.Load() { - return dialer.DialContext(ctx, network, address) - } + return utils.NewSafeDialContext(dialer, whitelist, func() bool { + return allowPrivateWebFetchHosts.Load() + }) +} - host, port, err := net.SplitHostPort(address) - if err != nil { - return nil, fmt.Errorf("invalid target address %q: %w", address, err) - } - if host == "" { - return nil, fmt.Errorf("empty target host") - } - if isAllowedFirstHopHost(ctx, host) { - return dialer.DialContext(ctx, network, address) - } +func newPrivateHostWhitelist(entries []string) (*utils.PrivateHostWhitelist, error) { + return utils.NewPrivateHostWhitelist(entries) +} - if ip := net.ParseIP(host); ip != nil { - if shouldBlockPrivateIP(ip, whitelist) { - return nil, fmt.Errorf("blocked private or local target: %s", host) - } - return dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port)) - } +func isObviousPrivateHost(host string, whitelist *utils.PrivateHostWhitelist) bool { + return utils.IsObviousPrivateHost(host, whitelist, func() bool { + return allowPrivateWebFetchHosts.Load() + }) +} - ipAddrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) - if err != nil { - return nil, fmt.Errorf("failed to resolve %s: %w", host, err) - } - - attempted := 0 - var lastErr error - for _, ipAddr := range ipAddrs { - if shouldBlockPrivateIP(ipAddr.IP, whitelist) { - continue - } - attempted++ - conn, err := dialer.DialContext( - ctx, - network, - net.JoinHostPort(ipAddr.IP.String(), port), - ) - if err == nil { - return conn, nil - } - lastErr = err - } - - if attempted == 0 { - return nil, fmt.Errorf( - "all resolved addresses for %s are private, restricted, or not whitelisted", - host, - ) - } - if lastErr != nil { - return nil, fmt.Errorf( - "failed connecting to public addresses for %s: %w", - host, - lastErr, - ) - } - return nil, fmt.Errorf("failed connecting to public addresses for %s", host) - } +func isPrivateOrRestrictedIP(ip net.IP) bool { + return utils.IsPrivateOrRestrictedIP(ip) } func allowConfiguredProxyFirstHop(req *http.Request, rt http.RoundTripper) { - if req == nil { - return - } - - transport, ok := rt.(*http.Transport) - if !ok || transport.Proxy == nil { - return - } - - proxyURL, err := transport.Proxy(req) - if err != nil || proxyURL == nil { - return - } - - host := normalizeAllowedFirstHopHost(proxyURL.Hostname()) - if host == "" { - return - } - - *req = *req.WithContext(context.WithValue( - req.Context(), - webFetchAllowedFirstHopHostKey{}, - host, - )) -} - -func isAllowedFirstHopHost(ctx context.Context, host string) bool { - allowed, ok := ctx.Value(webFetchAllowedFirstHopHostKey{}).(string) - if !ok || allowed == "" { - return false - } - return allowed == normalizeAllowedFirstHopHost(host) -} - -func normalizeAllowedFirstHopHost(host string) string { - host = strings.ToLower(strings.TrimSpace(host)) - return strings.TrimSuffix(host, ".") -} - -func newPrivateHostWhitelist(entries []string) (*privateHostWhitelist, error) { - if len(entries) == 0 { - return nil, nil - } - - whitelist := &privateHostWhitelist{ - exact: make(map[string]struct{}), - cidrs: make([]*net.IPNet, 0, len(entries)), - } - for _, entry := range entries { - entry = strings.TrimSpace(entry) - if entry == "" { - continue - } - if ip := net.ParseIP(entry); ip != nil { - whitelist.exact[normalizeWhitelistIP(ip).String()] = struct{}{} - continue - } - _, network, err := net.ParseCIDR(entry) - if err != nil { - return nil, fmt.Errorf("invalid entry %q: expected IP or CIDR", entry) - } - whitelist.cidrs = append(whitelist.cidrs, network) - } - - if len(whitelist.exact) == 0 && len(whitelist.cidrs) == 0 { - return nil, nil - } - return whitelist, nil -} - -func (w *privateHostWhitelist) Contains(ip net.IP) bool { - if w == nil || ip == nil { - return false - } - - normalized := normalizeWhitelistIP(ip) - if _, ok := w.exact[normalized.String()]; ok { - return true - } - for _, network := range w.cidrs { - if network.Contains(normalized) { - return true - } - } - return false -} - -func normalizeWhitelistIP(ip net.IP) net.IP { - if ip == nil { - return nil - } - if ip4 := ip.To4(); ip4 != nil { - return ip4 - } - return ip -} - -func shouldBlockPrivateIP(ip net.IP, whitelist *privateHostWhitelist) bool { - return isPrivateOrRestrictedIP(ip) && !whitelist.Contains(ip) -} - -// isObviousPrivateHost performs a lightweight, no-DNS check for obviously private hosts. -// It catches localhost, literal private IPs, and empty hosts. It does NOT resolve DNS — -// the real SSRF guard is newSafeDialContext which checks IPs at connect time. -func isObviousPrivateHost(host string, whitelist *privateHostWhitelist) bool { - if allowPrivateWebFetchHosts.Load() { - return false - } - - h := strings.ToLower(strings.TrimSpace(host)) - h = strings.TrimSuffix(h, ".") - if h == "" { - return true - } - - if h == "localhost" || strings.HasSuffix(h, ".localhost") { - return true - } - - if ip := net.ParseIP(h); ip != nil { - return shouldBlockPrivateIP(ip, whitelist) - } - - return false -} - -// isPrivateOrRestrictedIP returns true for IPs that should never be reached via web_fetch: -// RFC 1918, loopback, link-local (incl. cloud metadata 169.254.x.x), carrier-grade NAT, -// benchmark (198.18.0.0/15), IPv6 unique-local (fc00::/7), 6to4 (2002::/16), and -// Teredo (2001:0000::/32). -func isPrivateOrRestrictedIP(ip net.IP) bool { - if ip == nil { - return true - } - - if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || - ip.IsMulticast() || ip.IsUnspecified() { - return true - } - - if ip4 := ip.To4(); ip4 != nil { - // IPv4 private, loopback, link-local, and carrier-grade NAT ranges. - if ip4[0] == 10 || - ip4[0] == 127 || - ip4[0] == 0 || - (ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31) || - (ip4[0] == 192 && ip4[1] == 168) || - (ip4[0] == 169 && ip4[1] == 254) || - (ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127) || - (ip4[0] == 198 && ip4[1] >= 18 && ip4[1] <= 19) { - return true - } - return false - } - - if len(ip) == net.IPv6len { - // IPv6 unique local addresses (fc00::/7) - if (ip[0] & 0xfe) == 0xfc { - return true - } - // 6to4 addresses (2002::/16): check the embedded IPv4 at bytes [2:6]. - if ip[0] == 0x20 && ip[1] == 0x02 { - embedded := net.IPv4(ip[2], ip[3], ip[4], ip[5]) - return isPrivateOrRestrictedIP(embedded) - } - // Teredo (2001:0000::/32): client IPv4 is at bytes [12:16], XOR-inverted. - if ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x00 && ip[3] == 0x00 { - client := net.IPv4(ip[12]^0xff, ip[13]^0xff, ip[14]^0xff, ip[15]^0xff) - return isPrivateOrRestrictedIP(client) - } - } - - return false + utils.AllowConfiguredProxyFirstHop(req, rt) } diff --git a/pkg/utils/http_guard.go b/pkg/utils/http_guard.go new file mode 100644 index 00000000..bf80e811 --- /dev/null +++ b/pkg/utils/http_guard.go @@ -0,0 +1,318 @@ +package utils + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +type SafeHTTPClientOptions struct { + ProxyURL string + Timeout time.Duration + PrivateHostWhitelist []string + AllowPrivateHosts func() bool + MaxRedirects int +} + +type PrivateHostWhitelist struct { + exact map[string]struct{} + cidrs []*net.IPNet +} + +type allowedFirstHopHostKey struct{} + +func NewPrivateHostWhitelist(entries []string) (*PrivateHostWhitelist, error) { + if len(entries) == 0 { + return nil, nil + } + + whitelist := &PrivateHostWhitelist{ + exact: make(map[string]struct{}), + cidrs: make([]*net.IPNet, 0, len(entries)), + } + for _, entry := range entries { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + if ip := net.ParseIP(entry); ip != nil { + whitelist.exact[normalizeWhitelistIP(ip).String()] = struct{}{} + continue + } + _, network, err := net.ParseCIDR(entry) + if err != nil { + return nil, fmt.Errorf("invalid entry %q: expected IP or CIDR", entry) + } + whitelist.cidrs = append(whitelist.cidrs, network) + } + + if len(whitelist.exact) == 0 && len(whitelist.cidrs) == 0 { + return nil, nil + } + return whitelist, nil +} + +func (w *PrivateHostWhitelist) Contains(ip net.IP) bool { + if w == nil || ip == nil { + return false + } + + normalized := normalizeWhitelistIP(ip) + if _, ok := w.exact[normalized.String()]; ok { + return true + } + for _, network := range w.cidrs { + if network.Contains(normalized) { + return true + } + } + return false +} + +func CreateSafeHTTPClient(opts SafeHTTPClientOptions) (*http.Client, error) { + client, err := CreateHTTPClient(opts.ProxyURL, opts.Timeout) + if err != nil { + return nil, err + } + + whitelist, err := NewPrivateHostWhitelist(opts.PrivateHostWhitelist) + if err != nil { + return nil, err + } + + transport, ok := client.Transport.(*http.Transport) + if ok { + dialer := &net.Dialer{ + Timeout: 15 * time.Second, + KeepAlive: 30 * time.Second, + } + transport.DialContext = NewSafeDialContext(dialer, whitelist, opts.AllowPrivateHosts) + } + + maxRedirects := opts.MaxRedirects + if maxRedirects <= 0 { + maxRedirects = 10 + } + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= maxRedirects { + return fmt.Errorf("stopped after %d redirects", maxRedirects) + } + if IsObviousPrivateHost(req.URL.Hostname(), whitelist, opts.AllowPrivateHosts) { + return fmt.Errorf("redirect target is private or local network host") + } + AllowConfiguredProxyFirstHop(req, client.Transport) + return nil + } + + return client, nil +} + +func ValidateSafeHTTPURL(urlStr string, whitelist *PrivateHostWhitelist, allowPrivateHosts func() bool) error { + parsedURL, err := url.Parse(urlStr) + if err != nil { + return fmt.Errorf("invalid URL: %w", err) + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return fmt.Errorf("only http/https URLs are allowed") + } + if parsedURL.Host == "" { + return fmt.Errorf("missing domain in URL") + } + if IsObviousPrivateHost(parsedURL.Hostname(), whitelist, allowPrivateHosts) { + return fmt.Errorf("fetching private or local network hosts is not allowed") + } + return nil +} + +func NewSafeDialContext( + dialer *net.Dialer, + whitelist *PrivateHostWhitelist, + allowPrivateHosts func() bool, +) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + if allowPrivateHosts != nil && allowPrivateHosts() { + return dialer.DialContext(ctx, network, address) + } + + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, fmt.Errorf("invalid target address %q: %w", address, err) + } + if host == "" { + return nil, fmt.Errorf("empty target host") + } + if isAllowedFirstHopHost(ctx, host) { + return dialer.DialContext(ctx, network, address) + } + + if ip := net.ParseIP(host); ip != nil { + if shouldBlockPrivateIP(ip, whitelist) { + return nil, fmt.Errorf("blocked private or local target: %s", host) + } + return dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port)) + } + + ipAddrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) + if err != nil { + return nil, fmt.Errorf("failed to resolve %s: %w", host, err) + } + + attempted := 0 + var lastErr error + for _, ipAddr := range ipAddrs { + if shouldBlockPrivateIP(ipAddr.IP, whitelist) { + continue + } + attempted++ + conn, err := dialer.DialContext( + ctx, + network, + net.JoinHostPort(ipAddr.IP.String(), port), + ) + if err == nil { + return conn, nil + } + lastErr = err + } + + if attempted == 0 { + return nil, fmt.Errorf( + "all resolved addresses for %s are private, restricted, or not whitelisted", + host, + ) + } + if lastErr != nil { + return nil, fmt.Errorf( + "failed connecting to public addresses for %s: %w", + host, + lastErr, + ) + } + return nil, fmt.Errorf("failed connecting to public addresses for %s", host) + } +} + +func AllowConfiguredProxyFirstHop(req *http.Request, rt http.RoundTripper) { + if req == nil { + return + } + + transport, ok := rt.(*http.Transport) + if !ok || transport.Proxy == nil { + return + } + + proxyURL, err := transport.Proxy(req) + if err != nil || proxyURL == nil { + return + } + + host := normalizeAllowedFirstHopHost(proxyURL.Hostname()) + if host == "" { + return + } + + *req = *req.WithContext(context.WithValue( + req.Context(), + allowedFirstHopHostKey{}, + host, + )) +} + +func IsObviousPrivateHost( + host string, + whitelist *PrivateHostWhitelist, + allowPrivateHosts func() bool, +) bool { + if allowPrivateHosts != nil && allowPrivateHosts() { + return false + } + + h := strings.ToLower(strings.TrimSpace(host)) + h = strings.TrimSuffix(h, ".") + if h == "" { + return true + } + + if h == "localhost" || strings.HasSuffix(h, ".localhost") { + return true + } + + if ip := net.ParseIP(h); ip != nil { + return shouldBlockPrivateIP(ip, whitelist) + } + + return false +} + +func IsPrivateOrRestrictedIP(ip net.IP) bool { + if ip == nil { + return true + } + + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || + ip.IsMulticast() || ip.IsUnspecified() { + return true + } + + if ip4 := ip.To4(); ip4 != nil { + if ip4[0] == 10 || + ip4[0] == 127 || + ip4[0] == 0 || + (ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31) || + (ip4[0] == 192 && ip4[1] == 168) || + (ip4[0] == 169 && ip4[1] == 254) || + (ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127) || + (ip4[0] == 198 && ip4[1] >= 18 && ip4[1] <= 19) { + return true + } + return false + } + + if len(ip) == net.IPv6len { + if (ip[0] & 0xfe) == 0xfc { + return true + } + if ip[0] == 0x20 && ip[1] == 0x02 { + embedded := net.IPv4(ip[2], ip[3], ip[4], ip[5]) + return IsPrivateOrRestrictedIP(embedded) + } + if ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x00 && ip[3] == 0x00 { + client := net.IPv4(ip[12]^0xff, ip[13]^0xff, ip[14]^0xff, ip[15]^0xff) + return IsPrivateOrRestrictedIP(client) + } + } + + return false +} + +func isAllowedFirstHopHost(ctx context.Context, host string) bool { + allowed, ok := ctx.Value(allowedFirstHopHostKey{}).(string) + if !ok || allowed == "" { + return false + } + return allowed == normalizeAllowedFirstHopHost(host) +} + +func normalizeAllowedFirstHopHost(host string) string { + host = strings.ToLower(strings.TrimSpace(host)) + return strings.TrimSuffix(host, ".") +} + +func normalizeWhitelistIP(ip net.IP) net.IP { + if ip == nil { + return nil + } + if ip4 := ip.To4(); ip4 != nil { + return ip4 + } + return ip +} + +func shouldBlockPrivateIP(ip net.IP, whitelist *PrivateHostWhitelist) bool { + return IsPrivateOrRestrictedIP(ip) && !whitelist.Contains(ip) +} diff --git a/pkg/utils/http_guard_test.go b/pkg/utils/http_guard_test.go new file mode 100644 index 00000000..038cf91d --- /dev/null +++ b/pkg/utils/http_guard_test.go @@ -0,0 +1,234 @@ +package utils + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" +) + +func TestCreateSafeHTTPClient_AllowsLoopbackProxy(t *testing.T) { + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.String() != "http://example.com/proxied" { + t.Fatalf("proxy received URL %q, want %q", r.URL.String(), "http://example.com/proxied") + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("proxied")) + })) + defer proxy.Close() + + client, err := CreateSafeHTTPClient(SafeHTTPClientOptions{ + ProxyURL: proxy.URL, + Timeout: 5 * time.Second, + }) + if err != nil { + t.Fatalf("CreateSafeHTTPClient() error: %v", err) + } + + req, err := http.NewRequest(http.MethodGet, "http://example.com/proxied", nil) + if err != nil { + t.Fatalf("http.NewRequest() error: %v", err) + } + AllowConfiguredProxyFirstHop(req, client.Transport) + resp, err := client.Do(req) + if err != nil { + t.Fatalf("client.Do() error: %v", err) + } + defer resp.Body.Close() +} + +func TestCreateSafeHTTPClient_BlocksPrivateRedirect(t *testing.T) { + allowPrivateHosts := true + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://127.0.0.1/secret", http.StatusFound) + })) + defer server.Close() + + client, err := CreateSafeHTTPClient(SafeHTTPClientOptions{ + Timeout: 5 * time.Second, + AllowPrivateHosts: func() bool { + return allowPrivateHosts + }, + MaxRedirects: 5, + }) + if err != nil { + t.Fatalf("CreateSafeHTTPClient() error: %v", err) + } + + allowPrivateHosts = false + _, err = client.Get(server.URL) + if err == nil { + t.Fatal("expected redirect to private host to fail") + } + if !strings.Contains(err.Error(), "private or local network host") && + !strings.Contains(err.Error(), "blocked private or local target") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestValidateSafeHTTPURL_BlocksLoopback(t *testing.T) { + err := ValidateSafeHTTPURL("http://127.0.0.1:8080/file", nil, nil) + if err == nil { + t.Fatal("expected loopback URL to be blocked") + } + if !strings.Contains(err.Error(), "private or local network hosts") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestValidateSafeHTTPURL_AllowsWhitelistedPrivateHost(t *testing.T) { + whitelist, err := NewPrivateHostWhitelist([]string{"127.0.0.1"}) + if err != nil { + t.Fatalf("NewPrivateHostWhitelist() error: %v", err) + } + + err = ValidateSafeHTTPURL("http://127.0.0.1:8080/file", whitelist, nil) + if err != nil { + t.Fatalf("expected whitelisted private host to pass, got %v", err) + } +} + +func TestNewSafeDialContext_BlocksPrivateDNSResolutionWithoutWhitelist(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen on loopback: %v", err) + } + defer listener.Close() + + _, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatalf("failed to split listener address: %v", err) + } + + dialContext := NewSafeDialContext(&net.Dialer{Timeout: time.Second}, nil, nil) + _, err = dialContext(context.Background(), "tcp", net.JoinHostPort("localhost", port)) + if err == nil { + t.Fatal("expected localhost DNS resolution to be blocked without whitelist") + } + if !strings.Contains(err.Error(), "private") && !strings.Contains(err.Error(), "whitelisted") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestNewSafeDialContext_AllowsWhitelistedPrivateDNSResolution(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen on loopback: %v", err) + } + defer listener.Close() + + accepted := make(chan struct{}, 1) + go func() { + conn, acceptErr := listener.Accept() + if acceptErr != nil { + return + } + conn.Close() + accepted <- struct{}{} + }() + + _, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatalf("failed to split listener address: %v", err) + } + + whitelist, err := NewPrivateHostWhitelist([]string{"127.0.0.0/8"}) + if err != nil { + t.Fatalf("failed to parse whitelist: %v", err) + } + + dialContext := NewSafeDialContext(&net.Dialer{Timeout: time.Second}, whitelist, nil) + conn, err := dialContext(context.Background(), "tcp", net.JoinHostPort("localhost", port)) + if err != nil { + t.Fatalf("expected localhost DNS resolution to succeed with whitelist, got %v", err) + } + conn.Close() + + select { + case <-accepted: + case <-time.After(time.Second): + t.Fatal("expected localhost listener to accept a connection") + } +} + +func TestIsPrivateOrRestrictedIP_Table(t *testing.T) { + tests := []struct { + ip string + blocked bool + }{ + {"127.0.0.1", true}, + {"10.0.0.1", true}, + {"172.16.0.1", true}, + {"192.168.1.1", true}, + {"169.254.169.254", true}, + {"100.64.0.1", true}, + {"198.18.0.1", true}, + {"198.20.0.1", false}, + {"0.0.0.0", true}, + {"8.8.8.8", false}, + {"::1", true}, + {"::ffff:127.0.0.1", true}, + {"fc00::1", true}, + {"2002:7f00:0001::1", true}, + {"2002:0801:0101::1", false}, + {"2001:0000:4136:e378:8000:63bf:f5ff:fffe", true}, + {"2607:f8b0:4004:800::200e", false}, + } + + for _, tt := range tests { + t.Run(tt.ip, func(t *testing.T) { + ip := net.ParseIP(tt.ip) + if ip == nil { + t.Fatalf("failed to parse IP: %s", tt.ip) + } + got := IsPrivateOrRestrictedIP(ip) + if got != tt.blocked { + t.Fatalf("IsPrivateOrRestrictedIP(%s) = %v, want %v", tt.ip, got, tt.blocked) + } + }) + } +} + +func TestDownloadFile_DefaultAllowsLoopbackURL(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("local download")) + })) + defer server.Close() + + path := DownloadFile(server.URL, "file.txt", DownloadOptions{ + LoggerPrefix: "test", + Timeout: 5 * time.Second, + }) + if path == "" { + t.Fatal("expected default DownloadFile to allow loopback URL") + } + defer os.Remove(path) +} + +func TestDownloadFile_BlockPrivateTargetsBlocksRedirectToLoopback(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("secret")) + })) + defer target.Close() + + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer proxy.Close() + + path := DownloadFile("http://example.com/file.txt", "file.txt", DownloadOptions{ + LoggerPrefix: "test", + Timeout: 5 * time.Second, + ProxyURL: proxy.URL, + BlockPrivateTargets: true, + }) + if path != "" { + t.Fatalf("expected safe DownloadFile to block redirect to loopback, got %q", path) + } +} diff --git a/pkg/utils/media.go b/pkg/utils/media.go index a9c2497d..b30277b8 100644 --- a/pkg/utils/media.go +++ b/pkg/utils/media.go @@ -64,10 +64,11 @@ func SanitizeFilename(filename string) string { // DownloadOptions holds optional parameters for downloading files type DownloadOptions struct { - Timeout time.Duration - ExtraHeaders map[string]string - LoggerPrefix string - ProxyURL string + Timeout time.Duration + ExtraHeaders map[string]string + LoggerPrefix string + ProxyURL string + BlockPrivateTargets bool } // DownloadFile downloads a file from URL to a local temp directory. @@ -93,8 +94,45 @@ func DownloadFile(urlStr, filename string, opts DownloadOptions) string { safeName := SanitizeFilename(filename) localPath := filepath.Join(mediaDir, uuid.New().String()[:8]+"_"+safeName) - // Create HTTP request - req, err := http.NewRequest("GET", urlStr, nil) + var client *http.Client + var err error + if opts.BlockPrivateTargets { + if err := ValidateSafeHTTPURL(urlStr, nil, nil); err != nil { + logger.ErrorCF(opts.LoggerPrefix, "Blocked unsafe download URL", map[string]any{ + "error": err.Error(), + "url": urlStr, + }) + return "" + } + client, err = CreateSafeHTTPClient(SafeHTTPClientOptions{ + ProxyURL: opts.ProxyURL, + Timeout: opts.Timeout, + MaxRedirects: 10, + }) + if err != nil { + logger.ErrorCF(opts.LoggerPrefix, "Failed to create safe download client", map[string]any{ + "error": err.Error(), + }) + return "" + } + } else { + client = &http.Client{Timeout: opts.Timeout} + if opts.ProxyURL != "" { + proxyURL, parseErr := url.Parse(opts.ProxyURL) + if parseErr != nil { + logger.ErrorCF(opts.LoggerPrefix, "Invalid proxy URL for download", map[string]any{ + "error": parseErr.Error(), + "proxy": opts.ProxyURL, + }) + return "" + } + client.Transport = &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + } + } + } + + req, err := http.NewRequest(http.MethodGet, urlStr, nil) if err != nil { logger.ErrorCF(opts.LoggerPrefix, "Failed to create download request", map[string]any{ "error": err.Error(), @@ -106,21 +144,10 @@ func DownloadFile(urlStr, filename string, opts DownloadOptions) string { for key, value := range opts.ExtraHeaders { req.Header.Set(key, value) } - - client := &http.Client{Timeout: opts.Timeout} - if opts.ProxyURL != "" { - proxyURL, parseErr := url.Parse(opts.ProxyURL) - if parseErr != nil { - logger.ErrorCF(opts.LoggerPrefix, "Invalid proxy URL for download", map[string]any{ - "error": parseErr.Error(), - "proxy": opts.ProxyURL, - }) - return "" - } - client.Transport = &http.Transport{ - Proxy: http.ProxyURL(proxyURL), - } + if opts.BlockPrivateTargets { + AllowConfiguredProxyFirstHop(req, client.Transport) } + resp, err := client.Do(req) if err != nil { logger.ErrorCF(opts.LoggerPrefix, "Failed to download file", map[string]any{