fix(onebot): block private inbound media fetches
This commit is contained in:
lc6464
Guoguo
parent
a16a1e1535
commit
234bd03018
7 changed files with 798 additions and 289 deletions
|
|
@ -24,6 +24,7 @@ import (
|
|||
type OneBotChannel struct {
|
||||
*channels.BaseChannel
|
||||
config *config.OneBotSettings
|
||||
downloadFn func(urlStr, filename string) string
|
||||
conn *websocket.Conn
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
|
@ -795,9 +796,7 @@ func (c *OneBotChannel) parseMessageSegments(
|
|||
} else if n, ok := data["name"].(string); ok && n != "" {
|
||||
filename = n
|
||||
}
|
||||
localPath := utils.DownloadFile(url, filename, utils.DownloadOptions{
|
||||
LoggerPrefix: "onebot",
|
||||
})
|
||||
localPath := c.downloadInboundFile(url, filename)
|
||||
if localPath != "" {
|
||||
mediaRefs = append(mediaRefs, storeFile(localPath, filename))
|
||||
textParts = append(textParts, fmt.Sprintf("[%s]", segType))
|
||||
|
|
@ -809,9 +808,7 @@ func (c *OneBotChannel) parseMessageSegments(
|
|||
if data != nil {
|
||||
url, _ := data["url"].(string)
|
||||
if url != "" {
|
||||
localPath := utils.DownloadFile(url, "voice.amr", utils.DownloadOptions{
|
||||
LoggerPrefix: "onebot",
|
||||
})
|
||||
localPath := c.downloadInboundFile(url, "voice.amr")
|
||||
if localPath != "" {
|
||||
textParts = append(textParts, "[voice]")
|
||||
mediaRefs = append(mediaRefs, storeFile(localPath, "voice.amr"))
|
||||
|
|
@ -847,6 +844,16 @@ func (c *OneBotChannel) parseMessageSegments(
|
|||
}
|
||||
}
|
||||
|
||||
func (c *OneBotChannel) downloadInboundFile(urlStr, filename string) string {
|
||||
if c.downloadFn != nil {
|
||||
return c.downloadFn(urlStr, filename)
|
||||
}
|
||||
return utils.DownloadFile(urlStr, filename, utils.DownloadOptions{
|
||||
LoggerPrefix: "onebot",
|
||||
BlockPrivateTargets: true,
|
||||
})
|
||||
}
|
||||
|
||||
func (c *OneBotChannel) handleRawEvent(raw *oneBotRawEvent) {
|
||||
switch raw.PostType {
|
||||
case "message":
|
||||
|
|
|
|||
126
pkg/channels/onebot/onebot_test.go
Normal file
126
pkg/channels/onebot/onebot_test.go
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
package onebot
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/sipeed/picoclaw/pkg/media"
|
||||
)
|
||||
|
||||
func TestParseMessageSegments_BlocksLoopbackInboundMediaURL(t *testing.T) {
|
||||
ch := &OneBotChannel{}
|
||||
store := media.NewFileMediaStore()
|
||||
|
||||
raw := json.RawMessage(`[
|
||||
{"type":"text","data":{"text":"see attachment"}},
|
||||
{"type":"image","data":{"url":"http://127.0.0.1:8080/evil.png","file":"evil.png"}}
|
||||
]`)
|
||||
|
||||
result := ch.parseMessageSegments(raw, 0, store, "onebot:test:msg1")
|
||||
|
||||
if got := result.Text; got != "see attachment" {
|
||||
t.Fatalf("Text = %q, want %q", got, "see attachment")
|
||||
}
|
||||
if len(result.Media) != 0 {
|
||||
t.Fatalf("Media count = %d, want 0", len(result.Media))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageSegments_BlocksInboundMediaRedirectToLoopback(t *testing.T) {
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("secret"))
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got, want := r.URL.String(), "http://example.com/evil.png"; got != want {
|
||||
t.Fatalf("proxy request URL = %q, want %q", got, want)
|
||||
}
|
||||
http.Redirect(w, r, target.URL, http.StatusFound)
|
||||
}))
|
||||
defer proxy.Close()
|
||||
|
||||
t.Setenv("HTTP_PROXY", proxy.URL)
|
||||
t.Setenv("http_proxy", proxy.URL)
|
||||
t.Setenv("HTTPS_PROXY", "")
|
||||
t.Setenv("https_proxy", "")
|
||||
t.Setenv("ALL_PROXY", "")
|
||||
t.Setenv("all_proxy", "")
|
||||
t.Setenv("NO_PROXY", "")
|
||||
t.Setenv("no_proxy", "")
|
||||
|
||||
ch := &OneBotChannel{}
|
||||
store := media.NewFileMediaStore()
|
||||
|
||||
raw := json.RawMessage(`[
|
||||
{"type":"text","data":{"text":"see attachment"}},
|
||||
{"type":"image","data":{"url":"http://example.com/evil.png","file":"evil.png"}}
|
||||
]`)
|
||||
|
||||
result := ch.parseMessageSegments(raw, 0, store, "onebot:test:msg-redirect")
|
||||
|
||||
if got := result.Text; got != "see attachment" {
|
||||
t.Fatalf("Text = %q, want %q", got, "see attachment")
|
||||
}
|
||||
if len(result.Media) != 0 {
|
||||
t.Fatalf("Media count = %d, want 0", len(result.Media))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageSegments_StoresDownloadedMediaRef(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
localPath := filepath.Join(tmpDir, "image.png")
|
||||
if err := os.WriteFile(localPath, []byte("fake-image"), 0o600); err != nil {
|
||||
t.Fatalf("os.WriteFile() error = %v", err)
|
||||
}
|
||||
|
||||
ch := &OneBotChannel{
|
||||
downloadFn: func(urlStr, filename string) string {
|
||||
if urlStr != "https://cdn.example.com/image.png" {
|
||||
t.Fatalf("download url = %q, want %q", urlStr, "https://cdn.example.com/image.png")
|
||||
}
|
||||
if filename != "image.png" {
|
||||
t.Fatalf("download filename = %q, want %q", filename, "image.png")
|
||||
}
|
||||
return localPath
|
||||
},
|
||||
}
|
||||
store := media.NewFileMediaStore()
|
||||
|
||||
raw := json.RawMessage(`[
|
||||
{"type":"text","data":{"text":"see attachment"}},
|
||||
{"type":"image","data":{"url":"https://cdn.example.com/image.png","file":"image.png"}}
|
||||
]`)
|
||||
|
||||
result := ch.parseMessageSegments(raw, 0, store, "onebot:test:msg2")
|
||||
|
||||
if got := result.Text; got != "see attachment[image]" {
|
||||
t.Fatalf("Text = %q, want %q", got, "see attachment[image]")
|
||||
}
|
||||
if len(result.Media) != 1 {
|
||||
t.Fatalf("Media count = %d, want 1", len(result.Media))
|
||||
}
|
||||
if !strings.HasPrefix(result.Media[0], "media://") {
|
||||
t.Fatalf("media ref = %q, want media:// prefix", result.Media[0])
|
||||
}
|
||||
|
||||
resolvedPath, meta, err := store.ResolveWithMeta(result.Media[0])
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveWithMeta() error = %v", err)
|
||||
}
|
||||
if resolvedPath != localPath {
|
||||
t.Fatalf("resolved path = %q, want %q", resolvedPath, localPath)
|
||||
}
|
||||
if meta.Source != "onebot" {
|
||||
t.Fatalf("meta.Source = %q, want %q", meta.Source, "onebot")
|
||||
}
|
||||
if meta.Filename != "image.png" {
|
||||
t.Fatalf("meta.Filename = %q, want %q", meta.Filename, "image.png")
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,8 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
|
@ -213,6 +215,33 @@ func TestSendMedia_ImageFallbacksToDocumentOnInvalidDimensions(t *testing.T) {
|
|||
assert.Equal(t, "caption", constructor.calls[1].Parameters["caption"])
|
||||
}
|
||||
|
||||
func TestDownloadFileWithInfo_AllowsLocalConfiguredBaseURL(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got, want := r.URL.Path, "/file/bot"+testToken+"/photos/image"; got != want {
|
||||
t.Fatalf("request path = %q, want %q", got, want)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("telegram-local-bot-api"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
ch, err := NewTelegramChannel(
|
||||
&config.Channel{Type: config.ChannelTelegram, Enabled: true},
|
||||
&config.TelegramSettings{
|
||||
Token: *config.NewSecureString(testToken),
|
||||
BaseURL: server.URL,
|
||||
},
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
path := ch.downloadFileWithInfo(&telego.File{FilePath: "photos/image"}, "")
|
||||
if path == "" {
|
||||
t.Fatal("expected local base_url download to succeed")
|
||||
}
|
||||
defer os.Remove(path)
|
||||
}
|
||||
|
||||
func TestSendMedia_ImageNonDimensionErrorDoesNotFallback(t *testing.T) {
|
||||
constructor := &multipartRecordingConstructor{}
|
||||
caller := &stubCaller{
|
||||
|
|
|
|||
|
|
@ -1996,16 +1996,9 @@ type WebFetchTool struct {
|
|||
client *http.Client
|
||||
format string
|
||||
fetchLimitBytes int64
|
||||
whitelist *privateHostWhitelist
|
||||
whitelist *utils.PrivateHostWhitelist
|
||||
}
|
||||
|
||||
type privateHostWhitelist struct {
|
||||
exact map[string]struct{}
|
||||
cidrs []*net.IPNet
|
||||
}
|
||||
|
||||
type webFetchAllowedFirstHopHostKey struct{}
|
||||
|
||||
func NewWebFetchTool(maxChars int, format string, fetchLimitBytes int64) (*WebFetchTool, error) {
|
||||
// createHTTPClient cannot fail with an empty proxy string.
|
||||
return NewWebFetchToolWithConfig(maxChars, "", format, fetchLimitBytes, nil)
|
||||
|
|
@ -2035,31 +2028,22 @@ func NewWebFetchToolWithConfig(
|
|||
if maxChars <= 0 {
|
||||
maxChars = defaultMaxChars
|
||||
}
|
||||
whitelist, err := newPrivateHostWhitelist(privateHostWhitelist)
|
||||
whitelist, err := utils.NewPrivateHostWhitelist(privateHostWhitelist)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse web fetch private host whitelist: %w", err)
|
||||
}
|
||||
client, err := utils.CreateHTTPClient(proxy, fetchTimeout)
|
||||
client, err := utils.CreateSafeHTTPClient(utils.SafeHTTPClientOptions{
|
||||
ProxyURL: proxy,
|
||||
Timeout: fetchTimeout,
|
||||
PrivateHostWhitelist: privateHostWhitelist,
|
||||
AllowPrivateHosts: func() bool {
|
||||
return allowPrivateWebFetchHosts.Load()
|
||||
},
|
||||
MaxRedirects: maxRedirects,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create HTTP client for web fetch: %w", err)
|
||||
}
|
||||
if transport, ok := client.Transport.(*http.Transport); ok {
|
||||
dialer := &net.Dialer{
|
||||
Timeout: 15 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}
|
||||
transport.DialContext = newSafeDialContext(dialer, whitelist)
|
||||
}
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= maxRedirects {
|
||||
return fmt.Errorf("stopped after %d redirects", maxRedirects)
|
||||
}
|
||||
if isObviousPrivateHost(req.URL.Hostname(), whitelist) {
|
||||
return fmt.Errorf("redirect target is private or local network host")
|
||||
}
|
||||
allowConfiguredProxyFirstHop(req, client.Transport)
|
||||
return nil
|
||||
}
|
||||
if fetchLimitBytes <= 0 {
|
||||
fetchLimitBytes = 10 * 1024 * 1024 // Security Fallback
|
||||
}
|
||||
|
|
@ -2121,7 +2105,9 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
|
|||
// Lightweight pre-flight: block obvious localhost/literal-IP without DNS resolution.
|
||||
// The real SSRF guard is newSafeDialContext at connect time.
|
||||
hostname := parsedURL.Hostname()
|
||||
if isObviousPrivateHost(hostname, t.whitelist) {
|
||||
if utils.IsObviousPrivateHost(hostname, t.whitelist, func() bool {
|
||||
return allowPrivateWebFetchHosts.Load()
|
||||
}) {
|
||||
return ErrorResult("fetching private or local network hosts is not allowed")
|
||||
}
|
||||
|
||||
|
|
@ -2137,7 +2123,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
|
|||
if reqErr != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create request: %w", reqErr)
|
||||
}
|
||||
allowConfiguredProxyFirstHop(req, t.client.Transport)
|
||||
utils.AllowConfiguredProxyFirstHop(req, t.client.Transport)
|
||||
req.Header.Set("User-Agent", ua)
|
||||
resp, doErr := t.client.Do(req)
|
||||
if doErr != nil {
|
||||
|
|
@ -2325,247 +2311,29 @@ func (t *WebFetchTool) extractText(htmlContent string) string {
|
|||
return strings.Join(cleanLines, "\n")
|
||||
}
|
||||
|
||||
// newSafeDialContext re-resolves DNS at connect time to mitigate DNS rebinding (TOCTOU)
|
||||
// where a hostname resolves to a public IP during pre-flight but a private IP at connect time.
|
||||
func newSafeDialContext(
|
||||
dialer *net.Dialer,
|
||||
whitelist *privateHostWhitelist,
|
||||
whitelist *utils.PrivateHostWhitelist,
|
||||
) func(context.Context, string, string) (net.Conn, error) {
|
||||
return func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
if allowPrivateWebFetchHosts.Load() {
|
||||
return dialer.DialContext(ctx, network, address)
|
||||
}
|
||||
return utils.NewSafeDialContext(dialer, whitelist, func() bool {
|
||||
return allowPrivateWebFetchHosts.Load()
|
||||
})
|
||||
}
|
||||
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid target address %q: %w", address, err)
|
||||
}
|
||||
if host == "" {
|
||||
return nil, fmt.Errorf("empty target host")
|
||||
}
|
||||
if isAllowedFirstHopHost(ctx, host) {
|
||||
return dialer.DialContext(ctx, network, address)
|
||||
}
|
||||
func newPrivateHostWhitelist(entries []string) (*utils.PrivateHostWhitelist, error) {
|
||||
return utils.NewPrivateHostWhitelist(entries)
|
||||
}
|
||||
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if shouldBlockPrivateIP(ip, whitelist) {
|
||||
return nil, fmt.Errorf("blocked private or local target: %s", host)
|
||||
}
|
||||
return dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
||||
}
|
||||
func isObviousPrivateHost(host string, whitelist *utils.PrivateHostWhitelist) bool {
|
||||
return utils.IsObviousPrivateHost(host, whitelist, func() bool {
|
||||
return allowPrivateWebFetchHosts.Load()
|
||||
})
|
||||
}
|
||||
|
||||
ipAddrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve %s: %w", host, err)
|
||||
}
|
||||
|
||||
attempted := 0
|
||||
var lastErr error
|
||||
for _, ipAddr := range ipAddrs {
|
||||
if shouldBlockPrivateIP(ipAddr.IP, whitelist) {
|
||||
continue
|
||||
}
|
||||
attempted++
|
||||
conn, err := dialer.DialContext(
|
||||
ctx,
|
||||
network,
|
||||
net.JoinHostPort(ipAddr.IP.String(), port),
|
||||
)
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
|
||||
if attempted == 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"all resolved addresses for %s are private, restricted, or not whitelisted",
|
||||
host,
|
||||
)
|
||||
}
|
||||
if lastErr != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"failed connecting to public addresses for %s: %w",
|
||||
host,
|
||||
lastErr,
|
||||
)
|
||||
}
|
||||
return nil, fmt.Errorf("failed connecting to public addresses for %s", host)
|
||||
}
|
||||
func isPrivateOrRestrictedIP(ip net.IP) bool {
|
||||
return utils.IsPrivateOrRestrictedIP(ip)
|
||||
}
|
||||
|
||||
func allowConfiguredProxyFirstHop(req *http.Request, rt http.RoundTripper) {
|
||||
if req == nil {
|
||||
return
|
||||
}
|
||||
|
||||
transport, ok := rt.(*http.Transport)
|
||||
if !ok || transport.Proxy == nil {
|
||||
return
|
||||
}
|
||||
|
||||
proxyURL, err := transport.Proxy(req)
|
||||
if err != nil || proxyURL == nil {
|
||||
return
|
||||
}
|
||||
|
||||
host := normalizeAllowedFirstHopHost(proxyURL.Hostname())
|
||||
if host == "" {
|
||||
return
|
||||
}
|
||||
|
||||
*req = *req.WithContext(context.WithValue(
|
||||
req.Context(),
|
||||
webFetchAllowedFirstHopHostKey{},
|
||||
host,
|
||||
))
|
||||
}
|
||||
|
||||
func isAllowedFirstHopHost(ctx context.Context, host string) bool {
|
||||
allowed, ok := ctx.Value(webFetchAllowedFirstHopHostKey{}).(string)
|
||||
if !ok || allowed == "" {
|
||||
return false
|
||||
}
|
||||
return allowed == normalizeAllowedFirstHopHost(host)
|
||||
}
|
||||
|
||||
func normalizeAllowedFirstHopHost(host string) string {
|
||||
host = strings.ToLower(strings.TrimSpace(host))
|
||||
return strings.TrimSuffix(host, ".")
|
||||
}
|
||||
|
||||
func newPrivateHostWhitelist(entries []string) (*privateHostWhitelist, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
whitelist := &privateHostWhitelist{
|
||||
exact: make(map[string]struct{}),
|
||||
cidrs: make([]*net.IPNet, 0, len(entries)),
|
||||
}
|
||||
for _, entry := range entries {
|
||||
entry = strings.TrimSpace(entry)
|
||||
if entry == "" {
|
||||
continue
|
||||
}
|
||||
if ip := net.ParseIP(entry); ip != nil {
|
||||
whitelist.exact[normalizeWhitelistIP(ip).String()] = struct{}{}
|
||||
continue
|
||||
}
|
||||
_, network, err := net.ParseCIDR(entry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid entry %q: expected IP or CIDR", entry)
|
||||
}
|
||||
whitelist.cidrs = append(whitelist.cidrs, network)
|
||||
}
|
||||
|
||||
if len(whitelist.exact) == 0 && len(whitelist.cidrs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return whitelist, nil
|
||||
}
|
||||
|
||||
func (w *privateHostWhitelist) Contains(ip net.IP) bool {
|
||||
if w == nil || ip == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
normalized := normalizeWhitelistIP(ip)
|
||||
if _, ok := w.exact[normalized.String()]; ok {
|
||||
return true
|
||||
}
|
||||
for _, network := range w.cidrs {
|
||||
if network.Contains(normalized) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeWhitelistIP(ip net.IP) net.IP {
|
||||
if ip == nil {
|
||||
return nil
|
||||
}
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
return ip4
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
func shouldBlockPrivateIP(ip net.IP, whitelist *privateHostWhitelist) bool {
|
||||
return isPrivateOrRestrictedIP(ip) && !whitelist.Contains(ip)
|
||||
}
|
||||
|
||||
// isObviousPrivateHost performs a lightweight, no-DNS check for obviously private hosts.
|
||||
// It catches localhost, literal private IPs, and empty hosts. It does NOT resolve DNS —
|
||||
// the real SSRF guard is newSafeDialContext which checks IPs at connect time.
|
||||
func isObviousPrivateHost(host string, whitelist *privateHostWhitelist) bool {
|
||||
if allowPrivateWebFetchHosts.Load() {
|
||||
return false
|
||||
}
|
||||
|
||||
h := strings.ToLower(strings.TrimSpace(host))
|
||||
h = strings.TrimSuffix(h, ".")
|
||||
if h == "" {
|
||||
return true
|
||||
}
|
||||
|
||||
if h == "localhost" || strings.HasSuffix(h, ".localhost") {
|
||||
return true
|
||||
}
|
||||
|
||||
if ip := net.ParseIP(h); ip != nil {
|
||||
return shouldBlockPrivateIP(ip, whitelist)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// isPrivateOrRestrictedIP returns true for IPs that should never be reached via web_fetch:
|
||||
// RFC 1918, loopback, link-local (incl. cloud metadata 169.254.x.x), carrier-grade NAT,
|
||||
// benchmark (198.18.0.0/15), IPv6 unique-local (fc00::/7), 6to4 (2002::/16), and
|
||||
// Teredo (2001:0000::/32).
|
||||
func isPrivateOrRestrictedIP(ip net.IP) bool {
|
||||
if ip == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
||||
ip.IsMulticast() || ip.IsUnspecified() {
|
||||
return true
|
||||
}
|
||||
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
// IPv4 private, loopback, link-local, and carrier-grade NAT ranges.
|
||||
if ip4[0] == 10 ||
|
||||
ip4[0] == 127 ||
|
||||
ip4[0] == 0 ||
|
||||
(ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31) ||
|
||||
(ip4[0] == 192 && ip4[1] == 168) ||
|
||||
(ip4[0] == 169 && ip4[1] == 254) ||
|
||||
(ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127) ||
|
||||
(ip4[0] == 198 && ip4[1] >= 18 && ip4[1] <= 19) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if len(ip) == net.IPv6len {
|
||||
// IPv6 unique local addresses (fc00::/7)
|
||||
if (ip[0] & 0xfe) == 0xfc {
|
||||
return true
|
||||
}
|
||||
// 6to4 addresses (2002::/16): check the embedded IPv4 at bytes [2:6].
|
||||
if ip[0] == 0x20 && ip[1] == 0x02 {
|
||||
embedded := net.IPv4(ip[2], ip[3], ip[4], ip[5])
|
||||
return isPrivateOrRestrictedIP(embedded)
|
||||
}
|
||||
// Teredo (2001:0000::/32): client IPv4 is at bytes [12:16], XOR-inverted.
|
||||
if ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x00 && ip[3] == 0x00 {
|
||||
client := net.IPv4(ip[12]^0xff, ip[13]^0xff, ip[14]^0xff, ip[15]^0xff)
|
||||
return isPrivateOrRestrictedIP(client)
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
utils.AllowConfiguredProxyFirstHop(req, rt)
|
||||
}
|
||||
|
|
|
|||
318
pkg/utils/http_guard.go
Normal file
318
pkg/utils/http_guard.go
Normal file
|
|
@ -0,0 +1,318 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type SafeHTTPClientOptions struct {
|
||||
ProxyURL string
|
||||
Timeout time.Duration
|
||||
PrivateHostWhitelist []string
|
||||
AllowPrivateHosts func() bool
|
||||
MaxRedirects int
|
||||
}
|
||||
|
||||
type PrivateHostWhitelist struct {
|
||||
exact map[string]struct{}
|
||||
cidrs []*net.IPNet
|
||||
}
|
||||
|
||||
type allowedFirstHopHostKey struct{}
|
||||
|
||||
func NewPrivateHostWhitelist(entries []string) (*PrivateHostWhitelist, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
whitelist := &PrivateHostWhitelist{
|
||||
exact: make(map[string]struct{}),
|
||||
cidrs: make([]*net.IPNet, 0, len(entries)),
|
||||
}
|
||||
for _, entry := range entries {
|
||||
entry = strings.TrimSpace(entry)
|
||||
if entry == "" {
|
||||
continue
|
||||
}
|
||||
if ip := net.ParseIP(entry); ip != nil {
|
||||
whitelist.exact[normalizeWhitelistIP(ip).String()] = struct{}{}
|
||||
continue
|
||||
}
|
||||
_, network, err := net.ParseCIDR(entry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid entry %q: expected IP or CIDR", entry)
|
||||
}
|
||||
whitelist.cidrs = append(whitelist.cidrs, network)
|
||||
}
|
||||
|
||||
if len(whitelist.exact) == 0 && len(whitelist.cidrs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return whitelist, nil
|
||||
}
|
||||
|
||||
func (w *PrivateHostWhitelist) Contains(ip net.IP) bool {
|
||||
if w == nil || ip == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
normalized := normalizeWhitelistIP(ip)
|
||||
if _, ok := w.exact[normalized.String()]; ok {
|
||||
return true
|
||||
}
|
||||
for _, network := range w.cidrs {
|
||||
if network.Contains(normalized) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func CreateSafeHTTPClient(opts SafeHTTPClientOptions) (*http.Client, error) {
|
||||
client, err := CreateHTTPClient(opts.ProxyURL, opts.Timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
whitelist, err := NewPrivateHostWhitelist(opts.PrivateHostWhitelist)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
transport, ok := client.Transport.(*http.Transport)
|
||||
if ok {
|
||||
dialer := &net.Dialer{
|
||||
Timeout: 15 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}
|
||||
transport.DialContext = NewSafeDialContext(dialer, whitelist, opts.AllowPrivateHosts)
|
||||
}
|
||||
|
||||
maxRedirects := opts.MaxRedirects
|
||||
if maxRedirects <= 0 {
|
||||
maxRedirects = 10
|
||||
}
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= maxRedirects {
|
||||
return fmt.Errorf("stopped after %d redirects", maxRedirects)
|
||||
}
|
||||
if IsObviousPrivateHost(req.URL.Hostname(), whitelist, opts.AllowPrivateHosts) {
|
||||
return fmt.Errorf("redirect target is private or local network host")
|
||||
}
|
||||
AllowConfiguredProxyFirstHop(req, client.Transport)
|
||||
return nil
|
||||
}
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func ValidateSafeHTTPURL(urlStr string, whitelist *PrivateHostWhitelist, allowPrivateHosts func() bool) error {
|
||||
parsedURL, err := url.Parse(urlStr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
|
||||
return fmt.Errorf("only http/https URLs are allowed")
|
||||
}
|
||||
if parsedURL.Host == "" {
|
||||
return fmt.Errorf("missing domain in URL")
|
||||
}
|
||||
if IsObviousPrivateHost(parsedURL.Hostname(), whitelist, allowPrivateHosts) {
|
||||
return fmt.Errorf("fetching private or local network hosts is not allowed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewSafeDialContext(
|
||||
dialer *net.Dialer,
|
||||
whitelist *PrivateHostWhitelist,
|
||||
allowPrivateHosts func() bool,
|
||||
) func(context.Context, string, string) (net.Conn, error) {
|
||||
return func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
if allowPrivateHosts != nil && allowPrivateHosts() {
|
||||
return dialer.DialContext(ctx, network, address)
|
||||
}
|
||||
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid target address %q: %w", address, err)
|
||||
}
|
||||
if host == "" {
|
||||
return nil, fmt.Errorf("empty target host")
|
||||
}
|
||||
if isAllowedFirstHopHost(ctx, host) {
|
||||
return dialer.DialContext(ctx, network, address)
|
||||
}
|
||||
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if shouldBlockPrivateIP(ip, whitelist) {
|
||||
return nil, fmt.Errorf("blocked private or local target: %s", host)
|
||||
}
|
||||
return dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
||||
}
|
||||
|
||||
ipAddrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve %s: %w", host, err)
|
||||
}
|
||||
|
||||
attempted := 0
|
||||
var lastErr error
|
||||
for _, ipAddr := range ipAddrs {
|
||||
if shouldBlockPrivateIP(ipAddr.IP, whitelist) {
|
||||
continue
|
||||
}
|
||||
attempted++
|
||||
conn, err := dialer.DialContext(
|
||||
ctx,
|
||||
network,
|
||||
net.JoinHostPort(ipAddr.IP.String(), port),
|
||||
)
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
|
||||
if attempted == 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"all resolved addresses for %s are private, restricted, or not whitelisted",
|
||||
host,
|
||||
)
|
||||
}
|
||||
if lastErr != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"failed connecting to public addresses for %s: %w",
|
||||
host,
|
||||
lastErr,
|
||||
)
|
||||
}
|
||||
return nil, fmt.Errorf("failed connecting to public addresses for %s", host)
|
||||
}
|
||||
}
|
||||
|
||||
func AllowConfiguredProxyFirstHop(req *http.Request, rt http.RoundTripper) {
|
||||
if req == nil {
|
||||
return
|
||||
}
|
||||
|
||||
transport, ok := rt.(*http.Transport)
|
||||
if !ok || transport.Proxy == nil {
|
||||
return
|
||||
}
|
||||
|
||||
proxyURL, err := transport.Proxy(req)
|
||||
if err != nil || proxyURL == nil {
|
||||
return
|
||||
}
|
||||
|
||||
host := normalizeAllowedFirstHopHost(proxyURL.Hostname())
|
||||
if host == "" {
|
||||
return
|
||||
}
|
||||
|
||||
*req = *req.WithContext(context.WithValue(
|
||||
req.Context(),
|
||||
allowedFirstHopHostKey{},
|
||||
host,
|
||||
))
|
||||
}
|
||||
|
||||
func IsObviousPrivateHost(
|
||||
host string,
|
||||
whitelist *PrivateHostWhitelist,
|
||||
allowPrivateHosts func() bool,
|
||||
) bool {
|
||||
if allowPrivateHosts != nil && allowPrivateHosts() {
|
||||
return false
|
||||
}
|
||||
|
||||
h := strings.ToLower(strings.TrimSpace(host))
|
||||
h = strings.TrimSuffix(h, ".")
|
||||
if h == "" {
|
||||
return true
|
||||
}
|
||||
|
||||
if h == "localhost" || strings.HasSuffix(h, ".localhost") {
|
||||
return true
|
||||
}
|
||||
|
||||
if ip := net.ParseIP(h); ip != nil {
|
||||
return shouldBlockPrivateIP(ip, whitelist)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func IsPrivateOrRestrictedIP(ip net.IP) bool {
|
||||
if ip == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
||||
ip.IsMulticast() || ip.IsUnspecified() {
|
||||
return true
|
||||
}
|
||||
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
if ip4[0] == 10 ||
|
||||
ip4[0] == 127 ||
|
||||
ip4[0] == 0 ||
|
||||
(ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31) ||
|
||||
(ip4[0] == 192 && ip4[1] == 168) ||
|
||||
(ip4[0] == 169 && ip4[1] == 254) ||
|
||||
(ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127) ||
|
||||
(ip4[0] == 198 && ip4[1] >= 18 && ip4[1] <= 19) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if len(ip) == net.IPv6len {
|
||||
if (ip[0] & 0xfe) == 0xfc {
|
||||
return true
|
||||
}
|
||||
if ip[0] == 0x20 && ip[1] == 0x02 {
|
||||
embedded := net.IPv4(ip[2], ip[3], ip[4], ip[5])
|
||||
return IsPrivateOrRestrictedIP(embedded)
|
||||
}
|
||||
if ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x00 && ip[3] == 0x00 {
|
||||
client := net.IPv4(ip[12]^0xff, ip[13]^0xff, ip[14]^0xff, ip[15]^0xff)
|
||||
return IsPrivateOrRestrictedIP(client)
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func isAllowedFirstHopHost(ctx context.Context, host string) bool {
|
||||
allowed, ok := ctx.Value(allowedFirstHopHostKey{}).(string)
|
||||
if !ok || allowed == "" {
|
||||
return false
|
||||
}
|
||||
return allowed == normalizeAllowedFirstHopHost(host)
|
||||
}
|
||||
|
||||
func normalizeAllowedFirstHopHost(host string) string {
|
||||
host = strings.ToLower(strings.TrimSpace(host))
|
||||
return strings.TrimSuffix(host, ".")
|
||||
}
|
||||
|
||||
func normalizeWhitelistIP(ip net.IP) net.IP {
|
||||
if ip == nil {
|
||||
return nil
|
||||
}
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
return ip4
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
func shouldBlockPrivateIP(ip net.IP, whitelist *PrivateHostWhitelist) bool {
|
||||
return IsPrivateOrRestrictedIP(ip) && !whitelist.Contains(ip)
|
||||
}
|
||||
234
pkg/utils/http_guard_test.go
Normal file
234
pkg/utils/http_guard_test.go
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCreateSafeHTTPClient_AllowsLoopbackProxy(t *testing.T) {
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.String() != "http://example.com/proxied" {
|
||||
t.Fatalf("proxy received URL %q, want %q", r.URL.String(), "http://example.com/proxied")
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("proxied"))
|
||||
}))
|
||||
defer proxy.Close()
|
||||
|
||||
client, err := CreateSafeHTTPClient(SafeHTTPClientOptions{
|
||||
ProxyURL: proxy.URL,
|
||||
Timeout: 5 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateSafeHTTPClient() error: %v", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, "http://example.com/proxied", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("http.NewRequest() error: %v", err)
|
||||
}
|
||||
AllowConfiguredProxyFirstHop(req, client.Transport)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("client.Do() error: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
}
|
||||
|
||||
func TestCreateSafeHTTPClient_BlocksPrivateRedirect(t *testing.T) {
|
||||
allowPrivateHosts := true
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "http://127.0.0.1/secret", http.StatusFound)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := CreateSafeHTTPClient(SafeHTTPClientOptions{
|
||||
Timeout: 5 * time.Second,
|
||||
AllowPrivateHosts: func() bool {
|
||||
return allowPrivateHosts
|
||||
},
|
||||
MaxRedirects: 5,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateSafeHTTPClient() error: %v", err)
|
||||
}
|
||||
|
||||
allowPrivateHosts = false
|
||||
_, err = client.Get(server.URL)
|
||||
if err == nil {
|
||||
t.Fatal("expected redirect to private host to fail")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "private or local network host") &&
|
||||
!strings.Contains(err.Error(), "blocked private or local target") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSafeHTTPURL_BlocksLoopback(t *testing.T) {
|
||||
err := ValidateSafeHTTPURL("http://127.0.0.1:8080/file", nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected loopback URL to be blocked")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "private or local network hosts") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSafeHTTPURL_AllowsWhitelistedPrivateHost(t *testing.T) {
|
||||
whitelist, err := NewPrivateHostWhitelist([]string{"127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPrivateHostWhitelist() error: %v", err)
|
||||
}
|
||||
|
||||
err = ValidateSafeHTTPURL("http://127.0.0.1:8080/file", whitelist, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expected whitelisted private host to pass, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSafeDialContext_BlocksPrivateDNSResolutionWithoutWhitelist(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to listen on loopback: %v", err)
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
_, port, err := net.SplitHostPort(listener.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to split listener address: %v", err)
|
||||
}
|
||||
|
||||
dialContext := NewSafeDialContext(&net.Dialer{Timeout: time.Second}, nil, nil)
|
||||
_, err = dialContext(context.Background(), "tcp", net.JoinHostPort("localhost", port))
|
||||
if err == nil {
|
||||
t.Fatal("expected localhost DNS resolution to be blocked without whitelist")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "private") && !strings.Contains(err.Error(), "whitelisted") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSafeDialContext_AllowsWhitelistedPrivateDNSResolution(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to listen on loopback: %v", err)
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
accepted := make(chan struct{}, 1)
|
||||
go func() {
|
||||
conn, acceptErr := listener.Accept()
|
||||
if acceptErr != nil {
|
||||
return
|
||||
}
|
||||
conn.Close()
|
||||
accepted <- struct{}{}
|
||||
}()
|
||||
|
||||
_, port, err := net.SplitHostPort(listener.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to split listener address: %v", err)
|
||||
}
|
||||
|
||||
whitelist, err := NewPrivateHostWhitelist([]string{"127.0.0.0/8"})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse whitelist: %v", err)
|
||||
}
|
||||
|
||||
dialContext := NewSafeDialContext(&net.Dialer{Timeout: time.Second}, whitelist, nil)
|
||||
conn, err := dialContext(context.Background(), "tcp", net.JoinHostPort("localhost", port))
|
||||
if err != nil {
|
||||
t.Fatalf("expected localhost DNS resolution to succeed with whitelist, got %v", err)
|
||||
}
|
||||
conn.Close()
|
||||
|
||||
select {
|
||||
case <-accepted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("expected localhost listener to accept a connection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPrivateOrRestrictedIP_Table(t *testing.T) {
|
||||
tests := []struct {
|
||||
ip string
|
||||
blocked bool
|
||||
}{
|
||||
{"127.0.0.1", true},
|
||||
{"10.0.0.1", true},
|
||||
{"172.16.0.1", true},
|
||||
{"192.168.1.1", true},
|
||||
{"169.254.169.254", true},
|
||||
{"100.64.0.1", true},
|
||||
{"198.18.0.1", true},
|
||||
{"198.20.0.1", false},
|
||||
{"0.0.0.0", true},
|
||||
{"8.8.8.8", false},
|
||||
{"::1", true},
|
||||
{"::ffff:127.0.0.1", true},
|
||||
{"fc00::1", true},
|
||||
{"2002:7f00:0001::1", true},
|
||||
{"2002:0801:0101::1", false},
|
||||
{"2001:0000:4136:e378:8000:63bf:f5ff:fffe", true},
|
||||
{"2607:f8b0:4004:800::200e", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.ip, func(t *testing.T) {
|
||||
ip := net.ParseIP(tt.ip)
|
||||
if ip == nil {
|
||||
t.Fatalf("failed to parse IP: %s", tt.ip)
|
||||
}
|
||||
got := IsPrivateOrRestrictedIP(ip)
|
||||
if got != tt.blocked {
|
||||
t.Fatalf("IsPrivateOrRestrictedIP(%s) = %v, want %v", tt.ip, got, tt.blocked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFile_DefaultAllowsLoopbackURL(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("local download"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
path := DownloadFile(server.URL, "file.txt", DownloadOptions{
|
||||
LoggerPrefix: "test",
|
||||
Timeout: 5 * time.Second,
|
||||
})
|
||||
if path == "" {
|
||||
t.Fatal("expected default DownloadFile to allow loopback URL")
|
||||
}
|
||||
defer os.Remove(path)
|
||||
}
|
||||
|
||||
func TestDownloadFile_BlockPrivateTargetsBlocksRedirectToLoopback(t *testing.T) {
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("secret"))
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, target.URL, http.StatusFound)
|
||||
}))
|
||||
defer proxy.Close()
|
||||
|
||||
path := DownloadFile("http://example.com/file.txt", "file.txt", DownloadOptions{
|
||||
LoggerPrefix: "test",
|
||||
Timeout: 5 * time.Second,
|
||||
ProxyURL: proxy.URL,
|
||||
BlockPrivateTargets: true,
|
||||
})
|
||||
if path != "" {
|
||||
t.Fatalf("expected safe DownloadFile to block redirect to loopback, got %q", path)
|
||||
}
|
||||
}
|
||||
|
|
@ -64,10 +64,11 @@ func SanitizeFilename(filename string) string {
|
|||
|
||||
// DownloadOptions holds optional parameters for downloading files
|
||||
type DownloadOptions struct {
|
||||
Timeout time.Duration
|
||||
ExtraHeaders map[string]string
|
||||
LoggerPrefix string
|
||||
ProxyURL string
|
||||
Timeout time.Duration
|
||||
ExtraHeaders map[string]string
|
||||
LoggerPrefix string
|
||||
ProxyURL string
|
||||
BlockPrivateTargets bool
|
||||
}
|
||||
|
||||
// DownloadFile downloads a file from URL to a local temp directory.
|
||||
|
|
@ -93,8 +94,45 @@ func DownloadFile(urlStr, filename string, opts DownloadOptions) string {
|
|||
safeName := SanitizeFilename(filename)
|
||||
localPath := filepath.Join(mediaDir, uuid.New().String()[:8]+"_"+safeName)
|
||||
|
||||
// Create HTTP request
|
||||
req, err := http.NewRequest("GET", urlStr, nil)
|
||||
var client *http.Client
|
||||
var err error
|
||||
if opts.BlockPrivateTargets {
|
||||
if err := ValidateSafeHTTPURL(urlStr, nil, nil); err != nil {
|
||||
logger.ErrorCF(opts.LoggerPrefix, "Blocked unsafe download URL", map[string]any{
|
||||
"error": err.Error(),
|
||||
"url": urlStr,
|
||||
})
|
||||
return ""
|
||||
}
|
||||
client, err = CreateSafeHTTPClient(SafeHTTPClientOptions{
|
||||
ProxyURL: opts.ProxyURL,
|
||||
Timeout: opts.Timeout,
|
||||
MaxRedirects: 10,
|
||||
})
|
||||
if err != nil {
|
||||
logger.ErrorCF(opts.LoggerPrefix, "Failed to create safe download client", map[string]any{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return ""
|
||||
}
|
||||
} else {
|
||||
client = &http.Client{Timeout: opts.Timeout}
|
||||
if opts.ProxyURL != "" {
|
||||
proxyURL, parseErr := url.Parse(opts.ProxyURL)
|
||||
if parseErr != nil {
|
||||
logger.ErrorCF(opts.LoggerPrefix, "Invalid proxy URL for download", map[string]any{
|
||||
"error": parseErr.Error(),
|
||||
"proxy": opts.ProxyURL,
|
||||
})
|
||||
return ""
|
||||
}
|
||||
client.Transport = &http.Transport{
|
||||
Proxy: http.ProxyURL(proxyURL),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, urlStr, nil)
|
||||
if err != nil {
|
||||
logger.ErrorCF(opts.LoggerPrefix, "Failed to create download request", map[string]any{
|
||||
"error": err.Error(),
|
||||
|
|
@ -106,21 +144,10 @@ func DownloadFile(urlStr, filename string, opts DownloadOptions) string {
|
|||
for key, value := range opts.ExtraHeaders {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: opts.Timeout}
|
||||
if opts.ProxyURL != "" {
|
||||
proxyURL, parseErr := url.Parse(opts.ProxyURL)
|
||||
if parseErr != nil {
|
||||
logger.ErrorCF(opts.LoggerPrefix, "Invalid proxy URL for download", map[string]any{
|
||||
"error": parseErr.Error(),
|
||||
"proxy": opts.ProxyURL,
|
||||
})
|
||||
return ""
|
||||
}
|
||||
client.Transport = &http.Transport{
|
||||
Proxy: http.ProxyURL(proxyURL),
|
||||
}
|
||||
if opts.BlockPrivateTargets {
|
||||
AllowConfiguredProxyFirstHop(req, client.Transport)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
logger.ErrorCF(opts.LoggerPrefix, "Failed to download file", map[string]any{
|
||||
|
|
|
|||
Loading…
Reference in a new issue