21978860214

Committed 13 Feb 2026 07:45AM UTC coverage: 5.772% (+0.008%) from 5.764%

Build # 21978860214

Build Type

push

github

Committed by chaoyuepan

Commit Message

improve web fetch

Run Details

4 of 24 new or added lines in 10 files covered. (16.67%)

221 existing lines in 4 files now uncovered.

1517 of 26284 relevant lines covered (5.77%)

0.55 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/agent/tools/web.go

package tools

import (
        "bytes"
        "context"
        "encoding/json"
        "fmt"
        "io"
        "net/http"
        "net/url"
        "strings"
        "time"
)

// WebTool Web 工具
type WebTool struct {
        searchAPIKey string
        searchEngine string
        timeout      time.Duration
        client       *http.Client
}

// NewWebTool 创建 Web 工具
func NewWebTool(searchAPIKey, searchEngine string, timeout int) *WebTool {
        var t time.Duration
        if timeout > 0 {
                t = time.Duration(timeout) * time.Second
        } else {
                t = 10 * time.Second
        }

        return &WebTool{
                searchAPIKey: searchAPIKey,
                searchEngine: searchEngine,
                timeout:      t,
                client: &http.Client{
                        Timeout: t,
                },
        }
}

// WebSearch 网络搜索
func (t *WebTool) WebSearch(ctx context.Context, params map[string]interface{}) (string, error) {
        query, ok := params["query"].(string)
        if !ok {
                return "", fmt.Errorf("query parameter is required")
        }

        if t.searchAPIKey == "" {
                return fmt.Sprintf("Search results for: %s\n\n[Warning: Search API Key not configured. Please set search_api_key in config to get actual results.]", query), nil
        }

        // 默认使用 travily
        if t.searchEngine == "travily" || t.searchEngine == "tavily" || t.searchEngine == "" {
                return t.searchTavily(ctx, query)
        }

        if t.searchEngine == "serper" {
                return t.searchSerper(ctx, query)
        }

        if t.searchEngine == "google" {
                return t.searchGoogle(ctx, query)
        }

        return fmt.Sprintf("Search results for: %s\n\n[Warning: Search engine '%s' is not fully implemented. Using mock results.]", query, t.searchEngine), nil
}

func (t *WebTool) searchTavily(ctx context.Context, query string) (string, error) {
        apiURL := "https://api.tavily.com/search"
        maxResults := 5 // Default limit

        requestBody, err := json.Marshal(map[string]any{
                "query":          query,
                "search_depth":   "basic",
                "max_results":    maxResults,
                "include_images": true,
        })
        if err != nil {
                return "", fmt.Errorf("failed to marshal request body: %w", err)
        }

        req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewBuffer(requestBody))
        if err != nil {
                return "", fmt.Errorf("failed to create request: %w", err)
        }

        req.Header.Set("Content-Type", "application/json")
        req.Header.Set("Authorization", "Bearer "+t.searchAPIKey)

        resp, err := t.client.Do(req)
        if err != nil {
                return "", fmt.Errorf("failed to perform Tavily search: %w", err)
        }
        defer resp.Body.Close()

        if resp.StatusCode != http.StatusOK {
                body, _ := io.ReadAll(resp.Body)
                return "", fmt.Errorf("Tavily API returned status %d: %s", resp.StatusCode, string(body))
        }

        var result struct {
                Results []struct {
                        Title   string `json:"title"`
                        URL     string `json:"url"`
                        Content string `json:"content"`
                } `json:"results"`
                Images []string `json:"images"`
        }

        if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
                return "", fmt.Errorf("failed to decode Tavily response: %w", err)
        }

        var sb bytes.Buffer
        for _, item := range result.Results {
                sb.WriteString(fmt.Sprintf("Title: %s\nURL: %s\nContent: %s\n\n", item.Title, item.URL, item.Content))
        }

        if len(result.Images) > 0 {
                sb.WriteString("\nRelevant Images:\n")
                for _, imgURL := range result.Images {
                        sb.WriteString(fmt.Sprintf("- Image URL: %s\n", imgURL))
                }
                sb.WriteString("\n")
        }

        if sb.Len() == 0 {
                return "No results found.", nil
        }

        return sb.String(), nil
}

func (t *WebTool) searchGoogle(ctx context.Context, query string) (string, error) {
        // TODO: 实现 Google Custom Search API 调用
        if t.searchAPIKey == "" {
                return "", fmt.Errorf("google search api key is required")
        }
        return fmt.Sprintf("Google Search results for: %s\n\n1. Example Result (Mock)\n2. Another Result (Mock)", query), nil
}

func (t *WebTool) searchSerper(ctx context.Context, query string) (string, error) {
        url := "https://google.serper.dev/search"
        payload := strings.NewReader(fmt.Sprintf(`{"q": "%s"}`, query))

        req, err := http.NewRequestWithContext(ctx, "POST", url, payload)
        if err != nil {
                return "", err
        }

        req.Header.Add("X-API-KEY", t.searchAPIKey)
        req.Header.Add("Content-Type", "application/json")

        res, err := t.client.Do(req)
        if err != nil {
                return "", err
        }
        defer res.Body.Close()

        if res.StatusCode != http.StatusOK {
                return "", fmt.Errorf("search api returned status: %s", res.Status)
        }

        body, err := io.ReadAll(res.Body)
        if err != nil {
                return "", err
        }

        // 这里简单返回 JSON，实际应该解析并格式化
        return string(body), nil
}

// WebFetch 抓取网页
func (t *WebTool) WebFetch(ctx context.Context, params map[string]interface{}) (string, error) {
        urlStr, ok := params["url"].(string)
        if !ok {
                return "", fmt.Errorf("url parameter is required")
        }

        // 验证 URL
        parsedURL, err := url.Parse(urlStr)
        if err != nil {
                return "", fmt.Errorf("invalid URL: %w", err)
        }

        if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
                return "", fmt.Errorf("only http and https URLs are supported")
        }

        // 创建请求
        req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
        if err != nil {
                return "", fmt.Errorf("failed to create request: %w", err)
        }

        // 设置 User-Agent
        req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; goclaw/1.0)")
        // 设置 Accept header 优先获取 markdown 格式
        req.Header.Set("Accept", "text/markdown, text/html")

        // 发送请求
        resp, err := t.client.Do(req)
        if err != nil {
                return "", fmt.Errorf("failed to fetch URL: %w", err)
        }
        defer resp.Body.Close()

        // 检查状态码
        if resp.StatusCode != http.StatusOK {
                return "", fmt.Errorf("HTTP error: %s", resp.Status)
        }

        // 读取内容
        body, err := io.ReadAll(resp.Body)
        if err != nil {
                return "", fmt.Errorf("failed to read response body: %w", err)
        }

        // 简化实现：返回原始 HTML
        // 实际应该使用 Readability 转换为 Markdown
        return t.htmlToMarkdown(string(body)), nil
}

// htmlToMarkdown 简单的 HTML 到 Markdown 转换
func (t *WebTool) htmlToMarkdown(html string) string {
        // 移除脚本和样式
        html = removeHTMLTags(html, "script")
        html = removeHTMLTags(html, "style")

        // 简单转换
        content := strings.TrimSpace(html)
        if len(content) > 10000 {
                content = content[:10000] + "\n\n... (truncated)"
        }

        return content
}

// removeHTMLTags 移除指定的 HTML 标签
func removeHTMLTags(html, tag string) string {
        // 简化实现
        startTag := "<" + tag
        endTag := "</" + tag + ">"

        result := html
        inTag := false
        var sb strings.Builder

        for i := 0; i < len(result); i++ {
                if i+len(startTag) <= len(result) && result[i:i+len(startTag)] == startTag {
                        inTag = true
                        i += len(startTag) - 1
                        continue
                }

                if inTag && i+len(endTag) <= len(result) && result[i:i+len(endTag)] == endTag {
                        inTag = false
                        i += len(endTag) - 1
                        continue
                }

                if !inTag {
                        sb.WriteByte(result[i])
                }
        }

        return sb.String()
}

// GetTools 获取所有 Web 工具
func (t *WebTool) GetTools() []Tool {
        return []Tool{
                NewBaseTool(
                        "web_search",
                        "Search the web for information",
                        map[string]interface{}{
                                "type": "object",
                                "properties": map[string]interface{}{
                                        "query": map[string]interface{}{
                                                "type":        "string",
                                                "description": "Search query",
                                        },
                                },
                                "required": []string{"query"},
                        },
                        t.WebSearch,
                ),
                NewBaseTool(
                        "web_fetch",
                        "Fetch a web page and convert to markdown",
                        map[string]interface{}{
                                "type": "object",
                                "properties": map[string]interface{}{
                                        "url": map[string]interface{}{
                                                "type":        "string",
                                                "description": "URL to fetch",
                                        },
                                },
                                "required": []string{"url"},
                        },
                        t.WebFetch,
                ),
        }
}

1	package tools
2
3	import (
4	"bytes"
5	"context"
6	"encoding/json"
7	"fmt"
8	"io"
9	"net/http"
10	"net/url"
11	"strings"
12	"time"
13	)
14
15	// WebTool Web 工具
16	type WebTool struct {
17	searchAPIKey string
18	searchEngine string
19	timeout time.Duration
20	client *http.Client
21	}
22
23	// NewWebTool 创建 Web 工具
24	func NewWebTool(searchAPIKey, searchEngine string, timeout int) *WebTool {	×
25	var t time.Duration	×
26	if timeout > 0 {	×
27	t = time.Duration(timeout) * time.Second	×
28	} else {	×
29	t = 10 * time.Second	×
30	}	×
31
32	return &WebTool{	×
33	searchAPIKey: searchAPIKey,	×
34	searchEngine: searchEngine,	×
35	timeout: t,	×
36	client: &http.Client{	×
37	Timeout: t,	×
38	},	×
39	}	×
40	}
41
42	// WebSearch 网络搜索
43	func (t *WebTool) WebSearch(ctx context.Context, params map[string]interface{}) (string, error) {	×
44	query, ok := params["query"].(string)	×
45	if !ok {	×
46	return "", fmt.Errorf("query parameter is required")	×
47	}	×
48
49	if t.searchAPIKey == "" {	×
50	return fmt.Sprintf("Search results for: %s\n\n[Warning: Search API Key not configured. Please set search_api_key in config to get actual results.]", query), nil	×
51	}	×
52
53	// 默认使用 travily
54	if t.searchEngine == "travily" \|\| t.searchEngine == "tavily" \|\| t.searchEngine == "" {	×
55	return t.searchTavily(ctx, query)	×
56	}	×
57
58	if t.searchEngine == "serper" {	×
59	return t.searchSerper(ctx, query)	×
60	}	×
61
62	if t.searchEngine == "google" {	×
63	return t.searchGoogle(ctx, query)	×
64	}	×
65
66	return fmt.Sprintf("Search results for: %s\n\n[Warning: Search engine '%s' is not fully implemented. Using mock results.]", query, t.searchEngine), nil	×
67	}
68
69	func (t *WebTool) searchTavily(ctx context.Context, query string) (string, error) {	×
70	apiURL := "https://api.tavily.com/search"	×
71	maxResults := 5 // Default limit	×
72		×
73	requestBody, err := json.Marshal(map[string]any{	×
74	"query": query,	×
75	"search_depth": "basic",	×
76	"max_results": maxResults,	×
77	"include_images": true,	×
78	})	×
79	if err != nil {	×
80	return "", fmt.Errorf("failed to marshal request body: %w", err)	×
81	}	×
82
83	req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewBuffer(requestBody))	×
84	if err != nil {	×
85	return "", fmt.Errorf("failed to create request: %w", err)	×
86	}	×
87
88	req.Header.Set("Content-Type", "application/json")	×
89	req.Header.Set("Authorization", "Bearer "+t.searchAPIKey)	×
90		×
91	resp, err := t.client.Do(req)	×
92	if err != nil {	×
93	return "", fmt.Errorf("failed to perform Tavily search: %w", err)	×
94	}	×
95	defer resp.Body.Close()	×
96		×
97	if resp.StatusCode != http.StatusOK {	×
98	body, _ := io.ReadAll(resp.Body)	×
99	return "", fmt.Errorf("Tavily API returned status %d: %s", resp.StatusCode, string(body))	×
100	}	×
101
102	var result struct {	×
103	Results []struct {	×
104	Title string `json:"title"`	×
105	URL string `json:"url"`	×
106	Content string `json:"content"`	×
107	} `json:"results"`	×
108	Images []string `json:"images"`	×
109	}	×
110		×
111	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {	×
112	return "", fmt.Errorf("failed to decode Tavily response: %w", err)	×
113	}	×
114
115	var sb bytes.Buffer	×
116	for _, item := range result.Results {	×
117	sb.WriteString(fmt.Sprintf("Title: %s\nURL: %s\nContent: %s\n\n", item.Title, item.URL, item.Content))	×
118	}	×
119
120	if len(result.Images) > 0 {	×
121	sb.WriteString("\nRelevant Images:\n")	×
122	for _, imgURL := range result.Images {	×
123	sb.WriteString(fmt.Sprintf("- Image URL: %s\n", imgURL))	×
124	}	×
125	sb.WriteString("\n")	×
126	}
127
128	if sb.Len() == 0 {	×
129	return "No results found.", nil	×
130	}	×
131
132	return sb.String(), nil	×
133	}
134
135	func (t *WebTool) searchGoogle(ctx context.Context, query string) (string, error) {	×
136	// TODO: 实现 Google Custom Search API 调用	×
137	if t.searchAPIKey == "" {	×
138	return "", fmt.Errorf("google search api key is required")	×
139	}	×
140	return fmt.Sprintf("Google Search results for: %s\n\n1. Example Result (Mock)\n2. Another Result (Mock)", query), nil	×
141	}
142
143	func (t *WebTool) searchSerper(ctx context.Context, query string) (string, error) {	×
144	url := "https://google.serper.dev/search"	×
145	payload := strings.NewReader(fmt.Sprintf(`{"q": "%s"}`, query))	×
146		×
147	req, err := http.NewRequestWithContext(ctx, "POST", url, payload)	×
148	if err != nil {	×
149	return "", err	×
150	}	×
151
152	req.Header.Add("X-API-KEY", t.searchAPIKey)	×
153	req.Header.Add("Content-Type", "application/json")	×
154		×
155	res, err := t.client.Do(req)	×
156	if err != nil {	×
157	return "", err	×
158	}	×
159	defer res.Body.Close()	×
160		×
161	if res.StatusCode != http.StatusOK {	×
162	return "", fmt.Errorf("search api returned status: %s", res.Status)	×
163	}	×
164
165	body, err := io.ReadAll(res.Body)	×
166	if err != nil {	×
167	return "", err	×
168	}	×
169
170	// 这里简单返回 JSON，实际应该解析并格式化
171	return string(body), nil	×
172	}
173
174	// WebFetch 抓取网页
175	func (t *WebTool) WebFetch(ctx context.Context, params map[string]interface{}) (string, error) {	×
176	urlStr, ok := params["url"].(string)	×
177	if !ok {	×
178	return "", fmt.Errorf("url parameter is required")	×
179	}	×
180
181	// 验证 URL
182	parsedURL, err := url.Parse(urlStr)	×
183	if err != nil {	×
184	return "", fmt.Errorf("invalid URL: %w", err)	×
185	}	×
186
187	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {	×
188	return "", fmt.Errorf("only http and https URLs are supported")	×
189	}	×
190
191	// 创建请求
192	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)	×
193	if err != nil {	×
194	return "", fmt.Errorf("failed to create request: %w", err)	×
195	}	×
196
197	// 设置 User-Agent
198	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; goclaw/1.0)")	×
NEW 199	// 设置 Accept header 优先获取 markdown 格式	×
NEW 200	req.Header.Set("Accept", "text/markdown, text/html")	×
201		×
202	// 发送请求	×
203	resp, err := t.client.Do(req)	×
204	if err != nil {	×
205	return "", fmt.Errorf("failed to fetch URL: %w", err)	×
206	}	×
207	defer resp.Body.Close()	×
208		×
209	// 检查状态码	×
210	if resp.StatusCode != http.StatusOK {	×
211	return "", fmt.Errorf("HTTP error: %s", resp.Status)	×
212	}	×
213
214	// 读取内容
215	body, err := io.ReadAll(resp.Body)	×
216	if err != nil {	×
217	return "", fmt.Errorf("failed to read response body: %w", err)	×
218	}	×
219
220	// 简化实现：返回原始 HTML
221	// 实际应该使用 Readability 转换为 Markdown
222	return t.htmlToMarkdown(string(body)), nil	×
223	}
224
225	// htmlToMarkdown 简单的 HTML 到 Markdown 转换
226	func (t *WebTool) htmlToMarkdown(html string) string {	×
227	// 移除脚本和样式	×
228	html = removeHTMLTags(html, "script")	×
229	html = removeHTMLTags(html, "style")	×
230		×
231	// 简单转换	×
232	content := strings.TrimSpace(html)	×
233	if len(content) > 10000 {	×
234	content = content[:10000] + "\n\n... (truncated)"	×
235	}	×
236
237	return content	×
238	}
239
240	// removeHTMLTags 移除指定的 HTML 标签
241	func removeHTMLTags(html, tag string) string {	×
242	// 简化实现	×
243	startTag := "<" + tag	×
244	endTag := "</" + tag + ">"	×
245		×
246	result := html	×
247	inTag := false	×
248	var sb strings.Builder	×
249		×
250	for i := 0; i < len(result); i++ {	×
251	if i+len(startTag) <= len(result) && result[i:i+len(startTag)] == startTag {	×
252	inTag = true	×
253	i += len(startTag) - 1	×
254	continue	×
255	}
256
257	if inTag && i+len(endTag) <= len(result) && result[i:i+len(endTag)] == endTag {	×
258	inTag = false	×
259	i += len(endTag) - 1	×
260	continue	×
261	}
262
263	if !inTag {	×
264	sb.WriteByte(result[i])	×
265	}	×
266	}
267
268	return sb.String()	×
269	}
270
271	// GetTools 获取所有 Web 工具
272	func (t *WebTool) GetTools() []Tool {	×
273	return []Tool{	×
274	NewBaseTool(	×
275	"web_search",	×
276	"Search the web for information",	×
277	map[string]interface{}{	×
278	"type": "object",	×
279	"properties": map[string]interface{}{	×
280	"query": map[string]interface{}{	×
281	"type": "string",	×
282	"description": "Search query",	×
283	},	×
284	},	×
285	"required": []string{"query"},	×
286	},	×
287	t.WebSearch,	×
288	),	×
289	NewBaseTool(	×
290	"web_fetch",	×
291	"Fetch a web page and convert to markdown",	×
292	map[string]interface{}{	×
293	"type": "object",	×
294	"properties": map[string]interface{}{	×
295	"url": map[string]interface{}{	×
296	"type": "string",	×
297	"description": "URL to fetch",	×
298	},	×
299	},	×
300	"required": []string{"url"},	×
301	},	×
302	t.WebFetch,	×
303	),	×
304	}	×
305	}	×

smallnest / goclaw / 21978860214

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous