diff --git a/cmd/specstream/main.go b/cmd/specstream/main.go index b79104b..829ec63 100644 --- a/cmd/specstream/main.go +++ b/cmd/specstream/main.go @@ -12,21 +12,55 @@ // Instead of passing raw web content into the agent's context, the agent // calls specstream_extract and receives typed, structured data. // No raw text from untrusted sources ever enters the agent. +// +// Available MCP tools: +// specstream_extract — structured extraction (typed sections, no raw text) +// specstream_raw — raw HTML fetch with injection-pattern redaction +// specstream_seo — SEO-focused metadata extraction +// specstream_info — server metadata package main import ( + "crypto/sha256" "encoding/json" "fmt" + "io" "log" + "net" "net/http" "os" + "regexp" + "strings" + "time" + "github.com/PuerkitoBio/goquery" "github.com/sscoble/specstream/pkg/pipeline" "github.com/sscoble/specstream/pkg/types" ) var extractPipeline = pipeline.New() +// Restricted HTTP client for fetching URLs (shared across all fetch tools). +var fetchClient = &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 0, + }).DialContext, + DisableKeepAlives: true, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return fmt.Errorf("too many redirects") + } + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + return fmt.Errorf("blocked redirect scheme: %s", req.URL.Scheme) + } + return nil + }, +} + func main() { port := os.Getenv("SPECSTREAM_PORT") if port == "" { @@ -38,7 +72,7 @@ func main() { addr := "127.0.0.1:" + port log.Printf("[specstream] listening on %s", addr) - log.Printf("[specstream] MCP tools: specstream_extract, specstream_info") + log.Printf("[specstream] MCP tools: specstream_extract, specstream_raw, specstream_seo, specstream_info") if err := http.ListenAndServe(addr, mux); err != nil { log.Fatalf("[specstream] %v", err) } @@ -88,6 +122,10 @@ func jsonRPCHandler(w http.ResponseWriter, r *http.Request) { switch req.Method { case "specstream_extract": result, rpcErr = handleExtract(req.Params) + case "specstream_raw": + result, rpcErr = handleRaw(req.Params) + case "specstream_seo": + result, rpcErr = handleSEO(req.Params) case "specstream_info": result, rpcErr = handleInfo(req.Params) default: @@ -102,19 +140,54 @@ func jsonRPCHandler(w http.ResponseWriter, r *http.Request) { writeResult(w, req.ID, result) } -// ── Tool Handlers ────────────────────────── +// ── Shared URL Fetch ─────────────────────── + +type urlParam struct { + URL string `json:"url"` + MaxKb int `json:"max_kb"` // max response size in KB (0 = 1024) +} + +func fetchRaw(url string, maxKb int) ([]byte, int, error) { + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + return nil, 0, fmt.Errorf("blocked scheme: only http/https allowed") + } + + maxBytes := 1 << 20 // 1MB default + if maxKb > 0 && maxKb <= 5120 { + maxBytes = maxKb * 1024 + } + + resp, err := fetchClient.Get(url) + if err != nil { + return nil, 0, fmt.Errorf("http get: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + return nil, 0, fmt.Errorf("http %d", resp.StatusCode) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, int64(maxBytes))) + if err != nil { + return nil, 0, fmt.Errorf("read body: %w", err) + } + + return body, resp.StatusCode, nil +} + +// ── Tool: specstream_extract ───────────────── type extractParams struct { - URL string `json:"url"` - Content string `json:"content"` - Filename string `json:"filename"` - MaxSize int `json:"max_size"` - ExtractSections bool `json:"extract_sections"` - ExtractCode bool `json:"extract_code"` - ExtractLinks bool `json:"extract_links"` - ExtractImages bool `json:"extract_images"` - ExtractTables bool `json:"extract_tables"` - ExtractLists bool `json:"extract_lists"` + URL string `json:"url"` + Content string `json:"content"` + Filename string `json:"filename"` + MaxSize int `json:"max_size"` + ExtractSections bool `json:"extract_sections"` + ExtractCode bool `json:"extract_code"` + ExtractLinks bool `json:"extract_links"` + ExtractImages bool `json:"extract_images"` + ExtractTables bool `json:"extract_tables"` + ExtractLists bool `json:"extract_lists"` } func handleExtract(raw json.RawMessage) (interface{}, *rpcError) { @@ -138,7 +211,6 @@ func handleExtract(raw json.RawMessage) (interface{}, *rpcError) { Content: params.Content, Filename: params.Filename, MaxSize: params.MaxSize, - // Default to extracting everything unless explicitly restricted ExtractSections: !params.ExtractSections || params.ExtractSections, ExtractCode: !params.ExtractCode || params.ExtractCode, ExtractLinks: !params.ExtractLinks || params.ExtractLinks, @@ -165,16 +237,374 @@ func handleExtract(raw json.RawMessage) (interface{}, *rpcError) { return result, nil } +// ── Tool: specstream_raw ───────────────────── +// +// Fetches raw HTML from a URL. Runs the injection-pattern safety filter +// (redacts system-override, ignore-instructions, etc.) but preserves +// the full HTML structure. Intended for SEO analysis, site inspection, +// and working on our own sites' markup. +// +// This is an EXEMPTION from the "no raw text" policy — only because +// the HTML is from sites we control or need to diagnose. The injection +// filter still protects agents from accidentally executing instructions +// embedded in third-party content. + +type rawParams struct { + URL string `json:"url"` + MaxKb int `json:"max_kb"` +} + +type rawResult struct { + URL string `json:"url"` + Status int `json:"status"` + HTML string `json:"html"` + SizeBytes int `json:"size_bytes"` + SHA256 string `json:"sha256"` + InjectionHits map[string]int `json:"injection_hits"` + InjectionRedacted bool `json:"injection_redacted"` + ContentType string `json:"content_type"` +} + +// Injection patterns for raw-HTML redaction. +var rawInjectionPatterns = []struct { + name string + pattern *regexp.Regexp +}{ + {"ignore-previous", regexp.MustCompile(`(?i)(ignore|forget|disregard|override)\s+(all\s+)?(previous|prior|above)\s+(instructions|commands|directions|prompts?)`)}, + {"new-instructions", regexp.MustCompile(`(?i)(now|instead|from\s+now\s+on)\s+(you\s+are|you'?re|act\s+as|behave\s+as|pretend)\s+`)}, + {"system-override", regexp.MustCompile(`(?i)(system\s+prompt|system\s+message|SYSTEM:|system_message)\s*[:=]`)}, + {"role-switch", regexp.MustCompile(`(?i)^\s*(user|assistant|system)\s*[:]\s*`)}, + {"dangerous-command", regexp.MustCompile(`(?i)(rm\s+-(rf|r)\s+/|format\s+(drive|disk|volume)|dd\s+if=)\s*`)}, +} + +func handleRaw(raw json.RawMessage) (interface{}, *rpcError) { + var params rawParams + if len(raw) > 0 { + if err := json.Unmarshal(raw, ¶ms); err != nil { + return nil, &rpcError{Code: -32602, Message: "invalid params: " + err.Error()} + } + } + + if params.URL == "" { + return nil, &rpcError{Code: -32602, Message: "'url' is required"} + } + + maxKb := params.MaxKb + if maxKb <= 0 { + maxKb = 1024 + } + + body, status, err := fetchRaw(params.URL, maxKb) + if err != nil { + return nil, &rpcError{Code: -32000, Message: "fetch failed: " + err.Error()} + } + + htmlStr := string(body) + hash := sha256.Sum256(body) + fingerprint := fmt.Sprintf("%x", hash) + + // Detect content type from first 512 bytes + ct := http.DetectContentType(body) + // Normalize text/html detection + if strings.Contains(ct, "html") || strings.Contains(htmlStr[:min(len(htmlStr), 200)], " 0 { + hits[ip.name] = len(matches) + htmlStr = ip.pattern.ReplaceAllString(htmlStr, "[SPECSTREAM_REDACTED]") + redacted = true + } + } + + return rawResult{ + URL: params.URL, + Status: status, + HTML: htmlStr, + SizeBytes: len(htmlStr), + SHA256: fingerprint, + InjectionHits: hits, + InjectionRedacted: redacted, + ContentType: ct, + }, nil +} + +// ── Tool: specstream_seo ───────────────────── +// +// Extracts SEO-relevant metadata from a URL. Returns structured data +// suitable for audit, optimization, and analysis. Does NOT return raw HTML. +// +// Fields: title, description, canonical, h1-h6 structure, meta robots, +// og tags, twitter cards, structured data (JSON-LD), image alt counts, +// link counts, word count, headings summary. + +type seoParams struct { + URL string `json:"url"` + MaxKb int `json:"max_kb"` +} + +type seoResult struct { + URL string `json:"url"` + Status int `json:"status"` + Title string `json:"title"` + Description string `json:"description"` + Canonical string `json:"canonical"` + Robots string `json:"robots"` + Language string `json:"language"` + Charset string `json:"charset"` + Viewport string `json:"viewport"` + + // Open Graph + OGTitle string `json:"og_title,omitempty"` + OGDescription string `json:"og_description,omitempty"` + OGImage string `json:"og_image,omitempty"` + OGURL string `json:"og_url,omitempty"` + OGType string `json:"og_type,omitempty"` + OGSiteName string `json:"og_site_name,omitempty"` + + // Twitter Cards + TWCard string `json:"twitter_card,omitempty"` + TWSite string `json:"twitter_site,omitempty"` + TWTitle string `json:"twitter_title,omitempty"` + TWDescription string `json:"twitter_description,omitempty"` + TWImage string `json:"twitter_image,omitempty"` + + // Heading Structure + Headings []headingInfo `json:"headings"` + HeadingsTotal int `json:"headings_total"` + H1Count int `json:"h1_count"` + H1Texts []string `json:"h1_texts,omitempty"` + + // Content Stats + WordCount int `json:"word_count"` + CharCount int `json:"char_count"` + LinkCount int `json:"link_count"` + InternalLinks int `json:"internal_links"` + ExternalLinks int `json:"external_links"` + ImageCount int `json:"image_count"` + ImagesWithAlt int `json:"images_with_alt"` + ImagesWithoutAlt int `json:"images_without_alt"` + + // Structured Data + StructuredData []json.RawMessage `json:"structured_data,omitempty"` + + // Diagnostics + Favicon string `json:"favicon,omitempty"` + HasSitemap bool `json:"has_sitemap"` + MetaTags []metaTag `json:"meta_tags,omitempty"` +} + +type headingInfo struct { + Level int `json:"level"` + Text string `json:"text"` +} + +type metaTag struct { + Name string `json:"name,omitempty"` + Property string `json:"property,omitempty"` + Content string `json:"content"` +} + +func handleSEO(raw json.RawMessage) (interface{}, *rpcError) { + var params seoParams + if len(raw) > 0 { + if err := json.Unmarshal(raw, ¶ms); err != nil { + return nil, &rpcError{Code: -32602, Message: "invalid params: " + err.Error()} + } + } + + if params.URL == "" { + return nil, &rpcError{Code: -32602, Message: "'url' is required"} + } + + maxKb := params.MaxKb + if maxKb <= 0 { + maxKb = 1024 + } + + body, status, err := fetchRaw(params.URL, maxKb) + if err != nil { + return nil, &rpcError{Code: -32000, Message: "fetch failed: " + err.Error()} + } + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(body))) + if err != nil { + return nil, &rpcError{Code: -32000, Message: "parse failed: " + err.Error()} + } + + result := seoResult{ + URL: params.URL, + Status: status, + } + + // ── Basic meta ── + result.Title = strings.TrimSpace(doc.Find("title").First().Text()) + if desc, exists := doc.Find("meta[name=description]").Attr("content"); exists { + result.Description = strings.TrimSpace(desc) + } + if canonical, exists := doc.Find("link[rel=canonical]").Attr("href"); exists { + result.Canonical = canonical + } + if robots, exists := doc.Find("meta[name=robots]").Attr("content"); exists { + result.Robots = robots + } + if lang, exists := doc.Find("html").Attr("lang"); exists { + result.Language = lang + } + if charset, exists := doc.Find("meta[charset]").Attr("charset"); exists { + result.Charset = charset + } else if ct, exists := doc.Find("meta[http-equiv=Content-Type]").Attr("content"); exists { + if idx := strings.Index(strings.ToLower(ct), "charset="); idx >= 0 { + result.Charset = strings.TrimSpace(ct[idx+8:]) + } + } + if vp, exists := doc.Find("meta[name=viewport]").Attr("content"); exists { + result.Viewport = strings.TrimSpace(vp) + } + + // ── All meta tags ── + doc.Find("meta").Each(func(i int, s *goquery.Selection) { + mt := metaTag{} + if n, ok := s.Attr("name"); ok && n != "" { + mt.Name = n + } + if p, ok := s.Attr("property"); ok && p != "" { + mt.Property = p + } + if c, ok := s.Attr("content"); ok { + mt.Content = strings.TrimSpace(c) + } + if mt.Name != "" || mt.Property != "" { + result.MetaTags = append(result.MetaTags, mt) + } + }) + + // ── Open Graph ── + if og, exists := doc.Find("meta[property=og:title]").Attr("content"); exists { + result.OGTitle = og + } + if og, exists := doc.Find("meta[property=og:description]").Attr("content"); exists { + result.OGDescription = og + } + if og, exists := doc.Find("meta[property=og:image]").Attr("content"); exists { + result.OGImage = og + } + if og, exists := doc.Find("meta[property=og:url]").Attr("content"); exists { + result.OGURL = og + } + if og, exists := doc.Find("meta[property=og:type]").Attr("content"); exists { + result.OGType = og + } + if og, exists := doc.Find("meta[property=og:site_name]").Attr("content"); exists { + result.OGSiteName = og + } + + // ── Twitter Cards ── + if tc, exists := doc.Find("meta[name=twitter:card]").Attr("content"); exists { + result.TWCard = tc + } + if tc, exists := doc.Find("meta[name=twitter:site]").Attr("content"); exists { + result.TWSite = tc + } + if tc, exists := doc.Find("meta[name=twitter:title]").Attr("content"); exists { + result.TWTitle = tc + } + if tc, exists := doc.Find("meta[name=twitter:description]").Attr("content"); exists { + result.TWDescription = tc + } + if tc, exists := doc.Find("meta[name=twitter:image]").Attr("content"); exists { + result.TWImage = tc + } + + // ── Favicon ── + if icon, exists := doc.Find("link[rel=icon]").Attr("href"); exists { + result.Favicon = icon + } else if shortcut, exists := doc.Find("link[rel='shortcut icon']").Attr("href"); exists { + result.Favicon = shortcut + } + + // ── Heading structure ── + hostname := extractHostname(params.URL) + doc.Find("h1, h2, h3, h4, h5, h6").Each(func(i int, s *goquery.Selection) { + tag := goquery.NodeName(s) + level := int(tag[1] - '0') + text := strings.TrimSpace(s.Text()) + if text == "" { + return + } + result.Headings = append(result.Headings, headingInfo{Level: level, Text: text}) + if level == 1 { + result.H1Count++ + result.H1Texts = append(result.H1Texts, text) + } + }) + result.HeadingsTotal = len(result.Headings) + + // ── Content stats (from body text) ── + bodyText := strings.TrimSpace(doc.Find("body").Text()) + result.CharCount = len(bodyText) + words := strings.Fields(bodyText) + result.WordCount = len(words) + + // ── Links ── + doc.Find("a[href]").Each(func(i int, s *goquery.Selection) { + href, _ := s.Attr("href") + if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(href, "javascript:") { + return + } + result.LinkCount++ + if strings.HasPrefix(href, "/") || strings.HasPrefix(href, "#") || strings.Contains(href, hostname) { + result.InternalLinks++ + } else if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") { + result.ExternalLinks++ + } + }) + + // ── Images ── + doc.Find("img").Each(func(i int, s *goquery.Selection) { + result.ImageCount++ + if alt, exists := s.Attr("alt"); exists && strings.TrimSpace(alt) != "" { + result.ImagesWithAlt++ + } else { + result.ImagesWithoutAlt++ + } + }) + + // ── Structured Data (JSON-LD) ── + doc.Find("script[type='application/ld+json']").Each(func(i int, s *goquery.Selection) { + rawJSON := strings.TrimSpace(s.Text()) + if rawJSON == "" { + return + } + // Try to parse as JSON to validate + var parsed json.RawMessage + if err := json.Unmarshal([]byte(rawJSON), &parsed); err == nil { + result.StructuredData = append(result.StructuredData, parsed) + } + }) + + // ── Sitemap hint ── + result.HasSitemap = doc.Find("link[rel=sitemap]").Length() > 0 + + return result, nil +} + +// ── Tool: specstream_info ───────────────────── + type infoResult struct { - Version string `json:"version"` + Version string `json:"version"` Extractors []string `json:"extractors"` Filters []string `json:"filters"` MaxSize int64 `json:"max_size_bytes"` - Schema string `json:"schema"` + Schema string `json:"schema"` } func handleInfo(_ json.RawMessage) (interface{}, *rpcError) { - // Get filter names filterNames := []string{ "strip_null_bytes", "strip_control_chars", @@ -191,10 +621,30 @@ func handleInfo(_ json.RawMessage) (interface{}, *rpcError) { }, Filters: filterNames, MaxSize: 1 << 20, - Schema: "ExtractedContent with typed Sections, CodeBlocks, Links, Images, Lists, Tables — no raw text fields", + Schema: "ExtractedContent with typed Sections, CodeBlocks, Links, Images, Lists, Tables — no raw text fields. Additional tools: specstream_raw (injection-filtered HTML), specstream_seo (SEO metadata)", }, nil } +// ── Helpers ───────────────────────────────── + +func extractHostname(rawURL string) string { + // Simple hostname extraction from URL + // Strip protocol + u := rawURL + if idx := strings.Index(u, "://"); idx >= 0 { + u = u[idx+3:] + } + // Strip path/query + if idx := strings.Index(u, "/"); idx >= 0 { + u = u[:idx] + } + // Strip port + if idx := strings.Index(u, ":"); idx >= 0 { + u = u[:idx] + } + return u +} + // ── Response Helpers ─────────────────────── func writeResult(w http.ResponseWriter, id interface{}, result interface{}) {