sscoble/specstream
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add specstream_raw and specstream_seo MCP tools

Browse source
This commit is contained in:
agent-engineering 2026-05-25 22:26:05 -07:00
parent 2d7a770ff9
commit 38c0520629
1 changed files with 467 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

484
cmd/specstream/main.go
View file

@ -12,21 +12,55 @@
// Instead of passing raw web content into the agent's context, the agent
// calls specstream_extract and receives typed, structured data.
// No raw text from untrusted sources ever enters the agent.
//
// Available MCP tools:
// specstream_extract — structured extraction (typed sections, no raw text)
// specstream_raw — raw HTML fetch with injection-pattern redaction
// specstream_seo — SEO-focused metadata extraction
// specstream_info — server metadata
package main
import (
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"log"
"net"
"net/http"
"os"
"regexp"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/sscoble/specstream/pkg/pipeline"
"github.com/sscoble/specstream/pkg/types"
)
var extractPipeline = pipeline.New()
// Restricted HTTP client for fetching URLs (shared across all fetch tools).
var fetchClient = &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 0,
}).DialContext,
DisableKeepAlives: true,
},
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 5 {
return fmt.Errorf("too many redirects")
}
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
return fmt.Errorf("blocked redirect scheme: %s", req.URL.Scheme)
}
return nil
},
}
func main() {
port := os.Getenv("SPECSTREAM_PORT")
if port == "" {
@ -38,7 +72,7 @@ func main() {
addr := "127.0.0.1:" + port
log.Printf("[specstream] listening on %s", addr)
log.Printf("[specstream] MCP tools: specstream_extract, specstream_info")
log.Printf("[specstream] MCP tools: specstream_extract, specstream_raw, specstream_seo, specstream_info")
if err := http.ListenAndServe(addr, mux); err != nil {
log.Fatalf("[specstream] %v", err)
}
@ -88,6 +122,10 @@ func jsonRPCHandler(w http.ResponseWriter, r *http.Request) {
switch req.Method {
case "specstream_extract":
result, rpcErr = handleExtract(req.Params)
case "specstream_raw":
result, rpcErr = handleRaw(req.Params)
case "specstream_seo":
result, rpcErr = handleSEO(req.Params)
case "specstream_info":
result, rpcErr = handleInfo(req.Params)
default:
@ -102,19 +140,54 @@ func jsonRPCHandler(w http.ResponseWriter, r *http.Request) {
writeResult(w, req.ID, result)
}
// ── Tool Handlers ──────────────────────────
// ── Shared URL Fetch ───────────────────────
type urlParam struct {
URL string `json:"url"`
MaxKb int `json:"max_kb"` // max response size in KB (0 = 1024)
}
func fetchRaw(url string, maxKb int) ([]byte, int, error) {
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
return nil, 0, fmt.Errorf("blocked scheme: only http/https allowed")
}
maxBytes := 1 << 20 // 1MB default
if maxKb > 0 && maxKb <= 5120 {
maxBytes = maxKb * 1024
}
resp, err := fetchClient.Get(url)
if err != nil {
return nil, 0, fmt.Errorf("http get: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return nil, 0, fmt.Errorf("http %d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, int64(maxBytes)))
if err != nil {
return nil, 0, fmt.Errorf("read body: %w", err)
}
return body, resp.StatusCode, nil
}
// ── Tool: specstream_extract ─────────────────
type extractParams struct {
URL string `json:"url"`
Content string `json:"content"`
Filename string `json:"filename"`
MaxSize int `json:"max_size"`
ExtractSections bool `json:"extract_sections"`
ExtractCode bool `json:"extract_code"`
ExtractLinks bool `json:"extract_links"`
ExtractImages bool `json:"extract_images"`
ExtractTables bool `json:"extract_tables"`
ExtractLists bool `json:"extract_lists"`
URL string `json:"url"`
Content string `json:"content"`
Filename string `json:"filename"`
MaxSize int `json:"max_size"`
ExtractSections bool `json:"extract_sections"`
ExtractCode bool `json:"extract_code"`
ExtractLinks bool `json:"extract_links"`
ExtractImages bool `json:"extract_images"`
ExtractTables bool `json:"extract_tables"`
ExtractLists bool `json:"extract_lists"`
}
func handleExtract(raw json.RawMessage) (interface{}, *rpcError) {
@ -138,7 +211,6 @@ func handleExtract(raw json.RawMessage) (interface{}, *rpcError) {
Content: params.Content,
Filename: params.Filename,
MaxSize: params.MaxSize,
// Default to extracting everything unless explicitly restricted
ExtractSections: !params.ExtractSections || params.ExtractSections,
ExtractCode: !params.ExtractCode || params.ExtractCode,
ExtractLinks: !params.ExtractLinks || params.ExtractLinks,
@ -165,16 +237,374 @@ func handleExtract(raw json.RawMessage) (interface{}, *rpcError) {
return result, nil
}
// ── Tool: specstream_raw ─────────────────────
//
// Fetches raw HTML from a URL. Runs the injection-pattern safety filter
// (redacts system-override, ignore-instructions, etc.) but preserves
// the full HTML structure. Intended for SEO analysis, site inspection,
// and working on our own sites' markup.
//
// This is an EXEMPTION from the "no raw text" policy — only because
// the HTML is from sites we control or need to diagnose. The injection
// filter still protects agents from accidentally executing instructions
// embedded in third-party content.
type rawParams struct {
URL string `json:"url"`
MaxKb int `json:"max_kb"`
}
type rawResult struct {
URL string `json:"url"`
Status int `json:"status"`
HTML string `json:"html"`
SizeBytes int `json:"size_bytes"`
SHA256 string `json:"sha256"`
InjectionHits map[string]int `json:"injection_hits"`
InjectionRedacted bool `json:"injection_redacted"`
ContentType string `json:"content_type"`
}
// Injection patterns for raw-HTML redaction.
var rawInjectionPatterns = []struct {
name string
pattern *regexp.Regexp
}{
{"ignore-previous", regexp.MustCompile(`(?i)(ignore|forget|disregard|override)\s+(all\s+)?(previous|prior|above)\s+(instructions|commands|directions|prompts?)`)},
{"new-instructions", regexp.MustCompile(`(?i)(now|instead|from\s+now\s+on)\s+(you\s+are|you'?re|act\s+as|behave\s+as|pretend)\s+`)},
{"system-override", regexp.MustCompile(`(?i)(system\s+prompt|system\s+message|SYSTEM:|system_message)\s*[:=]`)},
{"role-switch", regexp.MustCompile(`(?i)^\s*(user|assistant|system)\s*[:]\s*`)},
{"dangerous-command", regexp.MustCompile(`(?i)(rm\s+-(rf|r)\s+/|format\s+(drive|disk|volume)|dd\s+if=)\s*`)},
}
func handleRaw(raw json.RawMessage) (interface{}, *rpcError) {
var params rawParams
if len(raw) > 0 {
if err := json.Unmarshal(raw, &params); err != nil {
return nil, &rpcError{Code: -32602, Message: "invalid params: " + err.Error()}
}
}
if params.URL == "" {
return nil, &rpcError{Code: -32602, Message: "'url' is required"}
}
maxKb := params.MaxKb
if maxKb <= 0 {
maxKb = 1024
}
body, status, err := fetchRaw(params.URL, maxKb)
if err != nil {
return nil, &rpcError{Code: -32000, Message: "fetch failed: " + err.Error()}
}
htmlStr := string(body)
hash := sha256.Sum256(body)
fingerprint := fmt.Sprintf("%x", hash)
// Detect content type from first 512 bytes
ct := http.DetectContentType(body)
// Normalize text/html detection
if strings.Contains(ct, "html") || strings.Contains(htmlStr[:min(len(htmlStr), 200)], "<!DOCTYPE html") || strings.Contains(htmlStr[:min(len(htmlStr), 200)], "<html") {
ct = "text/html"
}
// Run injection filter on the raw HTML
hits := make(map[string]int)
redacted := false
for _, ip := range rawInjectionPatterns {
matches := ip.pattern.FindAllString(htmlStr, -1)
if len(matches) > 0 {
hits[ip.name] = len(matches)
htmlStr = ip.pattern.ReplaceAllString(htmlStr, "[SPECSTREAM_REDACTED]")
redacted = true
}
}
return rawResult{
URL: params.URL,
Status: status,
HTML: htmlStr,
SizeBytes: len(htmlStr),
SHA256: fingerprint,
InjectionHits: hits,
InjectionRedacted: redacted,
ContentType: ct,
}, nil
}
// ── Tool: specstream_seo ─────────────────────
//
// Extracts SEO-relevant metadata from a URL. Returns structured data
// suitable for audit, optimization, and analysis. Does NOT return raw HTML.
//
// Fields: title, description, canonical, h1-h6 structure, meta robots,
// og tags, twitter cards, structured data (JSON-LD), image alt counts,
// link counts, word count, headings summary.
type seoParams struct {
URL string `json:"url"`
MaxKb int `json:"max_kb"`
}
type seoResult struct {
URL string `json:"url"`
Status int `json:"status"`
Title string `json:"title"`
Description string `json:"description"`
Canonical string `json:"canonical"`
Robots string `json:"robots"`
Language string `json:"language"`
Charset string `json:"charset"`
Viewport string `json:"viewport"`
// Open Graph
OGTitle string `json:"og_title,omitempty"`
OGDescription string `json:"og_description,omitempty"`
OGImage string `json:"og_image,omitempty"`
OGURL string `json:"og_url,omitempty"`
OGType string `json:"og_type,omitempty"`
OGSiteName string `json:"og_site_name,omitempty"`
// Twitter Cards
TWCard string `json:"twitter_card,omitempty"`
TWSite string `json:"twitter_site,omitempty"`
TWTitle string `json:"twitter_title,omitempty"`
TWDescription string `json:"twitter_description,omitempty"`
TWImage string `json:"twitter_image,omitempty"`
// Heading Structure
Headings []headingInfo `json:"headings"`
HeadingsTotal int `json:"headings_total"`
H1Count int `json:"h1_count"`
H1Texts []string `json:"h1_texts,omitempty"`
// Content Stats
WordCount int `json:"word_count"`
CharCount int `json:"char_count"`
LinkCount int `json:"link_count"`
InternalLinks int `json:"internal_links"`
ExternalLinks int `json:"external_links"`
ImageCount int `json:"image_count"`
ImagesWithAlt int `json:"images_with_alt"`
ImagesWithoutAlt int `json:"images_without_alt"`
// Structured Data
StructuredData []json.RawMessage `json:"structured_data,omitempty"`
// Diagnostics
Favicon string `json:"favicon,omitempty"`
HasSitemap bool `json:"has_sitemap"`
MetaTags []metaTag `json:"meta_tags,omitempty"`
}
type headingInfo struct {
Level int `json:"level"`
Text string `json:"text"`
}
type metaTag struct {
Name string `json:"name,omitempty"`
Property string `json:"property,omitempty"`
Content string `json:"content"`
}
func handleSEO(raw json.RawMessage) (interface{}, *rpcError) {
var params seoParams
if len(raw) > 0 {
if err := json.Unmarshal(raw, &params); err != nil {
return nil, &rpcError{Code: -32602, Message: "invalid params: " + err.Error()}
}
}
if params.URL == "" {
return nil, &rpcError{Code: -32602, Message: "'url' is required"}
}
maxKb := params.MaxKb
if maxKb <= 0 {
maxKb = 1024
}
body, status, err := fetchRaw(params.URL, maxKb)
if err != nil {
return nil, &rpcError{Code: -32000, Message: "fetch failed: " + err.Error()}
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(body)))
if err != nil {
return nil, &rpcError{Code: -32000, Message: "parse failed: " + err.Error()}
}
result := seoResult{
URL: params.URL,
Status: status,
}
// ── Basic meta ──
result.Title = strings.TrimSpace(doc.Find("title").First().Text())
if desc, exists := doc.Find("meta[name=description]").Attr("content"); exists {
result.Description = strings.TrimSpace(desc)
}
if canonical, exists := doc.Find("link[rel=canonical]").Attr("href"); exists {
result.Canonical = canonical
}
if robots, exists := doc.Find("meta[name=robots]").Attr("content"); exists {
result.Robots = robots
}
if lang, exists := doc.Find("html").Attr("lang"); exists {
result.Language = lang
}
if charset, exists := doc.Find("meta[charset]").Attr("charset"); exists {
result.Charset = charset
} else if ct, exists := doc.Find("meta[http-equiv=Content-Type]").Attr("content"); exists {
if idx := strings.Index(strings.ToLower(ct), "charset="); idx >= 0 {
result.Charset = strings.TrimSpace(ct[idx+8:])
}
}
if vp, exists := doc.Find("meta[name=viewport]").Attr("content"); exists {
result.Viewport = strings.TrimSpace(vp)
}
// ── All meta tags ──
doc.Find("meta").Each(func(i int, s *goquery.Selection) {
mt := metaTag{}
if n, ok := s.Attr("name"); ok && n != "" {
mt.Name = n
}
if p, ok := s.Attr("property"); ok && p != "" {
mt.Property = p
}
if c, ok := s.Attr("content"); ok {
mt.Content = strings.TrimSpace(c)
}
if mt.Name != "" || mt.Property != "" {
result.MetaTags = append(result.MetaTags, mt)
}
})
// ── Open Graph ──
if og, exists := doc.Find("meta[property=og:title]").Attr("content"); exists {
result.OGTitle = og
}
if og, exists := doc.Find("meta[property=og:description]").Attr("content"); exists {
result.OGDescription = og
}
if og, exists := doc.Find("meta[property=og:image]").Attr("content"); exists {
result.OGImage = og
}
if og, exists := doc.Find("meta[property=og:url]").Attr("content"); exists {
result.OGURL = og
}
if og, exists := doc.Find("meta[property=og:type]").Attr("content"); exists {
result.OGType = og
}
if og, exists := doc.Find("meta[property=og:site_name]").Attr("content"); exists {
result.OGSiteName = og
}
// ── Twitter Cards ──
if tc, exists := doc.Find("meta[name=twitter:card]").Attr("content"); exists {
result.TWCard = tc
}
if tc, exists := doc.Find("meta[name=twitter:site]").Attr("content"); exists {
result.TWSite = tc
}
if tc, exists := doc.Find("meta[name=twitter:title]").Attr("content"); exists {
result.TWTitle = tc
}
if tc, exists := doc.Find("meta[name=twitter:description]").Attr("content"); exists {
result.TWDescription = tc
}
if tc, exists := doc.Find("meta[name=twitter:image]").Attr("content"); exists {
result.TWImage = tc
}
// ── Favicon ──
if icon, exists := doc.Find("link[rel=icon]").Attr("href"); exists {
result.Favicon = icon
} else if shortcut, exists := doc.Find("link[rel='shortcut icon']").Attr("href"); exists {
result.Favicon = shortcut
}
// ── Heading structure ──
hostname := extractHostname(params.URL)
doc.Find("h1, h2, h3, h4, h5, h6").Each(func(i int, s *goquery.Selection) {
tag := goquery.NodeName(s)
level := int(tag[1] - '0')
text := strings.TrimSpace(s.Text())
if text == "" {
return
}
result.Headings = append(result.Headings, headingInfo{Level: level, Text: text})
if level == 1 {
result.H1Count++
result.H1Texts = append(result.H1Texts, text)
}
})
result.HeadingsTotal = len(result.Headings)
// ── Content stats (from body text) ──
bodyText := strings.TrimSpace(doc.Find("body").Text())
result.CharCount = len(bodyText)
words := strings.Fields(bodyText)
result.WordCount = len(words)
// ── Links ──
doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
href, _ := s.Attr("href")
if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(href, "javascript:") {
return
}
result.LinkCount++
if strings.HasPrefix(href, "/") || strings.HasPrefix(href, "#") || strings.Contains(href, hostname) {
result.InternalLinks++
} else if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
result.ExternalLinks++
}
})
// ── Images ──
doc.Find("img").Each(func(i int, s *goquery.Selection) {
result.ImageCount++
if alt, exists := s.Attr("alt"); exists && strings.TrimSpace(alt) != "" {
result.ImagesWithAlt++
} else {
result.ImagesWithoutAlt++
}
})
// ── Structured Data (JSON-LD) ──
doc.Find("script[type='application/ld+json']").Each(func(i int, s *goquery.Selection) {
rawJSON := strings.TrimSpace(s.Text())
if rawJSON == "" {
return
}
// Try to parse as JSON to validate
var parsed json.RawMessage
if err := json.Unmarshal([]byte(rawJSON), &parsed); err == nil {
result.StructuredData = append(result.StructuredData, parsed)
}
})
// ── Sitemap hint ──
result.HasSitemap = doc.Find("link[rel=sitemap]").Length() > 0
return result, nil
}
// ── Tool: specstream_info ─────────────────────
type infoResult struct {
Version string `json:"version"`
Version string `json:"version"`
Extractors []string `json:"extractors"`
Filters []string `json:"filters"`
MaxSize int64 `json:"max_size_bytes"`
Schema string `json:"schema"`
Schema string `json:"schema"`
}
func handleInfo(_ json.RawMessage) (interface{}, *rpcError) {
// Get filter names
filterNames := []string{
"strip_null_bytes",
"strip_control_chars",
@ -191,10 +621,30 @@ func handleInfo(_ json.RawMessage) (interface{}, *rpcError) {
},
Filters: filterNames,
MaxSize: 1 << 20,
Schema: "ExtractedContent with typed Sections, CodeBlocks, Links, Images, Lists, Tables — no raw text fields",
Schema: "ExtractedContent with typed Sections, CodeBlocks, Links, Images, Lists, Tables — no raw text fields. Additional tools: specstream_raw (injection-filtered HTML), specstream_seo (SEO metadata)",
}, nil
}
// ── Helpers ─────────────────────────────────
func extractHostname(rawURL string) string {
// Simple hostname extraction from URL
// Strip protocol
u := rawURL
if idx := strings.Index(u, "://"); idx >= 0 {
u = u[idx+3:]
}
// Strip path/query
if idx := strings.Index(u, "/"); idx >= 0 {
u = u[:idx]
}
// Strip port
if idx := strings.Index(u, ":"); idx >= 0 {
u = u[:idx]
}
return u
}
// ── Response Helpers ───────────────────────
func writeResult(w http.ResponseWriter, id interface{}, result interface{}) {