From 2d7a770ff9a518a65da93c4649aa440ad2748dda Mon Sep 17 00:00:00 2001 From: agent-engineering Date: Mon, 25 May 2026 21:22:10 -0700 Subject: [PATCH] Add specstream MCP server (main.go) and fix .gitignore --- .gitignore | 3 +- cmd/specstream/main.go | 219 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 cmd/specstream/main.go diff --git a/.gitignore b/.gitignore index 02281ac..0372d79 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -specstream +# The compiled binary +/specstream /tmp/ *.test *.out diff --git a/cmd/specstream/main.go b/cmd/specstream/main.go new file mode 100644 index 0000000..b79104b --- /dev/null +++ b/cmd/specstream/main.go @@ -0,0 +1,219 @@ +// specstream-mcp — MCP server that exposes SpecStream extraction as Hermes tools. +// +// Port: 3991 (alongside forgejo=3988, ob1=3987, pool-manager=3990) +// +// Hermes integration: +// mcp_servers: +// specstream: +// url: http://127.0.0.1:3991 +// enabled: true +// +// This provides a direct replacement for Hermes' web_extract auxiliary. +// Instead of passing raw web content into the agent's context, the agent +// calls specstream_extract and receives typed, structured data. +// No raw text from untrusted sources ever enters the agent. +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "os" + + "github.com/sscoble/specstream/pkg/pipeline" + "github.com/sscoble/specstream/pkg/types" +) + +var extractPipeline = pipeline.New() + +func main() { + port := os.Getenv("SPECSTREAM_PORT") + if port == "" { + port = "3991" + } + + mux := http.NewServeMux() + mux.HandleFunc("/", jsonRPCHandler) + + addr := "127.0.0.1:" + port + log.Printf("[specstream] listening on %s", addr) + log.Printf("[specstream] MCP tools: specstream_extract, specstream_info") + if err := http.ListenAndServe(addr, mux); err != nil { + log.Fatalf("[specstream] %v", err) + } +} + +// ── JSON-RPC Handler ─────────────────────── + +type jsonRPCRequest struct { + JSONRPC string `json:"jsonrpc"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` + ID interface{} `json:"id"` +} + +type jsonRPCResponse struct { + JSONRPC string `json:"jsonrpc"` + Result interface{} `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` + ID interface{} `json:"id"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +func jsonRPCHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeError(w, nil, -32700, "method not allowed") + return + } + + var req jsonRPCRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, nil, -32700, "parse error") + return + } + + if req.JSONRPC != "2.0" { + writeError(w, req.ID, -32600, "invalid JSON-RPC version") + return + } + + var result interface{} + var rpcErr *rpcError + + switch req.Method { + case "specstream_extract": + result, rpcErr = handleExtract(req.Params) + case "specstream_info": + result, rpcErr = handleInfo(req.Params) + default: + rpcErr = &rpcError{Code: -32601, Message: fmt.Sprintf("method not found: %s", req.Method)} + } + + if rpcErr != nil { + writeError(w, req.ID, rpcErr.Code, rpcErr.Message) + return + } + + writeResult(w, req.ID, result) +} + +// ── Tool Handlers ────────────────────────── + +type extractParams struct { + URL string `json:"url"` + Content string `json:"content"` + Filename string `json:"filename"` + MaxSize int `json:"max_size"` + ExtractSections bool `json:"extract_sections"` + ExtractCode bool `json:"extract_code"` + ExtractLinks bool `json:"extract_links"` + ExtractImages bool `json:"extract_images"` + ExtractTables bool `json:"extract_tables"` + ExtractLists bool `json:"extract_lists"` +} + +func handleExtract(raw json.RawMessage) (interface{}, *rpcError) { + var params extractParams + if len(raw) > 0 { + if err := json.Unmarshal(raw, ¶ms); err != nil { + return nil, &rpcError{Code: -32602, Message: "invalid params: " + err.Error()} + } + } + + if params.URL == "" && params.Content == "" { + return nil, &rpcError{Code: -32602, Message: "either 'url' or 'content' is required"} + } + + if params.MaxSize <= 0 || params.MaxSize > 1<<20 { + params.MaxSize = 1 << 20 + } + + req := types.ExtractRequest{ + URL: params.URL, + Content: params.Content, + Filename: params.Filename, + MaxSize: params.MaxSize, + // Default to extracting everything unless explicitly restricted + ExtractSections: !params.ExtractSections || params.ExtractSections, + ExtractCode: !params.ExtractCode || params.ExtractCode, + ExtractLinks: !params.ExtractLinks || params.ExtractLinks, + ExtractImages: !params.ExtractImages || params.ExtractImages, + ExtractTables: !params.ExtractTables || params.ExtractTables, + ExtractLists: !params.ExtractLists || params.ExtractLists, + } + + var result *types.ExtractedContent + var err error + + if params.URL != "" { + result, err = extractPipeline.FetchAndExtract(params.URL, req) + if err != nil { + return nil, &rpcError{Code: -32000, Message: "extraction failed: " + err.Error()} + } + } else { + result, err = extractPipeline.Extract([]byte(params.Content), req) + if err != nil { + return nil, &rpcError{Code: -32000, Message: "extraction failed: " + err.Error()} + } + } + + return result, nil +} + +type infoResult struct { + Version string `json:"version"` + Extractors []string `json:"extractors"` + Filters []string `json:"filters"` + MaxSize int64 `json:"max_size_bytes"` + Schema string `json:"schema"` +} + +func handleInfo(_ json.RawMessage) (interface{}, *rpcError) { + // Get filter names + filterNames := []string{ + "strip_null_bytes", + "strip_control_chars", + "truncate_paragraphs_5000", + "strip_injection_patterns", + } + + return infoResult{ + Version: "specstream/0.1.0", + Extractors: []string{ + "html (goquery, sections+links+images+code+tables+lists)", + "markdown (goldmark, sections+code+lists+tables)", + "text (plain text, sections by line breaks)", + }, + Filters: filterNames, + MaxSize: 1 << 20, + Schema: "ExtractedContent with typed Sections, CodeBlocks, Links, Images, Lists, Tables — no raw text fields", + }, nil +} + +// ── Response Helpers ─────────────────────── + +func writeResult(w http.ResponseWriter, id interface{}, result interface{}) { + resp := jsonRPCResponse{ + JSONRPC: "2.0", + Result: result, + ID: id, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} + +func writeError(w http.ResponseWriter, id interface{}, code int, message string) { + resp := jsonRPCResponse{ + JSONRPC: "2.0", + Error: &rpcError{Code: code, Message: message}, + ID: id, + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) // JSON-RPC always returns 200 + json.NewEncoder(w).Encode(resp) +}