ecf100b917
Go MCP server (port 3991) for agent content safety. Deterministic parsers, typed output, no raw text fields. - HTML (goquery), Markdown (goldmark), Text parsers - Injection pattern detection and redaction - SHA256 content fingerprinting
134 lines
5.3 KiB
Go
134 lines
5.3 KiB
Go
// Package types defines the SpecStream extraction data model.
|
|
//
|
|
// The central security invariant: every output type is structured and bounded.
|
|
// There is NO "RawText" or "Body" field. Every piece of content is categorized
|
|
// into typed, enumerated fields. The agent never receives free-form text
|
|
// from untrusted sources — only structured data extracted by deterministic
|
|
// (non-LLM) parsers.
|
|
|
|
package types
|
|
|
|
import "time"
|
|
|
|
// ExtractedContent is the top-level output of a SpecStream extraction.
|
|
// Every field is typed and bounded. No raw text bucket.
|
|
type ExtractedContent struct {
|
|
// Source metadata
|
|
Source Source `json:"source"`
|
|
Title string `json:"title,omitempty"`
|
|
Description string `json:"description,omitempty"`
|
|
Author string `json:"author,omitempty"`
|
|
PublishedAt string `json:"published_at,omitempty"`
|
|
Language string `json:"language,omitempty"`
|
|
SiteName string `json:"site_name,omitempty"`
|
|
|
|
// Structured content — the safe output
|
|
Sections []Section `json:"sections,omitempty"`
|
|
CodeBlocks []CodeBlock `json:"code_blocks,omitempty"`
|
|
Links []Link `json:"links,omitempty"`
|
|
Images []Image `json:"images,omitempty"`
|
|
Lists []List `json:"lists,omitempty"`
|
|
Tables []Table `json:"tables,omitempty"`
|
|
Metadata Metadata `json:"metadata"`
|
|
|
|
// Extraction diagnostics
|
|
ExtractedAt time.Time `json:"extracted_at"`
|
|
ContentType string `json:"content_type"` // "html", "markdown", "text", "json"
|
|
CharCount int `json:"char_count"`
|
|
SectionCnt int `json:"section_count"`
|
|
}
|
|
|
|
type Source struct {
|
|
URL string `json:"url,omitempty"`
|
|
Filename string `json:"filename,omitempty"`
|
|
MIMEType string `json:"mime_type,omitempty"`
|
|
ByteSize int64 `json:"byte_size,omitempty"`
|
|
IsTrusted bool `json:"is_trusted"` // false for all external content by default
|
|
}
|
|
|
|
// Section is a content section with a heading and bounded paragraphs.
|
|
type Section struct {
|
|
Heading string `json:"heading,omitempty"`
|
|
Level int `json:"level"` // h1-h6
|
|
Paragraphs []string `json:"paragraphs"` // one string per <p> — never the full body
|
|
IsCode bool `json:"is_code,omitempty"` // false = prose
|
|
}
|
|
|
|
// CodeBlock is an isolated code segment with known language.
|
|
type CodeBlock struct {
|
|
Language string `json:"language,omitempty"` // "go", "python", "javascript", etc.
|
|
Code string `json:"code"` // the code text — tagged, not inline
|
|
Caption string `json:"caption,omitempty"`
|
|
}
|
|
|
|
// Link is an extracted hyperlink.
|
|
type Link struct {
|
|
Text string `json:"text"`
|
|
Href string `json:"href"`
|
|
Rel string `json:"rel,omitempty"`
|
|
}
|
|
|
|
// Image is an extracted image reference.
|
|
type Image struct {
|
|
Alt string `json:"alt,omitempty"`
|
|
Src string `json:"src"`
|
|
Width int `json:"width,omitempty"`
|
|
}
|
|
|
|
// List is an ordered or unordered list.
|
|
type List struct {
|
|
Ordered bool `json:"ordered"`
|
|
Items []string `json:"items"`
|
|
}
|
|
|
|
// Table is a structured data table.
|
|
type Table struct {
|
|
Headers []string `json:"headers,omitempty"`
|
|
Rows [][]string `json:"rows"`
|
|
}
|
|
|
|
// Metadata holds extraction diagnostics and content hashes.
|
|
type Metadata struct {
|
|
ExtractorVersion string `json:"extractor_version"`
|
|
ExtractionTimeMs int64 `json:"extraction_time_ms"`
|
|
TagCounts map[string]int `json:"tag_counts,omitempty"`
|
|
Warnings []string `json:"warnings,omitempty"`
|
|
SHA256 string `json:"sha256,omitempty"`
|
|
StrippedPatterns map[string]int `json:"stripped_patterns,omitempty"` // count of filtered patterns
|
|
}
|
|
|
|
// ── Extraction Request ─────────────────────
|
|
|
|
// ExtractRequest is sent to SpecStream's MCP endpoint.
|
|
type ExtractRequest struct {
|
|
URL string `json:"url,omitempty"`
|
|
Content string `json:"content,omitempty"` // raw text content (for testing/offline)
|
|
Filename string `json:"filename,omitempty"` // hint for content type detection
|
|
MaxSize int `json:"max_size,omitempty"` // max bytes to process (default 1MB)
|
|
|
|
// Schema hints — what structured fields we want
|
|
ExtractSections bool `json:"extract_sections,omitempty"`
|
|
ExtractCode bool `json:"extract_code,omitempty"`
|
|
ExtractLinks bool `json:"extract_links,omitempty"`
|
|
ExtractImages bool `json:"extract_images,omitempty"`
|
|
ExtractTables bool `json:"extract_tables,omitempty"`
|
|
ExtractLists bool `json:"extract_lists,omitempty"`
|
|
}
|
|
|
|
// ── Safety Policy ──────────────────────────
|
|
|
|
// SafetyConfig controls what patterns to strip from extracted content.
|
|
// These run AFTER extraction — they remove instruction-like patterns
|
|
// that slipped through the structured extraction.
|
|
type SafetyConfig struct {
|
|
// StripMarkdownImages removes  from extracted text
|
|
StripMarkdownImages bool `json:"strip_md_images,omitempty"`
|
|
// StripInlineCode removes `code` from extracted paragraphs
|
|
StripInlineCode bool `json:"strip_inline_code,omitempty"`
|
|
// StripURLs removes http:// and https:// from extracted paragraph text
|
|
StripURLs bool `json:"strip_urls,omitempty"`
|
|
// MaxParagraphLength truncates paragraphs over this length
|
|
MaxParagraphLength int `json:"max_paragraph_length,omitempty"`
|
|
// BlocklistPatterns removes paragraphs matching these regexes
|
|
BlocklistPatterns []string `json:"blocklist_patterns,omitempty"`
|
|
}
|