ecf100b917
Go MCP server (port 3991) for agent content safety. Deterministic parsers, typed output, no raw text fields. - HTML (goquery), Markdown (goldmark), Text parsers - Injection pattern detection and redaction - SHA256 content fingerprinting
108 lines
2.6 KiB
Go
108 lines
2.6 KiB
Go
// Package extract_text implements a basic plain-text extractor.
|
|
// This is the fallback when no specialized extractor matches.
|
|
// Even plain text is structured into sections by blank lines.
|
|
package extract
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/sscoble/specstream/pkg/types"
|
|
)
|
|
|
|
type TextExtractor struct{}
|
|
|
|
func NewTextExtractor() *TextExtractor {
|
|
return &TextExtractor{}
|
|
}
|
|
|
|
func (e *TextExtractor) Detect(r io.ReaderAt, size int64) (bool, error) {
|
|
return true, nil // universal fallback
|
|
}
|
|
|
|
func (e *TextExtractor) MIMETypes() []string {
|
|
return []string{"text/plain"}
|
|
}
|
|
|
|
func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.ExtractedContent, error) {
|
|
start := time.Now()
|
|
|
|
raw, err := io.ReadAll(io.LimitReader(r, 1<<20))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read text: %w", err)
|
|
}
|
|
|
|
hash := sha256.Sum256(raw)
|
|
fingerprint := fmt.Sprintf("%x", hash)
|
|
|
|
content := &types.ExtractedContent{
|
|
Source: types.Source{
|
|
MIMEType: "text/plain",
|
|
ByteSize: int64(len(raw)),
|
|
IsTrusted: false,
|
|
},
|
|
ContentType: "text",
|
|
ExtractedAt: time.Now(),
|
|
Metadata: types.Metadata{
|
|
ExtractorVersion: "specstream/0.1.0",
|
|
SHA256: fingerprint,
|
|
StrippedPatterns: make(map[string]int),
|
|
TagCounts: make(map[string]int),
|
|
},
|
|
}
|
|
|
|
// Extract title from first non-empty line
|
|
text := string(raw)
|
|
lines := strings.Split(text, "\n")
|
|
for _, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
if trimmed != "" {
|
|
if len(trimmed) > 200 {
|
|
trimmed = trimmed[:200]
|
|
}
|
|
content.Title = trimmed
|
|
break
|
|
}
|
|
}
|
|
|
|
// Split into sections by double newline boundaries
|
|
sections := strings.Split(text, "\n\n")
|
|
for _, block := range sections {
|
|
block = strings.TrimSpace(block)
|
|
if block == "" {
|
|
continue
|
|
}
|
|
|
|
blockLines := strings.Split(block, "\n")
|
|
section := types.Section{}
|
|
|
|
// First line might be a heading (short line, no period at end)
|
|
first := strings.TrimSpace(blockLines[0])
|
|
if len(first) < 200 && !strings.HasSuffix(first, ".") && !strings.HasSuffix(first, "?") && !strings.HasSuffix(first, "!") {
|
|
section.Heading = first
|
|
blockLines = blockLines[1:]
|
|
}
|
|
|
|
// Remaining lines are paragraphs
|
|
for _, line := range blockLines {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
section.Paragraphs = append(section.Paragraphs, line)
|
|
}
|
|
|
|
if len(section.Paragraphs) > 0 {
|
|
content.Sections = append(content.Sections, section)
|
|
}
|
|
}
|
|
|
|
content.SectionCnt = len(content.Sections)
|
|
content.CharCount = countChars(content)
|
|
content.Metadata.ExtractionTimeMs = time.Since(start).Milliseconds()
|
|
|
|
return content, nil
|
|
}
|