// Package extract_text implements a basic plain-text extractor. // This is the fallback when no specialized extractor matches. // Even plain text is structured into sections by blank lines. package extract import ( "crypto/sha256" "fmt" "io" "strings" "time" "github.com/sscoble/specstream/pkg/types" ) type TextExtractor struct{} func NewTextExtractor() *TextExtractor { return &TextExtractor{} } func (e *TextExtractor) Detect(r io.ReaderAt, size int64) (bool, error) { return true, nil // universal fallback } func (e *TextExtractor) MIMETypes() []string { return []string{"text/plain"} } func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.ExtractedContent, error) { start := time.Now() raw, err := io.ReadAll(io.LimitReader(r, 1<<20)) if err != nil { return nil, fmt.Errorf("read text: %w", err) } hash := sha256.Sum256(raw) fingerprint := fmt.Sprintf("%x", hash) content := &types.ExtractedContent{ Source: types.Source{ MIMEType: "text/plain", ByteSize: int64(len(raw)), IsTrusted: false, }, ContentType: "text", ExtractedAt: time.Now(), Metadata: types.Metadata{ ExtractorVersion: "specstream/0.1.0", SHA256: fingerprint, StrippedPatterns: make(map[string]int), TagCounts: make(map[string]int), }, } // Extract title from first non-empty line text := string(raw) lines := strings.Split(text, "\n") for _, line := range lines { trimmed := strings.TrimSpace(line) if trimmed != "" { if len(trimmed) > 200 { trimmed = trimmed[:200] } content.Title = trimmed break } } // Split into sections by double newline boundaries sections := strings.Split(text, "\n\n") for _, block := range sections { block = strings.TrimSpace(block) if block == "" { continue } blockLines := strings.Split(block, "\n") section := types.Section{} // First line might be a heading (short line, no period at end) first := strings.TrimSpace(blockLines[0]) if len(first) < 200 && !strings.HasSuffix(first, ".") && !strings.HasSuffix(first, "?") && !strings.HasSuffix(first, "!") { section.Heading = first blockLines = blockLines[1:] } // Remaining lines are paragraphs for _, line := range blockLines { line = strings.TrimSpace(line) if line == "" { continue } section.Paragraphs = append(section.Paragraphs, line) } if len(section.Paragraphs) > 0 { content.Sections = append(content.Sections, section) } } content.SectionCnt = len(content.Sections) content.CharCount = countChars(content) content.Metadata.ExtractionTimeMs = time.Since(start).Milliseconds() return content, nil }