// Package extract_text implements a plain-text extractor with edge-case handling. // // Edge cases addressed: // • CRLF vs LF line endings (\r\n → \n normalisation) // • Code fence blocks (triple backticks with optional language) // • Indented code blocks (≥4 spaces or tab) // • List detection: bullet (-, *, ·) and numbered (1., 2)) lines // • Heading heuristics: markdown # prefix, short lines, ALL CAPS, trailing colon // • Prevents one-giant-section fallback when no double blank lines exist // // This is the fallback extractor — it must handle everything. package extract import ( "crypto/sha256" "fmt" "io" "strings" "time" "unicode" "github.com/sscoble/specstream/pkg/types" ) const maxTextBytes = 1 << 20 // 1MB type TextExtractor struct{} func NewTextExtractor() *TextExtractor { return &TextExtractor{} } func (e *TextExtractor) Detect(r io.ReaderAt, size int64) (bool, error) { return true, nil // universal fallback } func (e *TextExtractor) MIMETypes() []string { return []string{"text/plain"} } func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.ExtractedContent, error) { start := time.Now() raw, err := io.ReadAll(io.LimitReader(r, maxTextBytes)) if err != nil { return nil, fmt.Errorf("read text: %w", err) } hash := sha256.Sum256(raw) fingerprint := fmt.Sprintf("%x", hash) content := &types.ExtractedContent{ Source: types.Source{ MIMEType: "text/plain", ByteSize: int64(len(raw)), IsTrusted: false, URL: req.URL, }, ContentType: "text", ExtractedAt: time.Now(), Metadata: types.Metadata{ ExtractorVersion: "specstream/0.1.0", SHA256: fingerprint, StrippedPatterns: make(map[string]int), TagCounts: make(map[string]int), }, } // Normalise line endings text := strings.ReplaceAll(string(raw), "\r\n", "\n") text = strings.ReplaceAll(text, "\r", "\n") lines := strings.Split(text, "\n") if len(lines) == 0 { return content, nil } // Extract title from first non-empty line for _, line := range lines { trimmed := strings.TrimSpace(line) if trimmed != "" { content.Title = truncate(trimmed, 200) break } } // Parse into blocks blocks := splitBlocks(lines) for _, blk := range blocks { if len(blk.lines) == 0 { continue } trimmedFirst := strings.TrimSpace(blk.lines[0]) // Code fence block if blk.kind == blockKindCodeFence && len(blk.lines) >= 2 { lang := strings.TrimSpace(strings.TrimPrefix(trimmedFirst, "```")) codeBody := extractFenceBody(blk.lines) content.CodeBlocks = append(content.CodeBlocks, types.CodeBlock{ Language: lang, Code: strings.Join(codeBody, "\n"), }) content.Metadata.TagCounts["code_blocks"] = len(content.CodeBlocks) continue } // Indented code block if blk.kind == blockKindIndentedCode && len(blk.lines) > 0 { codeBody := []string{} for _, l := range blk.lines { if strings.HasPrefix(l, "\t") { codeBody = append(codeBody, strings.TrimPrefix(l, "\t")) } else if strings.HasPrefix(l, " ") { codeBody = append(codeBody, strings.TrimPrefix(l, " ")) } else { codeBody = append(codeBody, l) } } content.CodeBlocks = append(content.CodeBlocks, types.CodeBlock{ Language: "", Code: strings.Join(codeBody, "\n"), }) content.Metadata.TagCounts["code_blocks"] = len(content.CodeBlocks) continue } section := types.Section{} first := strings.TrimSpace(blk.lines[0]) if len(first) > 0 && isHeadingLine(first) { section.Heading = first section.Level = guessHeadingLevel(first) blk.lines = blk.lines[1:] } for _, l := range blk.lines { trimmed := strings.TrimSpace(l) if trimmed == "" { continue } if isListLine(trimmed) && trimmedFirst == "" { // List items: strip the bullet/number marker trimmed = stripListMarker(trimmed) } section.Paragraphs = append(section.Paragraphs, truncate(trimmed, 5000)) } if section.Heading != "" || len(section.Paragraphs) > 0 { content.Sections = append(content.Sections, section) } } // If zero sections despite having text, push everything into one fallback section if len(content.Sections) == 0 { buf := []string{} for _, l := range lines { t := strings.TrimSpace(l) if t != "" { buf = append(buf, t) } } if len(buf) > 0 { content.Sections = append(content.Sections, types.Section{ Paragraphs: buf, }) } } // Extract lists from text when present in sections extractLists(content) content.SectionCnt = len(content.Sections) content.CharCount = countChars(content) content.Metadata.ExtractionTimeMs = time.Since(start).Milliseconds() return content, nil } // ── Block splitting ── type blockKind int const ( blockKindText blockKind = iota blockKindCodeFence blockKindIndentedCode ) type block struct { kind blockKind lines []string } func splitBlocks(lines []string) []block { var blocks []block var cur []string var inFence bool var inIndented bool flush := func() { if len(cur) > 0 { kind := blockKindText if inFence { kind = blockKindCodeFence } else if inIndented { kind = blockKindIndentedCode } blocks = append(blocks, block{kind: kind, lines: append([]string(nil), cur...)}) cur = nil } } for i, line := range lines { trimmed := strings.TrimSpace(line) // Code fences if strings.HasPrefix(trimmed, "```") { if !inFence { flush() inFence = true _ = strings.TrimSpace(strings.TrimPrefix(trimmed, "```")) // lang hint ignored for block kind cur = append(cur, line) // If fence is closed on the same line or immediately next line, handle gracefully if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) == "```" { cur = append(cur, lines[i+1]) flush() inFence = false // skip the next line in outer loop — handled via peek, not elegant but works // Instead just let the next iteration see it and close; simpler logic: // Actually let loop flow naturally; next line starts with ```, will close. } } else { cur = append(cur, line) flush() inFence = false } continue } if inFence { cur = append(cur, line) continue } // Indented code indented := isIndentedCode(line) if indented { if !inIndented && len(cur) > 0 { flush() } inIndented = true cur = append(cur, line) continue } if inIndented && trimmed != "" { // A non-indented non-empty line ends indented block flush() inIndented = false } if trimmed == "" { flush() inIndented = false continue } // Heading on its own line → own block if isHeadingLine(trimmed) && len(cur) == 0 { cur = append(cur, line) flush() continue } // List item line if isListLine(trimmed) { if len(cur) > 0 && !isListLine(strings.TrimSpace(cur[len(cur)-1])) { flush() } cur = append(cur, line) continue } else if len(cur) > 0 && isListLine(strings.TrimSpace(cur[len(cur)-1])) { flush() } cur = append(cur, line) } flush() return blocks } func extractFenceBody(lines []string) []string { var body []string in := false for _, line := range lines { trimmed := strings.TrimSpace(line) if strings.HasPrefix(trimmed, "```") { if !in { in = true continue } break } if in { body = append(body, line) } } return body } // isHeadingLine returns true for short non-punctuated lines that look like headings. func isHeadingLine(s string) bool { if len(s) == 0 || len(s) > 120 { return false } last := s[len(s)-1] if last == '.' || last == '?' || last == '!' { return false } if strings.HasPrefix(s, "#") { return true } if len(s) <= 60 && isAllCaps(s) { return true } if len(s) <= 80 && last == ':' { return true } return false } func guessHeadingLevel(s string) int { if strings.HasPrefix(s, "###") { return 3 } if strings.HasPrefix(s, "##") { return 2 } if strings.HasPrefix(s, "#") { return 1 } if isAllCaps(s) && len(s) <= 40 { return 1 } if isAllCaps(s) || strings.HasSuffix(s, ":") { return 2 } return 2 } func isAllCaps(s string) bool { hasAlpha := false for _, r := range s { if unicode.IsLetter(r) { hasAlpha = true if unicode.IsLower(r) { return false } } } return hasAlpha } func isListLine(s string) bool { if len(s) == 0 { return false } if strings.HasPrefix(s, "- ") || strings.HasPrefix(s, "* ") || strings.HasPrefix(s, "• ") { return true } for i, r := range s { if unicode.IsDigit(r) { continue } if i > 0 && (r == '.' || r == ')') { if i+1 < len(s) && s[i+1] == ' ' { return true } } break } return false } func stripListMarker(s string) string { if strings.HasPrefix(s, "- ") || strings.HasPrefix(s, "* ") || strings.HasPrefix(s, "• ") { return strings.TrimSpace(s[2:]) } // Numbered: find first space after number marker for i := 0; i < len(s); i++ { if s[i] == ' ' { return strings.TrimSpace(s[i+1:]) } } return s } func isIndentedCode(line string) bool { if len(line) >= 4 && strings.HasPrefix(line, " ") { trimmed := strings.TrimSpace(line) if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") { return false } return true } if len(line) >= 1 && strings.HasPrefix(line, "\t") { trimmed := strings.TrimSpace(line) if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") { return false } return true } return false } func isOrderedListLine(s string) bool { for i, r := range s { if unicode.IsDigit(r) { continue } if i > 0 && (r == '.' || r == ')') { if i+1 < len(s) && s[i+1] == ' ' { return true } } break } return false } func extractLists(content *types.ExtractedContent) { // Simple list extraction for bullet/numbered items within sections // We scan paragraphs and group consecutive list-looking lines into types.List entries. for _, sec := range content.Sections { var currentList *types.List newParagraphs := []string{} for _, para := range sec.Paragraphs { if isListLine(para) { if currentList == nil { currentList = &types.List{ Ordered: isOrderedListLine(para), } } if currentList != nil { currentList.Items = append(currentList.Items, stripListMarker(para)) } } else { if currentList != nil { content.Lists = append(content.Lists, *currentList) content.Metadata.TagCounts["lists"] = len(content.Lists) currentList = nil } newParagraphs = append(newParagraphs, para) } } if currentList != nil { content.Lists = append(content.Lists, *currentList) content.Metadata.TagCounts["lists"] = len(content.Lists) } sec.Paragraphs = newParagraphs } } func truncate(s string, n int) string { if len(s) > n { return s[:n] } return s }