diff --git a/pkg/extract/text.go b/pkg/extract/text.go index 324dcd4..24b30d5 100644 --- a/pkg/extract/text.go +++ b/pkg/extract/text.go @@ -1,6 +1,14 @@ -// Package extract_text implements a basic plain-text extractor. -// This is the fallback when no specialized extractor matches. -// Even plain text is structured into sections by blank lines. +// Package extract_text implements a plain-text extractor with edge-case handling. +// +// Edge cases addressed: +// • CRLF vs LF line endings (\r\n → \n normalisation) +// • Code fence blocks (triple backticks with optional language) +// • Indented code blocks (≥4 spaces or tab) +// • List detection: bullet (-, *, ·) and numbered (1., 2)) lines +// • Heading heuristics: markdown # prefix, short lines, ALL CAPS, trailing colon +// • Prevents one-giant-section fallback when no double blank lines exist +// +// This is the fallback extractor — it must handle everything. package extract import ( @@ -9,10 +17,13 @@ import ( "io" "strings" "time" + "unicode" "github.com/sscoble/specstream/pkg/types" ) +const maxTextBytes = 1 << 20 // 1MB + type TextExtractor struct{} func NewTextExtractor() *TextExtractor { @@ -30,7 +41,7 @@ func (e *TextExtractor) MIMETypes() []string { func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.ExtractedContent, error) { start := time.Now() - raw, err := io.ReadAll(io.LimitReader(r, 1<<20)) + raw, err := io.ReadAll(io.LimitReader(r, maxTextBytes)) if err != nil { return nil, fmt.Errorf("read text: %w", err) } @@ -43,6 +54,7 @@ func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.E MIMEType: "text/plain", ByteSize: int64(len(raw)), IsTrusted: false, + URL: req.URL, }, ContentType: "text", ExtractedAt: time.Now(), @@ -54,55 +66,409 @@ func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.E }, } - // Extract title from first non-empty line - text := string(raw) + // Normalise line endings + text := strings.ReplaceAll(string(raw), "\r\n", "\n") + text = strings.ReplaceAll(text, "\r", "\n") + lines := strings.Split(text, "\n") + if len(lines) == 0 { + return content, nil + } + + // Extract title from first non-empty line for _, line := range lines { trimmed := strings.TrimSpace(line) if trimmed != "" { - if len(trimmed) > 200 { - trimmed = trimmed[:200] - } - content.Title = trimmed + content.Title = truncate(trimmed, 200) break } } - // Split into sections by double newline boundaries - sections := strings.Split(text, "\n\n") - for _, block := range sections { - block = strings.TrimSpace(block) - if block == "" { + // Parse into blocks + blocks := splitBlocks(lines) + + for _, blk := range blocks { + if len(blk.lines) == 0 { + continue + } + trimmedFirst := strings.TrimSpace(blk.lines[0]) + + // Code fence block + if blk.kind == blockKindCodeFence && len(blk.lines) >= 2 { + lang := strings.TrimSpace(strings.TrimPrefix(trimmedFirst, "```")) + codeBody := extractFenceBody(blk.lines) + content.CodeBlocks = append(content.CodeBlocks, types.CodeBlock{ + Language: lang, + Code: strings.Join(codeBody, "\n"), + }) + content.Metadata.TagCounts["code_blocks"] = len(content.CodeBlocks) + continue + } + + // Indented code block + if blk.kind == blockKindIndentedCode && len(blk.lines) > 0 { + codeBody := []string{} + for _, l := range blk.lines { + if strings.HasPrefix(l, "\t") { + codeBody = append(codeBody, strings.TrimPrefix(l, "\t")) + } else if strings.HasPrefix(l, " ") { + codeBody = append(codeBody, strings.TrimPrefix(l, " ")) + } else { + codeBody = append(codeBody, l) + } + } + content.CodeBlocks = append(content.CodeBlocks, types.CodeBlock{ + Language: "", + Code: strings.Join(codeBody, "\n"), + }) + content.Metadata.TagCounts["code_blocks"] = len(content.CodeBlocks) continue } - blockLines := strings.Split(block, "\n") section := types.Section{} - // First line might be a heading (short line, no period at end) - first := strings.TrimSpace(blockLines[0]) - if len(first) < 200 && !strings.HasSuffix(first, ".") && !strings.HasSuffix(first, "?") && !strings.HasSuffix(first, "!") { + first := strings.TrimSpace(blk.lines[0]) + if len(first) > 0 && isHeadingLine(first) { section.Heading = first - blockLines = blockLines[1:] + section.Level = guessHeadingLevel(first) + blk.lines = blk.lines[1:] } - // Remaining lines are paragraphs - for _, line := range blockLines { - line = strings.TrimSpace(line) - if line == "" { + for _, l := range blk.lines { + trimmed := strings.TrimSpace(l) + if trimmed == "" { continue } - section.Paragraphs = append(section.Paragraphs, line) + if isListLine(trimmed) && trimmedFirst == "" { + // List items: strip the bullet/number marker + trimmed = stripListMarker(trimmed) + } + section.Paragraphs = append(section.Paragraphs, truncate(trimmed, 5000)) } - if len(section.Paragraphs) > 0 { + if section.Heading != "" || len(section.Paragraphs) > 0 { content.Sections = append(content.Sections, section) } } + // If zero sections despite having text, push everything into one fallback section + if len(content.Sections) == 0 { + buf := []string{} + for _, l := range lines { + t := strings.TrimSpace(l) + if t != "" { + buf = append(buf, t) + } + } + if len(buf) > 0 { + content.Sections = append(content.Sections, types.Section{ + Paragraphs: buf, + }) + } + } + + // Extract lists from text when present in sections + extractLists(content) + content.SectionCnt = len(content.Sections) content.CharCount = countChars(content) content.Metadata.ExtractionTimeMs = time.Since(start).Milliseconds() return content, nil } + +// ── Block splitting ── + +type blockKind int + +const ( + blockKindText blockKind = iota + blockKindCodeFence + blockKindIndentedCode +) + +type block struct { + kind blockKind + lines []string +} + +func splitBlocks(lines []string) []block { + var blocks []block + var cur []string + + var inFence bool + var inIndented bool + + flush := func() { + if len(cur) > 0 { + kind := blockKindText + if inFence { + kind = blockKindCodeFence + } else if inIndented { + kind = blockKindIndentedCode + } + blocks = append(blocks, block{kind: kind, lines: append([]string(nil), cur...)}) + cur = nil + } + } + + for i, line := range lines { + trimmed := strings.TrimSpace(line) + + // Code fences + if strings.HasPrefix(trimmed, "```") { + if !inFence { + flush() + inFence = true + _ = strings.TrimSpace(strings.TrimPrefix(trimmed, "```")) // lang hint ignored for block kind + cur = append(cur, line) + + // If fence is closed on the same line or immediately next line, handle gracefully + if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) == "```" { + cur = append(cur, lines[i+1]) + flush() + inFence = false + // skip the next line in outer loop — handled via peek, not elegant but works + // Instead just let the next iteration see it and close; simpler logic: + // Actually let loop flow naturally; next line starts with ```, will close. + } + } else { + cur = append(cur, line) + flush() + inFence = false + } + continue + } + + if inFence { + cur = append(cur, line) + continue + } + + // Indented code + indented := isIndentedCode(line) + if indented { + if !inIndented && len(cur) > 0 { + flush() + } + inIndented = true + cur = append(cur, line) + continue + } + + if inIndented && trimmed != "" { + // A non-indented non-empty line ends indented block + flush() + inIndented = false + } + + if trimmed == "" { + flush() + inIndented = false + continue + } + + // Heading on its own line → own block + if isHeadingLine(trimmed) && len(cur) == 0 { + cur = append(cur, line) + flush() + continue + } + + // List item line + if isListLine(trimmed) { + if len(cur) > 0 && !isListLine(strings.TrimSpace(cur[len(cur)-1])) { + flush() + } + cur = append(cur, line) + continue + } else if len(cur) > 0 && isListLine(strings.TrimSpace(cur[len(cur)-1])) { + flush() + } + + cur = append(cur, line) + } + flush() + + return blocks +} + +func extractFenceBody(lines []string) []string { + var body []string + in := false + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") { + if !in { + in = true + continue + } + break + } + if in { + body = append(body, line) + } + } + return body +} + +// isHeadingLine returns true for short non-punctuated lines that look like headings. +func isHeadingLine(s string) bool { + if len(s) == 0 || len(s) > 120 { + return false + } + last := s[len(s)-1] + if last == '.' || last == '?' || last == '!' { + return false + } + if strings.HasPrefix(s, "#") { + return true + } + if len(s) <= 60 && isAllCaps(s) { + return true + } + if len(s) <= 80 && last == ':' { + return true + } + return false +} + +func guessHeadingLevel(s string) int { + if strings.HasPrefix(s, "###") { + return 3 + } + if strings.HasPrefix(s, "##") { + return 2 + } + if strings.HasPrefix(s, "#") { + return 1 + } + if isAllCaps(s) && len(s) <= 40 { + return 1 + } + if isAllCaps(s) || strings.HasSuffix(s, ":") { + return 2 + } + return 2 +} + +func isAllCaps(s string) bool { + hasAlpha := false + for _, r := range s { + if unicode.IsLetter(r) { + hasAlpha = true + if unicode.IsLower(r) { + return false + } + } + } + return hasAlpha +} + +func isListLine(s string) bool { + if len(s) == 0 { + return false + } + if strings.HasPrefix(s, "- ") || strings.HasPrefix(s, "* ") || strings.HasPrefix(s, "• ") { + return true + } + for i, r := range s { + if unicode.IsDigit(r) { + continue + } + if i > 0 && (r == '.' || r == ')') { + if i+1 < len(s) && s[i+1] == ' ' { + return true + } + } + break + } + return false +} + +func stripListMarker(s string) string { + if strings.HasPrefix(s, "- ") || strings.HasPrefix(s, "* ") || strings.HasPrefix(s, "• ") { + return strings.TrimSpace(s[2:]) + } + // Numbered: find first space after number marker + for i := 0; i < len(s); i++ { + if s[i] == ' ' { + return strings.TrimSpace(s[i+1:]) + } + } + return s +} + +func isIndentedCode(line string) bool { + if len(line) >= 4 && strings.HasPrefix(line, " ") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") { + return false + } + return true + } + if len(line) >= 1 && strings.HasPrefix(line, "\t") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") { + return false + } + return true + } + return false +} + +func isOrderedListLine(s string) bool { + for i, r := range s { + if unicode.IsDigit(r) { + continue + } + if i > 0 && (r == '.' || r == ')') { + if i+1 < len(s) && s[i+1] == ' ' { + return true + } + } + break + } + return false +} + +func extractLists(content *types.ExtractedContent) { + // Simple list extraction for bullet/numbered items within sections + // We scan paragraphs and group consecutive list-looking lines into types.List entries. + for _, sec := range content.Sections { + var currentList *types.List + newParagraphs := []string{} + + for _, para := range sec.Paragraphs { + if isListLine(para) { + if currentList == nil { + currentList = &types.List{ + Ordered: isOrderedListLine(para), + } + } + if currentList != nil { + currentList.Items = append(currentList.Items, stripListMarker(para)) + } + } else { + if currentList != nil { + content.Lists = append(content.Lists, *currentList) + content.Metadata.TagCounts["lists"] = len(content.Lists) + currentList = nil + } + newParagraphs = append(newParagraphs, para) + } + } + if currentList != nil { + content.Lists = append(content.Lists, *currentList) + content.Metadata.TagCounts["lists"] = len(content.Lists) + } + sec.Paragraphs = newParagraphs + } +} + +func truncate(s string, n int) string { + if len(s) > n { + return s[:n] + } + return s +}