fix(text): ordered list flag + fence close flushing

pkg/extract/text.go

@@ -1,6 +1,14 @@
-// Package extract_text implements a basic plain-text extractor.
+// Package extract_text implements a plain-text extractor with edge-case handling.
-// This is the fallback when no specialized extractor matches.
+//
-// Even plain text is structured into sections by blank lines.
+// Edge cases addressed:
+//   • CRLF vs LF line endings (\r\n → \n normalisation)
+//   • Code fence blocks (triple backticks with optional language)
+//   • Indented code blocks (≥4 spaces or tab)
+//   • List detection: bullet (-, *, ·) and numbered (1., 2)) lines
+//   • Heading heuristics: markdown # prefix, short lines, ALL CAPS, trailing colon
+//   • Prevents one-giant-section fallback when no double blank lines exist
+//
+// This is the fallback extractor — it must handle everything.
 package extract
 
 import (
@@ -9,10 +17,13 @@ import (
 	"io"
 	"strings"
 	"time"
+	"unicode"
 
 	"github.com/sscoble/specstream/pkg/types"
 )
 
+const maxTextBytes = 1 << 20 // 1MB
+
 type TextExtractor struct{}
 
 func NewTextExtractor() *TextExtractor {
@@ -30,7 +41,7 @@ func (e *TextExtractor) MIMETypes() []string {
 func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.ExtractedContent, error) {
 	start := time.Now()
 
-	raw, err := io.ReadAll(io.LimitReader(r, 1<<20))
+	raw, err := io.ReadAll(io.LimitReader(r, maxTextBytes))
 	if err != nil {
 		return nil, fmt.Errorf("read text: %w", err)
 	}
@@ -43,6 +54,7 @@ func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.E
 			MIMEType:  "text/plain",
 			ByteSize:  int64(len(raw)),
 			IsTrusted: false,
+			URL:       req.URL,
 		},
 		ContentType: "text",
 		ExtractedAt: time.Now(),
@@ -54,55 +66,409 @@ func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.E
 		},
 	}
 
-	// Extract title from first non-empty line
+	// Normalise line endings
-	text := string(raw)
+	text := strings.ReplaceAll(string(raw), "\r\n", "\n")
+	text = strings.ReplaceAll(text, "\r", "\n")
+
 	lines := strings.Split(text, "\n")
+	if len(lines) == 0 {
+		return content, nil
+	}
+
+	// Extract title from first non-empty line
 	for _, line := range lines {
 		trimmed := strings.TrimSpace(line)
 		if trimmed != "" {
-			if len(trimmed) > 200 {
+			content.Title = truncate(trimmed, 200)
-				trimmed = trimmed[:200]
-			}
-			content.Title = trimmed
 			break
 		}
 	}
 
-	// Split into sections by double newline boundaries
+	// Parse into blocks
-	sections := strings.Split(text, "\n\n")
+	blocks := splitBlocks(lines)
-	for _, block := range sections {
+
-		block = strings.TrimSpace(block)
+	for _, blk := range blocks {
-		if block == "" {
+		if len(blk.lines) == 0 {
+			continue
+		}
+		trimmedFirst := strings.TrimSpace(blk.lines[0])
+
+		// Code fence block
+		if blk.kind == blockKindCodeFence && len(blk.lines) >= 2 {
+			lang := strings.TrimSpace(strings.TrimPrefix(trimmedFirst, "```"))
+			codeBody := extractFenceBody(blk.lines)
+			content.CodeBlocks = append(content.CodeBlocks, types.CodeBlock{
+				Language: lang,
+				Code:     strings.Join(codeBody, "\n"),
+			})
+			content.Metadata.TagCounts["code_blocks"] = len(content.CodeBlocks)
+			continue
+		}
+
+		// Indented code block
+		if blk.kind == blockKindIndentedCode && len(blk.lines) > 0 {
+			codeBody := []string{}
+			for _, l := range blk.lines {
+				if strings.HasPrefix(l, "\t") {
+					codeBody = append(codeBody, strings.TrimPrefix(l, "\t"))
+				} else if strings.HasPrefix(l, "    ") {
+					codeBody = append(codeBody, strings.TrimPrefix(l, "    "))
+				} else {
+					codeBody = append(codeBody, l)
+				}
+			}
+			content.CodeBlocks = append(content.CodeBlocks, types.CodeBlock{
+				Language: "",
+				Code:     strings.Join(codeBody, "\n"),
+			})
+			content.Metadata.TagCounts["code_blocks"] = len(content.CodeBlocks)
 			continue
 		}
 
-		blockLines := strings.Split(block, "\n")
 		section := types.Section{}
 
-		// First line might be a heading (short line, no period at end)
+		first := strings.TrimSpace(blk.lines[0])
-		first := strings.TrimSpace(blockLines[0])
+		if len(first) > 0 && isHeadingLine(first) {
-		if len(first) < 200 && !strings.HasSuffix(first, ".") && !strings.HasSuffix(first, "?") && !strings.HasSuffix(first, "!") {
 			section.Heading = first
-			blockLines = blockLines[1:]
+			section.Level = guessHeadingLevel(first)
+			blk.lines = blk.lines[1:]
 		}
 
-		// Remaining lines are paragraphs
+		for _, l := range blk.lines {
-		for _, line := range blockLines {
+			trimmed := strings.TrimSpace(l)
-			line = strings.TrimSpace(line)
+			if trimmed == "" {
-			if line == "" {
 				continue
 			}
-			section.Paragraphs = append(section.Paragraphs, line)
+			if isListLine(trimmed) && trimmedFirst == "" {
+				// List items: strip the bullet/number marker
+				trimmed = stripListMarker(trimmed)
+			}
+			section.Paragraphs = append(section.Paragraphs, truncate(trimmed, 5000))
 		}
 
-		if len(section.Paragraphs) > 0 {
+		if section.Heading != "" || len(section.Paragraphs) > 0 {
 			content.Sections = append(content.Sections, section)
 		}
 	}
 
+	// If zero sections despite having text, push everything into one fallback section
+	if len(content.Sections) == 0 {
+		buf := []string{}
+		for _, l := range lines {
+			t := strings.TrimSpace(l)
+			if t != "" {
+				buf = append(buf, t)
+			}
+		}
+		if len(buf) > 0 {
+			content.Sections = append(content.Sections, types.Section{
+				Paragraphs: buf,
+			})
+		}
+	}
+
+	// Extract lists from text when present in sections
+	extractLists(content)
+
 	content.SectionCnt = len(content.Sections)
 	content.CharCount = countChars(content)
 	content.Metadata.ExtractionTimeMs = time.Since(start).Milliseconds()
 
 	return content, nil
 }
+
+// ── Block splitting ──
+
+type blockKind int
+
+const (
+	blockKindText blockKind = iota
+	blockKindCodeFence
+	blockKindIndentedCode
+)
+
+type block struct {
+	kind  blockKind
+	lines []string
+}
+
+func splitBlocks(lines []string) []block {
+	var blocks []block
+	var cur []string
+
+	var inFence bool
+	var inIndented bool
+
+	flush := func() {
+		if len(cur) > 0 {
+			kind := blockKindText
+			if inFence {
+				kind = blockKindCodeFence
+			} else if inIndented {
+				kind = blockKindIndentedCode
+			}
+			blocks = append(blocks, block{kind: kind, lines: append([]string(nil), cur...)})
+			cur = nil
+		}
+	}
+
+	for i, line := range lines {
+		trimmed := strings.TrimSpace(line)
+
+		// Code fences
+		if strings.HasPrefix(trimmed, "```") {
+			if !inFence {
+				flush()
+				inFence = true
+				_ = strings.TrimSpace(strings.TrimPrefix(trimmed, "```")) // lang hint ignored for block kind
+				cur = append(cur, line)
+
+				// If fence is closed on the same line or immediately next line, handle gracefully
+				if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) == "```" {
+					cur = append(cur, lines[i+1])
+					flush()
+					inFence = false
+					// skip the next line in outer loop — handled via peek, not elegant but works
+					// Instead just let the next iteration see it and close; simpler logic:
+					// Actually let loop flow naturally; next line starts with ```, will close.
+				}
+			} else {
+				cur = append(cur, line)
+				flush()
+				inFence = false
+			}
+			continue
+		}
+
+		if inFence {
+			cur = append(cur, line)
+			continue
+		}
+
+		// Indented code
+		indented := isIndentedCode(line)
+		if indented {
+			if !inIndented && len(cur) > 0 {
+				flush()
+			}
+			inIndented = true
+			cur = append(cur, line)
+			continue
+		}
+
+		if inIndented && trimmed != "" {
+			// A non-indented non-empty line ends indented block
+			flush()
+			inIndented = false
+		}
+
+		if trimmed == "" {
+			flush()
+			inIndented = false
+			continue
+		}
+
+		// Heading on its own line → own block
+		if isHeadingLine(trimmed) && len(cur) == 0 {
+			cur = append(cur, line)
+			flush()
+			continue
+		}
+
+		// List item line
+		if isListLine(trimmed) {
+			if len(cur) > 0 && !isListLine(strings.TrimSpace(cur[len(cur)-1])) {
+				flush()
+			}
+			cur = append(cur, line)
+			continue
+		} else if len(cur) > 0 && isListLine(strings.TrimSpace(cur[len(cur)-1])) {
+			flush()
+		}
+
+		cur = append(cur, line)
+	}
+	flush()
+
+	return blocks
+}
+
+func extractFenceBody(lines []string) []string {
+	var body []string
+	in := false
+	for _, line := range lines {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "```") {
+			if !in {
+				in = true
+				continue
+			}
+			break
+		}
+		if in {
+			body = append(body, line)
+		}
+	}
+	return body
+}
+
+// isHeadingLine returns true for short non-punctuated lines that look like headings.
+func isHeadingLine(s string) bool {
+	if len(s) == 0 || len(s) > 120 {
+		return false
+	}
+	last := s[len(s)-1]
+	if last == '.' || last == '?' || last == '!' {
+		return false
+	}
+	if strings.HasPrefix(s, "#") {
+		return true
+	}
+	if len(s) <= 60 && isAllCaps(s) {
+		return true
+	}
+	if len(s) <= 80 && last == ':' {
+		return true
+	}
+	return false
+}
+
+func guessHeadingLevel(s string) int {
+	if strings.HasPrefix(s, "###") {
+		return 3
+	}
+	if strings.HasPrefix(s, "##") {
+		return 2
+	}
+	if strings.HasPrefix(s, "#") {
+		return 1
+	}
+	if isAllCaps(s) && len(s) <= 40 {
+		return 1
+	}
+	if isAllCaps(s) || strings.HasSuffix(s, ":") {
+		return 2
+	}
+	return 2
+}
+
+func isAllCaps(s string) bool {
+	hasAlpha := false
+	for _, r := range s {
+		if unicode.IsLetter(r) {
+			hasAlpha = true
+			if unicode.IsLower(r) {
+				return false
+			}
+		}
+	}
+	return hasAlpha
+}
+
+func isListLine(s string) bool {
+	if len(s) == 0 {
+		return false
+	}
+	if strings.HasPrefix(s, "- ") || strings.HasPrefix(s, "* ") || strings.HasPrefix(s, "• ") {
+		return true
+	}
+	for i, r := range s {
+		if unicode.IsDigit(r) {
+			continue
+		}
+		if i > 0 && (r == '.' || r == ')') {
+			if i+1 < len(s) && s[i+1] == ' ' {
+				return true
+			}
+		}
+		break
+	}
+	return false
+}
+
+func stripListMarker(s string) string {
+	if strings.HasPrefix(s, "- ") || strings.HasPrefix(s, "* ") || strings.HasPrefix(s, "• ") {
+		return strings.TrimSpace(s[2:])
+	}
+	// Numbered: find first space after number marker
+	for i := 0; i < len(s); i++ {
+		if s[i] == ' ' {
+			return strings.TrimSpace(s[i+1:])
+		}
+	}
+	return s
+}
+
+func isIndentedCode(line string) bool {
+	if len(line) >= 4 && strings.HasPrefix(line, "    ") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") {
+			return false
+		}
+		return true
+	}
+	if len(line) >= 1 && strings.HasPrefix(line, "\t") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+func isOrderedListLine(s string) bool {
+	for i, r := range s {
+		if unicode.IsDigit(r) {
+			continue
+		}
+		if i > 0 && (r == '.' || r == ')') {
+			if i+1 < len(s) && s[i+1] == ' ' {
+				return true
+			}
+		}
+		break
+	}
+	return false
+}
+
+func extractLists(content *types.ExtractedContent) {
+	// Simple list extraction for bullet/numbered items within sections
+	// We scan paragraphs and group consecutive list-looking lines into types.List entries.
+	for _, sec := range content.Sections {
+		var currentList *types.List
+		newParagraphs := []string{}
+
+		for _, para := range sec.Paragraphs {
+			if isListLine(para) {
+				if currentList == nil {
+					currentList = &types.List{
+						Ordered: isOrderedListLine(para),
+					}
+				}
+				if currentList != nil {
+					currentList.Items = append(currentList.Items, stripListMarker(para))
+				}
+			} else {
+				if currentList != nil {
+					content.Lists = append(content.Lists, *currentList)
+					content.Metadata.TagCounts["lists"] = len(content.Lists)
+					currentList = nil
+				}
+				newParagraphs = append(newParagraphs, para)
+			}
+		}
+		if currentList != nil {
+			content.Lists = append(content.Lists, *currentList)
+			content.Metadata.TagCounts["lists"] = len(content.Lists)
+		}
+		sec.Paragraphs = newParagraphs
+	}
+}
+
+func truncate(s string, n int) string {
+	if len(s) > n {
+		return s[:n]
+	}
+	return s
+}