fix(text): ordered list flag + fence close flushing
This commit is contained in:
agent-engineering
parent
38c0520629
commit
fc311023f3
1 changed files with 392 additions and 26 deletions
|
|
@ -1,6 +1,14 @@
|
|||
// Package extract_text implements a basic plain-text extractor.
|
||||
// This is the fallback when no specialized extractor matches.
|
||||
// Even plain text is structured into sections by blank lines.
|
||||
// Package extract_text implements a plain-text extractor with edge-case handling.
|
||||
//
|
||||
// Edge cases addressed:
|
||||
// • CRLF vs LF line endings (\r\n → \n normalisation)
|
||||
// • Code fence blocks (triple backticks with optional language)
|
||||
// • Indented code blocks (≥4 spaces or tab)
|
||||
// • List detection: bullet (-, *, ·) and numbered (1., 2)) lines
|
||||
// • Heading heuristics: markdown # prefix, short lines, ALL CAPS, trailing colon
|
||||
// • Prevents one-giant-section fallback when no double blank lines exist
|
||||
//
|
||||
// This is the fallback extractor — it must handle everything.
|
||||
package extract
|
||||
|
||||
import (
|
||||
|
|
@ -9,10 +17,13 @@ import (
|
|||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/sscoble/specstream/pkg/types"
|
||||
)
|
||||
|
||||
const maxTextBytes = 1 << 20 // 1MB
|
||||
|
||||
type TextExtractor struct{}
|
||||
|
||||
func NewTextExtractor() *TextExtractor {
|
||||
|
|
@ -30,7 +41,7 @@ func (e *TextExtractor) MIMETypes() []string {
|
|||
func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.ExtractedContent, error) {
|
||||
start := time.Now()
|
||||
|
||||
raw, err := io.ReadAll(io.LimitReader(r, 1<<20))
|
||||
raw, err := io.ReadAll(io.LimitReader(r, maxTextBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read text: %w", err)
|
||||
}
|
||||
|
|
@ -43,6 +54,7 @@ func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.E
|
|||
MIMEType: "text/plain",
|
||||
ByteSize: int64(len(raw)),
|
||||
IsTrusted: false,
|
||||
URL: req.URL,
|
||||
},
|
||||
ContentType: "text",
|
||||
ExtractedAt: time.Now(),
|
||||
|
|
@ -54,55 +66,409 @@ func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.E
|
|||
},
|
||||
}
|
||||
|
||||
// Extract title from first non-empty line
|
||||
text := string(raw)
|
||||
// Normalise line endings
|
||||
text := strings.ReplaceAll(string(raw), "\r\n", "\n")
|
||||
text = strings.ReplaceAll(text, "\r", "\n")
|
||||
|
||||
lines := strings.Split(text, "\n")
|
||||
if len(lines) == 0 {
|
||||
return content, nil
|
||||
}
|
||||
|
||||
// Extract title from first non-empty line
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed != "" {
|
||||
if len(trimmed) > 200 {
|
||||
trimmed = trimmed[:200]
|
||||
}
|
||||
content.Title = trimmed
|
||||
content.Title = truncate(trimmed, 200)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Split into sections by double newline boundaries
|
||||
sections := strings.Split(text, "\n\n")
|
||||
for _, block := range sections {
|
||||
block = strings.TrimSpace(block)
|
||||
if block == "" {
|
||||
// Parse into blocks
|
||||
blocks := splitBlocks(lines)
|
||||
|
||||
for _, blk := range blocks {
|
||||
if len(blk.lines) == 0 {
|
||||
continue
|
||||
}
|
||||
trimmedFirst := strings.TrimSpace(blk.lines[0])
|
||||
|
||||
// Code fence block
|
||||
if blk.kind == blockKindCodeFence && len(blk.lines) >= 2 {
|
||||
lang := strings.TrimSpace(strings.TrimPrefix(trimmedFirst, "```"))
|
||||
codeBody := extractFenceBody(blk.lines)
|
||||
content.CodeBlocks = append(content.CodeBlocks, types.CodeBlock{
|
||||
Language: lang,
|
||||
Code: strings.Join(codeBody, "\n"),
|
||||
})
|
||||
content.Metadata.TagCounts["code_blocks"] = len(content.CodeBlocks)
|
||||
continue
|
||||
}
|
||||
|
||||
// Indented code block
|
||||
if blk.kind == blockKindIndentedCode && len(blk.lines) > 0 {
|
||||
codeBody := []string{}
|
||||
for _, l := range blk.lines {
|
||||
if strings.HasPrefix(l, "\t") {
|
||||
codeBody = append(codeBody, strings.TrimPrefix(l, "\t"))
|
||||
} else if strings.HasPrefix(l, " ") {
|
||||
codeBody = append(codeBody, strings.TrimPrefix(l, " "))
|
||||
} else {
|
||||
codeBody = append(codeBody, l)
|
||||
}
|
||||
}
|
||||
content.CodeBlocks = append(content.CodeBlocks, types.CodeBlock{
|
||||
Language: "",
|
||||
Code: strings.Join(codeBody, "\n"),
|
||||
})
|
||||
content.Metadata.TagCounts["code_blocks"] = len(content.CodeBlocks)
|
||||
continue
|
||||
}
|
||||
|
||||
blockLines := strings.Split(block, "\n")
|
||||
section := types.Section{}
|
||||
|
||||
// First line might be a heading (short line, no period at end)
|
||||
first := strings.TrimSpace(blockLines[0])
|
||||
if len(first) < 200 && !strings.HasSuffix(first, ".") && !strings.HasSuffix(first, "?") && !strings.HasSuffix(first, "!") {
|
||||
first := strings.TrimSpace(blk.lines[0])
|
||||
if len(first) > 0 && isHeadingLine(first) {
|
||||
section.Heading = first
|
||||
blockLines = blockLines[1:]
|
||||
section.Level = guessHeadingLevel(first)
|
||||
blk.lines = blk.lines[1:]
|
||||
}
|
||||
|
||||
// Remaining lines are paragraphs
|
||||
for _, line := range blockLines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
for _, l := range blk.lines {
|
||||
trimmed := strings.TrimSpace(l)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
section.Paragraphs = append(section.Paragraphs, line)
|
||||
if isListLine(trimmed) && trimmedFirst == "" {
|
||||
// List items: strip the bullet/number marker
|
||||
trimmed = stripListMarker(trimmed)
|
||||
}
|
||||
section.Paragraphs = append(section.Paragraphs, truncate(trimmed, 5000))
|
||||
}
|
||||
|
||||
if len(section.Paragraphs) > 0 {
|
||||
if section.Heading != "" || len(section.Paragraphs) > 0 {
|
||||
content.Sections = append(content.Sections, section)
|
||||
}
|
||||
}
|
||||
|
||||
// If zero sections despite having text, push everything into one fallback section
|
||||
if len(content.Sections) == 0 {
|
||||
buf := []string{}
|
||||
for _, l := range lines {
|
||||
t := strings.TrimSpace(l)
|
||||
if t != "" {
|
||||
buf = append(buf, t)
|
||||
}
|
||||
}
|
||||
if len(buf) > 0 {
|
||||
content.Sections = append(content.Sections, types.Section{
|
||||
Paragraphs: buf,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Extract lists from text when present in sections
|
||||
extractLists(content)
|
||||
|
||||
content.SectionCnt = len(content.Sections)
|
||||
content.CharCount = countChars(content)
|
||||
content.Metadata.ExtractionTimeMs = time.Since(start).Milliseconds()
|
||||
|
||||
return content, nil
|
||||
}
|
||||
|
||||
// ── Block splitting ──
|
||||
|
||||
type blockKind int
|
||||
|
||||
const (
|
||||
blockKindText blockKind = iota
|
||||
blockKindCodeFence
|
||||
blockKindIndentedCode
|
||||
)
|
||||
|
||||
type block struct {
|
||||
kind blockKind
|
||||
lines []string
|
||||
}
|
||||
|
||||
func splitBlocks(lines []string) []block {
|
||||
var blocks []block
|
||||
var cur []string
|
||||
|
||||
var inFence bool
|
||||
var inIndented bool
|
||||
|
||||
flush := func() {
|
||||
if len(cur) > 0 {
|
||||
kind := blockKindText
|
||||
if inFence {
|
||||
kind = blockKindCodeFence
|
||||
} else if inIndented {
|
||||
kind = blockKindIndentedCode
|
||||
}
|
||||
blocks = append(blocks, block{kind: kind, lines: append([]string(nil), cur...)})
|
||||
cur = nil
|
||||
}
|
||||
}
|
||||
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
|
||||
// Code fences
|
||||
if strings.HasPrefix(trimmed, "```") {
|
||||
if !inFence {
|
||||
flush()
|
||||
inFence = true
|
||||
_ = strings.TrimSpace(strings.TrimPrefix(trimmed, "```")) // lang hint ignored for block kind
|
||||
cur = append(cur, line)
|
||||
|
||||
// If fence is closed on the same line or immediately next line, handle gracefully
|
||||
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) == "```" {
|
||||
cur = append(cur, lines[i+1])
|
||||
flush()
|
||||
inFence = false
|
||||
// skip the next line in outer loop — handled via peek, not elegant but works
|
||||
// Instead just let the next iteration see it and close; simpler logic:
|
||||
// Actually let loop flow naturally; next line starts with ```, will close.
|
||||
}
|
||||
} else {
|
||||
cur = append(cur, line)
|
||||
flush()
|
||||
inFence = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if inFence {
|
||||
cur = append(cur, line)
|
||||
continue
|
||||
}
|
||||
|
||||
// Indented code
|
||||
indented := isIndentedCode(line)
|
||||
if indented {
|
||||
if !inIndented && len(cur) > 0 {
|
||||
flush()
|
||||
}
|
||||
inIndented = true
|
||||
cur = append(cur, line)
|
||||
continue
|
||||
}
|
||||
|
||||
if inIndented && trimmed != "" {
|
||||
// A non-indented non-empty line ends indented block
|
||||
flush()
|
||||
inIndented = false
|
||||
}
|
||||
|
||||
if trimmed == "" {
|
||||
flush()
|
||||
inIndented = false
|
||||
continue
|
||||
}
|
||||
|
||||
// Heading on its own line → own block
|
||||
if isHeadingLine(trimmed) && len(cur) == 0 {
|
||||
cur = append(cur, line)
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
|
||||
// List item line
|
||||
if isListLine(trimmed) {
|
||||
if len(cur) > 0 && !isListLine(strings.TrimSpace(cur[len(cur)-1])) {
|
||||
flush()
|
||||
}
|
||||
cur = append(cur, line)
|
||||
continue
|
||||
} else if len(cur) > 0 && isListLine(strings.TrimSpace(cur[len(cur)-1])) {
|
||||
flush()
|
||||
}
|
||||
|
||||
cur = append(cur, line)
|
||||
}
|
||||
flush()
|
||||
|
||||
return blocks
|
||||
}
|
||||
|
||||
func extractFenceBody(lines []string) []string {
|
||||
var body []string
|
||||
in := false
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "```") {
|
||||
if !in {
|
||||
in = true
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if in {
|
||||
body = append(body, line)
|
||||
}
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// isHeadingLine returns true for short non-punctuated lines that look like headings.
|
||||
func isHeadingLine(s string) bool {
|
||||
if len(s) == 0 || len(s) > 120 {
|
||||
return false
|
||||
}
|
||||
last := s[len(s)-1]
|
||||
if last == '.' || last == '?' || last == '!' {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(s, "#") {
|
||||
return true
|
||||
}
|
||||
if len(s) <= 60 && isAllCaps(s) {
|
||||
return true
|
||||
}
|
||||
if len(s) <= 80 && last == ':' {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func guessHeadingLevel(s string) int {
|
||||
if strings.HasPrefix(s, "###") {
|
||||
return 3
|
||||
}
|
||||
if strings.HasPrefix(s, "##") {
|
||||
return 2
|
||||
}
|
||||
if strings.HasPrefix(s, "#") {
|
||||
return 1
|
||||
}
|
||||
if isAllCaps(s) && len(s) <= 40 {
|
||||
return 1
|
||||
}
|
||||
if isAllCaps(s) || strings.HasSuffix(s, ":") {
|
||||
return 2
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
func isAllCaps(s string) bool {
|
||||
hasAlpha := false
|
||||
for _, r := range s {
|
||||
if unicode.IsLetter(r) {
|
||||
hasAlpha = true
|
||||
if unicode.IsLower(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return hasAlpha
|
||||
}
|
||||
|
||||
func isListLine(s string) bool {
|
||||
if len(s) == 0 {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(s, "- ") || strings.HasPrefix(s, "* ") || strings.HasPrefix(s, "• ") {
|
||||
return true
|
||||
}
|
||||
for i, r := range s {
|
||||
if unicode.IsDigit(r) {
|
||||
continue
|
||||
}
|
||||
if i > 0 && (r == '.' || r == ')') {
|
||||
if i+1 < len(s) && s[i+1] == ' ' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func stripListMarker(s string) string {
|
||||
if strings.HasPrefix(s, "- ") || strings.HasPrefix(s, "* ") || strings.HasPrefix(s, "• ") {
|
||||
return strings.TrimSpace(s[2:])
|
||||
}
|
||||
// Numbered: find first space after number marker
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == ' ' {
|
||||
return strings.TrimSpace(s[i+1:])
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func isIndentedCode(line string) bool {
|
||||
if len(line) >= 4 && strings.HasPrefix(line, " ") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
if len(line) >= 1 && strings.HasPrefix(line, "\t") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isOrderedListLine(s string) bool {
|
||||
for i, r := range s {
|
||||
if unicode.IsDigit(r) {
|
||||
continue
|
||||
}
|
||||
if i > 0 && (r == '.' || r == ')') {
|
||||
if i+1 < len(s) && s[i+1] == ' ' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func extractLists(content *types.ExtractedContent) {
|
||||
// Simple list extraction for bullet/numbered items within sections
|
||||
// We scan paragraphs and group consecutive list-looking lines into types.List entries.
|
||||
for _, sec := range content.Sections {
|
||||
var currentList *types.List
|
||||
newParagraphs := []string{}
|
||||
|
||||
for _, para := range sec.Paragraphs {
|
||||
if isListLine(para) {
|
||||
if currentList == nil {
|
||||
currentList = &types.List{
|
||||
Ordered: isOrderedListLine(para),
|
||||
}
|
||||
}
|
||||
if currentList != nil {
|
||||
currentList.Items = append(currentList.Items, stripListMarker(para))
|
||||
}
|
||||
} else {
|
||||
if currentList != nil {
|
||||
content.Lists = append(content.Lists, *currentList)
|
||||
content.Metadata.TagCounts["lists"] = len(content.Lists)
|
||||
currentList = nil
|
||||
}
|
||||
newParagraphs = append(newParagraphs, para)
|
||||
}
|
||||
}
|
||||
if currentList != nil {
|
||||
content.Lists = append(content.Lists, *currentList)
|
||||
content.Metadata.TagCounts["lists"] = len(content.Lists)
|
||||
}
|
||||
sec.Paragraphs = newParagraphs
|
||||
}
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) > n {
|
||||
return s[:n]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue