474 lines
11 KiB
Go
474 lines
11 KiB
Go
// Package extract_text implements a plain-text extractor with edge-case handling.
|
|
//
|
|
// Edge cases addressed:
|
|
// • CRLF vs LF line endings (\r\n → \n normalisation)
|
|
// • Code fence blocks (triple backticks with optional language)
|
|
// • Indented code blocks (≥4 spaces or tab)
|
|
// • List detection: bullet (-, *, ·) and numbered (1., 2)) lines
|
|
// • Heading heuristics: markdown # prefix, short lines, ALL CAPS, trailing colon
|
|
// • Prevents one-giant-section fallback when no double blank lines exist
|
|
//
|
|
// This is the fallback extractor — it must handle everything.
|
|
package extract
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
|
|
"github.com/sscoble/specstream/pkg/types"
|
|
)
|
|
|
|
const maxTextBytes = 1 << 20 // 1MB
|
|
|
|
type TextExtractor struct{}
|
|
|
|
func NewTextExtractor() *TextExtractor {
|
|
return &TextExtractor{}
|
|
}
|
|
|
|
func (e *TextExtractor) Detect(r io.ReaderAt, size int64) (bool, error) {
|
|
return true, nil // universal fallback
|
|
}
|
|
|
|
func (e *TextExtractor) MIMETypes() []string {
|
|
return []string{"text/plain"}
|
|
}
|
|
|
|
func (e *TextExtractor) Extract(r io.Reader, req types.ExtractRequest) (*types.ExtractedContent, error) {
|
|
start := time.Now()
|
|
|
|
raw, err := io.ReadAll(io.LimitReader(r, maxTextBytes))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read text: %w", err)
|
|
}
|
|
|
|
hash := sha256.Sum256(raw)
|
|
fingerprint := fmt.Sprintf("%x", hash)
|
|
|
|
content := &types.ExtractedContent{
|
|
Source: types.Source{
|
|
MIMEType: "text/plain",
|
|
ByteSize: int64(len(raw)),
|
|
IsTrusted: false,
|
|
URL: req.URL,
|
|
},
|
|
ContentType: "text",
|
|
ExtractedAt: time.Now(),
|
|
Metadata: types.Metadata{
|
|
ExtractorVersion: "specstream/0.1.0",
|
|
SHA256: fingerprint,
|
|
StrippedPatterns: make(map[string]int),
|
|
TagCounts: make(map[string]int),
|
|
},
|
|
}
|
|
|
|
// Normalise line endings
|
|
text := strings.ReplaceAll(string(raw), "\r\n", "\n")
|
|
text = strings.ReplaceAll(text, "\r", "\n")
|
|
|
|
lines := strings.Split(text, "\n")
|
|
if len(lines) == 0 {
|
|
return content, nil
|
|
}
|
|
|
|
// Extract title from first non-empty line
|
|
for _, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
if trimmed != "" {
|
|
content.Title = truncate(trimmed, 200)
|
|
break
|
|
}
|
|
}
|
|
|
|
// Parse into blocks
|
|
blocks := splitBlocks(lines)
|
|
|
|
for _, blk := range blocks {
|
|
if len(blk.lines) == 0 {
|
|
continue
|
|
}
|
|
trimmedFirst := strings.TrimSpace(blk.lines[0])
|
|
|
|
// Code fence block
|
|
if blk.kind == blockKindCodeFence && len(blk.lines) >= 2 {
|
|
lang := strings.TrimSpace(strings.TrimPrefix(trimmedFirst, "```"))
|
|
codeBody := extractFenceBody(blk.lines)
|
|
content.CodeBlocks = append(content.CodeBlocks, types.CodeBlock{
|
|
Language: lang,
|
|
Code: strings.Join(codeBody, "\n"),
|
|
})
|
|
content.Metadata.TagCounts["code_blocks"] = len(content.CodeBlocks)
|
|
continue
|
|
}
|
|
|
|
// Indented code block
|
|
if blk.kind == blockKindIndentedCode && len(blk.lines) > 0 {
|
|
codeBody := []string{}
|
|
for _, l := range blk.lines {
|
|
if strings.HasPrefix(l, "\t") {
|
|
codeBody = append(codeBody, strings.TrimPrefix(l, "\t"))
|
|
} else if strings.HasPrefix(l, " ") {
|
|
codeBody = append(codeBody, strings.TrimPrefix(l, " "))
|
|
} else {
|
|
codeBody = append(codeBody, l)
|
|
}
|
|
}
|
|
content.CodeBlocks = append(content.CodeBlocks, types.CodeBlock{
|
|
Language: "",
|
|
Code: strings.Join(codeBody, "\n"),
|
|
})
|
|
content.Metadata.TagCounts["code_blocks"] = len(content.CodeBlocks)
|
|
continue
|
|
}
|
|
|
|
section := types.Section{}
|
|
|
|
first := strings.TrimSpace(blk.lines[0])
|
|
if len(first) > 0 && isHeadingLine(first) {
|
|
section.Heading = first
|
|
section.Level = guessHeadingLevel(first)
|
|
blk.lines = blk.lines[1:]
|
|
}
|
|
|
|
for _, l := range blk.lines {
|
|
trimmed := strings.TrimSpace(l)
|
|
if trimmed == "" {
|
|
continue
|
|
}
|
|
if isListLine(trimmed) && trimmedFirst == "" {
|
|
// List items: strip the bullet/number marker
|
|
trimmed = stripListMarker(trimmed)
|
|
}
|
|
section.Paragraphs = append(section.Paragraphs, truncate(trimmed, 5000))
|
|
}
|
|
|
|
if section.Heading != "" || len(section.Paragraphs) > 0 {
|
|
content.Sections = append(content.Sections, section)
|
|
}
|
|
}
|
|
|
|
// If zero sections despite having text, push everything into one fallback section
|
|
if len(content.Sections) == 0 {
|
|
buf := []string{}
|
|
for _, l := range lines {
|
|
t := strings.TrimSpace(l)
|
|
if t != "" {
|
|
buf = append(buf, t)
|
|
}
|
|
}
|
|
if len(buf) > 0 {
|
|
content.Sections = append(content.Sections, types.Section{
|
|
Paragraphs: buf,
|
|
})
|
|
}
|
|
}
|
|
|
|
// Extract lists from text when present in sections
|
|
extractLists(content)
|
|
|
|
content.SectionCnt = len(content.Sections)
|
|
content.CharCount = countChars(content)
|
|
content.Metadata.ExtractionTimeMs = time.Since(start).Milliseconds()
|
|
|
|
return content, nil
|
|
}
|
|
|
|
// ── Block splitting ──
|
|
|
|
type blockKind int
|
|
|
|
const (
|
|
blockKindText blockKind = iota
|
|
blockKindCodeFence
|
|
blockKindIndentedCode
|
|
)
|
|
|
|
type block struct {
|
|
kind blockKind
|
|
lines []string
|
|
}
|
|
|
|
func splitBlocks(lines []string) []block {
|
|
var blocks []block
|
|
var cur []string
|
|
|
|
var inFence bool
|
|
var inIndented bool
|
|
|
|
flush := func() {
|
|
if len(cur) > 0 {
|
|
kind := blockKindText
|
|
if inFence {
|
|
kind = blockKindCodeFence
|
|
} else if inIndented {
|
|
kind = blockKindIndentedCode
|
|
}
|
|
blocks = append(blocks, block{kind: kind, lines: append([]string(nil), cur...)})
|
|
cur = nil
|
|
}
|
|
}
|
|
|
|
for i, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
|
|
// Code fences
|
|
if strings.HasPrefix(trimmed, "```") {
|
|
if !inFence {
|
|
flush()
|
|
inFence = true
|
|
_ = strings.TrimSpace(strings.TrimPrefix(trimmed, "```")) // lang hint ignored for block kind
|
|
cur = append(cur, line)
|
|
|
|
// If fence is closed on the same line or immediately next line, handle gracefully
|
|
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) == "```" {
|
|
cur = append(cur, lines[i+1])
|
|
flush()
|
|
inFence = false
|
|
// skip the next line in outer loop — handled via peek, not elegant but works
|
|
// Instead just let the next iteration see it and close; simpler logic:
|
|
// Actually let loop flow naturally; next line starts with ```, will close.
|
|
}
|
|
} else {
|
|
cur = append(cur, line)
|
|
flush()
|
|
inFence = false
|
|
}
|
|
continue
|
|
}
|
|
|
|
if inFence {
|
|
cur = append(cur, line)
|
|
continue
|
|
}
|
|
|
|
// Indented code
|
|
indented := isIndentedCode(line)
|
|
if indented {
|
|
if !inIndented && len(cur) > 0 {
|
|
flush()
|
|
}
|
|
inIndented = true
|
|
cur = append(cur, line)
|
|
continue
|
|
}
|
|
|
|
if inIndented && trimmed != "" {
|
|
// A non-indented non-empty line ends indented block
|
|
flush()
|
|
inIndented = false
|
|
}
|
|
|
|
if trimmed == "" {
|
|
flush()
|
|
inIndented = false
|
|
continue
|
|
}
|
|
|
|
// Heading on its own line → own block
|
|
if isHeadingLine(trimmed) && len(cur) == 0 {
|
|
cur = append(cur, line)
|
|
flush()
|
|
continue
|
|
}
|
|
|
|
// List item line
|
|
if isListLine(trimmed) {
|
|
if len(cur) > 0 && !isListLine(strings.TrimSpace(cur[len(cur)-1])) {
|
|
flush()
|
|
}
|
|
cur = append(cur, line)
|
|
continue
|
|
} else if len(cur) > 0 && isListLine(strings.TrimSpace(cur[len(cur)-1])) {
|
|
flush()
|
|
}
|
|
|
|
cur = append(cur, line)
|
|
}
|
|
flush()
|
|
|
|
return blocks
|
|
}
|
|
|
|
func extractFenceBody(lines []string) []string {
|
|
var body []string
|
|
in := false
|
|
for _, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
if strings.HasPrefix(trimmed, "```") {
|
|
if !in {
|
|
in = true
|
|
continue
|
|
}
|
|
break
|
|
}
|
|
if in {
|
|
body = append(body, line)
|
|
}
|
|
}
|
|
return body
|
|
}
|
|
|
|
// isHeadingLine returns true for short non-punctuated lines that look like headings.
|
|
func isHeadingLine(s string) bool {
|
|
if len(s) == 0 || len(s) > 120 {
|
|
return false
|
|
}
|
|
last := s[len(s)-1]
|
|
if last == '.' || last == '?' || last == '!' {
|
|
return false
|
|
}
|
|
if strings.HasPrefix(s, "#") {
|
|
return true
|
|
}
|
|
if len(s) <= 60 && isAllCaps(s) {
|
|
return true
|
|
}
|
|
if len(s) <= 80 && last == ':' {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func guessHeadingLevel(s string) int {
|
|
if strings.HasPrefix(s, "###") {
|
|
return 3
|
|
}
|
|
if strings.HasPrefix(s, "##") {
|
|
return 2
|
|
}
|
|
if strings.HasPrefix(s, "#") {
|
|
return 1
|
|
}
|
|
if isAllCaps(s) && len(s) <= 40 {
|
|
return 1
|
|
}
|
|
if isAllCaps(s) || strings.HasSuffix(s, ":") {
|
|
return 2
|
|
}
|
|
return 2
|
|
}
|
|
|
|
func isAllCaps(s string) bool {
|
|
hasAlpha := false
|
|
for _, r := range s {
|
|
if unicode.IsLetter(r) {
|
|
hasAlpha = true
|
|
if unicode.IsLower(r) {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return hasAlpha
|
|
}
|
|
|
|
func isListLine(s string) bool {
|
|
if len(s) == 0 {
|
|
return false
|
|
}
|
|
if strings.HasPrefix(s, "- ") || strings.HasPrefix(s, "* ") || strings.HasPrefix(s, "• ") {
|
|
return true
|
|
}
|
|
for i, r := range s {
|
|
if unicode.IsDigit(r) {
|
|
continue
|
|
}
|
|
if i > 0 && (r == '.' || r == ')') {
|
|
if i+1 < len(s) && s[i+1] == ' ' {
|
|
return true
|
|
}
|
|
}
|
|
break
|
|
}
|
|
return false
|
|
}
|
|
|
|
func stripListMarker(s string) string {
|
|
if strings.HasPrefix(s, "- ") || strings.HasPrefix(s, "* ") || strings.HasPrefix(s, "• ") {
|
|
return strings.TrimSpace(s[2:])
|
|
}
|
|
// Numbered: find first space after number marker
|
|
for i := 0; i < len(s); i++ {
|
|
if s[i] == ' ' {
|
|
return strings.TrimSpace(s[i+1:])
|
|
}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func isIndentedCode(line string) bool {
|
|
if len(line) >= 4 && strings.HasPrefix(line, " ") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
if len(line) >= 1 && strings.HasPrefix(line, "\t") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if strings.HasPrefix(trimmed, "- ") || strings.HasPrefix(trimmed, "* ") {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isOrderedListLine(s string) bool {
|
|
for i, r := range s {
|
|
if unicode.IsDigit(r) {
|
|
continue
|
|
}
|
|
if i > 0 && (r == '.' || r == ')') {
|
|
if i+1 < len(s) && s[i+1] == ' ' {
|
|
return true
|
|
}
|
|
}
|
|
break
|
|
}
|
|
return false
|
|
}
|
|
|
|
func extractLists(content *types.ExtractedContent) {
|
|
// Simple list extraction for bullet/numbered items within sections
|
|
// We scan paragraphs and group consecutive list-looking lines into types.List entries.
|
|
for _, sec := range content.Sections {
|
|
var currentList *types.List
|
|
newParagraphs := []string{}
|
|
|
|
for _, para := range sec.Paragraphs {
|
|
if isListLine(para) {
|
|
if currentList == nil {
|
|
currentList = &types.List{
|
|
Ordered: isOrderedListLine(para),
|
|
}
|
|
}
|
|
if currentList != nil {
|
|
currentList.Items = append(currentList.Items, stripListMarker(para))
|
|
}
|
|
} else {
|
|
if currentList != nil {
|
|
content.Lists = append(content.Lists, *currentList)
|
|
content.Metadata.TagCounts["lists"] = len(content.Lists)
|
|
currentList = nil
|
|
}
|
|
newParagraphs = append(newParagraphs, para)
|
|
}
|
|
}
|
|
if currentList != nil {
|
|
content.Lists = append(content.Lists, *currentList)
|
|
content.Metadata.TagCounts["lists"] = len(content.Lists)
|
|
}
|
|
sec.Paragraphs = newParagraphs
|
|
}
|
|
}
|
|
|
|
func truncate(s string, n int) string {
|
|
if len(s) > n {
|
|
return s[:n]
|
|
}
|
|
return s
|
|
}
|