diff --git a/examples/SQLOptimizer-runtime.ts b/examples/SQLOptimizer-runtime.ts new file mode 100644 index 0000000..1835a38 --- /dev/null +++ b/examples/SQLOptimizer-runtime.ts @@ -0,0 +1,596 @@ +// SQLOptimizer runtime — implements the actual SQL optimization logic +// that the SpeckDL state machine orchestrates. + +export type PlanOp = 'Scan' | 'Filter' | 'Join' | 'Project' | 'Aggregate' | 'Sort' | 'Limit'; + +export interface PlanNode { + id: string; + op: PlanOp; + table?: string; + predicate?: string; + joinType?: string; + leftChild?: string; + rightChild?: string; + onClause?: string; + columns?: string[]; + groupBy?: string; + orderBy?: string; + limit?: number; + estimatedRows: number; + children: string[]; +} + +export interface PlanTree { + nodes: Record; + root: string; +} + +export interface TableStats { + rowCount: number; + columns: Record; +} + +// ─── SQL Parser ───────────────────────────────────────────── + +interface SQLParts { + select: string; + from: string; + joins: { type: string; table: string; on: string }[]; + where: string; + groupBy: string; + orderBy: string; + limit: string; +} + +function parseSQL(sql: string): SQLParts { + const s = sql.trim().replace(/\s+/g, ' ').replace(/;$/, ''); + + // Extract clauses by keyword positions + const selectMatch = s.match(/\bSELECT\s+(.+?)\s+FROM\b/i); + const select = selectMatch ? selectMatch[1].trim() : '*'; + + // Extract FROM table (before any JOIN or WHERE or GROUP BY etc.) + const fromMatch = s.match(/\bFROM\s+(\w+)/i); + const from = fromMatch ? fromMatch[1] : ''; + + // Extract JOINs + const joins: { type: string; table: string; on: string }[] = []; + const joinRegex = /\b(INNER\s+JOIN|LEFT\s+JOIN|RIGHT\s+JOIN|FULL\s+JOIN|JOIN)\s+(\w+)(?:\s+AS\s+\w+)?\s+ON\s+(\w+\.\w+\s*=\s*\w+\.\w+)/gi; + let jm: RegExpExecArray | null; + while ((jm = joinRegex.exec(s)) !== null) { + joins.push({ + type: jm[1].toUpperCase().replace(/\s+/g, ' '), + table: jm[2], + on: jm[3].trim() + }); + } + + // Extract WHERE + const whereMatch = s.match(/\bWHERE\s+(.+?)(?:\s+GROUP\s+BY\b|\s+ORDER\s+BY\b|\s+LIMIT\b|\s*$)/i); + const where = whereMatch ? whereMatch[1].trim() : ''; + + // Extract GROUP BY + const groupMatch = s.match(/\bGROUP\s+BY\s+(.+?)(?:\s+ORDER\s+BY\b|\s+LIMIT\b|\s*$)/i); + const groupBy = groupMatch ? groupMatch[1].trim() : ''; + + // Extract ORDER BY + const orderMatch = s.match(/\bORDER\s+BY\s+(.+?)(?:\s+LIMIT\b|\s*$)/i); + const orderBy = orderMatch ? orderMatch[1].trim() : ''; + + // Extract LIMIT + const limitMatch = s.match(/\bLIMIT\s+(\d+)/i); + const limit = limitMatch ? limitMatch[1] : ''; + + return { select, from, joins, where, groupBy, orderBy, limit }; +} + +// ─── Plan Construction ────────────────────────────────────── + +let nodeIdCounter = 0; + +function nextId(prefix: string): string { + return `${prefix}_${nodeIdCounter++}`; +} + +export function resetIdCounter(): void { + nodeIdCounter = 0; +} + +export function buildInitialPlan(sql: string, stats: Record): PlanTree { + resetIdCounter(); + const parts = parseSQL(sql); + const nodes: Record = {}; + let currentRoot: string; + + // Build scan node for FROM table + const tableName = parts.from; + const tableRows = stats[tableName]?.rowCount ?? 1000; + const scanId = nextId('scan'); + nodes[scanId] = { id: scanId, op: 'Scan', table: tableName, estimatedRows: tableRows, children: [] }; + currentRoot = scanId; + + // Build JOIN nodes + for (const join of parts.joins) { + const joinRows = stats[join.table]?.rowCount ?? 1000; + const joinScanId = nextId('scan'); + nodes[joinScanId] = { id: joinScanId, op: 'Scan', table: join.table, estimatedRows: joinRows, children: [] }; + + const parentId = currentRoot; + const parentRows = nodes[parentId].estimatedRows; + const joinId = nextId('join'); + nodes[joinId] = { + id: joinId, op: 'Join', joinType: 'inner', + leftChild: parentId, rightChild: joinScanId, + onClause: join.on, + estimatedRows: Math.ceil(parentRows * joinRows * 0.1), + children: [parentId, joinScanId] + }; + currentRoot = joinId; + } + + // Add Filter if WHERE exists + if (parts.where) { + const filterId = nextId('filter'); + const selectivity = estimateSelectivity(parts.where); + const parentRows = nodes[currentRoot].estimatedRows; + nodes[filterId] = { + id: filterId, op: 'Filter', predicate: parts.where, + estimatedRows: Math.ceil(parentRows * selectivity), + children: [currentRoot] + }; + currentRoot = filterId; + } + + // Add Aggregate if GROUP BY exists + if (parts.groupBy) { + const aggId = nextId('aggregate'); + const parentRows = nodes[currentRoot].estimatedRows; + nodes[aggId] = { + id: aggId, op: 'Aggregate', groupBy: parts.groupBy, + estimatedRows: Math.min(parentRows, 100), + children: [currentRoot] + }; + currentRoot = aggId; + } + + // Add Sort if ORDER BY exists + if (parts.orderBy) { + const sortId = nextId('sort'); + const parentRows = nodes[currentRoot].estimatedRows; + nodes[sortId] = { + id: sortId, op: 'Sort', orderBy: parts.orderBy, + estimatedRows: parentRows, + children: [currentRoot] + }; + currentRoot = sortId; + } + + // Add Limit + if (parts.limit) { + const limitId = nextId('limit'); + nodes[limitId] = { + id: limitId, op: 'Limit', limit: Number(parts.limit), + estimatedRows: Number(parts.limit), + children: [currentRoot] + }; + currentRoot = limitId; + } + + // Add Project + if (parts.select && parts.select !== '*') { + const projectId = nextId('project'); + const parentRows = nodes[currentRoot].estimatedRows; + const columns = parts.select.split(',').map(c => c.trim()); + nodes[projectId] = { + id: projectId, op: 'Project', columns, + estimatedRows: parentRows, + children: [currentRoot] + }; + currentRoot = projectId; + } + + return { nodes, root: currentRoot }; +} + +// ─── Selectivity Estimation ───────────────────────────────── + +function estimateSelectivity(predicate: string): number { + if (!predicate) return 1.0; + // Each AND condition reduces by a factor + const conditions = predicate.split(/\band\b/i).length; + return Math.pow(0.3, conditions); +} + +// ─── Rewrite Rules ────────────────────────────────────────── + +export function pushDownFilters(tree: PlanTree): PlanTree { + const nodes = { ...tree.nodes }; + let root = tree.root; + + // Find Filter nodes directly above Join nodes and push predicates to the correct side + function pushDown(nodeId: string): string { + const node = nodes[nodeId]; + if (!node) return nodeId; + + // Recurse into children + node.children = node.children.map(pushDown); + + if (node.op === 'Filter' && nodes[node.children[0]]?.op === 'Join') { + const joinNode = nodes[node.children[0]]; + const predicate = node.predicate || ''; + + // Check which side the predicate references + const leftTable = nodes[joinNode.leftChild!]?.table ?? ''; + const rightTable = nodes[joinNode.rightChild!]?.table ?? ''; + const predCols = extractColumnRefs(predicate); + + const onlyLeft = predCols.every(c => c.startsWith(leftTable + '.') || !c.includes('.')); + const onlyRight = predCols.every(c => c.startsWith(rightTable + '.') || !c.includes('.')); + + if (onlyLeft && joinNode.leftChild) { + // Push filter to left child + const filterId = nextId('filter'); + nodes[filterId] = { + id: filterId, op: 'Filter', predicate, + estimatedRows: Math.ceil(nodes[joinNode.leftChild].estimatedRows * 0.3), + children: [joinNode.leftChild] + }; + joinNode.leftChild = filterId; + joinNode.children = [filterId, joinNode.rightChild!]; + joinNode.estimatedRows = Math.ceil(joinNode.estimatedRows * 0.7); + return joinNode.id; + } else if (onlyRight && joinNode.rightChild) { + // Push filter to right child + const filterId = nextId('filter'); + nodes[filterId] = { + id: filterId, op: 'Filter', predicate, + estimatedRows: Math.ceil(nodes[joinNode.rightChild].estimatedRows * 0.3), + children: [joinNode.rightChild] + }; + joinNode.rightChild = filterId; + joinNode.children = [joinNode.leftChild!, filterId]; + joinNode.estimatedRows = Math.ceil(joinNode.estimatedRows * 0.7); + return joinNode.id; + } + } + + return nodeId; + } + + root = pushDown(root); + return { nodes, root }; +} + +export function foldConstants(tree: PlanTree): PlanTree { + const nodes = { ...tree.nodes }; + + function fold(nodeId: string): void { + const node = nodes[nodeId]; + if (!node) return; + node.children.forEach(fold); + + if (node.op === 'Filter' && node.predicate) { + node.predicate = node.predicate + .replace(/\b(\d+)\s*\+\s*(\d+)\b/g, (_, a, b) => String(Number(a) + Number(b))) + .replace(/\b(\d+)\s*-\s*(\d+)\b/g, (_, a, b) => String(Number(a) - Number(b))) + .replace(/\b(\d+)\s*\*\s*(\d+)\b/g, (_, a, b) => String(Number(a) * Number(b))); + } + } + + fold(tree.root); + return { nodes, root: tree.root }; +} + +export function pruneUnusedColumns(tree: PlanTree): PlanTree { + // Collect all referenced columns from predicates, GROUP BY, ORDER BY + const nodes = { ...tree.nodes }; + const required = new Set(); + + function collectRefs(nodeId: string): void { + const node = nodes[nodeId]; + if (!node) return; + if (node.predicate) extractColumnRefs(node.predicate).forEach(c => required.add(c)); + if (node.groupBy) extractColumnRefs(node.groupBy).forEach(c => required.add(c)); + if (node.orderBy) extractColumnRefs(node.orderBy).forEach(c => required.add(c)); + if (node.columns) node.columns.forEach(c => required.add(c)); + node.children.forEach(collectRefs); + } + + collectRefs(tree.root); + + // Prune Project nodes + function prune(nodeId: string): void { + const node = nodes[nodeId]; + if (!node) return; + node.children.forEach(prune); + if (node.op === 'Project' && node.columns) { + node.columns = node.columns.filter(c => required.has(c) || c === '*'); + if (node.columns!.length === 0) node.columns = ['*']; + } + } + + prune(tree.root); + return { nodes, root: tree.root }; +} + +function extractColumnRefs(expr: string): string[] { + const refs: string[] = []; + const sqlKw = new Set(['select','from','where','and','or','not','join','inner','left','right', + 'full','on','as','is','null','in','between','like','order','by','group','limit', + 'having','distinct','asc','desc','count','sum','avg','min','max','true','false']); + const matches = expr.matchAll(/\b(\w+)\.(\w+)\b/g); + for (const m of matches) { + const ref = `${m[1]}.${m[2]}`; + if (!sqlKw.has(m[1].toLowerCase()) && !sqlKw.has(m[2].toLowerCase())) { + refs.push(ref); + } + } + return [...new Set(refs)]; +} + +// ─── Physical Plan Selection ──────────────────────────────── + +export function choosePhysicalOperators(tree: PlanTree, stats: Record): PlanTree { + const nodes = { ...tree.nodes }; + + function choose(nodeId: string): void { + const node = nodes[nodeId]; + if (!node) return; + node.children.forEach(choose); + + // Choose join algorithm based on input sizes + if (node.op === 'Join' && node.leftChild && node.rightChild) { + const leftRows = nodes[node.leftChild]?.estimatedRows ?? 1000; + const rightRows = nodes[node.rightChild]?.estimatedRows ?? 1000; + + if (leftRows > 10000 || rightRows > 10000) { + node.joinType = 'hash'; + } else if (leftRows < 100 || rightRows < 100) { + node.joinType = 'nested_loop'; + } else { + node.joinType = 'merge'; + } + } + + // Choose index scan for large tables with filters + if (node.op === 'Scan' && node.table) { + const parent = Object.values(nodes).find(n => n.children.includes(nodeId)); + if (parent?.op === 'Filter') { + const tableStats = stats[node.table]; + if (tableStats && tableStats.rowCount > 10000) { + // Mark as index scan + node.table = node.table + ' /* index_scan */'; + } + } + } + } + + choose(tree.root); + return { nodes, root: tree.root }; +} + +// ─── Cost Estimation ─────────────────────────────────────── + +export function estimateCost(nodes: Record, rootId: string): number { + const node = nodes[rootId]; + if (!node) return 0; + + const childCosts = node.children.reduce((sum, c) => sum + estimateCost(nodes, c), 0); + let selfCost = 0; + + switch (node.op) { + case 'Scan': + selfCost = node.estimatedRows; + break; + case 'Filter': + selfCost = Math.ceil(node.estimatedRows * 0.01); + break; + case 'Join': { + const leftRows = node.leftChild ? (nodes[node.leftChild]?.estimatedRows ?? 1000) : 1000; + const rightRows = node.rightChild ? (nodes[node.rightChild]?.estimatedRows ?? 1000) : 1000; + if (node.joinType === 'hash') { + selfCost = leftRows + rightRows + Math.ceil(leftRows * 0.5); + } else if (node.joinType === 'nested_loop') { + selfCost = leftRows * rightRows; + } else { + selfCost = leftRows + rightRows + Math.ceil(leftRows * 0.3); + } + break; + } + case 'Aggregate': + selfCost = Math.ceil(node.estimatedRows * 0.5); + break; + case 'Sort': + selfCost = Math.ceil(node.estimatedRows * Math.log2(Math.max(node.estimatedRows, 2))); + break; + case 'Project': + selfCost = Math.ceil(node.estimatedRows * 0.001); + break; + case 'Limit': + selfCost = 1; + break; + default: + selfCost = node.estimatedRows; + } + + return childCosts + selfCost; +} + +// ─── SQL Generation ──────────────────────────────────────── + +export function generateSQL(nodes: Record, rootId: string): string { + let select = '*'; + const fromTables: string[] = []; + const joins: string[] = []; + let where = ''; + let groupBy = ''; + let orderBy = ''; + let limit = ''; + + function walk(nid: string): void { + const n = nodes[nid]; + if (!n) return; + + switch (n.op) { + case 'Scan': + if (n.table) fromTables.push(n.table); + break; + case 'Filter': + if (where) { + where = `(${where}) AND (${n.predicate})`; + } else { + where = n.predicate ?? ''; + } + break; + case 'Join': { + const jt = n.joinType === 'hash' ? 'INNER' : n.joinType === 'nested_loop' ? 'INNER' : (n.joinType?.toUpperCase() ?? 'INNER'); + const rt = nodes[n.rightChild!]?.table ?? 'unknown'; + const lt = nodes[n.leftChild!]?.table ?? 'unknown'; + const onClause = n.onClause ?? `${rt}.id = ${lt}.id`; + joins.push(`${jt} JOIN ${rt} ON ${onClause}`); + if (n.leftChild) walk(n.leftChild); + if (n.rightChild) walk(n.rightChild); + return; + } + case 'Aggregate': + if (n.groupBy) { + select = `${n.groupBy}, COUNT(*)`; + groupBy = n.groupBy; + } else { + select = 'COUNT(*)'; + } + break; + case 'Sort': + orderBy = n.orderBy ?? ''; + break; + case 'Project': + if (n.columns && n.columns.length > 0 && !n.columns.includes('*')) { + select = n.columns.join(', '); + } + break; + case 'Limit': + limit = String(n.limit ?? ''); + break; + } + n.children.forEach(walk); + } + + walk(rootId); + + let sql = `SELECT ${select} FROM ${fromTables.join(', ')}`; + if (joins.length > 0) sql += ' ' + joins.join(' '); + if (where) sql += ` WHERE ${where}`; + if (groupBy) sql += ` GROUP BY ${groupBy}`; + if (orderBy) sql += ` ORDER BY ${orderBy}`; + if (limit) sql += ` LIMIT ${limit}`; + return sql + ';'; +} + +// ─── Utility ─────────────────────────────────────────────── + +export function stringOfPlan(nodes: Record, rootId: string, indent: number = 0): string { + const node = nodes[rootId]; + if (!node) return ''; + const prefix = ' '.repeat(indent); + let line = `${prefix}${node.op}`; + if (node.table) line += ` [${node.table}]`; + if (node.predicate) line += ` {${node.predicate}}`; + if (node.joinType) line += ` (${node.joinType})`; + line += ` rows=${node.estimatedRows}`; + + const children = node.children.map(c => stringOfPlan(nodes, c, indent + 1)).filter(Boolean); + return [line, ...children].join('\n'); +} + +// ─── Sample Statistics ────────────────────────────────────── + +export const sampleStats: Record = { + users: { + rowCount: 100000, + columns: { + id: { ndv: 100000, nullFraction: 0, avgWidth: 4 }, + name: { ndv: 95000, nullFraction: 0.01, avgWidth: 24 }, + email: { ndv: 100000, nullFraction: 0, avgWidth: 32 }, + created_at: { ndv: 90000, nullFraction: 0, avgWidth: 8 }, + } + }, + orders: { + rowCount: 500000, + columns: { + id: { ndv: 500000, nullFraction: 0, avgWidth: 4 }, + user_id: { ndv: 80000, nullFraction: 0, avgWidth: 4 }, + total: { ndv: 400000, nullFraction: 0.005, avgWidth: 8 }, + status: { ndv: 5, nullFraction: 0, avgWidth: 12 }, + created_at: { ndv: 450000, nullFraction: 0, avgWidth: 8 }, + } + }, + products: { + rowCount: 10000, + columns: { + id: { ndv: 10000, nullFraction: 0, avgWidth: 4 }, + name: { ndv: 9800, nullFraction: 0.01, avgWidth: 48 }, + price: { ndv: 8500, nullFraction: 0, avgWidth: 8 }, + category: { ndv: 50, nullFraction: 0, avgWidth: 24 }, + } + }, + order_items: { + rowCount: 1500000, + columns: { + id: { ndv: 1500000, nullFraction: 0, avgWidth: 4 }, + order_id: { ndv: 500000, nullFraction: 0, avgWidth: 4 }, + product_id: { ndv: 10000, nullFraction: 0, avgWidth: 4 }, + quantity: { ndv: 20, nullFraction: 0, avgWidth: 4 }, + price: { ndv: 5000, nullFraction: 0, avgWidth: 8 }, + } + } +}; + +// ─── Demo ─────────────────────────────────────────────────── + +export function demo(): void { + const sql = "SELECT users.name, COUNT(*) FROM users INNER JOIN orders ON users.id = orders.user_id WHERE users.created_at > '2024-01-01' AND orders.status = 'active' GROUP BY users.name ORDER BY COUNT(*) DESC LIMIT 10"; + + console.log('=== SQL Query Optimizer Demo ===\n'); + console.log('Input SQL:'); + console.log(sql); + console.log(); + + // Parse + const initialPlan = buildInitialPlan(sql, sampleStats); + console.log('Initial Plan:'); + console.log(stringOfPlan(initialPlan.nodes, initialPlan.root)); + const initialCost = estimateCost(initialPlan.nodes, initialPlan.root); + console.log(`Initial Cost: ${initialCost}\n`); + + // Rewrite: push down filters + const afterPushDown = pushDownFilters(initialPlan); + console.log('After Predicate Pushdown:'); + console.log(stringOfPlan(afterPushDown.nodes, afterPushDown.root)); + const pushDownCost = estimateCost(afterPushDown.nodes, afterPushDown.root); + console.log(`Cost after pushdown: ${pushDownCost}\n`); + + // Rewrite: fold constants + const afterFold = foldConstants(afterPushDown); + console.log('After Constant Folding:'); + console.log(stringOfPlan(afterFold.nodes, afterFold.root)); + const foldCost = estimateCost(afterFold.nodes, afterFold.root); + console.log(`Cost after folding: ${foldCost}\n`); + + // Physical plan + const physicalPlan = choosePhysicalOperators(afterFold, sampleStats); + console.log('Physical Plan:'); + console.log(stringOfPlan(physicalPlan.nodes, physicalPlan.root)); + const physicalCost = estimateCost(physicalPlan.nodes, physicalPlan.root); + console.log(`Physical Cost: ${physicalCost}\n`); + + // Generate optimized SQL + const optimizedSQL = generateSQL(physicalPlan.nodes, physicalPlan.root); + console.log('Optimized SQL:'); + console.log(optimizedSQL); + console.log(); + + console.log('=== Summary ==='); + console.log(`Initial cost: ${initialCost}`); + console.log(`Final cost: ${physicalCost}`); + console.log(`Improvement: ${((1 - physicalCost / initialCost) * 100).toFixed(1)}%`); +} \ No newline at end of file diff --git a/examples/SQLOptimizer.speckdl b/examples/SQLOptimizer.speckdl index 7dcf3dc..bd52313 100644 --- a/examples/SQLOptimizer.speckdl +++ b/examples/SQLOptimizer.speckdl @@ -1,63 +1,61 @@ -// SQLOptimizer.speckdl — A cost-based SQL query optimizer -// Models the full optimization pipeline: parse → rewrite → plan → execute -// Exercises: state-dependent transforms, cost models, map-based catalog, -// collection guards, conditional emits, and rollback semantics. +// SQLOptimizer.speckdl — A functional cost-based SQL query optimizer +// The generated TypeScript actually optimizes queries. +// It parses SQL, applies rewrite rules, estimates costs, and picks physical plans. speck SQLOptimizer { - version: "0.1.0" + version: "0.2.0" author: "sscoble" license: "MIT" // ─── State ──────────────────────────────────────────────── state: { phase: OptimizerPhase, - queryText: String, - logicalPlan: PlanNode, - physicalPlan: PlanNode, - estimatedCost: CostModel, - catalog: SchemaCatalog, - stats: TableStats, - rewriteRules: RuleSet, - planCache: PlanCache + originalSQL: String, + planNodes: PlanNodeMap, + planEdges: PlanEdgeMap, + rootPlanId: String, + catalog: CatalogMap, + stats: StatsMap, + rulesApplied: Nat, + bestCost: Nat, + resultSQL: String, + errors: StringList } init: { phase := Idle, - queryText := "", - logicalPlan := {}, - physicalPlan := {}, - estimatedCost := {}, + originalSQL := "", + planNodes := {}, + planEdges := {}, + rootPlanId := "", catalog := {}, stats := {}, - rewriteRules := {}, - planCache := {} + rulesApplied := 0, + bestCost := 0, + resultSQL := "", + errors := [] } // ─── Events ─────────────────────────────────────────────── event OptimizationStarted { - query: String, + sql: String, at: String } - event PlanRewritten { - rule: String, - before: String, - after: String, - costBefore: Nat, - costAfter: Nat, + event RuleApplied { + ruleName: String, + planBefore: String, + planAfter: String, + costDelta: Nat, at: String } - event PlanGenerated { - planType: PlanType, - estimatedRows: Nat, - estimatedCost: Nat, - at: String - } - - event OptimizationCompleted { - totalRulesApplied: Nat, - finalCost: Nat, + event PlanOptimized { + originalSQL: String, + optimizedSQL: String, + originalCost: Nat, + optimizedCost: Nat, + rulesApplied: Nat, at: String } @@ -68,226 +66,233 @@ speck SQLOptimizer { } // ─── Actions ────────────────────────────────────────────── - action optimize(query: String) { + + // Accept a SQL query string and begin optimization + action submit(sql: String) { require phase === Idle - require len(query) > 0 + require len(sql) > 0 this.phase := Parsing - this.queryText := query - this.logicalPlan := {} + this.originalSQL := sql + this.errors := [] - emit OptimizationStarted { query: query, at: now() } + emit OptimizationStarted { sql: sql, at: now() } } + // Parse SQL into a logical plan tree + // Builds PlanNode objects representing Scan, Filter, Join, Project, Aggregate, Sort action parse() { require phase === Parsing - require len(queryText) > 0 + require len(originalSQL) > 0 - // Lexical + syntactic analysis → raw AST - // Semantic validation: table/column existence, type checking - this.logicalPlan := parseSQL(queryText) + // Parse SELECT ... FROM ... WHERE ... GROUP BY ... ORDER BY ... + let sql := originalSQL - // Fail fast on syntax errors - if logicalPlan == {} { + // Extract FROM clause — identify table names + let fromClause := extractFrom(sql) + if len(fromClause) == 0 { this.phase := Failed - emit OptimizationFailed { phase: Parsing, message: "Syntax error in query", at: now() } + this.errors := ["No FROM clause found"] + emit OptimizationFailed { phase: Parsing, message: "No FROM clause found", at: now() } return } + // Extract WHERE clause — identify predicates + let whereClause := extractWhere(sql) + let groupByClause := extractGroupBy(sql) + let orderByClause := extractOrderBy(sql) + let selectClause := extractSelect(sql) + + // Build initial plan: Scan → Filter → Join → Project + let scanId := "scan_0" + this.planNodes := {} + this.planNodes := setPlanNode(planNodes, scanId, { op: "Scan", table: fromClause, estimatedRows: lookupRowCount(stats, fromClause), children: [], filter: "" }) + + let currentRoot := scanId + + // Add filter if WHERE exists + if len(whereClause) > 0 { + let filterId := "filter_0" + let filterRows := estimateFilterSelectivity(stats, fromClause, whereClause, lookupRowCount(stats, fromClause)) + this.planNodes := setPlanNode(planNodes, filterId, { op: "Filter", predicate: whereClause, estimatedRows: filterRows, children: [currentRoot] }) + currentRoot := filterId + } + + // Add GROUP BY if present + if len(groupByClause) > 0 { + let aggId := "aggregate_0" + this.planNodes := setPlanNode(planNodes, aggId, { op: "Aggregate", groupBy: groupByClause, estimatedRows: 1, children: [currentRoot] }) + currentRoot := aggId + } + + // Add ORDER BY if present + if len(orderByClause) > 0 { + let sortId := "sort_0" + this.planNodes := setPlanNode(planNodes, sortId, { op: "Sort", orderBy: orderByClause, estimatedRows: 1, children: [currentRoot] }) + currentRoot := sortId + } + + // Add projection + let projectId := "project_0" + this.planNodes := setPlanNode(planNodes, projectId, { op: "Project", columns: selectClause, estimatedRows: 1, children: [currentRoot] }) + currentRoot := projectId + + this.rootPlanId := currentRoot + this.bestCost := estimateCost(planNodes, rootPlanId) this.phase := Rewriting - emit PlanRewritten { rule: "parse", before: queryText, after: stringOf(logicalPlan), costBefore: 0, costAfter: estimatedCostOf(logicalPlan), at: now() } } + // Apply logical rewrite rules action rewrite() { require phase === Rewriting - // Apply logical rewrites: predicate pushdown, column pruning, - // subquery flattening, join reordering (logical), constant folding - let rulesApplied := 0 - let plan := logicalPlan - let prevCost := estimatedCostOf(plan) + let costBefore := bestCost + let rulesThisRound := 0 - // Predicate pushdown: move filters closer to scans - let pushed := pushDownPredicates(plan, catalog) - if estimatedCostOf(pushed) < prevCost { - plan := pushed - prevCost := estimatedCostOf(pushed) - rulesApplied := rulesApplied + 1 - emit PlanRewritten { rule: "predicate_pushdown", before: stringOf(logicalPlan), after: stringOf(pushed), costBefore: prevCost, costAfter: estimatedCostOf(pushed), at: now() } + // Rule 1: Predicate pushdown — move filters closer to scans + let pushed := pushDownFilters(planNodes, rootPlanId) + let pushedCost := estimateCost(pushed.nodes, pushed.root) + if pushedCost < bestCost { + this.planNodes := pushed.nodes + this.rootPlanId := pushed.root + this.bestCost := pushedCost + this.rulesApplied := rulesApplied + 1 + rulesThisRound := rulesThisRound + 1 + emit RuleApplied { ruleName: "predicate_pushdown", planBefore: stringOfPlan(planNodes, rootPlanId), planAfter: stringOfPlan(pushed.nodes, pushed.root), costDelta: costBefore - pushedCost, at: now() } + costBefore := pushedCost } - // Column pruning: remove unused projections - let pruned := pruneColumns(plan, catalog) - if estimatedCostOf(pruned) < prevCost { - plan := pruned - prevCost := estimatedCostOf(pruned) - rulesApplied := rulesApplied + 1 - emit PlanRewritten { rule: "column_pruning", before: stringOf(logicalPlan), after: stringOf(pruned), costBefore: prevCost, costAfter: estimatedCostOf(pruned), at: now() } + // Rule 2: Column pruning — remove unused columns from projections + let pruned := pruneUnusedColumns(planNodes, rootPlanId) + let prunedCost := estimateCost(pruned.nodes, pruned.root) + if prunedCost < bestCost { + this.planNodes := pruned.nodes + this.rootPlanId := pruned.root + this.bestCost := prunedCost + this.rulesApplied := rulesApplied + 1 + rulesThisRound := rulesThisRound + 1 + emit RuleApplied { ruleName: "column_pruning", planBefore: stringOfPlan(planNodes, rootPlanId), planAfter: stringOfPlan(pruned.nodes, pruned.root), costDelta: costBefore - prunedCost, at: now() } + costBefore := prunedCost } - // Subquery flattening: convert subqueries to joins where possible - let flattened := flattenSubqueries(plan, catalog) - if estimatedCostOf(flattened) < prevCost { - plan := flattened - prevCost := estimatedCostOf(flattened) - rulesApplied := rulesApplied + 1 - emit PlanRewritten { rule: "subquery_flatten", before: stringOf(logicalPlan), after: stringOf(flattened), costBefore: prevCost, costAfter: estimatedCostOf(flattened), at: now() } + // Rule 3: Constant folding — evaluate compile-time expressions + let folded := foldConstants(planNodes, rootPlanId) + let foldedCost := estimateCost(folded.nodes, folded.root) + if foldedCost < bestCost { + this.planNodes := folded.nodes + this.rootPlanId := folded.root + this.bestCost := foldedCost + this.rulesApplied := rulesApplied + 1 + rulesThisRound := rulesThisRound + 1 + emit RuleApplied { ruleName: "constant_folding", planBefore: stringOfPlan(planNodes, rootPlanId), planAfter: stringOfPlan(folded.nodes, folded.root), costDelta: costBefore - foldedCost, at: now() } } - // Constant folding: evaluate compile-time expressions - let folded := foldConstants(plan) - if estimatedCostOf(folded) < prevCost { - plan := folded - rulesApplied := rulesApplied + 1 - emit PlanRewritten { rule: "constant_folding", before: stringOf(logicalPlan), after: stringOf(folded), costBefore: prevCost, costAfter: estimatedCostOf(folded), at: now() } - } - - this.logicalPlan := plan this.phase := Planning } - action plan() { + // Choose physical operators and join order + action planPhysical() { require phase === Planning - // Cost-based join enumeration: consider join orderings - // Choose physical operators: hash join vs merge join vs nested loop - // Use catalog stats (row counts, cardinality, histograms) for cost estimation + // Choose join algorithm based on table sizes + let physical := choosePhysicalOperators(planNodes, rootPlanId, stats) + let physicalCost := estimateCost(physical.nodes, physical.root) - let bestPlan := choosePhysicalPlan(logicalPlan, catalog, stats) - let cost := estimatedCostOf(bestPlan) - - this.physicalPlan := bestPlan - this.estimatedCost := cost - - // Check plan cache for identical query shape - let cached := lookupPlan(planCache, logicalPlan) - if cached != {} and estimatedCostOf(cached) < cost { - this.physicalPlan := cached - this.estimatedCost := estimatedCostOf(cached) + if physicalCost < bestCost { + this.planNodes := physical.nodes + this.rootPlanId := physical.root + this.bestCost := physicalCost } - emit PlanGenerated { - planType: Physical, - estimatedRows: estimatedRowsOf(physicalPlan), - estimatedCost: cost.estimatedCost, + this.phase := Generating + } + + // Generate optimized SQL from the physical plan + action generate() { + require phase === Generating + + let optimized := generateSQL(planNodes, rootPlanId) + + this.resultSQL := optimized + this.phase := Done + + emit PlanOptimized { + originalSQL: originalSQL, + optimizedSQL: optimized, + originalCost: estimateCost(planNodes, rootPlanId), + optimizedCost: bestCost, + rulesApplied: rulesApplied, at: now() } - - this.phase := Ready - } - - action execute() { - require phase === Ready - - // Execute the physical plan against the storage engine - // Stream results, track actual vs estimated cardinalities - this.phase := Executing - } - - action complete() { - require phase === Executing - - // Collect execution statistics - // Update plan cache if actual cost diverges significantly from estimate - let rulesApplied := countRewrites(planCache) - - this.phase := Done - emit OptimizationCompleted { totalRulesApplied: rulesApplied, finalCost: estimatedCost.estimatedCost, at: now() } - } - - action fail(message: String) { - // Any phase can transition to Failed - this.phase := Failed - emit OptimizationFailed { phase: phase, message: message, at: now() } } + // Reset for next query action reset() { this.phase := Idle - this.queryText := "" - this.logicalPlan := {} - this.physicalPlan := {} - this.estimatedCost := {} + this.originalSQL := "" + this.planNodes := {} + this.planEdges := {} + this.rootPlanId := "" this.catalog := {} this.stats := {} - this.rewriteRules := defaultRules() - this.planCache := {} + this.rulesApplied := 0 + this.bestCost := 0 + this.resultSQL := "" + this.errors := [] } // ─── Constraints ────────────────────────────────────────── - constraint C1: phase === Idle || len(queryText) > 0 - // Non-idle phases require a query + constraint C1: phase === Idle || len(originalSQL) > 0 + // Non-idle implies a query is loaded - constraint C2: estimatedCost > 0 implies phase >= Planning - // A cost estimate implies planning has occurred + constraint C2: phase === Done implies len(resultSQL) > 0 + // Done implies we produced output - constraint C3: phase === Ready implies physicalPlan != {} - // Ready phase requires a physical plan + constraint C3: phase === Failed implies len(errors) > 0 + // Failure implies error messages exist - constraint C4: phase === Done implies estimatedCost > 0 - // Completion requires a cost estimate - - constraint C5: phase === Failed implies len(queryText) > 0 - // Failure requires a query was submitted + constraint C4: rulesApplied > 0 implies bestCost < estimateCost(planNodes, rootPlanId) + bestCost + // Rules only fire when they reduce cost (loose — cost can't increase from applying rules) // ─── Temporal Verifications ─────────────────────────────── verify V1: always(implies(phase === Parsing, eventually(phase === Rewriting || phase === Failed))) - // Parsing must complete or fail - verify V2: always(implies(phase === Rewriting, eventually(phase === Planning || phase === Failed))) - // Rewriting must reach planning or fail - - verify V3: always(implies(phase === Planning, eventually(phase === Ready || phase === Failed))) - // Planning must produce a plan or fail - - verify V4: always(implies(phase === Ready, eventually(phase === Done || phase === Failed))) - // A ready plan must execute or fail + verify V3: always(implies(phase === Generating, eventually(phase === Done))) // ─── Provenance ─────────────────────────────────────────── provenance { wasGeneratedBy(sql-optimizer, SQLOptimizer, "2026-05-13T00:00:00Z") used(SQLOptimizer, cascades-volcano, "framework") - used(SQLOptimizer, postgres-planner, "reference") - used(SQLOptimizer, z3, "verification") } // ─── BOM ────────────────────────────────────────────────── bom { - optimizer: "sql-optimizer@0.1.0" + optimizer: "sql-optimizer@0.2.0" runtime: "node@22.0.0" - solver: "z3@4.13.0" } // ─── Supporting Types ───────────────────────────────────── interface OptimizerPhase { - Idle, Parsing, Rewriting, Planning, Ready, Executing, Done, Failed + Idle, Parsing, Rewriting, Planning, Generating, Done, Failed } - interface PlanType { - Logical, Physical, Cached + interface PlanNodeMap { + // Map from plan node ID to PlanNode object } - interface PlanNode { - // Opaque — represents a query plan tree (scan, join, filter, project, aggregate, sort) + interface PlanEdgeMap { + // Map from parent ID to child IDs } - interface CostModel { - // Captures: estimated IO, CPU, memory, network cost - // Plus cardinality estimates per operator + interface CatalogMap { + // Map from table name to schema info } - interface SchemaCatalog { - // Table definitions, column types, indexes, constraints + interface StatsMap { + // Map from table name to row count and column statistics } - interface TableStats { - // Row counts, column histograms, NDV, null fractions - } - - interface RuleSet { - // Ordered list of rewrite rules with enable/disable flags - } - - interface PlanCache { - // Maps query fingerprints to cached physical plans + interface StringList { + // List of strings } } \ No newline at end of file