Phase 1 gate: Tier 0 corpus lexer invariants over the whole Jai distribution

Lexes all 714 .jai files (17.1M chars, 3.0M tokens) and asserts round-trip tiling, strict offset progress, and zero BAD_CHARACTER. Green on first run. jaitest: use cleanTest and retry once when a concurrent Gradle run clobbers build/test-results (java.io.EOFException with no XML).
2026-08-04 10:25:01 -04:00
parent 4469283024
commit ba5d8aba7b
2 changed files with 149 additions and 5 deletions
--- a/src/test/kotlin/dev/hgh/jai/lexer/JaiCorpusLexerTest.kt
+++ b/src/test/kotlin/dev/hgh/jai/lexer/JaiCorpusLexerTest.kt
@@ -0,0 +1,125 @@
+package dev.hgh.jai.lexer
+
+import com.intellij.psi.TokenType
+import org.junit.Assert.assertTrue
+import org.junit.Assume.assumeTrue
+import org.junit.Test
+import java.io.File
+
+/**
+ * Tier 0 — the flagship correctness gate (see `docs/BUILD_PLAN.md` §3).
+ *
+ * Lexes every `.jai` file in the local Jai distribution (~714 files, ~16 MB of real
+ * code written by the language's authors) and asserts the invariants an IntelliJ lexer
+ * must satisfy. Real-world coverage, no fixture boot, no human review.
+ */
+class JaiCorpusLexerTest {
+    private val corpusRoot = File(System.getProperty("user.home"), ".local/jai")
+
+    private fun corpusFiles(): List<File> = corpusRoot.walkTopDown().filter { it.isFile && it.extension == "jai" }.toList()
+
+    private fun assumeCorpusPresent() {
+        assumeTrue(
+            "Jai distribution not found at $corpusRoot — the corpus gate cannot run",
+            corpusRoot.isDirectory,
+        )
+    }
+
+    @Test
+    fun corpusIsSubstantial() {
+        assumeCorpusPresent()
+        val files = corpusFiles()
+        assertTrue("expected hundreds of corpus files, found ${files.size}", files.size > 500)
+    }
+
+    /**
+     * 1. Round-trip: the concatenation of all token texts reproduces the source exactly.
+     * 2. Progress: every token advances the offset (an infinite loop hangs the IDE
+     *    rather than failing visibly, so it must be caught here).
+     * 3. No BAD_CHARACTER anywhere in the distribution.
+     */
+    @Test
+    fun everyCorpusFileLexesCleanly() {
+        assumeCorpusPresent()
+        val problems = mutableListOf<String>()
+        var files = 0
+        var tokens = 0L
+        var bytes = 0L
+
+        for (file in corpusFiles()) {
+            val text = file.readText()
+            files++
+            bytes += text.length
+
+            val lexer = JaiLexer()
+            lexer.start(text, 0, text.length, 0)
+            var offset = 0
+            val rebuilt = StringBuilder(text.length)
+
+            while (true) {
+                val type = lexer.tokenType ?: break
+                tokens++
+
+                if (lexer.tokenStart != offset) {
+                    problems += "${where(file, text, offset)}: gap/overlap — " +
+                        "token starts at ${lexer.tokenStart}, previous ended at $offset"
+                    break
+                }
+                if (lexer.tokenEnd <= lexer.tokenStart) {
+                    problems += "${where(file, text, offset)}: no progress — " +
+                        "$type produced an empty token"
+                    break
+                }
+                if (type == TokenType.BAD_CHARACTER) {
+                    problems += "${where(file, text, offset)}: BAD_CHARACTER " +
+                        "'${text.substring(lexer.tokenStart, lexer.tokenEnd)}'"
+                }
+
+                rebuilt.append(text, lexer.tokenStart, lexer.tokenEnd)
+                offset = lexer.tokenEnd
+                lexer.advance()
+                if (problems.size > 25) break
+            }
+
+            if (offset != text.length) {
+                problems += "${where(file, text, offset)}: stopped at $offset of ${text.length}"
+            } else if (rebuilt.toString() != text) {
+                problems += "$file: round-trip mismatch at ${firstDifference(text, rebuilt)}"
+            }
+            if (problems.size > 25) break
+        }
+
+        assertTrue("no .jai files found under $corpusRoot", files > 0)
+        assertTrue(
+            "lexed $files files / $bytes chars / $tokens tokens; " +
+                "${problems.size} problem(s):\n" + problems.joinToString("\n"),
+            problems.isEmpty(),
+        )
+        println("Tier 0: lexed $files files, $bytes chars, $tokens tokens cleanly.")
+    }
+
+    private fun where(
+        file: File,
+        text: String,
+        offset: Int,
+    ): String {
+        var line = 1
+        var lineStart = 0
+        for (i in 0 until minOf(offset, text.length)) {
+            if (text[i] == '\n') {
+                line++
+                lineStart = i + 1
+            }
+        }
+        return "$file:$line:${offset - lineStart + 1}"
+    }
+
+    private fun firstDifference(
+        expected: String,
+        actual: CharSequence,
+    ): Int {
+        val n = minOf(expected.length, actual.length)
+        for (i in 0 until n) if (expected[i] != actual[i]) return i
+        return n
+    }
+}