Phase 1 gate: Tier 0 corpus lexer invariants over the whole Jai distribution

Lexes all 714 .jai files (17.1M chars, 3.0M tokens) and asserts round-trip
tiling, strict offset progress, and zero BAD_CHARACTER. Green on first run.

jaitest: use cleanTest and retry once when a concurrent Gradle run clobbers
build/test-results (java.io.EOFException with no XML).

jaitest

@@ -4,19 +4,38 @@
 # A green `test` task does not prove tests ran (see AGENTS.md), so this always
 # prints tests/failures/errors per suite and dumps failure messages.
 #
+# Two concurrent Gradle invocations (e.g. an editor/agent running tests in the
+# background at the same time) fight over build/test-results and the loser dies
+# with `java.io.EOFException` or a missing in-progress-results bin. That is an
+# infrastructure failure, not a test failure, so it is retried once.
+#
 # Usage: ./jaitest [gradle args...]      e.g. ./jaitest --tests '*Lexer*'
 set -uo pipefail
 
 cd "$(dirname "$0")" || exit 1
 
-rm -rf build/test-results/test
-./jaigradle test "$@" 2>&1 | grep -vE '^\[[0-9.]+s\]\[warning\]\[cds\]'
-gradle_status=${PIPESTATUS[0]}
+run_gradle() {
+    rm -rf build/reports/tests/test
+    ./jaigradle cleanTest test "$@" 2>&1 | grep -vE '^\[[0-9.]+s\]\[warning\]\[cds\]'
+    return "${PIPESTATUS[0]}"
+}
+
+run_gradle "$@"
+gradle_status=$?
+
+shopt -s nullglob
+xml=(build/test-results/test/*.xml)
+if [ "$gradle_status" -ne 0 ] && [ ${#xml[@]} -eq 0 ]; then
+    echo
+    echo "!!! no test XML and a failed build — retrying once (concurrent Gradle run?)"
+    sleep 3
+    run_gradle "$@"
+    gradle_status=$?
+    xml=(build/test-results/test/*.xml)
+fi
 
 echo
 echo "=== JUnit XML ==="
-shopt -s nullglob
-xml=(build/test-results/test/*.xml)
 if [ ${#xml[@]} -eq 0 ]; then
     echo "NO TEST XML PRODUCED — tests did not run."
     exit 1

src/test/kotlin/dev/hgh/jai/lexer/JaiCorpusLexerTest.kt

@@ -0,0 +1,125 @@
+package dev.hgh.jai.lexer
+
+import com.intellij.psi.TokenType
+import org.junit.Assert.assertTrue
+import org.junit.Assume.assumeTrue
+import org.junit.Test
+import java.io.File
+
+/**
+ * Tier 0 — the flagship correctness gate (see `docs/BUILD_PLAN.md` §3).
+ *
+ * Lexes every `.jai` file in the local Jai distribution (~714 files, ~16 MB of real
+ * code written by the language's authors) and asserts the invariants an IntelliJ lexer
+ * must satisfy. Real-world coverage, no fixture boot, no human review.
+ */
+class JaiCorpusLexerTest {
+    private val corpusRoot = File(System.getProperty("user.home"), ".local/jai")
+
+    private fun corpusFiles(): List<File> = corpusRoot.walkTopDown().filter { it.isFile && it.extension == "jai" }.toList()
+
+    private fun assumeCorpusPresent() {
+        assumeTrue(
+            "Jai distribution not found at $corpusRoot — the corpus gate cannot run",
+            corpusRoot.isDirectory,
+        )
+    }
+
+    @Test
+    fun corpusIsSubstantial() {
+        assumeCorpusPresent()
+        val files = corpusFiles()
+        assertTrue("expected hundreds of corpus files, found ${files.size}", files.size > 500)
+    }
+
+    /**
+     * 1. Round-trip: the concatenation of all token texts reproduces the source exactly.
+     * 2. Progress: every token advances the offset (an infinite loop hangs the IDE
+     *    rather than failing visibly, so it must be caught here).
+     * 3. No BAD_CHARACTER anywhere in the distribution.
+     */
+    @Test
+    fun everyCorpusFileLexesCleanly() {
+        assumeCorpusPresent()
+        val problems = mutableListOf<String>()
+        var files = 0
+        var tokens = 0L
+        var bytes = 0L
+
+        for (file in corpusFiles()) {
+            val text = file.readText()
+            files++
+            bytes += text.length
+
+            val lexer = JaiLexer()
+            lexer.start(text, 0, text.length, 0)
+            var offset = 0
+            val rebuilt = StringBuilder(text.length)
+
+            while (true) {
+                val type = lexer.tokenType ?: break
+                tokens++
+
+                if (lexer.tokenStart != offset) {
+                    problems += "${where(file, text, offset)}: gap/overlap — " +
+                        "token starts at ${lexer.tokenStart}, previous ended at $offset"
+                    break
+                }
+                if (lexer.tokenEnd <= lexer.tokenStart) {
+                    problems += "${where(file, text, offset)}: no progress — " +
+                        "$type produced an empty token"
+                    break
+                }
+                if (type == TokenType.BAD_CHARACTER) {
+                    problems += "${where(file, text, offset)}: BAD_CHARACTER " +
+                        "'${text.substring(lexer.tokenStart, lexer.tokenEnd)}'"
+                }
+
+                rebuilt.append(text, lexer.tokenStart, lexer.tokenEnd)
+                offset = lexer.tokenEnd
+                lexer.advance()
+                if (problems.size > 25) break
+            }
+
+            if (offset != text.length) {
+                problems += "${where(file, text, offset)}: stopped at $offset of ${text.length}"
+            } else if (rebuilt.toString() != text) {
+                problems += "$file: round-trip mismatch at ${firstDifference(text, rebuilt)}"
+            }
+            if (problems.size > 25) break
+        }
+
+        assertTrue("no .jai files found under $corpusRoot", files > 0)
+        assertTrue(
+            "lexed $files files / $bytes chars / $tokens tokens; " +
+                "${problems.size} problem(s):\n" + problems.joinToString("\n"),
+            problems.isEmpty(),
+        )
+        println("Tier 0: lexed $files files, $bytes chars, $tokens tokens cleanly.")
+    }
+
+    private fun where(
+        file: File,
+        text: String,
+        offset: Int,
+    ): String {
+        var line = 1
+        var lineStart = 0
+        for (i in 0 until minOf(offset, text.length)) {
+            if (text[i] == '\n') {
+                line++
+                lineStart = i + 1
+            }
+        }
+        return "$file:$line:${offset - lineStart + 1}"
+    }
+
+    private fun firstDifference(
+        expected: String,
+        actual: CharSequence,
+    ): Int {
+        val n = minOf(expected.length, actual.length)
+        for (i in 0 until n) if (expected[i] != actual[i]) return i
+        return n
+    }
+}