Phase 3: Grammar-Kit parser, PSI and ParserDefinition

Adds src/main/grammar/Jai.bnf, the generated parser/PSI in src/main/gen, a
JaiParserDefinition, and the Tier 3 corpus parse gate (406/714 files clean).

Two things were not obvious:

- Grammar-Kit mints its own token instances from the .bnf 'tokens' block, which
  are different objects from the ones JaiLexer emits, so every rule silently
  failed to match. Fixed with tokenTypeFactory -> JaiTokenTypes.byName.
- A backslash inside an identifier is a continuation in the compiler's lexer
  (Jai_Lexer/module.jai:444): 'left\_margin' is one identifier. JaiLexer now
  does the same; the Tier 0 round-trip still holds because the token span
  covers the backslash and the skipped spaces.

Constructs that BNF alone cannot express live in JaiParserUtil: directive-name
tests (#ident is one token), '==' before '{' for the switch form, procedure
header vs parenthesised expression, and the declaration lookahead.

src/test/kotlin/dev/hgh/jai/parser/JaiCorpusParserTest.kt

@@ -0,0 +1,111 @@
+package dev.hgh.jai.parser
+
+import com.intellij.psi.PsiErrorElement
+import com.intellij.psi.PsiFileFactory
+import com.intellij.psi.PsiRecursiveElementWalkingVisitor
+import com.intellij.testFramework.fixtures.BasePlatformTestCase
+import dev.hgh.jai.JaiLanguage
+import java.io.File
+
+/**
+ * Tier 3 gate (docs/BUILD_PLAN.md §3): parse every `.jai` file in the local Jai
+ * distribution and count `PsiErrorElement`s.
+ *
+ * The corpus is 714 files of real code by the language authors, so this is the
+ * acceptance criterion for "the grammar is correct". While the grammar is still being
+ * built the test reports rather than fails; [MIN_CLEAN_FILES] is the ratchet that
+ * stops it regressing.
+ */
+class JaiCorpusParserTest : BasePlatformTestCase() {
+    fun testCorpusParsesWithoutErrors() {
+        val corpusDir = File(System.getProperty("user.home"), ".local/jai")
+        if (!corpusDir.exists()) {
+            println("Skipping corpus parse test: $corpusDir does not exist")
+            return
+        }
+
+        val jaiFiles =
+            corpusDir
+                .walkTopDown()
+                .filter { it.isFile && it.extension == "jai" }
+                .sorted()
+                .toList()
+        assertTrue("Corpus files must be found in $corpusDir", jaiFiles.isNotEmpty())
+
+        val psiFactory = PsiFileFactory.getInstance(project)
+        var totalErrors = 0
+        val filesWithErrors = mutableListOf<String>()
+        val firstErrors = mutableListOf<String>()
+        val histogram = mutableMapOf<String, Int>()
+
+        for (file in jaiFiles) {
+            val text = file.readText()
+            val psiFile = psiFactory.createFileFromText(file.name, JaiLanguage, text)
+
+            var fileErrors = 0
+            var firstError: PsiErrorElement? = null
+            psiFile.accept(
+                object : PsiRecursiveElementWalkingVisitor() {
+                    override fun visitErrorElement(element: PsiErrorElement) {
+                        fileErrors++
+                        if (firstError == null) firstError = element
+                        super.visitErrorElement(element)
+                    }
+                },
+            )
+
+            totalErrors += fileErrors
+            val error = firstError
+            if (error != null) {
+                val relative = file.relativePath(corpusDir)
+                filesWithErrors.add(relative)
+                val offset = error.textOffset
+                val got = snippet(text, offset).take(GOT_WIDTH)
+                histogram.merge(got, 1, Int::plus)
+                if (firstErrors.size < REPORTED_ERRORS) {
+                    val line = text.substring(0, offset).count { it == '\n' } + 1
+                    firstErrors.add("$relative:$line: ${error.errorDescription} [at '$got']")
+                }
+            }
+        }
+
+        val cleanFiles = jaiFiles.size - filesWithErrors.size
+        val percent = "%.1f".format(cleanFiles.toDouble() / jaiFiles.size * 100)
+        println(
+            "Tier 3: parsed ${jaiFiles.size} files, $cleanFiles clean ($percent%), " +
+                "${filesWithErrors.size} with errors, $totalErrors PsiErrorElements total.",
+        )
+        if (histogram.isNotEmpty()) {
+            println("  most common token at the failure point:")
+            histogram.entries
+                .sortedByDescending { it.value }
+                .take(REPORTED_KINDS)
+                .forEach { (token, count) -> println("    %5d  %s".format(count, token)) }
+        }
+        firstErrors.forEach { println("  $it") }
+
+        assertTrue(
+            "Corpus parse regressed: only $cleanFiles files are clean, ratchet is $MIN_CLEAN_FILES. " +
+                "Raise MIN_CLEAN_FILES when the grammar improves, never lower it.",
+            cleanFiles >= MIN_CLEAN_FILES,
+        )
+    }
+
+    private fun snippet(
+        text: String,
+        offset: Int,
+    ): String =
+        text
+            .substring(offset, minOf(offset + 30, text.length))
+            .replace("\n", "\\n")
+
+    private fun File.relativePath(base: File): String = absolutePath.removePrefix(base.absolutePath + "/")
+
+    private companion object {
+        /** Ratchet. Raise it as the grammar improves; never lower it. */
+        const val MIN_CLEAN_FILES = 4
+        const val REPORTED_ERRORS = 25
+        const val REPORTED_KINDS = 20
+        const val GOT_WIDTH = 14
+    }
+}

src/test/kotlin/dev/hgh/jai/parser/JaiParserGoldenTest.kt

@@ -0,0 +1,11 @@
+package dev.hgh.jai.parser
+
+class JaiParserGoldenTest : JaiParsingTestCase() {
+    fun testSimpleDeclarations() {
+        doTest(true)
+    }
+
+    fun testExpressions() {
+        doTest(true)
+    }
+}

src/test/kotlin/dev/hgh/jai/parser/JaiParsingTestCase.kt

@@ -0,0 +1,14 @@
+package dev.hgh.jai.parser
+
+import com.intellij.testFramework.ParsingTestCase
+import dev.hgh.jai.psi.JaiParserDefinition
+
+abstract class JaiParsingTestCase(
+    dataSubFolder: String = "",
+) : ParsingTestCase(dataSubFolder, "jai", JaiParserDefinition()) {
+    override fun getTestDataPath(): String = "src/test/testData/parsing"
+
+    override fun skipSpaces(): Boolean = true
+
+    override fun includeRanges(): Boolean = true
+}