Parser: raise corpus parse rate from 57% to 69%

Grammar gains: array types, directive flags, declaration-level directive modifiers, parameter markers, backticked declaration names, notes after a block-valued declaration, 'if cond then stmt', 'ifx c else v', '#module_parameters' two-group form, '(.*)' prefix dereference, and '#asm' bodies consumed opaquely (they are x86-64 assembly, not Jai). Return items no longer take a default value: allowing one made 'f: (K) -> u32 = null' swallow the enclosing parameter's default and read the rest of the parameter list as extra return values.
2026-08-04 12:13:26 -04:00
parent 37809b8aa7
commit 840a503e40
19 changed files with 727 additions and 220 deletions
--- a/src/test/kotlin/dev/hgh/jai/parser/DebugParseTest.kt
+++ b/src/test/kotlin/dev/hgh/jai/parser/DebugParseTest.kt
@@ -0,0 +1,104 @@
+package dev.hgh.jai.parser
+
+import com.intellij.psi.PsiErrorElement
+import com.intellij.psi.PsiFileFactory
+import com.intellij.psi.PsiRecursiveElementWalkingVisitor
+import com.intellij.testFramework.fixtures.BasePlatformTestCase
+import dev.hgh.jai.JaiLanguage
+import java.io.File
+
+/**
+ * Scratch harness for grammar work. Not a gate — the gate is [JaiCorpusParserTest].
+ *
+ * Set `-Djai.debug.file=<path>` to dump every error in one corpus file instead.
+ */
+class DebugParseTest : BasePlatformTestCase() {
+    private val snippets =
+        listOf(
+            "tzname: **u8 #elsewhere libc;\n",
+            "S :: struct {\n    #as unknown: IUnknown;\n}\n",
+            "x.magic = (.*) cast(*u32) DATA.data;\n",
+            "M :: #library,system,link_always \"Metal\";\n",
+            "f :: (s: *u8, \$strict := false) -> u32 { return 1; }\n",
+            "#module_parameters (MAX := 4, VERBOSE := false);\n",
+            "Allocator_Proc :: #type (mode: int, old: *void) -> *void;\n",
+            "SimpleProcedure :: #type () -> ();\n",
+            "n := tprint(\"%\", ifx options.output_path else \".\");\n",
+            "v :: (info: *File_Visit_Info, modules : *[..] string) { }\n",
+            "B :: struct (type: Type, by_pointer := false) { }\n",
+            "e := ifx a then b;\n",
+        )
+
+    /** Corpus files to dump the first error of; edit freely while working. */
+    private val corpusFiles =
+        listOf(
+            "modules/Sound_Player/os/win32.jai",
+            "examples/module_info.jai",
+            "how_to/800_allocators.jai",
+            "modules/POSIX/module.jai",
+            "modules/Basic/module.jai",
+            "modules/Compiler/module.jai",
+            "modules/Window_Creation/module.jai",
+            "modules/String/module.jai",
+            "modules/Hash_Table.jai",
+            "modules/Math/module.jai",
+        )
+
+    fun testCorpusFiles() {
+        val root = File(System.getProperty("user.home"), ".local/jai")
+        if (!root.exists()) return
+        for (relative in corpusFiles) {
+            val file = File(root, relative)
+            if (file.exists()) dumpFile(file) else println("missing: $relative")
+        }
+    }
+
+    fun testSnippets() {
+        val debugFile = System.getProperty("jai.debug.file")
+        if (debugFile != null) {
+            dumpFile(File(debugFile))
+            return
+        }
+        val factory = PsiFileFactory.getInstance(project)
+        var failures = 0
+        for (text in snippets) {
+            val file = factory.createFileFromText("t.jai", JaiLanguage, text)
+            val error = firstError(file)
+            if (error == null) {
+                println("ok   ${text.replace("\n", "\\n")}")
+            } else {
+                failures++
+                val at = text.substring(error.textOffset).take(28).replace("\n", "\\n")
+                println("FAIL ${text.replace("\n", "\\n")}")
+                println("       ${error.errorDescription} | at '$at'")
+            }
+        }
+        println("$failures/${snippets.size} snippets failed")
+    }
+
+    private fun dumpFile(file: File) {
+        val text = file.readText()
+        val psi = PsiFileFactory.getInstance(project).createFileFromText(file.name, JaiLanguage, text)
+        val error = firstError(psi)
+        if (error == null) {
+            println("${file.name}: clean")
+            return
+        }
+        val line = text.substring(0, error.textOffset).count { it == '\n' } + 1
+        println("${file.name}:$line: ${error.errorDescription}")
+        println("  at '${text.substring(error.textOffset).take(80).replace("\n", "\\n")}'")
+    }
+
+    private fun firstError(file: com.intellij.psi.PsiFile): PsiErrorElement? {
+        var first: PsiErrorElement? = null
+        file.accept(
+            object : PsiRecursiveElementWalkingVisitor() {
+                override fun visitErrorElement(element: PsiErrorElement) {
+                    if (first == null) first = element
+                    super.visitErrorElement(element)
+                }
+            },
+        )
+        return first
+    }
+}