grant/intellijai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parser: corpus parse rate 69% -> 81%, and stop --tests from sticking

Browse Source 
Grammar: 'using,except(x)' modifiers, '#ifx' with block branches, unions with
parameters, multi-value initializers, mixed 'name:, lvalue = call()' targets,
'push_context' with no explicit context, 'operator []=', '#foreign lib "symbol"',
keyword-named arguments such as 'remove={...}', and '#insert (options) body'.
Directive flags now require adjacency: without it, in
'(callback: (*GUID) #c_call, lpContext: *void)' the flag swallowed ', lpContext'.

procModifier is now just directiveExpr — one rule for directives in both
positions, so fixes apply everywhere.

jaitest: a '--tests' filter persisted through Gradle's configuration cache, so a
later unfiltered ./jaitest silently re-ran only that class and printed OK.
Unfiltered runs now disable the configuration cache.

MIN_CLEAN_FILES ratchets the corpus gate at 575/714.
This commit is contained in:
hgranthorner
2026-08-04 12:20:36 -04:00
parent 840a503e40
commit 086ae993c5
24 changed files with 728 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/test/kotlin/dev/hgh/jai/parser/DebugParseTest.kt
View File

@@ -1,6 +1,7 @@
package dev.hgh.jai.parser
import com.intellij.psi.PsiErrorElement
import com.intellij.psi.PsiFile
import com.intellij.psi.PsiFileFactory
import com.intellij.psi.PsiRecursiveElementWalkingVisitor
import com.intellij.testFramework.fixtures.BasePlatformTestCase
@@ -9,39 +10,30 @@ import java.io.File
/**
* Scratch harness for grammar work. Not a gate — the gate is [JaiCorpusParserTest].
*
* Set `-Djai.debug.file=<path>` to dump every error in one corpus file instead.
* Edit [snippets] and [corpusFiles] freely while iterating on `Jai.bnf`.
*/
class DebugParseTest : BasePlatformTestCase() {
private val snippets =
listOf(
"tzname: **u8 #elsewhere libc;\n",
"S :: struct {\n #as unknown: IUnknown;\n}\n",
"x.magic = (.*) cast(*u32) DATA.data;\n",
"M :: #library,system,link_always \"Metal\";\n",
"f :: (s: *u8, \$strict := false) -> u32 { return 1; }\n",
"#module_parameters (MAX := 4, VERBOSE := false);\n",
"Allocator_Proc :: #type (mode: int, old: *void) -> *void;\n",
"SimpleProcedure :: #type () -> ();\n",
"n := tprint(\"%\", ifx options.output_path else \".\");\n",
"v :: (info: *File_Visit_Info, modules : *[..] string) { }\n",
"B :: struct (type: Type, by_pointer := false) { }\n",
"e := ifx a then b;\n",
"#scope_file\n\n\nbig_endian :: () -> bool {\n return true;\n}\n",
"S :: struct {\n #as using,except(vtable) iunknown: IUnknown;\n}\n",
"using,except(FT_Init) new_module :: #import \"freetype\";\n",
"operator []= :: inline (a: *B, i: int, v: bool) { }\n",
"inotify_init :: (flags: s32 = 0) -> s32 #foreign libc \"inotify_init1\";\n",
"F :: (a: *L) -> E #foreign ft #deprecated \"msg\";\n",
)
/** Corpus files to dump the first error of; edit freely while working. */
/** Corpus files to dump the first error of. */
private val corpusFiles =
listOf(
"modules/Sound_Player/os/win32.jai",
"examples/module_info.jai",
"how_to/800_allocators.jai",
"modules/POSIX/module.jai",
"modules/Basic/module.jai",
"modules/Compiler/module.jai",
"modules/Window_Creation/module.jai",
"modules/String/module.jai",
"modules/Hash_Table.jai",
"modules/Math/module.jai",
"modules/Basic/Array.jai",
"modules/Debug/windows.jai",
"modules/Simp/backend/gl.jai",
"modules/GetRect/system/occlusion.jai",
"modules/POSIX/compare_bindings.jai",
"modules/Android/Toolchain/apk.jai",
"modules/Curl/examples/ftp.jai",
"modules/Objective_C/LightweightRenderingView/module.jai",
)
fun testCorpusFiles() {
@@ -54,11 +46,6 @@ class DebugParseTest : BasePlatformTestCase() {
}
fun testSnippets() {
val debugFile = System.getProperty("jai.debug.file")
if (debugFile != null) {
dumpFile(File(debugFile))
return
}
val factory = PsiFileFactory.getInstance(project)
var failures = 0
for (text in snippets) {
@@ -89,7 +76,7 @@ class DebugParseTest : BasePlatformTestCase() {
println(" at '${text.substring(error.textOffset).take(80).replace("\n", "\\n")}'")
}
private fun firstError(file: com.intellij.psi.PsiFile): PsiErrorElement? {
private fun firstError(file: PsiFile): PsiErrorElement? {
var first: PsiErrorElement? = null
file.accept(
object : PsiRecursiveElementWalkingVisitor() {

13
src/test/kotlin/dev/hgh/jai/parser/JaiCorpusParserTest.kt
View File

@@ -36,6 +36,7 @@ class JaiCorpusParserTest : BasePlatformTestCase() {
var totalErrors = 0
val filesWithErrors = mutableListOf<String>()
val firstErrors = mutableListOf<String>()
val allErrors = mutableListOf<String>()
val histogram = mutableMapOf<String, Int>()
for (file in jaiFiles) {
@@ -62,10 +63,8 @@ class JaiCorpusParserTest : BasePlatformTestCase() {
val offset = error.textOffset
val got = snippet(text, offset).take(GOT_WIDTH)
histogram.merge(got, 1, Int::plus)
if (firstErrors.size < REPORTED_ERRORS) {
val line = text.substring(0, offset).count { it == '\n' } + 1
firstErrors.add("$relative:$line: ${error.errorDescription} [at '$got']")
}
val line = text.substring(0, offset).count { it == '\n' } + 1
allErrors.add("$relative:$line: ${error.errorDescription} [at '$got']")
}
}
@@ -82,6 +81,10 @@ class JaiCorpusParserTest : BasePlatformTestCase() {
.take(REPORTED_KINDS)
.forEach { (token, count) -> println(" %5d %s".format(count, token)) }
}
// A spread across the corpus, not the first N alphabetically: the failures
// cluster by directory, so an alphabetical head hides whole categories.
val stride = maxOf(1, allErrors.size / REPORTED_ERRORS)
allErrors.filterIndexed { index, _ -> index % stride == 0 }.take(REPORTED_ERRORS).forEach { println(" $it") }
firstErrors.forEach { println(" $it") }
assertTrue(
@@ -103,7 +106,7 @@ class JaiCorpusParserTest : BasePlatformTestCase() {
private companion object {
/** Ratchet. Raise it as the grammar improves; never lower it. */
const val MIN_CLEAN_FILES = 4
const val MIN_CLEAN_FILES = 575
const val REPORTED_ERRORS = 25
const val REPORTED_KINDS = 20
const val GOT_WIDTH = 14