Parser: raise corpus parse rate from 57% to 69%

Grammar gains: array types, directive flags, declaration-level directive
modifiers, parameter markers, backticked declaration names, notes after a
block-valued declaration, 'if cond then stmt', 'ifx c else v', '#module_parameters'
two-group form, '(.*)' prefix dereference, and '#asm' bodies consumed opaquely
(they are x86-64 assembly, not Jai).

Return items no longer take a default value: allowing one made
'f: (K) -> u32 = null' swallow the enclosing parameter's default and read the
rest of the parameter list as extra return values.
This commit is contained in:
hgranthorner
2026-08-04 12:13:26 -04:00
parent 37809b8aa7
commit 840a503e40
19 changed files with 727 additions and 220 deletions

View File

@@ -71,6 +71,7 @@ object JaiParserUtil : GeneratedParserUtilBase() {
"add_context",
"module_parameters",
"place",
"overlay",
"procedure_of_call",
"body_text",
"version",
@@ -85,7 +86,6 @@ object JaiParserUtil : GeneratedParserUtilBase() {
"elsewhere",
"expand",
"specified",
"discard",
"location",
"this",
)
@@ -286,9 +286,13 @@ object JaiParserUtil : GeneratedParserUtilBase() {
): Boolean = b.tokenType === T.LPAREN && !isProcHeaderAhead(b, level)
/**
* `[#directive|using]* IDENT (',' IDENT)* (':' | '::' | ':=')` — a declaration.
* Jai declarations have no introducer keyword, so this is what tells
* `[#directive [(…)] | using | `]* IDENT (',' IDENT)* (':' | '::' | ':=')` — a
* declaration. Jai declarations have no introducer keyword, so this is what tells
* `foo :: () {}` apart from `foo();`.
*
* The prefix skip covers `#as using base: Document;`,
* `#overlay (unknown.vtable) using vtable: *Vtable;` and the macro-exported
* `` `it := entry.value; ``.
*/
@JvmStatic
fun isDeclarationAhead(
@@ -296,11 +300,24 @@ object JaiParserUtil : GeneratedParserUtilBase() {
level: Int,
): Boolean {
var i = 0
while (true) {
val t = b.lookAhead(i) ?: return false
if (t === T.DIRECTIVE || t === T.KW_USING) i++ else break
loop@ while (true) {
when (b.lookAhead(i)) {
T.DIRECTIVE, T.KW_USING -> {
i++
}
T.LPAREN -> {
i = skipBalancedParens(b, i)
if (i < 0) return false
}
else -> {
break@loop
}
}
}
while (true) {
if (b.lookAhead(i) === T.BACKTICK) i++
if (b.lookAhead(i) !== T.IDENT) return false
i++
when (b.lookAhead(i)) {
@@ -311,6 +328,61 @@ object JaiParserUtil : GeneratedParserUtilBase() {
}
}
/** Index just past the `)` matching the `(` at [start], or -1 if unbalanced. */
private fun skipBalancedParens(
b: PsiBuilder,
start: Int,
): Int {
var depth = 0
var i = start
while (i - start <= MAX_LOOKAHEAD) {
when (b.lookAhead(i) ?: return -1) {
T.LPAREN -> {
depth++
}
T.RPAREN -> {
depth--
if (depth == 0) return i + 1
}
else -> {}
}
i++
}
return -1
}
/**
* Consumes a `{ … }` block as opaque tokens.
*
* Used for `#asm`, whose body is x86-64 assembly (`movd source:, byte;`) and not
* Jai at all. Giving it structure is a separate job; tiling it keeps the
* surrounding file parseable.
*/
@JvmStatic
fun opaqueBlock(
b: PsiBuilder,
level: Int,
): Boolean {
if (b.tokenType !== T.LBRACE) return false
var depth = 0
while (!b.eof()) {
val t = b.tokenType
if (t === T.LBRACE) {
depth++
} else if (t === T.RBRACE) {
depth--
if (depth == 0) {
b.advanceLexer()
return true
}
}
b.advanceLexer()
}
return false
}
/** `[$|$$] IDENT (',' [$|$$] IDENT)* (':' | '::' | ':=')` — a named parameter. */
@JvmStatic
fun isParamNameAhead(