Parser: raise corpus parse rate from 57% to 69%
Grammar gains: array types, directive flags, declaration-level directive modifiers, parameter markers, backticked declaration names, notes after a block-valued declaration, 'if cond then stmt', 'ifx c else v', '#module_parameters' two-group form, '(.*)' prefix dereference, and '#asm' bodies consumed opaquely (they are x86-64 assembly, not Jai). Return items no longer take a default value: allowing one made 'f: (K) -> u32 = null' swallow the enclosing parameter's default and read the rest of the parameter list as extra return values.
This commit is contained in:
@@ -71,6 +71,7 @@ object JaiParserUtil : GeneratedParserUtilBase() {
|
||||
"add_context",
|
||||
"module_parameters",
|
||||
"place",
|
||||
"overlay",
|
||||
"procedure_of_call",
|
||||
"body_text",
|
||||
"version",
|
||||
@@ -85,7 +86,6 @@ object JaiParserUtil : GeneratedParserUtilBase() {
|
||||
"elsewhere",
|
||||
"expand",
|
||||
"specified",
|
||||
"discard",
|
||||
"location",
|
||||
"this",
|
||||
)
|
||||
@@ -286,9 +286,13 @@ object JaiParserUtil : GeneratedParserUtilBase() {
|
||||
): Boolean = b.tokenType === T.LPAREN && !isProcHeaderAhead(b, level)
|
||||
|
||||
/**
|
||||
* `[#directive|using]* IDENT (',' IDENT)* (':' | '::' | ':=')` — a declaration.
|
||||
* Jai declarations have no introducer keyword, so this is what tells
|
||||
* `[#directive [(…)] | using | `]* IDENT (',' IDENT)* (':' | '::' | ':=')` — a
|
||||
* declaration. Jai declarations have no introducer keyword, so this is what tells
|
||||
* `foo :: () {}` apart from `foo();`.
|
||||
*
|
||||
* The prefix skip covers `#as using base: Document;`,
|
||||
* `#overlay (unknown.vtable) using vtable: *Vtable;` and the macro-exported
|
||||
* `` `it := entry.value; ``.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun isDeclarationAhead(
|
||||
@@ -296,11 +300,24 @@ object JaiParserUtil : GeneratedParserUtilBase() {
|
||||
level: Int,
|
||||
): Boolean {
|
||||
var i = 0
|
||||
while (true) {
|
||||
val t = b.lookAhead(i) ?: return false
|
||||
if (t === T.DIRECTIVE || t === T.KW_USING) i++ else break
|
||||
loop@ while (true) {
|
||||
when (b.lookAhead(i)) {
|
||||
T.DIRECTIVE, T.KW_USING -> {
|
||||
i++
|
||||
}
|
||||
|
||||
T.LPAREN -> {
|
||||
i = skipBalancedParens(b, i)
|
||||
if (i < 0) return false
|
||||
}
|
||||
|
||||
else -> {
|
||||
break@loop
|
||||
}
|
||||
}
|
||||
}
|
||||
while (true) {
|
||||
if (b.lookAhead(i) === T.BACKTICK) i++
|
||||
if (b.lookAhead(i) !== T.IDENT) return false
|
||||
i++
|
||||
when (b.lookAhead(i)) {
|
||||
@@ -311,6 +328,61 @@ object JaiParserUtil : GeneratedParserUtilBase() {
|
||||
}
|
||||
}
|
||||
|
||||
/** Index just past the `)` matching the `(` at [start], or -1 if unbalanced. */
|
||||
private fun skipBalancedParens(
|
||||
b: PsiBuilder,
|
||||
start: Int,
|
||||
): Int {
|
||||
var depth = 0
|
||||
var i = start
|
||||
while (i - start <= MAX_LOOKAHEAD) {
|
||||
when (b.lookAhead(i) ?: return -1) {
|
||||
T.LPAREN -> {
|
||||
depth++
|
||||
}
|
||||
|
||||
T.RPAREN -> {
|
||||
depth--
|
||||
if (depth == 0) return i + 1
|
||||
}
|
||||
|
||||
else -> {}
|
||||
}
|
||||
i++
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
/**
|
||||
* Consumes a `{ … }` block as opaque tokens.
|
||||
*
|
||||
* Used for `#asm`, whose body is x86-64 assembly (`movd source:, byte;`) and not
|
||||
* Jai at all. Giving it structure is a separate job; tiling it keeps the
|
||||
* surrounding file parseable.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun opaqueBlock(
|
||||
b: PsiBuilder,
|
||||
level: Int,
|
||||
): Boolean {
|
||||
if (b.tokenType !== T.LBRACE) return false
|
||||
var depth = 0
|
||||
while (!b.eof()) {
|
||||
val t = b.tokenType
|
||||
if (t === T.LBRACE) {
|
||||
depth++
|
||||
} else if (t === T.RBRACE) {
|
||||
depth--
|
||||
if (depth == 0) {
|
||||
b.advanceLexer()
|
||||
return true
|
||||
}
|
||||
}
|
||||
b.advanceLexer()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** `[$|$$] IDENT (',' [$|$$] IDENT)* (':' | '::' | ':=')` — a named parameter. */
|
||||
@JvmStatic
|
||||
fun isParamNameAhead(
|
||||
|
||||
Reference in New Issue
Block a user