grant/intellijai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parser: cover full Jai corpus

Browse Source
This commit is contained in:
hgranthorner
2026-08-04 13:44:59 -04:00
parent 8265389f64
commit bb54639ce5
21 changed files with 649 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
src/main/kotlin/dev/hgh/jai/parser/JaiParserUtil.kt
View File

@@ -37,7 +37,6 @@ object JaiParserUtil : GeneratedParserUtilBase() {
"insert",
"code",
"run_and_insert",
"compile_time",
"bytes",
"no_reset",
)
@@ -79,6 +78,9 @@ object JaiParserUtil : GeneratedParserUtilBase() {
"poke_name",
"dump",
"foreign",
"compiler",
"intrinsic",
"program_export",
"define",
"include",
"undef",
@@ -100,7 +102,40 @@ object JaiParserUtil : GeneratedParserUtilBase() {
* that declaration's scope. Taken from the corpus, where only these four appear
* directly in front of a `name :` / `name ::` / `name :=`.
*/
private val DIRECTIVES_ON_DECLARATIONS = setOf("as", "overlay", "add_context", "no_reset")
private val DIRECTIVES_ON_DECLARATIONS =
setOf("as", "overlay", "add_context", "no_reset", "program_export")
/** Scope switches are statements, never procedure modifiers. */
private val NON_PROC_DIRECTIVES = setOf("scope_file", "scope_module", "scope_export")
/** Keyword tokens accepted by the grammar's `keywordAsName_` name rule. */
private val DECL_NAME_KEYWORDS =
setOf(
T.KW_IF,
T.KW_FOR,
T.KW_CASE,
T.KW_ENUM,
T.KW_CAST,
T.KW_UNION,
T.KW_STRUCT,
T.KW_REMOVE,
T.KW_INLINE,
T.KW_CONTEXT,
T.KW_OPERATOR,
T.KW_INTERFACE,
T.KW_TYPE_INFO,
T.KW_USING,
T.KW_DEFER,
T.KW_RETURN,
T.KW_BREAK,
T.KW_CONTINUE,
T.KW_THEN,
T.KW_ELSE,
T.KW_WHILE,
T.KW_NULL,
T.KW_TRUE,
T.KW_FALSE,
)
// ---------------------------------------------------------------- directives
@@ -154,6 +189,22 @@ object JaiParserUtil : GeneratedParserUtilBase() {
return true
}
/**
* True when the current directive is a scope switch, which is a statement
* rather than a procedure modifier. Keeping this as a lookahead predicate lets
* the grammar's normal `directiveExpr` consume the modifier's operand, such
* as the library name in `#foreign kernel32`.
*/
@JvmStatic
fun nonProcDirectiveAhead(
b: PsiBuilder,
level: Int,
): Boolean {
if (b.tokenType !== T.DIRECTIVE) return false
val text = b.tokenText ?: return false
return text.removePrefix("#") in NON_PROC_DIRECTIVES
}
/**
* A directive that takes neither an operand nor a block, so it stands alone as a
* statement without a `;` — `#scope_file`, `#no_padding`, `#compiler`.
@@ -166,7 +217,12 @@ object JaiParserUtil : GeneratedParserUtilBase() {
if (b.tokenType !== T.DIRECTIVE) return false
val text = b.tokenText ?: return false
val name = text.removePrefix("#")
if (name in DIRECTIVES_WITH_OPERAND || name in DIRECTIVES_WITH_BLOCK) return false
if (name in DIRECTIVES_WITH_BLOCK) return false
if (name in DIRECTIVES_WITH_OPERAND &&
!(name == "program_export" && b.lookAhead(1) !== T.STRING)
) {
return false
}
b.advanceLexer()
return true
}
@@ -345,6 +401,8 @@ object JaiParserUtil : GeneratedParserUtilBase() {
* `#as using,except(vtable) iunknown: IUnknown;` and the macro-exported
* `` `it := entry.value; ``.
*/
private fun isDeclarationNameToken(token: IElementType?): Boolean = token === T.IDENT || token in DECL_NAME_KEYWORDS
@JvmStatic
fun isDeclarationAhead(
b: PsiBuilder,
@@ -352,17 +410,24 @@ object JaiParserUtil : GeneratedParserUtilBase() {
): Boolean {
var i = 0
var inPrefix = false
var sawUsingPrefix = false
loop@ while (true) {
when (b.lookAhead(i)) {
T.DIRECTIVE -> {
// After `using,except`, a directive is the operand of the
// using modifier (`using,except #run ... name: T`), not a
// second declaration modifier.
if (sawUsingPrefix && !inPrefix) break@loop
if (!isDeclarationDirective(b, i)) break@loop
i++
inPrefix = true
if (b.lookAhead(i) === T.STRING) i++
}
T.KW_USING -> {
i++
inPrefix = true
sawUsingPrefix = true
}
T.LPAREN, T.LBRACKET, T.BEGIN_ARRAY_LITERAL, T.BEGIN_STRUCT_LITERAL -> {
@@ -376,6 +441,17 @@ object JaiParserUtil : GeneratedParserUtilBase() {
if (!inPrefix || b.lookAhead(i + 1) !== T.IDENT) break@loop
i += 2
inPrefix = false
// `using,except SKIP_THESE orientation: T` has a bare
// identifier as the flag's operand. The declaration grammar
// consumes it through `exprNoAssign`; skip it here so the
// actual field name is what declNames sees.
if (sawUsingPrefix &&
b.lookAhead(i) === T.IDENT &&
b.lookAhead(i + 1) === T.IDENT &&
b.lookAhead(i + 2) === T.COLON
) {
i++
}
}
else -> {
@@ -383,14 +459,66 @@ object JaiParserUtil : GeneratedParserUtilBase() {
}
}
}
// A using-except operand can be a directive expression, for example:
// `using,except #run names_to_skip(Type) business: Business;`. The
// prefix loop intentionally stops at that directive, so find the first
// declaration name after its (possibly grouped) operand.
if (sawUsingPrefix && !isDeclarationNameToken(b.lookAhead(i))) {
var candidate = i
while (candidate <= MAX_LOOKAHEAD) {
when (b.lookAhead(candidate)) {
T.LPAREN, T.LBRACKET, T.BEGIN_ARRAY_LITERAL, T.BEGIN_STRUCT_LITERAL -> {
candidate = skipBalancedGroup(b, candidate)
if (candidate < 0) return false
continue
}
T.SEMICOLON, T.RBRACE -> {
return false
}
else -> {}
}
if (isDeclarationNameToken(b.lookAhead(candidate)) &&
b.lookAhead(candidate + 1) in setOf(T.COLON, T.COLON_COLON, T.COLON_EQ)
) {
return true
}
candidate++
}
}
while (true) {
if (b.lookAhead(i) === T.BACKTICK) i++
if (b.lookAhead(i) !== T.IDENT) return false
if (!isDeclarationNameToken(b.lookAhead(i))) return false
i++
when (b.lookAhead(i)) {
T.COLON, T.COLON_COLON, T.COLON_EQ -> return true
T.COMMA -> i++
else -> return false
T.COLON, T.COLON_COLON, T.COLON_EQ -> {
return true
}
// In a mixed declaration/assignment list, an existing lvalue is
// marked with `=` before the next declared name:
// `success, output=, error := adb(...)`.
T.EQ -> {
when (b.lookAhead(i + 1)) {
T.COMMA -> i += 2
// The final assigned target may be followed directly by
// the declaration's `:=` tail:
// `target, existing= := make_target()`.
T.COLON_EQ -> return true
else -> return false
}
}
T.COMMA -> {
i++
}
else -> {
return false
}
}
}
}
@@ -526,12 +654,43 @@ object JaiParserUtil : GeneratedParserUtilBase() {
return t !== T.DOLLAR && t !== T.DOUBLE_DOLLAR
}
/** `IDENT ':'` — a loop label. */
/** Consumes `if (condition) #if FLAG statement` as one nested statement. */
@JvmStatic
fun parseNestedInlineIfStatement(
b: PsiBuilder,
level: Int,
): Boolean {
val marker = b.mark()
fun fail(): Boolean {
marker.rollbackTo()
return false
}
if (b.tokenType !== T.KW_IF || b.lookAhead(1) !== T.LPAREN) return fail()
b.advanceLexer()
if (!JaiParser.parenExpr(b, level + 1)) return fail()
if (b.tokenType !== T.DIRECTIVE || b.tokenText != "#if") return fail()
b.advanceLexer()
if (b.tokenType !== T.IDENT) return fail()
b.advanceLexer()
if (!JaiParser.statement(b, level + 1)) return fail()
marker.drop()
return true
}
/** `IDENT ':'` or `IDENT ':='` — a loop label. */
@JvmStatic
fun isLabelAhead(
b: PsiBuilder,
level: Int,
): Boolean = b.tokenType === T.IDENT && b.lookAhead(1) === T.COLON
): Boolean {
val nameOffset = if (b.tokenType === T.BACKTICK) 1 else 0
if (b.lookAhead(nameOffset) !== T.IDENT) return false
val separator = b.lookAhead(nameOffset + 1)
return separator === T.COLON || separator === T.COLON_EQ
}
private const val MAX_LOOKAHEAD = 4096