grant/intellijai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parser: 91.7% of the corpus, operator symbols and #code statements

Browse Source 
operatorSymbol now tries multi-token forms first, so 'operator *[]' is no longer
cut short by the bare '*'. '#code'/'#insert' take a whole statement, and a
statement that already consumed its ';' satisfies the enclosing declaration.

Ratchet 650/714.
This commit is contained in:
hgranthorner
2026-08-04 12:40:20 -04:00
parent dc53033898
commit 47a2c5805e
7 changed files with 143 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
src/main/gen/dev/hgh/jai/parser/JaiParser.java
View File

@@ -714,6 +714,7 @@ public class JaiParser implements PsiParser, LightPsiParser {
// | <<dir "#foreign">> directiveFlag_* [<<exprNoAssign>>] [STRING]
// | <<dir "#elsewhere">> directiveFlag_* [<<exprNoAssign>>] [STRING]
// | <<dir "#insert">> directiveFlag_* ['(' [argList] ')'] [block | expr]
// | <<dir "#code">> directiveFlag_* (block | statement)
// | <<dir "#assert">> expr [expr]
// | <<dir "#module_parameters">> '(' [paramList] ')' ['(' [paramList] ')'] [block]
// | <<dirWithBlock>> directiveFlag_* block
@@ -732,6 +733,7 @@ public class JaiParser implements PsiParser, LightPsiParser {
if (!result_) result_ = directiveExpr_6(builder_, level_ + 1);
if (!result_) result_ = directiveExpr_7(builder_, level_ + 1);
if (!result_) result_ = directiveExpr_8(builder_, level_ + 1);
if (!result_) result_ = directiveExpr_9(builder_, level_ + 1);
exit_section_(builder_, level_, marker_, result_, false, null);
return result_;
}
@@ -942,111 +944,120 @@ public class JaiParser implements PsiParser, LightPsiParser {
return result_;
}
// <<dir "#assert">> expr [expr]
// <<dir "#code">> directiveFlag_* (block | statement)
private static boolean directiveExpr_4(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_4")) return false;
boolean result_;
Marker marker_ = enter_section_(builder_);
result_ = dir(builder_, level_ + 1, "#assert");
result_ = result_ && expr(builder_, level_ + 1, -1);
result_ = dir(builder_, level_ + 1, "#code");
result_ = result_ && directiveExpr_4_1(builder_, level_ + 1);
result_ = result_ && directiveExpr_4_2(builder_, level_ + 1);
exit_section_(builder_, marker_, null, result_);
return result_;
}
// [expr]
// directiveFlag_*
private static boolean directiveExpr_4_1(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_4_1")) return false;
while (true) {
int pos_ = current_position_(builder_);
if (!directiveFlag_(builder_, level_ + 1)) break;
if (!empty_element_parsed_guard_(builder_, "directiveExpr_4_1", pos_)) break;
}
return true;
}
// block | statement
private static boolean directiveExpr_4_2(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_4_2")) return false;
boolean result_;
result_ = block(builder_, level_ + 1);
if (!result_) result_ = statement(builder_, level_ + 1);
return result_;
}
// <<dir "#assert">> expr [expr]
private static boolean directiveExpr_5(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_5")) return false;
boolean result_;
Marker marker_ = enter_section_(builder_);
result_ = dir(builder_, level_ + 1, "#assert");
result_ = result_ && expr(builder_, level_ + 1, -1);
result_ = result_ && directiveExpr_5_2(builder_, level_ + 1);
exit_section_(builder_, marker_, null, result_);
return result_;
}
// [expr]
private static boolean directiveExpr_5_2(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_5_2")) return false;
expr(builder_, level_ + 1, -1);
return true;
}
// <<dir "#module_parameters">> '(' [paramList] ')' ['(' [paramList] ')'] [block]
private static boolean directiveExpr_5(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_5")) return false;
private static boolean directiveExpr_6(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_6")) return false;
boolean result_;
Marker marker_ = enter_section_(builder_);
result_ = dir(builder_, level_ + 1, "#module_parameters");
result_ = result_ && consumeToken(builder_, LPAREN);
result_ = result_ && directiveExpr_5_2(builder_, level_ + 1);
result_ = result_ && directiveExpr_6_2(builder_, level_ + 1);
result_ = result_ && consumeToken(builder_, RPAREN);
result_ = result_ && directiveExpr_5_4(builder_, level_ + 1);
result_ = result_ && directiveExpr_5_5(builder_, level_ + 1);
result_ = result_ && directiveExpr_6_4(builder_, level_ + 1);
result_ = result_ && directiveExpr_6_5(builder_, level_ + 1);
exit_section_(builder_, marker_, null, result_);
return result_;
}
// [paramList]
private static boolean directiveExpr_5_2(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_5_2")) return false;
private static boolean directiveExpr_6_2(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_6_2")) return false;
paramList(builder_, level_ + 1);
return true;
}
// ['(' [paramList] ')']
private static boolean directiveExpr_5_4(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_5_4")) return false;
directiveExpr_5_4_0(builder_, level_ + 1);
private static boolean directiveExpr_6_4(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_6_4")) return false;
directiveExpr_6_4_0(builder_, level_ + 1);
return true;
}
// '(' [paramList] ')'
private static boolean directiveExpr_5_4_0(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_5_4_0")) return false;
private static boolean directiveExpr_6_4_0(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_6_4_0")) return false;
boolean result_;
Marker marker_ = enter_section_(builder_);
result_ = consumeToken(builder_, LPAREN);
result_ = result_ && directiveExpr_5_4_0_1(builder_, level_ + 1);
result_ = result_ && directiveExpr_6_4_0_1(builder_, level_ + 1);
result_ = result_ && consumeToken(builder_, RPAREN);
exit_section_(builder_, marker_, null, result_);
return result_;
}
// [paramList]
private static boolean directiveExpr_5_4_0_1(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_5_4_0_1")) return false;
private static boolean directiveExpr_6_4_0_1(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_6_4_0_1")) return false;
paramList(builder_, level_ + 1);
return true;
}
// [block]
private static boolean directiveExpr_5_5(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_5_5")) return false;
private static boolean directiveExpr_6_5(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_6_5")) return false;
block(builder_, level_ + 1);
return true;
}
// <<dirWithBlock>> directiveFlag_* block
private static boolean directiveExpr_6(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_6")) return false;
boolean result_;
Marker marker_ = enter_section_(builder_);
result_ = dirWithBlock(builder_, level_ + 1);
result_ = result_ && directiveExpr_6_1(builder_, level_ + 1);
result_ = result_ && block(builder_, level_ + 1);
exit_section_(builder_, marker_, null, result_);
return result_;
}
// directiveFlag_*
private static boolean directiveExpr_6_1(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_6_1")) return false;
while (true) {
int pos_ = current_position_(builder_);
if (!directiveFlag_(builder_, level_ + 1)) break;
if (!empty_element_parsed_guard_(builder_, "directiveExpr_6_1", pos_)) break;
}
return true;
}
// <<dirWithOperand>> directiveFlag_* [expr]
private static boolean directiveExpr_7(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_7")) return false;
boolean result_;
Marker marker_ = enter_section_(builder_);
result_ = dirWithOperand(builder_, level_ + 1);
result_ = dirWithBlock(builder_, level_ + 1);
result_ = result_ && directiveExpr_7_1(builder_, level_ + 1);
result_ = result_ && directiveExpr_7_2(builder_, level_ + 1);
result_ = result_ && block(builder_, level_ + 1);
exit_section_(builder_, marker_, null, result_);
return result_;
}
@@ -1062,20 +1073,14 @@ public class JaiParser implements PsiParser, LightPsiParser {
return true;
}
// [expr]
private static boolean directiveExpr_7_2(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_7_2")) return false;
expr(builder_, level_ + 1, -1);
return true;
}
// <<dirPlain>> directiveFlag_*
// <<dirWithOperand>> directiveFlag_* [expr]
private static boolean directiveExpr_8(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_8")) return false;
boolean result_;
Marker marker_ = enter_section_(builder_);
result_ = dirPlain(builder_, level_ + 1);
result_ = dirWithOperand(builder_, level_ + 1);
result_ = result_ && directiveExpr_8_1(builder_, level_ + 1);
result_ = result_ && directiveExpr_8_2(builder_, level_ + 1);
exit_section_(builder_, marker_, null, result_);
return result_;
}
@@ -1091,6 +1096,35 @@ public class JaiParser implements PsiParser, LightPsiParser {
return true;
}
// [expr]
private static boolean directiveExpr_8_2(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_8_2")) return false;
expr(builder_, level_ + 1, -1);
return true;
}
// <<dirPlain>> directiveFlag_*
private static boolean directiveExpr_9(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_9")) return false;
boolean result_;
Marker marker_ = enter_section_(builder_);
result_ = dirPlain(builder_, level_ + 1);
result_ = result_ && directiveExpr_9_1(builder_, level_ + 1);
exit_section_(builder_, marker_, null, result_);
return result_;
}
// directiveFlag_*
private static boolean directiveExpr_9_1(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "directiveExpr_9_1")) return false;
while (true) {
int pos_ = current_position_(builder_);
if (!directiveFlag_(builder_, level_ + 1)) break;
if (!empty_element_parsed_guard_(builder_, "directiveExpr_9_1", pos_)) break;
}
return true;
}
/* ********************************************************** */
// <<directiveFlag>>
static boolean directiveFlag_(PsiBuilder builder_, int level_) {
@@ -1717,16 +1751,19 @@ public class JaiParser implements PsiParser, LightPsiParser {
}
/* ********************************************************** */
// '+' | '-' | '*' | '/' | '%' | '==' | '!=' | '<' | '>' | '<=' | '>='
// '*' '[' ']' | '[' ']' '=' | '[' ']'
// | '+' | '-' | '*' | '/' | '%' | '==' | '!=' | '<' | '>' | '<=' | '>='
// | '&' | '|' | '^' | '~' | '<<' | '>>' | '<<<' | '>>>'
// | '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^='
// | '<<=' | '>>=' | '<<<=' | '>>>='
// | '*' '[' ']' | '[' ']' '=' | '[' ']'
public static boolean operatorSymbol(PsiBuilder builder_, int level_) {
if (!recursion_guard_(builder_, level_, "operatorSymbol")) return false;
boolean result_;
Marker marker_ = enter_section_(builder_, level_, _NONE_, OPERATOR_SYMBOL, "<operator symbol>");
result_ = consumeToken(builder_, PLUS);
result_ = parseTokens(builder_, 0, STAR, LBRACKET, RBRACKET);
if (!result_) result_ = parseTokens(builder_, 0, LBRACKET, RBRACKET, EQ);
if (!result_) result_ = parseTokens(builder_, 0, LBRACKET, RBRACKET);
if (!result_) result_ = consumeToken(builder_, PLUS);
if (!result_) result_ = consumeToken(builder_, MINUS);
if (!result_) result_ = consumeToken(builder_, STAR);
if (!result_) result_ = consumeToken(builder_, SLASH);
@@ -1757,9 +1794,6 @@ public class JaiParser implements PsiParser, LightPsiParser {
if (!result_) result_ = consumeToken(builder_, SHR_EQ);
if (!result_) result_ = consumeToken(builder_, ROL_EQ);
if (!result_) result_ = consumeToken(builder_, ROR_EQ);
if (!result_) result_ = parseTokens(builder_, 0, STAR, LBRACKET, RBRACKET);
if (!result_) result_ = parseTokens(builder_, 0, LBRACKET, RBRACKET, EQ);
if (!result_) result_ = parseTokens(builder_, 0, LBRACKET, RBRACKET);
exit_section_(builder_, level_, marker_, result_, false, null);
return result_;
}

3
src/main/gen/dev/hgh/jai/psi/JaiDirectiveExpr.java
View File

@@ -19,4 +19,7 @@ public interface JaiDirectiveExpr extends JaiExpr {
@NotNull
List<JaiParameter> getParameterList();
@Nullable
JaiStatement getStatement();
}

6
src/main/gen/dev/hgh/jai/psi/impl/JaiDirectiveExprImpl.java
View File

@@ -51,4 +51,10 @@ public class JaiDirectiveExprImpl extends JaiExprImpl implements JaiDirectiveExp
return PsiTreeUtil.getChildrenOfTypeAsList(this, JaiParameter.class);
}
@Override
@Nullable
public JaiStatement getStatement() {
return findChildByClass(JaiStatement.class);
}
}

7
src/main/grammar/Jai.bnf
View File

@@ -235,11 +235,13 @@ initializer ::= '---' | expr
// `operator + :: (a: T, b: T) -> T { }`
operatorDeclaration ::= 'operator' operatorSymbol '::' expr semiWithNotes_
operatorSymbol ::= '+' | '-' | '*' | '/' | '%' | '==' | '!=' | '<' | '>' | '<=' | '>='
// Longest first: `*` would otherwise match the start of `*[]`, and `[]` the start
// of `[]=`. Single-token operators cannot collide, since `+=` is one token.
operatorSymbol ::= '*' '[' ']' | '[' ']' '=' | '[' ']'
| '+' | '-' | '*' | '/' | '%' | '==' | '!=' | '<' | '>' | '<=' | '>='
| '&' | '|' | '^' | '~' | '<<' | '>>' | '<<<' | '>>>'
| '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^='
| '<<=' | '>>=' | '<<<=' | '>>>='
| '*' '[' ']' | '[' ']' '=' | '[' ']'
// -------------------------------------------------------------------- expressions
@@ -411,6 +413,7 @@ directiveExpr ::= <<dir "#asm">> directiveFlag_* [expr (',' expr)*] <<opaqueBloc
| <<dir "#foreign">> directiveFlag_* [<<exprNoAssign>>] [STRING]
| <<dir "#elsewhere">> directiveFlag_* [<<exprNoAssign>>] [STRING]
| <<dir "#insert">> directiveFlag_* ['(' [argList] ')'] [block | expr]
| <<dir "#code">> directiveFlag_* (block | statement)
| <<dir "#assert">> expr [expr]
| <<dir "#module_parameters">> '(' [paramList] ')' ['(' [paramList] ')'] [block]
| <<dirWithBlock>> directiveFlag_* block

7
src/main/kotlin/dev/hgh/jai/parser/JaiParserUtil.kt
View File

@@ -237,8 +237,9 @@ object JaiParserUtil : GeneratedParserUtilBase() {
/**
* True if the previous token already closes a construct, so no `;` is needed:
* `}` ends a block, struct or enum body, and a here-string ends on its own
* terminator line (`BODY :: #string DONE … DONE`).
* `}` ends a block, struct or enum body, a here-string ends on its own terminator
* line (`BODY :: #string DONE … DONE`), and a `;` was already taken by a nested
* statement (`code :: #code a := 1;`).
*/
@JvmStatic
fun prevEndsConstruct(
@@ -249,7 +250,7 @@ object JaiParserUtil : GeneratedParserUtilBase() {
while (true) {
val t: IElementType = b.rawLookup(i) ?: return false
if (t !== TokenType.WHITE_SPACE && t !== T.LINE_COMMENT && t !== T.BLOCK_COMMENT) {
return t === T.RBRACE || t === T.HERE_STRING
return t === T.RBRACE || t === T.HERE_STRING || t === T.SEMICOLON
}
i--
}

47
src/test/kotlin/dev/hgh/jai/parser/DebugParseTest.kt
View File

@@ -15,24 +15,24 @@ import java.io.File
class DebugParseTest : BasePlatformTestCase() {
private val snippets =
listOf(
"f :: () -> a: int = 1;\n",
"f :: () -> a: int = 1 { }\n",
"f :: () -> a: int = xx { }\n",
"f :: () -> a: int = 1 #foreign lib;\n",
"g :: () { x := 1; }\n",
"T :: struct {\n p := .06;\n b: BG;\n b.shape.kind = .ABS;\n b.color = V4.{.00, .10, 1};\n}\n",
"T :: struct {\n orientation: enum u8 {\n H :: 0;\n V :: 1;\n }\n\n d: BT;\n}\n",
"T :: struct {\n using,except .[\"x\"] orientation: Quaternion;\n}\n",
"operator *[] :: (b: *Bucket, index: int) -> *int { return null; }\n",
"code :: #code a := Vector3.{1, 1, 1};\n",
)
/** Corpus files to dump the first error of. */
private val corpusFiles =
listOf(
"modules/Basic/Array.jai",
"modules/Debug/windows.jai",
"modules/Simp/backend/gl.jai",
"modules/GetRect/system/occlusion.jai",
"modules/POSIX/compare_bindings.jai",
"modules/Android/Toolchain/apk.jai",
"modules/Curl/examples/ftp.jai",
"modules/Objective_C/LightweightRenderingView/module.jai",
"modules/GetRect/widgets/color_picker.jai",
"modules/GetRect/widgets/slidable_region.jai",
"how_to/094_array_operators.jai",
"how_to/044_using_advanced/main.jai",
"how_to/630_compiler_get_nodes.jai",
"modules/Bucket_Array.jai",
"modules/Android/Toolchain/adb.jai",
"modules/Hash_Table.jai",
)
fun testCorpusFiles() {
@@ -71,14 +71,25 @@ class DebugParseTest : BasePlatformTestCase() {
private fun dumpFile(file: File) {
val text = file.readText()
val psi = PsiFileFactory.getInstance(project).createFileFromText(file.name, JaiLanguage, text)
val error = firstError(psi)
if (error == null) {
val errors = mutableListOf<PsiErrorElement>()
psi.accept(
object : PsiRecursiveElementWalkingVisitor() {
override fun visitErrorElement(element: PsiErrorElement) {
errors.add(element)
super.visitErrorElement(element)
}
},
)
if (errors.isEmpty()) {
println("${file.name}: clean")
return
}
val line = text.substring(0, error.textOffset).count { it == '\n' } + 1
println("${file.name}:$line: ${error.errorDescription}")
println(" at '${text.substring(error.textOffset).take(80).replace("\n", "\\n")}'")
println("${file.name}: ${errors.size} error(s)")
for (error in errors.take(6)) {
val line = text.substring(0, error.textOffset).count { it == '\n' } + 1
println(" :$line ${error.errorDescription}")
println(" at '${text.substring(error.textOffset).take(70).replace("\n", "\\n")}'")
}
}
private fun firstError(file: PsiFile): PsiErrorElement? {

2
src/test/kotlin/dev/hgh/jai/parser/JaiCorpusParserTest.kt
View File

@@ -106,7 +106,7 @@ class JaiCorpusParserTest : BasePlatformTestCase() {
private companion object {
/** Ratchet. Raise it as the grammar improves; never lower it. */
const val MIN_CLEAN_FILES = 645
const val MIN_CLEAN_FILES = 650
const val REPORTED_ERRORS = 25
const val REPORTED_KINDS = 20
const val GOT_WIDTH = 14