grant/intellijai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 3: Grammar-Kit parser, PSI and ParserDefinition

Browse Source 
Adds src/main/grammar/Jai.bnf, the generated parser/PSI in src/main/gen, a
JaiParserDefinition, and the Tier 3 corpus parse gate (406/714 files clean).

Two things were not obvious:

- Grammar-Kit mints its own token instances from the .bnf 'tokens' block, which
  are different objects from the ones JaiLexer emits, so every rule silently
  failed to match. Fixed with tokenTypeFactory -> JaiTokenTypes.byName.
- A backslash inside an identifier is a continuation in the compiler's lexer
  (Jai_Lexer/module.jai:444): 'left\_margin' is one identifier. JaiLexer now
  does the same; the Tier 0 round-trip still holds because the token span
  covers the backslash and the skipped spaces.

Constructs that BNF alone cannot express live in JaiParserUtil: directive-name
tests (#ident is one token), '==' before '{' for the switch form, procedure
header vs parenthesised expression, and the declaration lookahead.
This commit is contained in:
hgranthorner
2026-08-04 12:05:42 -04:00
parent 746647d27d
commit 37809b8aa7
169 changed files with 8328 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt
View File

@@ -302,7 +302,25 @@ open class JaiLexer : LexerBase() {
return q
}
private fun scanIdentifier(from: Int): Int = scanWhile(from) { continuesIdentifier(it) }
/**
* Scans an identifier body.
*
* A backslash inside an identifier is a *continuation*: the compiler's lexer
* (`Jai_Lexer/module.jai:444`) eats the `\` and any spaces after it and keeps
* appending, so `left\_margin` and `free\ _site_trace` are single identifiers.
* The token span still covers the backslash and the skipped spaces, so the Tier 0
* round-trip invariant holds.
*/
private fun scanIdentifier(from: Int): Int {
var q = scanWhile(from) { continuesIdentifier(it) }
while (q < bufEnd && ch(q) == '\\'.code) {
var r = q + 1
while (r < bufEnd && ch(r) == ' '.code) r++
if (r >= bufEnd || !continuesIdentifier(ch(r))) break
q = scanWhile(r) { continuesIdentifier(it) }
}
return q
}
private fun text(
from: Int,

37
src/main/kotlin/dev/hgh/jai/lexer/JaiTokenTypes.kt
View File

@@ -4,9 +4,17 @@ import com.intellij.psi.tree.IElementType
import com.intellij.psi.tree.TokenSet
import dev.hgh.jai.JaiLanguage
/**
* A Jai token type.
*
* [text] is the name used in `Jai.bnf`'s `tokens` block — the literal spelling for
* punctuation and keywords (`"::"`, `"if"`), and an upper-case label for the
* variable-text tokens (`"IDENT"`, `"NUMBER"`). Grammar-Kit's generated `JaiTypes`
* resolves tokens by that string through [JaiTokenTypes.byName].
*/
class JaiTokenType(
debugName: String,
) : IElementType(debugName, JaiLanguage) {
val text: String,
) : IElementType(text, JaiLanguage) {
override fun toString(): String = "Jai:" + super.toString()
}
@@ -387,4 +395,29 @@ object JaiTokenTypes {
HASH,
AT,
)
/**
* `Jai.bnf` token name -> token, for the Grammar-Kit `tokenTypeFactory` hook.
*
* Without this the generated `JaiTypes` would declare its *own*
* `new JaiTokenType("IDENT")`, which is a different object from the one the lexer
* emits, so every rule would silently fail to match and the whole file would parse
* as one error element. Grammar-Kit passes the token's *text* (`"::"`, `"if"`,
* `"IDENT"`), which is exactly [JaiTokenType.text]. Built reflectively over the
* fields above so it cannot drift.
*/
private val BY_NAME: Map<String, JaiTokenType> =
JaiTokenTypes::class.java.declaredFields
.filter { JaiTokenType::class.java.isAssignableFrom(it.type) }
.map { it.get(null) as JaiTokenType }
.associateBy { it.text }
/**
* Looks a token up by its `Jai.bnf` name. Throws on an unknown name, so a typo in
* the grammar fails loudly at class-load time instead of parsing as garbage.
*/
@JvmStatic
fun byName(name: String): JaiTokenType =
BY_NAME[name]
?: error("Unknown Jai token '$name'. Jai.bnf declares a token JaiTokenTypes does not define.")
}

366
src/main/kotlin/dev/hgh/jai/parser/JaiParserUtil.kt Normal file
View File

@@ -0,0 +1,366 @@
package dev.hgh.jai.parser
import com.intellij.lang.PsiBuilder
import com.intellij.lang.parser.GeneratedParserUtilBase
import com.intellij.psi.TokenType
import com.intellij.psi.tree.IElementType
import dev.hgh.jai.lexer.JaiTokenTypes as T
/**
* Hand-written helpers referenced from `Jai.bnf` as external rules.
*
* These exist because a few Jai constructs are not expressible as pure BNF over our
* token set:
*
* - `#ident` is a single DIRECTIVE token, so "is this `#if`?" is a *text* test.
* - `if x == { case ...; }` reuses `==` as a switch introducer, so the expression
* parser must refuse `==` when a `{` follows.
* - `(` starts both a parenthesised expression and a procedure header.
* - A declaration has no introducer keyword, so `foo :: ...` vs `foo(...)` needs
* lookahead.
*
* Everything here is pure lookahead except the `dir*` family, which consume exactly
* one DIRECTIVE token.
*/
object JaiParserUtil : GeneratedParserUtilBase() {
/**
* Directives whose operand is a brace-delimited block, e.g. `#asm { ... }`.
* Tried before [DIRECTIVES_WITH_OPERAND] so `#run { ... }` takes the block form
* while `#run foo()` falls through to the expression form.
*/
private val DIRECTIVES_WITH_BLOCK =
setOf(
"asm",
"modify",
"run",
"insert",
"code",
"run_and_insert",
"compile_time",
"bytes",
"no_reset",
)
/**
* Directives that take a single expression operand, e.g. `#import "Basic"`.
* The operand is optional: `#foreign` appears both bare and as `#foreign lib`.
* Anything not listed here is parsed as a bare directive so that, for example,
* `#scope_file` cannot swallow the declaration that follows it.
*/
private val DIRECTIVES_WITH_OPERAND =
setOf(
"import",
"load",
"run",
"assert",
"insert",
"code",
"type",
"char",
"library",
"system_library",
"foreign_library",
"foreign_system_library",
"align",
"placeholder",
"placeholders",
"bake",
"bake_arguments",
"bake_constants",
"deprecated",
"add_context",
"module_parameters",
"place",
"procedure_of_call",
"body_text",
"version",
"poke_name",
"dump",
"foreign",
"define",
"include",
"undef",
"ifdef",
"pragma",
"elsewhere",
"expand",
"specified",
"discard",
"location",
"this",
)
// ---------------------------------------------------------------- directives
/**
* Consumes the DIRECTIVE token if its text matches exactly.
*
* [text] includes the `#`: Grammar-Kit rewrites a `<<dir "if">>` argument into a
* reference to the `if` *token* because `'if'` is a declared token text, so the
* grammar has to say `<<dir "#if">>`.
*/
@JvmStatic
fun dir(
b: PsiBuilder,
level: Int,
text: String,
): Boolean {
if (b.tokenType !== T.DIRECTIVE) return false
if (b.tokenText != text) return false
b.advanceLexer()
return true
}
/** True (without consuming) if the current token is the directive [text] (with `#`). */
@JvmStatic
fun dirAhead(
b: PsiBuilder,
level: Int,
text: String,
): Boolean = b.tokenType === T.DIRECTIVE && b.tokenText == text
@JvmStatic
fun dirWithBlock(
b: PsiBuilder,
level: Int,
): Boolean = consumeDirectiveIn(b, DIRECTIVES_WITH_BLOCK)
@JvmStatic
fun dirWithOperand(
b: PsiBuilder,
level: Int,
): Boolean = consumeDirectiveIn(b, DIRECTIVES_WITH_OPERAND)
/** Any DIRECTIVE token; used for the bare `#complete` / `#scope_file` form. */
@JvmStatic
fun dirPlain(
b: PsiBuilder,
level: Int,
): Boolean {
if (b.tokenType !== T.DIRECTIVE) return false
b.advanceLexer()
return true
}
/**
* A directive that takes neither an operand nor a block, so it stands alone as a
* statement without a `;` — `#scope_file`, `#no_padding`, `#compiler`.
*/
@JvmStatic
fun dirBare(
b: PsiBuilder,
level: Int,
): Boolean {
if (b.tokenType !== T.DIRECTIVE) return false
val text = b.tokenText ?: return false
val name = text.removePrefix("#")
if (name in DIRECTIVES_WITH_OPERAND || name in DIRECTIVES_WITH_BLOCK) return false
b.advanceLexer()
return true
}
private fun consumeDirectiveIn(
b: PsiBuilder,
names: Set<String>,
): Boolean {
if (b.tokenType !== T.DIRECTIVE) return false
val text = b.tokenText ?: return false
if (text.length < 2 || text.substring(1) !in names) return false
b.advanceLexer()
return true
}
// ------------------------------------------------------------------ operators
/**
* An expression that stops short of `=`.
*
* Needed wherever a *type* is followed by `= value`: in `a : float = 37.0` the
* type slot must yield `float`, not the assignment `float = 37.0`. Grammar-Kit
* only ever calls the expression root at the lowest priority from BNF, so this
* re-enters it one level up.
*/
@JvmStatic
fun exprNoAssign(
b: PsiBuilder,
level: Int,
): Boolean = JaiParser.expr(b, level, ASSIGNMENT_PRIORITY)
/**
* Consumes `==` unless a `{` follows it, because `if x == { case ...; }` is the
* switch form and the `==` belongs to the statement, not to an equality expression.
*/
@JvmStatic
fun eqEqNotSwitch(
b: PsiBuilder,
level: Int,
): Boolean {
if (b.tokenType !== T.EQ_EQ) return false
if (b.lookAhead(1) === T.LBRACE) return false
b.advanceLexer()
return true
}
/**
* True if the previous token already closes a construct, so no `;` is needed:
* `}` ends a block, struct or enum body, and a here-string ends on its own
* terminator line (`BODY :: #string DONE … DONE`).
*/
@JvmStatic
fun prevEndsConstruct(
b: PsiBuilder,
level: Int,
): Boolean {
var i = -1
while (true) {
val t: IElementType = b.rawLookup(i) ?: return false
if (t !== TokenType.WHITE_SPACE && t !== T.LINE_COMMENT && t !== T.BLOCK_COMMENT) {
return t === T.RBRACE || t === T.HERE_STRING
}
i--
}
}
// ------------------------------------------------------------------ lookahead
/**
* `(`…`)` followed by `->`, a directive, or a `{` body, where the parameter list
* is either empty or contains a `:`/`$` at depth 1. Distinguishes a procedure
* header from `if (x) { }`, whose parenthesised condition looks similar.
*/
@JvmStatic
fun isProcHeaderAhead(
b: PsiBuilder,
level: Int,
): Boolean {
if (b.tokenType !== T.LPAREN) return false
var depth = 0
var i = 0
var sawParamMarker = false
var empty = true
while (true) {
val t = b.lookAhead(i) ?: return false
when (t) {
T.LPAREN, T.LBRACKET, T.BEGIN_ARRAY_LITERAL, T.LBRACE, T.BEGIN_STRUCT_LITERAL -> {
depth++
}
T.RPAREN -> {
depth--
if (depth == 0) {
val next = b.lookAhead(i + 1)
return when {
next === T.RIGHT_ARROW -> true
next === T.DIRECTIVE -> true
next === T.LBRACE -> empty || sawParamMarker
else -> false
}
}
}
T.RBRACKET, T.RBRACE -> {
depth--
}
else -> {
if (depth == 1) {
empty = false
if (t === T.COLON || t === T.DOLLAR || t === T.DOUBLE_DOLLAR ||
t === T.COLON_COLON || t === T.COLON_EQ
) {
sawParamMarker = true
}
}
}
}
i++
if (i > MAX_LOOKAHEAD) return false
}
}
/** `(` that starts a parenthesised *return list*, i.e. not a procedure type. */
@JvmStatic
fun isReturnGroupAhead(
b: PsiBuilder,
level: Int,
): Boolean = b.tokenType === T.LPAREN && !isProcHeaderAhead(b, level)
/**
* `[#directive|using]* IDENT (',' IDENT)* (':' | '::' | ':=')` — a declaration.
* Jai declarations have no introducer keyword, so this is what tells
* `foo :: () {}` apart from `foo();`.
*/
@JvmStatic
fun isDeclarationAhead(
b: PsiBuilder,
level: Int,
): Boolean {
var i = 0
while (true) {
val t = b.lookAhead(i) ?: return false
if (t === T.DIRECTIVE || t === T.KW_USING) i++ else break
}
while (true) {
if (b.lookAhead(i) !== T.IDENT) return false
i++
when (b.lookAhead(i)) {
T.COLON, T.COLON_COLON, T.COLON_EQ -> return true
T.COMMA -> i++
else -> return false
}
}
}
/** `[$|$$] IDENT (',' [$|$$] IDENT)* (':' | '::' | ':=')` — a named parameter. */
@JvmStatic
fun isParamNameAhead(
b: PsiBuilder,
level: Int,
): Boolean {
var i = 0
while (true) {
var t = b.lookAhead(i)
if (t === T.DOLLAR || t === T.DOUBLE_DOLLAR) {
i++
t = b.lookAhead(i)
}
if (t !== T.IDENT) return false
i++
when (b.lookAhead(i)) {
T.COLON, T.COLON_COLON, T.COLON_EQ -> return true
T.COMMA -> i++
else -> return false
}
}
}
/** `IDENT (',' IDENT)* ':'` — the named-variable prefix of a `for` header. */
@JvmStatic
fun isForNamesAhead(
b: PsiBuilder,
level: Int,
): Boolean {
var i = 0
while (true) {
if (b.lookAhead(i) !== T.IDENT) return false
i++
when (b.lookAhead(i)) {
T.COLON -> return true
T.COMMA -> i++
else -> return false
}
}
}
/** `IDENT ':'` — a loop label. */
@JvmStatic
fun isLabelAhead(
b: PsiBuilder,
level: Int,
): Boolean = b.tokenType === T.IDENT && b.lookAhead(1) === T.COLON
private const val MAX_LOOKAHEAD = 4096
/** Priority of `assignExpr` in the generated expression parser; see `Jai.bnf`. */
private const val ASSIGNMENT_PRIORITY = 0
}

8
src/main/kotlin/dev/hgh/jai/psi/JaiElementType.kt Normal file
View File

@@ -0,0 +1,8 @@
package dev.hgh.jai.psi
import com.intellij.psi.tree.IElementType
import dev.hgh.jai.JaiLanguage
class JaiElementType(
debugName: String,
) : IElementType(debugName, JaiLanguage)

15
src/main/kotlin/dev/hgh/jai/psi/JaiFile.kt Normal file
View File

@@ -0,0 +1,15 @@
package dev.hgh.jai.psi
import com.intellij.extapi.psi.PsiFileBase
import com.intellij.openapi.fileTypes.FileType
import com.intellij.psi.FileViewProvider
import dev.hgh.jai.JaiFileType
import dev.hgh.jai.JaiLanguage
class JaiFile(
viewProvider: FileViewProvider,
) : PsiFileBase(viewProvider, JaiLanguage) {
override fun getFileType(): FileType = JaiFileType
override fun toString(): String = "Jai File"
}

36
src/main/kotlin/dev/hgh/jai/psi/JaiParserDefinition.kt Normal file
View File

@@ -0,0 +1,36 @@
package dev.hgh.jai.psi
import com.intellij.lang.ASTNode
import com.intellij.lang.ParserDefinition
import com.intellij.lang.PsiParser
import com.intellij.lexer.Lexer
import com.intellij.openapi.project.Project
import com.intellij.psi.FileViewProvider
import com.intellij.psi.PsiElement
import com.intellij.psi.PsiFile
import com.intellij.psi.tree.IFileElementType
import com.intellij.psi.tree.TokenSet
import dev.hgh.jai.JaiLanguage
import dev.hgh.jai.lexer.JaiLexer
import dev.hgh.jai.lexer.JaiTokenTypes
import dev.hgh.jai.parser.JaiParser
class JaiParserDefinition : ParserDefinition {
override fun createLexer(project: Project?): Lexer = JaiLexer()
override fun createParser(project: Project?): PsiParser = JaiParser()
override fun getFileNodeType(): IFileElementType = FILE
override fun getCommentTokens(): TokenSet = JaiTokenTypes.COMMENTS
override fun getStringLiteralElements(): TokenSet = JaiTokenTypes.STRINGS
override fun createElement(node: ASTNode?): PsiElement = JaiTypes.Factory.createElement(node)
override fun createFile(viewProvider: FileViewProvider): PsiFile = JaiFile(viewProvider)
companion object {
val FILE = IFileElementType(JaiLanguage)
}
}