grant/intellijai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 2: syntax highlighting and colour settings page

Browse Source 
- JaiSyntaxHighlighter maps every one of the lexer's token types; asserted over
  the whole corpus (105 distinct token types seen, all mapped)
- JaiHighlightingLexer refines IDENT into built-in types and loop variables for
  colouring only, so the parser never sees them as reserved
- colour settings page with a demo file that is itself asserted to lex cleanly
This commit is contained in:
hgranthorner
2026-08-04 10:28:15 -04:00
parent ba5d8aba7b
commit 2c5d7f713b
7 changed files with 421 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
src/main/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPage.kt Normal file
View File

@@ -0,0 +1,90 @@
package dev.hgh.jai.highlighting
import com.intellij.openapi.editor.colors.TextAttributesKey
import com.intellij.openapi.fileTypes.SyntaxHighlighter
import com.intellij.openapi.options.colors.AttributesDescriptor
import com.intellij.openapi.options.colors.ColorDescriptor
import com.intellij.openapi.options.colors.ColorSettingsPage
import dev.hgh.jai.JaiIcons
import javax.swing.Icon
class JaiColorSettingsPage : ColorSettingsPage {
override fun getDisplayName(): String = "Jai"
override fun getIcon(): Icon = JaiIcons.FILE
override fun getHighlighter(): SyntaxHighlighter = JaiSyntaxHighlighter()
override fun getAttributeDescriptors(): Array<AttributesDescriptor> = DESCRIPTORS
override fun getColorDescriptors(): Array<ColorDescriptor> = ColorDescriptor.EMPTY_ARRAY
override fun getAdditionalHighlightingTagToDescriptorMap(): Map<String, TextAttributesKey>? = null
override fun getDemoText(): String = DEMO_TEXT
companion object {
private val DESCRIPTORS =
arrayOf(
AttributesDescriptor("Keyword", JaiColors.KEYWORD),
AttributesDescriptor("Built-in type", JaiColors.BUILTIN_TYPE),
AttributesDescriptor("Identifier", JaiColors.IDENTIFIER),
AttributesDescriptor("Loop variable (it, it_index)", JaiColors.LOOP_VARIABLE),
AttributesDescriptor("Compiler directive", JaiColors.DIRECTIVE),
AttributesDescriptor("Note", JaiColors.NOTE),
AttributesDescriptor("Number", JaiColors.NUMBER),
AttributesDescriptor("String", JaiColors.STRING),
AttributesDescriptor("Here-string", JaiColors.HERE_STRING),
AttributesDescriptor("Comments//Line comment", JaiColors.LINE_COMMENT),
AttributesDescriptor("Comments//Block comment", JaiColors.BLOCK_COMMENT),
AttributesDescriptor("Operator", JaiColors.OPERATOR),
AttributesDescriptor("Uninitialized (---)", JaiColors.UNINITIALIZED),
AttributesDescriptor("Polymorph (\$, \$\$)", JaiColors.POLYMORPH),
AttributesDescriptor("Backtick", JaiColors.BACKTICK),
AttributesDescriptor("Braces and operators//Parentheses", JaiColors.PARENTHESES),
AttributesDescriptor("Braces and operators//Braces", JaiColors.BRACES),
AttributesDescriptor("Braces and operators//Brackets", JaiColors.BRACKETS),
AttributesDescriptor("Braces and operators//Semicolon", JaiColors.SEMICOLON),
AttributesDescriptor("Braces and operators//Comma", JaiColors.COMMA),
AttributesDescriptor("Braces and operators//Dot", JaiColors.DOT),
AttributesDescriptor("Bad character", JaiColors.BAD_CHARACTER),
)
// Exercises every attribute above, and doubles as a lexer fixture in tests.
private val DEMO_TEXT =
"""
// A tour of Jai syntax.
/* Block comments /* nest */ like this. */
#import "Basic";
Vector2 :: struct {
x, y: float;
label: string = "origin"; @Cleanup
}
Fruit :: enum u32 #specified {
BANANA :: 5;
APPLE;
}
MESSAGE :: #string DONE
Here-strings are raw: "quotes" and /* comments */ mean nothing.
DONE
main :: (count: int, scale: ${'$'}T = 1.5e-7) -> bool {
total := 0x1f_ff + 0b1011 + 0h7fbf_ffff;
uninitialized: [8] u8 = ---;
point := Vector2.{ 1.0, 2.0 };
names := .[ "a", "b" ];
for < values {
if it_index == { case 0; continue; case; break; }
total += cast(int) it;
}
defer free(point);
return total >= 0 && !(total < 0);
}
""".trimIndent()
}
}

133
src/main/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighter.kt Normal file
View File

@@ -0,0 +1,133 @@
package dev.hgh.jai.highlighting
import com.intellij.lexer.Lexer
import com.intellij.openapi.editor.DefaultLanguageHighlighterColors
import com.intellij.openapi.editor.colors.TextAttributesKey
import com.intellij.openapi.editor.colors.TextAttributesKey.createTextAttributesKey
import com.intellij.openapi.fileTypes.SyntaxHighlighter
import com.intellij.openapi.fileTypes.SyntaxHighlighterBase
import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory
import com.intellij.openapi.project.Project
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.TokenType
import com.intellij.psi.tree.IElementType
import dev.hgh.jai.lexer.JaiLexer
import dev.hgh.jai.lexer.JaiTokenTypes
/** Text attribute keys for Jai. See `docs/JAI_LANGUAGE_REFERENCE.md` §13. */
object JaiColors {
val KEYWORD = key("JAI_KEYWORD", DefaultLanguageHighlighterColors.KEYWORD)
val BUILTIN_TYPE = key("JAI_BUILTIN_TYPE", DefaultLanguageHighlighterColors.CLASS_NAME)
val IDENTIFIER = key("JAI_IDENTIFIER", DefaultLanguageHighlighterColors.IDENTIFIER)
val LOOP_VARIABLE = key("JAI_LOOP_VARIABLE", DefaultLanguageHighlighterColors.LOCAL_VARIABLE)
val DIRECTIVE = key("JAI_DIRECTIVE", DefaultLanguageHighlighterColors.METADATA)
val NOTE = key("JAI_NOTE", DefaultLanguageHighlighterColors.METADATA)
val NUMBER = key("JAI_NUMBER", DefaultLanguageHighlighterColors.NUMBER)
val STRING = key("JAI_STRING", DefaultLanguageHighlighterColors.STRING)
val HERE_STRING = key("JAI_HERE_STRING", DefaultLanguageHighlighterColors.STRING)
val LINE_COMMENT = key("JAI_LINE_COMMENT", DefaultLanguageHighlighterColors.LINE_COMMENT)
val BLOCK_COMMENT = key("JAI_BLOCK_COMMENT", DefaultLanguageHighlighterColors.BLOCK_COMMENT)
val OPERATOR = key("JAI_OPERATOR", DefaultLanguageHighlighterColors.OPERATION_SIGN)
val UNINITIALIZED = key("JAI_UNINITIALIZED", DefaultLanguageHighlighterColors.KEYWORD)
val POLYMORPH = key("JAI_POLYMORPH", DefaultLanguageHighlighterColors.STATIC_FIELD)
val BACKTICK = key("JAI_BACKTICK", DefaultLanguageHighlighterColors.STATIC_FIELD)
val PARENTHESES = key("JAI_PARENTHESES", DefaultLanguageHighlighterColors.PARENTHESES)
val BRACES = key("JAI_BRACES", DefaultLanguageHighlighterColors.BRACES)
val BRACKETS = key("JAI_BRACKETS", DefaultLanguageHighlighterColors.BRACKETS)
val SEMICOLON = key("JAI_SEMICOLON", DefaultLanguageHighlighterColors.SEMICOLON)
val COMMA = key("JAI_COMMA", DefaultLanguageHighlighterColors.COMMA)
val DOT = key("JAI_DOT", DefaultLanguageHighlighterColors.DOT)
val BAD_CHARACTER = key("JAI_BAD_CHARACTER", com.intellij.openapi.editor.HighlighterColors.BAD_CHARACTER)
private fun key(
name: String,
fallback: TextAttributesKey,
): TextAttributesKey = createTextAttributesKey(name, fallback)
}
/**
* Token types produced *only* for highlighting.
*
* Built-in type names and the implicit loop variables are ordinary identifiers to the
* compiler (`docs/JAI_LANGUAGE_REFERENCE.md` §3, §14.9) and must stay that way for the
* parser. The highlighting lexer is a separate instance from the parsing lexer, so it
* can refine IDENT without reserving anything.
*/
object JaiHighlightingTokens {
val BUILTIN_TYPE: IElementType =
dev.hgh.jai.lexer
.JaiTokenType("BUILTIN_TYPE")
val LOOP_VARIABLE: IElementType =
dev.hgh.jai.lexer
.JaiTokenType("LOOP_VARIABLE")
}
/** [JaiLexer] plus the highlight-only refinement of IDENT. */
class JaiHighlightingLexer : JaiLexer() {
override fun getTokenType(): IElementType? {
val type = super.getTokenType()
if (type === JaiTokenTypes.IDENT) {
val name = bufferSequence.subSequence(tokenStart, tokenEnd).toString()
if (name in JaiTokenTypes.BUILTIN_TYPE_NAMES) return JaiHighlightingTokens.BUILTIN_TYPE
if (name in JaiTokenTypes.LOOP_VARIABLE_NAMES) return JaiHighlightingTokens.LOOP_VARIABLE
}
return type
}
}
class JaiSyntaxHighlighter : SyntaxHighlighterBase() {
override fun getHighlightingLexer(): Lexer = JaiHighlightingLexer()
override fun getTokenHighlights(tokenType: IElementType): Array<TextAttributesKey> = pack(ATTRIBUTES[tokenType])
companion object {
private val ATTRIBUTES: Map<IElementType, TextAttributesKey> =
buildMap {
JaiTokenTypes.KEYWORD_MAP.values.forEach { put(it, JaiColors.KEYWORD) }
put(JaiTokenTypes.IDENT, JaiColors.IDENTIFIER)
put(JaiHighlightingTokens.BUILTIN_TYPE, JaiColors.BUILTIN_TYPE)
put(JaiHighlightingTokens.LOOP_VARIABLE, JaiColors.LOOP_VARIABLE)
put(JaiTokenTypes.NUMBER, JaiColors.NUMBER)
put(JaiTokenTypes.STRING, JaiColors.STRING)
put(JaiTokenTypes.HERE_STRING, JaiColors.HERE_STRING)
put(JaiTokenTypes.DIRECTIVE, JaiColors.DIRECTIVE)
put(JaiTokenTypes.NOTE, JaiColors.NOTE)
put(JaiTokenTypes.LINE_COMMENT, JaiColors.LINE_COMMENT)
put(JaiTokenTypes.BLOCK_COMMENT, JaiColors.BLOCK_COMMENT)
put(JaiTokenTypes.LPAREN, JaiColors.PARENTHESES)
put(JaiTokenTypes.RPAREN, JaiColors.PARENTHESES)
put(JaiTokenTypes.LBRACE, JaiColors.BRACES)
put(JaiTokenTypes.RBRACE, JaiColors.BRACES)
put(JaiTokenTypes.LBRACKET, JaiColors.BRACKETS)
put(JaiTokenTypes.RBRACKET, JaiColors.BRACKETS)
put(JaiTokenTypes.SEMICOLON, JaiColors.SEMICOLON)
put(JaiTokenTypes.COMMA, JaiColors.COMMA)
put(JaiTokenTypes.DOUBLE_COMMA, JaiColors.COMMA)
put(JaiTokenTypes.DOT, JaiColors.DOT)
put(JaiTokenTypes.TRIPLE_MINUS, JaiColors.UNINITIALIZED)
put(JaiTokenTypes.DOLLAR, JaiColors.POLYMORPH)
put(JaiTokenTypes.DOUBLE_DOLLAR, JaiColors.POLYMORPH)
put(JaiTokenTypes.BACKTICK, JaiColors.BACKTICK)
put(TokenType.BAD_CHARACTER, JaiColors.BAD_CHARACTER)
// Everything else that is an operator or punctuation.
JaiTokenTypes.OPERATORS.types.forEach { putIfAbsent(it, JaiColors.OPERATOR) }
}
/** Every token type the highlighter knows how to colour. Used by tests. */
val HIGHLIGHTED_TOKENS: Set<IElementType> get() = ATTRIBUTES.keys
}
}
class JaiSyntaxHighlighterFactory : SyntaxHighlighterFactory() {
override fun getSyntaxHighlighter(
project: Project?,
virtualFile: VirtualFile?,
): SyntaxHighlighter = JaiSyntaxHighlighter()
}

2
src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt
View File

@@ -18,7 +18,7 @@ import com.intellij.psi.tree.IElementType
* boundary never carries information. That makes the lexer restartable at any token,
* which is what incremental re-highlighting needs.
*/
class JaiLexer : LexerBase() {
open class JaiLexer : LexerBase() {
private var buf: CharSequence = ""
private var bufEnd = 0
private var tokStart = 0

6
src/main/resources/META-INF/plugin.xml
View File

@@ -20,6 +20,12 @@
fieldName="INSTANCE"
language="Jai"
extensions="jai"/>
<colorSettingsPage implementation="dev.hgh.jai.highlighting.JaiColorSettingsPage"/>
<lang.syntaxHighlighterFactory
language="Jai"
implementationClass="dev.hgh.jai.highlighting.JaiSyntaxHighlighterFactory"/>
</extensions>
</idea-plugin>

10
src/test/kotlin/dev/hgh/jai/JaiFileTypeTest.kt
View File

@@ -13,6 +13,16 @@ class JaiFileTypeTest : BasePlatformTestCase() {
assertEquals(JaiFileType, type)
}
fun testSyntaxHighlighterIsRegisteredForJai() {
val highlighter =
com.intellij.openapi.fileTypes.SyntaxHighlighterFactory
.getSyntaxHighlighter(JaiLanguage, null, null)
assertTrue(
"plugin.xml must register our highlighter, got $highlighter",
highlighter is dev.hgh.jai.highlighting.JaiSyntaxHighlighter,
)
}
/**
* The *virtual* file already resolves to our type. The PSI file's language stays
* plain text until a ParserDefinition is registered (phase 3) — asserting that here

80
src/test/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPageTest.kt Normal file
View File

@@ -0,0 +1,80 @@
package dev.hgh.jai.highlighting
import com.intellij.openapi.editor.colors.TextAttributesKey
import com.intellij.psi.TokenType
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
class JaiColorSettingsPageTest {
private val page = JaiColorSettingsPage()
/**
* The demo text is what a user sees when picking colours, so it must be valid Jai:
* it has to tile cleanly and contain no bad characters.
*/
@Test
fun demoTextLexesCleanly() {
val text = page.demoText
val lexer = JaiHighlightingLexer()
lexer.start(text, 0, text.length, 0)
val rebuilt = StringBuilder()
var offset = 0
while (true) {
val type = lexer.tokenType ?: break
assertEquals("gap or overlap in demo text", offset, lexer.tokenStart)
assertTrue("empty token in demo text", lexer.tokenEnd > lexer.tokenStart)
assertTrue(
"demo text contains a bad character at ${lexer.tokenStart}",
type != TokenType.BAD_CHARACTER,
)
rebuilt.append(text, lexer.tokenStart, lexer.tokenEnd)
offset = lexer.tokenEnd
lexer.advance()
}
assertEquals(text, rebuilt.toString())
}
/** Every colour the highlighter can emit must be settable by the user. */
@Test
fun everyColorKeyIsExposedOnThePage() {
val exposed: Set<TextAttributesKey> = page.attributeDescriptors.map { it.key }.toSet()
val declared =
JaiColors::class.java.declaredFields
.filter { TextAttributesKey::class.java.isAssignableFrom(it.type) }
.map {
it.isAccessible = true
it.get(JaiColors) as TextAttributesKey
}
val missing = declared.filterNot { it in exposed }
assertEquals("colour keys missing from the settings page: $missing", emptyList<TextAttributesKey>(), missing)
}
@Test
fun demoTextExercisesTheInterestingTokens() {
val text = page.demoText
val lexer = JaiHighlightingLexer()
lexer.start(text, 0, text.length, 0)
val seen = mutableSetOf<String>()
while (true) {
seen += (lexer.tokenType ?: break).toString()
lexer.advance()
}
listOf(
"Jai:HERE_STRING",
"Jai:BLOCK_COMMENT",
"Jai:LINE_COMMENT",
"Jai:DIRECTIVE",
"Jai:NOTE",
"Jai:NUMBER",
"Jai:STRING",
"Jai:---",
"Jai:.{",
"Jai:.[",
"Jai:BUILTIN_TYPE",
"Jai:LOOP_VARIABLE",
"Jai:struct",
"Jai:$",
).forEach { assertTrue("demo text never produces $it", it in seen) }
}
}

101
src/test/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighterTest.kt Normal file
View File

@@ -0,0 +1,101 @@
package dev.hgh.jai.highlighting
import com.intellij.psi.TokenType
import com.intellij.psi.tree.IElementType
import dev.hgh.jai.lexer.JaiLexer
import dev.hgh.jai.lexer.JaiTokenTypes
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
import java.io.File
/**
* Phase 2 gate: the highlighter must have an attribute for every token type the lexer
* can produce — a token with no mapping renders as plain text, which is invisible in a
* screenshot review but obvious here.
*/
class JaiSyntaxHighlighterTest {
private val highlighter = JaiSyntaxHighlighter()
/** Every token type declared in the token holder, by reflection over the object. */
private fun allDeclaredTokenTypes(): List<IElementType> =
JaiTokenTypes::class.java.declaredFields
.filter { IElementType::class.java.isAssignableFrom(it.type) }
.map {
it.isAccessible = true
it.get(JaiTokenTypes) as IElementType
}
@Test
fun everyTokenTypeHasAHighlight() {
val unmapped =
allDeclaredTokenTypes().filter {
highlighter.getTokenHighlights(it).isEmpty()
}
assertEquals("token types with no text attributes: $unmapped", emptyList<IElementType>(), unmapped)
}
@Test
fun highlightOnlyTokensAreMapped() {
assertTrue(highlighter.getTokenHighlights(JaiHighlightingTokens.BUILTIN_TYPE).isNotEmpty())
assertTrue(highlighter.getTokenHighlights(JaiHighlightingTokens.LOOP_VARIABLE).isNotEmpty())
assertTrue(highlighter.getTokenHighlights(TokenType.BAD_CHARACTER).isNotEmpty())
}
@Test
fun whitespaceIsNotHighlighted() {
assertEquals(0, highlighter.getTokenHighlights(TokenType.WHITE_SPACE).size)
}
@Test
fun builtinTypesAndLoopVariablesAreRefinedIdentifiers() {
val types = lexTypes("x: int = 0; for v { print(it, it_index); } Foo :: struct {}")
assertTrue(JaiHighlightingTokens.BUILTIN_TYPE in types)
assertTrue(JaiHighlightingTokens.LOOP_VARIABLE in types)
assertTrue(JaiTokenTypes.IDENT in types)
// ...but the parsing lexer still calls them plain identifiers.
val parsing = JaiLexer()
parsing.start("int", 0, 3, 0)
assertEquals(JaiTokenTypes.IDENT, parsing.tokenType)
}
@Test
fun highlightingLexerCoversTheWholeCorpusWithNoUnhighlightedTokens() {
val root = File(System.getProperty("user.home"), ".local/jai")
org.junit.Assume.assumeTrue("Jai distribution not found at $root", root.isDirectory)
val seen = mutableSetOf<IElementType>()
var files = 0
root.walkTopDown().filter { it.isFile && it.extension == "jai" }.forEach { file ->
files++
val lexer = JaiHighlightingLexer()
val text = file.readText()
lexer.start(text, 0, text.length, 0)
while (true) {
val type = lexer.tokenType ?: break
seen += type
lexer.advance()
}
}
assertTrue("no corpus files scanned", files > 0)
val unmapped =
seen.filter {
it != TokenType.WHITE_SPACE && highlighter.getTokenHighlights(it).isEmpty()
}
assertEquals("corpus produced unhighlighted tokens: $unmapped", emptyList<IElementType>(), unmapped)
println("Highlighter: $files corpus files produced ${seen.size} distinct token types, all mapped.")
}
private fun lexTypes(code: String): Set<IElementType> {
val lexer = JaiHighlightingLexer()
lexer.start(code, 0, code.length, 0)
val out = mutableSetOf<IElementType>()
while (true) {
out += lexer.tokenType ?: break
lexer.advance()
}
return out
}
}