From 2c5d7f713bc8cbd710956b01db1ca1f3ef1ff415 Mon Sep 17 00:00:00 2001 From: hgranthorner <37941012+hgranthorner@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:28:15 -0400 Subject: [PATCH] Phase 2: syntax highlighting and colour settings page - JaiSyntaxHighlighter maps every one of the lexer's token types; asserted over the whole corpus (105 distinct token types seen, all mapped) - JaiHighlightingLexer refines IDENT into built-in types and loop variables for colouring only, so the parser never sees them as reserved - colour settings page with a demo file that is itself asserted to lex cleanly --- .../jai/highlighting/JaiColorSettingsPage.kt | 90 ++++++++++++ .../jai/highlighting/JaiSyntaxHighlighter.kt | 133 ++++++++++++++++++ src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt | 2 +- src/main/resources/META-INF/plugin.xml | 6 + .../kotlin/dev/hgh/jai/JaiFileTypeTest.kt | 10 ++ .../highlighting/JaiColorSettingsPageTest.kt | 80 +++++++++++ .../highlighting/JaiSyntaxHighlighterTest.kt | 101 +++++++++++++ 7 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPage.kt create mode 100644 src/main/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighter.kt create mode 100644 src/test/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPageTest.kt create mode 100644 src/test/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighterTest.kt diff --git a/src/main/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPage.kt b/src/main/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPage.kt new file mode 100644 index 0000000..0a7bc88 --- /dev/null +++ b/src/main/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPage.kt @@ -0,0 +1,90 @@ +package dev.hgh.jai.highlighting + +import com.intellij.openapi.editor.colors.TextAttributesKey +import com.intellij.openapi.fileTypes.SyntaxHighlighter +import com.intellij.openapi.options.colors.AttributesDescriptor +import com.intellij.openapi.options.colors.ColorDescriptor +import com.intellij.openapi.options.colors.ColorSettingsPage +import dev.hgh.jai.JaiIcons +import javax.swing.Icon + +class JaiColorSettingsPage : ColorSettingsPage { + override fun getDisplayName(): String = "Jai" + + override fun getIcon(): Icon = JaiIcons.FILE + + override fun getHighlighter(): SyntaxHighlighter = JaiSyntaxHighlighter() + + override fun getAttributeDescriptors(): Array = DESCRIPTORS + + override fun getColorDescriptors(): Array = ColorDescriptor.EMPTY_ARRAY + + override fun getAdditionalHighlightingTagToDescriptorMap(): Map? = null + + override fun getDemoText(): String = DEMO_TEXT + + companion object { + private val DESCRIPTORS = + arrayOf( + AttributesDescriptor("Keyword", JaiColors.KEYWORD), + AttributesDescriptor("Built-in type", JaiColors.BUILTIN_TYPE), + AttributesDescriptor("Identifier", JaiColors.IDENTIFIER), + AttributesDescriptor("Loop variable (it, it_index)", JaiColors.LOOP_VARIABLE), + AttributesDescriptor("Compiler directive", JaiColors.DIRECTIVE), + AttributesDescriptor("Note", JaiColors.NOTE), + AttributesDescriptor("Number", JaiColors.NUMBER), + AttributesDescriptor("String", JaiColors.STRING), + AttributesDescriptor("Here-string", JaiColors.HERE_STRING), + AttributesDescriptor("Comments//Line comment", JaiColors.LINE_COMMENT), + AttributesDescriptor("Comments//Block comment", JaiColors.BLOCK_COMMENT), + AttributesDescriptor("Operator", JaiColors.OPERATOR), + AttributesDescriptor("Uninitialized (---)", JaiColors.UNINITIALIZED), + AttributesDescriptor("Polymorph (\$, \$\$)", JaiColors.POLYMORPH), + AttributesDescriptor("Backtick", JaiColors.BACKTICK), + AttributesDescriptor("Braces and operators//Parentheses", JaiColors.PARENTHESES), + AttributesDescriptor("Braces and operators//Braces", JaiColors.BRACES), + AttributesDescriptor("Braces and operators//Brackets", JaiColors.BRACKETS), + AttributesDescriptor("Braces and operators//Semicolon", JaiColors.SEMICOLON), + AttributesDescriptor("Braces and operators//Comma", JaiColors.COMMA), + AttributesDescriptor("Braces and operators//Dot", JaiColors.DOT), + AttributesDescriptor("Bad character", JaiColors.BAD_CHARACTER), + ) + + // Exercises every attribute above, and doubles as a lexer fixture in tests. + private val DEMO_TEXT = + """ + // A tour of Jai syntax. + /* Block comments /* nest */ like this. */ + #import "Basic"; + + Vector2 :: struct { + x, y: float; + label: string = "origin"; @Cleanup + } + + Fruit :: enum u32 #specified { + BANANA :: 5; + APPLE; + } + + MESSAGE :: #string DONE + Here-strings are raw: "quotes" and /* comments */ mean nothing. + DONE + + main :: (count: int, scale: ${'$'}T = 1.5e-7) -> bool { + total := 0x1f_ff + 0b1011 + 0h7fbf_ffff; + uninitialized: [8] u8 = ---; + point := Vector2.{ 1.0, 2.0 }; + names := .[ "a", "b" ]; + + for < values { + if it_index == { case 0; continue; case; break; } + total += cast(int) it; + } + + defer free(point); + return total >= 0 && !(total < 0); + } + """.trimIndent() + } +} diff --git a/src/main/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighter.kt b/src/main/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighter.kt new file mode 100644 index 0000000..7d4969a --- /dev/null +++ b/src/main/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighter.kt @@ -0,0 +1,133 @@ +package dev.hgh.jai.highlighting + +import com.intellij.lexer.Lexer +import com.intellij.openapi.editor.DefaultLanguageHighlighterColors +import com.intellij.openapi.editor.colors.TextAttributesKey +import com.intellij.openapi.editor.colors.TextAttributesKey.createTextAttributesKey +import com.intellij.openapi.fileTypes.SyntaxHighlighter +import com.intellij.openapi.fileTypes.SyntaxHighlighterBase +import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory +import com.intellij.openapi.project.Project +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.psi.TokenType +import com.intellij.psi.tree.IElementType +import dev.hgh.jai.lexer.JaiLexer +import dev.hgh.jai.lexer.JaiTokenTypes + +/** Text attribute keys for Jai. See `docs/JAI_LANGUAGE_REFERENCE.md` §13. */ +object JaiColors { + val KEYWORD = key("JAI_KEYWORD", DefaultLanguageHighlighterColors.KEYWORD) + val BUILTIN_TYPE = key("JAI_BUILTIN_TYPE", DefaultLanguageHighlighterColors.CLASS_NAME) + val IDENTIFIER = key("JAI_IDENTIFIER", DefaultLanguageHighlighterColors.IDENTIFIER) + val LOOP_VARIABLE = key("JAI_LOOP_VARIABLE", DefaultLanguageHighlighterColors.LOCAL_VARIABLE) + val DIRECTIVE = key("JAI_DIRECTIVE", DefaultLanguageHighlighterColors.METADATA) + val NOTE = key("JAI_NOTE", DefaultLanguageHighlighterColors.METADATA) + val NUMBER = key("JAI_NUMBER", DefaultLanguageHighlighterColors.NUMBER) + val STRING = key("JAI_STRING", DefaultLanguageHighlighterColors.STRING) + val HERE_STRING = key("JAI_HERE_STRING", DefaultLanguageHighlighterColors.STRING) + val LINE_COMMENT = key("JAI_LINE_COMMENT", DefaultLanguageHighlighterColors.LINE_COMMENT) + val BLOCK_COMMENT = key("JAI_BLOCK_COMMENT", DefaultLanguageHighlighterColors.BLOCK_COMMENT) + val OPERATOR = key("JAI_OPERATOR", DefaultLanguageHighlighterColors.OPERATION_SIGN) + val UNINITIALIZED = key("JAI_UNINITIALIZED", DefaultLanguageHighlighterColors.KEYWORD) + val POLYMORPH = key("JAI_POLYMORPH", DefaultLanguageHighlighterColors.STATIC_FIELD) + val BACKTICK = key("JAI_BACKTICK", DefaultLanguageHighlighterColors.STATIC_FIELD) + val PARENTHESES = key("JAI_PARENTHESES", DefaultLanguageHighlighterColors.PARENTHESES) + val BRACES = key("JAI_BRACES", DefaultLanguageHighlighterColors.BRACES) + val BRACKETS = key("JAI_BRACKETS", DefaultLanguageHighlighterColors.BRACKETS) + val SEMICOLON = key("JAI_SEMICOLON", DefaultLanguageHighlighterColors.SEMICOLON) + val COMMA = key("JAI_COMMA", DefaultLanguageHighlighterColors.COMMA) + val DOT = key("JAI_DOT", DefaultLanguageHighlighterColors.DOT) + val BAD_CHARACTER = key("JAI_BAD_CHARACTER", com.intellij.openapi.editor.HighlighterColors.BAD_CHARACTER) + + private fun key( + name: String, + fallback: TextAttributesKey, + ): TextAttributesKey = createTextAttributesKey(name, fallback) +} + +/** + * Token types produced *only* for highlighting. + * + * Built-in type names and the implicit loop variables are ordinary identifiers to the + * compiler (`docs/JAI_LANGUAGE_REFERENCE.md` §3, §14.9) and must stay that way for the + * parser. The highlighting lexer is a separate instance from the parsing lexer, so it + * can refine IDENT without reserving anything. + */ +object JaiHighlightingTokens { + val BUILTIN_TYPE: IElementType = + dev.hgh.jai.lexer + .JaiTokenType("BUILTIN_TYPE") + val LOOP_VARIABLE: IElementType = + dev.hgh.jai.lexer + .JaiTokenType("LOOP_VARIABLE") +} + +/** [JaiLexer] plus the highlight-only refinement of IDENT. */ +class JaiHighlightingLexer : JaiLexer() { + override fun getTokenType(): IElementType? { + val type = super.getTokenType() + if (type === JaiTokenTypes.IDENT) { + val name = bufferSequence.subSequence(tokenStart, tokenEnd).toString() + if (name in JaiTokenTypes.BUILTIN_TYPE_NAMES) return JaiHighlightingTokens.BUILTIN_TYPE + if (name in JaiTokenTypes.LOOP_VARIABLE_NAMES) return JaiHighlightingTokens.LOOP_VARIABLE + } + return type + } +} + +class JaiSyntaxHighlighter : SyntaxHighlighterBase() { + override fun getHighlightingLexer(): Lexer = JaiHighlightingLexer() + + override fun getTokenHighlights(tokenType: IElementType): Array = pack(ATTRIBUTES[tokenType]) + + companion object { + private val ATTRIBUTES: Map = + buildMap { + JaiTokenTypes.KEYWORD_MAP.values.forEach { put(it, JaiColors.KEYWORD) } + + put(JaiTokenTypes.IDENT, JaiColors.IDENTIFIER) + put(JaiHighlightingTokens.BUILTIN_TYPE, JaiColors.BUILTIN_TYPE) + put(JaiHighlightingTokens.LOOP_VARIABLE, JaiColors.LOOP_VARIABLE) + + put(JaiTokenTypes.NUMBER, JaiColors.NUMBER) + put(JaiTokenTypes.STRING, JaiColors.STRING) + put(JaiTokenTypes.HERE_STRING, JaiColors.HERE_STRING) + put(JaiTokenTypes.DIRECTIVE, JaiColors.DIRECTIVE) + put(JaiTokenTypes.NOTE, JaiColors.NOTE) + + put(JaiTokenTypes.LINE_COMMENT, JaiColors.LINE_COMMENT) + put(JaiTokenTypes.BLOCK_COMMENT, JaiColors.BLOCK_COMMENT) + + put(JaiTokenTypes.LPAREN, JaiColors.PARENTHESES) + put(JaiTokenTypes.RPAREN, JaiColors.PARENTHESES) + put(JaiTokenTypes.LBRACE, JaiColors.BRACES) + put(JaiTokenTypes.RBRACE, JaiColors.BRACES) + put(JaiTokenTypes.LBRACKET, JaiColors.BRACKETS) + put(JaiTokenTypes.RBRACKET, JaiColors.BRACKETS) + put(JaiTokenTypes.SEMICOLON, JaiColors.SEMICOLON) + put(JaiTokenTypes.COMMA, JaiColors.COMMA) + put(JaiTokenTypes.DOUBLE_COMMA, JaiColors.COMMA) + put(JaiTokenTypes.DOT, JaiColors.DOT) + + put(JaiTokenTypes.TRIPLE_MINUS, JaiColors.UNINITIALIZED) + put(JaiTokenTypes.DOLLAR, JaiColors.POLYMORPH) + put(JaiTokenTypes.DOUBLE_DOLLAR, JaiColors.POLYMORPH) + put(JaiTokenTypes.BACKTICK, JaiColors.BACKTICK) + + put(TokenType.BAD_CHARACTER, JaiColors.BAD_CHARACTER) + + // Everything else that is an operator or punctuation. + JaiTokenTypes.OPERATORS.types.forEach { putIfAbsent(it, JaiColors.OPERATOR) } + } + + /** Every token type the highlighter knows how to colour. Used by tests. */ + val HIGHLIGHTED_TOKENS: Set get() = ATTRIBUTES.keys + } +} + +class JaiSyntaxHighlighterFactory : SyntaxHighlighterFactory() { + override fun getSyntaxHighlighter( + project: Project?, + virtualFile: VirtualFile?, + ): SyntaxHighlighter = JaiSyntaxHighlighter() +} diff --git a/src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt b/src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt index 1723a7f..8e18ec2 100644 --- a/src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt +++ b/src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt @@ -18,7 +18,7 @@ import com.intellij.psi.tree.IElementType * boundary never carries information. That makes the lexer restartable at any token, * which is what incremental re-highlighting needs. */ -class JaiLexer : LexerBase() { +open class JaiLexer : LexerBase() { private var buf: CharSequence = "" private var bufEnd = 0 private var tokStart = 0 diff --git a/src/main/resources/META-INF/plugin.xml b/src/main/resources/META-INF/plugin.xml index e3f1e0a..aa68520 100644 --- a/src/main/resources/META-INF/plugin.xml +++ b/src/main/resources/META-INF/plugin.xml @@ -20,6 +20,12 @@ fieldName="INSTANCE" language="Jai" extensions="jai"/> + + + + diff --git a/src/test/kotlin/dev/hgh/jai/JaiFileTypeTest.kt b/src/test/kotlin/dev/hgh/jai/JaiFileTypeTest.kt index ba50ea9..2e5b38f 100644 --- a/src/test/kotlin/dev/hgh/jai/JaiFileTypeTest.kt +++ b/src/test/kotlin/dev/hgh/jai/JaiFileTypeTest.kt @@ -13,6 +13,16 @@ class JaiFileTypeTest : BasePlatformTestCase() { assertEquals(JaiFileType, type) } + fun testSyntaxHighlighterIsRegisteredForJai() { + val highlighter = + com.intellij.openapi.fileTypes.SyntaxHighlighterFactory + .getSyntaxHighlighter(JaiLanguage, null, null) + assertTrue( + "plugin.xml must register our highlighter, got $highlighter", + highlighter is dev.hgh.jai.highlighting.JaiSyntaxHighlighter, + ) + } + /** * The *virtual* file already resolves to our type. The PSI file's language stays * plain text until a ParserDefinition is registered (phase 3) — asserting that here diff --git a/src/test/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPageTest.kt b/src/test/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPageTest.kt new file mode 100644 index 0000000..9ad6eda --- /dev/null +++ b/src/test/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPageTest.kt @@ -0,0 +1,80 @@ +package dev.hgh.jai.highlighting + +import com.intellij.openapi.editor.colors.TextAttributesKey +import com.intellij.psi.TokenType +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class JaiColorSettingsPageTest { + private val page = JaiColorSettingsPage() + + /** + * The demo text is what a user sees when picking colours, so it must be valid Jai: + * it has to tile cleanly and contain no bad characters. + */ + @Test + fun demoTextLexesCleanly() { + val text = page.demoText + val lexer = JaiHighlightingLexer() + lexer.start(text, 0, text.length, 0) + val rebuilt = StringBuilder() + var offset = 0 + while (true) { + val type = lexer.tokenType ?: break + assertEquals("gap or overlap in demo text", offset, lexer.tokenStart) + assertTrue("empty token in demo text", lexer.tokenEnd > lexer.tokenStart) + assertTrue( + "demo text contains a bad character at ${lexer.tokenStart}", + type != TokenType.BAD_CHARACTER, + ) + rebuilt.append(text, lexer.tokenStart, lexer.tokenEnd) + offset = lexer.tokenEnd + lexer.advance() + } + assertEquals(text, rebuilt.toString()) + } + + /** Every colour the highlighter can emit must be settable by the user. */ + @Test + fun everyColorKeyIsExposedOnThePage() { + val exposed: Set = page.attributeDescriptors.map { it.key }.toSet() + val declared = + JaiColors::class.java.declaredFields + .filter { TextAttributesKey::class.java.isAssignableFrom(it.type) } + .map { + it.isAccessible = true + it.get(JaiColors) as TextAttributesKey + } + val missing = declared.filterNot { it in exposed } + assertEquals("colour keys missing from the settings page: $missing", emptyList(), missing) + } + + @Test + fun demoTextExercisesTheInterestingTokens() { + val text = page.demoText + val lexer = JaiHighlightingLexer() + lexer.start(text, 0, text.length, 0) + val seen = mutableSetOf() + while (true) { + seen += (lexer.tokenType ?: break).toString() + lexer.advance() + } + listOf( + "Jai:HERE_STRING", + "Jai:BLOCK_COMMENT", + "Jai:LINE_COMMENT", + "Jai:DIRECTIVE", + "Jai:NOTE", + "Jai:NUMBER", + "Jai:STRING", + "Jai:---", + "Jai:.{", + "Jai:.[", + "Jai:BUILTIN_TYPE", + "Jai:LOOP_VARIABLE", + "Jai:struct", + "Jai:$", + ).forEach { assertTrue("demo text never produces $it", it in seen) } + } +} diff --git a/src/test/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighterTest.kt b/src/test/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighterTest.kt new file mode 100644 index 0000000..1883efa --- /dev/null +++ b/src/test/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighterTest.kt @@ -0,0 +1,101 @@ +package dev.hgh.jai.highlighting + +import com.intellij.psi.TokenType +import com.intellij.psi.tree.IElementType +import dev.hgh.jai.lexer.JaiLexer +import dev.hgh.jai.lexer.JaiTokenTypes +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.File + +/** + * Phase 2 gate: the highlighter must have an attribute for every token type the lexer + * can produce — a token with no mapping renders as plain text, which is invisible in a + * screenshot review but obvious here. + */ +class JaiSyntaxHighlighterTest { + private val highlighter = JaiSyntaxHighlighter() + + /** Every token type declared in the token holder, by reflection over the object. */ + private fun allDeclaredTokenTypes(): List = + JaiTokenTypes::class.java.declaredFields + .filter { IElementType::class.java.isAssignableFrom(it.type) } + .map { + it.isAccessible = true + it.get(JaiTokenTypes) as IElementType + } + + @Test + fun everyTokenTypeHasAHighlight() { + val unmapped = + allDeclaredTokenTypes().filter { + highlighter.getTokenHighlights(it).isEmpty() + } + assertEquals("token types with no text attributes: $unmapped", emptyList(), unmapped) + } + + @Test + fun highlightOnlyTokensAreMapped() { + assertTrue(highlighter.getTokenHighlights(JaiHighlightingTokens.BUILTIN_TYPE).isNotEmpty()) + assertTrue(highlighter.getTokenHighlights(JaiHighlightingTokens.LOOP_VARIABLE).isNotEmpty()) + assertTrue(highlighter.getTokenHighlights(TokenType.BAD_CHARACTER).isNotEmpty()) + } + + @Test + fun whitespaceIsNotHighlighted() { + assertEquals(0, highlighter.getTokenHighlights(TokenType.WHITE_SPACE).size) + } + + @Test + fun builtinTypesAndLoopVariablesAreRefinedIdentifiers() { + val types = lexTypes("x: int = 0; for v { print(it, it_index); } Foo :: struct {}") + assertTrue(JaiHighlightingTokens.BUILTIN_TYPE in types) + assertTrue(JaiHighlightingTokens.LOOP_VARIABLE in types) + assertTrue(JaiTokenTypes.IDENT in types) + + // ...but the parsing lexer still calls them plain identifiers. + val parsing = JaiLexer() + parsing.start("int", 0, 3, 0) + assertEquals(JaiTokenTypes.IDENT, parsing.tokenType) + } + + @Test + fun highlightingLexerCoversTheWholeCorpusWithNoUnhighlightedTokens() { + val root = File(System.getProperty("user.home"), ".local/jai") + org.junit.Assume.assumeTrue("Jai distribution not found at $root", root.isDirectory) + + val seen = mutableSetOf() + var files = 0 + root.walkTopDown().filter { it.isFile && it.extension == "jai" }.forEach { file -> + files++ + val lexer = JaiHighlightingLexer() + val text = file.readText() + lexer.start(text, 0, text.length, 0) + while (true) { + val type = lexer.tokenType ?: break + seen += type + lexer.advance() + } + } + + assertTrue("no corpus files scanned", files > 0) + val unmapped = + seen.filter { + it != TokenType.WHITE_SPACE && highlighter.getTokenHighlights(it).isEmpty() + } + assertEquals("corpus produced unhighlighted tokens: $unmapped", emptyList(), unmapped) + println("Highlighter: $files corpus files produced ${seen.size} distinct token types, all mapped.") + } + + private fun lexTypes(code: String): Set { + val lexer = JaiHighlightingLexer() + lexer.start(code, 0, code.length, 0) + val out = mutableSetOf() + while (true) { + out += lexer.tokenType ?: break + lexer.advance() + } + return out + } +}