Phase 2: syntax highlighting and colour settings page

- JaiSyntaxHighlighter maps every one of the lexer's token types; asserted over the whole corpus (105 distinct token types seen, all mapped) - JaiHighlightingLexer refines IDENT into built-in types and loop variables for colouring only, so the parser never sees them as reserved - colour settings page with a demo file that is itself asserted to lex cleanly
2026-08-04 10:28:15 -04:00
parent ba5d8aba7b
commit 2c5d7f713b
7 changed files with 421 additions and 1 deletions
--- a/src/main/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPage.kt
+++ b/src/main/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPage.kt
@@ -0,0 +1,90 @@
+package dev.hgh.jai.highlighting
+
+import com.intellij.openapi.editor.colors.TextAttributesKey
+import com.intellij.openapi.fileTypes.SyntaxHighlighter
+import com.intellij.openapi.options.colors.AttributesDescriptor
+import com.intellij.openapi.options.colors.ColorDescriptor
+import com.intellij.openapi.options.colors.ColorSettingsPage
+import dev.hgh.jai.JaiIcons
+import javax.swing.Icon
+
+class JaiColorSettingsPage : ColorSettingsPage {
+    override fun getDisplayName(): String = "Jai"
+
+    override fun getIcon(): Icon = JaiIcons.FILE
+
+    override fun getHighlighter(): SyntaxHighlighter = JaiSyntaxHighlighter()
+
+    override fun getAttributeDescriptors(): Array<AttributesDescriptor> = DESCRIPTORS
+
+    override fun getColorDescriptors(): Array<ColorDescriptor> = ColorDescriptor.EMPTY_ARRAY
+
+    override fun getAdditionalHighlightingTagToDescriptorMap(): Map<String, TextAttributesKey>? = null
+
+    override fun getDemoText(): String = DEMO_TEXT
+
+    companion object {
+        private val DESCRIPTORS =
+            arrayOf(
+                AttributesDescriptor("Keyword", JaiColors.KEYWORD),
+                AttributesDescriptor("Built-in type", JaiColors.BUILTIN_TYPE),
+                AttributesDescriptor("Identifier", JaiColors.IDENTIFIER),
+                AttributesDescriptor("Loop variable (it, it_index)", JaiColors.LOOP_VARIABLE),
+                AttributesDescriptor("Compiler directive", JaiColors.DIRECTIVE),
+                AttributesDescriptor("Note", JaiColors.NOTE),
+                AttributesDescriptor("Number", JaiColors.NUMBER),
+                AttributesDescriptor("String", JaiColors.STRING),
+                AttributesDescriptor("Here-string", JaiColors.HERE_STRING),
+                AttributesDescriptor("Comments//Line comment", JaiColors.LINE_COMMENT),
+                AttributesDescriptor("Comments//Block comment", JaiColors.BLOCK_COMMENT),
+                AttributesDescriptor("Operator", JaiColors.OPERATOR),
+                AttributesDescriptor("Uninitialized (---)", JaiColors.UNINITIALIZED),
+                AttributesDescriptor("Polymorph (\$, \$\$)", JaiColors.POLYMORPH),
+                AttributesDescriptor("Backtick", JaiColors.BACKTICK),
+                AttributesDescriptor("Braces and operators//Parentheses", JaiColors.PARENTHESES),
+                AttributesDescriptor("Braces and operators//Braces", JaiColors.BRACES),
+                AttributesDescriptor("Braces and operators//Brackets", JaiColors.BRACKETS),
+                AttributesDescriptor("Braces and operators//Semicolon", JaiColors.SEMICOLON),
+                AttributesDescriptor("Braces and operators//Comma", JaiColors.COMMA),
+                AttributesDescriptor("Braces and operators//Dot", JaiColors.DOT),
+                AttributesDescriptor("Bad character", JaiColors.BAD_CHARACTER),
+            )
+
+        // Exercises every attribute above, and doubles as a lexer fixture in tests.
+        private val DEMO_TEXT =
+            """
+            // A tour of Jai syntax.
+            /* Block comments /* nest */ like this. */
+            #import "Basic";
+
+            Vector2 :: struct {
+                x, y: float;
+                label: string = "origin";  @Cleanup
+            }
+
+            Fruit :: enum u32 #specified {
+                BANANA :: 5;
+                APPLE;
+            }
+
+            MESSAGE :: #string DONE
+            Here-strings are raw: "quotes" and /* comments */ mean nothing.
+            DONE
+
+            main :: (count: int, scale: ${'$'}T = 1.5e-7) -> bool {
+                total := 0x1f_ff + 0b1011 + 0h7fbf_ffff;
+                uninitialized: [8] u8 = ---;
+                point := Vector2.{ 1.0, 2.0 };
+                names := .[ "a", "b" ];
+
+                for < values {
+                    if it_index == { case 0; continue; case; break; }
+                    total += cast(int) it;
+                }
+
+                defer free(point);
+                return total >= 0 && !(total < 0);
+            }
+            """.trimIndent()
+    }
+}
--- a/src/main/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighter.kt
+++ b/src/main/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighter.kt
@@ -0,0 +1,133 @@
+package dev.hgh.jai.highlighting
+
+import com.intellij.lexer.Lexer
+import com.intellij.openapi.editor.DefaultLanguageHighlighterColors
+import com.intellij.openapi.editor.colors.TextAttributesKey
+import com.intellij.openapi.editor.colors.TextAttributesKey.createTextAttributesKey
+import com.intellij.openapi.fileTypes.SyntaxHighlighter
+import com.intellij.openapi.fileTypes.SyntaxHighlighterBase
+import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.psi.TokenType
+import com.intellij.psi.tree.IElementType
+import dev.hgh.jai.lexer.JaiLexer
+import dev.hgh.jai.lexer.JaiTokenTypes
+
+/** Text attribute keys for Jai. See `docs/JAI_LANGUAGE_REFERENCE.md` §13. */
+object JaiColors {
+    val KEYWORD = key("JAI_KEYWORD", DefaultLanguageHighlighterColors.KEYWORD)
+    val BUILTIN_TYPE = key("JAI_BUILTIN_TYPE", DefaultLanguageHighlighterColors.CLASS_NAME)
+    val IDENTIFIER = key("JAI_IDENTIFIER", DefaultLanguageHighlighterColors.IDENTIFIER)
+    val LOOP_VARIABLE = key("JAI_LOOP_VARIABLE", DefaultLanguageHighlighterColors.LOCAL_VARIABLE)
+    val DIRECTIVE = key("JAI_DIRECTIVE", DefaultLanguageHighlighterColors.METADATA)
+    val NOTE = key("JAI_NOTE", DefaultLanguageHighlighterColors.METADATA)
+    val NUMBER = key("JAI_NUMBER", DefaultLanguageHighlighterColors.NUMBER)
+    val STRING = key("JAI_STRING", DefaultLanguageHighlighterColors.STRING)
+    val HERE_STRING = key("JAI_HERE_STRING", DefaultLanguageHighlighterColors.STRING)
+    val LINE_COMMENT = key("JAI_LINE_COMMENT", DefaultLanguageHighlighterColors.LINE_COMMENT)
+    val BLOCK_COMMENT = key("JAI_BLOCK_COMMENT", DefaultLanguageHighlighterColors.BLOCK_COMMENT)
+    val OPERATOR = key("JAI_OPERATOR", DefaultLanguageHighlighterColors.OPERATION_SIGN)
+    val UNINITIALIZED = key("JAI_UNINITIALIZED", DefaultLanguageHighlighterColors.KEYWORD)
+    val POLYMORPH = key("JAI_POLYMORPH", DefaultLanguageHighlighterColors.STATIC_FIELD)
+    val BACKTICK = key("JAI_BACKTICK", DefaultLanguageHighlighterColors.STATIC_FIELD)
+    val PARENTHESES = key("JAI_PARENTHESES", DefaultLanguageHighlighterColors.PARENTHESES)
+    val BRACES = key("JAI_BRACES", DefaultLanguageHighlighterColors.BRACES)
+    val BRACKETS = key("JAI_BRACKETS", DefaultLanguageHighlighterColors.BRACKETS)
+    val SEMICOLON = key("JAI_SEMICOLON", DefaultLanguageHighlighterColors.SEMICOLON)
+    val COMMA = key("JAI_COMMA", DefaultLanguageHighlighterColors.COMMA)
+    val DOT = key("JAI_DOT", DefaultLanguageHighlighterColors.DOT)
+    val BAD_CHARACTER = key("JAI_BAD_CHARACTER", com.intellij.openapi.editor.HighlighterColors.BAD_CHARACTER)
+
+    private fun key(
+        name: String,
+        fallback: TextAttributesKey,
+    ): TextAttributesKey = createTextAttributesKey(name, fallback)
+}
+
+/**
+ * Token types produced *only* for highlighting.
+ *
+ * Built-in type names and the implicit loop variables are ordinary identifiers to the
+ * compiler (`docs/JAI_LANGUAGE_REFERENCE.md` §3, §14.9) and must stay that way for the
+ * parser. The highlighting lexer is a separate instance from the parsing lexer, so it
+ * can refine IDENT without reserving anything.
+ */
+object JaiHighlightingTokens {
+    val BUILTIN_TYPE: IElementType =
+        dev.hgh.jai.lexer
+            .JaiTokenType("BUILTIN_TYPE")
+    val LOOP_VARIABLE: IElementType =
+        dev.hgh.jai.lexer
+            .JaiTokenType("LOOP_VARIABLE")
+}
+
+/** [JaiLexer] plus the highlight-only refinement of IDENT. */
+class JaiHighlightingLexer : JaiLexer() {
+    override fun getTokenType(): IElementType? {
+        val type = super.getTokenType()
+        if (type === JaiTokenTypes.IDENT) {
+            val name = bufferSequence.subSequence(tokenStart, tokenEnd).toString()
+            if (name in JaiTokenTypes.BUILTIN_TYPE_NAMES) return JaiHighlightingTokens.BUILTIN_TYPE
+            if (name in JaiTokenTypes.LOOP_VARIABLE_NAMES) return JaiHighlightingTokens.LOOP_VARIABLE
+        }
+        return type
+    }
+}
+
+class JaiSyntaxHighlighter : SyntaxHighlighterBase() {
+    override fun getHighlightingLexer(): Lexer = JaiHighlightingLexer()
+
+    override fun getTokenHighlights(tokenType: IElementType): Array<TextAttributesKey> = pack(ATTRIBUTES[tokenType])
+
+    companion object {
+        private val ATTRIBUTES: Map<IElementType, TextAttributesKey> =
+            buildMap {
+                JaiTokenTypes.KEYWORD_MAP.values.forEach { put(it, JaiColors.KEYWORD) }
+
+                put(JaiTokenTypes.IDENT, JaiColors.IDENTIFIER)
+                put(JaiHighlightingTokens.BUILTIN_TYPE, JaiColors.BUILTIN_TYPE)
+                put(JaiHighlightingTokens.LOOP_VARIABLE, JaiColors.LOOP_VARIABLE)
+
+                put(JaiTokenTypes.NUMBER, JaiColors.NUMBER)
+                put(JaiTokenTypes.STRING, JaiColors.STRING)
+                put(JaiTokenTypes.HERE_STRING, JaiColors.HERE_STRING)
+                put(JaiTokenTypes.DIRECTIVE, JaiColors.DIRECTIVE)
+                put(JaiTokenTypes.NOTE, JaiColors.NOTE)
+
+                put(JaiTokenTypes.LINE_COMMENT, JaiColors.LINE_COMMENT)
+                put(JaiTokenTypes.BLOCK_COMMENT, JaiColors.BLOCK_COMMENT)
+
+                put(JaiTokenTypes.LPAREN, JaiColors.PARENTHESES)
+                put(JaiTokenTypes.RPAREN, JaiColors.PARENTHESES)
+                put(JaiTokenTypes.LBRACE, JaiColors.BRACES)
+                put(JaiTokenTypes.RBRACE, JaiColors.BRACES)
+                put(JaiTokenTypes.LBRACKET, JaiColors.BRACKETS)
+                put(JaiTokenTypes.RBRACKET, JaiColors.BRACKETS)
+                put(JaiTokenTypes.SEMICOLON, JaiColors.SEMICOLON)
+                put(JaiTokenTypes.COMMA, JaiColors.COMMA)
+                put(JaiTokenTypes.DOUBLE_COMMA, JaiColors.COMMA)
+                put(JaiTokenTypes.DOT, JaiColors.DOT)
+
+                put(JaiTokenTypes.TRIPLE_MINUS, JaiColors.UNINITIALIZED)
+                put(JaiTokenTypes.DOLLAR, JaiColors.POLYMORPH)
+                put(JaiTokenTypes.DOUBLE_DOLLAR, JaiColors.POLYMORPH)
+                put(JaiTokenTypes.BACKTICK, JaiColors.BACKTICK)
+
+                put(TokenType.BAD_CHARACTER, JaiColors.BAD_CHARACTER)
+
+                // Everything else that is an operator or punctuation.
+                JaiTokenTypes.OPERATORS.types.forEach { putIfAbsent(it, JaiColors.OPERATOR) }
+            }
+
+        /** Every token type the highlighter knows how to colour. Used by tests. */
+        val HIGHLIGHTED_TOKENS: Set<IElementType> get() = ATTRIBUTES.keys
+    }
+}
+
+class JaiSyntaxHighlighterFactory : SyntaxHighlighterFactory() {
+    override fun getSyntaxHighlighter(
+        project: Project?,
+        virtualFile: VirtualFile?,
+    ): SyntaxHighlighter = JaiSyntaxHighlighter()
+}
--- a/src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt
+++ b/src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt
@@ -18,7 +18,7 @@ import com.intellij.psi.tree.IElementType
 * boundary never carries information. That makes the lexer restartable at any token,
 * which is what incremental re-highlighting needs.
 */
-class JaiLexer : LexerBase() {
+open class JaiLexer : LexerBase() {
    private var buf: CharSequence = ""
    private var bufEnd = 0
    private var tokStart = 0