Phase 2: syntax highlighting and colour settings page

- JaiSyntaxHighlighter maps every one of the lexer's token types; asserted over
  the whole corpus (105 distinct token types seen, all mapped)
- JaiHighlightingLexer refines IDENT into built-in types and loop variables for
  colouring only, so the parser never sees them as reserved
- colour settings page with a demo file that is itself asserted to lex cleanly

src/main/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPage.kt

@@ -0,0 +1,90 @@
+package dev.hgh.jai.highlighting
+
+import com.intellij.openapi.editor.colors.TextAttributesKey
+import com.intellij.openapi.fileTypes.SyntaxHighlighter
+import com.intellij.openapi.options.colors.AttributesDescriptor
+import com.intellij.openapi.options.colors.ColorDescriptor
+import com.intellij.openapi.options.colors.ColorSettingsPage
+import dev.hgh.jai.JaiIcons
+import javax.swing.Icon
+
+class JaiColorSettingsPage : ColorSettingsPage {
+    override fun getDisplayName(): String = "Jai"
+
+    override fun getIcon(): Icon = JaiIcons.FILE
+
+    override fun getHighlighter(): SyntaxHighlighter = JaiSyntaxHighlighter()
+
+    override fun getAttributeDescriptors(): Array<AttributesDescriptor> = DESCRIPTORS
+
+    override fun getColorDescriptors(): Array<ColorDescriptor> = ColorDescriptor.EMPTY_ARRAY
+
+    override fun getAdditionalHighlightingTagToDescriptorMap(): Map<String, TextAttributesKey>? = null
+
+    override fun getDemoText(): String = DEMO_TEXT
+
+    companion object {
+        private val DESCRIPTORS =
+            arrayOf(
+                AttributesDescriptor("Keyword", JaiColors.KEYWORD),
+                AttributesDescriptor("Built-in type", JaiColors.BUILTIN_TYPE),
+                AttributesDescriptor("Identifier", JaiColors.IDENTIFIER),
+                AttributesDescriptor("Loop variable (it, it_index)", JaiColors.LOOP_VARIABLE),
+                AttributesDescriptor("Compiler directive", JaiColors.DIRECTIVE),
+                AttributesDescriptor("Note", JaiColors.NOTE),
+                AttributesDescriptor("Number", JaiColors.NUMBER),
+                AttributesDescriptor("String", JaiColors.STRING),
+                AttributesDescriptor("Here-string", JaiColors.HERE_STRING),
+                AttributesDescriptor("Comments//Line comment", JaiColors.LINE_COMMENT),
+                AttributesDescriptor("Comments//Block comment", JaiColors.BLOCK_COMMENT),
+                AttributesDescriptor("Operator", JaiColors.OPERATOR),
+                AttributesDescriptor("Uninitialized (---)", JaiColors.UNINITIALIZED),
+                AttributesDescriptor("Polymorph (\$, \$\$)", JaiColors.POLYMORPH),
+                AttributesDescriptor("Backtick", JaiColors.BACKTICK),
+                AttributesDescriptor("Braces and operators//Parentheses", JaiColors.PARENTHESES),
+                AttributesDescriptor("Braces and operators//Braces", JaiColors.BRACES),
+                AttributesDescriptor("Braces and operators//Brackets", JaiColors.BRACKETS),
+                AttributesDescriptor("Braces and operators//Semicolon", JaiColors.SEMICOLON),
+                AttributesDescriptor("Braces and operators//Comma", JaiColors.COMMA),
+                AttributesDescriptor("Braces and operators//Dot", JaiColors.DOT),
+                AttributesDescriptor("Bad character", JaiColors.BAD_CHARACTER),
+            )
+
+        // Exercises every attribute above, and doubles as a lexer fixture in tests.
+        private val DEMO_TEXT =
+            """
+            // A tour of Jai syntax.
+            /* Block comments /* nest */ like this. */
+            #import "Basic";
+
+            Vector2 :: struct {
+                x, y: float;
+                label: string = "origin";  @Cleanup
+            }
+
+            Fruit :: enum u32 #specified {
+                BANANA :: 5;
+                APPLE;
+            }
+
+            MESSAGE :: #string DONE
+            Here-strings are raw: "quotes" and /* comments */ mean nothing.
+            DONE
+
+            main :: (count: int, scale: ${'$'}T = 1.5e-7) -> bool {
+                total := 0x1f_ff + 0b1011 + 0h7fbf_ffff;
+                uninitialized: [8] u8 = ---;
+                point := Vector2.{ 1.0, 2.0 };
+                names := .[ "a", "b" ];
+
+                for < values {
+                    if it_index == { case 0; continue; case; break; }
+                    total += cast(int) it;
+                }
+
+                defer free(point);
+                return total >= 0 && !(total < 0);
+            }
+            """.trimIndent()
+    }
+}

src/main/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighter.kt

@@ -0,0 +1,133 @@
+package dev.hgh.jai.highlighting
+
+import com.intellij.lexer.Lexer
+import com.intellij.openapi.editor.DefaultLanguageHighlighterColors
+import com.intellij.openapi.editor.colors.TextAttributesKey
+import com.intellij.openapi.editor.colors.TextAttributesKey.createTextAttributesKey
+import com.intellij.openapi.fileTypes.SyntaxHighlighter
+import com.intellij.openapi.fileTypes.SyntaxHighlighterBase
+import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.psi.TokenType
+import com.intellij.psi.tree.IElementType
+import dev.hgh.jai.lexer.JaiLexer
+import dev.hgh.jai.lexer.JaiTokenTypes
+
+/** Text attribute keys for Jai. See `docs/JAI_LANGUAGE_REFERENCE.md` §13. */
+object JaiColors {
+    val KEYWORD = key("JAI_KEYWORD", DefaultLanguageHighlighterColors.KEYWORD)
+    val BUILTIN_TYPE = key("JAI_BUILTIN_TYPE", DefaultLanguageHighlighterColors.CLASS_NAME)
+    val IDENTIFIER = key("JAI_IDENTIFIER", DefaultLanguageHighlighterColors.IDENTIFIER)
+    val LOOP_VARIABLE = key("JAI_LOOP_VARIABLE", DefaultLanguageHighlighterColors.LOCAL_VARIABLE)
+    val DIRECTIVE = key("JAI_DIRECTIVE", DefaultLanguageHighlighterColors.METADATA)
+    val NOTE = key("JAI_NOTE", DefaultLanguageHighlighterColors.METADATA)
+    val NUMBER = key("JAI_NUMBER", DefaultLanguageHighlighterColors.NUMBER)
+    val STRING = key("JAI_STRING", DefaultLanguageHighlighterColors.STRING)
+    val HERE_STRING = key("JAI_HERE_STRING", DefaultLanguageHighlighterColors.STRING)
+    val LINE_COMMENT = key("JAI_LINE_COMMENT", DefaultLanguageHighlighterColors.LINE_COMMENT)
+    val BLOCK_COMMENT = key("JAI_BLOCK_COMMENT", DefaultLanguageHighlighterColors.BLOCK_COMMENT)
+    val OPERATOR = key("JAI_OPERATOR", DefaultLanguageHighlighterColors.OPERATION_SIGN)
+    val UNINITIALIZED = key("JAI_UNINITIALIZED", DefaultLanguageHighlighterColors.KEYWORD)
+    val POLYMORPH = key("JAI_POLYMORPH", DefaultLanguageHighlighterColors.STATIC_FIELD)
+    val BACKTICK = key("JAI_BACKTICK", DefaultLanguageHighlighterColors.STATIC_FIELD)
+    val PARENTHESES = key("JAI_PARENTHESES", DefaultLanguageHighlighterColors.PARENTHESES)
+    val BRACES = key("JAI_BRACES", DefaultLanguageHighlighterColors.BRACES)
+    val BRACKETS = key("JAI_BRACKETS", DefaultLanguageHighlighterColors.BRACKETS)
+    val SEMICOLON = key("JAI_SEMICOLON", DefaultLanguageHighlighterColors.SEMICOLON)
+    val COMMA = key("JAI_COMMA", DefaultLanguageHighlighterColors.COMMA)
+    val DOT = key("JAI_DOT", DefaultLanguageHighlighterColors.DOT)
+    val BAD_CHARACTER = key("JAI_BAD_CHARACTER", com.intellij.openapi.editor.HighlighterColors.BAD_CHARACTER)
+
+    private fun key(
+        name: String,
+        fallback: TextAttributesKey,
+    ): TextAttributesKey = createTextAttributesKey(name, fallback)
+}
+
+/**
+ * Token types produced *only* for highlighting.
+ *
+ * Built-in type names and the implicit loop variables are ordinary identifiers to the
+ * compiler (`docs/JAI_LANGUAGE_REFERENCE.md` §3, §14.9) and must stay that way for the
+ * parser. The highlighting lexer is a separate instance from the parsing lexer, so it
+ * can refine IDENT without reserving anything.
+ */
+object JaiHighlightingTokens {
+    val BUILTIN_TYPE: IElementType =
+        dev.hgh.jai.lexer
+            .JaiTokenType("BUILTIN_TYPE")
+    val LOOP_VARIABLE: IElementType =
+        dev.hgh.jai.lexer
+            .JaiTokenType("LOOP_VARIABLE")
+}
+
+/** [JaiLexer] plus the highlight-only refinement of IDENT. */
+class JaiHighlightingLexer : JaiLexer() {
+    override fun getTokenType(): IElementType? {
+        val type = super.getTokenType()
+        if (type === JaiTokenTypes.IDENT) {
+            val name = bufferSequence.subSequence(tokenStart, tokenEnd).toString()
+            if (name in JaiTokenTypes.BUILTIN_TYPE_NAMES) return JaiHighlightingTokens.BUILTIN_TYPE
+            if (name in JaiTokenTypes.LOOP_VARIABLE_NAMES) return JaiHighlightingTokens.LOOP_VARIABLE
+        }
+        return type
+    }
+}
+
+class JaiSyntaxHighlighter : SyntaxHighlighterBase() {
+    override fun getHighlightingLexer(): Lexer = JaiHighlightingLexer()
+
+    override fun getTokenHighlights(tokenType: IElementType): Array<TextAttributesKey> = pack(ATTRIBUTES[tokenType])
+
+    companion object {
+        private val ATTRIBUTES: Map<IElementType, TextAttributesKey> =
+            buildMap {
+                JaiTokenTypes.KEYWORD_MAP.values.forEach { put(it, JaiColors.KEYWORD) }
+
+                put(JaiTokenTypes.IDENT, JaiColors.IDENTIFIER)
+                put(JaiHighlightingTokens.BUILTIN_TYPE, JaiColors.BUILTIN_TYPE)
+                put(JaiHighlightingTokens.LOOP_VARIABLE, JaiColors.LOOP_VARIABLE)
+
+                put(JaiTokenTypes.NUMBER, JaiColors.NUMBER)
+                put(JaiTokenTypes.STRING, JaiColors.STRING)
+                put(JaiTokenTypes.HERE_STRING, JaiColors.HERE_STRING)
+                put(JaiTokenTypes.DIRECTIVE, JaiColors.DIRECTIVE)
+                put(JaiTokenTypes.NOTE, JaiColors.NOTE)
+
+                put(JaiTokenTypes.LINE_COMMENT, JaiColors.LINE_COMMENT)
+                put(JaiTokenTypes.BLOCK_COMMENT, JaiColors.BLOCK_COMMENT)
+
+                put(JaiTokenTypes.LPAREN, JaiColors.PARENTHESES)
+                put(JaiTokenTypes.RPAREN, JaiColors.PARENTHESES)
+                put(JaiTokenTypes.LBRACE, JaiColors.BRACES)
+                put(JaiTokenTypes.RBRACE, JaiColors.BRACES)
+                put(JaiTokenTypes.LBRACKET, JaiColors.BRACKETS)
+                put(JaiTokenTypes.RBRACKET, JaiColors.BRACKETS)
+                put(JaiTokenTypes.SEMICOLON, JaiColors.SEMICOLON)
+                put(JaiTokenTypes.COMMA, JaiColors.COMMA)
+                put(JaiTokenTypes.DOUBLE_COMMA, JaiColors.COMMA)
+                put(JaiTokenTypes.DOT, JaiColors.DOT)
+
+                put(JaiTokenTypes.TRIPLE_MINUS, JaiColors.UNINITIALIZED)
+                put(JaiTokenTypes.DOLLAR, JaiColors.POLYMORPH)
+                put(JaiTokenTypes.DOUBLE_DOLLAR, JaiColors.POLYMORPH)
+                put(JaiTokenTypes.BACKTICK, JaiColors.BACKTICK)
+
+                put(TokenType.BAD_CHARACTER, JaiColors.BAD_CHARACTER)
+
+                // Everything else that is an operator or punctuation.
+                JaiTokenTypes.OPERATORS.types.forEach { putIfAbsent(it, JaiColors.OPERATOR) }
+            }
+
+        /** Every token type the highlighter knows how to colour. Used by tests. */
+        val HIGHLIGHTED_TOKENS: Set<IElementType> get() = ATTRIBUTES.keys
+    }
+}
+
+class JaiSyntaxHighlighterFactory : SyntaxHighlighterFactory() {
+    override fun getSyntaxHighlighter(
+        project: Project?,
+        virtualFile: VirtualFile?,
+    ): SyntaxHighlighter = JaiSyntaxHighlighter()
+}

src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt

@@ -18,7 +18,7 @@ import com.intellij.psi.tree.IElementType
  * boundary never carries information. That makes the lexer restartable at any token,
  * which is what incremental re-highlighting needs.
  */
-class JaiLexer : LexerBase() {
+open class JaiLexer : LexerBase() {
     private var buf: CharSequence = ""
     private var bufEnd = 0
     private var tokStart = 0

src/main/resources/META-INF/plugin.xml

@@ -20,6 +20,12 @@
             fieldName="INSTANCE"
             language="Jai"
             extensions="jai"/>
+
+        <colorSettingsPage implementation="dev.hgh.jai.highlighting.JaiColorSettingsPage"/>
+
+        <lang.syntaxHighlighterFactory
+            language="Jai"
+            implementationClass="dev.hgh.jai.highlighting.JaiSyntaxHighlighterFactory"/>
     </extensions>
 
 </idea-plugin>

src/test/kotlin/dev/hgh/jai/JaiFileTypeTest.kt

@@ -13,6 +13,16 @@ class JaiFileTypeTest : BasePlatformTestCase() {
         assertEquals(JaiFileType, type)
     }
 
+    fun testSyntaxHighlighterIsRegisteredForJai() {
+        val highlighter =
+            com.intellij.openapi.fileTypes.SyntaxHighlighterFactory
+                .getSyntaxHighlighter(JaiLanguage, null, null)
+        assertTrue(
+            "plugin.xml must register our highlighter, got $highlighter",
+            highlighter is dev.hgh.jai.highlighting.JaiSyntaxHighlighter,
+        )
+    }
+
     /**
      * The *virtual* file already resolves to our type. The PSI file's language stays
      * plain text until a ParserDefinition is registered (phase 3) — asserting that here

src/test/kotlin/dev/hgh/jai/highlighting/JaiColorSettingsPageTest.kt

@@ -0,0 +1,80 @@
+package dev.hgh.jai.highlighting
+
+import com.intellij.openapi.editor.colors.TextAttributesKey
+import com.intellij.psi.TokenType
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class JaiColorSettingsPageTest {
+    private val page = JaiColorSettingsPage()
+
+    /**
+     * The demo text is what a user sees when picking colours, so it must be valid Jai:
+     * it has to tile cleanly and contain no bad characters.
+     */
+    @Test
+    fun demoTextLexesCleanly() {
+        val text = page.demoText
+        val lexer = JaiHighlightingLexer()
+        lexer.start(text, 0, text.length, 0)
+        val rebuilt = StringBuilder()
+        var offset = 0
+        while (true) {
+            val type = lexer.tokenType ?: break
+            assertEquals("gap or overlap in demo text", offset, lexer.tokenStart)
+            assertTrue("empty token in demo text", lexer.tokenEnd > lexer.tokenStart)
+            assertTrue(
+                "demo text contains a bad character at ${lexer.tokenStart}",
+                type != TokenType.BAD_CHARACTER,
+            )
+            rebuilt.append(text, lexer.tokenStart, lexer.tokenEnd)
+            offset = lexer.tokenEnd
+            lexer.advance()
+        }
+        assertEquals(text, rebuilt.toString())
+    }
+
+    /** Every colour the highlighter can emit must be settable by the user. */
+    @Test
+    fun everyColorKeyIsExposedOnThePage() {
+        val exposed: Set<TextAttributesKey> = page.attributeDescriptors.map { it.key }.toSet()
+        val declared =
+            JaiColors::class.java.declaredFields
+                .filter { TextAttributesKey::class.java.isAssignableFrom(it.type) }
+                .map {
+                    it.isAccessible = true
+                    it.get(JaiColors) as TextAttributesKey
+                }
+        val missing = declared.filterNot { it in exposed }
+        assertEquals("colour keys missing from the settings page: $missing", emptyList<TextAttributesKey>(), missing)
+    }
+
+    @Test
+    fun demoTextExercisesTheInterestingTokens() {
+        val text = page.demoText
+        val lexer = JaiHighlightingLexer()
+        lexer.start(text, 0, text.length, 0)
+        val seen = mutableSetOf<String>()
+        while (true) {
+            seen += (lexer.tokenType ?: break).toString()
+            lexer.advance()
+        }
+        listOf(
+            "Jai:HERE_STRING",
+            "Jai:BLOCK_COMMENT",
+            "Jai:LINE_COMMENT",
+            "Jai:DIRECTIVE",
+            "Jai:NOTE",
+            "Jai:NUMBER",
+            "Jai:STRING",
+            "Jai:---",
+            "Jai:.{",
+            "Jai:.[",
+            "Jai:BUILTIN_TYPE",
+            "Jai:LOOP_VARIABLE",
+            "Jai:struct",
+            "Jai:$",
+        ).forEach { assertTrue("demo text never produces $it", it in seen) }
+    }
+}

src/test/kotlin/dev/hgh/jai/highlighting/JaiSyntaxHighlighterTest.kt

@@ -0,0 +1,101 @@
+package dev.hgh.jai.highlighting
+
+import com.intellij.psi.TokenType
+import com.intellij.psi.tree.IElementType
+import dev.hgh.jai.lexer.JaiLexer
+import dev.hgh.jai.lexer.JaiTokenTypes
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import java.io.File
+
+/**
+ * Phase 2 gate: the highlighter must have an attribute for every token type the lexer
+ * can produce — a token with no mapping renders as plain text, which is invisible in a
+ * screenshot review but obvious here.
+ */
+class JaiSyntaxHighlighterTest {
+    private val highlighter = JaiSyntaxHighlighter()
+
+    /** Every token type declared in the token holder, by reflection over the object. */
+    private fun allDeclaredTokenTypes(): List<IElementType> =
+        JaiTokenTypes::class.java.declaredFields
+            .filter { IElementType::class.java.isAssignableFrom(it.type) }
+            .map {
+                it.isAccessible = true
+                it.get(JaiTokenTypes) as IElementType
+            }
+
+    @Test
+    fun everyTokenTypeHasAHighlight() {
+        val unmapped =
+            allDeclaredTokenTypes().filter {
+                highlighter.getTokenHighlights(it).isEmpty()
+            }
+        assertEquals("token types with no text attributes: $unmapped", emptyList<IElementType>(), unmapped)
+    }
+
+    @Test
+    fun highlightOnlyTokensAreMapped() {
+        assertTrue(highlighter.getTokenHighlights(JaiHighlightingTokens.BUILTIN_TYPE).isNotEmpty())
+        assertTrue(highlighter.getTokenHighlights(JaiHighlightingTokens.LOOP_VARIABLE).isNotEmpty())
+        assertTrue(highlighter.getTokenHighlights(TokenType.BAD_CHARACTER).isNotEmpty())
+    }
+
+    @Test
+    fun whitespaceIsNotHighlighted() {
+        assertEquals(0, highlighter.getTokenHighlights(TokenType.WHITE_SPACE).size)
+    }
+
+    @Test
+    fun builtinTypesAndLoopVariablesAreRefinedIdentifiers() {
+        val types = lexTypes("x: int = 0; for v { print(it, it_index); } Foo :: struct {}")
+        assertTrue(JaiHighlightingTokens.BUILTIN_TYPE in types)
+        assertTrue(JaiHighlightingTokens.LOOP_VARIABLE in types)
+        assertTrue(JaiTokenTypes.IDENT in types)
+
+        // ...but the parsing lexer still calls them plain identifiers.
+        val parsing = JaiLexer()
+        parsing.start("int", 0, 3, 0)
+        assertEquals(JaiTokenTypes.IDENT, parsing.tokenType)
+    }
+
+    @Test
+    fun highlightingLexerCoversTheWholeCorpusWithNoUnhighlightedTokens() {
+        val root = File(System.getProperty("user.home"), ".local/jai")
+        org.junit.Assume.assumeTrue("Jai distribution not found at $root", root.isDirectory)
+
+        val seen = mutableSetOf<IElementType>()
+        var files = 0
+        root.walkTopDown().filter { it.isFile && it.extension == "jai" }.forEach { file ->
+            files++
+            val lexer = JaiHighlightingLexer()
+            val text = file.readText()
+            lexer.start(text, 0, text.length, 0)
+            while (true) {
+                val type = lexer.tokenType ?: break
+                seen += type
+                lexer.advance()
+            }
+        }
+
+        assertTrue("no corpus files scanned", files > 0)
+        val unmapped =
+            seen.filter {
+                it != TokenType.WHITE_SPACE && highlighter.getTokenHighlights(it).isEmpty()
+            }
+        assertEquals("corpus produced unhighlighted tokens: $unmapped", emptyList<IElementType>(), unmapped)
+        println("Highlighter: $files corpus files produced ${seen.size} distinct token types, all mapped.")
+    }
+
+    private fun lexTypes(code: String): Set<IElementType> {
+        val lexer = JaiHighlightingLexer()
+        lexer.start(code, 0, code.length, 0)
+        val out = mutableSetOf<IElementType>()
+        while (true) {
+            out += lexer.tokenType ?: break
+            lexer.advance()
+        }
+        return out
+    }
+}