Phase 2: syntax highlighting and colour settings page
- JaiSyntaxHighlighter maps every one of the lexer's token types; asserted over the whole corpus (105 distinct token types seen, all mapped) - JaiHighlightingLexer refines IDENT into built-in types and loop variables for colouring only, so the parser never sees them as reserved - colour settings page with a demo file that is itself asserted to lex cleanly
This commit is contained in:
hgranthorner
@@ -0,0 +1,80 @@
|
||||
package dev.hgh.jai.highlighting
|
||||
|
||||
import com.intellij.openapi.editor.colors.TextAttributesKey
|
||||
import com.intellij.psi.TokenType
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
|
||||
class JaiColorSettingsPageTest {
|
||||
private val page = JaiColorSettingsPage()
|
||||
|
||||
/**
|
||||
* The demo text is what a user sees when picking colours, so it must be valid Jai:
|
||||
* it has to tile cleanly and contain no bad characters.
|
||||
*/
|
||||
@Test
|
||||
fun demoTextLexesCleanly() {
|
||||
val text = page.demoText
|
||||
val lexer = JaiHighlightingLexer()
|
||||
lexer.start(text, 0, text.length, 0)
|
||||
val rebuilt = StringBuilder()
|
||||
var offset = 0
|
||||
while (true) {
|
||||
val type = lexer.tokenType ?: break
|
||||
assertEquals("gap or overlap in demo text", offset, lexer.tokenStart)
|
||||
assertTrue("empty token in demo text", lexer.tokenEnd > lexer.tokenStart)
|
||||
assertTrue(
|
||||
"demo text contains a bad character at ${lexer.tokenStart}",
|
||||
type != TokenType.BAD_CHARACTER,
|
||||
)
|
||||
rebuilt.append(text, lexer.tokenStart, lexer.tokenEnd)
|
||||
offset = lexer.tokenEnd
|
||||
lexer.advance()
|
||||
}
|
||||
assertEquals(text, rebuilt.toString())
|
||||
}
|
||||
|
||||
/** Every colour the highlighter can emit must be settable by the user. */
|
||||
@Test
|
||||
fun everyColorKeyIsExposedOnThePage() {
|
||||
val exposed: Set<TextAttributesKey> = page.attributeDescriptors.map { it.key }.toSet()
|
||||
val declared =
|
||||
JaiColors::class.java.declaredFields
|
||||
.filter { TextAttributesKey::class.java.isAssignableFrom(it.type) }
|
||||
.map {
|
||||
it.isAccessible = true
|
||||
it.get(JaiColors) as TextAttributesKey
|
||||
}
|
||||
val missing = declared.filterNot { it in exposed }
|
||||
assertEquals("colour keys missing from the settings page: $missing", emptyList<TextAttributesKey>(), missing)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun demoTextExercisesTheInterestingTokens() {
|
||||
val text = page.demoText
|
||||
val lexer = JaiHighlightingLexer()
|
||||
lexer.start(text, 0, text.length, 0)
|
||||
val seen = mutableSetOf<String>()
|
||||
while (true) {
|
||||
seen += (lexer.tokenType ?: break).toString()
|
||||
lexer.advance()
|
||||
}
|
||||
listOf(
|
||||
"Jai:HERE_STRING",
|
||||
"Jai:BLOCK_COMMENT",
|
||||
"Jai:LINE_COMMENT",
|
||||
"Jai:DIRECTIVE",
|
||||
"Jai:NOTE",
|
||||
"Jai:NUMBER",
|
||||
"Jai:STRING",
|
||||
"Jai:---",
|
||||
"Jai:.{",
|
||||
"Jai:.[",
|
||||
"Jai:BUILTIN_TYPE",
|
||||
"Jai:LOOP_VARIABLE",
|
||||
"Jai:struct",
|
||||
"Jai:$",
|
||||
).forEach { assertTrue("demo text never produces $it", it in seen) }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package dev.hgh.jai.highlighting
|
||||
|
||||
import com.intellij.psi.TokenType
|
||||
import com.intellij.psi.tree.IElementType
|
||||
import dev.hgh.jai.lexer.JaiLexer
|
||||
import dev.hgh.jai.lexer.JaiTokenTypes
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* Phase 2 gate: the highlighter must have an attribute for every token type the lexer
|
||||
* can produce — a token with no mapping renders as plain text, which is invisible in a
|
||||
* screenshot review but obvious here.
|
||||
*/
|
||||
class JaiSyntaxHighlighterTest {
|
||||
private val highlighter = JaiSyntaxHighlighter()
|
||||
|
||||
/** Every token type declared in the token holder, by reflection over the object. */
|
||||
private fun allDeclaredTokenTypes(): List<IElementType> =
|
||||
JaiTokenTypes::class.java.declaredFields
|
||||
.filter { IElementType::class.java.isAssignableFrom(it.type) }
|
||||
.map {
|
||||
it.isAccessible = true
|
||||
it.get(JaiTokenTypes) as IElementType
|
||||
}
|
||||
|
||||
@Test
|
||||
fun everyTokenTypeHasAHighlight() {
|
||||
val unmapped =
|
||||
allDeclaredTokenTypes().filter {
|
||||
highlighter.getTokenHighlights(it).isEmpty()
|
||||
}
|
||||
assertEquals("token types with no text attributes: $unmapped", emptyList<IElementType>(), unmapped)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun highlightOnlyTokensAreMapped() {
|
||||
assertTrue(highlighter.getTokenHighlights(JaiHighlightingTokens.BUILTIN_TYPE).isNotEmpty())
|
||||
assertTrue(highlighter.getTokenHighlights(JaiHighlightingTokens.LOOP_VARIABLE).isNotEmpty())
|
||||
assertTrue(highlighter.getTokenHighlights(TokenType.BAD_CHARACTER).isNotEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun whitespaceIsNotHighlighted() {
|
||||
assertEquals(0, highlighter.getTokenHighlights(TokenType.WHITE_SPACE).size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun builtinTypesAndLoopVariablesAreRefinedIdentifiers() {
|
||||
val types = lexTypes("x: int = 0; for v { print(it, it_index); } Foo :: struct {}")
|
||||
assertTrue(JaiHighlightingTokens.BUILTIN_TYPE in types)
|
||||
assertTrue(JaiHighlightingTokens.LOOP_VARIABLE in types)
|
||||
assertTrue(JaiTokenTypes.IDENT in types)
|
||||
|
||||
// ...but the parsing lexer still calls them plain identifiers.
|
||||
val parsing = JaiLexer()
|
||||
parsing.start("int", 0, 3, 0)
|
||||
assertEquals(JaiTokenTypes.IDENT, parsing.tokenType)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun highlightingLexerCoversTheWholeCorpusWithNoUnhighlightedTokens() {
|
||||
val root = File(System.getProperty("user.home"), ".local/jai")
|
||||
org.junit.Assume.assumeTrue("Jai distribution not found at $root", root.isDirectory)
|
||||
|
||||
val seen = mutableSetOf<IElementType>()
|
||||
var files = 0
|
||||
root.walkTopDown().filter { it.isFile && it.extension == "jai" }.forEach { file ->
|
||||
files++
|
||||
val lexer = JaiHighlightingLexer()
|
||||
val text = file.readText()
|
||||
lexer.start(text, 0, text.length, 0)
|
||||
while (true) {
|
||||
val type = lexer.tokenType ?: break
|
||||
seen += type
|
||||
lexer.advance()
|
||||
}
|
||||
}
|
||||
|
||||
assertTrue("no corpus files scanned", files > 0)
|
||||
val unmapped =
|
||||
seen.filter {
|
||||
it != TokenType.WHITE_SPACE && highlighter.getTokenHighlights(it).isEmpty()
|
||||
}
|
||||
assertEquals("corpus produced unhighlighted tokens: $unmapped", emptyList<IElementType>(), unmapped)
|
||||
println("Highlighter: $files corpus files produced ${seen.size} distinct token types, all mapped.")
|
||||
}
|
||||
|
||||
private fun lexTypes(code: String): Set<IElementType> {
|
||||
val lexer = JaiHighlightingLexer()
|
||||
lexer.start(code, 0, code.length, 0)
|
||||
val out = mutableSetOf<IElementType>()
|
||||
while (true) {
|
||||
out += lexer.tokenType ?: break
|
||||
lexer.advance()
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user