Phase 1: hand-written Jai lexer + Tier 1 golden token tests
JaiLexer mirrors compose_new_token in the compiler's own lexer. Nested block comments and here-strings are each consumed inside a single token, so the lexer state is always 0 and it can restart at any token boundary. 13 plain-JUnit tests, one per gotcha in the language reference §14.
This commit is contained in:
hgranthorner
272
src/test/kotlin/dev/hgh/jai/lexer/JaiLexerTest.kt
Normal file
272
src/test/kotlin/dev/hgh/jai/lexer/JaiLexerTest.kt
Normal file
@@ -0,0 +1,272 @@
|
||||
package dev.hgh.jai.lexer
|
||||
|
||||
import com.intellij.psi.TokenType
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
|
||||
/**
|
||||
* Tier 1 lexer tests (see `docs/BUILD_PLAN.md` §3): one case per gotcha in
|
||||
* `docs/JAI_LANGUAGE_REFERENCE.md` §14. Plain JUnit — no IntelliJ fixture boots here.
|
||||
*/
|
||||
class JaiLexerTest {
|
||||
/** `TYPE 'text'` per token, whitespace dropped. */
|
||||
private fun dump(code: String): List<String> {
|
||||
val lexer = JaiLexer()
|
||||
lexer.start(code, 0, code.length, 0)
|
||||
val out = mutableListOf<String>()
|
||||
var previousStart = -1
|
||||
while (true) {
|
||||
val type = lexer.tokenType ?: break
|
||||
assertTrue(
|
||||
"lexer must make progress at offset ${lexer.tokenStart}",
|
||||
lexer.tokenStart > previousStart,
|
||||
)
|
||||
previousStart = lexer.tokenStart
|
||||
if (type != TokenType.WHITE_SPACE) {
|
||||
out += "$type '${code.substring(lexer.tokenStart, lexer.tokenEnd)}'"
|
||||
}
|
||||
lexer.advance()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
private fun assertTokens(
|
||||
code: String,
|
||||
vararg expected: String,
|
||||
) {
|
||||
assertEquals(expected.toList(), dump(code))
|
||||
assertRoundTrips(code)
|
||||
}
|
||||
|
||||
/** The flagship invariant: tokens must tile the input exactly. */
|
||||
private fun assertRoundTrips(code: String) {
|
||||
val lexer = JaiLexer()
|
||||
lexer.start(code, 0, code.length, 0)
|
||||
val sb = StringBuilder()
|
||||
var offset = 0
|
||||
while (lexer.tokenType != null) {
|
||||
assertEquals("token must start where the previous one ended", offset, lexer.tokenStart)
|
||||
sb.append(code, lexer.tokenStart, lexer.tokenEnd)
|
||||
offset = lexer.tokenEnd
|
||||
lexer.advance()
|
||||
}
|
||||
assertEquals(code, sb.toString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun declarationForms() {
|
||||
assertTokens(
|
||||
"a := 1;",
|
||||
"Jai:IDENT 'a'",
|
||||
"Jai::= ':='",
|
||||
"Jai:NUMBER '1'",
|
||||
"Jai:; ';'",
|
||||
)
|
||||
assertTokens(
|
||||
"main :: () { }",
|
||||
"Jai:IDENT 'main'",
|
||||
"Jai::: '::'",
|
||||
"Jai:( '('",
|
||||
"Jai:) ')'",
|
||||
"Jai:{ '{'",
|
||||
"Jai:} '}'",
|
||||
)
|
||||
assertTokens(
|
||||
"b : float;",
|
||||
"Jai:IDENT 'b'",
|
||||
"Jai:: ':'",
|
||||
"Jai:IDENT 'float'",
|
||||
"Jai:; ';'",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun keywordsAreKeywordsAndTypeNamesAreNot() {
|
||||
assertTokens("for", "Jai:for 'for'")
|
||||
assertTokens("xx", "Jai:xx 'xx'")
|
||||
assertTokens("initializer_of", "Jai:initializer_of 'initializer_of'")
|
||||
// §14.9: primitive type names are ordinary identifiers.
|
||||
assertTokens("int", "Jai:IDENT 'int'")
|
||||
assertTokens("string", "Jai:IDENT 'string'")
|
||||
// Not keywords, just similar.
|
||||
assertTokens("iffy", "Jai:IDENT 'iffy'")
|
||||
assertTokens("must", "Jai:IDENT 'must'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun nestedBlockComments() {
|
||||
assertTokens(
|
||||
"/* a /* b */ still */x",
|
||||
"Jai:BLOCK_COMMENT '/* a /* b */ still */'",
|
||||
"Jai:IDENT 'x'",
|
||||
)
|
||||
// Unterminated: claims the rest of the file.
|
||||
assertTokens("/* a /* b */", "Jai:BLOCK_COMMENT '/* a /* b */'")
|
||||
assertTokens("// a /* b\nx", "Jai:LINE_COMMENT '// a /* b'", "Jai:IDENT 'x'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun minusForms() {
|
||||
assertTokens("->", "Jai:-> '->'")
|
||||
assertTokens("---", "Jai:--- '---'")
|
||||
assertTokens("--", "Jai:-- '--'")
|
||||
assertTokens("-=", "Jai:-= '-='")
|
||||
assertTokens("-", "Jai:- '-'")
|
||||
assertTokens("----", "Jai:--- '---'", "Jai:- '-'")
|
||||
assertTokens(
|
||||
"x: T = ---;",
|
||||
"Jai:IDENT 'x'",
|
||||
"Jai:: ':'",
|
||||
"Jai:IDENT 'T'",
|
||||
"Jai:= '='",
|
||||
"Jai:--- '---'",
|
||||
"Jai:; ';'",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dotForms() {
|
||||
assertTokens(".{", "Jai:.{ '.{'")
|
||||
assertTokens(".[", "Jai:.[ '.['")
|
||||
assertTokens(".*", "Jai:.* '.*'")
|
||||
assertTokens(".APRICOT", "Jai:. '.'", "Jai:IDENT 'APRICOT'")
|
||||
assertTokens("ptr.*", "Jai:IDENT 'ptr'", "Jai:.* '.*'")
|
||||
assertTokens(
|
||||
"for 0..7",
|
||||
"Jai:for 'for'",
|
||||
"Jai:NUMBER '0'",
|
||||
"Jai:.. '..'",
|
||||
"Jai:NUMBER '7'",
|
||||
)
|
||||
assertTokens("...", "Jai:.. '..'", "Jai:. '.'")
|
||||
assertTokens("..Any", "Jai:.. '..'", "Jai:IDENT 'Any'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun numbers() {
|
||||
assertTokens("123", "Jai:NUMBER '123'")
|
||||
assertTokens("1_000_000", "Jai:NUMBER '1_000_000'")
|
||||
assertTokens("0xfade_deaf", "Jai:NUMBER '0xfade_deaf'")
|
||||
assertTokens("0b101101101", "Jai:NUMBER '0b101101101'")
|
||||
assertTokens("0h7fbf_ffff", "Jai:NUMBER '0h7fbf_ffff'")
|
||||
assertTokens("37.0", "Jai:NUMBER '37.0'")
|
||||
assertTokens("-10.0", "Jai:- '-'", "Jai:NUMBER '10.0'")
|
||||
assertTokens(".5", "Jai:NUMBER '.5'")
|
||||
assertTokens("1.5e-7", "Jai:NUMBER '1.5e-7'")
|
||||
assertTokens("1.5E+7", "Jai:NUMBER '1.5E+7'")
|
||||
// The compiler only takes an exponent after a decimal point.
|
||||
assertTokens("1e9", "Jai:NUMBER '1'", "Jai:IDENT 'e9'")
|
||||
// A range wins over a decimal point.
|
||||
assertTokens("1..3", "Jai:NUMBER '1'", "Jai:.. '..'", "Jai:NUMBER '3'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun strings() {
|
||||
assertTokens("\"Hello\"", "Jai:STRING '\"Hello\"'")
|
||||
assertTokens("\"a\\\"b\"", "Jai:STRING '\"a\\\"b\"'")
|
||||
assertTokens("\"a\\\\\"", "Jai:STRING '\"a\\\\\"'")
|
||||
// §14.2: there is no character literal; `#char "a"` is a directive + string.
|
||||
assertTokens("#char \"a\"", "Jai:DIRECTIVE '#char'", "Jai:STRING '\"a\"'")
|
||||
// Unterminated strings stop at the newline instead of bleeding.
|
||||
assertTokens("\"oops\nx", "Jai:STRING '\"oops'", "Jai:IDENT 'x'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun hereStrings() {
|
||||
assertTokens(
|
||||
"S :: #string DONE\nanything \"quoted\" /* not a comment */\nDONE\nx",
|
||||
"Jai:IDENT 'S'",
|
||||
"Jai::: '::'",
|
||||
"Jai:HERE_STRING '#string DONE\nanything \"quoted\" /* not a comment */\nDONE'",
|
||||
"Jai:IDENT 'x'",
|
||||
)
|
||||
// Arbitrary terminator, indented terminator line, and a near-miss line.
|
||||
assertTokens(
|
||||
"#string XY\nXYZ is not the end\n XY\n",
|
||||
"Jai:HERE_STRING '#string XY\nXYZ is not the end\n XY'",
|
||||
)
|
||||
assertTokens(
|
||||
"#string,cr END\nbody\nEND",
|
||||
"Jai:HERE_STRING '#string,cr END\nbody\nEND'",
|
||||
)
|
||||
// Unterminated: claim the rest rather than lexing the body as code.
|
||||
assertTokens("#string DONE\nbody", "Jai:HERE_STRING '#string DONE\nbody'")
|
||||
// Not a here-string at all: falls back to a directive token.
|
||||
assertTokens("#string;", "Jai:DIRECTIVE '#string'", "Jai:; ';'")
|
||||
assertTokens("#stringify", "Jai:DIRECTIVE '#stringify'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun directivesAndNotes() {
|
||||
assertTokens("#import \"Basic\";", "Jai:DIRECTIVE '#import'", "Jai:STRING '\"Basic\"'", "Jai:; ';'")
|
||||
assertTokens("#import,string", "Jai:DIRECTIVE '#import'", "Jai:, ','", "Jai:IDENT 'string'")
|
||||
assertTokens("#", "Jai:# '#'")
|
||||
assertTokens("@Cleanup", "Jai:NOTE '@Cleanup'")
|
||||
assertTokens("x; @Cleanup", "Jai:IDENT 'x'", "Jai:; ';'", "Jai:NOTE '@Cleanup'")
|
||||
assertTokens("@Note;", "Jai:NOTE '@Note'", "Jai:; ';'")
|
||||
assertTokens("@\"quoted note\"", "Jai:NOTE '@\"quoted note\"'")
|
||||
assertTokens("#!/usr/bin/env jai\nx", "Jai:LINE_COMMENT '#!/usr/bin/env jai'", "Jai:IDENT 'x'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun operators() {
|
||||
assertTokens(",,", "Jai:,, ',,'")
|
||||
assertTokens(",", "Jai:, ','")
|
||||
assertTokens("=>", "Jai:=> '=>'")
|
||||
assertTokens("===", "Jai:=== '==='")
|
||||
assertTokens("==", "Jai:== '=='")
|
||||
assertTokens("x == {", "Jai:IDENT 'x'", "Jai:== '=='", "Jai:{ '{'")
|
||||
assertTokens("<<<=", "Jai:<<<= '<<<='")
|
||||
assertTokens("<<<", "Jai:<<< '<<<'")
|
||||
assertTokens("<<=", "Jai:<<= '<<='")
|
||||
assertTokens("<<", "Jai:<< '<<'")
|
||||
assertTokens("<=", "Jai:<= '<='")
|
||||
assertTokens("<", "Jai:< '<'")
|
||||
assertTokens(">>>=", "Jai:>>>= '>>>='")
|
||||
assertTokens("&&=", "Jai:&&= '&&='")
|
||||
assertTokens("&&", "Jai:&& '&&'")
|
||||
assertTokens("&=", "Jai:&= '&='")
|
||||
assertTokens("&", "Jai:& '&'")
|
||||
assertTokens("||=", "Jai:||= '||='")
|
||||
assertTokens("\$\$", "Jai:\$\$ '\$\$'")
|
||||
assertTokens("\$T", "Jai:\$ '\$'", "Jai:IDENT 'T'")
|
||||
assertTokens("`it", "Jai:` '`'", "Jai:IDENT 'it'")
|
||||
assertTokens("`defer", "Jai:` '`'", "Jai:defer 'defer'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun forLoopModifiers() {
|
||||
assertTokens(
|
||||
"for < numbers {",
|
||||
"Jai:for 'for'",
|
||||
"Jai:< '<'",
|
||||
"Jai:IDENT 'numbers'",
|
||||
"Jai:{ '{'",
|
||||
)
|
||||
assertTokens(
|
||||
"for * teas {",
|
||||
"Jai:for 'for'",
|
||||
"Jai:* '*'",
|
||||
"Jai:IDENT 'teas'",
|
||||
"Jai:{ '{'",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun lexingASliceOfTheBufferStaysInBounds() {
|
||||
val text = "aaa 12345 bbb"
|
||||
val lexer = JaiLexer()
|
||||
lexer.start(text, 4, 9, 0)
|
||||
assertEquals(JaiTokenTypes.NUMBER, lexer.tokenType)
|
||||
assertEquals(4, lexer.tokenStart)
|
||||
assertEquals(9, lexer.tokenEnd)
|
||||
lexer.advance()
|
||||
assertEquals(null, lexer.tokenType)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun emptyInput() {
|
||||
assertTokens("")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user