Phase 1: hand-written Jai lexer + Tier 1 golden token tests
JaiLexer mirrors compose_new_token in the compiler's own lexer. Nested block comments and here-strings are each consumed inside a single token, so the lexer state is always 0 and it can restart at any token boundary. 13 plain-JUnit tests, one per gotcha in the language reference §14.
This commit is contained in:
hgranthorner
626
src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt
Normal file
626
src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt
Normal file
@@ -0,0 +1,626 @@
|
||||
package dev.hgh.jai.lexer
|
||||
|
||||
import com.intellij.lexer.LexerBase
|
||||
import com.intellij.psi.TokenType
|
||||
import com.intellij.psi.tree.IElementType
|
||||
|
||||
/**
|
||||
* Hand-written Jai lexer.
|
||||
*
|
||||
* Mirrors `compose_new_token` in the compiler's own lexer
|
||||
* (`~/.local/jai/modules/Jai_Lexer/module.jai:1503`). Hand-written rather than
|
||||
* generated because two Jai constructs cannot be expressed as a DFA: here-strings
|
||||
* capture their terminator at runtime, and block comments nest (see
|
||||
* `docs/BUILD_PLAN.md` §2.1).
|
||||
*
|
||||
* **Invariant:** the lexer state is always 0. Every construct that needs context —
|
||||
* nested comments, here-strings — is consumed inside a *single* token, so a token
|
||||
* boundary never carries information. That makes the lexer restartable at any token,
|
||||
* which is what incremental re-highlighting needs.
|
||||
*/
|
||||
class JaiLexer : LexerBase() {
|
||||
private var buf: CharSequence = ""
|
||||
private var bufEnd = 0
|
||||
private var tokStart = 0
|
||||
private var tokEnd = 0
|
||||
private var tokType: IElementType? = null
|
||||
|
||||
override fun start(
|
||||
buffer: CharSequence,
|
||||
startOffset: Int,
|
||||
endOffset: Int,
|
||||
initialState: Int,
|
||||
) {
|
||||
buf = buffer
|
||||
bufEnd = endOffset
|
||||
tokStart = startOffset
|
||||
tokEnd = startOffset
|
||||
tokType = null
|
||||
locateToken()
|
||||
}
|
||||
|
||||
override fun getState(): Int = 0
|
||||
|
||||
override fun getTokenType(): IElementType? = tokType
|
||||
|
||||
override fun getTokenStart(): Int = tokStart
|
||||
|
||||
override fun getTokenEnd(): Int = tokEnd
|
||||
|
||||
override fun getBufferSequence(): CharSequence = buf
|
||||
|
||||
override fun getBufferEnd(): Int = bufEnd
|
||||
|
||||
override fun advance() {
|
||||
tokStart = tokEnd
|
||||
locateToken()
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ core
|
||||
|
||||
/** Character at [i], or -1 past the end — mirrors the compiler's `peek_next_character`. */
|
||||
private fun ch(i: Int): Int = if (i in 0 until bufEnd) buf[i].code else -1
|
||||
|
||||
private fun locateToken() {
|
||||
if (tokStart >= bufEnd) {
|
||||
tokType = null
|
||||
tokEnd = tokStart
|
||||
return
|
||||
}
|
||||
|
||||
val p = tokStart
|
||||
val c = ch(p)
|
||||
|
||||
if (isSpace(c)) {
|
||||
tokType = TokenType.WHITE_SPACE
|
||||
tokEnd = scanWhile(p) { isSpace(it) }
|
||||
return
|
||||
}
|
||||
|
||||
if (startsIdentifier(c)) {
|
||||
val end = scanIdentifier(p)
|
||||
tokType = JaiTokenTypes.KEYWORD_MAP[text(p, end)] ?: JaiTokenTypes.IDENT
|
||||
tokEnd = end
|
||||
return
|
||||
}
|
||||
|
||||
if (isDigit(c)) {
|
||||
tokType = JaiTokenTypes.NUMBER
|
||||
tokEnd = scanNumber(p)
|
||||
return
|
||||
}
|
||||
|
||||
when (c) {
|
||||
'"'.code -> {
|
||||
emit(JaiTokenTypes.STRING, scanString(p))
|
||||
}
|
||||
|
||||
'/'.code -> {
|
||||
scanSlash(p)
|
||||
}
|
||||
|
||||
'#'.code -> {
|
||||
scanHash(p)
|
||||
}
|
||||
|
||||
'@'.code -> {
|
||||
scanNote(p)
|
||||
}
|
||||
|
||||
'.'.code -> {
|
||||
scanDot(p)
|
||||
}
|
||||
|
||||
'-'.code -> {
|
||||
scanMinus(p)
|
||||
}
|
||||
|
||||
':'.code -> {
|
||||
when (ch(p + 1)) {
|
||||
':'.code -> emit(JaiTokenTypes.COLON_COLON, p + 2)
|
||||
'='.code -> emit(JaiTokenTypes.COLON_EQ, p + 2)
|
||||
else -> emit(JaiTokenTypes.COLON, p + 1)
|
||||
}
|
||||
}
|
||||
|
||||
'+'.code -> {
|
||||
emitEq(p, JaiTokenTypes.PLUS, JaiTokenTypes.PLUS_EQ)
|
||||
}
|
||||
|
||||
'*'.code -> {
|
||||
emitEq(p, JaiTokenTypes.STAR, JaiTokenTypes.STAR_EQ)
|
||||
}
|
||||
|
||||
'%'.code -> {
|
||||
emitEq(p, JaiTokenTypes.PERCENT, JaiTokenTypes.PERCENT_EQ)
|
||||
}
|
||||
|
||||
'!'.code -> {
|
||||
emitEq(p, JaiTokenTypes.NOT, JaiTokenTypes.NOT_EQ)
|
||||
}
|
||||
|
||||
'^'.code -> {
|
||||
emitEq(p, JaiTokenTypes.XOR, JaiTokenTypes.XOR_EQ)
|
||||
}
|
||||
|
||||
'~'.code -> {
|
||||
emit(JaiTokenTypes.TILDE, p + 1)
|
||||
}
|
||||
|
||||
'&'.code -> {
|
||||
scanDoubleable(
|
||||
p,
|
||||
'&',
|
||||
JaiTokenTypes.AND,
|
||||
JaiTokenTypes.AND_EQ,
|
||||
JaiTokenTypes.AND_AND,
|
||||
JaiTokenTypes.AND_AND_EQ,
|
||||
)
|
||||
}
|
||||
|
||||
'|'.code -> {
|
||||
scanDoubleable(
|
||||
p,
|
||||
'|',
|
||||
JaiTokenTypes.OR,
|
||||
JaiTokenTypes.OR_EQ,
|
||||
JaiTokenTypes.OR_OR,
|
||||
JaiTokenTypes.OR_OR_EQ,
|
||||
)
|
||||
}
|
||||
|
||||
'='.code -> {
|
||||
scanEquals(p)
|
||||
}
|
||||
|
||||
'<'.code -> {
|
||||
scanShift(
|
||||
p,
|
||||
'<',
|
||||
JaiTokenTypes.LT,
|
||||
JaiTokenTypes.LT_EQ,
|
||||
JaiTokenTypes.SHL,
|
||||
JaiTokenTypes.SHL_EQ,
|
||||
JaiTokenTypes.ROL,
|
||||
JaiTokenTypes.ROL_EQ,
|
||||
)
|
||||
}
|
||||
|
||||
'>'.code -> {
|
||||
scanShift(
|
||||
p,
|
||||
'>',
|
||||
JaiTokenTypes.GT,
|
||||
JaiTokenTypes.GT_EQ,
|
||||
JaiTokenTypes.SHR,
|
||||
JaiTokenTypes.SHR_EQ,
|
||||
JaiTokenTypes.ROR,
|
||||
JaiTokenTypes.ROR_EQ,
|
||||
)
|
||||
}
|
||||
|
||||
','.code -> {
|
||||
if (ch(p + 1) == ','.code) {
|
||||
emit(JaiTokenTypes.DOUBLE_COMMA, p + 2)
|
||||
} else {
|
||||
emit(JaiTokenTypes.COMMA, p + 1)
|
||||
}
|
||||
}
|
||||
|
||||
'$'.code -> {
|
||||
if (ch(p + 1) == '$'.code) {
|
||||
emit(JaiTokenTypes.DOUBLE_DOLLAR, p + 2)
|
||||
} else {
|
||||
emit(JaiTokenTypes.DOLLAR, p + 1)
|
||||
}
|
||||
}
|
||||
|
||||
'('.code -> {
|
||||
emit(JaiTokenTypes.LPAREN, p + 1)
|
||||
}
|
||||
|
||||
')'.code -> {
|
||||
emit(JaiTokenTypes.RPAREN, p + 1)
|
||||
}
|
||||
|
||||
'['.code -> {
|
||||
emit(JaiTokenTypes.LBRACKET, p + 1)
|
||||
}
|
||||
|
||||
']'.code -> {
|
||||
emit(JaiTokenTypes.RBRACKET, p + 1)
|
||||
}
|
||||
|
||||
'{'.code -> {
|
||||
emit(JaiTokenTypes.LBRACE, p + 1)
|
||||
}
|
||||
|
||||
'}'.code -> {
|
||||
emit(JaiTokenTypes.RBRACE, p + 1)
|
||||
}
|
||||
|
||||
';'.code -> {
|
||||
emit(JaiTokenTypes.SEMICOLON, p + 1)
|
||||
}
|
||||
|
||||
'?'.code -> {
|
||||
emit(JaiTokenTypes.QUESTION, p + 1)
|
||||
}
|
||||
|
||||
'`'.code -> {
|
||||
emit(JaiTokenTypes.BACKTICK, p + 1)
|
||||
}
|
||||
|
||||
'\\'.code -> {
|
||||
emit(JaiTokenTypes.BACKSLASH, p + 1)
|
||||
}
|
||||
|
||||
else -> {
|
||||
emit(TokenType.BAD_CHARACTER, p + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun emit(
|
||||
type: IElementType,
|
||||
end: Int,
|
||||
) {
|
||||
tokType = type
|
||||
tokEnd = end
|
||||
}
|
||||
|
||||
/** `c` or `c=` — the compiler's `check_for_equals`. */
|
||||
private fun emitEq(
|
||||
p: Int,
|
||||
plain: IElementType,
|
||||
augmented: IElementType,
|
||||
) {
|
||||
if (ch(p + 1) == '='.code) emit(augmented, p + 2) else emit(plain, p + 1)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------- character classes
|
||||
|
||||
private fun isSpace(c: Int): Boolean =
|
||||
c == ' '.code || c == '\t'.code || c == '\n'.code || c == '\r'.code ||
|
||||
c == 0x0B || c == 0x0C
|
||||
|
||||
private fun isDigit(c: Int): Boolean = c >= '0'.code && c <= '9'.code
|
||||
|
||||
private fun isHexDigit(c: Int): Boolean = isDigit(c) || (c >= 'a'.code && c <= 'f'.code) || (c >= 'A'.code && c <= 'F'.code)
|
||||
|
||||
/** ASCII only, exactly like the compiler's `starts_identifier`. */
|
||||
private fun startsIdentifier(c: Int): Boolean = (c >= 'a'.code && c <= 'z'.code) || (c >= 'A'.code && c <= 'Z'.code) || c == '_'.code
|
||||
|
||||
private fun continuesIdentifier(c: Int): Boolean = startsIdentifier(c) || isDigit(c)
|
||||
|
||||
private inline fun scanWhile(
|
||||
from: Int,
|
||||
pred: (Int) -> Boolean,
|
||||
): Int {
|
||||
var q = from
|
||||
while (q < bufEnd && pred(ch(q))) q++
|
||||
return q
|
||||
}
|
||||
|
||||
private fun scanIdentifier(from: Int): Int = scanWhile(from) { continuesIdentifier(it) }
|
||||
|
||||
private fun text(
|
||||
from: Int,
|
||||
to: Int,
|
||||
): String = buf.subSequence(from, to).toString()
|
||||
|
||||
// --------------------------------------------------------------- scanners
|
||||
|
||||
/** `make_number`, span-wise. */
|
||||
private fun scanNumber(from: Int): Int {
|
||||
var p = from
|
||||
var isFloat = false
|
||||
|
||||
if (ch(p) == '.'.code) {
|
||||
isFloat = true
|
||||
p = scanWhile(p + 1) { isDigit(it) || it == '_'.code }
|
||||
} else {
|
||||
if (ch(p) == '0'.code) {
|
||||
when (ch(p + 1)) {
|
||||
// 0x hex, 0b binary, 0h hexfloat. Digit validity is the compiler's
|
||||
// problem; the lexer only needs the span.
|
||||
'x'.code, 'X'.code, 'h'.code, 'H'.code -> {
|
||||
return scanWhile(p + 2) { isHexDigit(it) || it == '_'.code }
|
||||
}
|
||||
|
||||
'b'.code, 'B'.code -> {
|
||||
return scanWhile(p + 2) { isDigit(it) || it == '_'.code }
|
||||
}
|
||||
}
|
||||
}
|
||||
p = scanWhile(p) { isDigit(it) || it == '_'.code }
|
||||
// `1..3` is a range, not a float: do not eat the first dot.
|
||||
if (ch(p) == '.'.code && ch(p + 1) != '.'.code) {
|
||||
isFloat = true
|
||||
p = scanWhile(p + 1) { isDigit(it) || it == '_'.code }
|
||||
}
|
||||
}
|
||||
|
||||
// The compiler only accepts an exponent after a decimal point, so `1e9` lexes
|
||||
// as NUMBER(1) IDENT(e9). Match that.
|
||||
if (isFloat && (ch(p) == 'e'.code || ch(p) == 'E'.code)) {
|
||||
var q = p + 1
|
||||
if (ch(q) == '+'.code || ch(q) == '-'.code) q++
|
||||
if (isDigit(ch(q))) p = scanWhile(q) { isDigit(it) || it == '_'.code }
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
/** From the opening quote to just past the closing one; stops at a newline or EOF. */
|
||||
private fun scanString(from: Int): Int {
|
||||
var q = from + 1
|
||||
while (q < bufEnd) {
|
||||
when (ch(q)) {
|
||||
'\n'.code -> return q
|
||||
|
||||
// unterminated: do not bleed into the next line
|
||||
'\\'.code -> q = minOf(q + 2, bufEnd)
|
||||
|
||||
'"'.code -> return q + 1
|
||||
|
||||
else -> q++
|
||||
}
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
private fun scanSlash(p: Int) {
|
||||
when (ch(p + 1)) {
|
||||
'/'.code -> emit(JaiTokenTypes.LINE_COMMENT, scanToEndOfLine(p + 2))
|
||||
'*'.code -> emit(JaiTokenTypes.BLOCK_COMMENT, scanBlockComment(p + 2))
|
||||
'='.code -> emit(JaiTokenTypes.SLASH_EQ, p + 2)
|
||||
else -> emit(JaiTokenTypes.SLASH, p + 1)
|
||||
}
|
||||
}
|
||||
|
||||
private fun scanToEndOfLine(from: Int): Int = scanWhile(from) { it != '\n'.code }
|
||||
|
||||
/**
|
||||
* `eat_input_due_to_block_comment` — comments nest, so this counts depth.
|
||||
* Note the compiler's own quirk: five slashes followed by a star opens a nested
|
||||
* comment (module.jai:1433 flags this as possibly undesirable). Matched here.
|
||||
*/
|
||||
private fun scanBlockComment(from: Int): Int {
|
||||
var q = from
|
||||
var depth = 1
|
||||
while (q < bufEnd && depth > 0) {
|
||||
when (ch(q)) {
|
||||
'/'.code -> {
|
||||
q++
|
||||
if (ch(q) == '*'.code) {
|
||||
q++
|
||||
depth++
|
||||
}
|
||||
}
|
||||
|
||||
'*'.code -> {
|
||||
q++
|
||||
if (ch(q) == '/'.code) {
|
||||
q++
|
||||
depth--
|
||||
}
|
||||
}
|
||||
|
||||
else -> {
|
||||
q++
|
||||
}
|
||||
}
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
private fun scanDot(p: Int) {
|
||||
when (ch(p + 1)) {
|
||||
'.'.code -> {
|
||||
emit(JaiTokenTypes.DOUBLE_DOT, p + 2)
|
||||
}
|
||||
|
||||
'*'.code -> {
|
||||
emit(JaiTokenTypes.POSTFIX_DEREFERENCE, p + 2)
|
||||
}
|
||||
|
||||
'['.code -> {
|
||||
emit(JaiTokenTypes.BEGIN_ARRAY_LITERAL, p + 2)
|
||||
}
|
||||
|
||||
'{'.code -> {
|
||||
emit(JaiTokenTypes.BEGIN_STRUCT_LITERAL, p + 2)
|
||||
}
|
||||
|
||||
else -> {
|
||||
if (isDigit(ch(p + 1))) {
|
||||
emit(JaiTokenTypes.NUMBER, scanNumber(p))
|
||||
} else {
|
||||
emit(JaiTokenTypes.DOT, p + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** `->` then `---` then `--` then `-=` then `-` (module.jai:1680). */
|
||||
private fun scanMinus(p: Int) {
|
||||
when (ch(p + 1)) {
|
||||
'>'.code -> {
|
||||
emit(JaiTokenTypes.RIGHT_ARROW, p + 2)
|
||||
}
|
||||
|
||||
'-'.code -> {
|
||||
if (ch(p + 2) == '-'.code) {
|
||||
emit(JaiTokenTypes.TRIPLE_MINUS, p + 3)
|
||||
} else {
|
||||
emit(JaiTokenTypes.DOUBLE_MINUS, p + 2)
|
||||
}
|
||||
}
|
||||
|
||||
'='.code -> {
|
||||
emit(JaiTokenTypes.MINUS_EQ, p + 2)
|
||||
}
|
||||
|
||||
else -> {
|
||||
emit(JaiTokenTypes.MINUS, p + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun scanEquals(p: Int) {
|
||||
when (ch(p + 1)) {
|
||||
'='.code -> {
|
||||
if (ch(p + 2) == '='.code) {
|
||||
emit(JaiTokenTypes.EQ_EQ_EQ, p + 3)
|
||||
} else {
|
||||
emit(JaiTokenTypes.EQ_EQ, p + 2)
|
||||
}
|
||||
}
|
||||
|
||||
'>'.code -> {
|
||||
emit(JaiTokenTypes.QUICK_LAMBDA, p + 2)
|
||||
}
|
||||
|
||||
else -> {
|
||||
emit(JaiTokenTypes.EQ, p + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** `&` `&=` `&&` `&&=` and the `|` equivalents. */
|
||||
private fun scanDoubleable(
|
||||
p: Int,
|
||||
c: Char,
|
||||
single: IElementType,
|
||||
singleEq: IElementType,
|
||||
double: IElementType,
|
||||
doubleEq: IElementType,
|
||||
) {
|
||||
if (ch(p + 1) == c.code) {
|
||||
if (ch(p + 2) == '='.code) emit(doubleEq, p + 3) else emit(double, p + 2)
|
||||
} else {
|
||||
emitEq(p, single, singleEq)
|
||||
}
|
||||
}
|
||||
|
||||
/** `<` `<=` `<<` `<<=` `<<<` `<<<=` and the `>` equivalents. */
|
||||
private fun scanShift(
|
||||
p: Int,
|
||||
c: Char,
|
||||
single: IElementType,
|
||||
singleEq: IElementType,
|
||||
shift: IElementType,
|
||||
shiftEq: IElementType,
|
||||
rotate: IElementType,
|
||||
rotateEq: IElementType,
|
||||
) {
|
||||
if (ch(p + 1) != c.code) {
|
||||
emitEq(p, single, singleEq)
|
||||
return
|
||||
}
|
||||
if (ch(p + 2) == c.code) {
|
||||
if (ch(p + 3) == '='.code) emit(rotateEq, p + 4) else emit(rotate, p + 3)
|
||||
} else {
|
||||
if (ch(p + 2) == '='.code) emit(shiftEq, p + 3) else emit(shift, p + 2)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `#ident` is one DIRECTIVE token. `#string` opens a here-string, which is also a
|
||||
* single token so that no lexer state has to survive a token boundary. `#!` at the
|
||||
* start of a line is a hashbang line.
|
||||
*/
|
||||
private fun scanHash(p: Int) {
|
||||
if (ch(p + 1) == '!'.code && (p == 0 || buf[p - 1] == '\n')) {
|
||||
emit(JaiTokenTypes.LINE_COMMENT, scanToEndOfLine(p + 2))
|
||||
return
|
||||
}
|
||||
if (!startsIdentifier(ch(p + 1))) {
|
||||
emit(JaiTokenTypes.HASH, p + 1)
|
||||
return
|
||||
}
|
||||
val identEnd = scanIdentifier(p + 1)
|
||||
if (text(p + 1, identEnd) == "string") {
|
||||
val hereEnd = scanHereString(identEnd)
|
||||
if (hereEnd > 0) {
|
||||
emit(JaiTokenTypes.HERE_STRING, hereEnd)
|
||||
return
|
||||
}
|
||||
}
|
||||
emit(JaiTokenTypes.DIRECTIVE, identEnd)
|
||||
}
|
||||
|
||||
/**
|
||||
* `parse_here_string` (module.jai:476). [from] is just past `#string`; returns the
|
||||
* end offset of the whole here-string, or -1 if this is not a well-formed one (in
|
||||
* which case the caller falls back to a plain `#string` directive token).
|
||||
*
|
||||
* Shape: `#string [,cr] IDENT \n body... \n [ws] IDENT`, and the token ends right
|
||||
* after the terminating identifier.
|
||||
*/
|
||||
private fun scanHereString(from: Int): Int {
|
||||
var p = scanWhile(from) { isSpace(it) }
|
||||
|
||||
// Optional `,cr` modifier.
|
||||
if (ch(p) == ','.code) {
|
||||
p = scanWhile(p + 1) { isSpace(it) }
|
||||
if (!startsIdentifier(ch(p))) return -1
|
||||
p = scanIdentifier(p)
|
||||
p = scanWhile(p) { isSpace(it) }
|
||||
}
|
||||
|
||||
if (!startsIdentifier(ch(p))) return -1
|
||||
val identStart = p
|
||||
p = scanIdentifier(p)
|
||||
val ident = text(identStart, p)
|
||||
|
||||
// Only whitespace may follow the terminator identifier, up to end of line.
|
||||
var sawNewline = false
|
||||
while (p < bufEnd) {
|
||||
val c = ch(p)
|
||||
if (!isSpace(c)) return -1
|
||||
p++
|
||||
if (c == '\n'.code) {
|
||||
sawNewline = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if (!sawNewline) return -1
|
||||
|
||||
// Body: scan line by line for a line whose first non-space run is the terminator.
|
||||
while (p < bufEnd) {
|
||||
var q = p
|
||||
while (q < bufEnd && ch(q) != '\n'.code && isSpace(ch(q))) q++
|
||||
if (matchesAt(q, ident) && !continuesIdentifier(ch(q + ident.length))) {
|
||||
return q + ident.length
|
||||
}
|
||||
while (q < bufEnd && ch(q) != '\n'.code) q++
|
||||
if (q < bufEnd) q++ // eat the newline
|
||||
p = q
|
||||
}
|
||||
// Unterminated: claim the rest of the file rather than lexing the body as code.
|
||||
return bufEnd
|
||||
}
|
||||
|
||||
private fun matchesAt(
|
||||
at: Int,
|
||||
s: String,
|
||||
): Boolean {
|
||||
if (at + s.length > bufEnd) return false
|
||||
for (i in s.indices) if (buf[at + i] != s[i]) return false
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* `make_note` (module.jai:716): `@"quoted"`, or `@` followed by everything up to
|
||||
* whitespace or `;`.
|
||||
*/
|
||||
private fun scanNote(p: Int) {
|
||||
if (ch(p + 1) == '"'.code) {
|
||||
emit(JaiTokenTypes.NOTE, scanString(p + 1))
|
||||
return
|
||||
}
|
||||
val end = scanWhile(p + 1) { !isSpace(it) && it != ';'.code && it != 0 }
|
||||
emit(if (end > p + 1) JaiTokenTypes.NOTE else JaiTokenTypes.AT, end)
|
||||
}
|
||||
}
|
||||
272
src/test/kotlin/dev/hgh/jai/lexer/JaiLexerTest.kt
Normal file
272
src/test/kotlin/dev/hgh/jai/lexer/JaiLexerTest.kt
Normal file
@@ -0,0 +1,272 @@
|
||||
package dev.hgh.jai.lexer
|
||||
|
||||
import com.intellij.psi.TokenType
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
|
||||
/**
|
||||
* Tier 1 lexer tests (see `docs/BUILD_PLAN.md` §3): one case per gotcha in
|
||||
* `docs/JAI_LANGUAGE_REFERENCE.md` §14. Plain JUnit — no IntelliJ fixture boots here.
|
||||
*/
|
||||
class JaiLexerTest {
|
||||
/** `TYPE 'text'` per token, whitespace dropped. */
|
||||
private fun dump(code: String): List<String> {
|
||||
val lexer = JaiLexer()
|
||||
lexer.start(code, 0, code.length, 0)
|
||||
val out = mutableListOf<String>()
|
||||
var previousStart = -1
|
||||
while (true) {
|
||||
val type = lexer.tokenType ?: break
|
||||
assertTrue(
|
||||
"lexer must make progress at offset ${lexer.tokenStart}",
|
||||
lexer.tokenStart > previousStart,
|
||||
)
|
||||
previousStart = lexer.tokenStart
|
||||
if (type != TokenType.WHITE_SPACE) {
|
||||
out += "$type '${code.substring(lexer.tokenStart, lexer.tokenEnd)}'"
|
||||
}
|
||||
lexer.advance()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
private fun assertTokens(
|
||||
code: String,
|
||||
vararg expected: String,
|
||||
) {
|
||||
assertEquals(expected.toList(), dump(code))
|
||||
assertRoundTrips(code)
|
||||
}
|
||||
|
||||
/** The flagship invariant: tokens must tile the input exactly. */
|
||||
private fun assertRoundTrips(code: String) {
|
||||
val lexer = JaiLexer()
|
||||
lexer.start(code, 0, code.length, 0)
|
||||
val sb = StringBuilder()
|
||||
var offset = 0
|
||||
while (lexer.tokenType != null) {
|
||||
assertEquals("token must start where the previous one ended", offset, lexer.tokenStart)
|
||||
sb.append(code, lexer.tokenStart, lexer.tokenEnd)
|
||||
offset = lexer.tokenEnd
|
||||
lexer.advance()
|
||||
}
|
||||
assertEquals(code, sb.toString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun declarationForms() {
|
||||
assertTokens(
|
||||
"a := 1;",
|
||||
"Jai:IDENT 'a'",
|
||||
"Jai::= ':='",
|
||||
"Jai:NUMBER '1'",
|
||||
"Jai:; ';'",
|
||||
)
|
||||
assertTokens(
|
||||
"main :: () { }",
|
||||
"Jai:IDENT 'main'",
|
||||
"Jai::: '::'",
|
||||
"Jai:( '('",
|
||||
"Jai:) ')'",
|
||||
"Jai:{ '{'",
|
||||
"Jai:} '}'",
|
||||
)
|
||||
assertTokens(
|
||||
"b : float;",
|
||||
"Jai:IDENT 'b'",
|
||||
"Jai:: ':'",
|
||||
"Jai:IDENT 'float'",
|
||||
"Jai:; ';'",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun keywordsAreKeywordsAndTypeNamesAreNot() {
|
||||
assertTokens("for", "Jai:for 'for'")
|
||||
assertTokens("xx", "Jai:xx 'xx'")
|
||||
assertTokens("initializer_of", "Jai:initializer_of 'initializer_of'")
|
||||
// §14.9: primitive type names are ordinary identifiers.
|
||||
assertTokens("int", "Jai:IDENT 'int'")
|
||||
assertTokens("string", "Jai:IDENT 'string'")
|
||||
// Not keywords, just similar.
|
||||
assertTokens("iffy", "Jai:IDENT 'iffy'")
|
||||
assertTokens("must", "Jai:IDENT 'must'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun nestedBlockComments() {
|
||||
assertTokens(
|
||||
"/* a /* b */ still */x",
|
||||
"Jai:BLOCK_COMMENT '/* a /* b */ still */'",
|
||||
"Jai:IDENT 'x'",
|
||||
)
|
||||
// Unterminated: claims the rest of the file.
|
||||
assertTokens("/* a /* b */", "Jai:BLOCK_COMMENT '/* a /* b */'")
|
||||
assertTokens("// a /* b\nx", "Jai:LINE_COMMENT '// a /* b'", "Jai:IDENT 'x'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun minusForms() {
|
||||
assertTokens("->", "Jai:-> '->'")
|
||||
assertTokens("---", "Jai:--- '---'")
|
||||
assertTokens("--", "Jai:-- '--'")
|
||||
assertTokens("-=", "Jai:-= '-='")
|
||||
assertTokens("-", "Jai:- '-'")
|
||||
assertTokens("----", "Jai:--- '---'", "Jai:- '-'")
|
||||
assertTokens(
|
||||
"x: T = ---;",
|
||||
"Jai:IDENT 'x'",
|
||||
"Jai:: ':'",
|
||||
"Jai:IDENT 'T'",
|
||||
"Jai:= '='",
|
||||
"Jai:--- '---'",
|
||||
"Jai:; ';'",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dotForms() {
|
||||
assertTokens(".{", "Jai:.{ '.{'")
|
||||
assertTokens(".[", "Jai:.[ '.['")
|
||||
assertTokens(".*", "Jai:.* '.*'")
|
||||
assertTokens(".APRICOT", "Jai:. '.'", "Jai:IDENT 'APRICOT'")
|
||||
assertTokens("ptr.*", "Jai:IDENT 'ptr'", "Jai:.* '.*'")
|
||||
assertTokens(
|
||||
"for 0..7",
|
||||
"Jai:for 'for'",
|
||||
"Jai:NUMBER '0'",
|
||||
"Jai:.. '..'",
|
||||
"Jai:NUMBER '7'",
|
||||
)
|
||||
assertTokens("...", "Jai:.. '..'", "Jai:. '.'")
|
||||
assertTokens("..Any", "Jai:.. '..'", "Jai:IDENT 'Any'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun numbers() {
|
||||
assertTokens("123", "Jai:NUMBER '123'")
|
||||
assertTokens("1_000_000", "Jai:NUMBER '1_000_000'")
|
||||
assertTokens("0xfade_deaf", "Jai:NUMBER '0xfade_deaf'")
|
||||
assertTokens("0b101101101", "Jai:NUMBER '0b101101101'")
|
||||
assertTokens("0h7fbf_ffff", "Jai:NUMBER '0h7fbf_ffff'")
|
||||
assertTokens("37.0", "Jai:NUMBER '37.0'")
|
||||
assertTokens("-10.0", "Jai:- '-'", "Jai:NUMBER '10.0'")
|
||||
assertTokens(".5", "Jai:NUMBER '.5'")
|
||||
assertTokens("1.5e-7", "Jai:NUMBER '1.5e-7'")
|
||||
assertTokens("1.5E+7", "Jai:NUMBER '1.5E+7'")
|
||||
// The compiler only takes an exponent after a decimal point.
|
||||
assertTokens("1e9", "Jai:NUMBER '1'", "Jai:IDENT 'e9'")
|
||||
// A range wins over a decimal point.
|
||||
assertTokens("1..3", "Jai:NUMBER '1'", "Jai:.. '..'", "Jai:NUMBER '3'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun strings() {
|
||||
assertTokens("\"Hello\"", "Jai:STRING '\"Hello\"'")
|
||||
assertTokens("\"a\\\"b\"", "Jai:STRING '\"a\\\"b\"'")
|
||||
assertTokens("\"a\\\\\"", "Jai:STRING '\"a\\\\\"'")
|
||||
// §14.2: there is no character literal; `#char "a"` is a directive + string.
|
||||
assertTokens("#char \"a\"", "Jai:DIRECTIVE '#char'", "Jai:STRING '\"a\"'")
|
||||
// Unterminated strings stop at the newline instead of bleeding.
|
||||
assertTokens("\"oops\nx", "Jai:STRING '\"oops'", "Jai:IDENT 'x'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun hereStrings() {
|
||||
assertTokens(
|
||||
"S :: #string DONE\nanything \"quoted\" /* not a comment */\nDONE\nx",
|
||||
"Jai:IDENT 'S'",
|
||||
"Jai::: '::'",
|
||||
"Jai:HERE_STRING '#string DONE\nanything \"quoted\" /* not a comment */\nDONE'",
|
||||
"Jai:IDENT 'x'",
|
||||
)
|
||||
// Arbitrary terminator, indented terminator line, and a near-miss line.
|
||||
assertTokens(
|
||||
"#string XY\nXYZ is not the end\n XY\n",
|
||||
"Jai:HERE_STRING '#string XY\nXYZ is not the end\n XY'",
|
||||
)
|
||||
assertTokens(
|
||||
"#string,cr END\nbody\nEND",
|
||||
"Jai:HERE_STRING '#string,cr END\nbody\nEND'",
|
||||
)
|
||||
// Unterminated: claim the rest rather than lexing the body as code.
|
||||
assertTokens("#string DONE\nbody", "Jai:HERE_STRING '#string DONE\nbody'")
|
||||
// Not a here-string at all: falls back to a directive token.
|
||||
assertTokens("#string;", "Jai:DIRECTIVE '#string'", "Jai:; ';'")
|
||||
assertTokens("#stringify", "Jai:DIRECTIVE '#stringify'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun directivesAndNotes() {
|
||||
assertTokens("#import \"Basic\";", "Jai:DIRECTIVE '#import'", "Jai:STRING '\"Basic\"'", "Jai:; ';'")
|
||||
assertTokens("#import,string", "Jai:DIRECTIVE '#import'", "Jai:, ','", "Jai:IDENT 'string'")
|
||||
assertTokens("#", "Jai:# '#'")
|
||||
assertTokens("@Cleanup", "Jai:NOTE '@Cleanup'")
|
||||
assertTokens("x; @Cleanup", "Jai:IDENT 'x'", "Jai:; ';'", "Jai:NOTE '@Cleanup'")
|
||||
assertTokens("@Note;", "Jai:NOTE '@Note'", "Jai:; ';'")
|
||||
assertTokens("@\"quoted note\"", "Jai:NOTE '@\"quoted note\"'")
|
||||
assertTokens("#!/usr/bin/env jai\nx", "Jai:LINE_COMMENT '#!/usr/bin/env jai'", "Jai:IDENT 'x'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun operators() {
|
||||
assertTokens(",,", "Jai:,, ',,'")
|
||||
assertTokens(",", "Jai:, ','")
|
||||
assertTokens("=>", "Jai:=> '=>'")
|
||||
assertTokens("===", "Jai:=== '==='")
|
||||
assertTokens("==", "Jai:== '=='")
|
||||
assertTokens("x == {", "Jai:IDENT 'x'", "Jai:== '=='", "Jai:{ '{'")
|
||||
assertTokens("<<<=", "Jai:<<<= '<<<='")
|
||||
assertTokens("<<<", "Jai:<<< '<<<'")
|
||||
assertTokens("<<=", "Jai:<<= '<<='")
|
||||
assertTokens("<<", "Jai:<< '<<'")
|
||||
assertTokens("<=", "Jai:<= '<='")
|
||||
assertTokens("<", "Jai:< '<'")
|
||||
assertTokens(">>>=", "Jai:>>>= '>>>='")
|
||||
assertTokens("&&=", "Jai:&&= '&&='")
|
||||
assertTokens("&&", "Jai:&& '&&'")
|
||||
assertTokens("&=", "Jai:&= '&='")
|
||||
assertTokens("&", "Jai:& '&'")
|
||||
assertTokens("||=", "Jai:||= '||='")
|
||||
assertTokens("\$\$", "Jai:\$\$ '\$\$'")
|
||||
assertTokens("\$T", "Jai:\$ '\$'", "Jai:IDENT 'T'")
|
||||
assertTokens("`it", "Jai:` '`'", "Jai:IDENT 'it'")
|
||||
assertTokens("`defer", "Jai:` '`'", "Jai:defer 'defer'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun forLoopModifiers() {
|
||||
assertTokens(
|
||||
"for < numbers {",
|
||||
"Jai:for 'for'",
|
||||
"Jai:< '<'",
|
||||
"Jai:IDENT 'numbers'",
|
||||
"Jai:{ '{'",
|
||||
)
|
||||
assertTokens(
|
||||
"for * teas {",
|
||||
"Jai:for 'for'",
|
||||
"Jai:* '*'",
|
||||
"Jai:IDENT 'teas'",
|
||||
"Jai:{ '{'",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun lexingASliceOfTheBufferStaysInBounds() {
|
||||
val text = "aaa 12345 bbb"
|
||||
val lexer = JaiLexer()
|
||||
lexer.start(text, 4, 9, 0)
|
||||
assertEquals(JaiTokenTypes.NUMBER, lexer.tokenType)
|
||||
assertEquals(4, lexer.tokenStart)
|
||||
assertEquals(9, lexer.tokenEnd)
|
||||
lexer.advance()
|
||||
assertEquals(null, lexer.tokenType)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun emptyInput() {
|
||||
assertTokens("")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user