From 4469283024b36c7f3f1ac56c63775172ca053c77 Mon Sep 17 00:00:00 2001 From: hgranthorner <37941012+hgranthorner@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:22:58 -0400 Subject: [PATCH] Phase 1: hand-written Jai lexer + Tier 1 golden token tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JaiLexer mirrors compose_new_token in the compiler's own lexer. Nested block comments and here-strings are each consumed inside a single token, so the lexer state is always 0 and it can restart at any token boundary. 13 plain-JUnit tests, one per gotcha in the language reference §14. --- src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt | 626 ++++++++++++++++++ .../kotlin/dev/hgh/jai/lexer/JaiLexerTest.kt | 272 ++++++++ 2 files changed, 898 insertions(+) create mode 100644 src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt create mode 100644 src/test/kotlin/dev/hgh/jai/lexer/JaiLexerTest.kt diff --git a/src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt b/src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt new file mode 100644 index 0000000..1723a7f --- /dev/null +++ b/src/main/kotlin/dev/hgh/jai/lexer/JaiLexer.kt @@ -0,0 +1,626 @@ +package dev.hgh.jai.lexer + +import com.intellij.lexer.LexerBase +import com.intellij.psi.TokenType +import com.intellij.psi.tree.IElementType + +/** + * Hand-written Jai lexer. + * + * Mirrors `compose_new_token` in the compiler's own lexer + * (`~/.local/jai/modules/Jai_Lexer/module.jai:1503`). Hand-written rather than + * generated because two Jai constructs cannot be expressed as a DFA: here-strings + * capture their terminator at runtime, and block comments nest (see + * `docs/BUILD_PLAN.md` §2.1). + * + * **Invariant:** the lexer state is always 0. Every construct that needs context — + * nested comments, here-strings — is consumed inside a *single* token, so a token + * boundary never carries information. That makes the lexer restartable at any token, + * which is what incremental re-highlighting needs. + */ +class JaiLexer : LexerBase() { + private var buf: CharSequence = "" + private var bufEnd = 0 + private var tokStart = 0 + private var tokEnd = 0 + private var tokType: IElementType? = null + + override fun start( + buffer: CharSequence, + startOffset: Int, + endOffset: Int, + initialState: Int, + ) { + buf = buffer + bufEnd = endOffset + tokStart = startOffset + tokEnd = startOffset + tokType = null + locateToken() + } + + override fun getState(): Int = 0 + + override fun getTokenType(): IElementType? = tokType + + override fun getTokenStart(): Int = tokStart + + override fun getTokenEnd(): Int = tokEnd + + override fun getBufferSequence(): CharSequence = buf + + override fun getBufferEnd(): Int = bufEnd + + override fun advance() { + tokStart = tokEnd + locateToken() + } + + // ------------------------------------------------------------------ core + + /** Character at [i], or -1 past the end — mirrors the compiler's `peek_next_character`. */ + private fun ch(i: Int): Int = if (i in 0 until bufEnd) buf[i].code else -1 + + private fun locateToken() { + if (tokStart >= bufEnd) { + tokType = null + tokEnd = tokStart + return + } + + val p = tokStart + val c = ch(p) + + if (isSpace(c)) { + tokType = TokenType.WHITE_SPACE + tokEnd = scanWhile(p) { isSpace(it) } + return + } + + if (startsIdentifier(c)) { + val end = scanIdentifier(p) + tokType = JaiTokenTypes.KEYWORD_MAP[text(p, end)] ?: JaiTokenTypes.IDENT + tokEnd = end + return + } + + if (isDigit(c)) { + tokType = JaiTokenTypes.NUMBER + tokEnd = scanNumber(p) + return + } + + when (c) { + '"'.code -> { + emit(JaiTokenTypes.STRING, scanString(p)) + } + + '/'.code -> { + scanSlash(p) + } + + '#'.code -> { + scanHash(p) + } + + '@'.code -> { + scanNote(p) + } + + '.'.code -> { + scanDot(p) + } + + '-'.code -> { + scanMinus(p) + } + + ':'.code -> { + when (ch(p + 1)) { + ':'.code -> emit(JaiTokenTypes.COLON_COLON, p + 2) + '='.code -> emit(JaiTokenTypes.COLON_EQ, p + 2) + else -> emit(JaiTokenTypes.COLON, p + 1) + } + } + + '+'.code -> { + emitEq(p, JaiTokenTypes.PLUS, JaiTokenTypes.PLUS_EQ) + } + + '*'.code -> { + emitEq(p, JaiTokenTypes.STAR, JaiTokenTypes.STAR_EQ) + } + + '%'.code -> { + emitEq(p, JaiTokenTypes.PERCENT, JaiTokenTypes.PERCENT_EQ) + } + + '!'.code -> { + emitEq(p, JaiTokenTypes.NOT, JaiTokenTypes.NOT_EQ) + } + + '^'.code -> { + emitEq(p, JaiTokenTypes.XOR, JaiTokenTypes.XOR_EQ) + } + + '~'.code -> { + emit(JaiTokenTypes.TILDE, p + 1) + } + + '&'.code -> { + scanDoubleable( + p, + '&', + JaiTokenTypes.AND, + JaiTokenTypes.AND_EQ, + JaiTokenTypes.AND_AND, + JaiTokenTypes.AND_AND_EQ, + ) + } + + '|'.code -> { + scanDoubleable( + p, + '|', + JaiTokenTypes.OR, + JaiTokenTypes.OR_EQ, + JaiTokenTypes.OR_OR, + JaiTokenTypes.OR_OR_EQ, + ) + } + + '='.code -> { + scanEquals(p) + } + + '<'.code -> { + scanShift( + p, + '<', + JaiTokenTypes.LT, + JaiTokenTypes.LT_EQ, + JaiTokenTypes.SHL, + JaiTokenTypes.SHL_EQ, + JaiTokenTypes.ROL, + JaiTokenTypes.ROL_EQ, + ) + } + + '>'.code -> { + scanShift( + p, + '>', + JaiTokenTypes.GT, + JaiTokenTypes.GT_EQ, + JaiTokenTypes.SHR, + JaiTokenTypes.SHR_EQ, + JaiTokenTypes.ROR, + JaiTokenTypes.ROR_EQ, + ) + } + + ','.code -> { + if (ch(p + 1) == ','.code) { + emit(JaiTokenTypes.DOUBLE_COMMA, p + 2) + } else { + emit(JaiTokenTypes.COMMA, p + 1) + } + } + + '$'.code -> { + if (ch(p + 1) == '$'.code) { + emit(JaiTokenTypes.DOUBLE_DOLLAR, p + 2) + } else { + emit(JaiTokenTypes.DOLLAR, p + 1) + } + } + + '('.code -> { + emit(JaiTokenTypes.LPAREN, p + 1) + } + + ')'.code -> { + emit(JaiTokenTypes.RPAREN, p + 1) + } + + '['.code -> { + emit(JaiTokenTypes.LBRACKET, p + 1) + } + + ']'.code -> { + emit(JaiTokenTypes.RBRACKET, p + 1) + } + + '{'.code -> { + emit(JaiTokenTypes.LBRACE, p + 1) + } + + '}'.code -> { + emit(JaiTokenTypes.RBRACE, p + 1) + } + + ';'.code -> { + emit(JaiTokenTypes.SEMICOLON, p + 1) + } + + '?'.code -> { + emit(JaiTokenTypes.QUESTION, p + 1) + } + + '`'.code -> { + emit(JaiTokenTypes.BACKTICK, p + 1) + } + + '\\'.code -> { + emit(JaiTokenTypes.BACKSLASH, p + 1) + } + + else -> { + emit(TokenType.BAD_CHARACTER, p + 1) + } + } + } + + private fun emit( + type: IElementType, + end: Int, + ) { + tokType = type + tokEnd = end + } + + /** `c` or `c=` — the compiler's `check_for_equals`. */ + private fun emitEq( + p: Int, + plain: IElementType, + augmented: IElementType, + ) { + if (ch(p + 1) == '='.code) emit(augmented, p + 2) else emit(plain, p + 1) + } + + // ------------------------------------------------------- character classes + + private fun isSpace(c: Int): Boolean = + c == ' '.code || c == '\t'.code || c == '\n'.code || c == '\r'.code || + c == 0x0B || c == 0x0C + + private fun isDigit(c: Int): Boolean = c >= '0'.code && c <= '9'.code + + private fun isHexDigit(c: Int): Boolean = isDigit(c) || (c >= 'a'.code && c <= 'f'.code) || (c >= 'A'.code && c <= 'F'.code) + + /** ASCII only, exactly like the compiler's `starts_identifier`. */ + private fun startsIdentifier(c: Int): Boolean = (c >= 'a'.code && c <= 'z'.code) || (c >= 'A'.code && c <= 'Z'.code) || c == '_'.code + + private fun continuesIdentifier(c: Int): Boolean = startsIdentifier(c) || isDigit(c) + + private inline fun scanWhile( + from: Int, + pred: (Int) -> Boolean, + ): Int { + var q = from + while (q < bufEnd && pred(ch(q))) q++ + return q + } + + private fun scanIdentifier(from: Int): Int = scanWhile(from) { continuesIdentifier(it) } + + private fun text( + from: Int, + to: Int, + ): String = buf.subSequence(from, to).toString() + + // --------------------------------------------------------------- scanners + + /** `make_number`, span-wise. */ + private fun scanNumber(from: Int): Int { + var p = from + var isFloat = false + + if (ch(p) == '.'.code) { + isFloat = true + p = scanWhile(p + 1) { isDigit(it) || it == '_'.code } + } else { + if (ch(p) == '0'.code) { + when (ch(p + 1)) { + // 0x hex, 0b binary, 0h hexfloat. Digit validity is the compiler's + // problem; the lexer only needs the span. + 'x'.code, 'X'.code, 'h'.code, 'H'.code -> { + return scanWhile(p + 2) { isHexDigit(it) || it == '_'.code } + } + + 'b'.code, 'B'.code -> { + return scanWhile(p + 2) { isDigit(it) || it == '_'.code } + } + } + } + p = scanWhile(p) { isDigit(it) || it == '_'.code } + // `1..3` is a range, not a float: do not eat the first dot. + if (ch(p) == '.'.code && ch(p + 1) != '.'.code) { + isFloat = true + p = scanWhile(p + 1) { isDigit(it) || it == '_'.code } + } + } + + // The compiler only accepts an exponent after a decimal point, so `1e9` lexes + // as NUMBER(1) IDENT(e9). Match that. + if (isFloat && (ch(p) == 'e'.code || ch(p) == 'E'.code)) { + var q = p + 1 + if (ch(q) == '+'.code || ch(q) == '-'.code) q++ + if (isDigit(ch(q))) p = scanWhile(q) { isDigit(it) || it == '_'.code } + } + return p + } + + /** From the opening quote to just past the closing one; stops at a newline or EOF. */ + private fun scanString(from: Int): Int { + var q = from + 1 + while (q < bufEnd) { + when (ch(q)) { + '\n'.code -> return q + + // unterminated: do not bleed into the next line + '\\'.code -> q = minOf(q + 2, bufEnd) + + '"'.code -> return q + 1 + + else -> q++ + } + } + return q + } + + private fun scanSlash(p: Int) { + when (ch(p + 1)) { + '/'.code -> emit(JaiTokenTypes.LINE_COMMENT, scanToEndOfLine(p + 2)) + '*'.code -> emit(JaiTokenTypes.BLOCK_COMMENT, scanBlockComment(p + 2)) + '='.code -> emit(JaiTokenTypes.SLASH_EQ, p + 2) + else -> emit(JaiTokenTypes.SLASH, p + 1) + } + } + + private fun scanToEndOfLine(from: Int): Int = scanWhile(from) { it != '\n'.code } + + /** + * `eat_input_due_to_block_comment` — comments nest, so this counts depth. + * Note the compiler's own quirk: five slashes followed by a star opens a nested + * comment (module.jai:1433 flags this as possibly undesirable). Matched here. + */ + private fun scanBlockComment(from: Int): Int { + var q = from + var depth = 1 + while (q < bufEnd && depth > 0) { + when (ch(q)) { + '/'.code -> { + q++ + if (ch(q) == '*'.code) { + q++ + depth++ + } + } + + '*'.code -> { + q++ + if (ch(q) == '/'.code) { + q++ + depth-- + } + } + + else -> { + q++ + } + } + } + return q + } + + private fun scanDot(p: Int) { + when (ch(p + 1)) { + '.'.code -> { + emit(JaiTokenTypes.DOUBLE_DOT, p + 2) + } + + '*'.code -> { + emit(JaiTokenTypes.POSTFIX_DEREFERENCE, p + 2) + } + + '['.code -> { + emit(JaiTokenTypes.BEGIN_ARRAY_LITERAL, p + 2) + } + + '{'.code -> { + emit(JaiTokenTypes.BEGIN_STRUCT_LITERAL, p + 2) + } + + else -> { + if (isDigit(ch(p + 1))) { + emit(JaiTokenTypes.NUMBER, scanNumber(p)) + } else { + emit(JaiTokenTypes.DOT, p + 1) + } + } + } + } + + /** `->` then `---` then `--` then `-=` then `-` (module.jai:1680). */ + private fun scanMinus(p: Int) { + when (ch(p + 1)) { + '>'.code -> { + emit(JaiTokenTypes.RIGHT_ARROW, p + 2) + } + + '-'.code -> { + if (ch(p + 2) == '-'.code) { + emit(JaiTokenTypes.TRIPLE_MINUS, p + 3) + } else { + emit(JaiTokenTypes.DOUBLE_MINUS, p + 2) + } + } + + '='.code -> { + emit(JaiTokenTypes.MINUS_EQ, p + 2) + } + + else -> { + emit(JaiTokenTypes.MINUS, p + 1) + } + } + } + + private fun scanEquals(p: Int) { + when (ch(p + 1)) { + '='.code -> { + if (ch(p + 2) == '='.code) { + emit(JaiTokenTypes.EQ_EQ_EQ, p + 3) + } else { + emit(JaiTokenTypes.EQ_EQ, p + 2) + } + } + + '>'.code -> { + emit(JaiTokenTypes.QUICK_LAMBDA, p + 2) + } + + else -> { + emit(JaiTokenTypes.EQ, p + 1) + } + } + } + + /** `&` `&=` `&&` `&&=` and the `|` equivalents. */ + private fun scanDoubleable( + p: Int, + c: Char, + single: IElementType, + singleEq: IElementType, + double: IElementType, + doubleEq: IElementType, + ) { + if (ch(p + 1) == c.code) { + if (ch(p + 2) == '='.code) emit(doubleEq, p + 3) else emit(double, p + 2) + } else { + emitEq(p, single, singleEq) + } + } + + /** `<` `<=` `<<` `<<=` `<<<` `<<<=` and the `>` equivalents. */ + private fun scanShift( + p: Int, + c: Char, + single: IElementType, + singleEq: IElementType, + shift: IElementType, + shiftEq: IElementType, + rotate: IElementType, + rotateEq: IElementType, + ) { + if (ch(p + 1) != c.code) { + emitEq(p, single, singleEq) + return + } + if (ch(p + 2) == c.code) { + if (ch(p + 3) == '='.code) emit(rotateEq, p + 4) else emit(rotate, p + 3) + } else { + if (ch(p + 2) == '='.code) emit(shiftEq, p + 3) else emit(shift, p + 2) + } + } + + /** + * `#ident` is one DIRECTIVE token. `#string` opens a here-string, which is also a + * single token so that no lexer state has to survive a token boundary. `#!` at the + * start of a line is a hashbang line. + */ + private fun scanHash(p: Int) { + if (ch(p + 1) == '!'.code && (p == 0 || buf[p - 1] == '\n')) { + emit(JaiTokenTypes.LINE_COMMENT, scanToEndOfLine(p + 2)) + return + } + if (!startsIdentifier(ch(p + 1))) { + emit(JaiTokenTypes.HASH, p + 1) + return + } + val identEnd = scanIdentifier(p + 1) + if (text(p + 1, identEnd) == "string") { + val hereEnd = scanHereString(identEnd) + if (hereEnd > 0) { + emit(JaiTokenTypes.HERE_STRING, hereEnd) + return + } + } + emit(JaiTokenTypes.DIRECTIVE, identEnd) + } + + /** + * `parse_here_string` (module.jai:476). [from] is just past `#string`; returns the + * end offset of the whole here-string, or -1 if this is not a well-formed one (in + * which case the caller falls back to a plain `#string` directive token). + * + * Shape: `#string [,cr] IDENT \n body... \n [ws] IDENT`, and the token ends right + * after the terminating identifier. + */ + private fun scanHereString(from: Int): Int { + var p = scanWhile(from) { isSpace(it) } + + // Optional `,cr` modifier. + if (ch(p) == ','.code) { + p = scanWhile(p + 1) { isSpace(it) } + if (!startsIdentifier(ch(p))) return -1 + p = scanIdentifier(p) + p = scanWhile(p) { isSpace(it) } + } + + if (!startsIdentifier(ch(p))) return -1 + val identStart = p + p = scanIdentifier(p) + val ident = text(identStart, p) + + // Only whitespace may follow the terminator identifier, up to end of line. + var sawNewline = false + while (p < bufEnd) { + val c = ch(p) + if (!isSpace(c)) return -1 + p++ + if (c == '\n'.code) { + sawNewline = true + break + } + } + if (!sawNewline) return -1 + + // Body: scan line by line for a line whose first non-space run is the terminator. + while (p < bufEnd) { + var q = p + while (q < bufEnd && ch(q) != '\n'.code && isSpace(ch(q))) q++ + if (matchesAt(q, ident) && !continuesIdentifier(ch(q + ident.length))) { + return q + ident.length + } + while (q < bufEnd && ch(q) != '\n'.code) q++ + if (q < bufEnd) q++ // eat the newline + p = q + } + // Unterminated: claim the rest of the file rather than lexing the body as code. + return bufEnd + } + + private fun matchesAt( + at: Int, + s: String, + ): Boolean { + if (at + s.length > bufEnd) return false + for (i in s.indices) if (buf[at + i] != s[i]) return false + return true + } + + /** + * `make_note` (module.jai:716): `@"quoted"`, or `@` followed by everything up to + * whitespace or `;`. + */ + private fun scanNote(p: Int) { + if (ch(p + 1) == '"'.code) { + emit(JaiTokenTypes.NOTE, scanString(p + 1)) + return + } + val end = scanWhile(p + 1) { !isSpace(it) && it != ';'.code && it != 0 } + emit(if (end > p + 1) JaiTokenTypes.NOTE else JaiTokenTypes.AT, end) + } +} diff --git a/src/test/kotlin/dev/hgh/jai/lexer/JaiLexerTest.kt b/src/test/kotlin/dev/hgh/jai/lexer/JaiLexerTest.kt new file mode 100644 index 0000000..690c93b --- /dev/null +++ b/src/test/kotlin/dev/hgh/jai/lexer/JaiLexerTest.kt @@ -0,0 +1,272 @@ +package dev.hgh.jai.lexer + +import com.intellij.psi.TokenType +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * Tier 1 lexer tests (see `docs/BUILD_PLAN.md` §3): one case per gotcha in + * `docs/JAI_LANGUAGE_REFERENCE.md` §14. Plain JUnit — no IntelliJ fixture boots here. + */ +class JaiLexerTest { + /** `TYPE 'text'` per token, whitespace dropped. */ + private fun dump(code: String): List { + val lexer = JaiLexer() + lexer.start(code, 0, code.length, 0) + val out = mutableListOf() + var previousStart = -1 + while (true) { + val type = lexer.tokenType ?: break + assertTrue( + "lexer must make progress at offset ${lexer.tokenStart}", + lexer.tokenStart > previousStart, + ) + previousStart = lexer.tokenStart + if (type != TokenType.WHITE_SPACE) { + out += "$type '${code.substring(lexer.tokenStart, lexer.tokenEnd)}'" + } + lexer.advance() + } + return out + } + + private fun assertTokens( + code: String, + vararg expected: String, + ) { + assertEquals(expected.toList(), dump(code)) + assertRoundTrips(code) + } + + /** The flagship invariant: tokens must tile the input exactly. */ + private fun assertRoundTrips(code: String) { + val lexer = JaiLexer() + lexer.start(code, 0, code.length, 0) + val sb = StringBuilder() + var offset = 0 + while (lexer.tokenType != null) { + assertEquals("token must start where the previous one ended", offset, lexer.tokenStart) + sb.append(code, lexer.tokenStart, lexer.tokenEnd) + offset = lexer.tokenEnd + lexer.advance() + } + assertEquals(code, sb.toString()) + } + + @Test + fun declarationForms() { + assertTokens( + "a := 1;", + "Jai:IDENT 'a'", + "Jai::= ':='", + "Jai:NUMBER '1'", + "Jai:; ';'", + ) + assertTokens( + "main :: () { }", + "Jai:IDENT 'main'", + "Jai::: '::'", + "Jai:( '('", + "Jai:) ')'", + "Jai:{ '{'", + "Jai:} '}'", + ) + assertTokens( + "b : float;", + "Jai:IDENT 'b'", + "Jai:: ':'", + "Jai:IDENT 'float'", + "Jai:; ';'", + ) + } + + @Test + fun keywordsAreKeywordsAndTypeNamesAreNot() { + assertTokens("for", "Jai:for 'for'") + assertTokens("xx", "Jai:xx 'xx'") + assertTokens("initializer_of", "Jai:initializer_of 'initializer_of'") + // §14.9: primitive type names are ordinary identifiers. + assertTokens("int", "Jai:IDENT 'int'") + assertTokens("string", "Jai:IDENT 'string'") + // Not keywords, just similar. + assertTokens("iffy", "Jai:IDENT 'iffy'") + assertTokens("must", "Jai:IDENT 'must'") + } + + @Test + fun nestedBlockComments() { + assertTokens( + "/* a /* b */ still */x", + "Jai:BLOCK_COMMENT '/* a /* b */ still */'", + "Jai:IDENT 'x'", + ) + // Unterminated: claims the rest of the file. + assertTokens("/* a /* b */", "Jai:BLOCK_COMMENT '/* a /* b */'") + assertTokens("// a /* b\nx", "Jai:LINE_COMMENT '// a /* b'", "Jai:IDENT 'x'") + } + + @Test + fun minusForms() { + assertTokens("->", "Jai:-> '->'") + assertTokens("---", "Jai:--- '---'") + assertTokens("--", "Jai:-- '--'") + assertTokens("-=", "Jai:-= '-='") + assertTokens("-", "Jai:- '-'") + assertTokens("----", "Jai:--- '---'", "Jai:- '-'") + assertTokens( + "x: T = ---;", + "Jai:IDENT 'x'", + "Jai:: ':'", + "Jai:IDENT 'T'", + "Jai:= '='", + "Jai:--- '---'", + "Jai:; ';'", + ) + } + + @Test + fun dotForms() { + assertTokens(".{", "Jai:.{ '.{'") + assertTokens(".[", "Jai:.[ '.['") + assertTokens(".*", "Jai:.* '.*'") + assertTokens(".APRICOT", "Jai:. '.'", "Jai:IDENT 'APRICOT'") + assertTokens("ptr.*", "Jai:IDENT 'ptr'", "Jai:.* '.*'") + assertTokens( + "for 0..7", + "Jai:for 'for'", + "Jai:NUMBER '0'", + "Jai:.. '..'", + "Jai:NUMBER '7'", + ) + assertTokens("...", "Jai:.. '..'", "Jai:. '.'") + assertTokens("..Any", "Jai:.. '..'", "Jai:IDENT 'Any'") + } + + @Test + fun numbers() { + assertTokens("123", "Jai:NUMBER '123'") + assertTokens("1_000_000", "Jai:NUMBER '1_000_000'") + assertTokens("0xfade_deaf", "Jai:NUMBER '0xfade_deaf'") + assertTokens("0b101101101", "Jai:NUMBER '0b101101101'") + assertTokens("0h7fbf_ffff", "Jai:NUMBER '0h7fbf_ffff'") + assertTokens("37.0", "Jai:NUMBER '37.0'") + assertTokens("-10.0", "Jai:- '-'", "Jai:NUMBER '10.0'") + assertTokens(".5", "Jai:NUMBER '.5'") + assertTokens("1.5e-7", "Jai:NUMBER '1.5e-7'") + assertTokens("1.5E+7", "Jai:NUMBER '1.5E+7'") + // The compiler only takes an exponent after a decimal point. + assertTokens("1e9", "Jai:NUMBER '1'", "Jai:IDENT 'e9'") + // A range wins over a decimal point. + assertTokens("1..3", "Jai:NUMBER '1'", "Jai:.. '..'", "Jai:NUMBER '3'") + } + + @Test + fun strings() { + assertTokens("\"Hello\"", "Jai:STRING '\"Hello\"'") + assertTokens("\"a\\\"b\"", "Jai:STRING '\"a\\\"b\"'") + assertTokens("\"a\\\\\"", "Jai:STRING '\"a\\\\\"'") + // §14.2: there is no character literal; `#char "a"` is a directive + string. + assertTokens("#char \"a\"", "Jai:DIRECTIVE '#char'", "Jai:STRING '\"a\"'") + // Unterminated strings stop at the newline instead of bleeding. + assertTokens("\"oops\nx", "Jai:STRING '\"oops'", "Jai:IDENT 'x'") + } + + @Test + fun hereStrings() { + assertTokens( + "S :: #string DONE\nanything \"quoted\" /* not a comment */\nDONE\nx", + "Jai:IDENT 'S'", + "Jai::: '::'", + "Jai:HERE_STRING '#string DONE\nanything \"quoted\" /* not a comment */\nDONE'", + "Jai:IDENT 'x'", + ) + // Arbitrary terminator, indented terminator line, and a near-miss line. + assertTokens( + "#string XY\nXYZ is not the end\n XY\n", + "Jai:HERE_STRING '#string XY\nXYZ is not the end\n XY'", + ) + assertTokens( + "#string,cr END\nbody\nEND", + "Jai:HERE_STRING '#string,cr END\nbody\nEND'", + ) + // Unterminated: claim the rest rather than lexing the body as code. + assertTokens("#string DONE\nbody", "Jai:HERE_STRING '#string DONE\nbody'") + // Not a here-string at all: falls back to a directive token. + assertTokens("#string;", "Jai:DIRECTIVE '#string'", "Jai:; ';'") + assertTokens("#stringify", "Jai:DIRECTIVE '#stringify'") + } + + @Test + fun directivesAndNotes() { + assertTokens("#import \"Basic\";", "Jai:DIRECTIVE '#import'", "Jai:STRING '\"Basic\"'", "Jai:; ';'") + assertTokens("#import,string", "Jai:DIRECTIVE '#import'", "Jai:, ','", "Jai:IDENT 'string'") + assertTokens("#", "Jai:# '#'") + assertTokens("@Cleanup", "Jai:NOTE '@Cleanup'") + assertTokens("x; @Cleanup", "Jai:IDENT 'x'", "Jai:; ';'", "Jai:NOTE '@Cleanup'") + assertTokens("@Note;", "Jai:NOTE '@Note'", "Jai:; ';'") + assertTokens("@\"quoted note\"", "Jai:NOTE '@\"quoted note\"'") + assertTokens("#!/usr/bin/env jai\nx", "Jai:LINE_COMMENT '#!/usr/bin/env jai'", "Jai:IDENT 'x'") + } + + @Test + fun operators() { + assertTokens(",,", "Jai:,, ',,'") + assertTokens(",", "Jai:, ','") + assertTokens("=>", "Jai:=> '=>'") + assertTokens("===", "Jai:=== '==='") + assertTokens("==", "Jai:== '=='") + assertTokens("x == {", "Jai:IDENT 'x'", "Jai:== '=='", "Jai:{ '{'") + assertTokens("<<<=", "Jai:<<<= '<<<='") + assertTokens("<<<", "Jai:<<< '<<<'") + assertTokens("<<=", "Jai:<<= '<<='") + assertTokens("<<", "Jai:<< '<<'") + assertTokens("<=", "Jai:<= '<='") + assertTokens("<", "Jai:< '<'") + assertTokens(">>>=", "Jai:>>>= '>>>='") + assertTokens("&&=", "Jai:&&= '&&='") + assertTokens("&&", "Jai:&& '&&'") + assertTokens("&=", "Jai:&= '&='") + assertTokens("&", "Jai:& '&'") + assertTokens("||=", "Jai:||= '||='") + assertTokens("\$\$", "Jai:\$\$ '\$\$'") + assertTokens("\$T", "Jai:\$ '\$'", "Jai:IDENT 'T'") + assertTokens("`it", "Jai:` '`'", "Jai:IDENT 'it'") + assertTokens("`defer", "Jai:` '`'", "Jai:defer 'defer'") + } + + @Test + fun forLoopModifiers() { + assertTokens( + "for < numbers {", + "Jai:for 'for'", + "Jai:< '<'", + "Jai:IDENT 'numbers'", + "Jai:{ '{'", + ) + assertTokens( + "for * teas {", + "Jai:for 'for'", + "Jai:* '*'", + "Jai:IDENT 'teas'", + "Jai:{ '{'", + ) + } + + @Test + fun lexingASliceOfTheBufferStaysInBounds() { + val text = "aaa 12345 bbb" + val lexer = JaiLexer() + lexer.start(text, 4, 9, 0) + assertEquals(JaiTokenTypes.NUMBER, lexer.tokenType) + assertEquals(4, lexer.tokenStart) + assertEquals(9, lexer.tokenEnd) + lexer.advance() + assertEquals(null, lexer.tokenType) + } + + @Test + fun emptyInput() { + assertTokens("") + } +}