From 427269ad782e62099575be58740c67553ac8e438 Mon Sep 17 00:00:00 2001 From: hgranthorner <37941012+hgranthorner@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:10:07 -0400 Subject: [PATCH] Implement Jai formatter --- AGENTS.md | 23 +- docs/BUILD_PLAN.md | 16 +- .../formatter/JaiFormattingModelBuilder.kt | 283 ++++++++++++++++++ src/main/resources/META-INF/plugin.xml | 1 + .../dev/hgh/jai/formatter/JaiFormatterTest.kt | 126 ++++++++ 5 files changed, 432 insertions(+), 17 deletions(-) create mode 100644 src/main/kotlin/dev/hgh/jai/formatter/JaiFormattingModelBuilder.kt create mode 100644 src/test/kotlin/dev/hgh/jai/formatter/JaiFormatterTest.kt diff --git a/AGENTS.md b/AGENTS.md index 820d735..a0bac9f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,7 @@ Last verified state (all green, `./jaigradle check` and `verifyPlugin` too): ```text dev.hgh.HarnessSmokeTest tests=2 dev.hgh.jai.JaiFileTypeTest tests=4 -dev.hgh.jai.completion.JaiCompletionTest tests=19 +dev.hgh.jai.completion.JaiCompletionTest tests=20 dev.hgh.jai.editor.JaiEditorSupportTest tests=7 dev.hgh.jai.editor.JaiFoldingBuilderTest tests=3 dev.hgh.jai.findusages.JaiFindUsagesTest tests=2 @@ -73,11 +73,12 @@ dev.hgh.jai.parser.JaiParserGoldenTest tests=5 <- Tier 2 golden trees dev.hgh.jai.parser.JaiParserLongTailTest tests=1 <- focused parser regressions dev.hgh.jai.parser.DebugParseTest tests=2 <- scratch harness, inert dev.hgh.jai.refactoring.JaiRenameTest tests=3 -dev.hgh.jai.reference.JaiReferenceTest tests=9 +dev.hgh.jai.reference.JaiReferenceTest tests=10 dev.hgh.jai.structure.JaiStructureViewTest tests=3 +dev.hgh.jai.formatter.JaiFormatterTest tests=4 dev.hgh.jai.settings.JaiProjectSettingsTest tests=3 dev.hgh.jai.settings.JaiConfiguredRootTest tests=6 --> total 93, failures+errors 0 +-> total 99, failures+errors 0 ``` The corpus gates report what they actually did; check both lines are still there: @@ -129,6 +130,10 @@ Tier 3: parsed 714 files, 714 clean (100.0%), 0 with errors, 0 PsiErrorElements sources. Headless tests cover state round-tripping, the Settings panel, custom-root `#import`/`#load` navigation and completion, and cross-root find-usages. +- **Phase 6** — PSI-aware formatter with operator/punctuation spacing, + block indentation, opaque `#asm` preservation, and a corpus-wide idempotence + gate. Headless formatter tests cover registration, representative formatting, + opaque/directive token preservation, and all 714 corpus files. ### Lexer design facts worth knowing before touching it @@ -197,16 +202,14 @@ Tier 3: parsed 714 files, 714 clean (100.0%), 0 with errors, 0 PsiErrorElements ### In progress — pick up here -1. **Phases 6+** — formatter, inspections, compiler integration, and other - optional work remain. `#asm` bodies are intentionally consumed opaquely, so - nothing inside them has PSI yet. +1. **Phases 7+** — inspections, compiler integration, and other optional work + remain. `#asm` bodies are intentionally consumed opaquely, so nothing inside + them has PSI yet. ### Next planned increment -- Decide whether to continue with the optional formatter, inspections, and - compiler integration phases (6–8). -- If formatter work begins, add the corpus-wide idempotence gate described in - `docs/BUILD_PLAN.md` before changing formatting behavior. +- Decide whether to continue with inspections and compiler integration phases + (7–8). ### Open questions for the user (unanswered) diff --git a/docs/BUILD_PLAN.md b/docs/BUILD_PLAN.md index 477e03d..f06f539 100644 --- a/docs/BUILD_PLAN.md +++ b/docs/BUILD_PLAN.md @@ -171,12 +171,14 @@ Each phase has a machine-checkable gate. Do not advance without a green gate. | **4** | Structure view, folding, `#import`/`#load` reference resolution + go-to-definition | Tier 4 fixture tests — **done** | | **5** | Completion (keywords, directives, module names, visible declarations, parameters, module aliases, incomplete-expression recovery), rename, find-usages | Tier 4 fixture tests — **done** (84-test suite) | | **5a** | Configurable Jai module/import roots and indexed external search scope | Tier 4 settings, resolution, completion, navigation, and find-usages tests — **done** (93-test suite) | -| **6** | Formatter, code style settings | Formatter round-trip: formatting the corpus is idempotent | +| **6** | Formatter, code style settings | Formatter round-trip — **done**, formatting all 714 corpus files is idempotent | | **7** | Inspections (e.g. `#must` misuse), quick fixes, live templates | Tier 4 + `verifyPlugin` | | **8** | Optional: run-configuration to invoke the `jai` compiler, parse its error output | Integration test against `~/.local/jai/bin` | Phase 6's idempotence check (format twice, assert no change) is another -corpus-scale invariant that needs no human judgment. +corpus-scale invariant that needs no human judgment. `JaiFormatterTest` runs this +check across all 714 files and also covers formatter registration, representative +spacing/indentation, directive flags, strings, and opaque `#asm` bodies. --- @@ -185,8 +187,8 @@ corpus-scale invariant that needs no human judgment. Phase 5 completion now covers keywords, compiler directives, `#import`/`#load` module and file paths, visible same-file or imported top-level declarations, procedure parameters, and members of imported module -aliases. Struct-field completion remains a possible follow-up before or after -Phase 6, but it is not currently a committed milestone. +aliases. Struct-field completion remains a possible follow-up, but it is not +currently a committed milestone. ## 5. Risks and how the plan handles them @@ -216,6 +218,6 @@ Phase 6, but it is not currently a committed milestone. setup, works today), or install a system JDK via Homebrew? I'd default to the wrapper. -Phases 0–5a are implemented and verified headlessly. The next decision is -whether to continue with the optional formatter, inspections, and compiler -integration work in Phases 6–8. +Phases 0–6 are implemented and verified headlessly. The next decision is +whether to continue with the optional inspections and compiler integration work +in Phases 7–8. diff --git a/src/main/kotlin/dev/hgh/jai/formatter/JaiFormattingModelBuilder.kt b/src/main/kotlin/dev/hgh/jai/formatter/JaiFormattingModelBuilder.kt new file mode 100644 index 0000000..9adc0f2 --- /dev/null +++ b/src/main/kotlin/dev/hgh/jai/formatter/JaiFormattingModelBuilder.kt @@ -0,0 +1,283 @@ +package dev.hgh.jai.formatter + +import com.intellij.formatting.Alignment +import com.intellij.formatting.Block +import com.intellij.formatting.ChildAttributes +import com.intellij.formatting.FormattingContext +import com.intellij.formatting.FormattingModel +import com.intellij.formatting.FormattingModelBuilder +import com.intellij.formatting.FormattingModelProvider +import com.intellij.formatting.Indent +import com.intellij.formatting.Spacing +import com.intellij.formatting.SpacingBuilder +import com.intellij.formatting.Wrap +import com.intellij.lang.ASTNode +import com.intellij.psi.TokenType +import com.intellij.psi.codeStyle.CodeStyleSettings +import com.intellij.psi.formatter.common.AbstractBlock +import com.intellij.psi.tree.IElementType +import com.intellij.psi.tree.TokenSet +import dev.hgh.jai.JaiLanguage +import dev.hgh.jai.lexer.JaiTokenTypes +import dev.hgh.jai.psi.JaiDirectiveExpr +import dev.hgh.jai.psi.JaiExpressionStatement +import dev.hgh.jai.psi.JaiPrefixExpr +import dev.hgh.jai.psi.JaiStatement +import dev.hgh.jai.psi.JaiTypes + +/** Basic PSI-aware formatter for Jai source files. */ +class JaiFormattingModelBuilder : FormattingModelBuilder { + override fun createModel(formattingContext: FormattingContext): FormattingModel { + val settings = formattingContext.codeStyleSettings + val spacingBuilder = createSpacingBuilder(settings) + val block = JaiFormattingBlock(formattingContext.node, null, null, spacingBuilder) + return FormattingModelProvider.createFormattingModelForPsiFile( + formattingContext.containingFile, + block, + settings, + ) + } + + private companion object { + private val BINARY_OPERATORS = + TokenSet.create( + JaiTokenTypes.PLUS, + JaiTokenTypes.MINUS, + JaiTokenTypes.STAR, + JaiTokenTypes.SLASH, + JaiTokenTypes.PERCENT, + JaiTokenTypes.PLUS_EQ, + JaiTokenTypes.MINUS_EQ, + JaiTokenTypes.STAR_EQ, + JaiTokenTypes.SLASH_EQ, + JaiTokenTypes.PERCENT_EQ, + JaiTokenTypes.EQ, + JaiTokenTypes.EQ_EQ, + JaiTokenTypes.EQ_EQ_EQ, + JaiTokenTypes.NOT_EQ, + JaiTokenTypes.LT, + JaiTokenTypes.GT, + JaiTokenTypes.LT_EQ, + JaiTokenTypes.GT_EQ, + JaiTokenTypes.AND_AND, + JaiTokenTypes.OR_OR, + JaiTokenTypes.AND_AND_EQ, + JaiTokenTypes.OR_OR_EQ, + JaiTokenTypes.AND, + JaiTokenTypes.OR, + JaiTokenTypes.XOR, + JaiTokenTypes.AND_EQ, + JaiTokenTypes.OR_EQ, + JaiTokenTypes.XOR_EQ, + JaiTokenTypes.SHL, + JaiTokenTypes.SHR, + JaiTokenTypes.ROL, + JaiTokenTypes.ROR, + JaiTokenTypes.SHL_EQ, + JaiTokenTypes.SHR_EQ, + JaiTokenTypes.ROL_EQ, + JaiTokenTypes.ROR_EQ, + JaiTokenTypes.RIGHT_ARROW, + JaiTokenTypes.QUICK_LAMBDA, + ) + + private fun createSpacingBuilder(settings: CodeStyleSettings): SpacingBuilder = + SpacingBuilder(settings, JaiLanguage) + .before(JaiTokenTypes.COMMA) + .none() + .after(JaiTokenTypes.COMMA) + .spaces(1) + .before(JaiTokenTypes.DOUBLE_COMMA) + .none() + .after(JaiTokenTypes.DOUBLE_COMMA) + .spaces(1) + .before(JaiTokenTypes.COLON) + .none() + .after(JaiTokenTypes.COLON) + .spaces(1) + .around(JaiTokenTypes.COLON_COLON) + .spaces(1) + .around(JaiTokenTypes.COLON_EQ) + .spaces(1) + .around(BINARY_OPERATORS) + .spaces(1) + .around(JaiTokenTypes.DOT) + .none() + .around(JaiTokenTypes.POSTFIX_DEREFERENCE) + .none() + .before(JaiTokenTypes.LBRACE) + .spaces(1) + .before(JaiTypes.BLOCK) + .spaces(1) + .withinPair(JaiTokenTypes.LPAREN, JaiTokenTypes.RPAREN) + .none() + .withinPair(JaiTokenTypes.LBRACKET, JaiTokenTypes.RBRACKET) + .none() + .withinPair(JaiTokenTypes.BEGIN_STRUCT_LITERAL, JaiTokenTypes.RBRACE) + .none() + .withinPair(JaiTokenTypes.BEGIN_ARRAY_LITERAL, JaiTokenTypes.RBRACKET) + .none() + } +} + +private class JaiFormattingBlock( + node: ASTNode, + wrap: Wrap?, + alignment: Alignment?, + private val spacingBuilder: SpacingBuilder, +) : AbstractBlock(node, wrap, alignment) { + private val astNode: ASTNode = node + + override fun buildChildren(): List { + if (isOpaqueAsm(myNode)) return emptyList() + + val blocks = mutableListOf() + var child = myNode.firstChildNode + while (child != null) { + if (child.elementType != TokenType.WHITE_SPACE && child.textLength > 0) { + blocks += JaiFormattingBlock(child, null, null, spacingBuilder) + } + child = child.treeNext + } + return blocks + } + + override fun getIndent(): Indent? { + val parent = myNode.treeParent ?: return Indent.getNoneIndent() + return if (parent.elementType == JaiTypes.BLOCK && + myNode.elementType != JaiTokenTypes.LBRACE && + myNode.elementType != JaiTokenTypes.RBRACE + ) { + Indent.getNormalIndent() + } else { + Indent.getNoneIndent() + } + } + + override fun getChildAttributes(newChildIndex: Int): ChildAttributes = + if (myNode.elementType == JaiTypes.BLOCK) { + ChildAttributes(Indent.getNormalIndent(), null) + } else { + ChildAttributes(Indent.getNoneIndent(), null) + } + + override fun getSpacing( + child1: Block?, + child2: Block, + ): Spacing? { + val firstNode = nodeOf(child1) + val secondNode = nodeOf(child2) + val firstType = firstNode?.elementType + val secondType = secondNode?.elementType + + if (myNode.elementType == JaiTypes.BLOCK && + firstType == JaiTokenTypes.LBRACE && + secondType == JaiTokenTypes.RBRACE + ) { + return NO_SPACE + } + + if (firstType == JaiTokenTypes.LINE_COMMENT) return LINE_COMMENT_END + if (secondType == JaiTokenTypes.LINE_COMMENT) return COMMENT_START + + if (firstType == JaiTokenTypes.COLON_COLON || secondType == JaiTokenTypes.COLON_COLON || + firstType == JaiTokenTypes.COLON_EQ || secondType == JaiTokenTypes.COLON_EQ + ) { + return ONE_SPACE + } + + if (secondType == JaiTokenTypes.COLON) return NO_SPACE + if (firstType == JaiTokenTypes.COLON) return ONE_SPACE + + if (firstType == JaiTokenTypes.COMMA && + ( + isDirectiveFlagComma(firstNode) || + (myNode.elementType == JaiTypes.DECLARATION && isUsingFlagComma(firstNode)) || + (myNode.elementType == JaiTypes.PARAMETER && isUsingFlagComma(firstNode)) + ) + ) { + return NO_SPACE + } + + if ((myNode.psi is JaiPrefixExpr || myNode.elementType == JaiTypes.PREFIX_EXPR) && + isPrefixOperator(firstType) + ) { + return NO_SPACE + } + + if (secondType == JaiTypes.STATEMENT && isBodyStatementParent(myNode.elementType)) { + return ONE_SPACE + } + + if (firstType == JaiTokenTypes.DOT || secondType == JaiTokenTypes.DOT || + firstType == JaiTokenTypes.POSTFIX_DEREFERENCE || secondType == JaiTokenTypes.POSTFIX_DEREFERENCE + ) { + return NO_SPACE + } + + return spacingBuilder.getSpacing(this, child1, child2) + } + + override fun isLeaf(): Boolean = isOpaqueAsm(myNode) || myNode.firstChildNode == null + + private fun isBodyStatementParent(type: IElementType): Boolean = + type == JaiTypes.IF_STATEMENT || + type == JaiTypes.WHILE_STATEMENT || + type == JaiTypes.FOR_STATEMENT || + type == JaiTypes.ELSE_CLAUSE || + type == JaiTypes.CASE_CLAUSE || + type == JaiTypes.DEFER_STATEMENT || + type == JaiTypes.PUSH_CONTEXT_STATEMENT + + private fun isUsingFlagComma(comma: ASTNode?): Boolean { + if (comma == null || comma.treeNext?.elementType != JaiTokenTypes.IDENT) return false + return comma.treePrev?.elementType == JaiTokenTypes.KW_USING + } + + private fun isPrefixOperator(type: IElementType?): Boolean = + type == JaiTokenTypes.PLUS || + type == JaiTokenTypes.MINUS || + type == JaiTokenTypes.STAR || + type == JaiTokenTypes.NOT || + type == JaiTokenTypes.TILDE || + type == JaiTokenTypes.DOLLAR || + type == JaiTokenTypes.DOUBLE_DOLLAR || + type == JaiTokenTypes.SHL || + type == JaiTokenTypes.SHR + + private fun nodeOf(block: Block?): ASTNode? = (block as? JaiFormattingBlock)?.astNode + + private fun isDirectiveFlagComma(comma: ASTNode?): Boolean { + if (comma == null) return false + if (comma.treeNext?.elementType != JaiTokenTypes.IDENT) return false + + val previous = comma.treePrev ?: return false + if (previous.elementType == TokenType.WHITE_SPACE) return false + if (previous.elementType == JaiTokenTypes.DIRECTIVE || + previous.elementType == JaiTokenTypes.KW_CAST || + previous.elementType == JaiTokenTypes.KW_XX || + previous.elementType == JaiTokenTypes.KW_PUSH_CONTEXT + ) { + return true + } + if (previous.elementType != JaiTokenTypes.IDENT) return false + + val previousComma = previous.treePrev ?: return false + return previousComma.elementType != TokenType.WHITE_SPACE && + previousComma.elementType == JaiTokenTypes.COMMA && + isDirectiveFlagComma(previousComma) + } + + private fun isOpaqueAsm(node: ASTNode): Boolean { + val psi = node.psi + return (psi is JaiDirectiveExpr || psi is JaiExpressionStatement || psi is JaiStatement) && + psi.text.trimStart().startsWith("#asm") + } + + private companion object { + val NO_SPACE: Spacing = Spacing.createSpacing(0, 0, 0, false, 0) + val ONE_SPACE: Spacing = Spacing.createSpacing(1, 1, 0, false, 0) + val COMMENT_START: Spacing = Spacing.createSpacing(1, 1, 0, true, 0) + val LINE_COMMENT_END: Spacing = Spacing.createSpacing(0, 0, 1, true, 0) + } +} diff --git a/src/main/resources/META-INF/plugin.xml b/src/main/resources/META-INF/plugin.xml index d1b1ded..1dc18eb 100644 --- a/src/main/resources/META-INF/plugin.xml +++ b/src/main/resources/META-INF/plugin.xml @@ -22,6 +22,7 @@ extensions="jai"/> + int{ + value:=1+2; + negative:=-69; + if value>0{ + return value; + } + } + """.trimIndent(), + ) + + WriteCommandAction.runWriteCommandAction(project) { + CodeStyleManager.getInstance(project).reformatText(myFixture.file, 0, myFixture.file.textLength) + } + myFixture.checkResult( + """ + main :: () -> int { + value := 1 + 2; + negative := -69; + if value > 0 { + return value; + } + } + """.trimIndent(), + ) + } + + fun testPreservesDirectiveFlagsOpaqueAssemblyAndStringText() { + myFixture.configureByText( + "opaque.jai", + """ + #import,file "library.jai"; + main::(){ + value:=cast,trunc(u32) input; + #asm { mov rax, rbx; } + text:=" keep "; + } + """.trimIndent(), + ) + + WriteCommandAction.runWriteCommandAction(project) { + CodeStyleManager.getInstance(project).reformatText(myFixture.file, 0, myFixture.file.textLength) + } + myFixture.checkResult( + """ + #import,file "library.jai"; + main :: () { + value := cast,trunc(u32) input; + #asm { mov rax, rbx; } + text := " keep "; + } + """.trimIndent(), + ) + } + + fun testCorpusFormattingIsIdempotent() { + val corpusDir = File(System.getProperty("user.home"), ".local/jai") + if (!corpusDir.exists()) { + println("Skipping formatter corpus test: $corpusDir does not exist") + return + } + + val jaiFiles = + corpusDir + .walkTopDown() + .filter { it.isFile && it.extension == "jai" } + .sorted() + .toList() + assertTrue("Corpus files must be found in $corpusDir", jaiFiles.isNotEmpty()) + + val nonIdempotent = mutableListOf() + jaiFiles.forEach { file -> + val normalizedText = file.readText().replace("\r\n", "\n").replace('\r', '\n') + val psiFile = PsiFileFactory.getInstance(project).createFileFromText(file.name, JaiLanguage, normalizedText) + val once = reformatPsiFile(psiFile) + val reparsed = PsiFileFactory.getInstance(project).createFileFromText(file.name, JaiLanguage, once) + val hasParseErrors = + com.intellij.psi.util.PsiTreeUtil + .findChildrenOfType(reparsed, com.intellij.psi.PsiErrorElement::class.java) + .isNotEmpty() + val twice = reformatPsiFile(psiFile) + if (hasParseErrors || once != twice) { + nonIdempotent += file.relativePath(corpusDir) + } + } + + println("Formatter corpus: checked ${jaiFiles.size} files, non-idempotent=${nonIdempotent.size}") + assertTrue( + "Formatter is not idempotent for ${nonIdempotent.take(20)}", + nonIdempotent.isEmpty(), + ) + } + + private fun reformatPsiFile(psiFile: com.intellij.psi.PsiFile): String = + WriteCommandAction.runWriteCommandAction( + project, + Computable { + CodeStyleManager.getInstance(project).reformatRange(psiFile, 0, psiFile.textLength) + psiFile.text + }, + ) + + private fun File.relativePath(base: File): String = absolutePath.removePrefix(base.absolutePath + "/") +}