diff --git a/grammars/prql-lezer/src/highlight.js b/grammars/prql-lezer/src/highlight.js index 068b8a78fd3a..06505ff20eef 100644 --- a/grammars/prql-lezer/src/highlight.js +++ b/grammars/prql-lezer/src/highlight.js @@ -1,7 +1,14 @@ import { styleTags, tags as t } from "@lezer/highlight"; -export const prqlHighlight = styleTags({ +// The selectors, separate from the `styleTags` call below so that +// `test/test-highlight.js` can check each one resolves to a term in the +// grammar: `styleTags` drops a selector that matches nothing without +// reporting it. +export const prqlHighlightSpec = { "CallExpression/Identifier": t.function(t.variableName), + // Keywords are named terms only because the grammar declares them through + // `kw<>`; see the note on the literal tokens in `prql.grammar`. + prql: t.keyword, module: t.moduleKeyword, let: t.definitionKeyword, case: t.controlKeyword, @@ -33,4 +40,6 @@ export const prqlHighlight = styleTags({ "[ ]": t.squareBracket, "{ }": t.brace, "| ,": t.separator, -}); +}; + +export const prqlHighlight = styleTags(prqlHighlightSpec); diff --git a/grammars/prql-lezer/src/prql.grammar b/grammars/prql-lezer/src/prql.grammar index a57a9586161e..33fff3aa064f 100644 --- a/grammars/prql-lezer/src/prql.grammar +++ b/grammars/prql-lezer/src/prql.grammar @@ -19,7 +19,7 @@ statements { newline* QueryDefinition? Module? Annotation? VariableDeclaration* pipelineStatement? end? } -QueryDefinition { @specialize NamedArg+ newline+ } +QueryDefinition { kw<"prql"> NamedArg+ newline+ } Module { kw<"module"> Identifier "{" statements "}" } @@ -46,7 +46,7 @@ Declaration { DeclarationItem { identPart } "=" expression } DeclarationTuple { DeclarationItem { identPart } "=" expression } CaseBranch { expression "=>" expression } // Possibly we could only accept case branches inside the TupleExpression? -CaseExpression { @specialize TupleExpression } +CaseExpression { kw<"case"> TupleExpression } NestedPipeline { "(" newline* Pipeline ~ambigNewline newline? ")" } @@ -239,6 +239,11 @@ LambdaParam { identPart TypeDefinition? (":" expression)? } "="[@name=Equals] + // Declaring these literals here names each one after itself, which is what + // makes them addressable from `highlight.js` — an undeclared literal is an + // anonymous term, so a style tag naming it silently matches nothing. + "(" ")" "[" "]" "{" "}" "," "|" + FString { interpolatedString<'f'> } RString { "r" (rawStringDouble | rawStringSingle) } SString { interpolatedString<'s'> } diff --git a/grammars/prql-lezer/test/misc.txt b/grammars/prql-lezer/test/misc.txt index 72877cbdc9ea..a53e0a0fec10 100644 --- a/grammars/prql-lezer/test/misc.txt +++ b/grammars/prql-lezer/test/misc.txt @@ -197,3 +197,24 @@ derive { ==> Query(Pipeline(CallExpression(Identifier,ArgList(TupleExpression(DeclarationTuple(DeclarationItem,Equals,NestedPipeline(Pipeline(Identifier,Identifier)))))))) + +# Query definition + +prql target:sql.duckdb + +from foo + +==> + +Query(QueryDefinition(prql,NamedArg(ArgumentName,Identifier)),Pipeline(CallExpression(Identifier,ArgList(Identifier)))) + +# Case expression + +derive x = case { + a => 1, + true => 2 +} + +==> + +Query(Pipeline(CallExpression(Identifier,ArgList(Declaration(DeclarationItem,Equals,CaseExpression(case,TupleExpression(CaseBranch(Identifier,Integer),CaseBranch(Boolean,Integer)))))))) diff --git a/grammars/prql-lezer/test/test-highlight.js b/grammars/prql-lezer/test/test-highlight.js new file mode 100644 index 000000000000..98537ecd24e9 --- /dev/null +++ b/grammars/prql-lezer/test/test-highlight.js @@ -0,0 +1,95 @@ +// A style tag in `highlight.js` whose selector matches no term in the grammar +// is silently dropped rather than reported, so these tests pin each token to +// the tag it should carry — a selector that stops matching turns up here as a +// failure instead of as unstyled output in a downstream editor. +import { parser } from "../dist/index.js"; +import { prqlHighlightSpec } from "../src/highlight.js"; +import { highlightTree, tagHighlighter, tags as t } from "@lezer/highlight"; +import * as assert from "assert"; + +// One class per tag, rather than `classHighlighter`'s groupings, so that a +// token styled with the wrong tag of a family is still a failure. +const highlighter = tagHighlighter( + [ + t.keyword, + t.moduleKeyword, + t.definitionKeyword, + t.controlKeyword, + t.operatorKeyword, + t.paren, + t.squareBracket, + t.brace, + t.separator, + ].map((tag) => ({ tag, class: tag.toString() })), +); + +// The tag each styled token carries, as `[text, tag]` pairs. Unstyled tokens +// are absent, which is what an unmatched style tag produces. +function tagged(source) { + const styled = []; + highlightTree(parser.parse(source), highlighter, (from, to, classes) => + styled.push([source.slice(from, to), classes]), + ); + return styled; +} + +describe("highlight", () => { + it("tags the keywords the grammar names through `kw<>`", () => { + assert.deepStrictEqual(tagged("prql target:sql.duckdb\nfrom x\n"), [ + ["prql", t.keyword.toString()], + ]); + assert.deepStrictEqual(tagged("module m {\nfrom x\n}\n"), [ + ["module", t.moduleKeyword.toString()], + ["{", t.brace.toString()], + ["}", t.brace.toString()], + ]); + assert.deepStrictEqual(tagged("let a = (from x)\n"), [ + ["let", t.definitionKeyword.toString()], + ["(", t.paren.toString()], + [")", t.paren.toString()], + ]); + assert.deepStrictEqual(tagged("from x\nderive b = case {a => 1}\n"), [ + ["case", t.controlKeyword.toString()], + ["{", t.brace.toString()], + ["}", t.brace.toString()], + ]); + assert.deepStrictEqual(tagged("from x\nfilter a in b\n"), [ + ["in", t.operatorKeyword.toString()], + ]); + }); + + it("tags brackets, braces and separators", () => { + assert.deepStrictEqual(tagged("from x\nselect {a, b}\n"), [ + ["{", t.brace.toString()], + [",", t.separator.toString()], + ["}", t.brace.toString()], + ]); + assert.deepStrictEqual(tagged("from x\nderive c = [1]\n"), [ + ["[", t.squareBracket.toString()], + ["]", t.squareBracket.toString()], + ]); + assert.deepStrictEqual(tagged("from x | select a\n"), [ + ["|", t.separator.toString()], + ]); + }); + + // The snippet cases above pin the tag each token carries, but only for the + // tokens they contain; this one covers every selector, including any added + // later for a token no snippet happens to use. + it("has no selector that matches no term in the grammar", () => { + const terms = new Set(parser.nodeSet.types.map((type) => type.name)); + // A key holds space-separated selectors, each a `/`-separated path, with + // `...` standing for any intervening nodes and a trailing `!` or `*` + // limiting how far down the tag applies. + const selectorTerms = Object.keys(prqlHighlightSpec) + .flatMap((key) => key.split(" ")) + .flatMap((selector) => selector.split("/")) + .map((part) => part.replace(/[!*]$/, "")) + .filter((part) => part !== "" && part !== "..."); + + assert.deepStrictEqual( + selectorTerms.filter((term) => !terms.has(term)), + [], + ); + }); +});