The demo test expects a list as it is printed

The check of several lines wanted the nodes as '((span ...) ...). ->str
writes a list without a quote, so the check failed. It wants
((span ...) ...) now.

🤖 Generated with [ECA](https://eca.dev) (anthropic/claude-opus-4-7)

Co-Authored-By: eca-agent <git@eca.dev>

grammars/bjolang/syntaxes/bjolang.tmLanguage.json

@@ -311,7 +311,7 @@
           }
         },
         {
-          "match": "(?<=\\()(spawn/detached|parameterize\\*|with-deadline|with-response|parameterize|syntax-match|syntax-quote|spawn/daemon|with-return|with-cancel|with-shield|record-set!|struct-set!|task->event|record-ref|record-set|struct-ref|struct-set|yield-from|bjoroutine|with-open|spawn-evt|let/mono|when-let|with-run|time-it|unless|letrec|if-let|some->|match|begin|yield|spawn|when|case|else|let\\*|loop|seql|set!|cast|cond|let|seq|try|fun|and|not|bjo|if|do|or)(?![^\\s()\\[\\]{},:\"';])",
+          "match": "(?<=\\()(please-hoist-i-promise-i-am-not-naughty|spawn/detached|parameterize\\*|with-deadline|with-response|parameterize|syntax-match|syntax-quote|spawn/daemon|with-return|with-cancel|with-shield|record-set!|struct-set!|task->event|record-ref|record-set|struct-ref|struct-set|yield-from|bjoroutine|with-open|spawn-evt|let/mono|when-let|with-run|time-it|try-get|unless|letrec|if-let|some->|update|match|begin|yield|spawn|when|case|else|let\\*|loop|seql|set!|cast|cond|let|seq|try|get|set|fun|and|not|bjo|if|do|or)(?![^\\s()\\[\\]{},:\"';])",
           "captures": {
             "1": {
               "name": "keyword.control.bjolang"

manifest.bjodat

@@ -1,6 +1,6 @@
 (package
   (name (textmate-bjolang))
-  (version "0.1.1")
+  (version "0.2.0")
   (description "Syntax highlighting for Bjolang: VS Code's TextMate grammars and themes, through TextMateSharp, as (text xml) nodes.")
   (license "MPL-2.0")
 

src/core.bjo

@@ -56,7 +56,7 @@
   highlight->nodes highlight->node code->node)
 
 (import (std prelude))
-(import (only (std set) list->set set-contains?))
+(import (only (std hashset) list->hashset hashset-contains?))
 (import (text xml))
 
 ;; ---------------------------------------------------------------------------
@@ -93,9 +93,9 @@
                          (-> RegistryOptions (System.Collections.Generic.List ClrLanguage))))
   (clr-language-id    (: TextMateSharp.Grammars.Language.Id (-> ClrLanguage string) #:get))
   (clr-language-aliases (: TextMateSharp.Grammars.Language.Aliases
-                           (-> ClrLanguage (System.Collections.Generic.List string)) #:get))
+                           (-> ClrLanguage (System.Collections.Generic.List System.String)) #:get))
   (clr-language-extensions (: TextMateSharp.Grammars.Language.Extensions
-                              (-> ClrLanguage (System.Collections.Generic.List string)) #:get))
+                              (-> ClrLanguage (System.Collections.Generic.List System.String)) #:get))
   (clr-scope-of       (: TextMateSharp.Grammars.RegistryOptions.GetScopeByLanguageId
                          (-> RegistryOptions string string)))
   (clr-load-grammar   (: TextMateSharp.Registry.Registry.LoadGrammar (-> Registry string Grammar)))
@@ -113,7 +113,7 @@
   (clr-encoded-state  (: TextMateSharp.Grammars.ITokenizeLineResult2.RuleStack (-> EncodedResult StateStack) #:get))
   (clr-token-start    (: TextMateSharp.Grammars.IToken.StartIndex (-> Token int) #:get))
   (clr-token-scopes   (: TextMateSharp.Grammars.IToken.Scopes
-                         (-> Token (System.Collections.Generic.List string)) #:get))
+                         (-> Token (System.Collections.Generic.List System.String)) #:get))
   (clr-meta-foreground (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetForeground (-> int int)))
   (clr-meta-background (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetBackground (-> int int)))
   (clr-meta-font-style (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetFontStyle (-> int FontStyle)))
@@ -121,16 +121,16 @@
   ;; --- the theme ---
   (clr-theme-color    (: TextMateSharp.Themes.Theme.GetColor (-> Theme int string)))
   (clr-theme-colours  (: TextMateSharp.Themes.Theme.GetColorMap
-                         (-> Theme (System.Collections.Generic.ICollection string))))
+                         (-> Theme (System.Collections.Generic.ICollection System.String))))
-  (clr-theme-gui      (: TextMateSharp.Themes.Theme.GetGuiColorDictionary (-> Theme (GuiColours string string))))
+  (clr-theme-gui      (: TextMateSharp.Themes.Theme.GetGuiColorDictionary (-> Theme (GuiColours System.String System.String))))
   (clr-theme-match    (: TextMateSharp.Themes.Theme.Match
-                         (-> Theme (System.Collections.Generic.IList string)
+                         (-> Theme (System.Collections.Generic.IList System.String)
                              (System.Collections.Generic.List ThemeRule))))
   (clr-rule-foreground (: TextMateSharp.Themes.ThemeTrieElementRule.foreground (-> ThemeRule int) #:get))
   (clr-rule-background (: TextMateSharp.Themes.ThemeTrieElementRule.background (-> ThemeRule int) #:get))
   (clr-rule-font-style (: TextMateSharp.Themes.ThemeTrieElementRule.fontStyle (-> ThemeRule FontStyle) #:get))
   (clr-rule-parents   (: TextMateSharp.Themes.ThemeTrieElementRule.parentScopes
-                         (-> ThemeRule (System.Collections.Generic.List string)) #:get))
+                         (-> ThemeRule (System.Collections.Generic.List System.String)) #:get))
   (clr-rule-depth     (: TextMateSharp.Themes.ThemeTrieElementRule.scopeDepth (-> ThemeRule int) #:get))
   (clr-has-flag       (: System.Enum.HasFlag (-> System.Enum System.Enum bool)))
 
@@ -156,31 +156,33 @@
 ;; ---------------------------------------------------------------------------
 
 ;; VS Code's themes, the ones TextMateSharp carries: #:theme 'dark-plus.
-(type (: TmTheme (Union (: ThemeDarkPlus #:tag dark-plus)
+(def/grammar TmTheme
-                        (: ThemeLightPlus #:tag light-plus)
+  (:or (: ThemeDarkPlus 'dark-plus)
-                        (: ThemeDark #:tag dark)
+       (: ThemeLightPlus 'light-plus)
-                        (: ThemeLight #:tag light)
+       (: ThemeDark 'dark)
-                        (: ThemeMonokai #:tag monokai)
+       (: ThemeLight 'light)
-                        (: ThemeDimmedMonokai #:tag dimmed-monokai)
+       (: ThemeMonokai 'monokai)
-                        (: ThemeOneDark #:tag one-dark)
+       (: ThemeDimmedMonokai 'dimmed-monokai)
-                        (: ThemeAtomOneDark #:tag atom-one-dark)
+       (: ThemeOneDark 'one-dark)
-                        (: ThemeAtomOneLight #:tag atom-one-light)
+       (: ThemeAtomOneDark 'atom-one-dark)
-                        (: ThemeDracula #:tag dracula)
+       (: ThemeAtomOneLight 'atom-one-light)
-                        (: ThemeSolarizedDark #:tag solarized-dark)
+       (: ThemeDracula 'dracula)
-                        (: ThemeSolarizedLight #:tag solarized-light)
+       (: ThemeSolarizedDark 'solarized-dark)
-                        (: ThemeQuietLight #:tag quiet-light)
+       (: ThemeSolarizedLight 'solarized-light)
-                        (: ThemeKimbieDark #:tag kimbie-dark)
+       (: ThemeQuietLight 'quiet-light)
-                        (: ThemeTomorrowNightBlue #:tag tomorrow-night-blue)
+       (: ThemeKimbieDark 'kimbie-dark)
-                        (: ThemeAbyss #:tag abyss)
+       (: ThemeTomorrowNightBlue 'tomorrow-night-blue)
-                        (: ThemeRed #:tag red)
+       (: ThemeAbyss 'abyss)
-                        (: ThemeHighContrastDark #:tag high-contrast-dark)
+       (: ThemeRed 'red)
-                        (: ThemeHighContrastLight #:tag high-contrast-light)
+       (: ThemeHighContrastDark 'high-contrast-dark)
-                        (: ThemeVisualStudioDark #:tag visual-studio-dark)
+       (: ThemeHighContrastLight 'high-contrast-light)
-                        (: ThemeVisualStudioLight #:tag visual-studio-light))))
+       (: ThemeVisualStudioDark 'visual-studio-dark)
+       (: ThemeVisualStudioLight 'visual-studio-light)))
 
 ;; How a node is coloured: #:style 'inline or #:style 'classes.
-(type (: TmStyleMode (Union (: StyleInline #:tag inline)
+(def/grammar TmStyleMode
-                            (: StyleClasses #:tag classes))))
+  (:or (: StyleInline 'inline)
+       (: StyleClasses 'classes)))
 
 ;; How a piece of text looks. None is the theme's own colour, which the pre
 ;; around the code already has.
@@ -285,12 +287,12 @@
 
 ;; A .NET list of strings, which TextMateSharp leaves null for none, in lower
 ;; case.
-(: lower-strings (-> (System.Collections.Generic.List string) (Vec string)))
+(: lower-strings (-> (System.Collections.Generic.List System.String) (Vec string)))
 (defun (lower-strings xs)
   (match (cast Object xs)
     ((:is System.Collections.IEnumerable e)
-     (loop (:for s (cast (Seq string) e))
+     (loop (:for s (cast (Seq System.String) e))
-           (:acc (vecing (string-downcase s)))))
+           (:acc (vecing (string-downcase (clr-string->string s))))))
     (_ (vec-empty))))
 
 (: language-table (-> RegistryOptions (Vec LanguageEntry)))
@@ -311,7 +313,7 @@
 
 (: gui-colour (-> Theme string (Option string)))
 (defun (gui-colour theme key)
-  (loop (:for kv (cast (Seq (KeyValuePair string string)) (clr-theme-gui theme)))
+  (loop (:for kv (cast (Seq (KeyValuePair System.String System.String)) (clr-theme-gui theme)))
         (:acc found (folding None (if (= (.-Key kv) key) (Some (.-Value kv)) found)))))
 
 (: colour-at (-> (Vec string) int string))
@@ -397,13 +399,18 @@
   (cond
     ((string-empty? unified) (vec-empty))
     ((string-ends-with? unified "\n")
-     (string-split (clr-substring unified 0 (- (string-length unified) 1)) "\n"))
+     (string-split (clr-substring unified 0 (- (utf16-length unified) 1)) "\n"))
     (else (string-split unified "\n"))))
 
 ;; Where each token's text lies in its line, as (index from to): from its
 ;; start to the next one's, the last to the end. TextMateSharp reads the line
 ;; with a newline added, so a start can lie past the text; and the first
 ;; token starts at 0, whatever it says. A token left with no text is dropped.
+;; TextMateSharp counts in UTF-16 units, and so does `clr-substring`, which
+;; works on the line converted to .NET's string; `string-length` counts bytes.
+(: utf16-length (-> string int))
+(defun (utf16-length s) (.-Length s))
+
 (: token-bounds (-> int int (-> int int) (Vec (Tuple int int int))))
 (defun (token-bounds count len start-of)
   (loop (:for i (range 0 count))
@@ -414,8 +421,8 @@
 
 (: scopes-of (-> Token (Vec string)))
 (defun (scopes-of t)
-  (loop (:for s (cast (Seq string) (clr-token-scopes t)))
+  (loop (:for s (cast (Seq System.String) (clr-token-scopes t)))
-        (:acc (vecing s))))
+        (:acc (vecing (clr-string->string s)))))
 
 (: scoped-line (-> string (Array Token) (Vec TmToken)))
 (defun (scoped-line line tokens)
@@ -424,7 +431,7 @@
                ((Tuple i from to)
                 (TmToken (text (clr-substring line from (- to from)))
                          (scopes (scopes-of (array-ref tokens i)))))))
-           (token-bounds (array-length tokens) (string-length line)
+           (token-bounds (array-length tokens) (utf16-length line)
                          (fun (i) (clr-token-start (array-ref tokens i))))))
 
 ;; Each line of some code, as tokens and their scopes. Needs no theme.
@@ -506,7 +513,7 @@
                   (plain-whitespace
                     (TmSpan (text (clr-substring line from (- to from)))
                             (style (style-of hl (array-ref encoded (+ (* 2 i) 1)))))))))
-             (token-bounds (/ (array-length encoded) 2) (string-length line)
+             (token-bounds (/ (array-length encoded) 2) (utf16-length line)
                            (fun (i) (array-ref encoded (* 2 i)))))))
 
 ;; Each line of some code, as spans coloured by the highlighter's theme.
@@ -553,8 +560,8 @@
         "entity.name.variable" "entity.other.inherited-class" "entity.other.attribute-name"
         "punctuation.section.embedded"))
 
-(: class-vocabulary (Set string))
+(: class-vocabulary (HashSet string))
-(def class-vocabulary (list->set class-scopes))
+(def class-vocabulary (list->hashset class-scopes))
 
 ;; The class scope a grammar scope comes under: the longest one it begins
 ;; with, a whole segment at a time. None for a scope that says nothing about
@@ -569,7 +576,7 @@
         (loop (:for k (range 1 (+ (min 3 (vec-length segments)) 1)))
               (:acc found (folding None
                                    (let ((prefix (string-join (vec-slice segments 0 k) ".")))
-                                     (if (set-contains? class-vocabulary prefix) (Some prefix) found))))))))
+                                     (if (hashset-contains? class-vocabulary prefix) (Some prefix) found))))))))
 
 (: class-name (-> string string))
 (defun (class-name scope) (str "tm-" (string-replace scope "." "-")))
@@ -622,7 +629,7 @@
 (defun (theme-css hl scope)
   (def rule (loop (:for r (cast (Seq ThemeRule)
                                 (clr-theme-match (record-ref hl theme)
-                                                 (cast (System.Collections.Generic.IList string) #[scope]))))
+                                                 (cast (System.Collections.Generic.IList System.String) #[(string->clr-string scope)]))))
                   (:acc found (folding None (if (and (none? found) (unconditional? r)) (Some r) found)))))
   (def fg (match rule ((Some r) (clr-rule-foreground r)) (None 0)))
   (def bg (match rule ((Some r) (if (> (clr-rule-background r) 2) (clr-rule-background r) 0)) (None 0)))

tests/demo.bjo

@@ -149,7 +149,7 @@
                " (code (@ (class \"language-csharp\")) (span (@ (style \"color:#B5CEA8\")) \"1\")))"))
   (expect "several lines, a newline between them"
           (->str (highlight->nodes cs "1\n2" #:style 'classes))
-          (str "'((span (@ (class \"tm-constant tm-constant-numeric\")) \"1\") \"\\n\""
+          (str "((span (@ (class \"tm-constant tm-constant-numeric\")) \"1\") \"\\n\""
                " (span (@ (class \"tm-constant tm-constant-numeric\")) \"2\"))"))
 
   ;; --- the stylesheet ---------------------------------------------------------