0.1.3

TextMateSharp's lists of strings are System.String now that Bjolang's
string is UTF-8, and its token offsets count UTF-16 units, so the lines
are measured in those.

🤖 Generated with [ECA](https://eca.dev)

Co-Authored-By: eca-agent <git@eca.dev>

manifest.bjodat

@@ -1,6 +1,6 @@
 (package
   (name (textmate-bjolang))
-  (version "0.1.2")
+  (version "0.1.3")
   (description "Syntax highlighting for Bjolang: VS Code's TextMate grammars and themes, through TextMateSharp, as (text xml) nodes.")
   (license "MPL-2.0")
 

src/core.bjo

@@ -93,9 +93,9 @@
                          (-> RegistryOptions (System.Collections.Generic.List ClrLanguage))))
   (clr-language-id    (: TextMateSharp.Grammars.Language.Id (-> ClrLanguage string) #:get))
   (clr-language-aliases (: TextMateSharp.Grammars.Language.Aliases
-                           (-> ClrLanguage (System.Collections.Generic.List string)) #:get))
+                           (-> ClrLanguage (System.Collections.Generic.List System.String)) #:get))
   (clr-language-extensions (: TextMateSharp.Grammars.Language.Extensions
-                              (-> ClrLanguage (System.Collections.Generic.List string)) #:get))
+                              (-> ClrLanguage (System.Collections.Generic.List System.String)) #:get))
   (clr-scope-of       (: TextMateSharp.Grammars.RegistryOptions.GetScopeByLanguageId
                          (-> RegistryOptions string string)))
   (clr-load-grammar   (: TextMateSharp.Registry.Registry.LoadGrammar (-> Registry string Grammar)))
@@ -113,7 +113,7 @@
   (clr-encoded-state  (: TextMateSharp.Grammars.ITokenizeLineResult2.RuleStack (-> EncodedResult StateStack) #:get))
   (clr-token-start    (: TextMateSharp.Grammars.IToken.StartIndex (-> Token int) #:get))
   (clr-token-scopes   (: TextMateSharp.Grammars.IToken.Scopes
-                         (-> Token (System.Collections.Generic.List string)) #:get))
+                         (-> Token (System.Collections.Generic.List System.String)) #:get))
   (clr-meta-foreground (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetForeground (-> int int)))
   (clr-meta-background (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetBackground (-> int int)))
   (clr-meta-font-style (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetFontStyle (-> int FontStyle)))
@@ -121,16 +121,16 @@
   ;; --- the theme ---
   (clr-theme-color    (: TextMateSharp.Themes.Theme.GetColor (-> Theme int string)))
   (clr-theme-colours  (: TextMateSharp.Themes.Theme.GetColorMap
-                         (-> Theme (System.Collections.Generic.ICollection string))))
+                         (-> Theme (System.Collections.Generic.ICollection System.String))))
-  (clr-theme-gui      (: TextMateSharp.Themes.Theme.GetGuiColorDictionary (-> Theme (GuiColours string string))))
+  (clr-theme-gui      (: TextMateSharp.Themes.Theme.GetGuiColorDictionary (-> Theme (GuiColours System.String System.String))))
   (clr-theme-match    (: TextMateSharp.Themes.Theme.Match
-                         (-> Theme (System.Collections.Generic.IList string)
+                         (-> Theme (System.Collections.Generic.IList System.String)
                              (System.Collections.Generic.List ThemeRule))))
   (clr-rule-foreground (: TextMateSharp.Themes.ThemeTrieElementRule.foreground (-> ThemeRule int) #:get))
   (clr-rule-background (: TextMateSharp.Themes.ThemeTrieElementRule.background (-> ThemeRule int) #:get))
   (clr-rule-font-style (: TextMateSharp.Themes.ThemeTrieElementRule.fontStyle (-> ThemeRule FontStyle) #:get))
   (clr-rule-parents   (: TextMateSharp.Themes.ThemeTrieElementRule.parentScopes
-                         (-> ThemeRule (System.Collections.Generic.List string)) #:get))
+                         (-> ThemeRule (System.Collections.Generic.List System.String)) #:get))
   (clr-rule-depth     (: TextMateSharp.Themes.ThemeTrieElementRule.scopeDepth (-> ThemeRule int) #:get))
   (clr-has-flag       (: System.Enum.HasFlag (-> System.Enum System.Enum bool)))
 
@@ -285,12 +285,12 @@
 
 ;; A .NET list of strings, which TextMateSharp leaves null for none, in lower
 ;; case.
-(: lower-strings (-> (System.Collections.Generic.List string) (Vec string)))
+(: lower-strings (-> (System.Collections.Generic.List System.String) (Vec string)))
 (defun (lower-strings xs)
   (match (cast Object xs)
     ((:is System.Collections.IEnumerable e)
-     (loop (:for s (cast (Seq string) e))
+     (loop (:for s (cast (Seq System.String) e))
-           (:acc (vecing (string-downcase s)))))
+           (:acc (vecing (string-downcase (clr-string->string s))))))
     (_ (vec-empty))))
 
 (: language-table (-> RegistryOptions (Vec LanguageEntry)))
@@ -311,7 +311,7 @@
 
 (: gui-colour (-> Theme string (Option string)))
 (defun (gui-colour theme key)
-  (loop (:for kv (cast (Seq (KeyValuePair string string)) (clr-theme-gui theme)))
+  (loop (:for kv (cast (Seq (KeyValuePair System.String System.String)) (clr-theme-gui theme)))
         (:acc found (folding None (if (= (.-Key kv) key) (Some (.-Value kv)) found)))))
 
 (: colour-at (-> (Vec string) int string))
@@ -397,13 +397,18 @@
   (cond
     ((string-empty? unified) (vec-empty))
     ((string-ends-with? unified "\n")
-     (string-split (clr-substring unified 0 (- (string-length unified) 1)) "\n"))
+     (string-split (clr-substring unified 0 (- (utf16-length unified) 1)) "\n"))
     (else (string-split unified "\n"))))
 
 ;; Where each token's text lies in its line, as (index from to): from its
 ;; start to the next one's, the last to the end. TextMateSharp reads the line
 ;; with a newline added, so a start can lie past the text; and the first
 ;; token starts at 0, whatever it says. A token left with no text is dropped.
+;; TextMateSharp counts in UTF-16 units, and so does `clr-substring`, which
+;; works on the line converted to .NET's string; `string-length` counts bytes.
+(: utf16-length (-> string int))
+(defun (utf16-length s) (.-Length s))
+
 (: token-bounds (-> int int (-> int int) (Vec (Tuple int int int))))
 (defun (token-bounds count len start-of)
   (loop (:for i (range 0 count))
@@ -414,8 +419,8 @@
 
 (: scopes-of (-> Token (Vec string)))
 (defun (scopes-of t)
-  (loop (:for s (cast (Seq string) (clr-token-scopes t)))
+  (loop (:for s (cast (Seq System.String) (clr-token-scopes t)))
-        (:acc (vecing s))))
+        (:acc (vecing (clr-string->string s)))))
 
 (: scoped-line (-> string (Array Token) (Vec TmToken)))
 (defun (scoped-line line tokens)
@@ -424,7 +429,7 @@
                ((Tuple i from to)
                 (TmToken (text (clr-substring line from (- to from)))
                          (scopes (scopes-of (array-ref tokens i)))))))
-           (token-bounds (array-length tokens) (string-length line)
+           (token-bounds (array-length tokens) (utf16-length line)
                          (fun (i) (clr-token-start (array-ref tokens i))))))
 
 ;; Each line of some code, as tokens and their scopes. Needs no theme.
@@ -506,7 +511,7 @@
                   (plain-whitespace
                     (TmSpan (text (clr-substring line from (- to from)))
                             (style (style-of hl (array-ref encoded (+ (* 2 i) 1)))))))))
-             (token-bounds (/ (array-length encoded) 2) (string-length line)
+             (token-bounds (/ (array-length encoded) 2) (utf16-length line)
                            (fun (i) (array-ref encoded (* 2 i)))))))
 
 ;; Each line of some code, as spans coloured by the highlighter's theme.
@@ -622,7 +627,7 @@
 (defun (theme-css hl scope)
   (def rule (loop (:for r (cast (Seq ThemeRule)
                                 (clr-theme-match (record-ref hl theme)
-                                                 (cast (System.Collections.Generic.IList string) #[scope]))))
+                                                 (cast (System.Collections.Generic.IList System.String) #[(string->clr-string scope)]))))
                   (:acc found (folding None (if (and (none? found) (unconditional? r)) (Some r) found)))))
   (def fg (match rule ((Some r) (clr-rule-foreground r)) (None 0)))
   (def bg (match rule ((Some r) (if (> (clr-rule-background r) 2) (clr-rule-background r) 0)) (None 0)))