diff --git a/manifest.bjodat b/manifest.bjodat index 4573821..cdba4d7 100644 --- a/manifest.bjodat +++ b/manifest.bjodat @@ -1,6 +1,6 @@ (package (name (textmate-bjolang)) - (version "0.1.2") + (version "0.1.3") (description "Syntax highlighting for Bjolang: VS Code's TextMate grammars and themes, through TextMateSharp, as (text xml) nodes.") (license "MPL-2.0") diff --git a/src/core.bjo b/src/core.bjo index bea8ee0..33bc35c 100644 --- a/src/core.bjo +++ b/src/core.bjo @@ -93,9 +93,9 @@ (-> RegistryOptions (System.Collections.Generic.List ClrLanguage)))) (clr-language-id (: TextMateSharp.Grammars.Language.Id (-> ClrLanguage string) #:get)) (clr-language-aliases (: TextMateSharp.Grammars.Language.Aliases - (-> ClrLanguage (System.Collections.Generic.List string)) #:get)) + (-> ClrLanguage (System.Collections.Generic.List System.String)) #:get)) (clr-language-extensions (: TextMateSharp.Grammars.Language.Extensions - (-> ClrLanguage (System.Collections.Generic.List string)) #:get)) + (-> ClrLanguage (System.Collections.Generic.List System.String)) #:get)) (clr-scope-of (: TextMateSharp.Grammars.RegistryOptions.GetScopeByLanguageId (-> RegistryOptions string string))) (clr-load-grammar (: TextMateSharp.Registry.Registry.LoadGrammar (-> Registry string Grammar))) @@ -113,7 +113,7 @@ (clr-encoded-state (: TextMateSharp.Grammars.ITokenizeLineResult2.RuleStack (-> EncodedResult StateStack) #:get)) (clr-token-start (: TextMateSharp.Grammars.IToken.StartIndex (-> Token int) #:get)) (clr-token-scopes (: TextMateSharp.Grammars.IToken.Scopes - (-> Token (System.Collections.Generic.List string)) #:get)) + (-> Token (System.Collections.Generic.List System.String)) #:get)) (clr-meta-foreground (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetForeground (-> int int))) (clr-meta-background (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetBackground (-> int int))) (clr-meta-font-style (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetFontStyle (-> int FontStyle))) @@ -121,16 +121,16 @@ ;; --- the theme --- (clr-theme-color (: TextMateSharp.Themes.Theme.GetColor (-> Theme int string))) (clr-theme-colours (: TextMateSharp.Themes.Theme.GetColorMap - (-> Theme (System.Collections.Generic.ICollection string)))) - (clr-theme-gui (: TextMateSharp.Themes.Theme.GetGuiColorDictionary (-> Theme (GuiColours string string)))) + (-> Theme (System.Collections.Generic.ICollection System.String)))) + (clr-theme-gui (: TextMateSharp.Themes.Theme.GetGuiColorDictionary (-> Theme (GuiColours System.String System.String)))) (clr-theme-match (: TextMateSharp.Themes.Theme.Match - (-> Theme (System.Collections.Generic.IList string) + (-> Theme (System.Collections.Generic.IList System.String) (System.Collections.Generic.List ThemeRule)))) (clr-rule-foreground (: TextMateSharp.Themes.ThemeTrieElementRule.foreground (-> ThemeRule int) #:get)) (clr-rule-background (: TextMateSharp.Themes.ThemeTrieElementRule.background (-> ThemeRule int) #:get)) (clr-rule-font-style (: TextMateSharp.Themes.ThemeTrieElementRule.fontStyle (-> ThemeRule FontStyle) #:get)) (clr-rule-parents (: TextMateSharp.Themes.ThemeTrieElementRule.parentScopes - (-> ThemeRule (System.Collections.Generic.List string)) #:get)) + (-> ThemeRule (System.Collections.Generic.List System.String)) #:get)) (clr-rule-depth (: TextMateSharp.Themes.ThemeTrieElementRule.scopeDepth (-> ThemeRule int) #:get)) (clr-has-flag (: System.Enum.HasFlag (-> System.Enum System.Enum bool))) @@ -285,12 +285,12 @@ ;; A .NET list of strings, which TextMateSharp leaves null for none, in lower ;; case. -(: lower-strings (-> (System.Collections.Generic.List string) (Vec string))) +(: lower-strings (-> (System.Collections.Generic.List System.String) (Vec string))) (defun (lower-strings xs) (match (cast Object xs) ((:is System.Collections.IEnumerable e) - (loop (:for s (cast (Seq string) e)) - (:acc (vecing (string-downcase s))))) + (loop (:for s (cast (Seq System.String) e)) + (:acc (vecing (string-downcase (clr-string->string s)))))) (_ (vec-empty)))) (: language-table (-> RegistryOptions (Vec LanguageEntry))) @@ -311,7 +311,7 @@ (: gui-colour (-> Theme string (Option string))) (defun (gui-colour theme key) - (loop (:for kv (cast (Seq (KeyValuePair string string)) (clr-theme-gui theme))) + (loop (:for kv (cast (Seq (KeyValuePair System.String System.String)) (clr-theme-gui theme))) (:acc found (folding None (if (= (.-Key kv) key) (Some (.-Value kv)) found))))) (: colour-at (-> (Vec string) int string)) @@ -397,13 +397,18 @@ (cond ((string-empty? unified) (vec-empty)) ((string-ends-with? unified "\n") - (string-split (clr-substring unified 0 (- (string-length unified) 1)) "\n")) + (string-split (clr-substring unified 0 (- (utf16-length unified) 1)) "\n")) (else (string-split unified "\n")))) ;; Where each token's text lies in its line, as (index from to): from its ;; start to the next one's, the last to the end. TextMateSharp reads the line ;; with a newline added, so a start can lie past the text; and the first ;; token starts at 0, whatever it says. A token left with no text is dropped. +;; TextMateSharp counts in UTF-16 units, and so does `clr-substring`, which +;; works on the line converted to .NET's string; `string-length` counts bytes. +(: utf16-length (-> string int)) +(defun (utf16-length s) (.-Length s)) + (: token-bounds (-> int int (-> int int) (Vec (Tuple int int int)))) (defun (token-bounds count len start-of) (loop (:for i (range 0 count)) @@ -414,8 +419,8 @@ (: scopes-of (-> Token (Vec string))) (defun (scopes-of t) - (loop (:for s (cast (Seq string) (clr-token-scopes t))) - (:acc (vecing s)))) + (loop (:for s (cast (Seq System.String) (clr-token-scopes t))) + (:acc (vecing (clr-string->string s))))) (: scoped-line (-> string (Array Token) (Vec TmToken))) (defun (scoped-line line tokens) @@ -424,7 +429,7 @@ ((Tuple i from to) (TmToken (text (clr-substring line from (- to from))) (scopes (scopes-of (array-ref tokens i))))))) - (token-bounds (array-length tokens) (string-length line) + (token-bounds (array-length tokens) (utf16-length line) (fun (i) (clr-token-start (array-ref tokens i)))))) ;; Each line of some code, as tokens and their scopes. Needs no theme. @@ -506,7 +511,7 @@ (plain-whitespace (TmSpan (text (clr-substring line from (- to from))) (style (style-of hl (array-ref encoded (+ (* 2 i) 1))))))))) - (token-bounds (/ (array-length encoded) 2) (string-length line) + (token-bounds (/ (array-length encoded) 2) (utf16-length line) (fun (i) (array-ref encoded (* 2 i))))))) ;; Each line of some code, as spans coloured by the highlighter's theme. @@ -622,7 +627,7 @@ (defun (theme-css hl scope) (def rule (loop (:for r (cast (Seq ThemeRule) (clr-theme-match (record-ref hl theme) - (cast (System.Collections.Generic.IList string) #[scope])))) + (cast (System.Collections.Generic.IList System.String) #[(string->clr-string scope)])))) (:acc found (folding None (if (and (none? found) (unconditional? r)) (Some r) found))))) (def fg (match rule ((Some r) (clr-rule-foreground r)) (None 0))) (def bg (match rule ((Some r) (if (> (clr-rule-background r) 2) (clr-rule-background r) 0)) (None 0)))