diff --git a/grammars/bjolang/syntaxes/bjolang.tmLanguage.json b/grammars/bjolang/syntaxes/bjolang.tmLanguage.json index e413851..267ace4 100644 --- a/grammars/bjolang/syntaxes/bjolang.tmLanguage.json +++ b/grammars/bjolang/syntaxes/bjolang.tmLanguage.json @@ -311,7 +311,7 @@ } }, { - "match": "(?<=\\()(spawn/detached|parameterize\\*|with-deadline|with-response|parameterize|syntax-match|syntax-quote|spawn/daemon|with-return|with-cancel|with-shield|record-set!|struct-set!|task->event|record-ref|record-set|struct-ref|struct-set|yield-from|bjoroutine|with-open|spawn-evt|let/mono|when-let|with-run|time-it|unless|letrec|if-let|some->|match|begin|yield|spawn|when|case|else|let\\*|loop|seql|set!|cast|cond|let|seq|try|fun|and|not|bjo|if|do|or)(?![^\\s()\\[\\]{},:\"';])", + "match": "(?<=\\()(please-hoist-i-promise-i-am-not-naughty|spawn/detached|parameterize\\*|with-deadline|with-response|parameterize|syntax-match|syntax-quote|spawn/daemon|with-return|with-cancel|with-shield|record-set!|struct-set!|task->event|record-ref|record-set|struct-ref|struct-set|yield-from|bjoroutine|with-open|spawn-evt|let/mono|when-let|with-run|time-it|try-get|unless|letrec|if-let|some->|update|match|begin|yield|spawn|when|case|else|let\\*|loop|seql|set!|cast|cond|let|seq|try|get|set|fun|and|not|bjo|if|do|or)(?![^\\s()\\[\\]{},:\"';])", "captures": { "1": { "name": "keyword.control.bjolang" diff --git a/manifest.bjodat b/manifest.bjodat index 856fcda..bb543f5 100644 --- a/manifest.bjodat +++ b/manifest.bjodat @@ -1,6 +1,6 @@ (package (name (textmate-bjolang)) - (version "0.1.1") + (version "0.2.0") (description "Syntax highlighting for Bjolang: VS Code's TextMate grammars and themes, through TextMateSharp, as (text xml) nodes.") (license "MPL-2.0") diff --git a/src/core.bjo b/src/core.bjo index c347633..1d6198f 100644 --- a/src/core.bjo +++ b/src/core.bjo @@ -56,7 +56,7 @@ highlight->nodes highlight->node code->node) (import (std prelude)) -(import (only (std set) list->set set-contains?)) +(import (only (std hashset) list->hashset hashset-contains?)) (import (text xml)) ;; --------------------------------------------------------------------------- @@ -93,9 +93,9 @@ (-> RegistryOptions (System.Collections.Generic.List ClrLanguage)))) (clr-language-id (: TextMateSharp.Grammars.Language.Id (-> ClrLanguage string) #:get)) (clr-language-aliases (: TextMateSharp.Grammars.Language.Aliases - (-> ClrLanguage (System.Collections.Generic.List string)) #:get)) + (-> ClrLanguage (System.Collections.Generic.List System.String)) #:get)) (clr-language-extensions (: TextMateSharp.Grammars.Language.Extensions - (-> ClrLanguage (System.Collections.Generic.List string)) #:get)) + (-> ClrLanguage (System.Collections.Generic.List System.String)) #:get)) (clr-scope-of (: TextMateSharp.Grammars.RegistryOptions.GetScopeByLanguageId (-> RegistryOptions string string))) (clr-load-grammar (: TextMateSharp.Registry.Registry.LoadGrammar (-> Registry string Grammar))) @@ -113,7 +113,7 @@ (clr-encoded-state (: TextMateSharp.Grammars.ITokenizeLineResult2.RuleStack (-> EncodedResult StateStack) #:get)) (clr-token-start (: TextMateSharp.Grammars.IToken.StartIndex (-> Token int) #:get)) (clr-token-scopes (: TextMateSharp.Grammars.IToken.Scopes - (-> Token (System.Collections.Generic.List string)) #:get)) + (-> Token (System.Collections.Generic.List System.String)) #:get)) (clr-meta-foreground (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetForeground (-> int int))) (clr-meta-background (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetBackground (-> int int))) (clr-meta-font-style (: TextMateSharp.Internal.Grammars.EncodedTokenAttributes.GetFontStyle (-> int FontStyle))) @@ -121,16 +121,16 @@ ;; --- the theme --- (clr-theme-color (: TextMateSharp.Themes.Theme.GetColor (-> Theme int string))) (clr-theme-colours (: TextMateSharp.Themes.Theme.GetColorMap - (-> Theme (System.Collections.Generic.ICollection string)))) - (clr-theme-gui (: TextMateSharp.Themes.Theme.GetGuiColorDictionary (-> Theme (GuiColours string string)))) + (-> Theme (System.Collections.Generic.ICollection System.String)))) + (clr-theme-gui (: TextMateSharp.Themes.Theme.GetGuiColorDictionary (-> Theme (GuiColours System.String System.String)))) (clr-theme-match (: TextMateSharp.Themes.Theme.Match - (-> Theme (System.Collections.Generic.IList string) + (-> Theme (System.Collections.Generic.IList System.String) (System.Collections.Generic.List ThemeRule)))) (clr-rule-foreground (: TextMateSharp.Themes.ThemeTrieElementRule.foreground (-> ThemeRule int) #:get)) (clr-rule-background (: TextMateSharp.Themes.ThemeTrieElementRule.background (-> ThemeRule int) #:get)) (clr-rule-font-style (: TextMateSharp.Themes.ThemeTrieElementRule.fontStyle (-> ThemeRule FontStyle) #:get)) (clr-rule-parents (: TextMateSharp.Themes.ThemeTrieElementRule.parentScopes - (-> ThemeRule (System.Collections.Generic.List string)) #:get)) + (-> ThemeRule (System.Collections.Generic.List System.String)) #:get)) (clr-rule-depth (: TextMateSharp.Themes.ThemeTrieElementRule.scopeDepth (-> ThemeRule int) #:get)) (clr-has-flag (: System.Enum.HasFlag (-> System.Enum System.Enum bool))) @@ -156,31 +156,33 @@ ;; --------------------------------------------------------------------------- ;; VS Code's themes, the ones TextMateSharp carries: #:theme 'dark-plus. -(type (: TmTheme (Union (: ThemeDarkPlus #:tag dark-plus) - (: ThemeLightPlus #:tag light-plus) - (: ThemeDark #:tag dark) - (: ThemeLight #:tag light) - (: ThemeMonokai #:tag monokai) - (: ThemeDimmedMonokai #:tag dimmed-monokai) - (: ThemeOneDark #:tag one-dark) - (: ThemeAtomOneDark #:tag atom-one-dark) - (: ThemeAtomOneLight #:tag atom-one-light) - (: ThemeDracula #:tag dracula) - (: ThemeSolarizedDark #:tag solarized-dark) - (: ThemeSolarizedLight #:tag solarized-light) - (: ThemeQuietLight #:tag quiet-light) - (: ThemeKimbieDark #:tag kimbie-dark) - (: ThemeTomorrowNightBlue #:tag tomorrow-night-blue) - (: ThemeAbyss #:tag abyss) - (: ThemeRed #:tag red) - (: ThemeHighContrastDark #:tag high-contrast-dark) - (: ThemeHighContrastLight #:tag high-contrast-light) - (: ThemeVisualStudioDark #:tag visual-studio-dark) - (: ThemeVisualStudioLight #:tag visual-studio-light)))) +(def/grammar TmTheme + (:or (: ThemeDarkPlus 'dark-plus) + (: ThemeLightPlus 'light-plus) + (: ThemeDark 'dark) + (: ThemeLight 'light) + (: ThemeMonokai 'monokai) + (: ThemeDimmedMonokai 'dimmed-monokai) + (: ThemeOneDark 'one-dark) + (: ThemeAtomOneDark 'atom-one-dark) + (: ThemeAtomOneLight 'atom-one-light) + (: ThemeDracula 'dracula) + (: ThemeSolarizedDark 'solarized-dark) + (: ThemeSolarizedLight 'solarized-light) + (: ThemeQuietLight 'quiet-light) + (: ThemeKimbieDark 'kimbie-dark) + (: ThemeTomorrowNightBlue 'tomorrow-night-blue) + (: ThemeAbyss 'abyss) + (: ThemeRed 'red) + (: ThemeHighContrastDark 'high-contrast-dark) + (: ThemeHighContrastLight 'high-contrast-light) + (: ThemeVisualStudioDark 'visual-studio-dark) + (: ThemeVisualStudioLight 'visual-studio-light))) ;; How a node is coloured: #:style 'inline or #:style 'classes. -(type (: TmStyleMode (Union (: StyleInline #:tag inline) - (: StyleClasses #:tag classes)))) +(def/grammar TmStyleMode + (:or (: StyleInline 'inline) + (: StyleClasses 'classes))) ;; How a piece of text looks. None is the theme's own colour, which the pre ;; around the code already has. @@ -285,12 +287,12 @@ ;; A .NET list of strings, which TextMateSharp leaves null for none, in lower ;; case. -(: lower-strings (-> (System.Collections.Generic.List string) (Vec string))) +(: lower-strings (-> (System.Collections.Generic.List System.String) (Vec string))) (defun (lower-strings xs) (match (cast Object xs) ((:is System.Collections.IEnumerable e) - (loop (:for s (cast (Seq string) e)) - (:acc (vecing (string-downcase s))))) + (loop (:for s (cast (Seq System.String) e)) + (:acc (vecing (string-downcase (clr-string->string s)))))) (_ (vec-empty)))) (: language-table (-> RegistryOptions (Vec LanguageEntry))) @@ -311,7 +313,7 @@ (: gui-colour (-> Theme string (Option string))) (defun (gui-colour theme key) - (loop (:for kv (cast (Seq (KeyValuePair string string)) (clr-theme-gui theme))) + (loop (:for kv (cast (Seq (KeyValuePair System.String System.String)) (clr-theme-gui theme))) (:acc found (folding None (if (= (.-Key kv) key) (Some (.-Value kv)) found))))) (: colour-at (-> (Vec string) int string)) @@ -397,13 +399,18 @@ (cond ((string-empty? unified) (vec-empty)) ((string-ends-with? unified "\n") - (string-split (clr-substring unified 0 (- (string-length unified) 1)) "\n")) + (string-split (clr-substring unified 0 (- (utf16-length unified) 1)) "\n")) (else (string-split unified "\n")))) ;; Where each token's text lies in its line, as (index from to): from its ;; start to the next one's, the last to the end. TextMateSharp reads the line ;; with a newline added, so a start can lie past the text; and the first ;; token starts at 0, whatever it says. A token left with no text is dropped. +;; TextMateSharp counts in UTF-16 units, and so does `clr-substring`, which +;; works on the line converted to .NET's string; `string-length` counts bytes. +(: utf16-length (-> string int)) +(defun (utf16-length s) (.-Length s)) + (: token-bounds (-> int int (-> int int) (Vec (Tuple int int int)))) (defun (token-bounds count len start-of) (loop (:for i (range 0 count)) @@ -414,8 +421,8 @@ (: scopes-of (-> Token (Vec string))) (defun (scopes-of t) - (loop (:for s (cast (Seq string) (clr-token-scopes t))) - (:acc (vecing s)))) + (loop (:for s (cast (Seq System.String) (clr-token-scopes t))) + (:acc (vecing (clr-string->string s))))) (: scoped-line (-> string (Array Token) (Vec TmToken))) (defun (scoped-line line tokens) @@ -424,7 +431,7 @@ ((Tuple i from to) (TmToken (text (clr-substring line from (- to from))) (scopes (scopes-of (array-ref tokens i))))))) - (token-bounds (array-length tokens) (string-length line) + (token-bounds (array-length tokens) (utf16-length line) (fun (i) (clr-token-start (array-ref tokens i)))))) ;; Each line of some code, as tokens and their scopes. Needs no theme. @@ -506,7 +513,7 @@ (plain-whitespace (TmSpan (text (clr-substring line from (- to from))) (style (style-of hl (array-ref encoded (+ (* 2 i) 1))))))))) - (token-bounds (/ (array-length encoded) 2) (string-length line) + (token-bounds (/ (array-length encoded) 2) (utf16-length line) (fun (i) (array-ref encoded (* 2 i))))))) ;; Each line of some code, as spans coloured by the highlighter's theme. @@ -553,8 +560,8 @@ "entity.name.variable" "entity.other.inherited-class" "entity.other.attribute-name" "punctuation.section.embedded")) -(: class-vocabulary (Set string)) -(def class-vocabulary (list->set class-scopes)) +(: class-vocabulary (HashSet string)) +(def class-vocabulary (list->hashset class-scopes)) ;; The class scope a grammar scope comes under: the longest one it begins ;; with, a whole segment at a time. None for a scope that says nothing about @@ -569,7 +576,7 @@ (loop (:for k (range 1 (+ (min 3 (vec-length segments)) 1))) (:acc found (folding None (let ((prefix (string-join (vec-slice segments 0 k) "."))) - (if (set-contains? class-vocabulary prefix) (Some prefix) found)))))))) + (if (hashset-contains? class-vocabulary prefix) (Some prefix) found)))))))) (: class-name (-> string string)) (defun (class-name scope) (str "tm-" (string-replace scope "." "-"))) @@ -622,7 +629,7 @@ (defun (theme-css hl scope) (def rule (loop (:for r (cast (Seq ThemeRule) (clr-theme-match (record-ref hl theme) - (cast (System.Collections.Generic.IList string) #[scope])))) + (cast (System.Collections.Generic.IList System.String) #[(string->clr-string scope)])))) (:acc found (folding None (if (and (none? found) (unconditional? r)) (Some r) found))))) (def fg (match rule ((Some r) (clr-rule-foreground r)) (None 0))) (def bg (match rule ((Some r) (if (> (clr-rule-background r) 2) (clr-rule-background r) 0)) (None 0))) diff --git a/tests/demo.bjo b/tests/demo.bjo index aba0bb9..669b618 100644 --- a/tests/demo.bjo +++ b/tests/demo.bjo @@ -149,7 +149,7 @@ " (code (@ (class \"language-csharp\")) (span (@ (style \"color:#B5CEA8\")) \"1\")))")) (expect "several lines, a newline between them" (->str (highlight->nodes cs "1\n2" #:style 'classes)) - (str "'((span (@ (class \"tm-constant tm-constant-numeric\")) \"1\") \"\\n\"" + (str "((span (@ (class \"tm-constant tm-constant-numeric\")) \"1\") \"\\n\"" " (span (@ (class \"tm-constant tm-constant-numeric\")) \"2\"))")) ;; --- the stylesheet ---------------------------------------------------------