;; bjo bench — benchmark files built, and run: as they are, from one menu over
;; several, or two versions of one against each other.
;;
;; What `bjo.bjo` owns, building a file and starting a program that has the
;; console, comes in as a `BenchTools`, so that this file does not import that
;; one and that one stays the command line and the compiler processes.
;;
;; A benchmark program is `(std bench)`'s, and what it does by itself — the
;; menu, the table, `--save` and `--compare` — is not repeated here. What is
;; here is what one program cannot do on its own: a menu over several files,
;; and an A/B run, which drives two programs over `--serve` and alternates
;; their samples.

(export BenchProgram BenchTools cmd-bench split-bench-args)

(import (std prelude))
(import (std bench))
(import (std bench store))
(import (std run))
(import (std random))
(import (text bjodat-core))
(import "project.bjo")

(import/extern
  (clr-file-name (: System.IO.Path.GetFileName (-> string string)))
  (clr-temp-path (: System.IO.Path.GetTempPath (-> string)))
  (clr-full-path (: System.IO.Path.GetFullPath (-> string string)))
  (clr-relative-path (: System.IO.Path.GetRelativePath (-> string string string)))
  (clr-change-extension (: System.IO.Path.ChangeExtension (-> string string string)))
  (clr-tick-count (: System.Environment.TickCount64 (-> long) #:get))
  (clr-process-id (: System.Environment.ProcessId (-> int) #:get))
  (console-input-redirected?  (: System.Console.IsInputRedirected (-> bool) #:get))
  (console-output-redirected? (: System.Console.IsOutputRedirected (-> bool) #:get))
  (error-message (: System.Exception.Message (-> Exception string) #:get)))

;; A benchmark file built, ready to run.
(type (: BenchProgram (Record (: source string) (: exe string))))

;; What `bjo.bjo` lends: building a file (an exit code when it fails, said
;; already), running a program on the console, and finding a source file.
(type (: BenchTools (Record (: build (-> string (Result int BenchProgram)))
                            (: exec (-> string (Vec string) int))
                            (: resolve (-> string (Option string))))))

(: (-> BenchProgram string))
(defun (program-exe p) (record-ref p exe))

(: (-> BenchProgram string))
(defun (program-source p) (record-ref p source))

(: (-> string string))
(defun (file-name-of path) (clr-file-name path))

(: (-> void))
(defun (flush!) (flush-port (parameter-ref current-output-port)))

;; ---------------------------------------------------------------------------
;; The arguments
;; ---------------------------------------------------------------------------

;; The flags of a benchmark program, and of an A/B run, that take a value, so
;; that the value is not taken for a file or a name.
(def bench-value-flags
  (list "--save" "--compare" "--params" "--min-samples" "--max-time" "--alpha"
        "--threshold" "--fail-slower" "--cases" "--alloc"
        "--against" "-N" "--processes" "--seed" "--against-warmup" "--against-sample"
        "--against-build"))

(: (-> string bool))
(defun (value-flag? a) (any? (fun (f) (= f a)) bench-value-flags))

;; The benchmark files among the arguments, and what goes to the programs.
(: (-> (-> string (Option string)) (List string) (Tuple (List string) (List string))))
(defun (split-bench-args resolve args)
  (let go ((left args) (files (list)) (passed (list)))
    (match left
      (Nil (Tuple (list-reverse files) (list-reverse passed)))
      ((Cons a more)
       (match (Tuple (value-flag? a) more)
         ((Tuple #t (Cons v after)) (go after files (cons v (cons a passed))))
         (_
          (match (if (string-ends-with? a ".bjo") (resolve a) None)
            ((Some file) (go more (cons file files) passed))
            (None (go more files (cons a passed))))))))))

;; Whether the arguments leave the choice to a menu: nothing named, and no
;; flag that says what to run or that the program is not run for a person.
(: (-> (List string) bool))
(defun (bench-menu-args? passed)
  (let go ((left passed))
    (match left
      (Nil #t)
      ((Cons a more)
       (cond
         ((value-flag? a) (go (match more ((Cons _ after) after) (Nil Nil))))
         ((or (not (string-starts-with? a "-"))
              (any? (fun (f) (= f a)) (list "--all" "--list" "--serve")))
          #f)
         (else (go more)))))))

(: (-> bool))
(defun (bench-at-terminal?)
  (or (= (get-environment-variable "BJO_BENCH_MENU") (Some "1"))
      (and (not console-input-redirected?) (not console-output-redirected?)
           (none? (get-environment-variable "CI")))))

(: (-> (List string) string bool))
(defun (has-flag? args flag) (any? (fun (a) (= a flag)) args))

;; ---------------------------------------------------------------------------
;; The command
;; ---------------------------------------------------------------------------

;; `bjo bench`: each benchmark file built in Release and run with the
;; arguments that are not files. One file is run as it is: at a terminal it
;; asks what to run itself. Several files, at a terminal and with nothing
;; named, get one menu of all their benchmarks, numbered across the files
;; under a heading for each; each file is then run with `--cases` and the
;; cases of the choice that are its own. Otherwise each file is run in turn
;; with the arguments, a table each. With `--against REF`, the one file is
;; run against itself as it is at `REF`.
(: cmd-bench #:sync (-> BenchTools (List string) (List string) int))
(defun (cmd-bench tools files passed)
  (with-return ret
    (when (has-flag? passed "--against")
      (ret (match files
             ((Cons file Nil) (cmd-against tools file passed))
             (_ (fail "bjo bench: --against compares one benchmark file; name one.")))))
    (def programs
      (loop (:for file files)
            (:let built ((record-ref tools build) file))
            (:do (match built ((Err status) (ret status)) (_ unit)))
            (:when-let (Ok p) built)
            (:acc out (listing p))))
    (def exec (record-ref tools exec))
    (match programs
      ((Cons p Nil) (exec "dotnet" (list->vec (cons (program-exe p) passed))))
      (_
       (when (or (has-flag? passed "--save") (has-flag? passed "--compare"))
         (ret (fail "bjo bench: --save and --compare take one benchmark file; name one.")))
       (if (and (bench-menu-args? passed) (bench-at-terminal?))
           (bench-files-menu tools programs passed)
           (let ()
             (def statuses
               (loop (:for p programs)
                     (:do (println "")
                          (println #"bjo bench: ${(file-name-of (program-source p))}")
                          (flush!))
                     (:acc out (listing (exec "dotnet" (list->vec (cons (program-exe p) passed)))))))
             (if (any? (fun (s) (not (= s 0))) statuses) 1 0)))))))

;; ---------------------------------------------------------------------------
;; What a program has
;; ---------------------------------------------------------------------------

;; One listed case: its number, its benchmark's name, labels, group, `once`
;; and skip.
(type (: Listed (Record (: index int)
                        (: name string)
                        (: labels (List (Tuple string string)))
                        (: group string)
                        (: once int)
                        (: skip string))))

(: (-> Listed string))
(defun (listed-name l) (record-ref l name))

(: (-> Listed (List (Tuple string string))))
(defun (listed-labels l) (record-ref l labels))

(: (-> Listed int))
(defun (listed-index l) (record-ref l index))

(: (-> Listed string))
(defun (listed-skip l) (record-ref l skip))

;; What a case is known by on both sides: its name and its parameters.
(: (-> Listed string))
(defun (listed-key l)
  (string-join (list->vec (cons (listed-name l)
                                (list-map (fun (t) (match t ((Tuple p v) (str p "=" v)))) (listed-labels l))))
               " "))

(: (-> (List Bjodat) string (Option Bjodat)))
(defun (clause-of clauses name)
  (match clauses
    ((Cons (BjoList (Cons (BjoSym s) (Cons v Nil))) more)
     (if (= (symbol->string s) name) (Some v) (clause-of more name)))
    ((Cons _ more) (clause-of more name))
    (Nil None)))

(: (-> Bjodat (Option Listed)))
(defun (listed-of d)
  (match d
    ((BjoList (Cons (BjoSym _) (Cons (BjoInt i) clauses)))
     (def (Some (BjoStr name)) (clause-of clauses "name") :leave-with None)
     (def labels
       (match (find (fun (c) (match c ((BjoList (Cons (BjoSym s) _)) (= (symbol->string s) "labels")) (_ #f)))
                    clauses)
         ((Some (BjoList (Cons _ pairs)))
          (loop (:for p pairs)
                (:acc (listing (match p
                                 ((BjoList (Cons (BjoStr a) (Cons (BjoStr b) Nil))) (Tuple a b))
                                 (_ (Tuple "" "")))))))
         (_ (list))))
     (Some (Listed (index (cast System.Int32 i)) (name name) (labels labels)
                   (group (match (clause-of clauses "group") ((Some (BjoStr g)) g) (_ "")))
                   (once (match (clause-of clauses "once") ((Some (BjoInt n)) (cast System.Int32 n)) (_ 0)))
                   (skip (match (clause-of clauses "skip") ((Some (BjoStr s)) s) (_ ""))))))
    (_ None)))

;; The cases of a `(cases ...)` reply.
(: (-> Bjodat (Option (List Listed))))
(defun (listed-cases-of reply)
  (match reply
    ((BjoList (Cons (BjoSym head) cases))
     (if (= (symbol->string head) "cases")
         (Some (loop (:for c cases) (:when-let (Some l) (listed-of c)) (:acc ls (listing l))))
         None))
    (_ None)))

;; What a benchmark program has, from its `--serve` `(list)`, asked once.
(: listed-cases #:sync (-> BenchProgram (List string) (Result string (List Listed))))
(defun (listed-cases p passed)
  (def commands (str (program-exe p) ".serve-input"))
  (file-write-text commands "(list)\n(quit)\n")
  (def answer (run/output '(from-file ,commands (dotnet [,(program-exe p) "--serve" ,@(list->vec passed)]))))
  (file-delete commands)
  (match answer
    ((Ok (Tuple 0 text))
     (match (parse-bjodat (string-trim text))
       ((Ok reply)
        (match (listed-cases-of reply)
          ((Some ls) (Ok ls))
          (None (Err #"bjo bench: ${(program-source p)} did not list its benchmarks: ${(string-trim text)}"))))
       (_ (Err #"bjo bench: ${(program-source p)} did not list its benchmarks: ${(string-trim text)}"))))
    (_ (Err #"bjo bench: ${(program-source p)} could not be asked for its benchmarks."))))

;; ---------------------------------------------------------------------------
;; One menu over several files
;; ---------------------------------------------------------------------------

(: (-> Bench string))
(defun (bench-name b) (record-ref b name))

(: (-> Bench string))
(defun (bench-module b) (record-ref b module))

(: (-> string (List Listed) (List Listed)))
(defun (take-while-name name xs)
  (match xs
    ((Cons x more) (if (= (listed-name x) name) (cons x (take-while-name name more)) (list)))
    (Nil (list))))

(: (-> int (List %a) (List %a)))
(defun (drop-count n xs)
  (match xs
    ((Cons _ more) #:when (> n 0) (drop-count (- n 1) more))
    (_ xs)))

;; The listed cases of one file as benchmarks, for `(std bench)`'s menu: one
;; per run of cases of one name, under a heading of the file and its group.
;; `module` is the program, and a case's labels find its number again.
(: (-> BenchProgram (List Listed) (List Bench)))
(defun (listed-benches p listed)
  (def shown (file-name-of (program-source p)))
  (let go ((left listed) (acc (list)))
    (match left
      (Nil (list-reverse acc))
      ((Cons first _)
       (def same (take-while-name (listed-name first) left))
       (def rest (drop-count (list-length same) left))
       (def group (record-ref first group))
       (def cases (list-map (fun (l) (BenchCase (labels (listed-labels l))
                                                (prepare (fun () (panic! "bjo bench: a listed case is run by its program")))))
                            same))
       (go rest
           (cons (Bench (name (listed-name first)) (module (program-exe p)) (file (program-source p))
                        (line 0) (group (if (string-empty? group) shown (str shown ": " group)))
                        (skip (listed-skip first)) (once (record-ref first once)) (alloc "")
                        (param-names (list-map (fun (t) (match t ((Tuple n _) n))) (listed-labels first)))
                        (cases (fun () cases)))
                 acc))))))

(: bench-files-menu #:sync (-> BenchTools (List BenchProgram) (List string) int))
(defun (bench-files-menu tools programs passed)
  (with-return ret
    (def per-file
      (loop (:for p programs)
            (:let listed (listed-cases p passed))
            (:do (match listed ((Err why) (ret (fail why))) (_ unit)))
            (:when-let (Ok ls) listed)
            (:acc out (listing (Tuple p ls)))))
    (def benches
      (fold (fun (pl acc) (match pl ((Tuple p ls) (list-append acc (listed-benches p ls))))) (list) per-file))
    (: (-> Bench BenchCase int))
    (defun (number-of b c)
      (match (find (fun (pl) (match pl ((Tuple p _) (= (program-exe p) (bench-module b))))) per-file)
        ((Some (Tuple _ ls))
         (match (find (fun (l) (and (= (listed-name l) (bench-name b))
                                    (= (listed-labels l) (case-labels c))))
                      ls)
           ((Some l) (listed-index l))
           (None -1)))
        (None -1)))
    (: (-> (List Selection) void))
    (defun (run-choice chosen)
      (loop (:for pl per-file)
            (:let (Tuple p _) pl)
            (:let numbers
                  (fold (fun (s acc)
                          (if (= (bench-module (sel-bench s)) (program-exe p))
                              (list-append acc (list-map (fun (c) (number-of (sel-bench s) c)) (sel-cases s)))
                              acc))
                        (list) chosen))
            (:do (unless (list-empty? numbers)
                   (println "")
                   (println #"bjo bench: ${(file-name-of (program-source p))}")
                   (flush!)
                   (ignore ((record-ref tools exec)
                            "dotnet"
                            (list->vec (cons (program-exe p)
                                             (cons "--cases"
                                                   (cons (string-join (list->vec (list-map int->string numbers)) ",")
                                                         passed))))))))))
    (def title (string-join (list->vec (list-map (fun (p) (file-name-of (program-source p))) programs)) ", "))
    (menu-loop (menu-lines title benches) benches
               (fun () (read-line/opt (parameter-ref current-input-port)))
               run-choice)
    0))

;; ---------------------------------------------------------------------------
;; A/B: --against REF
;; ---------------------------------------------------------------------------

;; What `--against` and the flags beside it ask for. Warm-up is seconds and
;; the sample target milliseconds, as the flags are given.
(type (: Against (Record (: ref string)
                         (: rounds int)
                         (: processes int)
                         (: seed ulong)
                         (: warmup-ms long)
                         (: sample-ms long)
                         (: build (Option string)))))

(: (-> string (Option int)))
(defun (parse-positive s)
  (match (try (string->int s) #:catch (System.Exception))
    ((Ok n) (if (> n 0) (Some n) None))
    (_ None)))

(: (-> string (Option double)))
(defun (parse-number s)
  (match (try (string->double s) #:catch (System.Exception))
    ((Ok x) (if (> x 0.0) (Some x) None))
    (_ None)))

;; The A/B flags read, and the arguments without them, which go on to the
;; programs and to `(std bench)`'s own reading of `--alpha` and the rest.
(: (-> (List string) (Result string (Tuple Against (List string)))))
(defun (against-options passed)
  (let go ((left passed)
           (a (Against (ref "") (rounds 20) (processes 1)
                       (seed (cast System.UInt64 (+ clr-tick-count (cast System.Int64 clr-process-id))))
                       (warmup-ms 1000L) (sample-ms 10L) (build None)))
           (rest (list)))
    (defun (number-flag flag v after (: k (-> int Against)))
      (match (parse-positive v)
        ((Some n) (go after (k n) rest))
        (None (Err #"bjo bench: ${flag} takes a positive whole number: ${v}"))))
    (match left
      (Nil (if (string-empty? (record-ref a ref))
               (Err "bjo bench: --against needs a git ref or a path.")
               (Ok (Tuple a (list-reverse rest)))))
      ((Cons "--against" (Cons v after)) (go after (record-set a (ref v)) rest))
      ((Cons "-N" (Cons v after)) (number-flag "-N" v after (fun (n) (record-set a (rounds n)))))
      ((Cons "--processes" (Cons v after)) (number-flag "--processes" v after (fun (n) (record-set a (processes n)))))
      ((Cons "--seed" (Cons v after))
       (match (try (cast System.UInt64 (string->bigint v)) #:catch (System.Exception))
         ((Ok s) (go after (record-set a (seed s)) rest))
         (_ (Err #"bjo bench: --seed takes a whole number: ${v}"))))
      ((Cons "--against-warmup" (Cons v after))
       (match (parse-number v)
         ((Some s) (go after (record-set a (warmup-ms (cast System.Int64 (* s 1000.0)))) rest))
         (None (Err #"bjo bench: --against-warmup takes seconds: ${v}"))))
      ((Cons "--against-sample" (Cons v after))
       (match (parse-number v)
         ((Some ms) (go after (record-set a (sample-ms (max 1L (cast System.Int64 ms)))) rest))
         (None (Err #"bjo bench: --against-sample takes milliseconds: ${v}"))))
      ((Cons "--against-build" (Cons v after)) (go after (record-set a (build (Some v))) rest))
      ((Cons a1 more) (go more a (cons a1 rest))))))

;; Runs git and answers its output, trimmed; the reason when it fails.
(: git-out #:sync (-> (List string) (Result string string)))
(defun (git-out args)
  (match (run/output '(git [,@(list->vec args)]))
    ((Ok (Tuple 0 out)) (Ok (string-trim out)))
    ((Ok (Tuple _ _)) (Err (str "git " (string-join (list->vec args) " ") " failed")))
    ((Err e) (Err (str "could not run git: " (error-message e))))))

(import/extern
  (clr-directory-name (: System.IO.Path.GetDirectoryName (-> string string))))

(: (-> string string))
(defun (file-directory file) (clr-directory-name (clr-full-path file)))

;; The other side's file: a path is taken as it is, a `.bjo` file or a
;; checkout holding this file at the same place in it; anything else is a git
;; ref, checked out in a temporary worktree, which is answered too so that it
;; can be removed.
(type (: OtherSide (Record (: file string)
                           (: worktree (Option (Tuple string string))))))

(: other-side #:sync (-> string string (Result string OtherSide)))
(defun (other-side file ref)
  (cond
    ((and (string-ends-with? ref ".bjo") (file-exists? ref))
     (Ok (OtherSide (file ref) (worktree None))))
    ((directory-exists? ref)
     (match (git-out (list "-C" (file-directory file) "rev-parse" "--show-toplevel"))
       ((Ok top)
        (Ok (OtherSide (file (path-combine ref (clr-relative-path top (clr-full-path file)))) (worktree None))))
       ((Err _)
        (Ok (OtherSide (file (path-combine ref (clr-file-name file))) (worktree None))))))
    (else
     (match (git-out (list "-C" (file-directory file) "rev-parse" "--show-toplevel"))
       ((Err why) (Err #"bjo bench: --against ${ref}: not a path, and ${file} is not in a git repository (${why})."))
       ((Ok top)
        (def tmp (path-combine (clr-temp-path) #"bjo-against-${clr-process-id}-${clr-tick-count}"))
        (match (git-out (list "-C" top "worktree" "add" "--detach" "--quiet" tmp ref))
          ((Err why) (Err #"bjo bench: --against ${ref}: ${why}; is it a commit, a branch or a tag?"))
          ((Ok _)
           (Ok (OtherSide (file (path-combine tmp (clr-relative-path top (clr-full-path file))))
                          (worktree (Some (Tuple top tmp))))))))))))

(: remove-worktree #:sync (-> OtherSide void))
(defun (remove-worktree side)
  (match (record-ref side worktree)
    ((Some (Tuple top tmp)) (ignore (git-out (list "-C" top "worktree" "remove" "--force" tmp))))
    (None unit)))

;; The other side built: by `bjo`, or by `--against-build CMD` run in its
;; file's directory, after which its program is where `bjo` would put it.
(: build-other #:sync (-> BenchTools Against string (Result int BenchProgram)))
(defun (build-other tools a file)
  (match (record-ref a build)
    (None ((record-ref tools build) file))
    ((Some command)
     (println #"bjo bench: building ${file} with: ${command}")
     (flush!)
     (def status ((record-ref tools exec) "sh" ["-c" (str "cd " (shell-quote (file-directory file)) " && " command)]))
     (def exe (clr-change-extension file ".exe"))
     (cond
       ((not (= status 0)) (Err (fail #"bjo bench: --against-build failed, with exit code ${status}.")))
       ((not (file-exists? exe)) (Err (fail #"bjo bench: --against-build left no ${exe}.")))
       (else (Ok (BenchProgram (source file) (exe exe))))))))

(: (-> string string))
(defun (shell-quote s) (str "'" (string-replace s "'" "'\\''") "'"))

;; --- Serving -----------------------------------------------------------------

;; A benchmark program running with `--serve`: a command written, a reply read.
(type (: Server (Record (: proc Proc) (: input OutputPort) (: output InputPort) (: what string))))

(: start-server #:sync (-> BenchProgram (List string) string (Result string Server)))
(defun (start-server p passed what)
  (match (run '(dotnet [,(program-exe p) "--serve" ,@(list->vec passed)]))
    ((Err e) (Err #"bjo bench: could not start ${what}: ${(error-message e)}"))
    ((Ok proc)
     (match (Tuple (proc-input proc) (proc-output proc))
       ((Tuple (Some in) (Some out)) (Ok (Server (proc proc) (input in) (output out) (what what))))
       (_ (Err #"bjo bench: ${what} has no pipes to talk to it over."))))))

;; One command and its reply. A reply that is `(error ...)`, or no reply, is
;; an `Err`.
(: ask #:sync (-> Server string (Result string Bjodat)))
(defun (ask s command)
  (write-string (record-ref s input) (str command "\n"))
  (flush-port (record-ref s input))
  (match (read-line/opt (record-ref s output))
    (None (Err #"bjo bench: ${(record-ref s what)} stopped answering, after ${command}"))
    ((Some line)
     (match (parse-bjodat line)
       ((Ok (BjoList (Cons (BjoSym head) (Cons (BjoStr why) Nil))))
        (if (= (symbol->string head) "error")
            (Err #"bjo bench: ${(record-ref s what)}: ${why}")
            (Err #"bjo bench: ${(record-ref s what)} answered ${line}")))
       ((Ok reply) (Ok reply))
       ((Err why) (Err #"bjo bench: ${(record-ref s what)} answered what is not bjodat: ${line}"))))))

(: stop-server #:sync (-> Server void))
(defun (stop-server s)
  (write-string (record-ref s input) "(quit)\n")
  (flush-port (record-ref s input))
  (ignore (wait (record-ref s proc))))

;; The value after keyword `name` in a reply.
(: (-> Bjodat string (Option Bjodat)))
(defun (reply-arg reply name)
  (match reply
    ((BjoList items)
     (let go ((left items))
       (match left
         ((Cons (BjoKey k) (Cons v more)) (if (= (keyword->string k) name) (Some v) (go more)))
         ((Cons _ more) (go more))
         (Nil None))))
    (_ None)))

(: (-> Bjodat string long))
(defun (reply-long reply name)
  (match (reply-arg reply name) ((Some (BjoInt n)) n) (_ -1L)))

;; Prepares case `i` on a server, and answers its batch size and whether it
;; got steady while it warmed up.
(: prepare-on #:sync (-> Server int Against (Result string (Tuple long bool))))
(defun (prepare-on s i a)
  (match (ask s #"(prepare ${i} :warmup ${(record-ref a warmup-ms)} :sample ${(record-ref a sample-ms)})")
    ((Ok reply)
     (Ok (Tuple (reply-long reply "k")
                (match (reply-arg reply "steady") ((Some (BjoBool b)) b) (_ #t)))))
    ((Err why) (Err why))))

;; One sample of `k` calls of case `i`, as nanoseconds per call.
(: sample-on #:sync (-> Server int long (Result string double)))
(defun (sample-on s i k)
  (match (ask s #"(sample ${i} :k ${k})")
    ((Ok reply) (Ok (/ (cast System.Double (reply-long reply "ns")) (cast System.Double k))))
    ((Err why) (Err why))))

;; --- The rounds ----------------------------------------------------------------

;; A case both sides have: its number on each.
(type (: Shared (Record (: listed Listed) (: a int) (: b int))))

(: (-> Shared Listed))
(defun (shared-listed s) (record-ref s listed))

;; Whether a filter chooses a case: its name contains one, or it is one of
;; its parameters as the report shows them.
(: (-> (List string) Listed bool))
(defun (chosen-by? names l)
  (or (list-empty? names)
      (any? (fun (f) (or (string-contains? (listed-name l) f)
                         (any? (fun (t) (match t ((Tuple p v) (= (str p "=" v) f)))) (listed-labels l))))
            names)))

;; What the programs are given as they are started: the flags of
;; `(std bench)` that decide what a case is, `--params` today.
(: (-> (List string) (List string)))
(defun (serve-flags rest)
  (let go ((left rest) (acc (list)))
    (match left
      ((Cons "--params" (Cons v more)) (go more (cons v (cons "--params" acc))))
      ((Cons _ more) (go more acc))
      (Nil (list-reverse acc)))))

;; The names among the arguments: what is not a flag or a flag's value.
(: (-> (List string) (List string)))
(defun (bench-names rest)
  (let go ((left rest) (acc (list)))
    (match left
      ((Cons a more)
       (cond
         ((value-flag? a) (go (match more ((Cons _ after) after) (Nil Nil)) acc))
         ((string-starts-with? a "-") (go more acc))
         (else (go more (cons a acc)))))
      (Nil (list-reverse acc)))))

;; How many of `rounds` fall to process pair `j` of `pairs`: as even as can
;; be, the first ones taking what is left over.
(: (-> int int int int))
(defun (rounds-of rounds pairs j)
  (+ (/ rounds pairs) (if (< j (% rounds pairs)) 1 0)))

;; One process pair: both started, every shared case prepared on both with
;; the smaller batch size, and its rounds taken, the order of the sides
;; alternating and the cases shuffled each round. Answers, per shared case,
;; A's and B's per-call times in round order.
(: run-pair #:sync (-> BenchProgram BenchProgram (List Shared) Against int (List string) RandomSource
                       (Box (List string))
                       (Result string (Vec (Tuple (Vec double) (Vec double))))))
(defun (run-pair pa pb shared a j flags rng unsteady)
  (with-return ret
    (def (Ok sa) (start-server pa flags (str "A (" (record-ref a ref) ")")) :leave ((Err why) (Err why)))
    (def (Ok sb) (start-server pb flags "B (this code)") :leave ((Err why) (stop-server sa) (Err why)))
    (defun (leave (: why string)) : (Result string (Vec (Tuple (Vec double) (Vec double))))
      (stop-server sa)
      (stop-server sb)
      (Err why))
    (def n (list-length shared))
    (def live (bench-at-terminal?))
    (def ks
      (loop (:for s shared)
            (:with i 0 (+ i 1))
            (:do (when live
                   (print (str "\r[pair " (int->string (+ j 1)) "/" (int->string (record-ref a processes))
                               "] preparing " (int->string (+ i 1)) "/" (int->string n) ": "
                               (listed-key (shared-listed s)) "\u{1B}[K"))
                   (flush!)))
            (:let ka (prepare-on sa (record-ref s a) a))
            (:let kb (prepare-on sb (record-ref s b) a))
            ;; The smaller batch on both sides, so that a sample of each is
            ;; the same number of calls.
            (:let k (match (Tuple ka kb)
                      ((Tuple (Ok (Tuple x steady-a)) (Ok (Tuple y steady-b)))
                       (unless steady-a (box-set! unsteady (cons (str (listed-key (shared-listed s)) " on A") (box-ref unsteady))))
                       (unless steady-b (box-set! unsteady (cons (str (listed-key (shared-listed s)) " on B") (box-ref unsteady))))
                       (Ok (min x y)))
                      ((Tuple (Err why) _) (Err why))
                      ((Tuple _ (Err why)) (Err why))))
            (:do (match k ((Err why) (ret (leave why))) (_ unit)))
            (:acc out (vecing (match k ((Ok x) x) (_ 1L))))))
    (def rounds (rounds-of (record-ref a rounds) (record-ref a processes) j))
    (def as (make-array n #:fill (vec-empty)))
    (def bs (make-array n #:fill (vec-empty)))
    (def order (list->vec (index-list n)))
    (loop (:for r (up-from 0 #:to rounds))
          (:do (when live
                 (print (str "\r[pair " (int->string (+ j 1)) "/" (int->string (record-ref a processes))
                             "] round " (int->string (+ r 1)) "/" (int->string rounds) "\u{1B}[K"))
                 (flush!)))
          (:subloop (:for i (shuffle-vec order #:rng rng))
                    (:let s (list-ref shared i))
                    (:let k (vec-ref ks i))
                    ;; A first in even rounds, B first in odd ones.
                    (:let first-a (= (% (+ r j) 2) 0))
                    (:let x1 (if first-a (sample-on sa (record-ref s a) k) (sample-on sb (record-ref s b) k)))
                    (:let x2 (if first-a (sample-on sb (record-ref s b) k) (sample-on sa (record-ref s a) k)))
                    (:do (match (Tuple x1 x2)
                           ((Tuple (Ok t1) (Ok t2))
                            (array-set! as i (vec-add (array-ref as i) (if first-a t1 t2)))
                            (array-set! bs i (vec-add (array-ref bs i) (if first-a t2 t1))))
                           ((Tuple (Err why) _) (ret (leave why)))
                           ((Tuple _ (Err why)) (ret (leave why)))))))
    (when live (print "\r\u{1B}[K") (flush!))
    (stop-server sa)
    (stop-server sb)
    (Ok {vecing (Tuple (array-ref as i) (array-ref bs i)) (:for i (range 0 n))})))

(: (-> int (List int)))
(defun (index-list n) {listing i (:for i (range 0 n))})

(: cmd-against #:sync (-> BenchTools string (List string) int))
(defun (cmd-against tools file passed)
  (with-return ret
    (def (Ok (Tuple a rest)) (against-options passed) :leave ((Err why) (fail why)))
    (def (Ok o) (parse-bench-args (list->vec (list-filter (fun (x) (not (= x "--serve"))) rest)))
      :leave ((Err why) (fail #"bjo bench: ${why}")))
    (when (some? (record-ref o compare))
      (ret (fail "bjo bench: --against and --compare are two comparisons; ask for one.")))
    (def (Ok pb) ((record-ref tools build) file) :leave ((Err status) status))
    (def (Ok side) (other-side file (record-ref a ref)) :leave ((Err why) (fail why)))
    (def code (against-with tools a o rest pb side))
    (remove-worktree side)
    code))

(: against-with #:sync (-> BenchTools Against BenchOptions (List string) BenchProgram OtherSide int))
(defun (against-with tools a o rest pb side)
  (with-return ret
    (def ref (record-ref a ref))
    (unless (file-exists? (record-ref side file))
      (ret (fail #"bjo bench: ${ref} has no ${(record-ref side file)}: the benchmark file is not there.")))
    (def (Ok pa) (build-other tools a (record-ref side file)) :leave ((Err status) status))
    (def flags (serve-flags rest))
    (def (Ok listed-a) (listed-cases pa flags) :leave ((Err why) (fail why)))
    (def (Ok listed-b) (listed-cases pb flags) :leave ((Err why) (fail why)))
    (def names (bench-names rest))
    (defun (runnable (: l Listed)) : bool (and (string-empty? (listed-skip l)) (chosen-by? names l)))
    (def wanted-a (list-filter runnable listed-a))
    (def wanted-b (list-filter runnable listed-b))
    (def shared
      (loop (:for l wanted-b)
            (:when-let (Some other) (find (fun (x) (= (listed-key x) (listed-key l))) wanted-a))
            (:acc out (listing (Shared (listed l) (a (listed-index other)) (b (listed-index l)))))))
    (def only-a (loop (:for l wanted-a)
                      (:acc out (listing (listed-key l))
                            #:when (none? (find (fun (x) (= (listed-key x) (listed-key l))) wanted-b)))))
    (def only-b (loop (:for l wanted-b)
                      (:acc out (listing (listed-key l))
                            #:when (none? (find (fun (x) (= (listed-key x) (listed-key l))) wanted-a)))))
    (when (list-empty? shared)
      (ret (fail "bjo bench: the two sides have no benchmark in common to compare.")))
    (def rng (make-random/seed64 (record-ref a seed)))
    (def pairs (record-ref a processes))
    ;; The cases that did not get steady while warming up, and on which side.
    (: (Box (List string)))
    (def unsteady (make-box (list)))
    (def taken
      (loop (:for j (up-from 0 #:to pairs))
            (:let got (run-pair pa pb shared a j flags rng unsteady))
            (:do (match got ((Err why) (ret (fail why))) (_ unit)))
            (:when-let (Ok v) got)
            (:acc out (listing v))))
    (def cases
      (loop (:for s shared)
            (:with i 0 (+ i 1))
            (:acc out (listing (PairedCase (name (listed-name (shared-listed s)))
                                           (labels (listed-labels (shared-listed s)))
                                           (a (list->vec (list-map (fun (v) (match (vec-ref v i) ((Tuple x _) x))) taken)))
                                           (b (list->vec (list-map (fun (v) (match (vec-ref v i) ((Tuple _ y) y))) taken))))))))
    (def result (compare-paired cases only-a only-b (record-ref o alpha) (record-ref o threshold)))
    (loop (:for l (header-lines (list-length shared))) (:do (println l)))
    (println (str "bench: A/B against " ref ", " (int->string (record-ref a rounds)) " rounds, "
                  (if (= pairs 1)
                      "paired (Wilcoxon signed-rank); "
                      #"${pairs} process pairs (t-test on their log ratios); ")
                  (comparisons-text (list-length shared))))
    (when (= pairs 1)
      (println "bench: one process per side: a difference between processes is not in the p-values (--processes P measures it)"))
    (println (str "bench: seed " (displayed (record-ref a seed)) " (--seed repeats the order of the cases)"))
    (unless (list-empty? (box-ref unsteady))
      (println (str "bench: warning: not steady after warming up, so the JIT may have changed them during the rounds: "
                    (string-join (list->vec (distinct-strings (list-reverse (box-ref unsteady)))) ", ")
                    ". --against-warmup S gives them longer.")))
    (def shown (if (file-exists? ref) (file-name-of ref) ref))
    (print-comparison-table result o shown "now" #"A (${shown})" #f)
    (match (record-ref o save)
      ((Some path) (save-sides path ref cases))
      (None unit))
    (if (fails-slower? result o) 1 0)))

(: (-> (List string) (List string)))
(defun (distinct-strings xs)
  (fold (fun (x acc) (if (any? (fun (y) (= y x)) acc) acc (list-append acc (list x)))) (list) xs))

;; `--save FILE` of an A/B run: this code's samples to FILE, the other side's
;; beside it, `FILE` with `.against` before its extension. Every process
;; pair's samples, one after another.
(: (-> string string (List PairedCase) void))
(defun (save-sides path ref cases)
  (def other (clr-change-extension path (str ".against" (path-extension-of path))))
  (defun (side (: pick (-> PairedCase (Vec (Vec double))))) : (List SavedCase)
    (list-map (fun (c) (SavedCase (name (record-ref c name)) (labels (record-ref c labels)) (k 0L) (once #f)
                                  (bytes 0.0) (samples (vec-fold (fun (v acc) (vec-append acc v)) (vec-empty) (pick c)))))
              cases))
  (write-saved-run! path (this-run (side (fun (c) (record-ref c b)))))
  (write-saved-run! other (this-run (side (fun (c) (record-ref c a)))))
  (println #"bench: saved this code's samples to ${path}, and ${ref}'s to ${other}"))

(: (-> string string))
(defun (path-extension-of path)
  (match (string-search-backward path ".")
    ((Some at) (substring path at (string-cursor-end path)))
    (None "")))
;; bjo — the command that runs Bjolang.
;;
;; The commands all end in the compiler: `run`, `build`, `check`, `repl`, and
;; `init` and `fetch`, which prepare what the others compile. What is here
;; rather than in the compiler is what a compiler cannot do for itself — decide
;; whether to start one at all, find the project a command was run inside,
;; fetch what its manifest names, and hand the console to what it starts.
;;
;; A *project* is a directory with a `manifest.bjodat`. Inside one, every
;; command compiles against the project's `.bjo/roots`, which says which
;; directory each package name means; outside one, every command behaves as it
;; did before there were projects. What a project is and how its dependencies
;; are resolved is in `project.bjo`, `resolve.bjo`, `git.bjo` and `lock.bjo` —
;; this file is the command line and the compiler processes, and stays that.
;;
;; `bjo` does not compile dependencies one at a time. The compiler builds an
;; imported module itself, in dependency order, so what `bjo` has to do is make
;; the modules findable — the roots file — and then compile the one entry.
;;
;; Staleness is *read*, not worked out. A build writes `<source>.bjobuild`
;; naming the mode it produced, the artefact, the compiler, and every source and
;; module it read; see `Build.writeBuildRecord`. So the question "is there
;; anything to do?" is a timestamp comparison over a list, and nothing here has
;; to know what an `include` is or where a module path resolves to. Those are
;; the compiler's to know. A driver that worked them out from the source
;; itself could disagree with the compiler about what a file depends on.
;;
;; Children inherit this process's console, which is why `(std run)` is not used
;; to run them: it redirects stdin and stdout in order to read them, and the
;; output of a program run under `bjo` belongs to whoever ran it. The REPL needs
;; a terminal on both ends for the same reason. The one child whose output `bjo`
;; reads is a benchmark program asked for its list, which nobody watches.
;;
;; Everything that waits for a child is `#:sync`, down from `dispatch`. Waiting
;; is what this command does and it has one thread to do it on, so a suspending
;; copy of any of it would be a body nothing here ever calls — reported, rightly,
;; as parking a thread that is nobody's to yield.
;;
;; A command line is a sequence of refusals before it is a command, so most of
;; what follows is written as guards: `def` binds the case that carries on and
;; says what the body produces when it does not, and `with-return` is what a
;; failure that ends the command leaves through. The alternative is the same
;; decisions as nesting, indented once per thing that can go wrong.
;;
;; Where the failing case has nothing to say for itself — two `Option`s in a row
;; and no answer but `None` — it is a `(do ...)` block instead, and neither
;; failure is written down at all.

(import (std prelude))
;; The manifest types, for the fields this file reads out of a `Package`:
;; `record-ref` needs the declaration, and an import of `project.bjo` alone
;; does not bring it. `(text bjodat)` comes with them — the manifest's own
;; instances are written against its traits.
(import (text bjodat))
(import "manifest.bjo")
(import "project.bjo")
(import "resolve.bjo")
(import "publish.bjo")
(import "bench.bjo")
(import "format.bjo")
(import "diff.bjo")

;; ---------------------------------------------------------------------------
;; Interop
;; ---------------------------------------------------------------------------

(import/class
  (Process (: System.Diagnostics.Process))
  (ProcessStartInfo (: System.Diagnostics.ProcessStartInfo (-> ProcessStartInfo)))
  (DateTime (: System.DateTime)))

(import/extern
  (settings-program!   (: System.Diagnostics.ProcessStartInfo.FileName
                          (-> ProcessStartInfo string void) #:set))
  (settings-use-shell! (: System.Diagnostics.ProcessStartInfo.UseShellExecute
                          (-> ProcessStartInfo bool void) #:set))
  (settings-argument   (: BjoProc.AddArgument
                          (-> ProcessStartInfo string ProcessStartInfo)))
  (process-start (: System.Diagnostics.Process.Start (-> ProcessStartInfo Process)
                    #:exceptions (System.ComponentModel.Win32Exception
                                  System.InvalidOperationException)))
  ;; The synchronous wait, and deliberately: `bjo` has nothing to do until the
  ;; child is done, and the thread it parks is the process's own.
  (process-wait  (: System.Diagnostics.Process.WaitForExit (-> Process void) #:blocking))
  (process-code  (: System.Diagnostics.Process.ExitCode (-> Process int) #:get))
  (error-message (: System.Exception.Message (-> Exception string) #:get))
  (clr-mtime     (: System.IO.File.GetLastWriteTimeUtc (-> string DateTime)))
  (clr-ticks     (: System.DateTime.Ticks (-> DateTime long) #:get))
  (string->long  (: System.Int64.TryParse (-> string (out long) (Option long))))
  (clr-change-extension (: System.IO.Path.ChangeExtension (-> string string string)))
  ;; Where this executable sits, for an installation that was not told where it
  ;; is. See `root`.
  (clr-base-dir  (: System.AppContext.BaseDirectory #:get)))


;; ---------------------------------------------------------------------------
;; Starting something, and waiting for it
;; ---------------------------------------------------------------------------

;; Runs `program` and answers its exit code.
;;
;; No redirection anywhere: the child writes to the terminal `bjo` was started
;; from, reads from the same stdin, and is not a pipe stage. `UseShellExecute`
;; off is what makes the program name resolve against PATH.
(: exec #:sync (-> string (Vec string) int))
(defun (exec program arguments)
  (def settings (ProcessStartInfo.))
  (settings-program! settings program)
  (settings-use-shell! settings #f)
  (loop (:for a arguments)
        (:do (ignore (settings-argument settings a))))
  (match (process-start settings)
    ((Err e)
     (println #"bjo: could not start ${program}: ${(error-message e)}")
     ;; What a shell answers for a command it could not run.
     127)
    ((Ok child)
     (process-wait child)
     (process-code child))))


;; ---------------------------------------------------------------------------
;; Timestamps
;; ---------------------------------------------------------------------------

;; A file's last write, in ticks. A file that is not there is as old as anything
;; can be, so a dependency that has been deleted is not newer than the artefact
;; and does not force a rebuild that would fail for the same reason.
(: (-> string long))
(defun (mtime path)
  (if (file-exists? path) (clr-ticks (clr-mtime path)) 0L))

(: (-> string string bool))
(defun (newer? a b) (> (mtime a) (mtime b)))


;; ---------------------------------------------------------------------------
;; The installation
;; ---------------------------------------------------------------------------

;; The directory the compiler and its sources live in.
;;
;; `BJO_ROOT` is what the launcher exports. Without it, this executable was
;; started some other way, and the one thing it knows about itself is where it
;; sits: a build puts the binary beside its source, so `bjo.exe` is in
;; `<root>/bjo`.
(: string)
(def root
  (match (get-environment-variable "BJO_ROOT")
    ((Some dir) dir)
    (None (path-absolute (path-combine clr-base-dir "..")))))

;; The compiler. `build_compiler.sh` is the one place that knows where it is,
;; and the launcher exports its answer as `BJOLANG_COMPILER`. The fallback is
;; that same answer written down, for a `bjo.exe` started without the launcher.
(: string)
(def compiler-dll
  (match (get-environment-variable "BJOLANG_COMPILER")
    ((Some path) path)
    (None (path-combine root "bin" "compiler" "Bjolang.dll"))))


;; ---------------------------------------------------------------------------
;; The compiler binary
;; ---------------------------------------------------------------------------

;; Is there a compiler to use at all?
(: (-> bool))
(defun (have-compiler?)
  (cond
    ((file-exists? compiler-dll) #t)
    (else
     (println "bjo: there is no compiler built here yet. Run 'bjo rebjo'.")
     #f)))


;; ---------------------------------------------------------------------------
;; The build record
;; ---------------------------------------------------------------------------

;; What a build wrote down about itself. `compiler`, `source` and `dep` lines
;; are read into one list: they are all files the artefact must be newer than,
;; and nothing here has a reason to tell them apart.
(type
  (: BuildRecord (Record (: mode string)
                         (: output string)
                         (: inputs (List string))
                         ;; The compiler that built it. Its timestamp is also
                         ;; in `inputs`; this is compared as a path, because a
                         ;; different compiler can be older than the artefact,
                         ;; and the timestamp check would then call the
                         ;; artefact current.
                         (: compiler string)
                         ;; The .NET shared frameworks the file's package
                         ;; declared when it was built. Not a file, so not an
                         ;; input: a declaration that is *removed* deletes the
                         ;; declarations file, and a file that is not there is
                         ;; newer than nothing.
                         (: frameworks (List string))
                         ;; The declarations file this was built against, or ""
                         ;; where there was none. It is an input as well — a
                         ;; rewrite is a change the timestamps can see — and
                         ;; this is for the change they cannot: the file
                         ;; appearing, or going away with the last declaration
                         ;; in the whole graph.
                         (: declaration-file string)
                         ;; The `--nuget` directory it was built with, or "".
                         ;; Compared for the same reason as `declaration-file`:
                         ;; packages removed from the manifest leave no file
                         ;; that a timestamp could show.
                         (: nuget string)
                         ;; For a program: its runtime settings, as
                         ;; `name=value`, sorted. Compared for the same reason
                         ;; as `nuget`: they are in the manifest, and a change
                         ;; to them changes no file that the build read.
                         (: runtime (List string))
                         ;; For a program built with `--aot`: its native
                         ;; executable. "" otherwise.
                         (: aot string)
                         ;; With `--aot-cpu`: the CPU it was built for. ""
                         ;; otherwise.
                         (: aot-cpu string)
                         ;; For a program: what its resolver loads, as the
                         ;; `link` and `native` lines name them. What
                         ;; `bjo publish` copies.
                         (: links (List (Tuple string string)))
                         (: natives (List (Tuple string string)))
                         ;; The artefact's last write when the record was
                         ;; written, 0 where it does not say. A record whose
                         ;; artefact was written since is another build's.
                         (: written long)
                         ;; For a library, the digest of what an importer is
                         ;; compiled against, or "".
                         (: interface string)
                         ;; Each linked module's digest as this build saw it, as
                         ;; `(digest . path)`.
                         (: interface-of (List (Tuple string string)))
                         ;; The `source` lines alone.
                         (: sources (List string)))))

(: (-> string string))
(defun (record-file file) (clr-change-extension file ".bjobuild"))

;; A line is `key value`, and the value may contain spaces — a path is allowed
;; them — so it is everything after the first one rather than a field of a
;; split. A line with no space at all is no line, which is the `None` the bind
;; short-circuits to.
(: (-> string (Option (Tuple string string))))
(defun (split-line line)
  (option-map (fun (space)
                (Tuple (substring line (string-cursor-start line) space)
                       (substring line (string-cursor-next line space) (string-cursor-end line))))
              (string-find #(char=? & #\space) line)))

;; A `link` or `native` value: a name, then a path. A value with no space in it
;; is a name with no path, which a publish reports as a file that is not there.
(: (-> string (Tuple string string)))
(defun (pair-of value)
  (match (split-line value)
    ((Some pair) pair)
    (None (Tuple value ""))))

;; The keys that name a file the artefact must be newer than. What they are
;; called apart is nothing this has to know.
;; `roots` is an input like any other: it says which directory each package name
;; means, so a build made against a different one is a build against different
;; code — and the `dep` lines here are the files the old one chose.
(: (-> string bool))
(defun (input-key? key)
  (case key
    (("source" "dep" "compiler" "roots" "frameworks" "nuget-list") #t)
    (else #f)))

;; `None` when there is no record, or when what is there names no artefact —
;; which is a record written by a newer `bjo` or a truncated one, and either way
;; not something to decide against.
(: (-> string (Option BuildRecord)))
(defun (read-record file)
  (with-return ret
    (def path (record-file file))
    (unless (file-exists? path) (ret None))

    (loop (:for line (string-split (file-read-text path) "\n"))
          ;; A line with no key at all is not one. `:when-let` skips it, and an
          ;; accumulator that is not stepped keeps what it had.
          (:when-let (Some (Tuple key value)) (split-line (string-trim line)))
          (:acc mode (folding "" (if (= key "mode") value mode)))
          (:acc artefact (folding "" (if (= key "output") value artefact)))
          (:acc compiler (folding "" (if (= key "compiler") value compiler)))
          ;; A key this `bjo` does not know is a record from a later one, and is
          ;; accumulated nowhere: the three fields here are what a decision is
          ;; made of, and they are all present.
          (:acc inputs (listing value) #:when (input-key? key))
          (:acc frameworks (listing value) #:when (= key "framework"))
          (:acc declarations (folding "" (if (= key "frameworks") value declarations)))
          (:acc nuget (folding "" (if (= key "nuget") value nuget)))
          (:acc runtime (listing value) #:when (= key "runtime"))
          (:acc aot (folding "" (if (= key "aot") value aot)))
          (:acc aot-cpu (folding "" (if (= key "aot-cpu") value aot-cpu)))
          ;; `link <name> <path>` and `native <key> <path>`: the value splits
          ;; at its first space the way a line does, and a path may hold more.
          (:acc links (listing (pair-of value)) #:when (= key "link"))
          (:acc natives (listing (pair-of value)) #:when (= key "native"))
          (:acc written (folding 0L (if (= key "written")
                                        (match (string->long value) ((Some n) n) (None 0L))
                                        written)))
          (:acc interface (folding "" (if (= key "interface") value interface)))
          (:acc interface-of (listing (pair-of value)) #:when (= key "interface-of"))
          (:acc sources (listing value) #:when (= key "source"))
      => (if (= artefact "")
             None
             (Some (BuildRecord (mode mode)
                                (output artefact)
                                (inputs inputs)
                                (compiler compiler)
                                (frameworks (list-sort frameworks))
                                (declaration-file declarations)
                                (nuget nuget)
                                (runtime (list-sort runtime))
                                (aot aot)
                                (aot-cpu aot-cpu)
                                (links links)
                                (natives natives)
                                (written written)
                                (interface interface)
                                (interface-of interface-of)
                                (sources sources)))))))

;; ---------------------------------------------------------------------------
;; Compiling
;; ---------------------------------------------------------------------------

;; The source a name refers to. A bare `foo` is `foo.bjo` when that exists,
;; which is what makes `bjo run examples/vec` work.
(: (-> string (Option string)))
(defun (resolve-source name)
  (cond
    ((file-exists? name) (Some name))
    ((file-exists? (str name ".bjo")) (Some (str name ".bjo")))
    (else None)))

;; `--roots <file>` where there is a project, and nothing where there is not.
;; The compiler records the file as an input of what it builds, so a build made
;; against other packages is stale by the same comparison every other change is.
;; What a compile is made against: where the packages are, and which of them
;; declared a .NET shared framework. Carried together because every command
;; that passes one passes the other, and two things a caller has to keep in
;; step are two things a caller can get out of step.
;;
;; Both are `None` outside a project, and the second is `None` inside one where
;; no package declared anything — which is the ordinary case, and is what keeps
;; such a project's command line exactly what it was before frameworks existed.
;;
;; `nuget` is the directory of the restored NuGet packages, and `None` unless
;; the manifest names some.
;;
;; `runtime` is the root manifest's runtime settings, as `name=value`, which a
;; program is built with, `aot` whether it is built as a Native AOT
;; executable too, and `aot-cpu` the CPU that executable is for, or "". Empty,
;; #f and "" outside a project.
(type (: Against (Record (: roots (Option string))
                         (: frameworks (Option string))
                         (: nuget (Option string))
                         (: runtime (List string))
                         (: aot bool)
                         (: aot-cpu string))))

(: Against)
(def nothing-against (Against (roots None) (frameworks None) (nuget None) (runtime Nil) (aot #f) (aot-cpu "")))

(: (-> Prepared Against))
(defun (prepared-against ready)
  (Against (roots (Some (prepared-roots-file ready)))
           (frameworks (prepared-frameworks-file ready))
           (nuget (prepared-nuget-dir ready))
           (runtime (prepared-runtime ready))
           (aot (prepared-aot ready))
           (aot-cpu (prepared-aot-cpu ready))))

(: (-> string (Option string) (List string)))
(defun (flag-for flag path)
  (match path
    ((Some p) (Cons flag (Cons p Nil)))
    (None Nil)))

;; `--runtime name=value` for each runtime setting.
(: (-> (List string) (List string)))
(defun (runtime-flags settings)
  (match settings
    (Nil Nil)
    ((Cons setting rest) (Cons "--runtime" (Cons setting (runtime-flags rest))))))

;; `--aot`, and `--aot-cpu` with it when the manifest names a CPU.
(: (-> Against (List string)))
(defun (aot-flags a)
  (cond ((not (record-ref a aot)) Nil)
        ((= (record-ref a aot-cpu) "") (list "--aot"))
        (else (list "--aot" "--aot-cpu" (record-ref a aot-cpu)))))

;; The compiler records both files as inputs of what it builds, so a build made
;; against other packages — or against another set of declarations — is stale by
;; the same comparison every other change is.
(: (-> Against (List string)))
(defun (against-flags a)
  ;; TODO list-append*.
  (list-append (flag-for "--roots" (record-ref a roots))
               (list-append (flag-for "--frameworks" (record-ref a frameworks))
                            (list-append (flag-for "--nuget" (record-ref a nuget))
                                         (list-append (aot-flags a)
                                                      (runtime-flags (record-ref a runtime)))))))

;; What the frameworks file says the package holding `file` declares.
;;
;; The longest directory that contains the file wins, which is the rule the
;; compiler uses to decide the same question — and has to be, because a
;; dependency fetched into `.bjo/pkg` lives *under* the root project.
(: (-> Against string (List string)))
(defun (declared-for against file)
  (def (Some path) (get against .frameworks) :leave-with Nil)
  (cond
    ((not (file-exists? path)) Nil)
    (:def full (path-absolute file))
    (:def lines (string-split (file-read-text path) "\n"))
    ;; Two passes rather than one: the package a file belongs to is the
    ;; longest directory containing it, and which that is cannot be known
    ;; until every line has been seen.
    (:def owner
          (loop (:for line lines)
                (:when-let (Some (Tuple dir name)) (split-tab (string-trim line)))
                (:acc out (folding "" (if (and (string-starts-with? full (str dir "/"))
                                               (> (string-length dir) (string-length out)))
                                          dir  
                                          out)))))
    ((= owner "") Nil)
    (else (list-sort
           (loop (:for line lines)
                 (:when-let (Some (Tuple dir name)) (split-tab (string-trim line)))
                 (:acc out (listing name) #:when (= dir owner)))))))

;; A frameworks-file line: a directory, a tab, a framework name.
(: (-> string (Option (Tuple string string))))
(defun (split-tab line)
  (def (Some tab) (string-find #(char=? & #\tab) line) :propagate)
  (Some (Tuple (substring line (string-cursor-start line) tab)
               (substring line (string-cursor-next line tab)
                          (string-cursor-end line)))))

;; Is there anything to do before the artefact can be used?
(: (-> string string Against bool))
(defun (stale? file wanted against)
  ;; No record at all is *stale*, not current: there is nothing that says what
  ;; the artefact was built from, so the only safe answer is to build it. `#f`
  ;; here makes `ensure-built` serve a record that is not there, and fail with
  ;; an exit code and no message.
  (def (Some rec) (read-record file) :leave-with #t)
  (or (built-otherwise? rec wanted against file)
      (any? #(newer? & (record-ref rec output)) (record-ref rec inputs))))

;; Whether the record describes a build other than the one asked for, whatever
;; the timestamps say.
(: (-> BuildRecord string Against string bool))
(defun (built-otherwise? rec wanted against file)
  (def artefact (record-ref rec output))
  (or (not (= (record-ref rec mode) wanted))
      (not (file-exists? artefact))
      ;; Built by another compiler than the one this would build it with, or by
      ;; one the record does not name.
      (= (record-ref rec compiler) "")
      (not (= (path-absolute (record-ref rec compiler)) (path-absolute compiler-dll)))
      ;; A changed declaration is a changed build: what a module may *name*
      ;; follows from it, so one built under another set has to be built again
      ;; — and told, if it now names something it may not.
      (not (= (record-ref rec frameworks) (declared-for against file)))
      ;; And the same for the file itself coming or going. The last
      ;; declaration in a graph being taken out deletes it, and a file that is
      ;; not there is newer than nothing — while the dependency that used to
      ;; declare it still has to be built again, which is the compiler's own
      ;; comparison and only runs if this one sends it there.
      (not (= (record-ref rec declaration-file)
              (match (record-ref against frameworks)
                ((Some path) path)
                (None ""))))
      ;; Packages added to or removed from the manifest.
      (not (= (record-ref rec nuget)
              (match (record-ref against nuget)
                ((Some dir) (path-absolute dir))
                (None ""))))
      ;; Runtime settings changed in the manifest. Only a program has them: a
      ;; library has no runtimeconfig, and its record no `runtime` lines.
      (and (string-ends-with? artefact ".exe")
           (not (= (record-ref rec runtime) (record-ref against runtime))))
      ;; `(aot ...)` changed in the manifest, or the native executable is gone.
      (and (string-ends-with? artefact ".exe")
           (not (= (not (= (record-ref rec aot) "")) (record-ref against aot))))
      ;; `(aot-cpu ...)` changed, for a program that is built as AOT.
      (and (string-ends-with? artefact ".exe")
           (record-ref against aot)
           (not (= (record-ref rec aot-cpu) (record-ref against aot-cpu))))
      (and (record-ref against aot)
           (not (file-exists? (record-ref rec aot))))))

;; A linked module whose own record vouches for the interface `entry` says the
;; build saw: the module and the sources it was built from. Nothing otherwise.
(: (-> (Tuple string string) (List string)))
(defun (interface-kept entry)
  (def (Tuple digest dll) entry)
  (def (Some dep) (read-record dll) :leave-with Nil)
  (if (and (= (record-ref dep written) (mtime dll))
           (= (record-ref dep interface) digest))
      (cons dll (record-ref dep sources))
      Nil))

;; Early cutoff, for an artefact whose imports have just been built: current if
;; everything newer than it is a linked module whose interface is the one it was
;; built against, or a source such a module was built from. The compiler decides
;; a library this way; this is the same for the entry it is never asked about.
(: (-> string string Against bool))
(defun (current-by-interface? file wanted against)
  (def* ((Some rec) (read-record file) :leave-with #f)
        (artefact (record-ref rec output)))
  (def excused
    (loop (:for entry (record-ref rec interface-of))
          (:acc out (folding Nil (list-append (interface-kept entry) out)))
      => out))
  (and (not (built-otherwise? rec wanted against file))
       (file-exists? artefact)
       (= (record-ref rec written) (mtime artefact))
       (all? (fun (input) (or (not (newer? input artefact)) (any? #(= & input) excused)))
             (record-ref rec inputs))))


(: compile #:sync (-> string bool bool Against int))
(defun (compile file debug? lib? roots)
  (def flags (list-append (if lib? (Cons "--lib" Nil) Nil)
                          (list-append (if debug? (Cons "-d" Nil) Nil)
                                       (against-flags roots))))
  (exec "dotnet" (list->vec (Cons compiler-dll (list-append flags (Cons file Nil))))))

;; Modules and everything they import, built in parallel, each as soon as what
;; it imports is built: the compiler's `--build-graph`. With `deps-only?` the
;; files themselves are left out, which is what a program's entry needs before
;; it is compiled as a program.
(: compile-graph #:sync (-> (List string) bool Against int))
(defun (compile-graph files deps-only? roots)
  (def flags (list-append (Cons "--build-graph" (Cons "--quiet" (if deps-only? (Cons "--deps-only" Nil) Nil)))
                          (against-flags roots)))
  (exec "dotnet" (list->vec (Cons compiler-dll (list-append flags files)))))

;; Every module of a library, in one compiler process, one after another.
;;
;; `--if-stale` leaves out what is current, and `--batch` pays the compiler's
;; half-second start once rather than once per module. What a debug build of a
;; library goes through: a graph build's workers build without `-d`.
(: compile-batch #:sync (-> (List string) bool Against int))
(defun (compile-batch files debug? roots)
  (def flags (list-append (Cons "--batch" (Cons "--if-stale" Nil))
                          (list-append (if debug? (Cons "-d" Nil) Nil)
                                       (against-flags roots))))
  (exec "dotnet" (list->vec (cons compiler-dll (list-append flags files)))))

;; The artefact as it already stands, for a file that needs no building.
;;
;; Only reached when `stale?` has answered from a record, so there is one to
;; read; `None` is the file having gone away between the two reads.
(: (-> string (Result int (Tuple BuildRecord bool))))
(defun (record-as-built file)
  (match (read-record file)
    ((Some rec) (Ok (rec . #f)))
    ;; Only reachable when `stale?` answered from a record that has gone away
    ;; since, and the exit code alone would be a command that failed without
    ;; saying anything at all.
    (None (Err (fail #"bjo: the build record for ${file} went away while it was being read.")))))

;; Whether what `file` imports should be built before it, by a graph build: when
;; nothing says what it was built from, or something other than the file itself
;; changed. An edit to the entry alone is the common case, and it compiles
;; straight away; the entry's own compile would build a stale import too, one
;; after another, which is what the graph saves.
(: (-> string bool))
(defun (imports-first? file)
  (def* ((Some rec) (read-record file) :leave-with #t)
        (artefact (record-ref rec output))
        (self (path-absolute file)))
  (or (not (file-exists? artefact))
      (any? #(and (not (= (path-absolute &) self)) (newer? & artefact)) (record-ref rec inputs))))

;; Compiles `file` if it needs it, and answers the record the build left — which
;; is where the artefact's name comes from. A file with no `main` is a library
;; whether or not `--lib` was passed, so the compiler is the only one that knows
;; whether it produced a `.exe` or a `.dll`.
;;
;; `Err` carries an exit code, so a failed compile leaves through here with the
;; compiler's own status rather than a made-up one. The `bool` says whether this
;; call is what built it.
(: ensure-built #:sync (-> string bool bool bool Against (Result int (Tuple BuildRecord bool))))
(defun (ensure-built file debug? lib? force? roots)
  (with-return ret
    (unless (have-compiler?) (ret (Err 1)))

    ;; A debug build is never served from the record: the dumps `-d` asks for
    ;; are produced by compiling, so a cached one has nothing to show.
    (def wanted (if debug? "debug" "release"))
    (unless (or force? debug? (stale? file wanted roots))
      (ret (record-as-built file)))

    (when (imports-first? file)
      (let ((imports (compile-graph (Cons file Nil) #t roots)))
        (unless (= imports 0) (ret (Err imports))))
      ;; What they changed may be nothing the entry was compiled against.
      (unless (or force? debug? (not (current-by-interface? file wanted roots)))
        (ret (record-as-built file))))

    (def status (compile file debug? lib? roots))
    (unless (= status 0) (ret (Err status)))

    ;; The compiler said it succeeded and left no record. It writes one after
    ;; every build that did, so this is a compiler too old for this `bjo`.
    (def (Some built) (read-record file)
      :leave (None (println #"bjo: ${file} compiled, but left no build record.")
                   (Err 1)))

    (Ok (Tuple built #t))))


;; ---------------------------------------------------------------------------
;; The commands
;; ---------------------------------------------------------------------------

;; Starts a built program with `arguments`: its Native AOT executable when the
;; record names one that is there, and the `.exe` under `dotnet` otherwise.
(: run-built #:sync (-> BuildRecord (List string) int))
(defun (run-built rec arguments)
  (def native (record-ref rec aot))
  (if (and (not (= native "")) (file-exists? native))
      (exec native (list->vec arguments))
      (exec "dotnet" (list->vec (Cons (record-ref rec output) arguments)))))

;; Everything after the source file belongs to the program, which is the only
;; reading that lets a program have flags of its own.
(: cmd-run #:sync (-> string (List string) bool bool Against int))
(defun (cmd-run file program-args debug? force? roots)
  (match (ensure-built file debug? #f force? roots)
    ((Err status) status)
    ((Ok (Tuple rec _)) #:when (string-ends-with? (record-ref rec output) ".dll")
     (println #"bjo: ${file} has no main, so it builds a library: ${(record-ref rec output)}")
     (println "bjo: there is nothing to run. Use 'bjo build' for a library.")
     1)
    ((Ok (Tuple rec _))
     (run-built rec program-args))))

(: cmd-build #:sync (-> string bool bool bool Against int))
(defun (cmd-build file debug? lib? force? roots)
  (match (ensure-built file debug? lib? force? roots)
    ((Err status) status)
    ((Ok (Tuple rec built?))
     (def what (if built? "Built" "Up to date"))
     (println #"${what}: ${(record-ref rec output)}")
     0)))

;; Nothing is written, so no record is read and none is left. A dependency is
;; still built — checking a module against an import means reading that import's
;; metadata, and the metadata is in the `.dll` — but that is the compiler's
;; business rather than this one's.
;;
;; Many files go to one compiler process: a cold one costs about half a second
;; before it does anything, and `--batch` pays that once instead of per file.
(: cmd-check #:sync (-> (List string) Against int))
(defun (cmd-check files roots)
  (cond
    ((not (have-compiler?)) 1)
    (else
     (def batch
       (match files
         ((Cons _ (Cons _ _)) (Cons "--batch" Nil))
         (_ Nil)))
     (exec "dotnet"
           (list->vec #fr(cons compiler-dll "--check"
                               (list-append (against-flags roots)
                                            (list-append batch files))))))))

;; `bjo test`: builds each test file and runs it, with `filters` as its
;; arguments, so that a test file ending in `(run-tests)` runs the tests whose
;; names contain one of them. A test file prints its failures and one line,
;; so this adds nothing for a single file; for several, one line at the end
;; says how many failed. Answers 1 if any file failed to build or to pass.
;;
;; `named?` says that the files were named on the command line. A file found
;; under tests/ that has no main is a module that the tests import, such as a
;; fixture, and is not run. A named file without a main is an error.
(: cmd-test #:sync (-> (List string) bool (List string) bool bool bool int))
(defun (cmd-test files named? filters debug? force? locked?)
  (defun (destructure-roots file)
    (def (Ok roots) (roots-for-file file locked?)
      :leave ((Err why) (Some (fail why))))
    (match (ensure-built file debug? #f force? roots)
      ((Err status) (Some status))
      ((Ok (Tuple rec _)) #:when (string-ends-with? (record-ref rec output) ".dll")
       (if named?
           (Some (fail #"bjo test: ${file} has no main. End a test file with (run-tests)."))
           None))
      ((Ok (Tuple rec _))
       (Some (exec "dotnet" (list->vec (Cons (record-ref rec output) filters)))))))
  (def statuses
    (loop (:for file files)
          (:let status (destructure-roots file))
          (:when-let (Some code) status)
          (:do (unless (= code 0) (println #"bjo test: ${file} failed.")))
          (:acc out (listing code))))
  (def failed (length (list-filter #(not (= & 0)) statuses)))
  (when (> (length statuses) 1)
    (println (if (= failed 0)
                 #"bjo test: ${(length statuses)} files, all passed."
                 #"bjo test: ${failed} of ${(length statuses)} files failed.")))
  (if (= failed 0) 0 1))

;; `bjo bench` is `bench.bjo`'s. What it needs from here is lent to it: a
;; benchmark file built in Release, whatever `-d` says; a program run on the
;; console; and a source file found.
(: build-bench #:sync (-> string bool bool (Result int BenchProgram)))
(defun (build-bench file force? locked?)
  (match (roots-for-file file locked?)
    ((Err why) (Err (fail why)))
    ((Ok roots)
     (match (ensure-built file #f #f force? roots)
       ((Err status) (Err status))
       ((Ok (Tuple rec _)) #:when (string-ends-with? (record-ref rec output) ".dll")
        (Err (fail #"bjo bench: ${file} has no main. End a benchmark file with (run-benches).")))
       ((Ok (Tuple rec _)) (Ok (BenchProgram (source file) (exe (record-ref rec output)))))))))

(: (-> bool bool BenchTools))
(defun (bench-tools force? locked?)
  (BenchTools (build (fun (file) (build-bench file force? locked?)))
              (exec exec)
              (resolve resolve-source)))

;; The compiler's REPL edits its own lines at a terminal, with completion and
;; history, so it is run as it is. Under a wrapper that edits lines itself,
;; such as `rlwrap`, set BJOLANG_REPL_PLAIN to have it read line by line.
(: cmd-repl #:sync (-> Against int))
(defun (cmd-repl roots)
  (def flags (against-flags roots))
  (if (not (have-compiler?))
      1
      (exec "dotnet" (list->vec (Cons compiler-dll
                                      (list-append flags (Cons "--repl" Nil)))))))


;; ---------------------------------------------------------------------------
;; Projects
;; ---------------------------------------------------------------------------

;; The flags a command was given, and what was left after them. Declared here
;; rather than beside the parser below it, because the project commands take
;; one and a signature cannot name a type that has not been declared yet.
(type
  (: Cmdline (Record (: debug bool)
                     (: force bool)
                     (: lib bool)
                     (: locked bool)
                     ;; `-o DIR`: where `publish` writes. Only `publish` takes it.
                     (: output (Option string))
                     ;; `--whole-stdlib`: `publish` copies all of the standard
                     ;; library, sources and documentation included.
                     (: whole-stdlib bool)
                     (: rest (List string)))))

;; The directory a file sits in, for the walk upward that finds a project.
(: (-> string string))
(defun (file-directory file)
  (match (parent-directory (path-absolute file))
    ((Some dir) dir)
    (None (path-absolute file))))

;; The project a directory is inside, if it is inside one.
(: project-around #:sync (-> string (Result string (Option Project))))
(defun (project-around where)
  (match (find-project-dir where)
    (None (Ok None))
    ((Some dir)
     (def (Ok project) (project-at dir) :propagate)
     (Ok (Some project)))))

;; The project a command that named no file was run in. Not finding one is an
;; error here, because the command had nothing else to work on.
(: project-here #:sync (-> string (Result string Project)))
(defun (project-here command)
  (def (Ok (Some project)) (project-around (current-directory))
    :leave ((Ok None)
            (Err (str #"bjo ${command}: this is not a project — there is no ${manifest-name} "
                      "here or in any directory above. Name a file, or make a project with 'bjo init'.")))
           ((Err why) (Err why)))
  (Ok project))

;; The roots a command compiles against, from where it was run.
;;
;; `None` outside a project, which is every command behaving as it did before
;; there were projects. Inside one, the project is resolved and fetched first:
;; a file in a project imports the project's packages whether it is reached as
;; `src/main.bjo` or as the entry.
(: roots-near #:sync (-> string bool (Result string Against)))
(defun (roots-near where locked?)
  (def (Ok (Some project)) (project-around where)
    :leave ((Ok None) (Ok nothing-against))
           ((Err why) (Err why)))
  (def (Ok ready) (prepare project locked?) :propagate)
  (Ok (prepared-against ready)))

(: roots-for-file #:sync (-> string bool (Result string Against)))
(defun (roots-for-file file locked?)
  (roots-near (file-directory file) locked?))

;; Builds a project, and answers the entry's build record where there is one.
;;
;; The entry decides the shape. A program is one compile of one file: the
;; compiler follows its imports and builds what they name, in order. A library
;; has no entry to follow, so every module under `src/` is named instead.
;;
;; `say?` is off for `run`, which has the program's own output to show and
;; nothing to add to it.
(: build-project #:sync (-> Project Cmdline bool (Result int (Option BuildRecord))))
(defun (build-project project line say?)
  (with-return ret
    (def (Ok ready) (prepare project (record-ref line locked))
      :leave ((Err why) (Err (fail why))))
    (def roots (prepared-against ready))

    (unless (project-has-entry? project)
      (ret (build-library project line roots say?)))

    (match (ensure-built (project-entry project)
                         (record-ref line debug) #f (record-ref line force) roots)
      ((Err status) (Err status))
      ((Ok (Tuple rec built?))
       (when say?
         (println (str (if built? "Built" "Up to date") ": " (record-ref rec output))))
       (Ok (Some rec))))))

(: (-> Project string))
(defun (package-name-of project)
  (name->string (record-ref (project-package project) name)))

(: build-library #:sync (-> Project Cmdline Against bool (Result int (Option BuildRecord))))
(defun (build-library project line roots say?)
  (with-return ret
    (def sources (sources-under (project-src project)))
    (when (empty? sources)
      (ret (Err (fail #"bjo build: there are no modules under ${(project-src project)}."))))

    (def status (if (record-ref line debug)
                    (compile-batch sources #t roots)
                    (compile-graph sources #f roots)))
    (unless (= status 0) (ret (Err status)))

    (when say?
      (def n (length sources))
      (println (str (package-name-of project) " is a library: "
                    (int->string n) (if (= n 1) " module" " modules")
                    " under src/, all current.")))
    (Ok None)))

(: build-and-say #:sync (-> Project Cmdline int))
(defun (build-and-say project line)
  (match (build-project project line #t)
    ((Err status) status)
    ((Ok _) 0)))

;; `bjo run` on a project: build it, then start the entry's artefact.
(: run-project #:sync (-> Project Cmdline (List string) int))
(defun (run-project project line arguments)
  (with-return ret
    (unless (project-has-entry? project)
      (ret (fail (str "bjo run: " (package-name-of project) " is a library — there is no "
                      (record-ref (project-package project) entry)
                      " to run. 'bjo build' builds it."))))

    (match (build-project project line #f)
      ((Err status) status)
      ;; A project with an entry always answers with its record.
      ((Ok None) 1)
      ((Ok (Some rec))
       (def out (record-ref rec output))
       (if (string-ends-with? out ".dll")
           (fail (str "bjo run: " (project-entry project) " has no main, so it builds a library: " out))
           (run-built rec arguments))))))

;; `bjo check` with no file, inside a project: everything the project holds.
(: check-project #:sync (-> Project Cmdline int))
(defun (check-project project line)
  (with-return ret
    (def (Ok ready) (prepare project (record-ref line locked))
      :leave ((Err why) (fail why)))
    (def sources (sources-under (project-src project)))
    (def files (if (and (project-has-entry? project)
                        (not (any? #(= & (project-entry project)) sources)))
                   (Cons (project-entry project) sources)
                   sources))
    (when (empty? files)
      (ret (fail #"bjo check: there are no modules under ${(project-src project)}.")))
    (cmd-check files (prepared-against ready))))


;; ---------------------------------------------------------------------------
;; Publishing
;; ---------------------------------------------------------------------------

;; The runtime the compiler links programs against, where the compiler finds
;; it (`Paths.runtimeDir`): the compiler's own directory, whose copies are
;; precompiled, or the Release build of BjolangRuntime when BJOLANG_NO_R2R
;; turns the precompiled code off.
(: string)
(def runtime-dir
  (let ((precompiled (path-combine root "bin" "compiler"))
        (off? (match (get-environment-variable "BJOLANG_NO_R2R")
                ((Some v) (not (or (= v "") (= v "0"))))
                (None #f))))
    (if (and (not off?) (file-exists? (path-combine precompiled "BjolangRuntime.dll")))
        precompiled
        (path-combine root "BjolangRuntime" "bin" "Release" target-framework))))

;; The packages a project was built from, with the resources their manifests
;; declare. The project itself is the one whose files go under `app/`.
(: (-> Project Prepared (List PublishedPackage)))
(defun (published-packages project ready)
  (def own-name (record-ref (project-package project) name))
  (loop (:for r (prepared-roots ready))
        (:when-let (Some dir) (root-package-dir r))
        (:let resources (match (root-package r)
                          ((Some package) (record-ref package resources))
                          (None Nil)))
        (:acc out (listing (PublishedPackage (name (root-name r))
                                             (dir dir)
                                             (resources resources)
                                             (own (= (root-name r) own-name)))))
    => out))

;; Publishes a program that has been built, and says where to.
(: publish-built #:sync (-> BuildRecord (List PublishedPackage) Cmdline string int))
(defun (publish-built rec packages line out)
  (def program (record-ref rec output))
  (if (string-ends-with? program ".dll")
      (fail (str "bjo publish: " program " has no main, so it is a library. Only a program can be published."))
      (match (publish (Publication (program program)
                                   (native-program (record-ref rec aot))
                                   (links (record-ref rec links))
                                   (natives (record-ref rec natives))
                                   (packages packages)
                                   (lib (lib-dir))
                                   (whole-stdlib (record-ref line whole-stdlib))
                                   (runtime runtime-dir)
                                   (out out)))
        ((Err why) (fail why))
        ((Ok n)
         (def dir (path-absolute out))
         (def (Some name) (path-filename program)
           :default program)
         ;; The copies of the program's modules, precompiled in place. The
         ;; builds stay IL, so that a rebuild after an edit runs no crossgen2.
         (def status (exec "dotnet" (list->vec (list compiler-dll "--precompile-published" dir))))
         (unless (= status 0)
           (println #"bjo publish: precompiling the published modules failed (exit code ${(int->string status)}). They are left as IL."))
         (println #"Published ${program} to ${dir}: ${n} files.")
         (def (Some native) (path-filename (record-ref rec aot))
           :default "")
         (if (= native "")
             (println #"Run it with: dotnet ${(path-combine dir name)}")
             (println #"Run it with: ${(path-combine dir native)}"))
         0))))

;; `bjo publish` in a project: a release build of its entry, then the copy.
(: publish-project #:sync (-> Project Cmdline int))
(defun (publish-project project line)
  (with-return ret
    (unless (project-has-entry? project)
      (ret (fail (str "bjo publish: " (package-name-of project) " is a library — there is no "
                      (record-ref (project-package project) entry) " to publish."))))
    (def (Ok ready) (prepare project (record-ref line locked))
      :leave ((Err why) (fail why)))
    (match (ensure-built (project-entry project) #f #f (record-ref line force) (prepared-against ready))
      ((Err status) status)
      ((Ok (Tuple rec _))
       (publish-built rec
                      (published-packages project ready)
                      line
                      (match (record-ref line output)
                        ((Some dir) dir)
                        (None (path-combine (project-dir project) "publish"))))))))

;; `bjo publish file.bjo`. A file inside a project is built against the
;; project's packages, and carries their resources, as `bjo build` would build
;; it; one outside a project has neither.
(: publish-file #:sync (-> string Cmdline int))
(defun (publish-file file line)
  (with-return ret
    (def (Ok around) (project-around (file-directory file))
      :leave ((Err why) (fail why)))
    (def context
      (match around
        (None (Ok None))
        ((Some project)
         (match (prepare project (record-ref line locked))
           ((Ok ready) (Ok (Some (Tuple project ready))))
           ((Err why) (Err why))))))
    (def (Ok prepared) context :leave ((Err why) (fail why)))
    (def roots (match prepared
                 ((Some (Tuple _ ready)) (prepared-against ready))
                 (None nothing-against)))
    (def packages (match prepared
                    ((Some (Tuple project ready)) (published-packages project ready))
                    (None Nil)))
    (match (ensure-built file #f #f (record-ref line force) roots)
      ((Err status) status)
      ((Ok (Tuple rec _))
       (publish-built rec
                      packages
                      line
                      (match (record-ref line output)
                        ((Some dir) dir)
                        (None (path-combine (file-directory file) "publish"))))))))


;; ---------------------------------------------------------------------------
;; The command line
;; ---------------------------------------------------------------------------

;; Flags come before the source file, and `--` ends them for a file whose name
;; begins with a dash. The first thing that is not a flag ends them too, and
;; everything from there on is positional — so `bjo run prog.bjo -d` passes `-d`
;; to the program rather than to the compiler.
;;
;; With `pass-unknown?`, a flag bjo does not know ends bjo's flags instead of
;; being an error: `bjo test --seed 7` gives `--seed 7` to the test files.
(: (-> (List string) bool (Result string Cmdline)))
(defun (parse-flags argv pass-unknown?)
  (let go ((left argv) (debug #f) (force #f) (lib #f) (locked #f) (output None) (whole-stdlib #f))
    (defun (done rest)
      (Cmdline (debug debug) (force force) (lib lib) (locked locked)
               (output output) (whole-stdlib whole-stdlib) (rest rest)))
    (match left
      (Nil (Ok (done Nil)))
      ((Cons a more)
       (case a
         ("--" (Ok (done more)))
         (("-d" "--debug") (go more #t force lib locked output whole-stdlib))
         (("-c" "--compile") (go more debug #t lib locked output whole-stdlib))
         ("--lib" (go more debug force #t locked output whole-stdlib))
         ("--locked" (go more debug force lib #t output whole-stdlib))
         ("--whole-stdlib" (go more debug force lib locked output #t))
         (("-o" "--output")
          (match more
            ((Cons dir after) (go after debug force lib locked (Some dir) whole-stdlib))
            (Nil (Err "bjo: -o needs a directory"))))
         (else
          (if (and (string-starts-with? a "-") (not pass-unknown?))
              (Err #"bjo: unknown flag '${a}'")
              (Ok (done left)))))))))

;; `bjo fmt`: each file laid out as bjo-mode lays it out (see `format.bjo`),
;; and written back if that changed it, or with `check?` only named. Answers 1
;; when a file could not be laid out, or, with `check?`, when one is not.
(: (-> (List string) bool bool int))
(defun (cmd-fmt files check? diff?)
  (def failures (list-foldl (fun (file n) (+ n (fmt-file file check? diff?))) 0 files))
  (if (> failures 0) 1 0))

;; Lays out one file, and answers 1 when that fails the command: the file
;; could not be laid out, or, with `check?`, it is not. A file that is laid out
;; already is not mentioned. With `diff?` nothing is written: what would change
;; is printed as a unified diff, which `patch -p0` takes, and so what is said
;; about a line goes to standard error, out of the diff's way.
(: (-> string bool bool int))
(defun (fmt-file file check? diff?)
  (def source (file-read-text file))
  (def shown (as-seen-from-here file))
  (match (format-source source)
    ((Err why) (println #"bjo fmt: ${shown}: ${why}") 1)
    ((Ok (Tuple text _)) #:when (= text source) 0)
    ((Ok (Tuple text long)) #:when diff?
     (display (unified-diff shown shown source text))
     (loop (:for at long)
           (:do (writeln (parameter-ref current-error-port) (noted shown at))))
     (if check? 1 0))
    ((Ok (Tuple _ long)) #:when check?
     (println #"${shown}: not formatted")
     (list-for-each #(println (noted shown &)) long)
     1)
    ((Ok (Tuple text long))
     (file-write-text file text)
     (println #"formatted ${shown}")
     (list-for-each #(println (noted shown &)) long)
     0)))

;; What `format-source` says about a line of `file`, as a line of output.
(: (-> string (Tuple int string) string))
(defun (noted file at)
  (def (Tuple line note) at)
  #"${file}:${(int->string line)}: ${note}")

;; A path under the current directory without it, as a person would type it.
(: (-> string string))
(defun (as-seen-from-here path)
  (def cwd (current-directory))
  (def here (if (string-ends-with? cwd "/") cwd (str cwd "/")))
  (if (string-starts-with? path here) (string-drop path (string-length here)) path))

;; `bjo fmt -`: standard input laid out to standard output, for an editor. What
;; is wrong goes to standard error, and nothing to standard output.
(: (-> bool int))
(defun (fmt-stdin diff?)
  (def input (get current-input-port))
  (def lines
    (let go ((acc Nil))
      (match (read-line/opt input)
        ((Some line) (go (Cons line acc)))
        (None (list-reverse acc)))))
  (def source (str (string-join (list->vec lines) "\n") "\n"))
  (match (format-source source)
    ((Ok (Tuple text long))
     (display (if diff? (unified-diff "-" "-" source text) text))
     (loop (:for at long)
           (:do (writeln (parameter-ref current-error-port) (noted "-" at))))
     0)
    ((Err why)
     (writeln (parameter-ref current-error-port) #"bjo fmt: ${why}")
     1)))

;; The files `bjo fmt` lays out: those named, a directory's named, or with
;; nothing named this project's src/, tests/ and benches/.
(: (-> (List string) (Result string (List string))))
(defun (fmt-targets named)
  (match named
    (Nil
     (def (Ok project) (project-here "fmt")
       :propagate)
     ;; TODO: this is not how to build a list :( 
     (Ok (loop (:for dir (list "src" "tests" "benches"))
               (:acc out (folding Nil (list-append out (sources-under (path-combine (project-dir project) dir)))))
           => out)))
    (_
     (defun (do-it name out)
       (def (Ok so-far) out :propagate)
       (cond
         ((directory-exists? name) (Ok (list-append so-far (sources-under name))))
         ((file-exists? name) (Ok (list-append so-far (list name))))
         (else (Err #"bjo fmt: no such file or directory '${name}'."))))
     
     (loop (:for name named)
           (:acc out (folding (Ok Nil) (do-it name  out)))
       => out))))

(: (-> void))
(defun (usage)
  (println "bjo - the Bjolang command")
  (println "")
  (println "Usage:")
  (println "  bjo init [name] [--lib]                 Make a project here. The name defaults to")
  (println "                                          this directory's; --lib makes a library.")
  (println "  bjo fetch [--locked]                    Fetch what the manifest names. Compiles nothing.")
  (println "  bjo run [-d] [-c] [args...]             Build this project and run it.")
  (println "  bjo run . [args...]                     The same, when the program takes arguments.")
  (println "  bjo run [-d] [-c] <file.bjo> [args...]  Compile one file if it needs it, then run it.")
  (println "                                          Arguments after the file go to the program.")
  (println "  bjo build [-d] [-c] [--locked]          Build this project.")
  (println "  bjo build [-d] [-c] [--lib] <file.bjo>  Compile one file if it needs it, and stop.")
  (println "  bjo check [<file.bjo>...]               Report errors without building anything.")
  (println "                                          With no file, every module of this project.")
  (println "  bjo test [-d] [-c] [files...] [names...]")
  (println "                                          Build and run test files: those named, or")
  (println "                                          every one under tests/. A name that is not")
  (println "                                          a file runs only the tests that contain it.")
  (println "  bjo bench [-c] [files...] [names...] [flags...]")
  (println "                                          Build benchmark files in Release and run them:")
  (println "                                          those named, or every one under benches/.")
  (println "                                          At a terminal with nothing named, a menu asks")
  (println "                                          what to run. The flags are the benchmark")
  (println "                                          programs': --all, --quick, --save FILE, ...")
  (println "  bjo publish [-c] [-o dir] [<file.bjo>]  Build a program for release, and copy it with")
  (println "                                          everything it loads into one directory that")
  (println "                                          runs from anywhere. Default: publish/ beside it.")
  (println "  bjo fmt [--check] [--diff] [files or dirs...]")
  (println "                                          Lay source out as bjo-mode does: those named,")
  (println "                                          or every file under src/, tests/ and benches/.")
  (println "                                          --check writes nothing and names what is not")
  (println "                                          laid out; --diff writes nothing and prints the")
  (println "                                          changes as a diff. bjo fmt - reads standard input.")
  (println "  bjo repl                                Start a REPL.")
  (println "  bjo rebjo                               Rebuild the compiler and bjo itself.")
  (println "  bjo help                                Show this message.")
  (println "")
  (println "Flags:")
  (println "  -d, --debug    Unoptimized, for a debugger.")
  (println "                 Always recompiles, since the dumps are made by compiling.")
  (println "  -c, --compile  Compile even when nothing has changed.")
  (println "  --lib          Build a .dll. A file with no main builds one anyway.")
  (println "  --locked       Refuse to change bjo.lock or packages.lock.json: any difference")
  (println "                 from them is an error. For a build that must be the one the")
  (println "                 locks describe.")
  (println "  -o <dir>       Where publish writes. An existing directory is replaced only")
  (println "                 when an earlier publish made it.")
  (println "  --whole-stdlib For publish: copy all of the standard library, sources and")
  (println "                 documentation included, for a program that reads Bjolang's")
  (println "                 own modules at run time. Otherwise only what it links.")
  (println "  -q, --quiet    Before the command: say nothing about building bjo itself")
  (println "                 unless that fails. For a tool reading what bjo writes.")
  (println "  --             End of flags, for a file whose name begins with a dash.")
  (println "")
  (println "A project is a directory with a manifest.bjodat in it, found by looking")
  (println "upward from where you stand. Its modules are in src/: src/lexer.bjo of a")
  (println "package called (myapp) is the module (myapp lexer). Outside a project every")
  (println "command works on the file it is given, as it always has.")
  (println "")
  (println "Nothing is rebuilt when the artefact is newer than everything the last")
  (println "build read: the compiler records that, and bjo compares timestamps.")
  (println "")
  (println "The compiler is not one of the things bjo keeps current: it is not asked")
  (println "whether it is behind its own sources, on the way to anything. Editing it")
  (println "is 'bjo rebjo', which builds it and this command — and that makes every")
  (println "artefact stale the way any other edit does."))

;; Every name resolved, or the first that names nothing. Resolved here rather
;; than left to the compiler, so that a misspelling is reported as one by the
;; command that was given it.
(: (-> string (List string) (Result string (List string))))
(defun (resolve-all command names)
  (with-return ret
    (Ok (loop (:for name names)
              (:let resolved (resolve-source name))
              ;; The one exit a loop cannot take by itself: not the end of the
              ;; loop but the end of the command. `ret` is a statement here
              ;; because that is the only place it may stand.
              (:do (unless (some? resolved)
                     (ret (Err #"bjo ${command}: no such file '${name}'."))))
              (:when-let (Some file) resolved)
              (:acc out (listing file))
          => out))))

(: dispatch #:sync (-> string Cmdline int))
(defun (dispatch command line)
  (def* (debug? (record-ref line debug))
        (force? (record-ref line force))
        (lib? (record-ref line lib))
        (locked? (record-ref line locked))
        (rest (record-ref line rest)))

  (case command
    ("init"
     (with-return ret
       (when (or debug? force? locked?)
         (ret (fail "bjo init: writes files and compiles nothing, so it takes no build flags.")))
       (cmd-init rest lib?)))

    ;; Resolve, fetch and write the roots file, and stop there. What `build`
    ;; does first, for a checkout that is about to be built offline or by
    ;; something that is not `bjo`.
    ("fetch"
     (with-return ret
       (when (or debug? force? lib?)
         (ret (fail "bjo fetch: compiles nothing, so it takes no build flags.")))
       (unless (empty? rest) (ret (fail "bjo fetch: takes no arguments.")))
       (def (Ok project) (project-here "fetch") :leave ((Err why) (fail why)))
       (def (Ok ready) (prepare project locked?) :leave ((Err why) (fail why)))
       (println (str (package-name-of project) ": "
                     (int->string (prepared-dependencies ready)) " dependencies ready."))
       0))

    ("run"
     (with-return ret
       (when lib?
         (ret (fail "bjo run: --lib builds a library, and a library does not run.")))
       ;; No file names this project. So does `.`, which exists only so that
       ;; arguments can follow it: everything after the file belongs to the
       ;; program, and a project with no file named has nothing to put them
       ;; after.
       (def (Cons name args) rest
         :leave (Nil (match (project-here "run")
                       ((Err why) (fail why))
                       ((Ok project) (run-project project line Nil)))))
       (when (= name ".")
         (ret (match (project-here "run")
                ((Err why) (fail why))
                ((Ok project) (run-project project line args)))))
       (def (Some file) (resolve-source name)
         :leave-with (fail #"bjo run: no such file '${name}'."))
       (def (Ok roots) (roots-for-file file locked?) :leave ((Err why) (fail why)))
       (cmd-run file args debug? force? roots)))

    ("build"
     (with-return ret
       (def (Cons name more) rest
         :leave (Nil (match (project-here "build")
                       ((Err why) (fail why))
                       ((Ok project) (build-and-say project line)))))
       (when (= name ".")
         (ret (cond
                ((not (empty? more))
                 (fail "bjo build: '.' is the whole project, so nothing may follow it."))
                (else (match (project-here "build")
                        ((Err why) (fail why))
                        ((Ok project) (build-and-say project line)))))))
       (unless (empty? more) (ret (fail "bjo build: takes one source file.")))
       (def (Some file) (resolve-source name)
         :leave-with (fail #"bjo build: no such file '${name}'."))
       (def (Ok roots) (roots-for-file file locked?) :leave ((Err why) (fail why)))
       (cmd-build file debug? lib? force? roots)))

    ("check"
     (with-return ret
       (when (or debug? force? lib?)
         (ret (fail "bjo check: builds nothing, so it takes no build flags.")))
       (when (empty? rest)
         (ret (match (project-here "check")
                ((Err why) (fail why))
                ((Ok project) (check-project project line)))))
       (def (Ok resolved) (resolve-all "check" rest)
         :leave ((Err message) (fail message)))
       (def (Cons first _) resolved
         :leave-with (fail "bjo check: no source file. Try 'bjo help'."))
       (def (Ok roots) (roots-for-file first locked?) :leave ((Err why) (fail why)))
       (cmd-check resolved roots)))

    ;; The test files named, or every one under the project's tests/. An
    ;; argument that is not a file is a filter on test names.
    ("test"
     (with-return ret
       (when lib? (ret (fail "bjo test: a test file is a program, so it takes no --lib.")))
       (def named (list-filter #(some? (resolve-source &)) rest))
       (def filters (list-filter #(none? (resolve-source &)) rest))
       (def found
         (if (empty? named)
             (match (project-here "test")
               ((Err why) (Err why))
               ((Ok project) (Ok (sources-under (path-combine (project-dir project) "tests")))))
             (Ok (loop (:for name named)
                       (:when-let (Some file) (resolve-source name))
                       (:acc out (listing file))))))
       (def (Ok files) found :leave ((Err why) (fail why)))
       (when (empty? files)
         (ret (fail "bjo test: there are no test files. Put them under tests/, or name them.")))
       (cmd-test files (not (empty? named)) filters debug? force? locked?)))

    ;; The benchmark files named, or every one under the project's benches/.
    ;; Always a Release build: a Debug one measures code the JIT did not
    ;; optimise.
    ("bench"
     (with-return ret
       (when lib? (ret (fail "bjo bench: a benchmark file is a program, so it takes no --lib.")))
       (when debug? (println "bjo bench: -d is ignored: benchmarks are always built in Release."))
       (def (Tuple named passed) (split-bench-args resolve-source rest))
       (def found
         (if (empty? named)
             (match (project-here "bench")
               ((Err why) (Err why))
               ((Ok project) (Ok (sources-under (path-combine (project-dir project) "benches")))))
             (Ok named)))
       (def (Ok files) found :leave ((Err why) (fail why)))
       (when (empty? files)
         (ret (fail "bjo bench: there are no benchmark files. Put them under benches/, or name them.")))
       (cmd-bench (bench-tools force? locked?) files passed)))

    ;; A release build of a program, and a copy of everything it loads in one
    ;; directory that can be moved anywhere. See `publish.bjo`.
    ("publish"
     (with-return ret
       (when (or debug? lib?)
         (ret (fail "bjo publish: publishes a release build of a program, so it takes no -d or --lib.")))
       (cond
         ((or (empty? rest) (= rest (list ".")))
          (match (project-here "publish")
            ((Err why) (fail why))
            ((Ok project) (publish-project project line))))
         ((= (length rest) 1)
          (def (Cons name _) rest :leave-with 1)
          (def (Some file) (resolve-source name)
            :leave-with (fail #"bjo publish: no such file '${name}'."))
          (publish-file file line))
         (else (fail "bjo publish: takes one source file, or none for this project.")))))

    ("repl"
     (with-return ret
       (when (or debug? force? lib?) (ret (fail "bjo repl: takes no build flags.")))
       (unless (empty? rest) (ret (fail "bjo repl: takes no arguments.")))
       ;; Inside a project the prompt sees the project's packages, which is
       ;; what makes `(import (myapp lexer))` work there.
       (def (Ok roots) (roots-near (current-directory) locked?)
         :leave ((Err why) (fail why)))
       (cmd-repl roots)))

    ;; Reached only when this binary was started directly. The launcher takes
    ;; `rebjo` before it gets here, and has to: the command rebuilds `bjo.exe`,
    ;; which is the file this process is running from.
    ("rebjo"
     (fail "bjo rebjo: run it through the bjo launcher — it rebuilds this binary, which is running."))

    ;; Source laid out as bjo-mode lays it out. See `format.bjo`.
    ("fmt"
     (with-return ret
       (when (or debug? force? lib?)
         (ret (fail "bjo fmt: builds nothing, so it takes no build flags.")))
       (def check? (not (empty? (list-filter #(= & "--check") rest))))
       (def diff? (not (empty? (list-filter #(= & "--diff") rest))))
       (def named (list-filter #(not (or (= & "--check") (= & "--diff"))) rest))
       (when (= named (list "-"))
         (ret (if check?
                  (fail "bjo fmt: - writes what it lays out, so it takes no --check.")
                  (fmt-stdin diff?))))
       (match (list-filter #(string-starts-with? & "-") named)
         ((Cons flag _) (ret (fail #"bjo fmt: unknown flag '${flag}'.")))
         (Nil (ignore 0)))
       (def (Ok files) (fmt-targets named) :leave ((Err why) (fail why)))
       (cmd-fmt files check? diff?)))

    (("help" "-h" "--help") (usage) 0)

    (else (fail #"bjo: '${command}' is not a command. Try 'bjo help'."))))

(defun (main args)
  ;; Before any git process is started. `(std run)` redirects a child's stdin
  ;; in order to write to it, so a repository that wants a password would read
  ;; from a pipe nobody writes to and wait there forever. With this set, git
  ;; fails instead, and says what it wanted.
  (set-environment-variable! "GIT_TERMINAL_PROMPT" "0")

  ;; The command comes first and is never a flag, so the flags of the command
  ;; are what follows it.
  (def (Cons command rest) (vec->list args)
    :leave (Nil (usage) 1))
  (match (parse-flags rest (or (= command "test") (= command "bench") (= command "fmt")))
    ((Err message) (fail message))
    ((Ok line) #:when (and (some? (record-ref line output)) (not (= command "publish")))
     (fail #"bjo ${command}: -o is for publish, which writes a directory; ${command} does not."))
    ((Ok line) (dispatch command line))))
;; bjo/diff.bjo — a unified diff of two texts, line by line, for
;; `bjo fmt --diff`.
;;
;; Made for what `bjo fmt` does to a file: it changes lines in place, joins a
;; line of closing brackets to the one before it, and drops blank lines. So
;; the lines are walked together, and where they part the next place two lines
;; in a row agree again is looked for a few lines ahead; where there is none,
;; the lines are taken as changed one for one, which is what a re-indented
;; block is. That is not always the shortest diff, but it is always a true one,
;; and it costs nothing on a long file.

(export unified-diff)

(import (std prelude))

;; What becomes of a line: kept, at its index in the old text and in the new;
;; or removed or added, at the index in the old and the new where it is.
(type (: Op (Union (: Same int int) (: Del int int) (: Add int int))))

;; How far ahead a place where the texts agree again is looked for.
(: reach int)
(def reach 40)

;; Lines of context around a change.
(: context int)
(def context 3)

(: lines (-> string (Vec string)))
(defun (lines s)
  (def parts (string-split s "\n"))
  (def n (vec-length parts))
  (if (and (> n 0) (string-empty? (vec-ref parts (- n 1)))) (vec-take parts (- n 1)) parts))

;; Whether old line `i` and new line `j` agree, and the lines after them too,
;; or one of the texts ends there: one equal line alone, a blank one or a `))`,
;; agrees anywhere.
(: agree? (-> (Vec string) (Vec string) int int bool))
(defun (agree? a b i j)
  (def n (vec-length a))
  (def m (vec-length b))
  (cond
    ((and (= i n) (= j m)) #t)
    ((or (>= i n) (>= j m)) #f)
    ((not (= (vec-ref a i) (vec-ref b j))) #f)
    ((or (= (+ i 1) n) (= (+ j 1) m)) #t)
    (else (= (vec-ref a (+ i 1)) (vec-ref b (+ j 1))))))

;; How many old lines to remove and new ones to add from `i` and `j` on for the
;; texts to agree again, the fewest first, if they do within `reach`.
(: resync (-> (Vec string) (Vec string) int int (Option (Tuple int int))))
(defun (resync a b i j)
  (let by-distance ((d 1))
    (if (> d reach)
        None
        (let by-split ((x 0))
          (cond
            ((> x d) (by-distance (+ d 1)))
            ((agree? a b (+ i x) (+ j (- d x))) (Some (Tuple x (- d x))))
            (else (by-split (+ x 1))))))))

;; `count` ops made by `make` from `from` on, onto `ops`, the last first.
(: ops-onto (-> (List Op) int int (-> int Op) (List Op)))
(defun (ops-onto ops from count make)
  (let go ((k 0) (ops ops))
    (if (>= k count) ops (go (+ k 1) (Cons (make (+ from k)) ops)))))

(: edit-script (-> (Vec string) (Vec string) (Vec Op)))
(defun (edit-script a b)
  (def n (vec-length a))
  (def m (vec-length b))
  (let go ((i 0) (j 0) (ops Nil))
    (cond
      ((and (>= i n) (>= j m)) (list->vec (list-reverse ops)))
      ((>= i n) (go i (+ j 1) (Cons (Add i j) ops)))
      ((>= j m) (go (+ i 1) j (Cons (Del i j) ops)))
      ((= (vec-ref a i) (vec-ref b j)) (go (+ i 1) (+ j 1) (Cons (Same i j) ops)))
      (else
       (match (resync a b i j)
         ((Some (Tuple x y))
          (def removed (ops-onto ops i x #(Del & j)))
          (go (+ i x) (+ j y) (ops-onto removed j y #(Add (+ i x) &))))
         (None (go (+ i 1) (+ j 1) (Cons (Add (+ i 1) j) (Cons (Del i j) ops)))))))))

(: changed? (-> Op bool))
(defun (changed? op)
  (match op
    ((Same _ _) #f)
    (_ #t)))

;; The old and the new index an op is at.
(: at-of (-> Op (Tuple int int)))
(defun (at-of op)
  (match op
    ((Same i j) (Tuple i j))
    ((Del i j) (Tuple i j))
    ((Add i j) (Tuple i j))))

;; The hunks: each as the first and the last op in it, changes no more than
;; twice the context apart sharing one.
(: hunks (-> (Vec Op) (List (Tuple int int))))
(defun (hunks ops)
  (def n (vec-length ops))
  (def changes (loop (:for k (range 0 n))
                     (:acc out (listing k) #:when (changed? (vec-ref ops k)))
                 => out))
  (defun (close-hunk first last) (Tuple (max0 (- first context)) (min-of (- n 1) (+ last context))))
  (match changes
    (Nil Nil)
    ((Cons start rest)
     (let go ((rest rest) (first start) (last start) (out Nil))
       (match rest
         (Nil (list-reverse (Cons (close-hunk first last) out)))
         ((Cons k more)
          (if (<= (- k last) (* 2 context))
              (go more first k out)
              (go more k k (Cons (close-hunk first last) out)))))))))

(: max0 (-> int int))
(defun (max0 x) (if (< x 0) 0 x))

(: min-of (-> int int int))
(defun (min-of x y) (if (< x y) x y))

;; `@@ -start,count +start,count @@`, where a side with no lines starts at the
;; line before.
(: hunk-header (-> (Vec Op) int int string))
(defun (hunk-header ops first last)
  (def (Tuple i j) (at-of (vec-ref ops first)))
  (def olds (loop (:for k (range first (+ last 1)))
                  (:acc c (counting k) #:when (match (vec-ref ops k) ((Add _ _) #f) (_ #t)))
              => c))
  (def news (loop (:for k (range first (+ last 1)))
                  (:acc c (counting k) #:when (match (vec-ref ops k) ((Del _ _) #f) (_ #t)))
              => c))
  (defun (side start count) #"${(int->string (if (= count 0) start (+ start 1)))},${(int->string count)}")
  #"@@ -${(side i olds)} +${(side j news)} @@")

;; The lines of one hunk: a run of changes as its removed lines and then its
;; added ones, as a unified diff writes it.
(: hunk-lines (-> (Vec string) (Vec string) (Vec Op) int int (List string)))
(defun (hunk-lines a b ops first last)
  ;; `out`, `dels` and `adds` are each the last first, so a run of changes goes
  ;; onto `out` as its additions and then its removals.
  (let go ((k first) (dels Nil) (adds Nil) (out Nil))
    (defun (flushed) (list-append adds (list-append dels out)))
    (if (> k last)
        (list-reverse (flushed))
        (match (vec-ref ops k)
          ((Same i _) (go (+ k 1) Nil Nil (Cons (str " " (vec-ref a i)) (flushed))))
          ((Del i _) (go (+ k 1) (Cons (str "-" (vec-ref a i)) dels) adds out))
          ((Add _ j) (go (+ k 1) dels (Cons (str "+" (vec-ref b j)) adds) out))))))

;; The changes from `old` to `new` as a unified diff, the old text named `from`
;; and the new `to`, or "" when they are the same.
(: unified-diff (-> string string string string string))
(defun (unified-diff from to old new)
  (def a (lines old))
  (def b (lines new))
  (def ops (edit-script a b))
  (def found (hunks ops))
  (cond
    ((= old new) "")
    ((empty? found)
     ;; The same lines, so what differs is the end of the file.
     #"--- ${from}\n+++ ${to}\n(the file ends in a newline, and it did not)\n")
    (else
     (def body
       (loop (:for h found)
             (:acc out (folding Nil
                                (match h
                                  ((Tuple first last)
                                   (list-append out
                                                (Cons (hunk-header ops first last)
                                                      (hunk-lines a b ops first last)))))))
         => out))
     (str #"--- ${from}\n+++ ${to}\n" (string-join (list->vec body) "\n") "\n"))))
;; bjo/format.bjo — `bjo fmt`: Bjolang source laid out as bjomode.el lays it
;; out.
;;
;; It re-indents and tidies, keeps every line break the source has but those
;; before a closing bracket, and adds the few that make code read the same
;; everywhere:
;;
;;   - every line is indented as `indent-region` in `bjo-mode` indents it;
;;   - a line of closing brackets goes up to the end of the line before it,
;;     Lisp style, unless that line ends in a comment;
;;   - a `def`'s failure part starts a line, as does each form of a `cond`,
;;     `begin`, `seq` or `try` over several lines, and each binding of a `let`
;;     whose bindings are over several lines (see "Reshaping");
;;   - a line wider than `wide` is broken in its lists, and a string that runs
;;     past the limit is split with a line continuation after the last space
;;     that fits (see "Breaking long lines");
;;   - trailing whitespace goes, as do spaces just inside a bracket; a tab
;;     becomes spaces, or `\t` in a string;
;;   - three or more blank lines in a row become two, and the file ends in one
;;     newline.
;;
;; The indentation follows Emacs's own algorithm rather than an idea of it,
;; because the two have to agree: `calculate-lisp-indent` for the column a
;; line's form lines up under, `lisp-indent-specform` and `lisp-indent-defform`
;; for a form with a body, the rules `bjo-indent-function` adds, and even the
;; column `lisp-indent-region` keeps for the following lines at the same depth.
;; The comments below say which of them each part is.
;;
;; A string's lines are text, not code. Its continued lines move with it, and
;; a long one is split, but what it stands for is never changed. What comes
;; out is checked against what went in: the same tokens in the same order, a
;; string compared by what it stands for, or the file is left alone.

(export format-source format-with wide)

(import (std prelude))

;; ---------------------------------------------------------------------------
;; Tokens

(type (: Tok (Union
              (: TOpen string)
              (: TClose string)
              ;; A symbol, a number, a keyword, a character, or a prefix such
              ;; as ' or #' — anything up to a space, a bracket, a quote or a
              ;; semicolon.
              (: TAtom string)
              ;; A string of any kind, as written, over several lines too.
              (: TStr string)
              (: TComment string)
              (: TSpace string)
              TNewline)))

(: text-between (-> (Vec char) int int string))
(defun (text-between cs from to)
  (def sb (stringbuilder-empty))
  (let go ((i from))
    (when (< i to)
      (stringbuilder-add! sb (vec-ref cs i))
      (go (+ i 1))))
  (stringbuilder->string sb))

(: char-at (-> (Vec char) int char))
(defun (char-at cs i)
  (if (< i (vec-length cs)) (vec-ref cs i) #\nul))

(: opening? (-> char bool))
(defun (opening? c) (or (= c #\() (= c #\[) (= c #\{)))

(: closing? (-> char bool))
(defun (closing? c) (or (= c #\)) (= c #\]) (= c #\})))

(: line-break? (-> (Vec char) int bool))
(defun (line-break? cs i)
  (or (= (char-at cs i) #\newline)
      (and (= (char-at cs i) #\return) (= (char-at cs (+ i 1)) #\newline))))

(: blank-char? (-> (Vec char) int bool))
(defun (blank-char? cs i)
  (and (< i (vec-length cs)) (char-whitespace? (vec-ref cs i)) (not (line-break? cs i))))

;; What the compiler's lexer counts as part of a name: a character literal's
;; name, `#\space`, is a run of them.
(: symbol-char? (-> char bool))
(defun (symbol-char? c)
  (not (or (char-whitespace? c) (opening? c) (closing? c)
           (= c #\,) (= c #\:) (= c #\") (= c #\;) (= c #\'))))

(: atom-end (-> (Vec char) int int))
(defun (atom-end cs i)
  (def c (char-at cs i))
  (if (or (>= i (vec-length cs))
          (char-whitespace? c) (opening? c) (closing? c) (= c #\") (= c #\;))
      i
      (atom-end cs (+ i 1))))

(: blank-end (-> (Vec char) int int))
(defun (blank-end cs i)
  (if (blank-char? cs i) (blank-end cs (+ i 1)) i))

(: line-end (-> (Vec char) int int))
(defun (line-end cs i)
  (if (or (>= i (vec-length cs)) (line-break? cs i)) i (line-end cs (+ i 1))))

(: after-break (-> (Vec char) int int))
(defun (after-break cs i)
  (if (= (char-at cs i) #\return) (+ i 2) (+ i 1)))

(: quotes-at (-> (Vec char) int int))
(defun (quotes-at cs i)
  (let go ((p i))
    (if (= (char-at cs p) #\") (go (+ p 1)) (- p i))))

(: line-of (-> (Vec char) int int))
(defun (line-of cs i)
  (let go ((k 0) (line 1))
    (cond
      ((>= k i) line)
      ((= (vec-ref cs k) #\newline) (go (+ k 1) (+ line 1)))
      (else (go (+ k 1) line)))))

(: unterminated (-> (Vec char) int string (Result string int)))
(defun (unterminated cs i what)
  (Err #"line ${(int->string (line-of cs i))}: ${what} that does not end"))

;; Past the end of the string at `i`, which is one of the three kinds the
;; lexer reads: plain, raw on one line, or raw over several lines, which ends
;; on the first line whose text begins with as many quotes as opened it.
(: string-end (-> (Vec char) int (Result string int)))
(defun (string-end cs i)
  (def n (vec-length cs))
  (def k (quotes-at cs i))
  (def opened (+ i k))
  (def w (blank-end cs opened))
  (cond
    ((and (>= k 3) (>= w n)) (unterminated cs i "a raw string"))
    ((and (>= k 3) (line-break? cs w))
     (let lines ((p (after-break cs w)))
       (def start (blank-end cs p))
       (def e (line-end cs p))
       (cond
         ((>= p n) (unterminated cs i "a raw string"))
         ((= (quotes-at cs start) k) (Ok (+ start k)))
         ((>= e n) (unterminated cs i "a raw string"))
         (else (lines (after-break cs e))))))
    ((>= k 3)
     (let one ((p opened))
       (def run (quotes-at cs p))
       (cond
         ((or (>= p n) (line-break? cs p)) (unterminated cs i "a raw string"))
         ((>= run k) (Ok (+ p k)))
         ((> run 0) (one (+ p run)))
         (else (one (+ p 1))))))
    (else
     (let plain ((p (+ i 1)))
       (cond
         ((>= p n) (unterminated cs i "a string"))
         ((= (vec-ref cs p) #\\) (plain (+ p 2)))
         ((= (vec-ref cs p) #\") (Ok (+ p 1)))
         (else (plain (+ p 1))))))))

;; Past the end of the `#"..."` at `i`. A `${...}` hole is code, delimited by
;; brace depth, with strings and `#\{` inside it skipped whole, as the lexer
;; does.
(: interpolated-end (-> (Vec char) int (Result string int)))
(defun (interpolated-end cs i)
  (def n (vec-length cs))
  (let outside ((p (+ i 2)))
    (cond
      ((>= p n) (unterminated cs i "an interpolated string"))
      ((= (vec-ref cs p) #\") (Ok (+ p 1)))
      ((= (vec-ref cs p) #\\) (outside (+ p 2)))
      ((and (= (vec-ref cs p) #\$) (= (char-at cs (+ p 1)) #\{))
       (let hole ((q (+ p 2)) (depth 1))
         (def c (char-at cs q))
         (cond
           ((>= q n) (unterminated cs i "an interpolated string"))
           ((= c #\")
            (match (string-end cs q)
              ((Ok e) (hole e depth))
              ((Err why) (Err why))))
           ((and (= c #\#) (= (char-at cs (+ q 1)) #\\)) (hole (+ q 3) depth))
           ((= c #\{) (hole (+ q 1) (+ depth 1)))
           ((and (= c #\}) (= depth 1)) (outside (+ q 1)))
           ((= c #\}) (hole (+ q 1) (- depth 1)))
           (else (hole (+ q 1) depth)))))
      (else (outside (+ p 1))))))

;; Past the end of the character literal at `i`: a name, `#\space`, or any one
;; character, `#\(` and `#\;` included.
(: char-literal-end (-> (Vec char) int int))
(defun (char-literal-end cs i)
  (def start (+ i 2))
  (def name-end
    (let go ((p start))
      (if (and (< p (vec-length cs)) (symbol-char? (vec-ref cs p))) (go (+ p 1)) p)))
  (cond
    ((> (- name-end start) 1) name-end)
    ((< start (vec-length cs)) (+ start 1))
    (else start)))

;; The token at `i`, and where the next one starts.
(: token-at (-> (Vec char) int (Result string (Tuple Tok int))))
(defun (token-at cs i)
  (def c (vec-ref cs i))
  (defun (to e) (Tuple (TAtom (text-between cs i e)) e))
  (cond
    ((line-break? cs i) (Ok (Tuple TNewline (after-break cs i))))
    ((blank-char? cs i)
     (def e (blank-end cs i))
     (Ok (Tuple (TSpace (text-between cs i e)) e)))
    ((= c #\;)
     (def e (line-end cs i))
     (Ok (Tuple (TComment (string-trim-right (text-between cs i e))) e)))
    ((opening? c) (Ok (Tuple (TOpen (char->string c)) (+ i 1))))
    ((closing? c) (Ok (Tuple (TClose (char->string c)) (+ i 1))))
    ((= c #\")
     (def (Ok e) (string-end cs i)
       :leave ((Err why) (Err why)))
     (Ok (Tuple (TStr (without-continued-indent (text-between cs i e) (column-at cs i))) e)))
    ((and (= c #\#) (= (char-at cs (+ i 1)) #\"))
     (def (Ok e) (interpolated-end cs i)
       :leave ((Err why) (Err why)))
     (Ok (Tuple (TStr (without-continued-indent (text-between cs i e) (column-at cs i))) e)))
    ((and (= c #\#) (= (char-at cs (+ i 1)) #\\)) (Ok (to (char-literal-end cs i))))
    (else (Ok (to (atom-end cs i))))))

(: tokenize (-> (Vec char) (Result string (List Tok))))
(defun (tokenize cs)
  (let go ((i 0) (out Nil))
    (if (>= i (vec-length cs))
        (Ok (list-reverse out))
        (match (token-at cs i)
          ((Err why) (Err why))
          ((Ok (Tuple tok next)) (go next (Cons tok out)))))))

;; The tokens that make the program: everything but the spacing, and a string
;; with its tabs written `\t`, as the layout writes them, and its continued
;; lines joined, since the layout may split a long one. A comment's words are
;; compared, its tabs being spaces once laid out.
(: significant (-> (List Tok) (List string)))
(defun (significant toks)
  (loop (:for t toks)
        (:acc out (listing (match t
                             ((TStr s) (escape-tabs (joined-continuations s)))
                             ((TComment s) (string-replace (string-replace s "\t" "") " " ""))
                             (_ (tok-text t))))
              #:when (meaningful? t))
    => out))

(: meaningful? (-> Tok bool))
(defun (meaningful? t)
  (match t
    ((TSpace _) #f)
    (TNewline #f)
    (_ #t)))

(: tok-text (-> Tok string))
(defun (tok-text t)
  (match t
    ((TOpen s) s)
    ((TClose s) s)
    ((TAtom s) s)
    ((TStr s) s)
    ((TComment s) s)
    ((TSpace s) s)
    (TNewline "\n")))

;; ---------------------------------------------------------------------------
;; Reshaping: the line breaks the layout adds
;;
;; Each rule says where a line has to break, and what it adds is a line break,
;; so that the tokens are what they were:
;;
;;   - a `def`'s failure part, `:leave`, `:leave-with`, `:default` or
;;     `:propagate`, starts a line of its own, so that the way out stands out;
;;   - in a `cond`, a `begin`, a `seq` or a `try` over several lines, each form
;;     after the head starts a line, but a keyword argument's value, which
;;     stays on the keyword's line: `#:finally (restore)`;
;;   - in a `let`'s bindings over several lines, each binding starts a line.
;;
;; A tab in a string, which is otherwise left as it is, is written `\t`.

;; The line breaks in a token.
(: breaks-in (-> Tok int))
(defun (breaks-in t)
  (match t
    (TNewline 1)
    ((TStr s) (snd (advance s 0)))
    (_ 0)))

;; For each token, the index of the token that closes it if it opens a list,
;; and how many line breaks come before it.
(: closers-and-lines (-> (Vec Tok) (Tuple (Vec int) (Vec int))))
(defun (closers-and-lines v)
  (def n (vec-length v))
  (def closes (vecbuilder-empty))
  (def lines (vecbuilder-empty))
  (let fill ((k 0) (line 0))
    (when (< k n)
      (vecbuilder-add! closes -1)
      (vecbuilder-add! lines line)
      (fill (+ k 1) (+ line (breaks-in (vec-ref v k))))))
  (let go ((k 0) (open Nil))
    (when (< k n)
      (match (vec-ref v k)
        ((TOpen _) (go (+ k 1) (Cons k open)))
        ((TClose _)
         (match open
           ((Cons o outer)
            (vecbuilder-set! closes o k)
            (go (+ k 1) outer))
           (Nil (go (+ k 1) open))))
        (_ (go (+ k 1) open)))))
  (Tuple (vecbuilder->vec closes) (vecbuilder->vec lines)))

;; A list being reshaped: its bracket, its head, what its second form is if it
;; is an atom, its last form so far, how many forms it has so far, whether it
;; spans lines, and whether it is the bindings of a `let`, the forms of which
;; go a line each.
(type (: Shape (Record (: bracket string)
                       (: head string)
                       (: second string)
                       (: last string)
                       (: count int)
                       (: spans bool)
                       (: bindings bool))))

;; Whether a form written `raw` is a keyword argument, `#:catch`, which the
;; form after it belongs with: the two stay on one line.
(: keyword-arg? (-> string bool))
(defun (keyword-arg? raw)
  (and (string-starts-with? raw "#:") (> (string-length raw) 2)))

(: body-only? (-> string bool))
(defun (body-only? head)
  (match head
    ((or "cond" "begin" "seq" "try") #t)
    (_ #f)))

;; Whether `s` is a `def` clause, of a `def`, a `:def` or a `def*`, in which a
;; failure part may be.
(: clause? (-> Shape (Option Shape) bool))
(defun (clause? s outer)
  (or (= (record-ref s head) "def")
      (= (record-ref s head) ":def")
      (and (= (record-ref s bracket) "(")
           (not (= (record-ref s head) "def*"))
           (match outer
             ((Some o) (= (record-ref o head) "def*"))
             (None #f)))))

;; Whether a list opening as the next form of `parent` is that form's bindings.
(: opens-bindings? (-> Shape string bool))
(defun (opens-bindings? parent bracket)
  (def head (record-ref parent head))
  (def named?
    (and (= head "let")
         (match (string->list (record-ref parent second))
           ((Cons c _) (char-alphabetic? c))
           (Nil #f))))
  (and (= bracket "(")
       (binding-form? head)
       (= (record-ref parent count) (if named? 2 1))))

;; `s` with one more form, written `raw`.
(: with-form (-> Shape string Shape))
(defun (with-form s raw)
  (def n (record-ref s count))
  (Shape (bracket (record-ref s bracket))
         (head (if (= n 0) raw (record-ref s head)))
         (second (if (= n 1) raw (record-ref s second)))
         (last raw)
         (count (+ n 1))
         (spans (record-ref s spans))
         (bindings (record-ref s bindings))))

;; Whether the form at position `n` of `s`, written `raw`, has to start a line.
;; One after a keyword argument stays with it.
(: breaks-before? (-> Shape (Option Shape) int string bool))
(defun (breaks-before? s outer n raw)
  (or (and (failure-keyword? raw) (clause? s outer))
      (and (>= n 1)
           (record-ref s spans)
           (or (body-only? (record-ref s head)) (record-ref s bindings))
           (not (keyword-arg? (record-ref s last))))))

;; The tokens with the line breaks the rules ask for, and their strings' tabs
;; written `\t`.
(: reshape (-> (List Tok) (List Tok)))
(defun (reshape toks)
  (def v (list->vec toks))
  (def n (vec-length v))
  (def (Tuple closes lines) (closers-and-lines v))
  (defun (spans? k)
    (def c (vec-ref closes k))
    (and (>= c 0) (> (vec-ref lines c) (vec-ref lines k))))
  ;; `shapes` are the lists open, the innermost first; `fresh?` whether
  ;; nothing but space is on the line yet; `prefixed?` whether a prefix
  ;; began the form this token belongs to.
  (let go ((k 0) (shapes Nil) (fresh? #t) (prefixed? #f) (out Nil))
    (if (>= k n)
        (list-reverse out)
        (let ((t (vec-ref v k)))
          (def rest (if (< (+ k 1) n) (list (vec-ref v (+ k 1))) Nil))
          (def starts? (and (starts-form? t) (not prefixed?)))
          (def raw
            (match t
              ((TStr _) "\"")
              (_ (tok-text t))))
          (def break?
            (and starts?
                 (not fresh?)
                 (match shapes
                   ((Cons s outer)
                    (breaks-before? s (match outer ((Cons o _) (Some o)) (Nil None)) (record-ref s count) raw))
                   (Nil #f))))
          (def counted
            (match shapes
              ((Cons s outer) #:when starts? (Cons (with-form s raw) outer))
              (_ shapes)))
          (def shapes-after
            (match t
              ((TOpen b)
               (Cons (Shape (bracket b) (head "") (second "") (last "") (count 0) (spans (spans? k))
                            (bindings (match shapes
                                        ((Cons s _) (opens-bindings? s b))
                                        (Nil #f))))
                     counted))
              ((TClose _)
               (match counted
                 ((Cons _ outer) outer)
                 (Nil Nil)))
              (_ counted)))
          (def written
            (match t
              ((TStr s) (TStr (escape-tabs s)))
              (_ t)))
          (def out-after (Cons written (if break? (Cons TNewline out) out)))
          (go (+ k 1)
              shapes-after
              (match t
                (TNewline #t)
                ((TSpace _) fresh?)
                (_ #f))
              (prefix-of-next? t rest)
              out-after)))))

;; The tokens reshaped until no rule asks for more. A line break one rule adds
;; can make a list span lines, which another rule asks a line break of: a
;; failure part inside a `begin` on one line. The rules only add line breaks,
;; so this ends.
(: reshaped (-> (List Tok) (List Tok)))
(defun (reshaped toks)
  (def again (reshape toks))
  (if (= (list-length again) (list-length toks)) again (reshaped again)))

;; A string's tabs written `\t`, as its value is unchanged by: outside the
;; holes of an interpolated string, and never in a raw one, which has no
;; escapes. A tab after a backslash is kept: the continuations are joined
;; already, so the lexer rejects it, and fmt does not make it mean something.
(: escape-tabs (-> string string))
(defun (escape-tabs s)
  (cond
    ((string-starts-with? s "\"\"\"") s)
    ((not (string-contains? s "\t")) s)
    (else
     (def cs (string->vec s))
     (def n (vec-length cs))
     (def sb (stringbuilder-empty))
     (defun (copy from to) (stringbuilder-add-string! sb (text-between cs from to)))
     (let go ((p 0))
       (when (< p n)
         (def c (vec-ref cs p))
         (cond
           ((= c #\\)
            (copy p (+ p 2))
            (go (+ p 2)))
           ((= c #\tab)
            (stringbuilder-add-string! sb "\\t")
            (go (+ p 1)))
           ((and (= c #\$) (= (char-at cs (+ p 1)) #\{) (string-starts-with? s "#\""))
            (def past (hole-end cs (+ p 2)))
            (copy p past)
            (go past))
           (else
            (stringbuilder-add! sb c)
            (go (+ p 1))))))
     (stringbuilder->string sb))))

;; ---------------------------------------------------------------------------
;; Line continuations
;;
;; A backslash that ends a line in a string stands for nothing, and neither
;; does the next line's indentation up to the column the string's text starts
;; at; indentation beyond that is text. What a continued line says therefore
;; depends on where the string starts, which the layout moves. So a string is
;; read without the indentation a continuation drops, and written with as much
;; as where it is written drops: the text it stands for stays the same.
;;
;; A raw string has no escapes. In the holes of an interpolated string nothing
;; is changed: a string in a hole that continues lines is left as it is.

;; The column of `i` as the compiler's lexer counts it: characters since the
;; line began, a tab one.
(: column-at (-> (Vec char) int int))
(defun (column-at cs i)
  (let go ((k i))
    (if (or (= k 0) (= (vec-ref cs (- k 1)) #\newline)) (- i k) (go (- k 1)))))

;; The column the text of the string `s` written at `col` starts at.
(: text-column (-> string int int))
(defun (text-column s col)
  (if (string-starts-with? s "#\"") (+ col 2) (+ col 1)))

;; Whether the backslash at `p` is a line continuation, and if so where the
;; line break after it starts.
(: continuation-at (-> (Vec char) int (Option int)))
(defun (continuation-at cs p)
  (let go ((k (+ p 1)))
    (cond
      ((line-break? cs k) (Some k))
      ((or (= (char-at cs k) #\space) (= (char-at cs k) #\tab)) (go (+ k 1)))
      (else None))))

;; The string `s`, written with its text from `text-col` on, each continuation
;; as `\` and the line break alone, and from each continued line the
;; indentation it drops: what it stands for wherever it is written. Indentation
;; it keeps is kept.
(: without-continued-indent (-> string int string))
(defun (without-continued-indent s col)
  (def text-col (text-column s col))
  (rewrite-continuations s #t (fun (sb cs at)
                             (let skip ((j at))
                               (if (and (< (- j at) text-col)
                                        (or (= (char-at cs j) #\space) (= (char-at cs j) #\tab)))
                                   (skip (+ j 1))
                                   j)))))

;; The string `s`, as `without-continued-indent` reads it, written at `col`:
;; each continued line indented to where its text starts.
(: with-continued-indent (-> string int string))
(defun (with-continued-indent s col)
  (def text-col (text-column s col))
  (rewrite-continuations s #t (fun (sb cs at)
                             (unless (or (>= at (vec-length cs)) (line-break? cs at))
                               (stringbuilder-add-string! sb (spaces text-col)))
                             at)))

;; `s` with each continuation written as `\` and its line break, and after it
;; what `after` writes, given where the continued line starts; it answers where
;; to go on from. Without `keep?` a continuation's `\` and line break are left
;; out, which joins the lines. A raw string is as it is.
(: rewrite-continuations (-> string bool (-> StringBuilder (Vec char) int int) string))
(defun (rewrite-continuations s keep? after)
  (cond
    ((string-starts-with? s "\"\"\"") s)
    ((not (string-contains? s "\\")) s)
    (else
     (def cs (string->vec s))
     (def n (vec-length cs))
     (def sb (stringbuilder-empty))
     (def interpolated? (string-starts-with? s "#\""))
     (let go ((p 0))
       (when (< p n)
         (def c (vec-ref cs p))
         (cond
           ((= c #\\)
            (match (continuation-at cs p)
              ((Some k)
               (def past (if (= (char-at cs k) #\return) (+ k 2) (+ k 1)))
               (when keep?
                 (stringbuilder-add! sb #\\)
                 (stringbuilder-add-string! sb (text-between cs k past)))
               (go (after sb cs past)))
              (None
               (stringbuilder-add-string! sb (text-between cs p (+ p 2)))
               (go (+ p 2)))))
           ((and interpolated? (= c #\$) (= (char-at cs (+ p 1)) #\{))
            (def past (hole-end cs (+ p 2)))
            (stringbuilder-add-string! sb (text-between cs p past))
            (go past))
           (else
            (stringbuilder-add! sb c)
            (go (+ p 1))))))
     (stringbuilder->string sb))))

;; The string `s`, as `without-continued-indent` reads it, with its continued
;; lines joined: what it stands for, written without continuations, which a
;; string split by the layout is compared by.
(: joined-continuations (-> string string))
(defun (joined-continuations s)
  (rewrite-continuations s #f (fun (sb cs at) at)))

;; A token as it is written at `col`: a string with its continued lines
;; indented to where its text then starts.
(: tok-written (-> Tok int string))
(defun (tok-written t col)
  (match t
    ((TStr s) (with-continued-indent s col))
    (_ (tok-text t))))

;; Past the `}` closing the hole whose code starts at `q`, as
;; `interpolated-end` reads one.
(: hole-end (-> (Vec char) int int))
(defun (hole-end cs q)
  (let hole ((q q) (depth 1))
    (def c (char-at cs q))
    (cond
      ((>= q (vec-length cs)) q)
      ((= c #\")
       (match (string-end cs q)
         ((Ok e) (hole e depth))
         ((Err _) (vec-length cs))))
      ((and (= c #\#) (= (char-at cs (+ q 1)) #\\)) (hole (+ q 3) depth))
      ((= c #\{) (hole (+ q 1) (+ depth 1)))
      ((and (= c #\}) (= depth 1)) (+ q 1))
      ((= c #\}) (hole (+ q 1) (- depth 1)))
      (else (hole (+ q 1) depth)))))

;; ---------------------------------------------------------------------------
;; Lines

;; A line's tokens, its own indentation taken off, and that indentation as it
;; was, which a `;;;` comment keeps.
(type (: Line (Record (: toks (List Tok)) (: indent string))))

(: lines-of (-> (List Tok) (List Line)))
(defun (lines-of toks)
  (let go ((toks toks) (here Nil) (out Nil))
    (match toks
      (Nil (list-reverse (Cons (tidy (list-reverse here)) out)))
      ((Cons TNewline rest) (go rest Nil (Cons (tidy (list-reverse here)) out)))
      ((Cons t rest) (go rest (Cons t here) out)))))

(: space? (-> Tok bool))
(defun (space? t)
  (match t
    ((TSpace _) #t)
    (_ #f)))

(: comment? (-> Tok bool))
(defun (comment? t)
  (match t
    ((TComment _) #t)
    (_ #f)))

(: close? (-> Tok bool))
(defun (close? t)
  (match t
    ((TClose _) #t)
    (_ #f)))

;; A line without the spaces at its ends and just inside its brackets.
(: tidy (-> (List Tok) Line))
(defun (tidy toks)
  (def indent
    (match toks
      ((Cons (TSpace s) _) s)
      (_ "")))
  (def trimmed (list-reverse (drop-spaces (list-reverse (drop-spaces toks)))))
  (Line (toks (inside-brackets trimmed)) (indent indent)))

(: drop-spaces (-> (List Tok) (List Tok)))
(defun (drop-spaces toks)
  (match toks
    ((Cons t rest) #:when (space? t) (drop-spaces rest))
    (_ toks)))

(: inside-brackets (-> (List Tok) (List Tok)))
(defun (inside-brackets toks)
  (match toks
    ((Cons (TOpen o) (Cons s rest)) #:when (and (space? s) (not (starts-comment? rest)))
     (inside-brackets (Cons (TOpen o) rest)))
    ((Cons s (Cons (TClose c) rest)) #:when (space? s)
     (inside-brackets (Cons (TClose c) rest)))
    ((Cons t rest) (Cons t (inside-brackets rest)))
    (Nil Nil)))

(: starts-comment? (-> (List Tok) bool))
(defun (starts-comment? toks)
  (match toks
    ((Cons t _) (comment? t))
    (Nil #f)))

(: blank? (-> Line bool))
(defun (blank? l) (empty? (record-ref l toks)))

(: drop-closers (-> (List Tok) (List Tok)))
(defun (drop-closers toks)
  (match toks
    ((Cons t rest) #:when (close? t) (drop-closers rest))
    (_ toks)))

;; Closing brackets alone on a line, perhaps with a comment after them.
(: closers-only? (-> Line bool))
(defun (closers-only? l)
  (def toks (record-ref l toks))
  (and (match toks
         ((Cons t _) (close? t))
         (Nil #f))
       (match (drop-closers toks)
         (Nil #t)
         ((Cons t Nil) (comment? t))
         ((Cons s (Cons t Nil)) (and (space? s) (comment? t)))
         (_ #f))))

(: ends-in-comment? (-> Line bool))
(defun (ends-in-comment? l)
  (match (list-reverse (record-ref l toks))
    ((Cons t _) (comment? t))
    (Nil #f)))

(: drop-blank-lines (-> (List Line) (List Line)))
(defun (drop-blank-lines lines)
  (match lines
    ((Cons l rest) #:when (blank? l) (drop-blank-lines rest))
    (_ lines)))

;; Lines of closing brackets go up to the end of the code before them, the
;; blank lines between going with them, unless that code ends in a comment.
(: join-closers (-> (List Line) (List Line)))
(defun (join-closers lines)
  ;; `done` is the lines so far, the last first.
  (let go ((lines lines) (done Nil))
    (match lines
      (Nil (list-reverse done))
      ((Cons l rest)
       (match (drop-blank-lines done)
         ((Cons before earlier) #:when (and (closers-only? l) (not (ends-in-comment? before)))
          (def joined (list-append (record-ref before toks) (record-ref l toks)))
          (go rest (Cons (record-set before (toks joined)) earlier)))
         (_ (go rest (Cons l done))))))))

;; No blank lines at the ends, and at most two in a row anywhere else.
(: squeeze-blank-lines (-> (List Line) (List Line)))
(defun (squeeze-blank-lines lines)
  (def trimmed (list-reverse (drop-blank-lines (list-reverse (drop-blank-lines lines)))))
  (let go ((lines trimmed) (run 0) (out Nil))
    (match lines
      (Nil (list-reverse out))
      ((Cons l rest) #:when (blank? l)
       (if (>= run 2) (go rest run out) (go rest (+ run 1) (Cons l out))))
      ((Cons l rest) (go rest 0 (Cons l out))))))

;; ---------------------------------------------------------------------------
;; Indentation

;; A form inside a list, as the indentation reads it: where it starts, a prefix
;; such as ' or #' included; the physical line it starts on; the column of the
;; first form that starts on that line; its text if it is an atom, or its
;; opening bracket; and for a list the column of that bracket.
(type (: Elem (Record (: col int)
                      (: line int)
                      (: line-first int)
                      (: raw string)
                      (: is-list bool)
                      (: bracket int))))

;; An open list: its bracket, the bracket's column and line, the forms in it so
;; far, the last first, and the first two of them.
(type (: Frame (Record (: bracket string)
                       (: col int)
                       (: line int)
                       (: elems (List Elem))
                       (: count int)
                       (: first (Option Elem))
                       (: second (Option Elem)))))

;; A column, and whether `lisp-indent-region` keeps it for the following lines
;; at the same depth, which it does for a plain number and not for the
;; (COLUMN START) a rule answers when the next line may differ.
(type (: Indent (Record (: col int) (: keep bool))))

(: indent (-> int bool Indent))
(defun (indent col keep) (Indent (col col) (keep keep)))

(: with-elem (-> Frame Elem Frame))
(defun (with-elem f e)
  (def n (record-ref f count))
  (Frame (bracket (record-ref f bracket))
         (col (record-ref f col))
         (line (record-ref f line))
         (elems (Cons e (record-ref f elems)))
         (count (+ n 1))
         (first (if (= n 0) (Some e) (record-ref f first)))
         (second (if (= n 1) (Some e) (record-ref f second)))))

(: raw-of (-> (Option Elem) string))
(defun (raw-of e)
  (match e
    ((Some x) (record-ref x raw))
    (None "")))

(: last-of (-> Frame Elem Elem))
(defun (last-of f head)
  (match (record-ref f elems)
    ((Cons e _) e)
    (Nil head)))

;; How a form with a body indents, from `bjo-indent-forms` in bjomode.el: how
;; many forms come before the body, or a rule of its own.
(type (: Rule (Union (: Body int) DefForm NamedLet DefStar)))

(: rule-of (-> string (Option Rule)))
(defun (rule-of head)
  (match head
    ((or "seq" "letrec" "export" "re-export" "import" "import/extern" "import/class"
         "type" "type-rec" "bjo" "begin" "spawn" "spawn/daemon" "spawn/detached"
         "with-shield" "cond" "time-it" "try")
     (Some (Body 0)))
    ;; A definition: everything after its head is the body, whatever else is on
    ;; the head's line.
    ((or "defun" "defbjo" "defbjouble" "impl" "impl/extern" ":doc") (Some DefForm))
    ((or "def/mutable" "def/trait" "def/macro" "def/pattern" "type/derive" "when"
         "unless" "match" "case" "with-open" "parameterize" "parameterize*" "fun" "bjoroutine"
         "let*" "syntax-match" "when-let" "if-let" "with-cancel" "with-deadline"
         "with-response" "with-run" "with-return")
     (Some (Body 1)))
    ("let/mono" (Some (Body 2)))
    ("if" (Some (Body 3)))
    ("def" (Some DefForm))
    ("let" (Some NamedLet))
    ("def*" (Some DefStar))
    (_ None)))

;; The characters Emacs reads as a prefix of the form after them.
(: prefix-char? (-> char bool))
(defun (prefix-char? c)
  (or (= c #\') (= c #\#) (= c #\,) (= c #\@) (= c #\`)))

(: list-drop-while (-> (-> %a bool) (List %a) (List %a)))
(defun (list-drop-while drop? xs)
  (match xs
    ((Cons x rest) #:when (drop? x) (list-drop-while drop? rest))
    (_ xs)))

(: without-prefix (-> string string))
(defun (without-prefix s)
  (list->string (list-drop-while prefix-char? (string->list s))))

(: prefix-length (-> string int))
(defun (prefix-length s)
  (- (string-length s) (string-length (without-prefix s))))

(: prefix-only? (-> string bool))
(defun (prefix-only? s)
  (and (not (string-empty? s)) (string-empty? (without-prefix s))))

;; Whether a form written `raw` begins with a word or a symbol character once
;; its prefix is past: a symbol or a number, not a list, a string or a
;; character literal.
(: symbolic? (-> string bool))
(defun (symbolic? raw)
  (match (string->list (without-prefix raw))
    ((Cons c _) (not (or (opening? c) (= c #\") (= c #\\) (= c #\;))))
    (Nil #f)))

;; `calculate-lisp-indent`'s own column, before any rule: under the first form
;; when it is a list, under the second when the forms so far are all on the
;; first one's line, and otherwise under the first form on the line of the
;; last of them.
(: normal-indent (-> Frame Elem int))
(defun (normal-indent f head)
  (def last (last-of f head))
  (cond
    ((record-ref head is-list) (record-ref head bracket))
    ((= (record-ref last line) (record-ref head line))
     (match (record-ref f second)
       ((Some s) (record-ref s col))
       (None (record-ref head col))))
    (else (record-ref last line-first))))

;; `lisp-indent-specform`: the first two of a form's `n` distinguished forms
;; four in, a third under them, and the body two in.
(: specform (-> int Frame int Indent))
(defun (specform n f normal)
  (def args (- (record-ref f count) 1))
  (def left (- n args))
  (def body (+ (record-ref f col) 2))
  (cond
    ((> left 0) (indent (if (<= args 1) (+ (record-ref f col) 4) normal) #f))
    ((or (and (= n 0) (= args 0)) (and (= left 0) (<= body normal))) (indent body #t))
    (else (indent normal #t))))

;; `lisp-indent-defform`: a body two in, when the form before it is on the
;; line of the bracket.
(: defform (-> Frame Elem (Option Indent)))
(defun (defform f head)
  (if (= (record-ref (last-of f head) line) (record-ref f line))
      (Some (indent (+ (record-ref f col) 2) #t))
      None))

;; Whether a `let` is a named one: a name after `let` on its line.
(: named-let? (-> Frame Elem bool))
(defun (named-let? f head)
  (match (record-ref f second)
    ((Some s) #:when (= (record-ref s line) (record-ref head line))
     (match (string->list (record-ref s raw))
       ((Cons c _) (char-alphabetic? c))
       (Nil #f)))
    (_ #f)))

;; `bjo-indent-function`: what the form's head says, if it says anything.
(: by-head (-> Frame Elem int (Option Indent)))
(defun (by-head f head normal)
  (def raw (record-ref head raw))
  (def name (without-prefix raw))
  (def head-col (+ (record-ref head col) (prefix-length raw)))
  (cond
    ((string-starts-with? raw "#:") (Some (specform 0 f normal)))
    ;; A list or a string as the head: the rest under the form after it on its
    ;; line, or under the head when nothing follows it there. A guard is part
    ;; of the pattern, so after `#:when` and its guard the form is the one
    ;; after those.
    ((not (symbolic? raw))
     (def forms (list-reverse (record-ref f elems)))
     (def after
       (match forms
         ((Cons _ (Cons w (Cons _ (Cons x _)))) #:when (= (record-ref w raw) "#:when") (Some x))
         ((Cons _ (Cons w _)) #:when (= (record-ref w raw) "#:when") None)
         ((Cons _ (Cons x _)) (Some x))
         (_ None)))
     (Some (match after
             ((Some s) #:when (= (record-ref s line) (record-ref head line)) (indent (record-ref s col) #t))
             (_ (indent (record-ref head col) #t)))))
    ((= name ":match") (Some (indent head-col #t)))
    (else
     (match (rule-of name)
       ((Some DefForm) (defform f head))
       ((Some (Body n)) (Some (specform n f normal)))
       ((Some NamedLet) (Some (specform (if (named-let? f head) 2 1) f normal)))
       ((Some DefStar) None)
       ;; A head that begins with `def` defines something and one that begins
       ;; with `with-` runs a body in something; both have a body.
       (None
        (if (and (> (string-length name) 3)
                 (or (string-starts-with? name "def") (string-starts-with? name "with-")))
            (defform f head)
            None))))))

;; A line that starts with a keyword goes under a keyword that starts the line
;; of the form before it, when that is not the list's first form.
(: keyword-aligned (-> Frame Elem int string int))
(defun (keyword-aligned f head normal here)
  (def last (last-of f head))
  (def on-its-line
    (list-filter #(= (record-ref & line) (record-ref last line)) (record-ref f elems)))
  (match (list-reverse on-its-line)
    ((Cons s _) #:when (and (string-starts-with? here ":")
                            (not (and (= (record-ref s col) (record-ref head col))
                                      (= (record-ref s line) (record-ref head line))))
                            (string-starts-with? (record-ref s raw) ":"))
     (record-ref s col))
    (_ normal)))

;; Where `def`'s failure part may be: the forms after a `def`'s head, or all
;; of a parenthesised clause of a `def*`. `None` for any other list.
(: clause-forms (-> Frame (Option Frame) (Option (List Elem))))
(defun (clause-forms f outer)
  (def forms (list-reverse (record-ref f elems)))
  (def head (raw-of (record-ref f first)))
  (def in-def-star?
    (match outer
      ((Some o) (= (raw-of (record-ref o first)) "def*"))
      (None #f)))
  (cond
    ((or (= head "def") (= head ":def"))
     (Some (match forms
             ((Cons _ rest) rest)
             (Nil Nil))))
    ((and in-def-star? (= (record-ref f bracket) "(") (not (= head "def*"))) (Some forms))
    (else None)))

(: failure-keyword? (-> string bool))
(defun (failure-keyword? s)
  (match (without-prefix s)
    ((or ":leave-with" ":leave" ":propagate" ":default") #t)
    (_ #f)))

;; `bjo--failure-indent`: a failure keyword two in from the clause's bracket,
;; and the arms after `:leave` under the first of them.
(: failure-indent (-> Frame (Option Frame) string bool (Option Indent)))
(defun (failure-indent f outer here here-form?)
  (match (clause-forms f outer)
    ((Some forms) #:when (>= (+ (list-length forms) (if here-form? 1 0)) 2)
     (cond
       ((and here-form? (failure-keyword? here)) (Some (indent (+ (record-ref f col) 2) #f)))
       (else
        (match (list-drop-while #(not (= (without-prefix (record-ref & raw)) ":leave")) forms)
          ((Cons _ (Cons arm _)) (Some (indent (record-ref arm col) #f)))
          ((Cons leave Nil) (Some (indent (record-ref leave col) #f)))
          (Nil None)))))
    (_ None)))

;; A form whose result a `=>` introduces, which goes two in, as a body does.
(: loop-form? (-> Frame bool))
(defun (loop-form? f)
  (match (raw-of (record-ref f first))
    ((or "loop" ":loop" "subloop" ":subloop" "seql") #t)
    (_ #f)))

(: binding-form? (-> string bool))
(defun (binding-form? head)
  (match head
    ((or "let" "let*" "letrec" "with-open" "parameterize" "parameterize*") #t)
    (_ #f)))

;; `bjo--binding-list-p`: whether `f` is the bindings of a `let` or a form like
;; it, `outer`: the list after the head, or after the name of a named `let`.
(: binding-list? (-> Frame (Option Frame) bool))
(defun (binding-list? f outer)
  (match outer
    ((Some o)
     (def head (raw-of (record-ref o first)))
     (def named?
       (and (= head "let")
            (match (string->list (raw-of (record-ref o second)))
              ((Cons c _) (char-alphabetic? c))
              (Nil #f))))
     ;; `f` is open inside `o`, so it is the last of `o`'s forms so far.
     (and (= (record-ref f bracket) "(")
          (binding-form? head)
          (= (record-ref o count) (if named? 3 2))))
    (None #f)))

;; Whether `bjo--calculate-indent` stops the column being kept: in a `def`
;; clause and in a `loop` or a `seql`, where the next line may indent
;; differently.
(: no-reuse? (-> Frame (Option Frame) bool))
(defun (no-reuse? f outer)
  (or (loop-form? f)
      (match (clause-forms f outer)
        ((Some forms) (not (empty? forms)))
        (None #f))))

;; The column for a line inside `f`, whose first token is `here` and starts a
;; form if `here-form?`.
(: indent-in (-> Frame (Option Frame) string bool Indent))
(defun (indent-in f outer here here-form?)
  (def calculated
    (match (record-ref f first)
      (None (indent (+ (record-ref f col) 1) #t))
      ;; A vec or an array is data, and a `let`'s bindings are a list of them:
      ;; every element goes under the first, whatever it is. The rules below
      ;; are for code.
      ((Some head) #:when (or (= (record-ref f bracket) "[") (binding-list? f outer))
       (indent (record-ref head col) #t))
      ((Some head)
       (def normal (normal-indent f head))
       (match (by-head f head normal)
         ((Some i) i)
         (None (indent (keyword-aligned f head normal here) #t))))))
  (match (failure-indent f outer here here-form?)
    ((Some i) i)
    (None
     (cond
       ((and (loop-form? f) (= here "=>")) (indent (+ (record-ref f col) 2) #f))
       ((and (record-ref calculated keep) (no-reuse? f outer)) (indent (record-ref calculated col) #f))
       (else calculated)))))

;; ---------------------------------------------------------------------------
;; Writing it out

;; What is known while the lines are written: the lists open, the innermost
;; first; the column kept at each depth, the innermost first, one more than
;; there are lists open, and how many lists were open when they were kept;
;; the physical line being written; and what is to be said about the lines so
;; far, each with its line, the last first.
(type (: State (Record (: frames (List Frame))
                       (: kept (List (Option int)))
                       (: kept-depth int)
                       (: line int)
                       (: notes (List (Tuple int string))))))

;; The width past which a line the layout made wider is pointed out: an
;; indenter only moves text right, and a line it moves too far wants breaking
;; by hand. Racket's style has the same limit.
(: wide int)
(def wide 102)

;; What is to be said about a line whose tokens are `toks` as it is written:
;; that it has a tab in a raw string, which no escape can write. How wide it
;; is is said once it is broken; see `width-notes`.
(: notes-on (-> (List Tok) (List string)))
(defun (notes-on toks)
  (if (any? (fun (t)
              (match t
                ((TStr s) (and (string-starts-with? s "\"\"\"") (string-contains? s "\t")))
                (_ #f)))
            toks)
      (list "a tab in a raw string, which has no escape for it; write the text another way")
      Nil))

;; The lines of `result` wider than `wide`, each with its width as a note, but
;; one that is part of a line of `source` as wide already: a string, a comment
;; or a table nothing here breaks, which a person has to, and has been told of.
(: width-notes (-> string string (List (Tuple int string))))
(defun (width-notes source result)
  (def wide-before
    (loop (:for l (string-split source "\n"))
          (:acc out (listing (string-trim l)) #:when (> (columns l) wide))
      => out))
  (def lines (string-split result "\n"))
  (loop (:for i (range 0 (vec-length lines)))
        (:acc out (listing (Tuple (+ i 1)
                                  #"${(int->string (columns (vec-ref lines i)))} columns, more than ${(int->string wide)}; break the line"))
              #:when (let ((l (vec-ref lines i)))
                       (and (> (columns l) wide)
                            (not (any? #(string-contains? & (string-trim l)) wide-before)))))
    => out))

(: spaces (-> int string))
(defun (spaces n)
  (def sb (stringbuilder-empty))
  (let go ((i 0))
    (when (< i n)
      (stringbuilder-add! sb #\space)
      (go (+ i 1))))
  (stringbuilder->string sb))

;; The column after `s` written at `col`, and how many line breaks it has.
(: advance (-> string int (Tuple int int)))
(defun (advance s col)
  (string-fold (fun (c at)
                 (match at
                   ((Tuple column breaks)
                    (if (= c #\newline) (Tuple 0 (+ breaks 1)) (Tuple (+ column 1) breaks)))))
               (Tuple col 0)
               s))

(: expand-tabs (-> string int string))
(defun (expand-tabs s col)
  (def sb (stringbuilder-empty))
  (ignore
   (string-fold (fun (c at)
                  (cond
                    ((= c #\tab)
                     (def to (* 8 (+ 1 (/ at 8))))
                     (stringbuilder-add-string! sb (spaces (- to at)))
                     to)
                    (else
                     (stringbuilder-add! sb c)
                     (+ at 1))))
                col
                s))
  (stringbuilder->string sb))

;; The kept columns adjusted from `from` lists open to `depth`, as
;; `lisp-indent-calc-next` adjusts its stack: one dropped per list closed
;; since, a fresh one per list opened.
(: kept-at (-> (List (Option int)) int int (List (Option int))))
(defun (kept-at kept from depth)
  (cond
    ((> from depth)
     (kept-at (match kept
                ((Cons _ rest) rest)
                (Nil Nil))
              (- from 1)
              depth))
    ((< from depth) (kept-at (Cons None kept) (+ from 1) depth))
    (else kept)))

;; The text of a line's first token, and whether it starts a form.
(: first-of-line (-> (List Tok) (Tuple string bool)))
(defun (first-of-line toks)
  (match toks
    ((Cons (TAtom s) _) (Tuple s #t))
    ((Cons (TOpen s) _) (Tuple s #t))
    ((Cons (TStr s) _) (Tuple s #t))
    ((Cons (TComment s) _) (Tuple s #f))
    ((Cons (TClose s) _) (Tuple s #f))
    (_ (Tuple "" #f))))

;; One line, indented, and the state after it.
(: write-line (-> StringBuilder State Line State))
(defun (write-line out st l)
  (def toks (record-ref l toks))
  (def frames (record-ref st frames))
  (def depth (list-length frames))
  (def kept (kept-at (record-ref st kept) (record-ref st kept-depth) depth))
  (def (Tuple here here-form?) (first-of-line toks))
  (def (Tuple col keep)
    (match kept
      ((Cons (Some c) _) (Tuple c #f))
      (_
       (match frames
         (Nil (Tuple 0 #t))
         ((Cons f rest)
          (def outer
            (match rest
              ((Cons o _) (Some o))
              (Nil None)))
          (def i (indent-in f outer here here-form?))
          (Tuple (record-ref i col) (record-ref i keep)))))))
  (def kept-now
    (match kept
      ((Cons None rest) #:when keep (Cons (Some col) rest))
      (_ kept)))
  ;; A `;;;` comment stays where it was, as `lisp-indent-line` leaves it.
  (def lead (if (string-starts-with? here ";;;") (record-ref l indent) (spaces col)))
  (def buffer (stringbuilder-empty))
  (unless (empty? toks) (stringbuilder-add-string! buffer lead))
  (def (Tuple frames-after line-after)
    (write-toks buffer toks frames (string-length lead) (record-ref st line)))
  (stringbuilder-add-string! out (stringbuilder->string buffer))
  (stringbuilder-add! out #\newline)
  (State (frames frames-after)
         (kept kept-now)
         (kept-depth depth)
         (line (+ line-after 1))
         (notes (list-foldl (fun (note acc) (Cons (Tuple (record-ref st line) note) acc))
                            (record-ref st notes)
                            (notes-on toks)))))

;; Whether the token is a prefix, ' or #' or #, written right before the form
;; it belongs to.
(: prefix-of-next? (-> Tok (List Tok) bool))
(defun (prefix-of-next? t rest)
  (match t
    ((TAtom s) #:when (prefix-only? s)
     (match rest
       ((Cons (TOpen _) _) #t)
       ((Cons (TStr _) _) #t)
       ((Cons (TAtom _) _) #t)
       (_ #f)))
    (_ #f)))

(: starts-form? (-> Tok bool))
(defun (starts-form? t)
  (match t
    ((TAtom _) #t)
    ((TOpen _) #t)
    ((TStr _) #t)
    (_ #f)))

;; The forms the token adds to the lists open, and the list it opens or closes.
(: after-tok (-> Tok (List Frame) Elem (List Frame)))
(defun (after-tok t frames e)
  (def added
    (match frames
      ((Cons f outer) (if (starts-form? t) (Cons (with-elem f e) outer) frames))
      (Nil Nil)))
  (match t
    ((TOpen s)
     (Cons (Frame (bracket s) (col (record-ref e bracket)) (line (record-ref e line))
                  (elems Nil) (count 0) (first None) (second None))
           added))
    ((TClose _)
     (match frames
       ((Cons _ outer) outer)
       (Nil Nil)))
    (_ added)))

;; The tokens of one line, written from `col` on physical line `line`, and the
;; lists open after them and the physical line they end on.
(: write-toks (-> StringBuilder (List Tok) (List Frame) int int (Tuple (List Frame) int)))
(defun (write-toks out toks frames col line)
  ;; `first` is the column of the first form on the physical line, once there
  ;; is one; `prefix` that of a prefix waiting for the form it belongs to.
  (let go ((toks toks) (frames frames) (col col) (line line) (first None) (prefix None))
    (match toks
      (Nil (Tuple frames line))
      ((Cons t rest)
       (def written
         (match t
           ((TSpace s) (expand-tabs s col))
           ((TComment s) (expand-tabs s col))
           (_ (tok-written t col))))
       (stringbuilder-add-string! out written)
       (def (Tuple next-col breaks) (advance written col))
       (def waits? (prefix-of-next? t rest))
       (def form-col
         (match prefix
           ((Some p) p)
           (None col)))
       (def line-first
         (match first
           ((Some c) c)
           (None form-col)))
       (def e (Elem (col form-col)
                    (line line)
                    (line-first line-first)
                    (raw (match t
                           ((TStr _) "\"")
                           (_ (tok-text t))))
                    (is-list (match t
                               ((TOpen _) #t)
                               (_ #f)))
                    (bracket col)))
       (def frames-after (if waits? frames (after-tok t frames e)))
       (def first-after
         (cond
           ((> breaks 0) None)
           ((and (or waits? (starts-form? t)) (none? first)) (Some form-col))
           (else first)))
       (def prefix-after
         (cond
           (waits? (Some form-col))
           (else None)))
       (go rest frames-after next-col (+ line breaks) first-after prefix-after)))))

;; ---------------------------------------------------------------------------
;; Brackets

(: partner (-> string string))
(defun (partner open)
  (match open
    ("(" ")")
    ("[" "]")
    (_ "}")))

;; Whether every bracket closes the one opened last: indentation means nothing
;; where they do not, and Emacs, asked to indent such a file, never finishes.
(: balanced (-> (List Tok) (Result string Unit)))
(defun (balanced toks)
  ;; `open` holds each bracket still open and its line, the innermost first.
  (let go ((toks toks) (open Nil) (line 1))
    (match toks
      (Nil
       (match open
         (Nil (Ok unit))
         ((Cons (Tuple b at) _)
          (Err #"line ${(int->string at)}: a ${b} that is never closed"))))
      ((Cons t rest)
       (match t
         (TNewline (go rest open (+ line 1)))
         ((TStr s) (go rest open (+ line (snd (advance s 0)))))
         ((TOpen b) (go rest (Cons (Tuple b line) open) line))
         ((TClose c)
          (match open
            (Nil (Err #"line ${(int->string line)}: a ${c} that closes nothing"))
            ((Cons (Tuple b at) outer)
             (if (= (partner b) c)
                 (go rest outer line)
                 (Err #"line ${(int->string line)}: a ${c} where the ${b} of line ${(int->string at)} wants a ${(partner b)}")))))
         (_ (go rest open line)))))))

;; ---------------------------------------------------------------------------
;; Breaking long lines
;;
;; A line wider than `wide` is broken in a list on it that runs past the limit,
;; the outermost that has somewhere to break, by putting its forms on lines of
;; their own the way the indentation then lays them out: a call's arguments
;; after the first under it, a body's forms two in, an `if`'s arms four in, a
;; clause's forms after its pattern. A call whose arguments are all small, and
;; a vec, are filled instead: broken only before the form that runs past the
;; limit. A list runs past the limit when it does with the closing brackets
;; right after it, so a line too wide only for the brackets that end it is
;; broken in its last list. A line that continues a list opened on a line
;; before is filled in that list. What none of that can take, a call's first
;; argument moving down gives a line more, in the call where that moves the
;; most left. A line none of that can break, a string that runs past the limit
;; is split on, with a line continuation after the last space that fits. Then
;; the text is laid out again and looked at again, until nothing more can be
;; broken.
;;
;; A keyword argument's value stays on the keyword's line, and a clause's
;; `#:when` and guard on the line of its pattern.
;;
;; A macro is left as it is: what its forms are is its own business. Those of
;; the prelude are named here; a head that begins with `with-` is one too. So
;; is everything in an `import/extern` or an `import/class`, which are tables
;; written to be read in columns.

(: macro-head? (-> string bool))
(defun (macro-head? head)
  (or (string-starts-with? head "with-")
      (match head
        ((or "cond" "get" "if-let" "in-scope" "let*" "own-disposable" "parameterize*" "scope-form"
             "set" "some->" "try->" "try-get" "type/derive" "update" "when-let")
         #t)
        (_ #f))))

(: table-head? (-> string bool))
(defun (table-head? head)
  (match head
    ((or "import/extern" "import/class") #t)
    (_ #f)))

;; How the forms of a list may start lines of their own.
(type (: Breaking (Union
                   ;; Each from this position on.
                   (: OneALine int)
                   ;; Only the one that runs past the limit, from this position on.
                   (: Filled int)
                   ;; A call: filled if its arguments are all small, else one a line.
                   (: Call int))))

;; How the forms of a list headed `head`, written `bracket`, may start lines of
;; their own, or `None` for a list to leave as it is.
(: breaking-of (-> string string (Option Breaking)))
(defun (breaking-of bracket head)
  (def name (without-prefix head))
  (cond
    ;; A vec and an export list are data, filled.
    ((= bracket "[") (Some (Filled 1)))
    ((or (= name "export") (= name "re-export")) (Some (Filled 1)))
    ((= bracket "{") None)
    ((or (macro-head? name) (table-head? name)) None)
    ((not (symbolic? head)) (Some (OneALine 1)))
    ((= name "if") (Some (OneALine 2)))
    (else
     (match (rule-of name)
       ((Some (Body n)) (Some (OneALine (+ n 1))))
       ((Some DefStar) (Some (OneALine 1)))
       ((Some _) (Some (OneALine 2)))
       (None
        (if (and (> (string-length name) 3) (string-starts-with? name "def"))
            (Some (OneALine 2))
            (Some (Call 2))))))))

;; A form of a list on the line looked at: its token, its position in the
;; list, the columns it starts and ends at, the closing brackets right after
;; it included; whether it is small: an atom, a string, or a list on one line
;; with no list in it; and whether it is glued to the form before it, which a
;; line break never comes between: the value of a keyword argument, and a
;; clause's `#:when`, which is part of its pattern.
(type (: Form (Record (: tok int) (: at int) (: from int) (: to int) (: small bool) (: glued bool))))

;; What the breaking reads of the laid-out tokens: each list's closing token,
;; each token's line and column, whether it is inside a table, and whether it
;; is inside a doc, whose strings may not continue lines: a published doc is
;; bjodat, which has no columns to go by.
(type (: Placed (Record (: toks (Vec Tok))
                        (: closes (Vec int))
                        (: lines (Vec int))
                        (: cols (Vec int))
                        (: tabled (Vec bool))
                        (: docked (Vec bool)))))

(: placed (-> (List Tok) Placed))
(defun (placed toks)
  (def v (list->vec toks))
  (def (Tuple closes _) (closers-and-lines v))
  (def lines (vecbuilder-empty))
  (def cols (vecbuilder-empty))
  (def tabled (vecbuilder-empty))
  (def docked (vecbuilder-empty))
  ;; `open` holds for each list open whether it is, or is in, a table, and
  ;; whether it is, or is in, a doc.
  (let go ((k 0) (line 0) (col 0) (open Nil))
    (when (< k (vec-length v))
      (def t (vec-ref v k))
      (def (Tuple in-table in-doc) (match open ((Cons b _) b) (Nil (Tuple #f #f))))
      (vecbuilder-add! lines line)
      (vecbuilder-add! cols col)
      (vecbuilder-add! tabled in-table)
      (vecbuilder-add! docked in-doc)
      (def (Tuple after breaks) (advance (tok-written t col) col))
      (def open-after
        (match t
          ((TOpen _)
           (def head (match (vec-ref v (+ k 1)) ((TAtom s) s) (_ "")))
           (Cons (Tuple (or in-table (table-head? head)) (or in-doc (= head ":doc"))) open))
          ((TClose _) (match open ((Cons _ outer) outer) (Nil Nil)))
          (_ open)))
      (go (+ k 1) (+ line breaks) after open-after)))
  (Placed (toks v) (closes closes) (lines (vecbuilder->vec lines))
          (cols (vecbuilder->vec cols)) (tabled (vecbuilder->vec tabled))
          (docked (vecbuilder->vec docked))))

(: tok-at (-> Placed int Tok))
(defun (tok-at p k) (vec-ref (record-ref p toks) k))

(: opens? (-> Tok bool))
(defun (opens? t)
  (match t
    ((TOpen _) #t)
    (_ #f)))

;; The column after the form starting at `j`, and after the closing brackets
;; right after it, which go where it goes.
(: form-end (-> Placed int int))
(defun (form-end p j)
  (closers-end p (if (opens? (tok-at p j)) (vec-ref (record-ref p closes) j) j)))

;; The column after token `k` and the closing brackets right after it.
(: closers-end (-> Placed int int))
(defun (closers-end p k)
  (def n (vec-length (record-ref p toks)))
  (let go ((k k))
    (if (and (< (+ k 1) n) (close? (tok-at p (+ k 1))))
        (go (+ k 1))
        (fst (advance (tok-written (tok-at p k) (vec-ref (record-ref p cols) k)) (vec-ref (record-ref p cols) k))))))

(: small-form? (-> Placed int bool))
(defun (small-form? p j)
  (def close (vec-ref (record-ref p closes) j))
  (cond
    ((not (opens? (tok-at p j))) #t)
    ((not (= (vec-ref (record-ref p lines) close) (vec-ref (record-ref p lines) j))) #f)
    (else (not (any? #(opens? (tok-at p &)) (range (+ j 1) close))))))

;; The forms of the list opening at `k`, each with its position, from the
;; first; those that start on line `ln`, unless `ln` is -1.
(: forms-of (-> Placed int int (List Form)))
(defun (forms-of p k ln)
  (def close (vec-ref (record-ref p closes) k))
  (def n (vec-length (record-ref p toks)))
  (def head-list? (and (< (+ k 1) n) (opens? (tok-at p (+ k 1)))))
  ;; `prev` is the text of the form before, if it is an atom.
  (let go ((j (+ k 1)) (at 0) (prev "") (out Nil))
    (cond
      ((or (< close 0) (>= j close)) (list-reverse out))
      ((not (starts-form? (tok-at p j))) (go (+ j 1) at prev out))
      (else
       ;; A prefix is the start of the form after it.
       (def body (if (and (< (+ j 1) n) (prefix-of-next? (tok-at p j) (list (tok-at p (+ j 1))))) (+ j 1) j))
       (def next (if (opens? (tok-at p body)) (+ (vec-ref (record-ref p closes) body) 1) (+ body 1)))
       (def here? (or (= ln -1) (= (vec-ref (record-ref p lines) j) ln)))
       (def text (if (opens? (tok-at p body)) "" (tok-text (tok-at p j))))
       (def form (Form (tok j) (at at) (from (vec-ref (record-ref p cols) j)) (to (form-end p body))
                       (small (small-form? p body))
                       (glued (or (keyword-arg? prev) (and (= at 1) head-list? (= text "#:when"))))))
       (go next (+ at 1) text (if here? (Cons form out) out))))))

(: head-text (-> Placed (List Form) string))
(defun (head-text p forms)
  (match forms
    ((Cons f _) #:when (= (record-ref f at) 0) (tok-text (tok-at p (record-ref f tok))))
    (_ "")))

;; The first of `forms` that runs past the limit, to break before, or, if it is
;; glued to the one before it, the first of them it is glued to.
(: first-past (-> (List Form) (List int)))
(defun (first-past forms)
  (def v (list->vec forms))
  (def n (vec-length v))
  (defun (past? i) (> (record-ref (vec-ref v i) to) wide))
  (defun (glued? i) (record-ref (vec-ref v i) glued))
  (defun (back i)
    (cond
      ((not (glued? i)) (list (record-ref (vec-ref v i) tok)))
      ((> i 0) (back (- i 1)))
      (else Nil)))
  (let find ((i 0))
    (cond
      ((>= i n) Nil)
      ((past? i) (back i))
      (else (find (+ i 1))))))

(: unglued (-> (List Form) (List int)))
(defun (unglued forms)
  (list-map #(record-ref & tok) (list-filter #(not (record-ref & glued)) forms)))

;; Where to break line `ln` in the list opening at `k` on it.
(: breaks-in-list (-> Placed int int (List int)))
(defun (breaks-in-list p k ln)
  (def all (forms-of p k -1))
  (def here (forms-of p k ln))
  (def bracket (tok-text (tok-at p k)))
  (defun (from-on n)
    (list-filter #(and (>= (record-ref & at) n) (> (record-ref & from) (+ (vec-ref (record-ref p cols) k) 1)))
                 here))
  (match (breaking-of bracket (head-text p all))
    (None Nil)
    ((Some (OneALine n)) (unglued (from-on n)))
    ((Some (Filled n)) (first-past (from-on n)))
    ((Some (Call n))
     (if (all? #(record-ref & small) (list-filter #(>= (record-ref & at) 1) all))
         (first-past (from-on n))
         (unglued (from-on n))))))

;; Whether the list opening at `k` runs past the limit on line `ln`, the
;; closing brackets right after it included: a line too wide only for the
;; brackets that end it is broken in the last list on it.
(: runs-past? (-> Placed int int bool))
(defun (runs-past? p k ln)
  (def c (vec-ref (record-ref p closes) k))
  (or (not (= (vec-ref (record-ref p lines) c) ln))
      (> (closers-end p c) wide)))

;; The list a line starts inside of: the innermost open at its first token.
(: enclosing (-> Placed int (Option int)))
(defun (enclosing p first)
  (let go ((k 0) (open Nil))
    (cond
      ((>= k first)
       (match open
         ((Cons o _) (Some o))
         (Nil None)))
      ((opens? (tok-at p k)) (go (+ k 1) (Cons k open)))
      ((close? (tok-at p k))
       (go (+ k 1)
           (match open
             ((Cons _ outer) outer)
             (Nil Nil))))
      (else (go (+ k 1) open)))))

;; A line that continues a list opened before it is filled in that list.
(: breaks-continuing (-> Placed int int (List int)))
(defun (breaks-continuing p first ln)
  (match (enclosing p first)
    ((Some k) #:when (not (vec-ref (record-ref p tabled) k))
     (match (breaking-of (tok-text (tok-at p k)) (head-text p (forms-of p k -1)))
       (None Nil)
       ((Some _)
        (first-past (list-filter #(> (record-ref & tok) first) (forms-of p k ln))))))
    (_ Nil)))

;; The first argument of a call of `lists` that has it on the line of its head,
;; moved down, when nothing else can be broken: the list the line continues,
;; which moves the whole line left, or one on the line, which moves what runs
;; past the limit left. Either moves it left by as much as the argument is to
;; the right of the call's bracket, so the call where that is the most is
;; taken, the first of those that gain as much.
(: first-argument-down (-> Placed (List int) (List int)))
(defun (first-argument-down p lists)
  (defun (call? k forms)
    (match (breaking-of (tok-text (tok-at p k)) (head-text p forms))
      ((Some (Call _)) #t)
      (_ #f)))
  ;; The argument of the call opening at `k` to move down, and how far left
  ;; that moves what comes after it.
  (defun (candidate k)
    (def forms (forms-of p k (vec-ref (record-ref p lines) k)))
    (match forms
      ((Cons h (Cons arg _)) #:when (and (= (record-ref h at) 0)
                                         (= (record-ref arg at) 1)
                                         (not (vec-ref (record-ref p tabled) k))
                                         (call? k forms))
       (Some (Tuple (record-ref arg tok) (- (record-ref arg from) (+ (vec-ref (record-ref p cols) k) 1)))))
      (_ None)))
  (def best
    (list-foldl (fun (k acc)
                  (match (Tuple (candidate k) acc)
                    ((Tuple (Some (Tuple tok gain)) (Some (Tuple _ most))) #:when (> gain most)
                     (Some (Tuple tok gain)))
                    ((Tuple (Some c) None) (Some c))
                    (_ acc)))
                None
                lists))
  (match best
    ((Some (Tuple tok gain)) #:when (> gain 0) (list tok))
    (_ Nil)))

;; A string that line `ln` has some of, split there with a line continuation,
;; as its token's index and new text; `None` when there is none to split. A raw
;; string has no continuations, a doc's strings may not have them, and a table
;; is left as it is.
(: split-for (-> Placed int (Option (Tuple int string))))
(defun (split-for p ln)
  (def n (vec-length (record-ref p toks)))
  (let find ((k 0))
    (cond
      ((or (>= k n) (> (vec-ref (record-ref p lines) k) ln)) None)
      (else
       (match (tok-at p k)
         ((TStr s) #:when (not (or (vec-ref (record-ref p tabled) k)
                                   (vec-ref (record-ref p docked) k)
                                   (string-starts-with? s "\"\"\"")))
          ;; The closing brackets right after it go where its end goes.
          (def closers (- (closers-end p k) (closers-end-alone p k)))
          (match (split-string s (vec-ref (record-ref p cols) k) (- ln (vec-ref (record-ref p lines) k)) closers)
            ((Some text) (Some (Tuple k text)))
            (None (find (+ k 1)))))
         (_ (find (+ k 1))))))))

;; The string `s`, written at `col`, split on its physical line `nth` with a
;; line continuation: after the last space on it that leaves room for the
;; backslash within `wide`, with text before it on the line and a word after
;; it. Never in an escape or an interpolation's hole. What it stands for is
;; the same: the space stays before the backslash, and the continued line
;; starts where the text does. `None` when the line has no such space.
(: split-string (-> string int int int (Option string)))
(defun (split-string s col nth closers)
  (def written (with-continued-indent s col))
  (def text-col (text-column s col))
  (def cs (string->vec written))
  (def n (vec-length cs))
  (def interpolated? (string-starts-with? s "#\""))
  (def opening (if interpolated? 2 1))
  ;; `seen` is whether the line has had text, the opening quote aside, and
  ;; `reach` the column the string reaches on line `nth`: one that does not run
  ;; past the limit there, with the `closers` columns of brackets after it if
  ;; it ends there, is not what makes the line too wide, and is left whole.
  (let go ((p opening) (line 0) (column (+ col opening)) (best None) (seen #f) (reach 0))
    (def c (char-at cs p))
    (def reach-now (if (= line nth) column reach))
    (cond
      ((or (>= p n) (> line nth))
       (match best
         ((Some at) #:when (> (if (and (>= p n) (= line nth)) (+ reach-now closers) reach-now) wide)
          (Some (without-continued-indent
                 (str (text-between cs 0 (+ at 1)) "\\\n" (spaces text-col) (text-between cs (+ at 1) n))
                 col)))
         (_ None)))
      ((= c #\newline) (go (+ p 1) (+ line 1) 0 best #f reach-now))
      ;; A continuation's line break is the next character's to deal with.
      ((and (= c #\\) (= (char-at cs (+ p 1)) #\newline)) (go (+ p 1) line (+ column 1) best #t reach-now))
      ((= c #\\) (go (+ p 2) line (+ column 2) best #t reach-now))
      ((and interpolated? (= c #\$) (= (char-at cs (+ p 1)) #\{))
       (def past (hole-end cs (+ p 2)))
       (def (Tuple after breaks) (advance (text-between cs p past) column))
       (go past (+ line breaks) after best #t reach-now))
      ((= c #\space)
       (def next (char-at cs (+ p 1)))
       (def fits? (<= (+ column 2) wide))
       (def word-after? (and (< (+ p 2) n) (not (or (= next #\space) (= next #\newline) (= next #\")))))
       (go (+ p 1) line (+ column 1)
           (if (and (= line nth) seen fits? word-after?) (Some p) best)
           seen
           reach-now))
      (else (go (+ p 1) line (+ column 1) best #t reach-now)))))

;; The column after token `k`, the closing brackets after it not counted.
(: closers-end-alone (-> Placed int int))
(defun (closers-end-alone p k)
  (def col (vec-ref (record-ref p cols) k))
  (fst (advance (tok-written (tok-at p k) col) col)))

;; `toks` with the strings `splits` names, by index, given their new text.
(: with-splits (-> (List Tok) (List (Tuple int string)) (List Tok)))
(defun (with-splits toks splits)
  (if (empty? splits)
      toks
      (let go ((toks toks) (k 0) (out Nil))
        (match toks
          (Nil (list-reverse out))
          ((Cons t rest)
           (def now
             (match (list-filter #(= (fst &) k) splits)
               ((Cons (Tuple _ text) _) (TStr text))
               (Nil t)))
           (go rest (+ k 1) (Cons now out)))))))

;; The tokens to put a line break before, to break line `ln`; and, when there
;; are none, those that move a first argument down, which is the last resort:
;; a string that runs past the limit is split before it is moved (see
;; `broken`).
(: breaks-for (-> Placed int (Tuple (List int) (List int))))
(defun (breaks-for p ln)
  (def n (vec-length (record-ref p toks)))
  (def on-line
    (loop (:for k (range 0 n))
          (:acc out (listing k) #:when (= (vec-ref (record-ref p lines) k) ln))
      => out))
  (def crossing
    (list-filter #(and (opens? (tok-at p &)) (not (vec-ref (record-ref p tabled) &)) (runs-past? p & ln))
                 on-line))
  (def first (match (list-filter #(starts-form? (tok-at p &)) on-line) ((Cons f _) f) (Nil -1)))
  (def around
    (match (if (< first 0) None (enclosing p first))
      ((Some k) (list k))
      (None Nil)))
  ;; The least column a form on the line can be moved to: in the list the
  ;; line continues, or past where the line starts.
  (def least
    (match around
      ((Cons k _)
       (def a (vec-ref (record-ref p cols) k))
       (def b (vec-ref (record-ref p cols) first))
       (+ (if (< a b) a b) 1))
      (Nil 1)))
  ;; A string or comment too wide to fit even moved there keeps the line wide
  ;; whatever is broken. One that can not be split leaves the line as it is.
  ;; One that can is split once the line's lists are broken (see `split-for`),
  ;; and moving it left, by filling or by moving a first argument down, would
  ;; only split it later.
  (def too-wide
    (list-filter #(and (not (opens? (tok-at p &))) (> (+ least (columns (tok-text (tok-at p &)))) wide))
                 on-line))
  (def splits-instead? (not (empty? too-wide)))
  (cond
    ((not (all? #(splittable? p &) too-wide)) (Tuple Nil Nil))
    (else
     (match (list-filter #(not (empty? &)) (list-map #(breaks-in-list p & ln) crossing))
       ((Cons found _) (Tuple found Nil))
       (Nil #:when splits-instead? (Tuple Nil Nil))
       (Nil
        (match (if (< first 0) Nil (breaks-continuing p first ln))
          (Nil (Tuple Nil (first-argument-down p (list-append around crossing))))
          (found (Tuple found Nil))))))))

;; Whether the token at `k` is a string `split-for` may split: not a raw one,
;; not one in a doc or a table, and with a space in it.
(: splittable? (-> Placed int bool))
(defun (splittable? p k)
  (match (tok-at p k)
    ((TStr s) (not (or (string-starts-with? s "\"\"\"")
                       (vec-ref (record-ref p tabled) k)
                       (vec-ref (record-ref p docked) k)
                       (not (string-contains? s " ")))))
    (_ #f)))

;; The laid-out text with its long lines broken, as far as they can be.
(: broken (-> (Tuple string (List (Tuple int string))) (Tuple string (List (Tuple int string)))))
(defun (broken laid)
  (let again ((laid laid) (rounds 0))
    (def (Tuple text _) laid)
    (match (tokenize (string->vec text))
      ((Err _) laid)
      ((Ok toks)
       (def p (placed toks))
       (def widths (vec-map columns (string-split text "\n")))
       (def long
         (loop (:for ln (range 0 (vec-length widths)))
               (:acc out (listing ln) #:when (> (vec-ref widths ln) wide))
           => out))
       ;; For each long line the line breaks that break it; when there are
       ;; none, a string that runs past the limit split with a line
       ;; continuation; and when there is none of those, a first argument
       ;; moved down.
       (def (Tuple wanted splits)
         (list-foldl (fun (ln acc)
                       (def (Tuple breaks split) acc)
                       (match (breaks-for p ln)
                         ((Tuple Nil last-resort)
                          (match (split-for p ln)
                            ((Some s) (Tuple breaks (Cons s split)))
                            (None (Tuple (list-append breaks last-resort) split))))
                         ((Tuple found _) (Tuple (list-append breaks found) split))))
                     (Tuple Nil Nil)
                     long))
       (if (or (and (empty? wanted) (empty? splits)) (>= rounds 40))
           laid
           (again (layout (with-breaks-before (with-splits toks splits) wanted)) (+ rounds 1)))))))

(: with-breaks-before (-> (List Tok) (List int) (List Tok)))
(defun (with-breaks-before toks at)
  (let go ((toks toks) (k 0) (out Nil))
    (match toks
      (Nil (list-reverse out))
      ((Cons t rest)
       (go rest (+ k 1) (Cons t (if (any? #(= & k) at) (Cons TNewline out) out)))))))

;; How many columns `s` takes, in characters: `string-length` counts the bytes
;; of its UTF-8.
(: columns (-> string int))
(defun (columns s) (fst (advance s 0)))

;; ---------------------------------------------------------------------------
;; The whole of it

;; `source` laid out, with what is to be said about its lines, each with its
;; line number in what is laid out (see `notes-on`); or why it cannot be: it
;; does not read, or, which would be a bug here, laying it out would change
;; its tokens.
(: format-source (-> string (Result string (Tuple string (List (Tuple int string))))))
(defun (format-source source) (format-with source #t))

;; `format-source`, its long lines broken if `break?`.
(: format-with (-> string bool (Result string (Tuple string (List (Tuple int string))))))
(defun (format-with source break?)
  (def cs (string->vec source))
  (def (Ok toks) (tokenize cs)
    :leave ((Err why) (Err why)))
  (def (Ok _) (balanced toks)
    :leave ((Err why) (Err why)))
  (def (Tuple result notes) (if break? (broken (layout toks)) (layout toks)))
  (def (Ok again) (tokenize (string->vec result))
    :leave ((Err why) (Err #"formatting made it unreadable: ${why}")))
  (if (= (significant again) (significant toks))
      (Ok (Tuple result (list-sort (list-append notes (width-notes source result)))))
      (Err "formatting would change its tokens, which is a bug in bjo fmt")))

;; The tokens laid out, and what is to be said about their lines.
(: layout (-> (List Tok) (Tuple string (List (Tuple int string)))))
(defun (layout toks)
  (def lines (squeeze-blank-lines (join-closers (lines-of (reshaped toks)))))
  (def out (stringbuilder-empty))
  (def done
    (list-foldl (fun (l st) (write-line out st l))
                (State (frames Nil) (kept (list None)) (kept-depth 0) (line 1) (notes Nil))
                lines))
  (Tuple (stringbuilder->string out) (list-reverse (record-ref done notes))))
;; bjo/git.bjo — the git a project is fetched with.
;;
;; One bare clone per repository URL, under `.bjo/git/`. Bare, because nothing
;; a resolve asks for needs a working tree: the tags, a manifest at a tag and a
;; tag's commit are all read out of the object database. A version is only ever
;; checked out when it has been chosen, and then into `.bjo/pkg/` rather than
;; into the clone.
;;
;; **The network is touched as little as possible.** A clone is made once and
;; fetched only when this run needs something it has not got: a lower bound
;; with no tag that satisfies it, or a commit the lock names and the clone does
;; not hold. A project whose dependencies are all here builds with the network
;; unplugged.
;;
;; Every command is run through `(std run)` rather than through `bjo`'s own
;; `exec`, because the output is read rather than shown. That redirects stdin,
;; which is why `bjo` sets `GIT_TERMINAL_PROMPT=0` before any of this runs: a
;; repository asking for a password would otherwise read from a pipe nobody
;; writes to and wait there for ever.

(export
  Tool tool-for git-on-path? tool-said
  clone-directory ensure-clone fetch-tags
  version-tags version->tag tag-commit commit-present? manifest-at
  package-directory materialise)

(import (std prelude))
(import (std run))
(import (text bjodat))
(import "manifest.bjo")
(import "project.bjo")

(import/extern
  (error-message (: System.Exception.Message (-> Exception string) #:get))
  ;; `git archive --format=zip` and this are the whole of a checkout: git
  ;; writes the tree, the runtime unpacks it. No third program — `tar` is not
  ;; everywhere — and no working tree, index or HEAD added to the bare clone.
  (zip-extract (: System.IO.Compression.ZipFile.ExtractToDirectory (-> string string void)
                  #:exceptions (System.IO.IOException
                                System.UnauthorizedAccessException
                                System.NotSupportedException)))
  ;; The last step of a checkout, and the reason there is one: a rename is
  ;; atomic enough that a directory which exists is a directory that is whole.
  (directory-move (: System.IO.Directory.Move (-> string string void)
                     #:exceptions (System.IO.IOException
                                   System.UnauthorizedAccessException))))

;; Where git's standard error goes. One scratch file, overwritten per command:
;; it is read only to quote git in a message about a command that failed.
(type (: Tool (Record (: errors string))))

(: tool-for #:sync (-> string Tool))
(defun (tool-for tmp)
  (directory-create tmp)
  (Tool (errors (path-combine tmp "git-stderr.txt"))))

(: (-> Tool string))
(defun (tool-said g)
  (if (file-exists? (record-ref g errors))
      (string-trim (file-read-text (record-ref g errors)))
      ""))

;; Is there a git to run at all? Asked before the first command, so that a
;; machine without one is told what is missing rather than what failed.
(: (-> bool))
(defun (git-on-path?) (some? (which "git")))

;; Runs git, and answers its exit code and its output. A non-zero code is an
;; ordinary answer here — "this tag has no manifest" is one — so it is not an
;; `Err`. `Err` is git not starting at all.
(: git-run #:sync (-> Tool (Vec string) (Result string (Tuple int string))))
(defun (git-run g arguments)
  (result-map-err (fun (e) (str "bjo: could not run git: " (error-message e)))
                  (run/output '(errors-into-file ,(record-ref g errors) (git ,arguments)))))

;; A command that has to work, with what git said when it did not.
(: git-must #:sync (-> Tool (Vec string) string (Result string string)))
(defun (git-must g arguments what)
  (def (Ok (Tuple code out)) (git-run g arguments) :propagate)
  (if (= code 0)
      (Ok out)
      (Err (str "bjo: " what " failed.\n  " (tool-said g)))))


;; ---------------------------------------------------------------------------
;; The bare clone
;; ---------------------------------------------------------------------------

;; One directory per URL, named after it. Every character that is not a letter
;; or a digit becomes `_`, so the name is a name on every filesystem and two
;; URLs that differ anywhere differ here.
(: (-> string string))
(defun (url->directory-name url)
  (loop (:for c url)
        (:acc out (stringing (if (or (char-alphabetic? c) (char-numeric? c)) c #\_)))
    => out))

(: (-> string string string))
(defun (clone-directory dot-bjo url)
  (path-combine dot-bjo "git" (url->directory-name url)))

;; The clone for a URL, made the first time it is wanted.
;;
;; A fresh clone brings every tag with it, so nothing fetches after one.
(: ensure-clone #:sync (-> Tool string string (List Symbol) (Result string string)))
(defun (ensure-clone g dot-bjo url name)
  (def dir (clone-directory dot-bjo url))
  (cond
    ((directory-exists? dir) (Ok dir))
    ((not (git-on-path?))
     (Err (str "bjo: git is not on PATH; it is needed to fetch "
               (name->string name) " from " url ".")))
    (else
     (directory-create (path-combine dot-bjo "git"))
     (def (Ok cloned) (git-must g ["clone" "--bare" "--quiet" url dir]
                                (str "cloning " url))
       :propagate)
     (Ok dir))))

;; New tags, for when this run needs something the clone has not got.
(: fetch-tags #:sync (-> Tool string (Result string Unit)))
(defun (fetch-tags g dir)
  (def (Ok fetched)
    (git-must g ["--git-dir" dir "fetch" "--tags" "--force" "--quiet"]
              (str "fetching tags into " dir))
    :propagate)
  (Ok unit))


;; ---------------------------------------------------------------------------
;; Versions, as tags
;; ---------------------------------------------------------------------------

(: (-> Version string))
(defun (version->tag v) (str "v" (version->string v)))

;; A tag as a version, if it is one.
;;
;; Exactly `vX.Y.Z`: the version is written back out and compared with what was
;; read, so `v1.2` and `v1.2.3-rc1` are not versions and are passed over. A tag
;; that is not a version is a name for something else — a branch point, a demo,
;; a talk — and a resolver that guessed at them would install one.
(: (-> string (Option Version)))
(defun (tag->version tag)
  (if (not (string-starts-with? tag "v"))
      None
      (let ((text (substring tag
                             (string-cursor-next tag (string-cursor-start tag))
                             (string-cursor-end tag))))
        (match (parse-version text)
          ((Err why) None)
          ((Ok v) (if (= (version->string v) text) (Some v) None))))))

;; Every version the clone holds, oldest first.
(: version-tags #:sync (-> Tool string (Result string (List Version))))
(defun (version-tags g dir)
  (def (Ok out) (git-must g ["--git-dir" dir "tag" "--list"] (str "reading the tags of " dir))
    :propagate)
  (Ok (list-sort-versions
       (loop (:for line (string-split out "\n"))
             (:let v (tag->version (string-trim line)))
             (:when-let (Some version) v)
             (:acc found (listing version))
         => found))))

;; Insertion sort, oldest first. A repository has tens of tags, not thousands,
;; and `Version` is a record with no ordering of its own.
(: (-> (List Version) (List Version)))
(defun (list-sort-versions versions)
  (match versions
    (Nil Nil)
    ((Cons v rest) (insert-version v (list-sort-versions rest)))))

(: (-> Version (List Version) (List Version)))
(defun (insert-version v sorted)
  (match sorted
    (Nil (Cons v Nil))
    ((Cons head rest)
     (if (<= (version-compare v head) 0)
         (Cons v sorted)
         (Cons head (insert-version v rest))))))

;; The commit a version's tag points at. `^{commit}` unwraps an annotated tag,
;; whose own object is not the commit.
(: tag-commit #:sync (-> Tool string Version (Result string string)))
(defun (tag-commit g dir v)
  (def (Ok out)
    (git-must g ["--git-dir" dir "rev-parse" (str (version->tag v) "^{commit}")]
              (str "reading the commit of " (version->tag v)))
    :propagate)
  (Ok (string-trim out)))

;; Is a commit in this clone? Asked about a commit the lock names, which a
;; clone made after it was written may not have.
(: commit-present? #:sync (-> Tool string string bool))
(defun (commit-present? g dir commit)
  (match (git-run g ["--git-dir" dir "cat-file" "-e" (str commit "^{commit}")])
    ((Err why) #f)
    ((Ok (Tuple code out)) (= code 0))))

;; A version's manifest, read straight out of the object database.
;;
;; Nothing is checked out for this: a resolve reads one file per version, and
;; most of the versions it reads are not the ones it ends up choosing.
(: manifest-at #:sync (-> Tool string Version (Result string string)))
(defun (manifest-at g dir v)
  (def (Ok (Tuple code out))
    (git-run g ["--git-dir" dir "show" (str (version->tag v) ":" manifest-name)])
    :propagate)
  (if (= code 0)
      (Ok out)
      (Err (str "bjo: " (version->tag v) " in " dir " has no " manifest-name
                ", so there is no package there.\n  " (tool-said g)))))


;; ---------------------------------------------------------------------------
;; Putting a version on disk
;; ---------------------------------------------------------------------------

;; `.bjo/pkg/bjorsec@1.2.3`, or `.bjo/pkg/my.app@1.2.3` for a name of several
;; segments. One directory per version, so two projects' dependencies never
;; share one and a version that is no longer wanted can be deleted by name.
(: (-> string (List Symbol) Version string))
(defun (package-directory dot-bjo name v)
  (path-combine dot-bjo "pkg"
                (str (string-join (list->vec (list-map symbol->string name)) ".")
                     "@" (version->string v))))

;; What a materialised directory says it holds.
(: (-> string string))
(defun (commit-marker dir) (path-combine dir ".bjo-commit"))

(: (-> string string))
(defun (marked-commit dir)
  (if (file-exists? (commit-marker dir))
      (string-trim (file-read-text (commit-marker dir)))
      ""))

;; Puts a commit's tree at `target`.
;;
;; A directory already there whose `.bjo-commit` matches is used as it stands;
;; one that does not match is deleted and replaced, because a tag that moved
;; must not leave the old tree under the new version's name. The new tree is
;; unpacked beside it and renamed into place, so a build never reads half of a
;; checkout.
(: materialise #:sync (-> Tool string string string string (Result string Unit)))
(defun (materialise g clone commit tmp target)
  (with-return ret
    (when (and (directory-exists? target) (= (marked-commit target) commit))
      (ret (Ok unit)))

    (directory-create tmp)
    (def archive (path-combine tmp (str commit ".zip")))
    (def unpacked (path-combine tmp (str "unpack-" commit)))

    (when (directory-exists? unpacked) (directory-delete-tree unpacked))
    (when (file-exists? archive) (file-delete archive))

    (def (Ok archived)
      (git-must g ["--git-dir" clone "archive" "--format=zip" "-o" archive commit]
                (str "writing an archive of " commit))
      :propagate)

    (def (Ok unzipped) (zip-extract archive unpacked)
      :leave ((Err e) (Err (str "bjo: could not unpack " archive ": " (error-message e)))))

    (file-write-text (commit-marker unpacked) commit)
    (file-delete archive)

    (when (directory-exists? target) (directory-delete-tree target))
    (directory-create (path-combine target ".."))

    (def (Ok moved) (directory-move unpacked target)
      :leave ((Err e) (Err (str "bjo: could not put " commit " at " target ": "
                                (error-message e)))))
    (Ok unit)))
;; bjo/lock.bjo — `bjo.lock`: the commit each git dependency resolved to.
;;
;;     (lock
;;       (locked
;;         (package (name (bjorsec)) (version "1.2.3")
;;                  (source (git (url "https://github.com/you/bjorsec")))
;;                  (commit "a1b2c3d4e5f6..."))))
;;
;; One entry per git package, sorted by name, so that a diff shows what
;; changed and not what was reordered. Path dependencies are not here: there is
;; nothing to pin about a directory on your own disk.
;;
;; An entry is written `(package ...)` under one `locked` clause, the way a
;; manifest writes `(depends (package ...) (package ...))`: a clause holds its
;; values, and a clause that repeated with the fields of a record spread
;; through it is not something the reader can express. It is the same record
;; either way — named, versioned, sourced, and pinned to a commit.
;;
;; The commit is the whole of the pinning. A version is a tag, a tag is a name
;; somebody can move, and a commit is the content — so what this file protects
;; against is a `v1.2.3` that is not the `v1.2.3` the last build read. There is
;; no separate hash of the tree: the commit already is one.

(export
  Locked Lock
  locked-name locked-version locked-source locked-commit
  read-lock lock-text write-lock lock-entry-for lock-changes)

(import (std prelude))
(import (text bjodat))
(import "manifest.bjo")
(import "project.bjo")

(def/bjodat-type
  (: Locked (Record #:tag package
                    (: name (List Symbol))
                    (: version Version)
                    (: source Source)
                    (: commit string))))

(def/bjodat-type
  (: Lock (Record #:tag lock
                  (: locked (List Locked) #:rest #:default Nil))))

(: (-> Locked (List Symbol)))
(defun (locked-name l) (record-ref l name))

(: (-> Locked Version))
(defun (locked-version l) (record-ref l version))

(: (-> Locked Source))
(defun (locked-source l) (record-ref l source))

(: (-> Locked string))
(defun (locked-commit l) (record-ref l commit))

;; What the last resolve wrote, or nothing at all.
;;
;; A missing file is an empty lock rather than an error: the first build of a
;; checkout that has never been built has no lock, and writing one is what it
;; is about to do.
(: (-> string (Result string (List Locked))))
(defun (read-lock file)
  (if (not (file-exists? file))
      (Ok Nil)
      (match (bjodat-parse-Lock (file-read-text file))
        ((Err why)
         (Err (str "bjo: " file ": " why
                   "\n  Delete it to resolve from the manifests again.")))
        ((Ok lock) (Ok (record-ref lock locked))))))

;; A source as a manifest writes it. The writer is the one in `(text bjodat)`,
;; so what comes out here is what a manifest may say.
(: (-> Source string))
(defun (source-text source) (bjodat->string (->bjodat source)))

(: (-> Locked string))
(defun (locked-text entry)
  (str "    (package (name " (name->string (locked-name entry)) ")"
       " (version \"" (version->string (locked-version entry)) "\")\n"
       "             (source " (source-text (locked-source entry)) ")\n"
       "             (commit \"" (locked-commit entry) "\"))"))

;; The whole file. Sorted by name: the entry begins with it, so sorting the
;; entries by their text sorts by name.
(: (-> (List Locked) string))
(defun (lock-text entries)
  (if (empty? entries)
      "(lock)\n"
      (str "(lock\n  (locked\n"
           (string-join (list->vec (list-sort (list-map locked-text entries))) "\n")
           "))\n")))

(: write-lock #:sync (-> string (List Locked) void))
(defun (write-lock file entries)
  (file-write-text file (lock-text entries)))

(: (-> (List Locked) (List Symbol) (Option Locked)))
(defun (lock-entry-for entries name)
  (find #(= (name->string (locked-name &)) (name->string name)) entries))

;; What changed between two locks, one line each, sorted.
;;
;; With minimal version selection this is empty unless a manifest changed: the
;; same manifests choose the same versions however many times they are read.
(: (-> (List Locked) (List Locked) (List string)))
(defun (lock-changes before after)
  (def added-or-changed
    (loop (:for entry after)
          (:let was (lock-entry-for before (locked-name entry)))
          (:let line
                (match was
                  (None (Some (str "added " (name->string (locked-name entry)) " "
                                   (version->string (locked-version entry)))))
                  ((Some old)
                   (if (= (version->string (locked-version old))
                          (version->string (locked-version entry)))
                       None
                       (Some (str (name->string (locked-name entry)) " "
                                  (version->string (locked-version entry))
                                  " (was " (version->string (locked-version old)) ")"))))))
          (:when-let (Some text) line)
          (:acc out (listing text))
          => out))
  (def removed
    (loop (:for entry before)
          (:when (not (some? (lock-entry-for after (locked-name entry)))))
          (:acc out (listing (str "removed " (name->string (locked-name entry)) " "
                                  (version->string (locked-version entry)))))
          => out))
  (list-sort (list-append added-or-changed removed)))
;; bjo/manifest.bjo
;;
;; What a `manifest.bjodat` says, as types.
;;
;;     (package
;;       (name (bjolang http))
;;       (version "1.2.3")
;;       (depends
;;         (package (name (bjolang cml)) (version (version-at-least "2.0.1")))))
;;
;; A manifest is a *tagged form*: the head symbol names the type and each clause
;; names a field. That is `#:tag` in `def/bjodat-type`, and everything below is
;; an ordinary use of it — there is no manifest-shaped special case anywhere in
;; `(text bjodat)`, and there should not be. See the docs of `def/bjodat-type`.
;;
;; Two things here are decisions rather than mechanism, and both come from the
;; language rather than from the format:
;;
;; **A module name is a parenthesised list of symbols.** `(std run)`,
;; `(text json)`, `(bjolang http)` — the same spelling `import` uses, so a name
;; in a manifest and a name in a source file are one thing written one way.
;; That makes a package name a `(List Symbol)`, which bjodat already reads: a
;; `(...)` is a list and a bare identifier is a symbol, so nothing had to be
;; added to the format to carry it. A string would have been a second spelling
;; of the same value and a thing to parse twice.
;;
;; **A version constraint is a tagged form of its own.** `(version-at-least
;; "1.2")` rather than `">=1.2"`, because a constraint written inside a string
;; is a small language smuggled past the reader — it needs its own parser, its
;; own errors, and its own position information that the reader already has.
;; Written as a form it is read by the reader, refused by the reader, and
;; located by the reader.

(export
  ;; Versions
  Version version version->string parse-version version-compare
  ;; Constraints. A union's cases travel with it, so `AtLeast` and its
  ;; neighbours are not named here — naming one asks for a signature it has no
  ;; way to have.
  VersionReq version-satisfies?
  ;; Where a package comes from
  Git Local Registry Source
  ;; The manifest
  Dep Package
  NuGetPackage nuget-id nuget-version nuget-precompile
  RuntimeSetting runtime-settings
  module-name->string
  ;; What `def/bjodat-type` generated for the two records people read.
  bjodat-parse-Package bjodat-read-Package bjodat->Package Package->bjodat
  bjodat-parse-Dep bjodat-read-Dep bjodat->Dep Dep->bjodat)

(import (std prelude))
(import (text bjodat))

(import/extern
  (string->long (: System.Int64.TryParse (-> string (out long) (Option long)))))

;; ---------------------------------------------------------------------------
;; A module name
;; ---------------------------------------------------------------------------
;; `(List Symbol)`, with no alias over it. An alias would read the same and buy
;; nothing: `(List Symbol)` already has both instances, so `(name (std run))`
;; decodes with no code here at all. This is only how to show one in a message.

(: (-> (List Symbol) string))
(defun (module-name->string m)
  (def sb (stringbuilder-empty))
  (stringbuilder-add! sb #\()
  (loop (:with first #t #f)
        (:for s m)
        (:do (unless first (stringbuilder-add! sb #\space))
             (stringbuilder-add-string! sb (symbol->string s))))
  (stringbuilder-add! sb #\))
  (stringbuilder->string sb))

;; ---------------------------------------------------------------------------
;; A version
;; ---------------------------------------------------------------------------
;; Three numbers. `"1.2"` is `1.2.0` and `"1"` is `1.0.0`, so a manifest may
;; write as much as it means and no more.

(type (: Version (Record (: major int) (: minor int) (: patch int))))

(: (-> int int int Version))
(defun (version ma mi pa) (Version (major ma) (minor mi) (patch pa)))

(: (-> Version string))
(defun (version->string v)
  (str (int->string (record-ref v major)) "."
       (int->string (record-ref v minor)) "."
       (int->string (record-ref v patch))))

(: (-> string string))
(defun (version-shape s)
  (str "\"" s "\" is not a version: a version is written \"1.2.3\""))

(: (-> (List int) string (Result string Version)))
(defun (finish-version out s)
  (match (list-reverse out)
    ((Cons a Nil) (Ok (version a 0 0)))
    ((Cons a (Cons b Nil)) (Ok (version a b 0)))
    ((Cons a (Cons b (Cons c Nil))) (Ok (version a b c)))
    (_ (Err (version-shape s)))))

;; Read by hand rather than by splitting on `.` and converting each piece: the
;; split would allocate a list of strings to throw away, and the conversion
;; would have to refuse a piece like `"1x"` all over again.
(: (-> string (Result string Version)))
(defun (parse-version s)
  (if (string-empty? s)
      (Err (version-shape s))
      (let go ((c (string-cursor-start s)) (acc 0) (digits 0) (out Nil))
        (if (string-cursor-end? s c)
            (if (= digits 0)
                (Err (version-shape s))
                (finish-version (Cons acc out) s))
            (let ((ch (string-cursor-ref s c))
                  (next (string-cursor-next s c)))
              (cond
                ((and (char>=? ch #\0) (char<=? ch #\9))
                 (go next (+ (* acc 10) (- (char->int ch) 48)) (+ digits 1) out))
                ((char=? ch #\.)
                 (if (= digits 0)
                     (Err (version-shape s))
                     (go next 0 0 (Cons acc out))))
                (else (Err (version-shape s)))))))))

;; Negative when `a` is the older, zero when they are the same version.
(: (-> Version Version int))
(defun (version-compare a b)
  (def d (- (record-ref a major) (record-ref b major)))
  (if (not (= d 0))
      d
      (let ((e (- (record-ref a minor) (record-ref b minor))))
        (if (not (= e 0))
            e
            (- (record-ref a patch) (record-ref b patch))))))

;; A version is written as a string, because it is one value and not a record.
(impl (->bjodat Version)
  (defun (->bjodat v) (BjoStr (version->string v))))

(impl (bjodat-> Version)
  (defun (bjodat-> v)
    (match v
      ((BjoStr s) (parse-version s))
      (_ (Err "expected a version, such as \"1.2.3\"")))))

;; ---------------------------------------------------------------------------
;; A version constraint
;; ---------------------------------------------------------------------------
;; `(version-at-least "1.2")`, `(version-at-most "2.0")`,
;; `(version-between "1.2" "2.0")`.
;;
;; **All three are inclusive.** `version-between` is exactly `version-at-least`
;; and `version-at-most` together, which is what its name says and the only
;; rule that does not need a footnote. A half-open range would read the same
;; and mean something the spelling does not admit to.

(type
  (: VersionReq (Union (: AtLeast Version)
                       (: AtMost Version)
                       (: Between Version Version))))

(: (-> Version VersionReq bool))
(defun (version-satisfies? v req)
  (match req
    ((AtLeast lo) (>= (version-compare v lo) 0))
    ((AtMost hi) (<= (version-compare v hi) 0))
    ((Between lo hi) (and (>= (version-compare v lo) 0)
                          (<= (version-compare v hi) 0)))))

(: string)
(def req-shape
  (str "expected a version constraint: (version-at-least \"1.2\"), "
       "(version-at-most \"2.0\") or (version-between \"1.2\" \"2.0\")"))

(: (-> (List Bjodat) string (Result string Version)))
(defun (one-version args tag)
  (match args
    ((Cons a Nil) (bjodat-> a))
    (_ (Err (str "(" tag " ...) takes one version")))))

;; The dispatch is a `cond` over the head symbol, written out rather than held
;; in a table. A table would have to be built at run time and looked up per
;; value, and there are three cases known when this is compiled.
(impl (bjodat-> VersionReq)
  (defun (bjodat-> v)
    (match v
      ((BjoList (Cons (BjoSym h) args))
       (let ((tag (symbol->string h)))
         (cond
           ((= tag "version-at-least")
            (match (one-version args tag)
              ((Err e) (Err e))
              ((Ok a) (Ok (AtLeast a)))))
           ((= tag "version-at-most")
            (match (one-version args tag)
              ((Err e) (Err e))
              ((Ok a) (Ok (AtMost a)))))
           ((= tag "version-between")
            (match args
              ((Cons lo (Cons hi Nil))
               (match (bjodat-> lo)
                 ((Err e) (Err e))
                 ((Ok l) (match (bjodat-> hi)
                           ((Err e) (Err e))
                           ((Ok h2) (Ok (Between l h2)))))))
              (_ (Err "(version-between ...) takes two versions"))))
           (else (Err (str "(" tag " ...) is not a version constraint: " req-shape))))))
      (_ (Err req-shape)))))

(impl (->bjodat VersionReq)
  (defun (->bjodat r)
    (match r
      ((AtLeast a)
       (BjoList (Cons (BjoSym (string->symbol "version-at-least"))
                      (Cons (->bjodat a) Nil))))
      ((AtMost a)
       (BjoList (Cons (BjoSym (string->symbol "version-at-most"))
                      (Cons (->bjodat a) Nil))))
      ((Between lo hi)
       (BjoList (Cons (BjoSym (string->symbol "version-between"))
                      (Cons (->bjodat lo) (Cons (->bjodat hi) Nil))))))))

;; ---------------------------------------------------------------------------
;; Where a package comes from
;; ---------------------------------------------------------------------------
;; Three tagged records, and a union that picks between them by their tags.

(def/bjodat-type
  (: Git (Record #:tag git
                 (: url string)
                 (: commit (Option string) #:optional)
                 (: branch (Option Symbol) #:optional))))

(def/bjodat-type
  (: Local (Record #:tag path (: dir string))))

(def/bjodat-type
  (: Registry (Record #:tag registry
                      (: host string #:default "https://packages.bjolang.org"))))

(type (: Source (Union (: FromGit Git) (: FromPath Local) (: FromRegistry Registry))))

;; This is the dispatch a manifest wants, and it is worth saying what it is not:
;; it is not the reader deciding what `(git ...)` means. The reader hands over a
;; list headed by a symbol and nothing more; the *expected type* — `Source`,
;; here — is what turns that into a case. The same list under a field expecting
;; a `(List Symbol)` is a module name, and neither reading is privileged.
(impl (bjodat-> Source)
  (defun (bjodat-> v)
    (match v
      ((BjoList (Cons (BjoSym h) _))
       (let ((tag (symbol->string h)))
         (cond
           ((= tag "git")
            (match (bjodat->Git v) ((Err e) (Err e)) ((Ok g) (Ok (FromGit g)))))
           ((= tag "path")
            (match (bjodat->Local v) ((Err e) (Err e)) ((Ok p) (Ok (FromPath p)))))
           ((= tag "registry")
            (match (bjodat->Registry v) ((Err e) (Err e)) ((Ok r) (Ok (FromRegistry r)))))
           (else (Err (str "(" tag " ...) is not a source: write (git (url \"...\")), "
                           "(path (dir \"...\")) or (registry)"))))))
      (_ (Err "expected a source, such as (git (url \"https://...\"))")))))

(impl (->bjodat Source)
  (defun (->bjodat s)
    (match s
      ((FromGit g) (->bjodat g))
      ((FromPath p) (->bjodat p))
      ((FromRegistry r) (->bjodat r)))))

;; ---------------------------------------------------------------------------
;; The manifest
;; ---------------------------------------------------------------------------
;; A dependency is written `(package ...)`, under the same tag as the manifest
;; itself, because that is what it is: a package, named and constrained rather
;; than described. Two types sharing one tag is not a collision — which record a
;; form decodes into is the field's to say, and `depends` says `Dep` exactly as
;; the file's root says `Package`.

(def/bjodat-type
  (: Dep (Record #:tag package
                 (: name (List Symbol))
                 (: version (Option VersionReq) #:optional)
                 (: source (Option Source) #:optional))))

;; A NuGet package: `(nuget (id "Npgsql") (version "9.0.3"))`.
;;
;; The version is NuGet's, passed to `PackageReference` as written: "9.0.3"
;; means 9.0.3 or newer, "[9.0.3]" exactly that, and ranges and floating
;; versions mean what NuGet says they mean. It is a string rather than a
;; `Version` because bjo never reads it.
;;
;; `(precompile #f)` keeps the package's assemblies as IL: they are not
;; precompiled with ReadyToRun. Absent, the package is precompiled. The root
;; project's entry for an id wins over a dependency's, so a root can turn it
;; back on with `(precompile #t)`. See `bjo/nuget.bjo`.
(def/bjodat-type
  (: NuGetPackage (Record #:tag nuget
                          (: id string)
                          (: version string)
                          (: precompile (Option bool) #:optional))))

(: (-> NuGetPackage string))
(defun (nuget-id p) (record-ref p id))

(: (-> NuGetPackage string))
(defun (nuget-version p) (record-ref p version))

;; What the entry says about precompiling: `None` when it says nothing.
(: (-> NuGetPackage (Option bool)))
(defun (nuget-precompile p) (record-ref p precompile))

;; One .NET runtime setting of the program, `(gc-server #t)`, as the compiler's
;; `--runtime` takes it: `name=value`. The names are those of
;; `Build.runtimeSettings`, and the two lists below must stay the same as it.
;; They are checked here too, so that a mistake is reported against the
;; manifest and not against a compiler flag.
(type (: RuntimeSetting (Record (: name string) (: value string))))

(def flag-settings
  (list "gc-concurrent" "gc-dynamic-adaptation" "gc-server" "invariant-globalization"
        "tiered-compilation" "tiered-pgo"))
(def count-settings (list "gc-heap-count"))
(def millis-settings (list "call-counting-delay"))

(: string)
(def known-settings
  (string-join (list->vec (list-sort (list-append flag-settings (list-append count-settings millis-settings))))
               ", "))

(impl (bjodat-> RuntimeSetting)
  (defun (bjodat-> v)
    (match v
      ((BjoList (Cons (BjoSym h) (Cons value Nil)))
       (let ((name (symbol->string h)))
         (cond
           ((any? #(= & name) flag-settings)
            (match value
              ((BjoBool b) (Ok (RuntimeSetting (name name) (value (if b "true" "false")))))
              (_ (Err (str "(" name " ...) is #t or #f")))))
           ((any? #(= & name) count-settings)
            (match value
              ((BjoInt n) #:when (>= n 1L) (Ok (RuntimeSetting (name name) (value (long->string n)))))
              (_ (Err (str "(" name " ...) is a whole number from 1")))))
           ((any? #(= & name) millis-settings)
            (match value
              ((BjoInt n) #:when (>= n 0L) (Ok (RuntimeSetting (name name) (value (long->string n)))))
              (_ (Err (str "(" name " ...) is a number of milliseconds from 0")))))
           (else (Err (str "(" name " ...) is not a runtime setting. The settings are: "
                           known-settings))))))
      (_ (Err "a runtime setting is (name value), such as (gc-server #t)")))))

(impl (->bjodat RuntimeSetting)
  (defun (->bjodat r)
    (def value (record-ref r value))
    (BjoList (list (BjoSym (string->symbol (record-ref r name)))
                   (case value
                     ("true" (BjoBool #t))
                     ("false" (BjoBool #f))
                     (else (BjoInt (match (string->long value) ((Some n) n) (None 0L)))))))))

;; The settings as the compiler's `--runtime` takes them, `name=value`, sorted
;; by name: also the form of the `runtime` lines of a program's build record.
(: (-> (List RuntimeSetting) (List string)))
(defun (runtime-settings settings)
  (list-sort (list-map #(str (record-ref & name) "=" (record-ref & value)) settings)))

(def/bjodat-type
  (: Package (Record #:tag package
                     (: name (List Symbol))
                     (: version Version)
                     (: description (Option string) #:optional)
                     (: license (Option string) #:optional)
                     (: authors (Vec string) #:default (vec-empty))
                     (: entry string #:default "src/main.bjo")
                     (: targets (Vec Symbol) #:default (vec-empty))
                     ;; The .NET shared frameworks this package's modules may
                     ;; name types out of: `(frameworks "Microsoft.AspNetCore.App")`.
                     ;;
                     ;; Optional, and allowed in any package, root or
                     ;; dependency. `Microsoft.NETCore.App` is always there and
                     ;; must not be listed — saying it would suggest that not
                     ;; saying it left it out. A package without the field
                     ;; behaves exactly as it always has.
                     ;;
                     ;; `#:rest` for the same reason `depends` has it: the
                     ;; clause holds as many names as it holds, and a `[...]`
                     ;; around them would be a bracket carrying no information.
                     (: frameworks (List string) #:rest #:default Nil)
                     ;; Files and directories the package's modules read at
                     ;; run time, relative to the package directory:
                     ;; `(resources "grammars")`. `bjo publish` copies them,
                     ;; keeping their place relative to the package's modules,
                     ;; so code that finds them from its own `.dll` still does.
                     (: resources (List string) #:rest #:default Nil)
                     ;; NuGet packages the package's modules may use through
                     ;; `import/extern` and `import/class`. Allowed in any
                     ;; package, root or dependency.
                     (: packages (List NuGetPackage) #:rest #:default Nil)
                     ;; Directories of C# sources, relative to the package
                     ;; directory: `(csharp "glue")`. bjo compiles them into
                     ;; one assembly per package, against the package's NuGet
                     ;; packages and the Bjolang runtime, and its modules use
                     ;; it through `import/extern` and `import/class`. Allowed
                     ;; in any package, root or dependency.
                     (: csharp (List string) #:rest #:default Nil)
                     ;; The .NET runtime settings of the program that the
                     ;; project builds: `(runtime (gc-server #t))`. Only the
                     ;; root project's count: they are settings of a process,
                     ;; and a dependency is not one.
                     (: runtime (List RuntimeSetting) #:rest #:default Nil)
                     ;; `(aot #t)`: the program is also built as a Native AOT
                     ;; executable, and `bjo run` runs that one. Only the root
                     ;; project's counts, as with `runtime`. Optional rather
                     ;; than defaulted, so that a manifest written back without
                     ;; it does not gain an `(aot #f)`.
                     (: aot (Option bool) #:optional)
                     ;; `(aot-cpu "native")`: the CPU the Native AOT executable
                     ;; is built for, passed as `--aot-cpu`. `native` is the
                     ;; machine that builds; `x86-64-v3` is any CPU with AVX2.
                     ;; Without it the executable runs on any x86-64 CPU and
                     ;; uses 128-bit vectors. Only `(aot #t)` reads it.
                     (: aot-cpu (Option string) #:optional)
                     (: source (Option Source) #:optional)
                     ;; `#:rest`: the clause holds as many packages as it holds,
                     ;; rather than one vec of them. This is the field that made
                     ;; `#:rest` worth having — `(depends (package ...) (package
                     ;; ...))` is how a person writes it, and a `[...]` around
                     ;; them would be a bracket that carries no information.
                     (: depends (List Dep) #:rest #:default Nil)
                     (: dev-depends (List Dep) #:rest #:default Nil))))
;; bjo/nuget.bjo — the NuGet packages a project's manifest names.
;;
;;     (package
;;       (name (shop))
;;       (version "0.1.0")
;;       (packages (nuget (id "Npgsql") (version "9.0.3"))))
;;
;; The packages are resolved by the .NET SDK rather than here. Which files of a
;; package a program compiles against and which it runs with depends on the
;; target framework, on the shared frameworks the program also uses (ASP.NET
;; already contains some packages, and its copy has to win), on `_._`
;; placeholders and on runtime-specific assets. MSBuild already knows all of
;; that, so `bjo` generates a small project in `.bjo/nuget/root/`, runs the
;; SDK's own restore and reference resolution on it, and has a target of its own
;; write the results into three files beside it:
;;
;;   compile.txt   the package assemblies the C# compile references
;;   runtime.txt   the package assemblies the program loads, and the type
;;                 checker reflects over
;;   native.txt    runtime-specific assets, for every runtime identifier: one
;;                 line each of package, asset type, RID and path. The compiler
;;                 picks the ones for the machine it runs on.
;;
;; A dependency that declares packages gets a generated project of its own in
;; `.bjo/nuget/packages/`, which the root project references. NuGet then
;; resolves versions across all of them, and the root's own declaration of a
;; package wins over a dependency's.
;;
;; A package whose manifest says `(csharp "glue")` has C# sources, its glue.
;; Each such package gets a generated project of its own in
;; `.bjo/nuget/glue/`, which compiles them into one assembly against the
;; package's NuGet packages and the Bjolang runtime's assemblies. The root
;; project references it, so the one restore resolves its packages along with
;; everything else's, and the lists name the assembly it builds. The glue is
;; built after the restore, by `dotnet build`.
;;
;; The compiler is given the directory with `--nuget`. The lists are written
;; only when their contents change, because the compiler records them as inputs
;; of every module it builds against them.
;;
;; Starting MSBuild costs a couple of seconds, so it is skipped when nothing it
;; depends on has changed. A stamp file holds a hash of the generated project —
;; which spells out the packages, the frameworks and the target framework — and
;; of `packages.lock.json`, and the lists have to name files that are still
;; there: a cleared NuGet cache is a restore to redo. A project that names no
;; packages starts no `dotnet` at all.

(export restore-packages)

(import (std prelude))
(import (std run))
(import (text bjodat))
(import "manifest.bjo")
(import "project.bjo")

(import/class (Encoding (: System.Text.Encoding)))

(import/extern
  (error-message (: System.Exception.Message (-> Exception string) #:get))
  (utf8 (: System.Text.Encoding.UTF8 #:get))
  (encode (: System.Text.Encoding.GetBytes (-> Encoding string (Array byte))))
  (sha256 (: System.Security.Cryptography.SHA256.HashData (-> (Array byte) (Array byte))))
  (hex (: System.Convert.ToHexString (-> (Array byte) string)))
  (index-of (: System.String.IndexOf (-> string string int)))
  (substring-from (: System.String.Substring (-> string int string)))
  (substring-range (: System.String.Substring (-> string int int string))))


;; ---------------------------------------------------------------------------
;; The generated project
;; ---------------------------------------------------------------------------

(: (-> string string))
(defun (xml-escape s)
  (string-replace
    (string-replace (string-replace (string-replace s "&" "&amp;") "<" "&lt;") ">" "&gt;")
    "\"" "&quot;"))

;; The target that writes the three lists.
;;
;; A package's compile assets are the `ReferencePath` items with a
;; `NuGetPackageId`. The shared frameworks' reference assemblies carry one too
;; (`Microsoft.NETCore.App.Ref`), so those are told apart by their
;; `FrameworkReferenceName`. Runtime assets are the `.dll`s among
;; `ReferenceCopyLocalPaths` that go in the output directory itself: that
;; leaves out symbols, documentation, satellite resource assemblies and
;; runtime-specific files, which all go in a subdirectory or are not `.dll`s.
;; Both filter on `NuGetPackageId` as well, which leaves out the generated
;; projects of dependencies: those are never built, and have nothing to offer.
;;
;; Runtime-specific assets are `RuntimeTargetsCopyLocalItems`, one per runtime
;; identifier a package has files for, with the RID and whether it is `native`
;; or `runtime` (managed) as metadata. `NativeCopyLocalItems` are native files
;; that are the same everywhere, and have no RID.
;;
;; `glue-dlls` are the assemblies the glue projects build. They go in both
;; lists by name, since the restore does not build them: they are built after
;; it, and are there before anything reads the lists.
(: (-> (List string) string))
(defun (resolve-target glue-dlls)
  (str "  <Target Name=\"BjoResolve\" DependsOnTargets=\"ResolveReferences\">\n"
       "    <ItemGroup>\n"
       "      <_BjoCompile Include=\"@(ReferencePath)\" Condition=\"'%(ReferencePath.NuGetPackageId)' != '' and '%(ReferencePath.FrameworkReferenceName)' == ''\" />\n"
       "      <_BjoRuntime Include=\"@(ReferenceCopyLocalPaths)\" Condition=\"'%(ReferenceCopyLocalPaths.NuGetPackageId)' != '' and '%(ReferenceCopyLocalPaths.Extension)' == '.dll' and '%(ReferenceCopyLocalPaths.DestinationSubDirectory)' == ''\" />\n"
       (string-join (list->vec (list-map #(str "      <_BjoCompile Include=\"" (xml-escape &) "\" />\n"
                                               "      <_BjoRuntime Include=\"" (xml-escape &) "\" />\n")
                                         glue-dlls))
                    "")
       "      <_BjoNative Include=\"@(NativeCopyLocalItems);@(RuntimeTargetsCopyLocalItems)\" />\n"
       "    </ItemGroup>\n"
       "    <WriteLinesToFile File=\"$(MSBuildProjectDirectory)/compile.txt\" Lines=\"@(_BjoCompile->'%(FullPath)')\" Overwrite=\"true\" WriteOnlyWhenDifferent=\"true\" />\n"
       "    <WriteLinesToFile File=\"$(MSBuildProjectDirectory)/runtime.txt\" Lines=\"@(_BjoRuntime->'%(FullPath)')\" Overwrite=\"true\" WriteOnlyWhenDifferent=\"true\" />\n"
       "    <WriteLinesToFile File=\"$(MSBuildProjectDirectory)/native.txt\" Lines=\"@(_BjoNative->'%(NuGetPackageId)&#9;%(AssetType)&#9;%(RuntimeIdentifier)&#9;%(FullPath)')\" Overwrite=\"true\" WriteOnlyWhenDifferent=\"true\" />\n"
       "    <WriteLinesToFile File=\"$(MSBuildProjectDirectory)/packages.txt\" Lines=\"@(_BjoRuntime->'%(NuGetPackageId)&#9;%(FullPath)')\" Overwrite=\"true\" WriteOnlyWhenDifferent=\"true\" />\n"
       "  </Target>\n"))

(: (-> (List NuGetPackage) (List string) string))
(defun (item-lines packages frameworks)
  (def package-lines
    (list-map #(str "    <PackageReference Include=\"" (xml-escape (nuget-id &))
                    "\" Version=\"" (xml-escape (nuget-version &)) "\" />\n")
              packages))
  (def framework-lines
    (list-map #(str "    <FrameworkReference Include=\"" (xml-escape &) "\" />\n") frameworks))
  (str (string-join (list->vec package-lines) "")
       (string-join (list->vec framework-lines) "")))

;; `Root.csproj`. A library with no sources; `CopyLocalLockFileAssemblies` is
;; what puts a library's package runtime assets in `ReferenceCopyLocalPaths`.
;; The shared frameworks are referenced so that the SDK's conflict resolution
;; drops a package assembly the framework already provides. `references` are
;; the generated projects of the dependencies that declare packages, and of
;; the glue; `glue-dlls` are what the glue projects build.
(: (-> (List NuGetPackage) (List string) (List string) (List string) string string))
(defun (csproj-text packages frameworks references glue-dlls lock-file)
  (def reference-lines
    (list-map #(str "    <ProjectReference Include=\"" (xml-escape &) "\" />\n") references))
  (str "<!-- Generated by bjo from manifest.bjodat. Do not edit. -->\n"
       "<Project Sdk=\"Microsoft.NET.Sdk\">\n"
       "  <PropertyGroup>\n"
       "    <TargetFramework>" target-framework "</TargetFramework>\n"
       "    <OutputType>Library</OutputType>\n"
       "    <EnableDefaultItems>false</EnableDefaultItems>\n"
       "    <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>\n"
       "    <RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>\n"
       "    <NuGetLockFilePath>" (xml-escape lock-file) "</NuGetLockFilePath>\n"
       "  </PropertyGroup>\n"
       "  <ItemGroup>\n"
       (item-lines packages frameworks)
       (string-join (list->vec reference-lines) "")
       "  </ItemGroup>\n"
       (resolve-target glue-dlls)
       "</Project>\n"))

;; The generated project of a dependency: its packages and frameworks, which
;; reach the root project through its `ProjectReference`. Never built — the
;; restore is run with `BuildProjectReferences=false`.
(: (-> string (List NuGetPackage) (List string) string))
(defun (dependency-csproj-text manifest packages frameworks)
  (str "<!-- Generated by bjo from " (xml-escape manifest) ". Do not edit. -->\n"
       "<Project Sdk=\"Microsoft.NET.Sdk\">\n"
       "  <PropertyGroup>\n"
       "    <TargetFramework>" target-framework "</TargetFramework>\n"
       "    <OutputType>Library</OutputType>\n"
       "    <EnableDefaultItems>false</EnableDefaultItems>\n"
       "  </PropertyGroup>\n"
       "  <ItemGroup>\n"
       (item-lines packages frameworks)
       "  </ItemGroup>\n"
       "</Project>\n"))

;; Writes `path` unless it already says `text`.
(: write-if-changed #:sync (-> string string void))
(defun (write-if-changed path text)
  (unless (and (file-exists? path) (= (file-read-text path) text))
    (file-write-text path text)))


;; ---------------------------------------------------------------------------
;; The glue
;; ---------------------------------------------------------------------------

;; A package of the build with C# sources: its name, its directory, and what
;; its manifest says.
(type (: Glue (Record (: name (List Symbol)) (: dir string) (: package Package))))

;; The packages of the build that have glue, the project itself included.
(: (-> (List Root) (List Glue)))
(defun (glues-of roots)
  (loop (:for r roots)
        (:when-let (Some dir) (root-package-dir r))
        (:when-let (Some package) (root-package r))
        (:when (not (empty? (record-ref package csharp))))
        (:acc out (listing (Glue (name (root-name r)) (dir (path-absolute dir)) (package package))))
        => out))

(: (-> Glue string))
(defun (glue-slug g)
  (string-join (list->vec (list-map symbol->string (record-ref g name))) "."))

;; The assembly's name, which is also its file's.
(: (-> Glue string))
(defun (glue-assembly g) (str "BjoGlue." (glue-slug g)))

(: (-> Project Glue string))
(defun (glue-project-dir project g)
  (path-combine (project-dot-bjo project) "nuget" "glue" (glue-slug g)))

(: (-> Project Glue string))
(defun (glue-csproj-file project g)
  (path-combine (glue-project-dir project g) (str (glue-assembly g) ".csproj")))

;; Where the glue project puts its assembly: a fixed place, so that the lists
;; can name it before it is built.
(: (-> Project Glue string))
(defun (glue-dll project g)
  (path-combine (glue-project-dir project g) "out" (str (glue-assembly g) ".dll")))

;; The source directories, as the manifest names them, made absolute.
(: (-> Glue (List string)))
(defun (glue-source-dirs g)
  (list-map #(path-absolute (path-combine (record-ref g dir) &)) (record-ref (record-ref g package) csharp)))

;; The Bjolang runtime's assemblies, which glue may use: `Option` and
;; `Result` from BjolangRuntime, `string` from BjoString, the collections.
;; Those the compiler's directory has.
(: (List string))
(def runtime-assembly-names
  (list "BjolangRuntime" "BjoString" "SchemeList" "BjoRx"))

(: (-> (List string)))
(defun (runtime-assemblies)
  (loop (:for name runtime-assembly-names)
        (:let path (path-combine (compiler-dir) (str name ".dll")))
        (:acc out (listing path) #:when (file-exists? path))
        => out))

;; The glue's project. It compiles every `.cs` file under its source
;; directories except those under `bin` and `obj`, against its package's NuGet
;; packages and frameworks and the runtime's assemblies. Those are not copied
;; next to it (`Private` false): a program loads them anyway.
(: (-> Glue string))
(defun (glue-csproj-text g)
  (def package (record-ref g package))
  (def compile-lines
    (list-map #(str "    <Compile Include=\"" (xml-escape (path-combine & "**" "*.cs"))
                    "\" Exclude=\"" (xml-escape (path-combine & "**" "bin" "**")) ";"
                    (xml-escape (path-combine & "**" "obj" "**")) "\" />\n")
              (glue-source-dirs g)))
  (def runtime-lines
    (list-map #(str "    <Reference Include=\"" (xml-escape (option-ref-or (path-filename &) &)) "\">"
                    "<HintPath>" (xml-escape &) "</HintPath><Private>false</Private></Reference>\n")
              (runtime-assemblies)))
  (str "<!-- Generated by bjo from " (xml-escape (path-combine (record-ref g dir) manifest-name))
       ". Do not edit. -->\n"
       "<Project Sdk=\"Microsoft.NET.Sdk\">\n"
       "  <PropertyGroup>\n"
       "    <TargetFramework>" target-framework "</TargetFramework>\n"
       "    <OutputType>Library</OutputType>\n"
       "    <AssemblyName>" (xml-escape (glue-assembly g)) "</AssemblyName>\n"
       "    <EnableDefaultItems>false</EnableDefaultItems>\n"
       "    <ImplicitUsings>enable</ImplicitUsings>\n"
       "    <Nullable>disable</Nullable>\n"
       "    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>\n"
       "    <OutputPath>$(MSBuildProjectDirectory)/out/</OutputPath>\n"
       "    <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>\n"
       "    <AppendRuntimeIdentifierToOutputPath>false</AppendRuntimeIdentifierToOutputPath>\n"
       "  </PropertyGroup>\n"
       "  <ItemGroup>\n"
       (string-join (list->vec compile-lines) "")
       (item-lines (record-ref package packages) (record-ref package frameworks))
       (string-join (list->vec runtime-lines) "")
       "  </ItemGroup>\n"
       "</Project>\n"))

;; What the glue's build reads that its project file does not say: every C#
;; source and its text, and when each runtime assembly was written. A change
;; to either is a build to redo.
(: (-> Glue string))
(defun (glue-fingerprint g)
  (def sources
    (loop (:for dir (glue-source-dirs g))
          (:when (directory-exists? dir))
          (:for file (directory-walk dir #:into? (fun (d) (not (any? #(= (path-filename d) (Some &))
                                                                      (list "bin" "obj"))))))
          (:when (string-ends-with? file ".cs"))
          (:acc out (listing (str file "\n" (file-read-text file))))
          => (list-sort out)))
  (def runtime
    (list-map #(str & " " (match (file-info &)
                            ((Some info) (long->string (record-ref info modified-ms)))
                            (None "absent")))
              (runtime-assemblies)))
  (string-join (list->vec (list-append sources runtime)) "\n"))

;; The C# compiler's errors in `output`, each once: `file(line,col): error
;; CS1002: ; expected`, without the project MSBuild appends.
(: (-> string (List string)))
(defun (compile-errors output)
  (def found
    (loop (:for line (string-split output "\n"))
          (:let at (index-of line ": error "))
          (:when (>= at 0))
          (:let end (index-of line " ["))
          (:acc out (listing (string-trim (if (> end at) (substring-range line 0 end) line))))
          => out))
  (loop (:for m found)
        (:acc out (folding Nil (if (any? #(= & m) out) out (Cons m out))))
        => (list-reverse out)))

;; Builds one glue project, after the restore that resolved its packages.
(: build-glue #:sync (-> Project Glue (Result string Unit)))
(defun (build-glue project g)
  (def dir (glue-project-dir project g))
  (def errors (path-combine dir "build-stderr.txt"))
  (def arguments
    (list->vec
      (list "build" (glue-csproj-file project g) "-c" "Release" "--no-restore" "-nologo"
            "-p:ImportDirectoryBuildProps=false"
            "-p:ImportDirectoryBuildTargets=false"
            "-p:ImportDirectoryPackagesProps=false")))
  (match (run/output '(errors-into-file ,errors (dotnet ,arguments)))
    ((Err e) (Err (str "bjo: could not run dotnet: " (error-message e))))
    ((Ok (Tuple 0 _)) (Ok unit))
    ((Ok (Tuple _ out))
     (def said (string-trim (str out "\n" (if (file-exists? errors) (file-read-text errors) ""))))
     (def name (name->string (record-ref g name)))
     (Err (match (compile-errors said)
            (Nil (str "bjo: building the C# glue of " name " failed. dotnet said:\n" said))
            (found (str "bjo: the C# glue of " name " does not compile:\n"
                        (string-join (list->vec (list-map #(str "  " &) found)) "\n"))))))))


;; ---------------------------------------------------------------------------
;; Skipping the restore
;; ---------------------------------------------------------------------------

(: (-> string string string))
(defun (stamp-of csproj lock-file)
  (def lock (if (file-exists? lock-file) (file-read-text lock-file) ""))
  (hex (sha256 (encode utf8 (str csproj "\n--- packages.lock.json\n" lock)))))

;; The paths in a list file, or `None` when there is no such file.
(: (-> string (Option (List string))))
(defun (listed file)
  (if (not (file-exists? file))
      None
      (Some (loop (:for line (string-split (file-read-text file) "\n"))
                  (:let path (string-trim line))
                  (:acc out (listing path) #:when (not (= path "")))
                  => out))))

(: (-> string bool))
(defun (list-current? file)
  (match (listed file)
    (None #f)
    ((Some paths) (all? file-exists? paths))))

;; The same for `native.txt`, whose path is the last of four fields.
(: (-> string bool))
(defun (native-current? file)
  (match (listed file)
    (None #f)
    ((Some lines)
     (all? (fun (line)
             (def fields (string-split line "\t"))
             (and (= (vec-length fields) 4) (file-exists? (vec-ref fields 3))))
           lines))))

;; Is the last restore still the answer?
(: (-> string string bool))
(defun (restore-current? dir stamp)
  (def stamp-file (path-combine dir "stamp"))
  (and (file-exists? stamp-file)
       (= (string-trim (file-read-text stamp-file)) stamp)
       (list-current? (path-combine dir "compile.txt"))
       (list-current? (path-combine dir "runtime.txt"))
       (native-current? (path-combine dir "native.txt"))
       ;; Written by a target added later: a restore without it is redone.
       (file-exists? (path-combine dir "packages.txt"))))


;; ---------------------------------------------------------------------------
;; Running MSBuild
;; ---------------------------------------------------------------------------

;; Runs the restore and the resolve target, and answers the exit code and
;; everything MSBuild said.
;;
;; The `Directory.*` imports are turned off because a project is often inside a
;; repository that has them, and they would apply to the generated project: a
;; `Directory.Packages.props` turns on central package management, which
;; refuses a `Version` on a `PackageReference`, and a `Directory.Build.props`
;; can change the target framework. `NuGet.config` and `global.json` are still
;; found by walking up from the generated project, which reaches the user's.
(: run-msbuild #:sync (-> string bool (Result string (Tuple int string))))
(defun (run-msbuild dir locked?)
  (def errors (path-combine dir "msbuild-stderr.txt"))
  (def arguments
    (list->vec
      (list-append
        (list "msbuild" (path-combine dir "Root.csproj") "-restore" "-t:BjoResolve" "-nologo"
              "-p:BuildProjectReferences=false"
              "-p:ImportDirectoryBuildProps=false"
              "-p:ImportDirectoryBuildTargets=false"
              "-p:ImportDirectoryPackagesProps=false")
        (if locked? (list "-p:RestoreLockedMode=true") Nil))))
  (set-environment-variable! "DOTNET_NOLOGO" "1")
  (match (run/output '(errors-into-file ,errors (dotnet ,arguments)))
    ((Err e) (Err (str "bjo: could not run dotnet: " (error-message e))))
    ((Ok (Tuple code out))
     (def said (if (file-exists? errors) (file-read-text errors) ""))
     (Ok (Tuple code (string-trim (str out "\n" said)))))))

;; Every distinct `NUxxxx: message` MSBuild reported with `code`, from the code
;; to the end of the line.
(: (-> string string (List string)))
(defun (messages-with output code)
  (def found
    (loop (:for line (string-split output "\n"))
          (:let at (index-of line (str code ": ")))
          (:acc out (listing (string-trim (substring-from line at))) #:when (>= at 0))
          => out))
  (loop (:for m found)
        (:acc out (folding Nil (if (any? #(= & m) out) out (Cons m out))))
        => (list-reverse out)))

;; What a failed restore is reported as. A NuGet error is quoted with its code
;; and a hint about what usually causes it; anything else is MSBuild's own
;; output, which is the only account there is.
(: (-> string string string (Option string)))
(defun (explain output code hint)
  (match (messages-with output code)
    (Nil None)
    (found (Some (str (string-join (list->vec (list-map #(str "  " &) found)) "\n")
                      "\n  hint: " hint)))))

(: (-> string bool string string))
(defun (failure-message output locked? lock-file)
  (def not-found "Check the package name and version, and the NuGet sources in your NuGet.config.")
  (def explained
    (loop (:for answer (list (explain output "NU1101" not-found)
                             (explain output "NU1102" not-found)
                             (explain output "NU1301"
                                      "The first restore of a package needs network access, or a local feed in NuGet.config.")
                             (explain output "NU1004"
                                      (if locked?
                                          (str "Run 'bjo build' without --locked to update " lock-file ".")
                                          (str lock-file " does not match the manifest.")))))
          (:when-let (Some text) answer)
          (:acc out (listing text))
          => out))
  (if (empty? explained)
      (str "bjo: restoring the NuGet packages failed. MSBuild said:\n" output)
      (str "bjo: restoring the NuGet packages failed:\n"
           (string-join (list->vec explained) "\n"))))

;; ---------------------------------------------------------------------------
;; ReadyToRun copies
;; ---------------------------------------------------------------------------
;;
;; After a restore, the package assemblies are precompiled with ReadyToRun by
;; the compiler's `--precompile-packages`, into a cache outside the NuGet
;; cache. `Build.fs` describes the cache and the files in the restore's
;; directory.
;;
;; - Each package is precompiled unless an entry for its id says
;;   `(precompile #f)`.
;; - The root project's entry wins. Without one, a dependency's `#f` turns it
;;   off.
;; - A package that is not declared (a transitive one) is opted out by
;;   declaring it with `(precompile #f)`.
;; - `BJOLANG_NO_R2R` turns it all off.

;; What the entries in `packages` for the id `key` (lowercase) say.
(: (-> (List NuGetPackage) string (List bool)))
(defun (said-about packages key)
  (loop (:for p packages)
        (:when (= (string-downcase (nuget-id p)) key))
        (:when-let (Some b) (nuget-precompile p))
        (:acc out (listing b))
        => out))

;; Whether the package `id` is precompiled.
(: (-> (List NuGetPackage) (List Declaring) string bool))
(defun (precompiled? own deps id)
  (def key (string-downcase id))
  (match (said-about own key)
    ((Cons b _) b)
    (Nil (not (any? #(any? not (said-about (declared-by &) key)) deps)))))

;; The NuGet packages a dependency declares.
(: (-> Declaring (List NuGetPackage)))
(defun (declared-by d) (record-ref (record-ref d package) packages))

;; The lines of `r2r-inputs.txt`: `r2r` or `il`, a tab, and the path, for each
;; package assembly in `packages.txt`. The glue's assemblies have no package
;; id there, and are left out.
(: (-> string (List NuGetPackage) (List Declaring) string))
(defun (precompile-inputs dir own deps)
  (match (listed (path-combine dir "packages.txt"))
    (None "")
    ((Some lines)
     (string-join
       (list->vec
         (loop (:for line lines)
               (:let fields (string-split line "\t"))
               (:when (and (= (vec-length fields) 2) (not (= (vec-ref fields 0) ""))))
               (:acc out (listing (str (if (precompiled? own deps (vec-ref fields 0)) "r2r" "il")
                                       "\t" (vec-ref fields 1))))
               => out))
       "\n"))))

;; Whether every copy `r2r.txt` names is still there.
(: (-> string bool))
(defun (copies-current? file)
  (match (listed file)
    (None #f)
    ((Some lines)
     (all? (fun (line)
             (def fields (string-split line "\t"))
             (and (= (vec-length fields) 2) (file-exists? (vec-ref fields 1))))
           lines))))

;; Fills the cache for the restore in `dir`, unless the last run had the same
;; inputs and its copies are all still there.
(: precompile-packages #:sync (-> string (List NuGetPackage) (List Declaring) (Result string Unit)))
(defun (precompile-packages dir own deps)
  (with-return ret
    (match (get-environment-variable "BJOLANG_NO_R2R")
      ((Some v) (unless (or (= v "") (= v "0")) (ret (Ok unit))))
      (None unit))
    (def inputs (precompile-inputs dir own deps))
    (def inputs-file (path-combine dir "r2r-inputs.txt"))
    (when (and (file-exists? inputs-file)
               (= (file-read-text inputs-file) inputs)
               (copies-current? (path-combine dir "r2r.txt")))
      (ret (Ok unit)))
    ;; The compiler reads the inputs from the file. It is written again after
    ;; a success, so that a failed run is tried again next time.
    (def pending (path-combine dir "r2r-inputs.pending"))
    (file-write-text pending inputs)
    (def arguments (list->vec (list (path-combine (compiler-dir) "Bjolang.dll") "--precompile-packages" dir pending)))
    (match (run/output '(dotnet ,arguments))
      ((Err e) (Err (str "bjo: could not run the compiler: " (error-message e))))
      ((Ok (Tuple 0 out))
       (unless (= (string-trim out) "") (println (string-trim out)))
       (file-write-text inputs-file inputs)
       (Ok unit))
      ((Ok (Tuple code out))
       (Err (str "bjo: precompiling the NuGet packages failed (exit code " (int->string code) "):\n"
                 (string-trim out)))))))

;; ---------------------------------------------------------------------------
;; restore-packages
;; ---------------------------------------------------------------------------

;; A package of the build that declares NuGet packages: its manifest, and what
;; the manifest says.
(type (: Declaring (Record (: name (List Symbol)) (: manifest string) (: package Package))))

;; The dependencies that declare packages. The project being built is not one
;; of them: its packages go in the root project itself.
(: (-> Project (List Root) (List Declaring)))
(defun (declaring-dependencies project roots)
  (def own (path-absolute (project-dir project)))
  (loop (:for r roots)
        (:when-let (Some dir) (root-package-dir r))
        (:when (not (= (path-absolute dir) own)))
        (:when-let (Some package) (root-package r))
        (:when (not (empty? (record-ref package packages))))
        (:acc out (listing (Declaring (name (root-name r))
                                      (manifest (path-combine dir manifest-name))
                                      (package package))))
        => out))

;; Where a dependency's generated project goes. The file name becomes the
;; project's name in the lock file, and is prefixed so that it cannot be taken
;; for a NuGet package of the same name.
(: (-> Project (List Symbol) string))
(defun (dependency-csproj-file project name)
  (def slug (string-join (list->vec (list-map symbol->string name)) "."))
  (path-combine (project-dot-bjo project) "nuget" "packages" slug
                (str "BjoPackage." slug ".csproj")))

(: (-> (List NuGetPackage) (Option string)))
(defun (duplicate-id packages)
  ;; NuGet ids are case-insensitive, so `npgsql` and `Npgsql` are one.
  (def ids (list-sort (list-map #(string-downcase (nuget-id &)) packages)))
  (let go ((left ids))
    (match left
      ((Cons a (Cons b more)) (if (= a b) (Some a) (go (Cons b more))))
      (_ None))))

(: (-> string Package (Option string)))
(defun (duplicate-in manifest package)
  (option-map (fun (id) (str "bjo: " manifest ": the NuGet package " id
                             " is listed twice under (packages ...)."))
              (duplicate-id (record-ref package packages))))

;; Restores the NuGet packages of the build, and answers the directory to hand
;; the compiler with `--nuget` — or `None` when no manifest names any.
;;
;; `roots` are every package of the build: a dependency's packages are the
;; build's packages too.
(: restore-packages #:sync (-> Project (List Root) bool (Result string (Option string))))
(defun (restore-packages project roots locked?)
  (with-return ret
    (def package (project-package project))
    (def packages (record-ref package packages))
    (def deps (declaring-dependencies project roots))
    (def glues (glues-of roots))
    (when (and (empty? packages) (empty? deps) (empty? glues)) (ret (Ok None)))

    (def own-manifest (path-combine (project-dir project) manifest-name))
    (def duplicates
      (loop (:for message (Cons (duplicate-in own-manifest package)
                                (list-map #(duplicate-in (record-ref & manifest) (record-ref & package))
                                          deps)))
            (:when-let (Some text) message)
            (:acc out (listing text))
            => out))
    (match duplicates
      ((Cons text _) (ret (Err text)))
      (Nil unit))

    (def dir (project-nuget-dir project))
    (def lock-file (project-nuget-lock-file project))
    (def dep-files (list-map #(dependency-csproj-file project (record-ref & name)) deps))
    (def dep-texts
      (list-map #(dependency-csproj-text (record-ref & manifest)
                                         (record-ref (record-ref & package) packages)
                                         (record-ref (record-ref & package) frameworks))
                deps))
    (def glue-files (list-map #(glue-csproj-file project &) glues))
    (def glue-texts (list-map glue-csproj-text glues))
    (def glue-dlls (list-sort (list-map #(glue-dll project &) glues)))
    (def csproj (csproj-text packages (record-ref package frameworks)
                             (list-sort (list-append dep-files glue-files)) glue-dlls lock-file))
    ;; Everything the restore and the glue's builds read that `bjo` wrote, and
    ;; the glue's sources, in an order that does not depend on the order of the
    ;; roots.
    (def generated
      (string-join (list->vec (Cons csproj
                                    (list-sort (list-append
                                                 (loop (:for file dep-files)
                                                       (:for text dep-texts)
                                                       (:acc out (listing (str file "\n" text)))
                                                       => out)
                                                 (loop (:for g glues)
                                                       (:for file glue-files)
                                                       (:for text glue-texts)
                                                       (:acc out (listing (str file "\n" text "\n"
                                                                               (glue-fingerprint g))))
                                                       => out)))))
                   "\n"))

    (when (restore-current? dir (stamp-of generated lock-file))
      (ret (match (precompile-packages dir packages deps)
             ((Err e) (Err e))
             ((Ok _) (Ok (Some dir))))))

    (unless (some? (which "dotnet"))
      (ret (Err (str "bjo: the build "
                     (if (empty? glues) "names NuGet packages" "has C# glue")
                     ", and "
                     (if (empty? glues) "restoring them" "compiling it")
                     " needs the .NET SDK's 'dotnet' command, which is not on PATH."))))

    (directory-create dir)
    (write-if-changed (path-combine dir "Root.csproj") csproj)
    (loop (:for file (list-append dep-files glue-files))
          (:for text (list-append dep-texts glue-texts))
          (:do (match (parent-directory file)
                 ((Some parent) (directory-create parent))
                 (None unit))
               (write-if-changed file text)))
    (def (Ok (Tuple code output)) (run-msbuild dir locked?) :propagate)
    (unless (= code 0)
      (ret (Err (failure-message output locked? lock-file))))

    ;; After the restore, which resolved the glue's packages.
    (loop (:for g glues)
          (:do (match (build-glue project g)
                 ((Err e) (ret (Err e)))
                 ((Ok _) unit))))

    ;; A rebuilt glue assembly may have changed what it offers, and the
    ;; modules that use it are compiled against what it had. The lists name
    ;; it and not its contents, so they are written again: the compiler
    ;; counts them as inputs of every module, which are then stale.
    (unless (empty? glues)
      (loop (:for list-file (list "compile.txt" "runtime.txt"))
            (:let path (path-combine dir list-file))
            (:when (file-exists? path))
            (:do (file-write-text path (file-read-text path)))))

    ;; After the restore, which may have written the lock file, and after the
    ;; glue is built: a build that failed is tried again next time.
    (file-write-text (path-combine dir "stamp") (stamp-of generated lock-file))
    (def (Ok _) (precompile-packages dir packages deps) :propagate)
    (Ok (Some dir))))
;; bjo/project.bjo — a project on disk, and where it is.
;;
;; A project is a directory with a `manifest.bjodat` in it. It is found by
;; walking upward from where the user stood, so that every command works
;; anywhere inside the tree rather than only at its top — the same rule git
;; uses for `.git`, and for the same reason.
;;
;; The layout is fixed:
;;
;;     manifest.bjodat   what the package is called, and what it needs
;;     bjo.lock          the commit each git dependency resolved to
;;     src/              the package's modules: src/lexer.bjo is (myapp lexer)
;;     tests/
;;     .bjo/             roots, clones, fetched packages. Never committed.
;;
;; `src/` is not a setting. A module's name is its path below the package
;; directory, so a second layout would be a second answer to "which file is
;; (myapp lexer)" — and the answer has to be the same for a package the author
;; wrote and one that was fetched, which nothing in the fetched copy can
;; configure after the fact.

(export
  ;; A project
  Project project-dir project-package
  manifest-name find-project-dir project-at read-package
  project-src project-dot-bjo project-tmp project-roots-file project-lock-file
  project-entry project-has-entry?

  ;; Package names
  reserved-name? parse-package-name name->string

  ;; The installed standard library
  lib-dir

  ;; The roots file
  Root root-name root-dir write-roots root-package-dir root-package
  project-frameworks-file write-frameworks

  ;; NuGet
  project-nuget-dir project-nuget-lock-file target-framework compiler-dir

  ;; Sources
  sources-under

  ;; Files, made once
  write-new parent-directory

  ;; Saying no
  fail

  ;; Finding a program
  which

  ;; The command
  cmd-init)

(import (std prelude))
;; The traits the manifest's own instances are written against. A module that
;; decodes a `Package` names them whether or not it says so, because the
;; generated reader calls them.
(import (text bjodat))
(import "manifest.bjo")

(import/extern
  (clr-directory-name (: System.IO.Path.GetDirectoryName (-> string string)))
  (clr-null-or-empty? (: System.String.IsNullOrEmpty (-> string bool)))
  ;; Where this executable sits, for the `lib` search below. `bjo.bjo` reads
  ;; the same thing for `BJO_ROOT`.
  (clr-base-dir (: System.AppContext.BaseDirectory #:get)))


;; ---------------------------------------------------------------------------
;; Saying no
;; ---------------------------------------------------------------------------

;; Prints a message and answers what a failed command answers, so that a guard
;; is `(ret (fail "..."))` — one expression, and the `when` it stands in has a
;; value to give rather than a dropped one.
(: (-> string int))
(defun (fail message)
  (println message)
  1)


;; ---------------------------------------------------------------------------
;; Paths
;; ---------------------------------------------------------------------------

;; The directory above this one, or `None` at the root of the filesystem, where
;; `GetDirectoryName` answers null rather than looping.
(: (-> string (Option string)))
(defun (parent-directory dir)
  (def up (clr-directory-name (path-absolute dir)))
  (if (clr-null-or-empty? up) None (Some up)))

;; A program on PATH, if there is one. Two ways to have no answer — no PATH at
;; all, and nothing on it by that name — and both of them propagate.
(: (-> string (Option string)))
(defun (which program)
  (def* ((Some path) (get-environment-variable "PATH") :propagate)
        ((Some dir) (find #(file-exists? (path-combine & program))
                          (string-split path ":")) :propagate))
  (Some (path-combine dir program)))

(: string)
(def manifest-name "manifest.bjodat")

;; The project a directory is in, if it is in one.
;;
;; Upward rather than downward: a project is the tree under the manifest, and a
;; command run three directories down is still run inside it.
(: (-> string (Option string)))
(defun (find-project-dir start)
  (let go ((dir (path-absolute start)))
    (if (file-exists? (path-combine dir manifest-name))
        (Some dir)
        (match (parent-directory dir)
          ((Some up) (go up))
          (None None)))))


;; ---------------------------------------------------------------------------
;; The manifest
;; ---------------------------------------------------------------------------

;; A project: the directory, and what its manifest says.
(type (: Project (Record (: dir string) (: package Package))))

(: (-> Project string))
(defun (project-dir p) (record-ref p dir))

(: (-> Project Package))
(defun (project-package p) (record-ref p package))

(: (-> Project string))
(defun (project-src p) (path-combine (project-dir p) "src"))

(: (-> Project string))
(defun (project-dot-bjo p) (path-combine (project-dir p) ".bjo"))

(: (-> Project string))
(defun (project-tmp p) (path-combine (project-dot-bjo p) "tmp"))

(: (-> Project string))
(defun (project-roots-file p) (path-combine (project-dot-bjo p) "roots"))

;; Which package declared which .NET shared framework. Beside the roots file,
;; because it answers the same kind of question about the same set of packages.
(: (-> Project string))
(defun (project-frameworks-file p) (path-combine (project-dot-bjo p) "frameworks"))

;; The generated project the root package's NuGet packages are restored with.
;; `root/` because a later dependency with packages gets a directory of its own.
(: (-> Project string))
(defun (project-nuget-dir p) (path-combine (project-dot-bjo p) "nuget" "root"))

;; The NuGet lock file. At the project root, beside the manifest, and committed.
(: (-> Project string))
(defun (project-nuget-lock-file p) (path-combine (project-dir p) "packages.lock.json"))

;; The target framework the compiler is built for and compiles programs for.
;; The compiler's own directory is named after it, and a restore has to resolve
;; packages for it.
(: string)
(def target-framework "net10.0")

;; The compiler's directory, where the Bjolang runtime's assemblies are: a
;; package's C# glue is compiled against them. The launcher exports the
;; compiler's path as `BJOLANG_COMPILER`; without it, the compiler is where
;; `build_compiler.sh` puts it.
(: (-> string))
(defun (compiler-dir)
  (match (get-environment-variable "BJOLANG_COMPILER")
    ((Some path) (option-ref-or (parent-directory (path-absolute path)) "."))
    (None
     (path-combine (match (get-environment-variable "BJO_ROOT")
                     ((Some dir) dir)
                     (None (path-absolute (path-combine clr-base-dir ".."))))
                   "bin" "compiler"))))

(: (-> Project string))
(defun (project-lock-file p) (path-combine (project-dir p) "bjo.lock"))

;; What the manifest says to build. Relative to the manifest, so that a project
;; means the same thing wherever it is checked out.
(: (-> Project string))
(defun (project-entry p)
  (path-combine (project-dir p) (record-ref (project-package p) entry)))

;; A package with no entry file is a library. Read from the disk rather than
;; from a flag in the manifest: `entry` has a default, and a library would
;; otherwise have to say twice that it has no program.
(: (-> Project bool))
(defun (project-has-entry? p) (file-exists? (project-entry p)))

;; The manifest at `file`, read into the types in `manifest.bjo`.
;;
;; The file name is in every message: a resolve reads the manifests of
;; dependencies too, and "expected a version" says nothing about which one.
(: (-> string (Result string Package)))
(defun (read-package file)
  (if (not (file-exists? file))
      (Err #"bjo: no manifest at ${file}.")
      (result-map-err (fun (why) #"bjo: ${file}: ${why}")
                      (bjodat-parse-Package (file-read-text file)))))

;; The project a directory holds, manifest read and nothing else checked yet.
(: (-> string (Result string Project)))
(defun (project-at dir)
  (def (Ok package) (read-package (path-combine dir manifest-name)) :propagate)
  (Ok (Project (dir (path-absolute dir)) (package package))))


;; ---------------------------------------------------------------------------
;; Package names
;; ---------------------------------------------------------------------------

;; `(myapp lexer)`, for a message.
(: (-> (List Symbol) string))
(defun (name->string name) (module-name->string name))

;; The installed `lib`, found the way the compiler finds it: an override first,
;; then the tree the binary is in, then the working directory, and a candidate
;; only counts if `std` is under it. The two have to agree, because a name this
;; one misses is a name the compiler refuses to give a directory to.
(: (-> (List string)))
(defun (lib-candidates)
  (def from-env
    (list-append
      (match (get-environment-variable "BJOLANG_LIB")
        ((Some dir) (Cons (path-absolute dir) Nil))
        (None Nil))
      (match (get-environment-variable "BJOLANG_HOME")
        ((Some dir) (Cons (path-absolute (path-combine dir "lib")) Nil))
        (None Nil))))
  (list-append from-env
               (list-append (ancestor-libs clr-base-dir)
                            (ancestor-libs (current-directory)))))

;; `<dir>/lib` for `dir` and every directory above it, deepest first.
(: (-> string (List string)))
(defun (ancestor-libs start)
  (let go ((dir (path-absolute start)))
    (Cons (path-combine dir "lib")
          (match (parent-directory dir)
            ((Some up) (go up))
            (None Nil)))))

(: (-> (Option string)))
(defun (lib-dir)
  (find #(directory-exists? (path-combine & "std")) (lib-candidates)))

;; A name whose first segment is a directory of `lib`.
;;
;; Refused rather than shadowed: the compiler gives the standard library its
;; roots whether or not a roots file was written, and refuses a roots file that
;; names one of them. A package called `(std)` would therefore build here and
;; fail at the compiler, one step later than the manifest that asked for it.
(: (-> (List Symbol) bool))
(defun (reserved-name? name)
  (match name
    (Nil #f)
    ((Cons first _)
     (match (lib-dir)
       (None #f)
       ((Some lib)
        (def head (symbol->string first))
        (any? #(match (path-filename &)
                 ((Some dir-name) (= dir-name head))
                 (None #f))
              (directory-subdirectories lib)))))))

;; The characters a name segment may be written with.
;;
;; A package name is read as symbols by the manifest reader and written as a
;; module path by an import, so a segment that needs quoting anywhere is a
;; segment that is spelled differently in two places. `init` is the only place
;; a name is made out of a string — a directory called `my.app` has no name
;; here, and has to be given one.
(: (-> char bool))
(defun (name-char? c)
  (or (and (char>=? c #\a) (char<=? c #\z))
      (and (char>=? c #\A) (char<=? c #\Z))
      (and (char>=? c #\0) (char<=? c #\9))
      (char=? c #\-)
      (char=? c #\_)))

(: (-> string bool))
(defun (valid-segment? s)
  (and (not (= s ""))
       (let ((first (string-cursor-ref s (string-cursor-start s))))
         (not (and (char>=? first #\0) (char<=? first #\9))))
       (all? name-char? (string->list s))))

;; The name a command line wrote: `myapp`, or `(my app)` for several segments.
(: (-> string (Result string (List Symbol))))
(defun (parse-package-name written)
  (def trimmed (string-trim written))
  (def inner (if (and (string-starts-with? trimmed "(") (string-ends-with? trimmed ")"))
                 (string-trim (substring trimmed
                                         (string-cursor-next trimmed (string-cursor-start trimmed))
                                         (string-cursor-prev trimmed (string-cursor-end trimmed))))
                 trimmed))
  (def segments (loop (:for part (string-split inner " "))
                      (:acc out (listing part) #:when (not (= part "")))
                      => out))
  (cond
    ((empty? segments) (Err #"bjo init: '${written}' is not a package name."))
    ((not (all? valid-segment? segments))
     (Err (str #"bjo init: '${written}' is not a package name. "
               "A name is one or more words of letters, digits, '-' and '_', "
               "written (my app) when there are several.")))
    (else (Ok (list-map string->symbol segments)))))


;; ---------------------------------------------------------------------------
;; The roots file
;; ---------------------------------------------------------------------------

;; One package, and the directory its modules are in.
(type (: Root (Record (: name (List Symbol)) (: dir string))))

(: (-> Root (List Symbol)))
(defun (root-name r) (record-ref r name))

(: (-> Root string))
(defun (root-dir r) (record-ref r dir))

;; A directory as a Bjolang string literal.
(: (-> string string))
(defun (quoted s)
  (str "\"" (string-replace (string-replace s "\\" "\\\\") "\"" "\\\"") "\""))

;; The roots file, sorted, so that a diff of two builds shows what changed
;; rather than what was reordered. The line begins with the name, so sorting
;; the lines sorts by name.
(: (-> (List Root) string))
(defun (roots-text roots)
  (def lines (list-sort (list-map #(str "  (root " (name->string (root-name &))
                                        " " (quoted (root-dir &)) ")")
                                  roots)))
  (str "(roots\n" (string-join (list->vec lines) "\n") ")\n"))

;; Writes the roots file, and answers whether it had to.
;;
;; **Only when the contents change.** The compiler records the roots file as an
;; input of every build made against it, so rewriting the same bytes with a new
;; timestamp makes every artefact in the project stale.
(: write-roots #:sync (-> string (List Root) bool))
(defun (write-roots path roots)
  (def text (roots-text roots))
  (def current (if (file-exists? path) (file-read-text path) ""))
  (cond
    ((= current text) #f)
    (else
     (match (parent-directory path)
       ((Some dir) (directory-create dir))
       (None unit))
     (file-write-text path text)
     #t)))


;; ---------------------------------------------------------------------------
;; The frameworks file
;; ---------------------------------------------------------------------------
;;
;; Which package declared which .NET shared framework, for the compiler's
;; `--frameworks`. One line per pair, two TAB-separated fields: the package's
;; own directory — the one its `manifest.bjodat` is in, not its `src` — and the
;; framework's name.
;;
;; A package's directory rather than its module root, because that is what the
;; compiler compares a source file against, and because a dependency fetched
;; into `.bjo/pkg` sits *under* the root project: the innermost directory
;; containing the file is the package the file belongs to, and it has to be the
;; package rather than a directory inside it.
;;
;; The declarations are read from each root's own manifest rather than carried
;; out of the resolver: a root's directory is `<package>/src`, its manifest is
;; beside it, and reading a handful of small files is cheaper than a second
;; answer to "what did this package say" that could disagree with the first.

;; The package directory a root's modules live under.
(: (-> Root (Option string)))
(defun (root-package-dir r)
  (parent-directory (root-dir r)))

;; The manifest of the package a root belongs to, or `None` when it has none.
;;
;; A manifest that does not read is also `None`. It is not this function's to
;; report: whatever is about to build against it says so, with the reason.
(: (-> Root (Option Package)))
(defun (root-package r)
  (def* ((Some dir) (root-package-dir r) :propagate)
        (manifest-file (path-combine dir manifest-name)))
  (if (not (file-exists? manifest-file))
      None
      (result->option (bjodat-parse-Package (file-read-text manifest-file)))))

;; What one root's package declares, or nothing when it has no manifest to say
;; it in — which is what a root without one means.
(: (-> Root (List string)))
(defun (root-frameworks r)
  (match (root-package r)
    (None Nil)
    ((Some package) (record-ref package frameworks))))

;; The file's text, sorted for the same reason the roots file is: a diff shows
;; what changed rather than what was reordered.
(: (-> (List Root) string))
(defun (frameworks-text roots)
  (def lines
    (loop (:for r roots)
          (:acc out (folding Nil
                             (match (root-package-dir r)
                               (None out)
                               ((Some dir)
                                (loop (:for name (root-frameworks r))
                                      (:acc inner (folding out (Cons (str dir "\t" name) inner)))
                                      => inner)))))
          => out))
  (if (empty? lines)
      ""
      (str (string-join (list->vec (list-sort lines)) "\n") "\n")))

;; Writes the frameworks file, and answers whether the build has one at all.
;;
;; No package declaring anything means no file and no flag, so a project that
;; uses no framework is compiled by exactly the command line it was before.
;; Written only when the contents change, for the reason `write-roots` gives:
;; the compiler records it as an input of every module built against it.
(: write-frameworks #:sync (-> string (List Root) bool))
(defun (write-frameworks path roots)
  (def text (frameworks-text roots))
  (def current (if (file-exists? path) (file-read-text path) ""))
  (cond
    ((= text "")
     (when (file-exists? path) (file-delete path))
     #f)
    ((= current text) #t)
    (else
     (match (parent-directory path)
       ((Some dir) (directory-create dir))
       (None unit))
     (file-write-text path text)
     #t)))


;; ---------------------------------------------------------------------------
;; Sources
;; ---------------------------------------------------------------------------

;; Every `.bjo` under a directory, at any depth, sorted.
;;
;; A library has no entry to start the compiler from, so this is the list the
;; batch is given. Sorted only so that a failure is reported in the same order
;; twice; the compiler builds imports on demand and does not need the order.
(: (-> string (List string)))
(defun (sources-under dir)
  (list-sort (collect-sources dir)))

(: (-> string (List string)))
(defun (collect-sources dir)
  (if (not (directory-exists? dir))
      Nil
      (let ((here (loop (:for file (directory-files dir))
                        (:acc out (listing file) #:when (string-ends-with? file ".bjo"))
                        => out)))
        (list-append here
                     (loop (:for sub (directory-subdirectories dir))
                           (:acc out (folding Nil (list-append out (collect-sources sub))))
                           => out)))))


;; ---------------------------------------------------------------------------
;; init
;; ---------------------------------------------------------------------------

;; Writes a file that was not there, and says so. A file that is there is left
;; alone: `init` in a directory someone has already written in is a command
;; that adds what is missing, not one that replaces what is not.
(: write-new #:sync (-> string string bool))
(defun (write-new path contents)
  (cond
    ((file-exists? path) #f)
    (else
     (match (parent-directory path)
       ((Some dir) (directory-create dir))
       (None unit))
     (file-write-text path contents)
     #t)))

(: (-> (List Symbol) string))
(defun (manifest-template name)
  (str "(package\n"
       "  (name " (name->string name) ")\n"
       "  (version \"0.1.0\")\n"
       "\n"
       "  ;; What this package needs. One (package ...) clause each:\n"
       "  ;;\n"
       "  ;; (depends\n"
       "  ;;   (package (name (bjorsec))\n"
       "  ;;            (version (version-at-least \"0.1\"))\n"
       "  ;;            (source (git (url \"https://github.com/bjoli/bjorsec\")))))\n"
       "  ;;\n"
       "  ;; A package on disk, instead of one to fetch:\n"
       "  ;;\n"
       "  ;; (depends\n"
       "  ;;   (package (name (bjorsec)) (source (path (dir \"../bjorsec\")))))\n"
       "  )\n"))

;; The name without its parentheses, for prose. `name->string` is how an
;; import spells a name, which is not how a greeting does.
(: (-> (List Symbol) string))
(defun (plain-name name)
  (string-join (list->vec (list-map symbol->string name)) " "))

(: (-> (List Symbol) string))
(defun (main-template name)
  (str ";; " (name->string name) " — the program.\n"
       "\n"
       "(import (std prelude))\n"
       "\n"
       "(defun (main)\n"
       "  (println \"hello from " (plain-name name) "\")\n"
       "  0)\n"))

(: (-> (List Symbol) string))
(defun (core-template name)
  (str ";; " (name->string name) " — imported as " (name->string (list-append name (Cons (string->symbol "core") Nil))) ".\n"
       "\n"
       "(import (std prelude))\n"
       "(export greet)\n"
       "\n"
       "(: greet (-> string string))\n"
       "(defun (greet who) (str \"hello, \" who))\n"))

;; `.bjo` holds everything fetched and everything generated, `publish` is where
;; `bjo publish` writes by default, and the rest is what the compiler writes
;; beside a source: one artefact, its debug symbols, the two files the runtime
;; needs to start an executable, and the build record.
(: string)
(def gitignore-template
  (str ".bjo/\n"
       "publish/\n"
       "*.dll\n"
       "*.exe\n"
       "*.pdb\n"
       "*.bjobuild\n"
       "*.runtimeconfig.json\n"
       "*.deps.json\n"))

;; Makes a project in the current directory.
;;
;; The name defaults to the directory's, because that is what the user already
;; chose when they made it. It is refused rather than mangled when it is not a
;; name: `bjo init my-app` is one word to type, and a guess at what `my.app`
;; was meant to be is a package name nobody wrote.
(: (-> (List string) bool int))
(defun (cmd-init arguments lib?)
  (with-return ret
    (def here (path-absolute (current-directory)))

    (when (file-exists? (path-combine here manifest-name))
      (ret (fail #"bjo init: there is already a ${manifest-name} here.")))

    ;; `--lib` is read here as well as among the leading flags: flag parsing
    ;; stops at the first thing that is not one, so `bjo init mylib --lib`
    ;; would otherwise be a name and a second argument.
    (def* ((None) (find #(and (string-starts-with? & "-") (not (= & "--lib"))) arguments)
            :leave ((Some flag) (fail #"bjo init: unknown flag '${flag}'.")))
          (library? (or lib? (any? #(= & "--lib") arguments)))
          (names (loop (:for a arguments)
                       (:acc out (listing a) #:when (not (string-starts-with? a "-")))
                       => out)))

    (def written
      (match names
        (Nil (match (path-filename here)
               ((Some dir-name) dir-name)
               (None "")))
        ((Cons only Nil) only)
        (_ "")))

    (when (= written "")
      (ret (fail "bjo init: takes one name, or none to use the directory's.")))

    (def (Ok name) (parse-package-name written)
      :leave ((Err why) (fail why)))

    (when (reserved-name? name)
      (ret (fail (str #"bjo init: ${(name->string name)} is part of the standard library, "
                      "so a package cannot be called that. Pick another name."))))

    (ignore (write-new (path-combine here manifest-name) (manifest-template name)))
    (ignore (write-new (path-combine here ".gitignore") gitignore-template))
    (directory-create (path-combine here "tests"))

    (if library?
        (ignore (write-new (path-combine here "src" "core.bjo") (core-template name)))
        (ignore (write-new (path-combine here "src" "main.bjo") (main-template name))))

    (println #"Created ${(name->string name)} in ${here}")
    0))
;; bjo/publish.bjo — a program and everything it loads, in one directory.
;;
;; A built program loads its modules, the runtime and its NuGet packages from
;; wherever they were when it was built, by absolute path. A published one
;; carries copies of them and loads only those: the compiler's resolver reads
;; `<program>.bjolinks` beside the program, when there is one, and loads
;; nothing that the table does not name.
;;
;; The layout keeps each file's place relative to its neighbours, because some
;; code finds files from its own `.dll` — a package's resources, or the
;; standard library's `lib/` from `(janitor modules)`:
;;
;;     <out>/<program>.exe            and its runtimeconfig, deps and pdb
;;     <out>/<program>                its Native AOT executable, when it has one
;;     <out>/<program>.bjolinks       assembly and native library -> path
;;     <out>/lib/...                  standard library modules, as under lib/
;;     <out>/runtime/...              the Bjolang runtime assemblies
;;     <out>/app/...                  the project's own modules and resources
;;     <out>/packages/<name>/...      each dependency, as under its directory
;;     <out>/external/...             NuGet packages, as below the NuGet
;;                                    cache; anything else below the deepest
;;                                    directory it shares with the rest

(export PublishedPackage Publication publish published-marker)

(import (std prelude))

(import/extern
  (clr-relative-path (: System.IO.Path.GetRelativePath (-> string string string)))
  (clr-path-rooted?  (: System.IO.Path.IsPathRooted (-> string bool)))
  (clr-directory-of  (: System.IO.Path.GetDirectoryName (-> string string)))
  (clr-file-name     (: System.IO.Path.GetFileName (-> string string)))
  (clr-file-stem     (: System.IO.Path.GetFileNameWithoutExtension (-> string string)))
  (clr-change-ext    (: System.IO.Path.ChangeExtension (-> string string string)))
  (clr-file-copy     (: System.IO.File.Copy (-> string string bool void)))
  (error-message     (: System.Exception.Message (-> Exception string) #:get)))

;; A package the program was built from: its name, the directory its manifest
;; is in, the resources that manifest declares, and whether it is the project
;; being published, whose files go under `app/`.
(type (: PublishedPackage (Record (: name (List Symbol))
                                  (: dir string)
                                  (: resources (List string))
                                  (: own bool))))

;; What to publish. `links` and `natives` are the build record's `link` and
;; `native` lines: a name or key, and the file's path. `native-program` is the
;; program's Native AOT executable, or "" when it has none.
;;
;; `whole-stdlib` copies all of the standard library, sources and documentation
;; included, rather than only the modules the program links. A program that
;; reads Bjolang's own modules at run time, such as one that shows their
;; documentation, needs that.
(type (: Publication (Record (: program string)
                             (: native-program string)
                             (: links (List (Tuple string string)))
                             (: natives (List (Tuple string string)))
                             (: packages (List PublishedPackage))
                             (: lib (Option string))
                             (: whole-stdlib bool)
                             (: runtime string)
                             (: out string))))

;; Written into every output directory. An existing directory is only replaced
;; when it has this file, so a mistyped `-o` cannot delete something else.
(: string)
(def published-marker ".bjopublish")

;; ---------------------------------------------------------------------------
;; Where a file goes
;; ---------------------------------------------------------------------------

;; `path` relative to `dir`, or `None` when it is not below it.
(: (-> string string (Option string)))
(defun (below dir path)
  (def rel (clr-relative-path (path-absolute dir) (path-absolute path)))
  (if (or (= rel ".") (string-starts-with? rel "..") (clr-path-rooted? rel))
      None
      (Some rel)))

(: (-> PublishedPackage string))
(defun (package-prefix p)
  (if (record-ref p own)
      "app"
      (path-combine "packages"
                    (string-join (list->vec (list-map symbol->string (record-ref p name))) "."))))

;; The package whose directory holds `path`, and the path relative to it. The
;; deepest directory wins, because fetched dependencies live in `.bjo/pkg`
;; inside the project's own directory.
(: (-> (List PublishedPackage) string (Option (Tuple PublishedPackage string))))
(defun (package-of packages path)
  (loop (:for p packages)
        (:when-let (Some rel) (below (record-ref p dir) path))
        (:acc best (folding None
                            (match best
                              ((Some (Tuple q _)) #:when (> (string-length (record-ref q dir))
                                                            (string-length (record-ref p dir)))
                               best)
                              (_ (Some (Tuple p rel))))))
        => best))

;; Where `path` goes in the output, when it is under the standard library,
;; the runtime or a package. `None` for anything else.
(: (-> Publication string (Option string)))
(defun (known-destination p path)
  (def in-lib (match (record-ref p lib)
                ((Some lib) (below lib path))
                (None None)))
  (match in-lib
    ((Some rel) (Some (path-combine "lib" rel)))
    (None
     (match (below (record-ref p runtime) path)
       ((Some rel) (Some (path-combine "runtime" rel)))
       (None
        (match (package-of (record-ref p packages) path)
          ((Some (Tuple package rel)) (Some (path-combine (package-prefix package) rel)))
          (None None)))))))

;; The NuGet cache, where `dotnet` puts packages.
(: (-> string))
(defun (nuget-packages-dir)
  (match (get-environment-variable "NUGET_PACKAGES")
    ((Some dir) #:when (not (= dir "")) dir)
    (_ (path-combine (option-ref-or (get-environment-variable "HOME") "/") ".nuget" "packages"))))

;; The path of a NuGet package's file below its store, or `None` for a file
;; that is in neither store:
;; - the NuGet cache
;; - the compiler's ReadyToRun cache, `.../bjolang/r2r/<rid>/<crossgen2
;;   version>/`, which has the same layout. See `Build.precompilePackages`.
;; A package's assemblies are in the second and its native libraries in the
;; first, so both are laid out as the NuGet cache is.
(: (-> string (Option string)))
(defun (package-store-path path)
  (match (below (nuget-packages-dir) path)
    ((Some rel) (Some rel))
    (None
     (let go ((parts (vec->list (string-split (path-absolute path) "/"))))
       (match parts
         ((Cons "bjolang" (Cons "r2r" (Cons _ (Cons _ (Cons first rest)))))
          (Some (string-join (list->vec (Cons first rest)) "/")))
         ((Cons _ more) (go more))
         (Nil None))))))

;; The longest list both start with.
(: (-> (List string) (List string) (List string)))
(defun (common-prefix a b)
  (match (Tuple a b)
    ((Tuple (Cons x xs) (Cons y ys)) #:when (= x y) (Cons x (common-prefix xs ys)))
    (_ Nil)))

;; The deepest directory that holds every one of `paths`.
(: (-> (List string) string))
(defun (common-directory paths)
  (match (list-map (fun (path) (vec->list (string-split (clr-directory-of (path-absolute path)) "/")))
                   paths)
    (Nil "/")
    ((Cons first rest)
     (def dir (string-join (list->vec (list-foldl common-prefix first rest)) "/"))
     (if (= dir "") "/" dir))))

;; ---------------------------------------------------------------------------
;; Copying
;; ---------------------------------------------------------------------------

;; Copies `from` to `rel` under `out`, making the directories on the way.
(: copy-to! #:sync (-> string string string void))
(defun (copy-to! out rel from)
  (def to (path-combine out rel))
  (directory-create (clr-directory-of to))
  (file-copy from to #:mode Truncate))

;; Copies a native executable. Through .NET's `File.Copy`, which keeps the file's
;; mode on Unix: a copy made by `file-copy` could not be run.
(: copy-executable! #:sync (-> string string string void))
(defun (copy-executable! out rel from)
  (def to (path-combine out rel))
  (directory-create (clr-directory-of to))
  (clr-file-copy from to #t))

;; Copies an assembly, and its `.pdb` when there is one, so that a stack trace
;; from the published program still names the source.
(: copy-assembly! #:sync (-> string string string void))
(defun (copy-assembly! out rel from)
  (copy-to! out rel from)
  (def pdb (clr-change-ext from ".pdb"))
  (when (file-exists? pdb)
    (copy-to! out (clr-change-ext rel ".pdb") pdb)))

;; Copies a directory and everything in it, leaving out the files `skip?`
;; answers true for.
(: copy-tree! #:sync (-> string string (-> string bool) void))
(defun (copy-tree! from to skip?)
  (directory-create to)
  (loop (:for f (directory-files from))
        (:when (not (skip? f)))
        (:do (file-copy f (path-combine to (clr-file-name f)) #:mode Truncate)))
  (loop (:for d (directory-subdirectories from))
        (:do (copy-tree! d (path-combine to (clr-file-name d)) skip?))))

;; The standard library is copied with its sources and documentation, but not
;; its build records, which name this machine's paths and which nothing reads
;; in a published program.
(: (-> string bool))
(defun (build-record? f) (string-ends-with? f ".bjobuild"))

(: (-> string bool))
(defun (nothing-skipped f) #f)

;; Copies a resource, a file or a directory, to `rel` under `out`.
(: copy-resource! #:sync (-> string string string void))
(defun (copy-resource! out rel from)
  (if (directory-exists? from)
      (copy-tree! from (path-combine out rel) nothing-skipped)
      (copy-to! out rel from)))

;; How many files are under `dir`, at any depth.
(: count-files #:sync (-> string int))
(defun (count-files dir)
  (+ (length (directory-files dir))
     (loop (:for d (directory-subdirectories dir))
           (:acc n (summing (count-files d)))
           => n)))

;; Empties the output directory, or makes it. A directory that is there and
;; holds files is replaced only when an earlier publish made it. Answers why
;; it refused, or `None` when the directory is ready.
(: prepare-output! #:sync (-> string (Option string)))
(defun (prepare-output! out)
  (cond
    ((not (directory-exists? out))
     (directory-create out)
     None)
    ((file-exists? (path-combine out published-marker))
     (directory-delete-tree out)
     (directory-create out)
     None)
    ((and (empty? (directory-files out)) (empty? (directory-subdirectories out)))
     None)
    (else
     (Some (str "bjo publish: " out " is not empty, and bjo publish did not make it. "
               "Give an empty or new directory with -o.")))))

;; Why a declared resource cannot be published, or nothing when it can.
(: (-> PublishedPackage string (Option string)))
(defun (resource-problem package resource)
  (def shown (str "the resource \"" resource "\" of " (record-ref package dir)))
  (def from (path-combine (record-ref package dir) resource))
  (cond
    ((or (clr-path-rooted? resource)
         (any? #(= & "..") (string-split (string-replace resource "\\" "/") "/")))
     (Some (str "bjo publish: " shown " is not a path inside the package. "
                "A resource is written relative to the package's directory.")))
    ((not (or (file-exists? from) (directory-exists? from)))
     (Some (str "bjo publish: " shown " is not there.")))
    (else None)))

;; Every resource problem, across all the packages.
(: (-> (List PublishedPackage) (List string)))
(defun (resource-problems packages)
  (loop (:for package packages)
        (:subloop)
        (:for resource (record-ref package resources))
        (:when-let (Some why) (resource-problem package resource))
        (:acc out (listing why))
        => out))

;; ---------------------------------------------------------------------------
;; Publishing
;; ---------------------------------------------------------------------------

;; Lays out the program in `out`, and answers how many files the output holds.
;;
;; Every resource is checked before anything is written, so that a manifest
;; naming one that is not there leaves the output as it was.
(: publish #:sync (-> Publication (Result string int)))
(defun (publish p)
  (with-return ret
    (when (empty? (record-ref p links))
      (ret (Err (str "bjo publish: the build record of " (record-ref p program)
                     " names nothing the program loads. Rebuild it with -c and publish again."))))

    (match (resource-problems (record-ref p packages))
      ((Cons why _) (ret (Err why)))
      (Nil unit))

    (def out (path-absolute (record-ref p out)))
    (match (prepare-output! out)
      ((Some why) (ret (Err why)))
      (None unit))

    (def files (list-append (list-map (fun (link) (match link ((Tuple name path) (Tuple "link" name path))))
                                      (record-ref p links))
                            (list-map (fun (native) (match native ((Tuple key path) (Tuple "native" key path))))
                                      (record-ref p natives))))
    (def external-root
      (common-directory (loop (:for (Tuple _ _ path) files)
                              (:acc paths (listing path) #:when (and (not (some? (known-destination p path)))
                                                                     (not (some? (package-store-path path)))))
                              => paths)))
    (defun (destination path)
      (match (known-destination p path)
        ((Some rel) rel)
        (None (path-combine "external"
                            (match (package-store-path path)
                              ((Some rel) rel)
                              (None (match (below external-root path)
                                      ((Some rel) rel)
                                      (None (clr-file-name path)))))))))

    (def program (record-ref p program))

    (def copied
      (try
        (let ()
          (loop (:for companion (list program
                                      (clr-change-ext program ".runtimeconfig.json")
                                      (clr-change-ext program ".deps.json")
                                      (clr-change-ext program ".pdb")))
                (:when (file-exists? companion))
                (:do (copy-to! out (clr-file-name companion) companion)))
          (def native (record-ref p native-program))
          (when (and (not (= native "")) (file-exists? native))
            (copy-executable! out (clr-file-name native) native))
          (match (record-ref p lib)
            ((Some lib) #:when (record-ref p whole-stdlib)
             (copy-tree! lib (path-combine out "lib") build-record?))
            (_ unit))
          (loop (:for (Tuple _ _ path) files)
                (:do (copy-assembly! out (destination path) path)))
          (loop (:for package (record-ref p packages))
                (:subloop)
                (:for resource (record-ref package resources))
                (:do (copy-resource! out
                                     (path-combine (package-prefix package) resource)
                                     (path-combine (record-ref package dir) resource))))
          ;; Relative to the program, with `/`, which .NET accepts on every
          ;; platform, so that the table means the same wherever the directory
          ;; is copied to.
          (def table
            (loop (:for (Tuple kind name path) files)
                  (:acc lines (listing (str kind "\t" name "\t" (string-replace (destination path) "\\" "/"))))
                  => lines))
          (file-write-text (path-combine out (str (clr-file-stem program) ".bjolinks"))
                           (str (string-join (list->vec table) "\n") "\n"))
          (file-write-text (path-combine out published-marker)
                           "Made by bjo publish. Publishing again replaces this directory.\n")
          (count-files out))
        #:catch (System.Exception)))

    (result-map-err (fun (e) (str "bjo publish: " (error-message e))) copied)))
;; bjo/resolve.bjo — what a project builds against.
;;
;; One entry point, `prepare`: it turns a project's manifest into the list of
;; packages the compiler is given, fetches whatever that needs, and writes
;; `.bjo/roots`. Every command that compiles anything inside a project goes
;; through it, so that `bjo run` and `bjo check` see the same packages.
;;
;; The walk
;; --------
;; A manifest names dependencies; each of those has a manifest of its own that
;; names more. The walk reads them breadth first, and a package is read once.
;;
;; A dependency may be written with no `source`, which means "somebody else in
;; this graph says where this comes from". So the walk cannot simply follow
;; edges: it keeps the dependencies whose source is still unknown in a waiting
;; list and retries them whenever it has read another manifest. A whole pass
;; that satisfies none of them is the end — and then a name with no source
;; anywhere is an error naming it.
;;
;; Two rules decide a source when there are several:
;;
;;   - the root manifest wins. That is how a user overrides a source a
;;     dependency chose: write the name in your own manifest with the source
;;     you want, and every manifest that asked for that name gets yours.
;;   - otherwise every source given for one name must be the same, compared as
;;     URLs with a trailing `/` and `.git` removed. Two different ones is an
;;     error listing both and which manifest gave each, because picking either
;;     silently would build code the other manifest never asked for.

(export
  Prepared prepared-roots-file prepared-frameworks-file prepared-nuget-dir
  prepared-runtime
  prepared-aot prepared-aot-cpu
  prepared-roots prepared-dependencies
  prepare)

(import (std prelude))
(import (text bjodat))
(import "manifest.bjo")
(import "project.bjo")
(import "lock.bjo")
(import "git.bjo")
(import "nuget.bjo")

;; What a project turned out to need: the roots file to hand the compiler, and
;; the packages it names.
;; `frameworks-file` is `None` when no package in the graph declared a .NET
;; shared framework, which is the ordinary case: then no file is written and no
;; flag is passed, and the build is exactly the build it was before.
;; `nuget-dir` is `None` the same way, when the manifest names no NuGet
;; packages.
;; `runtime` is the root manifest's runtime settings, as `name=value`, and
;; `aot` whether it asks for a Native AOT build, and `aot-cpu` the CPU that
;; build is for, or "" for the baseline.
(type (: Prepared (Record (: roots-file string)
                          (: frameworks-file (Option string))
                          (: nuget-dir (Option string))
                          (: runtime (List string))
                          (: aot bool)
                          (: aot-cpu string)
                          (: roots (List Root)))))

;; A manifest that has been read and whose dependencies have not been walked.
;;
;; `path-ok?` travels with it: a `(path ...)` source points at a directory on
;; the author's disk, which is a fact about a checkout rather than about a
;; package, so it may only be written by a manifest that was itself reached
;; through path sources.
(type (: Task (Record (: package Package)
                      (: dir string)
                      (: from string)
                      (: path-ok? bool)
                      (: dev? bool))))

;; A dependency that has been seen and cannot be placed yet, because nothing
;; has said where it comes from.
(type (: Want (Record (: dep Dep)
                      (: want-from string)
                      (: want-dir string)
                      (: want-path-ok? bool))))

;; A source, and the manifest that gave it.
(type (: Given (Record (: given Source) (: given-from string) (: given-root? bool))))

;; Somebody's constraint on a package's version, for the check after the
;; versions have been chosen.
(type (: Need (Record (: need-name (List Symbol))
                      (: need-req (Option VersionReq))
                      (: need-from string))))

;; A git package that has been chosen: everything the lock and the roots file
;; need to say about it.
(type (: Chosen (Record (: chosen-name (List Symbol))
                        (: chosen-version Version)
                        (: chosen-source Source)
                        (: chosen-clone string))))

;; The walk's state. Boxes rather than a threaded record: every step reads what
;; every earlier step decided, and a name is placed once however many manifests
;; asked for it.
;;
;; A path package is placed as it is found — there is one directory and nothing
;; to choose. A git package is not: every requirement on it makes a *node*, a
;; version that some manifest asked for, and which of them is used is only
;; known when the walk is over. See `chosen` below.
(type (: Walk (Record (: sources (Box (Map string Given)))
                      (: placed (Box (Map string Root)))
                      (: needs (Box (List Need)))
                      (: nodes (Box (Map string (List Version))))
                      (: names (Box (Map string (List Symbol))))
                      (: clones (Box (Map string string)))
                      (: origins (Box (Map string Source)))
                      (: walked (Box (Map string bool)))
                      (: fetched (Box (Map string bool)))
                      (: own (List Symbol))
                      (: own-from string)
                      (: tool Tool)
                      (: dot-bjo string)
                      (: was-locked (List Locked)))))

(: (-> Prepared string))
(defun (prepared-roots-file p) (record-ref p roots-file))

(: (-> Prepared (Option string)))
(defun (prepared-frameworks-file p) (record-ref p frameworks-file))

(: (-> Prepared (Option string)))
(defun (prepared-nuget-dir p) (record-ref p nuget-dir))

(: (-> Prepared (List string)))
(defun (prepared-runtime p) (record-ref p runtime))

(: (-> Prepared bool))
(defun (prepared-aot p) (record-ref p aot))

(: (-> Prepared string))
(defun (prepared-aot-cpu p) (record-ref p aot-cpu))

(: (-> Prepared (List Root)))
(defun (prepared-roots p) (record-ref p roots))

;; How many packages other than the project itself.
(: (-> Prepared int))
(defun (prepared-dependencies p) (- (length (prepared-roots p)) 1))


;; ---------------------------------------------------------------------------
;; Names and sources, as text
;; ---------------------------------------------------------------------------

;; A package name is the key everywhere here: two names are the same name when
;; they are written the same, and `(a b)` is a string a map can hold.
(: (-> (List Symbol) string))
(defun (key-of name) (name->string name))

(: (-> Source string))
(defun (source-shown s)
  (match s
    ((FromGit g) (str "(git (url \"" (record-ref g url) "\"))"))
    ((FromPath p) (str "(path (dir \"" (record-ref p dir) "\"))"))
    ((FromRegistry r) (str "(registry (host \"" (record-ref r host) "\"))"))))

(: (-> string string string))
(defun (without-suffix s suffix)
  (if (not (string-ends-with? s suffix))
      s
      (let go ((c (string-cursor-end s)) (left (string-count suffix)))
        (if (= left 0)
            (substring s (string-cursor-start s) c)
            (go (string-cursor-prev s c) (- left 1))))))

;; What two sources are compared as. A repository is the same repository under
;; `…/x`, `…/x/` and `…/x.git`, and a manifest that spells it either way means
;; the same place.
(: (-> Source string))
(defun (source-key s)
  (match s
    ((FromGit g) (str "git " (without-suffix (without-suffix (record-ref g url) "/") ".git")))
    ((FromPath p) (str "path " (record-ref p dir)))
    ((FromRegistry r) (str "registry " (record-ref r host)))))


;; ---------------------------------------------------------------------------
;; Reading a dependency's manifest
;; ---------------------------------------------------------------------------

;; The manifest of a package that was found on disk, checked against the name
;; it was asked for.
;;
;; A directory that answers to another name is refused rather than used: the
;; roots file would map the name that was asked for onto these modules, and
;; every module in it would then compile under a name its own author never
;; wrote.
(: (-> (List Symbol) string string (Result string Package)))
(defun (package-in-directory wanted dir asked-by)
  (def manifest-file (path-combine dir manifest-name))
  (cond
    ((not (directory-exists? dir))
     (Err (str "bjo: " asked-by " wants " (name->string wanted) " from " dir
               ", which is not a directory.")))
    ((not (file-exists? manifest-file))
     (Err (str "bjo: " asked-by " wants " (name->string wanted) " from " dir
               ", which has no " manifest-name ".")))
    (else
     (def (Ok package) (read-package manifest-file) :propagate)
     (if (not (= (key-of (record-ref package name)) (key-of wanted)))
         (Err (str "bjo: " asked-by " wants " (name->string wanted) ", but "
                   manifest-file " says the package there is "
                   (name->string (record-ref package name)) "."))
         (Ok package)))))


;; ---------------------------------------------------------------------------
;; The walk
;; ---------------------------------------------------------------------------

;; Records the source a manifest gave for a name.
;;
;; The root's wins and is never overwritten; anyone else's has to agree with
;; what is already there.
(: (-> Walk (List Symbol) Source string bool (Result string Unit)))
(defun (give-source! walk name source from root?)
  (def k (key-of name))
  (def given (Given (given source) (given-from from) (given-root? root?)))
  (match (map-try-ref (box-ref (record-ref walk sources)) k)
    (None
     (box-set! (record-ref walk sources)
               (map-set (box-ref (record-ref walk sources)) k given))
     (Ok unit))
    ((Some already)
     (cond
       ;; The root said where this comes from, so nothing else is consulted.
       ((record-ref already given-root?) (Ok unit))
       (root?
        (box-set! (record-ref walk sources)
                  (map-set (box-ref (record-ref walk sources)) k given))
        (Ok unit))
       ((= (source-key (record-ref already given)) (source-key source)) (Ok unit))
       (else
        (Err (str "bjo: " (name->string name) " is given two different sources.\n"
                  "  " (source-shown (record-ref already given))
                  " in " (record-ref already given-from) "\n"
                  "  " (source-shown source) " in " from "\n"
                  "  Name it in " (record-ref walk own-from)
                  " with the source you want: the root manifest's wins everywhere.")))))))

;; What a dependency clause has to satisfy before anything is fetched for it.
(: (-> Walk Dep string (Result string Unit)))
(defun (check-dep walk dep from)
  (def name (record-ref dep name))
  (cond
    ((empty? name) (Err (str "bjo: " from ": a dependency with no name.")))
    ((reserved-name? name)
     (Err (str "bjo: " from ": " (name->string name)
               " is part of the standard library, so it cannot be a dependency."
               " It is there already.")))
    ((= (key-of name) (key-of (record-ref walk own)))
     (Err (str "bjo: " from ": " (name->string name)
               " is this project itself, so it cannot be one of its dependencies.")))
    (else (Ok unit))))

;; Every dependency of one manifest: its clauses checked, its sources recorded,
;; and one `Want` each for the step that places them.
(: (-> Walk Task (Result string (List Want))))
(defun (dependencies-of walk task)
  (with-return ret
    (def package (record-ref task package))
    (def from (record-ref task from))
    (def clauses (if (record-ref task dev?)
                     (list-append (record-ref package depends)
                                  (record-ref package dev-depends))
                     (record-ref package depends)))
    (Ok (loop (:for dep clauses)
              (:do
               (match (check-dep walk dep from)
                 ((Err why) (ret (Err why)))
                 ((Ok u) unit))
               (box-set! (record-ref walk needs)
                         (Cons (Need (need-name (record-ref dep name))
                                     (need-req (record-ref dep version))
                                     (need-from from))
                               (box-ref (record-ref walk needs))))
               (match (record-ref dep source)
                 (None unit)
                 ((Some source)
                  (match (give-source! walk (record-ref dep name) source from
                                       (record-ref task dev?))
                    ((Err why) (ret (Err why)))
                    ((Ok u) unit)))))
              (:acc out (listing (Want (dep dep)
                                       (want-from from)
                                       (want-dir (record-ref task dir))
                                       (want-path-ok? (record-ref task path-ok?)))))
              => out))))

;; Places one dependency, now that its source is known: the package's directory
;; is decided, its manifest read, and a task made of it.
;;
;; `None` when the package is already placed. A name is one directory in the
;; roots file, so the second manifest that asks for it adds a requirement and
;; nothing else.
(: place #:sync (-> Walk Want Given (Result string (Option Task))))
(defun (place walk want given)
  (with-return ret
    (def dep (record-ref want dep))
    (def name (record-ref dep name))
    (def k (key-of name))
    (def asked-by (record-ref want want-from))

    (when (some? (map-try-ref (box-ref (record-ref walk placed)) k))
      (ret (Ok None)))

    (match (record-ref given given)
      ((FromRegistry r)
       (Err (str "bjo: " asked-by " wants " (name->string name)
                 " from a registry, and there are none yet."
                 " Use a git source, or a path source for a package on this disk.")))

      ((FromPath local)
       ;; Relative to the manifest that named it, never to the working
       ;; directory: the same manifest has to mean the same directory whoever
       ;; runs `bjo` and from wherever.
       (def dir (path-absolute (path-combine (record-ref want want-dir) (record-ref local dir))))
       (unless (record-ref want want-path-ok?)
         (ret (Err (str "bjo: " asked-by " gives " (name->string name)
                        " a path source, " (source-shown (record-ref given given))
                        ". A fetched package may not: the directory is on its author's"
                        " disk and says nothing about yours."
                        " Give " (name->string name) " a source in "
                        (record-ref walk own-from) " instead."))))
       (def (Ok package) (package-in-directory name dir asked-by) :propagate)
       (def src (path-combine dir "src"))
       (unless (directory-exists? src)
         (ret (Err (str "bjo: " (name->string name) " at " dir " has no src/."
                        " A package keeps its modules in src/, beside its manifest."))))
       (box-set! (record-ref walk placed)
                 (map-set (box-ref (record-ref walk placed)) k
                          (Root (name name) (dir src))))
       (Ok (Some (Task (package package)
                       (dir dir)
                       (from (path-combine dir manifest-name))
                       (path-ok? #t)
                       (dev? #f)))))

      ((FromGit source)
       (def (Ok made) (place-git walk want source) :propagate)
       (Ok made)))))

;; A git dependency: the node this requirement makes, and the manifest at that
;; version.
;;
;; The manifest is read out of the bare clone, so a version that is considered
;; and then superseded costs a `git show` and nothing else — no checkout, no
;; directory, no entry in the lock.
(: place-git #:sync (-> Walk Want Git (Result string (Option Task))))
(defun (place-git walk want source)
  (with-return ret
    (def dep (record-ref want dep))
    (def name (record-ref dep name))
    (def k (key-of name))
    (def asked-by (record-ref want want-from))
    (def url (record-ref source url))

    ;; Refused rather than ignored: a manifest that pins a commit is asking for
    ;; something this does not do, and building the tag instead would be a
    ;; different program from the one it asked for.
    (when (some? (record-ref source commit))
      (ret (Err (str "bjo: " asked-by " pins " (name->string name)
                     " to a commit, which bjo does not do yet."
                     " Use a version tag, or a path source for a copy on this disk."))))
    (when (some? (record-ref source branch))
      (ret (Err (str "bjo: " asked-by " pins " (name->string name)
                     " to a branch, which bjo does not do yet."
                     " Use a version tag, or a path source for a copy on this disk."))))

    (def (Ok clone) (ensure-clone (record-ref walk tool) (record-ref walk dot-bjo) url name)
      :propagate)
    (remember! walk k name (FromGit source) clone)

    (def (Ok required) (requirement-of walk dep asked-by clone) :propagate)
    (def (Ok version) (node-version walk clone name (lower-bound required) asked-by) :propagate)
    (add-node! walk k version)

    ;; Every node is walked once. Two manifests asking for the same version ask
    ;; for the same manifest, and its requirements are already in hand.
    (def node-key (str k "@" (version->string version)))
    (when (some? (map-try-ref (box-ref (record-ref walk walked)) node-key))
      (ret (Ok None)))
    (box-set! (record-ref walk walked)
              (map-set (box-ref (record-ref walk walked)) node-key #t))

    (def (Ok text) (manifest-at (record-ref walk tool) clone version) :propagate)
    (def shown (str url " " (version->tag version)))
    (def (Ok package) (package-at-tag name text shown) :propagate)

    (Ok (Some (Task (package package)
                    (dir clone)
                    (from shown)
                    (path-ok? #f)
                    (dev? #f))))))

;; A manifest read at a tag, checked against the name it was asked for.
;;
;; A version whose manifest says another name is refused: the roots file would
;; file these modules under the name that was asked for, and every one of them
;; would then compile under a name its author never wrote.
(: (-> (List Symbol) string string (Result string Package)))
(defun (package-at-tag wanted text shown)
  (match (bjodat-parse-Package text)
    ((Err why) (Err (str "bjo: " manifest-name " at " shown ": " why)))
    ((Ok package)
     (if (not (= (key-of (record-ref package name)) (key-of wanted)))
         (Err (str "bjo: " shown " was asked for as " (name->string wanted)
                   ", but its " manifest-name " says the package there is "
                   (name->string (record-ref package name)) "."))
         (Ok package)))))

;; What a name is called, where it comes from and where its clone is. Written
;; down when the first requirement on it is seen, and read again when the
;; chosen version is materialised and locked.
(: (-> Walk string (List Symbol) Source string Unit))
(defun (remember! walk k name source clone)
  (box-set! (record-ref walk names) (map-set (box-ref (record-ref walk names)) k name))
  (box-set! (record-ref walk origins) (map-set (box-ref (record-ref walk origins)) k source))
  (box-set! (record-ref walk clones) (map-set (box-ref (record-ref walk clones)) k clone))
  unit)

(: (-> Walk string Version Unit))
(defun (add-node! walk k version)
  (def had (match (map-try-ref (box-ref (record-ref walk nodes)) k)
             ((Some versions) versions)
             (None Nil)))
  (box-set! (record-ref walk nodes)
            (map-set (box-ref (record-ref walk nodes)) k (Cons version had)))
  unit)

;; A git dependency has to say which versions it will take.
;;
;; There is no default. "Whatever is newest" is a manifest that means something
;; different every week, and "whatever is oldest" is one that never gets a
;; fix — so the manifest says, and the tags that exist are listed here because
;; the answer is one of them.
(: requirement-of #:sync (-> Walk Dep string string (Result string VersionReq)))
(defun (requirement-of walk dep asked-by clone)
  (match (record-ref dep version)
    ((Some req) (Ok req))
    (None
     (def (Ok tags) (version-tags (record-ref walk tool) clone) :propagate)
     (Err (str "bjo: " asked-by " asks for " (name->string (record-ref dep name))
               " from git without saying which versions it takes.\n"
               "  Write one, as in (version (version-at-least \"0.1.0\")).\n"
               "  The versions that repository has: " (versions-shown tags))))))

(: (-> (List Version) string))
(defun (versions-shown versions)
  (if (empty? versions)
      "none — it has no vX.Y.Z tags at all."
      (string-join (list->vec (list-map version->tag versions)) ", ")))

;; The version a requirement asks for, as minimal version selection reads it:
;; the bottom of the range, and `0.0.0` where there is no bottom.
(: (-> VersionReq Version))
(defun (lower-bound req)
  (match req
    ((AtLeast lo) lo)
    ((Between lo hi) lo)
    ((AtMost hi) (version 0 0 0))))

;; The node a requirement makes: the *lowest* version that satisfies it.
;;
;; Not the newest. A manifest that says `(version-at-least "1.2")` says it
;; needs 1.2, not that it has been tested against everything after it — so the
;; build uses 1.2 unless something else in the graph asks for more. That is the
;; whole of minimal version selection, and it is why two builds of the same
;; manifests a year apart use the same code.
(: node-version #:sync (-> Walk string (List Symbol) Version string (Result string Version)))
(defun (node-version walk clone name lower asked-by)
  (def (Ok tags) (version-tags (record-ref walk tool) clone) :propagate)
  (match (find #(>= (version-compare & lower) 0) tags)
    ((Some found) (Ok found))
    (None
     ;; Only now: a lower bound nothing here satisfies is the one question the
     ;; clone cannot answer out of what it already has.
     (def (Ok fetched?) (fetch-once walk clone) :propagate)
     (def (Ok after) (version-tags (record-ref walk tool) clone) :propagate)
     (match (find #(>= (version-compare & lower) 0) after)
       ((Some found) (Ok found))
       (None
        (Err (str "bjo: " asked-by " asks for " (name->string name)
                  " at least " (version->string lower)
                  ", and that repository has no such version.\n"
                  "  The versions it has: " (versions-shown after))))))))

;; Fetches a clone once per run. A resolve asks several questions of one
;; repository, and a fetch that answered the first answers the rest.
(: fetch-once #:sync (-> Walk string (Result string bool)))
(defun (fetch-once walk clone)
  (cond
    ((some? (map-try-ref (box-ref (record-ref walk fetched)) clone)) (Ok #f))
    (else
     (box-set! (record-ref walk fetched)
               (map-set (box-ref (record-ref walk fetched)) clone #t))
     (def (Ok got) (fetch-tags (record-ref walk tool) clone) :propagate)
     (Ok #t))))

;; The dependencies that can be placed now, and the ones that still cannot.
(: (-> Walk (List Want) (Tuple (List Want) (List Want))))
(defun (ready-and-waiting walk wants)
  (def known (box-ref (record-ref walk sources)))
  (Tuple (list-filter #(some? (map-try-ref known (key-of (record-ref (record-ref & dep) name)))) wants)
         (list-filter #(not (some? (map-try-ref known (key-of (record-ref (record-ref & dep) name))))) wants)))

;; A name nobody said where to get.
(: (-> Walk Want string))
(defun (no-source-message walk want)
  (def name (record-ref (record-ref want dep) name))
  (str "bjo: " (record-ref want want-from) " wants " (name->string name)
       " and no manifest says where it comes from."
       " Give it a source: (source (git (url \"https://…\"))) to fetch it,"
       " or (source (path (dir \"../" (name->string name) "\"))) for a copy on this disk."))

;; Places every dependency that can be placed, and answers the tasks they made.
(: place-all #:sync (-> Walk (List Want) (Result string (List Task))))
(defun (place-all walk wants)
  (match wants
    (Nil (Ok Nil))
    ((Cons want rest)
     (def known (box-ref (record-ref walk sources)))
     (def (Ok made)
       (match (map-try-ref known (key-of (record-ref (record-ref want dep) name)))
         ;; Not this pass. It stays in the waiting list, which is the caller's.
         (None (Ok None))
         ((Some given) (place walk want given)))
       :propagate)
     (def (Ok more) (place-all walk rest) :propagate)
     (Ok (match made
           ((Some task) (Cons task more))
           (None more))))))

;; The whole walk: read a manifest, place what it names, repeat.
;;
;; `waiting` holds the dependencies with no source yet. When the queue empties,
;; the ones that have a source by now are placed and the walk goes on; a pass
;; that places none of them ends it, and whatever is still waiting is an error.
(: walk-graph #:sync (-> Walk (List Task) (List Want) (Result string Unit)))
(defun (walk-graph walk tasks waiting)
  (match tasks
    ((Cons task rest)
     (def (Ok found) (dependencies-of walk task) :propagate)
     (def (Tuple ready still) (ready-and-waiting walk (list-append waiting found)))
     (def (Ok made) (place-all walk ready) :propagate)
     (walk-graph walk (list-append rest made) still))
    (Nil
     (if (empty? waiting)
         (Ok unit)
         (let ((split (ready-and-waiting walk waiting)))
           (match split
             ((Tuple Nil (Cons first _)) (Err (no-source-message walk first)))
             ((Tuple ready still)
              (def (Ok made) (place-all walk ready) :propagate)
              (walk-graph walk made still))))))))


;; ---------------------------------------------------------------------------
;; The root package
;; ---------------------------------------------------------------------------

;; The root package's own entry in the roots file.
;;
;; `src/` has to exist: the compiler refuses a roots file naming a directory
;; that is not there, and refusing it here says which package and which
;; directory instead of naming a file the user never wrote.
(: (-> Project (Result string Root)))
(defun (root-package-root project)
  (def name (record-ref (project-package project) name))
  (def src (project-src project))
  (cond
    ((empty? name)
     (Err (str "bjo: " (path-combine (project-dir project) manifest-name)
               ": the package has no name.")))
    ((reserved-name? name)
     (Err (str "bjo: " (path-combine (project-dir project) manifest-name) ": "
               (name->string name)
               " is part of the standard library, so a package cannot be called that.")))
    ((not (directory-exists? src))
     (Err (str "bjo: " (name->string name) " has no " src "."
               " A package keeps its modules in src/, beside its manifest.")))
    (else (Ok (Root (name name) (dir src))))))


;; ---------------------------------------------------------------------------
;; Choosing versions
;; ---------------------------------------------------------------------------

;; The version each git package ends up at: the highest of its nodes.
;;
;; No search and no backtracking. Every node is a version some manifest asked
;; for, so the highest of them is the lowest version that satisfies all of
;; them — and since every reachable node counts, including versions that were
;; later superseded, adding a manifest to the graph can raise a version but
;; never lower one.
;;
;; A worked example. The app needs A at least 1.0 and B at least 1.0; B 1.0
;; needs A at least 1.1; A has tags v1.0.0, v1.1.0 and v1.2.0.
;;
;;   the app's (version-at-least "1.0") on A  -> node A 1.0.0
;;   the app's (version-at-least "1.0") on B  -> node B 1.0.0
;;   B 1.0.0's (version-at-least "1.1") on A  -> node A 1.1.0
;;
;; A's nodes are 1.0.0 and 1.1.0, so A is 1.1.0 — not 1.2.0, which nobody
;; asked for, and not 1.0.0, which B would not work with. Remove B and the
;; next build uses A 1.0.0 again.
(: (-> Walk (List Chosen)))
(defun (choose walk)
  (map-fold (fun (k versions found) (Cons (chosen-of walk k versions) found))
            Nil
            (box-ref (record-ref walk nodes))))

(: (-> Walk string (List Version) Chosen))
(defun (chosen-of walk k versions)
  (Chosen (chosen-name (map-ref (box-ref (record-ref walk names)) k))
          (chosen-version (highest versions))
          (chosen-source (map-ref (box-ref (record-ref walk origins)) k))
          (chosen-clone (map-ref (box-ref (record-ref walk clones)) k))))

;; `list-foldl` hands the item first and the accumulator second.
(: (-> (List Version) Version))
(defun (highest versions)
  (list-foldl (fun (v best) (if (> (version-compare v best) 0) v best))
              (version 0 0 0)
              versions))

;; A constraint as a manifest writes it.
(: (-> (Option VersionReq) string))
(defun (req-shown req)
  (match req
    (None "(no version)")
    ((Some r) (bjodat->string (->bjodat r)))))

;; Every upper bound, checked against the version that was chosen.
;;
;; An upper bound is a *check*, never a choice: it cannot make the resolver
;; take a lower version, because a version that is too low for one manifest is
;; a build that does not work at all, while a version that is too high for
;; another is a build the author of that one has not tried. So the graph
;; decides and the bounds say whether the answer is acceptable.
(: (-> Walk Chosen (Result string Unit)))
(defun (check-bounds walk c)
  (def name (record-ref c chosen-name))
  (def picked (record-ref c chosen-version))
  (def mine (list-filter #(= (key-of (record-ref & need-name)) (key-of name))
                         (box-ref (record-ref walk needs))))
  (def broken (list-filter (fun (n) (match (record-ref n need-req)
                                      (None #f)
                                      ((Some req) (not (version-satisfies? picked req)))))
                           mine))
  (if (empty? broken)
      (Ok unit)
      (Err (str "bjo: " (name->string name) " " (version->string picked)
                " does not satisfy everything asked of it.\n"
                "  What each manifest asks for:\n"
                (string-join (list->vec (list-sort (list-map #(need-line picked &) mine))) "\n")
                "\n  The highest version anybody asks for is the one that is used, and an"
                " upper bound never lowers it.\n"
                "  Raise the bound, or lower what asks for more."))))

(: (-> Version Need string))
(defun (need-line picked n)
  (def satisfied?
    (match (record-ref n need-req)
      (None #t)
      ((Some req) (version-satisfies? picked req))))
  (str "    " (req-shown (record-ref n need-req)) " in " (record-ref n need-from)
       (if satisfied? "" (str "   <- " (version->string picked) " breaks this"))))

(: (-> Walk (List Chosen) (Result string Unit)))
(defun (check-every-bound walk chosen)
  (match chosen
    (Nil (Ok unit))
    ((Cons c rest)
     (def (Ok held) (check-bounds walk c) :propagate)
     (check-every-bound walk rest))))


;; ---------------------------------------------------------------------------
;; The lock
;; ---------------------------------------------------------------------------

(: (-> (List Symbol) Version string string string string))
(defun (moved-tag-message name v locked-at now lock-file)
  (str "bjo: " (name->string name) " " (version->tag v)
       " no longer points at the commit " lock-file " names.\n"
       "    locked  " locked-at "\n"
       "    now     " now "\n"
       "  A version is a tag, and a tag can be moved. Nothing was built:"
       " the same version naming different code is what the lock exists to catch.\n"
       "  If the move was meant, remove that entry from " lock-file " and fetch again."))

;; What the lock should say about one chosen package.
;;
;; The check that matters is here: a package whose version has not changed must
;; still be the commit the lock names.
(: lock-entry #:sync (-> Walk Chosen string (Result string Locked)))
(defun (lock-entry walk c lock-file)
  (def name (record-ref c chosen-name))
  (def picked (record-ref c chosen-version))
  (def clone (record-ref c chosen-clone))
  (def was (lock-entry-for (record-ref walk was-locked) name))
  (def same-version?
    (match was
      (None #f)
      ((Some old) (= (version->string (locked-version old)) (version->string picked)))))

  ;; A commit the lock names and this clone has not got is the one thing worth
  ;; going to the network for: it existed when the lock was written.
  (def (Ok fetched-for-lock?)
    (match was
      ((Some old) (if (and same-version?
                           (not (commit-present? (record-ref walk tool) clone (locked-commit old))))
                      (fetch-once walk clone)
                      (Ok #f)))
      (None (Ok #f)))
    :propagate)

  (def (Ok commit) (tag-commit (record-ref walk tool) clone picked) :propagate)

  (def was-commit (match was ((Some old) (locked-commit old)) (None "")))
  (if (and same-version? (not (= was-commit commit)))
      (Err (moved-tag-message name picked was-commit commit lock-file))
      (Ok (Locked (name name) (version picked)
                  (source (record-ref c chosen-source))
                  (commit commit)))))

(: lock-entries #:sync (-> Walk (List Chosen) string (Result string (List Locked))))
(defun (lock-entries walk chosen lock-file)
  (match chosen
    (Nil (Ok Nil))
    ((Cons c rest)
     (def (Ok entry) (lock-entry walk c lock-file) :propagate)
     (def (Ok more) (lock-entries walk rest lock-file) :propagate)
     (Ok (Cons entry more)))))


;; ---------------------------------------------------------------------------
;; Putting the chosen versions on disk
;; ---------------------------------------------------------------------------

;; One directory per chosen version, and its `src/` in the roots file.
(: materialise-all #:sync (-> Walk (List Chosen) (List Locked) (Result string (List Root))))
(defun (materialise-all walk chosen entries)
  (match chosen
    (Nil (Ok Nil))
    ((Cons c rest)
     (def name (record-ref c chosen-name))
     (def picked (record-ref c chosen-version))
     (def (Some entry) (lock-entry-for entries name)
       :leave-with (Err (str "bjo: " (name->string name) " was chosen and not locked.")))
     (def where (package-directory (record-ref walk dot-bjo) name picked))
     (def (Ok put) (materialise (record-ref walk tool)
                                (record-ref c chosen-clone)
                                (locked-commit entry)
                                (path-combine (record-ref walk dot-bjo) "tmp")
                                where)
       :propagate)
     (def src (path-combine where "src"))
     (def (Ok more) (materialise-all walk rest entries) :propagate)
     (if (not (directory-exists? src))
         (Err (str "bjo: " (name->string name) " " (version->string picked)
                   " has no src/, so there are no modules in it."
                   " A package keeps its modules in src/, beside its manifest."))
         (Ok (Cons (Root (name name) (dir src)) more))))))


;; ---------------------------------------------------------------------------
;; prepare
;; ---------------------------------------------------------------------------

;; Resolves the project, fetches what that needs, and writes the roots file.
;;
;; `locked?` is `--locked`: any difference from `bjo.lock` is an error rather
;; than an update, which is what a build that must be the one the lock
;; describes wants.
(: prepare #:sync (-> Project bool (Result string Prepared)))
(defun (prepare project locked?)
  (def (Ok own) (root-package-root project) :propagate)
  (def lock-file (project-lock-file project))
  (def (Ok was-locked) (read-lock lock-file) :propagate)

  (def walk (Walk (sources (make-box (map-empty)))
                  (placed (make-box (map-empty)))
                  (needs (make-box Nil))
                  (nodes (make-box (map-empty)))
                  (names (make-box (map-empty)))
                  (clones (make-box (map-empty)))
                  (origins (make-box (map-empty)))
                  (walked (make-box (map-empty)))
                  (fetched (make-box (map-empty)))
                  (own (record-ref (project-package project) name))
                  (own-from (path-combine (project-dir project) manifest-name))
                  (tool (tool-for (project-tmp project)))
                  (dot-bjo (project-dot-bjo project))
                  (was-locked was-locked)))

  ;; The root task is the only one whose `dev-depends` count — a dependency's
  ;; are what its author needs in order to test it, and building it here runs
  ;; no tests — and the only one allowed to say `(path ...)`, which its own
  ;; path dependencies inherit: a directory beside yours is as much yours as
  ;; your own.
  (def first-task (Task (package (project-package project))
                        (dir (project-dir project))
                        (from (path-combine (project-dir project) manifest-name))
                        (path-ok? #t)
                        (dev? #t)))

  (def (Ok walked) (walk-graph walk (Cons first-task Nil) Nil) :propagate)

  (def chosen (choose walk))
  (def (Ok bounds-held) (check-every-bound walk chosen) :propagate)
  (def (Ok entries) (lock-entries walk chosen lock-file) :propagate)
  (def (Ok settled) (settle-lock project locked? was-locked entries) :propagate)
  (def (Ok fetched-roots) (materialise-all walk chosen entries) :propagate)

  (def path-roots (loop (:for entry (map-values (box-ref (record-ref walk placed))))
                        (:acc out (listing entry))
                        => out))
  (def roots (Cons own (list-append path-roots fetched-roots)))
  (def roots-path (project-roots-file project))
  (ignore (write-roots roots-path roots))

  ;; Beside the roots file, and written by the same rule: only when its
  ;; contents change, because the compiler records it as an input of every
  ;; module built against it.
  (def frameworks-path (project-frameworks-file project))
  (def declared? (write-frameworks frameworks-path roots))

  (def (Ok nuget-dir) (restore-packages project roots locked?) :propagate)

  (Ok (Prepared (roots-file roots-path)
                (frameworks-file (if declared? (Some frameworks-path) None))
                (nuget-dir nuget-dir)
                (runtime (runtime-settings (record-ref (project-package project) runtime)))
                (aot (match (record-ref (project-package project) aot)
                       ((Some aot?) aot?)
                       (None #f)))
                (aot-cpu (match (record-ref (project-package project) aot-cpu)
                           ((Some cpu) cpu)
                           (None "")))
                (roots roots))))

;; Writes the lock if it has to, and says what changed.
;;
;; With minimal version selection there is nothing to write unless a manifest
;; changed: the same manifests choose the same versions every time. So a line
;; printed here is a real change, and `--locked` refusing one is a build that
;; would not have been the build the lock describes.
(: settle-lock #:sync (-> Project bool (List Locked) (List Locked) (Result string Unit)))
(defun (settle-lock project locked? before after)
  (def changes (lock-changes before after))
  (def lock-file (project-lock-file project))
  (cond
    ((empty? changes) (Ok unit))
    (locked?
     (Err (str "bjo: --locked, and the manifests no longer agree with " lock-file ":\n"
               (string-join (list->vec (list-map #(str "    " &) changes)) "\n")
               "\n  Run 'bjo fetch' without --locked to bring the lock up to date.")))
    (else
     (write-lock lock-file after)
     (println (str "Updated " lock-file ":"))
     (loop (:for line changes)
           (:do (println (str "  " line))))
     (Ok unit))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; janitor/docs.bjo — a module's published docs, as records.
;;
;; The compiler publishes a module's `(:doc ...)` forms in its assembly, under
;; the metadata key `BjolangDocs` (Docs/Documentation.org, "Where the
;; documentation goes"). This reads them back without loading the assembly, so
;; a tool can read the docs of a module it also links, or of many modules,
;; without keeping them loaded.
;;
;; A clause this version does not know is kept as `DocOther` rather than
;; refused: docs written for a later compiler still read.

(export
  Doc DocSubject DocKind DocClause
  parse-docs assembly-docs module-docs
  doc-summary doc-named module-doc doc-subject->string)

(import (std prelude))
(import (text bjodat-core))
(import (janitor modules))

(import/extern
  (clr-read-metadata (: Bjolang.Runtime.BjoAssemblyMetadata.Read (-> string string string)
                        #:exceptions (System.IO.IOException
                                      System.UnauthorizedAccessException
                                      System.BadImageFormatException))))

(:doc #:module
  (summary "A module's published docs, read back as records.")
  (see module-docs assembly-docs Doc)
  (reference """
    Each (:doc ...) form a module was compiled with is a Doc. Its clauses are
    as written, in the order written, with what the compiler added taken out
    into the record: the kind of definition, and for a function or a value its
    signature, and for a defun its head as written.
    """))

;; ---------------------------------------------------------------------------
;; The records
;; ---------------------------------------------------------------------------

(:doc DocSubject
  (summary "What a doc is about.")
  (case SubjectName "A definition, by its name.")
  (case SubjectReader "A reader extension: (:doc name #:reader ...).")
  (case SubjectModule "The module itself: (:doc #:module ...)."))
(type (: DocSubject (Union (: SubjectName Symbol)
                           (: SubjectReader Symbol)
                           SubjectModule)))

(:doc DocKind
  (summary "What kind of definition a doc is about, as the compiler found it.")
  (case KindOther "A kind this version does not know, as published."))
(type (: DocKind (Union KindFunction KindValue KindRecord KindUnion KindAlias
                        KindTrait KindMacro KindReader KindModule
                        (: KindOther Symbol))))

(:doc DocClause
  (summary "One clause of a doc, as written.")
  (case DocArg "A positional parameter's name and text.")
  (case DocKey "A keyword parameter's name, without #:, and text.")
  (case DocRaises "The exception's type as written, and when.")
  (case DocTparam "The type parameter as written, %a, and its text.")
  (case DocForm "How a macro is written, as source text: (with-x name body ...).")
  (case DocOther "A clause this version does not know: its head and the rest."))
(type (: DocClause (Union
                    (: DocSummary string)
                    (: DocArg Symbol string)
                    (: DocRest Symbol string)
                    (: DocKey Symbol string)
                    (: DocReturns string)
                    (: DocRaises Symbol string)
                    (: DocSee (List Symbol))
                    (: DocExample string)
                    (: DocReference string)
                    (: DocField Symbol string)
                    (: DocCase Symbol string)
                    (: DocTparam Symbol string)
                    (: DocForm string)
                    (: DocLiteral (List Symbol))
                    (: DocOther Symbol (List Bjodat)))))

(:doc Doc
  (summary "One published doc.")
  (field subject "What it is about.")
  (field kind "What kind of definition that is.")
  (field signature "A function's or a value's type, as source writes it, when it has one.")
  (field definition "A defun's head as written, defaults and all: (random-int lo hi #:rng r).")
  (field clauses "Every other clause, in the order written."))
(type (: Doc (Record (: subject DocSubject)
                     (: kind DocKind)
                     (: signature (Option string))
                     (: definition (Option string))
                     (: clauses (List DocClause)))))

;; ---------------------------------------------------------------------------
;; Reading
;; ---------------------------------------------------------------------------

;; A message names the form, so a published doc that does not read says where.
(: (-> string Bjodat (Result string %a)))
(defun (malformed what form)
  (Err #"${what}: ${(bjodat->string form)}"))

(: (-> Symbol DocKind))
(defun (kind-named s)
  (match (symbol->string s)
    ("function" KindFunction)
    ("value" KindValue)
    ("record" KindRecord)
    ("union" KindUnion)
    ("alias" KindAlias)
    ("trait" KindTrait)
    ("macro" KindMacro)
    ("reader" KindReader)
    ("module" KindModule)
    (_ (KindOther s))))

(: (-> (List Bjodat) (Option (List Symbol))))
(defun (symbol-list xs)
  (match xs
    (Nil (Some Nil))
    ((Cons (BjoSym s) rest)
     (option-map (fun (more) (Cons s more)) (symbol-list rest)))
    (_ None)))

;; A clause by its head. `#:rng` and `rng` are the same key.
(: (-> Bjodat (Result string DocClause)))
(defun (read-clause c)
  (match c
    ((BjoList (Cons (BjoSym head) args))
     (match (Tuple (symbol->string head) args)
       ((Tuple "summary" (Cons (BjoStr t) Nil)) (Ok (DocSummary t)))
       ((Tuple "arg" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocArg n t)))
       ((Tuple "rest" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocRest n t)))
       ((Tuple "key" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocKey n t)))
       ((Tuple "key" (Cons (BjoKey k) (Cons (BjoStr t) Nil)))
        (Ok (DocKey (string->symbol (keyword->string k)) t)))
       ((Tuple "returns" (Cons (BjoStr t) Nil)) (Ok (DocReturns t)))
       ((Tuple "raises" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocRaises n t)))
       ((Tuple "example" (Cons (BjoStr t) Nil)) (Ok (DocExample t)))
       ((Tuple "reference" (Cons (BjoStr t) Nil)) (Ok (DocReference t)))
       ((Tuple "field" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocField n t)))
       ((Tuple "case" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocCase n t)))
       ((Tuple "tparam" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocTparam n t)))
       ((Tuple "form" (Cons (BjoStr t) Nil)) (Ok (DocForm t)))
       ;; As compilers before the string published it.
       ((Tuple "form" (Cons (BjoList shape) Nil)) (Ok (DocForm (bjodat->string (BjoList shape)))))
       ((Tuple "see" names)
        (match (symbol-list names)
          ((Some ns) (Ok (DocSee ns)))
          (None (malformed "a see clause names symbols" c))))
       ((Tuple "literal" names)
        (match (symbol-list names)
          ((Some ns) (Ok (DocLiteral ns)))
          (None (malformed "a literal clause names symbols" c))))
       ((Tuple (or "summary" "arg" "rest" "key" "returns" "raises" "example"
                   "reference" "field" "case" "tparam" "form") _)
        (malformed "a clause of the wrong shape" c))
       (_ (Ok (DocOther head args)))))
    (_ (malformed "a clause is a list headed by a symbol" c))))

;; The doc being built: the record's parts, before its clauses are reversed.
(type (: Partial (Record (: kind DocKind)
                         (: signature (Option string))
                         (: definition (Option string))
                         (: clauses (List DocClause)))))

;; The symbol a list is headed by, or "" for anything else.
(: (-> Bjodat string))
(defun (head-of b)
  (match b
    ((BjoList (Cons (BjoSym h) _)) (symbol->string h))
    (_ "")))

;; The added clauses go into the record, the rest into its clauses.
(: (-> Partial (List Bjodat) (Result string Partial)))
(defun (read-clauses p cs)
  (match cs
    (Nil (Ok p))
    ((Cons c rest)
     (match (Tuple (head-of c) c)
       ((Tuple "kind" (BjoList (Cons _ (Cons (BjoSym k) Nil))))
        (read-clauses (record-set p (kind (kind-named k))) rest))
       ((Tuple "signature" (BjoList (Cons _ (Cons (BjoStr s) Nil))))
        (read-clauses (record-set p (signature (Some s))) rest))
       ((Tuple "definition" (BjoList (Cons _ (Cons (BjoStr s) Nil))))
        (read-clauses (record-set p (definition (Some s))) rest))
       (_
        (def (Ok clause) (read-clause c) :propagate)
        (read-clauses (record-set p (clauses (Cons clause (record-ref p clauses)))) rest))))))

;; `(:doc name ...)`, `(:doc name #:reader ...)` or `(:doc #:module ...)`.
(: (-> Bjodat (Result string Doc)))
(defun (read-doc form)
  (def start (Partial (kind (KindOther (string->symbol "unknown")))
                      (signature None) (definition None) (clauses Nil)))
  (def subject+clauses
    (match form
      ((BjoList (Cons (BjoKey d) after))
       (if (not (= (keyword->string d) "doc"))
           None
           (match after
             ((Cons (BjoKey m) cs)
              (if (= (keyword->string m) "module") (Some (Tuple SubjectModule cs)) None))
             ((Cons (BjoSym n) (Cons (BjoKey r) cs))
              (if (= (keyword->string r) "reader") (Some (Tuple (SubjectReader n) cs)) None))
             ((Cons (BjoSym n) cs) (Some (Tuple (SubjectName n) cs)))
             (_ None))))
      (_ None)))
  (match subject+clauses
    (None (malformed "a doc is (:doc name clause ...)" form))
    ((Some (Tuple subject cs))
     (def (Ok p) (read-clauses start cs) :propagate)
     (Ok (Doc (subject subject)
              (kind (record-ref p kind))
              (signature (record-ref p signature))
              (definition (record-ref p definition))
              (clauses (list-reverse (record-ref p clauses))))))))

(: (-> (List Bjodat) (Result string (List Doc))))
(defun (read-docs forms)
  (match forms
    (Nil (Ok Nil))
    ((Cons f rest)
     (def (Ok doc) (read-doc f) :propagate)
     (def (Ok more) (read-docs rest) :propagate)
     (Ok (Cons doc more)))))

(:doc parse-docs
  (summary "Published docs, from the text the compiler wrote.")
  (arg text "A bjodat vec of (:doc ...) forms, or \"\" for a module with none.")
  (returns "The docs, in the order the module wrote them, or why the text does not read.")
  (see assembly-docs))
(: (-> string (Result string (Vec Doc))))
(defun (parse-docs text)
  (if (= (string-trim text) "")
      (Ok [])
      (match (parse-bjodat text)
        ((Err why) (Err why))
        ((Ok (BjoVec forms))
         (def (Ok docs) (read-docs (vec->list forms)) :propagate)
         (Ok (list->vec docs)))
        ((Ok other) (malformed "published docs are a vec" other)))))

(:doc assembly-docs
  (summary "The docs published in an assembly, read without loading it.")
  (arg path "The module's dll, or a program's exe.")
  (returns "Its docs, none when it has none, or why the file could not be read.")
  (see module-docs parse-docs))
(: (-> string (Result string (Vec Doc))))
(defun (assembly-docs path)
  (if (not (file-exists? path))
      (Err #"no such file: ${path}")
      (match (clr-read-metadata path "BjolangDocs")
        ((Err e) (Err #"${path}: ${(.-Message e)}"))
        ((Ok text)
         (result-map-err (fun (why) #"${path}: ${why}") (parse-docs text))))))

(:doc module-docs
  (summary "The docs of a module, by name.")
  (arg roots "The packages to look in: (standard-roots), and a project's from read-roots-file.")
  (arg name "The module: (std random).")
  (returns "Its docs, or why there are none to read.")
  (see assembly-docs find-module-assembly standard-roots read-roots-file))
(: (-> (List PackageRoot) (List Symbol) (Result string (Vec Doc))))
(defun (module-docs roots name)
  (def (Ok dll) (find-module-assembly roots name) :propagate)
  (assembly-docs dll))

;; ---------------------------------------------------------------------------
;; Asking
;; ---------------------------------------------------------------------------

(:doc doc-summary
  (summary "A doc's one-line summary.")
  (arg doc "The doc.")
  (returns "Its summary, or \"\" for one published without."))
(: (-> Doc string))
(defun (doc-summary doc)
  (loop (:for c (record-ref doc clauses))
        (:acc found (folding ""
                             (match c
                               ((DocSummary s) (if (= found "") s found))
                               (_ found))))))

(:doc doc-named
  (summary "The doc of a definition, by its name.")
  (arg docs "A module's docs.")
  (arg name "The definition's name. A reader extension's is written with its #: #pair.")
  (returns "The doc, or None when the module has none by that name.")
  (see module-doc))
(: (-> (Vec Doc) string (Option Doc)))
(defun (doc-named docs name)
  (find #(match (record-ref & subject)
           ((SubjectName n) (= (symbol->string n) name))
           ((SubjectReader n) (= (str "#" (symbol->string n)) name))
           (SubjectModule #f))
        docs))

(:doc module-doc
  (summary "The doc of the module itself: its (:doc #:module ...).")
  (arg docs "A module's docs.")
  (returns "The doc, or None when the module has none.")
  (see doc-named))
(: (-> (Vec Doc) (Option Doc)))
(defun (module-doc docs)
  (find #(match (record-ref & subject)
           (SubjectModule #t)
           (_ #f))
        docs))

(:doc doc-subject->string
  (summary "What a doc is about, as source writes it.")
  (arg subject "The subject.")
  (returns "random-int, #pair for a reader extension, or #:module."))
(: (-> DocSubject string))
(defun (doc-subject->string subject)
  (match subject
    ((SubjectName n) (symbol->string n))
    ((SubjectReader n) (str "#" (symbol->string n)))
    (SubjectModule "#:module")))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; janitor/modules.bjo — where a module name leads, found the way the compiler
;; finds it.
;;
;; `janitor` holds what tools need from the compiler's side of things. The
;; rules here are `Paths.fs`'s, written again: the installed `lib` is the first
;; candidate with a `std` in it, each directory of it is a package, a roots
;; file adds more, and the longest package name that is a proper prefix of a
;; module name owns it. A module's dll sits beside its source.

(export
  PackageRoot
  module-name->string
  lib-directory standard-roots read-roots-file
  find-module-source find-module-assembly)

(import (std prelude))
(import (text bjodat-core))

(import/extern
  (clr-base-dir       (: System.AppContext.BaseDirectory #:get))
  (clr-directory-name (: System.IO.Path.GetDirectoryName (-> string string)))
  (clr-change-ext     (: System.IO.Path.ChangeExtension (-> string string string)))
  (clr-null-or-empty? (: System.String.IsNullOrEmpty (-> string bool))))

(import/class
  (ClrType (: System.Type))
  (ClrAssembly (: System.Reflection.Assembly)))

(import/extern
  ;; Untyped: written (-> object ClrType), a use says object and object do
  ;; not match.
  (clr-type-of     (: System.Object.GetType))
  (clr-assembly-of (: System.Type.Assembly (-> ClrType ClrAssembly) #:get))
  (clr-location    (: System.Reflection.Assembly.Location (-> ClrAssembly string) #:get)))

;; A value of a type this module defines, whose assembly is this module's.
(type (: OwnMarker (Record (: unused int))))
(: OwnMarker)
(def own-marker (OwnMarker (unused 0)))

(:doc #:module
  (summary "Where a module name leads: its package, its source and its dll.")
  (see find-module-assembly read-roots-file)
  (reference """
    The rules are the compiler's. The installed lib is the first candidate
    that has a std in it: $BJOLANG_LIB, $BJOLANG_HOME/lib, then a lib in the
    directory of the running program or any directory above it, then the same
    from the working directory. Each directory of lib is a package of the
    standard library. A roots file, which bjo writes as .bjo/roots, adds the
    packages of a project. The longest package name that is a proper prefix of
    a module name owns the module, and the rest of the name is a path below
    the package's directory.
    """))

(:doc PackageRoot
  (summary "A package: its name, and the directory its modules are in.")
  (field name "As an import spells it: (std), (bjolang http).")
  (field directory "Absolute.")
  (field standard "Whether it is part of the installed standard library."))
(type (: PackageRoot (Record (: name (List Symbol))
                             (: directory string)
                             (: standard bool))))

(:doc module-name->string
  (summary "A module name as source writes it.")
  (arg name "The name.")
  (returns "(std random), for (std random)."))
(: (-> (List Symbol) string))
(defun (module-name->string name)
  (str "(" (string-join (list->vec (list-map symbol->string name)) " ") ")"))

;; ---------------------------------------------------------------------------
;; The installed library
;; ---------------------------------------------------------------------------

;; `None` at the root of the filesystem, where `GetDirectoryName` answers null.
(: (-> string (Option string)))
(defun (parent-of dir)
  (def up (clr-directory-name (path-absolute dir)))
  (if (clr-null-or-empty? up) None (Some up)))

;; `<dir>/lib` for `dir` and every directory above it, deepest first.
(: (-> string (List string)))
(defun (ancestor-libs start)
  (let go ((dir (path-absolute start)))
    (Cons (path-combine dir "lib")
          (match (parent-of dir)
            ((Some up) (go up))
            (None Nil)))))

(: (-> string string (List string)))
(defun (env-dir var below)
  (match (get-environment-variable var)
    ((Some dir) (Cons (path-absolute (if (= below "") dir (path-combine dir below))) Nil))
    (None Nil)))

;; The lib this module was loaded from: lib/janitor/modules.dll. A program run
;; from outside the installation finds no lib above itself or its working
;; directory, but it does link this.
(: (-> (List string)))
(defun (own-lib)
  (def own (clr-location (clr-assembly-of (clr-type-of own-marker))))
  (if (clr-null-or-empty? own)
      Nil
      (match (parent-of own)
        ((Some janitor) (match (parent-of janitor)
                          ((Some lib) (Cons lib Nil))
                          (None Nil)))
        (None Nil))))

(:doc lib-directory
  (summary "The installed lib directory, or None when there is none to find.")
  (returns "The first candidate with a std directory in it, as the compiler picks it.")
  (see standard-roots)
  (reference """
    The candidates are the compiler's, with one before them: the lib this
    module was loaded from, which a program run from anywhere has.
    """))
(: (-> (Option string)))
(defun (lib-directory)
  (find #(directory-exists? (path-combine & "std"))
        (list-append (env-dir "BJOLANG_LIB" "")
                     (list-append (env-dir "BJOLANG_HOME" "lib")
                                  (list-append (own-lib)
                                               (list-append (ancestor-libs clr-base-dir)
                                                            (ancestor-libs (current-directory))))))))

(:doc standard-roots
  (summary "The standard library's packages: one per directory of lib.")
  (returns "The packages, or none when there is no lib to find.")
  (see lib-directory))
(: (-> (List PackageRoot)))
(defun (standard-roots)
  (match (lib-directory)
    (None Nil)
    ((Some lib)
     (loop (:for dir (directory-subdirectories lib))
           (:acc out (listing (PackageRoot (name (Cons (string->symbol (option-ref-or (path-filename dir) "")) Nil))
                                           (directory (path-absolute dir))
                                           (standard #t))))
           => out))))

;; ---------------------------------------------------------------------------
;; A roots file
;; ---------------------------------------------------------------------------

;; `(root (pkg name) "dir")`, the directory relative to the file's.
(: (-> string Bjodat (Result string PackageRoot)))
(defun (root-entry base entry)
  (match entry
    ((BjoList (Cons (BjoSym head) (Cons (BjoList parts) (Cons (BjoStr dir) Nil))))
     (if (not (= (symbol->string head) "root"))
         (Err #"expected (root (name ...) \"directory\"), not ${(bjodat->string entry)}")
         (match (symbols-of parts)
           (None (Err #"a package name is a list of symbols: ${(bjodat->string entry)}"))
           ((Some Nil) (Err "a package name has at least one segment"))
           ((Some name)
            (Ok (PackageRoot (name name)
                             (directory (path-absolute (path-combine base dir)))
                             (standard #f)))))))
    (_ (Err #"expected (root (name ...) \"directory\"), not ${(bjodat->string entry)}"))))

(: (-> (List Bjodat) (Option (List Symbol))))
(defun (symbols-of parts)
  (match parts
    (Nil (Some Nil))
    ((Cons (BjoSym s) rest)
     (option-map (fun (more) (Cons s more)) (symbols-of rest)))
    (_ None)))

(: (-> string (List Bjodat) (Result string (List PackageRoot))))
(defun (root-entries base entries)
  (match entries
    (Nil (Ok Nil))
    ((Cons e rest)
     (def (Ok root) (root-entry base e) :propagate)
     (def (Ok more) (root-entries base rest) :propagate)
     (Ok (Cons root more)))))

(:doc read-roots-file
  (summary "The packages a roots file names, as bjo writes one in .bjo/roots.")
  (arg file "The roots file: (roots (root (name ...) \"directory\") ...).")
  (returns "Its packages, or why the file could not be read. A relative directory is relative to the file.")
  (see standard-roots))
(: (-> string (Result string (List PackageRoot))))
(defun (read-roots-file file)
  (if (not (file-exists? file))
      (Err #"no roots file at ${file}")
      (match (parse-bjodat (file-read-text file))
        ((Err why) (Err #"${file}: ${why}"))
        ((Ok (BjoList (Cons (BjoSym head) entries)))
         (if (= (symbol->string head) "roots")
             (result-map-err (fun (why) #"${file}: ${why}")
                             (root-entries (option-ref-or (parent-of file) ".") entries))
             (Err #"${file}: expected one (roots ...) form")))
        ((Ok _) (Err #"${file}: expected one (roots ...) form")))))

;; ---------------------------------------------------------------------------
;; Resolving
;; ---------------------------------------------------------------------------

;; `prefix` is a proper prefix of `name`: shorter, and equal as far as it goes.
(: (-> (List Symbol) (List Symbol) bool))
(defun (proper-prefix? prefix name)
  (match (Tuple prefix name)
    ((Tuple Nil (Cons _ _)) #t)
    ((Tuple (Cons p ps) (Cons n ns)) (and (= p n) (proper-prefix? ps ns)))
    (_ #f)))

(: (-> int (List Symbol) (List Symbol)))
(defun (drop-segments n name)
  (match name
    ((Cons _ rest) (if (> n 0) (drop-segments (- n 1) rest) name))
    (Nil Nil)))

(:doc find-module-source
  (summary "The .bjo a module name names, whether or not it exists.")
  (arg roots "The packages to look in, standard ones included.")
  (arg name "The module: (std random).")
  (returns "The source path, or None when no package owns the name.")
  (see find-module-assembly standard-roots))
(: (-> (List PackageRoot) (List Symbol) (Option string)))
(defun (find-module-source roots name)
  (def owner
    (loop (:for r roots)
          (:acc best
                (folding None
                         (if (and (proper-prefix? (record-ref r name) name)
                                  (match best
                                    (None #t)
                                    ((Some b) (> (list-length (record-ref r name))
                                                 (list-length (record-ref b name))))))
                             (Some r)
                             best)))))
  (match owner
    (None None)
    ((Some r)
     (def below (list-map symbol->string (drop-segments (list-length (record-ref r name)) name)))
     (Some (path-absolute (str (path-combine (record-ref r directory) (string-join (list->vec below) "/")) ".bjo"))))))

(:doc find-module-assembly
  (summary "The compiled dll of a module.")
  (arg roots "The packages to look in, standard ones included.")
  (arg name "The module: (std random).")
  (returns "The dll beside the module's source, or why there is none.")
  (see find-module-source))
(: (-> (List PackageRoot) (List Symbol) (Result string string)))
(defun (find-module-assembly roots name)
  (match (find-module-source roots name)
    (None (Err #"no package provides ${(module-name->string name)}"))
    ((Some source)
     (def dll (clr-change-ext source ".dll"))
     (cond
       ((file-exists? dll) (Ok dll))
       ((file-exists? source) (Err #"${(module-name->string name)} is not compiled: no ${dll}"))
       (else (Err #"no module ${(module-name->string name)}: no ${source}"))))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/array.bjo — whole-array operations on numbers.
;;
;; Each operation is one loop in the runtime (`BjoArray.cs`). The loop uses
;; the CPU's vector instructions when the element is a primitive number, so
;; one instruction does 4 doubles or 8 ints with AVX2. Other numbers, such as
;; `BigInteger`, go through an ordinary loop.
;;
;; Every function that needs arithmetic asks for `Num`, whose CLR constraint is
;; `INumber`. That is what the runtime's kernels ask for, so the C# call is
;; legal in a generic function too.
(import (std prelude))

(:doc #:file "array.bjodoc")

(export
  array-sum array-dot array-min array-max
  array-add! array-scale!
  array-count array-index-of
  array-fill!)

(import/extern
  (clr-sum      (: Bjolang.Runtime.BjoArray.Sum          (-> (Array %a) %a)))
  (clr-dot      (: Bjolang.Runtime.BjoArray.Dot          (-> (Array %a) (Array %a) %a)))
  (clr-min      (: Bjolang.Runtime.BjoArray.Min          (-> (Array %a) %a)))
  (clr-max      (: Bjolang.Runtime.BjoArray.Max          (-> (Array %a) %a)))
  (clr-add!     (: Bjolang.Runtime.BjoArray.AddInPlace   (-> (Array %a) (Array %a) void)))
  (clr-scale!   (: Bjolang.Runtime.BjoArray.ScaleInPlace (-> (Array %a) %a void)))
  (clr-count    (: Bjolang.Runtime.BjoArray.Count        (-> (Array %a) %a int)))
  (clr-index-of (: Bjolang.Runtime.BjoArray.IndexOf      (-> (Array %a) %a int)))
  (clr-fill!    (: Bjolang.Runtime.BjoArray.Fill         (-> (Array %a) %a void))))

;; ---------------------------------------------------------------------------
;; Reductions
;; ---------------------------------------------------------------------------

(: (-> (Array %a) %a) (where (Num %a)))
(defun (array-sum a) (clr-sum a))

(: (-> (Array %a) (Array %a) %a) (where (Num %a)))
(defun (array-dot a b) (clr-dot a b))

(: (-> (Array %a) %a) (where (Num %a)))
(defun (array-min a) (clr-min a))

(: (-> (Array %a) %a) (where (Num %a)))
(defun (array-max a) (clr-max a))

;; ---------------------------------------------------------------------------
;; In place
;; ---------------------------------------------------------------------------

(: (-> (Array %a) (Array %a) void) (where (Num %a)))
(defun (array-add! dst src) (clr-add! dst src))

(: (-> (Array %a) %a void) (where (Num %a)))
(defun (array-scale! a k) (clr-scale! a k))

;; Any element type: `Array.Fill` needs no arithmetic.
(: (-> (Array %a) %a void))
(defun (array-fill! a x) (clr-fill! a x))

;; ---------------------------------------------------------------------------
;; Search
;; ---------------------------------------------------------------------------

;; Equality is the number's `==`, as `=` is. `Eq` is not asked for: the kernel
;; cannot call a Bjolang implementation.
(: (-> (Array %a) %a int) (where (Num %a)))
(defun (array-count a x) (clr-count a x))

;; -1 when no element matches, as `mutablevec-index-of` answers.
(: (-> (Array %a) %a int) (where (Num %a)))
(defun (array-index-of a x) (clr-index-of a x))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/atomic.bjo — a cell that threads can read and write without a lock.
;;
;; An `(Atomic T)` is a `Bjolang.Runtime.Atomic<T>`. The runtime chooses the
;; class for T when the atomic is made: one class for each of int, uint, long
;; and ulong, `AtomicBox<T>` for other reference types and
;; `AtomicStructBox<T>` for other value types. See BjolangRuntime/BjoAtomic.cs.
;;
;; The integer operations are the methods of the trait `AtomicInteger`. A call
;; at a known integer type is inlined, and becomes a direct call to
;; `Interlocked`.
(import (std prelude))

(:doc #:file "atomic.bjodoc")

(export
  Atomic
  make-atomic atomic-ref atomic-set! atomic-swap! atomic-compare-and-swap!
  atomic-update!
  AtomicInteger atomic-increment! atomic-decrement! atomic-add! atomic-sub!
  atomic-array-increment!)

(import/class
  ((Atomic %a) (: Bjolang.Runtime.Atomic)))

;; ---------------------------------------------------------------------------
;; Any T
;; ---------------------------------------------------------------------------

(import/extern
  (make-atomic              (: Bjolang.Runtime.AtomicModule.Make           (-> %a (Atomic %a))))
  (atomic-ref               (: Bjolang.Runtime.AtomicModule.Ref            (-> (Atomic %a) %a)))
  (atomic-set!              (: Bjolang.Runtime.AtomicModule.Set            (-> (Atomic %a) %a void)))
  (atomic-swap!             (: Bjolang.Runtime.AtomicModule.Swap           (-> (Atomic %a) %a %a)))
  (atomic-compare-and-swap! (: Bjolang.Runtime.AtomicModule.CompareAndSwap (-> (Atomic %a) %a %a %a)))
  (atomic-update!           (: Bjolang.Runtime.AtomicModule.Update         (-> (Atomic %a) (-> %a %a) %a))))

;; ---------------------------------------------------------------------------
;; Integers
;; ---------------------------------------------------------------------------

;; TODO: make this trait closed once Bjolang supports closed traits.
(def/trait (AtomicInteger %a)
  (: atomic-increment! (-> (Atomic %a) %a))
  (: atomic-decrement! (-> (Atomic %a) %a))
  (: atomic-add! (-> (Atomic %a) %a %a))
  (: atomic-sub! (-> (Atomic %a) %a %a)))

(import/extern
  (int-increment!   (: Bjolang.Runtime.AtomicModule.Increment (-> (Atomic int) int)))
  (int-decrement!   (: Bjolang.Runtime.AtomicModule.Decrement (-> (Atomic int) int)))
  (int-add!         (: Bjolang.Runtime.AtomicModule.Add       (-> (Atomic int) int int)))
  (int-sub!         (: Bjolang.Runtime.AtomicModule.Sub       (-> (Atomic int) int int)))
  (uint-increment!  (: Bjolang.Runtime.AtomicModule.Increment (-> (Atomic uint) uint)))
  (uint-decrement!  (: Bjolang.Runtime.AtomicModule.Decrement (-> (Atomic uint) uint)))
  (uint-add!        (: Bjolang.Runtime.AtomicModule.Add       (-> (Atomic uint) uint uint)))
  (uint-sub!        (: Bjolang.Runtime.AtomicModule.Sub       (-> (Atomic uint) uint uint)))
  (long-increment!  (: Bjolang.Runtime.AtomicModule.Increment (-> (Atomic long) long)))
  (long-decrement!  (: Bjolang.Runtime.AtomicModule.Decrement (-> (Atomic long) long)))
  (long-add!        (: Bjolang.Runtime.AtomicModule.Add       (-> (Atomic long) long long)))
  (long-sub!        (: Bjolang.Runtime.AtomicModule.Sub       (-> (Atomic long) long long)))
  (ulong-increment! (: Bjolang.Runtime.AtomicModule.Increment (-> (Atomic ulong) ulong)))
  (ulong-decrement! (: Bjolang.Runtime.AtomicModule.Decrement (-> (Atomic ulong) ulong)))
  (ulong-add!       (: Bjolang.Runtime.AtomicModule.Add       (-> (Atomic ulong) ulong ulong)))
  (ulong-sub!       (: Bjolang.Runtime.AtomicModule.Sub       (-> (Atomic ulong) ulong ulong))))

(impl (AtomicInteger int)
  (defun (atomic-increment! a) (int-increment! a))
  (defun (atomic-decrement! a) (int-decrement! a))
  (defun (atomic-add! a n) (int-add! a n))
  (defun (atomic-sub! a n) (int-sub! a n)))

(impl (AtomicInteger uint)
  (defun (atomic-increment! a) (uint-increment! a))
  (defun (atomic-decrement! a) (uint-decrement! a))
  (defun (atomic-add! a n) (uint-add! a n))
  (defun (atomic-sub! a n) (uint-sub! a n)))

(impl (AtomicInteger long)
  (defun (atomic-increment! a) (long-increment! a))
  (defun (atomic-decrement! a) (long-decrement! a))
  (defun (atomic-add! a n) (long-add! a n))
  (defun (atomic-sub! a n) (long-sub! a n)))

(impl (AtomicInteger ulong)
  (defun (atomic-increment! a) (ulong-increment! a))
  (defun (atomic-decrement! a) (ulong-decrement! a))
  (defun (atomic-add! a n) (ulong-add! a n))
  (defun (atomic-sub! a n) (ulong-sub! a n)))

;; ---------------------------------------------------------------------------
;; Elements of an array
;; ---------------------------------------------------------------------------

;; Element `i` of the array plus 1, as one atomic step; the new value. For
;; counts kept in an array that several threads add to.
(import/extern
  (atomic-array-increment! (: Bjolang.Runtime.AtomicModule.IncrementAt (-> (Array int) int int))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/bench.bjo — benchmarks.
;;
;;     (import (std prelude))
;;     (import (std bench))
;;
;;     (def/bench "list-sort"
;;       #:params ((n [1000 100000]))
;;       #:setup ((xs (random-list n)))
;;       (list-sort xs))
;;
;;     (run-benches)
;;
;; `def/bench` registers a benchmark where it is written, and `(run-benches)`
;; is the file's `main`, as `deftest` and `(run-tests)` are. A run warms each
;; benchmark up until its times are steady, calibrates how many calls make a
;; sample, takes samples until the median is known well enough, and prints a
;; table: the median per call, its spread, the minimum, allocation, and
;; collections. See `Docs/Benchmark-design.org`, which this follows.
;;
;; The statistics are `(std bench stats)`; the runtime's part, the sink and
;; the collector's counters, is `BjoBench`.

(export def/bench run-benches time-it
        Bench BenchCase CaseRun BenchOptions Measured Settings BenchParam bench-param-read
        registered-benches module-benches parse-bench-args measure-case bench-params-set!
        bench-settings time-text bytes-text
        Selection parse-choice bench-menu menu-loop menu-lines run-selection run-and-report run-with-options bench-serve
        print-comparison-table comparisons-text fails-slower? header-lines header-facts this-run
        sel-bench sel-cases case-labels
        ;; Named by the templates of the macros, so they have to be reachable
        ;; where those expand.
        register-bench! bench-keep! bench-param-values bench-now bench-allocated
        run-registered-benches time-it-run time-it-fixed time-it-expansion time-it-keep)

;; Every export is documented there; the module doc is too.
(:doc #:file "bench.bjodoc")

(import (std prelude))
(import (std maths))
(import (std syntax-match))
(import (std bench stats))
(import (std fmt))
(import (std bench store))
(import (std random))
(import (text bjodat-core))

(import/extern
  (bench-keep!      (: Bjolang.Runtime.BjoBench.Consume (-> %a void)))
  (collect!         (: Bjolang.Runtime.BjoBench.Collect (-> void)))
  (gc-count         (: Bjolang.Runtime.BjoBench.Collections (-> int int)))
  (allocated-thread  (: Bjolang.Runtime.BjoBench.AllocatedOnThread (-> long)))
  (allocated-process (: Bjolang.Runtime.BjoBench.AllocatedInProcess (-> long)))
  (cpu-model        (: Bjolang.Runtime.BjoBench.CpuModel (-> string)))
  (core-count       (: Bjolang.Runtime.BjoBench.Cores (-> int)))
  (os-name          (: Bjolang.Runtime.BjoBench.OperatingSystemName (-> string)))
  (arch-name        (: Bjolang.Runtime.BjoBench.Architecture (-> string)))
  (dotnet-version   (: Bjolang.Runtime.BjoBench.DotnetVersion (-> string)))
  (server-gc?       (: Bjolang.Runtime.BjoBench.ServerGc (-> bool)))
  (governor         (: Bjolang.Runtime.BjoBench.Governor (-> string)))
  (debugger?        (: Bjolang.Runtime.BjoBench.DebuggerAttached (-> bool)))
  (debug-build?     (: Bjolang.Runtime.BjoBench.DebugBuild (-> bool)))
  (output-terminal? (: Bjolang.Runtime.BjoBench.OutputIsTerminal (-> bool)))
  (input-terminal?  (: Bjolang.Runtime.BjoBench.InputIsTerminal (-> bool)))
  (machine-key      (: Bjolang.Runtime.BjoBench.MachineKey (-> string)))
  (now-utc          (: Bjolang.Runtime.BjoBench.NowUtc (-> string)))
  (set-exit-code!   (: System.Environment.ExitCode (-> int void) #:set)))

;; ---------------------------------------------------------------------------
;; Small helpers
;; ---------------------------------------------------------------------------

(: (-> string (Option int)))
(defun (parse-int s)
  (match (try (string->int s) #:catch (System.Exception))
    ((Ok n) (Some n))
    (_ None)))

(: (-> string (Option long)))
(defun (parse-long s)
  (match (try (cast System.Int64 (string->bigint s)) #:catch (System.Exception))
    ((Ok n) (Some n))
    (_ None)))

(: (-> string (Option double)))
(defun (parse-double s)
  (match (try (string->double s) #:catch (System.Exception))
    ((Ok n) (Some n))
    (_ None)))

;; The text before and after the first `sep`, when it is there and has text
;; before it.
(: (-> string string (Option (Tuple string string))))
(defun (split-once s sep)
  (match (string-search s sep)
    ((Some at) #:when (not (string-cursor=? at (string-cursor-start s)))
     (Some (Tuple (substring s (string-cursor-start s) at)
                  (substring s (skip-chars s at (string-count sep)) (string-cursor-end s)))))
    (_ None)))

(: (-> string StringCursor int StringCursor))
(defun (skip-chars s at n)
  (if (= n 0) at (skip-chars s (string-cursor-next s at) (- n 1))))

(: (-> (List %a) (List %b) (List (Tuple %a %b))))
(defun (list-zip xs ys)
  (match (Tuple xs ys)
    ((Tuple (Cons x xm) (Cons y ym)) (cons (Tuple x y) (list-zip xm ym)))
    (_ (list))))

(: (-> int (List int)))
(defun (index-list n) {listing i (:for i (range 0 n))})

;; The clock, read as a stopwatch reads it, so that a test can bind
;; `current-clock` and get exact numbers.
(: (-> long))
(defun (bench-now) (monotonic-ns))

;; The sink as an ordinary function, for `time-it`: `(std stopwatch)`
;; re-exports what its expansion names, and a foreign import cannot be.
(: (-> %a void))
(defun (time-it-keep x) (bench-keep! x))

;; What `alloc/op` counts when a benchmark does not say: "thread" or
;; "process". `--alloc` sets it.
(: (Box string))
(def alloc-default (make-box "thread"))

;; A benchmark's `#:alloc`, or the run's default for "".
(: (-> string string))
(defun (alloc-counted mode)
  (if (string-empty? mode) (box-ref alloc-default) mode))

;; The bytes allocated so far, as `mode` counts them:
;;
;; - "thread": this thread's, exact, and blind to the work a body hands to
;;   fibers or the pool.
;; - "process": every thread's, also the JIT's background compiles and
;;   timers, so noisier. The read is precise, and not free.
;; - "": the run's default.
(: (-> string long))
(defun (bench-allocated mode)
  (if (= (alloc-counted mode) "process") (allocated-process) (allocated-thread)))

;; ---------------------------------------------------------------------------
;; Benchmarks and their registry
;; ---------------------------------------------------------------------------

;; What running one case comes to: `batch` runs the body `k` times; `call`,
;; for a `#:setup-every` benchmark, runs the setup and then one timed call, and
;; answers the call's nanoseconds and the bytes it allocated, the setup's
;; left out of both.
(type (: CaseRun (Record (: batch (-> long void))
                         (: call (Option (-> (Tuple long long))))
                         ;; The benchmark's `#:alloc`: "thread", "process", or
                         ;; "" for the run's default.
                         (: alloc string))))

;; One row: a benchmark at one combination of its parameters. `labels` are the
;; parameters' names and values as the report shows them; `prepare` runs the
;; setup and answers what to run.
(type (: BenchCase (Record (: labels (List (Tuple string string)))
                           (: prepare (-> CaseRun)))))

;; A registered benchmark. `once` is the number of cold runs of a `#:once`
;; benchmark, and 0 for a calibrated one. `cases` makes its rows; it is a
;; function so that the parameters' values are read when the run starts,
;; after the flags that may replace them.
(type (: Bench (Record (: name string)
                       (: module string)
                       (: file string)
                       (: line int)
                       (: group string)
                       (: skip string)
                       (: once int)
                       ;; Its `#:alloc`, or "".
                       (: alloc string)
                       (: param-names (List string))
                       (: cases (-> (List BenchCase))))))

;; Newest first, as `(std test)`'s registry.
(: (Box (List Bench)))
(def bench-registry (make-box (list)))

(: (-> string string string int string string int string (List string) (-> (List BenchCase)) int))
(defun (register-bench! name module file line group skip once alloc param-names cases)
  (box-set! bench-registry
            (cons (Bench (name name) (module module) (file file) (line line) (group group)
                         (skip skip) (once once) (alloc alloc) (param-names param-names) (cases cases))
                  (box-ref bench-registry)))
  0)

(: (-> (List Bench)))
(defun (registered-benches) (list-reverse (box-ref bench-registry)))

(: (-> string (List Bench)))
(defun (module-benches module)
  (list-filter (fun (b) (= (record-ref b module) module)) (registered-benches)))

;; ---------------------------------------------------------------------------
;; Parameters
;; ---------------------------------------------------------------------------

;; `--params n=10,100` replaces a parameter's values for the run, as text,
;; read in the parameter's own type.
(: (Box (Map string (Vec string))))
(def param-overrides (make-box (map-empty)))

;; How a parameter's value is read from `--params`.
(def/trait (BenchParam %a)
  (: bench-param-read (-> string (Option %a))))

(impl (BenchParam int) (defun (bench-param-read s) (parse-int s)))
(impl (BenchParam long) (defun (bench-param-read s) (parse-long s)))
(impl (BenchParam double) (defun (bench-param-read s) (parse-double s)))
(impl (BenchParam string) (defun (bench-param-read s) (Some s)))
(impl (BenchParam Symbol) (defun (bench-param-read s) (Some (string->symbol (string-trim-quote s)))))
(impl (BenchParam bool)
  (defun (bench-param-read s)
    (case s
      (("#t" "true") (Some #t))
      (("#f" "false") (Some #f))
      (else None))))

;; A symbol may be written 'sorted or sorted.
(: (-> string string))
(defun (string-trim-quote s)
  (if (string-starts-with? s "'")
      (substring s (string-cursor-next s (string-cursor-start s)) (string-cursor-end s))
      s))

;; Replaces parameters' values for the benchmarks run after it, as
;; `--params` does.
(: (-> (Map string (Vec string)) void))
(defun (bench-params-set! overrides) (box-set! param-overrides overrides))

;; The values of parameter `name`: its own, or those `--params` gave.
(: (-> string (Vec %a) (Vec %a)) (where (BenchParam %a)))
(defun (bench-param-values name values)
  (match (map-try-ref (box-ref param-overrides) name)
    (None values)
    ((Some texts)
     (vec-map (fun (t)
                (match (bench-param-read t)
                  ((Some v) v)
                  (None (panic! #"--params: ${name}=${t} is not a value of the parameter's type."))))
              texts))))

;; ---------------------------------------------------------------------------
;; def/bench
;; ---------------------------------------------------------------------------

;; What the options of one `def/bench` came to. The setups are bindings,
;; `(name expr)`, made in order as `let*` makes them.
(type (: BenchSyntax (Record (: setup (List Syntax))
                             (: setup-every (List Syntax))
                             (: params (List (Tuple Syntax Syntax)))
                             (: once Syntax)
                             (: group Syntax)
                             (: skip Syntax)
                             (: alloc Syntax)
                             (: body (List Syntax)))))

(: (-> BenchSyntax))
(defun (no-bench-options)
  (BenchSyntax (setup (list)) (setup-every (list)) (params (list)) (once #'0) (group #'"")
               (skip #'"") (alloc #'"") (body (list))))

(: (-> Syntax (List Syntax) BenchSyntax BenchSyntax))
(defun (bench-options form items o)
  (let ((go (fun (more o2) (bench-options form more o2))))
    (syntax-match (SList items)
      ((#:setup ((name e) ...) more ...)
       (go more (record-set o (setup (zip-bindings name e)))))
      ((#:setup bad more ...)
       (ignore (syntax-error bad "#:setup takes bindings, made once before the benchmark runs: #:setup ((deck (full-deck))). The body then names them."))
       o)
      ((#:setup-every ((name e) ...) more ...)
       (go more (record-set o (setup-every (zip-bindings name e)))))
      ((#:setup-every bad more ...)
       (ignore (syntax-error bad "#:setup-every takes bindings, made again before every call: #:setup-every ((cards (vec->list deck)))."))
       o)
      ((#:params ((name values) ...) more ...)
       (go more (record-set o (params (zip-syntax name values)))))
      ((#:once n more ...) (go more (record-set o (once n))))
      ((#:group g more ...) (go more (record-set o (group g))))
      ((#:skip why more ...) (go more (record-set o (skip why))))
      ((#:alloc mode more ...)
       (case (syntax->string mode)
         ("'thread" (go more (record-set o (alloc #'"thread"))))
         ("'process" (go more (record-set o (alloc #'"process"))))
         (else
          (ignore (syntax-error mode "#:alloc takes 'thread or 'process: what alloc/op counts, this thread's allocation or every thread's."))
          o)))
      ((first more ...) (record-set o (body (cons first more))))
      (_ (ignore (syntax-error form "def/bench takes a name and a body: (def/bench \"name\" body ...), with options such as #:setup, #:setup-every, #:params, #:once, #:group, #:skip and #:alloc after the name."))
         o))))

(: (-> (List Syntax) (List Syntax) (List (Tuple Syntax Syntax))))
(defun (zip-syntax xs ys)
  (match (Tuple xs ys)
    ((Tuple (Cons x xm) (Cons y ym)) (cons (Tuple x y) (zip-syntax xm ym)))
    (_ (list))))

(: (-> (List Syntax) (List Syntax) (List Syntax)))
(defun (zip-bindings names values)
  (list-map (fun (p) (match p ((Tuple n v) #'(,n ,v)))) (zip-syntax names values)))

;; `prepare` of one case: the setup's bindings, made once, and the runs. The
;; parameters are bound already, by the loop the cases are made in, so the
;; setup and the body name them as they are.
(: (-> Syntax BenchSyntax Syntax))
(defun (prepare-expansion form o)
  (def body #'(begin ,@(record-ref o body)))
  (def setup (SList (record-ref o setup)))
  (def alloc (record-ref o alloc))
  (match (record-ref o setup-every)
    (Nil
     #'(fun ()
         (let* ,setup
           (CaseRun (batch (fun (k)
                             (let lp ((i 0L))
                               (when (< i k)
                                 (bench-keep! ,body)
                                 (lp (+ i 1L))))))
                    (call None)
                    (alloc ,alloc)))))
    (every-bindings
     ;; Made again before every call, outside the time and the allocation:
     ;; each call is timed on its own.
     (def every (SList every-bindings))
     #'(fun ()
         (let* ,setup
           (CaseRun (batch (fun (k)
                             (let lp ((i 0L))
                               (when (< i k)
                                 (let* ,every (bench-keep! ,body))
                                 (lp (+ i 1L))))))
                    (call (Some (fun ()
                                  (let* ,every
                                    (let* ((b0 (bench-allocated ,alloc))
                                           (t0 (bench-now)))
                                      (bench-keep! ,body)
                                      (let ((t1 (bench-now)))
                                        (Tuple (- t1 t0) (- (bench-allocated ,alloc) b0))))))))
                    (alloc ,alloc)))))))

;; The cases: one per combination of the parameters' values, the last varying
;; fastest.
(: (-> Syntax BenchSyntax Syntax))
(defun (cases-expansion form o)
  (def params (record-ref o params))
  (def names (list-map (fun (p) (match p ((Tuple n _) n))) params))
  (def case-form
    #'(BenchCase (labels (list ,@(list-map (fun (n) #'(Tuple ,(SStr (syntax->string n)) (displayed ,n))) names)))
                 (prepare ,(prepare-expansion form o))))
  (if (list-empty? params)
      #'(fun () (list ,case-form))
      (let ()
        (def fors
          (list-map (fun (p)
                      (match p
                        ((Tuple n vs) #'(:for ,n (bench-param-values ,(SStr (syntax->string n)) ,vs)))))
                    params))
        ;; `(:subloop)` between the drivers, so that each is a level of its own
        ;; and they nest rather than walk in step.
        (def clauses (interleave-subloop fors))
        ;; One unnamed accumulator is the loop's value: a `=>` written in a
        ;; template would be renamed by hygiene, and no longer read as one.
        #'(fun ()
            (loop ,@clauses
                  (:acc (listing ,case-form)))))))

(: (-> (List Syntax) (List Syntax)))
(defun (interleave-subloop fors)
  (match fors
    (Nil (list))
    ((Cons f Nil) (list f))
    ((Cons f more) (cons f (cons #'(:subloop) (interleave-subloop more))))))

(: (-> Syntax Syntax BenchSyntax Syntax))
(defun (def-bench-expansion form name o)
  (when (list-empty? (record-ref o body))
    (ignore (syntax-error form "def/bench has no body: (def/bench \"name\" body ...).")))
  (def param-names (list-map (fun (p) (match p ((Tuple n _) (SStr (syntax->string n))))) (record-ref o params)))
  #'(def registered
      (register-bench! ,name (current-module) ,(SStr (syntax-file form)) ,(SInt (int->string (syntax-line form)))
                       ,(record-ref o group) ,(record-ref o skip) ,(record-ref o once) ,(record-ref o alloc)
                       (list ,@param-names)
                       ,(cases-expansion form o))))

;; `(def/bench "name" body ...)`, with options after the name in any order:
;; `#:setup ((name e) ...)`, `#:setup-every ((name e) ...)`,
;; `#:params ((p values) ...)`, `#:once n`, `#:group "g"`, `#:skip "why"` and
;; `#:alloc 'process`.
;; The body is forms, which the parameters and the setups' names are in
;; scope for. Registers the benchmark; `(run-benches)` runs it.
(def/macro (def/bench form inject compare)
  (syntax-match form
    ((_ name first more ...)
     (def-bench-expansion form name (bench-options form (cons first more) (no-bench-options))))
    (bad (syntax-error bad "def/bench takes a name and a body: (def/bench \"name\" body ...)."))))

;; ---------------------------------------------------------------------------
;; The options of a run
;; ---------------------------------------------------------------------------

;; What the arguments of a benchmark program asked for. `scale` multiplies the
;; time targets: 0.5 for `--quick`, 4 for `--long`.
(type (: BenchOptions (Record (: filters (Vec string))
                              (: all bool)
                              (: list bool)
                              (: scale double)
                              (: min-samples int)
                              (: max-time-ns long)
                              (: detail bool)
                              (: params (Map string (Vec string)))
                              ;; Saving and comparing.
                              (: save (Option string))
                              (: compare (Option string))
                              (: alpha double)
                              ;; A fraction: 0.02 for 2%.
                              (: threshold double)
                              (: p-raw bool)
                              (: fail-slower (Option double))
                              ;; Driven over standard input and output.
                              (: serve bool)
                              ;; Exactly these cases, numbered as `--serve`'s
                              ;; `(list)` numbers them: what `bjo bench` asks
                              ;; for after a menu over several files.
                              (: cases (Option (Vec int)))
                              ;; How time grows with a numeric parameter.
                              (: scaling bool)
                              ;; The cases of each `#:group` compared, in
                              ;; `rounds` interleaved rounds shuffled from
                              ;; `seed`.
                              (: compare-within bool)
                              (: rounds int)
                              (: seed (Option ulong))
                              ;; What alloc/op counts where a benchmark does
                              ;; not say: "thread" or "process".
                              (: alloc string))))

;; `2`, `2%` or `0.5` as a fraction of one: a percentage.
(: (-> string (Option double)))
(defun (parse-percent s)
  (def digits (if (string-ends-with? s "%")
                  (substring s (string-cursor-start s) (string-cursor-prev s (string-cursor-end s)))
                  s))
  (match (parse-double digits)
    ((Some x) #:when (>= x 0.0) (Some (/ x 100.0)))
    (_ None)))

(: (-> (Vec string) (Result string BenchOptions)))
(defun (parse-bench-args args)
  (let go ((left (vec->list args)) (o (BenchOptions (filters (vec-empty)) (all #f) (list #f) (scale 1.0)
                                                    (min-samples 20) (max-time-ns 0L) (detail #f)
                                                    (params (map-empty))
                                                    (save None) (compare None) (alpha 0.05)
                                                    (threshold 0.02) (p-raw #f) (fail-slower None) (serve #f)
                                                    (cases None) (scaling #f) (compare-within #f)
                                                    (rounds 20) (seed None) (alloc "thread"))))
    (defun (value-of flag more k)
      (match more
        ((Cons v after) (k v after))
        (Nil (Err #"${flag} needs a value."))))
    (match left
      (Nil (Ok o))
      ((Cons a more)
       (case a
         ("--all" (go more (record-set o (all #t))))
         ("--list" (go more (record-set o (list #t))))
         ("--quick" (go more (record-set o (scale 0.5))))
         ("--long" (go more (record-set o (scale 4.0))))
         ("--detail" (go more (record-set o (detail #t))))
         ("--p-raw" (go more (record-set o (p-raw #t))))
         ("--serve" (go more (record-set o (serve #t))))
         ("--scaling" (go more (record-set o (scaling #t))))
         ("--compare-within" (go more (record-set o (compare-within #t))))
         ("-N"
          (value-of a more
            (fun (v after)
              (match (parse-int v)
                ((Some n) #:when (> n 1) (go after (record-set o (rounds n))))
                (_ (Err #"-N takes a number of rounds above 1: ${v}"))))))
         ("--seed"
          (value-of a more
            (fun (v after)
              (match (parse-long v)
                ((Some n) (go after (record-set o (seed (Some (cast System.UInt64 n))))))
                (_ (Err #"--seed takes a whole number: ${v}"))))))
         ("--cases"
          (value-of a more
            (fun (v after)
              (match (all-ints (string-split v ","))
                ((Some is) (go after (record-set o (cases (Some is)))))
                (None (Err #"--cases takes case numbers, 0,3,4: ${v}"))))))
         ("--alloc"
          (value-of a more
            (fun (v after)
              (if (or (= v "thread") (= v "process"))
                  (go after (record-set o (alloc v)))
                  (Err #"--alloc takes thread or process: ${v}")))))
         ("--save" (value-of a more (fun (v after) (go after (record-set o (save (Some v)))))))
         ("--compare" (value-of a more (fun (v after) (go after (record-set o (compare (Some v)))))))
         ("--alpha"
          (value-of a more
            (fun (v after)
              (match (parse-double v)
                ((Some p) #:when (and (> p 0.0) (< p 1.0)) (go after (record-set o (alpha p))))
                (_ (Err #"--alpha takes a level between 0 and 1: ${v}"))))))
         ("--threshold"
          (value-of a more
            (fun (v after)
              (match (parse-percent v)
                ((Some t) (go after (record-set o (threshold t))))
                (_ (Err #"--threshold takes a percentage: ${v}"))))))
         ("--fail-slower"
          (value-of a more
            (fun (v after)
              (match (parse-percent v)
                ((Some t) (go after (record-set o (fail-slower (Some t)))))
                (_ (Err #"--fail-slower takes a percentage: ${v}"))))))
         ("--min-samples"
          (value-of a more
            (fun (v after)
              (match (parse-int v)
                ((Some n) #:when (> n 1) (go after (record-set o (min-samples n))))
                (_ (Err #"--min-samples takes a number above 1: ${v}"))))))
         ("--max-time"
          (value-of a more
            (fun (v after)
              (match (parse-double v)
                ((Some s) #:when (> s 0.0)
                 (go after (record-set o (max-time-ns (cast System.Int64 (* s 1e9))))))
                (_ (Err #"--max-time takes seconds: ${v}"))))))
         ("--params"
          (value-of a more
            (fun (v after)
              (match (parse-param-override v)
                ((Ok (Tuple name values))
                 (go after (record-set o (params (map-set (record-ref o params) name values)))))
                ((Err e) (Err e))))))
         (else
          (if (string-starts-with? a "--")
              (Err #"unknown flag ${a}. The flags are --all, --list, --quick, --long, --detail, --min-samples N, --max-time S, --params P=V,..., --save FILE, --compare FILE, --alpha P, --threshold PCT, --p-raw, --fail-slower PCT, --serve, --cases I,J,..., --scaling, --compare-within, -N ROUNDS, --seed N and --alloc thread|process")
              (go more (record-set o (filters (vec-add (record-ref o filters) a)))))))))))

;; Every text a number, or none of them.
(: (-> (Vec string) (Option (Vec int))))
(defun (all-ints texts)
  (def parsed (vec-map parse-int texts))
  (if (vec-iter some? parsed)
      (Some (vec-map (fun (x) (match x ((Some n) n) (None 0))) parsed))
      None))

;; `n=10,100` as `n` and its values.
(: (-> string (Result string (Tuple string (Vec string)))))
(defun (parse-param-override s)
  (match (split-once s "=")
    ((Some (Tuple name values)) (Ok (Tuple name (string-split values ","))))
    (None (Err #"--params takes NAME=VALUE,VALUE...: ${s}"))))

;; The targets of a run, in nanoseconds, from its options.
(type (: Settings (Record (: sample-ns long)
                          (: time-ns long)
                          (: warmup-ns long)
                          (: max-ns long)
                          (: min-samples int)
                          (: max-samples int))))

(: (-> BenchOptions Settings))
(defun (bench-settings o)
  (def scale (record-ref o scale))
  (def time-ns (cast System.Int64 (* 2e9 scale)))
  (Settings (sample-ns 10000000L)
            (time-ns time-ns)
            (warmup-ns (cast System.Int64 (* 1e9 scale)))
            (max-ns (if (> (record-ref o max-time-ns) 0L) (record-ref o max-time-ns) (* 3L time-ns)))
            (min-samples (record-ref o min-samples))
            (max-samples (max 100 (record-ref o min-samples)))))

;; ---------------------------------------------------------------------------
;; Measuring
;; ---------------------------------------------------------------------------

;; What measuring one case came to. `samples` are nanoseconds per call, in the
;; order they were taken; for a `#:once` benchmark, one per cold run.
;; `calls` is every call the samples timed, which the collections and the
;; allocation are per.
(type (: Measured (Record (: samples (Vec double))
                          (: k long)
                          (: calls long)
                          (: bytes long)
                          (: gc0 long)
                          (: gc1 long)
                          (: gc2 long)
                          (: steady bool)
                          (: once bool)
                          (: per-call bool))))

;; Where measuring says how far it has come: a phase, such as "warming up" or
;; "sampling 14/20 · median 6.21 ms". Nothing, unless the runner sets it: at a
;; terminal it rewrites one line.
(: (Box (-> string void)))
(def progress-box (make-box (fun (what) unit)))

(: (-> string void))
(defun (progress what) ((box-ref progress-box) what))

;; The nanoseconds of one batch of `k` calls.
(: (-> CaseRun long long))
(defun (time-batch run k)
  (def t0 (bench-now))
  ((record-ref run batch) k)
  (- (bench-now) t0))

;; One sample of `k` calls, after a collection: its nanoseconds, bytes and
;; collections.
(type (: Sample (Record (: ns long) (: bytes long) (: gc0 long) (: gc1 long) (: gc2 long))))

(: (-> CaseRun long Sample))
(defun (take-sample run k)
  (collect!)
  (def g0 (gc-count 0))
  (def g1 (gc-count 1))
  (def g2 (gc-count 2))
  (def alloc (record-ref run alloc))
  (def b0 (bench-allocated alloc))
  (def (Tuple ns bytes)
    (match (record-ref run call)
      ;; `#:setup-every`: each call on its own, its setup outside the time
      ;; and the allocation.
      ((Some call)
       (let go ((i 0L) (total 0L) (bytes 0L))
         (if (< i k)
             (match (call) ((Tuple t b) (go (+ i 1L) (+ total t) (+ bytes b))))
             (Tuple total bytes))))
      (None (let ((t (time-batch run k))) (Tuple t (- (bench-allocated alloc) b0))))))
  (Sample (ns ns) (bytes bytes)
          (gc0 (cast System.Int64 (- (gc-count 0) g0)))
          (gc1 (cast System.Int64 (- (gc-count 1) g1)))
          (gc2 (cast System.Int64 (- (gc-count 2) g2)))))

(: (-> long long double))
(defun (per-call-ns ns k) (/ (cast System.Double ns) (cast System.Double k)))

;; The batch size whose batch takes `target` nanoseconds: start at 1, and
;; multiply by what the last batch says is missing, at most by 10.
(: (-> CaseRun long long))
(defun (calibrate run target)
  (let go ((k 1L))
    (def t (time-batch run k))
    (cond
      ((>= t target) k)
      ((> k 1000000000000L) k)
      (else
       (def factor
         (if (<= t 0L)
             10L
             (min 10L (max 2L (cast System.Int64 (ceiling (/ (* 1.1 (cast System.Double target)) (cast System.Double t))))))))
       (go (* k factor))))))

;; Runs batches of `k` until the medians of the last two windows of five
;; batches agree within 2%, or the warm-up has had `limit` nanoseconds.
;; Answers whether it got steady.
(: (-> CaseRun long long bool))
(defun (warm-up run k limit)
  (def started (bench-now))
  (let go ((times (vec-empty)))
    (def t (cast System.Double (time-batch run k)))
    (def now-times (vec-add times t))
    (def n (vec-length now-times))
    (cond
      ;; Not before half the limit: the JIT recompiles a hot method only
      ;; once no new method has needed compiling for 100 ms, and then after
      ;; 30 more calls, so a body can be steady at its first tier for a
      ;; while before it gets faster.
      ((and (>= n 10)
            (>= (- (bench-now) started) (/ limit 2L))
            (let ((recent (median (vec-slice now-times (- n 5) 5)))
                  (before (median (vec-slice now-times (- n 10) 5))))
              (and (> before 0.0) (< (abs (- recent before)) (* 0.02 before)))))
       #t)
      ((>= (- (bench-now) started) limit) #f)
      (else (go now-times)))))

;; The half-width of a 95% interval of the median of sorted samples, from
;; the order statistics around it, as a fraction of the median.
(: (-> (Vec double) double))
(defun (median-precision samples)
  (def n (vec-length samples))
  (if (< n 5)
      1.0
      (let ()
        (def half (* 0.98 (sqrt (cast System.Double n))))
        (def lo (quantile samples (max 0.0 (- 0.5 (/ half (cast System.Double n))))))
        (def hi (quantile samples (min 1.0 (+ 0.5 (/ half (cast System.Double n))))))
        (def m (median samples))
        (if (<= m 0.0) 0.0 (/ (- hi lo) (* 2.0 m))))))

;; Measures one case, a calibrated one or `once` cold runs of one call.
(: (-> BenchCase int Settings Measured))
(defun (measure-case c once s)
  (def run ((record-ref c prepare)))
  (if (> once 0)
      (measure-once run once)
      (measure-calibrated run s)))

(: (-> CaseRun int Measured))
(defun (measure-once run n)
  (def taken
    (loop (:for i (up-from 0 #:to n))
          (:do (progress #"cold run ${(+ i 1)}/${n}"))
          (:acc out (vecing (take-sample run 1L)))
      => out))
  (summed taken 1L #t #t (record-ref run call)))

;; Calibrates to samples of `sample-ns` and warms up for at most `warmup-ns`:
;; the batch size, and whether the times got steady.
(: (-> CaseRun long long (Tuple long bool)))
(defun (prepare-run run sample-ns warmup-ns)
  (progress "calibrating")
  (def k0 (calibrate run sample-ns))
  (progress "warming up")
  (def steady (warm-up run k0 warmup-ns))
  ;; The JIT may have made the body much faster while it warmed up.
  (def k (let ((t (time-batch run k0)))
           (if (and (> t 0L) (< (* 2L t) sample-ns))
               (calibrate-from run k0 sample-ns)
               k0)))
  (Tuple k steady))

(: (-> CaseRun Settings Measured))
(defun (measure-calibrated run s)
  ;; A `#:setup-every` call is timed on its own, so a sample is k such calls,
  ;; and calibration counts them the same way.
  (def (Tuple k steady) (prepare-run run (record-ref s sample-ns) (record-ref s warmup-ns)))
  (def taken
    (let go ((samples (vec-empty)) (spent 0L))
      (def one (take-sample run k))
      (def now (vec-add samples one))
      (def spent2 (+ spent (record-ref one ns)))
      (def n (vec-length now))
      (def times (vec-map (fun (x) (per-call-ns (record-ref x ns) k)) now))
      ;; What is left, at the time the last sample took: the samples still
      ;; owed to the minimum, or the time still owed to the target.
      (def owed-ns (max (* (cast System.Int64 (max 0 (- (record-ref s min-samples) n))) (record-ref one ns))
                        (- (record-ref s time-ns) spent2)))
      (progress (str "sampling " (int->string n) "/" (int->string (record-ref s min-samples))
                     "   median " (time-text (median times))
                     (if (> owed-ns 0L) (str "   ~" (time-text (cast System.Double owed-ns)) " left") "")))
      (cond
        ((>= n (record-ref s max-samples)) now)
        ((>= spent2 (record-ref s max-ns)) now)
        ((or (< n (record-ref s min-samples)) (< spent2 (record-ref s time-ns))) (go now spent2))
        ;; Enough of both: more only while the median is not known to ±1%.
        ((> (median-precision times) 0.01) (go now spent2))
        (else now))))
  (summed taken k steady #f (record-ref run call)))

(: (-> CaseRun long long long))
(defun (calibrate-from run k target)
  (let go ((k k))
    (def t (time-batch run k))
    (if (or (>= t target) (> k 1000000000000L))
        k
        (go (* k (if (<= t 0L) 10L (min 10L (max 2L (cast System.Int64 (ceiling (/ (* 1.1 (cast System.Double target)) (cast System.Double t))))))))))))

(: (-> (Vec Sample) long bool bool (Option (-> (Tuple long long))) Measured))
(defun (summed taken k steady once call)
  (Measured (samples (vec-map (fun (x) (per-call-ns (record-ref x ns) k)) taken))
            (k k)
            (calls (* k (cast System.Int64 (vec-length taken))))
            (bytes (vec-fold (fun (x acc) (+ acc (record-ref x bytes))) 0L taken))
            (gc0 (vec-fold (fun (x acc) (+ acc (record-ref x gc0))) 0L taken))
            (gc1 (vec-fold (fun (x acc) (+ acc (record-ref x gc1))) 0L taken))
            (gc2 (vec-fold (fun (x acc) (+ acc (record-ref x gc2))) 0L taken))
            (steady steady)
            (once once)
            (per-call (some? call))))

;; ---------------------------------------------------------------------------
;; Reporting
;; ---------------------------------------------------------------------------

;; `v` with `digits` digits after the point.
(: (-> double int string))
(defun (fixed v digits)
  (def scale (pow 10.0 (cast System.Double digits)))
  (def n (cast System.Int64 (round (* (abs v) scale))))
  (def whole (/ n (cast System.Int64 scale)))
  (def frac (- n (* whole (cast System.Int64 scale))))
  (def sign (if (and (< v 0.0) (> n 0L)) "-" ""))
  (if (= digits 0)
      (str sign (long->string whole))
      (str sign (long->string whole) "." (string-pad-left (long->string frac) digits #:with #\0))))

;; Three significant digits.
(: (-> double string))
(defun (three v)
  (cond ((>= (abs v) 100.0) (fixed v 0))
        ((>= (abs v) 10.0) (fixed v 1))
        (else (fixed v 2))))

;; Nanoseconds, in the unit that reads best.
(: (-> double string))
(defun (time-text ns)
  (cond ((>= ns 1e9) (str (three (/ ns 1e9)) " s"))
        ((>= ns 1e6) (str (three (/ ns 1e6)) " ms"))
        ((>= ns 1e3) (str (three (/ ns 1e3)) " µs"))
        (else (str (three ns) " ns"))))

(: (-> double string))
(defun (bytes-text b)
  (cond ((>= b 1073741824.0) (str (three (/ b 1073741824.0)) " GB"))
        ((>= b 1048576.0) (str (three (/ b 1048576.0)) " MB"))
        ((>= b 1024.0) (str (three (/ b 1024.0)) " KB"))
        (else (str (fixed b 0) " B"))))

;; One row of the report: the cells, and the marks after them.
(type (: Row (Record (: name string)
                     (: labels (List (Tuple string string)))
                     (: cells (List string))
                     (: notes (List string))
                     (: extra (List string)))))

;; The samples outside Tukey's fences and also more than 2% from the median.
;; On a quiet machine the fences of a tight benchmark are a fraction of a
;; percent apart, and a sample past them is no news.
(: (-> (Vec double) double int))
(defun (far-outliers xs med)
  (def q1 (quantile xs 0.25))
  (def q3 (quantile xs 0.75))
  (def fence (max (* 1.5 (- q3 q1)) (* 0.02 med)))
  (loop (:for x xs)
        (:acc (counting x) #:when (or (< x (- q1 fence)) (> x (+ q3 fence))))))

;; ---------------------------------------------------------------------------
;; Pictures of samples, for --detail
;; ---------------------------------------------------------------------------

(def spark-levels "▁▂▃▄▅▆▇█")

;; The level of `x` between `lo` and `hi`, as one of the eight blocks.
(: (-> double double double string))
(defun (spark-char x lo hi)
  (def i (if (<= hi lo)
             3
             (min 7 (max 0 (cast System.Int32 (floor (* 8.0 (/ (- x lo) (- hi lo)))))))))
  (substring spark-levels (string-cursor-at spark-levels i) (string-cursor-at spark-levels (+ i 1))))

(: (-> string int StringCursor))
(defun (string-cursor-at s i)
  (let go ((c (string-cursor-start s)) (n i))
    (if (or (= n 0) (string-cursor-end? s c)) c (go (string-cursor-next s c) (- n 1)))))

;; The samples in the order they were taken, at most `width` of them: more
;; are averaged in runs. What shows a drift, such as a body that slows down
;; as its heap grows.
(: (-> (Vec double) int string))
(defun (sparkline xs width)
  (def n (vec-length xs))
  (def shown
    (if (<= n width)
        xs
        {vecing (mean (vec-slice xs (/ (* i n) width) (max 1 (- (/ (* (+ i 1) n) width) (/ (* i n) width)))))
                (:for i (range 0 width))}))
  (def lo (vec-fold min (vec-ref shown 0) shown))
  (def hi (vec-fold max (vec-ref shown 0) shown))
  (string-join (vec-map (fun (x) (spark-char x lo hi)) shown) ""))

;; How many samples fall in each of `bins` equal steps from the lowest to the
;; highest, a block each, a space for none. What shows two values rather
;; than a spread around one.
(: (-> (Vec double) int string))
(defun (histogram xs bins)
  (def lo (vec-fold min (vec-ref xs 0) xs))
  (def hi (vec-fold max (vec-ref xs 0) xs))
  (def counts (make-array bins #:fill 0))
  (loop (:for x xs)
        (:let i (if (<= hi lo) 0 (min (- bins 1) (cast System.Int32 (floor (* (cast System.Double bins) (/ (- x lo) (- hi lo))))))))
        (:do (array-set! counts i (+ 1 (array-ref counts i)))))
  (def most (cast System.Double (loop (:for i (up-from 0 #:to bins)) (:acc m (folding 0 (max m (array-ref counts i)))) => m)))
  (string-join {vecing (if (= (array-ref counts i) 0)
                           " "
                           (spark-char (cast System.Double (array-ref counts i)) 0.0 most))
                       (:for i (range 0 bins))}
               ""))

;; Collections per 1000 calls, or per call for a run of single calls.
(: (-> Measured string))
(defun (gc-text m)
  (def calls (cast System.Double (record-ref m calls)))
  (def per (if (record-ref m once) 1.0 1000.0))
  (defun (one (: n long)) : string
    (if (= n 0L) "0" (three (* per (/ (cast System.Double n) calls)))))
  (if (= 0L (+ (record-ref m gc0) (+ (record-ref m gc1) (record-ref m gc2))))
      "-"
      (str (one (record-ref m gc0)) "/" (one (record-ref m gc1)) "/" (one (record-ref m gc2)))))

(: (-> string (List (Tuple string string)) Measured bool Row))
(defun (measured-row name labels m detail)
  (def xs (record-ref m samples))
  (def med (median xs))
  (def spread (if (> med 0.0) (* 100.0 (/ (mad xs) med)) 0.0))
  (def lowest (vec-fold min (vec-ref xs 0) xs))
  (def outliers (far-outliers xs med))
  (def calls (cast System.Double (record-ref m calls)))
  (def notes
    (list-filter (fun (n) (not (string-empty? n)))
                 (list (if (and (not (record-ref m once)) (> spread 3.0)) "noisy" "")
                       (if (and (not (record-ref m once)) (not (record-ref m steady))) "unsteady" "")
                       (if (and (not (record-ref m once)) (> outliers 0))
                           (str (int->string outliers) (if (= outliers 1) " outlier" " outliers")) "")
                       (if (and (record-ref m per-call) (< med 1000.0)) "setup-every: below resolution" ""))))
  (def extra
    (if (record-ref m once)
        (list (str "(" (string-join (vec-map time-text xs) ", ") ")"))
        (if detail
            (list (str "mean " (time-text (mean xs)) ", sd " (time-text (std-dev xs))
                       ", q1 " (time-text (quantile xs 0.25)) ", q3 " (time-text (quantile xs 0.75))
                       ", " (int->string (vec-length xs)) " samples of " (long->string (record-ref m k)) " calls")
                  (str "in order   " (sparkline xs 60))
                  (str "histogram  " (time-text lowest) " " (histogram xs 24) " "
                       (time-text (vec-fold max (vec-ref xs 0) xs))))
            (list))))
  (Row (name name) (labels labels)
       (cells (list (time-text med)
                    (if (record-ref m once) "-" (str (fixed spread 1) "%"))
                    (time-text lowest)
                    (bytes-text (/ (cast System.Double (record-ref m bytes)) calls))
                    (gc-text m)))
       (notes notes)
       (extra extra)))

(: (-> int (List string)))
(defun (header-lines count)
  (def facts
    (str "bench: " (int->string count) (if (= count 1) " benchmark, " " benchmarks, ")
         (if (debug-build?) "Debug build, " "Release build, ")
         (dotnet-version) ", " (arch-name) ", " (int->string (core-count)) " cores, gcServer="
         (if (server-gc?) "true" "false")))
  (def gov (governor))
  (list-filter (fun (l) (not (string-empty? l)))
               (list facts
                     (str "bench: " (cpu-model) ", " (os-name))
                     (if (debug-build?) "bench: warning: a Debug build measures code the JIT did not optimize" "")
                     (if (debugger?) "bench: warning: a debugger is attached" "")
                     (if (and (not (string-empty? gov)) (not (= gov "performance")))
                         #"bench: warning: CPU frequency governor is '${gov}'; results will be noisy" ""))))

;; What alloc/op counts: a line when it is every thread's, by `--alloc` or by
;; a benchmark's `#:alloc`. The default, this thread's, says nothing.
(: (-> (List Selection) (List string)))
(defun (alloc-lines chosen)
  (def default (box-ref alloc-default))
  (def others
    (loop (:for p chosen)
          (:let b (sel-bench p))
          (:when (not (= (alloc-counted (record-ref b alloc)) default)))
          (:acc (listing (record-ref b name)))))
  (def what (fun (mode) (if (= mode "process") "every thread of the process" "the measuring thread")))
  (def names (string-join (list->vec others) ", "))
  (cond
    ((list-empty? others)
     (if (= default "process") (list #"bench: alloc/op counts ${(what default)}") (list)))
    (else
     (list #"bench: alloc/op counts ${(what default)}, and ${(what (if (= default "process") "thread" "process"))} for ${names} (#:alloc)"))))

;; What a run is asked to measure: benchmarks, each with the cases chosen.
(type (: Selection (Record (: bench Bench) (: cases (List BenchCase)))))

(: (-> Selection Bench))
(defun (sel-bench p) (record-ref p bench))

(: (-> Selection (List BenchCase)))
(defun (sel-cases p) (record-ref p cases))

;; The layout of a table: the parameters it has columns for, and the widths
;; and alignments of its columns, from `(std fmt)`. Fixed before the first
;; row, so that a row can be printed the moment it is measured and a long
;; run read while it goes.
(type (: Layout (Record (: params (List string))
                        (: heads (List string))
                        (: table TableLayout))))

;; The cells after the parameters are numbers, right-aligned, of a width
;; known in advance; a longer one pushes its row out rather than the column.
;; After them come the notes, unaligned.
(def number-heads (list "median" "±" "min" "alloc/op" "gc/1k 0/1/2"))
(def number-widths (list 9 6 9 9 11))

(: string)
(def table-sep "   ")

;; The label of parameter `p` among a case's, or nothing.
(: (-> (List (Tuple string string)) string string))
(defun (label-of labels p)
  (match (find (fun (l) (= (fst l) p)) labels)
    ((Some l) (snd l))
    (None "")))

(: (-> (List string) (Vec Block)))
(defun (blocks cells) (list->vec (list-map BStr cells)))

(: (-> (List Selection) bool Layout))
(defun (layout-for chosen scaling)
  (def params (selection-params chosen))
  (def heads (if scaling (list-append number-heads (list "scaling")) number-heads))
  (def head-widths (if scaling (list-append number-widths (list 14)) number-widths))
  ;; The name and parameter columns are as wide as their widest cell.
  (def label-rows
    (cons (blocks (cons "name" params))
          (fold (fun (p acc)
                  (list-append acc
                               (list-map (fun (c)
                                           (blocks (cons (record-ref (sel-bench p) name)
                                                         (list-map (fun (n) (label-of (case-labels c) n))
                                                                   params))))
                                         (sel-cases p))))
                (list) chosen)))
  (def widths
    (list->vec (list-append (vec->list (table-widths (list->vec label-rows)))
                            (list-map (fun (p) (match p ((Tuple h w) (max w (string-count h)))))
                                      (list-zip heads head-widths)))))
  (def aligns
    (list->vec (list-append (list-map (fun (n) AlignLeft) (cons "name" params))
                            (list-map (fun (h) AlignRight) heads))))
  (Layout (params params) (heads heads) (table (table-layout widths #:sep table-sep #:align aligns))))

(: (-> Layout void))
(defun (print-head l)
  (println "")
  (show [(table-row (record-ref l table)
                    (blocks (list-append (cons "name" (record-ref l params)) (record-ref l heads))))]))

(: (-> Layout Row void))
(defun (print-row l r)
  (def notes (record-ref r notes))
  (def cells
    (list-append (cons (record-ref r name)
                       (list-map (fun (p) (label-of (record-ref r labels) p)) (record-ref l params)))
                 (list-append (record-ref r cells)
                              (if (list-empty? notes) (list) (list (string-join (list->vec notes) ", "))))))
  (show [(table-row (record-ref l table) (blocks cells))])
  ;; Extra lines start under the first column after the name.
  (def indent (+ (vec-ref (record-ref (record-ref l table) widths) 0) (string-count table-sep)))
  (loop (:for e (record-ref r extra))
        (:do (println (str (string-pad-right "" indent) e)))))


;; ---------------------------------------------------------------------------
;; Running
;; ---------------------------------------------------------------------------

;; The parameters of what is chosen, each once, in the order first met.
(: (-> (List Selection) (List string)))
(defun (selection-params chosen)
  (fold (fun (p acc)
          (list-append acc (list-filter (fun (n) (not (any? (fun (m) (= m n)) acc)))
                                        (record-ref (sel-bench p) param-names))))
        (list) chosen))

(: (-> (List Selection) int))
(defun (case-count chosen) (fold (fun (p n) (+ n (list-length (sel-cases p)))) 0 chosen))

;; Whether a filter chooses a benchmark's case: its name, or its parameters as
;; the report shows them, `n=1000`.
(: (-> (Vec string) Bench BenchCase bool))
(defun (chosen? filters b c)
  (or (vec-empty? filters)
      (any? (fun (f)
              (or (string-contains? (record-ref b name) f)
                  (any? (fun (l) (= (str (fst l) "=" (snd l)) f)) (case-labels c))))
            filters)))

;; The benchmarks the filters choose, with their cases.
(: (-> (List Bench) (Vec string) (List Selection)))
(defun (filtered benches filters)
  (loop (:for b benches)
        (:let picked (list-filter (fun (c) (chosen? filters b c)) ((record-ref b cases))))
        (:when (not (list-empty? picked)))
        (:acc (listing (Selection (bench b) (cases picked))))))

;; Whether there is someone to show a progress line to: a terminal, and not
;; a comint buffer, which would show every rewrite as a line of its own.
(: (-> bool))
(defun (progress-line?)
  (and (output-terminal?)
       (none? (get-environment-variable "INSIDE_EMACS"))
       (none? (get-environment-variable "CI"))))

;; Measures what is chosen and prints the table, a row as each case is done.
;; Answers what was measured, as a saved run holds it; a skipped case is not
;; in it.
(: (-> (List Selection) BenchOptions (List SavedCase)))
(defun (run-selection chosen o)
  (def s (bench-settings o))
  (def count (case-count chosen))
  (loop (:for l (header-lines count)) (:do (println l)))
  (loop (:for l (alloc-lines chosen)) (:do (println l)))
  (def layout (layout-for chosen (record-ref o scaling)))
  (print-head layout)
  (flush-port (parameter-ref current-output-port))
  (def live (progress-line?))
  (def/mutable done 0)
  (def/mutable measured (list))
  ;; For --scaling: the benchmark, parameter value and median of the row
  ;; before.
  (: (Box (Option (Tuple string (Tuple double double)))))
  (def previous (make-box None))
  (loop (:for p chosen)
        (:let b (sel-bench p))
        (:subloop (:for c (sel-cases p))
                  (:do (set! done (+ done 1)))
                  (:let prefix
                        (str "[" (int->string done) "/" (int->string count) "] " (record-ref b name)
                             (string-join (list->vec (list-map (fun (l) (str " " (fst l) "=" (snd l)))
                                                               (case-labels c)))
                                          "")
                             "   "))
                  (:do (when live
                         (box-set! progress-box
                                   (fun (what)
                                     (print (str "\r" prefix what "\u{1B}[K"))
                                     (flush-port (parameter-ref current-output-port))))))
                  (:let m (if (string-empty? (record-ref b skip))
                              (Some (measure-case c (record-ref b once) s))
                              None))
                  (:let row
                         (match m
                           ((Some m)
                            (def r (measured-row (record-ref b name) (case-labels c) m (record-ref o detail)))
                            (if (record-ref o scaling)
                                (record-set r (cells (list-append (record-ref r cells)
                                                                  (list (scaling-text previous b c (median (record-ref m samples)))))))
                                r))
                           (None
                            (Row (name (record-ref b name)) (labels (case-labels c))
                                 (cells (list))
                                 (notes (list (str "skipped: " (record-ref b skip)))) (extra (list))))))
                  (:do (when live (print "\r\u{1B}[K"))
                       (print-row layout row)
                       (flush-port (parameter-ref current-output-port))
                       (match m
                         ((Some m) (set! measured (cons (saved-case (record-ref b name) (case-labels c) m) measured)))
                         (None unit)))))
  (box-set! progress-box (fun (what) unit))
  (list-reverse measured))

(: (-> string (List (Tuple string string)) Measured SavedCase))
(defun (saved-case name labels m)
  (SavedCase (name name) (labels labels) (k (record-ref m k)) (once (record-ref m once))
             (bytes (/ (cast System.Double (record-ref m bytes)) (cast System.Double (max 1L (record-ref m calls)))))
             (samples (record-ref m samples))))

;; `×98.2 for ×100`: how the median grew from the row before, against how
;; the benchmark's one numeric parameter did. Nothing for the first row of a
;; benchmark, or one whose parameters are not one number.
(: (-> (Box (Option (Tuple string (Tuple double double)))) Bench BenchCase double string))
(defun (scaling-text previous b c med)
  (def value
    (match (case-labels c)
      ((Cons (Tuple _ v) Nil) (parse-double v))
      (_ None)))
  (match value
    (None (box-set! previous None) "")
    ((Some x)
     (def text
       (match (box-ref previous)
         ((Some (Tuple name (Tuple x0 m0))) #:when (= name (record-ref b name))
          (if (and (> x0 0.0) (> m0 0.0))
              (str "×" (three (/ med m0)) " for ×" (three (/ x x0)))
              ""))
         (_ "")))
     (box-set! previous (Some (Tuple (record-ref b name) (Tuple x med))))
     text)))

;; The facts of this machine and build, as a saved run keeps them.
(: (-> (List (Tuple string string))))
(defun (header-facts)
  (list (Tuple "build" (if (debug-build?) "Debug" "Release"))
        (Tuple "dotnet" (dotnet-version))
        (Tuple "arch" (arch-name))
        (Tuple "cores" (int->string (core-count)))
        (Tuple "gc-server" (if (server-gc?) "true" "false"))
        (Tuple "cpu" (cpu-model))
        (Tuple "os" (os-name))
        (Tuple "governor" (governor))))

(: (-> (List SavedCase) SavedRun))
(defun (this-run cases)
  (SavedRun (saved (now-utc)) (machine (machine-key)) (header (header-facts)) (cases cases)))

;; `+8.9%`: a change, signed, to a tenth of a percent.
(: (-> double string))
(defun (change-text x)
  (def t (fixed (* 100.0 x) 1))
  (str (if (string-starts-with? t "-") "" "+") t "%"))

;; A percentage as it was given: `2%`, `1.5%`.
(: (-> double string))
(defun (percent-text x)
  (def v (* 100.0 x))
  (str (if (= v (round v)) (fixed v 0) (fixed v 1)) "%"))

(: (-> double string))
(defun (p-text p)
  (cond ((< p 0.001) "<0.001")
        ((< p 0.1) (fixed p 3))
        (else (fixed p 2))))

(: (-> Verdict double string))
(defun (verdict-text v threshold)
  (match v
    (Faster "faster")
    (Slower "SLOWER")
    (SmallChange #"no change (< ${(percent-text threshold)})")
    (NoChange "no change")))

(: (-> Comparison (List string) BenchOptions bool bool bool (List string)))
(defun (comparison-row c params o other spread? ratio?)
  (def labels (record-ref c labels))
  (list-append
    (cons (record-ref c name) (list-map (fun (p) (label-of labels p)) params))
    (list-append
      (list (time-text (record-ref c was))
            (time-text (record-ref c now))
            (change-text (record-ref c change))
            (str (change-text (record-ref c lo)) " .. " (change-text (record-ref c hi)))
            (p-text (record-ref c p)))
      (list-append
        (list-append
          (if ratio? (list (str "×" (fixed (+ 1.0 (record-ref c change)) 2))) (list))
          (if (record-ref o p-raw) (list (p-text (record-ref c p-raw))) (list)))
        (list-append
          (if spread? (list (str (fixed (* 100.0 (record-ref c spread)) 1) "%")) (list))
          (list (str (verdict-text (record-ref c verdict) (record-ref o threshold))
                     (if other " (other machine)" ""))))))))

(: (-> Comparison (List (Tuple string string))))
(defun (comparison-labels c) (record-ref c labels))

(: (-> Comparison double))
(defun (comparison-spread c) (record-ref c spread))

;; The comparison with a saved run, after the run's own table.
(: (-> (Option string) SavedRun CompareResult BenchOptions void))
(defun (print-comparison file saved result o)
  (def path (match file ((Some p) p) (None "")))
  (def n (list-length (record-ref result comparisons)))
  (println "")
  (println (str "bench: compared with " path ", saved " (record-ref saved saved)
                "; unpaired (Mann-Whitney U); " (comparisons-text n)))
  (println "bench: one process each: a difference between processes is not in the p-values")
  (print-comparison-table result o "was" "now" path #f))

;; `3 comparisons, Holm-corrected`.
(: (-> int string))
(defun (comparisons-text n)
  (str (int->string n) (if (= n 1) " comparison" " comparisons") (if (> n 1) ", Holm-corrected" "")))

;; A comparison's warnings, its table and what one side has alone, under the
;; header its caller printed: `was` and `now` name the two medians' columns,
;; and `other-side` is what the side `was` is from is called, for a case
;; only it has.
(: (-> CompareResult BenchOptions string string string bool void))
(defun (print-comparison-table result o was-head now-head other-side ratio?)
  (def comparisons (record-ref result comparisons))
  (def other (record-ref result other-machine))
  (def n (list-length comparisons))
  (when other
    (println "bench: warning: the saved run is from another machine: the numbers are shown, and no verdict is to be believed"))
  (loop (:for w (record-ref result warnings))
        (:do (println (str "bench: warning: " w))))
  (when (> n 0)
    (def params
      (fold (fun (c acc)
              (list-append acc (list-filter (fun (p) (not (has-string? acc p)))
                                            (list-map fst (comparison-labels c)))))
            (list) comparisons))
    (def spread? (any? (fun (c) (>= (comparison-spread c) 0.0)) comparisons))
    (def heads
      (list-append (cons "name" params)
                   (list-append (list was-head now-head "change" "95% interval" "p")
                                (list-append (list-append (if ratio? (list "ratio") (list))
                                                          (if (record-ref o p-raw) (list "p raw") (list)))
                                             (list-append (if spread? (list "between processes") (list))
                                                          (list "verdict"))))))
    (def numbers (+ 5 (if ratio? 1 0) (if (record-ref o p-raw) 1 0) (if spread? 1 0)))
    (def aligns
      (list->vec (list-append (list-map (fun (h) AlignLeft) (cons "name" params))
                              (list-append (list-map (fun (i) AlignRight) (index-list numbers))
                                           (list AlignLeft)))))
    (println "")
    (show [(tabular/rows (list->vec (cons (blocks heads)
                                          (list-map (fun (c) (blocks (comparison-row c params o other spread? ratio?)))
                                                    comparisons)))
                         #:sep table-sep #:align aligns)]))
  (unless (list-empty? (record-ref result only-now))
    (println (str "bench: only in this run: " (string-join (list->vec (record-ref result only-now)) ", "))))
  (unless (list-empty? (record-ref result only-saved))
    (println (str "bench: only in " other-side ": " (string-join (list->vec (record-ref result only-saved)) ", "))))
  (match (record-ref o fail-slower)
    ((Some _) #:when other
     (println "bench: --fail-slower does not apply: the saved run is from another machine"))
    (_ unit)))

;; Whether `--fail-slower PCT` fails the run: something SLOWER by more than
;; PCT, against a run from this machine.
(: (-> CompareResult BenchOptions bool))
(defun (fails-slower? result o)
  (match (record-ref o fail-slower)
    ((Some limit)
     (and (not (record-ref result other-machine))
          (any? (fun (c) (and (match (record-ref c verdict) (Slower #t) (_ #f))
                              (> (record-ref c change) limit)))
                (record-ref result comparisons))))
    (None #f)))

;; Runs what is chosen, then saves and compares as the options say. Answers
;; the exit code: 1 when `--fail-slower` is exceeded, 2 when the file to
;; compare with cannot be used, 0 otherwise.
(: (-> (List Selection) BenchOptions int))
(defun (run-and-report chosen o)
  ;; A comparison file that cannot be read is known before anything is run.
  (def baseline
    (option-map read-saved-run (record-ref o compare)))
  (match baseline
    ((Some (Err why)) (println #"bench: --compare: ${why}") 2)
    (_
     (def run (this-run (run-selection chosen o)))
     (match (record-ref o save)
       ((Some path)
        (match (try (write-saved-run! path run) #:catch (System.Exception))
          ((Ok _) (println #"bench: saved to ${path}"))
          (_ (println #"bench: --save: ${path} could not be written"))))
       (None unit))
     (when (record-ref o compare-within) (compare-within chosen o))
     (match baseline
       ((Some (Ok saved))
        (def result (compare-runs saved run (record-ref o alpha) (record-ref o threshold)))
        (print-comparison (record-ref o compare) saved result o)
        (if (fails-slower? result o) 1 0))
       (_ 0)))))

;; ---------------------------------------------------------------------------
;; --compare-within: the cases of a group against its first, in rounds
;; ---------------------------------------------------------------------------

;; The groups of what is chosen, in the order first met, each with its cases
;; that are not skipped: a benchmark's cases one after another.
(: (-> (List Selection) (List (Tuple string (List (Tuple Bench BenchCase))))))
(defun (chosen-groups chosen)
  (fold (fun (p acc)
          (def b (sel-bench p))
          (def g (record-ref b group))
          (def cases (list-map (fun (c) (Tuple b c)) (sel-cases p)))
          (cond
            ((or (string-empty? g) (not (string-empty? (record-ref b skip)))) acc)
            ((has-group? acc g)
             (list-map (fun (e) (if (= (fst-of e) g) (Tuple g (list-append (snd-of e) cases)) e)) acc))
            (else (list-append acc (list (Tuple g cases))))))
        (list) chosen))

(: (-> (List (Tuple string (List (Tuple Bench BenchCase)))) string bool))
(defun (has-group? groups g) (any? (fun (e) (= (fst-of e) g)) groups))

(: (-> (Tuple string (List (Tuple Bench BenchCase))) string))
(defun (fst-of e) (match e ((Tuple a _) a)))

(: (-> (Tuple string (List (Tuple Bench BenchCase))) (List (Tuple Bench BenchCase))))
(defun (snd-of e) (match e ((Tuple _ b) b)))

;; `sort n=1000`: a case as a comparison names it.
(: (-> Bench BenchCase string))
(defun (case-key-of b c)
  (string-join (list->vec (cons (record-ref b name)
                                (list-map (fun (l) (str (fst l) "=" (snd l))) (case-labels c))))
               " "))

;; Every group of two cases or more, measured again in `-N` rounds in this
;; process: each case prepared, and each round a sample of every case, in an
;; order shuffled from `--seed`. Each case is compared with the group's first
;; one, paired by round, as A/B pairs its sides; there is one process, so a
;; difference between processes is not in it to hide.
(: (-> (List Selection) BenchOptions void))
(defun (compare-within chosen o)
  (def s (bench-settings o))
  (def seed (match (record-ref o seed) ((Some n) n) (None (cast System.UInt64 (bench-now)))))
  (def rng (make-random/seed64 seed))
  (def rounds (record-ref o rounds))
  (def live (progress-line?))
  (loop (:for g (chosen-groups chosen))
        (:let (Tuple name members) g)
        (:when (>= (list-length members) 2))
        (:let cases (list->vec members))
        (:let n (vec-length cases))
        (:let runs
              (vec-map (fun (bc)
                         (match bc
                           ((Tuple b c)
                            (def run ((record-ref c prepare)))
                            (def k (if (> (record-ref b once) 0)
                                       1L
                                       (match (prepare-run run (record-ref s sample-ns) (record-ref s warmup-ns))
                                         ((Tuple k _) k))))
                            (Tuple run k))))
                       cases))
        (:let taken (make-array n #:fill (vec-empty)))
        (:do (loop (:for r (up-from 0 #:to rounds))
                   (:do (when live
                          (print (str "\r[" name "] round " (int->string (+ r 1)) "/" (int->string rounds) "\u{1B}[K"))
                          (flush-port (parameter-ref current-output-port))))
                   (:subloop (:for i (shuffle-vec (list->vec (index-list n)) #:rng rng))
                             (:let (Tuple run k) (vec-ref runs i))
                             (:let x (take-sample run k))
                             (:do (array-set! taken i (vec-add (array-ref taken i)
                                                               (per-call-ns (record-ref x ns) k))))))
             (when live (print "\r\u{1B}[K")))
        (:let (Tuple b0 c0) (vec-ref cases 0))
        (:let base (case-key-of b0 c0))
        (:let paired
              (loop (:for i (up-from 1 #:to n))
                    (:let (Tuple b c) (vec-ref cases i))
                    (:acc out (listing (PairedCase (name (record-ref b name)) (labels (case-labels c))
                                                   (a [(array-ref taken 0)]) (b [(array-ref taken i)]))))))
        (:let result (compare-paired paired (list) (list) (record-ref o alpha) (record-ref o threshold)))
        (:do (println "")
             (println (str "bench: within " name ", against " base ": " (int->string rounds)
                           " rounds in this process, paired (Wilcoxon signed-rank); "
                           (comparisons-text (list-length paired))))
             (println (str "bench: seed " (displayed seed) " (--seed repeats the order of the cases)"))
             (print-comparison-table result o base "this" base #t))))

;; ---------------------------------------------------------------------------
;; The menu
;; ---------------------------------------------------------------------------

;; The menu's lines: the benchmarks numbered from 0, under their groups, each
;; with its parameters and their values.
(: (-> string (List Bench) (List string)))
(defun (menu-lines file benches)
  (def/mutable last-group "")
  (def lines
    (loop (:for b benches)
          (:with i 0 (+ i 1))
          (:let group (record-ref b group))
          (:let heading
                (if (and (not (string-empty? group)) (not (= group last-group)))
                    (list (str "  " group))
                    (list)))
          (:do (set! last-group group))
          (:let indent (if (string-empty? group) "  " "    "))
          (:let params (param-summary b))
          (:acc out (folding (list) (list-append out (list-append heading
                                                                (list (string-trim-right
                                                                       (str indent "#" (int->string i) "  "
                                                                            (string-pad-right (record-ref b name) 18)
                                                                            params)))))))
      => out))
  (list-append (cons (str "Benchmarks in " file ":") (cons "" lines))
               (list ""
                     "Choose by number (0 2-4), by name (sort), with parameters (2 n=1000),"
                     "or * for all. Nothing quits, or repeats the last choice once there is one.")))

;; `n: 10 1000   order: sorted random`.
(: (-> (List string) string bool))
(defun (has-string? xs s) (any? (fun (x) (= x s)) xs))

(: (-> BenchCase (List (Tuple string string))))
(defun (case-labels c) (record-ref c labels))

(: (-> Bench string))
(defun (param-summary b)
  (: (List BenchCase))
  (def cases ((record-ref b cases)))
  (: (-> string (List string)))
  (defun (values-of n)
    (fold (fun (c acc)
            (match (find (fun (l) (= (fst l) n)) (case-labels c))
              ((Some l) (if (has-string? acc (snd l)) acc (list-append acc (list (snd l)))))
              (None acc)))
          (list) cases))
  (string-join
    (list->vec
      (list-map (fun (n) (str n ": " (string-join (list->vec (values-of n)) " ")))
                (record-ref b param-names)))
    "   "))

;; A menu choice: numbers, ranges of them, names, `*`, and after a benchmark,
;; `p=v,w` to narrow its cases to those values of parameter `p`.
(: (-> string (List Bench) (Result string (List Selection))))
(defun (parse-choice text benches)
  (def tokens (list-filter (fun (t) (not (string-empty? t))) (vec->list (string-split (string-trim text) " "))))
  (defun (all-of (: b Bench)) : Selection (Selection (bench b) (cases ((record-ref b cases)))))
  (let go ((left tokens) (acc (list)) (last (list)))
    ;; `acc` is what is chosen so far, `last` the selections the most recent
    ;; token made, which a `p=v` narrows.
    (match left
      (Nil (Ok (list-append acc last)))
      ((Cons t more)
       (cond
         ((= t "*") (go more (list-append acc last) (list-map all-of benches)))
         ((some? (split-once t "="))
          (def (Some (Tuple p vs)) (split-once t "=") :leave-with (Err t))
          (def wanted (vec->list (string-split vs ",")))
          (if (list-empty? last)
              (Err #"${t} narrows the benchmark before it, and there is none.")
              (go more acc
                  (list-map (fun (sel)
                              (Selection (bench (sel-bench sel))
                                         (cases (list-filter (fun (c)
                                                               (any? (fun (l) (and (= (fst l) p)
                                                                                   (any? (fun (w) (= w (snd l))) wanted)))
                                                                     (case-labels c)))
                                                             (sel-cases sel)))))
                            last))))
         (else
          (match (choice-indices t (list-length benches))
            ((Some is)
             (go more (list-append acc last)
                 (list-map (fun (i) (all-of (list-ref benches i))) is)))
            (None
             (def named (list-filter (fun (b) (string-contains? (record-ref b name) t)) benches))
             (cond
               ((some? (parse-int t))
                (Err #"There is no benchmark ${t}: they are numbered 0 to ${(- (list-length benches) 1)}."))
               ((list-empty? named)
                (Err #"Nothing is called ${t}, and it is not a number of one."))
               (else
                (go more (list-append acc last) (list-map all-of named))))))))))))

;; `3` or `2-4`, as indices below `n`.
(: (-> string int (Option (List int))))
(defun (choice-indices t n)
  (match (split-once t "-")
    ((Some (Tuple a b))
     (match (Tuple (parse-int a) (parse-int b))
       ((Tuple (Some lo) (Some hi)) #:when (and (<= 0 lo) (<= lo hi) (< hi n))
        (Some {listing i (:for i (range lo (+ hi 1)))}))
       (_ None)))
    (None
     (match (parse-int t)
       ((Some i) #:when (and (<= 0 i) (< i n)) (Some (list i)))
       (_ None)))))

;; The menu: ask, run, and ask again, until nothing is chosen with nothing to
;; repeat, `q`, or the end of the input.
(: (-> string (List Bench) BenchOptions (-> (Option string)) void))
(defun (bench-menu file benches o read)
  (menu-loop (menu-lines file benches) benches read
             (fun (chosen) (ignore (run-and-report chosen o)))))

;; The menu's asking, with the lines to show first and what to do with a
;; choice: `bench-menu` runs it here, and `bjo bench`, over several files,
;; runs each file with the cases that fall in it.
(: (-> (List string) (List Bench) (-> (Option string)) (-> (List Selection) void) void))
(defun (menu-loop lines benches read run)
  (loop (:for l lines) (:do (println l)))
  (let ask ((previous None))
    (print "bench> ")
    (flush-port (parameter-ref current-output-port))
    (match (read)
      (None (println ""))
      ((Some line)
       (def text (string-trim line))
       (cond
         ((= text "q") unit)
         ((string-empty? text)
          (match previous
            (None unit)
            ((Some chosen) (run chosen) (ask previous))))
         (else
          (match (parse-choice text benches)
            ((Err why) (println why) (ask previous))
            ((Ok chosen)
             (if (= (case-count chosen) 0)
                 (let () (println "That chooses nothing.") (ask previous))
                 (let () (run chosen) (ask (Some chosen))))))))))))

;; Whether to ask what to run: with no names and no `--all`, at a terminal,
;; and not in CI. `BJO_BENCH_MENU=1` asks anyway, which is how it is tested.
(: (-> BenchOptions bool))
(defun (menu? o)
  (and (vec-empty? (record-ref o filters))
       (not (record-ref o all))
       (none? (record-ref o cases))
       (or (= (get-environment-variable "BJO_BENCH_MENU") (Some "1"))
           (and (input-terminal?) (output-terminal?)
                (none? (get-environment-variable "CI"))))))

;; ---------------------------------------------------------------------------
;; --serve: driven by another program
;; ---------------------------------------------------------------------------
;;
;; A command a line on standard input, a reply a line on standard output,
;; both bjodat. Cases are numbered from 0 in the order `(list)` gives them.
;;
;;   (list)                               → (cases (case 0 (name "sort") (labels ("n" "10")) (once 0) (skip "")) ...)
;;   (prepare 3 :warmup 1000 :sample 10)   → (prepared 3 :k 4096 :steady #t)
;;   (sample 3 :k 4096)                    → (sample 3 :ns 41213 :bytes 0 :gc (0 0 0))
;;   (quit)                                → nothing; the program ends
;;
;; Keywords are bjodat's, `:k`; `#:k` reads as the same keyword. `:warmup`
;; and `:sample` are milliseconds, 1000 and 10 unless given. A sample's `:ns`
;; is the whole sample's, `k` calls. Anything wrong is `(error "why")`, and
;; the next command is read. `prepare`
;; runs the case's setup; a case is sampled only once it is prepared.

(: (-> string Bjodat))
(defun (sym s) (BjoSym (string->symbol s)))

(: (-> string Bjodat))
(defun (kw s) (BjoKey (string->keyword s)))

(: (-> string (List Bjodat) Bjodat))
(defun (form head xs) (BjoList (cons (sym head) xs)))

(: (-> int Bjodat))
(defun (long-of n) (BjoInt (cast System.Int64 n)))

;; The value after keyword `name` in a command's arguments.
(: (-> (List Bjodat) string (Option Bjodat)))
(defun (keyword-arg args name)
  (match args
    ((Cons (BjoKey k) (Cons v more)) (if (= (keyword->string k) name) (Some v) (keyword-arg more name)))
    ((Cons _ more) (keyword-arg more name))
    (Nil None)))

(: (-> (List Bjodat) string long (Option long)))
(defun (long-arg args name default)
  (match (keyword-arg args name)
    (None (Some default))
    ((Some (BjoInt n)) (Some n))
    (_ None)))

(: (-> string Bjodat))
(defun (serve-error why) (form "error" (list (BjoStr why))))

(: (-> int Bench BenchCase Bjodat))
(defun (case-listing i b c)
  (form "case" (list (long-of i)
                     (form "name" (list (BjoStr (record-ref b name))))
                     (form "labels" (list-map (fun (l) (BjoList (list (BjoStr (fst l)) (BjoStr (snd l)))))
                                              (case-labels c)))
                     (form "group" (list (BjoStr (record-ref b group))))
                     (form "once" (list (long-of (record-ref b once))))
                     (form "skip" (list (BjoStr (record-ref b skip)))))))

;; Answers one command. `runs` holds the prepared cases.
(: (-> Bjodat (Vec (Tuple Bench BenchCase)) (Array (Option CaseRun)) Bjodat))
(defun (serve-reply command cases runs)
  (def n (vec-length cases))
  (defun (case-at (: i long)) : (Option (Tuple Bench BenchCase))
    (if (and (>= i 0L) (< i (cast System.Int64 n))) (Some (vec-ref cases (cast System.Int32 i))) None))
  (match command
    ((BjoList (Cons (BjoSym head) args))
     (case (symbol->string head)
       ("list"
        (form "cases" (loop (:for p cases) (:with i 0 (+ i 1))
                            (:acc (listing (match p ((Tuple b c) (case-listing i b c))))))))
       ("prepare"
        (match (Tuple args (long-arg args "warmup" 1000L) (long-arg args "sample" 10L))
          ((Tuple (Cons (BjoInt i) _) (Some warmup-ms) (Some sample-ms))
           (match (case-at i)
             (None (serve-error #"there is no case ${i}"))
             ((Some (Tuple b c))
              (if (not (string-empty? (record-ref b skip)))
                  (serve-error #"case ${i} is skipped: ${(record-ref b skip)}")
                  (match (try (let ()
                                (def run ((record-ref c prepare)))
                                (array-set! runs (cast System.Int32 i) (Some run))
                                (if (> (record-ref b once) 0)
                                    (Tuple 1L #t)
                                    (prepare-run run (* sample-ms 1000000L) (* warmup-ms 1000000L))))
                              #:catch (System.Exception))
                    ((Ok (Tuple k steady))
                     (form "prepared" (list (BjoInt i) (kw "k") (BjoInt k) (kw "steady") (BjoBool steady))))
                    ((Err e) (serve-error #"case ${i} failed to prepare: ${(.-Message e)}")))))))
          (_ (serve-error "prepare takes a case number, and :warmup and :sample in milliseconds"))))
       ("sample"
        (match (Tuple args (long-arg args "k" 1L))
          ((Tuple (Cons (BjoInt i) _) (Some k))
           (match (if (some? (case-at i)) (array-ref runs (cast System.Int32 i)) None)
             (None (serve-error #"case ${i} is not prepared"))
             ((Some run)
              (match (try (take-sample run k) #:catch (System.Exception))
                ((Ok x)
                 (form "sample" (list (BjoInt i) (kw "ns") (BjoInt (record-ref x ns))
                                      (kw "bytes") (BjoInt (record-ref x bytes))
                                      (kw "gc") (BjoList (list (BjoInt (record-ref x gc0)) (BjoInt (record-ref x gc1))
                                                               (BjoInt (record-ref x gc2)))))))
                ((Err e) (serve-error #"case ${i} failed: ${(.-Message e)}"))))))
          (_ (serve-error "sample takes a case number, and :k"))))
       (else (serve-error #"unknown command ${(symbol->string head)}: the commands are list, prepare, sample and quit"))))
    (_ (serve-error "a command is a form: (list), (prepare i ...), (sample i ...) or (quit)"))))

(: (-> Bench (List BenchCase)))
(defun (cases-of b) ((record-ref b cases)))

;; Whether a command is `(quit)`.
(: (-> Bjodat bool))
(defun (quit? command)
  (match command
    ((BjoList (Cons (BjoSym head) _)) (= (symbol->string head) "quit"))
    (_ #f)))

;; Reads commands until `(quit)` or the end of the input.
(: (-> (List Bench) (-> (Option string)) void))
(defun (bench-serve benches read)
  (def cases
    (list->vec (fold (fun (b acc) (list-append acc (list-map (fun (c) (Tuple b c)) (cases-of b))))
                     (list) benches)))
  (def runs (make-array (vec-length cases) #:fill None))
  (let go ()
    (match (read)
      (None unit)
      ((Some line)
       (match (parse-bjodat line)
         ((Ok command)
          (unless (quit? command)
            (println (bjodat->string (serve-reply command cases runs)))
            (flush-port (parameter-ref current-output-port))
            (go)))
         ((Err why)
          (println (bjodat->string (serve-error why)))
          (flush-port (parameter-ref current-output-port))
          (go)))))))

(: (-> string (Vec string) int))
(defun (run-registered-benches module args)
  (match (parse-bench-args args)
    ((Err why) (println #"bench: ${why}") 2)
    ((Ok o) (run-with-options module o))))

(: (-> string BenchOptions int))
(defun (run-with-options module o)
  (bench-params-set! (record-ref o params))
  (box-set! alloc-default (record-ref o alloc))
  (def benches (module-benches module))
  (cond
    ((record-ref o serve)
     (bench-serve benches (fun () (read-line/opt (parameter-ref current-input-port))))
     0)
    ((record-ref o list)
     (loop (:for b benches)
           (:do (println (str (record-ref b name)
                              (if (list-empty? (record-ref b param-names))
                                  ""
                                  (str "   " (string-join (list->vec (record-ref b param-names)) " ")))))))
     0)
    ((list-empty? benches) (println "There are no benchmarks.") 0)
    ((menu? o)
     (def file (match benches ((Cons b _) (shown-file (record-ref b file))) (Nil module)))
     (bench-menu file benches o (fun () (read-line/opt (parameter-ref current-input-port))))
     0)
    ((some? (record-ref o cases))
     (def wanted (match (record-ref o cases) ((Some is) is) (None [])))
     (match (cases-numbered benches wanted)
       ((Ok chosen) (run-and-report chosen o))
       ((Err why) (println #"bench: --cases: ${why}") 2)))
    (else
     (def chosen (filtered benches (record-ref o filters)))
     (if (= (case-count chosen) 0)
         (let () (println "No benchmark's name contains any of the names given.") 0)
         (run-and-report chosen o)))))

;; The cases numbered `wanted`, in `(list)`'s numbering, as selections: a
;; benchmark's chosen cases together, in the order the file has them.
(: (-> (List Bench) (Vec int) (Result string (List Selection))))
(defun (cases-numbered benches wanted)
  (def total (fold (fun (b n) (+ n (list-length (cases-of b)))) 0 benches))
  (match (find (fun (i) (or (< i 0) (>= i total))) (vec->list wanted))
    ((Some i) (Err #"there is no case ${i}: there are ${total}"))
    (None
     (Ok (loop (:for b benches)
               (:with first 0 (+ first (list-length (cases-of b))))
               (:let picked
                     (loop (:for c (cases-of b))
                           (:with i first (+ i 1))
                           (:acc (listing c) #:when (any? (fun (w) (= w i)) wanted))))
               (:acc (listing (Selection (bench b) (cases picked)))
                     #:when (not (list-empty? picked))))))))

;; A path as a person reads it: its last part.
(: (-> string string))
(defun (shown-file path)
  (match (string-search-backward path "/")
    ((Some at) (substring path (string-cursor-next path at) (string-cursor-end path)))
    (None path)))

;; `(run-benches)` — the file's `main`: runs the benchmarks this module
;; registered.
(def/macro (run-benches form inject compare)
  (syntax-match form
    ((_)
     #'(begin
         (: ,(inject 'main) (-> (Vec string) int))
         (defun (,(inject 'main) args) (run-registered-benches (current-module) args))))
    (bad (syntax-error bad "run-benches takes nothing: (run-benches) at the end of a benchmark file."))))

;; ---------------------------------------------------------------------------
;; time-it
;; ---------------------------------------------------------------------------

;; `(time-it body ...)`: the body as an anonymous benchmark, measured at once
;; with `--quick`'s targets, and one line. A body whose first run takes more
;; than a second is not run again: that run is the measurement.
(: (-> string (-> long void) void))
(defun (time-it-run label batch)
  (def run (CaseRun (batch batch) (call None) (alloc "")))
  (def first (time-batch run 1L))
  (if (>= first 1000000000L)
      (println (str label ": " (time-text (cast System.Double first))
                    " (one cold run: a body over a second is run once)"))
      (let ()
        (def o (match (parse-bench-args (vec-empty)) ((Ok o) o) ((Err e) (panic! e))))
        (def m (measure-calibrated run (bench-settings (record-set o (scale 0.5)))))
        (def xs (record-ref m samples))
        (def med (median xs))
        (println (str label ": " (time-text med)
                      " ± " (fixed (if (> med 0.0) (* 100.0 (/ (mad xs) med)) 0.0) 1) "%"
                      " (min " (time-text (vec-fold min (vec-ref xs 0) xs)) "), "
                      (bytes-text (/ (cast System.Double (record-ref m bytes)) (cast System.Double (record-ref m calls))))
                      "/call, " (int->string (vec-length xs)) " samples")))))

;; The label of a `time-it`: the body as it was written, cut short.
(: (-> (List Syntax) Syntax))
(defun (time-it-label forms)
  (def text (string-join (list->vec (list-map syntax->string forms)) " "))
  (SStr (if (> (string-count text) 50) (str (list->string (list-take (string->list text) 47)) "...") text)))

(: (-> (List %a) int (List %a)))
(defun (list-take xs n)
  (match xs
    ((Cons x more) #:when (> n 0) (cons x (list-take more (- n 1))))
    (_ (list))))

;; A batch: the body `k` times, its value kept.
(: (-> (List Syntax) Syntax))
(defun (time-it-batch forms)
  #'(fun (k)
      (let lp ((i 0L))
        (when (< i k)
          (time-it-keep (begin ,@forms))
          (lp (+ i 1L))))))

;; `(time-it body ...)`: the body measured as an anonymous benchmark, on one
;; line. `#:times n` and `#:warm-up-rounds w` ask for exactly that many runs
;; instead, untimed then timed, as `time-it` always did.
;;
;; A function, so that `(std stopwatch)`'s `time-it` is this one too: a macro
;; is only seen by the modules that import its own, and programs import
;; `time-it` from `(std stopwatch)`.
(def/macro (time-it form inject compare)
  (time-it-expansion form))

(: (-> Syntax Syntax))
(defun (time-it-expansion form)
  (syntax-match form
    ((_ #:times n #:warm-up-rounds w first body ...)
     #'(time-it-fixed ,(time-it-label (cons first body)) ,n ,w ,(time-it-batch (cons first body))))
    ((_ #:warm-up-rounds w #:times n first body ...)
     #'(time-it-fixed ,(time-it-label (cons first body)) ,n ,w ,(time-it-batch (cons first body))))
    ((_ #:times n first body ...)
     #'(time-it-fixed ,(time-it-label (cons first body)) ,n 0 ,(time-it-batch (cons first body))))
    ((_ #:warm-up-rounds w first body ...)
     #'(time-it-fixed ,(time-it-label (cons first body)) 1 ,w ,(time-it-batch (cons first body))))
    ((_ first body ...)
     #'(time-it-run ,(time-it-label (cons first body)) ,(time-it-batch (cons first body))))
    (bad (syntax-error bad "time-it takes a body: (time-it [#:times n] [#:warm-up-rounds w] expr ...)"))))

;; `(time-it #:times n #:warm-up-rounds w body ...)`: exactly `w` untimed runs
;; and `n` timed ones, as `time-it` always did, reported on the new line.
(: (-> string int int (-> long void) void))
(defun (time-it-fixed label times warm batch)
  (def run (CaseRun (batch batch) (call None) (alloc "")))
  (loop (:for _ (up-from 0 #:to warm)) (:do ((record-ref run batch) 1L)))
  (def taken (loop (:for _ (up-from 0 #:to times)) (:acc out (vecing (take-sample run 1L))) => out))
  (def xs (vec-map (fun (x) (cast System.Double (record-ref x ns))) taken))
  (def bytes (vec-fold (fun (x acc) (+ acc (record-ref x bytes))) 0L taken))
  (println (str label ": " (time-text (median xs))
                (if (> times 1) (str " (min " (time-text (vec-fold min (vec-ref xs 0) xs)) ")") "")
                ", " (bytes-text (/ (cast System.Double bytes) (cast System.Double times))) "/call, "
                (int->string times) (if (= times 1) " run" " runs"))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/bench/stats.bjo — the statistics of a benchmark run.
;;
;; Summaries of one set of timings, and the comparison of two. Pure functions
;; over vecs of doubles, with nothing of a runner in them, so that they can be
;; tested against values computed elsewhere and used without `(std bench)`.
;;
;; Every test here makes no assumption about the shape of the timings, which
;; are skewed and have long tails: medians rather than means, rank tests
;; rather than t-tests, and the bootstrap for intervals. See
;; `Docs/Benchmark-design.org`, "The statistics of a comparison".

(export
  ;; One set of samples
  median quantile mean std-dev mad tukey-outliers
  ;; Intervals
  bootstrap-ratio bootstrap-ratio/paired
  ;; Tests
  wilcoxon-p mann-whitney-p t-test-p t-critical holm normal-sf)

;; Every export is documented there; the module doc is too.
(:doc #:file "stats.bjodoc")

(import (std prelude))
(import (std maths))
(import (std random))

(import/extern
  (sort-doubles! (: Bjolang.Runtime.BjoBench.SortDoubles (-> (Array double) void)))
  (arg-sort      (: Bjolang.Runtime.BjoBench.ArgSort (-> (Array double) (Array int))))
  (clr-vec->array (: BjolangRuntime.vecsubgtarray (-> (Vec double) (Array double)))))

;; ---------------------------------------------------------------------------
;; One set of samples
;; ---------------------------------------------------------------------------

;; A sorted copy, as an array: what every order statistic below reads.
(: (-> (Vec double) (Array double)))
(defun (sorted xs)
  (def a (clr-vec->array xs))
  (sort-doubles! a)
  a)

;; The `q`-quantile of a sorted array, interpolated linearly between the two
;; samples around it: R's type 7, which is NumPy's default.
(: (-> (Array double) double double))
(defun (sorted-quantile a q)
  (def n (array-length a))
  (cond
    ((= n 0) 0.0)
    ((= n 1) (array-ref a 0))
    (else
     (def h (* q (cast System.Double (- n 1))))
     (def lo (exact-floor h))
     (def hi (if (>= (+ lo 1) n) lo (+ lo 1)))
     (def frac (- h (cast System.Double lo)))
     (+ (array-ref a lo) (* frac (- (array-ref a hi) (array-ref a lo)))))))

(: (-> (Vec double) double double))
(defun (quantile xs q) (sorted-quantile (sorted xs) q))

(: (-> (Vec double) double))
(defun (median xs) (sorted-quantile (sorted xs) 0.5))

(: (-> (Vec double) double))
(defun (mean xs)
  (if (vec-empty? xs)
      0.0
      (/ (vec-fold + 0.0 xs) (cast System.Double (vec-length xs)))))

;; The sample standard deviation, over n - 1.
(: (-> (Vec double) double))
(defun (std-dev xs)
  (def n (vec-length xs))
  (if (< n 2)
      0.0
      (let ((m (mean xs)))
        (sqrt (/ (vec-fold (fun (x acc) (+ acc (* (- x m) (- x m)))) 0.0 xs)
                 (cast System.Double (- n 1)))))))

;; The median absolute deviation from the median, unscaled.
(: (-> (Vec double) double))
(defun (mad xs)
  (def m (median xs))
  (median (vec-map (fun (x) (abs (- x m))) xs)))

;; How many samples lie more than 1.5 inter-quartile ranges outside the
;; quartiles: Tukey's fences.
(: (-> (Vec double) int))
(defun (tukey-outliers xs)
  (def a (sorted xs))
  (def q1 (sorted-quantile a 0.25))
  (def q3 (sorted-quantile a 0.75))
  (def fence (* 1.5 (- q3 q1)))
  (loop (:for x xs)
        (:acc n (counting x) #:when (or (< x (- q1 fence)) (> x (+ q3 fence))))
    => n))

;; ---------------------------------------------------------------------------
;; The bootstrap
;; ---------------------------------------------------------------------------

;; The median of `n` draws, with replacement, from `from`, using `scratch`.
(: (-> (Array double) (Array double) RandomSource double))
(defun (resampled-median from scratch rng)
  (def n (array-length from))
  (loop (:for i (up-from 0 #:to n))
        (:do (array-set! scratch i (array-ref from (random-int 0 (- n 1) #:rng rng)))))
  (sort-doubles! scratch)
  (sorted-quantile scratch 0.5))

;; The 2.5th and 97.5th percentiles of `ratios`.
(: (-> (Array double) (Tuple double double)))
(defun (interval ratios)
  (sort-doubles! ratios)
  (Tuple (sorted-quantile ratios 0.025) (sorted-quantile ratios 0.975)))

;; A 95% interval for median(b) / median(a), from `resamples` resamplings of
;; each, independently, with replacement. Seeded, so a run is repeatable.
(: (-> (Vec double) (Vec double) (#:resamples int) (#:seed ulong)
       (Tuple double double)))
(defun (bootstrap-ratio a b #:resamples 10000 #:seed 1UL)
  (def rng (make-random/seed64 seed))
  (def aa (clr-vec->array a))
  (def ba (clr-vec->array b))
  (def sa (make-array (array-length aa)))
  (def sb (make-array (array-length ba)))
  (def ratios (make-array resamples))
  (loop (:for r (up-from 0 #:to resamples))
        (:do (array-set! ratios r (/ (resampled-median ba sb rng) (resampled-median aa sa rng)))))
  (interval ratios))

;; The same for paired samples, `a[i]` and `b[i]` taken together, as an A/B
;; run's rounds are: each resampling draws whole pairs, so that what a round
;; shared stays shared.
(: (-> (Vec double) (Vec double) (#:resamples int) (#:seed ulong)
       (Tuple double double)))
(defun (bootstrap-ratio/paired a b #:resamples 10000 #:seed 1UL)
  (def rng (make-random/seed64 seed))
  (def aa (clr-vec->array a))
  (def ba (clr-vec->array b))
  (def n (min (array-length aa) (array-length ba)))
  (def sa (make-array n))
  (def sb (make-array n))
  (def ratios (make-array resamples))
  (loop (:for r (up-from 0 #:to resamples))
        (:do (loop (:for i (up-from 0 #:to n))
                   (:let j (random-int 0 (- n 1) #:rng rng))
                   (:do (array-set! sa i (array-ref aa j))
                        (array-set! sb i (array-ref ba j))))
             (sort-doubles! sa)
             (sort-doubles! sb)
             (array-set! ratios r (/ (sorted-quantile sb 0.5) (sorted-quantile sa 0.5)))))
  (interval ratios))

;; ---------------------------------------------------------------------------
;; The normal distribution
;; ---------------------------------------------------------------------------

;; erfc(x), from Numerical Recipes' Chebyshev fit: a relative error below
;; 1.2e-7 everywhere, tails included, which is what a small p-value needs.
(: (-> double double))
(defun (erfc x)
  (def z (abs x))
  (def t (/ 1.0 (+ 1.0 (* 0.5 z))))
  (def poly
    (+ -1.26551223
       (* t (+ 1.00002368
               (* t (+ 0.37409196
                       (* t (+ 0.09678418
                               (* t (+ -0.18628806
                                       (* t (+ 0.27886807
                                               (* t (+ -1.13520398
                                                       (* t (+ 1.48851587
                                                               (* t (+ -0.82215223
                                                                       (* t 0.17087277)))))))))))))))))))
  (def r (* t (exp (+ (- 0.0 (* z z)) poly))))
  (if (>= x 0.0) r (- 2.0 r)))

;; P(Z > z) for a standard normal Z.
(: (-> double double))
(defun (normal-sf z) (* 0.5 (erfc (/ z (sqrt 2.0)))))

;; ---------------------------------------------------------------------------
;; Student's t
;; ---------------------------------------------------------------------------
;;
;; For a handful of values that are each a median of many samples, such as the
;; ratios of `bjo bench --against --processes P`: few enough that a rank test
;; cannot reach a small p (2/2^P at best), and close enough to normal that the
;; t-test is fair.

;; log Γ(x) for x > 0, Lanczos' approximation (g = 7, nine terms): about 15
;; correct digits.
(: (-> double double))
(defun (log-gamma x)
  (def cs [0.99999999999980993 676.5203681218851 -1259.1392167224028 771.32342877765313
           -176.61502916214059 12.507343278686905 -0.13857109526572012
           9.9843695780195716e-6 1.5056327351493116e-7])
  (def y (- x 1.0))
  (def a (loop (:for i (up-from 1 #:to 9))
               (:acc s (folding (vec-ref cs 0) (+ s (/ (vec-ref cs i) (+ y (cast System.Double i))))))
               => s))
  (def t (+ y 7.5))
  (+ (* 0.5 (loge (* 2.0 3.141592653589793)))
     (- (* (+ y 0.5) (loge t)) t)
     (loge a)))

;; The continued fraction of the incomplete beta function, by Lentz's method,
;; as Numerical Recipes' `betacf`.
(: (-> double double double double))
(defun (beta-fraction a b x)
  (def tiny 1e-300)
  (defun (floor-tiny (: v double)) : double (if (< (abs v) tiny) tiny v))
  (def qab (+ a b))
  (def qap (+ a 1.0))
  (def qam (- a 1.0))
  (let go ((m 1) (c 1.0) (d (/ 1.0 (floor-tiny (- 1.0 (/ (* qab x) qap))))) (h (/ 1.0 (floor-tiny (- 1.0 (/ (* qab x) qap))))))
    (if (> m 300)
        h
        (let ()
          (def fm (cast System.Double m))
          (def m2 (* 2.0 fm))
          (def aa (/ (* fm (- b fm) x) (* (+ qam m2) (+ a m2))))
          (def d1 (/ 1.0 (floor-tiny (+ 1.0 (* aa d)))))
          (def c1 (floor-tiny (+ 1.0 (/ aa c))))
          (def h1 (* h (* d1 c1)))
          (def ab (/ (- 0.0 (* (+ a fm) (+ qab fm) x)) (* (+ a m2) (+ qap m2))))
          (def d2 (/ 1.0 (floor-tiny (+ 1.0 (* ab d1)))))
          (def c2 (floor-tiny (+ 1.0 (/ ab c1))))
          (def del (* d2 c2))
          (def h2 (* h1 del))
          (if (< (abs (- del 1.0)) 1e-15) h2 (go (+ m 1) c2 d2 h2))))))

;; The regularized incomplete beta function I_x(a, b).
(: (-> double double double double))
(defun (incomplete-beta a b x)
  (cond
    ((<= x 0.0) 0.0)
    ((>= x 1.0) 1.0)
    (else
     (def front (exp (+ (- (log-gamma (+ a b)) (log-gamma a) (log-gamma b))
                        (* a (loge x)) (* b (loge (- 1.0 x))))))
     (if (< x (/ (+ a 1.0) (+ a b 2.0)))
         (/ (* front (beta-fraction a b x)) a)
         (- 1.0 (/ (* front (beta-fraction b a (- 1.0 x))) b))))))

;; P(|T| > t) for Student's T with `df` degrees of freedom.
(: (-> double int double))
(defun (t-two-sided t df)
  (def v (cast System.Double df))
  (incomplete-beta (* 0.5 v) 0.5 (/ v (+ v (* t t)))))

;; The two-sided p-value of a one-sample t-test of `xs` against a mean of 0.
;; SciPy's `ttest_1samp(xs, 0)` gives the same.
(: (-> (Vec double) double))
(defun (t-test-p xs)
  (def n (vec-length xs))
  (if (< n 2)
      1.0
      (let ()
        (def m (mean xs))
        (def sd (std-dev xs))
        (cond
          ((> sd 0.0) (t-two-sided (/ (abs m) (/ sd (sqrt (cast System.Double n)))) (- n 1)))
          ((= m 0.0) 1.0)
          (else 0.0)))))

;; The t with P(|T| > t) = alpha, for `df` degrees of freedom: what a mean is
;; widened by, times its standard error, for a 1 - alpha interval. SciPy's
;; `t.ppf(1 - alpha/2, df)`.
(: (-> double int double))
(defun (t-critical alpha df)
  ;; P(|T| > t) falls as t grows; bisect on it.
  (let go ((lo 0.0) (hi 1e6) (i 0))
    (def mid (* 0.5 (+ lo hi)))
    (cond
      ((>= i 200) mid)
      ((> (t-two-sided mid df) alpha) (go mid hi (+ i 1)))
      (else (go lo mid (+ i 1))))))

;; ---------------------------------------------------------------------------
;; Ranks
;; ---------------------------------------------------------------------------

;; The ranks of `xs`, 1-based, a tie getting the mean of the ranks it spans;
;; and the sum of t³ - t over the groups of t tied values, which the tests'
;; variances are corrected by.
(: (-> (Array double) (Tuple (Array double) double)))
(defun (ranks xs)
  (def n (array-length xs))
  ;; The indices in the order of their values.
  (def order (arg-sort xs))
  (def out (make-array n #:fill 0.0))
  (def/mutable ties 0.0)
  (let go ((i 0))
    (when (< i n)
      (def v (array-ref xs (array-ref order i)))
      (def j (let run ((j i))
               (if (and (< (+ j 1) n) (= (array-ref xs (array-ref order (+ j 1))) v)) (run (+ j 1)) j)))
      (def rank (/ (cast System.Double (+ i j 2)) 2.0))
      (loop (:for k (up-from i #:to (+ j 1)))
            (:do (array-set! out (array-ref order k) rank)))
      (def t (cast System.Double (+ (- j i) 1)))
      (set! ties (+ ties (- (* t t t) t)))
      (go (+ j 1))))
  (Tuple out ties))

;; ---------------------------------------------------------------------------
;; Wilcoxon signed-rank
;; ---------------------------------------------------------------------------

;; The two-sided p-value of the Wilcoxon signed-rank test that the
;; differences `ds` are centred on zero. A zero difference is dropped, as
;; Wilcoxon's own method does. Exact, from the distribution of the signed-rank
;; sum, when at most 50 differences are left and none are tied; otherwise the
;; normal approximation, its variance corrected for ties, with no continuity
;; correction. SciPy's `wilcoxon` with its defaults gives the same.
(: (-> (Vec double) double))
(defun (wilcoxon-p ds)
  (def nonzero (vec-filter (fun (d) (not (= d 0.0))) ds))
  (def n (vec-length nonzero))
  (if (= n 0)
      1.0
      (let ()
        (def (Tuple rs ties) (ranks (clr-vec->array (vec-map abs nonzero))))
        (def w-plus
          (loop (:for i (up-from 0 #:to n))
                (:acc s (summing (array-ref rs i)) #:when (> (vec-ref nonzero i) 0.0))
            => s))
        (def total (/ (cast System.Double (* n (+ n 1))) 2.0))
        (def w (min w-plus (- total w-plus)))
        (if (and (<= n 50) (= ties 0.0))
            (wilcoxon-exact-p n w)
            (let ()
              (def nd (cast System.Double n))
              (def mu (/ (* nd (+ nd 1.0)) 4.0))
              (def var (- (/ (* nd (+ nd 1.0) (+ (* 2.0 nd) 1.0)) 24.0) (/ ties 48.0)))
              (if (<= var 0.0)
                  1.0
                  (min 1.0 (* 2.0 (normal-sf (/ (abs (- w mu)) (sqrt var)))))))))))

;; 2·P(W ≤ w) for the signed-rank sum W of n untied differences: the number
;; of subsets of 1..n with each sum, by counting.
(: (-> int double double))
(defun (wilcoxon-exact-p n w)
  (def max-sum (/ (* n (+ n 1)) 2))
  (def counts (make-array (+ max-sum 1) #:fill 0.0))
  (array-set! counts 0 1.0)
  (loop (:for k (up-from 1 #:to (+ n 1)))
        (:do (loop (:for s (down-from max-sum #:to k))
                   (:do (array-set! counts s (+ (array-ref counts s) (array-ref counts (- s k))))))))
  (def total (pow 2.0 (cast System.Double n)))
  (def limit (exact-floor w))
  (def below
    (loop (:for s (up-from 0 #:to (+ limit 1)))
          (:acc c (summing (array-ref counts s)))
      => c))
  (min 1.0 (* 2.0 (/ below total))))

;; ---------------------------------------------------------------------------
;; Mann–Whitney U
;; ---------------------------------------------------------------------------

;; The two-sided p-value of the Mann–Whitney U test that `xs` and `ys` come
;; from the same distribution: the normal approximation, with the continuity
;; correction and the variance corrected for ties. SciPy's `mannwhitneyu`
;; with `method="asymptotic"` gives the same. Benchmark runs have tens of
;; samples or more, where the approximation is good.
(: (-> (Vec double) (Vec double) double))
(defun (mann-whitney-p xs ys)
  (def n1 (vec-length xs))
  (def n2 (vec-length ys))
  (if (or (= n1 0) (= n2 0))
      1.0
      (let ()
        (def both (clr-vec->array (vec-append xs ys)))
        (def (Tuple rs ties) (ranks both))
        (def r1 (loop (:for i (up-from 0 #:to n1)) (:acc s (summing (array-ref rs i))) => s))
        (def a (cast System.Double n1))
        (def b (cast System.Double n2))
        (def n (+ a b))
        (def u1 (- r1 (/ (* a (+ a 1.0)) 2.0)))
        (def mu (/ (* a b) 2.0))
        (def var (* (/ (* a b) 12.0) (- (+ n 1.0) (/ ties (* n (- n 1.0))))))
        (if (<= var 0.0)
            1.0
            (let ((z (/ (max 0.0 (- (abs (- u1 mu)) 0.5)) (sqrt var))))
              (min 1.0 (* 2.0 (normal-sf z))))))))

;; ---------------------------------------------------------------------------
;; Many comparisons
;; ---------------------------------------------------------------------------

;; Holm's step-down correction of `ps`, in their own order: the smallest is
;; multiplied by m, the next by m - 1, and so on, each at least the one
;; before it, none above 1.
(: (-> (Vec double) (Vec double)))
(defun (holm ps)
  (def m (vec-length ps))
  (def order (arg-sort (clr-vec->array ps)))
  (def out (make-array m #:fill 1.0))
  (let go ((k 0) (running 0.0))
    (when (< k m)
      (def i (array-ref order k))
      (def adjusted (min 1.0 (max running (* (cast System.Double (- m k)) (vec-ref ps i)))))
      (array-set! out i adjusted)
      (go (+ k 1) adjusted)))
  (array->vec out))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/bench/store.bjo — a benchmark run kept in a file, and a run compared
;; with one.
;;
;; A saved run is bjodat, one tagged form, a case to a line:
;;
;;     (bench-run
;;       (version 1)
;;       (saved "2026-10-16T12:00:00Z")
;;       (machine "3f9c...")
;;       (header (build "Release") (dotnet ".NET 10.0.4") ...)
;;       (case (name "sort") (labels ("n" "1000")) (k 512) (once #f)
;;             (bytes 0.0) (samples [41213.0 41180.5 ...]))
;;       ...)
;;
;; The samples rather than a summary: a comparison needs the distribution.
;; It is `(std bench)` that measures and prints; this module knows only runs
;; as data, so that the two do not import each other.

(export SavedRun SavedCase Comparison Verdict CompareResult PairedCase
        saved-run->string parse-saved-run write-saved-run! read-saved-run
        case-key compare-runs compare-paired)

;; Every export is documented there; the module doc is too.
(:doc #:file "store.bjodoc")

(import (std prelude))
(import (std maths))
(import (std bench stats))
(import (text bjodat-core))

;; ---------------------------------------------------------------------------
;; Runs as data
;; ---------------------------------------------------------------------------

;; One benchmark at one combination of its parameters. `samples` are
;; nanoseconds per call, in the order they were taken; `bytes` is allocated
;; bytes per call.
(type (: SavedCase (Record (: name string)
                           (: labels (List (Tuple string string)))
                           (: k long)
                           (: once bool)
                           (: bytes double)
                           (: samples (Vec double)))))

;; A run: when, where, and what it measured. `header` holds the facts of
;; the machine and the build, by name: `build`, `dotnet`, `arch`, `cores`,
;; `gc-server`, `cpu`, `os`, `governor`.
(type (: SavedRun (Record (: saved string)
                          (: machine string)
                          (: header (List (Tuple string string)))
                          (: cases (List SavedCase)))))

(: (-> SavedCase string))
(defun (case-name c) (record-ref c name))

(: (-> SavedCase (List (Tuple string string))))
(defun (case-labels c) (record-ref c labels))

(: (-> SavedCase (Vec double)))
(defun (case-samples c) (record-ref c samples))

;; What a case is known by across runs: its name and its parameters, as the
;; report shows them.
(: (-> SavedCase string))
(defun (case-key c)
  (string-join (list->vec (cons (case-name c)
                                (list-map (fun (l) (match l ((Tuple p v) (str p "=" v))))
                                          (case-labels c))))
               " "))

;; ---------------------------------------------------------------------------
;; Writing
;; ---------------------------------------------------------------------------

(def file-version 1L)

(: (-> string Bjodat))
(defun (sym s) (BjoSym (string->symbol s)))

(: (-> string Bjodat Bjodat))
(defun (clause name value) (BjoList (list (sym name) value)))

(: (-> (Tuple string string) Bjodat))
(defun (pair->bjodat p) (match p ((Tuple a b) (BjoList (list (BjoStr a) (BjoStr b))))))

(: (-> SavedCase Bjodat))
(defun (case->bjodat c)
  (BjoList (list (sym "case")
                 (clause "name" (BjoStr (case-name c)))
                 (BjoList (cons (sym "labels") (list-map pair->bjodat (case-labels c))))
                 (clause "k" (BjoInt (record-ref c k)))
                 (clause "once" (BjoBool (record-ref c once)))
                 (clause "bytes" (BjoFloat (record-ref c bytes)))
                 (clause "samples" (BjoVec (vec-map BjoFloat (case-samples c)))))))

(: (-> (List (Tuple string string)) Bjodat))
(defun (header->bjodat facts)
  (BjoList (cons (sym "header")
                 (list-map (fun (f) (match f ((Tuple n v) (clause n (BjoStr v))))) facts))))

;; The text of a saved run: a case to a line, so that a diff of two files
;; says which cases changed.
(: (-> SavedRun string))
(defun (saved-run->string r)
  (def lines
    (list-append
      (list (bjodat->string (clause "version" (BjoInt file-version)))
            (bjodat->string (clause "saved" (BjoStr (record-ref r saved))))
            (bjodat->string (clause "machine" (BjoStr (record-ref r machine))))
            (bjodat->string (header->bjodat (record-ref r header))))
      (list-map (fun (c) (bjodat->string (case->bjodat c))) (record-ref r cases))))
  (str "(bench-run\n"
       (string-join (list->vec (list-map (fun (l) (str "  " l)) lines)) "\n")
       ")\n"))

(: (-> string SavedRun void))
(defun (write-saved-run! path r) (file-write-text path (saved-run->string r)))

;; ---------------------------------------------------------------------------
;; Reading
;; ---------------------------------------------------------------------------

;; The value of clause `name` among `(name value)` clauses.
(: (-> (List Bjodat) string (Option Bjodat)))
(defun (clause-ref clauses name)
  (match clauses
    ((Cons (BjoList (Cons (BjoSym s) (Cons v Nil))) more)
     (if (= (symbol->string s) name) (Some v) (clause-ref more name)))
    ((Cons _ more) (clause-ref more name))
    (Nil None)))

;; The clauses of the form `(name ...)` among them, each without its name.
(: (-> (List Bjodat) string (List (List Bjodat))))
(defun (forms-named clauses name)
  (loop (:for c clauses)
        (:acc (listing (match c ((BjoList (Cons _ rest)) rest) (_ (list))))
              #:when (match c
                       ((BjoList (Cons (BjoSym s) _)) (= (symbol->string s) name))
                       (_ #f)))))

(: (-> Bjodat (Option double)))
(defun (number d)
  (match d
    ((BjoFloat x) (Some x))
    ((BjoInt n) (Some (cast System.Double n)))
    (_ None)))

(: (-> Bjodat (Option (Tuple string string))))
(defun (string-pair d)
  (match d
    ((BjoList (Cons (BjoStr a) (Cons (BjoStr b) Nil))) (Some (Tuple a b)))
    (_ None)))

(: (-> (List (Option %a)) (Option (List %a))))
(defun (all-some xs)
  (match xs
    (Nil (Some (list)))
    ((Cons (Some x) more) (option-map (fun (rest) (cons x rest)) (all-some more)))
    ((Cons None _) None)))

(: (-> (List Bjodat) (Result string SavedCase)))
(defun (bjodat->case clauses)
  (def labels
    (match (find (fun (c) (match c ((BjoList (Cons (BjoSym s) _)) (= (symbol->string s) "labels")) (_ #f)))
                 clauses)
      ((Some (BjoList (Cons _ pairs))) (all-some (list-map string-pair pairs)))
      (_ None)))
  (match (Tuple (clause-ref clauses "name") labels (clause-ref clauses "k")
                (clause-ref clauses "once") (clause-ref clauses "bytes") (clause-ref clauses "samples"))
    ((Tuple (Some (BjoStr name)) (Some labels) (Some (BjoInt k)) (Some (BjoBool once))
            (Some bytes) (Some (BjoVec samples)))
     (match (Tuple (number bytes) (all-some (list-map number (vec->list samples))))
       ((Tuple (Some b) (Some xs))
        (Ok (SavedCase (name name) (labels labels) (k k) (once once) (bytes b) (samples (list->vec xs)))))
       (_ (Err #"the case ${name} has a sample or a size that is not a number"))))
    (_ (Err "a case lacks one of name, labels, k, once, bytes and samples, or has one of the wrong kind"))))

(: (-> (List (Result string %a)) (Result string (List %a))))
(defun (results rs)
  (match rs
    (Nil (Ok (list)))
    ((Cons (Ok x) more) (match (results more) ((Ok rest) (Ok (cons x rest))) ((Err e) (Err e))))
    ((Cons (Err e) _) (Err e))))

(def not-a-run "it is not a saved benchmark run: that is one (bench-run ...) form")

(: (-> string (Result string SavedRun)))
(defun (parse-saved-run text)
  (match (parse-bjodat text)
    ((Err e) (Err e))
    ((Ok (BjoList (Cons (BjoSym head) clauses)))
     (if (not (= (symbol->string head) "bench-run"))
         (Err not-a-run)
         (match (Tuple (clause-ref clauses "version") (clause-ref clauses "saved") (clause-ref clauses "machine"))
           ((Tuple (Some (BjoInt v)) (Some (BjoStr saved)) (Some (BjoStr machine)))
            (if (> v file-version)
                (Err #"it is version ${v} of the format, and this program reads version ${file-version}")
                (saved-run-of clauses saved machine)))
           (_ (Err "it lacks its version, the time it was saved, or its machine")))))
    (_ (Err not-a-run))))


(: (-> (List Bjodat) string string (Result string SavedRun)))
(defun (saved-run-of clauses saved machine)
  (def header
    (match (forms-named clauses "header")
      ((Cons facts _)
       (loop (:for f facts)
             (:acc (listing (match f
                              ((BjoList (Cons (BjoSym n) (Cons (BjoStr v) Nil))) (Tuple (symbol->string n) v))
                              (_ (Tuple "" ""))))
                   #:when (match f ((BjoList (Cons (BjoSym _) (Cons (BjoStr _) Nil))) #t) (_ #f)))))
      (Nil (list))))
  (match (results (list-map bjodat->case (forms-named clauses "case")))
    ((Ok cases) (Ok (SavedRun (saved saved) (machine machine) (header header) (cases cases))))
    ((Err e) (Err e))))

;; The run saved in `path`, or what is wrong with it.
(: (-> string (Result string SavedRun)))
(defun (read-saved-run path)
  (if (not (file-exists? path))
      (Err #"${path} does not exist")
      (match (try (file-read-text path) #:catch (System.Exception))
        ((Ok text)
         (result-map-err (fun (e) #"${path}: ${e}") (parse-saved-run text)))
        (_ (Err #"${path} could not be read")))))

;; ---------------------------------------------------------------------------
;; Comparing
;; ---------------------------------------------------------------------------

;; What a comparison concludes: a change that is significant and larger
;; than the threshold, one that is significant but smaller, or none shown.
(type (: Verdict (Union Faster Slower SmallChange NoChange)))

;; One case compared. `change` is the ratio of the medians less one, `now`
;; against `was`, and `lo` and `hi` its 95% interval the same way; `p` is
;; corrected for the run's number of comparisons, `p-raw` is not.
(type (: Comparison (Record (: name string)
                            (: labels (List (Tuple string string)))
                            (: was double)
                            (: now double)
                            (: change double)
                            (: lo double)
                            (: hi double)
                            (: p-raw double)
                            (: p double)
                            (: verdict Verdict)
                            ;; With several process pairs, the standard
                            ;; deviation of their log ratios: how far apart
                            ;; processes were. -1 when it was not measured.
                            (: spread double))))

;; A run compared with a saved one. `only-now` and `only-saved` are the
;; cases one side has and the other has not, by `case-key`. `warnings` are
;; the facts of the two runs that differ; `other-machine` is set when the
;; machine keys do, and then no verdict is to be believed.
(type (: CompareResult (Record (: comparisons (List Comparison))
                               (: only-now (List string))
                               (: only-saved (List string))
                               (: warnings (List string))
                               (: other-machine bool))))

(: (-> (List (Tuple string string)) string string))
(defun (fact facts name)
  (match (find (fun (f) (match f ((Tuple n _) (= n name)))) facts)
    ((Some (Tuple _ v)) v)
    (None "")))

;; The facts that a machine key leaves out and that move times: said when
;; they differ.
(def watched-facts (list (Tuple "build" "the build")
                         (Tuple "dotnet" "the .NET version")
                         (Tuple "gc-server" "server GC")
                         (Tuple "governor" "the CPU frequency governor")))

(: (-> SavedRun SavedRun (List string)))
(defun (fact-warnings saved now)
  (loop (:for w watched-facts)
        (:let (Tuple name what) w)
        (:let a (fact (record-ref saved header) name))
        (:let b (fact (record-ref now header) name))
        (:acc (listing #"${what} differs: ${a} in the saved run, ${b} now")
              #:when (not (= a b)))))

(: (-> double double double double Verdict))
(defun (verdict p change alpha threshold)
  (cond
    ((>= p alpha) NoChange)
    ((<= (abs change) threshold) SmallChange)
    ((> change 0.0) Slower)
    (else Faster)))

(: (-> (List SavedCase) string (Option SavedCase)))
(defun (find-case cases key) (find (fun (c) (= (case-key c) key)) cases))

;; `now` compared with `saved`, case by case: the Mann–Whitney U test on the
;; two sets of samples, since a saved run has no pairs, and a bootstrap
;; interval of the ratio of the medians; the p-values corrected by Holm's
;; method for the number of cases compared.
(: (-> SavedRun SavedRun double double CompareResult))
(defun (compare-runs saved now alpha threshold)
  (def saved-cases (record-ref saved cases))
  (def now-cases (record-ref now cases))
  (def pairs
    (loop (:for c now-cases)
          (:acc (listing (Tuple (match (find-case saved-cases (case-key c)) ((Some s) s) (None c)) c))
                #:when (some? (find-case saved-cases (case-key c))))))
  (def raw
    (list->vec (list-map (fun (p) (match p ((Tuple a b) (mann-whitney-p (case-samples a) (case-samples b)))))
                         pairs)))
  (def corrected (holm raw))
  (def comparisons
    (loop (:for p pairs)
          (:with i 0 (+ i 1))
          (:let (Tuple a b) p)
          (:let was (median (case-samples a)))
          (:let now-median (median (case-samples b)))
          (:let change (if (> was 0.0) (- (/ now-median was) 1.0) 0.0))
          (:let (Tuple lo hi) (bootstrap-ratio (case-samples a) (case-samples b)))
          (:acc (listing (Comparison (name (case-name b)) (labels (case-labels b))
                                     (was was) (now now-median) (change change)
                                     (lo (- lo 1.0)) (hi (- hi 1.0))
                                     (p-raw (vec-ref raw i)) (p (vec-ref corrected i))
                                     (verdict (verdict (vec-ref corrected i) change alpha threshold))
                                     (spread -1.0))))))
  (CompareResult
    (comparisons comparisons)
    (only-now (loop (:for c now-cases)
                    (:acc (listing (case-key c)) #:when (none? (find-case saved-cases (case-key c))))))
    (only-saved (loop (:for c saved-cases)
                      (:acc (listing (case-key c)) #:when (none? (find-case now-cases (case-key c))))))
    (warnings (fact-warnings saved now))
    (other-machine (not (= (record-ref saved machine) (record-ref now machine))))))

;; ---------------------------------------------------------------------------
;; Comparing in rounds: A/B
;; ---------------------------------------------------------------------------

;; One case of an A/B run: for each process pair, A's and B's per-call times,
;; a sample of each a round, in the order of the rounds.
(type (: PairedCase (Record (: name string)
                            (: labels (List (Tuple string string)))
                            (: a (Vec (Vec double)))
                            (: b (Vec (Vec double))))))

(: (-> double double double))
(defun (log-ratio a b) (if (and (> a 0.0) (> b 0.0)) (loge (/ b a)) 0.0))

;; The comparison of one case before the correction: was, now, change, the
;; interval, the raw p, and the spread between processes.
(type (: Paired (Record (: was double) (: now double) (: change double)
                        (: lo double) (: hi double) (: p double) (: spread double))))

(: (-> PairedCase Paired))
(defun (paired-one c)
  (def pairs (vec-length (record-ref c a)))
  (if (= pairs 1)
      ;; One process a side: the rounds are the pairs. The Wilcoxon test on
      ;; each round's difference of log times, and a bootstrap that draws
      ;; whole rounds.
      (let ()
        (def a (vec-ref (record-ref c a) 0))
        (def b (vec-ref (record-ref c b) 0))
        (def n (min (vec-length a) (vec-length b)))
        (def was (median a))
        (def now (median b))
        (def (Tuple lo hi) (bootstrap-ratio/paired a b))
        (Paired (was was) (now now) (change (if (> was 0.0) (- (/ now was) 1.0) 0.0))
                (lo (- lo 1.0)) (hi (- hi 1.0))
                (p (wilcoxon-p {vecing (log-ratio (vec-ref a i) (vec-ref b i)) (:for i (range 0 n))}))
                (spread -1.0)))
      ;; Several process pairs: one ratio of medians each, and the t-test
      ;; and t-interval on their logs, which a difference between processes
      ;; is in.
      (let ()
        (def logs {vecing (log-ratio (median (vec-ref (record-ref c a) j)) (median (vec-ref (record-ref c b) j)))
                          (:for j (range 0 pairs))})
        (def m (mean logs))
        (def sd (std-dev logs))
        (def half (* (t-critical 0.05 (- pairs 1)) (/ sd (sqrt (cast System.Double pairs)))))
        (Paired (was (median (vec-map median (record-ref c a))))
                (now (median (vec-map median (record-ref c b))))
                (change (- (exp m) 1.0))
                (lo (- (exp (- m half)) 1.0)) (hi (- (exp (+ m half)) 1.0))
                (p (t-test-p logs))
                (spread sd)))))

;; An A/B run's cases compared, A as `was` and B as `now`, the p-values
;; corrected by Holm's method. `only-a` and `only-b` are the cases one side
;; has and the other has not.
(: (-> (List PairedCase) (List string) (List string) double double CompareResult))
(defun (compare-paired cases only-a only-b alpha threshold)
  (def results (list->vec (list-map paired-one cases)))
  (def corrected (holm (vec-map (fun (r) (record-ref r p)) results)))
  (CompareResult
    (comparisons
      (loop (:for c cases)
            (:with i 0 (+ i 1))
            (:let r (vec-ref results i))
            (:acc (listing (Comparison (name (record-ref c name)) (labels (record-ref c labels))
                                       (was (record-ref r was)) (now (record-ref r now))
                                       (change (record-ref r change))
                                       (lo (record-ref r lo)) (hi (record-ref r hi))
                                       (p-raw (record-ref r p)) (p (vec-ref corrected i))
                                       (verdict (verdict (vec-ref corrected i) (record-ref r change) alpha threshold))
                                       (spread (record-ref r spread)))))))
    (only-now only-b)
    (only-saved only-a)
    (warnings (list))
    (other-machine #f)))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; Ordering for common .NET types.
;;
;; The time types are ordered but their names are not exported, because
;; (std datetime) exports a DateTime of its own and a module may import both.
;; Code that names the .NET type writes System.DateTime or declares an alias;
;; the implementations here apply to the type whatever it is called.
(export Guid Version)

(import (std prelude))

;; Provides Ord trait implementations for common .NET types.
(import/class
  (DateTime       (: System.DateTime))
  (DateTimeOffset (: System.DateTimeOffset))
  (TimeSpan       (: System.TimeSpan))
  (DateOnly       (: System.DateOnly))
  (TimeOnly       (: System.TimeOnly))
  (Guid           (: System.Guid))
  (Version        (: System.Version)))

(:doc #:file "clr-ord.bjodoc")

;; Implements Ord using CompareTo.
(impl (Ord DateTime)
  (defun (compare a b) (.CompareTo a b)))

(impl (Ord DateTimeOffset)
  (defun (compare a b) (.CompareTo a b)))

(impl (Ord TimeSpan)
  (defun (compare a b) (.CompareTo a b)))

(impl (Ord DateOnly)
  (defun (compare a b) (.CompareTo a b)))

(impl (Ord TimeOnly)
  (defun (compare a b) (.CompareTo a b)))

(impl (Ord Guid)
  (defun (compare a b) (.CompareTo a b)))

;; Implements Ord for Version using CompareTo.
(impl (Ord Version)
  (defun (compare a b) (.CompareTo a b)))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; Values that combine in order: the result of a run over the elements of `a`
;; followed by those of `b`.
;;
;; - `combine` is associative, not commutative.
;; - `combine` changes neither argument. The result may share structure with
;;   them.
;; - Implemented only where a type has one meaning. A type that combines
;;   some other way is a type of its own.
;; - `Array` has none: it is mutable. Write the parts into one array instead.

(import (std prelude))
(import (std hashset))
(import (std orderedset))

(:doc #:file "combine.bjodoc")

(import/extern
  (iarray-adopt (: Bjolang.Runtime.IArrayModule.Adopt (-> (Array %a) (IArray %a)))))

(export Combinable combine combine-all)

;; - `combine a b`: `a`'s elements, then `b`'s.
;; - `combine-all xs`: the values of `xs` combined in order. `xs` is not
;;   empty. Equal to `combine` from the left; an implementation overrides it
;;   to copy once.
(def/trait (Combinable %a)
  (: combine (-> %a %a %a))
  (: combine-all (-> (Vec %a) %a))
  (defun (combine-all xs)
    (loop (:for i (range 1 (vec-length xs)))
          (:acc out (folding (vec-ref xs 0) (combine out (vec-ref xs i)))))))

(impl (Combinable Unit)
  (defun (combine a b) unit)
  (defun (combine-all xs) unit))

;; Concatenation shares the nodes of both vectors.
(impl (Combinable (Vec %a))
  (defun (combine a b) (vec-append a b)))

;; From the right: each list but the last is copied once.
(impl (Combinable (List %a))
  (defun (combine a b) (list-append a b))
  (defun (combine-all xs)
    (def n (vec-length xs))
    (loop (:for i (down-from (- n 2) #:to 0))
          (:acc out (folding (vec-ref xs (- n 1)) (list-append (vec-ref xs i) out))))))

(impl (Combinable string)
  (defun (combine a b) (string-append a b))
  (defun (combine-all xs) (string-join xs "")))

(impl (Combinable (IArray %a))
  (defun (combine a b) (combine-all [a b]))
  (defun (combine-all xs)
    (def out (make-array (loop (:for x xs) (:acc n (summing (iarray-length x))))))
    (loop (:for x xs)
          (:with at 0 (+ at (iarray-length x)))
          (:do (loop (:for i (range 0 (iarray-length x)))
                     (:do (array-set! out (+ at i) (iarray-ref x i))))))
    (iarray-adopt out)))

(impl (Combinable (HashSet %a))
  (defun (combine a b) (hashset-union a b)))

(impl (Combinable (OrderedSet %a))
  (defun (combine a b) (orderedset-union a b)))

;; Tuples combine element by element.

(impl (Combinable (Tuple %a %b))
  (where (Combinable %a) (Combinable %b))
  (defun (combine x y)
    (def (a1 b1) x)
    (def (a2 b2) y)
    (Tuple (combine a1 a2) (combine b1 b2))))

(impl (Combinable (Tuple %a %b %c))
  (where (Combinable %a) (Combinable %b) (Combinable %c))
  (defun (combine x y)
    (def (a1 b1 c1) x)
    (def (a2 b2 c2) y)
    (Tuple (combine a1 a2) (combine b1 b2) (combine c1 c2))))

(impl (Combinable (Tuple %a %b %c %d))
  (where (Combinable %a) (Combinable %b) (Combinable %c) (Combinable %d))
  (defun (combine x y)
    (def (a1 b1 c1 d1) x)
    (def (a2 b2 c2 d2) y)
    (Tuple (combine a1 a2) (combine b1 b2) (combine c1 c2) (combine d1 d2))))

(impl (Combinable (Tuple %a %b %c %d %e))
  (where (Combinable %a) (Combinable %b) (Combinable %c) (Combinable %d) (Combinable %e))
  (defun (combine x y)
    (def (a1 b1 c1 d1 e1) x)
    (def (a2 b2 c2 d2 e2) y)
    (Tuple (combine a1 a2) (combine b1 b2) (combine c1 c2) (combine d1 d2) (combine e1 e2))))

(impl (Combinable (Tuple %a %b %c %d %e %f))
  (where (Combinable %a) (Combinable %b) (Combinable %c) (Combinable %d) (Combinable %e) (Combinable %f))
  (defun (combine x y)
    (def (a1 b1 c1 d1 e1 f1) x)
    (def (a2 b2 c2 d2 e2 f2) y)
    (Tuple (combine a1 a2) (combine b1 b2) (combine c1 c2) (combine d1 d2) (combine e1 e2) (combine f1 f2))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/core-macros.bjo
;;
;; Macros below the prelude. Built without it, so the prelude and the files it
;; includes can use them. The prelude re-exports each one.

(import (std syntax-match))

(:doc #:file "core-macros.bjodoc")

(export)

;; `(cond (test body ...) ... (:def pattern expr) ... (else body ...))`
;;
;; Cond requires else.
;;
;; `(:match pattern expr body ...)`, with an optional `#:when guard` after
;; `expr`, is a test that binds: it answers when `expr` matches and the guard
;; holds, and falls through to the next clause otherwise. What it binds is in
;; scope in its guard and body only.
;;
;; Two clauses do not answer anything. `(:def pattern expr)` binds, and
;; `(:do body ...)` runs something; both are reached only when every test
;; above them has failed, and what a `:def` binds is in scope in every clause
;; below it. They exist because the alternative is a new `(else ...)` and a
;; new `cond` inside it for every value a later test needs — the same
;; decisions, indented once per binding.
;;
;; A `:def` clause *is* a `def`: the sequel it swallows is the rest of the
;; cond, so a refutable pattern takes the same failure part it takes anywhere
;; else: `(:def (Ok v) (parse s) :propagate)` makes the whole cond the `Err`,
;; and `(:def (Some n) (lookup k) :default 0)` binds `n` and never answers.
(def/macro (cond form inject compare)
  (syntax-match form
    ((_ clause ...) (cond-arms clause form))
    (bad (syntax-error bad "cond takes clauses: (cond (test body ...) ... (else body ...))"))))

(: (-> (List Syntax) Syntax Syntax))
(defun (cond-arms clauses whole)
  (match clauses
    (Nil
     (syntax-error whole "cond needs a final (else ...) clause: every arm answers something, so there has to be a last answer."))
    ((Cons clause rest)
     (syntax-match clause
       ;; Binds. The rest of the cond is the body the `def` swallows, which is
       ;; what puts the name in scope below and nowhere else.
       ((:def pattern expr fail ...)
        #'(let () (def ,pattern ,expr ,@fail) ,(cond-arms rest whole)))
       ((:def junk ...)
        (syntax-error clause "(:def ...) binds: (:def name expr), or (:def pattern expr failure-part), such as :default value or :propagate."))
       ;; Answers when `expr` matches, and falls through to the next clause
       ;; when it does not. Unlike `:def` it swallows nothing: what it binds
       ;; is in scope in its guard and body only. The rest of the cond is the
       ;; fallback arm, so it is written once.
       ((:match pattern expr #:when guard body ...)
        #'(match ,expr
            (,pattern #:when ,guard ,(one-form body clause))
            (,(SSym (string->symbol "_")) ,(cond-arms rest whole))))
       ((:match pattern expr body ...)
        #'(match ,expr
            (,pattern ,(one-form body clause))
            (,(SSym (string->symbol "_")) ,(cond-arms rest whole))))
       ((:match junk ...)
        (syntax-error clause "(:match ...) is (:match pattern expr body ...), or (:match pattern expr #:when guard body ...)."))
       ;; Runs.
       ((:do first body ...)
        #'(let () ,first ,@body ,(cond-arms rest whole)))
       ((:do)
        (syntax-error clause "(:do ...) needs something to run."))
       ;; Else clause. It always answers, so a clause after it could never be
       ;; reached.
       (('else body ...)
        (match rest
          (Nil (one-form body clause))
          ((Cons after _)
           (syntax-error after "This clause comes after (else ...), so it is never reached. Move it above the else, or remove it."))))
       ((test body ...) #'(if ,test ,(one-form body clause) ,(cond-arms rest whole)))
       (bad (syntax-error bad "A cond clause is (test body ...), (:match pattern expr body ...) to answer on a match, (:def pattern expr) to bind, (:do body ...) to run something, or (else body ...) for the last one."))))))

;; Several forms as one.
(: (-> (List Syntax) Syntax Syntax))
(defun (one-form body whole)
  (match body
    (Nil (syntax-error whole "This clause has nothing to do."))
    ((Cons only Nil) only)
    (_ #'(let () ,@body))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/crypto/hash.bjo — digests, HMAC and password hashing.
;;
;; The work is done by .NET's System.Security.Cryptography, called directly.
;; A string is hashed as its UTF-8 bytes.
;;
;; Passwords are hashed with PBKDF2, because it is the password hash that .NET
;; has. The result is a PHC string that holds the algorithm, the iteration
;; count and the salt, so a hash made with old parameters still verifies.
;; PBKDF2 at 600000 iterations takes a core for a few hundred milliseconds. In
;; a bjoroutine, the password functions run on a thread of their own, so the
;; fiber pool is not blocked.
;;
;; `derive-key` is PBKDF2 with a salt you give: the same bytes each time.

(export
  ;; Transparent: Md5, Sha1, Sha256, Sha384, Sha512, Sha3-256, Sha3-384, Sha3-512.
  HashAlgorithm
  hash-algorithm-supported? digest-length
  ;; One-shot
  hash-bytes hash-string hash-file
  hmac-bytes hmac-string
  ;; Incremental
  Hasher make-hasher make-hmac-hasher hasher-add! hasher-add-string! hasher-finish!
  ;; Comparing and printing
  constant-time-equal? bytes->hex hex->bytes
  ;; Passwords
  password-hash password-verify password-needs-rehash?
  default-password-iterations
  ;; Keys
  derive-key)

(import (std prelude))
(import (only (std random) random-bytes!))

(:doc #:file "hash.bjodoc")

;; MD5 and SHA-1 are broken for collision resistance. They are here for
;; protocols and checksums that need them.
(type (: HashAlgorithm (Union Md5 Sha1 Sha256 Sha384 Sha512 Sha3-256 Sha3-384 Sha3-512)))

(import/class
  (Hasher (: System.Security.Cryptography.IncrementalHash))
  (HashAlgorithmName (: System.Security.Cryptography.HashAlgorithmName (-> string HashAlgorithmName)))
  (Stream (: System.IO.Stream))
  ;; Path, mode, access, share, buffer size, and whether reads are async.
  (FileStream (: System.IO.FileStream (-> string FileMode FileAccess FileShare int bool FileStream)))
  (FileMode (: System.IO.FileMode))
  (FileAccess (: System.IO.FileAccess))
  (FileShare (: System.IO.FileShare))
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

(import/extern
  (clr-sha3-256?       (: System.Security.Cryptography.SHA3_256.IsSupported #:get))
  (clr-sha3-384?       (: System.Security.Cryptography.SHA3_384.IsSupported #:get))
  (clr-sha3-512?       (: System.Security.Cryptography.SHA3_512.IsSupported #:get))
  (clr-hash            (: System.Security.Cryptography.CryptographicOperations.HashData
                          (-> HashAlgorithmName (Array byte) (Array byte))))
  (clr-hash-stream     (: System.Security.Cryptography.CryptographicOperations.HashData
                          (-> HashAlgorithmName Stream (Array byte)) #:blocking))
  (clr-hash-stream-await (: System.Security.Cryptography.CryptographicOperations.HashDataAsync
                            (-> HashAlgorithmName Stream (Array byte)) #:async))
  (clr-hmac            (: System.Security.Cryptography.CryptographicOperations.HmacData
                          (-> HashAlgorithmName (Array byte) (Array byte) (Array byte))))
  (clr-make-hasher     (: System.Security.Cryptography.IncrementalHash.CreateHash
                          (-> HashAlgorithmName Hasher)))
  (clr-make-hmac       (: System.Security.Cryptography.IncrementalHash.CreateHMAC
                          (-> HashAlgorithmName (Array byte) Hasher)))
  (clr-hasher-add      (: System.Security.Cryptography.IncrementalHash.AppendData (-> Hasher (Array byte) void)))
  ;; The digest; the hasher is reset, and an HMAC hasher keeps its key.
  (clr-hasher-finish   (: System.Security.Cryptography.IncrementalHash.GetHashAndReset (-> Hasher (Array byte))))
  (clr-fixed-equal     (: System.Security.Cryptography.CryptographicOperations.FixedTimeEquals
                          (-> (RoSpan byte) (RoSpan byte) bool)))
  (clr-pbkdf2          (: System.Security.Cryptography.Rfc2898DeriveBytes.Pbkdf2
                          (-> (Array byte) (Array byte) int HashAlgorithmName int (Array byte))))
  (clr-to-hex          (: System.Convert.ToHexStringLower (-> (Array byte) string)))
  ;; Raises `FormatException` for an odd length or a character that is not a hex digit.
  (clr-from-hex        (: System.Convert.FromHexString (-> string (Array byte))))
  (clr-to-base64       (: System.Convert.ToBase64String (-> (Array byte) string)))
  ;; Raises `FormatException` for text that is not base64.
  (clr-from-base64     (: System.Convert.FromBase64String (-> string (Array byte)))))

;; .net algorithm names. 
(: (-> HashAlgorithm string))
(defun (algorithm-name a)
  (match a
    (Md5      "MD5")
    (Sha1     "SHA1")
    (Sha256   "SHA256")
    (Sha384   "SHA384")
    (Sha512   "SHA512")
    (Sha3-256 "SHA3-256")
    (Sha3-384 "SHA3-384")
    (Sha3-512 "SHA3-512")))

(: (-> HashAlgorithm HashAlgorithmName))
(defun (clr-name a) (HashAlgorithmName. (algorithm-name a)))

;; SHA-3 needs OpenSSL 1.1.1 or later, or Windows 11. The others are always there.
(: (-> HashAlgorithm bool))
(defun (hash-algorithm-supported? a)
  (match a
    (Sha3-256 clr-sha3-256?)
    (Sha3-384 clr-sha3-384?)
    (Sha3-512 clr-sha3-512?)
    (_ #t)))

;; In bytes.
(: (-> HashAlgorithm int))
(defun (digest-length a)
  (match a
    (Md5      16)
    (Sha1     20)
    (Sha256   32)
    (Sha384   48)
    (Sha512   64)
    (Sha3-256 32)
    (Sha3-384 48)
    (Sha3-512 64)))

;; ---------------------------------------------------------------------------
;; One-shot
;; ---------------------------------------------------------------------------

(: (-> (Array byte) (#:algorithm HashAlgorithm) (Array byte)))
(defun (hash-bytes data #:algorithm Sha256)
  (clr-hash (clr-name algorithm) data))

(: (-> string (#:algorithm HashAlgorithm) (Array byte)))
(defun (hash-string text #:algorithm Sha256)
  (clr-hash (clr-name algorithm) (string->bytes text)))

;; The file opened for reading, other readers allowed, as `File.OpenRead`
;; opens it. With `async`, reads do not park a thread.
(: (-> string bool FileStream))
(defun (open-for-hash path async)
  (FileStream. path FileMode.Open FileAccess.Read FileShare.Read 4096 async))

;; The file is read a buffer at a time, so a large file is never all in memory.
;; Raises as opening the file does, for example `FileNotFoundException`.
(: (-> string (#:algorithm HashAlgorithm) (Array byte)))
(defbjouble (hash-file path #:algorithm Sha256)
  (#:sync (let ((file (open-for-hash path #f)))
            (try (clr-hash-stream (clr-name algorithm) (cast Stream file))
                 #:finally (.Dispose file))))
  (#:bjo  (let ((file (open-for-hash path #t)))
            (try (clr-hash-stream-await (clr-name algorithm) (cast Stream file))
                 #:finally (.Dispose file)))))

(: (-> (Array byte) (Array byte) (#:algorithm HashAlgorithm) (Array byte)))
(defun (hmac-bytes key data #:algorithm Sha256)
  (clr-hmac (clr-name algorithm) key data))

(: (-> (Array byte) string (#:algorithm HashAlgorithm) (Array byte)))
(defun (hmac-string key text #:algorithm Sha256)
  (clr-hmac (clr-name algorithm) key (string->bytes text)))

;; ---------------------------------------------------------------------------
;; Incremental
;; ---------------------------------------------------------------------------
;;
;; A hasher is mutable. Do not share one between fibers without a lock.

(: (-> (#:algorithm HashAlgorithm) Hasher))
(defun (make-hasher #:algorithm Sha256)
  (clr-make-hasher (clr-name algorithm)))

(: (-> (Array byte) (#:algorithm HashAlgorithm) Hasher))
(defun (make-hmac-hasher key #:algorithm Sha256)
  (clr-make-hmac (clr-name algorithm) key))

(: (-> Hasher (Array byte) void))
(defun (hasher-add! h data) (clr-hasher-add h data))

(: (-> Hasher string void))
(defun (hasher-add-string! h text) (clr-hasher-add h (string->bytes text)))

;; The digest of all that was added. The hasher is then empty, and can be used
;; again. An HMAC hasher keeps its key.
(: (-> Hasher (Array byte)))
(defun (hasher-finish! h) (clr-hasher-finish h))

;; ---------------------------------------------------------------------------
;; Comparing and printing
;; ---------------------------------------------------------------------------

;; Use this to compare a secret, such as an HMAC tag or a token. The time does
;; not depend on where the arrays differ. Arrays of different lengths are not
;; equal, and the time can show the lengths.
(: (-> (Array byte) (Array byte) bool))
(defun (constant-time-equal? a b) (clr-fixed-equal (array->ro-span a) (array->ro-span b)))

;; Lower case.
(: (-> (Array byte) string))
(defun (bytes->hex bytes) (clr-to-hex bytes))

;; Either case. `None` for an odd length or a character that is not a hex digit.
(: (-> string (Option (Array byte))))
(defun (hex->bytes text)
  (result->option (try (clr-from-hex text) #:catch (System.FormatException))))

;; ---------------------------------------------------------------------------
;; Passwords
;; ---------------------------------------------------------------------------

;; The OWASP recommendation for PBKDF2-HMAC-SHA256 in 2023.
(: int)
(def default-password-iterations 600000)

;; 16 bytes is the size that NIST SP 800-132 recommends as a minimum.
(: int)
(def salt-length 16)

;; A stored password is a PHC string: `$pbkdf2-sha256$i=600000$<salt>$<hash>`.
;; The salt and the hash are standard base64 without padding, as the PHC
;; format specifies.

(: (-> string %a))
(defun (refuse message) (raise (cast Exception (ArgumentException. message))))

;; The PHC id of a password algorithm. Raises `ArgumentException` for an
;; algorithm that is not SHA-256, SHA-384 or SHA-512.
(: (-> HashAlgorithm string))
(defun (phc-id a)
  (match a
    (Sha256 "pbkdf2-sha256")
    (Sha384 "pbkdf2-sha384")
    (Sha512 "pbkdf2-sha512")
    (_ (refuse (str "Passwords are hashed with SHA256, SHA384 or SHA512, not "
                    (algorithm-name a) ".")))))

(: (-> string (Option HashAlgorithm)))
(defun (algorithm-of-phc-id id)
  (cond
    ((= id "pbkdf2-sha256") (Some Sha256))
    ((= id "pbkdf2-sha384") (Some Sha384))
    ((= id "pbkdf2-sha512") (Some Sha512))
    (else None)))

;; The length of a password hash: the digest length. Raises
;; `ArgumentException` for an algorithm that is not a password algorithm.
(: (-> HashAlgorithm int))
(defun (password-hash-length a)
  (match a
    ((or Sha256 Sha384 Sha512) (digest-length a))
    (_ (refuse (str "Not a password algorithm: " (algorithm-name a) ".")))))

;; Base64 without the `=` padding at the end.
(: (-> (Array byte) string))
(defun (to-base64 bytes)
  (def padding (match (% (array-length bytes) 3) (0 0) (1 2) (_ 1)))
  (string-drop-right (clr-to-base64 bytes) padding))

;; Base64 with or without its padding. `None` for text that is not base64.
(: (-> string (Option (Array byte))))
(defun (from-base64 text)
  (def padded
    (match (% (string-length text) 4)
      (0 (Some text))
      (2 (Some (str text "==")))
      (3 (Some (str text "=")))
      (_ None)))
  (match padded
    ((Some t)
     (result->option (try (clr-from-base64 t) #:catch (System.FormatException))))
    (None None)))

;; One or more ASCII digits, as an int of at least 1. `None` for anything
;; else: a sign, a space, or a number too big for an int.
(: (-> string (Option int)))
(defun (positive-int text)
  (def bytes (string->bytes text))
  (def n (array-length bytes))
  (let go ((i 0) (v 0L))
    (cond
      ((= i n) (if (and (> n 0) (>= v 1L)) (Some (cast int v)) None))
      (else
       (def b (cast int (array-ref bytes i)))
       (def next (+ (* v 10L) (cast long (- b 48))))
       (if (and (>= b 48) (<= b 57) (<= next 2147483647L))
           (go (+ i 1) next)
           None)))))

(: (-> string (Array byte) HashAlgorithm int int (Array byte)))
(defun (pbkdf2 password salt algorithm iterations length)
  (clr-pbkdf2 (string->bytes password) salt iterations (clr-name algorithm) length))

;; What a PHC string holds.
(type (: Phc (Record (: phc-algorithm HashAlgorithm)
                     (: phc-iterations int)
                     (: phc-salt (Array byte))
                     (: phc-hash (Array byte)))))

;; Raises `ArgumentException` when `encoded` is not a PHC string of PBKDF2
;; with SHA-256, SHA-384 or SHA-512, at least 1 iteration, and a salt and a
;; hash that are not empty.
(: (-> string Phc))
(defun (parse-phc encoded)
  (def parts (string-split encoded "$"))
  (def fail "Not a password hash: expected $pbkdf2-<sha256|sha384|sha512>$i=<n>$<salt>$<hash>.")
  ;; The leading `$` makes the first part empty.
  (if (not (and (= (vec-length parts) 5) (= (vec-ref parts 0) "")))
      (refuse fail)
      (let ((algorithm (algorithm-of-phc-id (vec-ref parts 1)))
            (count (if (string-starts-with? (vec-ref parts 2) "i=")
                       (positive-int (string-drop (vec-ref parts 2) 2))
                       None))
            (salt (from-base64 (vec-ref parts 3)))
            (hash (from-base64 (vec-ref parts 4))))
        (match (Tuple algorithm count salt hash)
          ((Tuple (Some a) (Some n) (Some s) (Some h))
           (if (or (= (array-length s) 0) (= (array-length h) 0))
               (refuse fail)
               (Phc (phc-algorithm a) (phc-iterations n) (phc-salt s) (phc-hash h))))
          (_ (refuse fail))))))

;; Raises `ArgumentException` for an algorithm that is not a password
;; algorithm, fewer than 1 iteration, or a salt shorter than 8 bytes.
(: (-> string (Array byte) HashAlgorithm int string))
(defun (encode-password-hash password salt algorithm iterations)
  (def id (phc-id algorithm))
  (when (< iterations 1) (refuse "The iteration count must be at least 1."))
  (when (< (array-length salt) 8) (refuse "The salt must be at least 8 bytes."))
  (def digest (pbkdf2 password salt algorithm iterations (password-hash-length algorithm)))
  (str "$" id "$i=" (int->string iterations) "$" (to-base64 salt) "$" (to-base64 digest)))

(: (-> string string bool))
(defun (password-verify-now password stored)
  (def phc (parse-phc stored))
  (def expected (record-ref phc phc-hash))
  (def candidate (pbkdf2 password (record-ref phc phc-salt) (record-ref phc phc-algorithm)
                         (record-ref phc phc-iterations) (array-length expected)))
  (constant-time-equal? candidate expected))

;; Raises `ArgumentException` for an algorithm that is not a password
;; algorithm, fewer than 1 iteration, or a length less than 1.
(: (-> string (Array byte) HashAlgorithm int int (Array byte)))
(defun (derive-key-now password salt algorithm iterations length)
  (ignore (password-hash-length algorithm))
  (when (< iterations 1) (refuse "The iteration count must be at least 1."))
  (when (< length 1) (refuse "The key length must be at least 1."))
  (pbkdf2 password salt algorithm iterations length))

(: (-> string HashAlgorithm int string))
(defun (password-hash-now password algorithm iterations)
  (def salt (make-array salt-length))
  (random-bytes! salt)
  (encode-password-hash password salt algorithm iterations))

;; The value of a thread's result, or its exception raised again.
(: (-> (Result Exception %a) %a))
(defun (unwrap-thread r)
  (match r
    ((Ok v) v)
    ((Err e) (raise e))))

(: (-> string (#:algorithm HashAlgorithm) (#:iterations int) string))
(defbjouble (password-hash password #:algorithm Sha256 #:iterations default-password-iterations)
  (#:sync (password-hash-now password algorithm iterations))
  (#:bjo  (unwrap-thread (sync (spawn/thread #(password-hash-now password algorithm iterations))))))

;; Raises `ArgumentException` when `stored` is not a hash from `password-hash`.
;; This is an error in the stored data, not a wrong password.
(: (-> string string bool))
(defbjouble (password-verify password stored)
  (#:sync (password-verify-now password stored))
  (#:bjo  (unwrap-thread (sync (spawn/thread #(password-verify-now password stored))))))

;; True when `stored` was made with another algorithm or with fewer iterations.
;; Check this after a correct `password-verify`, and store a new hash of the
;; password if it is true.
(: (-> string (#:algorithm HashAlgorithm) (#:iterations int) bool))
(defun (password-needs-rehash? stored #:algorithm Sha256 #:iterations default-password-iterations)
  (def phc (parse-phc stored))
  (or (not (= (algorithm-name (record-ref phc phc-algorithm)) (algorithm-name algorithm)))
      (< (record-ref phc phc-iterations) iterations)
      (not (= (array-length (record-ref phc phc-hash)) (password-hash-length algorithm)))))

;; ---------------------------------------------------------------------------
;; Keys
;; ---------------------------------------------------------------------------

;; PBKDF2 bytes of `password` with the salt given. Same password and salt,
;; same bytes. Runs on a thread of its own in a bjoroutine, as
;; `password-hash` does.
(: (-> string (Array byte) (#:algorithm HashAlgorithm) (#:iterations int) (#:length int) (Array byte)))
(defbjouble (derive-key password salt #:algorithm Sha256 #:iterations default-password-iterations
                        #:length 32)
  (#:sync (derive-key-now password salt algorithm iterations length))
  (#:bjo  (unwrap-thread
           (sync (spawn/thread #(derive-key-now password salt algorithm iterations length))))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/datetime.bjo — dates, times, instants, zones and durations.
;;
;; Every value lies between 0001-01-01T00:00 and 9999-12-31T23:59:59.9999999,
;; at a precision of 100 ns. A result outside that range raises an
;; OverflowException from a function that returns the value itself, and is None
;; from a constructor, a parser or an `/opt` function. Nothing is clamped
;; silently, except month arithmetic landing past the end of a month
;; (2026-01-31 + 1 month = 2026-02-28) and time-of-day arithmetic, which wraps
;; at midnight.
;;
;; Text is formatted and parsed here, digit by digit, rather than by .NET, so
;; no culture is involved.
;;
;; Its reference is its docs, in datetime.bjodoc.
;;
;; Open:
;; - ->weekday answers an int, so (= (->weekday d) 0) compiles and is never
;;   true. A Weekday union, Monday to Sunday, would make it a type error, at
;;   the cost of arithmetic on weekdays.
;; - date->text writes (week) beside (year) without a word, so
;;   '((year 4) "-W" (week 2)) writes 2027-01-01 as 2027-W53, a week 2027
;;   does not have, where text->date refuses that format. Beside (month) and
;;   (day) the week is right, so the writer cannot refuse it by itself.
;; - A format is checked each time it reads, as a list cannot carry the
;;   check, which is about a third of a reader's time; a reader costs about
;;   ten times an iso8601-> one. Reading has no optional parts either, such
;;   as seconds that may be missing; the ISO 8601 readers have those.
;; - utc-datetime->instant takes any DateTime, so a local time goes in and is
;;   read as UTC without a word.

(import (std prelude))

(import/class
  (ClrDate (: System.DateOnly (-> int int int ClrDate)))
  (ClrTime (: System.TimeOnly (-> long ClrTime)))
  (ClrDateTime (: System.DateTime (-> long DateTimeKind ClrDateTime)))
  (DateTimeKind (: System.DateTimeKind))
  (ClrTimeSpan (: System.TimeSpan (-> long ClrTimeSpan)))
  (TimeZoneInfo (: System.TimeZoneInfo))
  (OverflowException (: System.OverflowException (-> string OverflowException)))
  (ArgumentException (: System.ArgumentException (-> string ArgumentException)))
  (ClrLock (: System.Threading.Lock (-> ClrLock)))
  ;; A .NET StringBuilder of its own, because the prelude's cannot be given a
  ;; capacity, and it takes a UTF-16 unit, which a digit is.
  (TextBuilder (: System.Text.StringBuilder (-> int TextBuilder))))

;; A body an importer may copy (a trait method here, or a constrained
;; function) names .NET members through these aliases rather than through
;; helpers of this module: an alias travels under a name nobody can write,
;; while a helper would be exported under its own.
(import/extern
  (clr-date-from-day-number (: System.DateOnly.FromDayNumber (-> int ClrDate)))
  (clr-date-from-datetime (: System.DateOnly.FromDateTime (-> ClrDateTime ClrDate)))
  (clr-time-from-datetime (: System.TimeOnly.FromDateTime (-> ClrDateTime ClrTime)))
  (clr-time-add (: System.TimeOnly.Add (-> ClrTime ClrTimeSpan ClrTime)))
  (clr-time-hour (: System.TimeOnly.Hour (-> ClrTime int) #:get))
  (clr-time-minute (: System.TimeOnly.Minute (-> ClrTime int) #:get))
  (clr-time-second (: System.TimeOnly.Second (-> ClrTime int) #:get))
  (clr-time-millisecond (: System.TimeOnly.Millisecond (-> ClrTime int) #:get))
  (clr-time-microsecond (: System.TimeOnly.Microsecond (-> ClrTime int) #:get))
  (clr-time-nanosecond (: System.TimeOnly.Nanosecond (-> ClrTime int) #:get))
  (clr-date-day-number (: System.DateOnly.DayNumber (-> ClrDate int) #:get))
  (clr-date-year (: System.DateOnly.Year (-> ClrDate int) #:get))
  (clr-date-month (: System.DateOnly.Month (-> ClrDate int) #:get))
  (clr-date-day (: System.DateOnly.Day (-> ClrDate int) #:get))
  (clr-date-day-of-year (: System.DateOnly.DayOfYear (-> ClrDate int) #:get))
  (clr-date-add-months (: System.DateOnly.AddMonths (-> ClrDate int ClrDate)))
  (clr-date-add-years (: System.DateOnly.AddYears (-> ClrDate int ClrDate)))
  (clr-days-in-month (: System.DateTime.DaysInMonth (-> int int int)))
  (clr-time-ticks (: System.TimeOnly.Ticks (-> ClrTime long) #:get))
  (clr-datetime-ticks (: System.DateTime.Ticks (-> ClrDateTime long) #:get))
  (clr-timespan-ticks (: System.TimeSpan.Ticks (-> ClrTimeSpan long) #:get))
  (clr-tz-try-find
    (: System.TimeZoneInfo.TryFindSystemTimeZoneById
       (-> string (out TimeZoneInfo) (Option TimeZoneInfo))))
  (clr-tz-utc (: System.TimeZoneInfo.Utc #:get))
  (clr-tz-id (: System.TimeZoneInfo.Id (-> TimeZoneInfo string) #:get))
  (clr-tz-has-iana-id (: System.TimeZoneInfo.HasIanaId (-> TimeZoneInfo bool) #:get))
  (clr-tz-utc-offset (: System.TimeZoneInfo.GetUtcOffset (-> TimeZoneInfo ClrDateTime ClrTimeSpan)))
  (clr-at-utc (: BjolangRuntime.atsubtime (-> ClrDateTime (Event Unit))))
  (clr-lock-enter (: System.Threading.Lock.Enter (-> ClrLock void)))
  (clr-lock-exit (: System.Threading.Lock.Exit (-> ClrLock void)))
  (clr-add-unit (: System.Text.StringBuilder.Append (-> TextBuilder System.Char TextBuilder)))
  (clr-add-text (: System.Text.StringBuilder.Append (-> TextBuilder string TextBuilder)))
  (clr-builder-text (: System.Text.StringBuilder.ToString (-> TextBuilder string))))

;; Every export is documented there; the module doc is too.
(:doc #:file "datetime.bjodoc")

;; The prelude's, which timers take too. Offered here as well, so that an
;; import of the time types from this module names it as before.
(re-export Duration)

(export
  ;; Types
  Date Time DateTime Instant TimeZone Moment
  DatePeriod Period Gap Overlap Resolver
  ;; Traits
  HasDate ->date HasTime ->time HasInstant ->instant
  DateArith +date-period DateDiff ->date-in-zone-of
  TimeArith +duration
  ;; Construction
  date time datetime date+time->datetime ->datetime
  duration date-period period
  ;; Zones and moments
  timezone utc timezone-id timezone-offset-at
  resolver datetime->moment datetime->moment/opt instant->moment
  +date-period/resolver -date-period/resolver
  ->utc-offset ->timezone adjust-timezone
  ;; Instants
  instant->utc-datetime utc-datetime->instant
  instant->unix-seconds unix-seconds->instant
  instant->unix-milliseconds unix-milliseconds->instant ->unix
  instant->unix-microseconds unix-microseconds->instant
  date->unix-days unix-days->date
  ;; The clock
  system-clock-in clock-in fixed-clock manual-clock clock-advance!
  clock-timezone now now/moment today
  ;; Timers
  at-time
  ;; Accessors
  ->year ->month ->day ->weekday ->yday
  ->week ->week-year weeks-in-year week-date
  ->hours ->minutes ->seconds ->milliseconds ->nanoseconds
  ;; Arithmetic
  +years +months +weeks +days -years -months -weeks -days -date-period
  +hours +minutes +seconds +milliseconds
  -hours -minutes -seconds -milliseconds -duration
  +period -period
  duration->hours duration->minutes duration->seconds duration->milliseconds
  duration-negate
  ;; Differences
  years-between months-between weeks-between days-between
  duration-between hours-between minutes-between seconds-between milliseconds-between
  ;; Text
  iso8601->date iso8601->time iso8601->datetime iso8601->instant iso8601->moment
  iso8601->duration iso8601->date-period iso8601->period
  instant->http-date http-date->instant
  ;; Text of your own
  ;; DatePart is a grammar. Its export also exports DateField, DateNumber,
  ;; DateBareNumber, RomanNumber, TimePart, TimeField, TimeBareNumber,
  ;; DateTimePart, DateTimeField, MomentPart and InstantPart.
  DatePart DateNames
  date->text time->text datetime->text moment->text instant->text
  text->date text->time text->datetime text->instant text->moment text->moment/resolver
  english-names swedish-names current-date-names)


;; ===========================================================================
;; Types
;; ===========================================================================

;; The value types are opaque so that every value comes from a function that
;; checked it: a Moment's offset is always one its zone has at that time.
(type (: Date #:opaque (Struct (: v ClrDate))))
(type (: Time #:opaque (Struct (: v ClrTime))))
;; Kind is always Unspecified.
(type (: DateTime #:opaque (Struct (: v ClrDateTime))))
;; UTC ticks, not a UTC DateTime, because DateTime equality ignores Kind.
(type (: Instant #:opaque (Struct (: ticks long))))
(type (: TimeZone #:opaque (Struct (: tz TimeZoneInfo))))
(type (: Moment #:opaque (Struct (: local DateTime) (: offset Duration) (: zone TimeZone))))

;; Any mix of amounts is a valid period, so these are transparent. Structs,
;; so that the one +days builds is not an allocation.
(type (: DatePeriod (Struct (: years int) (: months int) (: weeks int) (: days int))))
(type (: Period (Struct (: calendar DatePeriod) (: exact Duration))))

;; What datetime->moment does with a local time that a zone skips (a gap) or
;; has twice (an overlap).
(type (: Gap (Union GapPush GapNextValid)))
(type (: Overlap (Union OverlapEarlier OverlapLater)))
(type (: Resolver (Record (: gap Gap) (: overlap Overlap))))


;; ===========================================================================
;; Ranges
;; ===========================================================================

(: long)
(def ticks-per-day 864000000000L)
(: long)
(def ticks-per-hour 36000000000L)
(: long)
(def ticks-per-minute 600000000L)
(: long)
(def ticks-per-second 10000000L)
(: long)
(def ticks-per-millisecond 10000L)

;; 9999-12-31T23:59:59.9999999 in ticks, and 9999-12-31 as a day number.
(: long)
(def max-ticks 3155378975999999999L)
(: long)
(def max-day-number 3652058L)

;; 1970-01-01T00:00Z in ticks.
(: long)
(def unix-epoch-ticks 621355968000000000L)

;; Every Duration stays within ±long-max ticks, about ±29 227 years. Leaving
;; out long's minimum gives every duration a negation, so each one prints and
;; reads back.
(: long)
(def long-max 9223372036854775807L)

(: (-> int long))
(defun (long-of n) (cast System.Int64 n))

(: (-> long int))
(defun (int-of n) (cast System.Int32 n))

(: (-> string %a))
(defun (range-error what)
  (raise (cast Exception (OverflowException. (str "(std datetime): " what)))))

(: (-> string %a))
(defun (calendar-error what)
  (range-error (str what " is outside 0001-01-01T00:00 to 9999-12-31T23:59:59.9999999")))

(: (-> %a))
(defun (duration-error)
  (range-error "a duration is outside ±29 227 years (±9 223 372 036 854 775 807 ticks)"))

(: (-> long bool))
(defun (in-calendar? t) (and (>= t 0L) (<= t max-ticks)))

;; n × unit for a positive unit, or None past the range of a Duration.
(: (-> long long (Option long)))
(defun (checked-mul n unit)
  (def limit (/ long-max unit))
  (if (or (> n limit) (< n (- limit))) None (Some (* n unit))))

;; a + b, or None past the range of a Duration.
(: (-> long long (Option long)))
(defun (checked-add a b)
  (if (if (> b 0L) (> a (- long-max b)) (< a (- (- long-max) b)))
      None
      (Some (+ a b))))

(: (-> long long long))
(defun (units->ticks n unit)
  (match (checked-mul n unit)
    ((Some t) t)
    (None (duration-error))))

(: (-> long long long))
(defun (plus-ticks a b)
  (match (checked-add a b)
    ((Some t) t)
    (None (duration-error))))

;; A tick count moved by a duration, for DateTime and Instant. The sum can
;; wrap only above long's maximum, and then it lands below zero, so the one
;; range check catches it.
(: (-> long long string long))
(defun (shift-ticks t d what)
  (def moved (+ t d))
  (if (in-calendar? moved) moved (calendar-error what)))


;; ===========================================================================
;; Construction
;; ===========================================================================

(: (-> long DateTime))
(defun (ticks->datetime t) (DateTime (v (ClrDateTime. t DateTimeKind.Unspecified))))

(: (-> DateTime long))
(defun (datetime-ticks dt) (clr-datetime-ticks (struct-ref dt v)))

(: (-> Date long))
(defun (day-number d) (long-of (clr-date-day-number (struct-ref d v))))

(: (-> long Date))
(defun (day-number->date n) (Date (v (clr-date-from-day-number (int-of n)))))

(: (-> int int int bool))
(defun (valid-date? y m d)
  (and (>= y 1) (<= y 9999) (>= m 1) (<= m 12) (>= d 1) (<= d (clr-days-in-month y m))))

(: (-> int int int int bool))
(defun (valid-time? h m s ns)
  (and (>= h 0) (< h 24) (>= m 0) (< m 60) (>= s 0) (< s 60)
       (>= ns 0) (< ns 1000000000)))

;; Nanoseconds are cut to the 100 ns a tick holds.
(: (-> int int int int long))
(defun (time-of-day-ticks h m s ns)
  (+ (+ (* (long-of h) ticks-per-hour) (* (long-of m) ticks-per-minute))
     (+ (* (long-of s) ticks-per-second) (long-of (/ ns 100)))))

(: (-> int int int (Option Date)))
(defun (date y m d)
  (if (valid-date? y m d) (Some (Date (v (ClrDate. y m d)))) None))

(: (-> int int (#:seconds int) (#:nanoseconds int) (Option Time)))
(defun (time h m #:seconds 0 #:nanoseconds 0)
  (if (valid-time? h m seconds nanoseconds)
      (Some (Time (v (ClrTime. (time-of-day-ticks h m seconds nanoseconds)))))
      None))

(: (-> int int int (#:hours int) (#:minutes int) (#:seconds int)
       (#:nanoseconds int) (Option DateTime)))
(defun (datetime y mo d #:hours 0 #:minutes 0 #:seconds 0 #:nanoseconds 0)
  (if (and (valid-date? y mo d) (valid-time? hours minutes seconds nanoseconds))
      (Some (ticks->datetime
              (+ (* (long-of (clr-date-day-number (ClrDate. y mo d))) ticks-per-day)
                 (time-of-day-ticks hours minutes seconds nanoseconds))))
      None))

(: (-> Date Time DateTime))
(defun (date+time->datetime d t)
  (ticks->datetime (+ (* (day-number d) ticks-per-day) (clr-time-ticks (struct-ref t v)))))

(: (-> (#:days long) (#:hours long) (#:minutes long) (#:seconds long)
       (#:milliseconds long) (#:nanoseconds long) Duration))
(defun (duration #:days 0L #:hours 0L #:minutes 0L #:seconds 0L #:milliseconds 0L
                 #:nanoseconds 0L)
  (ticks->duration
    (plus-ticks (plus-ticks (plus-ticks (units->ticks days ticks-per-day)
                                        (units->ticks hours ticks-per-hour))
                            (plus-ticks (units->ticks minutes ticks-per-minute)
                                        (units->ticks seconds ticks-per-second)))
                (plus-ticks (units->ticks milliseconds ticks-per-millisecond)
                            (/ nanoseconds 100L)))))

(: (-> (#:years int) (#:months int) (#:weeks int) (#:days int) DatePeriod))
(defun (date-period #:years 0 #:months 0 #:weeks 0 #:days 0)
  (DatePeriod (years years) (months months) (weeks weeks) (days days)))

(: (-> (#:years int) (#:months int) (#:weeks int) (#:days int)
       (#:hours long) (#:minutes long) (#:seconds long)
       (#:milliseconds long) (#:nanoseconds long) Period))
(defun (period #:years 0 #:months 0 #:weeks 0 #:days 0
               #:hours 0L #:minutes 0L #:seconds 0L #:milliseconds 0L #:nanoseconds 0L)
  (Period (calendar (DatePeriod (years years) (months months) (weeks weeks) (days days)))
          (exact (duration #:hours hours #:minutes minutes #:seconds seconds
                           #:milliseconds milliseconds #:nanoseconds nanoseconds))))


;; ===========================================================================
;; Traits
;; ===========================================================================

(def/trait (HasDate %a)
  (: ->date (-> %a Date)))

(def/trait (HasTime %a)
  (: ->time (-> %a Time)))

(def/trait (HasInstant %a)
  (: ->instant (-> %a Instant)))

(def/trait (DateArith %a)
  (: +date-period (-> %a DatePeriod %a)))

;; The date of b where a is, which calendar differences compare with a's own:
;; a moment b is first moved into a's zone, so two moments of one instant are
;; the same day wherever they are.
(def/trait (DateDiff %a)
  (: ->date-in-zone-of (-> %a %a Date)))

(def/trait (TimeArith %a)
  (: +duration (-> %a Duration %a)))


;; ===========================================================================
;; Date
;; ===========================================================================

;; Each step is checked before .NET takes it, because DateOnly would throw its
;; own error, and because weeks × 7 overflows an int.
(: (-> ClrDate int ClrDate))
(defun (plus-years c n)
  (def y (+ (long-of (clr-date-year c)) (long-of n)))
  (cond ((= n 0) c)
        ((and (>= y 1L) (<= y 9999L)) (clr-date-add-years c n))
        (else (calendar-error "a date plus years"))))

;; Months are counted from year 0, so year 1 January is month 12.
(: (-> ClrDate int ClrDate))
(defun (plus-months c n)
  (def m (+ (+ (* (long-of (clr-date-year c)) 12L) (long-of (- (clr-date-month c) 1)))
            (long-of n)))
  (cond ((= n 0) c)
        ((and (>= m 12L) (<= m 119999L)) (clr-date-add-months c n))
        (else (calendar-error "a date plus months"))))

(: (-> ClrDate long ClrDate))
(defun (plus-days c n)
  (def dn (+ (long-of (clr-date-day-number c)) n))
  (cond ((= n 0L) c)
        ((and (>= dn 0L) (<= dn max-day-number)) (clr-date-from-day-number (int-of dn)))
        (else (calendar-error "a date plus days"))))

;; Largest unit first: years, then months, then weeks and days together.
(: (-> ClrDate DatePeriod ClrDate))
(defun (clr-date-plus-period c p)
  (plus-days (plus-months (plus-years c (record-ref p years)) (record-ref p months))
             (+ (* (long-of (record-ref p weeks)) 7L) (long-of (record-ref p days)))))

;; This and the other `type+unit` functions are the trait methods at one type.
;; They are exported because the impls' bodies, which importers copy, call
;; them.
(: (-> Date DatePeriod Date))
(defun (date+date-period d p) (Date (v (clr-date-plus-period (struct-ref d v) p))))

(impl (HasDate Date)
  (defun (->date d) d))

(impl (DateArith Date)
  (defun (+date-period d p) (date+date-period d p)))

(impl (DateDiff Date)
  (defun (->date-in-zone-of b a) b))

(impl (Ord Date)
  (defun (compare a b)
    (compare (clr-date-day-number (struct-ref a v)) (clr-date-day-number (struct-ref b v)))))

(impl (Written Date)
  (defun (written d) (date->string (struct-ref d v)))
  (defun (write-into d b) (stringbuilder-add-string! b (written d))))


;; ===========================================================================
;; Time
;; ===========================================================================

(impl (HasTime Time)
  (defun (->time t) t))

;; Wraps at midnight, as TimeOnly.Add does.
(impl (TimeArith Time)
  (defun (+duration t d) (Time (v (clr-time-add (struct-ref t v) (ClrTimeSpan. (duration-ticks d)))))))

(impl (Ord Time)
  (defun (compare a b)
    (compare (clr-time-ticks (struct-ref a v)) (clr-time-ticks (struct-ref b v)))))

(impl (Written Time)
  (defun (written t) (time-of-day->string (clr-time-ticks (struct-ref t v))))
  (defun (write-into t b) (stringbuilder-add-string! b (written t))))


;; ===========================================================================
;; DateTime
;; ===========================================================================

(impl (HasDate DateTime)
  (defun (->date dt) (Date (v (clr-date-from-datetime (struct-ref dt v))))))

(impl (HasTime DateTime)
  (defun (->time dt) (Time (v (clr-time-from-datetime (struct-ref dt v))))))

(: (-> DateTime DatePeriod DateTime))
(defun (datetime+date-period dt p)
  (def t (datetime-ticks dt))
  (def day (clr-date-plus-period (clr-date-from-day-number (int-of (/ t ticks-per-day))) p))
  (ticks->datetime (+ (* (long-of (clr-date-day-number day)) ticks-per-day)
                      (% t ticks-per-day))))

(: (-> DateTime Duration DateTime))
(defun (datetime+duration dt d)
  (ticks->datetime (shift-ticks (datetime-ticks dt) (duration-ticks d)
                                "a datetime plus a duration")))

(impl (DateArith DateTime)
  (defun (+date-period dt p) (datetime+date-period dt p)))

(impl (DateDiff DateTime)
  (defun (->date-in-zone-of b a) (->date b)))

(impl (TimeArith DateTime)
  (defun (+duration dt d) (datetime+duration dt d)))

(impl (Ord DateTime)
  (defun (compare a b) (compare (datetime-ticks a) (datetime-ticks b))))

(impl (Written DateTime)
  (defun (written dt) (ticks->iso-datetime (datetime-ticks dt)))
  (defun (write-into dt b) (stringbuilder-add-string! b (written dt))))


;; ===========================================================================
;; Instant
;; ===========================================================================

(: (-> Instant long))
(defun (instant-ticks i) (struct-ref i ticks))

(: (-> long string Instant))
(defun (ticks->instant t what)
  (if (in-calendar? t) (Instant (ticks t)) (calendar-error what)))

(: (-> Instant Duration Instant))
(defun (instant+duration i d)
  (Instant (ticks (shift-ticks (instant-ticks i) (duration-ticks d)
                               "an instant plus a duration"))))

(impl (HasInstant Instant)
  (defun (->instant i) i))

(impl (TimeArith Instant)
  (defun (+duration i d) (instant+duration i d)))

(impl (Ord Instant)
  (defun (compare a b) (compare (instant-ticks a) (instant-ticks b))))

(impl (Written Instant)
  (defun (written i) (ticks->iso-instant (instant-ticks i)))
  (defun (write-into i b) (stringbuilder-add-string! b (written i))))

(: (-> Instant DateTime))
(defun (instant->utc-datetime i) (ticks->datetime (instant-ticks i)))

(: (-> DateTime Instant))
(defun (utc-datetime->instant dt) (Instant (ticks (datetime-ticks dt))))

;; Rounds toward the past, so the second before 1970 is -1 rather than 0.
(: (-> long long long))
(defun (floor-div a b)
  (def q (/ a b))
  (if (< (% a b) 0L) (- q 1L) q))

(: (-> Instant long))
(defun (instant->unix-seconds i)
  (floor-div (- (instant-ticks i) unix-epoch-ticks) ticks-per-second))

(: (-> Instant long))
(defun (instant->unix-milliseconds i)
  (floor-div (- (instant-ticks i) unix-epoch-ticks) ticks-per-millisecond))

;; The bounds are 0001-01-01T00:00Z and the last whole unit of 9999, so the
;; multiplication cannot overflow.
(: (-> long (Option Instant)))
(defun (unix-seconds->instant s)
  (if (and (>= s -62135596800L) (<= s 253402300799L))
      (Some (Instant (ticks (+ (* s ticks-per-second) unix-epoch-ticks))))
      None))

(: (-> long (Option Instant)))
(defun (unix-milliseconds->instant ms)
  (if (and (>= ms -62135596800000L) (<= ms 253402300799999L))
      (Some (Instant (ticks (+ (* ms ticks-per-millisecond) unix-epoch-ticks))))
      None))

(: (-> Instant long))
(defun (instant->unix-microseconds i)
  (floor-div (- (instant-ticks i) unix-epoch-ticks) 10L))

(: (-> long (Option Instant)))
(defun (unix-microseconds->instant us)
  (if (and (>= us -62135596800000000L) (<= us 253402300799999999L))
      (Some (Instant (ticks (+ (* us 10L) unix-epoch-ticks))))
      None))

;; Days since 1970-01-01, which is day number 719 162. A column of dates in
;; Apache Arrow, Polars and R holds these.
(: long)
(def unix-epoch-day-number 719162L)

(: (-> Date long))
(defun (date->unix-days d) (- (day-number d) unix-epoch-day-number))

(: (-> long (Option Date)))
(defun (unix-days->date n)
  (def dn (+ n unix-epoch-day-number))
  (if (and (>= dn 0L) (<= dn max-day-number)) (Some (day-number->date dn)) None))

;; Unix seconds of an instant or a moment, rounded as instant->unix-seconds
;; rounds, which it calls because an importer copies this body.
(: (-> %a long) (where (HasInstant %a)))
(defun (->unix x) (instant->unix-seconds (->instant x)))


;; ===========================================================================
;; Duration
;; ===========================================================================

;; Checked here rather than by TimeSpan.Add, which lets a sum reach -2^63
;; ticks: the one TimeSpan with no negation, which could not be printed or
;; read back.
(: (-> Duration Duration Duration))
(defun (duration+duration a b) (ticks->duration (plus-ticks (duration-ticks a) (duration-ticks b))))

;; (+hours (duration #:minutes 30) 1) is PT1H30M.
(impl (TimeArith Duration)
  (defun (+duration a b) (duration+duration a b)))

;; Whole units, toward zero.
(: (-> Duration long))
(defun (duration->hours d) (/ (duration-ticks d) ticks-per-hour))

(: (-> Duration long))
(defun (duration->minutes d) (/ (duration-ticks d) ticks-per-minute))

(: (-> Duration long))
(defun (duration->seconds d) (/ (duration-ticks d) ticks-per-second))

(: (-> Duration long))
(defun (duration->milliseconds d) (/ (duration-ticks d) ticks-per-millisecond))

;; Never raises, as no duration reaches -2^63 ticks.
(: (-> Duration Duration))
(defun (duration-negate d) (ticks->duration (- (duration-ticks d))))


;; ===========================================================================
;; Periods
;; ===========================================================================

(: (-> DatePeriod DatePeriod))
(defun (date-period-negate p)
  (DatePeriod (years (- (record-ref p years))) (months (- (record-ref p months)))
              (weeks (- (record-ref p weeks))) (days (- (record-ref p days)))))

(impl (Written DatePeriod)
  (defun (written p) (date-period->string p))
  (defun (write-into p b) (stringbuilder-add-string! b (written p))))

(impl (Written Period)
  (defun (written p) (period->string p))
  (defun (write-into p b) (stringbuilder-add-string! b (written p))))


;; ===========================================================================
;; Accessors and arithmetic
;; ===========================================================================

(: (-> %a DateTime) (where (HasDate %a) (HasTime %a)))
(defun (->datetime x) (date+time->datetime (->date x) (->time x)))

(: (-> %a int) (where (HasDate %a)))
(defun (->year x) (clr-date-year (struct-ref (->date x) v)))

(: (-> %a int) (where (HasDate %a)))
(defun (->month x) (clr-date-month (struct-ref (->date x) v)))

(: (-> %a int) (where (HasDate %a)))
(defun (->day x) (clr-date-day (struct-ref (->date x) v)))

;; ISO 8601: 1 is Monday, 7 is Sunday. Day number 0, 0001-01-01, was a Monday.
(: (-> %a int) (where (HasDate %a)))
(defun (->weekday x) (+ (% (clr-date-day-number (struct-ref (->date x) v)) 7) 1))

(: (-> %a int) (where (HasDate %a)))
(defun (->yday x) (clr-date-day-of-year (struct-ref (->date x) v)))

;; ISO 8601 weeks run Monday to Sunday, and a week belongs to the year its
;; Thursday is in, so week 1 holds the year's first Thursday. n - n mod 7 is
;; the Monday, as day number 0 was one. Every date in range has its Thursday
;; in range, since 0001-01-01 was a Monday and 9999-12-31 a Friday.
(: (-> %a int) (where (HasDate %a)))
(defun (->week x)
  (def n (clr-date-day-number (struct-ref (->date x) v)))
  (+ (/ (- (clr-date-day-of-year (clr-date-from-day-number (+ (- n (% n 7)) 3))) 1) 7) 1))

;; The year ->week counts in. It is not always ->year: 2027-01-01 is in week
;; 53 of 2026.
(: (-> %a int) (where (HasDate %a)))
(defun (->week-year x)
  (def n (clr-date-day-number (struct-ref (->date x) v)))
  (clr-date-year (clr-date-from-day-number (+ (- n (% n 7)) 3))))

;; 52 or 53. December 28 is always in its year's last week.
(: (-> int int))
(defun (weeks-in-year y)
  (if (and (>= y 1) (<= y 9999))
      (->week (Date (v (ClrDate. y 12 28))))
      (range-error (str "year " (displayed y) " is outside 1 to 9999"))))

;; Day d (1 is Monday) of week w of week-year y. Week 1 starts on the Monday
;; on or before January 4.
(: (-> int int int (Option Date)))
(defun (week-date y w d)
  (if (and (>= y 1) (<= y 9999) (>= w 1) (<= w (weeks-in-year y)) (>= d 1) (<= d 7))
      (let* ((jan4 (long-of (clr-date-day-number (ClrDate. y 1 4))))
             (n (+ (- jan4 (% jan4 7L)) (long-of (+ (* (- w 1) 7) (- d 1))))))
        ;; The last week of 9999 ends in 10000.
        (if (<= n max-day-number) (Some (day-number->date n)) None))
      None))

(: (-> %a int) (where (HasTime %a)))
(defun (->hours x) (clr-time-hour (struct-ref (->time x) v)))

(: (-> %a int) (where (HasTime %a)))
(defun (->minutes x) (clr-time-minute (struct-ref (->time x) v)))

(: (-> %a int) (where (HasTime %a)))
(defun (->seconds x) (clr-time-second (struct-ref (->time x) v)))

(: (-> %a int) (where (HasTime %a)))
(defun (->milliseconds x) (clr-time-millisecond (struct-ref (->time x) v)))

;; The whole fraction of the second, so always a multiple of 100.
(: (-> %a int) (where (HasTime %a)))
(defun (->nanoseconds x)
  (def t (struct-ref (->time x) v))
  (+ (* (clr-time-millisecond t) 1000000)
     (+ (* (clr-time-microsecond t) 1000) (clr-time-nanosecond t))))

(: (-> %a int %a) (where (DateArith %a)))
(defun (+years x n) (+date-period x (DatePeriod (years n) (months 0) (weeks 0) (days 0))))

(: (-> %a int %a) (where (DateArith %a)))
(defun (+months x n) (+date-period x (DatePeriod (years 0) (months n) (weeks 0) (days 0))))

(: (-> %a int %a) (where (DateArith %a)))
(defun (+weeks x n) (+date-period x (DatePeriod (years 0) (months 0) (weeks n) (days 0))))

(: (-> %a int %a) (where (DateArith %a)))
(defun (+days x n) (+date-period x (DatePeriod (years 0) (months 0) (weeks 0) (days n))))

(: (-> %a int %a) (where (DateArith %a)))
(defun (-years x n) (+years x (- n)))

(: (-> %a int %a) (where (DateArith %a)))
(defun (-months x n) (+months x (- n)))

(: (-> %a int %a) (where (DateArith %a)))
(defun (-weeks x n) (+weeks x (- n)))

(: (-> %a int %a) (where (DateArith %a)))
(defun (-days x n) (+days x (- n)))

(: (-> %a DatePeriod %a) (where (DateArith %a)))
(defun (-date-period x p)
  (+date-period x (DatePeriod (years (- (record-ref p years))) (months (- (record-ref p months)))
                              (weeks (- (record-ref p weeks))) (days (- (record-ref p days))))))

(: (-> %a long %a) (where (TimeArith %a)))
(defun (+hours x n) (+duration x (duration #:hours n)))

(: (-> %a long %a) (where (TimeArith %a)))
(defun (+minutes x n) (+duration x (duration #:minutes n)))

(: (-> %a long %a) (where (TimeArith %a)))
(defun (+seconds x n) (+duration x (duration #:seconds n)))

(: (-> %a long %a) (where (TimeArith %a)))
(defun (+milliseconds x n) (+duration x (duration #:milliseconds n)))

;; Negated after the range check, so long's minimum is refused rather than
;; wrapped back to itself.
(: (-> %a long %a) (where (TimeArith %a)))
(defun (-hours x n) (+duration x (duration-negate (duration #:hours n))))

(: (-> %a long %a) (where (TimeArith %a)))
(defun (-minutes x n) (+duration x (duration-negate (duration #:minutes n))))

(: (-> %a long %a) (where (TimeArith %a)))
(defun (-seconds x n) (+duration x (duration-negate (duration #:seconds n))))

(: (-> %a long %a) (where (TimeArith %a)))
(defun (-milliseconds x n) (+duration x (duration-negate (duration #:milliseconds n))))

(: (-> %a Duration %a) (where (TimeArith %a)))
(defun (-duration x d) (+duration x (duration-negate d)))

(: (-> %a Period %a) (where (DateArith %a) (TimeArith %a)))
(defun (+period x p)
  (+duration (+date-period x (record-ref p calendar)) (record-ref p exact)))

(: (-> %a Period %a) (where (DateArith %a) (TimeArith %a)))
(defun (-period x p)
  (-duration (-date-period x (record-ref p calendar)) (record-ref p exact)))


;; ===========================================================================
;; Differences
;; ===========================================================================

;; Calendar differences read the dates only: 23:00 to 01:00 the next day is
;; one day. Both arguments have one type, and b's date is read where a is.
(: (-> %a %a int) (where (HasDate %a) (DateDiff %a)))
(defun (days-between a b)
  (- (clr-date-day-number (struct-ref (->date-in-zone-of b a) v))
     (clr-date-day-number (struct-ref (->date a) v))))

(: (-> %a %a int) (where (HasDate %a) (DateDiff %a)))
(defun (weeks-between a b) (/ (days-between a b) 7))

;; A month counts once the day of the month is reached, as in java.time, so
;; swapping the arguments only flips the sign: 2026-01-31 to 2026-02-28 is 0.
(: (-> %a %a int) (where (HasDate %a) (DateDiff %a)))
(defun (months-between a b)
  (def x (struct-ref (->date a) v))
  (def y (struct-ref (->date-in-zone-of b a) v))
  (def months (- (+ (* (clr-date-year y) 12) (clr-date-month y))
                 (+ (* (clr-date-year x) 12) (clr-date-month x))))
  (def days (- (clr-date-day y) (clr-date-day x)))
  (cond ((and (> months 0) (< days 0)) (- months 1))
        ((and (< months 0) (> days 0)) (+ months 1))
        (else months)))

(: (-> %a %a int) (where (HasDate %a) (DateDiff %a)))
(defun (years-between a b) (/ (months-between a b) 12))

;; The time that passed from a to b, between their instants, so a moment and
;; an instant mix, and a DateTime, which has no instant, does not compile. Two
;; instants are less than 10 000 years apart, so the difference fits.
(: (-> %a %b Duration) (where (HasInstant %a) (HasInstant %b)))
(defun (duration-between a b)
  (ticks->duration (- (struct-ref (->instant b) ticks) (struct-ref (->instant a) ticks))))

(: (-> %a %b long) (where (HasInstant %a) (HasInstant %b)))
(defun (hours-between a b) (duration->hours (duration-between a b)))

(: (-> %a %b long) (where (HasInstant %a) (HasInstant %b)))
(defun (minutes-between a b) (duration->minutes (duration-between a b)))

(: (-> %a %b long) (where (HasInstant %a) (HasInstant %b)))
(defun (seconds-between a b) (duration->seconds (duration-between a b)))

(: (-> %a %b long) (where (HasInstant %a) (HasInstant %b)))
(defun (milliseconds-between a b) (duration->milliseconds (duration-between a b)))


;; ===========================================================================
;; Zones
;; ===========================================================================

(: TimeZone)
(def utc (TimeZone (tz clr-tz-utc)))

;; IANA ids only: .NET would also take a Windows id such as "W. Europe
;; Standard Time", which HasIanaId refuses.
(: (-> string (Option TimeZone)))
(defun (timezone id)
  (match (clr-tz-try-find id)
    ((Some z) (if (clr-tz-has-iana-id z) (Some (TimeZone (tz z))) None))
    (None None)))

(: (-> TimeZone string))
(defun (timezone-id z) (clr-tz-id (struct-ref z tz)))

(impl (Eq TimeZone)
  (defun (= a b) (= (timezone-id a) (timezone-id b)))
  (defun (eq-hash z) (eq-hash (timezone-id z))))

(impl (Written TimeZone)
  (defun (written z) (timezone-id z))
  (defun (write-into z b) (stringbuilder-add-string! b (written z))))

;; The offset in z at the UTC tick count u, in ticks. u is clamped into the
;; calendar, because local-fit looks a day past a local time on the first or
;; last day; no zone changes offset within a day of either end, so the answer
;; is the same.
(: (-> TimeZoneInfo long long))
(defun (offset-at z u)
  (def clamped (if (< u 0L) 0L (if (> u max-ticks) max-ticks u)))
  (clr-timespan-ticks (clr-tz-utc-offset z (ClrDateTime. clamped DateTimeKind.Utc))))

(: (-> TimeZone Instant Duration))
(defun (timezone-offset-at z i) (ticks->duration (offset-at (struct-ref z tz) (instant-ticks i))))

;; Where a local time lands in a zone: at one offset, or two in an overlap
;; (the larger first, since its instant is the earlier), or in a gap between
;; the offset before and the offset after.
(type (: Fit (Union (: Fits long long) (: Skipped long long))))

(: (-> long (List long) bool))
(defun (member-long? x xs)
  (and (not (list-empty? xs))
       (or (= x (list-head xs)) (member-long? x (list-tail xs)))))

;; An offset o fits the local tick count l when the zone has offset o at the
;; instant l - o. The candidates are the offsets a day either side of l, which
;; is every offset in reach while a zone changes offset at most once in 48 h:
;; the tz database's closest two changes are 95 h apart. When a lookup answers
;; with a third offset it becomes a candidate too, which catches most layouts
;; with two changes in 48 h, but not all.
(: (-> TimeZoneInfo long Fit))
(defun (local-fit z l)
  (def before (offset-at z (- l ticks-per-day)))
  (def after (offset-at z (+ l ticks-per-day)))
  (def at-before (offset-at z (- l before)))
  (if (and (= before after) (= at-before before))
      (Fits before before)
      (let* ((at-after (offset-at z (- l after)))
             (candidates (list-foldl (fun (o seen) (if (member-long? o seen) seen (Cons o seen)))
                                     '() (list before after at-before at-after)))
             (fits (list-filter (fun (o) (= (offset-at z (- l o)) o)) candidates)))
        (if (list-empty? fits)
            (Skipped before at-before)
            (Fits (list-foldl max (list-head fits) fits) (list-foldl min (list-head fits) fits))))))

;; The first instant at offset `after`, by bisection between an instant still
;; before the change and one after it.
(: (-> TimeZoneInfo long long long long))
(defun (change-ticks z lo hi after)
  (if (<= (- hi lo) 1L)
      hi
      (let ((mid (+ lo (/ (- hi lo) 2L))))
        (if (= (offset-at z mid) after)
            (change-ticks z lo mid after)
            (change-ticks z mid hi after)))))


;; ===========================================================================
;; Moments
;; ===========================================================================

(: (-> Moment long))
(defun (moment-local-ticks m) (datetime-ticks (struct-ref m local)))

(: (-> Moment long))
(defun (moment-offset-ticks m) (duration-ticks (struct-ref m offset)))

(: (-> Moment long))
(defun (moment-instant-ticks m) (- (moment-local-ticks m) (moment-offset-ticks m)))

;; A moment from a local time and an offset that fits it. Near either end of
;; the calendar the instant can fall outside it: 0001-01-01T00:00 in Tokyo is
;; the day before.
(: (-> long long TimeZone Moment))
(defun (fitted-moment l o z)
  (if (in-calendar? (- l o))
      (Moment (local (ticks->datetime l)) (offset (ticks->duration o)) (zone z))
      (calendar-error "the instant of a local time")))

(: (-> Instant TimeZone Moment))
(defun (instant->moment i z)
  (def u (instant-ticks i))
  (def o (offset-at (struct-ref z tz) u))
  (if (in-calendar? (+ u o))
      (Moment (local (ticks->datetime (+ u o))) (offset (ticks->duration o)) (zone z))
      (calendar-error "the local time of an instant")))

(: (-> Gap Overlap Resolver))
(defun (resolver g o) (Resolver (gap g) (overlap o)))

;; GapPush moves the local time forward by the length of the gap: the instant
;; it names at the offset before the gap, read at the offset after it.
(: (-> long TimeZone Resolver Moment))
(defun (resolve-local l z r)
  (match (local-fit (struct-ref z tz) l)
    ((Fits earlier later)
     (fitted-moment l (match (record-ref r overlap)
                        (OverlapEarlier earlier)
                        (OverlapLater later))
                    z))
    ((Skipped before after)
     (match (record-ref r gap)
       (GapPush (instant->moment (ticks->instant (- l before) "the instant of a local time") z))
       (GapNextValid
        (instant->moment (Instant (ticks (change-ticks (struct-ref z tz) (- l after) (- l before) after)))
                         z))))))

(: (-> DateTime TimeZone Resolver Moment))
(defun (datetime->moment dt z r) (resolve-local (datetime-ticks dt) z r))

(: (-> DateTime TimeZone (Option Moment)))
(defun (datetime->moment/opt dt z)
  (def l (datetime-ticks dt))
  (match (local-fit (struct-ref z tz) l)
    ((Fits earlier later)
     (if (and (= earlier later) (in-calendar? (- l earlier)))
         (Some (Moment (local dt) (offset (ticks->duration earlier)) (zone z)))
         None))
    (_ None)))

(: (-> Moment DatePeriod Resolver Moment))
(defun (+date-period/resolver m p r)
  (datetime->moment (datetime+date-period (struct-ref m local) p) (struct-ref m zone) r))

(: (-> Moment DatePeriod Resolver Moment))
(defun (-date-period/resolver m p r) (+date-period/resolver m (date-period-negate p) r))

;; Date arithmetic on a moment keeps its offset through an overlap if that
;; offset still fits, takes the earlier one if not, and pushes forward through
;; a gap.
(: (-> Moment DatePeriod Moment))
(defun (moment+date-period m p)
  (def l (datetime-ticks (datetime+date-period (struct-ref m local) p)))
  (def z (struct-ref m zone))
  (def kept (moment-offset-ticks m))
  (match (local-fit (struct-ref z tz) l)
    ((Fits earlier later)
     (fitted-moment l (if (or (= kept earlier) (= kept later)) kept earlier) z))
    ((Skipped before after)
     (instant->moment (ticks->instant (- l before) "the instant of a local time") z))))

(: (-> Moment Duration))
(defun (->utc-offset m) (struct-ref m offset))

(: (-> Moment TimeZone))
(defun (->timezone m) (struct-ref m zone))

(: (-> Moment TimeZone Moment))
(defun (adjust-timezone m z)
  (instant->moment (Instant (ticks (moment-instant-ticks m))) z))

;; Exact: through the instant, and back into the same zone.
(: (-> Moment Duration Moment))
(defun (moment+duration m d)
  (instant->moment
    (Instant (ticks (shift-ticks (moment-instant-ticks m) (duration-ticks d)
                                 "a moment plus a duration")))
    (struct-ref m zone)))

(impl (HasDate Moment)
  (defun (->date m) (->date (struct-ref m local))))

(impl (HasTime Moment)
  (defun (->time m) (->time (struct-ref m local))))

(impl (HasInstant Moment)
  (defun (->instant m)
    (Instant (ticks (- (clr-datetime-ticks (struct-ref (struct-ref m local) v))
                       (duration-ticks (struct-ref m offset)))))))

(impl (DateArith Moment)
  (defun (+date-period m p) (moment+date-period m p)))

;; Near either end of the calendar, the move can raise as instant->moment does.
(impl (DateDiff Moment)
  (defun (->date-in-zone-of b a)
    (if (= (->timezone b) (->timezone a))
        (->date b)
        (->date (adjust-timezone b (->timezone a))))))

(impl (TimeArith Moment)
  (defun (+duration m d) (moment+duration m d)))

;; Every field, so 12:00+02:00[Europe/Stockholm] and 10:00+00:00[UTC] are two
;; values although they are one instant.
(impl (Eq Moment)
  (defun (= a b)
    (and (= (moment-local-ticks a) (moment-local-ticks b))
         (= (moment-offset-ticks a) (moment-offset-ticks b))
         (= (struct-ref a zone) (struct-ref b zone))))
  (defun (eq-hash m)
    (hash-combine (eq-hash (moment-local-ticks m))
                  (hash-combine (eq-hash (moment-offset-ticks m)) (eq-hash (struct-ref m zone))))))

;; By instant, then local time, then zone id, so that compare is 0 exactly
;; when = holds.
(impl (Ord Moment)
  (defun (compare a b)
    (def by-instant (compare (moment-instant-ticks a) (moment-instant-ticks b)))
    (def by-local (compare (moment-local-ticks a) (moment-local-ticks b)))
    (cond ((not (= by-instant 0)) by-instant)
          ((not (= by-local 0)) by-local)
          (else (compare (timezone-id (struct-ref a zone)) (timezone-id (struct-ref b zone)))))))

(impl (Written Moment)
  (defun (written m) (moment->string m))
  (defun (write-into m b) (stringbuilder-add-string! b (written m))))

(impl (Written Gap)
  (defun (written g) (match g (GapPush "GapPush") (GapNextValid "GapNextValid")))
  (defun (write-into g b) (stringbuilder-add-string! b (written g))))

(impl (Written Overlap)
  (defun (written o) (match o (OverlapEarlier "OverlapEarlier") (OverlapLater "OverlapLater")))
  (defun (write-into o b) (stringbuilder-add-string! b (written o))))

(impl (Written Resolver)
  (defun (written r)
    (str "(resolver " (written (record-ref r gap)) " " (written (record-ref r overlap)) ")"))
  (defun (write-into r b) (stringbuilder-add-string! b (written r))))


;; ===========================================================================
;; The clock
;; ===========================================================================

;; The functions that read the time take #:clock, defaulting to the value of
;; the prelude's current-clock at the call, so a program that binds the
;; parameter changes no call site. They are the only functions here that are
;; not pure.

(: (-> TimeZone (-> TimeZoneInfo)))
(defun (zone-reader z)
  (def info (struct-ref z tz))
  (fun () info))

;; c, with today in z.
(: (-> Clock TimeZone Clock))
(defun (clock-in c z) (record-set c (read-zone (zone-reader z))))

;; The system's time in zone z, so that a program on a machine set to UTC can
;; bind current-clock once and have (today) be z's date everywhere.
(: (-> TimeZone Clock))
(defun (system-clock-in z) (clock-in system-clock z))

;; Always the same instant. Its monotonic reading stands still too.
(: (-> Instant TimeZone Clock))
(defun (fixed-clock i z)
  (def t (instant-ticks i))
  (make-clock #:monotonic-ms (fun () 0L)
              #:utc-ticks (fun () t)
              #:zone (zone-reader z)))

(: (-> ClrLock (-> %a) %a))
(defun (locked lock body)
  (clr-lock-enter lock)
  (try (body) #:finally (clr-lock-exit lock)))

;; Stands still until clock-advance! moves it. Read and written under its lock,
;; so fibers on other threads may share one: an advance is whole, and visible
;; to the next read.
;;
;; The monotonic reading starts at 0 and moves only on a positive advance, so a
;; step back of the wall clock never makes an elapsed time negative.
(: (-> Instant TimeZone Clock))
(defun (manual-clock i z)
  (def wall (make-box i))
  (def mono (make-box 0L))
  (def lock (ClrLock.))
  (make-clock
    #:monotonic-ms (fun () (/ (locked lock (fun () (box-ref mono))) ticks-per-millisecond))
    #:monotonic-ns (Some (fun () (* (locked lock (fun () (box-ref mono))) 100L)))
    #:utc-ticks (fun () (instant-ticks (locked lock (fun () (box-ref wall)))))
    #:zone (zone-reader z)
    #:advance (Some (fun (t)
                      (clr-lock-enter lock)
                      (try (let ()
                             (box-set! wall (instant+duration (box-ref wall) (ticks->duration t)))
                             (when (> t 0L) (box-set! mono (+ (box-ref mono) t))))
                        #:finally (clr-lock-exit lock))))))

;; A negative duration steps the clock back, as a corrected wall clock does.
(: (-> Clock Duration void))
(defun (clock-advance! c d)
  (match (record-ref c advance-ticks)
    ((Some advance) (advance (duration-ticks d)))
    (None (raise (cast Exception
                       (ArgumentException.
                         "(std datetime): clock-advance! moves a clock made with #:advance, as a manual clock is, and this one has none."))))))

(: (-> (#:clock Clock) TimeZone))
(defun (clock-timezone #:clock (parameter-ref current-clock))
  (TimeZone (tz ((record-ref clock read-zone)))))

(: (-> (#:clock Clock) Instant))
(defun (now #:clock (parameter-ref current-clock))
  (ticks->instant ((record-ref clock read-utc-ticks)) "a clock's reading"))

(: (-> (#:clock Clock) (#:tz TimeZone) Moment))
(defun (now/moment #:clock (parameter-ref current-clock) #:tz (clock-timezone #:clock clock))
  (instant->moment (now #:clock clock) tz))

(: (-> (#:clock Clock) (#:tz TimeZone) Date))
(defun (today #:clock (parameter-ref current-clock) #:tz (clock-timezone #:clock clock))
  (->date (now/moment #:clock clock #:tz tz)))


;; ===========================================================================
;; Timers
;; ===========================================================================

;; Ready at an instant. The time left is worked out at each sync against the
;; system's clock, which is what a timer waits by, and not against
;; `current-clock`: a clock that is fixed or moved by hand does not move a
;; timer.
(: (-> Instant (Event Unit)))
(defun (at-time i) (clr-at-utc (ClrDateTime. (struct-ref i ticks) DateTimeKind.Utc)))


;; ===========================================================================
;; Writing ISO 8601
;; ===========================================================================
;;
;; A writer appends to one StringBuilder sized for its text, a digit at a time,
;; so writing a value allocates the builder and the string, not a string per
;; field.

(: (-> int TextBuilder))
(defun (new-text capacity) (TextBuilder. capacity))

(: (-> TextBuilder string))
(defun (text-of sb) (clr-builder-text sb))

(: (-> TextBuilder string void))
(defun (add-text! sb s) (ignore (clr-add-text sb s)))

(: (-> TextBuilder long void))
(defun (add-digit! sb d) (ignore (clr-add-unit sb (cast System.Char (+ 48 (int-of d))))))

;; The number of decimal digits in n >= 0, 0 having one.
(: (-> long int))
(defun (digit-count n) (if (< n 10L) 1 (+ 1 (digit-count (/ n 10L)))))

;; The largest power of ten not above n >= 0, or 1.
(: (-> long long))
(defun (top-power n) (if (< n 10L) 1L (* 10L (top-power (/ n 10L)))))

;; n >= 0 in decimal, with zeros in front up to width digits.
(: (-> TextBuilder long int void))
(defun (add-padded! sb n width)
  (let zeros ((k (digit-count n)))
    (when (< k width)
      (add-digit! sb 0L)
      (zeros (+ k 1))))
  (let digits ((p (top-power n)))
    (when (> p 0L)
      (add-digit! sb (% (/ n p) 10L))
      (digits (/ p 10L)))))

(: (-> TextBuilder ClrDate void))
(defun (add-date! sb c)
  (add-padded! sb (long-of (clr-date-year c)) 4)
  (add-text! sb "-")
  (add-padded! sb (long-of (clr-date-month c)) 2)
  (add-text! sb "-")
  (add-padded! sb (long-of (clr-date-day c)) 2))

;; ".5" for half a second: up to seven digits, without trailing zeros, and
;; nothing for a whole second.
(: (-> TextBuilder long void))
(defun (add-fraction! sb f)
  (unless (= f 0L)
    (let loop ((f f) (width 7))
      (if (= (% f 10L) 0L)
          (loop (/ f 10L) (- width 1))
          (begin
            (add-text! sb ".")
            (add-padded! sb f width))))))

;; hh:mm:ss of a tick count within a day.
(: (-> TextBuilder long void))
(defun (add-clock! sb t)
  (add-padded! sb (/ t ticks-per-hour) 2)
  (add-text! sb ":")
  (add-padded! sb (% (/ t ticks-per-minute) 60L) 2)
  (add-text! sb ":")
  (add-padded! sb (% (/ t ticks-per-second) 60L) 2))

(: (-> TextBuilder long void))
(defun (add-time-of-day! sb t)
  (add-clock! sb t)
  (add-fraction! sb (% t ticks-per-second)))

(: (-> TextBuilder long void))
(defun (add-datetime! sb t)
  (add-date! sb (clr-date-from-day-number (int-of (/ t ticks-per-day))))
  (add-text! sb "T")
  (add-time-of-day! sb (% t ticks-per-day)))

;; ±hh:mm, and :ss when there are seconds.
(: (-> TextBuilder long void))
(defun (add-offset! sb o)
  (def a (abs o))
  (def s (% (/ a ticks-per-second) 60L))
  (add-text! sb (if (< o 0L) "-" "+"))
  (add-padded! sb (/ a ticks-per-hour) 2)
  (add-text! sb ":")
  (add-padded! sb (% (/ a ticks-per-minute) 60L) 2)
  (unless (= s 0L)
    (add-text! sb ":")
    (add-padded! sb s 2)))

(: (-> ClrDate string))
(defun (date->string c)
  (def sb (new-text 10))
  (add-date! sb c)
  (text-of sb))

(: (-> long string))
(defun (time-of-day->string t)
  (def sb (new-text 16))
  (add-time-of-day! sb t)
  (text-of sb))

(: (-> long string))
(defun (ticks->iso-datetime t)
  (def sb (new-text 27))
  (add-datetime! sb t)
  (text-of sb))

(: (-> long string))
(defun (ticks->iso-instant t)
  (def sb (new-text 28))
  (add-datetime! sb t)
  (add-text! sb "Z")
  (text-of sb))

;; 38 characters hold the longest local time and offset, and the brackets.
(: (-> Moment string))
(defun (moment->string m)
  (def id (timezone-id (struct-ref m zone)))
  (def sb (new-text (+ 38 (string-length id))))
  (add-datetime! sb (moment-local-ticks m))
  (add-offset! sb (moment-offset-ticks m))
  (add-text! sb "[")
  (add-text! sb id)
  (add-text! sb "]")
  (text-of sb))

;; n and its unit, with a minus when n is negative, and nothing for 0.
(: (-> TextBuilder long string void))
(defun (add-amount! sb n unit)
  (unless (= n 0L)
    (when (< n 0L) (add-text! sb "-"))
    (add-padded! sb (abs n) 1)
    (add-text! sb unit)))

;; The time components of a non-negative tick count, each prefixed by sign.
(: (-> TextBuilder long string void))
(defun (add-time-parts! sb a sign)
  (def h (/ a ticks-per-hour))
  (def m (% (/ a ticks-per-minute) 60L))
  (def s (% (/ a ticks-per-second) 60L))
  (def f (% a ticks-per-second))
  (unless (= h 0L)
    (add-text! sb sign)
    (add-padded! sb h 1)
    (add-text! sb "H"))
  (unless (= m 0L)
    (add-text! sb sign)
    (add-padded! sb m 1)
    (add-text! sb "M"))
  (unless (and (= s 0L) (= f 0L))
    (add-text! sb sign)
    (add-padded! sb s 1)
    (add-fraction! sb f)
    (add-text! sb "S")))

;; Each component carries its own sign, since a period may mix them: P1M-3D.
(: (-> TextBuilder DatePeriod void))
(defun (add-date-parts! sb p)
  (add-amount! sb (long-of (record-ref p years)) "Y")
  (add-amount! sb (long-of (record-ref p months)) "M")
  (add-amount! sb (long-of (record-ref p weeks)) "W")
  (add-amount! sb (long-of (record-ref p days)) "D"))

(: (-> DatePeriod bool))
(defun (date-period-zero? p)
  (and (= (record-ref p years) 0) (= (record-ref p months) 0)
       (= (record-ref p weeks) 0) (= (record-ref p days) 0)))

(: (-> DatePeriod string))
(defun (date-period->string p)
  (if (date-period-zero? p)
      "P0D"
      (let ((sb (new-text 48)))
        (add-text! sb "P")
        (add-date-parts! sb p)
        (text-of sb))))

;; P1DT3H. A negative exact part signs each of its components: P1DT-3H.
(: (-> Period string))
(defun (period->string p)
  (def dates (record-ref p calendar))
  (def t (duration-ticks (record-ref p exact)))
  (if (and (date-period-zero? dates) (= t 0L))
      "P0D"
      (let ((sb (new-text 80)))
        (add-text! sb "P")
        (add-date-parts! sb dates)
        (unless (= t 0L)
          (add-text! sb "T")
          (add-time-parts! sb (abs t) (if (< t 0L) "-" "")))
        (text-of sb))))


;; ===========================================================================
;; Reading ISO 8601
;; ===========================================================================
;;
;; Each reader takes the string and a cursor into it, and answers what it read
;; with the cursor after it, or None. Option and Tuple are structs, so reading
;; allocates nothing but a moment's zone id. Only the extended format is read
;; (2026-10-25T02:30:00, not 20261025T023000), with an upper-case T and Z.

(: (-> char int))
(defun (ascii-digit c)
  (def n (char->int c))
  (if (and (>= n 48) (<= n 57)) (- n 48) -1))

(: (-> string StringCursor int))
(defun (next-digit s c)
  (if (string-cursor-end? s c) -1 (ascii-digit (string-cursor-ref s c))))

(: (-> string StringCursor char bool))
(defun (starts-with-char? s c ch)
  (and (not (string-cursor-end? s c)) (char=? (string-cursor-ref s c) ch)))

(: (-> string StringCursor char (Option StringCursor)))
(defun (skip-char s c ch)
  (if (starts-with-char? s c ch) (Some (string-cursor-next s c)) None))

;; Exactly n digits.
(: (-> string StringCursor int (Option (Tuple int StringCursor))))
(defun (read-fixed s c n)
  (let loop ((c c) (left n) (acc 0))
    (def d (next-digit s c))
    (cond ((= left 0) (Some (Tuple acc c)))
          ((< d 0) None)
          (else (loop (string-cursor-next s c) (- left 1) (+ (* acc 10) d))))))

;; One or more digits, or None past long's range.
(: (-> string StringCursor (Option (Tuple long StringCursor))))
(defun (read-number s c)
  (let loop ((c c) (count 0) (acc 0L))
    (def d (long-of (next-digit s c)))
    (cond ((< d 0L) (if (= count 0) None (Some (Tuple acc c))))
          ((> acc (/ (- long-max d) 10L)) None)
          (else (loop (string-cursor-next s c) (+ count 1) (+ (* acc 10L) d))))))

;; The digits after a decimal point or comma, as ticks. Digits past the
;; seventh are read and dropped, as a tick holds no less than 100 ns.
(: (-> string StringCursor (Option (Tuple long StringCursor))))
(defun (read-fraction s c)
  (let loop ((c c) (count 0) (acc 0L))
    (def d (long-of (next-digit s c)))
    (cond ((>= d 0L)
           (loop (string-cursor-next s c) (+ count 1) (if (< count 7) (+ (* acc 10L) d) acc)))
          ((= count 0) None)
          (else (Some (Tuple (scale-fraction acc count) c))))))

(: (-> long int long))
(defun (scale-fraction f digits)
  (if (>= digits 7) f (scale-fraction (* f 10L) (+ digits 1))))

;; An optional fraction: nothing, or a separator and at least one digit.
(: (-> string StringCursor (Option (Tuple long StringCursor))))
(defun (read-optional-fraction s c)
  (if (or (starts-with-char? s c #\.) (starts-with-char? s c #\,))
      (read-fraction s (string-cursor-next s c))
      (Some (Tuple 0L c))))

;; YYYY-MM-DD, as a day number.
(: (-> string StringCursor (Option (Tuple long StringCursor))))
(defun (read-date s c)
  (def* ((Some (Tuple y c1)) (read-fixed s c 4) :propagate)
        ((Some c2) (skip-char s c1 #\-) :propagate)
        ((Some (Tuple m c3)) (read-fixed s c2 2) :propagate)
        ((Some c4) (skip-char s c3 #\-) :propagate)
        ((Some (Tuple d c5)) (read-fixed s c4 2) :propagate))
  (if (valid-date? y m d)
      (Some (Tuple (long-of (clr-date-day-number (ClrDate. y m d))) c5))
      None))

;; hh:mm, hh:mm:ss or hh:mm:ss.fff, as ticks since midnight. No leap second.
(: (-> string StringCursor (Option (Tuple long StringCursor))))
(defun (read-time s c)
  (def* ((Some (Tuple h c1)) (read-fixed s c 2) :propagate)
        ((Some c2) (skip-char s c1 #\:) :propagate)
        ((Some (Tuple m c3)) (read-fixed s c2 2) :propagate))
  (cond ((not (and (< h 24) (< m 60))) None)
        ((not (starts-with-char? s c3 #\:)) (Some (Tuple (time-of-day-ticks h m 0 0) c3)))
        (:def (Some (Tuple sec c4)) (read-fixed s (string-cursor-next s c3) 2) :propagate)
        (:def (Some (Tuple f c5)) (read-optional-fraction s c4) :propagate)
        ((>= sec 60) None)
        (else (Some (Tuple (+ (time-of-day-ticks h m sec 0) f) c5)))))

;; A date, T, and a time, as local ticks.
(: (-> string StringCursor (Option (Tuple long StringCursor))))
(defun (read-datetime s c)
  (def* ((Some (Tuple day c1)) (read-date s c) :propagate)
        ((Some c2) (skip-char s c1 #\T) :propagate)
        ((Some (Tuple t c3)) (read-time s c2) :propagate))
  (Some (Tuple (+ (* day ticks-per-day) t) c3)))

;; Z, or ±hh:mm with an optional :ss, as ticks.
(: (-> string StringCursor (Option (Tuple long StringCursor))))
(defun (read-offset s c)
  (cond ((starts-with-char? s c #\Z) (Some (Tuple 0L (string-cursor-next s c))))
        ((starts-with-char? s c #\+) (read-hhmm s (string-cursor-next s c) 1L))
        ((starts-with-char? s c #\-) (read-hhmm s (string-cursor-next s c) -1L))
        (else None)))

(: (-> string StringCursor long (Option (Tuple long StringCursor))))
(defun (read-hhmm s c sign)
  (def* ((Some (Tuple h c1)) (read-fixed s c 2) :propagate)
        ((Some c2) (skip-char s c1 #\:) :propagate)
        ((Some (Tuple m c3)) (read-fixed s c2 2) :propagate))
  (cond ((not (and (< h 24) (< m 60))) None)
        ((not (starts-with-char? s c3 #\:))
         (Some (Tuple (* sign (time-of-day-ticks h m 0 0)) c3)))
        (:def (Some (Tuple sec c4)) (read-fixed s (string-cursor-next s c3) 2) :propagate)
        ((>= sec 60) None)
        (else (Some (Tuple (* sign (time-of-day-ticks h m sec 0)) c4)))))

;; The text up to the character ch, and the cursor after ch.
(: (-> string StringCursor char (Option (Tuple string StringCursor))))
(defun (read-until s c ch)
  (let loop ((e c))
    (cond ((string-cursor-end? s e) None)
          ((char=? (string-cursor-ref s e) ch)
           (Some (Tuple (substring s c e) (string-cursor-next s e))))
          (else (loop (string-cursor-next s e))))))

;; A reader's value when it read to the end of s.
(: (-> string (Option (Tuple %a StringCursor)) (Option %a)))
(defun (whole s r)
  (match r
    ((Some (Tuple v rest)) (if (string-cursor-end? s rest) (Some v) None))
    (None None)))

(: (-> string (Option Date)))
(defun (iso8601->date s)
  (def (Some day) (whole s (read-date s (string-cursor-start s))) :propagate)
  (Some (day-number->date day)))

(: (-> string (Option Time)))
(defun (iso8601->time s)
  (def (Some t) (whole s (read-time s (string-cursor-start s))) :propagate)
  (Some (Time (v (ClrTime. t)))))

(: (-> string (Option DateTime)))
(defun (iso8601->datetime s)
  (def (Some t) (whole s (read-datetime s (string-cursor-start s))) :propagate)
  (Some (ticks->datetime t)))

;; Any offset, converted to UTC.
(: (-> string (Option Instant)))
(defun (iso8601->instant s)
  (def* ((Some (Tuple l c)) (read-datetime s (string-cursor-start s)) :propagate)
        ((Some o) (whole s (read-offset s c)) :propagate))
  (if (in-calendar? (- l o)) (Some (Instant (ticks (- l o)))) None))

;; The offset has to be one the zone has at that local time, so an overlap
;; reads either way and a gap reads neither.
(: (-> string (Option Moment)))
(defun (iso8601->moment s)
  (def* ((Some (Tuple l c1)) (read-datetime s (string-cursor-start s)) :propagate)
        ((Some (Tuple o c2)) (read-offset s c1) :propagate)
        ((Some c3) (skip-char s c2 #\[) :propagate)
        ((Some id) (whole s (read-until s c3 #\])) :propagate)
        ((Some z) (timezone id) :propagate))
  (if (and (in-calendar? (- l o)) (= (offset-at (struct-ref z tz) (- l o)) o))
      (Some (Moment (local (ticks->datetime l)) (offset (ticks->duration o)) (zone z)))
      None))

;; An optional sign: -1 for a minus, 1 otherwise.
(: (-> string StringCursor (Tuple long StringCursor)))
(defun (read-sign s c)
  (cond ((starts-with-char? s c #\-) (Tuple -1L (string-cursor-next s c)))
        ((starts-with-char? s c #\+) (Tuple 1L (string-cursor-next s c)))
        (else (Tuple 1L c))))

;; A unit's place in P1Y2M3W4D, or -1.
(: (-> string StringCursor int))
(defun (date-unit s c)
  (cond ((starts-with-char? s c #\Y) 0)
        ((starts-with-char? s c #\M) 1)
        ((starts-with-char? s c #\W) 2)
        ((starts-with-char? s c #\D) 3)
        (else -1)))

;; A unit's place in T1H2M3S, or -1.
(: (-> string StringCursor int))
(defun (time-unit s c)
  (cond ((starts-with-char? s c #\H) 0)
        ((starts-with-char? s c #\M) 1)
        ((starts-with-char? s c #\S) 2)
        (else -1)))

;; The date components of a period, each optional and in the order Y M W D,
;; each with its own sign, which multiplies with the period's. Stops at T or
;; at the end. next is the place of the first unit still allowed.
(: (-> string StringCursor long (Option (Tuple (Tuple DatePeriod int) StringCursor))))
(defun (read-date-parts s c sign)
  (let loop ((c c) (next 0) (p (date-period)) (count 0))
    (if (or (string-cursor-end? s c) (starts-with-char? s c #\T))
        (Some (Tuple (Tuple p count) c))
        (let ((signed (read-sign s c)))
          (def (Tuple own c1) signed)
          (def (Some (Tuple n c2)) (read-number s c1) :propagate)
          (def value (* (* sign own) n))
          (def unit (date-unit s c2))
          (if (or (< unit next) (> value 2147483647L) (< value -2147483648L))
              None
              (loop (string-cursor-next s c2) (+ unit 1)
                    (set-date-part p unit (int-of value)) (+ count 1)))))))

(: (-> DatePeriod int int DatePeriod))
(defun (set-date-part p unit n)
  (cond ((= unit 0) (record-set p (years n)))
        ((= unit 1) (record-set p (months n)))
        ((= unit 2) (record-set p (weeks n)))
        (else (record-set p (days n)))))

;; The time components after T, in the order H M S, as ticks. Only seconds
;; take a fraction.
(: (-> string StringCursor long (Option (Tuple (Tuple long int) StringCursor))))
(defun (read-time-parts s c sign)
  (let loop ((c c) (next 0) (total 0L) (count 0))
    (if (string-cursor-end? s c)
        (Some (Tuple (Tuple total count) c))
        (let ((signed (read-sign s c)))
          (def (Tuple own c1) signed)
          (def* ((Some (Tuple n c2)) (read-number s c1) :propagate)
                ((Some (Tuple f c3)) (read-optional-fraction s c2) :propagate))
          (def unit (time-unit s c3))
          (def per (cond ((= unit 0) ticks-per-hour)
                         ((= unit 1) ticks-per-minute)
                         (else ticks-per-second)))
          (cond ((< unit next) None)
                ((and (> f 0L) (not (= unit 2))) None)
                (:def (Some whole-ticks) (checked-mul n per) :propagate)
                (:def (Some magnitude) (checked-add whole-ticks f) :propagate)
                (:def (Some sum) (checked-add total (* (* sign own) magnitude)) :propagate)
                (else (loop (string-cursor-next s c3) (+ unit 1) sum (+ count 1))))))))

;; [sign] P [date parts] [T time parts]: the date period, the ticks, and
;; whether each part was there. A struct, so reading one allocates nothing.
(type (: PeriodText (Struct (: dates DatePeriod) (: date-count int)
                            (: ticks long) (: time-count int) (: has-t bool))))

(: (-> string (Option PeriodText)))
(defun (read-period-text s)
  (def (Tuple sign c0) (read-sign s (string-cursor-start s)))
  (def* ((Some c1) (skip-char s c0 #\P) :propagate)
        ((Some (Tuple (Tuple dates date-count) c2)) (read-date-parts s c1 sign) :propagate))
  (if (string-cursor-end? s c2)
      (Some (PeriodText (dates dates) (date-count date-count) (ticks 0L) (time-count 0)
                        (has-t #f)))
      (let ((after-t (string-cursor-next s c2)))
        (def (Some (Tuple (Tuple t time-count) c3)) (read-time-parts s after-t sign) :propagate)
        (Some (PeriodText (dates dates) (date-count date-count) (ticks t)
                          (time-count time-count) (has-t #t))))))

;; PT26H30M. Days are a calendar amount, so P1D is not a duration.
(: (-> string (Option Duration)))
(defun (iso8601->duration s)
  (def (Some p) (read-period-text s) :propagate)
  (if (and (= (record-ref p date-count) 0) (> (record-ref p time-count) 0))
      (Some (ticks->duration (record-ref p ticks)))
      None))

(: (-> string (Option DatePeriod)))
(defun (iso8601->date-period s)
  (def (Some p) (read-period-text s) :propagate)
  (if (and (> (record-ref p date-count) 0) (not (record-ref p has-t)))
      (Some (record-ref p dates))
      None))

;; P1DT3H. A T needs a time part after it.
(: (-> string (Option Period)))
(defun (iso8601->period s)
  (def (Some p) (read-period-text s) :propagate)
  (if (and (> (+ (record-ref p date-count) (record-ref p time-count)) 0)
           (or (not (record-ref p has-t)) (> (record-ref p time-count) 0)))
      (Some (Period (calendar (record-ref p dates)) (exact (ticks->duration (record-ref p ticks)))))
      None))


;; ===========================================================================
;; HTTP dates
;; ===========================================================================
;;
;; IMF-fixdate (RFC 9110): Sun, 25 Oct 2026 01:30:00 GMT. The two obsolete
;; forms, RFC 850 and asctime, are not read.

;; Numbered as ->weekday numbers them.
(: (-> int string))
(defun (day-name n)
  (match n (1 "Mon") (2 "Tue") (3 "Wed") (4 "Thu") (5 "Fri") (6 "Sat") (_ "Sun")))

(: (-> int string))
(defun (month-name n)
  (match n
    (1 "Jan") (2 "Feb") (3 "Mar") (4 "Apr") (5 "May") (6 "Jun")
    (7 "Jul") (8 "Aug") (9 "Sep") (10 "Oct") (11 "Nov") (_ "Dec")))

;; The fraction of a second is dropped, as the format has none.
(: (-> Instant string))
(defun (instant->http-date i)
  (def t (instant-ticks i))
  (def d (day-number->date (/ t ticks-per-day)))
  (def c (struct-ref d v))
  (def sb (new-text 29))
  (add-text! sb (day-name (->weekday d)))
  (add-text! sb ", ")
  (add-padded! sb (long-of (clr-date-day c)) 2)
  (add-text! sb " ")
  (add-text! sb (month-name (clr-date-month c)))
  (add-text! sb " ")
  (add-padded! sb (long-of (clr-date-year c)) 4)
  (add-text! sb " ")
  (add-clock! sb (% t ticks-per-day))
  (add-text! sb " GMT")
  (text-of sb))

;; The cursor n characters on, or None when s ends first.
(: (-> string StringCursor int (Option StringCursor)))
(defun (skip-chars s c n)
  (cond ((= n 0) (Some c))
        ((string-cursor-end? s c) None)
        (else (skip-chars s (string-cursor-next s c) (- n 1)))))

;; The cursor after the text want, when s has it at c.
(: (-> string StringCursor string (Option StringCursor)))
(defun (skip-string s c want)
  (let loop ((c c) (w (string-cursor-start want)))
    (cond ((string-cursor-end? want w) (Some c))
          ((starts-with-char? s c (string-cursor-ref want w))
           (loop (string-cursor-next s c) (string-cursor-next want w)))
          (else None))))

;; A month's name, as its number.
(: (-> string StringCursor (Option (Tuple int StringCursor))))
(defun (read-month s c)
  (let loop ((m 1))
    (if (> m 12)
        None
        (match (skip-string s c (month-name m))
          ((Some after) (Some (Tuple m after)))
          (None (loop (+ m 1)))))))

;; The day name must be the date's own, so it is checked once the date is read.
(: (-> string (Option Instant)))
(defun (http-date->instant s)
  (def c0 (string-cursor-start s))
  (def* ((Some c1) (skip-chars s c0 3) :propagate)
        ((Some c2) (skip-string s c1 ", ") :propagate)
        ((Some (Tuple d c3)) (read-fixed s c2 2) :propagate)
        ((Some c4) (skip-char s c3 #\space) :propagate)
        ((Some (Tuple m c5)) (read-month s c4) :propagate)
        ((Some c6) (skip-char s c5 #\space) :propagate)
        ((Some (Tuple y c7)) (read-fixed s c6 4) :propagate)
        ((Some c8) (skip-char s c7 #\space) :propagate)
        ((Some (Tuple h c9)) (read-fixed s c8 2) :propagate)
        ((Some c10) (skip-char s c9 #\:) :propagate)
        ((Some (Tuple mi c11)) (read-fixed s c10 2) :propagate)
        ((Some c12) (skip-char s c11 #\:) :propagate)
        ((Some (Tuple sec c13)) (read-fixed s c12 2) :propagate)
        ((Some c14) (skip-string s c13 " GMT") :propagate))
  (if (and (string-cursor-end? s c14) (valid-date? y m d) (valid-time? h mi sec 0))
      (let ((day (long-of (clr-date-day-number (ClrDate. y m d)))))
        (if (some? (skip-string s c0 (day-name (->weekday (day-number->date day)))))
            (Some (Instant (ticks (+ (* day ticks-per-day) (time-of-day-ticks h mi sec 0)))))
            None))
      None))


;; ===========================================================================
;; Text of your own
;; ===========================================================================
;;
;; A format is a quoted list. The grammar `DatePart` reads it into the parts
;; below. A keyword names a field, and a string or a character is written as
;; it is.
;;
;;   (date->text d '((year 4) "-" (month 2) "-" (day 2)))      ;; 2026-03-05
;;
;; A number after the name of a field is a width, filled with zeros. A bare
;; name, such as `year`, has no width. There is one production for each kind
;; of value, so a format that asks a date for its hours is a compile error
;; where it is written.
;;
;; The text and the characters are in the parts only: `DatePart`,
;; `TimePart`, `DateTimePart`, `MomentPart` and `InstantPart`. The fields are
;; in `DateField`, `TimeField` and `DateTimeField`, which have no text. Thus
;; the grammar knows the part that a string goes in.
(def/grammar DatePart
  (DatePart (:or (: DateOf DateField) (: DateText string) (: DateChar char)))

  ;; A date's numbers, its Roman numerals and its names, from a DateNames.
  (DateField (:or (: DateNum DateNumber)
                  (: DateBare DateBareNumber)
                  (: DateRoman (roman RomanNumber))
                  (: MonthName 'month-name)
                  (: MonthAbbr 'month-abbr)
                  (: WeekdayName 'weekday-name)
                  (: WeekdayAbbr 'weekday-abbr)))

  ;; The numbers of a date, with a width: (year 4). A width list that is
  ;; empty, (year), is the same as a bare year.
  (DateNumber (:or (: NumYear (year int ...))
                   (: NumMonth (month int ...))
                   (: NumDay (day int ...))
                   (: NumWeekday (weekday int ...))
                   (: NumYday (yday int ...))
                   (: NumWeek (week int ...))
                   (: NumWeekYear (week-year int ...))))

  ;; The numbers of a date without a width: year.
  (DateBareNumber (:or (: BareYear 'year)
                       (: BareMonth 'month)
                       (: BareDay 'day)
                       (: BareWeekday 'weekday)
                       (: BareYday 'yday)
                       (: BareWeek 'week)
                       (: BareWeekYear 'week-year)))

  ;; What (roman ...) writes. No width, as a numeral has no zeros to fill
  ;; with.
  (RomanNumber (:or (: RomanYear 'year) (: RomanMonth 'month) (: RomanDay 'day)))

  (TimePart (:or (: TimeOf TimeField) (: TimeText string) (: TimeChar char)))

  ;; (fraction n) is n digits of the second's fraction, cut, not rounded,
  ;; with zeros past the seventh. A bare fraction is ISO 8601's, point
  ;; included: .5, and nothing for a whole second.
  (TimeField (:or (: TimeHours (hours int ...))
                  (: TimeHours12 (hours12 int ...))
                  (: TimeMinutes (minutes int ...))
                  (: TimeSeconds (seconds int ...))
                  (: TimeFraction (fraction int ...))
                  (: TimeAmPm 'am-pm)
                  (: TimeBare TimeBareNumber)))

  ;; The numbers of a time without a width: hours.
  (TimeBareNumber (:or (: BareHours 'hours)
                       (: BareHours12 'hours12)
                       (: BareMinutes 'minutes)
                       (: BareSeconds 'seconds)
                       (: BareFraction 'fraction)))

  (DateTimePart (:or (: DateTimeOf DateTimeField) (: DateTimeText string) (: DateTimeChar char)))

  ;; The fields of a date and of a time.
  (DateTimeField (:or (: DateTimeDate DateField) (: DateTimeTime TimeField)))

  ;; A date and a time, the offset as ISO 8601 writes it (+02:00), the zone's
  ;; id, and the Unix time in seconds or milliseconds, rounded toward the
  ;; past.
  (MomentPart (:or (: MomentDateTime DateTimeField)
                   (: MomentOffset 'offset)
                   (: MomentZone 'zone)
                   (: MomentUnix 'unix)
                   (: MomentUnixMs 'unix-ms)
                   (: MomentText string)
                   (: MomentChar char)))

  ;; A date and a time in UTC, whose offset is written Z, and the Unix time.
  (InstantPart (:or (: InstantDateTime DateTimeField)
                    (: InstantOffset 'offset)
                    (: InstantUnix 'unix)
                    (: InstantUnixMs 'unix-ms)
                    (: InstantText string)
                    (: InstantChar char))))

;; A bare number of a date is the number without a width.
(: (-> DateBareNumber DateNumber))
(defun (bare-date-number b)
  (match b
    (BareYear (NumYear Nil))
    (BareMonth (NumMonth Nil))
    (BareDay (NumDay Nil))
    (BareWeekday (NumWeekday Nil))
    (BareYday (NumYday Nil))
    (BareWeek (NumWeek Nil))
    (BareWeekYear (NumWeekYear Nil))))

;; A bare number of a time is the number without a width.
(: (-> TimeBareNumber TimeField))
(defun (bare-time-field b)
  (match b
    (BareHours (TimeHours Nil))
    (BareHours12 (TimeHours12 Nil))
    (BareMinutes (TimeMinutes Nil))
    (BareSeconds (TimeSeconds Nil))
    (BareFraction (TimeFraction Nil))))

;; Twelve months from January and seven weekdays from Monday, and what am-pm
;; writes. A table of other lengths raises when a name is taken from it.
(type (: DateNames (Struct (: months (Vec string)) (: months-short (Vec string))
                           (: weekdays (Vec string)) (: weekdays-short (Vec string))
                           (: am string) (: pm string))))

(: DateNames)
(def english-names
  (DateNames (months ["January" "February" "March" "April" "May" "June" "July"
                      "August" "September" "October" "November" "December"])
             (months-short ["Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct"
                            "Nov" "Dec"])
             (weekdays ["Monday" "Tuesday" "Wednesday" "Thursday" "Friday" "Saturday"
                        "Sunday"])
             (weekdays-short ["Mon" "Tue" "Wed" "Thu" "Fri" "Sat" "Sun"])
             (am "AM")
             (pm "PM")))

(: DateNames)
(def swedish-names
  (DateNames (months ["januari" "februari" "mars" "april" "maj" "juni" "juli"
                      "augusti" "september" "oktober" "november" "december"])
             (months-short ["jan" "feb" "mar" "apr" "maj" "jun" "jul" "aug" "sep" "okt"
                            "nov" "dec"])
             (weekdays ["måndag" "tisdag" "onsdag" "torsdag" "fredag" "lördag" "söndag"])
             (weekdays-short ["mån" "tis" "ons" "tor" "fre" "lör" "sön"])
             (am "fm")
             (pm "em")))

;; After english-names, which is its first value.
(: (Param DateNames))
(def current-date-names (make-parameter english-names))

(: (-> string %a))
(defun (argument-error what)
  (raise (cast Exception (ArgumentException. (str "(std datetime): " what)))))

;; The width a part was written with, 0 for none.
(: (-> (List int) string int))
(defun (part-width ws tag)
  (cond ((list-empty? ws) 0)
        ((list-empty? (list-tail ws)) (list-head ws))
        (else (argument-error (str "(" tag " ...) takes one width, and was given "
                                   (displayed (list-length ws)) ".")))))

;; A table of names, which must have count of them.
(: (-> (Vec string) int string (Vec string)))
(defun (full-table names count what)
  (if (= (vec-length names) count)
      names
      (argument-error (str "a DateNames has " (displayed count) " " what ", and this one has "
                           (displayed (vec-length names)) "."))))

;; The i:th name, counting from 1, of a table that must have count of them.
(: (-> (Vec string) int int string string))
(defun (name-at names count i what) (vec-ref (full-table names count what) (- i 1)))

;; MMXXVI. Numerals stop at MMMCMXCIX, and a larger year goes on adding Ms,
;; 4000 being MMMM, since nothing else is standard.
(: (List (Tuple int string)))
(def roman-steps
  (list (Tuple 1000 "M") (Tuple 900 "CM") (Tuple 500 "D") (Tuple 400 "CD")
        (Tuple 100 "C") (Tuple 90 "XC") (Tuple 50 "L") (Tuple 40 "XL")
        (Tuple 10 "X") (Tuple 9 "IX") (Tuple 5 "V") (Tuple 4 "IV") (Tuple 1 "I")))

(: (-> TextBuilder int void))
(defun (add-roman! sb n)
  (let loop ((n n) (steps roman-steps))
    (unless (list-empty? steps)
      (match (list-head steps)
        ((Tuple value numeral)
         (if (>= n value)
             (begin
               (add-text! sb numeral)
               (loop (- n value) steps))
             (loop n (list-tail steps))))))))

;; The first n digits of a second's fraction f, in ticks: cut, not rounded,
;; and zeros past the seventh.
(: (-> TextBuilder long int void))
(defun (add-fraction-digits! sb f n)
  (let loop ((i 0) (scale 1000000L))
    (when (< i n)
      (add-digit! sb (if (> scale 0L) (% (/ f scale) 10L) 0L))
      (loop (+ i 1) (/ scale 10L)))))

(: (-> TextBuilder Date DateNumber void))
(defun (add-date-number! sb d n)
  (match n
    ((NumYear w) (add-padded! sb (long-of (->year d)) (part-width w "year")))
    ((NumMonth w) (add-padded! sb (long-of (->month d)) (part-width w "month")))
    ((NumDay w) (add-padded! sb (long-of (->day d)) (part-width w "day")))
    ((NumWeekday w) (add-padded! sb (long-of (->weekday d)) (part-width w "weekday")))
    ((NumYday w) (add-padded! sb (long-of (->yday d)) (part-width w "yday")))
    ((NumWeek w) (add-padded! sb (long-of (->week d)) (part-width w "week")))
    ((NumWeekYear w) (add-padded! sb (long-of (->week-year d)) (part-width w "week-year")))))

(: (-> TextBuilder Date DateNames DateField void))
(defun (add-date-field! sb d names p)
  (match p
    ((DateNum n) (add-date-number! sb d n))
    ((DateBare b) (add-date-number! sb d (bare-date-number b)))
    ((DateRoman RomanYear) (add-roman! sb (->year d)))
    ((DateRoman RomanMonth) (add-roman! sb (->month d)))
    ((DateRoman RomanDay) (add-roman! sb (->day d)))
    (MonthName (add-text! sb (name-at (record-ref names months) 12 (->month d) "months")))
    (MonthAbbr (add-text! sb (name-at (record-ref names months-short) 12 (->month d) "months")))
    (WeekdayName (add-text! sb (name-at (record-ref names weekdays) 7 (->weekday d) "weekdays")))
    (WeekdayAbbr
     (add-text! sb (name-at (record-ref names weekdays-short) 7 (->weekday d) "weekdays")))))

(: (-> TextBuilder Date DateNames DatePart void))
(defun (add-date-part! sb d names p)
  (match p
    ((DateOf f) (add-date-field! sb d names f))
    ((DateText s) (add-text! sb s))
    ((DateChar c) (add-text! sb (char->string c)))))

;; t is the ticks since midnight.
(: (-> TextBuilder long DateNames TimeField void))
(defun (add-time-field! sb t names p)
  (def h (/ t ticks-per-hour))
  (match p
    ((TimeHours w) (add-padded! sb h (part-width w "hours")))
    ((TimeHours12 w) (add-padded! sb (if (= (% h 12L) 0L) 12L (% h 12L)) (part-width w "hours12")))
    ((TimeMinutes w) (add-padded! sb (% (/ t ticks-per-minute) 60L) (part-width w "minutes")))
    ((TimeSeconds w) (add-padded! sb (% (/ t ticks-per-second) 60L) (part-width w "seconds")))
    ((TimeFraction w)
     (if (list-empty? w)
         (add-fraction! sb (% t ticks-per-second))
         (add-fraction-digits! sb (% t ticks-per-second) (part-width w "fraction"))))
    (TimeAmPm (add-text! sb (if (< h 12L) (record-ref names am) (record-ref names pm))))
    ((TimeBare b) (add-time-field! sb t names (bare-time-field b)))))

;; t is the ticks since midnight.
(: (-> TextBuilder long DateNames TimePart void))
(defun (add-time-part! sb t names p)
  (match p
    ((TimeOf f) (add-time-field! sb t names f))
    ((TimeText s) (add-text! sb s))
    ((TimeChar c) (add-text! sb (char->string c)))))

;; t is local ticks.
(: (-> TextBuilder long DateNames DateTimeField void))
(defun (add-datetime-field! sb t names p)
  (match p
    ((DateTimeDate df) (add-date-field! sb (day-number->date (/ t ticks-per-day)) names df))
    ((DateTimeTime tf) (add-time-field! sb (% t ticks-per-day) names tf))))

;; t is local ticks.
(: (-> TextBuilder long DateNames DateTimePart void))
(defun (add-datetime-part! sb t names p)
  (match p
    ((DateTimeOf f) (add-datetime-field! sb t names f))
    ((DateTimeText s) (add-text! sb s))
    ((DateTimeChar c) (add-text! sb (char->string c)))))

;; n in decimal, with a minus when it is negative.
(: (-> TextBuilder long void))
(defun (add-signed! sb n)
  (when (< n 0L) (add-text! sb "-"))
  (add-padded! sb (abs n) 1))

(: (-> TextBuilder Moment DateNames MomentPart void))
(defun (add-moment-part! sb m names p)
  (match p
    ((MomentDateTime dtf) (add-datetime-field! sb (moment-local-ticks m) names dtf))
    (MomentOffset (add-offset! sb (moment-offset-ticks m)))
    (MomentZone (add-text! sb (timezone-id (struct-ref m zone))))
    (MomentUnix
     (add-signed! sb (instant->unix-seconds (Instant (ticks (moment-instant-ticks m))))))
    (MomentUnixMs
     (add-signed! sb (instant->unix-milliseconds (Instant (ticks (moment-instant-ticks m))))))
    ((MomentText s) (add-text! sb s))
    ((MomentChar c) (add-text! sb (char->string c)))))

(: (-> TextBuilder Instant DateNames InstantPart void))
(defun (add-instant-part! sb i names p)
  (match p
    ((InstantDateTime dtf) (add-datetime-field! sb (instant-ticks i) names dtf))
    (InstantOffset (add-text! sb "Z"))
    (InstantUnix (add-signed! sb (instant->unix-seconds i)))
    (InstantUnixMs (add-signed! sb (instant->unix-milliseconds i)))
    ((InstantText s) (add-text! sb s))
    ((InstantChar c) (add-text! sb (char->string c)))))

;; One function per type rather than one over the traits, so a moment's date
;; is written with (date->text (->date m) ...), which says whose date it is.
(: (-> Date (List DatePart) (#:names DateNames) string))
(defun (date->text d parts #:names (parameter-ref current-date-names))
  (def sb (new-text 32))
  (let loop ((ps parts))
    (unless (list-empty? ps)
      (add-date-part! sb d names (list-head ps))
      (loop (list-tail ps))))
  (text-of sb))

(: (-> Time (List TimePart) (#:names DateNames) string))
(defun (time->text t parts #:names (parameter-ref current-date-names))
  (def sb (new-text 32))
  (let loop ((ps parts))
    (unless (list-empty? ps)
      (add-time-part! sb (clr-time-ticks (struct-ref t v)) names (list-head ps))
      (loop (list-tail ps))))
  (text-of sb))

(: (-> DateTime (List DateTimePart) (#:names DateNames) string))
(defun (datetime->text dt parts #:names (parameter-ref current-date-names))
  (def sb (new-text 48))
  (let loop ((ps parts))
    (unless (list-empty? ps)
      (add-datetime-part! sb (datetime-ticks dt) names (list-head ps))
      (loop (list-tail ps))))
  (text-of sb))

(: (-> Moment (List MomentPart) (#:names DateNames) string))
(defun (moment->text m parts #:names (parameter-ref current-date-names))
  (def sb (new-text 64))
  (let loop ((ps parts))
    (unless (list-empty? ps)
      (add-moment-part! sb m names (list-head ps))
      (loop (list-tail ps))))
  (text-of sb))

;; The date and time are UTC's, so the format that writes a moment's local
;; time writes the instant as `written` does.
(: (-> Instant (List InstantPart) (#:names DateNames) string))
(defun (instant->text i parts #:names (parameter-ref current-date-names))
  (def sb (new-text 48))
  (let loop ((ps parts))
    (unless (list-empty? ps)
      (add-instant-part! sb i names (list-head ps))
      (loop (list-tail ps))))
  (text-of sb))


;; ===========================================================================
;; Reading text of your own
;; ===========================================================================
;;
;; A reader sees each part of a format as a Field, whichever union it came
;; from, so one reader serves every format. It checks the format first, so a
;; format that cannot be read raises whatever the text, and then walks the
;; text: the parts in order, and the whole text used. The fields read so far
;; are kept in a Seen, a struct, and put together at the end, so reading
;; allocates nothing but a zone id.

;; What a part reads. A match on it tests its arms in order, so text,
;; characters and the common fields come first where that matters.
(type (: FieldKind (Union FText FChar
                          FYear FMonth FDay FWeekday FYday FWeek FWeekYear
                          FRomanYear FRomanMonth FRomanDay
                          FMonthName FMonthAbbr FWeekdayName FWeekdayAbbr
                          FHours FHours12 FMinutes FSeconds FFraction FAmPm
                          FOffset FZone FUnix FUnixMs)))

;; A part as a reader sees it: its width, 0 for none; the most digits it
;; reads, 0 when it is not a number; and its text or character when it is
;; one. A bare fraction, which can be nothing at all, has the width -1, so
;; that it is told apart from (fraction 0). The functions that make a Field
;; know its kind, so they say its most digits, and no reader asks again.
(type (: Field (Struct (: kind FieldKind) (: width int) (: most int) (: text string) (: ch char))))

(: (-> FieldKind int Field))
(defun (field k w) (Field (kind k) (width w) (most 0) (text "") (ch #\nul)))

(: (-> FieldKind int int Field))
(defun (number-field k most w) (Field (kind k) (width w) (most most) (text "") (ch #\nul)))

(: (-> string Field))
(defun (text-field s) (Field (kind FText) (width 0) (most 0) (text s) (ch #\nul)))

(: (-> char Field))
(defun (char-field c) (Field (kind FChar) (width 0) (most 0) (text "") (ch c)))

(: (-> DateNumber Field))
(defun (date-number-field n)
  (match n
    ((NumYear w) (number-field FYear 4 (part-width w "year")))
    ((NumMonth w) (number-field FMonth 2 (part-width w "month")))
    ((NumDay w) (number-field FDay 2 (part-width w "day")))
    ((NumWeekday w) (number-field FWeekday 1 (part-width w "weekday")))
    ((NumYday w) (number-field FYday 3 (part-width w "yday")))
    ((NumWeek w) (number-field FWeek 2 (part-width w "week")))
    ((NumWeekYear w) (number-field FWeekYear 4 (part-width w "week-year")))))

(: (-> DateField Field))
(defun (date-field-field p)
  (match p
    ((DateNum n) (date-number-field n))
    ((DateBare b) (date-number-field (bare-date-number b)))
    ((DateRoman RomanYear) (field FRomanYear 0))
    ((DateRoman RomanMonth) (field FRomanMonth 0))
    ((DateRoman RomanDay) (field FRomanDay 0))
    (MonthName (field FMonthName 0))
    (MonthAbbr (field FMonthAbbr 0))
    (WeekdayName (field FWeekdayName 0))
    (WeekdayAbbr (field FWeekdayAbbr 0))))

(: (-> DatePart Field))
(defun (date-part-field p)
  (match p
    ((DateText s) (text-field s))
    ((DateChar c) (char-field c))
    ((DateOf f) (date-field-field f))))

(: (-> TimeField Field))
(defun (time-field-field p)
  (match p
    ((TimeHours w) (number-field FHours 2 (part-width w "hours")))
    ((TimeHours12 w) (number-field FHours12 2 (part-width w "hours12")))
    ((TimeMinutes w) (number-field FMinutes 2 (part-width w "minutes")))
    ((TimeSeconds w) (number-field FSeconds 2 (part-width w "seconds")))
    ((TimeFraction w)
     (field FFraction (if (list-empty? w) -1 (max (part-width w "fraction") 0))))
    (TimeAmPm (field FAmPm 0))
    ((TimeBare b) (time-field-field (bare-time-field b)))))

(: (-> TimePart Field))
(defun (time-part-field p)
  (match p
    ((TimeText s) (text-field s))
    ((TimeChar c) (char-field c))
    ((TimeOf f) (time-field-field f))))

(: (-> DateTimeField Field))
(defun (datetime-field-field p)
  (match p
    ((DateTimeDate df) (date-field-field df))
    ((DateTimeTime tf) (time-field-field tf))))

(: (-> DateTimePart Field))
(defun (datetime-part-field p)
  (match p
    ((DateTimeText s) (text-field s))
    ((DateTimeChar c) (char-field c))
    ((DateTimeOf f) (datetime-field-field f))))

(: (-> MomentPart Field))
(defun (moment-part-field p)
  (match p
    ((MomentText s) (text-field s))
    ((MomentChar c) (char-field c))
    ((MomentDateTime dtf) (datetime-field-field dtf))
    (MomentOffset (field FOffset 0))
    (MomentZone (field FZone 0))
    (MomentUnix (field FUnix 0))
    (MomentUnixMs (field FUnixMs 0))))

(: (-> InstantPart Field))
(defun (instant-part-field p)
  (match p
    ((InstantText s) (text-field s))
    ((InstantChar c) (char-field c))
    ((InstantDateTime dtf) (datetime-field-field dtf))
    (InstantOffset (field FOffset 0))
    (InstantUnix (field FUnix 0))
    (InstantUnixMs (field FUnixMs 0))))

;; Each format union's parts as Fields, text first, as separators are the
;; most common part. A trait rather than a function argument, so that each
;; reader is compiled for its own union.
(def/trait (FormatPart %p)
  (: part-field (-> %p Field)))

(impl (FormatPart DatePart)
  (defun (part-field p) (date-part-field p)))

(impl (FormatPart TimePart)
  (defun (part-field p) (time-part-field p)))

(impl (FormatPart DateTimePart)
  (defun (part-field p) (datetime-part-field p)))

(impl (FormatPart MomentPart)
  (defun (part-field p) (moment-part-field p)))

(impl (FormatPart InstantPart)
  (defun (part-field p) (instant-part-field p)))

(: (-> string int string))
(defun (with-width name w) (if (> w 0) (str "(" name " " (displayed w) ")") name))

;; A part as a format writes it, for a message.
(: (-> Field string))
(defun (field-name f)
  (def w (struct-ref f width))
  (match (struct-ref f kind)
    (FText (str "the text \"" (struct-ref f text) "\""))
    (FChar (str "the character " (char->string (struct-ref f ch))))
    (FYear (with-width "year" w))
    (FMonth (with-width "month" w))
    (FDay (with-width "day" w))
    (FWeekday (with-width "weekday" w))
    (FYday (with-width "yday" w))
    (FWeek (with-width "week" w))
    (FWeekYear (with-width "week-year" w))
    (FRomanYear "(roman year)")
    (FRomanMonth "(roman month)")
    (FRomanDay "(roman day)")
    (FMonthName "month-name")
    (FMonthAbbr "month-abbr")
    (FWeekdayName "weekday-name")
    (FWeekdayAbbr "weekday-abbr")
    (FHours (with-width "hours" w))
    (FHours12 (with-width "hours12" w))
    (FMinutes (with-width "minutes" w))
    (FSeconds (with-width "seconds" w))
    (FFraction (if (< w 0) "fraction" (str "(fraction " (displayed w) ")")))
    (FAmPm "am-pm")
    (FOffset "offset")
    (FZone "zone")
    (FUnix "unix")
    (FUnixMs "unix-ms")))

;; The kinds of field a format has, one bit per kind that the checks below
;; tell apart.
(: int)
(def b-year 1)
;; month, day and yday, by number, numeral or name: the fields within a year.
(: int)
(def b-in-year 2)
(: int)
(def b-week-year 4)
(: int)
(def b-week 8)
(: int)
(def b-weekday 16)
(: int)
(def b-hours 32)
(: int)
(def b-hours12 64)
(: int)
(def b-am-pm 128)
;; minutes, seconds and the fraction: the fields within an hour.
(: int)
(def b-in-hour 256)
(: int)
(def b-offset 512)
(: int)
(def b-zone 1024)
(: int)
(def b-unix 2048)
(: int)
(def b-unix-ms 4096)

(: (-> FieldKind int))
(defun (field-bits k)
  (match k
    (FText 0)
    (FChar 0)
    (FYear b-year)
    (FRomanYear b-year)
    (FMonth b-in-year)
    (FDay b-in-year)
    (FYday b-in-year)
    (FRomanMonth b-in-year)
    (FRomanDay b-in-year)
    (FMonthName b-in-year)
    (FMonthAbbr b-in-year)
    (FWeekYear b-week-year)
    (FWeek b-week)
    (FWeekday b-weekday)
    (FWeekdayName b-weekday)
    (FWeekdayAbbr b-weekday)
    (FHours b-hours)
    (FHours12 b-hours12)
    (FAmPm b-am-pm)
    (FMinutes b-in-hour)
    (FSeconds b-in-hour)
    (FFraction b-in-hour)
    (FOffset b-offset)
    (FZone b-zone)
    (FUnix b-unix)
    (FUnixMs b-unix-ms)))

(: (-> int int bool))
(defun (has? bits b) (> (bitwise-and bits b) 0))

;; The characters a part can go on taking once it has begun, as bits: a
;; part of one fixed length takes none.
(: int)
(def t-digits 1)
(: int)
(def t-roman 2)
;; A decimal point or comma, which a bare fraction may begin with.
(: int)
(def t-point 4)
;; The characters of an IANA id.
(: int)
(def t-zone 8)
;; The colon before an offset's seconds.
(: int)
(def t-colon 16)

(: (-> char int))
(defun (roman-value c)
  (cond ((char=? c #\I) 1)
        ((char=? c #\V) 5)
        ((char=? c #\X) 10)
        ((char=? c #\L) 50)
        ((char=? c #\C) 100)
        ((char=? c #\D) 500)
        ((char=? c #\M) 1000)
        (else 0)))

;; ASCII letters and digits, and / _ - +, as in Etc/GMT+5 and
;; America/Port-au-Prince.
(: (-> char bool))
(defun (zone-char? c)
  (def n (char->int c))
  (or (and (>= n 65) (<= n 90)) (and (>= n 97) (<= n 122)) (and (>= n 48) (<= n 57))
      (= n 47) (= n 95) (= n 45) (= n 43)))

(: (-> int char bool))
(defun (takes? taken c)
  (or (and (has? taken t-digits) (>= (ascii-digit c) 0))
      (and (has? taken t-roman) (> (roman-value c) 0))
      (and (has? taken t-point) (or (char=? c #\.) (char=? c #\,)))
      (and (has? taken t-zone) (zone-char? c))
      (and (has? taken t-colon) (char=? c #\:))))

;; A number reads from its width, or 1, up to its most digits or its width,
;; so it has one length only when its width is at least its most. A bare
;; fraction is nothing, or a point and as many digits as there are, and an
;; offset may have seconds after a colon.
(: (-> Field int))
(defun (field-takes f)
  (def k (struct-ref f kind))
  (def w (struct-ref f width))
  (def most (struct-ref f most))
  (if (> most 0)
      (if (< (max w 1) most) t-digits 0)
      (match k
        (FText 0)
        (FChar 0)
        (FRomanYear t-roman)
        (FRomanMonth t-roman)
        (FRomanDay t-roman)
        (FFraction (if (< w 0) (bitwise-ior t-digits t-point) 0))
        (FOffset t-colon)
        (FZone t-zone)
        (FUnix t-digits)
        (FUnixMs t-digits)
        (_ 0))))

(: (-> Field bool))
(defun (can-be-empty? f)
  (match (struct-ref f kind)
    (FText (string-empty? (struct-ref f text)))
    (FFraction (<= (struct-ref f width) 0))
    (_ #f)))

(: (-> string char))
(defun (first-char s) (string-cursor-ref s (string-cursor-start s)))

(: (-> string int bool))
(defun (name-starts? name taken)
  (and (not (string-empty? name)) (takes? taken (first-char name))))

(: (-> (Vec string) int bool))
(defun (any-name-starts? table taken)
  (let loop ((i 0))
    (and (< i (vec-length table))
         (or (name-starts? (vec-ref table i) taken) (loop (+ i 1))))))

;; The table a name reads from, which must be whole.
(: (-> DateNames FieldKind (Vec string)))
(defun (name-table names k)
  (match k
    (FMonthName (full-table (record-ref names months) 12 "months"))
    (FMonthAbbr (full-table (record-ref names months-short) 12 "months"))
    (FWeekdayName (full-table (record-ref names weekdays) 7 "weekdays"))
    (_ (full-table (record-ref names weekdays-short) 7 "weekdays"))))

(: (-> FieldKind bool))
(defun (name-kind? k)
  (match k
    (FText #f)
    (FChar #f)
    (FMonthName #t)
    (FMonthAbbr #t)
    (FWeekdayName #t)
    (FWeekdayAbbr #t)
    (_ #f)))

;; Whether a part can begin with a character in taken. Digits, Roman
;; letters and the letters an id begins with are each one class, so one of
;; them stands for all: M is a Roman letter and begins MST.
(: (-> Field DateNames int bool))
(defun (starts-within? f names taken)
  (def k (struct-ref f kind))
  (def w (struct-ref f width))
  (cond ((> (struct-ref f most) 0) (takes? taken #\0))
        ((name-kind? k) (any-name-starts? (name-table names k) taken))
        (else
         (match k
           (FText (name-starts? (struct-ref f text) taken))
           (FChar (takes? taken (struct-ref f ch)))
           (FRomanYear (takes? taken #\M))
           (FRomanMonth (takes? taken #\M))
           (FRomanDay (takes? taken #\M))
           (FFraction (cond ((< w 0) (or (takes? taken #\.) (takes? taken #\,)))
                            ((> w 0) (takes? taken #\0))
                            (else #f)))
           (FAmPm (or (name-starts? (record-ref names am) taken)
                      (name-starts? (record-ref names pm) taken)))
           (FOffset (or (takes? taken #\Z) (takes? taken #\+) (takes? taken #\-)))
           (FZone (takes? taken #\M))
           (FUnix (or (takes? taken #\-) (takes? taken #\0)))
           (FUnixMs (or (takes? taken #\-) (takes? taken #\0)))
           (_ #f)))))

(: (-> Field Field %a))
(defun (two-readings-error before after)
  (def most (struct-ref before most))
  (argument-error
    (str (field-name before) " can be of more than one length, and " (field-name after)
         " after it can begin with a character " (field-name before)
         " could also take, so a text could be read two ways. "
         (if (> most 0)
             (str "A width of " (displayed most) ", or text between the two, reads one way.")
             "Text between the two reads one way."))))

;; The kinds of field a format has, as bits, once it is known that no text
;; can be read with it two ways: no part that can be of more than one length
;; is followed by a part that can begin with a character it could also take.
;; A part that can be empty passes what could be taken on to the next.
(: (-> (List %p) DateNames int) (where (FormatPart %p)))
(defun (format-bits parts names)
  (let loop ((ps parts) (bits 0) (taken 0) (before (text-field "")))
    (if (list-empty? ps)
        bits
        (let ((f (part-field (list-head ps))))
          (def k (struct-ref f kind))
          (when (name-kind? k) (ignore (name-table names k)))
          (when (and (> taken 0) (starts-within? f names taken))
            (two-readings-error before f))
          (def own (field-takes f))
          (loop (list-tail ps)
                (bitwise-ior bits (field-bits k))
                (if (can-be-empty? f) (bitwise-ior taken own) own)
                (if (> own 0) f before))))))

;; An offset or a Unix time not read, which no reading can be.
(: long)
(def no-ticks -9223372036854775808L)

;; The fields read so far, each -1 until it is read, or no-ticks for those
;; in ticks. pm is 0 for the morning's word and 1 for the afternoon's, the
;; fraction is in ticks, read to fraction-digits digits, and unix is the
;; instant's UTC ticks, from either kind of Unix time. The zone is the span of
;; text its id was read from, empty until then, and is looked up once at the
;; end. So a Seen holds no reference, and .NET need not clear every copy of
;; it a function might make each time the function is called.
(type (: Seen (Struct (: year int) (: month int) (: day int) (: weekday int) (: yday int)
                      (: week int) (: week-year int)
                      (: hours int) (: hours12 int) (: pm int) (: minutes int) (: seconds int)
                      (: fraction long) (: fraction-digits int)
                      (: offset long) (: unix long)
                      (: zone-start StringCursor) (: zone-end StringCursor))))

(: Seen)
(def unseen
  (Seen (year -1) (month -1) (day -1) (weekday -1) (yday -1) (week -1) (week-year -1)
        (hours -1) (hours12 -1) (pm -1) (minutes -1) (seconds -1)
        (fraction -1L) (fraction-digits 0)
        (offset no-ticks) (unix no-ticks)
        (zone-start (string-cursor-start "")) (zone-end (string-cursor-start ""))))

;; From least to most ASCII digits, as many as there are. The value stops
;; growing at eighteen digits, which no field reaches.
(: (-> string StringCursor int int (Option (Tuple long StringCursor))))
(defun (read-digits s c least most)
  (let loop ((c c) (count 0) (acc 0L))
    (def d (next-digit s c))
    (if (or (= count most) (< d 0))
        (if (>= count least) (Some (Tuple acc c)) None)
        (loop (string-cursor-next s c) (+ count 1)
              (if (>= acc 100000000000000000L) acc (+ (* acc 10L) (long-of d)))))))

;; Exactly n digits of a second's fraction, as ticks. Digits past the
;; seventh are read and dropped, as a tick holds no less than 100 ns.
(: (-> string StringCursor int (Option (Tuple long StringCursor))))
(defun (read-fraction-digits s c n)
  (let loop ((c c) (i 0) (acc 0L))
    (def d (next-digit s c))
    (cond ((= i n) (Some (Tuple (scale-fraction acc (min n 7)) c)))
          ((< d 0) None)
          (else (loop (string-cursor-next s c) (+ i 1)
                      (if (< i 7) (+ (* acc 10L) (long-of d)) acc))))))

;; The cursor after n's numeral as add-roman! writes it, when s has it at c.
(: (-> string StringCursor int (Option StringCursor)))
(defun (skip-roman s c n)
  (let loop ((c c) (n n) (steps roman-steps))
    (if (list-empty? steps)
        (Some c)
        (match (list-head steps)
          ((Tuple value numeral)
           (cond ((< n value) (loop c n (list-tail steps)))
                 (:def (Some after) (skip-string s c numeral) :propagate)
                 (else (loop after (- n value) steps))))))))

;; A run of Roman letters, read only as add-roman! writes its value: MCM,
;; not MDCCCC, and IV, not IIII. The value stops growing past 100 000, and
;; is None there, as no field reaches it.
(: (-> string StringCursor (Option (Tuple long StringCursor))))
(defun (read-roman s c)
  (let loop ((e c) (total 0) (prev 0))
    (def v (if (string-cursor-end? s e) 0 (roman-value (string-cursor-ref s e))))
    (cond ((> v 0)
           (loop (string-cursor-next s e)
                 (cond ((> total 100000) total)
                       ((> v prev) (+ total (- v (* 2 prev))))
                       (else (+ total v)))
                 v))
          ((or (< total 1) (> total 100000)) None)
          (:def (Some after) (skip-roman s c total) :propagate)
          ((string-cursor=? after e) (Some (Tuple (long-of total) e)))
          (else None))))

;; The number, from 1, of the longest name in the table that s has at c.
(: (-> string StringCursor (Vec string) (Option (Tuple long StringCursor))))
(defun (read-name s c table)
  (let loop ((i 0) (best 0) (best-end c))
    (if (= i (vec-length table))
        (if (= best 0) None (Some (Tuple (long-of best) best-end)))
        (match (skip-string s c (vec-ref table i))
          ((Some e) (if (or (= best 0) (string-cursor>? e best-end))
                        (loop (+ i 1) (+ i 1) e)
                        (loop (+ i 1) best best-end)))
          (None (loop (+ i 1) best best-end))))))

;; 0 for the morning's word and 1 for the afternoon's, the longer when both
;; match.
(: (-> string StringCursor DateNames (Option (Tuple long StringCursor))))
(defun (read-am-pm s c names)
  (match (Tuple (skip-string s c (record-ref names am)) (skip-string s c (record-ref names pm)))
    ((Tuple (Some a) (Some p)) (if (string-cursor>? p a) (Some (Tuple 1L p)) (Some (Tuple 0L a))))
    ((Tuple (Some a) None) (Some (Tuple 0L a)))
    ((Tuple None (Some p)) (Some (Tuple 1L p)))
    (_ None)))

;; A Unix time, an optional minus and up to 12 digits of seconds or 15 of
;; milliseconds, as UTC ticks: None outside the years 1 to 9999, as
;; unix-seconds->instant answers.
(: (-> string StringCursor bool (Option (Tuple long StringCursor))))
(defun (read-unix s c ms)
  (def minus (starts-with-char? s c #\-))
  (def (Some (Tuple n c1))
    (read-digits s (if minus (string-cursor-next s c) c) 1 (if ms 15 12)) :propagate)
  (def signed (if minus (- n) n))
  (def (Some i)
    (if ms (unix-milliseconds->instant signed) (unix-seconds->instant signed)) :propagate)
  (Some (Tuple (instant-ticks i) c1)))

;; The end of the longest run, at least one long, of the characters an IANA
;; id is made of.
(: (-> string StringCursor (Option StringCursor)))
(defun (skip-zone s c)
  (let loop ((e c))
    (cond ((and (not (string-cursor-end? s e)) (zone-char? (string-cursor-ref s e)))
           (loop (string-cursor-next s e)))
          ((string-cursor=? e c) None)
          (else (Some e)))))

;; Whether s has the same text from a to a-end as from b to b-end.
(: (-> string StringCursor StringCursor StringCursor StringCursor bool))
(defun (same-span? s a a-end b b-end)
  (cond ((string-cursor=? a a-end) (string-cursor=? b b-end))
        ((string-cursor=? b b-end) #f)
        ((char=? (string-cursor-ref s a) (string-cursor-ref s b))
         (same-span? s (string-cursor-next s a) a-end (string-cursor-next s b) b-end))
        (else #f)))

(: (-> (Option StringCursor) (Option (Tuple long StringCursor))))
(defun (zero-after r)
  (option-map (fun (c) (Tuple 0L c)) r))

;; What one part reads at c, as a number, and the cursor after it. Text and
;; characters read as 0, and a zone is read by skip-zone.
(: (-> string StringCursor DateNames Field (Option (Tuple long StringCursor))))
(defun (read-value s c names f)
  (def k (struct-ref f kind))
  (def w (struct-ref f width))
  (def most (struct-ref f most))
  (cond ((> most 0) (read-digits s c (max w 1) (max w most)))
        ((name-kind? k) (read-name s c (name-table names k)))
        (else
         (match k
           (FText (zero-after (skip-string s c (struct-ref f text))))
           (FChar (zero-after (skip-char s c (struct-ref f ch))))
           (FFraction (if (< w 0) (read-optional-fraction s c) (read-fraction-digits s c w)))
           (FAmPm (read-am-pm s c names))
           (FOffset (read-offset s c))
           (FUnix (read-unix s c #f))
           (FUnixMs (read-unix s c #t))
           (FZone None)
           (_ (read-roman s c))))))

;; Whether a field read as old, or not read when -1, agrees with v.
(: (-> int int bool))
(defun (agrees? old v) (or (< old 0) (= old v)))

(: (-> long long bool))
(defun (ticks-agree? old t) (or (= old no-ticks) (= old t)))

;; The ticks in the last of 7 - digits places, which a fraction read to
;; digits digits does not say.
(: (-> int long))
(defun (fraction-unit digits) (if (>= digits 7) 1L (* 10L (fraction-unit (+ digits 1)))))

;; A fraction read twice has to agree to the fewer digits of the two, and
;; the one with more is kept: .5 and .5000 agree, and .5 and .4 do not.
(: (-> Seen long int (Option Seen)))
(defun (put-fraction seen f digits)
  (def old (struct-ref seen fraction))
  (def old-digits (struct-ref seen fraction-digits))
  (def unit (fraction-unit (min digits old-digits)))
  (cond ((or (< old 0L) (> digits old-digits))
         (if (or (< old 0L) (= (/ old unit) (/ f unit)))
             (Some (record-set seen (fraction f) (fraction-digits digits)))
             None))
        ((= (/ old unit) (/ f unit)) (Some seen))
        (else None)))

;; seen with the field f read as v, or None when it was read before as
;; something else. A value too large for any field stays too large. The
;; check comes first, so a Seen, which is large, is copied only to change it.
(: (-> Seen Field long (Option Seen)))
(defun (put-value seen f v)
  (def n (if (or (> v 1000000000L) (< v 0L)) 1000000000 (int-of v)))
  (def w (struct-ref f width))
  (match (struct-ref f kind)
    (FYear (if (agrees? (struct-ref seen year) n) (Some (record-set seen (year n))) None))
    (FMonth (if (agrees? (struct-ref seen month) n) (Some (record-set seen (month n))) None))
    (FDay (if (agrees? (struct-ref seen day) n) (Some (record-set seen (day n))) None))
    (FHours (if (agrees? (struct-ref seen hours) n) (Some (record-set seen (hours n))) None))
    (FMinutes (if (agrees? (struct-ref seen minutes) n) (Some (record-set seen (minutes n))) None))
    (FSeconds (if (agrees? (struct-ref seen seconds) n) (Some (record-set seen (seconds n))) None))
    ;; A bare fraction says every digit there is: .5 is .5000000.
    (FFraction (put-fraction seen v (if (< w 0) 7 (min w 7))))
    (FOffset (if (ticks-agree? (struct-ref seen offset) v) (Some (record-set seen (offset v))) None))
    (FUnix (if (ticks-agree? (struct-ref seen unix) v) (Some (record-set seen (unix v))) None))
    (FUnixMs (if (ticks-agree? (struct-ref seen unix) v) (Some (record-set seen (unix v))) None))
    (FRomanYear (if (agrees? (struct-ref seen year) n) (Some (record-set seen (year n))) None))
    (FRomanMonth (if (agrees? (struct-ref seen month) n) (Some (record-set seen (month n))) None))
    (FRomanDay (if (agrees? (struct-ref seen day) n) (Some (record-set seen (day n))) None))
    (FMonthName (if (agrees? (struct-ref seen month) n) (Some (record-set seen (month n))) None))
    (FMonthAbbr (if (agrees? (struct-ref seen month) n) (Some (record-set seen (month n))) None))
    (FWeekday (if (agrees? (struct-ref seen weekday) n) (Some (record-set seen (weekday n))) None))
    (FWeekdayName
     (if (agrees? (struct-ref seen weekday) n) (Some (record-set seen (weekday n))) None))
    (FWeekdayAbbr
     (if (agrees? (struct-ref seen weekday) n) (Some (record-set seen (weekday n))) None))
    (FYday (if (agrees? (struct-ref seen yday) n) (Some (record-set seen (yday n))) None))
    (FWeek (if (agrees? (struct-ref seen week) n) (Some (record-set seen (week n))) None))
    (FWeekYear
     (if (agrees? (struct-ref seen week-year) n) (Some (record-set seen (week-year n))) None))
    (FHours12 (if (agrees? (struct-ref seen hours12) n) (Some (record-set seen (hours12 n))) None))
    (FAmPm (if (agrees? (struct-ref seen pm) n) (Some (record-set seen (pm n))) None))
    ;; Text is read by read-format, and a zone by put-zone.
    (_ (Some seen))))

;; seen with the zone read from c to e, when no other was read before.
(: (-> string Seen StringCursor StringCursor (Option Seen)))
(defun (put-zone s seen c e)
  (def start (struct-ref seen zone-start))
  (def end (struct-ref seen zone-end))
  (cond ((string-cursor=? start end) (Some (record-set seen (zone-start c) (zone-end e))))
        ((same-span? s start end c e) (Some seen))
        (else None)))

;; The fields of a text in a format's parts, when the text has every part
;; and nothing after. Text is only skipped, and a field's value is read
;; before it is put, so a Seen is copied no more than it has to be.
(: (-> string (List %p) DateNames (Option Seen)) (where (FormatPart %p)))
(defun (read-format s parts names)
  (let loop ((ps parts) (c (string-cursor-start s)) (seen unseen))
    (if (list-empty? ps)
        (if (string-cursor-end? s c) (Some seen) None)
        (let ((f (part-field (list-head ps))))
          (match (struct-ref f kind)
            (FText
             (match (skip-string s c (struct-ref f text))
               ((Some c1) (loop (list-tail ps) c1 seen))
               (None None)))
            (FChar
             (match (skip-char s c (struct-ref f ch))
               ((Some c1) (loop (list-tail ps) c1 seen))
               (None None)))
            (FZone
             (def* ((Some c1) (skip-zone s c) :propagate)
                   ((Some seen1) (put-zone s seen c c1) :propagate))
             (loop (list-tail ps) c1 seen1))
            (_
             (def* ((Some (Tuple v c1)) (read-value s c names f) :propagate)
                   ((Some seen1) (put-value seen f v) :propagate))
             (loop (list-tail ps) c1 seen1)))))))

;; A date needs its largest unit. Month, day and yday count within year,
;; and week within week-year; either may be read beside a date the other
;; kind of year makes, and is checked against it.
(: (-> int void))
(defun (check-date-bits bits)
  (def year? (has? bits b-year))
  (def week-year? (has? bits b-week-year))
  (def in-year? (has? bits b-in-year))
  (def week? (has? bits b-week))
  (unless (or year? week-year?)
    (argument-error "a format read as a date needs year, (roman year) or week-year."))
  (when (and in-year? (not year?) (not week?))
    (argument-error
      "month, day and yday count within year, and this format has week-year instead: write year, or read a week date with week-year and week."))
  (when (and week? (not week-year?) (not in-year?))
    (argument-error
      "week counts within week-year, and this format has year instead: write week-year, or read a calendar date with year, month and day.")))

;; A time needs its hours: hours, or hours12 with am-pm, since 1:05 on a
;; twelve-hour clock could be either. A datetime's format may leave the time
;; out, which is midnight, and then optional is true.
(: (-> int bool void))
(defun (check-time-bits bits optional)
  (def hours? (or (has? bits b-hours) (has? bits b-hours12)))
  (when (and (has? bits b-hours12) (not (has? bits b-am-pm)))
    (argument-error "hours12 needs am-pm beside it, as 1:05 could be either."))
  (unless (or hours? optional)
    (argument-error "a format read as a time needs hours, or hours12 with am-pm."))
  (when (and (not hours?) (has? bits (bitwise-ior b-in-hour b-am-pm)))
    (argument-error
      "minutes, seconds, fraction and am-pm count within hours, and this format has no hours or hours12.")))

;; An instant is its Unix time alone, or a date and a time. A Unix time
;; beside another field of the instant could say another instant, so a
;; format with one has no other field but a moment's zone.
(: (-> int void))
(defun (check-instant-bits bits)
  (def unix (bitwise-ior b-unix b-unix-ms))
  (def others (- bits (bitwise-and bits (bitwise-ior unix b-zone))))
  (when (and (has? bits b-unix) (has? bits b-unix-ms))
    (argument-error "unix and unix-ms each say the whole instant, so a format reads one of them."))
  (when (and (has? bits unix) (> others 0))
    (argument-error
      "unix and unix-ms say the whole instant, and a field beside them could say another, so a format that reads one has no other field but text and a moment's zone."))
  (unless (has? bits unix)
    (check-date-bits bits)
    (check-time-bits bits #t)))

;; A moment needs its zone, which is never guessed.
(: (-> int void))
(defun (check-moment-bits bits)
  (unless (has? bits b-zone)
    (argument-error
      "a format read as a moment needs zone. A text with an offset and no zone is read with text->instant, and put in a zone with instant->moment."))
  (check-instant-bits bits))

(: (-> int int))
(defun (first-value n) (if (< n 0) 1 n))

(: (-> int int int (Option long)))
(defun (calendar-day y m d)
  (if (valid-date? y m d) (Some (long-of (clr-date-day-number (ClrDate. y m d)))) None))

(: (-> int int (Option long)))
(defun (ordinal-day y yd)
  (if (and (>= y 1) (<= y 9999) (>= yd 1) (<= yd (clr-date-day-of-year (ClrDate. y 12 31))))
      (Some (+ (long-of (clr-date-day-number (ClrDate. y 1 1))) (long-of (- yd 1))))
      None))

;; The day the date fields make: year, month and day; year and yday; or
;; week-year, week and weekday. A field left out is its first value.
(: (-> Seen (Option long)))
(defun (seen-day-number seen)
  (def y (struct-ref seen year))
  (def m (struct-ref seen month))
  (def d (struct-ref seen day))
  (def yd (struct-ref seen yday))
  (def wy (struct-ref seen week-year))
  (cond ((and (>= y 0) (or (>= m 0) (>= d 0))) (calendar-day y (first-value m) (first-value d)))
        ((and (>= y 0) (>= yd 0)) (ordinal-day y yd))
        ((>= wy 0)
         (option-map day-number (week-date wy (first-value (struct-ref seen week))
                                           (first-value (struct-ref seen weekday)))))
        ((>= y 0) (calendar-day y 1 1))
        (else None)))

;; Whether every date field read is the date's own. A field not read is -1,
;; and its accessor is not called.
(: (-> Seen Date bool))
(defun (date-agrees? seen d)
  (and (or (< (struct-ref seen year) 0) (= (struct-ref seen year) (->year d)))
       (or (< (struct-ref seen month) 0) (= (struct-ref seen month) (->month d)))
       (or (< (struct-ref seen day) 0) (= (struct-ref seen day) (->day d)))
       (or (< (struct-ref seen weekday) 0) (= (struct-ref seen weekday) (->weekday d)))
       (or (< (struct-ref seen yday) 0) (= (struct-ref seen yday) (->yday d)))
       (or (< (struct-ref seen week) 0) (= (struct-ref seen week) (->week d)))
       (or (< (struct-ref seen week-year) 0) (= (struct-ref seen week-year) (->week-year d)))))

(: (-> Seen (Option Date)))
(defun (seen->date seen)
  (def (Some n) (seen-day-number seen) :propagate)
  (def d (day-number->date n))
  (if (date-agrees? seen d) (Some d) None))

;; The time of day the fields make, in ticks since midnight: hours, or
;; hours12 and am-pm, where 12 AM is 00 and 12 PM is 12; minutes, seconds
;; and the fraction left out are 0. am-pm beside hours has to agree with it.
(: (-> Seen (Option long)))
(defun (seen-time-ticks seen)
  (def h24 (struct-ref seen hours))
  (def h12 (struct-ref seen hours12))
  (def pm (struct-ref seen pm))
  (def h (cond ((< h12 0) (max h24 0))
               ((or (< h12 1) (> h12 12)) -1)
               (else (+ (% h12 12) (* (max pm 0) 12)))))
  (def m (max (struct-ref seen minutes) 0))
  (def sec (max (struct-ref seen seconds) 0))
  (cond ((not (and (agrees? h24 h) (valid-time? h m sec 0))) None)
        ((and (>= pm 0) (not (= pm (/ h 12)))) None)
        (else (Some (+ (time-of-day-ticks h m sec 0) (max (struct-ref seen fraction) 0L))))))

;; Local ticks: a date and a time, midnight when the format has none.
(: (-> Seen (Option long)))
(defun (seen-local-ticks seen)
  (def* ((Some d) (seen->date seen) :propagate)
        ((Some t) (seen-time-ticks seen) :propagate))
  (Some (+ (* (day-number d) ticks-per-day) t)))

;; The instant's UTC ticks: its Unix time, or the local time at the offset
;; read, or in UTC when the format has none, as instant->text writes it.
(: (-> Seen (Option long)))
(defun (seen-instant-ticks seen)
  (def u (struct-ref seen unix))
  (def o (struct-ref seen offset))
  (cond ((not (= u no-ticks)) (Some u))
        (:def (Some l) (seen-local-ticks seen) :propagate)
        ((= o no-ticks) (Some l))
        ((in-calendar? (- l o)) (Some (- l o)))
        (else None)))

;; resolve-local, None rather than raising where a local time and its
;; instant cannot both be in the calendar.
(: (-> long TimeZone Resolver (Option Moment)))
(defun (resolve-local/opt l z r)
  (result->option (try (resolve-local l z r) #:catch (System.OverflowException))))

;; A moment in the zone read from s: the Unix time's instant there; or the
;; local time at the offset read, when the zone has that offset then; or the
;; local time alone, which in a gap or an overlap is None unless a resolver
;; says. An id the machine does not know is None.
(: (-> string Seen (Option Resolver) (Option Moment)))
(defun (seen->moment s seen r)
  (def (Some z)
    (timezone (substring s (struct-ref seen zone-start) (struct-ref seen zone-end)))
    :propagate)
  (def zi (struct-ref z tz))
  (def u (struct-ref seen unix))
  (def o (struct-ref seen offset))
  (cond ((not (= u no-ticks))
         (let ((uo (offset-at zi u)))
           (if (in-calendar? (+ u uo))
               (Some (Moment (local (ticks->datetime (+ u uo))) (offset (ticks->duration uo))
                             (zone z)))
               None)))
        (:def (Some l) (seen-local-ticks seen) :propagate)
        ((not (= o no-ticks))
         (if (and (in-calendar? (- l o)) (= (offset-at zi (- l o)) o))
             (Some (Moment (local (ticks->datetime l)) (offset (ticks->duration o)) (zone z)))
             None))
        (else
         (match r
           ((Some resolve) (resolve-local/opt l z resolve))
           (None (datetime->moment/opt (ticks->datetime l) z))))))

;; The reader of date->text's formats.
(: (-> string (List DatePart) (#:names DateNames) (Option Date)))
(defun (text->date s parts #:names (parameter-ref current-date-names))
  (check-date-bits (format-bits parts names))
  (def (Some seen) (read-format s parts names) :propagate)
  (seen->date seen))

(: (-> string (List TimePart) (#:names DateNames) (Option Time)))
(defun (text->time s parts #:names (parameter-ref current-date-names))
  (check-time-bits (format-bits parts names) #f)
  (def* ((Some seen) (read-format s parts names) :propagate)
        ((Some t) (seen-time-ticks seen) :propagate))
  (Some (Time (v (ClrTime. t)))))

(: (-> string (List DateTimePart) (#:names DateNames) (Option DateTime)))
(defun (text->datetime s parts #:names (parameter-ref current-date-names))
  (def bits (format-bits parts names))
  (check-date-bits bits)
  (check-time-bits bits #t)
  (def* ((Some seen) (read-format s parts names) :propagate)
        ((Some l) (seen-local-ticks seen) :propagate))
  (Some (ticks->datetime l)))

;; The fields are UTC's unless the format has an offset, as instant->text
;; writes them.
(: (-> string (List InstantPart) (#:names DateNames) (Option Instant)))
(defun (text->instant s parts #:names (parameter-ref current-date-names))
  (check-instant-bits (format-bits parts names))
  (def* ((Some seen) (read-format s parts names) :propagate)
        ((Some u) (seen-instant-ticks seen) :propagate))
  (Some (Instant (ticks u))))

;; With an offset, the offset has to be one the zone has at that local time,
;; as iso8601->moment demands. Without one, a local time in a gap or an
;; overlap is None.
(: (-> string (List MomentPart) (#:names DateNames) (Option Moment)))
(defun (text->moment s parts #:names (parameter-ref current-date-names))
  (check-moment-bits (format-bits parts names))
  (def (Some seen) (read-format s parts names) :propagate)
  (seen->moment s seen None))

;; A local time without an offset found as the resolver says, as
;; datetime->moment finds it.
(: (-> string (List MomentPart) Resolver (#:names DateNames) (Option Moment)))
(defun (text->moment/resolver s parts r #:names (parameter-ref current-date-names))
  (check-moment-bits (format-bits parts names))
  (def (Some seen) (read-format s parts names) :propagate)
  (seen->moment s seen (Some r)))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/effect.bjo — tail-resumptive effect handlers.
;;
;; An effect operation is a parameter holding a function. `with-handler` is
;; `parameterize`, and performing an operation reads the parameter and calls the
;; function on the performer's own stack. No continuation is captured, so a
;; handler can return once (resume) or throw (abort) and nothing else.
;;
;; What that rules out, and rules out for good on this backend: generators,
;; backtracking, and resuming an operation twice. Nothing here can be extended
;; to give them — there is no continuation to hold on to.
;;
;; This module imports neither `prelude` nor anything that imports it, and that
;; is what lets `prelude` declare effects of its own: a macro cannot be used in
;; the module that defines it, so `defeffect` has to live below the module with
;; the most to declare. The parameter machinery a template reaches for —
;; `make-parameter`, `make-cold-parameter`, `parameter-ref`, `Param` and
;; `parameterize` — is built in rather than imported, so an expansion resolves
;; wherever it lands.

(import (std eq))
(import (std syntax-match))

(import/extern
  (str-concat (: System.String.Concat)))

(import/class
  (Exception (: System.Exception))
  (Unhandled (: Bjolang.Runtime.Unhandled (-> string Unhandled))))

(:doc #:file "effect.bjodoc")

(export unhandled-effect)

(: (-> string string string))
(defun (cat a b) (str-concat a b))

;; What an operation declared with no `#:default` does when it is performed.
;; Written here rather than in the template so that the message is in one place.
(: (-> string %a))
(defun (unhandled-effect operation)
  (raise (cast Exception (Unhandled. operation))))

;; ---------------------------------------------------------------------------
;; Reading a declaration
;; ---------------------------------------------------------------------------

(: (-> Syntax string))
(defun (ident-name s)
  (match s
    ((SSym name) (symbol->string name))
    (_ "")))

;; The head of an arrow: "->" or "-bjo->".
(: (-> Syntax string))
(defun (arrow-head arrow)
  (match arrow
    ((SList (Cons head _)) (ident-name head))
    (_ "")))

(: (-> Syntax bool))
(defun (suspending-arrow? arrow)
  (= (arrow-head arrow) "-bjo->"))

(: (-> Syntax bool))
(defun (arrow? arrow)
  (if (= (arrow-head arrow) "->") #t (suspending-arrow? arrow)))

;; Everything but the head and the return type.
(: (-> Syntax int))
(defun (arrow-arity arrow)
  (match arrow
    ((SList items) (- (list-length items) 2))
    (_ 0)))

;; The parameter that holds an operation's handler.
;;
;; `op/log` is the cell and `log` is the wrapper that reads it. Both are
;; ordinary module-level names: a module that wants another module to be able to
;; handle its effect exports both.
;;
;; Built through `inject`, so the name is *not* renamed. `defeffect` and
;; `with-handler` are separate invocations with separate rename tables, and a
;; renamed cell would give each of them a spelling the other could not write.
(: (-> (-> Symbol Syntax) Syntax Syntax))
(defun (op-cell inject name)
  (inject (string->symbol (cat "op/" (ident-name name)))))

;; The wrapper's parameters. Like every binder a macro introduces they are
;; renamed, to `effect-arg0__164` and the like, which is harmless here: the only
;; thing in scope inside the wrapper is the wrapper. A doc of the operation names
;; them itself, as it would a builtin's.
(: (-> int int (List Syntax)))
(defun (wrapper-args i n)
  (if (>= i n)
      Nil
      (Cons (SSym (string->symbol (cat "effect-arg" (.ToString i))))
            (wrapper-args (+ i 1) n))))

;; ---------------------------------------------------------------------------
;; The clauses of a `defeffect`
;; ---------------------------------------------------------------------------

(: (-> (List Syntax) (List Syntax)))
(defun (rest-after clauses)
  (match clauses
    (Nil Nil)
    ((Cons _ rest) rest)))

;; The items of the form following a keyword clause.
(: (-> (List Syntax) (List Syntax)))
(defun (clause-items clauses)
  (match clauses
    (Nil Nil)
    ((Cons c _)
     (match c
       ((SList items) items)
       (_ (list (syntax-error c "a #:default or #:cold clause takes a parenthesized list.")))))))

;; The operation declarations, which is everything that is not a keyword clause
;; or the form one introduces.
(: (-> (List Syntax) (List Syntax)))
(defun (operation-clauses clauses)
  (match clauses
    (Nil Nil)
    ((Cons c rest)
     (syntax-match c
       (#:default (operation-clauses (rest-after rest)))
       (#:cold (operation-clauses (rest-after rest)))
       (#:adopt (operation-clauses (rest-after rest)))
       (_ (Cons c (operation-clauses rest)))))))

(: (-> (List Syntax) (List Syntax)))
(defun (default-clauses clauses)
  (match clauses
    (Nil Nil)
    ((Cons c rest)
     (syntax-match c
       (#:default (clause-items rest))
       (_ (default-clauses rest))))))

(: (-> (List Syntax) (List Syntax)))
(defun (cold-clauses clauses)
  (match clauses
    (Nil Nil)
    ((Cons c rest)
     (syntax-match c
       (#:cold (clause-items rest))
       (_ (cold-clauses rest))))))

(: (-> (List Syntax) (List Syntax)))
(defun (adopt-clauses clauses)
  (match clauses
    (Nil Nil)
    ((Cons c rest)
     (syntax-match c
       (#:adopt (clause-items rest))
       (_ (adopt-clauses rest))))))

(: (-> Syntax (List Syntax) bool))
(defun (named-in? name names)
  (match names
    (Nil #f)
    ((Cons n rest) (if (syntax-ident=? n name) #t (named-in? name rest)))))

(: (-> Syntax (List Syntax) (Option Syntax)))
(defun (default-for name pairs)
  (match pairs
    (Nil None)
    ((Cons pair rest)
     (syntax-match pair
       ((op handler) (if (syntax-ident=? op name) (Some handler) (default-for name rest)))
       (bad (Some (syntax-error bad "a #:default clause pairs an operation with a handler: (log write-it)")))))))

;; ---------------------------------------------------------------------------
;; What one operation expands to
;; ---------------------------------------------------------------------------

;; The default is a lambda of the operation's own colour: colours are not
;; subtypes here, so an ordinary `fun` does not stand in for a `(-bjo-> ...)`.
(: (-> Syntax Syntax (List Syntax) (List Syntax) Syntax))
(defun (handler-default name arrow args defaults)
  (match (default-for name defaults)
    ((Some handler) handler)
    (None
     (if (suspending-arrow? arrow)
         #'(bjoroutine (,@args) (unhandled-effect ,(SStr (ident-name name))))
         #'(fun (,@args) (unhandled-effect ,(SStr (ident-name name))))))))

;; Three definitions per operation, and nothing per effect: the cell, the
;; wrapper's signature, and the wrapper.
;;
;; The wrapper is a named function rather than a bare parameter read so that the
;; declared arrow is checked at every call, and so that the colour machinery
;; sees a function: a `(-bjo-> ...)` operation performed inside a `defun` is then
;; the ordinary "cannot suspend here" error.
;;
;; The handler goes into a local before it is applied. A suspending call is only
;; awaited when its callee is a name, so `((parameter-ref cell) x)` would emit a
;; `Fiber<T>` where the value was wanted.
;;
;; `#:adopt` intercepts the value returned by a handler and passes it through a
;; named function before returning it to the caller. It is for operations that
;; return *resources*: the resource can be registered on the current scope, so
;; that one a handler made is cleaned up as a real one would be.
(: (-> Syntax Syntax (List Syntax) (List Syntax) Syntax))
(defun (wrapper-body name cell args adopts)
  (match (default-for name adopts)
    (None #'(let ((handler (parameter-ref ,cell))) (handler ,@args)))
    ((Some adopt) #'(let ((handler (parameter-ref ,cell))) (,adopt (handler ,@args))))))

(: (-> (-> Symbol Syntax) Syntax (List Syntax) (List Syntax) (List Syntax) (List Syntax)))
(defun (operation-decls inject clause defaults colds adopts)
  (syntax-match clause
    ((name arrow)
     (if (not (arrow? arrow))
         (list (syntax-error arrow "an operation's type is an arrow: (-> ...) for one performable anywhere, (-bjo-> ...) for one only a bjoroutine may perform."))
         (let ((cell (op-cell inject name))
               (args (wrapper-args 0 (arrow-arity arrow))))
           (list #'(: ,cell (Param ,arrow))
                 (if (named-in? name colds)
                     #'(def ,cell (make-cold-parameter ,(handler-default name arrow args defaults)))
                     #'(def ,cell (make-parameter ,(handler-default name arrow args defaults))))
                 #'(: ,name ,arrow)
                 (if (suspending-arrow? arrow)
                     #'(defbjo (,name ,@args) ,(wrapper-body name cell args adopts))
                     #'(defun (,name ,@args) ,(wrapper-body name cell args adopts)))))))
    (bad (list (syntax-error bad "an operation is written (name arrow), as in (log (-> string void)).")))))

(: (-> (-> Symbol Syntax) (List Syntax) (List Syntax) (List Syntax) (List Syntax) (List Syntax)))
(defun (effect-decls inject ops defaults colds adopts)
  (match ops
    (Nil Nil)
    ((Cons op rest)
     (syntax-splice (operation-decls inject op defaults colds adopts)
                    (effect-decls inject rest defaults colds adopts)))))

;; ---------------------------------------------------------------------------
;; The macros
;; ---------------------------------------------------------------------------

;; `(defeffect Name (op arrow) ... #:default ((op handler) ...) #:cold (op ...))`
;;
;;     (defeffect Log
;;       (log   (-> string void))
;;       (audit (-bjo-> string void))
;;       #:default ((log (fun (s) (display s))))
;;       #:cold (audit))
;;
;; The effect name is documentation: nothing is emitted for it, and there is no
;; form that installs a whole effect at once. Handlers are installed one
;; operation at a time with `with-handler`.
;;
;; An operation with no `#:default` raises `Bjolang.Runtime.Unhandled` when it is
;; performed with nothing installed. Give a default when there is one sensible
;; meaning; leave it out when running with no handler should fail loudly.
;;
;; An operation's colour is its declared arrow, and a handler has to match it.
;; A `(-> ...)` operation takes `fun` handlers and can be performed anywhere,
;; including inside a `defun`. A `(-bjo-> ...)` operation takes `bjoroutine`
;; handlers and can only be performed from a bjoroutine; an ordinary `fun` is
;; *not* lifted into one, because the two arrows are different calling
;; conventions rather than a subtype pair.
;;
;; Prefer a sync operation. Anything a `defun` has to be able to perform must be
;; one, and an operation that has to return something suspendable can return a
;; value whose own methods are colour-aware — a port, say — rather than
;; suspending itself.
;;
;; Every operation spends one of the dynamic environment's 31 cheap parameter
;; ids. `#:cold` gives that claim up, for an operation its author knows is not
;; performed in a loop: the ids are a budget, and a program with many rarely
;; performed effects should not push its ports out of the cheap slots.
;;
;; `#:adopt ((op fn))` intercepts the value returned by a handler and passes it
;; through `fn` before returning it to the caller. An operation that returns a
;; *resource* needs it when a handler may build one that nothing owns - a
;; string port, say - since otherwise that resource is never released and a
;; leak test over the handler means nothing. An operation that returns the raw
;; material instead, and leaves building and registering the resource to the
;; caller, does not: the prelude's `FS` openers answer a `ClrStream` for that
;; reason.
(def/macro (defeffect form inject compare)
  (syntax-match form
    ((_ name clause ...)
     (if (list-empty? clause)
         (syntax-error form "defeffect needs at least one operation: (defeffect Log (log (-> string void)))")
         #'(begin
             ,@(effect-decls inject
                             (operation-clauses clause)
                             (default-clauses clause)
                             (cold-clauses clause)
                             (adopt-clauses clause)))))
    (bad (syntax-error bad "defeffect takes an effect name and one or more operations: (defeffect Log (log (-> string void)) #:default ((log write-it)))"))))

;; `(with-handler ((op handler) ...) body ...)` — install handlers for the
;; extent of the body.
;;
;; This is `parameterize` on the operations' cells: the bindings are
;; simultaneous, and the environment is restored however the body ends.
;;
;; A handler runs on the *performer's* stack. So a handler that throws unwinds
;; the performer rather than the installer, and inside a spawned fiber that is a
;; child failure like any other.
;;
;; # A handler frame is not a scope
;;
;; A scope waits for its children; this does not. So
;;
;;     (with-cancel (cancel)
;;       (with-handler ((log (collect-into buf)))
;;         (spawn (work)))
;;       (report buf))
;;
;; returns from `with-handler` as soon as the spawn is queued, and `report`
;; reads a buffer the child is still filling. The child does not lose the
;; handler — a spawn takes the environment as it stood at the spawn, so the
;; handler travels with it and outlives the form that installed it — which is
;; why the read is early rather than empty.
;;
;; **A handler that collects must enclose the scope that owns the performers.**
;; Put the `with-handler` outside the `with-cancel`, and read after both have
;; returned. `parameterize` behaves the same way; a collecting handler is only
;; where it shows.
;;
;; # Forwarding
;;
;; A handler runs with itself installed, so one that performs its own operation
;; calls itself:
;;
;;     (with-handler ((log (fun (s) (log (str "[app] " s))))) ...)   ;; loops
;;
;; The handler installed before it is `(handler-of log)`. The bindings of a
;; `with-handler` are evaluated before any of them is installed, so it can be
;; read in the binding itself:
;;
;;     (with-handler ((log (let ((outer (handler-of log)))
;;                           (fun (s) (outer (str "[app] " s))))))
;;       ...)
;;
;; # The cell is found by spelling
;;
;; `with-handler` derives the cell from the operation's written name: `log`
;; binds `op/log`. That is a convention, not a binding, so an operation imported
;; under a prefix or a rename has no cell of that spelling and `with-handler`
;; cannot find it. The fallback is the expansion itself:
;;
;;     (parameterize ((p:op/log collect)) ...)
;;
;; `op/log` is an ordinary exported binding and follows every rename.
;;
;; The same convention is why a macro that writes a `with-handler` has to
;; `inject` the operation name: a template renames `log`, and a renamed `log`
;; derives a cell nothing declares.
;;
;;     (with-handler ((,(inject (string->symbol "log")) collect)) ...)
;;
;; A name taken from the macro's own input is already unrenamed.
;; `std/simpletest`'s `with-fake-fs` is the worked example.
(def/macro (with-handler form inject compare)
  (syntax-match form
    ((_ ((op handler) ...) first body ...)
     (if (list-empty? op)
         (syntax-error form "with-handler installs at least one handler: (with-handler ((log collect)) body ...)")
         #'(parameterize ,(SList (handler-bindings inject op handler)) ,first ,@body)))
    (bad (syntax-error bad "with-handler takes a list of (operation handler) pairs and a body: (with-handler ((log collect)) body ...)"))))

(: (-> (-> Symbol Syntax) (List Syntax) (List Syntax) (List Syntax)))
(defun (handler-bindings inject ops handlers)
  (match ops
    (Nil Nil)
    ((Cons op orest)
     (match handlers
       (Nil Nil)
       ((Cons h hrest) (Cons #'(,(op-cell inject op) ,h) (handler-bindings inject orest hrest)))))))

;; `(handler-of op)` - the handler installed for `op` here.
;;
;; `(parameter-ref op/x)`, with the cell derived the way `with-handler` derives
;; it. Read it where a `with-handler` is written to forward to the handler that
;; one replaces.
(def/macro (handler-of form inject compare)
  (syntax-match form
    ((_ op) #'(parameter-ref ,(op-cell inject op)))
    (bad (syntax-error bad "handler-of takes an operation name: (handler-of log)"))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/eq.bjo — equality and hashing.
(export = eq-hash
        Ord compare less? greater? at-most? at-least?
        least greatest list-sort vec-sort vec-sort-by array-sort! array-sort-by!
        list-sort-unstable vec-sort-unstable array-sort-unstable!
        list-min list-max
        not)

(:doc #:file "eq.bjodoc")

;; Here, and not in the prelude, so that the modules the prelude imports
;; have it too. The prelude re-exports it.
(: (-> bool bool))
(defun (not b) #:inline (if b #f #t))

;; ---------------------------------------------------------------------------
;; Eq — the one equality
;; ---------------------------------------------------------------------------
;; Trait for equality and hashing.
(def/trait (Eq %a)
  (: = (-> %a %a bool))
  (: eq-hash (-> %a int)))


;; ---------------------------------------------------------------------------
;; Ord — the one ordering
;; ---------------------------------------------------------------------------
;; Trait for ordering.
(def/trait (Ord %a)
  (where (Eq %a))
  (: compare (-> %a %a int)))

;; Ord predicates.
(: (-> %a %a bool) (where (Ord %a)))
(defun (less? a b) (< (compare a b) 0))

(: (-> %a %a bool) (where (Ord %a)))
(defun (greater? a b) (> (compare a b) 0))

(: (-> %a %a bool) (where (Ord %a)))
(defun (at-most? a b) (<= (compare a b) 0))

(: (-> %a %a bool) (where (Ord %a)))
(defun (at-least? a b) (>= (compare a b) 0))

;; least and greatest (ties to left).
(: (-> %a %a %a) (where (Ord %a)))
(defun (least a b) (if (at-most? a b) a b))

(: (-> %a %a %a) (where (Ord %a)))
(defun (greatest a b) (if (at-least? a b) a b))

;; --- Sorting ---------------------------------------------------------------
;; Two kinds, in the order of the elements' Ord:
;;
;; - `array-sort!`, `list-sort`, `vec-sort` (and `mutablevec-sort!`): stable,
;;   equal elements keep their order. The merge sort in `array-sort!`. It is
;;   exported, so the compiler copies it into each caller at the element type,
;;   and `compare` is a direct call.
;; - The `-unstable` ones: .NET's introsort (`Bjolang.Runtime.SortModule`),
;;   with `(ord-comparer)`. Equal elements can change places. Faster, and for
;;   a type whose equal values are the same value, such as an int, the result
;;   is the same.
;;
;; The list and vec sorts sort a copy, so when `compare` raises their input is
;; as it was.

(import/class
  ((Comparer %a) (: System.Collections.Generic.IComparer)))

(import/extern
  (clr-sort-unstable! (: Bjolang.Runtime.SortModule.SortUnstable (-> (Array %a) (Comparer %a) void)))
  (clr-vec->array     (: BjolangRuntime.vecsubgtarray (-> (Vec %a) (Array %a))))
  (clr-list->array    (: BjolangRuntime.listsubgtarray (-> (List %a) (Array %a)))))

;; Sorts `arr` in place, in the order of the elements' Ord. Stable.
;;
;; - Insertion sort over runs of 16, then bottom-up merges between `arr` and
;;   one buffer. A merge takes from the left run on a tie, which keeps the sort
;;   stable.
;; - When `compare` raises, the contents of `arr` are unspecified: an element
;;   can be lost or be there twice.
(: (-> (Array %a) void) (where (Ord %a)))
(defun (array-sort! arr)
  (def n (array-length arr))
  (def run 16)

  (defun (smaller (: a int) b) (if (< a b) a b))

  ;; The elements in [lo, hi), in order.
  (defun (insert-run lo hi)
    (let next ((i (+ lo 1)))
      (when (< i hi)
        (let ((x (array-ref arr i)))
          (let shift ((j (- i 1)))
            (if (if (>= j lo) (> (compare (array-ref arr j) x) 0) #f)
                (let ()
                  (array-set! arr (+ j 1) (array-ref arr j))
                  (shift (- j 1)))
                (array-set! arr (+ j 1) x))))
        (next (+ i 1)))))

  ;; src[lo, mid) and src[mid, hi) into dst[lo, hi).
  (defun (merge-runs (: src (Array %a)) dst lo mid hi)
    (let go ((l lo) (r mid) (k lo))
      (when (< k hi)
        (if (if (< l mid)
                (if (< r hi) (<= (compare (array-ref src l) (array-ref src r)) 0) #t)
                #f)
            (let ()
              (array-set! dst k (array-ref src l))
              (go (+ l 1) r (+ k 1)))
            (let ()
              (array-set! dst k (array-ref src r))
              (go l (+ r 1) (+ k 1)))))))

  (defun (merge-pass src dst width)
    (let go ((lo 0))
      (if (< lo n)
          (let ()
            (merge-runs src dst lo (smaller (+ lo width) n) (smaller (+ lo (* 2 width)) n))
            (go (+ lo (* 2 width))))
          dst)))

  (let runs ((lo 0))
    (when (< lo n)
      (insert-run lo (smaller (+ lo run) n))
      (runs (+ lo run))))
  (when (> n run)
    (def sorted
      (let go ((src arr) (dst (make-array n)) (width run))
        (if (< width n)
            (go (merge-pass src dst width) src (* width 2))
            src)))
    (when (not (eq? sorted arr))
      (let copy ((i 0))
        (when (< i n)
          (array-set! arr i (array-ref sorted i))
          (copy (+ i 1)))))))

;; Sorts `arr` in place, in the order of the elements' Ord. Not stable.
;;
;; - When `compare` raises, it raises the same exception, and the contents of
;;   `arr` are unspecified.
(: (-> (Array %a) void) (where (Ord %a)))
(defun (array-sort-unstable! arr) (clr-sort-unstable! arr (ord-comparer)))

(: (-> (List %a) (List %a)) (where (Ord %a)))
(defun (list-sort xs)
  (def n (list-length xs))

  (defun (fill arr)
    (let go ((i 0) (rest xs))
      (if (< i n)
          (let ()
            (array-set! arr i (list-head rest))
            (go (+ i 1) (list-tail rest)))
          arr)))

  (defun (sorted)
    (def (: arr (Array %a)) (fill (make-array n)))
    (array-sort! arr)
    (let go ((i (- n 1)) (acc Nil))
      (if (< i 0)
          acc
          (go (- i 1) (Cons (array-ref arr i) acc)))))

  (if (< n 2) xs (sorted)))

;; The same merge sort over an array, with the comparison passed in: negative
;; when the first comes first, zero when they tie, positive otherwise. Stable,
;; since ties take the left run. Answers the array that holds the result, which
;; is arr or one of the same length.
(: (-> (-> %a %a int) (Array %a) (Array %a)))
;;
;; The local functions take the array and the comparison as arguments, and
;; capture nothing. A captured variable is a field that .NET reads again
;; after each call of `cmp`, and checks the index against again.
(defun (sort-array-by cmp arr)
  (def n (array-length arr))
  (def run 16)

  (defun (smaller (: a int) b) (if (< a b) a b))

  ;; The elements in [lo, hi), in order, as `array-sort!` sorts its runs.
  (defun (insert-run (: c (-> %a %a int)) (: a (Array %a)) lo hi)
    (let next ((i (+ lo 1)))
      (when (< i hi)
        (let ((x (array-ref a i)))
          (let shift ((j (- i 1)))
            (if (if (>= j lo) (> (c (array-ref a j) x) 0) #f)
                (let ()
                  (array-set! a (+ j 1) (array-ref a j))
                  (shift (- j 1)))
                (array-set! a (+ j 1) x))))
        (next (+ i 1)))))

  (defun (merge-runs (: c (-> %a %a int)) (: src (Array %a)) dst lo mid hi)
    (let go ((l lo) (r mid) (k lo))
      (if (< k hi)
          (if (if (< l mid)
                  (if (< r hi) (<= (c (array-ref src l) (array-ref src r)) 0) #t)
                  #f)
              (let ()
                (array-set! dst k (array-ref src l))
                (go (+ l 1) r (+ k 1)))
              (let ()
                (array-set! dst k (array-ref src r))
                (go l (+ r 1) (+ k 1))))
          k)))

  (defun (merge-pass (: c (-> %a %a int)) src dst (: m int) width)
    (let go ((lo 0))
      (if (< lo m)
          (go (merge-runs c src dst lo (smaller (+ lo width) m) (smaller (+ lo (* 2 width)) m)))
          dst)))

  (let runs ((lo 0))
    (when (< lo n)
      (insert-run cmp arr lo (smaller (+ lo run) n))
      (runs (+ lo run))))
  (if (> n run)
      (let go ((src arr) (dst (make-array n)) (width run))
        (if (< width n)
            (go (merge-pass cmp src dst n width) src (* width 2))
            src))
      arr))

;; Sorts `arr` in place, in the order of a comparison, as `sort-array-by`
;; orders it. Stable. When `cmp` raises, the contents of `arr` are
;; unspecified.
(: (-> (-> %a %a int) (Array %a) void))
(defun (array-sort-by! cmp arr)
  (def sorted (sort-array-by cmp arr))
  (when (not (eq? sorted arr))
    (let copy ((i 0))
      (when (< i (array-length arr))
        (array-set! arr i (array-ref sorted i))
        (copy (+ i 1))))))

;; A vec in order of a comparison, as `sort-array-by` orders an array.
(: (-> (-> %a %a int) (Vec %a) (Vec %a)))
(defun (vec-sort-by cmp v)
  (def n (vec-length v))
  (if (< n 2)
      v
      (let ()
        (def arr (make-array n))
        (let fill ((i 0))
          (when (< i n)
            (array-set! arr i (vec-ref v i))
            (fill (+ i 1))))
        (array->vec (sort-array-by cmp arr)))))

;; A vec in the order of its elements' Ord. Stable.
(: (-> (Vec %a) (Vec %a)) (where (Ord %a)))
(defun (vec-sort v)
  (if (< (vec-length v) 2)
      v
      (let ((arr (clr-vec->array v)))
        (array-sort! arr)
        (array->vec arr))))

;; A vec in the order of its elements' Ord. Not stable.
(: (-> (Vec %a) (Vec %a)) (where (Ord %a)))
(defun (vec-sort-unstable v)
  (if (< (vec-length v) 2)
      v
      (let ((arr (clr-vec->array v)))
        (array-sort-unstable! arr)
        (array->vec arr))))

;; A list in the order of its elements' Ord. Not stable.
(: (-> (List %a) (List %a)) (where (Ord %a)))
(defun (list-sort-unstable xs)
  (if (if (list-empty? xs) #t (list-empty? (list-tail xs)))
      xs
      (let ((arr (clr-list->array xs)))
        (array-sort-unstable! arr)
        (let go ((i (- (array-length arr) 1)) (acc Nil))
          (if (< i 0)
              acc
              (go (- i 1) (Cons (array-ref arr i) acc)))))))

;; Private left fold for list min/max.
(: (-> (-> %a %b %b) %b (List %a) %b))
(defun (fold-left f init xs)
  (let go ((rest xs) (acc init))
    (if (list-empty? rest)
        acc
        (go (list-tail rest) (f (list-head rest) acc)))))

(: (-> (List %a) (Option %a)) (where (Ord %a)))
(defun (list-min xs)
  (if (list-empty? xs)
      None
      (Some (fold-left least (list-head xs) (list-tail xs)))))

(: (-> (List %a) (Option %a)) (where (Ord %a)))
(defun (list-max xs)
  (if (list-empty? xs)
      None
      (Some (fold-left greatest (list-head xs) (list-tail xs)))))


;; --- The blanket -----------------------------------------------------------
;; Default to .NET equality.
(impl (Eq %a)
  (defun (= a b) (clr-equals a b))
  (defun (eq-hash a) (clr-hash a)))


;; --- The primitives --------------------------------------------------------
;; Eq for primitives.

(impl (Eq int)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

(impl (Eq long)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

(impl (Eq short)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

(impl (Eq byte)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

(impl (Eq uint)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

(impl (Eq ulong)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

(impl (Eq ushort)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

;; Eq for double uses Equals (NaN == NaN).
(impl (Eq double)
  (defun (= a b) (clr-equals a b))
  (defun (eq-hash a) (clr-hash a)))

(impl (Eq bool)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

;; Eq for string.
(impl (Eq string)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

(impl (Eq char)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

;; Eq for Keyword and Symbol.
(impl (Eq Keyword)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

(impl (Eq Symbol)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

(impl (Eq StringCursor)
  (defun (= a b) (clr-eq a b))
  (defun (eq-hash a) (clr-hash a)))

;; Eq for void.
(impl (Eq void)
  (defun (= a b) #t)
  (defun (eq-hash a) 0))


;; --- Ord for the primitives ------------------------------------------------
;; Delegates to IComparable.

(impl (Ord int)    (defun (compare a b) (.CompareTo a b)))
(impl (Ord long)   (defun (compare a b) (.CompareTo a b)))
(impl (Ord short)  (defun (compare a b) (.CompareTo a b)))
(impl (Ord byte)   (defun (compare a b) (.CompareTo a b)))
(impl (Ord uint)   (defun (compare a b) (.CompareTo a b)))
(impl (Ord ulong)  (defun (compare a b) (.CompareTo a b)))
(impl (Ord ushort) (defun (compare a b) (.CompareTo a b)))

;; Ord for double uses CompareTo.
(impl (Ord double) (defun (compare a b) (.CompareTo a b)))

(impl (Ord bool)   (defun (compare a b) (.CompareTo a b)))

;; Ord for string compares code points, never the ambient culture: the same
;; answer everywhere, and the order an OrderedMap of strings keeps.
(impl (Ord string) (defun (compare a b) (.CompareTo a b)))

(impl (Ord char)   (defun (compare a b) (.CompareTo a b)))

;; Ord for the four numbers past the eight. Their Eq is the blanket's, which is
;; their own Equals; their order is their own CompareTo, as for the primitives.
(impl (Ord decimal) (defun (compare a b) (.CompareTo a b)))
(impl (Ord bigint)  (defun (compare a b) (.CompareTo a b)))
(impl (Ord rational) (defun (compare a b) (.CompareTo a b)))
(impl (Ord int128)  (defun (compare a b) (.CompareTo a b)))
(impl (Ord uint128) (defun (compare a b) (.CompareTo a b)))

;; And for sbyte, which has no name of its own: a program imports it with
;; import/class. The alias is not exported.
(import/class (ClrSByte (: System.SByte)))
(impl (Ord ClrSByte) (defun (compare a b) (.CompareTo a b)))


;; --- The containers --------------------------------------------------------
;; Container equality using element Eq. Uses builtins only.

;; A loop rather than recursion through `and`, so a long list does not
;; overflow the stack. Two lists that share a tail stop comparing there.
(impl (Eq (List %a))
  (where (Eq %a))
  (defun (= xs ys)
    (let go ((a xs) (b ys))
      (if (eq? a b)
          #t
          (if (list-empty? a)
              (list-empty? b)
              (if (list-empty? b)
                  #f
                  (if (= (list-head a) (list-head b))
                      (go (list-tail a) (list-tail b))
                      #f))))))
  (defun (eq-hash xs)
    (fold-left (fun (x h) (hash-combine h (eq-hash x))) 17 xs)))

;; Two cursors walk the leaves in step: no tree lookup per element, and no
;; recursion per element through `and`.
(impl (Eq (Vec %a))
  (where (Eq %a))
  (defun (= xs ys)
    (if (eq? xs ys)
        #t
        (if (= (vec-length xs) (vec-length ys))
            (let go ((cx (vec-cursor xs)) (cy (vec-cursor ys)))
              (if (vec-cursor-done? cx)
                  #t
                  (if (vec-cursor-done? cy)
                      #f
                      (if (= (vec-cursor-current cx) (vec-cursor-current cy))
                          (go (vec-cursor-next cx) (vec-cursor-next cy))
                          #f))))
            #f)))
  ;; Manual indexing.
  (defun (eq-hash xs)
    (defun (hash-from i h)
      (if (= i (vec-length xs))
          h
          (hash-from (+ i 1) (hash-combine h (eq-hash (vec-ref xs i))))))
    (hash-from 0 17)))

(impl (Eq (Option %a))
  (where (Eq %a))
  (defun (= a b)
    (match a
      ((Some x)
       (match b
         ((Some y) (= x y))
         (None #f)))
      (None (none? b))))
  (defun (eq-hash o)
    (match o
      ((Some x) (hash-combine 1 (eq-hash x)))
      (None 0))))

;; Eq for Tuples.
(impl (Eq (Tuple %a %b))
  (where (Eq %a) (Eq %b))
  (defun (= x y)
    (def (a1 b1) x)
    (def (a2 b2) y)
    (and (= a1 a2) (= b1 b2)))
  (defun (eq-hash x)
    (def (a b) x)
    (hash-combine (eq-hash a) (eq-hash b))))

(impl (Eq (Tuple %a %b %c))
  (where (Eq %a) (Eq %b) (Eq %c))
  (defun (= x y)
    (def (a1 b1 c1) x)
    (def (a2 b2 c2) y)
    (and (= a1 a2) (and (= b1 b2) (= c1 c2))))
  (defun (eq-hash x)
    (def (a b c) x)
    (hash-combine (hash-combine (eq-hash a) (eq-hash b)) (eq-hash c))))

(impl (Eq (Tuple %a %b %c %d))
  (where (Eq %a) (Eq %b) (Eq %c) (Eq %d))
  (defun (= x y)
    (def (a1 b1 c1 d1) x)
    (def (a2 b2 c2 d2) y)
    (and (and (= a1 a2) (= b1 b2)) (and (= c1 c2) (= d1 d2))))
  (defun (eq-hash x)
    (def (a b c d) x)
    (hash-combine (hash-combine (eq-hash a) (eq-hash b))
                  (hash-combine (eq-hash c) (eq-hash d)))))

;; Ord for Tuples: lexicographic. The first items decide, and a later item is
;; compared only when every item before it is equal.
(impl (Ord (Tuple %a %b))
  (where (Ord %a) (Ord %b))
  (defun (compare x y)
    (def (a1 b1) x)
    (def (a2 b2) y)
    (def c (compare a1 a2))
    (if (= c 0) (compare b1 b2) c)))

(impl (Ord (Tuple %a %b %c))
  (where (Ord %a) (Ord %b) (Ord %c))
  (defun (compare x y)
    (def (a1 b1 c1) x)
    (def (a2 b2 c2) y)
    (def ca (compare a1 a2))
    (if (= ca 0)
        (let ((cb (compare b1 b2)))
          (if (= cb 0) (compare c1 c2) cb))
        ca)))

(impl (Ord (Tuple %a %b %c %d))
  (where (Ord %a) (Ord %b) (Ord %c) (Ord %d))
  (defun (compare x y)
    (def (a1 b1 c1 d1) x)
    (def (a2 b2 c2 d2) y)
    (def ca (compare a1 a2))
    (if (= ca 0)
        (let ((cb (compare b1 b2)))
          (if (= cb 0)
              (let ((cc (compare c1 c2)))
                (if (= cc 0) (compare d1 d2) cc))
              cb))
        ca)))

(impl (Eq (Result %e %a))
  (where (Eq %e) (Eq %a))
  (defun (= a b)
    (match a
      ((Ok x)
       (match b
         ((Ok y) (= x y))
         ((Err _) #f)))
      ((Err x)
       (match b
         ((Ok _) #f)
         ((Err y) (= x y))))))
  (defun (eq-hash r)
    (match r
      ((Ok x) (hash-combine 1 (eq-hash x)))
      ((Err e) (hash-combine 2 (eq-hash e))))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/fmt.bjo — formatting. Layer 1: Interpolation. Layer 2: Layout.
(import (std prelude))
(import (std syntax-match))
(export
  ;; Exported types.
  Disp Sink Block Align TableLayout TableStyle TableRules BorderChars ColumnWidth Overflow

  ;; Layer 1 — interpolation
  println*
  ;; Running blocks
  render show render->string run-block run-blocks
  ;; Settings
  with-fmt with-fmt* with-pad
  current-pad current-width current-ellipsis current-string-width
  current-precision current-comma current-comma-sep current-decimal-sep
  current-signed terminal-width
  ;; The block algebra
  nothing each captured width-of
  ;; Vertical and horizontal space
  nl fl space-to tab-to
  ;; Fitting to a width
  padded/left padded/right padded/both
  trimmed/left trimmed/right trimmed/both
  fitted/left fitted/right fitted/both
  ;; Sequences
  joined joined/prefix joined/suffix joined/last
  ;; Numbers
  numeric numeric/comma bytes percent bar sparkline
  ;; Paragraphs and tables
  wrapped justified columnar tabular tabular/rows
  table-layout table-widths table-row table-top table-rule table-bottom
  ;; A table as a literal
  Table TableCell table-block show-table)

;; Every export is documented there; the module doc is too.
(:doc #:file "fmt.bjodoc")


;; ===========================================================================
;; Layer 1 — interpolation
;; ===========================================================================
;;
;; What an interpolation literal can hold: one atom of each of these types.
(def/grammar Disp
  (:or (: DStr string) (: DInt int) (: DDouble double) (: DChar char) (: DBool bool)))

(: (-> Disp string))
(defun (disp->string d)
  (match d
    ((DStr s) s)
    ((DInt n) (int->string n))
    ((DDouble x) (double->string x))
    ((DChar c) (char->string c))
    ((DBool b) (if b "#t" "#f"))))

;; Formats to string.
(: (-> (Vec Disp) string))
(defun (str* ds)
  (let ((b (stringbuilder-empty)))
    (ignore
     (fold (fun (d acc) (stringbuilder-add-string! acc (disp->string d)) acc)
           b
           ds))
    (stringbuilder->string b)))

(: (-> (Vec Disp) void))
(defun (println* ds) (displayln (str* ds)))


;; ===========================================================================
;; Layer 2 — layout
;; ===========================================================================

;; Output sink.
(type (: Sink (-> string void)))

;; Layout blocks. A vec of blocks is a block: its blocks in order. BRun runs
;; combinators.
(def/grammar Block
  (:or (: BStr string) (: BInt int) (: BChar char) (: BMany (Vec Block))
       (: BRun ,(-> Sink int int))))

;; Computes column position.
(: (-> int string int))
(defun (column-after col s)
  (string-fold (fun (ch c) (if (char=? ch #\newline) 0 (+ c 1))) col s))

(: (-> Sink int Block int))
(defun (run-block out col b)
  (match b
    ((BStr s) (let () (out s) (column-after col s)))
    ((BInt n) (run-block out col (BStr (int->string n))))
    ((BChar c) (run-block out col (BStr (char->string c))))
    ((BMany bs) (run-blocks out col bs))
    ((BRun f) (f out col))))

(: (-> Sink int (Vec Block) int))
(defun (run-blocks out col bs)
  (fold (fun (b c) (run-block out c b)) col bs))

;; `#:sync`, and it cannot be otherwise: the sink is stored in a `Block`'s
;; `BRun` case, and a stored function's colour is its type — so no `-?->` can
;; go there and no suspending copy of this could ever await. Rendering from a
;; fiber is `render->string` plus a write you make yourself.
(: render #:sync (-> OutputPort Block void))
(defun (render port b)
  (ignore (run-block #(write-string port &1) 0 b)))

(: (-> Block string))
(defun (render->string b)
  (let ((sb (stringbuilder-empty)))
    (ignore (run-block #(stringbuilder-add-string! sb &1) 0 b))
    (stringbuilder->string sb)))

;; Renders to current output port. `#:sync` for `render`'s reason, and stated
;; here too rather than left to follow from it: this is the exported one, and
;; what it promises a caller should be readable at it.
;; 
(: show #:sync (-> Block void))
(defun (show b) (render (parameter-ref current-output-port) b))


;; --- Settings --------------------------------------------------------------
(: (-> (Param %a) %a Block Block))
(defun (with-fmt p v b)
  (BRun (fun (out col)
          (parameterize ((p v))
            (run-block out col b)))))

;; What padding is made of.
(: (Param char))
(def current-pad (make-parameter #\space))

(: (-> char Block Block))
(defun (with-pad c b) (with-fmt current-pad c b))

;; `(with-fmt* ((current-width 30) (current-pad #\.)) block ...)`: with-fmt
;; for each setting, the first outermost, around the blocks in order.
(def/macro (with-fmt* form inject compare)
  (syntax-match form
    ((_ () body ...)
     #'[,@body])
    ((_ ((p v) more ...) body ...)
     #'(with-fmt ,p ,v (with-fmt* (,@more) ,@body)))
    (bad (syntax-error bad "with-fmt* takes settings and blocks: (with-fmt* ((current-width 30)) block ...)"))))

;; The right margin: what `wrapped`, `justified` and `columnar` fill up to.
(: (Param int))
(def current-width (make-parameter 78))

;; Put in place of whatever a `trimmed` cut off. Empty means cut silently.
(: (Param string))
(def current-ellipsis (make-cold-parameter ""))

;; Calculates string width in columns.
(: (Param (-> string int)))
(def current-string-width (make-cold-parameter #(string-count &1)))

;; --- Number settings -------------------------------------------------------

;; Digits after the decimal separator. Negative means as many as the number
;; needs, which is what `double->string` would have given.
(: (Param int))
(def current-precision (make-cold-parameter -1))

;; Digits per group in the integer part. Zero means no grouping.
(: (Param int))
(def current-comma (make-cold-parameter 0))

(: (Param char))
(def current-comma-sep (make-cold-parameter #\,))

(: (Param char))
(def current-decimal-sep (make-cold-parameter #\.))

;; A leading `+` on a number that has no sign of its own.
(: (Param bool))
(def current-signed (make-cold-parameter #f))


;; --- Measuring -------------------------------------------------------------

;; Returns width of string.
(: (-> string int))
(defun (width-of s)
  (let ((measure (parameter-ref current-string-width)))
    (measure s)))

;; Generates padding of specified width.
(: (-> int string))
(defun (padding w)
  (def* (c (parameter-ref current-pad))
        (cw (width-of (char->string c)))
        (b (stringbuilder-empty)))
  (when (> cw 0)
    (let loop ((room w))
      (when (>= room cw)
        (stringbuilder-add! b c)
        (loop (- room cw)))))
  (stringbuilder->string b))

;; Gets prefix/suffix fitting w columns.
(: (-> string int string))
(defun (prefix-of-width s w)
  (let loop ((c (string-cursor-start s)) (room w))
    (if (string-cursor-end? s c)
        s
        (let ((cw (width-of (char->string (string-cursor-ref s c)))))
          (if (<= cw room)
              (loop (string-cursor-next s c) (- room cw))
              (substring s (string-cursor-start s) c))))))

(: (-> string int string))
(defun (suffix-of-width s w)
  (let loop ((c (string-cursor-end s)) (room w))
    (cond
      ((string-cursor=? c (string-cursor-start s))
       s)
      (:def p (string-cursor-prev s c))
      (:def cw (width-of (char->string (string-cursor-ref s p))))
      ((<= cw room) (loop p (- room cw)))
      (else (substring s c (string-cursor-end s))))))
    ;; (if (string-cursor=? c (string-cursor-start s))
    ;;     s
    ;;     (let* ((p (string-cursor-prev s c))
    ;;            (cw (width-of (char->string (string-cursor-ref s p)))))
    ;;           (if (<= cw room)
    ;;               (loop p (- room cw))
    ;;               (substring s c (string-cursor-end s)))))))

(: (-> int string string))
(defun (pad/left w s) (string-append (padding (- w (width-of s))) s))

(: (-> int string string))
(defun (pad/right w s) (string-append s (padding (- w (width-of s)))))

;; Centred, with the odd column on the right — SRFI 166's rule.
(: (-> int string string))
(defun (pad/both w s)
  (let* ((missing (- w (width-of s)))
         (left (/ missing 2)))
    (string-append (padding left) (string-append s (padding (- missing left))))))

;; Trims string to w columns.
(: (-> int string string))
(defun (trim/left w s)
  (if (<= (width-of s) w)
      s
      (let ((e (prefix-of-width (parameter-ref current-ellipsis) w)))
        (string-append e (suffix-of-width s (- w (width-of e)))))))

(: (-> int string string))
(defun (trim/right w s)
  (if (<= (width-of s) w)
      s
      (let ((e (prefix-of-width (parameter-ref current-ellipsis) w)))
        (string-append (prefix-of-width s (- w (width-of e))) e))))

;; The middle survives, and both ends say so.
(: (-> int string string))
(defun (trim/both w s)
  (if (<= (width-of s) w)
      s
      (let* ((e (prefix-of-width (parameter-ref current-ellipsis) (/ w 2)))
             (room (- w (* 2 (width-of e))))
             (dropped (- (width-of s) (max room 0)))
             (kept (suffix-of-width s (- (width-of s) (/ dropped 2)))))
        (string-append e (string-append (prefix-of-width kept (max room 0)) e)))))


;; --- The block algebra -----------------------------------------------------

;; Nothing, written. What a conditional arm answers when it has nothing to say.
(: Block)
(def nothing (BRun (fun (out col) col)))

;; Blocks in order, as one. What a vec of blocks is anyway; the name is for
;; where a function is wanted, as in (vec-map each rows).
(: (-> (Vec Block) Block))
(defun (each bs) (BMany bs))

;; Renders the content and applies f.
(: (-> (-> string Block) Block Block))
(defun (captured f b)
  (BRun (fun (out col) (run-block out col (f (render->string b))))))


;; --- Vertical and horizontal space -----------------------------------------

(: Block)
(def nl (BStr "\n"))

;; Outputs newline if not at start of line.
(: Block)
(def fl
  (BRun (fun (out col)
          (if (= col 0) col (run-block out col nl)))))

;; Pads to specified column.
(: (-> int Block))
(defun (space-to goal)
  (BRun (fun (out col)
          (if (>= col goal)
              col
              (run-block out col (BStr (padding (- goal col))))))))

;; Moves to next tab stop.
(: (-> (#:width int) Block))
(defun (tab-to #:width 8)
  (BRun (fun (out col)
          (run-block out col (BStr (padding (- (* width (+ 1 (/ col width)))
                                               col)))))))


;; --- Fitting to a width ----------------------------------------------------

(: (-> int Block Block))
(defun (padded/left w b) (captured (fun (s) (BStr (pad/left w s))) b))

(: (-> int Block Block))
(defun (padded/right w b) (captured (fun (s) (BStr (pad/right w s))) b))

(: (-> int Block Block))
(defun (padded/both w b) (captured (fun (s) (BStr (pad/both w s))) b))

(: (-> int Block Block))
(defun (trimmed/left w b) (captured (fun (s) (BStr (trim/left w s))) b))

(: (-> int Block Block))
(defun (trimmed/right w b) (captured (fun (s) (BStr (trim/right w s))) b))

(: (-> int Block Block))
(defun (trimmed/both w b) (captured (fun (s) (BStr (trim/both w s))) b))

(: (-> int Block Block))
(defun (fitted/left w b)
  (captured (fun (s) (BStr (pad/left w (trim/left w s)))) b))

(: (-> int Block Block))
(defun (fitted/right w b)
  (captured (fun (s) (BStr (pad/right w (trim/right w s)))) b))

(: (-> int Block Block))
(defun (fitted/both w b)
  (captured (fun (s) (BStr (pad/both w (trim/both w s)))) b))


;; --- Sequences -------------------------------------------------------------

;; Every element, with `sep` between them.
(: (-> (-> %a Block) (List %a) Block Block))
(defun (joined f xs sep) (joined/last f f xs sep))

;; `sep` before each element, including the first.
(: (-> (-> %a Block) (List %a) Block Block))
(defun (joined/prefix f xs sep)
  (BRun (fun (out col)
          (list-foldl (fun (x c) (run-block out (run-block out c sep) (f x)))
                      col
                      xs))))

;; Appends separator after elements.
(: (-> (-> %a Block) (List %a) Block Block))
(defun (joined/suffix f xs sep)
  (BRun (fun (out col)
          (list-foldl (fun (x c) (run-block out (run-block out c (f x)) sep))
                      col
                      xs))))

;; Joins elements with custom last element formatter.
(: (-> (-> %a Block) (-> %a Block) (List %a) Block Block))
(defun (joined/last f last-f xs sep)
  (BRun (fun (out col)
          (let loop ((ys xs) (c col) (first? #t))
            (match ys
              (Nil c)
              ((Cons x Nil)
               (run-block out (if first? c (run-block out c sep)) (last-f x)))
              ((Cons x rest)
               (loop rest
                     (run-block out (if first? c (run-block out c sep)) (f x))
                     #f)))))))


;; --- Numbers ---------------------------------------------------------------
(import/extern
  (clr-invariant (: System.Globalization.CultureInfo.InvariantCulture #:get)))

;; `n` digits at a time from the right, `sep` between the groups.
(: (-> string int char string))
(defun (grouped digits n sep)
  (if (<= n 0)
      digits
      (let ((b (stringbuilder-empty)))
        ;; Backwards, because the groups are counted from the right; the string
        ;; comes out reversed and is turned around once at the end.
        (let loop ((c (string-cursor-end digits)) (i 0))
          (unless (string-cursor=? c (string-cursor-start digits))
            (let ((p (string-cursor-prev digits c)))
              (when (and (> i 0) (= 0 (% i n)))
                (stringbuilder-add! b sep))
              (stringbuilder-add! b (string-cursor-ref digits p))
              (loop p (+ i 1)))))
        (string-reverse (stringbuilder->string b)))))

;; Formats number blocks.
;;
;; It is `double` rather than anything wider: an `int` crosses with
;; `(cast double n)`, and a number being formatted is on its way to a string.
(: (-> double (#:precision int) (#:comma int) (#:comma-sep char)
       (#:decimal-sep char) (#:signed bool) Block))
(defun (numeric x #:precision -1 #:comma -1 #:comma-sep #\nul
                #:decimal-sep #\nul #:signed #f)
  (BRun (fun (out col)
          (run-block
           out
           col
           (BStr (formatted x
                            (if (< precision 0) (parameter-ref current-precision) precision)
                            (if (< comma 0) (parameter-ref current-comma) comma)
                            (if (char=? comma-sep #\nul)
                                (parameter-ref current-comma-sep)
                                comma-sep)
                            (if (char=? decimal-sep #\nul)
                                (parameter-ref current-decimal-sep)
                                decimal-sep)
                            (or signed (parameter-ref current-signed))))))))

(: (-> double int int char char bool string))
(defun (formatted x precision comma comma-sep decimal-sep signed)
  (let* ((raw (if (< precision 0)
                  (.ToString x clr-invariant)
                  (.ToString x (string-append "F" (int->string precision)) clr-invariant)))
         (negative? (string-starts-with? raw "-"))
         (body (if negative? (string-drop-first raw) raw))
         (parts (split-at-point body))
         (whole (first-of parts))
         (fraction (second-of parts)))
    (string-append
     (if negative? "-" (if signed "+" ""))
     (string-append
      (grouped whole comma comma-sep)
      (if (string-empty? fraction)
          ""
          (string-append (char->string decimal-sep) fraction))))))

;; The common case, and the reason `comma` is spelled that way: thousands.
(: (-> double Block))
(defun (numeric/comma x) (numeric x #:comma 3))

;; The invariant culture's decimal point is `.`, so this is a split on one
;; character and not a search for whatever the machine calls a separator.
(: (-> string (Tuple string string)))
(defun (split-at-point s)
  (match (string-find (fun (c) (char=? c #\.)) s)
    ((Some c) (Tuple (substring s (string-cursor-start s) c)
                     (substring s (string-cursor-next s c) (string-cursor-end s))))
    (None (Tuple s ""))))

(: (-> (Tuple string string) string))
(defun (first-of t) (let (((a _) t)) a))

(: (-> (Tuple string string) string))
(defun (second-of t) (let (((_ b) t)) b))

(: (-> string string))
(defun (string-drop-first s)
  (if (string-empty? s)
      s
      (substring s
                 (string-cursor-next s (string-cursor-start s))
                 (string-cursor-end s))))


;; --- Sizes, shares, bars ----------------------------------------------------

;; A number of bytes in the largest unit it is at least one of: `512 B`,
;; `1.2 KiB`, `3.4 GiB`. With `#:si`, the units are powers of 1000: `1.2 kB`.
(: (-> long (#:si bool) (#:precision int) Block))
(defun (bytes n #:si #f #:precision 1)
  (BRun (fun (out col)
          (def step (if si 1000.0 1024.0))
          (def units (if si ["B" "kB" "MB" "GB" "TB" "PB" "EB"] ["B" "KiB" "MiB" "GiB" "TiB" "PiB" "EiB"]))
          (def negative? (< n 0L))
          (def size (cast double (if negative? (- 0L n) n)))
          (def text
            (let lp ((x size) (u 0))
              (cond
                ((= u 0) (if (< x step) (str (long->string (cast long x)) " B") (lp (/ x step) 1)))
                ((and (>= x step) (< u (- (vec-length units) 1))) (lp (/ x step) (+ u 1)))
                (else (str (formatted x (max precision 0) 0 #\, (parameter-ref current-decimal-sep) #f)
                           " " (vec-ref units u))))))
          (run-block out col (BStr (if negative? (str "-" text) text))))))

;; A share, written as a percentage: 0.125 is `12.5%`.
(: (-> double (#:precision int) Block))
(defun (percent x #:precision 1)
  (BRun (fun (out col)
          (run-block out col
                     (BStr (str (formatted (* x 100.0) (max precision 0) 0 #\,
                                           (parameter-ref current-decimal-sep) #f)
                                "%"))))))

;; The eighths of a block, from one eighth to seven.
(def eighths ["▏" "▎" "▍" "▌" "▋" "▊" "▉"])

;; A bar `width` columns wide, filled to `fraction` of it to an eighth of a
;; column, and `empty` in the rest. A fraction outside 0 to 1 is the nearest
;; end.
(: (-> double (#:width int) (#:empty string) Block))
(defun (bar fraction #:width 10 #:empty " ")
  (BRun (fun (out col)
          (def* (f (max (min fraction 1.0) 0.0))
                (total (cast int (+ (* f (cast double (* width 8))) 0.5)))
                (full (/ total 8))
                (part (% total 8))
                (b (stringbuilder-empty)))
          (loop (:for _ (range 0 full)) (:do (stringbuilder-add-string! b "█")))
          (when (> part 0) (stringbuilder-add-string! b (vec-ref eighths (- part 1))))
          (def used (+ full (if (> part 0) 1 0)))
          (loop (:for _ (range 0 (- width used))) (:do (stringbuilder-add-string! b empty)))
          (run-block out col (BStr (stringbuilder->string b))))))

(def levels ["▁" "▂" "▃" "▄" "▅" "▆" "▇" "█"])

;; The values as a line of bars of eight heights, the smallest the lowest and
;; the largest the highest. Values that are all the same are all in the
;; middle.
(: (-> (Vec double) Block))
(defun (sparkline xs)
  (BRun (fun (out col)
          (def* (first (if (> (vec-length xs) 0) (vec-ref xs 0) 0.0))
                (lo (vec-fold min first xs))
                (hi (vec-fold max first xs))
                (b (stringbuilder-empty)))
          (loop (:for x xs)
                (:do (stringbuilder-add-string!
                      b
                      (if (<= hi lo)
                          (vec-ref levels 3)
                          (vec-ref levels (min 7 (cast int (* 8.0 (/ (- x lo) (- hi lo))))))))))
          (run-block out col (BStr (stringbuilder->string b))))))


;; --- Paragraphs ------------------------------------------------------------

;; The runs of non-whitespace in `s`, in order.
(: (-> string (Vec string)))
(defun (words-of s)
  (def out (vecbuilder-empty))
  (let lp ((c (string-cursor-start s)) (start (string-cursor-start s)) (in? #f))
    (if (string-cursor-end? s c)
        ;; todo: add a (cond ...) here
        (when in? (vecbuilder-add! out (substring s start c)))
        (if (char-whitespace? (string-cursor-ref s c))
            (let ()
              (when in? (vecbuilder-add! out (substring s start c)))
              (lp (string-cursor-next s c) (string-cursor-next s c) #f))
            (lp (string-cursor-next s c) (if in? start c) #t))))
  (vecbuilder->vec out))

;; Packs words into lines.
(: (-> (Vec string) int int (List (List string))))
(defun (packed ws first width)
  (let lp ((i 0) (line Nil) (used 0) (budget first) (lines Nil))
    (if (>= i (vec-length ws))
        (list-reverse (if (list-empty? line) lines (Cons (list-reverse line) lines)))
        (let* ((w (vec-ref ws i))
               (needed (if (= used 0) (width-of w) (+ used 1 (width-of w)))))
          (if (or (= used 0) (<= needed budget))
              (lp (+ i 1) (Cons w line) needed budget lines)
              (lp i Nil 0 width (Cons (list-reverse line) lines)))))))

;; The words with a single pad character between them.
(: (-> (List string) string))
(defun (spaced ws)
  (def b (stringbuilder-empty))
  (def gap (padding 1))
  (loop (:with first? #t #f)
        (:for w ws)
        (:do
         (unless first? (stringbuilder-add-string! b gap))
         (stringbuilder-add-string! b w)))
  (stringbuilder->string b))

;; Stretches spaced words to budget.
(: (-> (List string) int string))
(defun (stretched ws budget)
  (let ((gaps (- (list-length ws) 1)))
    (if (<= gaps 0)
        (spaced ws)
        (let* ((text (list-foldl (fun (w acc) (+ acc (width-of w))) 0 ws))
               (room (max (- budget text) gaps))
               (base (/ room gaps))
               (extra (% room gaps))
               (b (stringbuilder-empty)))
              ;; TODO: rewrite as a {folding ...}
              (ignore (list-foldl (fun (w i)
                                    (when (> i 0)
                                      (stringbuilder-add-string!
                                       b
                                       (padding (+ base (if (<= i extra) 1 0)))))
                                    (stringbuilder-add-string! b w)
                                    (+ i 1))
                                  0
                                  ws))
              (stringbuilder->string b)))))

(: (-> (Vec string) bool Block))
(defun (wrap ws justify?)
  (BRun (fun (out col)
          (let* ((width (parameter-ref current-width))
                 (lines (packed ws (- width col) width)))
                (let loop ((ls lines) (c col) (first? #t))
                  (match ls
                    (Nil c)
                    ((Cons line rest)
                     ;; Todo: rewrite using (def ...)
                     (let* ((c1 (if first? c (run-block out c nl)))
                            (budget (if first? (- width col) width))
                            (text (if (and justify? (not (list-empty? rest)))
                                      (stretched line budget)
                                      (spaced line))))
                       (loop rest (run-block out c1 (BStr text)) #f)))))))))

;; Filled to the right margin, ragged on the right.
(: (-> Block Block))
(defun (wrapped b) (captured (fun (s) (wrap (words-of s) #f)) b))

;; Filled to the right margin and flush against both, the last line excepted.
(: (-> Block Block))
(defun (justified b) (captured (fun (s) (wrap (words-of s) #t)) b))


;; --- Tables ----------------------------------------------------------------
;;
;; They differ only in where the widths come from: `columnar` divides
;; `current-width` evenly and cuts whatever does not fit, `tabular` gives each
;; column the width its own content asks for and cuts nothing.

;; The lines of `s`, without the empty one a trailing newline leaves behind.
(: (-> string (Vec string)))
(defun (lines-of s)
  (def* (parts (string-split s "\n"))
        (n (vec-length parts)))
  (if (and (> n 1) (string-empty? (vec-ref parts (- n 1))))
      (vec-slice parts 0 (- n 1))
      parts))

;; Where a cell sits in its column. `(on #\.)` puts a character of every cell
;; in one column, as the decimal point is in a column of numbers.
(def/grammar Align
  (:or (: AlignLeft 'left) (: AlignRight 'right) (: AlignCentre 'centre) (: AlignOn (on char))))

;; The alignment of column `i`: left unless said otherwise.
(: (-> (Vec Align) int Align))
(defun (align-at aligns i)
  (if (< i (vec-length aligns)) (vec-ref aligns i) AlignLeft))

;; A cell fitted to its column. A cut keeps the end an alignment leans on,
;; as `fitted/left` and the rest do. The last column is not padded on the
;; right: there is nothing after it to line up.
(: (-> Align int bool bool string string))
(defun (fit-cell a w cut? last? text)
  (def s (if cut?
             (match a
               (AlignLeft (trim/right w text))
               (AlignRight (trim/left w text))
               (AlignCentre (trim/both w text))
               ((AlignOn _) (trim/left w text)))
             text))
  (match a
    (AlignLeft (if last? s (pad/right w s)))
    (AlignRight (pad/left w s))
    (AlignCentre (if last?
                     (string-append (padding (/ (- w (width-of s)) 2)) s)
                     (pad/both w s)))
    ;; Without the widths on either side of the character, which these
    ;; tables do not measure, the cell is aligned to the right.
    ((AlignOn _) (pad/left w s))))

(: (-> (Vec (Vec string)) (Vec int) (Vec Align) string bool Block))
(defun (laid-out cells widths aligns sep cut?)
  (BRun (fun (out col)
          (def rows (vec-fold (fun (ls acc) (max acc (vec-length ls))) 0 cells))
          (let lp ((r 0) (c (run-block out col fl)))
            (if (>= r rows)
                c
                (lp (+ r 1)
                    (run-block out
                               (run-block out c (BStr (row-of cells widths aligns sep cut? r)))
                               nl)))))))

;; Line `r` of a row. Empty cells at the end of it are left out, separators
;; and all, so that no line ends in blanks.
(: (-> (Vec (Vec string)) (Vec int) (Vec Align) string bool int string))
(defun (row-of cells widths aligns sep cut? r)
  (def* (b (stringbuilder-empty))
        (texts (vec-map (fun (ls) (if (< r (vec-length ls)) (vec-ref ls r) "")) cells))
        (n (let lp ((n (vec-length texts)))
             (if (and (> n 0) (string-empty? (vec-ref texts (- n 1)))) (lp (- n 1)) n))))
  (let lp ((i 0))
    (when (< i n)
      (when (> i 0) (stringbuilder-add-string! b sep))
      (stringbuilder-add-string!
       b
       (fit-cell (align-at aligns i) (vec-ref widths i) cut? (= i (- n 1)) (vec-ref texts i)))
      (lp (+ i 1))))
  (stringbuilder->string b))

;; Each cell's lines.
(: (-> (Vec (Vec Block)) (Vec (Vec string))))
(defun (rendered-cells cells)
  (vec-map (fun (c) (lines-of (render->string c))) cells))

;; The lines of a row's cells, a block each.
(: (-> (Vec Block) (Vec (Vec string))))
(defun (rendered-row row)
  (vec-map (fun (c) (lines-of (render->string [c]))) row))

(: (-> (Vec string) int))
(defun (widest ls) (vec-fold (fun (l acc) (max acc (width-of l))) 0 ls))

;; Renders columns.
(: (-> (Vec (Vec Block)) (#:sep string) (#:align (Vec Align)) Block))
(defun (columnar cols #:sep " " #:align [])
  (BRun (fun (out col)
          (let* ((cells (rendered-cells cols))
                 (n (vec-length cells))
                 (room (- (parameter-ref current-width)
                          (* (width-of sep) (max (- n 1) 0))))
                 (each-width (if (> n 0) (max (/ room n) 0) 0)))
            (run-block out
                       col
                       (laid-out cells (vec-map (fun (ls) each-width) cells) align sep #t))))))

;; --- Borders -----------------------------------------------------------------
;;
;; A bordered table is lines of three kinds: the cells of a row, a horizontal
;; line made of a fill repeated to each column's width, and the edges and
;; joins around both. `BorderChars` is every piece of them, so a style is a
;; value and a custom one is written as one.

(type (: BorderChars (Record (: top-left string) (: top string) (: top-join string) (: top-right string)
                             (: left string) (: column string) (: right string)
                             (: rule-left string) (: rule string) (: rule-join string) (: rule-right string)
                             (: bottom-left string) (: bottom string) (: bottom-join string)
                             (: bottom-right string)
                             (: marks bool))))

(def/grammar TableStyle
  (:or (: StyleNone 'none) (: StyleSimple 'simple) (: StyleAscii 'ascii) (: StyleLight 'light)
       (: StyleHeavy 'heavy) (: StyleDouble 'double) (: StyleRounded 'rounded)
       (: StyleMarkdown 'markdown) (: StyleCustom ,BorderChars)))

;; Which horizontal rules a table has inside it: under the header, between
;; every two rows, or none.
(def/grammar TableRules
  (:or (: RulesHeader 'header) (: RulesEvery 'every) (: RulesNone 'none)))

;; How wide a column is: as its content asks, a number of columns, at least
;; or at most a number, or a share of what the other columns leave of
;; `current-width`.
(def/grammar ColumnWidth
  (:or (: WidthAuto 'auto) (: WidthFixed int) (: WidthAtLeast (at-least int))
       (: WidthAtMost (at-most int)) (: WidthShare (share int))))

;; What a cell wider than its column does: pushes the rest of its row out,
;; is cut, or is wrapped onto more lines. A table that cuts or wraps is made
;; to fit `current-width`.
(def/grammar Overflow
  (:or (: OverflowPush 'push) (: OverflowCut 'cut) (: OverflowWrap 'wrap)))

;; `n` spaces, whatever the pad character is: a border's spacing is part of
;; the border.
(: (-> int string))
(defun (spaces n)
  (def b (stringbuilder-empty))
  (loop (:for _ (range 0 n)) (:do (stringbuilder-add! b #\space)))
  (stringbuilder->string b))

;; A box drawn with the line `h`, the line `v`, and the nine places they meet,
;; as on a keypad: the top row, the middle row and the bottom row, each left,
;; middle and right. A cell has a space on each side of it.
(: (-> string string string string string string string string string string string BorderChars))
(defun (box-chars h v tl tj tr ml mj mr bl bj br)
  (BorderChars (top-left (str tl h)) (top h) (top-join (str h tj h)) (top-right (str h tr))
               (left (str v " ")) (column (str " " v " ")) (right (str " " v))
               (rule-left (str ml h)) (rule h) (rule-join (str h mj h)) (rule-right (str h mr))
               (bottom-left (str bl h)) (bottom h) (bottom-join (str h bj h)) (bottom-right (str h br))
               (marks #f)))

;; No edges: `sep` between the columns, and a rule of `fill` under the header
;; when `fill` is not empty.
(: (-> string string BorderChars))
(defun (open-chars sep fill)
  (BorderChars (top-left "") (top "") (top-join "") (top-right "")
               (left "") (column sep) (right "")
               (rule-left "") (rule fill) (rule-join (spaces (width-of sep))) (rule-right "")
               (bottom-left "") (bottom "") (bottom-join "") (bottom-right "")
               (marks #f)))

;; The pieces a style is drawn with. `sep` is what `StyleNone` puts between
;; the columns.
(: (-> TableStyle string BorderChars))
(defun (border-chars style sep)
  (match style
    (StyleNone (open-chars sep ""))
    (StyleSimple (open-chars "  " "─"))
    (StyleAscii (box-chars "-" "|" "+" "+" "+" "+" "+" "+" "+" "+" "+"))
    (StyleLight (box-chars "─" "│" "┌" "┬" "┐" "├" "┼" "┤" "└" "┴" "┘"))
    (StyleHeavy (box-chars "━" "┃" "┏" "┳" "┓" "┣" "╋" "┫" "┗" "┻" "┛"))
    (StyleDouble (box-chars "═" "║" "╔" "╦" "╗" "╠" "╬" "╣" "╚" "╩" "╝"))
    (StyleRounded (box-chars "─" "│" "╭" "┬" "╮" "├" "┼" "┤" "╰" "┴" "╯"))
    (StyleMarkdown
     (BorderChars (top-left "") (top "") (top-join "") (top-right "")
                  (left "| ") (column " | ") (right " |")
                  (rule-left "| ") (rule "-") (rule-join " | ") (rule-right " |")
                  (bottom-left "") (bottom "") (bottom-join "") (bottom-right "")
                  (marks #t)))
    ((StyleCustom chars) chars)))

;; Whether a table is laid out as the tables without borders always were:
;; no borders, no header, no column aligned on a character, which only the
;; bordered layout measures, and no width rules or overflow.
(: (-> TableStyle (Vec Block) (Vec Align) (Vec ColumnWidth) Overflow bool))
(defun (plain-table? style titles aligns widths overflow)
  (and (match style (StyleNone #t) (_ #f))
       (= (vec-length titles) 0)
       (not (any? (fun (a) (match a ((AlignOn _) #t) (_ #f))) aligns))
       (= (vec-length widths) 0)
       (match overflow (OverflowPush #t) (_ #f))))

;; A column's width, and for one aligned on a character, the widest part of a
;; line before the character and the widest from it on.
(type (: ColumnFit (Record (: width int) (: before int) (: after int))))

;; A line cut where `c` first is: the part before it, and the rest from it
;; on. A line without one is all before.
(: (-> char string (Tuple string string)))
(defun (split-on c s)
  (match (string-find (fun (x) (char=? x c)) s)
    ((Some at) (Tuple (substring s (string-cursor-start s) at) (substring s at (string-cursor-end s))))
    (None (Tuple s ""))))

(: (-> (Vec (Vec string)) int (Vec string)))
(defun (cell-at cells i)
  (if (< i (vec-length cells)) (vec-ref cells i) []))

;; Every line of column `i`, in all the rows.
(: (-> (Vec (Vec (Vec string))) int (Vec string)))
(defun (column-lines rows i)
  (def b (vecbuilder-empty))
  (loop (:for row rows)
        (:do (loop (:for l (cell-at row i)) (:do (vecbuilder-add! b l)))))
  (vecbuilder->vec b))

(: (-> Align (Vec string) (Vec string) bool ColumnFit))
(defun (column-fit a title lines marks)
  (def* (parts (match a
                 ((AlignOn c)
                  (vec-fold (fun (l acc)
                              (def (Tuple b af) (split-on c l))
                              (Tuple (max (fst acc) (width-of b)) (max (snd acc) (width-of af))))
                            (Tuple 0 0) lines))
                 (_ (Tuple 0 0))))
        (content (match a
                   ((AlignOn _) (+ (fst parts) (snd parts)))
                   (_ (widest lines)))))
  (ColumnFit (width (max content (max (widest title) (if marks 3 0))))
             (before (fst parts))
             (after (snd parts))))

(: (-> (Vec Align) (Vec (Vec string)) (Vec (Vec (Vec string))) bool (Vec ColumnFit)))
(defun (column-fits aligns titles rows marks)
  (def n (vec-fold (fun (r acc) (max acc (vec-length r))) (vec-length titles) rows))
  {vecing (column-fit (align-at aligns i) (cell-at titles i) (column-lines rows i) marks)
          (:for i (range 0 n))})

;; A title is aligned as its column is, and one over a column aligned on a
;; character to the right, where the numbers under it end.
(: (-> Align Align))
(defun (title-align a)
  (match a
    ((AlignOn _) AlignRight)
    (_ a)))

;; One line of a cell, as wide as its column. `open-end?` is a last cell with
;; nothing after it, which is not padded on the right.
(: (-> Align ColumnFit bool string string))
(defun (fit-line a fit open-end? text)
  (def w (record-ref fit width))
  (match a
    ((AlignOn c)
     (def* (parts (split-on c text))
           (before (fst parts))
           (span (+ (record-ref fit before) (record-ref fit after)))
           (core (str (padding (- (record-ref fit before) (width-of before))) before (snd parts))))
     (str (padding (- w span)) (if open-end? core (pad/right span core))))
    (_ (fit-cell a w #f open-end? text))))

;; One line of a row: the edges, the separators and each cell's line `k`.
;; Without edges, empty cells at the end are left out and the last one is not
;; padded, as in a table without borders.
(: (-> BorderChars (Vec Align) (Vec ColumnFit) (Vec string) string))
(defun (row-line chars aligns fits texts)
  (def* (open? (and (string-empty? (record-ref chars left)) (string-empty? (record-ref chars right))))
        (n (if open?
               (let lp ((n (vec-length texts)))
                 (if (and (> n 0) (string-empty? (vec-ref texts (- n 1)))) (lp (- n 1)) n))
               (vec-length fits)))
        (b (stringbuilder-empty)))
  (stringbuilder-add-string! b (record-ref chars left))
  (loop (:for i (range 0 n))
        (:do (when (> i 0) (stringbuilder-add-string! b (record-ref chars column)))
             (stringbuilder-add-string!
              b
              (fit-line (align-at aligns i) (vec-ref fits i) (and open? (= i (- n 1)))
                        (if (< i (vec-length texts)) (vec-ref texts i) "")))))
  (stringbuilder-add-string! b (record-ref chars right))
  (stringbuilder->string b))

;; The lines of a row, as many as its tallest cell has.
(: (-> BorderChars (Vec Align) (Vec ColumnFit) (Vec (Vec string)) (Vec string)))
(defun (row-lines chars aligns fits cells)
  (def height (vec-fold (fun (c acc) (max acc (vec-length c))) 1 cells))
  {vecing (row-line chars aligns fits
                    (vec-map (fun (c) (if (< k (vec-length c)) (vec-ref c k) "")) cells))
          (:for k (range 0 height))})

;; `fill` repeated to `w` columns.
(: (-> string int string))
(defun (repeated fill w)
  (def* (fw (width-of fill))
        (b (stringbuilder-empty)))
  (when (> fw 0)
    (loop (:for _ (range 0 (/ w fw))) (:do (stringbuilder-add-string! b fill))))
  (stringbuilder->string b))

;; A column's part of the rule under a Markdown header, which says how the
;; column is aligned.
(: (-> Align int string))
(defun (marked a w)
  (match a
    (AlignLeft (str ":" (repeated "-" (- w 1))))
    (AlignCentre (str ":" (repeated "-" (- w 2)) ":"))
    (_ (str (repeated "-" (- w 1)) ":"))))

;; A horizontal line: `fill` across each column, `join` where the columns
;; meet, and the two ends.
(: (-> string string string string bool (Vec Align) (Vec ColumnFit) string))
(defun (hline lft fill join rgt marks aligns fits)
  (def b (stringbuilder-empty))
  (stringbuilder-add-string! b lft)
  (loop (:for i (range 0 (vec-length fits)))
        (:do (when (> i 0) (stringbuilder-add-string! b join))
             (def w (record-ref (vec-ref fits i) width))
             (stringbuilder-add-string! b (if marks (marked (align-at aligns i) w) (repeated fill w)))))
  (stringbuilder-add-string! b rgt)
  (stringbuilder->string b))

(: (-> BorderChars (Vec Align) (Vec ColumnFit) string))
(defun (top-line chars aligns fits)
  (hline (record-ref chars top-left) (record-ref chars top) (record-ref chars top-join)
         (record-ref chars top-right) #f aligns fits))

(: (-> BorderChars bool (Vec Align) (Vec ColumnFit) string))
(defun (rule-line chars marks aligns fits)
  (hline (record-ref chars rule-left) (record-ref chars rule) (record-ref chars rule-join)
         (record-ref chars rule-right) marks aligns fits))

(: (-> BorderChars (Vec Align) (Vec ColumnFit) string))
(defun (bottom-line chars aligns fits)
  (hline (record-ref chars bottom-left) (record-ref chars bottom) (record-ref chars bottom-join)
         (record-ref chars bottom-right) #f aligns fits))

;; The lines, each ended by a newline, starting on a fresh line.
(: (-> (Vec string) Block))
(defun (lines-block lines)
  (BRun (fun (out col)
          (vec-fold (fun (l c) (run-block out (run-block out c (BStr l)) nl)) (run-block out col fl) lines))))

;; --- Column widths and overflow ----------------------------------------------

(: (-> (Vec ColumnWidth) int ColumnWidth))
(defun (width-rule-at rules i)
  (if (< i (vec-length rules)) (vec-ref rules i) WidthAuto))

(: (-> ColumnWidth bool))
(defun (shared? rule)
  (match rule ((WidthShare _) #t) (_ #f)))

;; The narrowest a column may be made to fit the table into its width.
(: (-> ColumnWidth int))
(defun (narrowest rule)
  (match rule
    ((WidthFixed n) (max n 1))
    ((WidthAtLeast n) (max n 1))
    (_ 1)))

;; The width of each column: its content's, `natural`, as its rule says, with
;; the shares dividing what the others leave of `room`. When `fit?`, the
;; widest column that may shrink loses a column until the table fits `room`.
(: (-> (Vec int) (Vec ColumnWidth) int bool (Vec int)))
(defun (settled-widths natural rules room fit?)
  (def* (n (vec-length natural))
        (base {vecing (match (width-rule-at rules i)
                        ((WidthFixed k) (max k 1))
                        ((WidthAtLeast k) (max (vec-ref natural i) k))
                        ((WidthAtMost k) (max (min (vec-ref natural i) k) 1))
                        ((WidthShare _) 0)
                        (WidthAuto (vec-ref natural i)))
                      (:for i (range 0 n))})
        (shares (loop (:for i (range 0 n))
                      (:acc (summing (match (width-rule-at rules i) ((WidthShare s) (max s 1)) (_ 0)))))))
  (defun (shrinkable? ws i)
    (and (not (shared? (width-rule-at rules i)))
         (match (width-rule-at rules i) ((WidthFixed _) #f) (_ #t))
         (> (vec-ref ws i) (narrowest (width-rule-at rules i)))))
  (def* (fitted (if (not fit?)
                    base
                    (let lp ((ws base))
                      (def used (vec-fold + 0 ws))
                      (def wanted (+ used (if (> shares 0) n 0)))
                      (def widest-i
                        (loop (:for i (range 0 n))
                              (:when (shrinkable? ws i))
                              (:acc best (folding -1 (if (or (< best 0) (> (vec-ref ws i) (vec-ref ws best))) i best)))))
                      (if (or (<= wanted room) (< widest-i 0))
                          ws
                          (lp (vec-set ws widest-i (- (vec-ref ws widest-i) 1))))))))
  (if (= shares 0)
      fitted
      (let ()
        (def left (max (- room (vec-fold + 0 fitted)) 0))
        {vecing (match (width-rule-at rules i)
                  ((WidthShare s) (max (/ (* left (max s 1)) shares) 1))
                  (_ (vec-ref fitted i)))
                (:for i (range 0 n))})))

;; `word` in pieces no wider than `w`. A character wider than `w` is a piece
;; of its own.
(: (-> int string (List string)))
(defun (chopped w word)
  (let lp ((c (string-cursor-start word)) (start (string-cursor-start word)) (used 0) (acc Nil))
    (cond
      ((string-cursor-end? word c)
       (list-reverse (if (string-cursor=? start c) acc (Cons (substring word start c) acc))))
      (:def cw (width-of (char->string (string-cursor-ref word c))))
      ((and (> used 0) (> (+ used cw) w)) (lp c c 0 (Cons (substring word start c) acc)))
      (else (lp (string-cursor-next word c) start (+ used cw) acc)))))

;; A line broken at its spaces into lines no wider than `w`.
(: (-> int string (Vec string)))
(defun (wrapped-line w line)
  (if (<= (width-of line) w)
      [line]
      (let ()
        (def pieces (vecbuilder-empty))
        (loop (:for word (words-of line))
              (:do (loop (:for p (chopped w word)) (:do (vecbuilder-add! pieces p)))))
        (list->vec (list-map spaced (packed (vecbuilder->vec pieces) w w))))))

;; A cell's lines made to fit a column of `w`, as `overflow` says.
(: (-> Overflow Align int (Vec string) (Vec string)))
(defun (overflowed overflow a w lines)
  (match (Tuple overflow a)
    ((Tuple _ (AlignOn _)) lines)
    ((Tuple OverflowPush _) lines)
    ((Tuple OverflowCut AlignLeft) (vec-map (fun (l) (trim/right w l)) lines))
    ((Tuple OverflowCut AlignCentre) (vec-map (fun (l) (trim/both w l)) lines))
    ((Tuple OverflowCut _) (vec-map (fun (l) (trim/left w l)) lines))
    ((Tuple OverflowWrap _)
     (vec-fold (fun (l acc) (vec-append acc (wrapped-line w l))) [] lines))))

;; A row's cells made to fit their columns.
(: (-> Overflow (Vec Align) (Vec int) (Vec (Vec string)) (Vec (Vec string))))
(defun (overflowed-row overflow aligns widths cells)
  {vecing (overflowed overflow (align-at aligns i)
                      (if (< i (vec-length widths)) (vec-ref widths i) 1)
                      (vec-ref cells i))
          (:for i (range 0 (vec-length cells)))})

;; How many columns the edges and separators of a row of `n` cells take.
(: (-> BorderChars int int))
(defun (border-width chars n)
  (+ (width-of (record-ref chars left))
     (+ (width-of (record-ref chars right))
        (* (max (- n 1) 0) (width-of (record-ref chars column))))))

;; A table with borders, its cells rendered already: a header of `titles`
;; when there are any, and `rows`. A horizontal line is drawn where its fill
;; is not empty. The columns are as wide as `widths` says, and a cell wider
;; than its column does what `overflow` says.
(: (-> BorderChars (Vec Align) (Vec (Vec string)) (Vec (Vec (Vec string))) TableRules
       (Vec ColumnWidth) Overflow Block))
(defun (bordered chars aligns titles rows rules widths overflow)
  (def* (marks (record-ref chars marks))
        (natural (column-fits aligns titles rows marks))
        (fit? (match overflow (OverflowPush #f) (_ #t)))
        (settled (vec-map (fun (w) (if marks (max w 3) w))
                          (settled-widths (vec-map (fun (f) (record-ref f width)) natural) widths
                                          (- (parameter-ref current-width)
                                             (border-width chars (vec-length natural)))
                                          fit?)))
        (fits {vecing (ColumnFit (width (vec-ref settled i))
                                 (before (record-ref (vec-ref natural i) before))
                                 (after (record-ref (vec-ref natural i) after)))
                      (:for i (range 0 (vec-length natural)))})
        (title-aligns {vecing (title-align (align-at aligns i)) (:for i (range 0 (vec-length fits)))})
        (titles (overflowed-row overflow title-aligns settled titles))
        (rows (vec-map (fun (row) (overflowed-row overflow aligns settled row)) rows))
        (ruled? (not (string-empty? (record-ref chars rule))))
        (under-header? (match rules (RulesNone #f) (_ #t)))
        (between-rows? (match rules (RulesEvery #t) (_ #f)))
        (lines (vecbuilder-empty)))
  (defun (add-all ls) (loop (:for l ls) (:do (vecbuilder-add! lines l))))
  (unless (string-empty? (record-ref chars top))
    (vecbuilder-add! lines (top-line chars aligns fits)))
  (when (> (vec-length titles) 0)
    (add-all (row-lines chars title-aligns fits titles))
    (when (and ruled? under-header?)
      (vecbuilder-add! lines (rule-line chars marks aligns fits))))
  (loop (:for (Tuple r row) (enumerate rows))
        (:do (when (and (> r 0) ruled? (not marks) between-rows?)
               (vecbuilder-add! lines (rule-line chars #f aligns fits)))
             (add-all (row-lines chars aligns fits row))))
  (unless (string-empty? (record-ref chars bottom))
    (vecbuilder-add! lines (bottom-line chars aligns fits)))
  (lines-block (vecbuilder->vec lines)))

;; Columns of lines as rows of one-line cells: the `r`th line of each column
;; is row `r`.
(: (-> (Vec (Vec string)) (Vec (Vec (Vec string)))))
(defun (columns->rows cols)
  (def height (vec-fold (fun (c acc) (max acc (vec-length c))) 0 cols))
  {vecing (vec-map (fun (c) (if (< r (vec-length c)) [(vec-ref c r)] [])) cols)
          (:for r (range 0 height))})

(: (-> (Vec (Vec Block)) (#:sep string) (#:align (Vec Align)) (#:style TableStyle)
       (#:titles (Vec Block)) (#:rules TableRules) (#:widths (Vec ColumnWidth)) (#:overflow Overflow)
       Block))
(defun (tabular cols #:sep " " #:align [] #:style StyleNone #:titles [] #:rules RulesHeader
                #:widths [] #:overflow OverflowPush)
  (BRun (fun (out col)
          (def cells (rendered-cells cols))
          (if (plain-table? style titles align widths overflow)
              (run-block out col (laid-out cells (vec-map widest cells) align sep #f))
              (run-block out col (bordered (border-chars style sep) align (rendered-row titles)
                                           (columns->rows cells) rules widths overflow))))))

;; --- Tables a row at a time --------------------------------------------------
;;
;; A table whose rows are known one by one: the widths are settled first,
;; and each row is then laid out on its own, so it can be written the moment
;; it is known.

(type (: TableLayout (Record (: widths (Vec int))
                             (: aligns (Vec Align))
                             (: sep string)
                             (: style TableStyle))))

(: (-> (Vec int) (#:sep string) (#:align (Vec Align)) (#:style TableStyle) TableLayout))
(defun (table-layout widths #:sep " " #:align [] #:style StyleNone)
  (TableLayout (widths widths) (aligns align) (sep sep) (style style)))

;; A layout's columns as fits. The widths on either side of a character are
;; not known a row at a time, so a column aligned on one is aligned right.
(: (-> TableLayout (Vec Align)))
(defun (layout-aligns layout)
  (def aligns (record-ref layout aligns))
  {vecing (title-align (align-at aligns i)) (:for i (range 0 (vec-length (record-ref layout widths))))})

(: (-> TableLayout (Vec ColumnFit)))
(defun (layout-fits layout)
  (vec-map (fun (w) (ColumnFit (width w) (before 0) (after 0))) (record-ref layout widths)))

(: (-> TableLayout BorderChars))
(defun (layout-chars layout)
  (border-chars (record-ref layout style) (record-ref layout sep)))

;; The line above a table laid out a row at a time, or nothing for a style
;; without one.
(: (-> TableLayout Block))
(defun (table-top layout)
  (def chars (layout-chars layout))
  (if (string-empty? (record-ref chars top))
      nothing
      (lines-block [(top-line chars (layout-aligns layout) (layout-fits layout))])))

;; A rule across the table: under a header row, or between two rows.
(: (-> TableLayout Block))
(defun (table-rule layout)
  (def chars (layout-chars layout))
  (if (string-empty? (record-ref chars rule))
      nothing
      (lines-block [(rule-line chars (record-ref chars marks) (layout-aligns layout) (layout-fits layout))])))

;; The line under a table laid out a row at a time.
(: (-> TableLayout Block))
(defun (table-bottom layout)
  (def chars (layout-chars layout))
  (if (string-empty? (record-ref chars bottom))
      nothing
      (lines-block [(bottom-line chars (layout-aligns layout) (layout-fits layout))])))

;; The width each column of these rows asks for: its widest line.
(: (-> (Vec (Vec Block)) (Vec int)))
(defun (table-widths rows)
  (vec-fold (fun (row acc)
              (def ws (vec-map widest (rendered-row row)))
              (def n (max (vec-length acc) (vec-length ws)))
              {vecing (max (if (< i (vec-length acc)) (vec-ref acc i) 0)
                           (if (< i (vec-length ws)) (vec-ref ws i) 0))
                      (:for i (range 0 n))})
            [] rows))

;; One row, laid out to `layout`. A cell wider than its column is not cut:
;; it pushes the rest of its row out. Cells past the layout's columns get
;; no width and are left as they are.
;;
;; With a style that has borders, a row is drawn between them, every cell
;; padded to its column, and a cell past the layout's columns is left out.
(: (-> TableLayout (Vec Block) Block))
(defun (table-row layout row)
  (BRun (fun (out col)
          (def cells (rendered-row row))
          (if (plain-table? (record-ref layout style) [] [] [] OverflowPush)
              (let ()
                (def ws (record-ref layout widths))
                (def widths {vecing (if (< i (vec-length ws)) (vec-ref ws i) 0)
                                    (:for i (range 0 (vec-length cells)))})
                (run-block out col (laid-out cells widths (record-ref layout aligns) (record-ref layout sep) #f)))
              (run-block out col (lines-block (row-lines (layout-chars layout) (layout-aligns layout)
                                                         (layout-fits layout) cells)))))))

;; `tabular`, given rows rather than columns.
(: (-> (Vec (Vec Block)) (#:sep string) (#:align (Vec Align)) (#:style TableStyle)
       (#:titles (Vec Block)) (#:rules TableRules) (#:widths (Vec ColumnWidth)) (#:overflow Overflow)
       Block))
(defun (tabular/rows rows #:sep " " #:align [] #:style StyleNone #:titles [] #:rules RulesHeader
                     #:widths [] #:overflow OverflowPush)
  ;; Measured when it runs, where the settings it is run under are bound.
  (BRun (fun (out col)
          (if (plain-table? style titles align widths overflow)
              (let ()
                (def layout (table-layout (table-widths rows) #:sep sep #:align align))
                (run-blocks out col (vec-map (fun (row) (table-row layout row)) rows)))
              (run-block out col (bordered (border-chars style sep) align (rendered-row titles)
                                           (vec-map rendered-row rows) rules widths overflow))))))

;; --- Width in a terminal ---------------------------------------------------
;;
;; The code points whose width in a terminal is not one column, as triples of
;; the first, the last and the width: 0 for combining marks, format and
;; control characters and the skin tone modifiers, which join what is before
;; them, and 2 for East Asian Wide and Fullwidth characters and the regional
;; indicators. Generated from Python's `unicodedata` (Unicode 16.0) and checked
;; in, as `(std rx)` checks in its tables.
(: (Array int))
(def width-ranges
  #[
    0 31 0 127 159 0 173 173 0 768 879 0 1155 1161 0 1425 1469 0 1471 1471 0 1473 1474 0
    1476 1477 0 1479 1479 0 1536 1541 0 1552 1562 0 1564 1564 0 1611 1631 0 1648 1648 0
    1750 1757 0 1759 1764 0 1767 1768 0 1770 1773 0 1807 1807 0 1809 1809 0 1840 1866 0
    1958 1968 0 2027 2035 0 2045 2045 0 2070 2073 0 2075 2083 0 2085 2087 0 2089 2093 0
    2137 2139 0 2192 2193 0 2199 2207 0 2250 2306 0 2362 2362 0 2364 2364 0 2369 2376 0
    2381 2381 0 2385 2391 0 2402 2403 0 2433 2433 0 2492 2492 0 2497 2500 0 2509 2509 0
    2530 2531 0 2558 2558 0 2561 2562 0 2620 2620 0 2625 2626 0 2631 2632 0 2635 2637 0
    2641 2641 0 2672 2673 0 2677 2677 0 2689 2690 0 2748 2748 0 2753 2757 0 2759 2760 0
    2765 2765 0 2786 2787 0 2810 2815 0 2817 2817 0 2876 2876 0 2879 2879 0 2881 2884 0
    2893 2893 0 2901 2902 0 2914 2915 0 2946 2946 0 3008 3008 0 3021 3021 0 3072 3072 0
    3076 3076 0 3132 3132 0 3134 3136 0 3142 3144 0 3146 3149 0 3157 3158 0 3170 3171 0
    3201 3201 0 3260 3260 0 3263 3263 0 3270 3270 0 3276 3277 0 3298 3299 0 3328 3329 0
    3387 3388 0 3393 3396 0 3405 3405 0 3426 3427 0 3457 3457 0 3530 3530 0 3538 3540 0
    3542 3542 0 3633 3633 0 3636 3642 0 3655 3662 0 3761 3761 0 3764 3772 0 3784 3790 0
    3864 3865 0 3893 3893 0 3895 3895 0 3897 3897 0 3953 3966 0 3968 3972 0 3974 3975 0
    3981 3991 0 3993 4028 0 4038 4038 0 4141 4144 0 4146 4151 0 4153 4154 0 4157 4158 0
    4184 4185 0 4190 4192 0 4209 4212 0 4226 4226 0 4229 4230 0 4237 4237 0 4253 4253 0
    4352 4447 2 4448 4607 0 4957 4959 0 5906 5908 0 5938 5939 0 5970 5971 0 6002 6003 0
    6068 6069 0 6071 6077 0 6086 6086 0 6089 6099 0 6109 6109 0 6155 6159 0 6277 6278 0
    6313 6313 0 6432 6434 0 6439 6440 0 6450 6450 0 6457 6459 0 6679 6680 0 6683 6683 0
    6742 6742 0 6744 6750 0 6752 6752 0 6754 6754 0 6757 6764 0 6771 6780 0 6783 6783 0
    6832 6862 0 6912 6915 0 6964 6964 0 6966 6970 0 6972 6972 0 6978 6978 0 7019 7027 0
    7040 7041 0 7074 7077 0 7080 7081 0 7083 7085 0 7142 7142 0 7144 7145 0 7149 7149 0
    7151 7153 0 7212 7219 0 7222 7223 0 7376 7378 0 7380 7392 0 7394 7400 0 7405 7405 0
    7412 7412 0 7416 7417 0 7616 7679 0 8203 8207 0 8234 8238 0 8288 8292 0 8294 8303 0
    8400 8432 0 8986 8987 2 9001 9002 2 9193 9196 2 9200 9200 2 9203 9203 2 9725 9726 2
    9748 9749 2 9776 9783 2 9800 9811 2 9855 9855 2 9866 9871 2 9875 9875 2 9889 9889 2
    9898 9899 2 9917 9918 2 9924 9925 2 9934 9934 2 9940 9940 2 9962 9962 2 9970 9971 2
    9973 9973 2 9978 9978 2 9981 9981 2 9989 9989 2 9994 9995 2 10024 10024 2 10060 10060 2
    10062 10062 2 10067 10069 2 10071 10071 2 10133 10135 2 10160 10160 2 10175 10175 2
    11035 11036 2 11088 11088 2 11093 11093 2 11503 11505 0 11647 11647 0 11744 11775 0
    11904 11929 2 11931 12019 2 12032 12245 2 12272 12329 2 12330 12333 0 12334 12350 2
    12353 12438 2 12441 12442 0 12443 12543 2 12549 12591 2 12593 12686 2 12688 12773 2
    12783 12830 2 12832 12871 2 12880 42124 2 42128 42182 2 42607 42610 0 42612 42621 0
    42654 42655 0 42736 42737 0 43010 43010 0 43014 43014 0 43019 43019 0 43045 43046 0
    43052 43052 0 43204 43205 0 43232 43249 0 43263 43263 0 43302 43309 0 43335 43345 0
    43360 43388 2 43392 43394 0 43443 43443 0 43446 43449 0 43452 43453 0 43493 43493 0
    43561 43566 0 43569 43570 0 43573 43574 0 43587 43587 0 43596 43596 0 43644 43644 0
    43696 43696 0 43698 43700 0 43703 43704 0 43710 43711 0 43713 43713 0 43756 43757 0
    43766 43766 0 44005 44005 0 44008 44008 0 44013 44013 0 44032 55203 2 63744 64255 2
    64286 64286 0 65024 65039 0 65040 65049 2 65056 65071 0 65072 65106 2 65108 65126 2
    65128 65131 2 65279 65279 0 65281 65376 2 65504 65510 2 65529 65531 0 66045 66045 0
    66272 66272 0 66422 66426 0 68097 68099 0 68101 68102 0 68108 68111 0 68152 68154 0
    68159 68159 0 68325 68326 0 68900 68903 0 68969 68973 0 69291 69292 0 69372 69375 0
    69446 69456 0 69506 69509 0 69633 69633 0 69688 69702 0 69744 69744 0 69747 69748 0
    69759 69761 0 69811 69814 0 69817 69818 0 69821 69821 0 69826 69826 0 69837 69837 0
    69888 69890 0 69927 69931 0 69933 69940 0 70003 70003 0 70016 70017 0 70070 70078 0
    70089 70092 0 70095 70095 0 70191 70193 0 70196 70196 0 70198 70199 0 70206 70206 0
    70209 70209 0 70367 70367 0 70371 70378 0 70400 70401 0 70459 70460 0 70464 70464 0
    70502 70508 0 70512 70516 0 70587 70592 0 70606 70606 0 70608 70608 0 70610 70610 0
    70625 70626 0 70712 70719 0 70722 70724 0 70726 70726 0 70750 70750 0 70835 70840 0
    70842 70842 0 70847 70848 0 70850 70851 0 71090 71093 0 71100 71101 0 71103 71104 0
    71132 71133 0 71219 71226 0 71229 71229 0 71231 71232 0 71339 71339 0 71341 71341 0
    71344 71349 0 71351 71351 0 71453 71453 0 71455 71455 0 71458 71461 0 71463 71467 0
    71727 71735 0 71737 71738 0 71995 71996 0 71998 71998 0 72003 72003 0 72148 72151 0
    72154 72155 0 72160 72160 0 72193 72202 0 72243 72248 0 72251 72254 0 72263 72263 0
    72273 72278 0 72281 72283 0 72330 72342 0 72344 72345 0 72752 72758 0 72760 72765 0
    72767 72767 0 72850 72871 0 72874 72880 0 72882 72883 0 72885 72886 0 73009 73014 0
    73018 73018 0 73020 73021 0 73023 73029 0 73031 73031 0 73104 73105 0 73109 73109 0
    73111 73111 0 73459 73460 0 73472 73473 0 73526 73530 0 73536 73536 0 73538 73538 0
    73562 73562 0 78896 78912 0 78919 78933 0 90398 90409 0 90413 90415 0 92912 92916 0
    92976 92982 0 94031 94031 0 94095 94098 0 94176 94179 2 94180 94180 0 94192 94193 2
    94208 100343 2 100352 101589 2 101631 101640 2 110576 110579 2 110581 110587 2
    110589 110590 2 110592 110882 2 110898 110898 2 110928 110930 2 110933 110933 2
    110948 110951 2 110960 111355 2 113821 113822 0 113824 113827 0 118528 118573 0
    118576 118598 0 119143 119145 0 119155 119170 0 119173 119179 0 119210 119213 0
    119362 119364 0 119552 119638 2 119648 119670 2 121344 121398 0 121403 121452 0
    121461 121461 0 121476 121476 0 121499 121503 0 121505 121519 0 122880 122886 0
    122888 122904 0 122907 122913 0 122915 122916 0 122918 122922 0 123023 123023 0
    123184 123190 0 123566 123566 0 123628 123631 0 124140 124143 0 124398 124399 0
    125136 125142 0 125252 125258 0 126980 126980 2 127183 127183 2 127374 127374 2
    127377 127386 2 127462 127490 2 127504 127547 2 127552 127560 2 127568 127569 2
    127584 127589 2 127744 127776 2 127789 127797 2 127799 127868 2 127870 127891 2
    127904 127946 2 127951 127955 2 127968 127984 2 127988 127988 2 127992 127994 2
    127995 127999 0 128000 128062 2 128064 128064 2 128066 128252 2 128255 128317 2
    128331 128334 2 128336 128359 2 128378 128378 2 128405 128406 2 128420 128420 2
    128507 128591 2 128640 128709 2 128716 128716 2 128720 128722 2 128725 128727 2
    128732 128735 2 128747 128748 2 128756 128764 2 128992 129003 2 129008 129008 2
    129292 129338 2 129340 129349 2 129351 129535 2 129648 129660 2 129664 129673 2
    129679 129734 2 129742 129756 2 129759 129769 2 129776 129784 2 131072 196605 2
    196608 262141 2 917505 917505 0 917536 917631 0 917760 917999 0])

;; The width of one code point, by a binary search of `width-ranges`.
(: (-> int int))
(defun (code-point-width cp)
  (if (and (>= cp 32) (< cp 127))
      1
      (searched-width cp)))

(: (-> int int))
(defun (searched-width cp)
  (let lp ((lo 0) (hi (- (/ (array-length width-ranges) 3) 1)))
    (if (> lo hi)
        1
        (let ()
          (def mid (/ (+ lo hi) 2))
          (cond
            ((< cp (ref width-ranges (* 3 mid))) (lp lo (- mid 1)))
            ((> cp (ref width-ranges (+ (* 3 mid) 1))) (lp (+ mid 1) hi))
            (else (ref width-ranges (+ (* 3 mid) 2))))))))

;; The cursor after the escape sequence that starts at `c`, an ESC: a CSI
;; ends at a character from @ to ~, an OSC at BEL or at ESC and a backslash,
;; and any other is ESC and one character.
(: (-> string StringCursor StringCursor))
(defun (after-escape s c)
  (def next (string-cursor-next s c))
  (defun (char-at k) (if (string-cursor-end? s k) 0 (char->int (string-cursor-ref s k))))
  (cond
    ((string-cursor-end? s next) next)
    ((= (char-at next) 91)
     (let lp ((k (string-cursor-next s next)))
       (cond
         ((string-cursor-end? s k) k)
         ((and (>= (char-at k) 64) (<= (char-at k) 126)) (string-cursor-next s k))
         (else (lp (string-cursor-next s k))))))
    ((= (char-at next) 93)
     (let lp ((k (string-cursor-next s next)))
       (cond
         ((string-cursor-end? s k) k)
         ((= (char-at k) 7) (string-cursor-next s k))
         ((and (= (char-at k) 27)
               (let ((k2 (string-cursor-next s k)))
                 (and (not (string-cursor-end? s k2)) (= (char-at k2) 92))))
          (string-cursor-next s (string-cursor-next s k)))
         (else (lp (string-cursor-next s k))))))
    (else (string-cursor-next s next))))

;; How many columns `s` takes in a terminal.
(: (-> string int))
(defun (terminal-width s)
  (let lp ((c (string-cursor-start s)) (total 0) (last 0) (joined? #f) (open-flag? #f))
    (if (string-cursor-end? s c)
        total
        (let ()
          (def cp (char->int (string-cursor-ref s c)))
          (def next (string-cursor-next s c))
          (cond
            ((= cp 27) (lp (after-escape s c) total last joined? open-flag?))
            ((= cp 8205) (lp next total last #t #f))
            ((= cp 65039)
             (if (= last 1) (lp next (+ total 1) 2 #f #f) (lp next total last #f #f)))
            (joined? (lp next total last #f #f))
            ((and (>= cp 127462) (<= cp 127487))
             (if open-flag? (lp next total last #f #f) (lp next (+ total 2) 2 #f #t)))
            (else
             (def w (code-point-width cp))
             (lp next (+ total w) (if (> w 0) w last) #f #f)))))))

;; ===========================================================================
;; A table as a literal
;; ===========================================================================
;;
;; `(show-table '(table (style rounded) header (column "Item") ... (rows ,rows)))`.
;; The grammar checks the literal where it is written; `table-block` lays out
;; the value with `tabular/rows`.

(def/grammar Table
  (Table (table (: options TableOptions) (: columns TableColumn ...+) (: rows (rows TableRow ...))))
  (TableOptions (:each (: style    (:? (style TableStyle)))
                       (: header   (:? 'header))
                       (: rules    (:? (rules TableRules)))
                       (: width    (:? (width int)))
                       (: overflow (:? (overflow Overflow)))))
  (TableColumn (column (: title string) (: options ColumnOptions)))
  (ColumnOptions (:each (: align (:? (align Align)))
                        (: width (:? (width ColumnWidth)))
                        (: kind  (:? ColumnKind))))
  (ColumnKind (:or (: KindNumber (number NumberFormat))
                   (: KindBytes 'bytes)
                   (: KindPercent (percent (:? (precision int))))
                   (: KindBar (bar BarFormat))))
  (NumberFormat (:each (: precision (:? (precision int)))
                       (: comma     (:? (comma int)))
                       (: signed    (:? 'signed))))
  (BarFormat (:each (: limit (:? (max double)))
                    (: size  (:? (width int)))))
  (TableRow (:or (: RowOf (row TableCell ...))
                 (: RowSpliced ,(Vec TableCell))
                 (: RowsSpliced ,(Vec (Vec TableCell)))))
  (TableCell (:or (: CellText string) (: CellInt int) (: CellNum double) (: CellBlocks ,(Vec Block)))))

;; A cell shown as it is.
(: (-> TableCell Block))
(defun (plain-cell c)
  (match c
    ((CellText s) (BStr s))
    ((CellInt n) (BInt n))
    ((CellNum x) (numeric x))
    ((CellBlocks bs) (each bs))))

(: (-> TableCell (Option double)))
(defun (cell-number c)
  (match c
    ((CellInt n) (Some (cast double n)))
    ((CellNum x) (Some x))
    (_ None)))

;; A cell shown as its column's kind says. A cell that is not a number is
;; shown as it is in any column.
(: (-> (Option ColumnKind) TableCell Block))
(defun (kind-cell kind c)
  (match (Tuple kind (cell-number c))
    ((Tuple (Some (KindNumber f)) (Some x))
     (numeric x #:precision (option-ref-or (record-ref f precision) -1)
                #:comma (option-ref-or (record-ref f comma) -1)
                #:signed (record-ref f signed)))
    ((Tuple (Some KindBytes) (Some x)) (bytes (cast long x)))
    ((Tuple (Some (KindPercent p)) (Some x)) (percent x #:precision (option-ref-or p 1)))
    ((Tuple (Some (KindBar b)) (Some x))
     (bar (/ x (option-ref-or (record-ref b limit) 1.0)) #:width (option-ref-or (record-ref b size) 10)))
    (_ (plain-cell c))))

;; A column's alignment: the one it says, or to the right for a column of
;; numbers, sizes or shares, and to the left for anything else.
(: (-> TableColumn Align))
(defun (column-align col)
  (def o (record-ref col options))
  (match (record-ref o align)
    ((Some a) a)
    (None (match (record-ref o kind)
            ((Some (KindBar _)) AlignLeft)
            ((Some _) AlignRight)
            (None AlignLeft)))))

;; The table's rows, a block a cell, each shown as its column's kind says.
(: (-> (Vec TableColumn) (Vec (Vec TableCell)) (Vec (Vec Block))))
(defun (table-cells cols rows)
  (vec-map (fun (row)
             {vecing (kind-cell (if (< i (vec-length cols))
                                    (record-ref (record-ref (vec-ref cols i) options) kind)
                                    None)
                                (vec-ref row i))
                     (:for i (range 0 (vec-length row)))})
           rows))

;; A table literal as a block, laid out as `tabular/rows` lays out a table.
(: (-> Table Block))
(defun (table-block t)
  (def* (o (record-ref t options))
        (cols (list->vec (record-ref t columns)))
        (rows (list-foldl (fun (r acc)
                            (match r
                              ((RowOf cells) (vec-add acc (list->vec cells)))
                              ((RowSpliced cells) (vec-add acc cells))
                              ((RowsSpliced more) (vec-append acc more))))
                          []
                          (record-ref t rows)))
        (laid (tabular/rows (table-cells cols rows)
                            #:titles (if (record-ref o header)
                                         (vec-map (fun (c) (BStr (record-ref c title))) cols)
                                         [])
                            #:style (option-ref-or (record-ref o style) StyleNone)
                            #:rules (option-ref-or (record-ref o rules) RulesHeader)
                            #:align (vec-map column-align cols)
                            #:widths (vec-map (fun (c) (option-ref-or (record-ref (record-ref c options) width)
                                                                      WidthAuto))
                                              cols)
                            #:overflow (option-ref-or (record-ref o overflow) OverflowPush))))
  (match (record-ref o width)
    ((Some w) (with-fmt current-width w [laid]))
    (None laid)))

;; Shows a table literal on the current output port. `#:sync` for `show`'s
;; reason.
(: show-table #:sync (-> Table void))
(defun (show-table t) (show [(table-block t)]))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/hashset.bjo — the unordered hash set.
;;
;; A CHAMP trie: a node holds up to 32 slots, picked by 5 bits of the
;; element's hash at its depth. A slot holds an element or a child.
;;
;; - The nodes are C# classes in BjolangRuntime/HashTries, one class for
;;   each size, so that a node is one object. `NodeOps` does the operations
;;   on one node. This module does everything that spans nodes.
;; - A leaf holds only elements. A branch holds elements and children. A
;;   collision node holds elements whose full hashes are equal.
;; - A child is never a leaf of one element: removal puts such an element in
;;   the parent's slot. The empty set is the empty leaf.
;; - A transient owns the nodes it copied, and edits those in place.

(export
  ;; The types.
  HashSet HashSetBuilder TransientHashSet HashSetCursor HashSetting

  ;; Sets
  hashset-empty hashset-add hashset-add-all hashset-remove hashset-remove-all
  hashset-contains? hashset-length hashset-empty? hashset-clear hashset-union
  hashset-map hashset-filter hashset-fold hashset-for-each hashset-iter hashset-any? hashset-find
  hashset-split
  ;; Conversions
  list->hashset hashset->list vec->hashset hashset->vec seq->hashset hashset->seq
  hashset->str
  ;; Builders — the same shape as ListBuilder
  hashsetbuilder-empty hashset->hashsetbuilder hashsetbuilder-add! hashsetbuilder-add-all!
  hashsetbuilder-length hashsetbuilder->hashset
  ;; Transients — the set's interface, in place
  transienthashset-empty hashset->transienthashset transienthashset->hashset
  transienthashset-add! transienthashset-add-all! transienthashset-remove!
  transienthashset-remove-all! transienthashset-contains? transienthashset-length
  transienthashset-empty?)

(import (std prelude))

(:doc #:file "hashset.bjodoc")

(import/class
  ((Node %a) (: Bjolang.Runtime.HashTries.Node))
  ;; What a set compares its elements with. See `eq-comparer`.
  ((EqualityComparer %a) (: System.Collections.Generic.IEqualityComparer)))

(import/extern
  (owner-next     (: Bjolang.Runtime.HashTries.NodeOps.OwnerNext     (-> long)))
  (empty-node     (: Bjolang.Runtime.HashTries.NodeOps.Empty         (-> (Node %a))))
  (node-kind      (: Bjolang.Runtime.HashTries.NodeOps.Kind          (-> (Node %a) int)))
  (node-size      (: Bjolang.Runtime.HashTries.NodeOps.Size          (-> (Node %a) int)))
  (child-count    (: Bjolang.Runtime.HashTries.NodeOps.ChildCount    (-> (Node %a) int)))
  (element        (: Bjolang.Runtime.HashTries.NodeOps.Element       (-> (Node %a) int %a)))
  (child          (: Bjolang.Runtime.HashTries.NodeOps.Child         (-> (Node %a) int (Node %a))))
  (element-map    (: Bjolang.Runtime.HashTries.NodeOps.ElementMap    (-> (Node %a) int)))
  (child-map      (: Bjolang.Runtime.HashTries.NodeOps.ChildMap      (-> (Node %a) int)))
  (collision-hash (: Bjolang.Runtime.HashTries.NodeOps.CollisionHash (-> (Node %a) int)))
  (same-node?     (: Bjolang.Runtime.HashTries.NodeOps.Same          (-> (Node %a) (Node %a) bool)))
  (fragment       (: Bjolang.Runtime.HashTries.NodeOps.Fragment      (-> int int int)))
  (hash-of        (: Bjolang.Runtime.HashTries.NodeOps.Hash          (-> (EqualityComparer %a) %a int)))
  (same-element?  (: Bjolang.Runtime.HashTries.NodeOps.Equal         (-> (EqualityComparer %a) %a %a bool)))
  (same-comparer? (: Bjolang.Runtime.HashTries.NodeOps.SameComparer  (-> (EqualityComparer %a) (EqualityComparer %a) bool)))
  (probe          (: Bjolang.Runtime.HashTries.NodeOps.Probe         (-> (Node %a) %a int int (EqualityComparer %a) int)))
  (element-at     (: Bjolang.Runtime.HashTries.NodeOps.ElementAt     (-> (Node %a) int int %a)))
  (with-element   (: Bjolang.Runtime.HashTries.NodeOps.WithElement   (-> (Node %a) %a int int long (Node %a))))
  (with-subtree   (: Bjolang.Runtime.HashTries.NodeOps.WithSubtree   (-> (Node %a) int int (Node %a) long (Node %a))))
  (with-child     (: Bjolang.Runtime.HashTries.NodeOps.WithChild     (-> (Node %a) int (Node %a) long (Node %a))))
  (leaf-of-one    (: Bjolang.Runtime.HashTries.NodeOps.Single        (-> %a int int long (Node %a))))
  (leaf-of-two    (: Bjolang.Runtime.HashTries.NodeOps.Pair          (-> %a int %a int int long (Node %a))))
  (branch-around  (: Bjolang.Runtime.HashTries.NodeOps.Wrap          (-> (Node %a) int int long (Node %a))))
  (branch-of-two  (: Bjolang.Runtime.HashTries.NodeOps.Fork          (-> %a int (Node %a) int int long (Node %a))))
  (collision-of-two (: Bjolang.Runtime.HashTries.NodeOps.Collision   (-> %a %a int long (Node %a))))
  (without-element (: Bjolang.Runtime.HashTries.NodeOps.WithoutElement
                      (-> (Node %a) %a int int (EqualityComparer %a) long (Node %a))))
  (without-child  (: Bjolang.Runtime.HashTries.NodeOps.WithoutChild  (-> (Node %a) int int int long (Node %a))))
  (with-child-inlined (: Bjolang.Runtime.HashTries.NodeOps.WithChildInlined
                         (-> (Node %a) int %a int int long (Node %a))))
  (assemble       (: Bjolang.Runtime.HashTries.NodeOps.Assemble
                     (-> int (Array %a) int (Array (Node %a)) int long (Node %a))))
  (collision-of   (: Bjolang.Runtime.HashTries.NodeOps.CollisionOf   (-> (Array %a) int int long (Node %a)))))

;; The kinds `node-kind` returns.
(def leaf-kind 0)
(def collision-kind 2)

;; The owner of the nodes no transient owns. They are never edited in place.
(: no-owner long)
(def no-owner 0)

(: (-> (Node %a) bool))
(defun (leaf? n) (= (node-kind n) leaf-kind))

(: (-> (Node %a) bool))
(defun (collision? n) (= (node-kind n) collision-kind))

;; The elements under `n`. Nodes hold no counts, so this visits every node.
(: (-> (Node %a) int))
(defun (count-under n)
  (loop (:for i (range 0 (child-count n)))
        (:acc c (folding (node-size n) (+ c (count-under (child n i)))))
        => c))

;; ---------------------------------------------------------------------------
;; The set
;; ---------------------------------------------------------------------------

;; - `count` is -1 for a part made by `hashset-split` until it is first asked
;;   for, so that the part counts itself in its own fiber. It is written at
;;   most once, with the same value by every thread that writes it.
(type (: (HashSet %a) #:opaque
         (Record (: root (Node %a))
                 (: count int #:mutable)
                 (: comparer (EqualityComparer %a)))))

(: (-> (EqualityComparer %a) (HashSet %a)))
(defun (empty-with cmp) (HashSet (root (empty-node)) (count 0) (comparer cmp)))

;; What makes a set out of nothing asks for the element's Eq, and the set
;; compares elements with it — the same answer `=` gives, also for two cases of
;; a union that `=` equates. One made from a set keeps that set's.
(: (-> (HashSet %a)) (where (Eq %a)))
(defun (hashset-empty) (empty-with (eq-comparer)))

;; An empty set that compares elements as `s` does.
(: (-> (HashSet %a) (HashSet %a)))
(defun (hashset-clear s) (empty-with (.comparer s)))

(: (-> (HashSet %a) int))
(defun (hashset-length s)
  (def n (.count s))
  (if (>= n 0)
      n
      (begin
        (def counted (count-under (.root s)))
        (record-set! s (count counted))
        counted)))

(: (-> (HashSet %a) bool))
(defun (hashset-empty? s) (= (hashset-length s) 0))

;; ---------------------------------------------------------------------------
;; Lookup
;; ---------------------------------------------------------------------------

(: (-> (Node %a) %a int int (EqualityComparer %a) bool))
(defun (contains-in? n x h shift cmp)
  (def c (probe n x h shift cmp))
  (if (>= c 0)
      (contains-in? (child n c) x h (+ shift 5) cmp)
      (= c -1)))

(: (-> (HashSet %a) %a bool))
(defun (hashset-contains? s x)
  (def cmp (.comparer s))
  (contains-in? (.root s) x (hash-of cmp x) 0 cmp))

;; ---------------------------------------------------------------------------
;; Add and remove
;; ---------------------------------------------------------------------------

;; A node at `shift` of the two different elements `a` and `b`: a leaf if
;; their fragments differ there, a collision node if their hashes are equal,
;; else a branch around the node one level down.
(: (-> %a int %a int int long (Node %a)))
(defun (join a ha b hb shift owner)
  (cond
    ((= ha hb) (collision-of-two a b ha owner))
    ((= (fragment ha shift) (fragment hb shift))
     (branch-around (join a ha b hb (+ shift 5) owner) ha shift owner))
    (else (leaf-of-two a ha b hb shift owner))))

;; The collision node `col`, at `shift`, and `x`, whose hash differs from its
;; elements' hash, under one node at `shift`.
(: (-> (Node %a) %a int int long (Node %a)))
(defun (split-collision col x hx shift owner)
  (def hc (collision-hash col))
  (if (= (fragment hc shift) (fragment hx shift))
      (branch-around (split-collision col x hx (+ shift 5) owner) hc shift owner)
      (branch-of-two x hx col hc shift owner)))

;; Adds `x` to the subtree `n` at `shift`. Returns the new subtree, and
;; whether `x` is new. An equal element already there is kept, and `n`
;; itself returned.
(: (-> (Node %a) %a int int (EqualityComparer %a) long (Tuple (Node %a) bool)))
(defun (insert n x h shift cmp owner)
  (def c (probe n x h shift cmp))
  (cond
    ((>= c 0)
     (def ch (child n c))
     (def (Tuple r added?) (insert ch x h (+ shift 5) cmp owner))
     (Tuple (if (same-node? r ch) n (with-child n c r owner)) added?))
    ((= c -1) (Tuple n #f))
    ((= c -2) (Tuple (with-element n x h shift owner) #t))
    ((collision? n) (Tuple (split-collision n x h shift owner) #t))
    (else
     (def y (element-at n h shift))
     (Tuple (with-subtree n h shift (join y (hash-of cmp y) x h (+ shift 5) owner) owner) #t))))

;; Removes the element equal to `x` from the subtree `n` at `shift`. Returns
;; the new subtree, and whether there was one. A child left empty is
;; dropped, and a child left with one element is put in its parent's slot.
(: (-> (Node %a) %a int int (EqualityComparer %a) long (Tuple (Node %a) bool)))
(defun (delete n x h shift cmp owner)
  (def c (probe n x h shift cmp))
  (cond
    ((= c -1) (Tuple (without-element n x h shift cmp owner) #t))
    ((< c 0) (Tuple n #f))
    (else
     (def ch (child n c))
     (def (Tuple r removed?) (delete ch x h (+ shift 5) cmp owner))
     (cond
       ((not removed?) (Tuple n #f))
       ((and (leaf? r) (= (node-size r) 0)) (Tuple (without-child n c h shift owner) #t))
       ((and (leaf? r) (= (node-size r) 1)) (Tuple (with-child-inlined n c (element r 0) h shift owner) #t))
       ((same-node? r ch) (Tuple n #t))
       (else (Tuple (with-child n c r owner) #t))))))

(: (-> (HashSet %a) %a (HashSet %a)))
(defun (hashset-add s x)
  (def cmp (.comparer s))
  (def (Tuple r added?) (insert (.root s) x (hash-of cmp x) 0 cmp no-owner))
  (if added?
      (HashSet (root r) (count (+ (hashset-length s) 1)) (comparer cmp))
      s))

(: (-> (HashSet %a) %a (HashSet %a)))
(defun (hashset-remove s x)
  (def cmp (.comparer s))
  (def (Tuple r removed?) (delete (.root s) x (hash-of cmp x) 0 cmp no-owner))
  (if removed?
      (HashSet (root r) (count (- (hashset-length s) 1)) (comparer cmp))
      s))

(: (-> (HashSet %a) (Seq %a) (HashSet %a)))
(defun (hashset-add-all s xs)
  (def t (hashset->transienthashset s))
  (transienthashset-add-all! t xs)
  (transienthashset->hashset t))

(: (-> (HashSet %a) (Seq %a) (HashSet %a)))
(defun (hashset-remove-all s xs)
  (def t (hashset->transienthashset s))
  (transienthashset-remove-all! t xs)
  (transienthashset->hashset t))

;; ---------------------------------------------------------------------------
;; Transients
;; ---------------------------------------------------------------------------

;; A set that edits in place the nodes it copied.
;; `transienthashset->hashset` returns a set of its contents and gives it a
;; new owner, so that the set does not change after.
(type (: (TransientHashSet %a) #:opaque
         (Record (: root (Node %a) #:mutable)
                 (: count int #:mutable)
                 (: owner long #:mutable)
                 (: comparer (EqualityComparer %a)))))

(: (-> (TransientHashSet %a)) (where (Eq %a)))
(defun (transienthashset-empty) (hashset->transienthashset (hashset-empty)))

(: (-> (HashSet %a) (TransientHashSet %a)))
(defun (hashset->transienthashset s)
  (TransientHashSet (root (.root s)) (count (hashset-length s)) (owner (owner-next)) (comparer (.comparer s))))

(: (-> (TransientHashSet %a) (HashSet %a)))
(defun (transienthashset->hashset t)
  (record-set! t (owner (owner-next)))
  (HashSet (root (.root t)) (count (.count t)) (comparer (.comparer t))))

(: (-> (TransientHashSet %a) %a void))
(defun (transienthashset-add! t x)
  (def cmp (.comparer t))
  (def (Tuple r added?) (insert (.root t) x (hash-of cmp x) 0 cmp (.owner t)))
  (record-set! t (root r))
  (when added? (record-set! t (count (+ (.count t) 1)))))

(: (-> (TransientHashSet %a) %a void))
(defun (transienthashset-remove! t x)
  (def cmp (.comparer t))
  (def (Tuple r removed?) (delete (.root t) x (hash-of cmp x) 0 cmp (.owner t)))
  (when removed? (record-set! t (root r) (count (- (.count t) 1)))))

(: (-> (TransientHashSet %a) (Seq %a) void))
(defun (transienthashset-add-all! t xs)
  (loop (:for x xs) (:do (transienthashset-add! t x))))

(: (-> (TransientHashSet %a) (Seq %a) void))
(defun (transienthashset-remove-all! t xs)
  (loop (:for x xs) (:do (transienthashset-remove! t x))))

(: (-> (TransientHashSet %a) %a bool))
(defun (transienthashset-contains? t x)
  (def cmp (.comparer t))
  (contains-in? (.root t) x (hash-of cmp x) 0 cmp))

(: (-> (TransientHashSet %a) int))
(defun (transienthashset-length t) (.count t))

(: (-> (TransientHashSet %a) bool))
(defun (transienthashset-empty? t) (= (.count t) 0))

;; ---------------------------------------------------------------------------
;; Cursors
;; ---------------------------------------------------------------------------

;; A position in an iteration of a set, a struct, so that an iteration
;; allocates nothing. The elements come in the trie's order: a node's own
;; elements, then those under its children, in order.
;;
;; - The cursor is on element `i` of `node`, of `end` elements.
;; - `path` is the child index at each level above `node`, 5 bits a level
;;   above the low 4 bits, which hold the depth of `node`. `last-run` marks a
;;   node with no elements after it: a root that is a leaf.
;; - At the end of a node, `advance` follows `path` from the root again.
(type (: (HashSetCursor %a) #:opaque
         (Struct (: node (Node %a))
                 (: i int)
                 (: end int)
                 (: root (Node %a))
                 (: path long))))

(def depth-bits 4)
(def index-bits 5)
(: last-run long)
(def last-run -1)

(: (-> long int int))
(defun (index-at path level)
  (cast System.Int32 (bitwise-and (shift-right-logical path (+ depth-bits (* index-bits level))) 31)))

;; `path` with the index `index` at `level`, and the depth `depth`.
(: (-> long int int int long))
(defun (path-to path level index depth)
  (def shift (+ depth-bits (* index-bits level)))
  (def (: mask long) (bitwise-ior (shift-left 31 shift) 15))
  (bitwise-ior (bitwise-and path (bitwise-not mask))
               (shift-left (cast System.Int64 index) shift)
               (cast System.Int64 depth)))

(: (-> (Node %a) (HashSetCursor %a)))
(defun (cursor-of root)
  (if (leaf? root)
      (HashSetCursor (node root) (i 0) (end (node-size root)) (root root) (path last-run))
      (enter root root 0)))

;; The cursor on the first element of `n`, at `path`, or after it if it has
;; none.
(: (-> (Node %a) (Node %a) long (HashSetCursor %a)))
(defun (enter n root path)
  (def size (node-size n))
  (if (> size 0)
      (HashSetCursor (node n) (i 0) (end size) (root root) (path path))
      (advance root path)))

;; The cursor on the first element after the node at `path`: under its first
;; child, else under the next sibling of it or of its nearest ancestor that
;; has one. A done cursor if there is none.
(: (-> (Node %a) long (HashSetCursor %a)))
(defun (advance root path)
  (def depth (cast System.Int32 (bitwise-and path 15)))
  ;; The deepest level whose child index can move right, and its node.
  (def/mutable level -1)
  (def/mutable parent root)
  (def/mutable n root)
  (loop (:for d (range 0 depth))
        (:do (def i (index-at path d))
             (when (< (+ i 1) (child-count n))
               (set! level d)
               (set! parent n))
             (set! n (child n i))))
  (cond
    ((> (child-count n) 0) (enter (child n 0) root (path-to path depth 0 (+ depth 1))))
    ((>= level 0)
     (def next (+ (index-at path level) 1))
     (enter (child parent next) root (path-to path level next (+ level 1))))
    (else (HashSetCursor (node root) (i 0) (end 0) (root root) (path last-run)))))

(: (-> (HashSetCursor %a) bool))
(defun (cursor-done? c) (>= (.i c) (.end c)))

(: (-> (HashSetCursor %a) %a))
(defun (cursor-element c) (element (.node c) (.i c)))

(: (-> (HashSetCursor %a) (HashSetCursor %a)))
(defun (cursor-step c)
  (def i (+ (.i c) 1))
  (if (or (< i (.end c)) (= (.path c) last-run))
      (record-set c (i i))
      (advance (.root c) (.path c))))

(impl (Iterable (HashSet %a))
  (type %elem %a)
  (type %cursor (HashSetCursor %a))
  (defun (iterable-start s) (cursor-of (.root s)))
  (defun (iterable-done? s c) (cursor-done? c))
  (defun (iterable-current s c) (cursor-element c))
  (defun (iterable-next s c) (cursor-step c))
  (defun (count s) (hashset-length s)))

;; ---------------------------------------------------------------------------
;; Iteration and rebuilding
;; ---------------------------------------------------------------------------

;; Folds `f` over the elements in the trie's order. Recursive over the trie,
;; which reads each node once.
(: (-> (-?-> %a %s %s) %s (HashSet %a) %s))
(defun (hashset-fold f init s) (fold-node f init (.root s)))

(: (-> (-?-> %a %s %s) %s (Node %a) %s))
(defun (fold-node f acc n)
  (def own (loop (:for i (range 0 (node-size n)))
                 (:acc s (folding acc (f (element n i) s)))
                 => s))
  (loop (:for i (range 0 (child-count n)))
        (:acc s (folding own (fold-node f s (child n i))))
        => s))

(: (-> (-?-> %a void) (HashSet %a) void))
(defun (hashset-for-each f s)
  (loop (:for x s) (:do (f x))))

;; Calls `f` until it returns #f. Returns whether it never did.
(: (-> (-?-> %a bool) (HashSet %a) bool))
(defun (hashset-iter f s)
  (let go ((c (cursor-of (.root s))))
    (cond
      ((cursor-done? c) #t)
      ((f (cursor-element c)) (go (cursor-step c)))
      (else #f))))

(: (-> (-?-> %a bool) (HashSet %a) bool))
(defun (hashset-any? pred s)
  (let go ((c (cursor-of (.root s))))
    (cond
      ((cursor-done? c) #f)
      ((pred (cursor-element c)) #t)
      (else (go (cursor-step c))))))

;; The first element in the trie's order that `pred` accepts.
(: (-> (-?-> %a bool) (HashSet %a) (Option %a)))
(defun (hashset-find pred s)
  (let go ((c (cursor-of (.root s))))
    (cond
      ((cursor-done? c) None)
      (:def x (cursor-element c))
      ((pred x) (Some x))
      (else (go (cursor-step c))))))

;; `f` of each element, compared by `%b`'s Eq.
(: (-> (-?-> %a %b) (HashSet %a) (HashSet %b)) (where (Eq %b)))
(defun (hashset-map f s)
  (def b (builder-with (eq-comparer) (max 16 (hashset-length s))))
  (loop (:for x s) (:do (hashsetbuilder-add! b (f x))))
  (hashsetbuilder->hashset b))

;; The elements `keep?` accepts. Returns `s` when it accepts every element.
(: (-> (-?-> %a bool) (HashSet %a) (HashSet %a)))
(defun (hashset-filter keep? s)
  (def b (builder-with (.comparer s) (max 16 (hashset-length s))))
  (loop (:for x s) (:do (when (keep? x) (hashsetbuilder-add! b x))))
  (if (= (.count b) (hashset-length s)) s (hashsetbuilder->hashset b)))

;; ---------------------------------------------------------------------------
;; Union
;; ---------------------------------------------------------------------------

(: (-> int int bool))
(defun (has? map bit) (not (= (bitwise-and map bit) 0)))

;; The levels of leaves and branches: shifts 0 to 30. Below them are only
;; collision nodes.
(def levels 7)

;; The union of the subtrees `a` and `b`, both at `shift`. Of two equal
;; elements the one in `a` is kept. Returns the union, and how many elements
;; of `b` are in `a`. Returns `a` itself when `b` adds nothing to it.
;; - `xs` and `kids` hold 32 slots for each level, where a node at that level
;;   gathers its elements and children.
(: (-> (EqualityComparer %a) (Node %a) (Node %a) int (Array %a) (Array (Node %a)) (Tuple (Node %a) int)))
(defun (union-of cmp a b shift xs kids)
  (cond
    ((same-node? a b) (Tuple a (count-under a)))
    ((collision? a) (union-collision cmp a b shift))
    ((collision? b) (union-into-collision cmp a b shift))
    (else (union-nodes cmp a b shift xs kids))))

;; The union of the collision node `a` and the node `b`.
(: (-> (EqualityComparer %a) (Node %a) (Node %a) int (Tuple (Node %a) int)))
(defun (union-collision cmp a b shift)
  (def h (collision-hash a))
  (cond
    ((and (collision? b) (= (collision-hash b) h))
     (loop (:for i (range 0 (node-size b)))
           (:acc r (folding (Tuple a 0)
                            (begin
                              (def (Tuple n o) r)
                              (def y (element b i))
                              (if (contains-in? n y h shift cmp)
                                  (Tuple n (+ o 1))
                                  (Tuple (with-element n y h shift no-owner) o)))))
           => r))
    ((and (leaf? b) (= (node-size b) 1))
     (def y (element b 0))
     (def hy (hash-of cmp y))
     (cond
       ((not (= hy h)) (Tuple (split-collision a y hy shift no-owner) 0))
       ((contains-in? a y h shift cmp) (Tuple a 1))
       (else (Tuple (with-element a y h shift no-owner) 0))))
    (else
     ;; The elements of `a` into `b`, each replacing an equal one in `b`.
     (loop (:for i (range 0 (node-size a)))
           (:acc r (folding (Tuple b 0)
                            (begin
                              (def (Tuple n o) r)
                              (def x (element a i))
                              (def (Tuple without removed?) (delete n x h shift cmp no-owner))
                              (def (Tuple added-to _) (insert without x h shift cmp no-owner))
                              (Tuple added-to (if removed? (+ o 1) o)))))
           => r))))

;; The union of the leaf or branch `a` and the collision node `b`.
(: (-> (EqualityComparer %a) (Node %a) (Node %a) int (Tuple (Node %a) int)))
(defun (union-into-collision cmp a b shift)
  (def h (collision-hash b))
  (if (and (leaf? a) (= (node-size a) 1))
      (begin
        (def x (element a 0))
        (def hx (hash-of cmp x))
        (cond
          ((not (= hx h)) (Tuple (split-collision b x hx shift no-owner) 0))
          ((contains-in? b x h shift cmp)
           (def n (node-size b))
           (def (: xs (Array %a)) (make-array n))
           (loop (:for i (range 0 n))
                 (:do (def y (element b i))
                      (array-set! xs i (if (same-element? cmp x y) x y))))
           (Tuple (collision-of xs n h no-owner) 1))
          (else (Tuple (with-element b x h shift no-owner) 0))))
      ;; Insert keeps an equal element of `a`.
      (loop (:for i (range 0 (node-size b)))
            (:acc r (folding (Tuple a 0)
                             (begin
                               (def (Tuple n o) r)
                               (def (Tuple added-to added?) (insert n (element b i) h shift cmp no-owner))
                               (Tuple added-to (if added? o (+ o 1))))))
            => r)))

;; The union of two leaves or branches, slot by slot.
(: (-> (EqualityComparer %a) (Node %a) (Node %a) int (Array %a) (Array (Node %a)) (Tuple (Node %a) int)))
(defun (union-nodes cmp a b shift xs kids)
  (def ea (element-map a))
  (def ca (child-map a))
  (def eb (element-map b))
  (def cb (child-map b))
  (def down (+ shift 5))
  (def base (* 32 (/ shift 5)))
  (def/mutable ia 0)
  (def/mutable ja 0)
  (def/mutable ib 0)
  (def/mutable jb 0)
  ;; The next free slots of this level in `xs` and `kids`.
  (def/mutable ne base)
  (def/mutable nk base)
  (def/mutable em 0)
  (def/mutable cm 0)
  (def/mutable changed? #f)
  (def/mutable overlap 0)
  ;; The occupied slots, lowest first.
  (def/mutable m (bitwise-ior (bitwise-ior ea ca) (bitwise-ior eb cb)))
  (loop (:while (not (= m 0)))
        (:do (def bit (bitwise-and m (- 0 m)))
             (set! m (bitwise-xor m bit))
             (def ea? (has? ea bit))
             (def ca? (has? ca bit))
             (def eb? (has? eb bit))
             (def cb? (has? cb bit))
             (cond
               ((and ea? eb?)
                (def x (element a ia))
                (def y (element b ib))
                (if (same-element? cmp x y)
                    (begin (set! overlap (+ overlap 1))
                           (array-set! xs ne x) (set! ne (+ ne 1)) (set! em (bitwise-ior em bit)))
                    (begin
                      (set! changed? #t)
                      (array-set! kids nk (join x (hash-of cmp x) y (hash-of cmp y) down no-owner))
                      (set! nk (+ nk 1))
                      (set! cm (bitwise-ior cm bit)))))
               ((or ca? cb?)
                ;; A child on either side, and a child or an element on the other.
                (def left (cond (ca? (child a ja))
                                (ea? (begin (def x (element a ia)) (leaf-of-one x (hash-of cmp x) down no-owner)))
                                (else (child b jb))))
                (def right (cond (cb? (child b jb))
                                 (eb? (begin (def y (element b ib)) (leaf-of-one y (hash-of cmp y) down no-owner)))
                                 (else left)))
                (def (Tuple n o)
                  (if (and (or ca? ea?) (or cb? eb?))
                      (union-of cmp left right down xs kids)
                      (Tuple left 0)))
                (set! overlap (+ overlap o))
                (unless (and ca? (same-node? n (child a ja))) (set! changed? #t))
                (array-set! kids nk n)
                (set! nk (+ nk 1))
                (set! cm (bitwise-ior cm bit)))
               (ea? (array-set! xs ne (element a ia)) (set! ne (+ ne 1)) (set! em (bitwise-ior em bit)))
               (eb? (set! changed? #t)
                    (array-set! xs ne (element b ib)) (set! ne (+ ne 1)) (set! em (bitwise-ior em bit)))
               (else unit))
             (when ea? (set! ia (+ ia 1)))
             (when ca? (set! ja (+ ja 1)))
             (when eb? (set! ib (+ ib 1)))
             (when cb? (set! jb (+ jb 1)))))
  (if changed?
      (Tuple (assemble em xs cm kids base no-owner) overlap)
      (Tuple a overlap)))



;; The elements of `s` and of `other`. Of two equal elements the one in `s`
;; is kept, and the union compares elements as `s` does. Two sets that
;; compare alike share the nodes that only one of them has.
(: (-> (HashSet %a) (HashSet %a) (HashSet %a)))
(defun (hashset-union s other)
  (def cmp (.comparer s))
  (cond
    ((hashset-empty? other) s)
    ((not (same-comparer? cmp (.comparer other)))
     (def t (hashset->transienthashset s))
     (loop (:for x other) (:do (transienthashset-add! t x)))
     (transienthashset->hashset t))
    ((hashset-empty? s) other)
    (else
     (def (Tuple r overlap) (union-of cmp (.root s) (.root other) 0 (make-array (* 32 levels)) (make-array (* 32 levels))))
     (cond
       ((same-node? r (.root s)) s)
       ((same-node? r (.root other)) other)
       (else (HashSet (root r)
                      (count (- (+ (hashset-length s) (hashset-length other)) overlap))
                      (comparer cmp)))))))

;; ---------------------------------------------------------------------------
;; Builders
;; ---------------------------------------------------------------------------

;; Elements added with their hashes, made into a trie at the end by sorting
;; them on 5 bits of hash a level.
(type (: (HashSetBuilder %a) #:opaque
         (Record (: hashes (Array int) #:mutable)
                 (: elements (Array %a) #:mutable)
                 (: count int #:mutable)
                 (: comparer (EqualityComparer %a)))))

(: (-> (EqualityComparer %a) int (HashSetBuilder %a)))
(defun (builder-with cmp capacity)
  (HashSetBuilder (hashes (make-array capacity)) (elements (make-array capacity)) (count 0) (comparer cmp)))

(: (-> (HashSetBuilder %a)) (where (Eq %a)))
(defun (hashsetbuilder-empty) (builder-with (eq-comparer) 16))

;; A builder that holds the elements of `s`.
(: (-> (HashSet %a) (HashSetBuilder %a)))
(defun (hashset->hashsetbuilder s)
  (def b (builder-with (.comparer s) (max 16 (hashset-length s))))
  (loop (:for x s) (:do (hashsetbuilder-add! b x)))
  b)

(: (-> (HashSetBuilder %a) %a void))
(defun (hashsetbuilder-add! b x)
  (def n (.count b))
  (when (= n (array-length (.elements b)))
    (record-set! b (hashes (array-resize (.hashes b) (* 2 n))) (elements (array-resize (.elements b) (* 2 n)))))
  (array-set! (.hashes b) n (hash-of (.comparer b) x))
  (array-set! (.elements b) n x)
  (record-set! b (count (+ n 1))))

(: (-> (HashSetBuilder %a) (Seq %a) void))
(defun (hashsetbuilder-add-all! b xs)
  (loop (:for x xs) (:do (hashsetbuilder-add! b x))))

;; The number of elements added, an element added twice counted twice.
(: (-> (HashSetBuilder %a) int))
(defun (hashsetbuilder-length b) (.count b))

(: (-> (Array int) int int bool))
(defun (one-hash? hs from n)
  (def h (array-ref hs from))
  (loop (:for i (range (+ from 1) (+ from n)))
        (:acc ok (folding #t (and ok (= (array-ref hs i) h))))
        => ok))

;; The distinct elements of `xs` from `from`, `n` of them, into the start of
;; `out`. Of equal elements the last one is kept. Returns how many.
(: (-> (EqualityComparer %a) (Array %a) int int (Array %a) int))
(defun (distinct-into cmp xs from n out)
  (def/mutable k 0)
  (loop (:for i (range from (+ from n)))
        (:do (def x (array-ref xs i))
             (def at (let find ((j 0))
                       (cond ((= j k) -1)
                             ((same-element? cmp (array-ref out j) x) j)
                             (else (find (+ j 1))))))
             (if (< at 0)
                 (begin (array-set! out k x) (set! k (+ k 1)))
                 (array-set! out at x))))
  k)

;; Groups this small are sorted in place by insertion rather than counted
;; into 32 buckets.
(def small-group 12)

;; Where `build-node` sorts and gathers: 32 slots for each level.
(type (: (BuildScratch %a) #:opaque
         (Record (: counts (Array int))
                 (: starts (Array int))
                 (: ends (Array int))
                 (: own (Array %a))
                 (: kids (Array (Node %a))))))

;; The node at `shift` of the `n` entries of `hs` and `xs` from `from`, which
;; have at least two hashes. Sorts them into `to-hs` and `to-xs` on the
;; fragment at `shift`, and builds each group's child from there, with the
;; arrays swapped. Returns the node and how many elements it holds.
(: (-> (EqualityComparer %a) (Array int) (Array %a) (Array int) (Array %a) int int int (BuildScratch %a)
       (Tuple (Node %a) int)))
(defun (build-node cmp hs xs to-hs to-xs from n shift sc)
  (if (<= n small-group)
      (build-small cmp hs xs to-hs to-xs from n shift sc)
      (build-sorted cmp hs xs to-hs to-xs from n shift sc)))

;; `build-node` for a small group: sorts it in place on the fragment at
;; `shift`, then takes its runs of one fragment in order.
(: (-> (EqualityComparer %a) (Array int) (Array %a) (Array int) (Array %a) int int int (BuildScratch %a)
       (Tuple (Node %a) int)))
(defun (build-small cmp hs xs to-hs to-xs from n shift sc)
  (def end (+ from n))
  (loop (:for i (range (+ from 1) end))
        (:do (def h (array-ref hs i))
             (def x (array-ref xs i))
             (def f (fragment h shift))
             (def/mutable j i)
             (loop (:while (and (> j from) (> (fragment (array-ref hs (- j 1)) shift) f)))
                   (:do (array-set! hs j (array-ref hs (- j 1)))
                        (array-set! xs j (array-ref xs (- j 1)))
                        (set! j (- j 1))))
             (array-set! hs j h)
             (array-set! xs j x)))
  (def base (* 32 (/ shift 5)))
  (def own (.own sc))
  (def kids (.kids sc))
  (def/mutable ne base)
  (def/mutable nk base)
  (def/mutable em 0)
  (def/mutable cm 0)
  (def/mutable total 0)
  (def/mutable i from)
  (loop (:while (< i end))
        (:do (def f (fragment (array-ref hs i) shift))
             (def/mutable j (+ i 1))
             (loop (:while (and (< j end) (= (fragment (array-ref hs j) shift) f))) (:do (set! j (+ j 1))))
             (def c (- j i))
             (def bit (shift-left 1 f))
             (cond
               ((= c 1)
                (set! total (+ total 1))
                (array-set! own ne (array-ref xs i))
                (set! ne (+ ne 1))
                (set! em (bitwise-ior em bit)))
               ((one-hash? hs i c)
                (def (: group (Array %a)) (make-array c))
                (def k (distinct-into cmp xs i c group))
                (set! total (+ total k))
                (if (= k 1)
                    (begin (array-set! own ne (array-ref group 0)) (set! ne (+ ne 1)) (set! em (bitwise-ior em bit)))
                    (begin (array-set! kids nk (collision-of group k (array-ref hs i) no-owner))
                           (set! nk (+ nk 1))
                           (set! cm (bitwise-ior cm bit)))))
               (else
                (def (Tuple kid k) (build-node cmp hs xs to-hs to-xs i c (+ shift 5) sc))
                (set! total (+ total k))
                (array-set! kids nk kid)
                (set! nk (+ nk 1))
                (set! cm (bitwise-ior cm bit))))
             (set! i j)))
  (Tuple (assemble em own cm kids base no-owner) total))

;; `build-node` for a big group: counts it into 32 buckets on the fragment
;; at `shift` and moves it into `to-hs` and `to-xs` by bucket.
(: (-> (EqualityComparer %a) (Array int) (Array %a) (Array int) (Array %a) int int int (BuildScratch %a)
       (Tuple (Node %a) int)))
(defun (build-sorted cmp hs xs to-hs to-xs from n shift sc)
  (def base (* 32 (/ shift 5)))
  (def counts (.counts sc))
  (def starts (.starts sc))
  (def ends (.ends sc))
  (def own (.own sc))
  (def kids (.kids sc))
  (loop (:for f (range base (+ base 32))) (:do (array-set! counts f 0)))
  (loop (:for i (range from (+ from n)))
        (:do (def f (+ base (fragment (array-ref hs i) shift)))
             (array-set! counts f (+ (array-ref counts f) 1))))
  (def/mutable o from)
  (loop (:for f (range base (+ base 32)))
        (:do (array-set! starts f o)
             (array-set! ends f o)
             (set! o (+ o (array-ref counts f)))))
  (loop (:for i (range from (+ from n)))
        (:do (def f (+ base (fragment (array-ref hs i) shift)))
             (def p (array-ref ends f))
             (array-set! to-hs p (array-ref hs i))
             (array-set! to-xs p (array-ref xs i))
             (array-set! ends f (+ p 1))))
  (def/mutable ne base)
  (def/mutable nk base)
  (def/mutable em 0)
  (def/mutable cm 0)
  (def/mutable total 0)
  (loop (:for f (range 0 32))
        (:do (def c (array-ref counts (+ base f)))
             (def s (array-ref starts (+ base f)))
             (def bit (shift-left 1 f))
             (cond
               ((= c 0) unit)
               ((= c 1)
                (set! total (+ total 1))
                (array-set! own ne (array-ref to-xs s))
                (set! ne (+ ne 1))
                (set! em (bitwise-ior em bit)))
               ((one-hash? to-hs s c)
                (def (: group (Array %a)) (make-array c))
                (def k (distinct-into cmp to-xs s c group))
                (set! total (+ total k))
                (if (= k 1)
                    (begin (array-set! own ne (array-ref group 0)) (set! ne (+ ne 1)) (set! em (bitwise-ior em bit)))
                    (begin (array-set! kids nk (collision-of group k (array-ref to-hs s) no-owner))
                           (set! nk (+ nk 1))
                           (set! cm (bitwise-ior cm bit)))))
               (else
                (def (Tuple kid k) (build-node cmp to-hs to-xs hs xs s c (+ shift 5) sc))
                (set! total (+ total k))
                (array-set! kids nk kid)
                (set! nk (+ nk 1))
                (set! cm (bitwise-ior cm bit))))))
  (Tuple (assemble em own cm kids base no-owner) total))

(: (-> (HashSetBuilder %a) (HashSet %a)))
(defun (hashsetbuilder->hashset b)
  (def cmp (.comparer b))
  (def n (.count b))
  (def hs (.hashes b))
  (def xs (.elements b))
  (cond
    ((= n 0) (empty-with cmp))
    ((one-hash? hs 0 n)
     (def (: group (Array %a)) (make-array n))
     (def k (distinct-into cmp xs 0 n group))
     (def h (array-ref hs 0))
     (HashSet (root (if (= k 1) (leaf-of-one (array-ref group 0) h 0 no-owner) (collision-of group k h no-owner)))
              (count k)
              (comparer cmp)))
    (else
     ;; The build reorders the builder's entries, which leaves the builder
     ;; holding the same elements.
     (def sc (BuildScratch (counts (make-array (* 32 levels))) (starts (make-array (* 32 levels)))
                           (ends (make-array (* 32 levels))) (own (make-array (* 32 levels)))
                           (kids (make-array (* 32 levels)))))
     (def (Tuple root k) (build-node cmp hs xs (make-array n) (make-array n) 0 n 0 sc))
     (HashSet (root root) (count k) (comparer cmp)))))

;; ---------------------------------------------------------------------------
;; Splitting
;; ---------------------------------------------------------------------------

;; A slot of some node, for `hashset-split`: an element, or a child.
;; - `path` holds the fragments of the nodes above, 5 bits a level, the
;;   root's lowest. `depth` is the level of the node; 0 is the root.
;; - `node` is the child, or the node that holds the element.
;; - `index` is the element's index in `node`, or -1 for a child.
(type (: (SplitUnit %a) #:opaque
         (Struct (: path long)
                 (: node (Node %a))
                 (: depth int)
                 (: bit int)
                 (: index int))))

(: (-> (SplitUnit %a) bool))
(defun (unit-element? u) (>= (.index u) 0))

;; Whether the unit is a leaf or a branch, which can be replaced by its slots.
(: (-> (SplitUnit %a) bool))
(defun (expands? u) (and (not (unit-element? u)) (not (collision? (.node u)))))

;; The slots of `n`, at `depth` under `path`, onto `out`: its elements, then
;; its children, as iteration has them.
(: (-> (Node %a) int long (ListBuilder (SplitUnit %a)) void))
(defun (add-units n depth path out)
  (def/mutable m (element-map n))
  (def/mutable i 0)
  (loop (:while (not (= m 0)))
        (:do (listbuilder-add! out (SplitUnit (path path) (node n) (depth depth) (bit (trailing-zero-count m)) (index i)))
             (set! i (+ i 1))
             (set! m (bitwise-and m (- m 1)))))
  (set! m (child-map n))
  (set! i 0)
  (loop (:while (not (= m 0)))
        (:do (listbuilder-add! out (SplitUnit (path path) (node (child n i)) (depth depth) (bit (trailing-zero-count m)) (index -1)))
             (set! i (+ i 1))
             (set! m (bitwise-and m (- m 1))))))

(: (-> int int))
(defun (trailing-zero-count m)
  (let go ((m m) (k 0)) (if (= (bitwise-and m 1) 1) k (go (shift-right-logical m 1) (+ k 1)))))

;; The node at `depth` for the units from `from` below `to`, which all lie
;; at or below it on one path. A node of one element and no child, below the
;; root, is returned as a leaf of it with #t, for the parent's slot, as
;; removal would put it.
(: (-> (Array (SplitUnit %a)) int int int (Tuple (Node %a) bool)))
(defun (build-part units from to depth)
  (def (: own (Array %a)) (make-array 32))
  (def (: kids (Array (Node %a))) (make-array 32))
  ;; The slots by fragment first, then packed in fragment order.
  (def (: by-bit-x (Array %a)) (make-array 32))
  (def (: by-bit-n (Array (Node %a))) (make-array 32))
  (def/mutable em 0)
  (def/mutable cm 0)
  (let go ((i from))
    (when (< i to)
      (def u (array-ref units i))
      (if (= (.depth u) depth)
          (begin
            (if (unit-element? u)
                (begin (array-set! by-bit-x (.bit u) (element (.node u) (.index u)))
                       (set! em (bitwise-ior em (shift-left 1 (.bit u)))))
                (begin (array-set! by-bit-n (.bit u) (.node u))
                       (set! cm (bitwise-ior cm (shift-left 1 (.bit u))))))
            (go (+ i 1)))
          (begin
            (def f (fragment-of (.path u) depth))
            (def j (let scan ((j (+ i 1)))
                     (if (and (< j to)
                              (> (.depth (array-ref units j)) depth)
                              (= (fragment-of (.path (array-ref units j)) depth) f))
                         (scan (+ j 1))
                         j)))
            (def (Tuple piece inline?) (build-part units i j (+ depth 1)))
            (if inline?
                (begin (array-set! by-bit-x f (element piece 0))
                       (set! em (bitwise-ior em (shift-left 1 f))))
                (begin (array-set! by-bit-n f piece)
                       (set! cm (bitwise-ior cm (shift-left 1 f)))))
            (go j)))))
  (def/mutable ne 0)
  (def/mutable nk 0)
  (loop (:for f (range 0 32))
        (:do (def bit (shift-left 1 f))
             (when (has? em bit) (array-set! own ne (array-ref by-bit-x f)) (set! ne (+ ne 1)))
             (when (has? cm bit) (array-set! kids nk (array-ref by-bit-n f)) (set! nk (+ nk 1)))))
  (def n (assemble em own cm kids 0 no-owner))
  (Tuple n (and (> depth 0) (= cm 0) (= ne 1))))

(: (-> (ListBuilder %a) (Array %a)))
(defun (array-of b) (loop (:for x (listbuilder->list b)) (:acc (arraying x))))

(: (-> long int int))
(defun (fragment-of path depth)
  (cast System.Int32 (bitwise-and (shift-right-logical path (* 5 depth)) 31)))

;; About `k` sets that together hold each element of `s` once, for work on
;; them at the same time. They share the nodes of `s`; only the nodes on the
;; way down to a cut are new.
;;
;; - The units are the slots of the root. While a child holds more than
;;   count / k elements on average, every leaf and branch child is replaced
;;   by its own slots. A collision node stays one unit, of its known size.
;; - The parts are runs of units, cut where the running weight passes
;;   i * count / k. Hashes that are not spread make parts of different
;;   sizes, not wrong ones.
;; - A part's count is not known until it is first asked for.
(: (-> (HashSet %a) int (Array (HashSet %a))))
(defun (hashset-split s k)
  (def size (hashset-length s))
  (def root (.root s))
  (def cmp (.comparer s))
  (if (or (<= k 1) (<= size 1) (collision? root))
      (vec->array [s])
      (begin
        (def target (/ (cast double size) (cast double k)))
        (def (: units-of-root (ListBuilder (SplitUnit %a))) (listbuilder-empty))
        (add-units root 0 0 units-of-root)
        (: child-weight (-> (Array (SplitUnit %a)) (Tuple double int)))
        (defun (child-weight us)
          (def/mutable rest size)
          (def/mutable children 0)
          (loop (:for u us)
                (:do (cond ((unit-element? u) (set! rest (- rest 1)))
                           ((collision? (.node u)) (set! rest (- rest (node-size (.node u)))))
                           (else (set! children (+ children 1))))))
          (Tuple (if (= children 0) 0.0 (max 1.0 (/ (cast double rest) (cast double children)))) children))
        (def (Tuple units weight)
          (let expand ((us (array-of units-of-root)))
            (def (Tuple w children) (child-weight us))
            (if (or (<= w target) (= children 0))
                (Tuple us w)
                (begin
                  (def out (listbuilder-empty))
                  (loop (:for u us)
                        (:do (if (expands? u)
                                 (add-units (.node u) (+ (.depth u) 1)
                                            (bitwise-ior (.path u) (shift-left (cast System.Int64 (.bit u)) (* 5 (.depth u))))
                                            out)
                                 (listbuilder-add! out u))))
                  (expand (array-of out))))))
        (def weights (loop (:for u units)
                           (:acc (arraying (cond ((unit-element? u) 1.0)
                                                 ((collision? (.node u)) (cast double (node-size (.node u))))
                                                 (else weight))))))
        (def total (loop (:for w weights) (:acc (summing w))))
        (def n (array-length units))
        (def parts (min k n))
        (def out (listbuilder-empty))
        (def/mutable made 0)
        (def/mutable start 0)
        (def/mutable done 0.0)
        (loop (:for j (range 0 n))
              (:do (set! done (+ done (array-ref weights j)))
                   (when (or (= j (- n 1))
                             (and (< made (- parts 1))
                                  (>= done (/ (* total (cast double (+ made 1))) (cast double parts)))))
                     (def (Tuple part _) (build-part units start (+ j 1) 0))
                     (listbuilder-add! out (HashSet (root part) (count -1) (comparer cmp)))
                     (set! made (+ made 1))
                     (set! start (+ j 1)))))
        (array-of out))))

;; ---------------------------------------------------------------------------
;; Conversions
;; ---------------------------------------------------------------------------

(: (-> (Seq %a) (HashSet %a)) (where (Eq %a)))
(defun (seq->hashset xs)
  (def b (hashsetbuilder-empty))
  (hashsetbuilder-add-all! b xs)
  (hashsetbuilder->hashset b))

(: (-> (HashSet %a) (Seq %a)))
(defun (hashset->seq s) (->seq s))

(: (-> (List %a) (HashSet %a)) (where (Eq %a)))
(defun (list->hashset xs) (loop (:for x xs) (:acc (hash-setting x))))

(: (-> (HashSet %a) (List %a)))
(defun (hashset->list s) (->list s))

(: (-> (Vec %a) (HashSet %a)) (where (Eq %a)))
(defun (vec->hashset v) (loop (:for x v) (:acc (hash-setting x))))

(: (-> (HashSet %a) (Vec %a)))
(defun (hashset->vec s) (->vec s))

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------

(impl (Refable (HashSet %a))
  (type %index %a)
  (type %out bool)
  (defun (ref set key)
    (hashset-contains? set key)))

(impl (Refable (TransientHashSet %a))
  (type %index %a)
  (type %out bool)
  (defun (ref set key)
    (transienthashset-contains? set key)))

(impl (Collection (HashSet %a))
  (type %content %a)
  (defun (length s) (hashset-length s))
  (defun (empty? s) (hashset-empty? s)))

(type (: (HashSetting %a) (Union hash-setting)))

(impl (Collector (HashSetting %a))
  (where (Eq %a))
  (type %elem %a)
  (type %acc (HashSetBuilder %a))
  (type %out (HashSet %a))
  (defun (collector-init c) (hashsetbuilder-empty))
  (defun (collector-step c acc x) (hashsetbuilder-add! acc x) acc)
  (defun (collector-finish c acc) (hashsetbuilder->hashset acc)))

;; The mutable sinks, one element at a time. `Addable!` is the prelude's
;; trait; this module owns the types, which is what licenses the impls.
(impl (Addable! (HashSetBuilder %a))
      (type %elem %a)
      (defun (add! sb elem)
        (hashsetbuilder-add! sb elem)))

(impl (Addable! (TransientHashSet %a))
      (type %elem %a)
      (defun (add! ts elem)
        (transienthashset-add! ts elem)))

;; And the persistent set itself.
(impl (Addable (HashSet %a))
      (type %elem %a)
      (defun (add s elem)
        (hashset-add s elem)))

;; Equality for sets.
(impl (Eq (HashSet %a))
  (where (Eq %a))
  (defun (= a b)
    (and (= (hashset-length a) (hashset-length b))
         (hashset-iter (fun (x) (hashset-contains? b x)) a)))
  (defun (eq-hash s)
    (hashset-fold (fun (x h) (bitwise-xor h (eq-hash x))) 0 s)))

;; `#{a b c}`, in the order of the trie.
(impl (Written (HashSet %a))
  (where (Written %a))
  (defun (write-into s b) (elements-print-into "#{" s "}" b (fun (x b) (write-into x b)))))

(impl (Displayed (HashSet %a))
  (where (Displayed %a))
  (defun (display-into s b) (elements-print-into "#{" s "}" b (fun (x b) (display-into x b)))))

(: (-> (HashSet %a) string) (where (Displayed %a)))
(defun (hashset->str s) (displayed s))

(:deprecated hashset->str "use `displayed`")
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/http.bjo — HTTP requests.
;;
;; A wrapper over System.Net.Http
;; 
;; Two things are deliberately not what .NET does. A status code is a value,
;; never an exception — `EnsureSuccessStatusCode` is not imported and 404 is
;; not a failure. And `HttpClient.Timeout` is switched off, because a deadline
;; belongs to the CML system bjolang uses.

(export
  ;; The types
  Method Body Custom Pair Request Response Client

  ;; Writing a request
  request http-get http-post

  ;; Sending one
  fetch fetching

  ;; Reading the answer
  response-status response-ok? response-reason
  response-header response-headers
  response-text response-port response-bytes-port response-close

  ;; Scoped
  with-response

  ;; The client
  make-client current-http-client)

(import (std prelude))
(import (std syntax-match))
(import (only (text encodings) utf-8 encoding-named))
(import (only (std url) encode-query-part encode-form))

;; Every export is documented there; the module doc is too.
(:doc #:file "http.bjodoc")

;; ===========================================================================
;; Interop
;; ===========================================================================

(import/class
  (SocketsHandler (: System.Net.Http.SocketsHttpHandler (-> SocketsHandler)))
  (MessageHandler (: System.Net.Http.HttpMessageHandler))
  (Client         (: System.Net.Http.HttpClient (-> MessageHandler Client)))
  (Verb           (: System.Net.Http.HttpMethod (-> string Verb)))
  (Message        (: System.Net.Http.HttpRequestMessage (-> Verb Uri Message)))
  (Reply          (: System.Net.Http.HttpResponseMessage))
  (Content        (: System.Net.Http.HttpContent))
  (StringContent  (: System.Net.Http.StringContent (-> string Encoding StringContent)))
  (Headers        (: System.Net.Http.Headers.HttpHeaders))
  (RequestHeaders (: System.Net.Http.Headers.HttpRequestHeaders))
  (ReplyHeaders   (: System.Net.Http.Headers.HttpResponseHeaders))
  (ContentHeaders (: System.Net.Http.Headers.HttpContentHeaders))
  (Completion     (: System.Net.Http.HttpCompletionOption))
  (StatusCode     (: System.Net.HttpStatusCode))
  (Decompression  (: System.Net.DecompressionMethods))
  (Span           (: System.TimeSpan))
  (ClrStream      (: System.IO.Stream))
  (Uri            (: System.Uri (-> string Uri) #:exceptions (System.UriFormatException)))
  (Failure        (: System.Net.Http.HttpRequestException (-> string Failure)))
  (Object         (: System.Object))
  ;; How .NET hands over a header collection: one name, several values.
  ((Pairing %k %v) (: System.Collections.Generic.KeyValuePair)))

(import/extern
  ;; Enum values and other statics.
  (headers-read   (: System.Net.Http.HttpCompletionOption.ResponseHeadersRead #:get))
  (decompress-all (: System.Net.DecompressionMethods.All #:get))
  (no-timeout     (: System.Threading.Timeout.InfiniteTimeSpan #:get))
  ;; Bytes in a declared encoding as an input port: read as they are when
  ;; they are UTF-8, transcoded to UTF-8 otherwise.
  (text-port-over (: Bjolang.Runtime.InputPorts.Over (-> ClrStream Encoding InputPort)))
  ;; Named `span-seconds` rather than `seconds`: `std/prelude` exports a
  ;; `seconds` that answers milliseconds, for a deadline, and two bindings of
  ;; one name that differ only in what they return is the kind of collision
  ;; nothing reports usefully.
  (span-seconds   (: System.TimeSpan.FromSeconds (-> double Span)))

  ;; The client and the handler under it.
  (handler-lifetime!   (: System.Net.Http.SocketsHttpHandler.PooledConnectionLifetime
                          (-> SocketsHandler Span void) #:set))
  (handler-decompress! (: System.Net.Http.SocketsHttpHandler.AutomaticDecompression
                          (-> SocketsHandler Decompression void) #:set))
  (handler-redirects!  (: System.Net.Http.SocketsHttpHandler.AllowAutoRedirect
                          (-> SocketsHandler bool void) #:set))
  (client-timeout!     (: System.Net.Http.HttpClient.Timeout (-> Client Span void) #:set))

  ;; The call. Declared with two arguments after the receiver, so the overload
  ;; taking a third — the CancellationToken — is the one selected, and the
  ;; ambient token is filled in at the call site.
  (send (: System.Net.Http.HttpClient.SendAsync (-> Client Message Completion Reply) #:async))

  ;; Building a request.
  (url-scheme       (: System.Uri.Scheme (-> Uri string) #:get))
  (message-headers  (: System.Net.Http.HttpRequestMessage.Headers (-> Message RequestHeaders) #:get))
  (message-content! (: System.Net.Http.HttpRequestMessage.Content (-> Message Content void) #:set))
  (content-headers  (: System.Net.Http.HttpContent.Headers (-> Content ContentHeaders) #:get))
  (header-add!      (: System.Net.Http.Headers.HttpHeaders.TryAddWithoutValidation
                       (-> Headers string string bool)))
  (header-remove!   (: System.Net.Http.Headers.HttpHeaders.Remove (-> Headers string bool)))

  ;; Reading a reply.
  (reply-status   (: System.Net.Http.HttpResponseMessage.StatusCode (-> Reply StatusCode) #:get))
  (reply-reason   (: System.Net.Http.HttpResponseMessage.ReasonPhrase (-> Reply string) #:get))
  (reply-headers  (: System.Net.Http.HttpResponseMessage.Headers (-> Reply ReplyHeaders) #:get))
  (reply-content  (: System.Net.Http.HttpResponseMessage.Content (-> Reply Content) #:get))

  ;; The body, both ways. A `defbjouble` pair: the synchronous one parks a
  ;; thread, which is right for a plain `defun`, and the other suspends.
  (clr-stream       (: System.Net.Http.HttpContent.ReadAsStream (-> Content ClrStream)))
  (clr-stream-await (: System.Net.Http.HttpContent.ReadAsStreamAsync (-> Content ClrStream) #:async)))

;; Nothing in the language builds an `Exception`, and every rejection below is
;; this module's own rather than .NET's.
(: (-> string Exception))
(defun (http-error message)
  (cast Exception (Failure. message)))

;; .NET says null where it has nothing: a reason phrase over HTTP/2, a header
;; that is not there. A .NET string becomes a Bjolang one at the call, and a
;; null becomes "" there.
(: (-> string string))
(defun (text-or-empty s) s)


;; ===========================================================================
;; The client
;; ===========================================================================

;; One handler, one client, and a connection that does not live forever: a
;; pooled connection outlives DNS, so a client that never recycles keeps
;; talking to an address that stopped being right.
(: (-> (#:connection-seconds double) (#:redirects bool) Client))
(defun (make-client #:connection-seconds 120.0 #:redirects #t)
  (def handler (SocketsHandler.))
  (handler-lifetime! handler (span-seconds connection-seconds))
  (handler-decompress! handler decompress-all)
  (handler-redirects! handler redirects)
  (def client (Client. (cast MessageHandler handler)))
  ;; Off, on purpose. It applies to every call made through the client,
  ;; cannot be varied per request, and arrives as a cancellation that
  ;; cannot be told from a real one. Race a `(timeout d)` instead.
  (client-timeout! client no-timeout)
  client)

;; Parameterize another one in for a proxy, a test double or a second set of
;; connection settings.
(: (Param Client))
(def current-http-client (make-parameter (make-client)))


;; ===========================================================================
;; What you write
;; ===========================================================================

;; A name and a value: a header, a query parameter or a form field. Written
;; `("Accept" . "application/json")`.
(type (: Pair (Alias (Tuple string string))))

(type (: Method (Enum GET POST PUT PATCH DELETE HEAD)))

;; A body .NET has no name for: your own content type, your own text.
(type (: Custom (Record (: content-type string) (: text string))))

(type (: Body (Union Empty
                     (: Text string)
                     ;; Already serialized: there is no JSON writer to call yet,
                     ;; and inventing one here would be the wrong module. Named
                     ;; `JsonText` rather than `Json` because `(text json)`
                     (: JsonText string)
                     (: Form (Vec Pair))
                     (: Other Custom))))

(type (: Request (Record (: method Method)
                         (: url string)
                         (: query (Vec Pair))
                         (: headers (Vec Pair))
                         (: body Body))))

(: (-> Method string (#:query (Vec Pair)) (#:headers (Vec Pair)) (#:body Body) Request))
(defun (request method url #:query (vec-empty) #:headers (vec-empty) #:body Empty)
  (Request (method method) (url url) (query query) (headers headers) (body body)))

(: (-> string (#:query (Vec Pair)) (#:headers (Vec Pair)) Request))
(defun (http-get url #:query (vec-empty) #:headers (vec-empty))
  (request GET url #:query query #:headers headers))

(: (-> string Body (#:query (Vec Pair)) (#:headers (Vec Pair)) Request))
(defun (http-post url body #:query (vec-empty) #:headers (vec-empty))
  (request POST url #:query query #:headers headers #:body body))

(: (-> Method string))
(defun (method-name m)
  (match m
    (GET "GET")
    (POST "POST")
    (PUT "PUT")
    (PATCH "PATCH")
    (DELETE "DELETE")
    (HEAD "HEAD")))


;; ===========================================================================
;; Turning one into an HttpRequestMessage
;; ===========================================================================

;; Pairs as `encode` writes each name and value, joined with `&`. A query is
;; encoded by `encode-query-part` and a form by `encode-form`, from (std url):
;; both encode `&`, `=` and `+`, so a value cannot begin another pair.
(: (-> (-> string string) (Vec Pair) string))
(defun (encode-pairs encode pairs)
  (string-join (vec-map (fun (pair)
                          (match pair
                            ((Tuple name value) (str (encode name) "=" (encode value)))))
                        pairs)
               "&"))

(: (-> Request string))
(defun (full-url req)
  (def base (record-ref req url))
  (def query (record-ref req query))
  (cond
    ((empty? query) base)
    (else
     ;; The query goes before a fragment. After the `#` it would be part of
     ;; the fragment, which is not sent.
     (def parts (string-split base "#"))
     (def head (vec-ref parts 0))
     (def fragment (if (> (vec-length parts) 1) (str "#" (string-join (vec-slice parts 1 (- (vec-length parts) 1)) "#")) ""))
     ;; A url may already carry a query, in which case ours is more of it.
     (str head (if (string-contains? head "?") "&" "?")
          (encode-pairs encode-query-part query)
          fragment))))

;; Attaches a body. Helper function for the header functions below
(: (-> Message Body (Option Content)))
(defun (attach-body! message body)
  (match body
    (Empty None)
    ((Text text) (Some (attach! message text "text/plain; charset=utf-8")))
    ((JsonText text) (Some (attach! message text "application/json; charset=utf-8")))
    ((Form fields) (Some (attach! message (encode-pairs encode-form fields)
                                  "application/x-www-form-urlencoded")))
    ((Other custom) (Some (attach! message
                                   (record-ref custom text)
                                   (record-ref custom content-type))))))

(: (-> Message string string Content))
(defun (attach! message text content-type)
  (def content (cast Content (StringContent. text utf-8)))
  (message-content! message content)
  ;; StringContent has already put `text/plain` there.
  (ignore (header-remove! (cast Headers (content-headers content)) "Content-Type"))
  (ignore (header-add! (cast Headers (content-headers content)) "Content-Type" content-type))
  content)

;; Which end of the request a header belongs on is .NET's business, not the
;; caller's: `Accept` goes on the request and `Content-Type` on the content.
;; `TryAddWithoutValidation` returning false is how it says "the other one".
(: (-> Message (Option Content) Pair (Result Exception Unit)))
(defun (add-header! message content pair)
  (match pair
    ((Tuple name value)
     (if (header-add! (cast Headers (message-headers message)) name value)
         (Ok unit)
         (match content
           (None (Err (http-error
                       (str "'" name "' is a header about a body, and this request has none"))))
           ((Some body)
            ;; Remove first, or the content type set above is sent twice.
            (ignore (header-remove! (cast Headers (content-headers body)) name))
            (if (header-add! (cast Headers (content-headers body)) name value)
                (Ok unit)
                (Err (http-error (str "not a usable header name: " name))))))))))

(: (-> Message (Option Content) (Vec Pair) (Result Exception Unit)))
(defun (add-headers! message content headers)
  (loop (:for pair headers)
        (:acc outcome (folding (Ok unit)
                               (match outcome
                                 ((Err e) (Err e))
                                 ((Ok done) (add-header! message content pair)))))))

(: (-> Uri bool))
(defun (web-url? url)
  (def scheme (url-scheme url))
  (or (= scheme "http") (= scheme "https")))

(: (-> Request (Result Exception Message)))
(defun (message-of req)
  (match (Uri. (full-url req))
    ((Err e) (Err (http-error (str "not a url: " (record-ref req url)))))
    ((Ok url)
     ;; `new Uri` is happy with `file:///etc/passwd`, which is not something an
     ;; HTTP library should be sending anywhere.
     (if (not (web-url? url))
         (Err (http-error (str "not an http or https url: " (record-ref req url))))
         (let* ((message (Message. (Verb. (method-name (record-ref req method))) url))
                (content (attach-body! message (record-ref req body))))
           (match (add-headers! message content (record-ref req headers))
             ((Err e) (Err e))
             ((Ok added) (Ok message))))))))


;; ===========================================================================
;; The answer
;; ===========================================================================

;; Opaque: the CLR reply inside owns a live connection, and handing it out
;; would make closing it somebody else's problem by accident.
(type
  (: Response #:opaque
     (Record (: status int)
             (: reason string)
             (: headers (Map string string))
             (: raw Reply))))

(: (-> (Seq System.String) string))
(defun (join-values values)
  (string-join (loop (:for value values) (:acc all (vecing (clr-string->string value)))) ", "))

;; Makes both headers and into  lower-cased, folded with ", ". Header names are
;; case-insensitive on the wire 
;;
;; Set-Cookie is the exception the RFC makes and this does not: folded cookies
;; cannot be taken apart again. When something here reads cookies it needs a
;; call of its own.
(: (-> (Seq (Pairing System.String (Seq System.String))) (Map string string) (Map string string)))
(defun (collect-headers headers into)
  (loop (:for header headers)
        (:acc all (folding into
                           (map-set all
                                    (string-downcase (.-Key header))
                                    (join-values (.-Value header)))))))

(: (-> Reply Response))
(defun (response-of raw)
  (Response (status (cast int (reply-status raw)))
            (reason (text-or-empty (reply-reason raw)))
            (headers (collect-headers
                      (cast (Seq (Pairing System.String (Seq System.String))) (content-headers (reply-content raw)))
                      (collect-headers
                       (cast (Seq (Pairing System.String (Seq System.String))) (reply-headers raw))
                       (map-empty))))
            (raw raw)))

(: (-> Response int))
(defun (response-status r) (record-ref r status))

;; 2xx. Nothing here throws on a 404: which codes are failures is the caller's
;; business, and plenty of them are not.
(: (-> Response bool))
(defun (response-ok? r)
  (def code (record-ref r status))
  (and (>= code 200) (< code 300)))

;; Empty over HTTP/2, which has no reason phrase. Do not parse it.
(: (-> Response string))
(defun (response-reason r) (record-ref r reason))

(: (-> Response (Map string string)))
(defun (response-headers r) (record-ref r headers))

;; The name is lower-cased for you, so "Content-Type" and "content-type"
;; find the same header.
(: (-> Response string (Option string)))
(defun (response-header r name)
  (map-try-ref (record-ref r headers) (string-downcase name)))


;; ===========================================================================
;; The body
;; ===========================================================================

(: (-> Content ClrStream))
(defbjouble (body-stream content)
  (#:sync (clr-stream content))
  (#:bjo  (clr-stream-await content)))

;; What the server said the body was encoded as: any name or alias
;; `encoding-named` knows, and UTF-8 when it named none. A charset nothing
;; here knows is an error rather than a guess, since text decoded by the wrong
;; encoding is wrong without a word; `response-bytes-port` reads such a body.
(: (-> Response Encoding))
(defun (charset-of r)
  (match (response-header r "content-type")
    (None utf-8)
    ((Some content-type)
     (match (charset-name content-type)
       (None utf-8)
       ((Some name)
        (match (encoding-named name)
          ((Some found) found)
          (None
           (raise (http-error
                   (str "the response declares charset=" name
                        ", which is not an encoding (text encodings) knows. "
                        "response-bytes-port reads its bytes as they are"))))))))))

(: (-> string (Option string)))
(defun (charset-name content-type)
  (loop (:for part (string-split content-type ";"))
        (:acc found (folding None
                             (let ((trimmed (string-trim part)))
                               (if (string-starts-with? (string-downcase trimmed) "charset=")
                                   (Some (strip-quotes (string-trim (substring-after trimmed "="))))
                                   found))))))

(: (-> string string string))
(defun (substring-after s sep)
  (def parts (string-split s sep))
  (if (< (length parts) 2)
      ""
      (vec-ref parts 1)))

;; charset="utf-8" is as legal as charset=utf-8, and an encoding name has
;; no quotes of its own to protect.
(: (-> string string))
(defun (strip-quotes s)
  (string-replace s "\"" ""))

;; The body as a port, so that `read-line`, `port->seq`, `port->stream` and
;; `read-json` all work on it. Reading suspends inside a bjoroutine and blocks
;; outside one, exactly as reading a file does.
;;
;; Decoded by the charset the server declared, and only by it: a byte order
;; mark is not treated specially. A UTF-8 body is the port's bytes as they
;; arrived; another charset is transcoded to UTF-8 on the way in. A charset
;; that no encoding has raises.
(: (-> Response InputPort))
(defun (response-port r)
  (def body (reply-content (record-ref r raw)))
  (text-port-over (body-stream body) (charset-of r)))

;; The body's bytes as they arrived, whatever the charset says, as Racket's
;; HTTP client hands them over: for a body that is not text, or text to decode
;; yourself with `reencode-input-port` or `bytes->string`.
(: (-> Response InputPort))
(defun (response-bytes-port r)
  (def body (reply-content (record-ref r raw)))
  (text-port-over (body-stream body) utf-8))

(: (-> Response string))
(defun (response-text r)
  (read-all (response-port r)))

;; A response holds a connection until it is closed. `ignore` does this, so a
;; response dropped in statement position releases it rather than leaking it.
(: (-> Response void))
(defun (response-close r)
  (.Dispose (record-ref r raw)))

(impl (Discard Response)
  (defun (ignore r) (response-close r)))


;; ===========================================================================
;; Sending one
;; ===========================================================================

;; The request as a cml event: nothing is sent until the branch is reached, and a
;; branch that loses cancels the request it started.
;;
;; `guard` is not decoration. An `HttpRequestMessage` may be sent once, and an
;; event may be synced any number of times — every retry, every losing branch
;; that is tried again. Building the message inside the generator is what makes
;; the second sync work.
(: (-> Request (Event (Result Exception Response))))
(defun (fetching req)
  (guard
   (fun ()
     (match (message-of req)
       ((Err e) (always (Err e)))
       ((Ok message)
        (wrap (task->event (send (parameter-ref current-http-client) message headers-read))
              (fun (outcome)
                (match outcome
                  ((Ok raw) (Ok (response-of raw)))
                  ((Err e) (Err e))))))))))

;; Send it and wait for the answer, in whichever colour the caller is.
(: (-> Request (Result Exception Response)))
(defbjouble (fetch req)
  (#:sync (sync/blocking (fetching req)))
  (#:bjo  (sync (fetching req))))


;; ===========================================================================
;; Scoped
;; ===========================================================================

;; `(with-response (r (fetch (http-get url))) body ...)` — closes the response on
;; the way out, however the body leaves.
(def/macro (with-response form inject compare)
  (syntax-match form
    ((_ (name spec) first body ...)
     #'(let ()
         (def answer ,spec)
         (try
           (let ((,name answer))
             ,first
             ,@body)
           #:finally (match answer
                       ((Ok got) (response-close got))
                       ((Err failed) unit)))))
    (bad (syntax-error bad "with-response names a response and takes a body: (with-response (r (fetch (http-get url))) body ...)"))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/inbox.bjo — the door .NET code knocks on.
;;
;; A thread that is not a fiber — Kestrel's, a timer's, a `FileSystemWatcher`'s
;; — hands an item to an inbox and returns. A fiber receives it, with its own
;; dynamic environment: its handlers, its parameters, its cancellation scope.
;;
;; THE RULE FOR A LAMBDA .NET CALLS. It may call the functions in this module
;; and nothing else. Everything here that a foreign thread uses is an ordinary
;; `defun`, not a bjoroutine, and reads neither parameters nor effect handlers —
;; because a thread that is not a fiber has none. A `(log ...)` written in such
;; a lambda does not fail: it quietly goes to the default handler, even inside a
;; `with-handler`. Post the work, and do the logging in the fiber.
;;
;; See `BjolangRuntime/Cml/Inbox.cs` for how it is done, and the Readme's Inbox
;; section for what to write.

(export
  ;; Making one, and the policy it needs. `WhenFull` is transparent: its two
  ;; cases are written wherever a `make-inbox` is.
  WhenFull
  make-inbox

  ;; Putting something in. `inbox-post!` and `inbox-send` are safe from any
  ;; thread; `inbox-send-evt` is the fiber's form.
  inbox-post! inbox-send inbox-send/token inbox-send-evt

  ;; Taking it out, and the end of the stream.
  inbox-recv inbox-closed

  ;; Ending it.
  inbox-close! inbox-fail! inbox-abort!

  ;; What was lost, and subscribing to an `IObservable`.
  inbox-dropped-count inbox-observer make-observer IObserver

  ;; Request and reply.
  inbox-call inbox-call/token
  inbox-try-call inbox-try-call/token
  call-request call-reply! call-fail! call-abandoned

  ;; The token an `inbox-call/token` is given: a caller's own, such as
  ;; Kestrel's `RequestAborted`.
  CancellationToken)

(import (std prelude))

(:doc #:file "inbox.bjodoc")

(import/class
  (CancellationToken (: System.Threading.CancellationToken))
  ((IObserver %a)    (: System.IObserver)))

;; `Inbox`, `Call`, `Task` and `Event` are spelled by the compiler: they are
;; runtime types the language has a name for, like `Chan` and `Promise`. See
;; `DotNetInterop.genericTypeCorrespondence`.
(import/extern
  (clr-make-inbox    (: Bjoml.InboxModule.Make            (-> int bool (Inbox %a))))
  (clr-post          (: Bjoml.InboxModule.Post            (-> (Inbox %a) %a bool)))
  (clr-send          (: Bjoml.InboxModule.Send            (-> (Inbox %a) %a (Task Unit))))
  (clr-send/token    (: Bjoml.InboxModule.SendWithToken   (-> (Inbox %a) %a CancellationToken (Task Unit))))
  (clr-send-evt      (: Bjoml.InboxModule.SendEvent       (-> (Inbox %a) %a (Event Unit))))
  (clr-recv          (: Bjoml.InboxModule.Receive         (-> (Inbox %a) (Event %a))))
  (clr-closed        (: Bjoml.InboxModule.Closed          (-> (Inbox %a) (Event (Option Exception)))))
  (clr-close!        (: Bjoml.InboxModule.Close           (-> (Inbox %a) void)))
  (clr-fail!         (: Bjoml.InboxModule.Fail            (-> (Inbox %a) Exception void)))
  (clr-abort!        (: Bjoml.InboxModule.Abort           (-> (Inbox %a) void)))
  (clr-dropped-count (: Bjoml.InboxModule.DroppedCount    (-> (Inbox %a) int)))
  (clr-make-observer (: Bjoml.InboxModule.MakeObserver
                        (-> (-> %a void) (-> Exception void) (-> void) (IObserver %a))))

  (clr-call          (: Bjoml.InboxModule.CallInbox          (-> (Inbox (Call %q %r)) %q (Task %r))))
  (clr-call/token    (: Bjoml.InboxModule.CallInboxWithToken
                        (-> (Inbox (Call %q %r)) %q CancellationToken (Task %r))))
  (clr-try-call      (: Bjoml.InboxModule.TryCallInbox
                        (-> (Inbox (Call %q %r)) %q (Option (Task %r)))))
  (clr-try-call/token (: Bjoml.InboxModule.TryCallInboxWithToken
                        (-> (Inbox (Call %q %r)) %q CancellationToken (Option (Task %r)))))
  (clr-call-request  (: Bjoml.InboxModule.CallRequest      (-> (Call %q %r) %q)))
  (clr-call-reply!   (: Bjoml.InboxModule.CallReply        (-> (Call %q %r) %r bool)))
  (clr-call-fail!    (: Bjoml.InboxModule.CallFail         (-> (Call %q %r) Exception bool)))
  (clr-call-abandoned (: Bjoml.InboxModule.CallAbandoned   (-> (Call %q %r) (Event Unit)))))

;; ---------------------------------------------------------------------------
;; Making one
;; ---------------------------------------------------------------------------

;; What a full inbox does to an item `inbox-post!` or `inbox-call` brings.
;;
;; It says nothing about `inbox-send` or `inbox-send-evt`: those wait for room
;; and never drop anything.
(type (: WhenFull (Union DropNewest DropOldest)))

;; An inbox holding at most `capacity` queued items, with `capacity > 0`.
;;
;; There is no unbounded inbox and no default capacity, because an event source
;; faster than its consumer must not grow memory without limit — a
;; `FileSystemWatcher` during a large `git checkout` fires thousands of events.
;; The number is yours to pick: for an HTTP server it is how many requests may
;; wait before the answer is 503.
(: (-> int WhenFull (Inbox %a)))
(defun (make-inbox capacity policy)
  (clr-make-inbox capacity (match policy
                             (DropNewest #f)
                             (DropOldest #t))))

;; ---------------------------------------------------------------------------
;; Putting something in
;; ---------------------------------------------------------------------------

;; Hands `x` over without ever waiting. Safe from any thread, including one with
;; no fiber context at all.
;;
;; `#t` when the item was queued or handed straight to a waiting receiver. `#f`
;; when the inbox is closed, or full under `DropNewest`.
(: (-> (Inbox %a) %a bool))
(defun (inbox-post! ib x) (clr-post ib x))

;; The same handover for a caller that can wait, because it returns a `Task`:
;; the task completes once `x` is in. On a full inbox it stays pending until a
;; receiver makes room, which is what makes this real backpressure — each
;; waiting producer holds exactly one item.
;;
;; On a closed inbox, or an `inbox-abort!` while waiting, the task is cancelled.
(: (-> (Inbox %a) %a (Task Unit)))
(defun (inbox-send ib x) (clr-send ib x))

;; The same, with the caller's own token. If it fires while the send is waiting
;; for room, the item never enters the queue and the task is cancelled.
(: (-> (Inbox %a) %a CancellationToken (Task Unit)))
(defun (inbox-send/token ib x token) (clr-send/token ib x token))

;; The fiber's form of a send, as a CML event: it commits when there is room,
;; and in a `choose` that another branch wins, `x` is *not* queued.
;;
;; That is why a fiber gets an event rather than the task above. A task cannot
;; be withdrawn, so `(task->event (inbox-send ib x))` in a losing branch would
;; queue `x` anyway, later, with nobody expecting it.
;;
;; On a closed inbox it never commits. A fiber that may meet one should choose
;; it together with `inbox-closed`.
(: (-> (Inbox %a) %a (Event Unit)))
(defun (inbox-send-evt ib x) (clr-send-evt ib x))

;; ---------------------------------------------------------------------------
;; Taking it out
;; ---------------------------------------------------------------------------

;; A real CML event: in a `choose` that another branch wins, nothing is
;; consumed and the item stays for the next receiver, exactly as a channel
;; receive does.
;;
;; On a closed, empty inbox it never fires. `inbox-closed` is how the end is
;; noticed.
(: (-> (Inbox %a) (Event %a)))
(defun (inbox-recv ib) (clr-recv ib))

;; Fires once the inbox is closed *and* empty, and from then on commits
;; immediately on every `sync`. `None` after `inbox-close!` or `inbox-abort!`,
;; and `(Some e)` after `(inbox-fail! ib e)`.
;;
;; The usual receive loop is
;;
;;   (sync (choose (wrap (inbox-recv ib) Some)
;;                 (wrap (inbox-closed ib) (fun (_) None))))
;;
;; It is a separate event rather than an `Option` from `inbox-recv` so that code
;; which never closes its inbox — an HTTP worker stopped by cancelling its scope
;; — keeps the simple form.
(: (-> (Inbox %a) (Event (Option Exception))))
(defun (inbox-closed ib) (clr-closed ib))

;; ---------------------------------------------------------------------------
;; Ending it
;; ---------------------------------------------------------------------------

;; A graceful end. Later posts answer `#f` and later sends and calls get a
;; cancelled task, but what is already queued stays and receivers can still take
;; it; a sender already waiting for room still gets in. `inbox-closed` fires
;; once all of that has been received.
(: (-> (Inbox %a) void))
(defun (inbox-close! ib) (clr-close! ib))

;; The same as `inbox-close!`, but `inbox-closed` yields `(Some e)`.
;;
;; The error is kept beside the queue rather than in it, so no full policy can
;; drop the one item that says what went wrong.
(: (-> (Inbox %a) Exception void))
(defun (inbox-fail! ib e) (clr-fail! ib e))

;; A hard stop: closed, and everything queued dropped. Queued calls and waiting
;; `inbox-send` tasks are cancelled, a waiting `inbox-send-evt` never commits,
;; and `inbox-closed` then fires with `None` — or with the earlier error, if
;; `inbox-fail!` came first.
(: (-> (Inbox %a) void))
(defun (inbox-abort! ib) (clr-abort! ib))

;; ---------------------------------------------------------------------------
;; What was lost
;; ---------------------------------------------------------------------------

;; How many items the two dropping policies have thrown away, so that a program
;; can report its losses: "dropped 37 progress updates". `inbox-abort!` is not
;; counted — that is the program stopping, not the inbox overflowing.
(: (-> (Inbox %a) int))
(defun (inbox-dropped-count ib) (clr-dropped-count ib))

;; ---------------------------------------------------------------------------
;; Subscribing to an IObservable
;; ---------------------------------------------------------------------------

;; An `IObserver` made of three functions. Subscribing to an `IObservable`
;; needs an object implementing the interface, which Bjolang cannot write, and
;; base .NET has no `Subscribe(Action<T>)`.
;;
;; All three run on the observable's thread, so they follow the rule every
;; foreign callback does: post, close or fail, and nothing else.
(: (-> (-> %a void) (-> Exception void) (-> void) (IObserver %a)))
(defun (make-observer on-next on-error on-completed)
  (clr-make-observer on-next on-error on-completed))

;; The inbox as an observer, for `(.Subscribe observable (inbox-observer ib))`.
;; `OnNext` posts, so the inbox's policy applies; `OnError` fails it; and
;; `OnCompleted` closes it.
(: (-> (Inbox %a) (IObserver %a)))
(defun (inbox-observer ib)
  (make-observer (fun (x) (ignore (inbox-post! ib x)))
                 (fun (e) (inbox-fail! ib e))
                 (fun () (inbox-close! ib))))

;; ---------------------------------------------------------------------------
;; Request and reply
;; ---------------------------------------------------------------------------
;;
;; A request/reply inbox is an inbox whose items are `Call`s, so capacity, the
;; full policy, closing and receiving work exactly as they do for any other.
;;
;; A fiber that takes a `Call` must answer it — typically in `#:finally`, so
;; that a raise answers it too. Otherwise the .NET caller waits forever: a
;; browser request that never finishes.

;; Posts a call and hands back the task its caller waits on. Safe from any
;; thread. When the inbox is full the policy decides, and the task faults with a
;; `Bjoml.InboxFullException`; when it is closed, the task is cancelled.
(: (-> (Inbox (Call %q %r)) %q (Task %r)))
(defun (inbox-call ib request) (clr-call ib request))

;; The same, with the caller's own `CancellationToken` — Kestrel's
;; `RequestAborted`. What the token does depends on where the call is:
;;
;;   * still queued — it is removed and the task is cancelled, so no fiber ever
;;     sees a call nobody is waiting for;
;;   * already taken — the task is *not* completed. Only `call-reply!` and
;;     `call-fail!` complete it, and `call-abandoned` fires instead, for the
;;     receiver to do as it sees fit. This is what keeps an HTTP handler from
;;     writing into a context that has already been recycled into the next
;;     request.
(: (-> (Inbox (Call %q %r)) %q CancellationToken (Task %r)))
(defun (inbox-call/token ib request token) (clr-call/token ib request token))

;; The same two, with the refusal as an answer rather than as a fault on the
;; task: `None` when the inbox was full under `DropNewest` or closed, and
;; nothing was posted.
;;
;; This is `inbox-post!`'s `#f` for a call, and it is here because a faulted
;; task is the wrong shape for the caller who most needs to know. A .NET
;; callback that hands its task straight back — an HTTP middleware is the case
;; — never awaits it, so it cannot see the fault, and the exception goes to
;; whatever the framework does with an unhandled one. A web server that answers
;; 500 where it meant 503 is telling the client to give up rather than to come
;; back:
;;
;;     (match (inbox-try-call/token requests ctx (request-aborted ctx))
;;       ((Some answering) (cast Task answering))
;;       (None (set-status! (response-of ctx) 503) completed-task))
;;
;; Under `DropOldest` the answer is `(Some ...)`: this call went in, and it is
;; an *older* one whose task faults. `inbox-post!` means the same thing by `#t`.
(: (-> (Inbox (Call %q %r)) %q (Option (Task %r))))
(defun (inbox-try-call ib request) (clr-try-call ib request))

(: (-> (Inbox (Call %q %r)) %q CancellationToken (Option (Task %r))))
(defun (inbox-try-call/token ib request token) (clr-try-call/token ib request token))

;; What the caller asked.
(: (-> (Call %q %r) %q))
(defun (call-request c) (clr-call-request c))

;; Answers the call. The first completion wins and returns `#t`; a later one
;; returns `#f` and does nothing, so a timeout and a handler can race to answer
;; safely.
(: (-> (Call %q %r) %r bool))
(defun (call-reply! c value) (clr-call-reply! c value))

;; The same, with an exception for the caller to receive.
(: (-> (Call %q %r) Exception bool))
(defun (call-fail! c e) (clr-call-fail! c e))

;; Fires once the caller's token has fired *after* this call was taken. It never
;; fires for a call made without a token, and never for one cancelled while it
;; was still queued — nobody had it to be told.
(: (-> (Call %q %r) (Event Unit)))
(defun (call-abandoned c) (clr-call-abandoned c))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/interop.bjo — what .NET wants and Bjolang cannot write.
;;
;; A Bjolang function is a Func. A .NET API that wants an Action, an
;; EventHandler or a delegate type of its own gets one from `as-delegate`.
;; A parameter that wants null gets it from `clr-default`. The work is in the
;; runtime (`BjoInterop.cs`).
(import (std prelude))

(:doc #:file "interop.bjodoc")

(export Delegate ClrType
  as-delegate as-delegate/type
  add-event-handler! remove-event-handler!
  clr-default)

(import/class
  (Delegate (: System.Delegate))
  (ClrType  (: System.Type)))

(import/extern
  (clr-default           (: Bjolang.Runtime.BjoInterop.Default (-> %a)))
  (as-delegate           (: Bjolang.Runtime.BjoInterop.As      (-> %f %d)))
  (as-delegate/type      (: Bjolang.Runtime.BjoInterop.Convert (-> ClrType %f Delegate)))
  (add-event-handler!    (: Bjolang.Runtime.BjoInterop.AddEventHandler
                            (-> %t string %f Delegate)))
  (remove-event-handler! (: Bjolang.Runtime.BjoInterop.RemoveEventHandler
                            (-> %t string Delegate void))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/maths.bjo — Math functions.

;; Import Eq trait for `=`.
(import (std eq))

(:doc #:file "maths.bjodoc")

(export
  ;; Num
  abs sign min max clamp zero? even? odd?
  sqr positive? negative?
  ;; Integer division, and the integer functions
  quotient remainder modulo quotient/remainder
  gcd lcm expt integer-sqrt
  ;; Bits
  Bits popcount trailing-zeros leading-zeros
  ;; Doubles: what they are, and pi
  nan? infinite? finite? pi
  ;; Exponential and logarithmic
  cbrt exp pow loge log-base log2 log10
  ;; Rounding, and rounding to an int
  floor ceiling truncate round
  exact-floor exact-ceiling exact-truncate exact-round
  ;; Trigonometric
  sin cos tan asin acos atan atan2
  ;; Hyperbolic
  sinh cosh tanh asinh acosh atanh)

;; ---------------------------------------------------------------------------
;; Num
;; ---------------------------------------------------------------------------

;; `Num` interface. A number can be compared with `<`: `INumber` extends
;; `IComparisonOperators`, which is `Ordered`.
(def/trait (Num %a)
  (where (Ordered %a))
  (#:clr-constraint (System.Numerics.INumber %a))
  (: abs   (-> %a %a)        #:clr-member Abs)
  (: sign  (-> %a int)       #:clr-member Sign)
  (: min   (-> %a %a %a)     #:clr-member Min)
  (: max   (-> %a %a %a)     #:clr-member Max)
  (: clamp (-> %a %a %a %a)  #:clr-member Clamp)
  ;; Parity tests.
  (: zero? (-> %a bool)      #:clr-member IsZero)
  (: even? (-> %a bool)      #:clr-member IsEvenInteger)
  (: odd?  (-> %a bool)      #:clr-member IsOddInteger))


;; Todo: `min`/`max` over a sequence.

;; At any unit: the square of a length is an area.
(: (-> (num %a %u) (num %a %u^2)) (where (Num %a)))
(defun (sqr x) (* x x))

;; Not `INumber.IsPositive`, which holds for zero. At any unit: zero is a
;; number of every unit.
(: (-> (num %a %u) bool) (where (Num %a)))
(defun (positive? x) (> x 0))

(: (-> (num %a %u) bool) (where (Num %a)))
(defun (negative? x) (< x 0))

;; ---------------------------------------------------------------------------
;; Integer division
;; ---------------------------------------------------------------------------
;;
;; Racket's names. `quotient` and `remainder` are `/` and `%` at an integer
;; type, named, so that they can be passed and so that a reader sees which
;; convention is meant; `modulo` is the one the operators do not give. R7RS's
;; `truncate/` and `floor/` families are the same operations under other names.

(import/class
  (Exception (: System.Exception))
  (ArgumentOutOfRangeException (: System.ArgumentOutOfRangeException)))

(: (-> string string %a))
(defun (out-of-range parameter why)
  (raise (cast Exception (ArgumentOutOfRangeException. parameter why))))

(: (-> %a %a %a) (where (Integral %a)))
(defun (quotient a b) (/ a b))

(: (-> %a %a %a) (where (Integral %a)))
(defun (remainder a b) (% a b))

;; The remainder moved into the divisor's sign: a nonzero remainder of the
;; other sign is one divisor away from it.
(: (-> %a %a %a) (where (Integral %a)))
(defun (modulo a b)
  (let ((r (% a b)))
    (if (and (not (zero? r)) (not (= (< r 0) (< b 0))))
        (+ r b)
        r)))

(: (-> %a %a (Tuple %a %a)) (where (Integral %a)))
(defun (quotient/remainder a b)
  (Tuple (/ a b) (% a b)))

;; ---------------------------------------------------------------------------
;; The integer functions
;; ---------------------------------------------------------------------------

;; Euclid's algorithm, on the magnitudes.
(: (-> %a %a %a) (where (Integral %a)))
(defun (gcd a b)
  (let go ((x (abs a)) (y (abs b)))
    (if (zero? y) x (go y (% x y)))))

;; Divided before it is multiplied, so that the product overflows only when the
;; answer does.
(: (-> %a %a %a) (where (Integral %a)))
(defun (lcm a b)
  (if (or (zero? a) (zero? b))
      0
      (abs (* (/ a (gcd a b)) b))))

;; By squaring: one multiplication per bit of the exponent, and one more for
;; each bit that is set.
(: (-> %a int %a) (where (Integral %a)))
(defun (expt base n)
  (when (< n 0)
    (out-of-range "n" "expt: the exponent is negative, and an integer power has to be one."))
  (let go ((b base) (k n) (acc 1))
    (if (= k 0)
        acc
        (go (* b b) (shift-right k 1) (if (odd? k) (* acc b) acc)))))

;; Newton's method from above. The start, n/2 + 1, is at least the root for
;; every n of 2 or more, and nothing on the way adds past n, so it cannot
;; overflow even at the type's largest value.
(: (-> %a %a) (where (Integral %a)))
(defun (integer-sqrt n)
  (when (< n 0)
    (out-of-range "n" "integer-sqrt: the number is negative."))
  (if (< n 2)
      n
      (let go ((x (+ (/ n 2) 1)))
        (let ((y (/ (+ x (/ n x)) 2)))
          (if (>= y x) x (go y))))))

;; ---------------------------------------------------------------------------
;; Bits
;; ---------------------------------------------------------------------------

;; The bits of an integer, answered by .NET's generic math for every integer
;; type: `(popcount 13)` is 3 and `(popcount 13L)` is 3L.
(def/trait (Bits %a)
  (#:clr-constraint (System.Numerics.IBinaryInteger %a))
  (: popcount (-> %a %a) #:clr-member PopCount)
  (: trailing-zeros (-> %a %a) #:clr-member TrailingZeroCount)
  (: leading-zeros (-> %a %a) #:clr-member LeadingZeroCount))

;; ---------------------------------------------------------------------------
;; The double functions
;; ---------------------------------------------------------------------------

;; What a double is.
(import/extern
  (nan?      (: System.Double.IsNaN))
  (infinite? (: System.Double.IsInfinity))
  (finite?   (: System.Double.IsFinite))
  (pi        (: System.Math.PI #:get)))

;; Exponential and logarithmic.
;;
;; One name per base, and none of them overloaded. `System.Math.Log` is two
;; methods rather than one — the natural logarithm and the log to a given base —
;; and binding both to `log` made the meaning depend on how many arguments the
;; call happened to pass. `loge` completes `log2` and `log10`, and `log-base`
;; says what its second argument is.
(import/extern
  (clr-cbrt (: System.Math.Cbrt (-> double double)))
  (exp      (: System.Math.Exp))
  (pow      (: System.Math.Pow))
  (loge     (: System.Math.Log (-> double double)))
  (log-base (: System.Math.Log (-> double double double)))
  (log2     (: System.Math.Log2))
  (log10    (: System.Math.Log10)))

;; The cube root of a number in m^3 is in m.
(: cbrt (-> (double %u^3) (double %u)))
(defun (cbrt x) (cast (double %u) (clr-cbrt (cast double x))))

;; Rounding.
(import/extern
  (floor    (: System.Math.Floor))
  (ceiling  (: System.Math.Ceiling))
  (truncate (: System.Math.Truncate))
  (round    (: System.Math.Round)))

;; Rounding to an int. `Convert.ToInt32` refuses NaN, the infinities and
;; anything out of an int's range with an OverflowException, where a cast
;; would answer a number. The value it is given is already whole, so its own
;; rounding never applies.
(import/extern
  (double->int (: System.Convert.ToInt32 (-> double int))))

(: (-> double int))
(defun (exact-floor x) (double->int (floor x)))

(: (-> double int))
(defun (exact-ceiling x) (double->int (ceiling x)))

(: (-> double int))
(defun (exact-truncate x) (double->int (truncate x)))

(: (-> double int))
(defun (exact-round x) (double->int (round x)))

;; Trigonometric.
(import/extern
  (sin   (: System.Math.Sin))
  (cos   (: System.Math.Cos))
  (tan   (: System.Math.Tan))
  (asin  (: System.Math.Asin))
  (acos  (: System.Math.Acos))
  (atan  (: System.Math.Atan))
  (atan2 (: System.Math.Atan2)))

;; Hyperbolic.
(import/extern
  (sinh  (: System.Math.Sinh))
  (cosh  (: System.Math.Cosh))
  (tanh  (: System.Math.Tanh))
  (asinh (: System.Math.Asinh))
  (acosh (: System.Math.Acosh))
  (atanh (: System.Math.Atanh)))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/monad.bjo
;;
;; Monad trait, stock instances, and (do ...) macro.
(import (std prelude))
(import (std syntax-match))

(:doc #:file "monad.bjodoc")

(export pure bind)

;; Monad trait definition.
(def/trait (Monad (%m %a))
  (: pure (-> %a (%m %a)))
  (: bind (-> (%m %a) (-?-> %a (%m %b)) (%m %b))))

;; `k` is handed to `list-map` rather than called inside a lambda. A lambda's
;; text is fixed and its colour is not — `(fun ...)` is wrong in the suspending
;; copy and `(bjoroutine ...)` in the ordinary one — so a callback written here
;; could only ever be one of them.
(impl (Monad (List %a))
  (defun (pure x) (Cons x Nil))
  (defun (bind m k)
    (list-foldr (fun (xs ys) (list-append xs ys)) Nil (list-map k m))))

(impl (Monad (Option %a))
  (defun (pure x) (Some x))
  (defun (bind m k)
    (match m
      ((Some v) (k v))
      (None None))))

;; Short-circuits on Err.
(impl (Monad (Result %e %a))
  (defun (pure x) (Ok x))
  (defun (bind m k)
    (match m
      ((Ok v) (k v))
      ((Err e) (Err e)))))

;; ---------------------------------------------------------------------------
;; The block
;; ---------------------------------------------------------------------------
;; Macro for (do ...) blocks.
(def/macro (do form inject compare)
  (syntax-match form
    ((_ step ...) (do-block step form))
    (bad (syntax-error bad "do takes a block of forms: (do (:bind x m) ... (:return e))"))))

;; Builds the block chain.
(: (-> (List Syntax) Syntax Syntax))
(defun (do-block steps whole)
  (match steps
    (Nil (syntax-error whole "A (do ...) block is empty. It has to answer something."))
    ((Cons step Nil) (do-last step))
    ((Cons step rest) (do-step step (do-block rest whole)))))

(: (-> Syntax bool))
(defun (identifier? s)
  (match s
    ((SSym _) #t)
    (_ #f)))

;; Checks if form is keyword-headed.
(: (-> Syntax bool))
(defun (clause? s)
  (match s
    ((SList (Cons head _))
     (match head
       ((SKey _) #t)
       (_ #f)))
    (_ #f)))

(: (-> Syntax Syntax))
(defun (unknown-clause step)
  (syntax-error step "This is not a (do ...) form. They are :bind, :match, :let, :then and :return."))

;; The block's last form, which is its value.
(: (-> Syntax Syntax))
(defun (do-last step)
  (syntax-match step
    ((:return expr) #'(pure ,expr))
    ((:return bad ...) (syntax-error step "(:return expr) takes one expression."))
    ((:bind bad ...)
     (syntax-error step "A (do ...) block cannot end with (:bind ...). Its last form is the block's value."))
    ((:match bad ...)
     (syntax-error step "A (do ...) block cannot end with (:match ...). Its last form is the block's value."))
    ((:let bad ...)
     (syntax-error step "A (do ...) block cannot end with (:let ...). Its last form is the block's value."))
    ((:then bad ...)
     (syntax-error step "A (do ...) block cannot end with (:then ...). Its last form is the block's value."))
    ;; Reports keyword errors.
    (last (if (clause? last) (unknown-clause last) last))))

;; A step, and everything after it.
(: (-> Syntax Syntax Syntax))
(defun (do-step step rest)
  (syntax-match step
    ;; :bind takes an identifier.
    ((:bind name expr)
     (if (identifier? name)
         #'(bind ,expr (fun (,name) ,rest))
         (syntax-error step "(:bind name expr) takes a plain identifier, not a pattern.")))
    ((:bind bad ...) (syntax-error step "(:bind name expr) takes an identifier and one expression."))

    ;; :match takes a pattern.
    ((:match pattern expr)
     #'(bind ,expr (fun (bound) (match bound (,pattern ,rest)))))
    ((:match bad ...) (syntax-error step "(:match pattern expr) takes one pattern and one expression."))

    ((:let name expr)
     (if (identifier? name)
         #'(let ((,name ,expr)) ,rest)
         (syntax-error step "(:let name expr) takes a plain identifier, not a pattern.")))
    ((:let bad ...) (syntax-error step "(:let name expr) takes an identifier and one expression."))

    ;; :then for variable-less bind.
    ((:then expr) #'(bind ,expr (fun (_) ,rest)))
    ((:then bad ...) (syntax-error step "(:then expr) takes one expression."))

    ((:return bad ...)
     (syntax-error step "(:return ...) is the last form of its (do ...) block, and this one is not."))

    ;; Plain form statement.
    (other (if (clause? other) (unknown-clause other) #'(let ((_ ,other)) ,rest)))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; A double-ended queue that is written in place.
;;
;; Two lists: `front` holds the front of the deque in order, `back` holds the
;; rear reversed, so the deque's contents are `front ++ reverse back`. Pushing
;; at either end is a cons. Popping takes the head of the list on that end, and
;; when that list is empty the other one is split to refill it.
;;
;; **This is the mutable one.** `(deque-push-back! d x)` writes `d`; it answers
;; nothing, and everyone holding `d` sees the change. The persistent shape of
;; the same structure is what `List` and `Vec` already are, and a program that
;; wants a queue it can keep old versions of should use one of those with an
;; explicit fold. What this buys is the thing they cannot: a worklist that a
;; recursive walk pushes onto and a loop pops from, without either of them
;; having to thread a new deque back out.
;;
;; **Split in half, not moved wholesale.** Refilling an empty side by moving
;; *all* of the other one is the textbook queue trick, and it is quadratic on a
;; deque: pop one end, pop the other, and every pop moves the whole structure
;; back again. Moving half leaves both ends stocked, so the next n operations at
;; either end are free. Every operation is O(1) amortized.
;;
;; **Not a key.** `Deque` has mutable fields, so `eq-hash` refuses and a deque
;; cannot go in a `Map` or a `HashSet`. See the `Eq` implementation at the bottom.
(import (std prelude))

(:doc #:file "deque.bjodoc")

(export
  Deque
  deque deque-length deque-empty? deque-clear!
  deque-push-front! deque-push-back!
  deque-pop-first! deque-pop-last! deque-try-pop-first! deque-try-pop-last!
  deque-peek-front! deque-peek-back!
  deque-contains?
  list->deque deque->list deque-fold deque-for-each
  deque->str)

;; `#:opaque`, so the two lists and the count stay here: which half an element
;; is in is a fact about the representation and not about the deque, and a
;; caller that could read `front` would be reading an implementation detail that
;; every push and pop rearranges. The name still crosses — a signature may say
;; `(Deque int)`, and the trait implementations below still dispatch — so what
;; is lost is only the ability to take one apart.
;;
;; That also settles the mutable fields. A mutable field may be written only by
;; the module that declared it; opacity makes reading one the same rule, and the
;; operations below are the whole interface.
;;
;; `size` is carried rather than counted so that `deque-length` is O(1). It is
;; the one field that is not derivable from the others in constant time.
(type
  (: (Deque %a) #:opaque
     (Record (: front (List %a) #:mutable)
             (: back (List %a) #:mutable)
             (: size int #:mutable))))

(: (-> (Deque %a)))
(defun (deque)
  (Deque (front Nil) (back Nil) (size 0)))

(: (-> (Deque %a) int))
(defun (deque-length d) (record-ref d size))

(: (-> (Deque %a) bool))
(defun (deque-empty? d) (= (record-ref d size) 0))

(: (-> (Deque %a) void))
(defun (deque-clear! d)
  (record-set! d (front Nil) (back Nil) (size 0)))

;; ---------------------------------------------------------------------------
;; Splitting
;; ---------------------------------------------------------------------------
;;
;; The prelude has no `list-take` or `list-drop`, deliberately — they invite the
;; O(n) indexing a cons list cannot support — so the two walks a split needs are
;; here. Both run once per rebalance, which is once per n operations.

;; Collected reversed and turned once, so a long prefix does not recurse once
;; per element.
(: (-> int (List %a) (List %a)))
(defun (take-n k xs)
  (let go ((k k) (rest xs) (acc Nil))
    (if (= k 0)
        (list-reverse acc)
        (match rest
          (Nil (list-reverse acc))
          ((Cons x more) (go (- k 1) more (Cons x acc)))))))

(: (-> int (List %a) (List %a)))
(defun (drop-n k xs)
  (if (= k 0)
      xs
      (match xs
        (Nil Nil)
        ((Cons _ rest) (drop-n (- k 1) rest)))))

;; `front` is empty and everything is on `back`, reversed. Keep the rear half
;; where it is and hand the front half over, in order.
;;
;; `k` elements stay on `back`, and since `back` is the deque reversed its first
;; `k` are the deque's *last* `k` — so the ones to move are the rest, reversed
;; back into order. With `k = n / 2` the front is left non-empty for any n ≥ 1:
;; a single element has `k = 0` and moves across whole.
;;
;; Both fields are written by one `record-set!`, so the deque is never briefly
;; holding an element twice.
(: (-> (Deque %a) void))
(defun (refill-front! d)
  (when (and (list-empty? (record-ref d front))
             (not (list-empty? (record-ref d back))))
    (def b (record-ref d back))
    (def k (/ (record-ref d size) 2))
    (record-set! d
                 (front (list-reverse (drop-n k b)))
                 (back (take-n k b)))))

;; The mirror image: everything is on `front`, in order, and the rear half moves
;; across reversed.
(: (-> (Deque %a) void))
(defun (refill-back! d)
  (when (and (list-empty? (record-ref d back))
             (not (list-empty? (record-ref d front))))
    (def f (record-ref d front))
    (def k (/ (record-ref d size) 2))
    (record-set! d
                 (front (take-n k f))
                 (back (list-reverse (drop-n k f))))))

;; ---------------------------------------------------------------------------
;; The ends
;; ---------------------------------------------------------------------------

(: (-> (Deque %a) %a void))
(defun (deque-push-front! d x)
  (record-set! d
               (front (Cons x (record-ref d front)))
               (size (+ (record-ref d size) 1))))

(: (-> (Deque %a) %a void))
(defun (deque-push-back! d x)
  (record-set! d
               (back (Cons x (record-ref d back)))
               (size (+ (record-ref d size) 1))))

(: (-> (Deque %a) (Option %a)))
(defun (deque-try-pop-first! d)
  (refill-front! d)
  (match (record-ref d front)
    (Nil None)
    ((Cons x rest)
     (record-set! d
                  (front rest)
                  (size (- (record-ref d size) 1)))
     (Some x))))

(: (-> (Deque %a) (Option %a)))
(defun (deque-try-pop-last! d)
  (refill-back! d)
  (match (record-ref d back)
    (Nil None)
    ((Cons x rest)
     (record-set! d
                  (back rest)
                  (size (- (record-ref d size) 1)))
     (Some x))))

(import/class
  (EmptyDeque (: System.InvalidOperationException (-> string EmptyDeque))))

;; The front element, removed.
;; Raises InvalidOperationException when the deque is empty.
(: (-> (Deque %a) %a))
(defun (deque-pop-first! d)
  (match (deque-try-pop-first! d)
    ((Some x) x)
    (None (raise (cast Exception (EmptyDeque. "deque-pop-first!: the deque is empty."))))))

;; The back element, removed.
;; Raises InvalidOperationException when the deque is empty.
(: (-> (Deque %a) %a))
(defun (deque-pop-last! d)
  (match (deque-try-pop-last! d)
    ((Some x) x)
    (None (raise (cast Exception (EmptyDeque. "deque-pop-last!: the deque is empty."))))))

;; A look that does not consume — but it still rebalances, hence the `!`.
;;
;; Refilling changes how the deque is *stored* and not what it holds, so this is
;; a read as far as the caller is concerned. Reading the front of an empty
;; `front` without refilling would mean walking to the end of `back`, which is
;; the O(n) this structure exists to avoid.
(: (-> (Deque %a) (Option %a)))
(defun (deque-peek-front! d)
  (refill-front! d)
  (match (record-ref d front)
    (Nil None)
    ((Cons x _) (Some x))))

(: (-> (Deque %a) (Option %a)))
(defun (deque-peek-back! d)
  (refill-back! d)
  (match (record-ref d back)
    (Nil None)
    ((Cons x _) (Some x))))

;; ---------------------------------------------------------------------------
;; Conversions and walks
;; ---------------------------------------------------------------------------
;;
;; None of these rebalance: they read both halves as they stand, so a walk is
;; not a write and a deque may be folded from anywhere.

;; `front` reversed, then consed element by element onto `back` in order: the
;; front lands first, and nothing recurses per element.
(: (-> (Deque %a) (List %a)))
(defun (deque->list d)
  (list-foldl (fun (x acc) (Cons x acc))
              (list-reverse (record-ref d back))
              (list-reverse (record-ref d front))))

(: (-> (List %a) (Deque %a)))
(defun (list->deque xs)
  (Deque (front xs) (back Nil) (size (list-length xs))))

;; Front to back. The rear half is reversed once rather than per element.
(: (-> (-?-> %a %b %b) %b (Deque %a) %b))
(defun (deque-fold f init d)
  (list-foldl f (list-foldl f init (record-ref d front))
              (list-reverse (record-ref d back))))

(: (-> (-?-> %a void) (Deque %a) void))
(defun (deque-for-each f d)
  (list-for-each f (record-ref d front))
  (list-for-each f (list-reverse (record-ref d back))))

;; Whether the deque holds `x`, compared with `=`. The order of the two lists
;; does not matter here, so neither is reversed.
(: (-> (Deque %a) %a bool) (where (Eq %a)))
(defun (deque-contains? d x)
  (or (list-contains? (record-ref d front) x) (list-contains? (record-ref d back) x)))

;; ---------------------------------------------------------------------------
;; Equality, and the hash there is not
;; ---------------------------------------------------------------------------
;;
;; Two deques are equal when they hold the same elements in the same order.
;; *Where* those elements sit is a fact about the representation: pushing three
;; items on the back and popping one leaves a deque that is `=` to one built by
;; three pushes on the front, and it would be absurd for it not to be.
;;
;; This implementation is not decoration, then — without it a deque would fall
;; to the blanket, which is .NET's equality, which for this record is a
;; field-wise comparison of `front`, `back` and `size`. Two deques holding the
;; same elements but split differently would have compared unequal, silently.
;;
;; `eq-hash` refuses. There is no honest number for a value whose fields move:
;; a `Map` keyed on a deque would lose the entry at the next push. The
;; implementation is compiled into `Deque<T>` as its `Equals` and
;; `GetHashCode`, so .NET's equality is this one too, and a `Map` or a `HashSet`
;; that asks for the hash gets the refusal.
(impl (Eq (Deque %a))
  (where (Eq %a))
  (defun (= a b)
    (and (= (deque-length a) (deque-length b))
         (= (deque->list a) (deque->list b))))
  (defun (eq-hash d) (unhashable "Deque")))

;; `#<a b c>`, front first.
(impl (Written (Deque %a))
  (where (Written %a))
  (defun (write-into d b)
    (elements-print-into "#<" (deque->list d) ">" b (fun (x b) (write-into x b)))))

(impl (Displayed (Deque %a))
  (where (Displayed %a))
  (defun (display-into d b)
    (elements-print-into "#<" (deque->list d) ">" b (fun (x b) (display-into x b)))))

(: (-> (Deque %a) string) (where (Displayed %a)))
(defun (deque->str d) (displayed d))

(:deprecated deque->str "use `displayed`")
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/mutable/heap.bjo — the priority queue.
;;
;; A `MutableHeap` *is* a .NET `PriorityQueue<T,P>`: an array-backed quaternary
;; min-heap. Pushing and popping are O(log n), reading the least is O(1).
;;
;; **An element and a priority, kept apart.** `(mutableheap-add! h task 3)`
;; files `task` under priority 3, and the two come back together. Ordering the
;; element itself would mean every element type needing an order, and a
;; scheduler's task has none — the priority is the thing being ordered, and this
;; is the one structure in the library where they are different values.
;;
;; **Least first.** `mutableheap-pop!` answers the smallest priority. A largest
;; -first heap is the same structure under a negated comparison, which
;; `mutableheap-by` takes.
;;
;; **Ordering is .NET's unless you say otherwise.** `mutableheap` uses
;; `Comparer<P>.Default`, which is what `Map` and `HashSet` do for equality.
;; `mutableheap-by` takes a comparison, and is how a priority is ordered by the
;; `Ord` trait or by anything else.
;;
;; **No `Eq`, and deliberately.** Two heaps holding the same pairs are not
;; distinguishable from the outside without draining both, and draining is a
;; write. `=` therefore falls to the blanket, which for a `PriorityQueue<T,P>`
;; is reference identity — which is honest, unlike a structural answer nothing
;; could compute without destroying its arguments.
(import (std prelude))
(import (only (std interop) as-delegate))

(:doc #:file "heap.bjodoc")

(export
  MutableHeap
  mutableheap mutableheap-by
  mutableheap-length mutableheap-empty? mutableheap-clear!
  mutableheap-add! mutableheap-add-all! mutableheap-add-pop!
  mutableheap-peek mutableheap-pop! mutableheap-try-pop!
  mutableheap-copy
  list->mutableheap mutableheap->unordered-list
  seq->mutableheap mutableheap->unordered-seq
  mutableheap->str)

;; The type is .NET's, so a `MutableHeap` handed to any other `import/extern`
;; taking a `PriorityQueue<T,P>` is accepted as it stands.
(import/class
  ((MutableHeap %a %p) (: System.Collections.Generic.PriorityQueue))
  ((UnorderedItems %a %p) (: System.Collections.Generic.PriorityQueue`2+UnorderedItemsCollection))
  ((Comparison %a) (: System.Comparison))
  ((Comparer %a) (: System.Collections.Generic.Comparer))
  ((IComparer %a) (: System.Collections.Generic.IComparer)))

;; ---------------------------------------------------------------------------
;; The heap
;; ---------------------------------------------------------------------------

;; `PriorityQueue<T,P>`'s own members.
(import/extern
  (mutableheap-length   (: System.Collections.Generic.PriorityQueue.Count        (-> (MutableHeap %a %p) int) #:get))
  (mutableheap-add!     (: System.Collections.Generic.PriorityQueue.Enqueue      (-> (MutableHeap %a %p) %a %p void)))
  (mutableheap-add-all! (: System.Collections.Generic.PriorityQueue.EnqueueRange (-> (MutableHeap %a %p) (Seq (Tuple %a %p)) void)))
  ;; Pushes and pops as one rebalance, answering what came off — which may be
  ;; the element just pushed. This is how a heap is held to a bound.
  (mutableheap-add-pop! (: System.Collections.Generic.PriorityQueue.EnqueueDequeue (-> (MutableHeap %a %p) %a %p %a)))
  (mutableheap-clear!   (: System.Collections.Generic.PriorityQueue.Clear        (-> (MutableHeap %a %p) void)))
  ;; The least element and its priority, left where they are.
  (mutableheap-peek (: System.Collections.Generic.PriorityQueue.TryPeek
                       (-> (MutableHeap %a %p) (out %a) (out %p) (Option (Tuple %a %p)))))
  ;; The least element and its priority, removed. `None` leaves the heap as it
  ;; was, so a drain can pop until it stops.
  (mutableheap-try-pop! (: System.Collections.Generic.PriorityQueue.TryDequeue
                       (-> (MutableHeap %a %p) (out %a) (out %p) (Option (Tuple %a %p)))))
  (clr-items    (: System.Collections.Generic.PriorityQueue.UnorderedItems (-> (MutableHeap %a %p) (UnorderedItems %a %p)) #:get))
  (clr-comparer (: System.Collections.Generic.PriorityQueue.Comparer (-> (MutableHeap %a %p) (IComparer %p)) #:get))
  (clr-comparer-of (: System.Collections.Generic.Comparer.Create (-> (Comparison %p) (Comparer %p)))))

;; Ordered by `Comparer<P>.Default`, least priority first.
(: (-> (MutableHeap %a %p)))
(defun (mutableheap) (MutableHeap.))

(import/class
  (EmptyHeap (: System.InvalidOperationException (-> string EmptyHeap))))

;; The least element and its priority, removed.
;; Raises InvalidOperationException when the heap is empty.
(: (-> (MutableHeap %a %p) (Tuple %a %p)))
(defun (mutableheap-pop! h)
  (match (mutableheap-try-pop! h)
    ((Some entry) entry)
    (None (raise (cast Exception (EmptyHeap. "mutableheap-pop!: the heap is empty."))))))

;; Ordered by `cmp`, which is how a priority is ordered by the `Ord` trait
;; rather than by .NET.
(: (-> (-> %p %p int) (MutableHeap %a %p)))
(defun (mutableheap-by cmp)
  (MutableHeap. (cast (IComparer %p) (clr-comparer-of (: (as-delegate cmp) (Comparison %p))))))

(: (-> (MutableHeap %a %p) bool))
(defun (mutableheap-empty? h) #:inline (= (mutableheap-length h) 0))

;; In *no* order: heap order is only observable by popping, and a walk that
;; looked ordered would be a lie the structure cannot tell.
(: (-> (MutableHeap %a %p) (Seq (Tuple %a %p))))
(defun (mutableheap->unordered-seq h) (cast (Seq (Tuple %a %p)) (clr-items h)))

(: (-> (Seq (Tuple %a %p)) (MutableHeap %a %p)))
(defun (seq->mutableheap xs)
  (def h (mutableheap))
  (mutableheap-add-all! h xs)
  h)

;; The copy keeps the comparer, so a heap built by `mutableheap-by` copies to
;; one ordered the same way.
(: (-> (MutableHeap %a %p) (MutableHeap %a %p)))
(defun (mutableheap-copy h) (MutableHeap. (mutableheap->unordered-seq h) (clr-comparer h)))

;; ---------------------------------------------------------------------------
;; Conversions
;; ---------------------------------------------------------------------------

(: (-> (List (Tuple %a %p)) (MutableHeap %a %p)))
(defun (list->mutableheap xs) (seq->mutableheap (list->seq xs)))

(: (-> (MutableHeap %a %p) (List (Tuple %a %p))))
(defun (mutableheap->unordered-list h) (->list (mutableheap->unordered-seq h)))

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------
;;
;; `Collection`, `Written` and `Displayed` only. There is no `Iterable`: the walk a heap has is
;; unordered, and a caller reaching for `for-each` would get an order it had
;; every reason to expect was the heap's. `mutableheap->unordered-seq` is that
;; walk under a name that says so, and is `Iterable` by being a `Seq`.
;;
;; No `Addable!` either, since adding to a heap takes a priority as well as an
;; element and the trait's `add!` has one argument.

(impl (Collection (MutableHeap %a %p))
  (type %content (Tuple %a %p))
  (defun (length h) (mutableheap-length h))
  (defun (empty? h) (mutableheap-empty? h)))

;; `#mheap(element priority ...)`, unordered, for the reason above. Each
;; element and priority is added by the function given for it.
(: (-> (MutableHeap %a %p) StringBuilder (-> %a StringBuilder void) (-> %p StringBuilder void) void))
(defun (mutableheap-print-into h b each-elem each-priority)
  (stringbuilder-add-string! b "#mheap(")
  (loop (:with first? #t #f)
        (:for entry (mutableheap->unordered-seq h))
        (:do (def (x p) entry)
             (unless first? (stringbuilder-add-string! b " "))
             (each-elem x b)
             (stringbuilder-add-string! b " ")
             (each-priority p b)))
  (stringbuilder-add-string! b ")"))

(impl (Written (MutableHeap %a %p))
  (where (Written %a) (Written %p))
  (defun (write-into h b)
    (mutableheap-print-into h b (fun (x b) (write-into x b)) (fun (p b) (write-into p b)))))

(impl (Displayed (MutableHeap %a %p))
  (where (Displayed %a) (Displayed %p))
  (defun (display-into h b)
    (mutableheap-print-into h b (fun (x b) (display-into x b)) (fun (p b) (display-into p b)))))

(: (-> (MutableHeap %a %p) string) (where (Displayed %a) (Displayed %p)))
(defun (mutableheap->str h) (displayed h))

(:deprecated mutableheap->str "use `displayed`")
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/mutable/map.bjo — the hash table, written in place.
;;
;; A `MutableMap` *is* a .NET `Dictionary<K,V>`: open addressing over buckets,
;; O(1) expected for every operation below. `mutablemap` makes a subclass that
;; also keeps an entry for the key `null`, which a `Dictionary` refuses. The
;; one case of a union that carries nothing is often `null` at run time, and
;; this lets it be a key.
;;
;; **This is the mutable one.** `(mutablemap-set! m k v)` writes `m`; it answers
;; nothing, and everyone holding `m` sees the change. `Map` is the persistent
;; structure of the same shape.
;;
;; **What `TransientMap` is not.** A transient is a `Map` being built: it is
;; made from a persistent map, filled, and frozen back into one. This is a table
;; that is never frozen — entries go in and come out for as long as the program
;; holds it, which is what a cache, an index or a counter wants.
;;
;; **Keys are filed under .NET's equality**, `EqualityComparer<K>.Default`, not
;; under the `Eq` implementation in `std/eq`. That is the same equality `Map` and
;; `HashSet` file their keys under, so the three agree with each other — but a
;; hand-written `Eq` that departs from .NET's is not consulted here.
(import (std prelude))

(:doc #:file "map.bjodoc")

(export
  MutableMap MutableMapEnumerator
  mutablemap mutablemap-with-capacity
  mutablemap-length mutablemap-empty?
  mutablemap-ref mutablemap-try-ref mutablemap-ref-or
  mutablemap-set! mutablemap-add-new! mutablemap-try-add-new!
  mutablemap-remove! mutablemap-clear!
  mutablemap-alter! mutablemap-update!
  mutablemap-contains? mutablemap-has-key? mutablemap-keys mutablemap-values mutablemap-copy
  mutablemap-enumerator mutablemap-enumerator-next! mutablemap-enumerator-current
  mutablemap-for-each
  list->mutablemap mutablemap->list vec->mutablemap mutablemap->vec
  seq->mutablemap mutablemap->seq map->mutablemap mutablemap->map
  mutablemap->str)

;; The type is .NET's, so a `MutableMap` handed to any other `import/extern`
;; taking a `Dictionary<K,V>` is accepted as it stands. Such an extern does not
;; see the entry for a `null` key.
(import/class
  ((MutableMap %k %v) (: System.Collections.Generic.Dictionary))
  ((MutableMapEnumerator %k %v) (: Bjolang.Runtime.MutableMapEnumerator)))

;; ---------------------------------------------------------------------------
;; The map
;; ---------------------------------------------------------------------------

(import/extern
  (mutablemap               (: Bjolang.Runtime.MutableMapModule.Empty        (-> (MutableMap %k %v))))
  (mutablemap-with-capacity (: Bjolang.Runtime.MutableMapModule.WithCapacity (-> int (MutableMap %k %v))))
  (mutablemap-length        (: Bjolang.Runtime.MutableMapModule.Count        (-> (MutableMap %k %v) int)))
  (mutablemap-empty?        (: Bjolang.Runtime.MutableMapModule.IsEmpty      (-> (MutableMap %k %v) bool)))
  ;; Throws when the key is absent, which is what `Refable` asks. The two
  ;; answers that are not an exception are below it.
  (mutablemap-ref           (: Bjolang.Runtime.MutableMapModule.Get          (-> (MutableMap %k %v) %k %v)))
  (mutablemap-ref-or        (: Bjolang.Runtime.MutableMapModule.GetOr        (-> (MutableMap %k %v) %k %v %v)))
  ;; Replaces whatever was under the key.
  (mutablemap-set!          (: Bjolang.Runtime.MutableMapModule.Set          (-> (MutableMap %k %v) %k %v void)))
  ;; Adds a key that must be new, and raises when it is not. To add or replace,
  ;; there is `mutablemap-set!`, which is also what `Addable!`'s `add!` does.
  (mutablemap-add-new!      (: Bjolang.Runtime.MutableMapModule.AddNew       (-> (MutableMap %k %v) %k %v void)))
  ;; The value already under the key, or None when the entry was added. What
  ;; `mutablemap-try-add-new!` is made of.
  (add-or-existing!         (: Bjolang.Runtime.MutableMapModule.AddOrExisting (-> (MutableMap %k %v) %k %v (out %v) (Option %v))))
  (mutablemap-remove!       (: Bjolang.Runtime.MutableMapModule.Remove       (-> (MutableMap %k %v) %k bool)))
  (mutablemap-clear!        (: Bjolang.Runtime.MutableMapModule.Clear        (-> (MutableMap %k %v) void)))
  (mutablemap-contains?     (: Bjolang.Runtime.MutableMapModule.ContainsKey  (-> (MutableMap %k %v) %k bool)))
  ;; Views, not copies: each walks the map as it stands when it is walked, so
  ;; writing the map during one is an error .NET raises.
  (mutablemap-keys          (: Bjolang.Runtime.MutableMapModule.Keys         (-> (MutableMap %k %v) (Seq %k))))
  (mutablemap-values        (: Bjolang.Runtime.MutableMapModule.Values       (-> (MutableMap %k %v) (Seq %v))))
  (mutablemap-copy          (: Bjolang.Runtime.MutableMapModule.Copy         (-> (MutableMap %k %v) (MutableMap %k %v))))
  (mutablemap->seq          (: Bjolang.Runtime.MutableMapModule.AsEnumerable (-> (MutableMap %k %v) (Seq (Tuple %k %v)))))
  ;; Later pairs win, so this and repeated `mutablemap-set!` agree.
  (seq->mutablemap          (: Bjolang.Runtime.MutableMapModule.FromEnumerable (-> (Seq (Tuple %k %v)) (MutableMap %k %v))))

  (mutablemap-try-ref       (: Bjolang.Runtime.MutableMapModule.TryGet       (-> (MutableMap %k %v) %k (out %v) (Option %v))))

  ;; A walk with .NET's protocol: `next!` is `MoveNext`, `current` is `Current`.
  (mutablemap-enumerator         (: Bjolang.Runtime.MutableMapModule.Enumerator (-> (MutableMap %k %v) (MutableMapEnumerator %k %v))))
  (mutablemap-enumerator-next!   (: Bjolang.Runtime.MutableMapModule.MoveNext   (-> (MutableMapEnumerator %k %v) bool)))
  (mutablemap-enumerator-current (: Bjolang.Runtime.MutableMapModule.Current    (-> (MutableMapEnumerator %k %v) (Tuple %k %v)))))

;; `mutablemap-add-new!` with the failure as a value: `(Err v)` carries the
;; value the key already has, which the lookup found anyway.
(: (-> (MutableMap %k %v) %k %v (Result %v Unit)))
(defun (mutablemap-try-add-new! m k v)
  (match (add-or-existing! m k v)
    ((Some old) (Err old))
    (None (Ok unit))))

;; The entry of `k` decided by `f`, written in place:
;; - `f` is given `(Some v)` for the value of `k`, or `None` if `k` is absent.
;; - `(Some v)` from `f` sets `k` to `v`, and `None` removes `k`.
;; - A lookup, then a set or a remove.
(: (-> (MutableMap %k %v) %k (-?-> (Option %v) (Option %v)) void))
(defun (mutablemap-alter! m k f)
  (def old (mutablemap-try-ref m k))
  (match (f old)
    ((Some v) (mutablemap-set! m k v))
    (None (when (some? old) (ignore (mutablemap-remove! m k))))))

;; `k` set to `f` of its value, in place. A missing key raises
;; `KeyNotFoundException`, unless `#:default (Some d)` is given: then `f` is
;; applied to `d`.
(: (-> (MutableMap %k %v) %k (-?-> %v %v) (#:default (Option %v)) void))
(defun (mutablemap-update! m k f #:default None)
  (def old (match (mutablemap-try-ref m k)
             ((Some v) v)
             (None (match default
                     ((Some d) d)
                     ;; Raises KeyNotFoundException.
                     (None (mutablemap-ref m k))))))
  (mutablemap-set! m k (f old)))

;; The key and the value as two arguments, which is what a caller of a map
;; almost always wants and what the `Iterable` walk over pairs cannot give.
(: (-> (-?-> %k %v void) (MutableMap %k %v) void))
(defun (mutablemap-for-each f m)
  (def e (mutablemap-enumerator m))
  (let go ()
    (when (mutablemap-enumerator-next! e)
      (def (k v) (mutablemap-enumerator-current e))
      (f k v)
      (go))))

;; ---------------------------------------------------------------------------
;; Conversions
;; ---------------------------------------------------------------------------

(: (-> (List (Tuple %k %v)) (MutableMap %k %v)))
(defun (list->mutablemap xs) (seq->mutablemap (list->seq xs)))

(: (-> (MutableMap %k %v) (List (Tuple %k %v))))
(defun (mutablemap->list m) (->list (mutablemap->seq m)))

(: (-> (Vec (Tuple %k %v)) (MutableMap %k %v)))
(defun (vec->mutablemap v) (seq->mutablemap (vec->seq v)))

(: (-> (MutableMap %k %v) (Vec (Tuple %k %v))))
(defun (mutablemap->vec m) (->vec (mutablemap->seq m)))

(: (-> (Map %k %v) (MutableMap %k %v)))
(defun (map->mutablemap m) (seq->mutablemap (map->seq m)))

(: (-> (MutableMap %k %v) (Map %k %v)))
(defun (mutablemap->map m) (->map (mutablemap->seq m)))

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------

(: (-> (MutableMap %k %v) %k bool))
(defun (mutablemap-has-key? m k) (mutablemap-contains? m k))

(impl (Refable (MutableMap %k %v))
  (type %index %k)
  (type %out %v)
  (defun (ref m key)
    (mutablemap-ref m key))
  (defun (try-ref m key)
    (mutablemap-try-ref m key)))

;; Written in place, so `(set! m k v)` writes `m`, and adds `k` if it was not
;; there.
(impl (Settable! (MutableMap %k %v))
  (type %index %k)
  (type %in %v)
  (defun (ref-set! m k v)
    (mutablemap-set! m k v)))

(impl (Iterable (MutableMap %k %v))
  (type %elem (Tuple %k %v))
  ;; The enumerator is the cursor, and it steps in `done?`, which the
  ;; protocol calls once per step before `current`. See the `Seq` impl.
  (type %cursor (MutableMapEnumerator %k %v))
  (defun (iterable-start m) (mutablemap-enumerator m))
  (defun (iterable-done? m e) (not (mutablemap-enumerator-next! e)))
  (defun (iterable-current m e) (mutablemap-enumerator-current e))
  (defun (iterable-next m e) e))

(impl (Collection (MutableMap %k %v))
  (type %content (Tuple %k %v))
  (defun (length m) (mutablemap-length m))
  (defun (empty? m) (mutablemap-empty? m)))

;; Set semantics, as `Addable`'s do everywhere: a key already there is replaced.
(impl (Addable! (MutableMap %k %v))
  (type %elem (Tuple %k %v))
  (defun (add! m pair)
    (def (k v) pair)
    (mutablemap-set! m k v)))

;; Two maps are equal when they hold the same keys with equal values.
;;
;; Without this they would fall to the blanket, which for a `Dictionary<K,V>` is
;; reference identity. The walk is one-sided because the lengths are compared
;; first: every key of `a` being in `b` with an equal value, at equal counts,
;; leaves `b` no room for a key of its own.
;;
;; The *keys* are matched by .NET's equality, since finding one is a lookup;
;; only the values go through `Eq`. `eq-hash` refuses, as `Deque`'s does.
(impl (Eq (MutableMap %k %v))
  (where (Eq %v))
  (defun (= a b)
    (and (= (mutablemap-length a) (mutablemap-length b))
         (let ((e (mutablemap-enumerator a)))
           (let go ()
             (cond
               ((not (mutablemap-enumerator-next! e)) #t)
               (else
                (def (k v) (mutablemap-enumerator-current e))
                (match (mutablemap-try-ref b k)
                  (None #f)
                  ((Some w) (if (= v w) (go) #f)))))))))
  (defun (eq-hash m) (unhashable "MutableMap")))

;; `#mmap(k v k v)`: each key and its value, flat, with a space between all.
;; Each key and value is added by the function given for it.
(: (-> (MutableMap %k %v) StringBuilder (-> %k StringBuilder void) (-> %v StringBuilder void) void))
(defun (mutablemap-print-into m b each-key each-value)
  (stringbuilder-add-string! b "#mmap(")
  (loop (:with first? #t #f)
        (:for entry m)
        (:do (def (k v) entry)
             (unless first? (stringbuilder-add-string! b " "))
             (each-key k b)
             (stringbuilder-add-string! b " ")
             (each-value v b)))
  (stringbuilder-add-string! b ")"))

(impl (Written (MutableMap %k %v))
  (where (Written %k) (Written %v))
  (defun (write-into m b)
    (mutablemap-print-into m b (fun (k b) (write-into k b)) (fun (v b) (write-into v b)))))

(impl (Displayed (MutableMap %k %v))
  (where (Displayed %k) (Displayed %v))
  (defun (display-into m b)
    (mutablemap-print-into m b (fun (k b) (display-into k b)) (fun (v b) (display-into v b)))))

(: (-> (MutableMap %k %v) string) (where (Displayed %k) (Displayed %v)))
(defun (mutablemap->str m) (displayed m))

(:deprecated mutablemap->str "use `displayed`")

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/mutable/set.bjo — the hash set, written in place.
;;
;; A `MutableSet` *is* a .NET `HashSet<T>`. O(1) expected membership, and set
;; algebra that rewrites the receiver rather than building a new set.
;;
;; **This is the mutable one.** `(mutableset-add! s x)` writes `s`; everyone
;; holding `s` sees the change. `HashSet` is the persistent structure of the same
;; shape, and `TransientHashSet` is a `HashSet` being built — made from one, filled, and
;; frozen back. This is the one that is never frozen.
;;
;; **The set algebra writes the left-hand side.** `(mutableset-union! a b)` is
;; `a` becoming the union, not a third set: rebuilding is what the persistent
;; `hashset-union` is for, and doing it in place is the reason to reach for this.
;;
;; **Members are filed under .NET's equality**, `EqualityComparer<T>.Default`,
;; not under the `Eq` implementation in `std/eq` — the same equality `Map` and
;; `HashSet` file their keys under.
(import (std prelude))
;; For `hashset->seq` and `seq->hashset`, which the two conversions at the bottom are
;; written with. The persistent set is a module of its own.
(import (std hashset))
(import (only (std interop) as-delegate))

(:doc #:file "set.bjodoc")

(export
  MutableSet MutableSetEnumerator
  mutableset
  mutableset-length mutableset-empty?
  mutableset-add! mutableset-add-new! mutableset-try-add-new!
  mutableset-add-all! mutableset-remove! mutableset-remove-all!
  mutableset-remove-where! mutableset-clear! mutableset-contains?
  mutableset-union! mutableset-intersect! mutableset-difference!
  mutableset-symmetric-difference!
  mutableset-subset? mutableset-superset? mutableset-overlaps?
  mutableset-copy
  mutableset-enumerator mutableset-enumerator-next! mutableset-enumerator-current
  mutableset-for-each
  list->mutableset mutableset->list vec->mutableset mutableset->vec
  seq->mutableset mutableset->seq hashset->mutableset mutableset->hashset
  mutableset->str)

;; The type is .NET's, so a `MutableSet` handed to any other `import/extern`
;; taking a `HashSet<T>` is accepted as it stands.
(import/class
  ((MutableSet %a) (: System.Collections.Generic.HashSet))
  ((MutableSetEnumerator %a) (: Bjolang.Runtime.MutableSetEnumerator))
  ((Predicate %a) (: System.Predicate))
  (ClrObject (: System.Object))
  (ArgumentException (: System.ArgumentException (-> string string ArgumentException))))

;; ---------------------------------------------------------------------------
;; The set
;; ---------------------------------------------------------------------------

;; `HashSet<T>`'s own members. The ones that take another collection take a
;; `Seq`, so a set is given to them as one.
(import/extern
  (mutableset-length     (: System.Collections.Generic.HashSet.Count        (-> (MutableSet %a) int) #:get))
  (clr-add               (: System.Collections.Generic.HashSet.Add          (-> (MutableSet %a) %a bool)))
  (clr-try-get           (: System.Collections.Generic.HashSet.TryGetValue  (-> (MutableSet %a) %a (out %a) (Option %a))))
  (mutableset-add-all!   (: System.Collections.Generic.HashSet.UnionWith    (-> (MutableSet %a) (Seq %a) void)))
  (mutableset-remove!    (: System.Collections.Generic.HashSet.Remove       (-> (MutableSet %a) %a bool)))
  (mutableset-remove-all! (: System.Collections.Generic.HashSet.ExceptWith  (-> (MutableSet %a) (Seq %a) void)))
  (clr-remove-where      (: System.Collections.Generic.HashSet.RemoveWhere  (-> (MutableSet %a) (Predicate %a) int)))
  (mutableset-clear!     (: System.Collections.Generic.HashSet.Clear        (-> (MutableSet %a) void)))
  (mutableset-contains?  (: System.Collections.Generic.HashSet.Contains     (-> (MutableSet %a) %a bool)))
  (clr-intersect         (: System.Collections.Generic.HashSet.IntersectWith (-> (MutableSet %a) (Seq %a) void)))
  (clr-symmetric-except  (: System.Collections.Generic.HashSet.SymmetricExceptWith (-> (MutableSet %a) (Seq %a) void)))
  (clr-subset?           (: System.Collections.Generic.HashSet.IsSubsetOf   (-> (MutableSet %a) (Seq %a) bool)))
  (clr-superset?         (: System.Collections.Generic.HashSet.IsSupersetOf (-> (MutableSet %a) (Seq %a) bool)))
  (clr-overlaps?         (: System.Collections.Generic.HashSet.Overlaps     (-> (MutableSet %a) (Seq %a) bool)))
  (clr-text              (: System.Object.ToString                          (-> ClrObject string)))

  ;; A walk with .NET's protocol: `next!` is `MoveNext`, `current` is `Current`.
  (mutableset-enumerator         (: Bjolang.Runtime.MutableSetModule.Enumerator (-> (MutableSet %a) (MutableSetEnumerator %a))))
  (mutableset-enumerator-next!   (: Bjolang.Runtime.MutableSetModule.MoveNext   (-> (MutableSetEnumerator %a) bool)))
  (mutableset-enumerator-current (: Bjolang.Runtime.MutableSetModule.Current    (-> (MutableSetEnumerator %a) %a))))

(: (-> (MutableSet %a)))
(defun (mutableset) (MutableSet.))

(: (-> (Seq %a) (MutableSet %a)))
(defun (seq->mutableset xs) (MutableSet. xs))

;; The set itself, seen as a sequence: nothing is copied.
(: (-> (MutableSet %a) (Seq %a)))
(defun (mutableset->seq s) #:inline (cast (Seq %a) s))

;; A set with the same members. .NET copies another `HashSet` with the same
;; equality without hashing its members again.
(: (-> (MutableSet %a) (MutableSet %a)))
(defun (mutableset-copy s) (MutableSet. (mutableset->seq s)))

(: (-> (MutableSet %a) bool))
(defun (mutableset-empty? s) #:inline (= (mutableset-length s) 0))

;; Makes x a member. Adding one that is already a member is not a failure:
;; afterwards it is a member, which is all `add!` says.
(: (-> (MutableSet %a) %a void))
(defun (mutableset-add! s x) #:inline (ignore (clr-add s x)))

;; The equal element already there, or None when x was added. What
;; `mutableset-try-add-new!` is made of.
(: (-> (MutableSet %a) %a (Option %a)))
(defun (add-or-existing! s x)
  (match (clr-try-get s x)
    ((Some old) (Some old))
    (None (ignore (clr-add s x)) None)))

;; Adds an element that must be new, and raises when it is not.
(: (-> (MutableSet %a) %a void))
(defun (mutableset-add-new! s x)
  (when (not (clr-add s x))
    (raise (cast Exception
                 (ArgumentException. (str "The element is already in the set: "
                                          (clr-text (cast ClrObject x)))
                                     "value")))))

;; Removes every member that `pred` accepts, and returns how many it removed.
(: (-> (MutableSet %a) (-> %a bool) int))
(defun (mutableset-remove-where! s pred)
  (clr-remove-where s (: (as-delegate pred) (Predicate %a))))

;; --- Set algebra, written into the first argument ---

(: (-> (MutableSet %a) (MutableSet %a) void))
(defun (mutableset-union! a b) (mutableset-add-all! a (mutableset->seq b)))

(: (-> (MutableSet %a) (MutableSet %a) void))
(defun (mutableset-intersect! a b) (clr-intersect a (mutableset->seq b)))

(: (-> (MutableSet %a) (MutableSet %a) void))
(defun (mutableset-difference! a b) (mutableset-remove-all! a (mutableset->seq b)))

(: (-> (MutableSet %a) (MutableSet %a) void))
(defun (mutableset-symmetric-difference! a b) (clr-symmetric-except a (mutableset->seq b)))

;; --- Comparisons, which write nothing ---

(: (-> (MutableSet %a) (MutableSet %a) bool))
(defun (mutableset-subset? a b) (clr-subset? a (mutableset->seq b)))

(: (-> (MutableSet %a) (MutableSet %a) bool))
(defun (mutableset-superset? a b) (clr-superset? a (mutableset->seq b)))

(: (-> (MutableSet %a) (MutableSet %a) bool))
(defun (mutableset-overlaps? a b) (clr-overlaps? a (mutableset->seq b)))

;; `mutableset-add-new!` with the failure as a value: `(Err y)` carries the
;; equal element already in the set, which is what interning a value needs.
(: (-> (MutableSet %a) %a (Result %a Unit)))
(defun (mutableset-try-add-new! s x)
  (match (add-or-existing! s x)
    ((Some old) (Err old))
    (None (Ok unit))))

;; A walk whose callback may suspend. The `Iterable` default would do as well;
;; this one names the set first, matching `hashset-for-each`.
(: (-> (-?-> %a void) (MutableSet %a) void))
(defun (mutableset-for-each f s)
  (def e (mutableset-enumerator s))
  (let go ()
    (when (mutableset-enumerator-next! e)
      (f (mutableset-enumerator-current e))
      (go))))

;; ---------------------------------------------------------------------------
;; Conversions
;; ---------------------------------------------------------------------------

(: (-> (List %a) (MutableSet %a)))
(defun (list->mutableset xs) (seq->mutableset (list->seq xs)))

(: (-> (MutableSet %a) (List %a)))
(defun (mutableset->list s) (->list (mutableset->seq s)))

(: (-> (Vec %a) (MutableSet %a)))
(defun (vec->mutableset v) (seq->mutableset (vec->seq v)))

(: (-> (MutableSet %a) (Vec %a)))
(defun (mutableset->vec s) (->vec (mutableset->seq s)))

(: (-> (HashSet %a) (MutableSet %a)))
(defun (hashset->mutableset s) (seq->mutableset (hashset->seq s)))

(: (-> (MutableSet %a) (HashSet %a)))
(defun (mutableset->hashset s) (seq->hashset (mutableset->seq s)))

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------

(impl (Refable (MutableSet %a))
  (type %index %a)
  (type %out bool)
  (defun (ref s elem)
    (mutableset-contains? s elem)))

(impl (Iterable (MutableSet %a))
  (type %elem %a)
  ;; The enumerator is the cursor, and it steps in `done?`, which the
  ;; protocol calls once per step before `current`. See the `Seq` impl.
  (type %cursor (MutableSetEnumerator %a))
  (defun (iterable-start s) (mutableset-enumerator s))
  (defun (iterable-done? s e) (not (mutableset-enumerator-next! e)))
  (defun (iterable-current s e) (mutableset-enumerator-current e))
  (defun (iterable-next s e) e))

(impl (Collection (MutableSet %a))
  (type %content %a)
  (defun (length s) (mutableset-length s))
  (defun (empty? s) (mutableset-empty? s)))

(impl (Addable! (MutableSet %a))
  (type %elem %a)
  (defun (add! s elem)
    (mutableset-add! s elem)))

;; Two sets are equal when they hold the same elements. Same size, and every
;; element of `a` in `b` — which leaves `b` no room for one of its own.
;;
;; Membership is .NET's equality, since finding an element is a lookup. Without
;; this implementation the whole comparison would be .NET's too, and for a
;; `HashSet<T>` that is reference identity. `eq-hash` refuses, as `Deque`'s does.
(impl (Eq (MutableSet %a))
  (defun (= a b)
    (and (= (mutableset-length a) (mutableset-length b))
         (let ((e (mutableset-enumerator a)))
           (let go ()
             (cond
               ((not (mutableset-enumerator-next! e)) #t)
               ((mutableset-contains? b (mutableset-enumerator-current e)) (go))
               (else #f))))))
  (defun (eq-hash s) (unhashable "MutableSet")))

;; `#mset(a b c)`, in whatever order the table holds them.
(impl (Written (MutableSet %a))
  (where (Written %a))
  (defun (write-into s b) (elements-print-into "#mset(" s ")" b (fun (x b) (write-into x b)))))

(impl (Displayed (MutableSet %a))
  (where (Displayed %a))
  (defun (display-into s b) (elements-print-into "#mset(" s ")" b (fun (x b) (display-into x b)))))

(: (-> (MutableSet %a) string) (where (Displayed %a)))
(defun (mutableset->str s) (displayed s))

(:deprecated mutableset->str "use `displayed`")
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/mutable/vec.bjo — the growable array, written in place.
;;
;; A `MutableVec` *is* a .NET `List<T>`: a contiguous array that doubles when it
;; fills. Indexing and appending are O(1), inserting and removing at an index
;; are O(n) in what follows the index.
;;
;; **This is the mutable one.** `(mutablevec-add! v x)` writes `v`; it answers
;; nothing, and everyone holding `v` sees the change. `Vec` is the persistent
;; structure of the same shape, and a program that wants to keep old versions
;; should use that.
;;
;; **What the existing builders are not.** `VecBuilder` and `ListBuilder` are
;; append-only: an element goes in and nothing comes out until the collection is
;; finished. This is the one that can be written at an index, sorted, and had
;; elements removed from the middle — a scratch buffer rather than a funnel.
;;
;; **Not a key.** The `Eq` implementation at the bottom is structural, and
;; `eq-hash` refuses: there is no honest number for a value whose elements move.
(import (std prelude))
(import (only (std interop) as-delegate))

(:doc #:file "vec.bjodoc")

(export
  MutableVec
  mutablevec mutablevec-with-capacity
  mutablevec-length mutablevec-empty?
  mutablevec-ref mutablevec-try-ref mutablevec-set!
  mutablevec-add! mutablevec-add-all! mutablevec-insert!
  mutablevec-remove-at! mutablevec-remove! mutablevec-remove-where!
  mutablevec-pop! mutablevec-try-pop! mutablevec-clear!
  mutablevec-contains? mutablevec-index-of
  mutablevec-reverse! mutablevec-sort! mutablevec-sort-unstable! mutablevec-sort-by!
  mutablevec-copy
  mutablevec-grow! mutablevec-span
  list->mutablevec mutablevec->list vec->mutablevec mutablevec->vec
  mutablevec->iarray
  seq->mutablevec mutablevec->seq mutablevec->array
  mutablevec->str)

;; The type is .NET's, not an alias for a record of our own: a `MutableVec`
;; handed to any other `import/extern` that takes a `List<T>` is accepted as it
;; stands, with nothing to unwrap.
(import/class
  ((MutableVec %a) (: System.Collections.Generic.List))
  ((Predicate %a) (: System.Predicate)))

;; ---------------------------------------------------------------------------
;; The vector
;; ---------------------------------------------------------------------------

;; `List<T>`'s own members. `get_Item` and `set_Item` are its indexer.
(import/extern
  (mutablevec-length        (: System.Collections.Generic.List.Count     (-> (MutableVec %a) int) #:get))
  ;; Throws when the index is out of range, which is what `Refable` asks of an
  ;; indexed collection. `mutablevec-try-ref` is the checked one.
  (mutablevec-ref           (: System.Collections.Generic.List.get_Item  (-> (MutableVec %a) int %a)))
  (mutablevec-set!          (: System.Collections.Generic.List.set_Item  (-> (MutableVec %a) int %a void)))
  (mutablevec-add!          (: System.Collections.Generic.List.Add       (-> (MutableVec %a) %a void)))
  (mutablevec-add-all!      (: System.Collections.Generic.List.AddRange  (-> (MutableVec %a) (Seq %a) void)))
  (mutablevec-insert!       (: System.Collections.Generic.List.Insert    (-> (MutableVec %a) int %a void)))
  (mutablevec-remove-at!    (: System.Collections.Generic.List.RemoveAt  (-> (MutableVec %a) int void)))
  ;; Removes the first element equal to `x`, and answers whether there was one.
  (mutablevec-remove!       (: System.Collections.Generic.List.Remove    (-> (MutableVec %a) %a bool)))
  (mutablevec-clear!        (: System.Collections.Generic.List.Clear     (-> (MutableVec %a) void)))
  (mutablevec-contains?     (: System.Collections.Generic.List.Contains  (-> (MutableVec %a) %a bool)))
  ;; -1 when there is none, which is what .NET answers and what an `Option`
  ;; would cost an allocation to say.
  (mutablevec-index-of      (: System.Collections.Generic.List.IndexOf   (-> (MutableVec %a) %a int)))
  (mutablevec-reverse!      (: System.Collections.Generic.List.Reverse   (-> (MutableVec %a) void)))
  (mutablevec->array        (: System.Collections.Generic.List.ToArray   (-> (MutableVec %a) (Array %a))))
  (clr-remove-all           (: System.Collections.Generic.List.RemoveAll (-> (MutableVec %a) (Predicate %a) int)))
  ;; The way to build an `IArray` one element at a time: add to a vector, then
  ;; copy it once. `iarray-add` copies the whole array each time.
  (mutablevec->iarray       (: Bjolang.Runtime.IArrayModule.FromList (-> (MutableVec %a) (IArray %a))))
  ;; The elements where the vector keeps them. Valid until the vector grows.
  (clr-as-span              (: System.Runtime.InteropServices.CollectionsMarshal.AsSpan (-> (MutableVec %a) (Span %a))))
  (clr-set-count            (: System.Runtime.InteropServices.CollectionsMarshal.SetCount (-> (MutableVec %a) int void))))

(: (-> (MutableVec %a)))
(defun (mutablevec) (MutableVec.))

;; Room for `capacity` elements before the first regrow. The length is still 0.
(: (-> int (MutableVec %a)))
(defun (mutablevec-with-capacity capacity) (MutableVec. capacity))

(: (-> (Seq %a) (MutableVec %a)))
(defun (seq->mutablevec xs) (MutableVec. xs))

;; The vector itself, seen as a sequence: nothing is copied.
(: (-> (MutableVec %a) (Seq %a)))
(defun (mutablevec->seq v) #:inline (cast (Seq %a) v))

(: (-> (MutableVec %a) (MutableVec %a)))
(defun (mutablevec-copy v) (MutableVec. (mutablevec->seq v)))

(: (-> (MutableVec %a) bool))
(defun (mutablevec-empty? v) #:inline (= (mutablevec-length v) 0))

(: (-> (MutableVec %a) int (Option %a)))
(defun (mutablevec-try-ref v i) #:inline
  (if (and (>= i 0) (< i (mutablevec-length v))) (Some (mutablevec-ref v i)) None))

;; The last element, removed. `None` leaves the vector as it was, so a
;; stack-shaped loop can pop until it stops.
(: (-> (MutableVec %a) (Option %a)))
(defun (mutablevec-try-pop! v)
  (def last (- (mutablevec-length v) 1))
  (if (< last 0)
      None
      (let ((x (mutablevec-ref v last)))
        (mutablevec-remove-at! v last)
        (Some x))))

(import/class
  (EmptyVec (: System.InvalidOperationException (-> string EmptyVec))))

;; The last element, removed.
;; Raises InvalidOperationException when the vector is empty.
(: (-> (MutableVec %a) %a))
(defun (mutablevec-pop! v)
  (match (mutablevec-try-pop! v)
    ((Some x) x)
    (None (raise (cast Exception (EmptyVec. "mutablevec-pop!: the vector is empty."))))))

;; Removes every element that `pred` accepts, and returns how many it removed.
(: (-> (MutableVec %a) (-> %a bool) int))
(defun (mutablevec-remove-where! v pred)
  (clr-remove-all v (: (as-delegate pred) (Predicate %a))))

(import/class
  (ArgumentOutOfRangeException (: System.ArgumentOutOfRangeException
                                  (-> string string ArgumentOutOfRangeException))))

;; The elements of `sorted`, which has the vector's length, into the vector.
(: (-> (MutableVec %a) (Array %a) void))
(defun (write-back! xs sorted)
  (def items (clr-as-span xs))
  (loop (:for i (range 0 (array-length sorted)))
        (:do (span-set! items i (array-ref sorted i)))))

;; Sorted by the element's own `Ord`. `mutablevec-sort-by!` takes the comparison
;; directly, for an order that is not the type's. Both are stable, as
;; `list-sort` is: equal elements keep their order. A copy is sorted with
;; `array-sort!` and written back once the sort is done, so when a comparison
;; raises, the vector is as it was.
(: (-> (MutableVec %a) void) (where (Ord %a)))
(defun (mutablevec-sort! xs)
  (def sorted (mutablevec->array xs))
  (array-sort! sorted)
  (write-back! xs sorted))

;; As `mutablevec-sort!`, with `array-sort-unstable!`: equal elements can
;; change places. When a comparison raises, the vector is as it was.
(: (-> (MutableVec %a) void) (where (Ord %a)))
(defun (mutablevec-sort-unstable! xs)
  (def sorted (mutablevec->array xs))
  (array-sort-unstable! sorted)
  (write-back! xs sorted))

;; As `mutablevec-sort!`, in the order of `cmp`, with `array-sort-by!`.
;; Stable. When `cmp` raises, the vector is as it was.
(: (-> (MutableVec %a) (-> %a %a int) void))
(defun (mutablevec-sort-by! v cmp)
  (def sorted (mutablevec->array v))
  (array-sort-by! cmp sorted)
  (write-back! v sorted))

;; ---------------------------------------------------------------------------
;; Blocks
;; ---------------------------------------------------------------------------
;;
;; A loop that fills a vector a block at a time grows the vector, and then
;; writes the new elements through a span.

;; Adds `n` elements at the end. Their values are unspecified: the storage
;; can keep values from before. Write each one before you read it.
;; Raises ArgumentOutOfRangeException when `n` is less than 0.
(: (-> (MutableVec %a) int void))
(defun (mutablevec-grow! xs n)
  (when (< n 0)
    (raise (cast Exception
                 (ArgumentOutOfRangeException.
                  "n" (str "mutablevec-grow!: the count is " (int->string n) ", and it must be 0 or more.")))))
  (clr-set-count xs (+ (mutablevec-length xs) n)))

;; The `n` elements from index `start`, as a span over the storage of the
;; vector. A write through the span writes the vector. The span is valid
;; until the vector changes its length.
;; Raises ArgumentOutOfRangeException when the range is not in the vector.
(: (-> (MutableVec %a) int int (Span %a)))
(defun (mutablevec-span xs start n)
  (span-slice (clr-as-span xs) #:from start #:to (+ start n)))

;; ---------------------------------------------------------------------------
;; Conversions
;; ---------------------------------------------------------------------------

(: (-> (List %a) (MutableVec %a)))
(defun (list->mutablevec xs) (seq->mutablevec (list->seq xs)))

(: (-> (MutableVec %a) (List %a)))
(defun (mutablevec->list xs) (->list (mutablevec->seq xs)))

(: (-> (Vec %a) (MutableVec %a)))
(defun (vec->mutablevec v) (seq->mutablevec (vec->seq v)))

(: (-> (MutableVec %a) (Vec %a)))
(defun (mutablevec->vec xs) (->vec (mutablevec->seq xs)))

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------

(impl (Refable (MutableVec %a))
  (type %index int)
  (type %out %a)
  (defun (ref xs index)
    (mutablevec-ref xs index))
  (defun (try-ref xs index)
    (mutablevec-try-ref xs index)))

;; Written in place, so `(set! v i x)` writes `v`. An index out of range throws.
(impl (Settable! (MutableVec %a))
  (type %index int)
  (type %in %a)
  (defun (ref-set! v i x)
    (mutablevec-set! v i x)))

;; An `int` cursor, as `Array` has: an index and an array read is what this
;; collection makes cheap, so there is nothing a cursor object would save.
;;
;; A walk reads the length at every step rather than once, so a vector written
;; *during* its own walk is walked as it now stands. Adding an element inside a
;; `for-each` over the same vector therefore does not end the loop early — nor
;; does it end at all, which is the usual hazard of writing a collection you
;; are reading. Shrinking it past the cursor ends the walk.
(impl (Iterable (MutableVec %a))
  (type %elem %a)
  (type %cursor int)
  (defun (iterable-start xs) 0)
  (defun (iterable-done? xs index) (>= index (mutablevec-length xs)))
  (defun (iterable-current xs index) (mutablevec-ref xs index))
  (defun (iterable-next xs index) (+ index 1)))

;; The elements are one .NET array, so a loop over them can be a vector loop.
;; Such a loop does not add to the vector, so the span stays valid.
(impl (Contiguous (MutableVec %a))
  (type %value %a)
  (defun (contiguous? xs) #t)
  (defun (contiguous-values xs) (span->ro-span (clr-as-span xs)))
  (defun (contiguous-valid xs) (array->ro-span (make-array 0))))

(impl (Collection (MutableVec %a))
  (type %content %a)
  (defun (length xs) (mutablevec-length xs))
  (defun (empty? xs) (mutablevec-empty? xs)))

(impl (Addable! (MutableVec %a))
  (type %elem %a)
  (defun (add! xs elem)
    (mutablevec-add! xs elem)))

;; Two vectors are equal when they hold the same elements in the same order.
;;
;; Without this they would fall to the blanket, which is .NET's equality, which
;; for a `List<T>` is reference identity — two vectors built the same way from
;; the same elements would have compared unequal.
;;
;; `eq-hash` refuses, exactly as `Deque`'s does: there is no honest number for a
;; value whose elements move, and a `Map` keyed on one would ask .NET rather
;; than asking this.
(impl (Eq (MutableVec %a))
  (where (Eq %a))
  (defun (= a b)
    (def len (mutablevec-length a))
    (if (= len (mutablevec-length b))
        (let lp ((i 0))
          (cond
            ((= i len) #t)
            ((= (mutablevec-ref a i) (mutablevec-ref b i)) (lp (+ i 1)))
            (else #f)))
        #f))
  (defun (eq-hash xs) (unhashable "MutableVec")))

;; `#mvec(a b c)`, in order.
(impl (Written (MutableVec %a))
  (where (Written %a))
  (defun (write-into xs b) (elements-print-into "#mvec(" xs ")" b (fun (x b) (write-into x b)))))

(impl (Displayed (MutableVec %a))
  (where (Displayed %a))
  (defun (display-into xs b) (elements-print-into "#mvec(" xs ")" b (fun (x b) (display-into x b)))))

(: (-> (MutableVec %a) string) (where (Displayed %a)))
(defun (mutablevec->str xs) (displayed xs))

(:deprecated mutablevec->str "use `displayed`")
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/net.bjo — TCP, as two byte ports.
;;
;; Connecting gives you a `(Tuple InputPort OutputPort)` and nothing
;; else — no socket object, which is why this module is `(std net)` and not
;; `(std socket)`. That is the whole design: a protocol written against
;; `(std ports)` runs over TCP without knowing it is TCP, and `byte-pipe` can
;; stand in for a network in a test.
;;
;; Everything you do with a connection comes from `(std ports)` and the prelude:
;; `read-some`, `read-bytes-evt`, `write-bytes!`, `flush-port`, `shutdown!`,
;; `limited`, and `read-line` on the same input port as `read-bytes`. Import
;; both.
;;
;;     (match (tcp-connect "127.0.0.1" 8080)
;;       ((Ok (Tuple in out))
;;        (write-bytes! out request)
;;        (flush-port out)
;;        (shutdown! out)              ; a FIN: "I have finished asking"
;;        (read-some in))              ; ...and the answer still arrives
;;       ((Err e) ...))
;;
;; THE INVARIANT THE ACCEPT SIDE TURNS ON, which is the byte port's one layer
;; out:
;;
;;   A connection the operating system has handed over belongs to the listener,
;;   not to the event that asked for it. An accept never takes a connection from
;;   the kernel — it takes one from the listener's slot, and only when it wins.
;;
;; So `accept-evt` is a real CML event: a branch of a `choose` that loses accepts
;; nothing, and the connection is still there for the next acceptor.
;;
;; `BjolangRuntime/BjoNet.cs` is where that is done.

(export
  ;; Connecting.
  tcp-connect

  ;; A connection somebody else made.
  stream->ports

  ;; Listening.
  Listener
  tcp-listen accept-evt accept
  listener-closed listener-port close-listener!

  ;; Who is at the other end.
  peer-address)

(import (std prelude))
(import (std ports))

(:doc #:file "net.bjodoc")

(import/class
  (Listener   (: Bjolang.Runtime.BjoTcpListener))
  ;; Opaque, and deliberately not exported: a connection *is* its two ports, and
  ;; a third value to pass around would be the thing this module exists not to
  ;; have. It appears below only between the runtime and the tuple.
  (Connection (: Bjolang.Runtime.BjoConnection (-> ClrStream string Connection)))
  (Exception  (: System.Exception)))

(import/extern
  ;; Connecting. `#:async`, so the ambient cancellation token is filled in at
  ;; the call site and `with-deadline` works on a connect with no keyword of its
  ;; own. `OperationCanceledException` is deliberately *not* in the list: a
  ;; deadline is a raise, not a `Result`.
  (clr-connect (: Bjolang.Runtime.Net.Connect (-> string int Connection) #:async
                  #:exceptions (System.Net.Sockets.SocketException
                                System.ArgumentOutOfRangeException)))

  ;; Listening.
  (clr-listen (: Bjolang.Runtime.Net.Listen (-> string int int Listener)
                 #:exceptions (System.Net.Sockets.SocketException
                               System.ArgumentOutOfRangeException)))
  (clr-accept-evt     (: Bjolang.Runtime.Net.AcceptEvent
                         (-> Listener (Event (Result Exception Connection)))))
  (clr-closed-evt     (: Bjolang.Runtime.Net.ClosedEvent    (-> Listener (Event Unit))))
  (clr-listener-port  (: Bjolang.Runtime.Net.ListenerPort   (-> Listener int)))
  (clr-close-listener (: Bjolang.Runtime.Net.CloseListener  (-> Listener void)))

  ;; A connection's two halves, and who is behind them.
  (clr-conn-in  (: Bjolang.Runtime.Net.ConnectionInput  (-> Connection InputPort)))
  (clr-conn-out (: Bjolang.Runtime.Net.ConnectionOutput (-> Connection OutputPort)))
  (clr-peer     (: Bjolang.Runtime.Net.PeerAddress (-> InputPort (Option string))))

  ;; Ownership. Called from here, on the fiber, and never from inside the
  ;; awaited connect: after an await the continuation is on a pool thread, where
  ;; the ambient scope is not this fiber's and there would be nothing to
  ;; register on.
  (clr-own-connection (: BjolangRuntime.OwnConnection (-> Connection Connection)))
  (clr-own-listener   (: BjolangRuntime.OwnListener   (-> Listener Listener))))

;; ---------------------------------------------------------------------------
;; A connection is its two ports
;; ---------------------------------------------------------------------------

;; The socket under the pair goes when **both** halves have been closed, or when
;; the scope that owns the connection ends, whichever comes first. That is what
;; makes the obvious spelling mean the obvious thing:
;;
;;     (with-open ((i in) (o out))
;;       ...)                         ; both closed here, so the handle goes here
;;
;; Closing one half does not close the other — "I have finished listening, now
;; let me answer" has to work, and it is the same fact `shutdown!` exists for.
(: (-> Connection (Tuple InputPort OutputPort)))
(defun (connection->ports c)
  (Tuple (clr-conn-in c) (clr-conn-out c)))

;; ---------------------------------------------------------------------------
;; Connecting
;; ---------------------------------------------------------------------------

;; Connects to `host` on `port`, and answers the two ports over it.
;;
;; `host` may be an address or a name. A name is resolved and its addresses are
;; tried in order, the first that answers winning; there is no happy-eyeballs
;; racing, and an address literal asks nothing of the resolver at all.
;;
;; The connection is owned by the ambient scope, so a program that forgets to
;; close it still closes it when the scope returns. There is no `#:deadline`:
;; `sync` watches the ambient token, so `with-deadline` already bounds this.
;;
;; Nagle is switched off on the socket. A byte port has a write buffer of its
;; own and `flush-port` is the program saying "send this now", so leaving Nagle on
;; would hold a flushed small write back waiting for an ACK — exactly wrong for
;; a request/response protocol.
(: (-bjo-> string int (Result Exception (Tuple InputPort OutputPort))))
(defbjo (tcp-connect host port)
  ;; Raises here if there is no scope, before a socket exists to leak.
  (ignore (current-scope))
  (match (clr-connect host port)
    ((Ok c) (Ok (connection->ports (clr-own-connection c))))
    ((Err e) (Err e))))

;; ---------------------------------------------------------------------------
;; A connection somebody else made
;; ---------------------------------------------------------------------------

;; The two ports over a duplex `ClrStream` that something other than this module
;; connected: the stream an HTTP server hands over after `101 Switching
;; Protocols`, or a `NetworkStream`.
;;
;; `stream->input-port` and `stream->output-port` each own the stream
;; they wrap, so the first half closed would take the stream from the other.
;; This shares it the way `tcp-connect` does: the stream is disposed when both
;; halves have been closed, or when the ambient scope ends.
;;
;; `peer` is what `peer-address` answers for the input port; `""` means `None`.
(: (-> ClrStream (#:peer string) (Tuple InputPort OutputPort)))
(defun (stream->ports s #:peer "")
  (ignore (current-scope))
  (connection->ports (clr-own-connection (Connection. s peer))))

;; ---------------------------------------------------------------------------
;; Listening
;; ---------------------------------------------------------------------------

;; Binds and listens. `port` may be 0, in which case the operating system picks
;; one and `listener-port` says which.
;;
;; The listener is owned by the ambient scope, and what the scope does at the end
;; is the *hard* stop — see `close-listener!` for the other one.
;;
;; A host name here is resolved synchronously, because binding happens once at
;; startup. An address literal — `"127.0.0.1"`, `"0.0.0.0"`, `"::1"` — asks
;; nothing of the resolver.
(: (-> string int (#:backlog int) (Result Exception Listener)))
(defun (tcp-listen host port #:backlog 128)
  (ignore (current-scope))
  (match (clr-listen host port backlog)
    ((Ok l) (Ok (clr-own-listener l)))
    ((Err e) (Err e))))

;; The port actually bound, which is the interesting question when 0 was asked
;; for. Still answerable after the listener is closed.
(: (-> Listener int))
(defun (listener-port l) (clr-listener-port l))

;; A real CML event: in a `choose` that another branch wins, nothing is accepted
;; and the connection stays for the next acceptor, exactly as a channel receive
;; does. It is also *ephemeral* — each `sync` asks the listener a new question,
;; so a server loop accepts successive connections rather than winning with a
;; remembered one forever.
;;
;; WHY THE VALUE IS A `Result`. A listening socket can fail, and the CML resume
;; path carries a value rather than a raise — so a failure that lands while a
;; fiber is parked in a `choose` has to arrive as one. `promise-join`, `blocking`
;; and the byte ports' reads all answer a `Result` for the same reason.
;;
;; On a closed listener it never commits. `listener-closed` is how a loop
;; notices, exactly as `inbox-closed` is for an inbox:
;;
;;     (let go ()
;;       (match (sync (choose (wrap (accept-evt l) Some)
;;                            (wrap (listener-closed l) (fun (u) None))))
;;         ((Some (Ok conn)) (spawn (handle conn)) (go))
;;         ((Some (Err e))   (log (str "accept failed: " (.-Message e))) (go))
;;         (None             'stopped)))
(: (-> Listener (Event (Result Exception (Tuple InputPort OutputPort)))))
(defun (accept-evt l)
  (wrap (clr-accept-evt l)
        #(match &
           ((Ok c) (Ok (connection->ports c)))
           ((Err e) (Err e)))))

;; Fires once the listener has stopped accepting, and from then on commits
;; immediately on every `sync`.
(: (-> Listener (Event Unit)))
(defun (listener-closed l) (clr-closed-evt l))

(: (-> (Result Exception (Tuple InputPort OutputPort))
       (Tuple InputPort OutputPort)))
(defun (accept-or-raise r)
  (match r
    ((Ok conn) conn)
    ((Err e) (raise e))))

;; One connection, for when there is nothing to choose between. Defined by
;; syncing the event above: a convenience, not a second implementation. The
;; failure is raised here, on the caller's own stack.
;;
;; On a closed listener this waits forever, because the event never commits.
;; A program that may meet one uses `accept-evt` under a `choose` with
;; `listener-closed`.
(: (-> Listener (Tuple InputPort OutputPort)))
(defbjouble (accept l)
  (#:sync (accept-or-raise (sync/blocking (accept-evt l))))
  (#:bjo  (accept-or-raise (sync (accept-evt l)))))

;; Stops accepting, and says so.
;;
;; GRACEFUL, and that is the whole difference from letting the scope end:
;; connections already accepted are **not** touched, because they belong to
;; whoever is handling them and killing a request half-answered is not what
;; "stop listening" means. That is `inbox-close!`'s meaning, one layer out.
;;
;; So a server drains by closing the listener, letting its handlers finish, and
;; leaving the scope — which closes the listening socket and anything still open.
(: (-> Listener void))
(defun (close-listener! l) (clr-close-listener l))

;; ---------------------------------------------------------------------------
;; Who is at the other end
;; ---------------------------------------------------------------------------

;; The peer, as the address and port it connected from, for a log line.
;;
;; It hangs off the input port because the tuple is fixed at two and there is
;; nowhere else to put it. `None` for a port that is not a connection's — a
;; file, a pipe, a `limited` view — and for a `stream->ports` given no peer.
(: (-> InputPort (Option string)))
(defun (peer-address p)
  (match (clr-peer p)
    ((Some "") None)
    (found found)))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/orderedmap.bjo — the sorted map: a persistent B+ tree, keyed in the keys'
;; order. Lookups are O(log n).
;;
;; - The tree is a B+ tree over C# nodes in the runtime, through
;;   `Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps`. The nodes stay in C#
;;   because an internal node holds its keys and children in inline arrays.
;; - Everything that spans several nodes is here: descent, splits, merges,
;;   rotations, iteration, building and joining trees.
;; - A leaf holds up to 64 entries, an internal node up to 31 keys and 32
;;   children. An edit that leaves a node other than the root with fewer
;;   than 8 keys takes a key from a sibling or merges with it. A tree that is
;;   built, split or joined may have smaller nodes at its right edge.
;; - An edit copies the nodes on its path. A transient's owner lets it edit
;;   the nodes it copied in place. The owner `owner-none` makes `editable`
;;   copy every node it is given.
;; - A leaf's value type is not in its C# base type. Every leaf is annotated
;;   with `(Leaf %k %v)` where a `%v` is in scope, and a function with no
;;   `%v` in its signature otherwise takes `vt`, a `(Option %v)` that is
;;   always `None` and only names the type.

(import (std prelude))

(:doc #:file "orderedmap.bjodoc")

(export
  OrderedMap OrderedMapCursor OrderedMapRange OrderedMapBuilder TransientOrderedMap OrderedMapping
  ;; Maps
  orderedmap-empty orderedmap-length orderedmap-empty? orderedmap-clear
  orderedmap-ref orderedmap-ref-or orderedmap-try-ref orderedmap-contains? orderedmap-has-key?
  orderedmap-set orderedmap-set-all orderedmap-remove orderedmap-remove-all
  orderedmap-alter orderedmap-update
  orderedmap-keys orderedmap-values
  orderedmap-merge orderedmap-merge-with
  ;; Order
  orderedmap-min orderedmap-max orderedmap-successor orderedmap-predecessor
  orderedmap-range orderedmap-from orderedmap-until
  ;; Iteration and rebuilding
  orderedmap-map orderedmap-map-values orderedmap-filter orderedmap-fold
  orderedmap-for-each orderedmap-iter orderedmap-any? orderedmap-find
  orderedmap-cursor orderedmap-cursor-done? orderedmap-cursor-current orderedmap-cursor-next
  ;; Parts, for work on them at the same time
  orderedmap-split orderedmap-concat
  ;; Conversions
  list->orderedmap orderedmap->list vec->orderedmap orderedmap->vec seq->orderedmap orderedmap->seq
  map->orderedmap orderedmap->map orderedmap->str
  ;; Builders
  orderedmapbuilder-empty orderedmap->orderedmapbuilder orderedmapbuilder-add! orderedmapbuilder-add-all!
  orderedmapbuilder-length orderedmapbuilder->orderedmap
  ;; Transients
  transientorderedmap-empty orderedmap->transientorderedmap transientorderedmap->orderedmap
  transientorderedmap-set! transientorderedmap-set-all! transientorderedmap-remove!
  transientorderedmap-alter! transientorderedmap-update!
  transientorderedmap-remove-all! transientorderedmap-contains? transientorderedmap-has-key? transientorderedmap-ref
  transientorderedmap-ref-or transientorderedmap-try-ref transientorderedmap-length
  transientorderedmap-empty? transientorderedmap-filter!)

(import/class
  (Exception (: System.Exception))
  (KeyNotFoundException (: System.Collections.Generic.KeyNotFoundException))
  (ArgumentException (: System.ArgumentException))
  (InvalidOperationException (: System.InvalidOperationException))
  ((Node %k) (: Bjolang.Runtime.OrderedTrees.MapNodes.Node))
  ((Leaf %k %v) (: Bjolang.Runtime.OrderedTrees.MapNodes.LeafNode))
  (Owner (: Bjolang.Runtime.OrderedTrees.MapNodes.OwnerId))
  ((Comparer %a) (: System.Collections.Generic.IComparer)))

(import/extern
  (owner-none      (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.OwnerNone    (-> Owner)))
  (owner-next      (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.OwnerNext    (-> Owner)))
  (editable        (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Editable     (-> (Node %k) Owner (Node %k))))
  (new-leaf        (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.NewLeaf      (-> Owner (Leaf %k %v))))
  (new-internal    (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.NewInternal  (-> Owner (Node %k))))
  (as-leaf         (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.AsLeaf       (-> (Node %k) (Leaf %k %v))))
  (leaf-node       (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafAsNode   (-> (Leaf %k %v) (Node %k))))
  (leaf?           (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.IsLeaf       (-> (Node %k) bool)))
  (node-count      (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Count        (-> (Node %k) int)))
  (leaf-key        (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafKey      (-> (Leaf %k %v) int %k)))
  (leaf-value      (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafValue    (-> (Leaf %k %v) int %v)))
  (leaf-keys       (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafKeys     (-> (Leaf %k %v) (Array %k))))
  (leaf-values     (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafValues   (-> (Leaf %k %v) (Array %v))))
  (node-key        (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Key          (-> (Node %k) int %k)))
  (key-of          (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.KeyOf        (-> (Node %k) int %k)))
  (child           (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Child        (-> (Node %k) int (Node %k))))
  (leaf-find       (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafFind     (-> (Leaf %k %v) %k (Comparer %k) int)))
  (route           (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Route        (-> (Node %k) %k (Comparer %k) int)))
  (compare-keys    (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Compare      (-> (Comparer %k) %k %k int)))
  (set-count!      (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.SetCount     (-> (Node %k) int void)))
  (leaf-set!       (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafSet      (-> (Leaf %k %v) int %k %v void)))
  (leaf-set-value! (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafSetValue (-> (Leaf %k %v) int %v void)))
  (set-key!        (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.SetKey       (-> (Node %k) int %k void)))
  (set-child!      (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.SetChild     (-> (Node %k) int (Node %k) void)))
  (leaf-move!      (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafMove     (-> (Leaf %k %v) int (Leaf %k %v) int int void)))
  (key-move!       (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.KeyMove      (-> (Node %k) int (Node %k) int int void)))
  (child-move!     (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.ChildMove    (-> (Node %k) int (Node %k) int int void))))

;; Raise the .NET exceptions the C# map raised, so that a caller's handler
;; still catches them.
(: (-> %a))
(defun (key-not-found)
  (raise (cast Exception (KeyNotFoundException. "The key is not in the map."))))

(: (-> string %a))
(defun (bad-argument why) (raise (cast Exception (ArgumentException. why))))

(def leaf-capacity 64)
(def internal-capacity 32)
(def merge-threshold 8)

;; ---------------------------------------------------------------------------
;; The map
;; ---------------------------------------------------------------------------

;; `%v` is the value type of the leaves under `root`.
(type (: (OrderedMap %k %v) #:opaque
         (Record (: root (Node %k))
                 (: count int)
                 (: comparer (Comparer %k)))))

(: (-> (Comparer %k) (OrderedMap %k %v)))
(defun (empty-with cmp)
  (def (: l (Leaf %k %v)) (new-leaf (owner-none)))
  (OrderedMap (root (leaf-node l)) (count 0) (comparer cmp)))

(: (-> (OrderedMap %k %v)) (where (Ord %k)))
(defun (orderedmap-empty) (empty-with (ord-comparer)))

;; An empty map with the order of `m`.
(: (-> (OrderedMap %k %v) (OrderedMap %k %v)))
(defun (orderedmap-clear m) (empty-with (.comparer m)))

(: (-> (OrderedMap %k %v) int))
(defun (orderedmap-length m) (.count m))

(: (-> (OrderedMap %k %v) bool))
(defun (orderedmap-empty? m) (= (.count m) 0))

;; ---------------------------------------------------------------------------
;; Lookups
;; ---------------------------------------------------------------------------

;; The leaf under `node` that `key` belongs in.
(: (-> (Comparer %k) (Node %k) %k (Leaf %k %v)))
(defun (leaf-for cmp node key)
  (if (leaf? node)
      (as-leaf node)
      (leaf-for cmp (child node (route node key cmp)) key)))

(: (-> (Comparer %k) (Node %k) %k (Option %v)))
(defun (try-ref-in cmp root key)
  (def (: l (Leaf %k %v)) (leaf-for cmp root key))
  (def i (leaf-find l key cmp))
  (if (>= i 0) (Some (leaf-value l i)) None))

(: (-> (Comparer %k) (Node %k) %k %v %v))
(defun (ref-or-in cmp root key fallback)
  (def (: l (Leaf %k %v)) (leaf-for cmp root key))
  (def i (leaf-find l key cmp))
  (if (>= i 0) (leaf-value l i) fallback))

(: (-> (Comparer %k) (Node %k) %k %v))
(defun (ref-in cmp root key)
  (def (: l (Leaf %k %v)) (leaf-for cmp root key))
  (def i (leaf-find l key cmp))
  (if (>= i 0) (leaf-value l i) (key-not-found)))

(: (-> (Comparer %k) (Node %k) %k (Option %v) bool))
(defun (contains-in? cmp root key vt)
  (def (: l (Leaf %k %v)) (leaf-for cmp root key))
  (>= (leaf-find l key cmp) 0))

(: (-> (OrderedMap %k %v) %k (Option %v)))
(defun (orderedmap-try-ref m k) (try-ref-in (.comparer m) (.root m) k))

;; Raises when `key` is not in `m`.
(: (-> (OrderedMap %k %v) %k %v))
(defun (orderedmap-ref m k) (ref-in (.comparer m) (.root m) k))

(: (-> (OrderedMap %k %v) %k %v %v))
(defun (orderedmap-ref-or m k default) (ref-or-in (.comparer m) (.root m) k default))

(: (-> (OrderedMap %k %v) %k bool))
(defun (orderedmap-contains? m k)
  (def (: vt (Option %v)) None)
  (contains-in? (.comparer m) (.root m) k vt))

(: (-> (OrderedMap %k %v) %k bool))
(defun (orderedmap-has-key? m key) (orderedmap-contains? m key))

(: (-> (Node %k) (Node %k)))
(defun (leftmost node) (if (leaf? node) node (leftmost (child node 0))))

(: (-> (Node %k) (Node %k)))
(defun (rightmost node) (if (leaf? node) node (rightmost (child node (node-count node)))))

(: (-> (Node %k) %k))
(defun (first-key node) (key-of (leftmost node) 0))

(: (-> (Node %k) (Option (Tuple %k %v))))
(defun (min-in node)
  (def (: l (Leaf %k %v)) (as-leaf (leftmost node)))
  (if (= (leaf-count l) 0) None (Some (Tuple (leaf-key l 0) (leaf-value l 0)))))

(: (-> (Node %k) (Option (Tuple %k %v))))
(defun (max-in node)
  (def (: l (Leaf %k %v)) (as-leaf (rightmost node)))
  (def n (leaf-count l))
  (if (= n 0) None (Some (Tuple (leaf-key l (- n 1)) (leaf-value l (- n 1))))))

(: (-> (OrderedMap %k %v) (Option (Tuple %k %v))))
(defun (orderedmap-min m) (min-in (.root m)))

(: (-> (OrderedMap %k %v) (Option (Tuple %k %v))))
(defun (orderedmap-max m) (max-in (.root m)))

;; The entry with the least key greater than `key`.
(: (-> (OrderedMap %k %v) %k (Option (Tuple %k %v))))
(defun (orderedmap-successor m k) (successor-in (.comparer m) (.root m) k))

(: (-> (Comparer %k) (Node %k) %k (Option (Tuple %k %v))))
(defun (successor-in cmp node key)
  (if (leaf? node)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf node))
        (def i (leaf-find l key cmp))
        (def at (if (>= i 0) (+ i 1) (- -1 i)))
        (if (< at (leaf-count l)) (Some (Tuple (leaf-key l at) (leaf-value l at))) None))
      (begin
        (def r (route node key cmp))
        (def found (successor-in cmp (child node r) key))
        (if (and (none? found) (< r (node-count node)))
            (min-in (child node (+ r 1)))
            found))))

;; The entry with the greatest key less than `key`.
(: (-> (OrderedMap %k %v) %k (Option (Tuple %k %v))))
(defun (orderedmap-predecessor m k) (predecessor-in (.comparer m) (.root m) k))

(: (-> (Comparer %k) (Node %k) %k (Option (Tuple %k %v))))
(defun (predecessor-in cmp node key)
  (if (leaf? node)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf node))
        (def i (leaf-find l key cmp))
        (def at (if (>= i 0) i (- -1 i)))
        (if (> at 0) (Some (Tuple (leaf-key l (- at 1)) (leaf-value l (- at 1)))) None))
      (begin
        (def r (route node key cmp))
        (def found (predecessor-in cmp (child node r) key))
        (if (and (none? found) (> r 0))
            (max-in (child node (- r 1)))
            found))))

;; ---------------------------------------------------------------------------
;; Slots in one node
;; ---------------------------------------------------------------------------

(: (-> (Leaf %k %v) int))
(defun (leaf-count l) (node-count (leaf-node l)))

(: (-> (Leaf %k %v) int %k %v void))
(defun (leaf-insert! l i key value)
  (def n (leaf-count l))
  (when (< i n) (leaf-move! l i l (+ i 1) (- n i)))
  (leaf-set! l i key value)
  (set-count! (leaf-node l) (+ n 1)))

(: (-> (Leaf %k %v) int void))
(defun (leaf-remove! l i)
  (def n (leaf-count l))
  (when (> (- n i 1) 0) (leaf-move! l (+ i 1) l i (- n i 1)))
  (set-count! (leaf-node l) (- n 1)))

;; Puts `sep` at key slot `i` and `right` at child slot `i + 1`.
(: (-> (Node %k) int %k (Node %k) void))
(defun (internal-insert! node i sep right)
  (def n (node-count node))
  (when (< i n)
    (key-move! node i node (+ i 1) (- n i))
    (child-move! node (+ i 1) node (+ i 2) (- n i)))
  (set-key! node i sep)
  (set-child! node (+ i 1) right)
  (set-count! node (+ n 1)))

;; ---------------------------------------------------------------------------
;; Set
;; ---------------------------------------------------------------------------

;; What `insert!` did:
;; - `(Tuple c leaf _)` with `c` negative: the key is at slot `-1 - c` of
;;   `leaf`, and its value is not written. The path to `leaf` is editable.
;; - `(Tuple 1 _ _)`: the key is added.
;; - `(Tuple 2 right sep)`: the key is added and the node split. `right` is
;;   the new right half, and `sep` its first key.
(: (-> (Comparer %k) (Node %k) %k %v Owner (Tuple int (Node %k) %k)))
(defun (insert! cmp node key value owner)
  (if (leaf? node)
      (insert-leaf! cmp (as-leaf node) key value owner)
      (begin
        (def i (route node key cmp))
        (def c (editable (child node i) owner))
        (set-child! node i c)
        (def r (insert! cmp c key value owner))
        (def (Tuple code right sep) r)
        (cond
          ((< code 2) r)
          ((< (node-count node) (- internal-capacity 1))
           (internal-insert! node i sep right)
           (Tuple 1 node key))
          (else (split-internal! node i sep right owner))))))

(: (-> (Comparer %k) (Leaf %k %v) %k %v Owner (Tuple int (Node %k) %k)))
(defun (insert-leaf! cmp l key value owner)
  (def i (leaf-find l key cmp))
  (cond
    ((>= i 0) (Tuple (- -1 i) (leaf-node l) key))
    ((< (leaf-count l) leaf-capacity)
     (leaf-insert! l (- -1 i) key value)
     (Tuple 1 (leaf-node l) key))
    (else (split-leaf! l (- -1 i) key value owner))))

;; Splits a full leaf and inserts at slot `i`. An insert at either end
;; leaves the full half as it is, so that ascending or descending inserts
;; fill their leaves.
(: (-> (Leaf %k %v) int %k %v Owner (Tuple int (Node %k) %k)))
(defun (split-leaf! left i key value owner)
  (def (: right (Leaf %k %v)) (new-leaf owner))
  (def total (leaf-count left))
  (def at (cond ((= i total) total) ((= i 0) 0) (else (shift-right-logical total 1))))
  (def moving (- total at))
  (when (> moving 0) (leaf-move! left at right 0 moving))
  (set-count! (leaf-node left) at)
  (set-count! (leaf-node right) moving)
  (if (or (< i at) (and (= at 0) (= i 0)))
      (leaf-insert! left i key value)
      (leaf-insert! right (- i at) key value))
  (Tuple 2 (leaf-node right) (leaf-key right 0)))

;; Splits a full internal node, and puts `sep` and `new-child` at slot `i`.
(: (-> (Node %k) int %k (Node %k) Owner (Tuple int (Node %k) %k)))
(defun (split-internal! left i sep new-child owner)
  (def right (new-internal owner))
  (def n (node-count left))
  (def at (shift-right-logical n 1))
  (def up (node-key left at))
  (def moving (- n at 1))
  (key-move! left (+ at 1) right 0 moving)
  (child-move! left (+ at 1) right 0 (+ moving 1))
  (set-count! left at)
  (set-count! right moving)
  (if (<= i at)
      (internal-insert! left i sep new-child)
      (internal-insert! right (- i (+ at 1)) sep new-child))
  (Tuple 2 right up))

(: (-> (Node %k) %k (Node %k) Owner (Node %k)))
(defun (new-root left sep right owner)
  (def r (new-internal owner))
  (set-key! r 0 sep)
  (set-child! r 0 left)
  (set-child! r 1 right)
  (set-count! r 1)
  r)

;; Sets `key` in the tree under `root`. Returns the new root, and whether
;; the key is new.
(: (-> (Comparer %k) (Node %k) %k %v Owner (Tuple (Node %k) bool)))
(defun (set-in cmp root key value owner)
  (def r (editable root owner))
  (def (Tuple code right sep) (insert! cmp r key value owner))
  (cond
    ((< code 0)
     (def (: l (Leaf %k %v)) (as-leaf right))
     (leaf-set-value! l (- -1 code) value)
     (Tuple r #f))
    ((= code 2) (Tuple (new-root r sep right owner) #t))
    (else (Tuple r #t))))

(: (-> (OrderedMap %k %v) %k %v (OrderedMap %k %v)))
(defun (orderedmap-set m k v)
  (def (Tuple root added?) (set-in (.comparer m) (.root m) k v (owner-none)))
  (OrderedMap (root root)
        (count (if added? (+ (.count m) 1) (.count m)))
        (comparer (.comparer m))))

;; ---------------------------------------------------------------------------
;; Remove
;; ---------------------------------------------------------------------------

;; Removes `key` under `node`, which `owner` may edit. Returns 0 if the key
;; is not there, 1 if it is removed, and 2 if it is removed and `node` is
;; left with fewer than `merge-threshold` keys.
(: (-> (Comparer %k) (Node %k) %k Owner (Option %v) int))
(defun (remove! cmp node key owner vt)
  (if (leaf? node)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf node))
        (def i (leaf-find l key cmp))
        (cond
          ((< i 0) 0)
          (else
           (leaf-remove! l i)
           (if (< (leaf-count l) merge-threshold) 2 1))))
      (begin
        (def i (route node key cmp))
        (def c (editable (child node i) owner))
        (set-child! node i c)
        (def r (remove! cmp c key owner vt))
        (cond
          ((< r 2) r)
          ((underflow! node i owner vt) 2)
          (else 1)))))

(: (-> (Node %k) bool))
(defun (can-lend? node) (> (node-count node) (+ merge-threshold 1)))

;; Child `i` of `parent` has too few keys. Takes one from a sibling that has
;; enough, or else merges with a sibling. Returns whether `parent` has too few
;; keys after.
(: (-> (Node %k) int Owner (Option %v) bool))
(defun (underflow! parent i owner vt)
  (cond
    ((< i (node-count parent))
     (def right (editable (child parent (+ i 1)) owner))
     (set-child! parent (+ i 1) right)
     (balance! parent i (child parent i) right #t vt))
    ((> i 0)
     (def left (editable (child parent (- i 1)) owner))
     (set-child! parent (- i 1) left)
     (balance! parent (- i 1) left (child parent i) #f vt))
    (else #t)))

;; `left` and `right` are the children on each side of key `s` of `parent`.
;; The one `from-right?` names lends a key, or the two merge.
(: (-> (Node %k) int (Node %k) (Node %k) bool (Option %v) bool))
(defun (balance! parent s left right from-right? vt)
  (cond
    ((and from-right? (can-lend? right)) (rotate-left! parent s left right vt) #f)
    ((and (not from-right?) (can-lend? left)) (rotate-right! parent s left right vt) #f)
    (else
     (merge! parent s left right vt)
     (< (node-count parent) merge-threshold))))

;; Moves the first entry of `right` to the end of `left`.
(: (-> (Node %k) int (Node %k) (Node %k) (Option %v) void))
(defun (rotate-left! parent s left right vt)
  (if (leaf? left)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf left))
        (def (: r (Leaf %k %v)) (as-leaf right))
        (leaf-insert! l (leaf-count l) (leaf-key r 0) (leaf-value r 0))
        (leaf-remove! r 0)
        (set-key! parent s (leaf-key r 0)))
      (begin
        (internal-insert! left (node-count left) (node-key parent s) (child right 0))
        (set-key! parent s (node-key right 0))
        (def rn (node-count right))
        (child-move! right 1 right 0 rn)
        (when (> rn 1) (key-move! right 1 right 0 (- rn 1)))
        (set-count! right (- rn 1)))))

;; Moves the last entry of `left` to the front of `right`.
(: (-> (Node %k) int (Node %k) (Node %k) (Option %v) void))
(defun (rotate-right! parent s left right vt)
  (if (leaf? left)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf left))
        (def (: r (Leaf %k %v)) (as-leaf right))
        (def last (- (leaf-count l) 1))
        (leaf-insert! r 0 (leaf-key l last) (leaf-value l last))
        (leaf-remove! l last)
        (set-key! parent s (leaf-key r 0)))
      (begin
        (def last (- (node-count left) 1))
        (def rn (node-count right))
        (key-move! right 0 right 1 rn)
        (set-key! right 0 (node-key parent s))
        (child-move! right 0 right 1 (+ rn 1))
        (set-child! right 0 (child left (+ last 1)))
        (set-count! right (+ rn 1))
        (set-key! parent s (node-key left last))
        (set-count! left last))))

;; Moves all of `right` to the end of `left`, and removes key `s` of
;; `parent` and the child after it.
(: (-> (Node %k) int (Node %k) (Node %k) (Option %v) void))
(defun (merge! parent s left right vt)
  (def ln (node-count left))
  (def rn (node-count right))
  (if (leaf? left)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf left))
        (def (: r (Leaf %k %v)) (as-leaf right))
        (leaf-move! r 0 l ln rn)
        (set-count! left (+ ln rn)))
      (begin
        (set-key! left ln (node-key parent s))
        (key-move! right 0 left (+ ln 1) rn)
        (child-move! right 0 left (+ ln 1) (+ rn 1))
        (set-count! left (+ ln 1 rn))))
  (def pn (node-count parent))
  (def moving (- pn s 1))
  (when (> moving 0)
    (key-move! parent (+ s 1) parent s moving)
    (child-move! parent (+ s 2) parent (+ s 1) moving))
  (set-count! parent (- pn 1)))

;; Removes `key` from the tree under `root`. Returns the new root, and
;; whether the key was there. When it was not, the root is `root`.
(: (-> (Comparer %k) (Node %k) %k Owner (Option %v) (Tuple (Node %k) bool)))
(defun (remove-in cmp root key owner vt)
  (def r (editable root owner))
  (def code (remove! cmp r key owner vt))
  (cond
    ((= code 0) (Tuple root #f))
    ((and (= code 2) (not (leaf? r)) (= (node-count r) 0)) (Tuple (child r 0) #t))
    (else (Tuple r #t))))

(: (-> (OrderedMap %k %v) %k (OrderedMap %k %v)))
(defun (orderedmap-remove m k)
  (def (: vt (Option %v)) None)
  (def (Tuple root removed?) (remove-in (.comparer m) (.root m) k (owner-none) vt))
  (if removed?
      (OrderedMap (root root) (count (- (.count m) 1)) (comparer (.comparer m)))
      m))

;; ---------------------------------------------------------------------------
;; Alter and update
;; ---------------------------------------------------------------------------
;;
;; A read descent, then a set or a remove. The read allocates nothing, and the
;; write copies the path as `orderedmap-set` and `orderedmap-remove` do.

;; The value an update applies its function to: the key's value, else the
;; default. Raises `KeyNotFoundException` when there is neither.
(: (-> (Option %v) (Option %v) %v))
(defun (update-start old default)
  (match old
    ((Some v) v)
    (None (match default
            ((Some d) d)
            (None (key-not-found))))))

;; `m` with the entry of `k` decided by `f`:
;; - `f` is given `(Some v)` for the value of `k`, or `None` if `k` is absent.
;; - `(Some v)` from `f` sets `k` to `v`, and `None` removes `k`.
;; - `None` for an absent key returns `m` itself.
(: (-> (OrderedMap %k %v) %k (-?-> (Option %v) (Option %v)) (OrderedMap %k %v)))
(defun (orderedmap-alter m k f)
  (def old (orderedmap-try-ref m k))
  (match (f old)
    ((Some v) (orderedmap-set m k v))
    (None (if (some? old) (orderedmap-remove m k) m))))

;; `m` with `k` set to `f` of its value. A missing key raises
;; `KeyNotFoundException`, unless `#:default (Some d)` is given: then `f` is
;; applied to `d`.
(: (-> (OrderedMap %k %v) %k (-?-> %v %v) (#:default (Option %v)) (OrderedMap %k %v)))
(defun (orderedmap-update m k f #:default None)
  (orderedmap-set m k (f (update-start (orderedmap-try-ref m k) default))))

;; ---------------------------------------------------------------------------
;; The cursor
;; ---------------------------------------------------------------------------

;; A position in an iteration in key order. A value: `orderedmap-cursor-next` returns
;; the next one.
;;
;; - It holds the key and value arrays of one leaf, so a step inside a leaf
;;   is an increment and a compare.
;; - `path` is the child index at each level above the leaf, 6 bits a level
;;   above the low 4 bits, which hold the leaf's depth. `last-leaf` marks a
;;   leaf with no leaf after it.
;; - At the end of a leaf, `cursor-after` follows `path` from the root again.
(type (: (OrderedMapCursor %k %v) #:opaque
         (Struct (: keys (Array %k))
                 (: values (Array %v))
                 (: i int)
                 (: end int)
                 (: root (Node %k))
                 (: path long))))

(def depth-bits 4)
(def index-bits 6)
(def max-depth 10)
(: last-leaf long)
(def last-leaf -1)

(: (-> long int int))
(defun (index-at path level)
  (cast System.Int32 (bitwise-and (shift-right-logical path (+ depth-bits (* index-bits level))) 63)))

(: (-> long int int long))
(defun (with-index path level index)
  (def shift (+ depth-bits (* index-bits level)))
  (def (: mask long) (shift-left 63 shift))
  (bitwise-ior (bitwise-and path (bitwise-not mask))
               (shift-left (cast System.Int64 index) shift)))

(: (-> long int long))
(defun (with-depth path depth)
  (def (: depth-mask long) 15)
  (bitwise-ior (bitwise-and path (bitwise-not depth-mask)) (cast System.Int64 depth)))

(: (-> (Leaf %k %v) int (Node %k) long (OrderedMapCursor %k %v)))
(defun (cursor-at l i root path)
  (OrderedMapCursor (keys (leaf-keys l)) (values (leaf-values l)) (i i) (end (leaf-count l))
              (root root) (path path)))

(: (-> (Node %k) (OrderedMapCursor %k %v)))
(defun (cursor-of root)
  (if (leaf? root)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf root))
        (cursor-at l 0 root last-leaf))
      (descend root root 0 0)))

(: (-> (OrderedMap %k %v) (OrderedMapCursor %k %v)))
(defun (orderedmap-cursor m) (cursor-of (.root m)))

;; The cursor on the first entry of the leftmost leaf under `node`, which is
;; at `level`, or on the entry after it if that leaf is empty.
(: (-> (Node %k) (Node %k) int long (OrderedMapCursor %k %v)))
(defun (descend node root level path)
  (cond
    ((leaf? node)
     (def (: l (Leaf %k %v)) (as-leaf node))
     (def at (with-depth path level))
     (if (> (leaf-count l) 0)
         (cursor-at l 0 root at)
         (advance l root at)))
    ((= level max-depth)
     (raise (cast Exception (InvalidOperationException. "The map is deeper than a cursor can walk."))))
    (else (descend (child node 0) root (+ level 1) (with-index path level 0)))))

;; The cursor on the first entry after the leaf `l` at `path`, or a done
;; cursor on `l`.
(: (-> (Leaf %k %v) (Node %k) long (OrderedMapCursor %k %v)))
(defun (advance l root path)
  (def depth (cast System.Int32 (bitwise-and path 15)))
  ;; The deepest level whose child index can move right, and its node.
  (def/mutable level -1)
  (def/mutable parent root)
  (def/mutable node root)
  (loop (:for d (range 0 depth))
        (:do (def i (index-at path d))
             (when (< i (node-count node))
               (set! level d)
               (set! parent node))
             (set! node (child node i))))
  (if (< level 0)
      (cursor-at l (leaf-count l) root last-leaf)
      (begin
        (def next (+ (index-at path level) 1))
        (descend (child parent next) root (+ level 1) (with-index path level next)))))

;; The cursor on the first entry whose key is not less than `key`, or with
;; `after?` greater than `key`.
(: (-> (Comparer %k) (Node %k) (Node %k) %k bool int long (OrderedMapCursor %k %v)))
(defun (cursor-from cmp node root key after? level path)
  (if (leaf? node)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf node))
        (def i (leaf-find l key cmp))
        (def at (cond ((< i 0) (- -1 i)) (after? (+ i 1)) (else i)))
        (def p (if (= level 0) last-leaf (with-depth path level)))
        (if (or (< at (leaf-count l)) (= level 0))
            (cursor-at l at root p)
            (advance l root p)))
      (begin
        (def r (route node key cmp))
        (cursor-from cmp (child node r) root key after? (+ level 1) (with-index path level r)))))

(: (-> (OrderedMapCursor %k %v) bool))
(defun (orderedmap-cursor-done? c) (>= (.i c) (.end c)))

(: (-> (OrderedMapCursor %k %v) (Tuple %k %v)))
(defun (orderedmap-cursor-current c)
  (Tuple (array-ref (.keys c) (.i c)) (array-ref (.values c) (.i c))))

(: (-> (OrderedMapCursor %k %v) %k))
(defun (cursor-key c) (array-ref (.keys c) (.i c)))

(: (-> (OrderedMapCursor %k %v) (OrderedMapCursor %k %v)))
(defun (orderedmap-cursor-next c)
  (def i (+ (.i c) 1))
  (if (or (< i (.end c)) (= (.path c) last-leaf))
      (record-set c (i i))
      (cursor-after c)))

;; Out of line, so that a step inside a leaf stays small.
(: (-> (OrderedMapCursor %k %v) (OrderedMapCursor %k %v)))
(defun (cursor-after c)
  (def root (.root c))
  (def depth (cast System.Int32 (bitwise-and (.path c) 15)))
  (def/mutable node root)
  (loop (:for d (range 0 depth)) (:do (set! node (child node (index-at (.path c) d)))))
  (def (: l (Leaf %k %v)) (as-leaf node))
  (advance l root (.path c)))

(impl (Iterable (OrderedMap %k %v))
  (type %elem (Tuple %k %v))
  (type %cursor (OrderedMapCursor %k %v))
  (defun (iterable-start m) (orderedmap-cursor m))
  (defun (iterable-done? m c) (orderedmap-cursor-done? c))
  (defun (iterable-current m c) (orderedmap-cursor-current c))
  (defun (iterable-next m c) (orderedmap-cursor-next c))
  (defun (count m) (orderedmap-length m)))

;; ---------------------------------------------------------------------------
;; Ranges
;; ---------------------------------------------------------------------------

;; The entries from `from` to `to`, both included. `orderedmap-range`,
;; `orderedmap-from` and `orderedmap-until` return one, not a `Seq`: a loop over it
;; inlines the cursor. `(->seq r)` returns a `Seq` of it.
;;
;; - `start` is the cursor on the first entry.
;; - `end-i` and `end-path` are the position of the first entry after the
;;   range. A cursor's position is its slot and path, and two cursors on one
;;   entry have the same position, so a step compares two numbers and no key.
;; - With no upper bound, `end-i` is -1, which no cursor has.
;; - When `from` is greater than `to`, `start` is the end.
(type (: (OrderedMapRange %k %v) #:opaque
         (Struct (: start (OrderedMapCursor %k %v)) (: end-i int) (: end-path long))))

(impl (Iterable (OrderedMapRange %k %v))
  (type %elem (Tuple %k %v))
  (type %cursor (OrderedMapCursor %k %v))
  (defun (iterable-start r) (.start r))
  (defun (iterable-done? r c)
    (or (orderedmap-cursor-done? c) (and (= (.i c) (.end-i r)) (= (.path c) (.end-path r)))))
  (defun (iterable-current r c) (orderedmap-cursor-current c))
  (defun (iterable-next r c) (orderedmap-cursor-next c)))

(: (-> (OrderedMap %k %v) %k bool (OrderedMapCursor %k %v)))
(defun (cursor-at-key m key after?) (cursor-from (.comparer m) (.root m) (.root m) key after? 0 0))

(: (-> (OrderedMapCursor %k %v) (OrderedMapCursor %k %v) (OrderedMapRange %k %v)))
(defun (range-of start end)
  (OrderedMapRange (start start) (end-i (.i end)) (end-path (.path end))))

(: (-> (OrderedMap %k %v) %k %k (OrderedMapRange %k %v)))
(defun (orderedmap-range m lo hi)
  (def end (cursor-at-key m hi #t))
  (range-of (if (> (compare-keys (.comparer m) lo hi) 0) end (cursor-at-key m lo #f)) end))

(: (-> (OrderedMap %k %v) %k (OrderedMapRange %k %v)))
(defun (orderedmap-from m lo)
  (OrderedMapRange (start (cursor-at-key m lo #f)) (end-i -1) (end-path 0)))

(: (-> (OrderedMap %k %v) %k (OrderedMapRange %k %v)))
(defun (orderedmap-until m hi) (range-of (orderedmap-cursor m) (cursor-at-key m hi #t)))

(: (-> (OrderedMap %k %v) (Seq %k)))
(defun (orderedmap-keys m) (seql (:for (Tuple k _) m) (:yield k)))

(: (-> (OrderedMap %k %v) (Seq %v)))
(defun (orderedmap-values m) (seql (:for (Tuple _ v) m) (:yield v)))

;; ---------------------------------------------------------------------------
;; Building trees
;; ---------------------------------------------------------------------------

;; One tree over nodes of one depth, in key order: internal nodes of up to 32
;; children each, as even as they can be, level by level up to one root.
(: (-> (Array (Node %k)) (Node %k)))
(defun (over nodes)
  (def n (array-length nodes))
  (if (= n 1)
      (array-ref nodes 0)
      (begin
        (def parents (/ (+ n (- internal-capacity 1)) internal-capacity))
        (def up (make-array parents))
        (def/mutable offset 0)
        (loop (:for p (range 0 parents))
              (:do (def size (+ (/ n parents) (if (< p (% n parents)) 1 0)))
                   (def parent (new-internal (owner-none)))
                   (loop (:for c (range 0 size))
                         (:do (def node (array-ref nodes (+ offset c)))
                              (set-child! parent c node)
                              (when (> c 0) (set-key! parent (- c 1) (first-key node)))))
                   (set-count! parent (- size 1))
                   (array-set! up p parent)
                   (set! offset (+ offset size))))
        (over up))))

;; A map of the first `n` keys and values, in ascending key order, in full
;; leaves.
(: (-> (Array %k) (Array %v) int (Comparer %k) (OrderedMap %k %v)))
(defun (from-sorted ks vs n cmp)
  (if (= n 0)
      (empty-with cmp)
      (begin
        (def count (/ (+ n (- leaf-capacity 1)) leaf-capacity))
        (def leaves (make-array count))
        (loop (:for i (range 0 count))
              (:do (def (: l (Leaf %k %v)) (new-leaf (owner-none)))
                   (def at (* i leaf-capacity))
                   (def size (min leaf-capacity (- n at)))
                   (array-copy! ks at (leaf-keys l) 0 size)
                   (array-copy! vs at (leaf-values l) 0 size)
                   (set-count! (leaf-node l) size)
                   (array-set! leaves i (leaf-node l))))
        (OrderedMap (root (over leaves)) (count n) (comparer cmp)))))

;; ---------------------------------------------------------------------------
;; Builders
;; ---------------------------------------------------------------------------

;; Entries added in any order, made into a map at the end. The last value of
;; a key wins. Entries added in ascending key order need no sort.
(type (: (OrderedMapBuilder %k %v) #:opaque
         (Record (: keys (Array %k) #:mutable)
                 (: values (Array %v) #:mutable)
                 (: count int #:mutable)
                 (: ascending? bool #:mutable)
                 (: comparer (Comparer %k)))))

(: (-> (Comparer %k) int (OrderedMapBuilder %k %v)))
(defun (builder-with cmp capacity)
  (OrderedMapBuilder (keys (make-array capacity)) (values (make-array capacity)) (count 0)
               (ascending? #t) (comparer cmp)))

(: (-> (OrderedMapBuilder %k %v)) (where (Ord %k)))
(defun (orderedmapbuilder-empty) (builder-with (ord-comparer) 16))

;; A builder that holds the entries of `m`.
(: (-> (OrderedMap %k %v) (OrderedMapBuilder %k %v)))
(defun (orderedmap->orderedmapbuilder m)
  (def b (builder-with (.comparer m) (max 16 (.count m))))
  (loop (:for (Tuple k v) m) (:do (orderedmapbuilder-add! b k v)))
  b)

(: (-> (OrderedMapBuilder %k %v) %k %v void))
(defun (orderedmapbuilder-add! b k v)
  (def n (.count b))
  (when (= n (array-length (.keys b)))
    (record-set! b (keys (array-resize (.keys b) (* 2 n))) (values (array-resize (.values b) (* 2 n)))))
  (when (and (.ascending? b) (> n 0)
             (>= (compare-keys (.comparer b) (array-ref (.keys b) (- n 1)) k) 0))
    (record-set! b (ascending? #f)))
  (array-set! (.keys b) n k)
  (array-set! (.values b) n v)
  (record-set! b (count (+ n 1))))

(: (-> (OrderedMapBuilder %k %v) (Seq (Tuple %k %v)) void))
(defun (orderedmapbuilder-add-all! b kvs)
  (loop (:for (Tuple k v) kvs) (:do (orderedmapbuilder-add! b k v))))

;; The number of entries added, a key added twice counted twice.
(: (-> (OrderedMapBuilder %k %v) int))
(defun (orderedmapbuilder-length b) (.count b))

(: (-> (OrderedMapBuilder %k %v) (OrderedMap %k %v)))
(defun (orderedmapbuilder->orderedmap b)
  (def cmp (.comparer b))
  (def n (.count b))
  (def ks (.keys b))
  (def vs (.values b))
  (if (.ascending? b)
      (from-sorted ks vs n cmp)
      (begin
        ;; The slots in key order, equal keys in the order they were added.
        (def order (loop (:for i (range 0 n)) (:acc (arraying i))))
        (array-sort-by! (fun (x y) (compare-keys cmp (array-ref ks x) (array-ref ks y))) order)
        (def uk (make-array n))
        (def uv (make-array n))
        (def/mutable u 0)
        (loop (:for j order)
              (:do (when (or (= u 0) (not (= (compare-keys cmp (array-ref ks j) (array-ref uk (- u 1))) 0)))
                     (set! u (+ u 1)))
                   (array-set! uk (- u 1) (array-ref ks j))
                   (array-set! uv (- u 1) (array-ref vs j))))
        (from-sorted uk uv u cmp))))

;; ---------------------------------------------------------------------------
;; Transients
;; ---------------------------------------------------------------------------

;; A map that edits in place the nodes it copied. `transientorderedmap->orderedmap`
;; returns a map of its contents and gives it a new owner, so that the map
;; does not change after.
(type (: (TransientOrderedMap %k %v) #:opaque
         (Record (: root (Node %k) #:mutable)
                 (: count int #:mutable)
                 (: owner Owner #:mutable)
                 (: comparer (Comparer %k)))))

(: (-> (TransientOrderedMap %k %v)) (where (Ord %k)))
(defun (transientorderedmap-empty) (orderedmap->transientorderedmap (orderedmap-empty)))

(: (-> (OrderedMap %k %v) (TransientOrderedMap %k %v)))
(defun (orderedmap->transientorderedmap m)
  (TransientOrderedMap (root (.root m)) (count (.count m)) (owner (owner-next)) (comparer (.comparer m))))

(: (-> (TransientOrderedMap %k %v) (OrderedMap %k %v)))
(defun (transientorderedmap->orderedmap t)
  (record-set! t (owner (owner-next)))
  (OrderedMap (root (.root t)) (count (.count t)) (comparer (.comparer t))))

(: (-> (TransientOrderedMap %k %v) %k %v void))
(defun (transientorderedmap-set! t k v)
  (def (Tuple root added?) (set-in (.comparer t) (.root t) k v (.owner t)))
  (record-set! t (root root))
  (when added? (record-set! t (count (+ (.count t) 1)))))

(: (-> (TransientOrderedMap %k %v) %k void))
(defun (transientorderedmap-remove! t k)
  (def (: vt (Option %v)) None)
  (def (Tuple root removed?) (remove-in (.comparer t) (.root t) k (.owner t) vt))
  (when removed? (record-set! t (root root) (count (- (.count t) 1)))))

;; `orderedmap-alter` in place.
(: (-> (TransientOrderedMap %k %v) %k (-?-> (Option %v) (Option %v)) void))
(defun (transientorderedmap-alter! t k f)
  (def old (transientorderedmap-try-ref t k))
  (match (f old)
    ((Some v) (transientorderedmap-set! t k v))
    (None (when (some? old) (transientorderedmap-remove! t k)))))

;; `orderedmap-update` in place.
(: (-> (TransientOrderedMap %k %v) %k (-?-> %v %v) (#:default (Option %v)) void))
(defun (transientorderedmap-update! t k f #:default None)
  (transientorderedmap-set! t k (f (update-start (transientorderedmap-try-ref t k) default))))

(: (-> (TransientOrderedMap %k %v) (Seq (Tuple %k %v)) void))
(defun (transientorderedmap-set-all! t kvs)
  (loop (:for (Tuple k v) kvs) (:do (transientorderedmap-set! t k v))))

(: (-> (TransientOrderedMap %k %v) (Seq %k) void))
(defun (transientorderedmap-remove-all! t ks)
  (loop (:for k ks) (:do (transientorderedmap-remove! t k))))

(: (-> (TransientOrderedMap %k %v) %k bool))
(defun (transientorderedmap-contains? t k)
  (def (: vt (Option %v)) None)
  (contains-in? (.comparer t) (.root t) k vt))

(: (-> (TransientOrderedMap %k %v) %k bool))
(defun (transientorderedmap-has-key? t k) (transientorderedmap-contains? t k))

(: (-> (TransientOrderedMap %k %v) %k %v))
(defun (transientorderedmap-ref t k) (ref-in (.comparer t) (.root t) k))

(: (-> (TransientOrderedMap %k %v) %k %v %v))
(defun (transientorderedmap-ref-or t k default) (ref-or-in (.comparer t) (.root t) k default))

(: (-> (TransientOrderedMap %k %v) %k (Option %v)))
(defun (transientorderedmap-try-ref t k) (try-ref-in (.comparer t) (.root t) k))

(: (-> (TransientOrderedMap %k %v) int))
(defun (transientorderedmap-length t) (.count t))

(: (-> (TransientOrderedMap %k %v) bool))
(defun (transientorderedmap-empty? t) (= (.count t) 0))

;; Removes the entries `keep?` rejects. Calls it in key order, before it
;; removes any.
(: (-> (-?-> %k %v bool) (TransientOrderedMap %k %v) void))
(defun (transientorderedmap-filter! keep? t)
  (def/mutable dropped '())
  (loop (:for (Tuple k v) (OrderedMap (root (.root t)) (count (.count t)) (comparer (.comparer t))))
        (:do (unless (keep? k v) (set! dropped (cons k dropped)))))
  (loop (:for k dropped) (:do (transientorderedmap-remove! t k))))

;; ---------------------------------------------------------------------------
;; Iteration
;; ---------------------------------------------------------------------------

;; Folds `f` over the entries in key order. Recursive over the tree, which
;; is about twice as fast as the cursor: it reads each leaf's arrays once.
(: (-> (-?-> %k %v %s %s) %s (OrderedMap %k %v) %s))
(defun (orderedmap-fold f init m) (fold-node f init (.root m)))

(: (-> (-?-> %k %v %s %s) %s (Node %k) %s))
(defun (fold-node f acc node)
  (if (leaf? node)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf node))
        (def ks (leaf-keys l))
        (def vs (leaf-values l))
        (loop (:for i (range 0 (leaf-count l)))
              (:acc s (folding acc (f (array-ref ks i) (array-ref vs i) s)))
              => s))
      (loop (:for i (range 0 (+ (node-count node) 1)))
            (:acc s (folding acc (fold-node f s (child node i))))
            => s)))

(: (-> (-?-> %k %v void) (OrderedMap %k %v) void))
(defun (orderedmap-for-each f m)
  (loop (:for (Tuple k v) m) (:do (f k v))))

;; Calls `f` in key order until it returns #f. Returns whether it never did.
(: (-> (-?-> %k %v bool) (OrderedMap %k %v) bool))
(defun (orderedmap-iter f m)
  (let go ((c (orderedmap-cursor m)))
    (cond
      ((orderedmap-cursor-done? c) #t)
      (:def (Tuple k v) (orderedmap-cursor-current c))
      ((f k v) (go (orderedmap-cursor-next c)))
      (else #f))))

(: (-> (-?-> %k %v bool) (OrderedMap %k %v) bool))
(defun (orderedmap-any? pred m)
  (let go ((c (orderedmap-cursor m)))
    (cond
      ((orderedmap-cursor-done? c) #f)
      (:def (Tuple k v) (orderedmap-cursor-current c))
      ((pred k v) #t)
      (else (go (orderedmap-cursor-next c))))))

;; The first entry in key order that `pred` accepts.
(: (-> (-?-> %k %v bool) (OrderedMap %k %v) (Option (Tuple %k %v))))
(defun (orderedmap-find pred m)
  (let go ((c (orderedmap-cursor m)))
    (cond
      ((orderedmap-cursor-done? c) None)
      (:def (Tuple k v) (orderedmap-cursor-current c))
      ((pred k v) (Some (Tuple k v)))
      (else (go (orderedmap-cursor-next c))))))

;; A map of the same keys, with `f` of each entry. A tree of the same shape:
;; the keys are copied and `f` is called in key order.
(: (-> (-?-> %k %v %w) (OrderedMap %k %v) (OrderedMap %k %w)))
(defun (orderedmap-map f m)
  (def (: vt (Option %v)) None)
  (def (: wt (Option %w)) None)
  (OrderedMap (root (map-node f (.root m) vt wt)) (count (.count m)) (comparer (.comparer m))))

(: (-> (-?-> %k %v %w) (Node %k) (Option %v) (Option %w) (Node %k)))
(defun (map-node f node vt wt)
  (if (leaf? node)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf node))
        (def (: out (Leaf %k %w)) (new-leaf (owner-none)))
        (def n (leaf-count l))
        (def ks (leaf-keys l))
        (def vs (leaf-values l))
        (def ws (leaf-values out))
        (array-copy! ks 0 (leaf-keys out) 0 n)
        (loop (:for i (range 0 n)) (:do (array-set! ws i (f (array-ref ks i) (array-ref vs i)))))
        (set-count! (leaf-node out) n)
        (leaf-node out))
      (begin
        (def out (new-internal (owner-none)))
        (def n (node-count node))
        (key-move! node 0 out 0 n)
        (loop (:for i (range 0 (+ n 1))) (:do (set-child! out i (map-node f (child node i) vt wt))))
        (set-count! out n)
        out)))

(: (-> (-?-> %v %w) (OrderedMap %k %v) (OrderedMap %k %w)))
(defun (orderedmap-map-values f m)
  (def (: vt (Option %v)) None)
  (def (: wt (Option %w)) None)
  (OrderedMap (root (map-values-node f (.root m) vt wt)) (count (.count m)) (comparer (.comparer m))))

(: (-> (-?-> %v %w) (Node %k) (Option %v) (Option %w) (Node %k)))
(defun (map-values-node f node vt wt)
  (if (leaf? node)
      (begin
        (def (: l (Leaf %k %v)) (as-leaf node))
        (def (: out (Leaf %k %w)) (new-leaf (owner-none)))
        (def n (leaf-count l))
        (def vs (leaf-values l))
        (def ws (leaf-values out))
        (array-copy! (leaf-keys l) 0 (leaf-keys out) 0 n)
        (loop (:for i (range 0 n)) (:do (array-set! ws i (f (array-ref vs i)))))
        (set-count! (leaf-node out) n)
        (leaf-node out))
      (begin
        (def out (new-internal (owner-none)))
        (def n (node-count node))
        (key-move! node 0 out 0 n)
        (loop (:for i (range 0 (+ n 1))) (:do (set-child! out i (map-values-node f (child node i) vt wt))))
        (set-count! out n)
        out)))

;; The entries `keep?` accepts, in new leaves filled one after the other.
;; Returns `m` when it accepts every entry.
(: (-> (-?-> %k %v bool) (OrderedMap %k %v) (OrderedMap %k %v)))
(defun (orderedmap-filter keep? m)
  (def n (.count m))
  (def (: ks (Array %k)) (make-array n))
  (def (: vs (Array %v)) (make-array n))
  (def/mutable kept 0)
  (loop (:for (Tuple k v) m)
        (:do (when (keep? k v)
               (array-set! ks kept k)
               (array-set! vs kept v)
               (set! kept (+ kept 1)))))
  (if (= kept n) m (from-sorted ks vs kept (.comparer m))))

;; ---------------------------------------------------------------------------
;; Merges
;; ---------------------------------------------------------------------------

;; The entries of all the maps. The last map's value of a key wins.
(: (-> (OrderedMap %k %v) (OrderedMap %k %v) #:rest (OrderedMap %k %v) (OrderedMap %k %v)))
(defun (orderedmap-merge m other #:rest more)
  (merge-all (fun (k a b) b) (inputs-of m other more)))

;; The entries of all the maps. The values of a key are combined by `f`,
;; first to last.
(: (-> (-?-> %k %v %v %v) (OrderedMap %k %v) (OrderedMap %k %v) #:rest (OrderedMap %k %v) (OrderedMap %k %v)))
(defun (orderedmap-merge-with f m other #:rest more)
  (merge-all f (inputs-of m other more)))

(: (-> (OrderedMap %k %v) (OrderedMap %k %v) (Array (OrderedMap %k %v)) (Array (OrderedMap %k %v))))
(defun (inputs-of m other more)
  (def all (cons m (cons other (loop (:for x more) (:acc (listing x))))))
  (def kept (loop (:for x all) (:when (> (.count x) 0)) (:acc (arraying x))))
  (if (= (array-length kept) 0) (vec->array [m]) kept))

;; At most this many inputs are iterated at once, and only when the inputs
;; other than the biggest hold this percent of the keys or more. Else the
;; others are set into a transient of the biggest.
(def merge-max-inputs 32)
(def merge-percent 30)

(: (-> (-?-> %k %v %v %v) (Array (OrderedMap %k %v)) (OrderedMap %k %v)))
(defun (merge-all f inputs)
  (def k (array-length inputs))
  (cond
    ((= k 1) (array-ref inputs 0))
    (else
     (def total (loop (:for x inputs) (:acc (summing (.count x)))))
     (def biggest (loop (:for x inputs) (:acc b (folding 0 (max b (.count x)))) => b))
     (if (and (<= k merge-max-inputs) (>= (* 100 (- total biggest)) (* merge-percent total)))
         (merge-walk f inputs total)
         (merge-into f inputs)))))

;; The inputs set one after the other into a transient of the first, the
;; values of a key combined in input order.
(: (-> (-?-> %k %v %v %v) (Array (OrderedMap %k %v)) (OrderedMap %k %v)))
(defun (merge-into f inputs)
  (def t (orderedmap->transientorderedmap (array-ref inputs 0)))
  (loop (:for i (range 1 (array-length inputs)))
        (:do (loop (:for (Tuple k v) (array-ref inputs i))
                   (:do (combine! f t k v)))))
  (transientorderedmap->orderedmap t))

;; Sets `k` in `t` to `v`, or to `(f k old v)` if it is there, by one
;; descent.
(: (-> (-?-> %k %v %v %v) (TransientOrderedMap %k %v) %k %v void))
(defun (combine! f t k v)
  (def owner (.owner t))
  (def r (editable (.root t) owner))
  (def (Tuple code right sep) (insert! (.comparer t) r k v owner))
  (cond
    ((< code 0)
     (record-set! t (root r))
     (def (: l (Leaf %k %v)) (as-leaf right))
     (def i (- -1 code))
     (leaf-set-value! l i (f k (leaf-value l i) v)))
    (else
     (record-set! t (root (if (= code 2) (new-root r sep right owner) r))
                    (count (+ (.count t) 1))))))

;; The inputs iterated in key order at once into a builder, which needs no
;; sort. The least key is found by a scan of the cursors.
(: (-> (-?-> %k %v %v %v) (Array (OrderedMap %k %v)) int (OrderedMap %k %v)))
(defun (merge-walk f inputs total)
  (def k (array-length inputs))
  (def cmp (.comparer (array-ref inputs 0)))
  (def cursors (loop (:for x inputs) (:acc (arraying (orderedmap-cursor x)))))
  (def b (builder-with cmp total))
  (def/mutable live k)
  (loop (:while (> live 0))
        (:do
         ;; The least key of the cursors that are not done.
         (def/mutable least -1)
         (loop (:for i (range 0 k))
               (:do (def c (array-ref cursors i))
                    (when (and (not (orderedmap-cursor-done? c))
                               (or (< least 0)
                                   (< (compare-keys cmp (cursor-key c) (cursor-key (array-ref cursors least))) 0)))
                      (set! least i))))
         (def key (cursor-key (array-ref cursors least)))
         ;; Its values, in input order, and the cursors stepped past it.
         (def/mutable value (tuple-second (orderedmap-cursor-current (array-ref cursors least))))
         (array-set! cursors least (orderedmap-cursor-next (array-ref cursors least)))
         (when (orderedmap-cursor-done? (array-ref cursors least)) (set! live (- live 1)))
         (loop (:for i (range (+ least 1) k))
               (:do (def c (array-ref cursors i))
                    (when (and (not (orderedmap-cursor-done? c)) (= (compare-keys cmp (cursor-key c) key) 0))
                      (set! value (f key value (tuple-second (orderedmap-cursor-current c))))
                      (array-set! cursors i (orderedmap-cursor-next c))
                      (when (orderedmap-cursor-done? (array-ref cursors i)) (set! live (- live 1))))))
         (orderedmapbuilder-add! b key value)))
  (orderedmapbuilder->orderedmap b))

(: (-> (Tuple %a %b) %b))
(defun (tuple-second t) (def (Tuple _ b) t) b)

;; ---------------------------------------------------------------------------
;; Parts
;; ---------------------------------------------------------------------------

(: (-> (Node %k) int))
(defun (count-keys node)
  (if (leaf? node)
      (node-count node)
      (loop (:for i (range 0 (+ (node-count node) 1))) (:acc (summing (count-keys (child node i)))))))

;; The leaves under an internal node, or the keys of a leaf.
(: (-> (Node %k) int))
(defun (weight node)
  (cond
    ((leaf? node) (node-count node))
    ((leaf? (child node 0)) (+ (node-count node) 1))
    (else (loop (:for i (range 0 (+ (node-count node) 1))) (:acc (summing (weight (child node i))))))))

;; About `k` maps in key order that together hold the entries of `m` once.
;; They share its nodes: a part is a run of the nodes of the highest level
;; with at least `k` nodes, under new internal nodes. The runs are cut where
;; the running weight passes i * total / k, so the parts are close in size.
(: (-> (OrderedMap %k %v) int (Array (OrderedMap %k %v))))
(defun (orderedmap-split m k)
  (def root (.root m))
  (if (or (<= k 1) (leaf? root))
      (vec->array [m])
      (begin
        (def level (level-of (vec->array [root]) k))
        (def n (array-length level))
        (if (= n 1)
            (vec->array [m])
            (begin
              (def weights (loop (:for x level) (:acc (arraying (weight x)))))
              (def total (loop (:for w weights) (:acc (summing w))))
              (def parts (min k n))
              (def/mutable out '())
              (def/mutable made 0)
              (def/mutable start 0)
              (def/mutable done 0)
              (loop (:for j (range 0 n))
                    (:do (set! done (+ done (array-ref weights j)))
                         (when (or (= j (- n 1))
                                   (and (< made (- parts 1))
                                        (>= (* (cast System.Int64 done) (cast System.Int64 parts))
                                            (* (cast System.Int64 total) (cast System.Int64 (+ made 1))))))
                           (def node (over (loop (:for x (range start (+ j 1))) (:acc (arraying (array-ref level x))))))
                           (set! out (cons (OrderedMap (root node) (count (count-keys node)) (comparer (.comparer m))) out))
                           (set! made (+ made 1))
                           (set! start (+ j 1)))))
              (loop (:for x (list-reverse out)) (:acc (arraying x))))))))

;; The highest level, from `level` down, with at least `k` nodes, or the
;; leaves.
(: (-> (Array (Node %k)) int (Array (Node %k))))
(defun (level-of level k)
  (if (or (>= (array-length level) k) (leaf? (array-ref level 0)))
      level
      (begin
        (def/mutable below '())
        (loop (:for x level)
              (:do (loop (:for i (range 0 (+ (node-count x) 1)))
                         (:do (set! below (cons (child x i) below))))))
        (level-of (list->nodes (list-reverse below)) k))))

;; One map of the maps' entries, in order. Each map's keys must be less than
;; every key of the next. The leaves are kept: an empty one is dropped, and
;; a small one is copied together with its neighbour when the two fit in one
;; leaf.
(: (-> (Array (OrderedMap %k %v)) (OrderedMap %k %v)))
(defun (orderedmap-concat maps)
  (when (= (array-length maps) 0) (bad-argument "orderedmap-concat needs at least one map."))
  (def cmp (.comparer (array-ref maps 0)))
  (def (: vt (Option %v)) None)
  (def/mutable leaves '())
  (def/mutable count 0)
  (def/mutable (: last (Option %k)) None)
  (loop (:for m maps)
        (:when (> (.count m) 0))
        (:do (def lo (first-key (.root m)))
             (when (and (some? last) (>= (compare-keys cmp (option-ref-or last lo) lo) 0))
               (bad-argument "The maps are not in key order: a key of one is not less than every key of the next."))
             (def hi (key-of (rightmost (.root m)) (- (node-count (rightmost (.root m))) 1)))
             (set! last (Some hi))
             (set! leaves (leaves-of (.root m) leaves))
             (set! count (+ count (.count m)))))
  (def joined (join-small (list-reverse leaves) vt))
  (if (list-empty? joined)
      (empty-with cmp)
      (OrderedMap (root (over (list->nodes joined))) (count count) (comparer cmp))))

(: (-> (List (Node %k)) (Array (Node %k))))
(defun (list->nodes xs) (loop (:for x xs) (:acc (arraying x))))

;; The non-empty leaves under `node`, in reverse key order, onto `acc`.
(: (-> (Node %k) (List (Node %k)) (List (Node %k))))
(defun (leaves-of node acc)
  (cond
    ((leaf? node) (if (> (node-count node) 0) (cons node acc) acc))
    (else (loop (:for i (range 0 (+ (node-count node) 1)))
                (:acc a (folding acc (leaves-of (child node i) a)))
                => a))))

;; `leaves` with each small leaf copied together with the one before it when
;; the two fit in one leaf.
(: (-> (List (Node %k)) (Option %v) (List (Node %k))))
(defun (join-small leaves vt)
  (list-reverse
   (loop (:for x leaves)
         (:acc out (folding '()
                            (if (and (not (list-empty? out))
                                     (or (< (node-count (list-head out)) merge-threshold)
                                         (< (node-count x) merge-threshold))
                                     (<= (+ (node-count (list-head out)) (node-count x)) leaf-capacity))
                                (cons (both (list-head out) x vt) (list-tail out))
                                (cons x out))))
         => out)))

;; A new leaf with the entries of `a` and then those of `b`.
(: (-> (Node %k) (Node %k) (Option %v) (Node %k)))
(defun (both a b vt)
  (def (: la (Leaf %k %v)) (as-leaf a))
  (def (: lb (Leaf %k %v)) (as-leaf b))
  (def (: l (Leaf %k %v)) (new-leaf (owner-none)))
  (def na (leaf-count la))
  (def nb (leaf-count lb))
  (leaf-move! la 0 l 0 na)
  (leaf-move! lb 0 l na nb)
  (set-count! (leaf-node l) (+ na nb))
  (leaf-node l))

;; ---------------------------------------------------------------------------
;; Conversions
;; ---------------------------------------------------------------------------

;; Sets each entry, in order, by a transient.
(: (-> (OrderedMap %k %v) (Seq (Tuple %k %v)) (OrderedMap %k %v)))
(defun (orderedmap-set-all m kvs)
  (def t (orderedmap->transientorderedmap m))
  (transientorderedmap-set-all! t kvs)
  (transientorderedmap->orderedmap t))

(: (-> (OrderedMap %k %v) (Seq %k) (OrderedMap %k %v)))
(defun (orderedmap-remove-all m ks)
  (def t (orderedmap->transientorderedmap m))
  (transientorderedmap-remove-all! t ks)
  (transientorderedmap->orderedmap t))

(: (-> (List (Tuple %k %v)) (OrderedMap %k %v)) (where (Ord %k)))
(defun (list->orderedmap xs)
  (def b (orderedmapbuilder-empty))
  (loop (:for (Tuple k v) xs) (:do (orderedmapbuilder-add! b k v)))
  (orderedmapbuilder->orderedmap b))

(: (-> (Vec (Tuple %k %v)) (OrderedMap %k %v)) (where (Ord %k)))
(defun (vec->orderedmap v)
  (def b (orderedmapbuilder-empty))
  (loop (:for (Tuple key value) v) (:do (orderedmapbuilder-add! b key value)))
  (orderedmapbuilder->orderedmap b))

(: (-> (Seq (Tuple %k %v)) (OrderedMap %k %v)) (where (Ord %k)))
(defun (seq->orderedmap xs)
  (def b (orderedmapbuilder-empty))
  (loop (:for (Tuple k v) xs) (:do (orderedmapbuilder-add! b k v)))
  (orderedmapbuilder->orderedmap b))

(: (-> (Map %k %v) (OrderedMap %k %v)) (where (Ord %k)))
(defun (map->orderedmap m)
  (def b (orderedmapbuilder-empty))
  (map-for-each (fun (k v) (orderedmapbuilder-add! b k v)) m)
  (orderedmapbuilder->orderedmap b))

(: (-> (OrderedMap %k %v) (List (Tuple %k %v))))
(defun (orderedmap->list m) (->list m))

(: (-> (OrderedMap %k %v) (Vec (Tuple %k %v))))
(defun (orderedmap->vec m) (->vec m))

(: (-> (OrderedMap %k %v) (Seq (Tuple %k %v))))
(defun (orderedmap->seq m) (->seq m))

(: (-> (OrderedMap %k %v) (Map %k %v)))
(defun (orderedmap->map m)
  (def mb (mapbuilder-empty))
  (loop (:for (Tuple k v) m) (:do (mapbuilder-add! mb k v)))
  (mapbuilder->map mb))

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------

(impl (Refable (OrderedMap %k %v))
  (type %index %k)
  (type %out %v)
  (defun (ref m key) (orderedmap-ref m key))
  (defun (try-ref m key) (orderedmap-try-ref m key)))

(impl (Settable (OrderedMap %k %v))
  (type %index %k)
  (type %in %v)
  (defun (ref-set m key v) (orderedmap-set m key v)))

(impl (Refable (TransientOrderedMap %k %v))
  (type %index %k)
  (type %out %v)
  (defun (ref t key) (transientorderedmap-ref t key))
  (defun (try-ref t key) (transientorderedmap-try-ref t key)))

(impl (Collection (OrderedMap %k %v))
  (type %content (Tuple %k %v))
  (defun (length m) (orderedmap-length m))
  (defun (empty? m) (orderedmap-empty? m)))

(type (: (OrderedMapping %k %v) (Union ordered-mapping)))

(impl (Collector (OrderedMapping %k %v))
  (where (Ord %k))
  (type %elem (Tuple %k %v))
  (type %acc (OrderedMapBuilder %k %v))
  (type %out (OrderedMap %k %v))
  (defun (collector-init c) (orderedmapbuilder-empty))
  (defun (collector-step c acc kv)
    (def (Tuple k v) kv)
    (orderedmapbuilder-add! acc k v)
    acc)
  (defun (collector-finish c acc) (orderedmapbuilder->orderedmap acc)))

(impl (Addable! (OrderedMapBuilder %k %v))
  (type %elem (Tuple %k %v))
  (defun (add! b kv) (def (Tuple k v) kv) (orderedmapbuilder-add! b k v)))

(impl (Addable! (TransientOrderedMap %k %v))
  (type %elem (Tuple %k %v))
  (defun (add! t kv) (def (Tuple k v) kv) (transientorderedmap-set! t k v)))

(impl (Addable (OrderedMap %k %v))
  (type %elem (Tuple %k %v))
  (defun (add m kv) (def (Tuple k v) kv) (orderedmap-set m k v)))

;; In the type's module, so that it is the map's .NET `Equals` and
;; `GetHashCode` too.
(impl (Eq (OrderedMap %k %v))
  (where (Eq %k) (Eq %v))
  (defun (= a b)
    (and (= (orderedmap-length a) (orderedmap-length b))
         (= (orderedmap->list a) (orderedmap->list b))))
  (defun (eq-hash m)
    (orderedmap-fold (fun (k v h) (hash-combine h (hash-combine (eq-hash k) (eq-hash v)))) 17 m)))

;; `#map((k v) ...)` in ascending key order, as a `Map` prints.
(impl (Written (OrderedMap %k %v))
  (where (Written %k) (Written %v))
  (defun (write-into m b)
    (entries-print-into "#map(" m ")" b (fun (k b) (write-into k b)) (fun (v b) (write-into v b)))))

(impl (Displayed (OrderedMap %k %v))
  (where (Displayed %k) (Displayed %v))
  (defun (display-into m b)
    (entries-print-into "#map(" m ")" b (fun (k b) (display-into k b)) (fun (v b) (display-into v b)))))

(: (-> (OrderedMap %k %v) string) (where (Displayed %k) (Displayed %v)))
(defun (orderedmap->str m) (displayed m))

(:deprecated orderedmap->str "use `displayed`")
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/orderedset.bjo — the sorted set: a persistent B+ tree of elements, in
;; their order. Lookups are O(log n).
;;
;; - The tree of `(std orderedmap)` without values, over C# nodes in the
;;   runtime, through `Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps`. The
;;   nodes stay in C# because an internal node holds its keys and children
;;   in inline arrays.
;; - Everything that spans several nodes is here: descent, splits, merges,
;;   rotations, iteration, building and joining trees, and set algebra.
;; - A leaf holds up to 64 elements, an internal node up to 31 keys and 32
;;   children. An edit that leaves a node other than the root with fewer
;;   than 8 keys takes a key from a sibling or merges with it. A tree that is
;;   built, split or joined may have smaller nodes at its right edge.
;; - An edit copies the nodes on its path. A transient's owner lets it edit
;;   the nodes it copied in place. The owner `owner-none` makes `editable`
;;   copy every node it is given.

(import (std prelude))

(:doc #:file "orderedset.bjodoc")

(export
  ;; The types.
  OrderedSet OrderedSetBuilder TransientOrderedSet OrderedSetCursor OrderedSetRange
  OrderedSetting

  ;; Sets
  orderedset-empty orderedset-add orderedset-add-all orderedset-remove
  orderedset-remove-all orderedset-contains? orderedset-length orderedset-empty?
  orderedset-clear
  ;; Set algebra
  orderedset-union orderedset-intersection orderedset-difference
  orderedset-symmetric-difference
  ;; Order — what the tree is for
  orderedset-min orderedset-max orderedset-successor orderedset-predecessor
  orderedset-range orderedset-from orderedset-until
  ;; Walking and rebuilding
  orderedset-map orderedset-filter orderedset-fold orderedset-for-each
  orderedset-iter orderedset-any? orderedset-find
  ;; Parts, for work on them at the same time
  orderedset-split orderedset-concat
  ;; Conversions
  list->orderedset orderedset->list vec->orderedset orderedset->vec
  seq->orderedset orderedset->seq
  orderedset->str
  ;; Builders — the same shape as ListBuilder
  orderedsetbuilder-empty orderedset->orderedsetbuilder orderedsetbuilder-add!
  orderedsetbuilder-add-all! orderedsetbuilder-length orderedsetbuilder->orderedset
  ;; Transients — the set's interface, in place
  transientorderedset-empty orderedset->transientorderedset
  transientorderedset->orderedset transientorderedset-add!
  transientorderedset-add-all! transientorderedset-remove!
  transientorderedset-remove-all! transientorderedset-contains?
  transientorderedset-length transientorderedset-empty?
  transientorderedset-min transientorderedset-max
  transientorderedset-filter!)

(import/class
  (Exception (: System.Exception))
  (ArgumentException (: System.ArgumentException))
  (InvalidOperationException (: System.InvalidOperationException))
  ((Node %k) (: Bjolang.Runtime.OrderedTrees.SetNodes.Node))
  ((Leaf %k) (: Bjolang.Runtime.OrderedTrees.SetNodes.LeafNode))
  (Owner (: Bjolang.Runtime.OrderedTrees.SetNodes.OwnerId))
  ;; What a set orders its elements by. See `ord-comparer`.
  ((Comparer %a) (: System.Collections.Generic.IComparer)))

(import/extern
  (owner-none   (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.OwnerNone   (-> Owner)))
  (owner-next   (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.OwnerNext   (-> Owner)))
  (editable     (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Editable    (-> (Node %k) Owner (Node %k))))
  (new-leaf     (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.NewLeaf     (-> Owner (Leaf %k))))
  (new-internal (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.NewInternal (-> Owner (Node %k))))
  (as-leaf      (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.AsLeaf      (-> (Node %k) (Leaf %k))))
  (leaf-node    (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafAsNode  (-> (Leaf %k) (Node %k))))
  (leaf?        (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.IsLeaf      (-> (Node %k) bool)))
  (node-count   (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Count       (-> (Node %k) int)))
  (leaf-key     (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafKey     (-> (Leaf %k) int %k)))
  (leaf-keys    (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafKeys    (-> (Leaf %k) (Array %k))))
  (node-key     (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Key         (-> (Node %k) int %k)))
  (key-of       (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.KeyOf       (-> (Node %k) int %k)))
  (child        (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Child       (-> (Node %k) int (Node %k))))
  (leaf-find    (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafFind    (-> (Leaf %k) %k (Comparer %k) int)))
  (route        (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Route       (-> (Node %k) %k (Comparer %k) int)))
  (compare-keys (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Compare     (-> (Comparer %k) %k %k int)))
  (set-count!   (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.SetCount    (-> (Node %k) int void)))
  (leaf-set!    (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafSet     (-> (Leaf %k) int %k void)))
  (set-key!     (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.SetKey      (-> (Node %k) int %k void)))
  (set-child!   (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.SetChild    (-> (Node %k) int (Node %k) void)))
  (leaf-move!   (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafMove    (-> (Leaf %k) int (Leaf %k) int int void)))
  (key-move!    (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.KeyMove     (-> (Node %k) int (Node %k) int int void)))
  (child-move!  (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.ChildMove   (-> (Node %k) int (Node %k) int int void)))
  (log2         (: System.Math.Log2               (-> double double))))

;; Raises the .NET exception the C# set raised, so that a caller's handler
;; still catches it.
(: (-> string %a))
(defun (bad-argument why) (raise (cast Exception (ArgumentException. why))))

(def leaf-capacity 64)
(def internal-capacity 32)
(def merge-threshold 8)

;; ---------------------------------------------------------------------------
;; The set
;; ---------------------------------------------------------------------------

(type (: (OrderedSet %a) #:opaque
         (Record (: root (Node %a))
                 (: count int)
                 (: comparer (Comparer %a)))))

(: (-> (Comparer %a) (OrderedSet %a)))
(defun (empty-with cmp)
  (def (: l (Leaf %a)) (new-leaf (owner-none)))
  (OrderedSet (root (leaf-node l)) (count 0) (comparer cmp)))

(: (-> (OrderedSet %a)) (where (Ord %a)))
(defun (orderedset-empty) (empty-with (ord-comparer)))

;; An empty set with the order of `s`.
(: (-> (OrderedSet %a) (OrderedSet %a)))
(defun (orderedset-clear s) (empty-with (.comparer s)))

(: (-> (OrderedSet %a) int))
(defun (orderedset-length s) (.count s))

(: (-> (OrderedSet %a) bool))
(defun (orderedset-empty? s) (= (.count s) 0))

;; ---------------------------------------------------------------------------
;; Lookups
;; ---------------------------------------------------------------------------

;; The leaf under `node` that `key` belongs in.
(: (-> (Comparer %k) (Node %k) %k (Leaf %k)))
(defun (leaf-for cmp node key)
  (if (leaf? node)
      (as-leaf node)
      (leaf-for cmp (child node (route node key cmp)) key)))

(: (-> (Comparer %k) (Node %k) %k bool))
(defun (contains-in? cmp root key)
  (>= (leaf-find (leaf-for cmp root key) key cmp) 0))

(: (-> (OrderedSet %a) %a bool))
(defun (orderedset-contains? s x) (contains-in? (.comparer s) (.root s) x))

(: (-> (Node %k) (Node %k)))
(defun (leftmost node) (if (leaf? node) node (leftmost (child node 0))))

(: (-> (Node %k) (Node %k)))
(defun (rightmost node) (if (leaf? node) node (rightmost (child node (node-count node)))))

(: (-> (Node %k) %k))
(defun (first-key node) (key-of (leftmost node) 0))

(: (-> (Node %k) (Option %k)))
(defun (min-in node)
  (def l (as-leaf (leftmost node)))
  (if (= (leaf-count l) 0) None (Some (leaf-key l 0))))

(: (-> (Node %k) (Option %k)))
(defun (max-in node)
  (def l (as-leaf (rightmost node)))
  (def n (leaf-count l))
  (if (= n 0) None (Some (leaf-key l (- n 1)))))

(: (-> (OrderedSet %a) (Option %a)))
(defun (orderedset-min s) (min-in (.root s)))

(: (-> (OrderedSet %a) (Option %a)))
(defun (orderedset-max s) (max-in (.root s)))

;; The least element greater than `x`.
(: (-> (OrderedSet %a) %a (Option %a)))
(defun (orderedset-successor s x) (successor-in (.comparer s) (.root s) x))

(: (-> (Comparer %k) (Node %k) %k (Option %k)))
(defun (successor-in cmp node key)
  (if (leaf? node)
      (begin
        (def l (as-leaf node))
        (def i (leaf-find l key cmp))
        (def at (if (>= i 0) (+ i 1) (- -1 i)))
        (if (< at (leaf-count l)) (Some (leaf-key l at)) None))
      (begin
        (def r (route node key cmp))
        (def found (successor-in cmp (child node r) key))
        (if (and (none? found) (< r (node-count node)))
            (min-in (child node (+ r 1)))
            found))))

;; The greatest element less than `x`.
(: (-> (OrderedSet %a) %a (Option %a)))
(defun (orderedset-predecessor s x) (predecessor-in (.comparer s) (.root s) x))

(: (-> (Comparer %k) (Node %k) %k (Option %k)))
(defun (predecessor-in cmp node key)
  (if (leaf? node)
      (begin
        (def l (as-leaf node))
        (def i (leaf-find l key cmp))
        (def at (if (>= i 0) i (- -1 i)))
        (if (> at 0) (Some (leaf-key l (- at 1))) None))
      (begin
        (def r (route node key cmp))
        (def found (predecessor-in cmp (child node r) key))
        (if (and (none? found) (> r 0))
            (max-in (child node (- r 1)))
            found))))

;; ---------------------------------------------------------------------------
;; Slots in one node
;; ---------------------------------------------------------------------------

(: (-> (Leaf %k) int))
(defun (leaf-count l) (node-count (leaf-node l)))

(: (-> (Leaf %k) int %k void))
(defun (leaf-insert! l i key)
  (def n (leaf-count l))
  (when (< i n) (leaf-move! l i l (+ i 1) (- n i)))
  (leaf-set! l i key)
  (set-count! (leaf-node l) (+ n 1)))

(: (-> (Leaf %k) int void))
(defun (leaf-remove! l i)
  (def n (leaf-count l))
  (when (> (- n i 1) 0) (leaf-move! l (+ i 1) l i (- n i 1)))
  (set-count! (leaf-node l) (- n 1)))

;; Puts `sep` at key slot `i` and `right` at child slot `i + 1`.
(: (-> (Node %k) int %k (Node %k) void))
(defun (internal-insert! node i sep right)
  (def n (node-count node))
  (when (< i n)
    (key-move! node i node (+ i 1) (- n i))
    (child-move! node (+ i 1) node (+ i 2) (- n i)))
  (set-key! node i sep)
  (set-child! node (+ i 1) right)
  (set-count! node (+ n 1)))

;; ---------------------------------------------------------------------------
;; Add
;; ---------------------------------------------------------------------------

;; What `insert!` did:
;; - `(Tuple 0 _ _)`: the key was there.
;; - `(Tuple 1 _ _)`: the key is added.
;; - `(Tuple 2 right sep)`: the key is added and the node split. `right` is
;;   the new right half, and `sep` its first key.
(: (-> (Comparer %k) (Node %k) %k Owner (Tuple int (Node %k) %k)))
(defun (insert! cmp node key owner)
  (if (leaf? node)
      (insert-leaf! cmp (as-leaf node) key owner)
      (begin
        (def i (route node key cmp))
        (def c (editable (child node i) owner))
        (set-child! node i c)
        (def r (insert! cmp c key owner))
        (def (Tuple code right sep) r)
        (cond
          ((< code 2) r)
          ((< (node-count node) (- internal-capacity 1))
           (internal-insert! node i sep right)
           (Tuple 1 node key))
          (else (split-internal! node i sep right owner))))))

(: (-> (Comparer %k) (Leaf %k) %k Owner (Tuple int (Node %k) %k)))
(defun (insert-leaf! cmp l key owner)
  (def i (leaf-find l key cmp))
  (cond
    ((>= i 0) (Tuple 0 (leaf-node l) key))
    ((< (leaf-count l) leaf-capacity)
     (leaf-insert! l (- -1 i) key)
     (Tuple 1 (leaf-node l) key))
    (else (split-leaf! l (- -1 i) key owner))))

;; Splits a full leaf and inserts at slot `i`. An insert at either end
;; leaves the full half as it is, so that ascending or descending inserts
;; fill their leaves.
(: (-> (Leaf %k) int %k Owner (Tuple int (Node %k) %k)))
(defun (split-leaf! left i key owner)
  (def (: right (Leaf %k)) (new-leaf owner))
  (def total (leaf-count left))
  (def at (cond ((= i total) total) ((= i 0) 0) (else (shift-right-logical total 1))))
  (def moving (- total at))
  (when (> moving 0) (leaf-move! left at right 0 moving))
  (set-count! (leaf-node left) at)
  (set-count! (leaf-node right) moving)
  (if (or (< i at) (and (= at 0) (= i 0)))
      (leaf-insert! left i key)
      (leaf-insert! right (- i at) key))
  (Tuple 2 (leaf-node right) (leaf-key right 0)))

;; Splits a full internal node, and puts `sep` and `new-child` at slot `i`.
(: (-> (Node %k) int %k (Node %k) Owner (Tuple int (Node %k) %k)))
(defun (split-internal! left i sep new-child owner)
  (def right (new-internal owner))
  (def n (node-count left))
  (def at (shift-right-logical n 1))
  (def up (node-key left at))
  (def moving (- n at 1))
  (key-move! left (+ at 1) right 0 moving)
  (child-move! left (+ at 1) right 0 (+ moving 1))
  (set-count! left at)
  (set-count! right moving)
  (if (<= i at)
      (internal-insert! left i sep new-child)
      (internal-insert! right (- i (+ at 1)) sep new-child))
  (Tuple 2 right up))

(: (-> (Node %k) %k (Node %k) Owner (Node %k)))
(defun (new-root left sep right owner)
  (def r (new-internal owner))
  (set-key! r 0 sep)
  (set-child! r 0 left)
  (set-child! r 1 right)
  (set-count! r 1)
  r)

;; Adds `key` to the tree under `root`. Returns the new root, and whether
;; the key is new.
(: (-> (Comparer %k) (Node %k) %k Owner (Tuple (Node %k) bool)))
(defun (add-in cmp root key owner)
  (def r (editable root owner))
  (def (Tuple code right sep) (insert! cmp r key owner))
  (cond
    ((= code 0) (Tuple r #f))
    ((= code 2) (Tuple (new-root r sep right owner) #t))
    (else (Tuple r #t))))

(: (-> (OrderedSet %a) %a (OrderedSet %a)))
(defun (orderedset-add s x)
  (def (Tuple root added?) (add-in (.comparer s) (.root s) x (owner-none)))
  (OrderedSet (root root)
              (count (if added? (+ (.count s) 1) (.count s)))
              (comparer (.comparer s))))

;; ---------------------------------------------------------------------------
;; Remove
;; ---------------------------------------------------------------------------

;; Removes `key` under `node`, which `owner` may edit. Returns 0 if the key
;; is not there, 1 if it is removed, and 2 if it is removed and `node` is
;; left with fewer than `merge-threshold` keys.
(: (-> (Comparer %k) (Node %k) %k Owner int))
(defun (remove! cmp node key owner)
  (if (leaf? node)
      (begin
        (def l (as-leaf node))
        (def i (leaf-find l key cmp))
        (cond
          ((< i 0) 0)
          (else
           (leaf-remove! l i)
           (if (< (leaf-count l) merge-threshold) 2 1))))
      (begin
        (def i (route node key cmp))
        (def c (editable (child node i) owner))
        (set-child! node i c)
        (def r (remove! cmp c key owner))
        (cond
          ((< r 2) r)
          ((underflow! node i owner) 2)
          (else 1)))))

(: (-> (Node %k) bool))
(defun (can-lend? node) (> (node-count node) (+ merge-threshold 1)))

;; Child `i` of `parent` has too few keys. Takes one from a sibling that has
;; enough, or else merges with a sibling. Returns whether `parent` has too few
;; keys after.
(: (-> (Node %k) int Owner bool))
(defun (underflow! parent i owner)
  (cond
    ((< i (node-count parent))
     (def right (editable (child parent (+ i 1)) owner))
     (set-child! parent (+ i 1) right)
     (balance! parent i (child parent i) right #t))
    ((> i 0)
     (def left (editable (child parent (- i 1)) owner))
     (set-child! parent (- i 1) left)
     (balance! parent (- i 1) left (child parent i) #f))
    (else #t)))

;; `left` and `right` are the children on each side of key `s` of `parent`.
;; The one `from-right?` names lends a key, or the two merge.
(: (-> (Node %k) int (Node %k) (Node %k) bool bool))
(defun (balance! parent s left right from-right?)
  (cond
    ((and from-right? (can-lend? right)) (rotate-left! parent s left right) #f)
    ((and (not from-right?) (can-lend? left)) (rotate-right! parent s left right) #f)
    (else
     (merge! parent s left right)
     (< (node-count parent) merge-threshold))))

;; Moves the first key of `right` to the end of `left`.
(: (-> (Node %k) int (Node %k) (Node %k) void))
(defun (rotate-left! parent s left right)
  (if (leaf? left)
      (begin
        (def l (as-leaf left))
        (def r (as-leaf right))
        (leaf-insert! l (leaf-count l) (leaf-key r 0))
        (leaf-remove! r 0)
        (set-key! parent s (leaf-key r 0)))
      (begin
        (internal-insert! left (node-count left) (node-key parent s) (child right 0))
        (set-key! parent s (node-key right 0))
        (def rn (node-count right))
        (child-move! right 1 right 0 rn)
        (when (> rn 1) (key-move! right 1 right 0 (- rn 1)))
        (set-count! right (- rn 1)))))

;; Moves the last key of `left` to the front of `right`.
(: (-> (Node %k) int (Node %k) (Node %k) void))
(defun (rotate-right! parent s left right)
  (if (leaf? left)
      (begin
        (def l (as-leaf left))
        (def r (as-leaf right))
        (def last (- (leaf-count l) 1))
        (leaf-insert! r 0 (leaf-key l last))
        (leaf-remove! l last)
        (set-key! parent s (leaf-key r 0)))
      (begin
        (def last (- (node-count left) 1))
        (def rn (node-count right))
        (key-move! right 0 right 1 rn)
        (set-key! right 0 (node-key parent s))
        (child-move! right 0 right 1 (+ rn 1))
        (set-child! right 0 (child left (+ last 1)))
        (set-count! right (+ rn 1))
        (set-key! parent s (node-key left last))
        (set-count! left last))))

;; Moves all of `right` to the end of `left`, and removes key `s` of
;; `parent` and the child after it.
(: (-> (Node %k) int (Node %k) (Node %k) void))
(defun (merge! parent s left right)
  (def ln (node-count left))
  (def rn (node-count right))
  (if (leaf? left)
      (begin
        (leaf-move! (as-leaf right) 0 (as-leaf left) ln rn)
        (set-count! left (+ ln rn)))
      (begin
        (set-key! left ln (node-key parent s))
        (key-move! right 0 left (+ ln 1) rn)
        (child-move! right 0 left (+ ln 1) (+ rn 1))
        (set-count! left (+ ln 1 rn))))
  (def pn (node-count parent))
  (def moving (- pn s 1))
  (when (> moving 0)
    (key-move! parent (+ s 1) parent s moving)
    (child-move! parent (+ s 2) parent (+ s 1) moving))
  (set-count! parent (- pn 1)))

;; Removes `key` from the tree under `root`. Returns the new root, and
;; whether the key was there. When it was not, the root is `root`.
(: (-> (Comparer %k) (Node %k) %k Owner (Tuple (Node %k) bool)))
(defun (remove-in cmp root key owner)
  (def r (editable root owner))
  (def code (remove! cmp r key owner))
  (cond
    ((= code 0) (Tuple root #f))
    ((and (= code 2) (not (leaf? r)) (= (node-count r) 0)) (Tuple (child r 0) #t))
    (else (Tuple r #t))))

(: (-> (OrderedSet %a) %a (OrderedSet %a)))
(defun (orderedset-remove s x)
  (def (Tuple root removed?) (remove-in (.comparer s) (.root s) x (owner-none)))
  (if removed?
      (OrderedSet (root root) (count (- (.count s) 1)) (comparer (.comparer s)))
      s))

;; ---------------------------------------------------------------------------
;; The cursor
;; ---------------------------------------------------------------------------

;; A position in an iteration in order. A value: the next step returns the
;; next one.
;;
;; - It holds the key array of one leaf, so a step inside a leaf is an
;;   increment and a compare.
;; - `path` is the child index at each level above the leaf, 6 bits a level
;;   above the low 4 bits, which hold the leaf's depth. `last-leaf` marks a
;;   leaf with no leaf after it.
;; - At the end of a leaf, `cursor-after` follows `path` from the root again.
(type (: (OrderedSetCursor %a) #:opaque
         (Struct (: keys (Array %a))
                 (: i int)
                 (: end int)
                 (: root (Node %a))
                 (: path long))))

(def depth-bits 4)
(def index-bits 6)
(def max-depth 10)
(: last-leaf long)
(def last-leaf -1)

(: (-> long int int))
(defun (index-at path level)
  (cast System.Int32 (bitwise-and (shift-right-logical path (+ depth-bits (* index-bits level))) 63)))

(: (-> long int int long))
(defun (with-index path level index)
  (def shift (+ depth-bits (* index-bits level)))
  (def (: mask long) (shift-left 63 shift))
  (bitwise-ior (bitwise-and path (bitwise-not mask))
               (shift-left (cast System.Int64 index) shift)))

(: (-> long int long))
(defun (with-depth path depth)
  (def (: depth-mask long) 15)
  (bitwise-ior (bitwise-and path (bitwise-not depth-mask)) (cast System.Int64 depth)))

(: (-> (Leaf %k) int (Node %k) long (OrderedSetCursor %k)))
(defun (cursor-at l i root path)
  (OrderedSetCursor (keys (leaf-keys l)) (i i) (end (leaf-count l)) (root root) (path path)))

(: (-> (Node %k) (OrderedSetCursor %k)))
(defun (cursor-of root)
  (if (leaf? root)
      (cursor-at (as-leaf root) 0 root last-leaf)
      (descend root root 0 0)))

;; The cursor on the first key of the leftmost leaf under `node`, which is
;; at `level`, or on the key after it if that leaf is empty.
(: (-> (Node %k) (Node %k) int long (OrderedSetCursor %k)))
(defun (descend node root level path)
  (cond
    ((leaf? node)
     (def l (as-leaf node))
     (def at (with-depth path level))
     (if (> (leaf-count l) 0)
         (cursor-at l 0 root at)
         (advance l root at)))
    ((= level max-depth)
     (raise (cast Exception (InvalidOperationException. "The set is deeper than a cursor can walk."))))
    (else (descend (child node 0) root (+ level 1) (with-index path level 0)))))

;; The cursor on the first key after the leaf `l` at `path`, or a done
;; cursor on `l`.
(: (-> (Leaf %k) (Node %k) long (OrderedSetCursor %k)))
(defun (advance l root path)
  (def depth (cast System.Int32 (bitwise-and path 15)))
  ;; The deepest level whose child index can move right, and its node.
  (def/mutable level -1)
  (def/mutable parent root)
  (def/mutable node root)
  (loop (:for d (range 0 depth))
        (:do (def i (index-at path d))
             (when (< i (node-count node))
               (set! level d)
               (set! parent node))
             (set! node (child node i))))
  (if (< level 0)
      (cursor-at l (leaf-count l) root last-leaf)
      (begin
        (def next (+ (index-at path level) 1))
        (descend (child parent next) root (+ level 1) (with-index path level next)))))

;; The cursor on the first key not less than `key`, or with `after?`
;; greater than `key`.
(: (-> (Comparer %k) (Node %k) (Node %k) %k bool int long (OrderedSetCursor %k)))
(defun (cursor-from cmp node root key after? level path)
  (if (leaf? node)
      (begin
        (def l (as-leaf node))
        (def i (leaf-find l key cmp))
        (def at (cond ((< i 0) (- -1 i)) (after? (+ i 1)) (else i)))
        (def p (if (= level 0) last-leaf (with-depth path level)))
        (if (or (< at (leaf-count l)) (= level 0))
            (cursor-at l at root p)
            (advance l root p)))
      (begin
        (def r (route node key cmp))
        (cursor-from cmp (child node r) root key after? (+ level 1) (with-index path level r)))))

(: (-> (OrderedSetCursor %k) bool))
(defun (cursor-done? c) (>= (.i c) (.end c)))

(: (-> (OrderedSetCursor %k) %k))
(defun (cursor-key c) (array-ref (.keys c) (.i c)))

(: (-> (OrderedSetCursor %k) (OrderedSetCursor %k)))
(defun (cursor-step c)
  (def i (+ (.i c) 1))
  (if (or (< i (.end c)) (= (.path c) last-leaf))
      (record-set c (i i))
      (cursor-after c)))

;; Out of line, so that a step inside a leaf stays small.
(: (-> (OrderedSetCursor %k) (OrderedSetCursor %k)))
(defun (cursor-after c)
  (def root (.root c))
  (def depth (cast System.Int32 (bitwise-and (.path c) 15)))
  (def/mutable node root)
  (loop (:for d (range 0 depth)) (:do (set! node (child node (index-at (.path c) d)))))
  (advance (as-leaf node) root (.path c)))

(impl (Iterable (OrderedSet %a))
  (type %elem %a)
  (type %cursor (OrderedSetCursor %a))
  (defun (iterable-start s) (cursor-of (.root s)))
  (defun (iterable-done? s c) (cursor-done? c))
  (defun (iterable-current s c) (cursor-key c))
  (defun (iterable-next s c) (cursor-step c))
  (defun (count s) (orderedset-length s)))

;; ---------------------------------------------------------------------------
;; Ranges
;; ---------------------------------------------------------------------------

;; The elements from one bound to another, both included. `orderedset-range`,
;; `orderedset-from` and `orderedset-until` return one, not a `Seq`: a loop
;; over it inlines the cursor. `(->seq r)` returns a `Seq` of it.
;;
;; - `start` is the cursor on the first element.
;; - `end-i` and `end-path` are the position of the first element after the
;;   range. A cursor's position is its slot and path, and two cursors on one
;;   element have the same position, so a step compares two numbers and no
;;   element.
;; - With no upper bound, `end-i` is -1, which no cursor has.
;; - When the lower bound is greater than the upper, `start` is the end.
(type (: (OrderedSetRange %a) #:opaque
         (Struct (: start (OrderedSetCursor %a)) (: end-i int) (: end-path long))))

(impl (Iterable (OrderedSetRange %a))
  (type %elem %a)
  (type %cursor (OrderedSetCursor %a))
  (defun (iterable-start r) (.start r))
  (defun (iterable-done? r c)
    (or (cursor-done? c) (and (= (.i c) (.end-i r)) (= (.path c) (.end-path r)))))
  (defun (iterable-current r c) (cursor-key c))
  (defun (iterable-next r c) (cursor-step c)))

(: (-> (OrderedSet %a) %a bool (OrderedSetCursor %a)))
(defun (cursor-at-key s key after?) (cursor-from (.comparer s) (.root s) (.root s) key after? 0 0))

(: (-> (OrderedSetCursor %a) (OrderedSetCursor %a) (OrderedSetRange %a)))
(defun (range-of start end)
  (OrderedSetRange (start start) (end-i (.i end)) (end-path (.path end))))

(: (-> (OrderedSet %a) %a %a (OrderedSetRange %a)))
(defun (orderedset-range s lo hi)
  (def end (cursor-at-key s hi #t))
  (range-of (if (> (compare-keys (.comparer s) lo hi) 0) end (cursor-at-key s lo #f)) end))

(: (-> (OrderedSet %a) %a (OrderedSetRange %a)))
(defun (orderedset-from s lo)
  (OrderedSetRange (start (cursor-at-key s lo #f)) (end-i -1) (end-path 0)))

(: (-> (OrderedSet %a) %a (OrderedSetRange %a)))
(defun (orderedset-until s hi) (range-of (cursor-of (.root s)) (cursor-at-key s hi #t)))

;; ---------------------------------------------------------------------------
;; Building trees
;; ---------------------------------------------------------------------------

;; One tree over nodes of one depth, in order: internal nodes of up to 32
;; children each, as even as they can be, level by level up to one root.
(: (-> (Array (Node %k)) (Node %k)))
(defun (over nodes)
  (def n (array-length nodes))
  (if (= n 1)
      (array-ref nodes 0)
      (begin
        (def parents (/ (+ n (- internal-capacity 1)) internal-capacity))
        (def up (make-array parents))
        (def/mutable offset 0)
        (loop (:for p (range 0 parents))
              (:do (def size (+ (/ n parents) (if (< p (% n parents)) 1 0)))
                   (def parent (new-internal (owner-none)))
                   (loop (:for c (range 0 size))
                         (:do (def node (array-ref nodes (+ offset c)))
                              (set-child! parent c node)
                              (when (> c 0) (set-key! parent (- c 1) (first-key node)))))
                   (set-count! parent (- size 1))
                   (array-set! up p parent)
                   (set! offset (+ offset size))))
        (over up))))

;; A set of the first `n` elements of `ks`, which are in ascending order and
;; distinct, in full leaves.
(: (-> (Array %a) int (Comparer %a) (OrderedSet %a)))
(defun (from-sorted ks n cmp)
  (if (= n 0)
      (empty-with cmp)
      (begin
        (def count (/ (+ n (- leaf-capacity 1)) leaf-capacity))
        (def leaves (make-array count))
        (loop (:for i (range 0 count))
              (:do (def (: l (Leaf %a)) (new-leaf (owner-none)))
                   (def at (* i leaf-capacity))
                   (def size (min leaf-capacity (- n at)))
                   (array-copy! ks at (leaf-keys l) 0 size)
                   (set-count! (leaf-node l) size)
                   (array-set! leaves i (leaf-node l))))
        (OrderedSet (root (over leaves)) (count n) (comparer cmp)))))

;; The elements of `s`, in order, in a new array.
(: (-> (OrderedSet %a) (Array %a)))
(defun (elements-of s)
  (def (: out (Array %a)) (make-array (.count s)))
  (def/mutable i 0)
  (loop (:for x s) (:do (array-set! out i x) (set! i (+ i 1))))
  out)

;; ---------------------------------------------------------------------------
;; Builders
;; ---------------------------------------------------------------------------

;; Elements added in any order, made into a set at the end. Elements added
;; in ascending order need no sort.
(type (: (OrderedSetBuilder %a) #:opaque
         (Record (: keys (Array %a) #:mutable)
                 (: count int #:mutable)
                 (: ascending? bool #:mutable)
                 (: comparer (Comparer %a)))))

(: (-> (Comparer %a) int (OrderedSetBuilder %a)))
(defun (builder-with cmp capacity)
  (OrderedSetBuilder (keys (make-array capacity)) (count 0) (ascending? #t) (comparer cmp)))

(: (-> (OrderedSetBuilder %a)) (where (Ord %a)))
(defun (orderedsetbuilder-empty) (builder-with (ord-comparer) 16))

;; A builder that holds the elements of `s`.
(: (-> (OrderedSet %a) (OrderedSetBuilder %a)))
(defun (orderedset->orderedsetbuilder s)
  (def b (builder-with (.comparer s) (max 16 (.count s))))
  (loop (:for x s) (:do (orderedsetbuilder-add! b x)))
  b)

;; "Ascending" means strictly: an element equal to the one before makes the
;; build sort, which drops the repeat.
(: (-> (OrderedSetBuilder %a) %a void))
(defun (orderedsetbuilder-add! b x)
  (def n (.count b))
  (when (= n (array-length (.keys b)))
    (record-set! b (keys (array-resize (.keys b) (* 2 n)))))
  (when (and (.ascending? b) (> n 0)
             (>= (compare-keys (.comparer b) (array-ref (.keys b) (- n 1)) x) 0))
    (record-set! b (ascending? #f)))
  (array-set! (.keys b) n x)
  (record-set! b (count (+ n 1))))

(: (-> (OrderedSetBuilder %a) (Seq %a) void))
(defun (orderedsetbuilder-add-all! b xs)
  (loop (:for x xs) (:do (orderedsetbuilder-add! b x))))

;; The number of elements added, an element added twice counted twice.
(: (-> (OrderedSetBuilder %a) int))
(defun (orderedsetbuilder-length b) (.count b))

(: (-> (OrderedSetBuilder %a) (OrderedSet %a)))
(defun (orderedsetbuilder->orderedset b)
  (def cmp (.comparer b))
  (def n (.count b))
  (if (.ascending? b)
      (from-sorted (.keys b) n cmp)
      (begin
        (def ks (array-resize (.keys b) n))
        (array-sort-by! (fun (x y) (compare-keys cmp x y)) ks)
        ;; Drops repeats in place.
        (def/mutable u 0)
        (loop (:for i (range 0 n))
              (:do (when (or (= u 0) (not (= (compare-keys cmp (array-ref ks i) (array-ref ks (- u 1))) 0)))
                     (array-set! ks u (array-ref ks i))
                     (set! u (+ u 1)))))
        (from-sorted ks u cmp))))

;; ---------------------------------------------------------------------------
;; Transients
;; ---------------------------------------------------------------------------

;; A set that edits in place the nodes it copied.
;; `transientorderedset->orderedset` returns a set of its contents and gives
;; it a new owner, so that the set does not change after.
(type (: (TransientOrderedSet %a) #:opaque
         (Record (: root (Node %a) #:mutable)
                 (: count int #:mutable)
                 (: owner Owner #:mutable)
                 (: comparer (Comparer %a)))))

(: (-> (TransientOrderedSet %a)) (where (Ord %a)))
(defun (transientorderedset-empty) (orderedset->transientorderedset (orderedset-empty)))

(: (-> (OrderedSet %a) (TransientOrderedSet %a)))
(defun (orderedset->transientorderedset s)
  (TransientOrderedSet (root (.root s)) (count (.count s)) (owner (owner-next)) (comparer (.comparer s))))

(: (-> (TransientOrderedSet %a) (OrderedSet %a)))
(defun (transientorderedset->orderedset t)
  (record-set! t (owner (owner-next)))
  (OrderedSet (root (.root t)) (count (.count t)) (comparer (.comparer t))))

(: (-> (TransientOrderedSet %a) %a void))
(defun (transientorderedset-add! t x)
  (def (Tuple root added?) (add-in (.comparer t) (.root t) x (.owner t)))
  (record-set! t (root root))
  (when added? (record-set! t (count (+ (.count t) 1)))))

(: (-> (TransientOrderedSet %a) %a void))
(defun (transientorderedset-remove! t x)
  (def (Tuple root removed?) (remove-in (.comparer t) (.root t) x (.owner t)))
  (when removed? (record-set! t (root root) (count (- (.count t) 1)))))

(: (-> (TransientOrderedSet %a) (Seq %a) void))
(defun (transientorderedset-add-all! t xs)
  (loop (:for x xs) (:do (transientorderedset-add! t x))))

(: (-> (TransientOrderedSet %a) (Seq %a) void))
(defun (transientorderedset-remove-all! t xs)
  (loop (:for x xs) (:do (transientorderedset-remove! t x))))

(: (-> (TransientOrderedSet %a) %a bool))
(defun (transientorderedset-contains? t x) (contains-in? (.comparer t) (.root t) x))

(: (-> (TransientOrderedSet %a) int))
(defun (transientorderedset-length t) (.count t))

(: (-> (TransientOrderedSet %a) bool))
(defun (transientorderedset-empty? t) (= (.count t) 0))

(: (-> (TransientOrderedSet %a) (Option %a)))
(defun (transientorderedset-min t) (min-in (.root t)))

(: (-> (TransientOrderedSet %a) (Option %a)))
(defun (transientorderedset-max t) (max-in (.root t)))

;; Removes the elements `keep?` rejects. Calls it in order, before it
;; removes any.
(: (-> (-?-> %a bool) (TransientOrderedSet %a) void))
(defun (transientorderedset-filter! keep? t)
  (def/mutable dropped '())
  (loop (:for x (OrderedSet (root (.root t)) (count (.count t)) (comparer (.comparer t))))
        (:do (unless (keep? x) (set! dropped (cons x dropped)))))
  (loop (:for x dropped) (:do (transientorderedset-remove! t x))))

(: (-> (OrderedSet %a) (Seq %a) (OrderedSet %a)))
(defun (orderedset-add-all s xs)
  (def t (orderedset->transientorderedset s))
  (transientorderedset-add-all! t xs)
  (transientorderedset->orderedset t))

(: (-> (OrderedSet %a) (Seq %a) (OrderedSet %a)))
(defun (orderedset-remove-all s xs)
  (def t (orderedset->transientorderedset s))
  (transientorderedset-remove-all! t xs)
  (transientorderedset->orderedset t))

;; ---------------------------------------------------------------------------
;; Iteration and rebuilding
;; ---------------------------------------------------------------------------

;; Folds `f` over the elements in order. Recursive over the tree, which is
;; about twice as fast as the cursor: it reads each leaf's array once.
(: (-> (-?-> %a %s %s) %s (OrderedSet %a) %s))
(defun (orderedset-fold f init s) (fold-node f init (.root s)))

(: (-> (-?-> %a %s %s) %s (Node %a) %s))
(defun (fold-node f acc node)
  (if (leaf? node)
      (begin
        (def l (as-leaf node))
        (def ks (leaf-keys l))
        (loop (:for i (range 0 (leaf-count l)))
              (:acc s (folding acc (f (array-ref ks i) s)))
              => s))
      (loop (:for i (range 0 (+ (node-count node) 1)))
            (:acc s (folding acc (fold-node f s (child node i))))
            => s)))

(: (-> (-?-> %a void) (OrderedSet %a) void))
(defun (orderedset-for-each f s)
  (loop (:for x s) (:do (f x))))

;; Calls `f` in order until it returns #f. Returns whether it never did.
(: (-> (-?-> %a bool) (OrderedSet %a) bool))
(defun (orderedset-iter f s)
  (let go ((c (cursor-of (.root s))))
    (cond
      ((cursor-done? c) #t)
      ((f (cursor-key c)) (go (cursor-step c)))
      (else #f))))

(: (-> (-?-> %a bool) (OrderedSet %a) bool))
(defun (orderedset-any? pred s)
  (let go ((c (cursor-of (.root s))))
    (cond
      ((cursor-done? c) #f)
      ((pred (cursor-key c)) #t)
      (else (go (cursor-step c))))))

;; The first element in order that `pred` accepts.
(: (-> (-?-> %a bool) (OrderedSet %a) (Option %a)))
(defun (orderedset-find pred s)
  (let go ((c (cursor-of (.root s))))
    (cond
      ((cursor-done? c) None)
      (:def x (cursor-key c))
      ((pred x) (Some x))
      (else (go (cursor-step c))))))

;; `f` of each element, ordered by `%b`'s order. `f` is called in order.
(: (-> (-?-> %a %b) (OrderedSet %a) (OrderedSet %b)) (where (Ord %b)))
(defun (orderedset-map f s)
  (def b (builder-with (ord-comparer) (max 16 (.count s))))
  (loop (:for x s) (:do (orderedsetbuilder-add! b (f x))))
  (orderedsetbuilder->orderedset b))

;; The elements `keep?` accepts, in new leaves filled one after the other.
;; Returns `s` when it accepts every element.
(: (-> (-?-> %a bool) (OrderedSet %a) (OrderedSet %a)))
(defun (orderedset-filter keep? s)
  (def n (.count s))
  (def (: ks (Array %a)) (make-array n))
  (def/mutable kept 0)
  (loop (:for x s)
        (:do (when (keep? x)
               (array-set! ks kept x)
               (set! kept (+ kept 1)))))
  (if (= kept n) s (from-sorted ks kept (.comparer s))))

;; ---------------------------------------------------------------------------
;; Set algebra
;; ---------------------------------------------------------------------------

;; The sets that have elements, or `s` alone when none has.
(: (-> (OrderedSet %a) (OrderedSet %a) (Array (OrderedSet %a)) (Array (OrderedSet %a))))
(defun (inputs-of s other more)
  (def all (cons s (cons other (loop (:for x more) (:acc (listing x))))))
  (def kept (loop (:for x all) (:when (> (.count x) 0)) (:acc (arraying x))))
  (if (= (array-length kept) 0) (vec->array [s]) kept))

;; At most this many sets are iterated at once, and only when the sets other
;; than the biggest hold this percent of the elements or more. Else the
;; others are added to a transient of the biggest.
(def union-max-inputs 32)
(def union-percent 30)

;; The elements of any of the sets.
(: (-> (OrderedSet %a) (OrderedSet %a) #:rest (OrderedSet %a) (OrderedSet %a)))
(defun (orderedset-union s other #:rest more)
  (def inputs (inputs-of s other more))
  (def k (array-length inputs))
  (cond
    ((= k 1) (array-ref inputs 0))
    (else
     (def total (loop (:for x inputs) (:acc (summing (.count x)))))
     (def/mutable biggest 0)
     (loop (:for i (range 1 k))
           (:do (when (> (.count (array-ref inputs i)) (.count (array-ref inputs biggest)))
                  (set! biggest i))))
     (if (and (<= k union-max-inputs)
              (>= (* 100 (- total (.count (array-ref inputs biggest)))) (* union-percent total)))
         (union-walk inputs total)
         (union-into inputs biggest)))))

;; The sets other than `biggest` added to a transient of it.
(: (-> (Array (OrderedSet %a)) int (OrderedSet %a)))
(defun (union-into inputs biggest)
  (def t (orderedset->transientorderedset (array-ref inputs biggest)))
  (loop (:for i (range 0 (array-length inputs)))
        (:do (unless (= i biggest)
               (loop (:for x (array-ref inputs i)) (:do (transientorderedset-add! t x))))))
  (transientorderedset->orderedset t))

;; The sets iterated in order at once into an array. The least element is
;; found by a scan of the cursors.
(: (-> (Array (OrderedSet %a)) int (OrderedSet %a)))
(defun (union-walk inputs total)
  (def k (array-length inputs))
  (def cmp (.comparer (array-ref inputs 0)))
  (def cursors (loop (:for x inputs) (:acc (arraying (cursor-of (.root x))))))
  (def (: out (Array %a)) (make-array total))
  (def/mutable n 0)
  (def/mutable live k)
  (loop (:while (> live 0))
        (:do
         (def/mutable least -1)
         (loop (:for i (range 0 k))
               (:do (def c (array-ref cursors i))
                    (when (and (not (cursor-done? c))
                               (or (< least 0)
                                   (< (compare-keys cmp (cursor-key c) (cursor-key (array-ref cursors least))) 0)))
                      (set! least i))))
         (def x (cursor-key (array-ref cursors least)))
         (array-set! out n x)
         (set! n (+ n 1))
         ;; Every cursor on `x` steps past it.
         (loop (:for i (range least k))
               (:do (def c (array-ref cursors i))
                    (when (and (not (cursor-done? c)) (= (compare-keys cmp (cursor-key c) x) 0))
                      (def next (cursor-step c))
                      (array-set! cursors i next)
                      (when (cursor-done? next) (set! live (- live 1))))))))
  (from-sorted out n cmp))

;; Keeps the first `n` elements of `kept`, in order, that `against` has
;; (`keep?` #t) or lacks (#f), at the front of `kept`. Returns how many.
;;
;; - A probe of `against` costs a descent, about 20 steps of an iteration of
;;   it. So few elements are probed, and many are compared with an iteration
;;   of `against` in order.
(: (-> (Array %a) int (OrderedSet %a) bool int))
(defun (keep-in! kept n against keep?)
  (def cmp (.comparer against))
  (def/mutable w 0)
  (if (< (* (cast long n) 20) (cast long (.count against)))
      (loop (:for i (range 0 n))
            (:do (def x (array-ref kept i))
                 (when (= (contains-in? cmp (.root against) x) keep?)
                   (array-set! kept w x)
                   (set! w (+ w 1)))))
      (begin
        (def/mutable c (cursor-of (.root against)))
        (loop (:for i (range 0 n))
              (:do (def x (array-ref kept i))
                   (loop (:while (and (not (cursor-done? c)) (< (compare-keys cmp (cursor-key c) x) 0)))
                         (:do (set! c (cursor-step c))))
                   (def there? (and (not (cursor-done? c)) (= (compare-keys cmp (cursor-key c) x) 0)))
                   (when (= there? keep?)
                     (array-set! kept w x)
                     (set! w (+ w 1)))))))
  w)

;; The elements in every set. Starts from the smallest set, since the result
;; can only shrink, and keeps what each other set has.
(: (-> (OrderedSet %a) (OrderedSet %a) #:rest (OrderedSet %a) (OrderedSet %a)))
(defun (orderedset-intersection s other #:rest more)
  (def all (loop (:for x (cons s (cons other (loop (:for x more) (:acc (listing x)))))) (:acc (arraying x))))
  (def/mutable least 0)
  (loop (:for i (range 1 (array-length all)))
        (:do (when (< (.count (array-ref all i)) (.count (array-ref all least)))
               (set! least i))))
  (def smallest (array-ref all least))
  (def size (.count smallest))
  (def kept (elements-of smallest))
  (def/mutable n size)
  (loop (:for i (range 0 (array-length all)))
        (:while (> n 0))
        (:do (unless (= i least) (set! n (keep-in! kept n (array-ref all i) #t)))))
  (if (= n size) smallest (from-sorted kept n (.comparer smallest))))

;; The elements of `s` that no other set has.
;;
;; - Removes the others' elements from a transient of `s` when they are
;;   few: a removal costs a descent and copies of nodes, about three probes.
;; - Else keeps each element of `s` that each other set lacks.
(: (-> (OrderedSet %a) (OrderedSet %a) #:rest (OrderedSet %a) (OrderedSet %a)))
(defun (orderedset-difference s other #:rest more)
  (def others
    (loop (:for x (cons other (loop (:for x more) (:acc (listing x)))))
          (:when (> (.count x) 0))
          (:acc (arraying x))))
  (def size (.count s))
  (cond
    ((or (= size 0) (= (array-length others) 0)) s)
    (else
     (def depth (log2 (cast double (+ size 2))))
     (def removing
       (loop (:for o others) (:acc (summing (* 3.0 (cast double (.count o)) depth)))))
     (def filtering
       (loop (:for o others)
             (:acc (summing (min (* (cast double size) (log2 (cast double (+ (.count o) 2))))
                                 (cast double (+ size (.count o))))))))
     (if (< removing filtering)
         (begin
           (def t (orderedset->transientorderedset s))
           (loop (:for o others) (:do (loop (:for x o) (:do (transientorderedset-remove! t x)))))
           (transientorderedset->orderedset t))
         (begin
           (def kept (elements-of s))
           (def/mutable n size)
           (loop (:for o others)
                 (:while (> n 0))
                 (:do (set! n (keep-in! kept n o #f))))
           (if (= n size) s (from-sorted kept n (.comparer s))))))))

;; The elements of one of the two sets and not of both, by an iteration of
;; both in order.
(: (-> (OrderedSet %a) (OrderedSet %a) (OrderedSet %a)))
(defun (orderedset-symmetric-difference s other)
  (def cmp (.comparer s))
  (def (: out (Array %a)) (make-array (+ (.count s) (.count other))))
  (def/mutable n 0)
  (: put! (-> %a void))
  (defun (put! x) (array-set! out n x) (set! n (+ n 1)))
  (let go ((a (cursor-of (.root s))) (c (cursor-of (.root other))))
    (cond
      ((and (cursor-done? a) (cursor-done? c)) unit)
      ((cursor-done? a) (put! (cursor-key c)) (go a (cursor-step c)))
      ((cursor-done? c) (put! (cursor-key a)) (go (cursor-step a) c))
      (else
       (def order (compare-keys cmp (cursor-key a) (cursor-key c)))
       (cond
         ((< order 0) (put! (cursor-key a)) (go (cursor-step a) c))
         ((> order 0) (put! (cursor-key c)) (go a (cursor-step c)))
         (else (go (cursor-step a) (cursor-step c)))))))
  (from-sorted out n cmp))

;; ---------------------------------------------------------------------------
;; Parts
;; ---------------------------------------------------------------------------

(: (-> (Node %k) int))
(defun (count-keys node)
  (if (leaf? node)
      (node-count node)
      (loop (:for i (range 0 (+ (node-count node) 1))) (:acc (summing (count-keys (child node i)))))))

;; The leaves under an internal node, or the keys of a leaf.
(: (-> (Node %k) int))
(defun (weight node)
  (cond
    ((leaf? node) (node-count node))
    ((leaf? (child node 0)) (+ (node-count node) 1))
    (else (loop (:for i (range 0 (+ (node-count node) 1))) (:acc (summing (weight (child node i))))))))

;; About `k` sets in order that together hold the elements of `s` once.
;; They share its nodes: a part is a run of the nodes of the highest level
;; with at least `k` nodes, under new internal nodes. The runs are cut where
;; the running weight passes i * total / k, so the parts are close in size.
(: (-> (OrderedSet %a) int (Array (OrderedSet %a))))
(defun (orderedset-split s k)
  (def root (.root s))
  (if (or (<= k 1) (leaf? root))
      (vec->array [s])
      (begin
        (def level (level-of (vec->array [root]) k))
        (def n (array-length level))
        (if (= n 1)
            (vec->array [s])
            (begin
              (def weights (loop (:for x level) (:acc (arraying (weight x)))))
              (def total (loop (:for w weights) (:acc (summing w))))
              (def parts (min k n))
              (def/mutable out '())
              (def/mutable made 0)
              (def/mutable start 0)
              (def/mutable done 0)
              (loop (:for j (range 0 n))
                    (:do (set! done (+ done (array-ref weights j)))
                         (when (or (= j (- n 1))
                                   (and (< made (- parts 1))
                                        (>= (* (cast System.Int64 done) (cast System.Int64 parts))
                                            (* (cast System.Int64 total) (cast System.Int64 (+ made 1))))))
                           (def node (over (loop (:for x (range start (+ j 1))) (:acc (arraying (array-ref level x))))))
                           (set! out (cons (OrderedSet (root node) (count (count-keys node)) (comparer (.comparer s))) out))
                           (set! made (+ made 1))
                           (set! start (+ j 1)))))
              (loop (:for x (list-reverse out)) (:acc (arraying x))))))))

;; The highest level, from `level` down, with at least `k` nodes, or the
;; leaves.
(: (-> (Array (Node %k)) int (Array (Node %k))))
(defun (level-of level k)
  (if (or (>= (array-length level) k) (leaf? (array-ref level 0)))
      level
      (begin
        (def/mutable below '())
        (loop (:for x level)
              (:do (loop (:for i (range 0 (+ (node-count x) 1)))
                         (:do (set! below (cons (child x i) below))))))
        (level-of (list->nodes (list-reverse below)) k))))

;; One set of the sets' elements, in order. Each set's elements must be less
;; than every element of the next. The leaves are kept: an empty one is
;; dropped, and a small one is copied together with its neighbour when the
;; two fit in one leaf.
(: (-> (Array (OrderedSet %a)) (OrderedSet %a)))
(defun (orderedset-concat sets)
  (when (= (array-length sets) 0) (bad-argument "orderedset-concat needs at least one set."))
  (def cmp (.comparer (array-ref sets 0)))
  (def/mutable leaves '())
  (def/mutable count 0)
  (def/mutable (: last (Option %a)) None)
  (loop (:for s sets)
        (:when (> (.count s) 0))
        (:do (def lo (first-key (.root s)))
             (when (and (some? last) (>= (compare-keys cmp (option-ref-or last lo) lo) 0))
               (bad-argument "The sets are not in order: an element of one is not less than every element of the next."))
             (def hi (key-of (rightmost (.root s)) (- (node-count (rightmost (.root s))) 1)))
             (set! last (Some hi))
             (set! leaves (leaves-of (.root s) leaves))
             (set! count (+ count (.count s)))))
  (def joined (join-small (list-reverse leaves)))
  (if (list-empty? joined)
      (empty-with cmp)
      (OrderedSet (root (over (list->nodes joined))) (count count) (comparer cmp))))

(: (-> (List (Node %k)) (Array (Node %k))))
(defun (list->nodes xs) (loop (:for x xs) (:acc (arraying x))))

;; The non-empty leaves under `node`, in reverse order, onto `acc`.
(: (-> (Node %k) (List (Node %k)) (List (Node %k))))
(defun (leaves-of node acc)
  (cond
    ((leaf? node) (if (> (node-count node) 0) (cons node acc) acc))
    (else (loop (:for i (range 0 (+ (node-count node) 1)))
                (:acc a (folding acc (leaves-of (child node i) a)))
                => a))))

;; `leaves` with each small leaf copied together with the one before it when
;; the two fit in one leaf.
(: (-> (List (Node %k)) (List (Node %k))))
(defun (join-small leaves)
  (list-reverse
   (loop (:for x leaves)
         (:acc out (folding '()
                            (if (and (not (list-empty? out))
                                     (or (< (node-count (list-head out)) merge-threshold)
                                         (< (node-count x) merge-threshold))
                                     (<= (+ (node-count (list-head out)) (node-count x)) leaf-capacity))
                                (cons (both (list-head out) x) (list-tail out))
                                (cons x out))))
         => out)))

;; A new leaf with the keys of `a` and then those of `b`.
(: (-> (Node %k) (Node %k) (Node %k)))
(defun (both a b)
  (def la (as-leaf a))
  (def lb (as-leaf b))
  (def (: l (Leaf %k)) (new-leaf (owner-none)))
  (def na (leaf-count la))
  (def nb (leaf-count lb))
  (leaf-move! la 0 l 0 na)
  (leaf-move! lb 0 l na nb)
  (set-count! (leaf-node l) (+ na nb))
  (leaf-node l))

;; ---------------------------------------------------------------------------
;; Conversions
;; ---------------------------------------------------------------------------

(: (-> (List %a) (OrderedSet %a)) (where (Ord %a)))
(defun (list->orderedset xs)
  (def b (orderedsetbuilder-empty))
  (loop (:for x xs) (:do (orderedsetbuilder-add! b x)))
  (orderedsetbuilder->orderedset b))

(: (-> (OrderedSet %a) (List %a)))
(defun (orderedset->list s) (->list s))

(: (-> (Vec %a) (OrderedSet %a)) (where (Ord %a)))
(defun (vec->orderedset v)
  (def b (orderedsetbuilder-empty))
  (loop (:for x v) (:do (orderedsetbuilder-add! b x)))
  (orderedsetbuilder->orderedset b))

(: (-> (OrderedSet %a) (Vec %a)))
(defun (orderedset->vec s) (->vec s))

(: (-> (Seq %a) (OrderedSet %a)) (where (Ord %a)))
(defun (seq->orderedset xs)
  (def b (orderedsetbuilder-empty))
  (orderedsetbuilder-add-all! b xs)
  (orderedsetbuilder->orderedset b))

(: (-> (OrderedSet %a) (Seq %a)))
(defun (orderedset->seq s) (->seq s))

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------

(impl (Refable (OrderedSet %a))
  (type %index %a)
  (type %out bool)
  (defun (ref set key)
    (orderedset-contains? set key)))

(impl (Refable (TransientOrderedSet %a))
  (type %index %a)
  (type %out bool)
  (defun (ref set key)
    (transientorderedset-contains? set key)))

(impl (Collection (OrderedSet %a))
  (type %content %a)
  (defun (length s) (orderedset-length s))
  (defun (empty? s) (orderedset-empty? s)))

(type (: (OrderedSetting %a) (Union ordered-setting)))

(impl (Collector (OrderedSetting %a))
  (where (Ord %a))
  (type %elem %a)
  (type %acc (OrderedSetBuilder %a))
  (type %out (OrderedSet %a))
  (defun (collector-init c) (orderedsetbuilder-empty))
  (defun (collector-step c acc x) (orderedsetbuilder-add! acc x) acc)
  (defun (collector-finish c acc) (orderedsetbuilder->orderedset acc)))

;; `Addable!`, adds an element at a time.
(impl (Addable! (OrderedSetBuilder %a))
  (type %elem %a)
  (defun (add! sb elem)
    (orderedsetbuilder-add! sb elem)))

(impl (Addable! (TransientOrderedSet %a))
  (type %elem %a)
  (defun (add! ts elem)
    (transientorderedset-add! ts elem)))

;; `Addable` adds an element immutably.
(impl (Addable (OrderedSet %a))
  (type %elem %a)
  (defun (add s elem)
    (orderedset-add s elem)))

;; In the type's module, so that it is the set's .NET `Equals` and
;; `GetHashCode` too.
(impl (Eq (OrderedSet %a))
  (where (Eq %a))
  (defun (= a b)
    (and (= (orderedset-length a) (orderedset-length b))
         (= (orderedset->list a) (orderedset->list b))))
  (defun (eq-hash s)
    (orderedset-fold (fun (x h) (hash-combine h (eq-hash x))) 17 s)))

;; `#{a b c}`, in ascending order.
(impl (Written (OrderedSet %a))
  (where (Written %a))
  (defun (write-into s b) (elements-print-into "#{" s "}" b (fun (x b) (write-into x b)))))

(impl (Displayed (OrderedSet %a))
  (where (Displayed %a))
  (defun (display-into s b) (elements-print-into "#{" s "}" b (fun (x b) (display-into x b)))))

(: (-> (OrderedSet %a) string) (where (Displayed %a)))
(defun (orderedset->str s) (displayed s))

(:deprecated orderedset->str "use `displayed`")
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; The operating system the program runs on, the machine and the process, and
;; the user's standard folders.
;;
;; - Which system: .NET's `OperatingSystem.IsXYZ`. Each check is a call of the
;;   .NET method itself, at the call. .NET's core library is built for one
;;   OS, so the method returns a constant, and the JIT removes the branch that
;;   cannot run. A program is built once and chooses at run time.
;; - The machine and the process: `RuntimeInformation`, `Environment` and the
;;   GC's memory limit.
;; - The folders: `Environment.GetFolderPath`, which follows the XDG
;;   variables on Linux, and the cache folder, which it does not have.
;; - The signals: .NET's `PosixSignalRegistration`, through
;;   `Bjolang.Runtime.SignalGroup`, and an inbox for each kind of signal.

(import (std prelude))
(import (std syntax-match))
(import (std inbox))

(:doc #:file "os.bjodoc")

(export
  ;; Which system.
  windows? linux? macos? freebsd? android? ios? mac-catalyst? tvos? watchos? browser? wasi?
  os-platform?
  windows-version-at-least? macos-version-at-least? freebsd-version-at-least?
  android-version-at-least? ios-version-at-least? mac-catalyst-version-at-least?
  tvos-version-at-least? watchos-version-at-least? os-platform-version-at-least?
  ;; The machine and the process.
  os-description os-version os-architecture process-architecture runtime-identifier
  dotnet-description processor-count available-memory machine-name user-name
  process-id process-path privileged?
  ;; The user's folders.
  home-directory config-directory data-directory cache-directory
  documents-directory desktop-directory temp-directory make-temp-directory
  ;; Signals.
  Signal Signals signal-subscribe signal-recv signal-unsubscribe! signal-name
  on-signal with-signal-cancel
  ;; Named by the template of `with-signal-cancel`, so they have to be
  ;; reachable where it expands.
  signal-cancel-arm! signal-cancel-result)

(import/extern
  (windows?      (: System.OperatingSystem.IsWindows (-> bool)))
  (linux?        (: System.OperatingSystem.IsLinux (-> bool)))
  (macos?        (: System.OperatingSystem.IsMacOS (-> bool)))
  (freebsd?      (: System.OperatingSystem.IsFreeBSD (-> bool)))
  (android?      (: System.OperatingSystem.IsAndroid (-> bool)))
  (ios?          (: System.OperatingSystem.IsIOS (-> bool)))
  (mac-catalyst? (: System.OperatingSystem.IsMacCatalyst (-> bool)))
  (tvos?         (: System.OperatingSystem.IsTvOS (-> bool)))
  (watchos?      (: System.OperatingSystem.IsWatchOS (-> bool)))
  (browser?      (: System.OperatingSystem.IsBrowser (-> bool)))
  (wasi?         (: System.OperatingSystem.IsWasi (-> bool)))
  (os-platform?  (: System.OperatingSystem.IsOSPlatform (-> string bool))))

;; The version checks take .NET's trailing optional parameters as keywords.
;; Each wrapper is copied into its calls, so the call is .NET's.
(import/extern
  (clr-windows-at-least      (: System.OperatingSystem.IsWindowsVersionAtLeast (-> int int int int bool)))
  (clr-macos-at-least        (: System.OperatingSystem.IsMacOSVersionAtLeast (-> int int int bool)))
  (clr-freebsd-at-least      (: System.OperatingSystem.IsFreeBSDVersionAtLeast (-> int int int int bool)))
  (clr-android-at-least      (: System.OperatingSystem.IsAndroidVersionAtLeast (-> int int int int bool)))
  (clr-ios-at-least          (: System.OperatingSystem.IsIOSVersionAtLeast (-> int int int bool)))
  (clr-mac-catalyst-at-least (: System.OperatingSystem.IsMacCatalystVersionAtLeast (-> int int int bool)))
  (clr-tvos-at-least         (: System.OperatingSystem.IsTvOSVersionAtLeast (-> int int int bool)))
  (clr-watchos-at-least      (: System.OperatingSystem.IsWatchOSVersionAtLeast (-> int int int bool)))
  (clr-platform-at-least     (: System.OperatingSystem.IsOSPlatformVersionAtLeast
                                (-> string int int int int bool))))

(: (-> int (#:minor int) (#:build int) (#:revision int) bool))
(defun (windows-version-at-least? major #:minor 0 #:build 0 #:revision 0) #:inline
  (clr-windows-at-least major minor build revision))

(: (-> int (#:minor int) (#:build int) bool))
(defun (macos-version-at-least? major #:minor 0 #:build 0) #:inline
  (clr-macos-at-least major minor build))

(: (-> int (#:minor int) (#:build int) (#:revision int) bool))
(defun (freebsd-version-at-least? major #:minor 0 #:build 0 #:revision 0) #:inline
  (clr-freebsd-at-least major minor build revision))

(: (-> int (#:minor int) (#:build int) (#:revision int) bool))
(defun (android-version-at-least? major #:minor 0 #:build 0 #:revision 0) #:inline
  (clr-android-at-least major minor build revision))

(: (-> int (#:minor int) (#:build int) bool))
(defun (ios-version-at-least? major #:minor 0 #:build 0) #:inline
  (clr-ios-at-least major minor build))

(: (-> int (#:minor int) (#:build int) bool))
(defun (mac-catalyst-version-at-least? major #:minor 0 #:build 0) #:inline
  (clr-mac-catalyst-at-least major minor build))

(: (-> int (#:minor int) (#:build int) bool))
(defun (tvos-version-at-least? major #:minor 0 #:build 0) #:inline
  (clr-tvos-at-least major minor build))

(: (-> int (#:minor int) (#:build int) bool))
(defun (watchos-version-at-least? major #:minor 0 #:build 0) #:inline
  (clr-watchos-at-least major minor build))

(: (-> string int (#:minor int) (#:build int) (#:revision int) bool))
(defun (os-platform-version-at-least? platform major #:minor 0 #:build 0 #:revision 0) #:inline
  (clr-platform-at-least platform major minor build revision))

;; ---------------------------------------------------------------------------
;; The machine and the process
;; ---------------------------------------------------------------------------

(import/class
  (Architecture (: System.Runtime.InteropServices.Architecture))
  (ClrVersion (: System.Version))
  (ClrOperatingSystem (: System.OperatingSystem))
  (GCMemoryInfo (: System.GCMemoryInfo)))

;; A static property is imported as a value. Each is given as a function, as
;; the checks above are, and copied into its calls, so the call is the read of
;; the property.
(import/extern
  (clr-os-description   (: System.Runtime.InteropServices.RuntimeInformation.OSDescription (-> string) #:get))
  (clr-runtime-identifier
   (: System.Runtime.InteropServices.RuntimeInformation.RuntimeIdentifier (-> string) #:get))
  (clr-dotnet-description
   (: System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription (-> string) #:get))
  (clr-os-architecture  (: System.Runtime.InteropServices.RuntimeInformation.OSArchitecture (-> Architecture) #:get))
  (clr-process-architecture
   (: System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture (-> Architecture) #:get))
  (clr-os               (: System.Environment.OSVersion (-> ClrOperatingSystem) #:get))
  (clr-os-version       (: System.OperatingSystem.Version (-> ClrOperatingSystem ClrVersion) #:get))
  (clr-major            (: System.Version.Major (-> ClrVersion int) #:get))
  (clr-minor            (: System.Version.Minor (-> ClrVersion int) #:get))
  (clr-build            (: System.Version.Build (-> ClrVersion int) #:get))
  (clr-revision         (: System.Version.Revision (-> ClrVersion int) #:get))
  (clr-processor-count  (: System.Environment.ProcessorCount (-> int) #:get))
  (clr-machine-name     (: System.Environment.MachineName (-> string) #:get))
  (clr-user-name        (: System.Environment.UserName (-> string) #:get))
  (clr-process-id       (: System.Environment.ProcessId (-> int) #:get))
  (clr-process-path     (: System.Environment.ProcessPath (-> string) #:get))
  (clr-privileged?      (: System.Environment.IsPrivilegedProcess (-> bool) #:get))
  (clr-memory-info      (: System.GC.GetGCMemoryInfo (-> GCMemoryInfo)))
  (clr-total-available  (: System.GCMemoryInfo.TotalAvailableMemoryBytes (-> GCMemoryInfo long) #:get)))

(: (-> string))
(defun (os-description) #:inline clr-os-description)

(: (-> string))
(defun (runtime-identifier) #:inline clr-runtime-identifier)

(: (-> string))
(defun (dotnet-description) #:inline clr-dotnet-description)

(: (-> int))
(defun (processor-count) #:inline clr-processor-count)

(: (-> string))
(defun (machine-name) #:inline clr-machine-name)

(: (-> string))
(defun (user-name) #:inline clr-user-name)

(: (-> int))
(defun (process-id) #:inline clr-process-id)

(: (-> bool))
(defun (privileged?) #:inline clr-privileged?)

;; .NET's name of an architecture, in lower case, as a runtime identifier
;; writes it.
(: (-> Architecture string))
(defun (architecture-name a) (string-downcase (.ToString a)))

(: (-> string))
(defun (os-architecture) (architecture-name clr-os-architecture))

(: (-> string))
(defun (process-architecture) (architecture-name clr-process-architecture))

;; The parts of the version that .NET has: a part it does not know is -1, and
;; is left out.
(: (-> (Vec int)))
(defun (os-version)
  (def v (clr-os-version clr-os))
  (vec-filter (fun (n) (>= n 0)) [(clr-major v) (clr-minor v) (clr-build v) (clr-revision v)]))

;; The memory the GC may use: the machine's, or the limit of the container or
;; of the process.
(: (-> long))
(defun (available-memory) (clr-total-available (clr-memory-info)))

;; .NET has null when it cannot tell, which arrives as "".
(: (-> (Option string)))
(defun (process-path)
  (def p clr-process-path)
  (if (string-empty? p) None (Some p)))

;; ---------------------------------------------------------------------------
;; The user's folders
;; ---------------------------------------------------------------------------

(import/class
  (SpecialFolder (: System.Environment+SpecialFolder))
  (SpecialFolderOption (: System.Environment+SpecialFolderOption))
  (ClrDirectoryInfo (: System.IO.DirectoryInfo))
  (InvalidOperationException (: System.InvalidOperationException (-> string InvalidOperationException))))

(import/extern
  (clr-folder-path       (: System.Environment.GetFolderPath (-> SpecialFolder SpecialFolderOption string)))
  (clr-temp-path         (: System.IO.Path.GetTempPath (-> string)))
  (clr-trim-separator    (: System.IO.Path.TrimEndingDirectorySeparator (-> string string)))
  (clr-rooted?           (: System.IO.Path.IsPathRooted (-> string bool)))
  (clr-create-temp       (: System.IO.Directory.CreateTempSubdirectory (-> string ClrDirectoryInfo)))
  (clr-full-name         (: System.IO.DirectoryInfo.FullName (-> ClrDirectoryInfo string) #:get)))

;; The path of a special folder, whether or not it exists yet. Raises when
;; the system has none, as when HOME is not set and the user has no home in
;; the password database.
(: (-> SpecialFolder string string))
(defun (folder-path folder what)
  (def p (clr-folder-path folder SpecialFolderOption.DoNotVerify))
  (if (string-empty? p)
      (raise (cast Exception (InvalidOperationException. (str "This system has no " what "."))))
      p))

(: (-> string))
(defun (home-directory) (folder-path SpecialFolder.UserProfile "home directory"))

(: (-> string))
(defun (config-directory) (folder-path SpecialFolder.ApplicationData "folder for configuration"))

(: (-> string))
(defun (data-directory) (folder-path SpecialFolder.LocalApplicationData "folder for data"))

(: (-> string))
(defun (documents-directory) (folder-path SpecialFolder.MyDocuments "folder for documents"))

(: (-> string))
(defun (desktop-directory) (folder-path SpecialFolder.Desktop "desktop folder"))

;; .NET has no special folder for a cache. On Windows it is the local data
;; folder, on macOS `~/Library/Caches`, and elsewhere `XDG_CACHE_HOME`, or
;; `~/.cache` when that is not set or not an absolute path, as the XDG Base
;; Directory specification says.
(: (-> string))
(defun (cache-directory)
  (cond
    ((windows?) (data-directory))
    ((macos?) (path-combine (home-directory) "Library" "Caches"))
    (else
     (match (get-environment-variable "XDG_CACHE_HOME")
       ((Some p) (if (clr-rooted? p) p (path-combine (home-directory) ".cache")))
       (None (path-combine (home-directory) ".cache"))))))

;; The folder for temporary files: `TMPDIR` on Unix, `TMP` or `TEMP` on
;; Windows, else the system's. Without the separator .NET ends it with.
(: (-> string))
(defun (temp-directory) (clr-trim-separator (clr-temp-path)))

;; A new, empty directory in the temp folder, with a name of its own after
;; `prefix`. On Unix only its owner may read or write it.
(: (-> (#:prefix string) string))
(defun (make-temp-directory #:prefix "")
  (clr-full-name (clr-create-temp prefix)))

;; ---------------------------------------------------------------------------
;; Signals
;; ---------------------------------------------------------------------------
;;
;; - While a subscription lives, its signals do not do their default action,
;;   as ending or stopping the process. The ambient scope owns the
;;   subscription, so the default action is back when the scope ends.
;; - .NET calls the handler on a thread that is not a fiber. The handler posts
;;   the signal to an inbox with room for one item. Each kind of signal has
;;   its own inbox, so a signal of a kind that already waits is merged with
;;   it, as in OCaml and Racket. A fiber receives from the inboxes.
;; - `with-signal-cancel` cancels its scope at the first signal and gives the
;;   signals back to the system, so a second signal does the default action.

(type (: Signal (Union Hangup Interrupt Quit Terminate Child Continue WindowChanged
                       TerminalInput TerminalOutput TerminalStop User1 User2
                       (: OtherSignal int))))

(import/class
  (SignalGroup (: Bjolang.Runtime.SignalGroup (-> SignalGroup)))
  (PlatformNotSupportedException
   (: System.PlatformNotSupportedException (-> string PlatformNotSupportedException)))
  (ArgumentException (: System.ArgumentException (-> string ArgumentException)))
  (HandlerFailed (: System.InvalidOperationException (-> string Exception HandlerFailed))))

(import/extern
  (clr-group-add!    (: Bjolang.Runtime.SignalGroup.Add (-> SignalGroup int bool (-> void) void)))
  (clr-cancellation? (: Bjolang.Runtime.SignalGroup.IsCancellation (-> Exception bool))))

(type (: Signals #:opaque (Struct (: group SignalGroup) (: inboxes (Vec (Inbox Signal))))))

(: (-> string PlatformNotSupportedException))
(defun (unsupported message) (PlatformNotSupportedException. message))

(: (-> Signal string))
(defun (signal-name s)
  (match s
    (Hangup "SIGHUP")
    (Interrupt "SIGINT")
    (Quit "SIGQUIT")
    (Terminate "SIGTERM")
    (Child "SIGCHLD")
    (Continue "SIGCONT")
    (WindowChanged "SIGWINCH")
    (TerminalInput "SIGTTIN")
    (TerminalOutput "SIGTTOU")
    (TerminalStop "SIGTSTP")
    (User1 "SIGUSR1")
    (User2 "SIGUSR2")
    ((OtherSignal n) (str "signal " (int->string n)))))

;; The four that Windows has: the console's close button, Ctrl-C, Ctrl-Break,
;; and log off or shut down.
(: (-> Signal bool))
(defun (windows-signal? s)
  (match s
    (Hangup #t)
    (Interrupt #t)
    (Quit #t)
    (Terminate #t)
    (_ #f)))

;; Whether the default action of the signal ends or stops the process, which
;; a subscription prevents. The default action of the other three is to do
;; nothing, and .NET's own handling of them stays.
(: (-> Signal bool))
(defun (cancels-default? s)
  (match s
    (Child #f)
    (Continue #f)
    (WindowChanged #f)
    (_ #t)))

;; SIGUSR1 and SIGUSR2 have one number on Linux and another on the BSDs and
;; Apple's systems.
(: (-> Signal int int int))
(defun (user-signal-code s linux bsd)
  (cond
    ((or (linux?) (android?)) linux)
    ((or (macos?) (freebsd?) (ios?) (tvos?) (watchos?) (mac-catalyst?)) bsd)
    (else
     (raise (cast Exception (unsupported (str "The number of " (signal-name s) " on this system"
                                              " is not known. Use (OtherSignal n).")))))))

;; The number of the signal for .NET's `PosixSignal`: negative for .NET's own
;; names, and the system's number for the others.
;;
;; - Raises when this system cannot have the signal.
;; - Raises for `(OtherSignal n)` with `n` below 1: .NET reads a negative
;;   number as one of its own names.
(: (-> Signal int))
(defun (signal-code s)
  (cond
    ((or (browser?) (wasi?))
     (raise (cast Exception (unsupported "This system has no signals."))))
    ((and (windows?) (not (windows-signal? s)))
     (raise (cast Exception (unsupported (str "Windows has no " (signal-name s) ".")))))
    (else
     (match s
       (Hangup -1)
       (Interrupt -2)
       (Quit -3)
       (Terminate -4)
       (Child -5)
       (Continue -6)
       (WindowChanged -7)
       (TerminalInput -8)
       (TerminalOutput -9)
       (TerminalStop -10)
       (User1 (user-signal-code s 10 30))
       (User2 (user-signal-code s 12 31))
       ((OtherSignal n)
        (if (> n 0)
            n
            (raise (cast Exception (ArgumentException.
                                    (str "OtherSignal takes a signal number above 0, not "
                                         (int->string n) "."))))))))))

;; Each signal once, in the order given.
(: (-> (Vec Signal) (Vec Signal)))
(defun (distinct-signals signals)
  (let go ((rest (vec->list signals)) (kept []))
    (match rest
      (Nil kept)
      ((Cons s more) (go more (if (vec-contains? kept s) kept (vec-add kept s)))))))

;; Registers the handler in `handlers` for each signal in `kinds`, whose
;; numbers are `codes`. Raises when the system cannot handle one, as SIGKILL.
;; The caller disposes the group then.
(: (-> SignalGroup (Vec Signal) (Vec int) (Vec (-> void)) void))
(defun (add-handlers! group kinds codes handlers)
  (loop (:for s kinds)
        (:for code codes)
        (:for handler handlers)
        (:do (match (try (clr-group-add! group code (cancels-default? s) handler)
                      #:catch (System.Exception))
               ((Ok _) unit)
               ((Err e)
                (raise (cast Exception (unsupported (str "This system cannot handle " (signal-name s)
                                                         ": " (.-Message e))))))))))

;; Subscribes to `signals`.
;;
;; - While the subscription lives, the signals do not do their default
;;   action. The ambient scope owns it, and ends it when the scope ends.
;; - Raises before it registers anything when a signal is not on this
;;   system, and when there is no ambient scope.
(: (-> (Vec Signal) Signals))
(defun (signal-subscribe signals)
  (def kinds (distinct-signals signals))
  (def codes (->vec (map signal-code kinds)))
  (def inboxes (->vec (map (fun (_) (make-inbox 1 DropNewest)) kinds)))
  (def subscription (Signals (group (SignalGroup.)) (inboxes inboxes)))
  (def owned (own! (fun () (signal-unsubscribe! subscription))))
  (def handlers
    (loop (:for s kinds)
          (:for ib inboxes)
          (:acc hs (vecing (fun () (ignore (inbox-post! ib s)))))))
  (match (try (add-handlers! (struct-ref subscription group) kinds codes handlers)
              #:catch (System.Exception))
    ((Ok _) subscription)
    ((Err e) (release! owned) (raise e))))

;; The event of the next signal. In a `choose` that another branch wins, it
;; takes nothing. When signals of more than one kind wait, which one comes
;; first is not given. After the subscription ends, it never fires.
(: (-> Signals (Event Signal)))
(defun (signal-recv s)
  (match (vec->list (struct-ref s inboxes))
    (Nil (never))
    ((Cons ib more)
     (let go ((rest more) (e (inbox-recv ib)))
       (match rest
         (Nil e)
         ((Cons next after) (go after (choose e (inbox-recv next)))))))))

;; Ends the subscription now: the signals do their default action again, and
;; signals that wait are dropped. A second call does nothing.
(: (-> Signals void))
(defun (signal-unsubscribe! s)
  (.Dispose (struct-ref s group))
  (for-each inbox-abort! (struct-ref s inboxes)))

;; Calls `handler` with each of `signals` that comes, for as long as the
;; ambient scope lives.
;;
;; - A daemon of the scope calls it, so it has the dynamic environment of
;;   this call, and the scope does not wait for it. The subscription and the
;;   daemon end together, with the scope.
;; - One call at a time. A signal that comes during a call waits, merged
;;   with the others of its kind.
;; - The handler is a bjoroutine, and an ordinary function is one too. Not
;;   `-?->`: with a bjoroutine, that would make this a bjoroutine, though it
;;   only gives the handler to the daemon, and `main` could not call it.
(: (-> (Vec Signal) (-bjo-> Signal void) void))
(defun (on-signal signals handler)
  (def subscription (signal-subscribe signals))
  (spawn/daemon (handle-signals subscription handler)))

;; The daemon of `on-signal`. When the handler raises, it ends the
;; subscription, so the signals do their default action again, and raises
;; for the report of the daemon. Nothing else would receive the signals, and
;; they would be lost without a word. A cancellation is raised as it is: the
;; scope ends.
(: (-bjo-> Signals (-bjo-> Signal void) void))
(defbjo (handle-signals subscription handler)
  (def s (sync (signal-recv subscription)))
  (match (try (handler s) #:catch (System.Exception))
    ((Ok _) (handle-signals subscription handler))
    ((Err e)
     (signal-unsubscribe! subscription)
     (if (clr-cancellation? e)
         (raise e)
         (raise (cast Exception (HandlerFailed. (str "The handler of " (signal-name s) " raised, so "
                                                     "its signals do their default action again.")
                                                e)))))))

;; `(with-signal-cancel signals body ...)` - a scope that the first of
;; `signals` cancels.
;;
;; - Returns `(Ok v)` when the body returns `v`, and `(Err s)` when the signal
;;   `s` stopped the body. The scope's reason is `(Requested "SIGINT")` and
;;   the like.
;; - At the first signal it gives the signals back to the system, so a second
;;   signal does the default action, as ending the process at once.
;; - A failure of the body is raised again, also after a signal.
(def/macro (with-signal-cancel form inject compare)
  (syntax-match form
    ((_ signals first body ...)
     #'(let ()
         (def kinds ,signals)
         (def scope (scope-open! 0))
         (signal-cancel-result
          kinds
          (try (scope-form scope (cancel)
                 (signal-cancel-arm! kinds cancel)
                 ,first ,@body)
               #:catch (System.Exception))
          (scope-token scope))))
    (bad (syntax-error bad "with-signal-cancel takes a Vec of signals and a body: (with-signal-cancel [Interrupt Terminate] body ...)"))))

;; Subscribes the scope that `cancel` cancels to `signals`. The first signal
;; ends the subscription and then cancels the scope, so the signals do their
;; default action again before anything sees the cancellation.
(: (-> (Vec Signal) (-> CancelReason void) void))
(defun (signal-cancel-arm! signals cancel)
  (def kinds (distinct-signals signals))
  (def codes (->vec (map signal-code kinds)))
  (def group (own-disposable (SignalGroup.)))
  (add-handlers! group kinds codes
                 (->vec (map (fun (s) (fun ()
                                        (.Dispose group)
                                        (cancel (Requested (signal-name s)))))
                             kinds))))

;; The value of `with-signal-cancel`, from the outcome of its scope and the
;; scope's token.
;;
;; - The body returned: its value.
;; - One of `signals` cancelled the scope, and the body left because of it:
;;   that signal.
;; - Else: the body's exception, raised again.
(: (-> (Vec Signal) (Result Exception %a) CancelToken (Result Signal %a)))
(defun (signal-cancel-result signals outcome token)
  (match outcome
    ((Ok v) (Ok v))
    ((Err e)
     (match (cancel-reason token)
       ((Some (Requested who))
        (match (find (fun (s) (= (signal-name s) who)) signals)
          ((Some s) (if (clr-cancellation? e) (Err s) (raise e)))
          (None (raise e))))
       (_ (raise e))))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; Work on the parts of a collection at the same time.
;;
;; A `Chunkable` collection splits into parts that together hold each element
;; once, in the collection's order when it has one. Both run a function on
;; each part on threads made for work that uses a core:
;;
;; - `in-parts` returns the results in the order of the parts.
;; - `in-parallel` returns them combined in that order, by `Combinable`.
;;
;; The functions over a whole collection join the parts for the caller:
;;
;; - `par-map` and `par-filter` return a collection of the type they are
;;   given. Each type has its own way to join its parts.
;; - `par-for-each` runs a function on each element.
;;
;; An array splits into views, not copies. A part of an array gives a span of
;; its own elements, so the parts of one array can be written in place at the
;; same time. A span cannot go to another thread, so each part makes its own.
;; A vector, a map or a set splits into collections of its own type, which
;; share its nodes.

(import (std prelude))
(import (std hashset))
(import (std orderedmap))
(import (std orderedset))
(import (std atomic))
(import (std inbox))
(import (std combine))

(:doc #:file "parallel.bjodoc")

(import/extern
  (processor-count (: System.Environment.ProcessorCount #:get))
  (parse-int (: System.Int32.TryParse (-> string (out int) (Option int))))
  (make-jobs-inbox (: Bjoml.InboxModule.MakeNewestFirst (-> int bool (Inbox %a))))
  (spin-before-parking (: BjolangRuntime.SpinBeforeParking (-> int void)))
  (iarray-adopt (: Bjolang.Runtime.IArrayModule.Adopt (-> (Array %a) (IArray %a)))))

(import/class
  ;; The accumulator of `arraying`, which the prelude does not export.
  ((ArrayBuilder %a) (: Bjolang.Runtime.ArrayBuilder)))

(import/extern
  (arraybuilder-empty (: Bjolang.Runtime.ArrayBuilderModule.Empty (-> (ArrayBuilder %a))))
  (arraybuilder-add   (: Bjolang.Runtime.ArrayBuilderModule.Add   (-> (ArrayBuilder %a) %a (ArrayBuilder %a))))
  (arraybuilder-count (: Bjolang.Runtime.ArrayBuilderModule.Count (-> (ArrayBuilder %a) int)))
  (arraybuilder-span  (: Bjolang.Runtime.ArrayBuilderModule.Span  (-> (ArrayBuilder %a) int int (Span %a)))))

(re-export combine combine-all)

(export Chunkable chunk-size chunks
        in-parts in-parallel computing-each compute-thread-count
        ArrayPart part-array part-lo part-hi part-length part->span part->ro-span
        IArrayPart iarray-part-lo iarray-part-hi iarray-part-length
        par-for-each
        ParallelMap par-map ParallelFilter par-filter)

;; ---------------------------------------------------------------------------
;; The trait
;; ---------------------------------------------------------------------------

;; A collection that splits into parts.
;;
;; - `chunk-size`: the number of elements, or a good estimate.
;; - `chunks c k`: about k parts, in the collection's order when it has one,
;;   that hold each element once. Fewer when there are fewer elements than k,
;;   and at least one, which may be empty.
(def/trait (Chunkable %c)
  (type %chunk)
  (: chunk-size (-> %c int))
  (: chunks (-> %c int (Vec %chunk))))

;; The number of parts to make of n elements when k are asked for: at least
;; one, and no more than there are elements.
(: (-> int int int))
(defun (part-count n k)
  (max 1 (min k n)))

;; Where part i of k starts, as an offset into n elements. Parts differ in size
;; by one at most. In a long, because i times n can pass `int-max`.
(: (-> int int int int))
(defun (part-start n k i)
  (cast int (/ (* (cast long n) (cast long i)) (cast long k))))

;; ---------------------------------------------------------------------------
;; The compute threads
;; ---------------------------------------------------------------------------

;; Threads for work that uses a core, made at the first call and kept.
;;
;; - Each is a `spawn/thread` that takes jobs from `jobs` for ever. A
;;   detached fiber syncs its event, as `spawn/thread` starts its thread when
;;   synced. The fiber is suspended for ever, and holds nothing.
;; - They are not the threads of the run queues, so a job holds no pump, and
;;   fibers go on running while the jobs compute.
;; - `jobs` is an inbox, which several threads can wait on, and which a post
;;   does not wait for. It wakes the thread that waited last, which is the
;;   one most likely to be spinning still, as a thread spins a little before
;;   it parks: then a post makes no system call to wake it.
;; - A thread that has no job spins for 50 us before it parks. A call that
;;   comes within that time finds it awake: it starts at once, and the post
;;   makes no system call. Measured on 24 empty parts after 10 us of other
;;   work: 14 to 6 us for each call. The cost is a core for 50 us after the
;;   last job of each thread.
;; - A job is a closure over the dynamic environment of the code that made
;;   it, and runs in that environment.
;; - A job posted to a full inbox is dropped. That is safe because a call
;;   always has a thread that takes its parts: in ordinary code the calling
;;   thread, and in a bjoroutine the first runner, which waits for room
;;   when the inbox is full.

;; How many compute threads there are: BJOLANG_COMPUTE_THREADS, or one less
;; than the logical cores, and at least one. The core left over is for the
;; thread that calls, which takes parts too in ordinary code, and for the
;; fibers. Read once, when the module is loaded.
;;
;; Measured with bjostat's bench/latency.bjo, 24 logical cores: with 24
;; compute threads, a fiber that sleeps 1 ms woke up to 7.2-8.6 ms late in
;; every run; with 23, up to 1.9-2.7 ms, at the same number of loops.
(def default-thread-count (max 1 (- processor-count 1)))

(def thread-count
  (match (get-environment-variable "BJOLANG_COMPUTE_THREADS")
    ((Some s) (match (parse-int s)
                ((Some n) (if (> n 0) n default-thread-count))
                (None default-thread-count)))
    (None default-thread-count)))

(: (-> int))
(defun (compute-thread-count) thread-count)

(def jobs (: (make-jobs-inbox 4096 #f) (Inbox (-> Unit))))
(def started (make-atomic 0))

;; A compute thread: runs jobs for ever. A job raises nothing: the parts it
;; runs keep their exceptions.
(def spin-us 50)

(: (-> Unit))
(defun (compute-loop)
  (spin-before-parking spin-us)
  (let go ()
    ((sync/blocking (inbox-recv jobs)))
    (go)))

(: (-bjo-> void))
(defbjo (keep-compute-thread)
  (ignore (sync (spawn/thread compute-loop))))

(: (-> void))
(defun (start-compute-threads!)
  (when (= (atomic-compare-and-swap! started 0 1) 0)
    (loop (:for _ (range 0 thread-count))
          (:do (spawn/detached (keep-compute-thread))))))

;; ---------------------------------------------------------------------------
;; Running the parts
;; ---------------------------------------------------------------------------

;; Runs `(work k)` for each k from 0 to n - 1 on the compute threads, and
;; returns the values in the order of k.
;;
;; - A runner is a job that takes parts by number until none is left. The
;;   caller posts runners one after the other while a part is left that no
;;   runner has taken, up to one for each compute thread, or for each part
;;   when there are fewer. A post that wakes a thread that sleeps is a system
;;   call of some microseconds, and the thread starts some microseconds
;;   later: so when the parts are small, the first runners take them all and
;;   the caller stops posting. A runner that a thread takes after the parts
;;   are all taken ends at once.
;; - In a bjoroutine the fiber is suspended until the parts have run, and
;;   holds nothing.
;; - In ordinary code the thread posts one runner fewer, then takes the parts
;;   that are left itself, and then waits only for the parts that other
;;   threads have taken. So a part can call `computing-each` itself: a call
;;   does not wait for runners that no thread is free to take, and there is
;;   no deadlock when calls nest.
;; - A part runs in the dynamic environment of the call.
;; - When a part raises, the parts not started do not run, and the exception
;;   is raised here when the parts that started have ended. A part that runs
;;   cannot be stopped. In a bjoroutine, the parts not started when the scope
;;   is cancelled do not run, and `Cancelled` is raised.
(: computing-each (-> int (-> int %a) (Vec %a)))
(defun (computing-each n work)
  (if (<= n 0)
      []
      (let ()
        (start-compute-threads!)
        (def results (make-array n))
        (def next (make-atomic -1))
        (def ended (make-atomic 0))
        (def failed (make-atomic 0))
        (def failure (: (make-array 1) (Array Exception)))
        (def skipped (make-atomic 0))
        (def done (: (make-inbox 1 DropNewest) (Inbox Unit)))
        (def token (parameter-ref current-cancel))
        (def env (current-dynamic-environment))
        ;; Takes parts until none is left.
        (def run-parts
          (fun ()
            (let go ()
              (def k (atomic-increment! next))
              (when (< k n)
                (cond
                  ((not (= (atomic-ref failed) 0)) unit)
                  ((cancelled? token) (ignore (atomic-increment! skipped)))
                  (else (match (try (array-set! results k (work k)) #:catch (System.Exception))
                          ((Ok _) unit)
                          ((Err e) (when (= (atomic-compare-and-swap! failed 0 1) 0)
                                     (array-set! failure 0 e))))))
                (when (= (atomic-increment! ended) n)
                  (ignore (inbox-post! done unit)))
                (go)))
            unit))
        (def runner (fun () (call-with-dynamic-environment env run-parts)))
        ;; At most one runner for each part and one for each compute thread,
        ;; and in ordinary code one less, as the calling thread takes parts
        ;; too.
        (def runners (by-colour (#:sync (min (- n 1) thread-count))
                                (#:bjo (min n thread-count))))
        (def posted (make-atomic 0))
        ;; Posts runners while a part is left that no runner has taken.
        (def post-runners
          (fun ()
            (let go ()
              (when (and (< (+ (atomic-ref next) 1) n)
                         (<= (atomic-increment! posted) runners))
                (ignore (inbox-post! jobs runner))
                (go)))))
        (by-colour
          (#:sync (begin
                    (post-runners)
                    (run-parts)
                    (sync/blocking (inbox-recv done))))
          (#:bjo (begin
                   (ignore (atomic-increment! posted))
                   (unless (inbox-post! jobs runner)
                     (sync (inbox-send-evt jobs runner)))
                   (post-runners)
                   (sync (inbox-recv done))
                   (when (> (atomic-ref skipped) 0) (sync (: (never) (Event Unit)))))))
        (when (= (atomic-ref failed) 1) (raise (array-ref failure 0)))
        (array->vec results))))

;; How many parts to ask of a collection of `n` elements.
;; - `size` > 0: enough parts that none need be longer than `size`.
;; - else: `parts`.
(: (-> int int int int))
(defun (parts-asked n parts size)
  (if (> size 0)
      (max 1 (+ (/ n size) (if (= 0 (% n size)) 0 1)))
      parts))

;; Runs `f` on each part of `c` on the compute threads, and returns the
;; results in the order of the parts. The parts run by `computing-each`.
;;
;; - `#:parts`: how many parts to ask for. Default: four for each compute
;;   thread. The collection may make fewer.
;; - `#:size`: instead, ceil(n / size) parts of n elements, so that the parts
;;   do not depend on the machine.
(: in-parts (-> %c (-> %p %r) (#:parts int) (#:size int) (Vec %r)) (where (Chunkable %c #:chunk %p)))
(defun (in-parts c f #:parts (* 4 thread-count) #:size 0)
  (def ps (chunks c (parts-asked (chunk-size c) parts size)))
  (computing-each (vec-length ps) (fun (i) (f (vec-ref ps i)))))

;; `in-parts`, with the results combined in the order of the parts.
(: in-parallel (-> %c (-> %p %r) (#:parts int) (#:size int) %r)
   (where (Chunkable %c #:chunk %p) (Combinable %r)))
(defun (in-parallel c f #:parts (* 4 thread-count) #:size 0)
  (combine-all (in-parts c f #:parts parts #:size size)))

;; ---------------------------------------------------------------------------
;; Ranges and vectors
;; ---------------------------------------------------------------------------

(impl (Chunkable Range)
  (type %chunk Range)
  (defun (chunk-size r) (max 0 (- (record-ref r hi) (record-ref r lo))))
  (defun (chunks r k)
    (def lo (record-ref r lo))
    (def n (max 0 (- (record-ref r hi) lo)))
    (def m (part-count n k))
    (loop (:for i (range 0 m))
          (:acc parts (vecing (range (+ lo (part-start n m i))
                                     (+ lo (part-start n m (+ i 1)))))))))

;; A part of a vector is a slice of it, which shares its nodes.
(impl (Chunkable (Vec %a))
  (type %chunk (Vec %a))
  (defun (chunk-size v) (vec-length v))
  (defun (chunks v k)
    (def n (vec-length v))
    (def m (part-count n k))
    (loop (:for i (range 0 m))
          (:acc parts (vecing (vec-slice v
                                         (part-start n m i)
                                         (- (part-start n m (+ i 1)) (part-start n m i))))))))

;; ---------------------------------------------------------------------------
;; Arrays: parts are views
;; ---------------------------------------------------------------------------

;; The elements of `array` from `lo` up to `hi`. A view: it holds the array,
;; not a copy of the elements.
(type (: (ArrayPart %a) (Struct (: array (Array %a)) (: lo int) (: hi int))))

(: (-> (ArrayPart %a) (Array %a)))
(defun (part-array p) (record-ref p array))

;; The index in the array of the part's first element.
(: (-> (ArrayPart %a) int))
(defun (part-lo p) (record-ref p lo))

;; The index in the array after the part's last element.
(: (-> (ArrayPart %a) int))
(defun (part-hi p) (record-ref p hi))

(: (-> (ArrayPart %a) int))
(defun (part-length p) (- (record-ref p hi) (record-ref p lo)))

;; A span of the part's elements, which writes into the array. Index 0 is the
;; part's first element. Made where it is used: a span cannot go to another
;; thread or into a closure.
(: (-> (ArrayPart %a) (Span %a)))
(defun (part->span p)
  (array->span (record-ref p array) #:from (record-ref p lo) #:to (record-ref p hi)))

(: (-> (ArrayPart %a) (RoSpan %a)))
(defun (part->ro-span p)
  (array->ro-span (record-ref p array) #:from (record-ref p lo) #:to (record-ref p hi)))

;; The part's elements, in order. The cursor is the index into the array.
(impl (Iterable (ArrayPart %a))
  (type %elem %a)
  (type %cursor int)
  (defun (iterable-start p) (record-ref p lo))
  (defun (iterable-done? p index) (>= index (record-ref p hi)))
  (defun (iterable-current p index) (array-ref (record-ref p array) index))
  (defun (iterable-next p index) (+ index 1))
  (defun (count p) (- (record-ref p hi) (record-ref p lo))))

(impl (Chunkable (Array %a))
  (type %chunk (ArrayPart %a))
  (defun (chunk-size a) (array-length a))
  (defun (chunks a k)
    (def n (array-length a))
    (def m (part-count n k))
    (loop (:for i (range 0 m))
          (:acc parts (vecing (ArrayPart (array a)
                                         (lo (part-start n m i))
                                         (hi (part-start n m (+ i 1)))))))))

;; ---------------------------------------------------------------------------
;; Immutable arrays: parts are views too
;; ---------------------------------------------------------------------------

(type (: (IArrayPart %a) (Struct (: iarray (IArray %a)) (: lo int) (: hi int))))

(: (-> (IArrayPart %a) int))
(defun (iarray-part-lo p) (record-ref p lo))

(: (-> (IArrayPart %a) int))
(defun (iarray-part-hi p) (record-ref p hi))

(: (-> (IArrayPart %a) int))
(defun (iarray-part-length p) (- (record-ref p hi) (record-ref p lo)))

(impl (Iterable (IArrayPart %a))
  (type %elem %a)
  (type %cursor int)
  (defun (iterable-start p) (record-ref p lo))
  (defun (iterable-done? p index) (>= index (record-ref p hi)))
  (defun (iterable-current p index) (iarray-ref (record-ref p iarray) index))
  (defun (iterable-next p index) (+ index 1))
  (defun (count p) (- (record-ref p hi) (record-ref p lo))))

(impl (Chunkable (IArray %a))
  (type %chunk (IArrayPart %a))
  (defun (chunk-size a) (iarray-length a))
  (defun (chunks a k)
    (def n (iarray-length a))
    (def m (part-count n k))
    (loop (:for i (range 0 m))
          (:acc parts (vecing (IArrayPart (iarray a)
                                          (lo (part-start n m i))
                                          (hi (part-start n m (+ i 1)))))))))

;; ---------------------------------------------------------------------------
;; Maps and sets: parts are maps and sets
;; ---------------------------------------------------------------------------

;; The split is in the runtime. It cuts along the tree's own nodes, so the
;; parts share them, and their sizes are close but not equal. A tree too small
;; to cut, one leaf, is one part.

;; The parts of a hash map or a hash set are in no order that means
;; anything, as the map's own order does not.
(impl (Chunkable (Map %k %v))
  (type %chunk (Map %k %v))
  (defun (chunk-size m) (map-length m))
  (defun (chunks m k) (array->vec (map-split m k))))

(impl (Chunkable (HashSet %a))
  (type %chunk (HashSet %a))
  (defun (chunk-size s) (hashset-length s))
  (defun (chunks s k) (array->vec (hashset-split s k))))

;; The parts of an ordered map or set are in key order: every key of a part
;; is less than every key of the next.
(impl (Chunkable (OrderedMap %k %v))
  (type %chunk (OrderedMap %k %v))
  (defun (chunk-size m) (orderedmap-length m))
  (defun (chunks m k) (array->vec (orderedmap-split m k))))

(impl (Chunkable (OrderedSet %a))
  (type %chunk (OrderedSet %a))
  (defun (chunk-size s) (orderedset-length s))
  (defun (chunks s k) (array->vec (orderedset-split s k))))

;; ---------------------------------------------------------------------------
;; Over the elements of a whole collection
;; ---------------------------------------------------------------------------

;; Runs `f` on each element of `c`, on the compute threads. The elements of
;; a part are taken in order, and the parts at the same time.
(: par-for-each (-> (-> %e void) %c void)
   (where (Chunkable %c #:chunk %p)
          (Iterable %p #:elem %e)))
(defun (par-for-each f c)
  (ignore (in-parallel c
                       (fun (part)
                         (let go ((cur (iterable-start part)))
                           (if (iterable-done? part cur)
                               unit
                               (begin
                                 (f (iterable-current part cur))
                                 (go (iterable-next part cur)))))))))

;; The parts joined from the first to the last: `(join (join p0 p1) p2)`.
;; There is at least one part.
(: (-> (Vec %a) (-> %a %a %a) %a))
(defun (joined parts join)
  (loop (:for i (range 1 (vec-length parts)))
        (:acc out (folding (vec-ref parts 0) (join out (vec-ref parts i))))))

;; ---------------------------------------------------------------------------
;; A collection of the type given
;; ---------------------------------------------------------------------------

;; A collection whose elements a function can change on the compute threads.
;;
;; - `par-map f c`: a collection of the same type, with `(f x)` in place of
;;   each element x, in the same order. `f` runs on the compute threads.
;; - Of a map, the values are the elements: the keys stay as they are.
;; - A set is not a `ParallelMap`, as a set of the values of `f` needs their
;;   `Eq` or `Ord`. Use `in-parallel` with a loop into `hash-setting` or
;;   `ordered-setting`.
(def/trait (ParallelMap (%c %a))
  (: par-map (-> (-> %a %b) (%c %a) (%c %b))))

;; A collection whose elements a test can select on the compute threads.
;;
;; - `par-filter keep? c`: a collection of the same type, with the elements
;;   for which `(keep? x)` is true, in the same order. `keep?` runs on the
;;   compute threads.
;; - Of a map, `keep?` is given the value. To test the key too, use
;;   `in-parts` with `map-filter`, and `map-merge` the parts.
(def/trait (ParallelFilter (%c %a))
  (: par-filter (-> (-> %a bool) (%c %a) (%c %a))))

;; --- Vectors: the parts are joined by concatenation, which shares their
;; nodes.

(impl (ParallelMap (Vec %a))
  (defun (par-map f v)
    (in-parallel v (fun (p) (vec-map f p)))))

(impl (ParallelFilter (Vec %a))
  (defun (par-filter keep? v)
    (in-parallel v (fun (p) (vec-filter keep? p)))))

;; --- Arrays: `par-map` makes the result first, and each part writes its
;; own range of it. `par-filter` collects each part's elements, and then
;; copies them into an array of their number.

;; Writes `(f x)` into `out` for each element x of part `p` of `a`, at the
;; same index.
(: (-> (-> %a %b) (ArrayPart %a) (Array %b) Unit))
(defun (map-part-into f p out)
  (def a (record-ref p array))
  (loop (:for i (range (record-ref p lo) (record-ref p hi)))
        (:do (array-set! out i (f (array-ref a i)))))
  unit)

(: (-> (-> %a %b) (IArrayPart %a) (Array %b) Unit))
(defun (map-iarray-part-into f p out)
  (def a (record-ref p iarray))
  (loop (:for i (range (record-ref p lo) (record-ref p hi)))
        (:do (array-set! out i (f (iarray-ref a i)))))
  unit)

;; The elements of the builders, one builder after the other, in an array of
;; their number.
(: (-> (Vec (ArrayBuilder %a)) (Array %a)))
(defun (builders->array parts)
  (def out (make-array (loop (:for b parts) (:acc n (summing (arraybuilder-count b))))))
  (loop (:for b parts)
        (:with at 0 (+ at (arraybuilder-count b)))
        (:do (let ((from (arraybuilder-span b 0 (arraybuilder-count b))))
               (loop (:for i (range 0 (span-length from)))
                     (:do (array-set! out (+ at i) (span-ref from i)))))))
  out)

(: (-> (-> %a bool) (ArrayPart %a) (ArrayBuilder %a)))
(defun (filter-part keep? p)
  (loop (:for x p)
        (:when (keep? x))
        (:acc kept (folding (arraybuilder-empty) (arraybuilder-add kept x)))))

(: (-> (-> %a bool) (IArrayPart %a) (ArrayBuilder %a)))
(defun (filter-iarray-part keep? p)
  (loop (:for x p)
        (:when (keep? x))
        (:acc kept (folding (arraybuilder-empty) (arraybuilder-add kept x)))))

(impl (ParallelMap (Array %a))
  (defun (par-map f a)
    (def out (make-array (array-length a)))
    (ignore (in-parts a (fun (p) (map-part-into f p out))))
    out))

(impl (ParallelFilter (Array %a))
  (defun (par-filter keep? a)
    (builders->array (in-parts a (fun (p) (filter-part keep? p))))))

;; The array that the parts write is not seen by any other code, so it
;; becomes the immutable array with no copy.
(impl (ParallelMap (IArray %a))
  (defun (par-map f a)
    (def out (make-array (iarray-length a)))
    (ignore (in-parts a (fun (p) (map-iarray-part-into f p out))))
    (iarray-adopt out)))

(impl (ParallelFilter (IArray %a))
  (defun (par-filter keep? a)
    (iarray-adopt (builders->array (in-parts a (fun (p) (filter-iarray-part keep? p)))))))

;; --- Hash maps and sets: the parts have no key in common, so a merge
;; shares their nodes.

(impl (ParallelMap (Map %k %a))
  (defun (par-map f m)
    (joined (in-parts m (fun (p) (map-map-values f p))) (fun (a b) (map-merge a b)))))

(impl (ParallelFilter (Map %k %a))
  (defun (par-filter keep? m)
    (joined (in-parts m (fun (p) (map-filter (fun (k v) (keep? v)) p)))
            (fun (a b) (map-merge a b)))))

(impl (ParallelFilter (HashSet %a))
  (defun (par-filter keep? s)
    (in-parallel s (fun (p) (hashset-filter keep? p)))))

;; --- Ordered maps and sets: the parts are in key order, so a concat joins
;; them by their leaves, under new internal nodes. It compares one pair of
;; keys at each join, and copies only small leaves, together with a
;; neighbour.

(impl (ParallelMap (OrderedMap %k %a))
  (defun (par-map f m)
    (orderedmap-concat (vec->array (in-parts m (fun (p) (orderedmap-map-values f p)))))))

(impl (ParallelFilter (OrderedMap %k %a))
  (defun (par-filter keep? m)
    (orderedmap-concat (vec->array (in-parts m (fun (p) (orderedmap-filter (fun (k v) (keep? v)) p)))))))

(impl (ParallelFilter (OrderedSet %a))
  (defun (par-filter keep? s)
    (orderedset-concat (vec->array (in-parts s (fun (p) (orderedset-filter keep? p)))))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/ports.bjo — draining and streaming a port, its bytes, and byte output.
;;
;; An `InputPort` is bytes, as a Racket port is: `read-line` and `read-char`
;; decode UTF-8 from it, and `read-some`, `read-bytes` and `peek-bytes` take
;; the bytes themselves, on the same port and in any order. The byte reads are
;; CML events, so a read can be a branch of a `choose`.
;;
;; The invariant the port turns on, and the reason a read is safe inside a
;; `choose`:
;;
;;   Every byte that leaves the operating system lands in the port's buffer. A
;;   read never takes bytes from the underlying stream — it takes them from the
;;   buffer, and only when it wins.
;;
;; `BjolangRuntime/BjoInputPort.cs` is where that is done.
;;
;; An `OutputPort` is bytes too: `write-string` and `write-char` write UTF-8
;; to it, and `write-bytes!` writes bytes as they are, on the same port and in
;; any order.

(export
  ;; Draining
  port->list port->vec port->lines
  ;; Streaming
  port->seq file->seq
  port->stream

  ;; --- Bytes ---------------------------------------------------------------

  ;; The types. `Encoding` is the prelude's, its values `(text encodings)`'s.
  ClrStream

  ;; Reading, as events. These are what go in a `choose`.
  read-some-evt read-bytes-evt eof-evt
  ;; Reading, for when there is nothing to choose between.
  read-some read-bytes
  ;; Looking without taking.
  peek-bytes

  ;; Writing.
  write-bytes!

  ;; The half-close.
  shutdown!

  ;; Composition.
  limited byte-pipe

  ;; Sources.
  stream->input-port stream->output-port

  ;; Encodings.
  reencode-input-port reencode-output-port)

(import (std prelude))
(import (std stream))

(:doc #:file "ports.bjodoc")

;; Exception import.
(import/class
  (Exception (: System.Exception)))

;; ---------------------------------------------------------------------------
;; Draining
;; ---------------------------------------------------------------------------
;;
;; Drains items.


;; Drains a port.
(: (-> (-?-> InputPort %a) InputPort (List %a)))
(defun (port->list reader p)
  (loop (:with i 1 1)
        (:finish (port-eof? p))
        (:acc lst (listing (reader p)))))


(: (-> (-?-> InputPort %a) InputPort (Vec %a)))
(defun (port->vec reader p)
  (loop (:with i 1 1)
        (:finish (port-eof? p))
        (:acc vec (vecing (reader p)))))

;; The lines left in `port`, ended as `#:mode` says (`Any` by default), as
;; Racket's `port->lines` takes a `#:line-mode`.
(: (-> InputPort (#:mode LineMode) (Vec string)))
(defun (port->lines port #:mode Any)
  (port->vec (fun (p) (read-line p #:mode mode)) port))

;; ---------------------------------------------------------------------------
;; Streaming
;; ---------------------------------------------------------------------------

;; Streams items from port as a Seq.
(: port->seq #:sync (-> (-> InputPort %a) InputPort (Seq %a)))
(defun (port->seq reader p)
  (seql (:with i 0 0)
        (:finish (port-eof? p))
        (:yield (reader p))))

;; ---------------------------------------------------------------------------
;; Streaming that suspends
;; ---------------------------------------------------------------------------

;; The items `reader` reads from `p` until EOF, as a `Stream`.
;;
;; - Each item is read when the loop asks for it, so a loop left early leaves
;;   the port just after the last item it took.
;; - What `reader` raises is raised in the loop.
;; - The port is not closed.
(: (-> (-bjo-> InputPort %a) InputPort (Stream %a)))
(defun (port->stream reader p)
  (make-stream (bjoroutine (out)
                 (let go ()
                   (unless (port-eof? p)
                     (emit! out (reader p))
                     (go))))))

;; Streams file contents as Seq.
(: file->seq #:sync (-> (-> InputPort %a) string (Result Exception (Seq %a))))
(defun (file->seq reader path)
  (match (open-input-file path)
    ((Ok probe)
     ;; First we just check if the file exists
     (close-input-port probe)
     (Ok (file-read/seq reader path)))
    ((Err e) (Err e))))

;; ===========================================================================
;; Bytes
;; ===========================================================================
;;
;; An input port owns its buffer, and a read is a cursor move inside that
;; buffer and nothing else — which is what makes a read safe inside a
;; `choose`, and why a refill that lands after its branch has already lost is
;; harmless rather than data loss.
;;
;; The port is a class in `BjolangRuntime/BjoInputPort.cs`, the way `Inbox` is
;; a class with `(std inbox)` over it, and its type, `InputPort`, is the
;; prelude's. What is here is the typed surface of its byte reads.
;;
;; This module still imports `(std prelude)` and nothing else, so it keeps its
;; place in `build_std.sh` above `(std inbox)`: the event vocabulary is all
;; prelude builtins, and the runtime class is reached by `import/extern`, which
;; is not a module dependency.

(import/class
  ;; The pipe object, which exists only so that `byte-pipe` can take its two
  ;; halves off it. Not exported: a pipe *is* its two ports.
  (BytePipe       (: Bjolang.Runtime.BjoBytePipe))
  (ClrStream      (: System.IO.Stream)))

(import/extern
  ;; Sources.
  (clr-from-stream (: Bjolang.Runtime.BytePorts.FromStream (-> ClrStream InputPort)))
  (clr-to-stream   (: Bjolang.Runtime.OutputPorts.FromStream (-> ClrStream OutputPort)))
  (clr-make-pipe   (: Bjolang.Runtime.BytePorts.MakePipe   (-> BytePipe)))
  (clr-pipe-in     (: Bjolang.Runtime.BytePorts.PipeInput  (-> BytePipe InputPort)))
  (clr-pipe-out    (: Bjolang.Runtime.BytePorts.PipeOutput (-> BytePipe OutputPort)))
  (clr-limited     (: Bjolang.Runtime.BytePorts.Limited    (-> InputPort int InputPort)))

  ;; The events.
  (clr-read-some-evt  (: Bjolang.Runtime.BytePorts.ReadSomeEvent
                         (-> InputPort (Event (Result Exception (Option (Array byte)))))))
  (clr-read-bytes-evt (: Bjolang.Runtime.BytePorts.ReadBytesEvent
                         (-> InputPort int (Event (Result Exception (Option (Array byte)))))))
  (clr-eof-evt        (: Bjolang.Runtime.BytePorts.EofEvent (-> InputPort (Event Unit))))

  ;; The questions that are not events, in sync/async pairs.
  (clr-peek           (: Bjolang.Runtime.BytePorts.PeekBytes
                         (-> InputPort int int (Option (Array byte))) #:blocking))
  (clr-peek-await     (: Bjolang.Runtime.BytePorts.PeekBytesAsync
                         (-> InputPort int int (Option (Array byte))) #:async))

  ;; Writing.
  (clr-write-bytes       (: Bjolang.Runtime.BjoOutputPort.WriteBytes
                            (-> OutputPort (Array byte) void) #:blocking))
  (clr-write-bytes-await (: Bjolang.Runtime.BjoOutputPort.WriteBytesAsync
                            (-> OutputPort (Array byte) void) #:async))
  (clr-shutdown          (: Bjolang.Runtime.BjoOutputPort.Shutdown (-> OutputPort void) #:blocking))
  (clr-shutdown-await    (: Bjolang.Runtime.BjoOutputPort.ShutdownAsync (-> OutputPort void) #:async))

  ;; Encodings.
  (clr-reencode  (: Bjolang.Runtime.InputPorts.Reencode (-> InputPort Encoding InputPort)))
  (clr-reencode-out (: Bjolang.Runtime.OutputPorts.Reencode (-> OutputPort Encoding OutputPort))))

;; ---------------------------------------------------------------------------
;; Reading, as events
;; ---------------------------------------------------------------------------
;;
;; These are what go in a `choose`, and they are real CML events: a branch that
;; loses consumes nothing, and the bytes stay in the buffer for the next reader.
;; They are also *ephemeral* — each `sync` asks the port a new question, so a
;; loop over one reads successive bytes rather than winning with a remembered
;; answer forever. That is exactly what `(task->event ...)` cannot do, which is
;; why a read is not one.
;;
;; WHY THE VALUE IS A `Result`. A stream can fail, and the CML resume path
;; carries a value rather than a raise — so a failure that lands while a fiber
;; is parked in a `choose` has to arrive as one. `promise-join`, `blocking` and
;; `task->event` all answer a `Result` for the same reason. `read-some` and
;; `read-bytes` below unwrap it and raise on the fiber's own stack, which is
;; what an ordinary read wants.

;; Whatever is available, never an empty array, and `None` once there is no more.
(: (-> InputPort (Event (Result Exception (Option (Array byte))))))
(defun (read-some-evt p) (clr-read-some-evt p))

;; Exactly `n` bytes. At end of input the short remainder comes back as
;; `(Some ...)` and the call after it as `None`, so the two are separate facts
;; rather than one ambiguous answer.
;;
;; `n` must be at least 1.
(: (-> InputPort int (Event (Result Exception (Option (Array byte))))))
(defun (read-bytes-evt p n) (clr-read-bytes-evt p n))

;; Fires once the port is finished — cleanly, or because it failed. It says only
;; that, which is what makes it the thing to wait on for "the other end has gone
;; away"; a read beside it is what says which of the two happened.
(: (-> InputPort (Event Unit)))
(defun (eof-evt p) (clr-eof-evt p))

;; ---------------------------------------------------------------------------
;; Reading, for when there is nothing to choose between
;; ---------------------------------------------------------------------------
;;
;; Defined by syncing the events above: conveniences, not a second
;; implementation. The `defbjouble` is what picks the right kind of wait —
;; `sync/blocking` parks the calling thread for an ordinary `defun`, `sync`
;; suspends the fiber — and the raise happens after either of them returns, on
;; the caller's own stack.

(: (-> (Result Exception (Option (Array byte))) (Option (Array byte))))
(defun (byte-read-or-raise r)
  (match r
    ((Ok bytes) bytes)
    ((Err e) (raise e))))

;; Whatever is available, waiting for at least one byte. `None` at end of input.
(: (-> InputPort (Option (Array byte))))
(defbjouble (read-some p)
  (#:sync (byte-read-or-raise (sync/blocking (clr-read-some-evt p))))
  (#:bjo  (byte-read-or-raise (sync (clr-read-some-evt p)))))

;; Exactly `n` bytes, or the short remainder at end of input, or `None`.
(: (-> InputPort int (Option (Array byte))))
(defbjouble (read-bytes p n)
  (#:sync (byte-read-or-raise (sync/blocking (clr-read-bytes-evt p n))))
  (#:bjo  (byte-read-or-raise (sync (clr-read-bytes-evt p n)))))

;; ---------------------------------------------------------------------------
;; Looking without taking
;; ---------------------------------------------------------------------------

;; `skip` bytes past the cursor, then `count` of them, leaving the cursor where
;; it was. Short at end of input, and `None` when there is nothing at all beyond
;; the skip.
;;
;; This is the mechanism a read is, with the cursor move left out — which is why
;; "peek, decide, then read" loses nothing: the sniffed bytes are still in the
;; buffer, for `read-bytes` and `read-line` alike.
(: (-> InputPort int int (Option (Array byte))))
(defbjouble (peek-bytes p skip count)
  (#:sync (clr-peek p skip count))
  (#:bjo  (clr-peek-await p skip count)))

;; ---------------------------------------------------------------------------
;; Writing
;; ---------------------------------------------------------------------------
;;
;; Buffered, so most writes are a copy. A write that fills the buffer drains it,
;; and that is a syscall — which is why these have suspending twins at all.

(: (-> OutputPort (Array byte) void))
(defbjouble (write-bytes! p bytes)
  (#:sync (clr-write-bytes p bytes))
  (#:bjo  (clr-write-bytes-await p bytes)))


;; `flush-port` and `close-output-port` are the prelude's: there is one kind of
;; output port.

;; ---------------------------------------------------------------------------
;; The half-close
;; ---------------------------------------------------------------------------

;; Ends the write side and leaves the read side open.
;;
;; A protocol needs this and `close` cannot say it: "I have finished speaking,
;; now answer me" is a half-close, not a closed connection. Everything buffered
;; goes out first, and a write after it is refused rather than dropped. A stream
;; with no half-close of its own gets the flush and nothing more, which is the
;; most that can honestly be done for a file.
(: (-> OutputPort void))
(defbjouble (shutdown! p)
  (#:sync (clr-shutdown p))
  (#:bjo  (clr-shutdown-await p)))

;; ---------------------------------------------------------------------------
;; Composition
;; ---------------------------------------------------------------------------

;; At most `n` bytes of `p`, and then end of input. Once the bound is reached,
;; `p` itself is positioned exactly where the limit ended — the limited port
;; reads *through* `p` rather than around it, so nothing it buffered is lost or
;; double-counted.
;;
;; The result owns nothing: closing it does not close `p`.
(: (-> InputPort int InputPort))
(defun (limited p n) (clr-limited p n))

;; A port pair with nothing underneath: what is written to the output can be
;; read from the input.
;;
;; Unbounded and in memory, which is what a string port is one layer up — the GC
;; prices it, nothing registers it on a scope, and a write never waits. Closing
;; the output half, or `shutdown!` on it, is what the input half sees as end of
;; input.
(: (-> (Tuple InputPort OutputPort)))
(defun (byte-pipe)
  (def pipe (clr-make-pipe))
  (Tuple (clr-pipe-in pipe) (clr-pipe-out pipe)))

;; ---------------------------------------------------------------------------
;; Sources
;; ---------------------------------------------------------------------------
;;
;; A file is opened with the prelude's `open-input-file` and `open-output-file`:
;; there is one kind of input port and one kind of output port, and a file port
;; takes bytes as readily as text.

;; A port over a `ClrStream` somebody else handed over — a `NetworkStream`, a
;; request body, a `MemoryStream`. This pair is what `(std net)` and a web
;; framework reach for, and it is the only way into this module from outside it.
;;
;; Owned like a file, because a `ClrStream` is a handle in every case that matters.
(: (-> ClrStream InputPort))
(defun (stream->input-port s)
  (ignore (current-scope))
  (clr-from-stream s))

(: (-> ClrStream OutputPort))
(defun (stream->output-port s)
  (ignore (current-scope))
  (clr-to-stream s))

;; ---------------------------------------------------------------------------
;; Encodings
;; ---------------------------------------------------------------------------

;; A port whose bytes are `p`'s read as `encoding` and re-encoded as UTF-8, as
;; Racket's `reencode-input-port` makes one: the way to read text that is not
;; UTF-8, since every port's text reads decode UTF-8.
;;
;; Reads *through* `p`, so whatever `p` had buffered — from a `peek-bytes`, or
;; a refill that overshot — is not skipped: sniff the first bytes, then
;; re-encode the rest. Closing the new port does not close `p`.
;;
;; `encoding` is any value from `(text encodings)`. Bytes that do not decode
;; read as U+FFFD, as they do on every port; `bytes->string` is the strict
;; one. A byte order mark is not treated specially.
(: (-> InputPort Encoding InputPort))
(defun (reencode-input-port p encoding)
  (clr-reencode p encoding))

;; Output's counterpart, as Racket's `reencode-output-port` makes one: what is
;; written to the new port is UTF-8 re-encoded as `encoding` on its way into
;; `p`. A character the encoding lacks is written as its replacement. Closing
;; the new port flushes it into `p` and leaves `p` open.
(: (-> OutputPort Encoding OutputPort))
(defun (reencode-output-port p encoding)
  (clr-reencode-out p encoding))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/prelude.bjo
;; Standard library entry point
;; Macros defined here CANNOT be used in this file. Those of (std core-macros),
;; such as cond, can.
(export
  ;; Types.
  Map MapBuilder TransientMap MapCursor TransientVec
  Range RangeBy
  ArraySlice ReverseArraySlice VecSlice ReverseVecSlice
  VecBackCursor StringWalk ListTails PortReads UpFrom DownFrom
  Enumerated Zipped Cycled Pairwise Repeated Chained
  Listing ListingReverse Vecing VecingSmall Arraying Mapping Stringing Summing Counting Folding
  Maximizing Minimizing MaximizingPairs MinimizingPairs Tallying GroupingPairs
  VecingInto ListingInto ListingReverseInto MappingInto
  InputPort OutputPort Exception Encoding

  ;; Iterable
  iterable-start iterable-done? iterable-current iterable-next iterable-end iterable-cleanup
  fold map filter take drop take-while drop-while
  seq-enumerate flat-map any? all? find for-each ->seq
  ->list ->vec count first ->map
  ;; Adapters
  enumerate zip in-cycle pairwise in-repeat chain
  ;; Cursor.
  cursor-start cursor-end cursor-end? cursor-ref cursor-next cursor-prev
  ;; Collector
  collector-init collector-step collector-finish
  ;; Vector forms
  contiguous? contiguous-values contiguous-valid
  block-count block-grow block-span block-none!
  ;; `#:into` on a collector
  collector-into
  ;; Collection
  length empty?
  ;; Addable! and Addable
  add! add-all!
  add add-all
  ;; Discard.
  ignore
  ;; Range
  range range-by
  ;; Iterating part of a collection, or backwards, or without one.
  ;; Written in iterators.protobjo, which this file includes.
  in-array in-reverse-array in-vec in-reverse-vec
  over-list in-port up-from down-from
  int-max int-min
  ;; The cursors those walk with, named because an inlined impl body reaches them
  vec-back-cursor vec-back-cursor-done? vec-back-cursor-current vec-back-cursor-next
  string-walk string-walk-done? string-walk-current string-walk-next

  print println ->str list->str
  ;; Written and Displayed
  written write-into displayed display-into
  elements-print-into entries-print-into
  list-append

  ;; List operations.
  list-map list-filter list-foldl list-foldr list-for-each list-unfold
  list-contains?

  ;; Vec operations.
  vec-take vec-drop vec-reverse vec->array vec-try-pop vec-try-pop-first
  vec-contains? array-contains? iarray-contains? transientvec-contains?
  vec-add vec-add-first vec-insert vec-remove-at vec-slice vec-split
  vec-map vec-filter vec-fold vec-reduce
  vec-for-each vec-iter

  ;; Seq operations.
  seq-fold seq-for-each

  ;; Maps. 
  map-empty map-ref map-ref-or map-try-ref
  map-set map-add-new map-try-add-new map-set-all map-remove map-remove-all
  map-alter map-update
  map-contains? map-has-key? map-length map-empty? map-clear
  map-keys map-values map-merge map-merge-with
  ;; Walking and rebuilding
  map-map map-map-values map-filter map-fold
  map-for-each map-iter map-any? map-find map-split
  ;; The cursor `Iterable` walks with
  map-cursor map-cursor-done? map-cursor-current map-cursor-next
  ;; Conversions.
  list->map list->vec map->list vec->list vec->map map->vec seq->map map->seq
  map->str map-entry->str
  ;; Builders.
  mapbuilder-empty map->mapbuilder mapbuilder-add! mapbuilder-add-all!
  mapbuilder-length mapbuilder->map
  ;; The vec's transient.
  transientvec-empty vec->transientvec transientvec->vec
  transientvec-length transientvec-empty? transientvec-ref transientvec-set!
  transientvec-add! transientvec-add-all! transientvec-pop! transientvec-try-pop!
  transientvec-add-first! transientvec-pop-first! transientvec-try-pop-first!
  ;; The immutable array.
  IArray
  iarray-empty iarray-length iarray-empty? iarray-ref iarray-try-ref
  iarray-set iarray-add iarray-map
  subarray->iarray array->iarray iarray->array seq->iarray iarray->seq
  vec->iarray iarray->vec list->iarray iarray->list
  ;; Arrays.
  array-copy!
  ;; Spans.
  Span RoSpan
  array->span array->ro-span span->ro-span
  span-length span-ref span-set! span-slice span->array span->
  ro-span-length ro-span-ref ro-span-slice ro-span->array ro-span->
  ;; Transients.
  transientmap-empty map->transientmap transientmap->map
  transientmap-set! transientmap-set-all! transientmap-remove!
  transientmap-alter! transientmap-update!
  transientmap-remove-all! transientmap-contains? transientmap-has-key? transientmap-ref
  transientmap-ref-or transientmap-try-ref transientmap-length
  transientmap-empty? transientmap-filter!

  ;; Results
  ok? err? result-ref result-ref-or result-map result-map-err result->option
  option-map option-filter option-or option->result
  ;; Pairs
  fst snd

  ;; Box
  Box
  make-box box-ref box-set!

  ;; Refable, Settable, Settable!, Derefable, Derefable!
  ref try-ref
  ref-set
  ref-set!
  deref try-deref
  deref-set!

  ;; Strings
  str string-append string-length
  string-contains? string-starts-with? string-ends-with? string-replace
  string-upcase string-downcase
  string-trim string-trim-left string-trim-right
  string-trim/slice string-trim-left/slice string-trim-right/slice string-copy
  string-pad-left string-pad-right
  string-split string-split/slice string-join
  string-split-once string-split-once/slice
  ;; Taking and dropping characters.
  string-take string-drop string-take-right string-drop-right
  string-take/slice string-drop/slice string-take-right/slice string-drop-right/slice
  string-cursor-forward string-cursor-back
  ;; Characters.
  char=? char<? char>? char<=? char>=?
  char-ci=? char-ci<? char-ci>? char-ci<=? char-ci>=?
  ;; String cursors.
  string-cursor=? string-cursor<? string-cursor>? string-cursor<=? string-cursor>=?
  string-fold string-fold-right string-for-each string-find
  string->list list->string string->vec string-reverse
  ;; Conversions
  int->string long->string double->string byte->string number->string
  string->int string->long string->double
  decimal->string string->decimal bigint->string string->bigint
  int128->string string->int128 uint128->string string->uint128
  bytes->string string->bytes bytes-utf-8?
  Utf8Chunk bytes->utf8-chunk utf8-chunk-length utf8-chunk-valid-end utf8-chunk-substring
  ;; Math.
  sqrt
  ;; Files
  file-read-text file-write-text file-append-text file-exists? file-delete file-move
  file-info file-copy file-read-bytes file-write-bytes file-append-bytes
  file-read-lines file-write-lines
  file-read/seq file-read-lines/seq
  ;; Paths
  path-absolute path-combine path-filename path-file-extension
  ;; Directories
  directory-exists? directory-create directory-delete directory-delete-tree
  directory-files directory-subdirectories directory-walk
  ;; The environment
  get-environment-variable set-environment-variable!
  current-directory set-current-directory!
  ;; Open ports.
  open-input-file open-input-string open-output-file open-output-string
  ;; Ports — closing
  close-input-port close-output-port flush-port
  ;; Ports — the openers past the FS effect
  open-input-file/raw open-output-file/raw WriteMode
  call-with-input-file call-with-output-file
  ;; The FS effect: its operations, their cells, and the defaults
  FsKind FsInfo MoveMode
  open-read-stream op/open-read-stream open-read-stream/raw
  open-write-stream op/open-write-stream open-write-stream/raw
  fs-info op/fs-info fs-info/raw
  fs-list op/fs-list fs-list/raw
  fs-create-directory op/fs-create-directory fs-create-directory/raw
  fs-delete-file op/fs-delete-file fs-delete-file/raw
  fs-delete-directory op/fs-delete-directory fs-delete-directory/raw
  fs-move op/fs-move fs-move/raw
  ;; Ports — input
  port-eof? read-line read-line/opt LineMode read-char read-char/opt
  peek-char peek-char/opt read-all port-fold
  ;; Ports — output
  write-string write-char writeln get-output-string get-output-bytes

  ;; Durations, which every timer takes.
  Duration ticks->duration duration-ticks milliseconds seconds minutes hours
  timeout duration->timer-ms

  ;; Effects the standard library performs. The `op/` name of each is the
  ;; parameter `with-handler` binds, and is exported for that reason.
  warn op/warn
  log op/log
  getenv op/getenv

  ;; The clock
  Clock make-clock system-clock current-clock monotonic-ms monotonic-ns

  ;; The dynamic environment, taken here and run in on another thread.
  DynamicEnvironment current-dynamic-environment call-with-dynamic-environment

  promise-join/sync panic!

  ;; What a grammar's checked literal calls. See (def/grammar ...).
  grammar-checked)


(import/extern (clr-panic (: BjolangRuntime.Panic (-> string int %a))))


(: (-> string (#:exit-code int) %a))
(defun (panic! msg #:exit-code 1)
  (clr-panic msg exit-code))

;; The value of a grammar's check on a literal. The compiler ran the same
;; check on the same literal and refused an Err, so the Err case is not
;; reached unless the check gives different results for one value.
(: (-> (Result string %a) %a))
(defun (grammar-checked r)
  (match r
    ((Ok v) v)
    ((Err e) (panic! (str "a check of a grammar gave Err for a literal it gave Ok for when compiled: " e)))))


;; The `written` fallback, invariant. `System.Convert.ToString` reads the ambient
;; culture, which is how a `short` or a `float` — types with no conversion of
;; their own below — would print differently on a differently configured
;; machine. Generic, because a union case that is `null` at run time is named
;; by the type the value has, and the value alone does not say it.
(import/extern
  (clr-show (: Bjolang.Runtime.BjoNum.Show (-> %a string))))

;; The docs: the prelude's own definitions and the builtins beside them, by
;; area. The builtins are documented here because every program gets them with
;; the prelude, and their docs are published in its assembly.
(:doc #:file "prelude.bjodoc")
(:doc #:file "prelude-core.bjodoc")
(:doc #:file "prelude-seqs.bjodoc")
(:doc #:file "prelude-lists.bjodoc")
(:doc #:file "prelude-vecs.bjodoc")
(:doc #:file "prelude-maps.bjodoc")
(:doc #:file "prelude-strings.bjodoc")
(:doc #:file "prelude-chars.bjodoc")
(:doc #:file "prelude-files.bjodoc")
(:doc #:file "prelude-ports.bjodoc")
(:doc #:file "prelude-concurrency.bjodoc")
(:doc #:file "prelude-syntax.bjodoc")

;; Before the includes, which use them: cond and the rest of the macros the
;; prelude needs itself.
(import (std core-macros))
(re-export cond)

;; First, and before the includes. = is a trait and needs to be imported
(import (std eq))
;; re-export ord and eq traits, and not
(re-export = eq-hash not
           compare less? greater? at-most? at-least?
                       least greatest list-sort vec-sort vec-sort-by array-sort! array-sort-by!
                       list-sort-unstable vec-sort-unstable array-sort-unstable!
                       list-min list-max)


;; List append.
(: (-> (List %a) (List %a) #:rest (List %a) (List %a)))
(defun (list-append xs ys #:rest more)
  ;; Each list but the last is copied, once, and the last is shared: the
  ;; lists are joined from the right.
  (def n (array-length more))
  (if (= n 0)
      (list-foldr Cons ys xs)
      (loop (:for i (down-from (- n 2) #:to 0))
            (:acc acc (folding (array-ref more (- n 1))
                               (list-foldr Cons acc (array-ref more i))))
            => (list-foldr Cons (list-foldr Cons acc ys) xs))))

;; ---------------------------------------------------------------------------
;; Written and Displayed
;; ---------------------------------------------------------------------------
;; `written` is a value as the reader reads it, as Scheme's `write` writes
;; it: a string in quotes, a char as `#\a`, a long as `21L`. `displayed` is
;; the value for a person to read, as `display` writes it: a string is its
;; text. Every type is written; a type is displayed as it is written unless
;; it says otherwise.
;;
;; Each trait has two methods. `write-into` adds the value to a builder, and
;; is what a container calls on its elements, so that a nested value is
;; written into one buffer. `written` answers a string, through `write-into`
;; and a builder the thread keeps. A type whose text it has at hand, such as
;; a number, gives `written` too, and no builder is used.
(import/extern
  (clr-builder-rent   (: Bjolang.Runtime.BjoShow.Rent))
  (clr-builder-finish (: Bjolang.Runtime.BjoShow.Finish))
  (clr-write-string-into (: Bjolang.Runtime.BjoShow.WriteStringInto))
  (clr-written-char   (: Bjolang.Runtime.BjoShow.WrittenChar))
  (clr-written-double (: Bjolang.Runtime.BjoShow.WrittenDouble))
  (clr-written-number (: Bjolang.Runtime.BjoShow.WrittenNumber (-> %a string string))))

(def/trait (Written %a)
  (: write-into (-> %a StringBuilder void))
  (: written (-> %a string))
  (defun (written x)
    (def b (clr-builder-rent))
    (write-into x b)
    (clr-builder-finish b)))

(def/trait (Displayed %a)
  (where (Written %a))
  (: display-into (-> %a StringBuilder void))
  (: displayed (-> %a string))
  (defun (displayed x)
    (def b (clr-builder-rent))
    (display-into x b)
    (clr-builder-finish b)))

;; A .NET type is written as it writes itself, in the invariant culture.
(impl (Written %a)
  (defun (written x) (clr-show x))
  (defun (write-into x b) (stringbuilder-add-string! b (clr-show x))))

;; Every type is displayed as it is written, unless it says otherwise.
(impl (Displayed %a)
  (defun (displayed x) (written x))
  (defun (display-into x b) (write-into x b)))

;; A function, as Scheme writes one. No blanket reaches a function type, so it
;; is written and displayed by these, inside a container as well.
(impl (Written Procedure)
  (defun (written f) "#<procedure>")
  (defun (write-into f b) (stringbuilder-add-string! b "#<procedure>")))

(impl (Displayed Procedure)
  (defun (displayed f) "#<procedure>")
  (defun (display-into f b) (stringbuilder-add-string! b "#<procedure>")))

;; Each number with the suffix that gives its type back, and displayed
;; without it.
(impl (Written int)
  (defun (written n) (int->string n))
  (defun (write-into n b) (stringbuilder-add-string! b (int->string n))))

(impl (Written long)
  (defun (written n) (clr-written-number n "L"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed long)
  (defun (displayed n) (long->string n))
  (defun (display-into n b) (stringbuilder-add-long! b n)))

(impl (Written byte)
  (defun (written n) (clr-written-number n "uy"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed byte)
  (defun (displayed n) (byte->string n))
  (defun (display-into n b) (stringbuilder-add-string! b (byte->string n))))

(impl (Written short)
  (defun (written n) (clr-written-number n "s"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed short)
  (defun (displayed n) (clr-show n))
  (defun (display-into n b) (stringbuilder-add-string! b (clr-show n))))

(impl (Written ushort)
  (defun (written n) (clr-written-number n "us"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed ushort)
  (defun (displayed n) (clr-show n))
  (defun (display-into n b) (stringbuilder-add-string! b (clr-show n))))

(impl (Written uint)
  (defun (written n) (clr-written-number n "u"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed uint)
  (defun (displayed n) (clr-show n))
  (defun (display-into n b) (stringbuilder-add-string! b (clr-show n))))

(impl (Written ulong)
  (defun (written n) (clr-written-number n "UL"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed ulong)
  (defun (displayed n) (clr-show n))
  (defun (display-into n b) (stringbuilder-add-string! b (clr-show n))))

(impl (Written decimal)
  (defun (written n) (clr-written-number n "M"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed decimal)
  (defun (displayed n) (clr-show n))
  (defun (display-into n b) (stringbuilder-add-string! b (clr-show n))))

(impl (Written bigint)
  (defun (written n) (clr-written-number n "I"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed bigint)
  (defun (displayed n) (clr-show n))
  (defun (display-into n b) (stringbuilder-add-string! b (clr-show n))))

;; A float, an int128 and a uint128 have no suffix, so each is written with
;; its type, as a value with its type written is read: `(: 2.5 float)`.
(import/extern
  (clr-written-single (: Bjolang.Runtime.BjoShow.WrittenSingle)))

(impl (Written float)
  (defun (written n) (str "(: " (clr-written-single n) " float)"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed float)
  (defun (displayed n) (clr-show n))
  (defun (display-into n b) (stringbuilder-add-string! b (clr-show n))))

(import/extern
  (clr-written-rational (: Bjolang.Runtime.BjoRational.Written (-> rational string))))

(impl (Written rational)
  (defun (written n) (clr-written-rational n))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed rational)
  (defun (displayed n) (clr-show n))
  (defun (display-into n b) (stringbuilder-add-string! b (clr-show n))))

(impl (Written int128)
  (defun (written n) (str "(: " (clr-written-number n "") " int128)"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed int128)
  (defun (displayed n) (clr-show n))
  (defun (display-into n b) (stringbuilder-add-string! b (clr-show n))))

(impl (Written uint128)
  (defun (written n) (str "(: " (clr-written-number n "") " uint128)"))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(impl (Displayed uint128)
  (defun (displayed n) (clr-show n))
  (defun (display-into n b) (stringbuilder-add-string! b (clr-show n))))

;; A double is written with a point, so that it is not read back as an int.
(impl (Written double)
  (defun (written n) (clr-written-double n))
  (defun (write-into n b) (stringbuilder-add-string! b (clr-written-double n))))

(impl (Displayed double)
  (defun (displayed n) (double->string n))
  (defun (display-into n b) (stringbuilder-add-double! b n)))

;; As the language writes a boolean, not as .NET does.
(impl (Written bool)
  (defun (written b) (if b "#t" "#f"))
  (defun (write-into x b) (stringbuilder-add-string! b (if x "#t" "#f"))))

;; A string in quotes, and displayed as its text.
(impl (Written string)
  (defun (write-into s b) (clr-write-string-into b s)))

(impl (Displayed string)
  (defun (displayed s) s)
  (defun (display-into s b) (stringbuilder-add-string! b s)))

(impl (Written Char)
  (defun (written c) (clr-written-char c))
  (defun (write-into c b) (stringbuilder-add-string! b (clr-written-char c))))

(impl (Displayed Char)
  (defun (displayed c) (char->string c))
  (defun (display-into c b) (stringbuilder-add! b c)))

(impl (Written Keyword)
  (defun (written k) (str "#:" (keyword->string k)))
  (defun (write-into k b)
    (stringbuilder-add-string! b "#:")
    (stringbuilder-add-string! b (keyword->string k))))

(impl (Displayed Keyword)
  (defun (displayed k) (keyword->string k))
  (defun (display-into k b) (stringbuilder-add-string! b (keyword->string k))))

(impl (Written Symbol)
  (defun (written s) (symbol->string s))
  (defun (write-into s b) (stringbuilder-add-string! b (symbol->string s))))

;; Container formats are further down.

;; ---------------------------------------------------------------------------
;; ->str
;; ---------------------------------------------------------------------------
;; What `displayed` was called before there was `written`. An implementation
;; of its own is still called by `->str`, but not by `displayed`, `println`
;; or an interpolation, which is why it is deprecated.
(def/trait (->str %a)
  (where (Displayed %a))
  (: ->str (-> %a string)))

(impl (->str %a)
  (defun (->str x) (displayed x)))

(:deprecated ->str "use `displayed`, and implement `Displayed` or `Written`")


;; ---------------------------------------------------------------------------
;; Discard
;; ---------------------------------------------------------------------------
;; Discard value.
(def/trait (Discard %a)
  (: ignore (-> %a void)))

(impl (Discard %a)
  (defun (ignore x) unit))

;; Detach promise.
(impl (Discard (Promise %a))
  (defun (ignore p) (detach p)))

;; Writes a value as `displayed` gives it.
(: (-> %a void))
(defun (print x)
  (display (displayed x)))

(: (-> %a void))
(defun (println x)
  (displayln (displayed x)))


;; ---------------------------------------------------------------------------
;; TransientVec — a vec changed in place
;; ---------------------------------------------------------------------------
;; The builder a `VecBuilder` is, under its own name, with the ends a builder
;; has no use for.
(import/class
  ((TransientVec %a) (: Bjolang.Runtime.Rrb.RrbTransient)))

(import/extern
  (transientvec-empty      (: Bjolang.Runtime.Rrb.TransientVecModule.Empty    (-> (TransientVec %a))))
  (vec->transientvec       (: Bjolang.Runtime.Rrb.TransientVecModule.FromList (-> (Vec %a) (TransientVec %a))))
  (transientvec->vec       (: Bjolang.Runtime.Rrb.TransientVecModule.ToList   (-> (TransientVec %a) (Vec %a))))
  (transientvec-length     (: Bjolang.Runtime.Rrb.TransientVecModule.Count    (-> (TransientVec %a) int)))
  (transientvec-empty?     (: Bjolang.Runtime.Rrb.TransientVecModule.IsEmpty  (-> (TransientVec %a) bool)))
  (transientvec-ref        (: Bjolang.Runtime.Rrb.TransientVecModule.Ref      (-> (TransientVec %a) int %a)))
  (transientvec-set!       (: Bjolang.Runtime.Rrb.TransientVecModule.Set      (-> (TransientVec %a) int %a void)))
  (transientvec-add!       (: Bjolang.Runtime.Rrb.TransientVecModule.Add      (-> (TransientVec %a) %a void)))
  (transientvec-add-all!   (: Bjolang.Runtime.Rrb.TransientVecModule.AddRange (-> (TransientVec %a) (Seq %a) void)))
  (transientvec-add-first! (: Bjolang.Runtime.Rrb.TransientVecModule.Prepend  (-> (TransientVec %a) %a void)))
  (transientvec-pop!       (: Bjolang.Runtime.Rrb.TransientVecModule.Pop      (-> (TransientVec %a) %a)))
  (transientvec-pop-first! (: Bjolang.Runtime.Rrb.TransientVecModule.PopFirst (-> (TransientVec %a) %a))))

;; The last element, removed, or `None` when the transient is empty. `None`
;; leaves the transient as it was.
(: (-> (TransientVec %a) (Option %a)))
(defun (transientvec-try-pop! t)
  (if (transientvec-empty? t) None (Some (transientvec-pop! t))))

;; The first element, removed, or `None` when the transient is empty. `None`
;; leaves the transient as it was.
(: (-> (TransientVec %a) (Option %a)))
(defun (transientvec-try-pop-first! t)
  (if (transientvec-empty? t) None (Some (transientvec-pop-first! t))))

;; ---------------------------------------------------------------------------
;; IArray — the immutable array
;; ---------------------------------------------------------------------------
;; A struct around .NET's `ImmutableArray<T>`: one array that is not written
;; after it is made. Indexing is an array read, and the value is one reference
;; wide. Every change copies the whole array, so it is for values that are
;; built once and then read. `Vec` is the one to change. The impls are with
;; those of `Array`. Why it is a wrapper is in `BjoIArray.cs`.
(import/class
  ((IArray %a) (: Bjolang.Runtime.IArray)))

(import/extern
  (iarray-empty     (: Bjolang.Runtime.IArrayModule.Empty          (-> (IArray %a))))
  (iarray-length    (: Bjolang.Runtime.IArrayModule.Count          (-> (IArray %a) int)))
  (iarray-empty?    (: Bjolang.Runtime.IArrayModule.IsEmpty        (-> (IArray %a) bool)))
  ;; Throws when the index is out of range. `iarray-try-ref` is the checked one.
  (iarray-ref       (: Bjolang.Runtime.IArrayModule.Get            (-> (IArray %a) int %a)))
  (iarray-try-ref   (: Bjolang.Runtime.IArrayModule.TryGet         (-> (IArray %a) int (out %a) (Option %a))))
  ;; A copy with one element replaced, or one added at the end.
  (iarray-set       (: Bjolang.Runtime.IArrayModule.SetItem        (-> (IArray %a) int %a (IArray %a))))
  (iarray-add       (: Bjolang.Runtime.IArrayModule.Add            (-> (IArray %a) %a (IArray %a))))
  (iarray-map       (: Bjolang.Runtime.IArrayModule.Map            (-> (-> %a %b) (IArray %a) (IArray %b))))
  ;; The array, start and count, as `subarray->vec` takes them. The elements
  ;; are copied, so the array can be written again.
  (subarray->iarray (: Bjolang.Runtime.IArrayModule.FromSubarray   (-> (Array %a) int int (IArray %a))))
  (array->iarray    (: Bjolang.Runtime.IArrayModule.FromArray      (-> (Array %a) (IArray %a))))
  ;; A new array, which the caller can write.
  (iarray->array    (: Bjolang.Runtime.IArrayModule.ToArray        (-> (IArray %a) (Array %a))))
  (seq->iarray      (: Bjolang.Runtime.IArrayModule.FromEnumerable (-> (Seq %a) (IArray %a))))
  (iarray->seq      (: Bjolang.Runtime.IArrayModule.AsEnumerable   (-> (IArray %a) (Seq %a)))))

(: (-> (Vec %a) (IArray %a)))
(defun (vec->iarray v) (seq->iarray (vec->seq v)))

(: (-> (List %a) (IArray %a)))
(defun (list->iarray xs) (seq->iarray (list->seq xs)))

;; ---------------------------------------------------------------------------
;; Arrays
;; ---------------------------------------------------------------------------
;; The other array operations (`make-array`, `array-ref`, `array-resize` ...)
;; are compiler builtins.
(import/extern
  (array-copy! (: Bjolang.Runtime.BjoArray.Copy (-> (Array %a) int (Array %a) int int void))))

;; ---------------------------------------------------------------------------
;; Spans
;; ---------------------------------------------------------------------------
;; A view of part of an array, which is valid only while the call that made
;; it runs. `SpanCheck` refuses one where it could outlive that call. A
;; `Span` and a `RoSpan` have separate functions, because neither converts to
;; the other implicitly; `span->ro-span` converts explicitly.
(import/class
  ((Span %a)   (: System.Span))
  ((RoSpan %a) (: System.ReadOnlySpan)))

(import/extern
  (clr-array->span    (: Bjolang.Runtime.Spans.FromArray         (-> (Array %a) int int (Span %a))))
  (clr-array->ro-span (: Bjolang.Runtime.Spans.ReadOnlyFromArray (-> (Array %a) int int (RoSpan %a))))
  (clr-span-slice     (: Bjolang.Runtime.Spans.Slice             (-> (Span %a) int int (Span %a))))
  (clr-ro-span-slice  (: Bjolang.Runtime.Spans.Slice             (-> (RoSpan %a) int int (RoSpan %a))))
  (span-length        (: Bjolang.Runtime.Spans.Length            (-> (Span %a) int)))
  (span-ref           (: Bjolang.Runtime.Spans.Ref               (-> (Span %a) int %a)))
  (span-set!          (: Bjolang.Runtime.Spans.Set               (-> (Span %a) int %a void)))
  (span->array        (: Bjolang.Runtime.Spans.ToArray           (-> (Span %a) (Array %a))))
  (span->ro-span      (: Bjolang.Runtime.Spans.ReadOnly          (-> (Span %a) (RoSpan %a))))
  (ro-span-length     (: Bjolang.Runtime.Spans.Length            (-> (RoSpan %a) int)))
  (ro-span-ref        (: Bjolang.Runtime.Spans.Ref               (-> (RoSpan %a) int %a)))
  (ro-span->array     (: Bjolang.Runtime.Spans.ToArray           (-> (RoSpan %a) (Array %a))))
  (clr-span->         (: Bjolang.Runtime.Spans.As                (-> (Span %a) (Span %b))))
  (clr-ro-span->      (: Bjolang.Runtime.Spans.As                (-> (RoSpan %a) (RoSpan %b)))))

;; The same memory as numbers of another type, as a (Span byte) as a
;; (Span long). The use gives the type. Nothing is copied: a write through
;; one span is a write through the other, and a read-only span gives a
;; read-only one. Only plain numbers: the integers, float and double, which
;; have no references in them, and whose every bit pattern is a value. A
;; decimal or a bigint, and a length in bytes that is not a whole number of
;; the new elements, is a panic.
(: (-> (Span %a) (Span %b)) (where (Num %a) (Num %b)))
(defun (span-> s) (clr-span-> s))

(: (-> (RoSpan %a) (RoSpan %b)) (where (Num %a) (Num %b)))
(defun (ro-span-> s) (clr-ro-span-> s))

(: (-> (Array %a) (#:from int) (#:to int) (Span %a)))
(defun (array->span a #:from 0 #:to (array-length a))
  (clr-array->span a from to))

(: (-> (Array %a) (#:from int) (#:to int) (RoSpan %a)))
(defun (array->ro-span a #:from 0 #:to (array-length a))
  (clr-array->ro-span a from to))

(: (-> (Span %a) (#:from int) (#:to int) (Span %a)))
(defun (span-slice s #:from 0 #:to (span-length s))
  (clr-span-slice s from to))

(: (-> (RoSpan %a) (#:from int) (#:to int) (RoSpan %a)))
(defun (ro-span-slice s #:from 0 #:to (ro-span-length s))
  (clr-ro-span-slice s from to))

;; ---------------------------------------------------------------------------
;; Map — the unordered map
;; ---------------------------------------------------------------------------
;;
;; A hash trie written in Bjolang, over the C# nodes of
;; `Bjolang.Runtime.HashTries`. Included rather than imported: the prelude
;; defines `Map`, and a module of its own would have to import the prelude
;; that is still being compiled.
(include "hashmap.protobjo")

(: (-> (Map %k %v) %k bool))
(defun (map-has-key? m key) (map-contains? m key))

(: (-> (TransientMap %k %v) %k bool))
(defun (transientmap-has-key? m key) (transientmap-contains? m key))

;; `map-add-new` with the failure as a value: `(Err v)` carries the value the
;; key already has.
(: (-> (Map %k %v) %k %v (Result %v (Map %k %v))))
(defun (map-try-add-new m k v)
  (match (map-try-ref m k)
    ((Some old) (Err old))
    (None (Ok (map-set m k v)))))

;; Two maps are equal when they hold the same entries.

(impl (Eq (Map %k %v))
  (where (Eq %k) (Eq %v))
  (defun (= a b)
    (and (= (map-length a) (map-length b))
         (map-iter (fun (k v)
                     (match (map-try-ref b k)
                       ((Some w) (= v w))
                       (None #f)))
                   a)))
  ;; Hash map.
  (defun (eq-hash m)
    (map-fold (fun (k v h) (bitwise-xor h (hash-combine (eq-hash k) (eq-hash v)))) 0 m)))

;; Two immutable arrays are equal when they hold equal elements in the same
;; order. The value cannot change, so it has a hash.
(impl (Eq (IArray %a))
  (where (Eq %a))
  (defun (= a b)
    (def len (iarray-length a))
    (if (= len (iarray-length b))
        (let go ((i 0))
          (if (= i len)
              #t
              (if (= (iarray-ref a i) (iarray-ref b i))
                  (go (+ i 1))
                  #f)))
        #f))
  (defun (eq-hash xs)
    (def len (iarray-length xs))
    (let go ((i 0) (h 17))
      (if (= i len)
          h
          (go (+ i 1) (hash-combine h (eq-hash (iarray-ref xs i))))))))




;; ---------------------------------------------------------------------------
;; Collector
;; ---------------------------------------------------------------------------

;; Collector interface.
(def/trait (Collector %c)
  (type %elem)
  (type %acc)
  (type %out)
  (: collector-init (-> %c %acc))
  (: collector-step (-> %c %acc %elem %acc))
  (: collector-finish (-> %c %acc %out)))

;; ---------------------------------------------------------------------------
;; Vector forms of drivers and collectors
;; ---------------------------------------------------------------------------
;;
;; A loop whose drivers are Contiguous and whose collectors are
;; BlockCollectors can be compiled to vector instructions. Each implementation
;; is a promise of its author: the compiler does not prove it. See
;; Docs/Vectorisation-design.org.

;; An Iterable whose values are in memory one after another. The Iterable's
;; cursor is an int that starts at 0 and steps by 1, and the walk is done when
;; the cursor is the length of the values. The element at cursor i is
;; values[i], or, when the element is an (Option %value), (Some values[i]) if
;; bit i of the valid bitmap is set and None if it is not.
(def/trait (Contiguous %s)
  (type %value)
  ;; Whether the values are in one span now. If not, the loop walks them one
  ;; by one.
  (: contiguous? (-> %s bool))
  (: contiguous-values (-> %s (RoSpan %value)))
  ;; The valid bitmap, bit i of byte i/8 for element i, the least
  ;; significant bit first. Empty when no element is None.
  (: contiguous-valid (-> %s (RoSpan byte))))

;; A Collector whose accumulator can take a block of elements at a time. The
;; collector's element is %value or (Option %value).
(def/trait (BlockCollector %c)
  (type %value)
  (type %acc)
  ;; The number of elements taken in so far.
  (: block-count (-> %c %acc int))
  ;; The accumulator with room for n more elements at the end. They are as if
  ;; taken in: not None, with values that are written next.
  (: block-grow (-> %c %acc int %acc))
  ;; The elements from start, n of them, to write.
  (: block-span (-> %c %acc int int (Span %value)))
  ;; Makes element i None.
  (: block-none! (-> %c %acc int void)))

;; Collectors.

(type (: (Listing %a) (Union listing)))

;; Listing.
(impl (Collector (Listing %a))
  (type %elem %a)
  (type %acc (ListBuilder %a))
  (type %out (List %a))
  (defun (collector-init c) (listbuilder-empty))
  (defun (collector-step c acc x) (listbuilder-add! acc x) acc)
  (defun (collector-finish c acc) (listbuilder->list acc)))

;; Listing in reverse: each value consed onto the list so far, so the list is
;; the values last first, and there is no builder to finish.
(type (: (ListingReverse %a) (Union listing/reverse)))

(impl (Collector (ListingReverse %a))
  (type %elem %a)
  (type %acc (List %a))
  (type %out (List %a))
  (defun (collector-init c) Nil)
  (defun (collector-step c acc x) (Cons x acc))
  (defun (collector-finish c acc) acc))

;; Vecing.
(type (: (Vecing %a) (Union vecing)))

(impl (Collector (Vecing %a))
  (type %elem %a)
  (type %acc (VecBuilder %a))
  (type %out (Vec %a))
  (defun (collector-init c) (vecbuilder-empty))
  (defun (collector-step c acc x) (vecbuilder-add! acc x) acc)
  (defun (collector-finish c acc) (vecbuilder->vec acc)))

;; Arraying: the elements in an array. Its accumulator takes a block of
;; elements at a time, so a loop of numbers into it can be a vector loop.
(type (: (Arraying %a) (Union arraying)))

(import/class
  ((ArrayBuilder %a) (: Bjolang.Runtime.ArrayBuilder)))

(import/extern
  (arraybuilder-empty     (: Bjolang.Runtime.ArrayBuilderModule.Empty   (-> (ArrayBuilder %a))))
  (arraybuilder-add       (: Bjolang.Runtime.ArrayBuilderModule.Add     (-> (ArrayBuilder %a) %a (ArrayBuilder %a))))
  (arraybuilder-count     (: Bjolang.Runtime.ArrayBuilderModule.Count   (-> (ArrayBuilder %a) int)))
  (arraybuilder-grow      (: Bjolang.Runtime.ArrayBuilderModule.Grow    (-> (ArrayBuilder %a) int (ArrayBuilder %a))))
  (arraybuilder-span      (: Bjolang.Runtime.ArrayBuilderModule.Span    (-> (ArrayBuilder %a) int int (Span %a))))
  (arraybuilder->array    (: Bjolang.Runtime.ArrayBuilderModule.ToArray (-> (ArrayBuilder %a) (Array %a))))
  (arraybuilder-no-none!  (: Bjolang.Runtime.ArrayBuilderModule.NoNone  (-> (ArrayBuilder %a) int void))))

(impl (Collector (Arraying %a))
  (type %elem %a)
  (type %acc (ArrayBuilder %a))
  (type %out (Array %a))
  (defun (collector-init c) (arraybuilder-empty))
  (defun (collector-step c acc x) (arraybuilder-add acc x))
  (defun (collector-finish c acc) (arraybuilder->array acc)))

(impl (BlockCollector (Arraying %a))
  (type %value %a)
  (type %acc (ArrayBuilder %a))
  (defun (block-count c acc) (arraybuilder-count acc))
  (defun (block-grow c acc n) (arraybuilder-grow acc n))
  (defun (block-span c acc start n) (arraybuilder-span acc start n))
  (defun (block-none! c acc i) (arraybuilder-no-none! acc i)))

;; Vecing into a builder for a vec that is likely small: its tail starts at 4
;; items and doubles, where `vecing` takes a leaf of 32 at the first item.
(type (: (VecingSmall %a) (Union vecing/small)))

(impl (Collector (VecingSmall %a))
  (type %elem %a)
  (type %acc (VecBuilder %a))
  (type %out (Vec %a))
  (defun (collector-init c) (vecbuilder-empty/small))
  (defun (collector-step c acc x) (vecbuilder-add! acc x) acc)
  (defun (collector-finish c acc) (vecbuilder->vec acc)))

;; Mapping.
(type (: (Mapping %k %v) (Union mapping)))

(impl (Collector (Mapping %k %v))
  (where (Eq %k))
  (type %elem (Tuple %k %v))
  (type %acc (MapBuilder %k %v))
  (type %out (Map %k %v))
  (defun (collector-init c) (mapbuilder-empty))
  (defun (collector-step c acc kv)
    (def (k v) kv)
    (mapbuilder-add! acc k v)
    acc)
  (defun (collector-finish c acc) (mapbuilder->map acc)))

;; --- Starting from a collection: `#:into` ---
;;
;; `(vecing x #:into v)` adds to the back of `v` rather than to an empty vec,
;; and so do `listing` to a list and `mapping` to a map; `listing/reverse`
;; conses onto the front of its list, which costs nothing at all. The loop
;; makes the collector as `(collector-into vecing v)`, and the builder is made
;; from `v` once, before the first element: every step is the step it was.
;;
;; A collector takes `#:into` by implementing this, which says what it can
;; start from and what collector that makes. One that does not is a type error
;; naming it.
(def/trait (CollectorInto %c)
  (type %seed)
  (type %into)
  (: collector-into (-> %c %seed %into)))

(type (: (VecingInto %a) (Struct (: seed (Vec %a)))))

(impl (Collector (VecingInto %a))
  (type %elem %a)
  (type %acc (VecBuilder %a))
  (type %out (Vec %a))
  (defun (collector-init c) (vec->vecbuilder (record-ref c seed)))
  (defun (collector-step c acc x) (vecbuilder-add! acc x) acc)
  (defun (collector-finish c acc) (vecbuilder->vec acc)))

(impl (CollectorInto (Vecing %a))
  (type %seed (Vec %a))
  (type %into (VecingInto %a))
  (defun (collector-into c seed) (VecingInto (seed seed))))

;; The vec it starts from says how big it is, so `/small' has nothing to add.
(impl (CollectorInto (VecingSmall %a))
  (type %seed (Vec %a))
  (type %into (VecingInto %a))
  (defun (collector-into c seed) (VecingInto (seed seed))))

;; A list grows only at the front, so adding to its back copies it, once.
(type (: (ListingInto %a) (Struct (: seed (List %a)))))

(impl (Collector (ListingInto %a))
  (type %elem %a)
  (type %acc (ListBuilder %a))
  (type %out (List %a))
  (defun (collector-init c) (list->listbuilder (record-ref c seed)))
  (defun (collector-step c acc x) (listbuilder-add! acc x) acc)
  (defun (collector-finish c acc) (listbuilder->list acc)))

(impl (CollectorInto (Listing %a))
  (type %seed (List %a))
  (type %into (ListingInto %a))
  (defun (collector-into c seed) (ListingInto (seed seed))))

;; Consing onto the front shares the list it starts from: nothing is copied.
(type (: (ListingReverseInto %a) (Struct (: seed (List %a)))))

(impl (Collector (ListingReverseInto %a))
  (type %elem %a)
  (type %acc (List %a))
  (type %out (List %a))
  (defun (collector-init c) (record-ref c seed))
  (defun (collector-step c acc x) (Cons x acc))
  (defun (collector-finish c acc) acc))

(impl (CollectorInto (ListingReverse %a))
  (type %seed (List %a))
  (type %into (ListingReverseInto %a))
  (defun (collector-into c seed) (ListingReverseInto (seed seed))))

;; The map's own key comparer goes with it, so no `Eq' is asked for.
(type (: (MappingInto %k %v) (Struct (: seed (Map %k %v)))))

(impl (Collector (MappingInto %k %v))
  (type %elem (Tuple %k %v))
  (type %acc (MapBuilder %k %v))
  (type %out (Map %k %v))
  (defun (collector-init c) (map->mapbuilder (record-ref c seed)))
  (defun (collector-step c acc kv)
    (def (k v) kv)
    (mapbuilder-add! acc k v)
    acc)
  (defun (collector-finish c acc) (mapbuilder->map acc)))

(impl (CollectorInto (Mapping %k %v))
  (type %seed (Map %k %v))
  (type %into (MappingInto %k %v))
  (defun (collector-into c seed) (MappingInto (seed seed))))

;; Stringing.
(type (: Stringing (Union stringing)))

(impl (Collector Stringing)
  (type %elem char)
  (type %acc StringBuilder)
  (type %out string)
  (defun (collector-init c) (stringbuilder-empty))
  (defun (collector-step c acc x) (stringbuilder-add! acc x) acc)
  (defun (collector-finish c acc) (stringbuilder->string acc)))

;; Summing and counting work at every numeric type. A sum has the type of what
;; it adds. A count has the type its use asks for, and is an int when nothing
;; asks, as a literal is.
(type (: (Summing %a) (Union summing)))

(impl (Collector (Summing %a))
  (where (Num %a))
  (type %elem %a)
  (type %acc %a)
  (type %out %a)
  (defun (collector-init c) 0)
  (defun (collector-step c acc x) (+ acc x))
  (defun (collector-finish c acc) acc))

(type (: (Counting %a %n) (Union counting)))

(impl (Collector (Counting %a %n))
  (where (Num %n))
  (type %elem %a)
  (type %acc %n)
  (type %out %n)
  (defun (collector-init c) 0)
  (defun (collector-step c acc x) (+ acc 1))
  (defun (collector-finish c acc) acc))

;; Folding.
(type (: (Folding %a) (Union (: folding %a))))

(impl (Collector (Folding %a))
  (type %elem %a)
  (type %acc %a)
  (type %out %a)
  (defun (collector-init c) (match c ((folding s) s)))
  (defun (collector-step c acc x) x)
  (defun (collector-finish c acc) acc))




;; ---------------------------------------------------------------------------
;; Iterable
;; ---------------------------------------------------------------------------
;; Iterable protocol. What the compiler does, is that it fuses
;; subsequent seql, so doing (-> an-iterable map map map filter map take) should fuse
;; to only ONE IEnumerable, meaning only one of the shitty awful state machines.
(def/trait (Iterable %s)
  (type %elem)
  (type %cursor)
  (: iterable-start (-> %s %cursor))
  ;; A walk calls this once per step, before `current`. So a cursor may be a
  ;; .NET enumerator that steps here, as the `Seq` impl's does; its
  ;; `iterable-next` then answers the same enumerator.
  (: iterable-done? (-> %s %cursor bool))
  (: iterable-current (-> %s %cursor %elem))
  (: iterable-next (-> %s %cursor %cursor))
  ;; A loop calls this on each cursor it stops before `done?` answered true:
  ;; at a `:finish`, an `:abandon`, or when another driver of the level ended
  ;; first. It may also come after `done?` answered true, or a second time,
  ;; and must then do nothing: a `zip` ends the longer side when the shorter
  ;; one ends. A cursor that holds a resource for the walk releases it here.
  (: iterable-end (-> %s %cursor void))
  ;; A loop calls this, in place of `iterable-end`, on each cursor that is
  ;; live when an exception or a non-local exit (such as `ret` of a
  ;; `with-return`) leaves it. A cursor that holds a resource releases it here
  ;; as well. With the default, which does nothing, the loop has no `try`
  ;; around it and keeps no record of its cursors.
  (: iterable-cleanup (-> %s %cursor void))
  (: ->seq  (-> %s (Seq %elem)))
  (: fold (-> (-> %elem %acc %acc) %acc %s %acc))
  (: map    (-> (-> %elem %b) %s (Seq %b)))
  (: filter (-> (-> %elem bool) %s (Seq %elem)))
  (: take (-> int %s (Seq %elem)))
  (: drop (-> int %s (Seq %elem)))
  (: take-while (-> (-> %elem bool) %s (Seq %elem)))
  (: drop-while (-> (-> %elem bool) %s (Seq %elem)))
  (: seq-enumerate (-> %s (Seq (Tuple int %elem))))
  (: flat-map (-> (-> %elem (Seq %b)) %s (Seq %b)))
  ;; The methods below return a value rather than a sequence, so their defaults
  ;; are ordinary loops instead of `seql`s. A `seql` becomes a C# iterator,
  ;; which cannot await, so only these can take a callback of either colour.
  (: any? (-> (-?-> %elem bool) %s bool))
  (: all? (-> (-?-> %elem bool) %s bool))
  (: find (-> (-?-> %elem bool) %s (Option %elem)))
  (: for-each (-> (-?-> %elem void) %s void))
  (: ->list (-> %s (List %elem)))
  (: ->vec (-> %s (Vec %elem)))
  (: count (-> %s int))
  (: first (-> %s (Option %elem)))

  ;; Most cursors hold nothing that needs release.
  (defun (iterable-end s c) (when #f ()))
  (defun (iterable-cleanup s c) (when #f ()))
  (defun (->seq s)
    (seql (:for e s)
          (:yield e)))
  (defun (fold f acc coll)
    (loop (:for elem coll)
          (:acc res (folding acc (f elem res)))))
  (defun (map f s)
    (seql (:for e s)
          (:yield (f e))))
  (defun (filter f s)
    (seql (:for e s)
          (:when (f e))
          (:yield e)))
  ;; The count is tested before the walk pulls, so the element after the n-th
  ;; is never asked for: over a port it stays for the next reader.
  (defun (take n s)
    (seql (:with i 0 (+ i 1) (>= i n))
          (:for elem s)
          (:yield elem)))
  ;; this is of course not efficient, but hey. 
  (defun (drop n s)
    (seql (:with i 1 (+ i 1))
          (:for elem s)
          (:when (> i n))
          (:yield elem)))
  (defun (take-while f s)
    (seql (:for elem s)
          (:finish (not (f elem)))
          (:yield elem)))

  ;; The flag is made inside the `seq`, whose body runs once per walk, so that
  ;; walking the result twice drops the same elements both times.
  (defun (drop-while f s)
    (seq
      (def/mutable dropping #t)
      (loop (:for elem s)
            (:do (when (and dropping (not (f elem)))
                   (set! dropping #f)))
            (:when (not dropping))
            (:do (yield elem)))))

  ;; A `seql`, so that a loop over it fuses with a chain of others, as
  ;; `(seq-enumerate (map f xs))` does. `enumerate` makes an adapter instead.
  (defun (seq-enumerate s)
    (seql (:with i 0 (+ i 1))
          (:for e s)
          (:yield (Tuple i e))))

  (defun (flat-map f s)
    (seql (:for e s)
          (:subloop)
          (:for x (f e))
          (:yield x)))

  ;; `folding` keeps the last stepped value, so the accumulator holds the
  ;; current element's answer and `:finish` stops on it.
  (defun (any? f s)
    (loop (:for e s)
          (:acc found (folding #f (f e)))
          (:finish found)))

  (defun (all? f s)
    (loop (:for e s)
          (:acc ok (folding #t (f e)))
          (:finish (not ok))))

  (defun (find f s)
    (loop (:for e s)
          (:acc got (folding None (if (f e) (Some e) None)))
          (:finish (match got ((Some _) #t) (None #f)))))

  (defun (for-each f s)
    (loop (:for e s)
          (:do (ignore (f e)))))

  (defun (->list s)
    (loop (:for e s)
          (:acc xs (listing e))))

  (defun (->vec s)
    (loop (:for e s)
          (:acc xs (vecing e))))

  ;; Walks the whole of `s`. A collection that knows its length answers it.
  (defun (count s)
    (loop (:for e s)
          (:acc n (counting e))))

  ;; Stops on the first element, so the walk pulls only that one.
  (defun (first s)
    (loop (:for e s)
          (:acc got (folding None (Some e)))
          (:finish #t)))
  )

;; Declared before the other implementations because the inherited
;; `flat-map` default walks the `(Seq %b)` its callback returns, and an
;; implementation is only in scope for the ones checked after it.
(impl (Iterable (Seq %a))
  (type %elem %a)
  ;; The walk's cursor is the enumerator itself. The protocol calls `done?`
  ;; once per step, before `current`, so `done?` is where it steps, and
  ;; `next` has nothing left to do.
  (type %cursor (SeqEnumerator %a))
  (defun (iterable-start s) (seq-enumerator s))
  (defun (iterable-done? s e) (not (seq-enumerator-next! e)))
  (defun (iterable-current s e) (seq-enumerator-current e))
  (defun (iterable-next s e) e)
  ;; Disposes the enumerator, which closes a file the sequence opened for
  ;; this walk.
  (defun (iterable-end s e) (seq-enumerator-end! e))
  ;; The same on an exception, so that a file is closed then too.
  (defun (iterable-cleanup s e) (seq-enumerator-end! e))

  (defun (->seq s) s))

(impl (Iterable (List %a))
  (type %elem %a)
  (type %cursor (List %a))
  (defun (iterable-start s) s)
  (defun (iterable-done? s c) (list-empty? c))
  (defun (iterable-current s c) (list-head c))
  (defun (iterable-next s c) (list-tail c))
  (defun (->list s) s))

;; A struct cursor that walks the vec's leaf arrays: an array read per
;; element, and nothing allocated for the walk.
(impl (Iterable (Vec %a))
  (type %elem %a)
  (type %cursor (VecCursor %a))
  (defun (iterable-start s) (vec-cursor s))
  (defun (iterable-done? s c) (vec-cursor-done? c))
  (defun (iterable-current s c) (vec-cursor-current c))
  (defun (iterable-next s c) (vec-cursor-next c))
  (defun (->vec s) s)
  (defun (count s) (vec-length s)))

;; `>=` rather than `=`: the JIT drops the bounds check on `array-ref` only
;; when the loop test is `index < length`.
(impl (Iterable (Array %a))
  (type %elem %a)
  (type %cursor int)
  (defun (iterable-start s) 0)
  (defun (iterable-done? s index) (>= index (array-length s)))
  (defun (iterable-current s index) (array-ref s index))
  (defun (iterable-next s index) (+ index 1))
  (defun (count s) (array-length s)))

;; An `int` cursor, as `Array` has.
(impl (Iterable (IArray %a))
  (type %elem %a)
  (type %cursor int)
  (defun (iterable-start s) 0)
  (defun (iterable-done? s index) (>= index (iarray-length s)))
  (defun (iterable-current s index) (iarray-ref s index))
  (defun (iterable-next s index) (+ index 1))
  (defun (count s) (iarray-length s)))

(: (-> (IArray %a) (Vec %a)))
(defun (iarray->vec xs) (->vec xs))

(: (-> (IArray %a) (List %a)))
(defun (iarray->list xs) (->list xs))

;; Range of ints, in steps of one. A step of one reaches `hi` before it can
;; pass `int-max`, so the walk tests `(>= c hi)`, the test the JIT knows from a
;; loop over an array's indices.
(type (: Range (Struct (: lo int) (: hi int))))

;; Inlined, so that a loop over a range has its two ints and no struct (see
;; `Simplify`, "A record that is only read").
(: (-> int int Range))
(defun (range lo hi) #:inline
  (Range (lo lo) (hi hi)))

(impl (Iterable Range)
  (type %elem int)
  (type %cursor int)
  (defun (iterable-start s) (record-ref s lo))
  (defun (iterable-done? s c) (>= c (record-ref s hi)))
  (defun (iterable-current s c) c)
  (defun (iterable-next s c) (+ c 1)))

;; Range of ints, in steps of `by`. A step past the last value can pass
;; `int-max`, so the walk counts in a long, where it cannot wrap round below
;; `hi`: a value below `hi` and a step of at most `int-max` stay far below
;; `long-max`.
(type (: RangeBy (Struct (: lo int) (: hi int) (: by int))))

;; A range counts up, so a step of zero or less would never reach `hi`. The
;; message is made in `range-by-panic`, so that `range-by` is small enough for
;; the JIT to inline at the start of a loop.
(: (-> int int int RangeBy))
(defun (range-by lo hi by)
  (when (<= by 0)
    (range-by-panic by))
  (RangeBy (lo lo) (hi hi) (by by)))

(: (-> int void))
(defun (range-by-panic by)
  (panic! #"range-by: the step is ${by}, and a range counts up from lo to hi. down-from counts down."))

;; How many values a non-empty range holds, in longs, where `hi - lo` cannot
;; overflow.
(: (-> int int int long))
(defun (range-by-count lo hi by)
  (+ (/ (- (- (cast long hi) (cast long lo)) 1L) (cast long by)) 1L))

(impl (Iterable RangeBy)
  (type %elem int)
  (type %cursor long)
  (defun (iterable-start s) (cast long (record-ref s lo)))
  (defun (iterable-done? s c) (>= c (cast long (record-ref s hi))))
  (defun (iterable-current s c) (cast int c))
  (defun (iterable-next s c) (+ c (cast long (record-ref s by)))))

;; Map elements.
(impl (Iterable (Map %k %v))
  (type %elem (Tuple %k %v))
  (type %cursor (MapCursor %k %v))
  (defun (iterable-start s) (map-cursor s))
  (defun (iterable-done? s c) (map-cursor-done? c))
  (defun (iterable-current s c) (map-cursor-current c))
  (defun (iterable-next s c) (map-cursor-next c))
  (defun (count s) (map-length s)))

;; String characters. Not a `StringCursor`, which asks the string at every
;; step whether it is a slice: a walk takes its bytes once and decodes each
;; character as it steps onto it.
(import/class
  (StringWalk (: Bjolang.Runtime.StringWalk)))

(import/extern
  (string-walk         (: Bjolang.Runtime.StringWalks.Start   (-> string StringWalk)))
  (string-walk-done?   (: Bjolang.Runtime.StringWalks.Done    (-> StringWalk bool)))
  (string-walk-current (: Bjolang.Runtime.StringWalks.Current (-> StringWalk char)))
  (string-walk-next    (: Bjolang.Runtime.StringWalks.Next    (-> StringWalk StringWalk))))

(impl (Iterable string)
  (type %elem char)
  (type %cursor StringWalk)
  (defun (iterable-start s) (string-walk s))
  (defun (iterable-done? s c) (string-walk-done? c))
  (defun (iterable-current s c) (string-walk-current c))
  (defun (iterable-next s c) (string-walk-next c)))

;; An iterable of pairs as a map. Not a method of `Iterable`: its where clause
;; says the element is a pair, and a member may not constrain the
;; implementor. Copied into the call, so that it fuses with a chain under it.
(: (-> %s (Map %k %v)) (where (Eq %k) (Iterable %s #:elem (Tuple %k %v))))
(defun (->map s) #:inline
  (loop (:for e s)
        (:acc m (mapping e))))

;; ---------------------------------------------------------------------------
;; Adapters
;; ---------------------------------------------------------------------------
;;
;; An adapter is a struct around iterables, and an iterable itself: its
;; element and cursor types are made from theirs, and its methods call
;; theirs. A loop over one at a known type inlines it with what it wraps, so
;; it allocates nothing; through a dictionary its class holds the dictionaries
;; of what it wraps. `iterable-end` and `iterable-cleanup` are passed on, so
;; an adapter over a sequence that holds a file closes it as the sequence
;; would, and one over iterables that hold nothing keeps the loop free of a
;; guard. The cursor is taken apart with a `def` rather than `fst` and `snd`
;; where it is passed on: `Simplify` can see that the binding does nothing,
;; and so that a cleanup which does nothing leaves no guard.

;; The elements of `inner`, each with its index from 0.
(type (: (Enumerated %s) (Struct (: inner %s))))

(: (-> %s (Enumerated %s)) (where (Iterable %s)))
(defun (enumerate s) (Enumerated (inner s)))

(impl (Iterable (Enumerated %s)) (where (Iterable %s))
  (type %elem (Tuple int (assoc Iterable elem %s)))
  (type %cursor (Tuple int (assoc Iterable cursor %s)))
  (defun (iterable-start e) (Tuple 0 (iterable-start (record-ref e inner))))
  (defun (iterable-done? e c) (iterable-done? (record-ref e inner) (snd c)))
  (defun (iterable-current e c) (Tuple (fst c) (iterable-current (record-ref e inner) (snd c))))
  (defun (iterable-next e c) (Tuple (+ (fst c) 1) (iterable-next (record-ref e inner) (snd c))))
  (defun (iterable-end e c)
    (def (Tuple _ k) c)
    (iterable-end (record-ref e inner) k))
  (defun (iterable-cleanup e c)
    (def (Tuple _ k) c)
    (iterable-cleanup (record-ref e inner) k)))

;; The elements of `left` and `right` in pairs, until either ends.
(type (: (Zipped %l %r) (Struct (: left %l) (: right %r))))

(: (-> %l %r (Zipped %l %r)) (where (Iterable %l) (Iterable %r)))
(defun (zip l r) (Zipped (left l) (right r)))

;; `done?` asks the left side first. When one side ends, the other is ended
;; there and then, so that a sequence holding a file is closed although its
;; walk did not finish; the loop ends no cursor whose `done?` answered true.
;; When the right side is the shorter, the left has read one element more,
;; which nothing uses.
(impl (Iterable (Zipped %l %r)) (where (Iterable %l) (Iterable %r))
  (type %elem (Tuple (assoc Iterable elem %l) (assoc Iterable elem %r)))
  (type %cursor (Tuple (assoc Iterable cursor %l) (assoc Iterable cursor %r)))
  (defun (iterable-start z)
    (Tuple (iterable-start (record-ref z left)) (iterable-start (record-ref z right))))
  ;; Written with `or` and `and` rather than `if`s: where both ends do nothing,
  ;; `Simplify` drops them, and the test is one expression, as two lockstep
  ;; `:for`s would make it.
  (defun (iterable-done? z c)
    (def (Tuple kl kr) c)
    (or (and (iterable-done? (record-ref z left) kl)
             (begin (iterable-end (record-ref z right) kr) #t))
        (and (iterable-done? (record-ref z right) kr)
             (begin (iterable-end (record-ref z left) kl) #t))))
  (defun (iterable-current z c)
    (Tuple (iterable-current (record-ref z left) (fst c))
           (iterable-current (record-ref z right) (snd c))))
  (defun (iterable-next z c)
    (Tuple (iterable-next (record-ref z left) (fst c))
           (iterable-next (record-ref z right) (snd c))))
  (defun (iterable-end z c)
    (def (Tuple kl kr) c)
    (iterable-end (record-ref z left) kl)
    (iterable-end (record-ref z right) kr))
  (defun (iterable-cleanup z c)
    (def (Tuple kl kr) c)
    (iterable-cleanup (record-ref z left) kl)
    (iterable-cleanup (record-ref z right) kr)))

;; Each element of `inner` with the one after it: (Tuple a b), one fewer
;; than `inner` has, in one walk of it.
(type (: (Pairwise %s) (Struct (: inner %s))))

(: (-> %s (Pairwise %s)) (where (Iterable %s)))
(defun (pairwise s) (Pairwise (inner s)))

;; The cursor is the element before and the inner cursor at the one after.
;; `start` takes the first element, so it asks the inner `done?` of the first
;; cursor itself; `None` before says the inner iterable was empty, and no
;; `done?` is asked again of that cursor. `current` and `next` both read the
;; element at the inner cursor.
(impl (Iterable (Pairwise %s)) (where (Iterable %s))
  (type %elem (Tuple (assoc Iterable elem %s) (assoc Iterable elem %s)))
  (type %cursor (Tuple (Option (assoc Iterable elem %s)) (assoc Iterable cursor %s)))
  ;; One tuple, not one per branch, and `done?` with `or` rather than a
  ;; `match`: over an array, the loop is then faster than one that indexes
  ;; the array twice.
  (defun (iterable-start w)
    (def k (iterable-start (record-ref w inner)))
    (def empty (iterable-done? (record-ref w inner) k))
    (Tuple (if empty None (Some (iterable-current (record-ref w inner) k)))
           (if empty k (iterable-next (record-ref w inner) k))))
  (defun (iterable-done? w c)
    (def (Tuple before k) c)
    (or (none? before) (iterable-done? (record-ref w inner) k)))
  (defun (iterable-current w c)
    (def (Tuple before k) c)
    (Tuple (option-ref before) (iterable-current (record-ref w inner) k)))
  (defun (iterable-next w c)
    (def (Tuple _ k) c)
    (Tuple (Some (iterable-current (record-ref w inner) k))
           (iterable-next (record-ref w inner) k)))
  (defun (iterable-end w c)
    (def (Tuple _ k) c)
    (iterable-end (record-ref w inner) k))
  (defun (iterable-cleanup w c)
    (def (Tuple _ k) c)
    (iterable-cleanup (record-ref w inner) k)))

;; The elements of `first` and then those of `second`: two iterables with one
;; element type, of any two kinds.
(type (: (Chained %a %b) (Struct (: first %a) (: second %b))))

(: (-> %a %b (Chained %a %b)) (where (Iterable %a #:elem %e) (Iterable %b #:elem %e)))
(defun (chain a b) (Chained (first a) (second b)))

;; The cursor is `(Tuple on-second? cur-a cur-b)`.
;;
;; Because `iterable-done?` cannot mutate the cursor, transitions from `first` to
;; `second` are determined ahead of time:
;; - `iterable-start` and `iterable-next` eagerly check if `first` has finished,
;;   setting `on-second?` to `#t` when moving to `second`.
;; - `iterable-done?` only checks completion of the active side (via `cur-b` once
;;   `on-second?` is true).
;; - `iterable-cleanup` safely cleans up both iterables when the walk terminates.
(impl (Iterable (Chained %a %b))
  (where (Iterable %a) (Iterable %b #:elem (assoc Iterable elem %a)))
  (type %elem (assoc Iterable elem %a))
  (type %cursor (Tuple bool (assoc Iterable cursor %a) (assoc Iterable cursor %b)))
  (defun (iterable-start h)
    (def ka (iterable-start (record-ref h first)))
    (Tuple (iterable-done? (record-ref h first) ka) ka (iterable-start (record-ref h second))))
  (defun (iterable-done? h c)
    (def (Tuple on-second _ kb) c)
    (and on-second (iterable-done? (record-ref h second) kb)))
  (defun (iterable-current h c)
    (def (Tuple on-second ka kb) c)
    (if on-second
        (iterable-current (record-ref h second) kb)
        (iterable-current (record-ref h first) ka)))
  (defun (iterable-next h c)
    (def (Tuple on-second ka kb) c)
    (if on-second
        (Tuple #t ka (iterable-next (record-ref h second) kb))
        (let ((next-a (iterable-next (record-ref h first) ka)))
          (Tuple (iterable-done? (record-ref h first) next-a) next-a kb))))
  ;; Both sides, whichever the walk is on: an end of a cursor whose `done?`
  ;; answered true does nothing, as the protocol says, and with no test
  ;; here, ends that do nothing leave the loop no guard.
  (defun (iterable-end h c)
    (def (Tuple _ ka kb) c)
    (iterable-end (record-ref h first) ka)
    (iterable-end (record-ref h second) kb))
  (defun (iterable-cleanup h c)
    (def (Tuple _ ka kb) c)
    (iterable-cleanup (record-ref h first) ka)
    (iterable-cleanup (record-ref h second) kb)))

;; `item`, `times` times, or without end when `times` is `repeat-endless`,
;; which is what `in-repeat` gives when no count is written. It is
;; `int-min`, which no count written as a count is.
(type (: (Repeated %a) (Struct (: item %a) (: times int))))

(: int)
(def repeat-endless -2147483648)

(: (-> %a (#:times int) (Repeated %a)))
(defun (in-repeat x #:times repeat-endless) (Repeated (item x) (times times)))

(impl (Iterable (Repeated %a))
  (type %elem %a)
  (type %cursor int)
  (defun (iterable-start r) 0)
  (defun (iterable-done? r i)
    (and (not (= (record-ref r times) repeat-endless)) (>= i (record-ref r times))))
  (defun (iterable-current r i) (record-ref r item))
  (defun (iterable-next r i) (+ i 1)))

;; The elements of `inner`, over and over: when a walk of it ends, the next
;; one starts. It ends when a walk yields nothing, so the cycle of an empty
;; iterable is empty, and so is the next pass over a port that is used up.
(type (: (Cycled %s) (Struct (: inner %s))))

(: (-> %s (Cycled %s)) (where (Iterable %s)))
(defun (in-cycle s) (Cycled (inner s)))

;; `done?` of the inner iterable is asked once for each of its cursors, as
;; the protocol wants: a cursor over a sequence pulls when asked. So `next`
;; asks it of the cursor it makes, and the flag in the cursor says that it
;; answered false; when it answered true, `next` starts the next walk, and
;; `done?` asks of that cursor in turn. `next` therefore pulls the element
;; after the current one before a loop asks for it, as `zip` does on its
;; left side.
(impl (Iterable (Cycled %s)) (where (Iterable %s))
  (type %elem (assoc Iterable elem %s))
  (type %cursor (Tuple bool (assoc Iterable cursor %s)))
  (defun (iterable-start y) (Tuple #f (iterable-start (record-ref y inner))))
  (defun (iterable-done? y c)
    (def (Tuple asked k) c)
    (and (not asked) (iterable-done? (record-ref y inner) k)))
  (defun (iterable-current y c) (iterable-current (record-ref y inner) (snd c)))
  (defun (iterable-next y c)
    (def (Tuple _ k) c)
    (def n (iterable-next (record-ref y inner) k))
    (if (iterable-done? (record-ref y inner) n)
        (Tuple #f (iterable-start (record-ref y inner)))
        (Tuple #t n)))
  (defun (iterable-end y c)
    (def (Tuple _ k) c)
    (iterable-end (record-ref y inner) k))
  (defun (iterable-cleanup y c)
    (def (Tuple _ k) c)
    (iterable-cleanup (record-ref y inner) k)))


;; ---------------------------------------------------------------------------
;; Cursor
;; ---------------------------------------------------------------------------

;; Cursor.
(def/trait (Cursor %s)
  (type %elem)
  (type %cursor)
  (: cursor-start (-> %s %cursor))
  (: cursor-end (-> %s %cursor))
  (: cursor-end? (-> %s %cursor bool))
  (: cursor-ref (-> %s %cursor %elem))
  (: cursor-next (-> %s %cursor %cursor))
  (: cursor-prev (-> %s %cursor %cursor)))

(impl (Cursor string)
  (type %elem char)
  (type %cursor StringCursor)
  (defun (cursor-start s) (string-cursor-start s))
  (defun (cursor-end s) (string-cursor-end s))
  (defun (cursor-end? s c) (string-cursor-end? s c))
  (defun (cursor-ref s c) (string-cursor-ref s c))
  (defun (cursor-next s c) (string-cursor-next s c))
  (defun (cursor-prev s c) (string-cursor-prev s c)))


;; ---------------------------------------------------------------------------
;; Lists, where the callback may suspend
;; ---------------------------------------------------------------------------
;; The walks below are written #:inline where their body is past the size the
;; compiler copies by itself: copied, the loop is in the caller with the
;; callback's body in it, and no delegate is made or called.
;;
;; List map.
(: (-> (-?-> %a %b) (List %a) (List %b)))
(defun (list-map f xs)
  (loop (:for elem xs)
        (:acc lst (listing (f elem)))))

(: (-> (-?-> %a bool) (List %a) (List %a)))
(defun (list-filter keep? xs) #:inline
  (loop (:for elem xs)
        (:when (keep? elem))
        (:acc acc (listing elem))))

;; List foldl.
(: (-> (-?-> %a %b %b) %b (List %a) %b))
(defun (list-foldl f init xs) #:inline
  (loop (:for elem xs)
        (:acc acc (folding init (f elem acc)))))

;; List foldr.
(: (-> (-?-> %a %b %b) %b (List %a) %b))
(defun (list-foldr f init xs) #:inline
  (loop (:for elem (list-reverse xs))
        (:acc acc (folding init (f elem acc)))))

(: (-> (-?-> %a Unit) (List %a) Unit))
(defun (list-for-each f xs)
  (loop (:for elem xs)
        (:do (f elem))))

;; The list seq-unfold would make, without the lazy seq. The update is
;; evaluated after the iteration, so it reads the state the :finish-let bound.
(: (-> (-?-> %s (Option (Tuple %a %s))) %s (List %a)))
(defun (list-unfold f state)
  (loop (:with step (f state) (f next))
        (:finish-let (Some (Tuple x next)) step)
        (:acc out (listing x))))

;; ---------------------------------------------------------------------------
;; Vecs, where the callback may suspend
;; ---------------------------------------------------------------------------
;; Vec functions.

(import/extern
  (clr-vec-map            (: BjolangRuntime.RrbMap          (-> (-> %a %b) (Vec %a) (Vec %b))))
  ;; A vec of the same shape as the first, holding the array's items in
  ;; order: what `clr-vec-map` makes, for items made by a function that may
  ;; suspend, which no extern can call.
  (clr-vec-map-shape      (: Bjolang.Runtime.Rrb.RrbFun.MapShape    (-> (Vec %a) (Array %b) (Vec %b))))
  (clr-vec-filter         (: BjolangRuntime.RrbFilter       (-> (-> %a bool) (Vec %a) (Vec %a))))
  (clr-vec-fold           (: BjolangRuntime.RrbFold         (-> (-> %a %b %b) %b (Vec %a) %b)))
  (clr-vec-reduce         (: BjolangRuntime.RrbReduce       (-> (-> %a %a %a) (Vec %a) %a)))
  (clr-vec-for-each       (: BjolangRuntime.RrbForEach      (-> (-> %a Unit) (Vec %a) int int Unit)))
  (clr-vec-iter           (: BjolangRuntime.RrbIter         (-> (-> %a bool) (Vec %a) bool)))
  ;; The checks the ordinary copies make, for the suspending copies to make
  ;; too, so that both raise the same thing.
  (clr-vec-check-reduce   (: BjolangRuntime.RrbCheckReduce  (-> (Vec %a) Unit)))
  (clr-vec-check-range    (: BjolangRuntime.RrbCheckRange   (-> (Vec %a) int int Unit)))
  ;; Append one element at the end — the persistent counterpart of
  ;; `vecbuilder-add!`, and what `Addable`'s impl for Vec delegates to.
  (vec-add                (: Bjolang.Runtime.Rrb.RrbFun.Add         (-> (Vec %a) %a (Vec %a))))
  ;; Add one element at the front — the persistent counterpart of
  ;; `transientvec-add-first!`, as `vec-pop-first` is of `transientvec-pop-first!`.
  (vec-add-first          (: Bjolang.Runtime.Rrb.RrbFun.Prepend     (-> (Vec %a) %a (Vec %a)))))

;; Vec map.
(: (-> (-?-> %a %b) (Vec %a) (Vec %b)))
;;
;; Not inlined: `clr-vec-map` copies the vec's arrays at their exact size, and
;; for the short vecs most programs map that is faster than a loop into a
;; builder, even with the call through the delegate.
;;
;; The suspending copy keeps the vec's shape too: the items go into an array
;; in order, and `clr-vec-map-shape` lays them out as `v` is laid out. So the
;; halves of a split, mapped in bjoroutines, merge back into a dense vec.
(defbjouble (vec-map f v)
  #:no-inline
  (#:sync (clr-vec-map f v))
  (#:bjo
    (def out (make-array (vec-length v)))
    (loop (:for elem v)
          (:with i 0 (+ i 1))
          (:do (array-set! out i (f elem))))
    (clr-vec-map-shape v out)))

(: (-> (-?-> %a bool) (Vec %a) (Vec %a)))
(defbjouble (vec-filter keep? v)
  #:inline
  (#:sync (clr-vec-filter keep? v))
  (#:bjo
    (loop (:for elem v)
          (:when (keep? elem))
          (:acc acc (vecing/small elem)))))

(: (-> (-?-> %a %b %b) %b (Vec %a) %b))
(defbjouble (vec-fold f init v)
  #:inline
  (#:sync (clr-vec-fold f init v))
  (#:bjo
    (loop (:for elem v)
          (:acc acc (folding init (f elem acc))))))

;; Vec reduce.
(: (-> (-?-> %a %a %a) (Vec %a) %a))
(defbjouble (vec-reduce f v)
  (#:sync (clr-vec-reduce f v))
  (#:bjo
    (ignore (clr-vec-check-reduce v))
    (let go ((c (vec-cursor-slice v 1 (- (vec-length v) 1))) (acc (vec-ref v 0)))
      (if (vec-cursor-done? c)
          acc
          (go (vec-cursor-next c) (f (vec-cursor-current c) acc))))))

;; The elements from `#:from` up to `#:to`, front to back. Raises
;; ArgumentOutOfRangeException, before `f` is first called, unless
;; 0 <= from <= to <= length.
(: (-> (-?-> %a Unit) (Vec %a) (#:from int) (#:to int) Unit))
(defbjouble (vec-for-each f v #:from 0 #:to (vec-length v))
  #:inline
  (#:sync (clr-vec-for-each f v from to))
  (#:bjo
    (ignore (clr-vec-check-range v from to))
    (let go ((c (vec-cursor-slice v from (- to from))))
      (if (vec-cursor-done? c)
          unit
          (let ()
            (ignore (f (vec-cursor-current c)))
            (go (vec-cursor-next c)))))))

;; Vec iter.
(: (-> (-?-> %a bool) (Vec %a) bool))
(defbjouble (vec-iter f v)
  #:inline
  (#:sync (clr-vec-iter f v))
  (#:bjo
    (loop (:for elem v)
          (:acc acc (folding #t (f elem)))
          (:finish (not acc)))))

;; ---------------------------------------------------------------------------
;; Seqs, where the callback may suspend
;; ---------------------------------------------------------------------------
;;
;; The eager two. `seq-unfold` is lazy and stays a builtin: its callback runs
;; when the walker pulls, which is somewhere else.

(: (-> (-?-> %a %b %b) %b (Seq %a) %b))
(defun (seq-fold f init s)
  (let ((e (seq-enumerator s)))
    (let go ((acc init))
      (if (seq-enumerator-next! e)
          (go (f (seq-enumerator-current e) acc))
          acc))))

(: (-> (-?-> %a Unit) (Seq %a) Unit))
(defun (seq-for-each f s)
  (let ((e (seq-enumerator s)))
    (let go ()
      (if (seq-enumerator-next! e)
          (let ()
            (ignore (f (seq-enumerator-current e)))
            (go))
          unit))))


;; ---------------------------------------------------------------------------
;; Conversions
;; ---------------------------------------------------------------------------
;; Conversions.

(: (-> (List (Tuple %k %v)) (Map %k %v)))
(defun (list->map xs)
  (loop (:for kvp xs)
        (:acc map (mapping kvp))))

(: (-> (List %a) (Vec %a)))
(defun (list->vec lst)
  {vecing e (:for e lst)})

(: (-> (Map %k %v) (List (Tuple %k %v))))
(defun (map->list m)
  {listing kvp (:for kvp m)})

(: (-> (Vec (Tuple %k %v)) (Map %k %v)))
(defun (vec->map v)
  (loop (:for kvp v)
        (:acc map (mapping kvp))))

;; Faster than a loop over the indices: `vec-for-each` walks the vec itself,
;; and the builder adds in place.
(: (-> (Vec %a) (List %a)))
(defun (vec->list v)
  (def b (listbuilder-empty))
  (vec-for-each (fun (elem) (listbuilder-add! b elem)) v )
  (listbuilder->list b))

(: (-> (Map %k %v) (Vec (Tuple %k %v))))
(defun (map->vec m)
  {vecing kvp (:for kvp m)})

;; vec-insert, vec-remove-at, vec-slice and vec-split, over the runtime's RRB
;; nodes.
(include "vec.protobjo")


;; --------------------------------------------------------------------------
;; Results and options
;; --------------------------------------------------------------------------

;; `some?` and `none?`, for the other two-case builtin.

(: (-> (Result %e %a) bool))
(defun (ok? r)
  (match r
    ((Ok v) #t)
    ((Err e) #f)))

(: (-> (Result %e %a) bool))
(defun (err? r)
  (match r
    ((Ok v) #f)
    ((Err e) #t)))

;; The `Ok` value through `f`, which may suspend; an `Err` returns itself.
(: (-> (-?-> %a %b) (Result %e %a) (Result %e %b)))
(defun (result-map f r)
  (match r
    ((Ok v) (Ok (f v)))
    ((Err e) (Err e))))

;; The `Err` value through `f`, which may suspend; an `Ok` returns itself.
(: (-> (-?-> %e %f) (Result %e %a) (Result %f %a)))
(defun (result-map-err f r)
  (match r
    ((Ok v) (Ok v))
    ((Err e) (Err (f e)))))

(import/class
  (ResultRefError (: System.InvalidOperationException (-> string ResultRefError))))

;; The `Ok` value. An `Err` raises InvalidOperationException, as
;; `option-ref` does for `None`.
(: (-> (Result %e %a) %a))
(defun (result-ref r)
  (match r
    ((Ok v) v)
    ((Err _) (raise (cast Exception (ResultRefError. "result-ref on Err"))))))

;; The `Ok` value, or `fallback` for an `Err`. `fallback` is evaluated
;; first, as for any call.
(: (-> (Result %e %a) %a %a))
(defun (result-ref-or r fallback)
  (match r
    ((Ok v) v)
    ((Err _) fallback)))

;; `(Some v)` for `(Ok v)`, and `None` for an `Err`, whose value is dropped.
(: (-> (Result %e %a) (Option %a)))
(defun (result->option r)
  (match r
    ((Ok v) (Some v))
    ((Err _) None)))

;; The `Some` value through `f`, which may suspend; `None` returns itself.
(: (-> (-?-> %a %b) (Option %a) (Option %b)))
(defun (option-map f opt)
  (match opt
    ((Some v) (Some (f v)))
    (None None)))

;; `opt` if `pred` holds for its value, else `None`. `pred` may suspend.
(: (-> (-?-> %a bool) (Option %a) (Option %a)))
(defun (option-filter pred opt)
  (match opt
    ((Some v) (if (pred v) opt None))
    (None None)))

;; `opt` if it is a `Some`, else `other`. Both are evaluated, as for any
;; call.
(: (-> (Option %a) (Option %a) (Option %a)))
(defun (option-or opt other)
  (match opt
    ((Some _) opt)
    (None other)))

;; `(Ok v)` for `(Some v)`, and `(Err e)` for `None`.
(: (-> (Option %a) %e (Result %e %a)))
(defun (option->result opt e)
  (match opt
    ((Some v) (Ok v))
    (None (Err e))))

;; --- Pairs ---

(: (-> (Tuple %a %b) %a))
(defun (fst p)
  (match p ((Tuple a _) a)))

(: (-> (Tuple %a %b) %b))
(defun (snd p)
  (match p ((Tuple _ b) b)))


;; --------------------------------------------------------------------------
;; Boxes - a 1-width array
;; --------------------------------------------------------------------------

(type (: (Box %a) #:opaque (Struct (: field (Array %a)))))
(impl (Written (Box %a))
  (where (Written %a))
  (defun (write-into box b)
    (stringbuilder-add-string! b "(Box ")
    (write-into (box-ref box) b)
    (stringbuilder-add-string! b ")")))

(impl (Displayed (Box %a))
  (where (Displayed %a))
  (defun (display-into box b)
    (stringbuilder-add-string! b "(Box ")
    (display-into (box-ref box) b)
    (stringbuilder-add-string! b ")")))
(: (-> %a (Box %a)))
(defun (make-box elem)
  (def arr (make-array 1))
  (array-set! arr 0 elem)
  (Box (field arr)))

(: (-> (Box %a) %a))
(defun (box-ref b)
  (-> (struct-ref b field)
      (array-ref & 0)))

(: (-> (Box %a) %a Unit))
(defun (box-set! box val)
  (def arr (struct-ref box field))
  (array-set! arr 0 val))



;; --------------------------------------------------------------------------
;; Refable and Derefable trait - a "generic" ref function
;; --------------------------------------------------------------------------
(def/trait (Refable %c)
  (type %index)
  (type %out)
  (: ref (-> %c %index %out))
  (: try-ref (-> %c %index (Option %out)))

  ;; The default: `ref`'s answer as a `Some`, so an impl whose `ref` raises out
  ;; of bounds needs no `try-ref` of its own.
  (defun (try-ref coll i)
    (Some (ref coll i))))

(impl (Refable (Array %i))
  (type %index int)
  (type %out %i)
  (defun (ref arr i)
    (array-ref arr i)))

(impl (Refable (IArray %i))
  (type %index int)
  (type %out %i)
  (defun (ref xs i)
    (iarray-ref xs i))
  (defun (try-ref xs i)
    (iarray-try-ref xs i)))

(impl (Refable (Vec %i))
  (type %index int)
  (type %out %i)
  (defun (ref vec i)
    (vec-ref vec i))
  (defun (try-ref vec i)
    (if (and (>= i 0) (< i (vec-length vec)))
        (Some (vec-ref vec i))
        None)))

(impl (Refable (Map %k %v))
  (type %index %k)
  (type %out %v)
  (defun (ref map k)
    (map-ref map k))
  (defun (try-ref map k)
    (map-try-ref map k)))

(impl (Refable string)
  (type %index StringCursor)
  (type %out char)
  (defun (ref str cur)
    (string-cursor-ref str cur))
  (defun (try-ref str cur)
    (if (string-cursor-end? str cur)
        None
        (Some (string-cursor-ref str cur)))))

;; The collections written in place, read by the same `ref`, so that `get`
;; and `try-get` read them and a `set!` path can go through them.
(impl (Refable (VecBuilder %i))
  (type %index int)
  (type %out %i)
  (defun (ref vb i)
    (vecbuilder-ref vb i))
  (defun (try-ref vb i)
    (if (and (>= i 0) (< i (vecbuilder-length vb)))
        (Some (vecbuilder-ref vb i))
        None)))

(impl (Refable (TransientVec %i))
  (type %index int)
  (type %out %i)
  (defun (ref t i)
    (transientvec-ref t i))
  (defun (try-ref t i)
    (if (and (>= i 0) (< i (transientvec-length t)))
        (Some (transientvec-ref t i))
        None)))

(impl (Refable (TransientMap %k %v))
  (type %index %k)
  (type %out %v)
  (defun (ref t k)
    (transientmap-ref t k))
  (defun (try-ref t k)
    (transientmap-try-ref t k)))

;; Settable — the persistent write that goes with `ref`
;;
;; `(ref-set coll i x)` answers a copy of `coll` with `x` at `i`, and leaves
;; `coll` alone. It is a trait of its own rather than a method of `Refable`
;; because many things that can be read by index cannot sensibly be copied
;; with one slot changed: a string read by cursor, a set read by membership,
;; or a mutable collection, whose write is `!` and in place.
;;
;; `set` and `update` write through it. A Map gains a key it did not have; a
;; Vec index out of range throws.
(def/trait (Settable %c)
  (type %index)
  (type %in)
  (: ref-set (-> %c %index %in %c)))

(impl (Settable (Vec %i))
  (type %index int)
  (type %in %i)
  (defun (ref-set vec i x)
    (vec-set vec i x)))

(impl (Settable (Map %k %v))
  (type %index %k)
  (type %in %v)
  (defun (ref-set map k v)
    (map-set map k v)))

;; Settable! — the write in place that goes with `ref`
;;
;; `(ref-set! coll i x)` writes `x` at `i` in `coll` itself, as `Addable!`
;; adds to a collection in place where `Addable` answers a copy. `set!` with a
;; path writes its last key through it. A persistent collection has no impl,
;; because it has no place to write: `ref-set` answers its copy.
;;
;; A Map, transient or mutable, gains a key it did not have; an Array or a Vec
;; index out of range throws.
(def/trait (Settable! %c)
  (type %index)
  (type %in)
  (: ref-set! (-> %c %index %in void)))

(impl (Settable! (Array %i))
  (type %index int)
  (type %in %i)
  (defun (ref-set! arr i x)
    (array-set! arr i x)))

(impl (Settable! (VecBuilder %i))
  (type %index int)
  (type %in %i)
  (defun (ref-set! vb i x)
    (vecbuilder-set! vb i x)))

(impl (Settable! (TransientVec %i))
  (type %index int)
  (type %in %i)
  (defun (ref-set! t i x)
    (transientvec-set! t i x)))

(impl (Settable! (TransientMap %k %v))
  (type %index %k)
  (type %in %v)
  (defun (ref-set! t k v)
    (transientmap-set! t k v)))

(def/trait (Derefable %a)
  (type %out)
  (: deref (-> %a %out))
  (: try-deref (-> %a (Option %out)))
  (defun (try-deref p) (Some (deref p))))

(impl (Derefable (Param %b))
  (type %out %b)
  (defun (deref p) (parameter-ref p)))

(impl (Derefable (Box %b))
  (type %out %b)
  (defun (deref b) (box-ref b)))

;; Derefable! — the write in place that goes with `deref`
;;
;; `(deref-set! cell x)` puts `x` in `cell`, as `Settable!` goes with
;; `Settable`. `(set! name value)` writes through it when `name` is not a
;; `def/mutable`: the binding cannot change, so what it holds does. A Param has
;; no impl, since a Param is rebound for an extent with `parameterize` and
;; not written.
(def/trait (Derefable! %a)
  (type %in)
  (: deref-set! (-> %a %in void)))

(impl (Derefable! (Box %b))
  (type %in %b)
  (defun (deref-set! b x) (box-set! b x)))


;; ---------------------------------------------------------------------------
;; Maths
;; ---------------------------------------------------------------------------

;; Math.

(import (std maths))

;; Remember: exporting one trait function exposes ALL functions.
(re-export abs sign min max clamp zero?)

;; Square root. The root of a number in m^2 is in m.
(import/extern
  (clr-sqrt (: System.Math.Sqrt (-> double double))))

(: sqrt (-> (double %u^2) (double %u)))
(defun (sqrt x) (cast (double %u) (clr-sqrt (cast double x))))

;; ---------------------------------------------------------------------------
;; Strings, conversions, files and paths
;; ---------------------------------------------------------------------------
;; Wrappers.

;; --- Strings ---

;; Invariant throughout. The ambient culture reads and writes "1.5" as "1,5"
;; where the decimal separator is a comma, and writes -100 as "−100" with
;; U+2212 where sv-SE does, so a number written by one machine is unreadable on
;; the next.
(import/extern
  (clr-concat        (: BjoString.Utf8String.Concat))
  (clr-concat-all    (: BjolangRuntime.StringConcat))
  (clr-parse-int     (: BjoString.Utf8Number.ParseInt (-> string int)))
  (clr-parse-long    (: Bjolang.Runtime.BjoNum.ParseLong))
  (clr-parse-double  (: BjoString.Utf8Number.ParseDouble (-> string double)))
  (clr-double-string (: BjoString.Utf8Number.Format (-> double string)))
  (clr-int-string    (: BjoString.Utf8Number.Format (-> int string)))
  (clr-long-string   (: BjoString.Utf8Number.Format (-> long string)))
  (clr-byte-string   (: BjoString.Utf8Number.Format (-> byte string)))
  (clr-parse-decimal (: Bjolang.Runtime.BjoNum.ParseDecimal))
  (clr-parse-bigint  (: Bjolang.Runtime.BjoNum.ParseInteger (-> string bigint)))
  (clr-parse-int128  (: Bjolang.Runtime.BjoNum.ParseInteger (-> string int128)))
  (clr-parse-uint128 (: Bjolang.Runtime.BjoNum.ParseInteger (-> string uint128)))
  (clr-format-number (: Bjolang.Runtime.BjoNum.FormatNumber (-> %a string))))

(: (-> string string string))
(defun (string-append a b) (clr-concat a b))

;; Variadic concatenation.
(: (-> #:rest string string))
(defun (str #:rest parts) (clr-concat-all parts))

;; String length in storage units: bytes of UTF-8.
(: (-> string int))
(defun (string-length s) (.-ByteLength s))

;; --- Searching and rewriting -----------------------------------------------
;; Ordinal: bytes of UTF-8 compared as they are.

(import/extern
  (clr-contains    (: BjolangRuntime.StringContains))
  (clr-starts-with (: BjolangRuntime.StringStartsWith))
  (clr-ends-with   (: BjolangRuntime.StringEndsWith))
  (clr-replace     (: BjolangRuntime.StringReplace)))

(: (-> string string bool))
(defun (string-contains? s needle) (clr-contains s needle))

(: (-> string string bool))
(defun (string-starts-with? s prefix) (clr-starts-with s prefix))

(: (-> string string bool))
(defun (string-ends-with? s suffix) (clr-ends-with s suffix))

;; String replace.
(: (-> string string string string))
(defun (string-replace s old new) (clr-replace s old new))

;; --- Case ------------------------------------------------------------------
;; Invariant case.
(: (-> string string))
(defun (string-upcase s) (.ToUpperInvariant s))

(: (-> string string))
(defun (string-downcase s) (.ToLowerInvariant s))

;; --- Trimming and padding --------------------------------------------------
;; Trim whitespace.
(: (-> string string))
(defun (string-trim s) (.Trim s))

(: (-> string string))
(defun (string-trim-left s) (.TrimStart s))

(: (-> string string))
(defun (string-trim-right s) (.TrimEnd s))

;; The /slice forms share the string's bytes rather than copying them, so
;; what they answer keeps the whole string alive. `string-copy` gives such a
;; string bytes of its own, and is the string itself when it has them.
(: (-> string string))
(defun (string-trim/slice s) (.TrimSlice s))

(: (-> string string))
(defun (string-trim-left/slice s) (.TrimStartSlice s))

(: (-> string string))
(defun (string-trim-right/slice s) (.TrimEndSlice s))

(: (-> string string))
(defun (string-copy s) (.Copy s))

;; --- Taking and dropping characters ----------------------------------------
;; A count of characters from one end of a string. The cursors walk the count,
;; so the cost is in the count and not in the length of the string. A count
;; past the end takes or drops the whole string; a negative count is refused.
;; The /slice forms share the string's bytes, as the trims' do.

(: (-> string int void))
(defun (check-count who n)
  (when (< n 0)
    (count-panic who n)))

(: (-> string int void))
(defun (count-panic who n)
  (panic! #"${who}: the count is ${n}, and a count of characters or elements is 0 or more."))

;; The cursor n characters after c, or the end when there are fewer.
(: (-> string StringCursor int StringCursor))
(defun (forward-at-most s c n)
  (let go ((c c) (k n))
    (if (or (= k 0) (string-cursor-end? s c))
        c
        (go (string-cursor-next s c) (- k 1)))))

;; The cursor n characters before c, or the start when there are fewer.
(: (-> string StringCursor int StringCursor))
(defun (back-at-most s c n)
  (def start (string-cursor-start s))
  (let go ((c c) (k n))
    (if (or (= k 0) (string-cursor=? c start))
        c
        (go (string-cursor-prev s c) (- k 1)))))

(: (-> string StringCursor int StringCursor))
(defun (string-cursor-forward s c n)
  (check-count "string-cursor-forward" n)
  (let go ((c c) (k n))
    (if (= k 0) c (go (string-cursor-next s c) (- k 1)))))

(: (-> string StringCursor int StringCursor))
(defun (string-cursor-back s c n)
  (check-count "string-cursor-back" n)
  (let go ((c c) (k n))
    (if (= k 0) c (go (string-cursor-prev s c) (- k 1)))))

(: (-> string int string))
(defun (string-take s n)
  (check-count "string-take" n)
  (substring s (string-cursor-start s) (forward-at-most s (string-cursor-start s) n)))

(: (-> string int string))
(defun (string-drop s n)
  (check-count "string-drop" n)
  (substring s (forward-at-most s (string-cursor-start s) n) (string-cursor-end s)))

(: (-> string int string))
(defun (string-take-right s n)
  (check-count "string-take-right" n)
  (substring s (back-at-most s (string-cursor-end s) n) (string-cursor-end s)))

(: (-> string int string))
(defun (string-drop-right s n)
  (check-count "string-drop-right" n)
  (substring s (string-cursor-start s) (back-at-most s (string-cursor-end s) n)))

(: (-> string int string))
(defun (string-take/slice s n)
  (check-count "string-take/slice" n)
  (substring/slice s (string-cursor-start s) (forward-at-most s (string-cursor-start s) n)))

(: (-> string int string))
(defun (string-drop/slice s n)
  (check-count "string-drop/slice" n)
  (substring/slice s (forward-at-most s (string-cursor-start s) n) (string-cursor-end s)))

(: (-> string int string))
(defun (string-take-right/slice s n)
  (check-count "string-take-right/slice" n)
  (substring/slice s (back-at-most s (string-cursor-end s) n) (string-cursor-end s)))

(: (-> string int string))
(defun (string-drop-right/slice s n)
  (check-count "string-drop-right/slice" n)
  (substring/slice s (string-cursor-start s) (back-at-most s (string-cursor-end s) n)))

;; The text before the first sep and the text after it, or None when sep is
;; not in s.
(: (-> string string (Option (Tuple string string))))
(defun (string-split-once s sep)
  (option-map (fun (c)
                (Tuple (substring s (string-cursor-start s) c)
                       (substring s (string-cursor-forward s c (string-count sep)) (string-cursor-end s))))
              (string-search s sep)))

(: (-> string string (Option (Tuple string string))))
(defun (string-split-once/slice s sep)
  (option-map (fun (c)
                (Tuple (substring/slice s (string-cursor-start s) c)
                       (substring/slice s (string-cursor-forward s c (string-count sep)) (string-cursor-end s))))
              (string-search s sep)))

;; Pad width.
(: (-> string int (#:with char) string))
(defun (string-pad-left s width #:with #\space)
  (let ((b (stringbuilder-empty)))
    (string-pad-fill! b with (- width (string-count s)))
    (stringbuilder-add-string! b s)
    (stringbuilder->string b)))

(: (-> string int (#:with char) string))
(defun (string-pad-right s width #:with #\space)
  (let ((b (stringbuilder-empty)))
    (stringbuilder-add-string! b s)
    (string-pad-fill! b with (- width (string-count s)))
    (stringbuilder->string b)))

;; Pad fill.
(: (-> StringBuilder char int void))
(defun (string-pad-fill! b c n)
  (let fill ((i 0))
    (when (< i n)
      (stringbuilder-add! b c)
      (fill (+ i 1)))))

;; --- Splitting and joining -------------------------------------------------

;; String split, empty fields kept. The runtime gives the array the split
;; makes to the vec without copying it, which only it can do safely.
(import/extern
  (clr-split        (: BjolangRuntime.SplitToVec (-> string string (Vec string))))
  (clr-split/slices (: BjolangRuntime.SplitSlicesToVec (-> string string (Vec string)))))

(: (-> string string (Vec string)))
(defun (string-split s sep) (clr-split s sep))

;; The fields as slices of `s`, sharing its bytes.
(: (-> string string (Vec string)))
(defun (string-split/slice s sep) (clr-split/slices s sep))

(: (-> (Vec string) string string))
(defun (string-join parts sep)
  (def b (stringbuilder-empty))
  (loop (:with first? #t #f)
        (:for part parts)
        (:do (unless first? (stringbuilder-add-string! b sep))
             (stringbuilder-add-string! b part)))
  (stringbuilder->string b))

;; --- Conversions ---
;; ToString wrappers. Each is invariant, so each round-trips through the
;; matching reader on any machine. See the externs above.

(: (-> int string))
(defun (int->string n) (clr-int-string n))

(: (-> long string))
(defun (long->string n) (clr-long-string n))

(: (-> double string))
(defun (double->string n) (clr-double-string n))

(: (-> byte string))
(defun (byte->string n) (clr-byte-string n))

;; Number to string.
(: (-> int string))
(defun (number->string n) (int->string n))

(: (-> string int))
(defun (string->int s) (clr-parse-int s))

(: (-> string long))
(defun (string->long s) (clr-parse-long s))

(: (-> string double))
(defun (string->double s) (clr-parse-double s))

;; The four numbers past the eight. A decimal is written and read with a point
;; and no exponent when it has none: (decimal->string 1.50M) is "1.50", as
;; .NET keeps a decimal's scale.
(: (-> decimal string))
(defun (decimal->string n) (clr-format-number n))

(: (-> string decimal))
(defun (string->decimal s) (clr-parse-decimal s))

(: (-> bigint string))
(defun (bigint->string n) (clr-format-number n))

(: (-> string bigint))
(defun (string->bigint s) (clr-parse-bigint s))

(: (-> int128 string))
(defun (int128->string n) (clr-format-number n))

(: (-> string int128))
(defun (string->int128 s) (clr-parse-int128 s))

(: (-> uint128 string))
(defun (uint128->string n) (clr-format-number n))

(: (-> string uint128))
(defun (string->uint128 s) (clr-parse-uint128 s))

;; --- Bytes and strings ---
;;
;; An encoding is a .NET `System.Text.Encoding`. `(text encodings)` has one
;; value for each encoding .NET knows, and finds them by name; UTF-8 is the
;; default here and needs no import.
;;
;; Both directions are strict, as Racket's `bytes->string/utf-8` is: what
;; does not decode, or has no bytes in the encoding, is an error naming its
;; offset. `#:replace (Some c)` puts `c` there instead. A port is the
;; lenient one: it reads what does not decode as U+FFFD.

(type (: Encoding System.Text.Encoding))

(import/extern
  (clr-utf-8          (: Bjolang.Runtime.Encodings.Cp65001 #:get))
  (clr-decode         (: Bjolang.Runtime.Encodings.Decode (-> (Array byte) int int Encoding string)))
  (clr-decode/replace (: Bjolang.Runtime.Encodings.Decode (-> (Array byte) int int Encoding char string)))
  (clr-encode         (: Bjolang.Runtime.Encodings.Encode (-> string Encoding (Array byte))))
  (clr-encode/replace (: Bjolang.Runtime.Encodings.Encode (-> string Encoding char (Array byte))))
  ;; Whether the bytes from `start` to `end` are UTF-8, which is what
  ;; `bytes->string` would accept. Vectorized: a reader that copies runs of
  ;; bytes checks a run with it in one call.
  (bytes-utf-8?       (: Bjolang.Runtime.Encodings.IsUtf8 (-> (Array byte) int int bool))))

;; The text of the bytes from `#:start` to `#:end`, in `#:encoding`. A string
;; is UTF-8, so from UTF-8 this checks the bytes and copies them. A byte order
;; mark is a character like any other.
(: (-> (Array byte) (#:encoding Encoding) (#:start int) (#:end int) (#:replace (Option char)) string))
(defun (bytes->string bytes #:encoding clr-utf-8 #:start 0 #:end (array-length bytes) #:replace None)
  (match replace
    (None (clr-decode bytes start end encoding))
    ((Some c) (clr-decode/replace bytes start end encoding c))))

;; The string's bytes in `#:encoding`, without a byte order mark. To UTF-8
;; this is a copy of the string's own bytes.
(: (-> string (#:encoding Encoding) (#:replace (Option char)) (Array byte)))
(defun (string->bytes s #:encoding clr-utf-8 #:replace None)
  (match replace
    (None (clr-encode s encoding))
    ((Some c) (clr-encode/replace s encoding c))))

;; A copy of bytes, checked to be UTF-8 once, for a reader that makes many
;; strings from one buffer: each string is taken by byte offset and is not
;; checked again. The chunk owns its copy, and takes a string only between
;; character boundaries in its valid part, so a string made from it is valid
;; whatever the offsets are. `BjolangRuntime/Utf8Chunk.cs` says why.
(type (: Utf8Chunk Bjolang.Runtime.Utf8Chunk))

(import/extern
  (clr-utf8-chunk       (: Bjolang.Runtime.Utf8Chunk.FromBytes (-> (Array byte) int int Utf8Chunk)))
  (utf8-chunk-length    (: Bjolang.Runtime.Utf8Chunk.Length (-> Utf8Chunk int) #:get))
  (utf8-chunk-valid-end (: Bjolang.Runtime.Utf8Chunk.ValidEnd (-> Utf8Chunk int) #:get))
  (utf8-chunk-substring (: Bjolang.Runtime.Utf8Chunk.Substring (-> Utf8Chunk int int string))))

(: (-> (Array byte) (#:start int) (#:end int) Utf8Chunk))
(defun (bytes->utf8-chunk bytes #:start 0 #:end (array-length bytes))
  (clr-utf8-chunk bytes start end))

;; ---------------------------------------------------------------------------
;; Characters
;; ---------------------------------------------------------------------------
;; Character ordering.
(: (-> char char bool))
(defun (char=? a b) (= a b))

(: (-> char char bool))
(defun (char<? a b) (< a b))

(: (-> char char bool))
(defun (char>? a b) (> a b))

(: (-> char char bool))
(defun (char<=? a b) (<= a b))

(: (-> char char bool))
(defun (char>=? a b) (>= a b))

;; Case-insensitive comparison.
(: (-> char char bool))
(defun (char-ci=? a b) (= (char-foldcase a) (char-foldcase b)))

(: (-> char char bool))
(defun (char-ci<? a b) (< (char-foldcase a) (char-foldcase b)))

(: (-> char char bool))
(defun (char-ci>? a b) (> (char-foldcase a) (char-foldcase b)))

(: (-> char char bool))
(defun (char-ci<=? a b) (<= (char-foldcase a) (char-foldcase b)))

(: (-> char char bool))
(defun (char-ci>=? a b) (>= (char-foldcase a) (char-foldcase b)))

;; ---------------------------------------------------------------------------
;; String cursors
;; ---------------------------------------------------------------------------
;; String cursors.
(: (-> StringCursor StringCursor bool))
(defun (string-cursor=? a b) (= a b))

(: (-> StringCursor StringCursor bool))
(defun (string-cursor<? a b) (< a b))

(: (-> StringCursor StringCursor bool))
(defun (string-cursor>? a b) (> a b))

(: (-> StringCursor StringCursor bool))
(defun (string-cursor<=? a b) (<= a b))

(: (-> StringCursor StringCursor bool))
(defun (string-cursor>=? a b) (>= a b))

;; --- Traversal -------------------------------------------------------------

;; String fold.
;; The forward walks below ask for the character and the step in one call, so
;; that the unit at the cursor is read and bounds-checked once per character
;; rather than twice.
(: (-> (-> char %a %a) %a string %a))
(defun (string-fold f init s)
  (let loop ((c (string-cursor-start s)) (acc init))
    (if (string-cursor-end? s c)
        acc
        (match (string-cursor-ref+next s c)
          ((Tuple ch next) (loop next (f ch acc)))))))

;; String fold right.
(: (-> (-> char %a %a) %a string %a))
(defun (string-fold-right f init s)
  (let loop ((c (string-cursor-end s)) (acc init))
    (if (string-cursor=? c (string-cursor-start s))
        acc
        (let ((p (string-cursor-prev s c)))
          (loop p (f (string-cursor-ref s p) acc))))))

(: (-> (-> char void) string void))
(defun (string-for-each f s)
  ;; (loop (:for ch s)
  ;;       (:do (f ch)))
  (let loop ((c (string-cursor-start s)))
    (unless (string-cursor-end? s c)
      (def (Tuple ch next) (string-cursor-ref+next s c))
      (f ch)
      (loop next))))

;; The cursor on the first character of `s` that `pred` holds for.
(: (-> (-> char bool) string (Option StringCursor)))
(defun (string-find pred s)
  (let loop ((c (string-cursor-start s)))
    (if (string-cursor-end? s c)
        None
        (match (string-cursor-ref+next s c)
          ((Tuple ch next)
           (if (pred ch)
               (Some c)
               (loop next)))))))

;; --- Conversions -----------------------------------------------------------

(: (-> string (List char)))
(defun (string->list s)
  (string-fold-right cons Nil s))

(: (-> (List char) string))
(defun (list->string cs)
  (def b (stringbuilder-empty))
  (list-for-each #(stringbuilder-add! b &1) cs)
  (stringbuilder->string b))

(: (-> string (Vec char)))
(defun (string->vec s)
  (def b (vecbuilder-empty))
  (string-for-each #(vecbuilder-add! b &1) s)
  (vecbuilder->vec b))

;; Reverse string.
(: (-> string string))
(defun (string-reverse s) (.Reverse s))

;; --- File operations.
;;
;; Everything in this section and the directory one below reaches the disk
;; through the operations of the `FS` effect, declared with the ports further
;; down, so a fake filesystem answers for all of it.

;; How an output file is opened.
;;
;; - `Truncate`: create the file, or empty it if it exists.
;; - `Append`: create the file, or write after what it holds.
;; - `CreateNew`: create the file, and fail if it exists. The check and the
;;   create are one system call, so two writers cannot both succeed.
(type (: WriteMode (Union Truncate Append CreateNew)))

;; What a path names, when it names anything.
(type (: FsKind (Union FsFile FsDirectory)))

;; What a move does when something is already at the target.
;;
;; - `MoveNew`: fails, and changes nothing.
;; - `MoveReplace`: a file at the target is replaced, in the one rename that
;;   moves the new one there, so nothing ever sees the target missing. A
;;   directory is never replaced, whichever of the two it is.
(type (: MoveMode (Union MoveNew MoveReplace)))

;; What `(fs-info path)` answers about a path that names something. One answer
;; to one question, so that a handler cannot say a path is a file and then
;; forget to say how big it is.
;;
;; - `size`: in bytes. 0 for a directory.
;; - `modified-ms`: the last write, in Unix milliseconds, UTC. Milliseconds
;;   because `(std datetime)`'s `unix-milliseconds->instant` takes them, and
;;   the prelude cannot name an `Instant` itself.
;; - `link?`: the path itself is a symbolic link. The other fields are what
;;   .NET reports for it; `directory-walk` does not enter one, so a link back up
;;   the tree cannot loop it.
(type (: FsInfo (Record (: kind FsKind)
                        (: size long)
                        (: modified-ms long)
                        (: link? bool))))

;; The operations answer a `Result`; the functions in these two sections raise
;; the `Err`, as the .NET methods they replaced did.
(: (-> (Result System.Exception %a) %a))
(defun (raise-err r)
  (match r
    ((Ok v) v)
    ((Err e) (raise e))))

;; Read all text.
(: (-> string string))
(defun (file-read-text path)
  (match (open-input-file path)
    ((Ok opened) (with-open ((p opened)) (read-all p)))
    ((Err e) (raise e))))

;; The flush is explicit so that the suspending copy awaits it. Closing would
;; flush too, but synchronously, on whatever thread runs the close.
(: (-> string string WriteMode void))
(defun (write-whole-file path contents mode)
  (match (open-output-file path #:mode mode)
    ((Ok opened) (with-open ((p opened))
                   (write-string p contents)
                   (flush-port p)))
    ((Err e) (raise e))))

;; Write text.
(: (-> string string void))
(defun (file-write-text path contents) (write-whole-file path contents Truncate))

(: (-> string string void))
(defun (file-append-text path contents) (write-whole-file path contents Append))

(: (-> string (Option FsInfo)))
(defun (file-info path) (fs-info path))

;; What a path names, for the functions below that only ask that.
(: (-> string (Option FsKind)))
(defun (kind-of path)
  (option-map (fun (info) (record-ref info kind)) (fs-info path)))

(: (-> string bool))
(defun (file-exists? path)
  (match (kind-of path)
    ((Some FsFile) #t)
    (_ #f)))

;; A missing file is not an error, as with `File.Delete`.
(: (-> string void))
(defun (file-delete path) (raise-err (fs-delete-file path)))

;; Moves or renames a file or a directory. Fails if `to` exists, unless
;; `#:mode MoveReplace` says to replace a file there.
(: (-> string string (#:mode MoveMode) void))
(defun (file-move from to #:mode MoveNew) (raise-err (fs-move from to mode)))

;; --- Paths ---
;; Path strings.

(import/extern
  (clr-full-path      (: System.IO.Path.GetFullPath))
  (clr-path-combine   (: System.IO.Path.Combine))
  (clr-file-name      (: System.IO.Path.GetFileName))
  (clr-file-extension (: System.IO.Path.GetExtension)))

(: (-> string string))
(defun (path-absolute path) (clr-full-path path))

;; Path combine.
(: (-> #:rest string string))
(defun (path-combine #:rest parts) (clr-path-combine parts))

;; Path filename.
(: (-> string (Option string)))
(defun (path-filename path)
  (def name (clr-file-name path))
  (if (string-empty? name) None (Some name)))

;; File extension.
(: (-> string (Option string)))
(defun (path-file-extension path)
  (def ext (clr-file-extension path))
  (if (string-empty? ext) None (Some ext)))

;; --- Directories ---

(: (-> string bool))
(defun (directory-exists? path)
  (match (kind-of path)
    ((Some FsDirectory) #t)
    (_ #f)))

;; Create directory.
(: (-> string void))
(defun (directory-create path) (raise-err (fs-create-directory path)))

;; Delete directory.
(: (-> string void))
(defun (directory-delete path) (raise-err (fs-delete-directory path)))

;; Built from the operations rather than handed to `Directory.Delete` with its
;; recursive flag, so that a handler sees every deletion.
;;
;; A symbolic link to a directory is removed as a link and never entered, and
;; that includes `path` itself. Entering one would delete what it points at: a
;; link to the home directory inside a build directory would empty the home
;; directory, and a link back up the tree would never end.
(: (-> string void))
(defun (directory-delete-tree path)
  (if (directory-link? path)
      (directory-delete path)
      (let ()
        (loop (:for entry (raise-err (fs-list path)))
              (:do (match (fs-info entry)
                     ((Some info)
                      (match (record-ref info kind)
                        (FsDirectory (directory-delete-tree entry))
                        (FsFile (file-delete entry))))
                     ;; A link to nothing, which `file-delete` removes.
                     (None (file-delete entry)))))
        (directory-delete path))))

(: (-> string bool))
(defun (directory-link? path)
  (match (fs-info path)
    ((Some info) (and (record-ref info link?)
                      (match (record-ref info kind) (FsDirectory #t) (FsFile #f))))
    (None #f)))

(: (-> string FsKind (Vec string)))
(defun (entries-of-kind path kind)
  (loop (:for entry (raise-err (fs-list path)))
        (:when (match (Tuple (kind-of entry) kind)
                 ((Tuple (Some FsFile) FsFile) #t)
                 ((Tuple (Some FsDirectory) FsDirectory) #t)
                 (_ #f)))
        (:acc found (vecing entry))))

;; Directory files.
(: (-> string (Vec string)))
(defun (directory-files path) (entries-of-kind path FsFile))

(: (-> string (Vec string)))
(defun (directory-subdirectories path) (entries-of-kind path FsDirectory))

;; --- The environment ---

(import/extern
  (clr-get-env      (: System.Environment.GetEnvironmentVariable))
  (clr-set-env      (: System.Environment.SetEnvironmentVariable))
  (clr-empty?       (: System.String.IsNullOrEmpty))
  (clr-cwd          (: System.Environment.CurrentDirectory #:get))
  (clr-set-cwd      (: System.Environment.CurrentDirectory #:set)))

(: (-> string (Option string)))
(defun (get-environment-variable name)
  (def value (clr-get-env name))
  (if (clr-empty? value)
      None
      (Some value)))

;; Set environment variable.
(: (-> string string void))
(defun (set-environment-variable! name value) (clr-set-env name value))

(: (-> string))
(defun (current-directory) clr-cwd)

(: (-> string void))
(defun (set-current-directory! path) (clr-set-cwd path))

;; ---------------------------------------------------------------------------
;; Ports
;; ---------------------------------------------------------------------------

(import/class
  ;; What the `FS` effect's openers answer, and what a file port is over.
  (ClrStream (: System.IO.Stream))
  ;; `IOException` covers `CreateNew` finding the file already there, as well
  ;; as the not-found exceptions below it.
  (FileStream (: System.IO.FileStream (-> string FileMode FileAccess FileShare FileStream)
                 #:exceptions (System.IO.IOException
                               System.UnauthorizedAccessException)))
  (FileMode (: System.IO.FileMode))
  (FileAccess (: System.IO.FileAccess))
  (FileShare (: System.IO.FileShare))

  (Exception (: System.Exception)))

;; Port types.
;;
;; One input port type and one output port type, as in Racket: a port is
;; bytes. Text reads decode UTF-8 from an input port, and text writes encode
;; UTF-8 into an output port, so byte and text operations mix on one port.
;; They are also a `System.IO.TextReader` and a `System.IO.TextWriter`, so a
;; .NET API that takes one takes a port.
(type
  (: InputPort Bjolang.Runtime.BjoInputPort)
  (: OutputPort Bjolang.Runtime.BjoOutputPort))

;; What ends a line: Racket's `read-line` modes. `Any` is the default
;; everywhere a line is read.
(type (: LineMode (Union Any AnyOne Linefeed Return ReturnLinefeed)))

;; --- Opening ---
;;
;; A file port is owned by the ambient scope: when the scope returns, the file
;; is closed, whether the program said so or not. A string port is not - it is
;; memory, and the GC prices memory.
;;
;; The scope is read before the file is opened, so a program with no scope to
;; put the port in fails without having opened anything.

(import/extern
  ;; A port over a stream, registered on the ambient scope.
  (owned-port-over (: Bjolang.Runtime.BytePorts.FromStream (-> ClrStream InputPort)))
  (string-port  (: Bjolang.Runtime.InputPorts.FromString (-> string InputPort)))
  (owned-port-out (: Bjolang.Runtime.OutputPorts.FromStream (-> ClrStream OutputPort)))
  (string-port-out (: Bjolang.Runtime.OutputPorts.OpenString (-> OutputPort)))
  (output-bytes (: Bjolang.Runtime.OutputPorts.GetBytes (-> OutputPort (Array byte))))
  (close-in     (: BjolangRuntime.CloseInput  (-> InputPort void)))
  (close-out    (: BjolangRuntime.CloseOutput (-> OutputPort void)))
  (close-out-await (: BjolangRuntime.CloseOutputAsync (-> OutputPort void) #:async)))

;; An input port over a stream an opener answered, registered on the ambient
;; scope. Whatever the stream is - a file, or a `MemoryStream` a fake
;; filesystem made - the port is owned the same way.
(: (-> (Result Exception ClrStream) (Result Exception InputPort)))
(defun (stream->input-port opened)
  (match opened
    ((Ok s) (Ok (owned-port-over s)))
    ((Err e) (Err e))))

;; The same for an output port: a file port is buffered a block at a time, and
;; closing it, or its scope ending, writes what is left.
(: (-> (Result Exception ClrStream) (Result Exception OutputPort)))
(defun (stream->output-port opened)
  (match opened
    ((Ok s) (Ok (owned-port-out s)))
    ((Err e) (Err e))))

;; Open input string: a port over the string's own UTF-8 bytes, shared rather
;; than copied. Memory only, so nothing registers it.
(: (-> string InputPort))
(defun (open-input-string contents)
  (string-port contents))

;; Open output string: a port that keeps what is written to it, read back with
;; `get-output-string` or `get-output-bytes`. Memory only, so nothing
;; registers it.
(: (-> OutputPort))
(defun (open-output-string)
  (string-port-out))

;; --- Closing ---
;;
;; Closing goes through the owner handle rather than calling `.Dispose`. A
;; direct dispose would close the stream and leave the registration on the
;; scope's list, so a long-lived scope would collect one spent entry per file it
;; had already finished with.
;;
;; On a port nothing owns this disposes, which is what a string port wants. On
;; one of the standard ports it flushes and stops: nothing in the language
;; should be able to take stdout away from the rest of the program.
(: (-> InputPort void))
(defun (close-input-port p) (close-in p))

;; What is still buffered goes out first. On a fiber that write suspends, so the
;; close works over a stream that refuses blocking I/O, as Kestrel's do; the
;; port is closed even when the write fails, and the failure raised after.
;; `with-open` and a scope ending close with a blocking write, which after a
;; `flush-port` has nothing left to do.
(: (-> OutputPort void))
(defbjouble (close-output-port p)
  (#:sync (close-out p))
  (#:bjo  (close-out-await p)))

;; --- The filesystem is an effect ---
;;
;; All filesystem operations route through the `FS` algebraic effect, allowing
;; callers to intercept disk operations (e.g., in-memory mocks for tests, sandbox
;; write restrictions, or build dependency tracking). When unhandled, operations
;; fall back to their `/raw` defaults using standard .NET calls.
;;
;; Design:
;; - Openers return raw `ClrStream`s rather than ports. Higher-level functions
;;   (like `open-input-file` or `(std ports)` byte openers) construct ports and
;;   register them in the ambient scope, allowing mock handlers to work uniformly
;;   across text and byte I/O.
;; - Effect operations are synchronous; coroutine suspension occurs only when reading
;;   or writing through the underlying streams/ports.
;; - Effect operations return `(Result Exception %a)` values, whereas higher-level
;;   prelude wrappers unwrap them and raise exceptions on error.

(import/class
  (DirectoryInfo (: System.IO.DirectoryInfo (-> string DirectoryInfo)))
  (FileInfo (: System.IO.FileInfo (-> string FileInfo))))

(import/extern
  (clr-file-exists (: System.IO.File.Exists (-> string bool)))
  (clr-dir-exists  (: System.IO.Directory.Exists (-> string bool)))
  (clr-dir-entries (: System.IO.Directory.GetFileSystemEntries (-> string (Array string))
                      #:exceptions (System.IO.IOException System.UnauthorizedAccessException)))
  (clr-dir-create  (: System.IO.Directory.CreateDirectory (-> string DirectoryInfo)
                      #:exceptions (System.IO.IOException System.UnauthorizedAccessException)))
  (clr-file-delete (: System.IO.File.Delete (-> string void)
                      #:exceptions (System.IO.IOException System.UnauthorizedAccessException)))
  (clr-dir-delete  (: System.IO.Directory.Delete (-> string void)
                      #:exceptions (System.IO.IOException System.UnauthorizedAccessException)))
  (clr-file-move   (: System.IO.File.Move (-> string string bool void)
                      #:exceptions (System.IO.IOException System.UnauthorizedAccessException)))
  (clr-dir-move    (: System.IO.Directory.Move (-> string string void)
                      #:exceptions (System.IO.IOException System.UnauthorizedAccessException))))

;; Read-only, and other readers allowed alongside, as `StreamReader`'s own
;; path constructor opens.
(: (-> string (Result Exception ClrStream)))
(defun (open-read-stream/raw path)
  (match (FileStream. path FileMode.Open FileAccess.Read FileShare.Read)
    ((Ok s) (Ok (cast ClrStream s)))
    ((Err e) (Err e))))

(: (-> WriteMode FileMode))
(defun (write-mode->file-mode mode)
  (match mode
    (Truncate FileMode.Create)
    (Append FileMode.Append)
    (CreateNew FileMode.CreateNew)))

;; Write-only, readers allowed alongside, as `StreamWriter`'s path constructor
;; opens.
(: (-> string WriteMode (Result Exception ClrStream)))
(defun (open-write-stream/raw path mode)
  (match (FileStream. path (write-mode->file-mode mode) FileAccess.Write FileShare.Read)
    ((Ok s) (Ok (cast ClrStream s)))
    ((Err e) (Err e))))

;; .NET ticks at 1970-01-01T00:00Z, and per millisecond.
(: long)
(def unix-epoch-ticks 621355968000000000L)

;; Rounds toward the past, so a write just before 1970 is -1 and not 0.
(: (-> long long))
(defun (ticks->unix-ms ticks)
  (def d (- ticks unix-epoch-ticks))
  (def q (/ d 10000L))
  (if (< (% d 10000L) 0L) (- q 1L) q))

;; `File.Exists` and `Directory.Exists` first, because they never throw: a
;; path that is no path at all, "" say, is `None` rather than an
;; `ArgumentException` out of the `FileInfo` constructor. The info object then
;; reads the path once, on its first property, so size and time agree with
;; each other; a file deleted in between is `None`.
;; Where a chain of symbolic links ends: `path` itself when it is no link. A
;; link's target is relative to the directory the link is in. At most 40
;; steps, as Linux allows, so that a loop of links ends. Read step by step
;; because `ResolveLinkTarget` answers null for a link to nothing, and there is
;; no null to test here.
(: (-> string string))
(defun (link-end path)
  (let loop ((p path) (steps 0))
    (def target (.-LinkTarget (FileInfo. p)))
    (if (or (clr-empty? target) (= steps 40))
        p
        (loop (match (path-directory p)
                ((Some dir) (path-combine dir target))
                (None target))
              (+ steps 1)))))

;; A link's size and time are those of the file it leads to, which is what
;; reading it reads; `link?` is what says it is one. A link to nothing keeps
;; its own.
(: (-> string (Option FsInfo)))
(defun (file-info/raw path)
  (def own (FileInfo. path))
  (def link (not (clr-empty? (.-LinkTarget own))))
  (def end (if link (FileInfo. (link-end path)) own))
  (def f (if (.-Exists end) end own))
  (if (.-Exists own)
      (Some (FsInfo (kind FsFile)
                    (size (.-Length f))
                    (modified-ms (ticks->unix-ms (.-Ticks (.-LastWriteTimeUtc f))))
                    (link? link)))
      None))

(: (-> string (Option FsInfo)))
(defun (directory-info/raw path)
  (def own (DirectoryInfo. path))
  (def link (not (clr-empty? (.-LinkTarget own))))
  (def end (if link (DirectoryInfo. (link-end path)) own))
  (def d (if (.-Exists end) end own))
  (if (.-Exists own)
      (Some (FsInfo (kind FsDirectory)
                    (size 0L)
                    (modified-ms (ticks->unix-ms (.-Ticks (.-LastWriteTimeUtc d))))
                    (link? link)))
      None))

(: (-> string (Option FsInfo)))
(defun (fs-info/raw path)
  (if (clr-file-exists path)
      (file-info/raw path)
      (if (clr-dir-exists path) (directory-info/raw path) None)))

;; Whole paths, files and directories both, in the order the OS gives them.
(: (-> string (Result Exception (Vec string))))
(defun (fs-list/raw path)
  (match (clr-dir-entries path)
    ((Ok names) (Ok (loop (:for name names) (:acc found (vecing name)))))
    ((Err e) (Err e))))

;; Creates every missing directory along the path. One that is already there is
;; not an error.
(: (-> string (Result Exception Unit)))
(defun (fs-create-directory/raw path)
  (match (clr-dir-create path)
    ((Ok _) (Ok unit))
    ((Err e) (Err e))))

;; What a guarded `void` call answers, as the `Unit` an operation declares.
(: (-> (Result Exception %a) (Result Exception Unit)))
(defun (unit-result r)
  (match r
    ((Ok _) (Ok unit))
    ((Err e) (Err e))))

;; A missing file is not an error.
(: (-> string (Result Exception Unit)))
(defun (fs-delete-file/raw path) (unit-result (clr-file-delete path)))

;; An empty directory. A missing or non-empty one is an `IOException`.
(: (-> string (Result Exception Unit)))
(defun (fs-delete-directory/raw path) (unit-result (clr-dir-delete path)))

;; A file or a directory. Fails if `to` exists, unless `MoveReplace` asks to
;; replace a file there. `Directory.Move` never replaces, which is the rule.
(: (-> string string MoveMode (Result Exception Unit)))
(defun (fs-move/raw from to mode)
  (unit-result (if (clr-dir-exists from)
                   (clr-dir-move from to)
                   (clr-file-move from to (match mode (MoveNew #f) (MoveReplace #t))))))

(defeffect FS
  (open-read-stream    (-> string (Result Exception ClrStream)))
  (open-write-stream   (-> string WriteMode (Result Exception ClrStream)))
  (fs-info             (-> string (Option FsInfo)))
  (fs-list             (-> string (Result Exception (Vec string))))
  (fs-create-directory (-> string (Result Exception Unit)))
  (fs-delete-file      (-> string (Result Exception Unit)))
  (fs-delete-directory (-> string (Result Exception Unit)))
  (fs-move             (-> string string MoveMode (Result Exception Unit)))
  #:default ((open-read-stream open-read-stream/raw)
             (open-write-stream open-write-stream/raw)
             (fs-info fs-info/raw)
             (fs-list fs-list/raw)
             (fs-create-directory fs-create-directory/raw)
             (fs-delete-file fs-delete-file/raw)
             (fs-delete-directory fs-delete-directory/raw)
             (fs-move fs-move/raw))
  #:cold (open-read-stream open-write-stream fs-info fs-list
          fs-create-directory fs-delete-file fs-delete-directory fs-move))

;; --- Opening a file port ---
;;
;; The scope is read before the perform, so a program with nowhere to put the
;; port fails without having opened anything.

(: (-> string (Result Exception InputPort)))
(defun (open-input-file path)
  (ignore (current-scope))
  (stream->input-port (open-read-stream path)))

;; `(open-output-file path)` truncates; `#:mode Append` or `#:mode CreateNew`
;; says otherwise.
(: (-> string (#:mode WriteMode) (Result Exception OutputPort)))
(defun (open-output-file path #:mode Truncate)
  (ignore (current-scope))
  (stream->output-port (open-write-stream path mode)))

;; The same, past whatever handler is installed. The `/raw` says so.
(: (-> string (Result Exception InputPort)))
(defun (open-input-file/raw path)
  (ignore (current-scope))
  (stream->input-port (open-read-stream/raw path)))

(: (-> string WriteMode (Result Exception OutputPort)))
(defun (open-output-file/raw path mode)
  (ignore (current-scope))
  (stream->output-port (open-write-stream/raw path mode)))

;; Opens a file, passes the open port to `proc`, and guarantees port closure
;; on normal return or exception via `with-open`.
;;
;; Unlike `with-scope`, `with-open` avoids allocating a new fiber scope, allowing
;; synchronous functions to call it without requiring a coroutine context.
;;
;; `proc` is suspension-polymorphic (`-?->`): both synchronous and async/suspending
;; variants are compiled, ensuring suspending bodies receive identical cleanup.
(: (-> string (-?-> InputPort %a) (Result Exception %a)))
(defun (call-with-input-file path proc)
  (match (open-input-file path)
    ((Ok p) (with-open ((it p)) (Ok (proc it))))
    ((Err e) (Err e))))

(: (-> string (-?-> OutputPort %a) (#:mode WriteMode) (Result Exception %a)))
(defun (call-with-output-file path proc #:mode Truncate)
  (match (open-output-file path #:mode mode)
    ((Ok p) (with-open ((it p)) (Ok (proc it))))
    ((Err e) (Err e))))

;; --- Bytes, copies and lines ---
;;
;; Because the prelude has no byte port type, byte I/O and file copying operate
;; directly on `ClrStream`s returned by the `FS` effect openers (`open-read-stream`
;; and `open-write-stream`).
;;
;; Like the text port functions, these helpers provide both sync and suspending
;; (async) variants backed by the streams' async methods. When used in `with-open`,
;; a stream is explicitly flushed before exiting so pending writes are awaited
;; asynchronously rather than blocking during synchronous disposal.

(import/class
  (MemoryStream (: System.IO.MemoryStream (-> MemoryStream)))
  (IOException (: System.IO.IOException (-> string IOException))))

(import/extern
  (clr-copy-to            (: System.IO.Stream.CopyTo      (-> ClrStream ClrStream void) #:blocking))
  (clr-copy-to-await      (: System.IO.Stream.CopyToAsync (-> ClrStream ClrStream void) #:async))
  (clr-stream-write       (: System.IO.Stream.Write       (-> ClrStream (Array byte) int int void) #:blocking))
  (clr-stream-write-await (: System.IO.Stream.WriteAsync  (-> ClrStream (Array byte) int int void) #:async))
  (clr-stream-flush       (: System.IO.Stream.Flush       (-> ClrStream void) #:blocking))
  (clr-stream-flush-await (: System.IO.Stream.FlushAsync  (-> ClrStream void) #:async)))

(: (-> ClrStream ClrStream void))
(defbjouble (stream-copy! from to)
  (#:sync (clr-copy-to from to))
  (#:bjo  (clr-copy-to-await from to)))

(: (-> ClrStream (Array byte) void))
(defbjouble (stream-write! s bytes)
  (#:sync (clr-stream-write s bytes 0 (array-length bytes)))
  (#:bjo  (clr-stream-write-await s bytes 0 (array-length bytes))))

(: (-> ClrStream void))
(defbjouble (stream-flush! s)
  (#:sync (clr-stream-flush s))
  (#:bjo  (clr-stream-flush-await s)))

(import/extern
  (clr-stream-read       (: System.IO.Stream.Read      (-> ClrStream (Array byte) int int int) #:blocking))
  (clr-stream-read-await (: System.IO.Stream.ReadAsync (-> ClrStream (Array byte) int int int) #:async))
  (clr-stream-length     (: System.IO.Stream.Length (-> ClrStream long) #:get))
  (clr-new-guid          (: System.Guid.NewGuid (-> System.Guid)))
  (clr-guid-text         (: System.Guid.ToString (-> System.Guid string string))))

;; Up to `n` bytes into the start of `buffer`; the number read, 0 at the end.
(: (-> ClrStream (Array byte) int int))
(defbjouble (stream-read-some! s buffer n)
  (#:sync (clr-stream-read s buffer 0 n))
  (#:bjo  (clr-stream-read-await s buffer 0 n)))

;; The first `n` bytes of `buffer`.
(: (-> ClrStream (Array byte) int void))
(defbjouble (stream-write-some! s buffer n)
  (#:sync (clr-stream-write s buffer 0 n))
  (#:bjo  (clr-stream-write-await s buffer 0 n)))

;; At most `count` bytes: what the source held when it was opened.
;;
;; - `Stream.CopyTo` copies to the end of the source. When the target is the
;;   source under another name, opened to append, the end moves away as fast
;;   as the copy writes, and the file grows until the disk is full. Copying
;;   the length that the source had when it was opened ends.
;; - Stops early when the source ends before `count` bytes.
(: (-> ClrStream ClrStream long void))
(defun (stream-copy-count! from to count)
  (def buffer (make-array (cast int (min 81920L (max count 1L)))))
  (let go ((left count))
    (when (> left 0L)
      (def n (stream-read-some! from buffer (cast int (min (cast long (array-length buffer)) left))))
      (when (> n 0)
        (stream-write-some! to buffer n)
        (go (- left (cast long n)))))))

;; Copies a file from `from` to `to`.
;;
;; `#:mode` controls destination behavior:
;; - `CreateNew` (default): Fails if `to` exists, preventing accidental overwrite.
;; - `Truncate`: Overwrites `to` via an atomic replace.
;; - `Append`: Appends to `to`.
;;
;; Safety & Aliasing:
;; - Direct self-copies are rejected.
;; - The source is opened first; a missing source leaves `to` untouched.
;; - If `to` is an alias or symlink to `from`, in-place truncation could empty
;;   the source before copying. To prevent this:
;;   * `Truncate` copies to a temporary sibling file and moves it over `to`
;;     (replacing any symlink rather than writing through it).
;;   * `Append` copies only the source's initial byte length, preventing an
;;     infinite loop when appending a file to itself.
(: (-> string string (#:mode WriteMode) void))
(defun (file-copy from to #:mode CreateNew)
  (when (= (path-absolute from) (path-absolute to))
    (raise (cast Exception (IOException. (string-append "cannot copy a file onto itself: " from)))))
  (with-open ((in (raise-err (open-read-stream from))))
    (def count (clr-stream-length in))
    (match mode
      (Truncate
       (def fresh (str to ".copy-" (clr-guid-text (clr-new-guid) "N")))
       (match (try (begin
                     (with-open ((out (raise-err (open-write-stream fresh CreateNew))))
                       (stream-copy-count! in out count)
                       (stream-flush! out))
                     (raise-err (fs-move fresh to MoveReplace)))
                   #:catch (System.Exception))
         ((Ok _) unit)
         ((Err e)
          (ignore (fs-delete-file fresh))
          (raise e))))
      (_
       (with-open ((out (raise-err (open-write-stream to mode))))
         (stream-copy-count! in out count)
         (stream-flush! out))))))

(: (-> string (Array byte)))
(defun (file-read-bytes path)
  (with-open ((in (raise-err (open-read-stream path))))
    (def buffer (MemoryStream.))
    (stream-copy! in (cast ClrStream buffer))
    (.ToArray buffer)))

(: (-> string (Array byte) WriteMode void))
(defun (write-whole-bytes path bytes mode)
  (with-open ((out (raise-err (open-write-stream path mode))))
    (stream-write! out bytes)
    (stream-flush! out)))

(: (-> string (Array byte) void))
(defun (file-write-bytes path bytes) (write-whole-bytes path bytes Truncate))

(: (-> string (Array byte) void))
(defun (file-append-bytes path bytes) (write-whole-bytes path bytes Append))

;; The lines, without their line ends. What ends a line is `#:mode`'s to say;
;; by default "\n", "\r" and "\r\n" all do. A last line needs no end, and a
;; final line end makes no empty line after it.
(: (-> string (#:mode LineMode) (Vec string)))
(defun (file-read-lines path #:mode Any)
  (match (open-input-file path)
    ((Ok opened) (with-open ((p opened))
                   (def lines (vecbuilder-empty))
                   (let next ()
                     (unless (port-eof? p)
                       (vecbuilder-add! lines (read-line p #:mode mode))
                       (next)))
                   (vecbuilder->vec lines)))
    ((Err e) (raise e))))

;; Each line followed by "\n", the last one too, on every OS: the same lines
;; make the same bytes wherever they are written, and `file-read-lines` gives
;; them back as long as none holds a line end of its own.
(: (-> string (Vec string) void))
(defun (file-write-lines path lines)
  (match (open-output-file path)
    ((Ok opened) (with-open ((p opened))
                   (loop (:for line lines)
                         (:do (write-string p line)
                              (write-string p "\n")))
                   (flush-port p)))
    ((Err e) (raise e))))
 
;; Returns a lazy `Seq` of items produced by repeatedly calling `(reader port)`.
;;
;; The file is opened inside the `seq`, meaning each iteration re-opens the file
;; from the start. The file is closed when iteration reaches EOF, stops early,
;; or encounters an exception. If an iterator is abandoned without cleanup, the
;; port remains open until the ambient scope ends.
;;
;; Because opening happens inside the sequence, open errors (such as a missing
;; file) raise on the first pull rather than when `file-read/seq` is called.
(: file-read/seq #:sync (-> (-> InputPort %a) string (Seq %a)))
(defun (file-read/seq reader path)
  (seq
    (def file (raise-err (open-input-file path)))
    (with-open ((p file))
      (let loop ()
        (unless (port-eof? p)
          (yield (reader p))
          (loop))))))

(: file-read-lines/seq #:sync (-> string (#:mode LineMode) (Seq string)))
(defun (file-read-lines/seq path #:mode Any)
  (file-read/seq (fun (p) (read-line p #:mode mode)) path))

(: (-> string (List string)))
(defun (sorted-entries dir) (list-sort (vec->list (raise-err (fs-list dir)))))

;; Lazily yields all files under `dir` in depth-first order.
;;
;; Behavior & Guarantees:
;; - Deterministic: Directory entries are sorted ordinally, guaranteeing identical
;;   traversal order across all operating systems and virtual filesystems.
;; - Filtering (`#:into?`): Called on each directory before entering; return `#f`
;;   to skip a subtree (e.g. `.git`).
;; - Cycle Prevention: Directory symlinks are never followed.
;; - Lazy Traversal: Subdirectories are listed on demand using the ambient `FS`
;;   handler, so stopping iteration early avoids unneeded filesystem I/O.
(: directory-walk #:sync (-> string (#:into? (-> string bool)) (Seq string)))
(defun (directory-walk dir #:into? (fun (d) #t))
  (seq
    (let loop ((todo (sorted-entries dir)))
      (unless (list-empty? todo)
        (def entry (list-head todo))
        (def rest (list-tail todo))
        (match (fs-info entry)
          ((Some info)
           (match (record-ref info kind)
             (FsFile (yield entry)
                     (loop rest))
             (FsDirectory
              (if (and (not (record-ref info link?)) (into? entry))
                  (loop (list-append (sorted-entries entry) rest))
                  (loop rest)))))
          ;; Gone since it was listed.
          (None (loop rest)))))))

;; Flush.
(import/extern
  (clr-flush       (: Bjolang.Runtime.BjoOutputPort.Flush           (-> OutputPort void) #:blocking))
  (clr-flush-await (: Bjolang.Runtime.BjoOutputPort.FlushValueAsync (-> OutputPort void) #:async)))

(: (-> OutputPort void))
(defbjouble (flush-port p)
  (#:sync (clr-flush p))
  (#:bjo  (clr-flush-await p)))

;; --- Input ---
;; These are the leaves of our IO. The bottom calls, so to say. 
(import/extern
  (clr-eof        (: Bjolang.Runtime.BjoInputPort.Eof                 (-> InputPort bool)   #:blocking))
  (clr-eof-await  (: Bjolang.Runtime.BjoInputPort.EofAsync            (-> InputPort bool)   #:async))
  (clr-line       (: Bjolang.Runtime.InputPorts.ReadLineOrThrow    (-> InputPort int string) #:blocking))
  (clr-line-await (: Bjolang.Runtime.InputPorts.ReadLineOrThrowAsync (-> InputPort int string) #:async))
  (clr-char       (: Bjolang.Runtime.InputPorts.ReadCharOrThrow    (-> InputPort char)   #:blocking))
  (clr-char-await (: Bjolang.Runtime.InputPorts.ReadCharOrThrowAsync (-> InputPort char) #:async))
  (clr-rest       (: Bjolang.Runtime.BjoInputPort.ReadToEndUtf8       (-> InputPort string) #:blocking))
  (clr-rest-await (: Bjolang.Runtime.BjoInputPort.ReadToEndUtf8Async  (-> InputPort string) #:async))
  (clr-line-opt       (: Bjolang.Runtime.InputPorts.ReadLineOpt      (-> InputPort int (Option string)) #:blocking))
  (clr-line-opt-await (: Bjolang.Runtime.InputPorts.ReadLineOptAsync (-> InputPort int (Option string)) #:async))
  (clr-char-opt       (: Bjolang.Runtime.InputPorts.ReadCharOpt      (-> InputPort (Option char)) #:blocking))
  (clr-char-opt-await (: Bjolang.Runtime.InputPorts.ReadCharOptAsync (-> InputPort (Option char)) #:async))
  (clr-peek-char       (: Bjolang.Runtime.InputPorts.PeekCharOrThrow      (-> InputPort char) #:blocking))
  (clr-peek-char-await (: Bjolang.Runtime.InputPorts.PeekCharOrThrowAsync (-> InputPort char) #:async))
  (clr-peek-char-opt       (: Bjolang.Runtime.InputPorts.PeekCharOpt      (-> InputPort (Option char)) #:blocking))
  (clr-peek-char-opt-await (: Bjolang.Runtime.InputPorts.PeekCharOptAsync (-> InputPort (Option char)) #:async)))

;; Port eof.
(: (-> InputPort bool))
(defbjouble (port-eof? p)
  (#:sync (clr-eof p))
  (#:bjo  (clr-eof-await p)))

;; `LineMode` as `Bjolang.Runtime.LineMode` numbers it.
(: (-> LineMode int))
(defun (line-mode-code mode)
  (match mode
    (Any 0)
    (AnyOne 1)
    (Linefeed 2)
    (Return 3)
    (ReturnLinefeed 4)))

;; Where a reader of one argument is expected, `read-line` is the function
;; that calls it with the port, and the mode is `Any`: `(port->list read-line
;; p)`. For another mode, hand it over as `(fun (p) (read-line p #:mode m))`.
(: (-> InputPort (#:mode LineMode) string))
(defbjouble (read-line p #:mode Any)
  (#:sync (clr-line p (line-mode-code mode)))
  (#:bjo  (clr-line-await p (line-mode-code mode))))

;; One read, not `port-eof?` and then `read-line`: between those two another
;; fiber reading the same port can take the last line.
(: (-> InputPort (#:mode LineMode) (Option string)))
(defbjouble (read-line/opt p #:mode Any)
  (#:sync (clr-line-opt p (line-mode-code mode)))
  (#:bjo  (clr-line-opt-await p (line-mode-code mode))))

(: (-> InputPort char))
(defbjouble (read-char p)
  (#:sync (clr-char p))
  (#:bjo  (clr-char-await p)))

;; One read, for the reason `read-line/opt` is.
(: (-> InputPort (Option char)))
(defbjouble (read-char/opt p)
  (#:sync (clr-char-opt p))
  (#:bjo  (clr-char-opt-await p)))

;; The character `read-char` would answer, left in the port, as Racket's
;; `peek-char`. Decoded from the port's buffer: it waits for the bytes a
;; character needs and takes none of them.
(: (-> InputPort char))
(defbjouble (peek-char p)
  (#:sync (clr-peek-char p))
  (#:bjo  (clr-peek-char-await p)))

(: (-> InputPort (Option char)))
(defbjouble (peek-char/opt p)
  (#:sync (clr-peek-char-opt p))
  (#:bjo  (clr-peek-char-opt-await p)))

;; Drains a port
(: (-> InputPort string))
(defbjouble (read-all p)
  (#:sync (clr-rest p))
  (#:bjo  (clr-rest-await p)))

;; Port fold.
(: (-> (-?-> InputPort %a %a) %a InputPort %a))
(defun (port-fold f init port)
  (let loop ((acc init))
    (if (port-eof? port)
        acc
        (loop (f port acc)))))

;; --- Output ---

;; Port write operations provide both synchronous (`#:sync`) and suspending (`#:async`)
;; implementations via `defbjouble`.
;;
;; Writes are buffered in memory and non-blocking until the buffer fills. When a buffer
;; drains (triggering a syscall), the async variant suspends the coroutine rather than
;; blocking the host thread.
;;
;; In contrast, pure string formatters (`written` and `displayed`) output to a
;; thread-local `StringBuilder`, remaining strictly synchronous and side-effect free.
(import/extern
  (clr-write         (: Bjolang.Runtime.BjoOutputPort.WriteText          (-> OutputPort string void) #:blocking))
  (clr-write-await   (: Bjolang.Runtime.BjoOutputPort.WriteTextAsync     (-> OutputPort string void) #:async))
  (clr-writec        (: Bjolang.Runtime.BjoOutputPort.WriteScalar        (-> OutputPort char void)   #:blocking))
  (clr-writec-await  (: Bjolang.Runtime.BjoOutputPort.WriteScalarAsync   (-> OutputPort char void)   #:async))
  (clr-writeln       (: Bjolang.Runtime.BjoOutputPort.WriteTextLine      (-> OutputPort string void) #:blocking))
  (clr-writeln-await (: Bjolang.Runtime.BjoOutputPort.WriteTextLineAsync (-> OutputPort string void) #:async)))

(: (-> OutputPort string void))
(defbjouble (write-string p s)
  (#:sync (clr-write p s))
  (#:bjo  (clr-write-await p s)))

;; Write char.
(: (-> OutputPort char void))
(defbjouble (write-char p c)
  (#:sync (clr-writec p c))
  (#:bjo  (clr-writec-await p c)))

(: (-> OutputPort string void))
(defbjouble (writeln p s)
  (#:sync (clr-writeln p s))
  (#:bjo  (clr-writeln-await p s)))

;; What a string port has been given, as text. Bytes written to it with
;; `write-bytes!` that are not UTF-8 read as U+FFFD, as Racket's do.
(: (-> OutputPort string))
(defun (get-output-string p) (writer->string p))

;; What a string port has been given, as bytes.
(: (-> OutputPort (Array byte)))
(defun (get-output-bytes p) (output-bytes p))


;; ---------------------------------------------------------------------------
;; Effects
;; ---------------------------------------------------------------------------
;;
;; An operation is a parameter holding a function, and performing an effect
;; calls that function on the performer's own stack.
;; (std effect) has the whole of the mechanism; what is here is the handful
;; of operations the standard library performs itself.
;;
;; What is an effect and what is a parameter?
;; If the thing is a value the caller reads, it is a parameter. If it is a
;; behaviour the caller decides - with a default worth naming, a possible
;; abort, and a signature worth checking - it is an effect. So errors stay
;; try/raise, state stays parameterize and def/mutable, the three
;; standard ports stay parameters, and random stays the parameter
;; (std random) already has: it is swappable, it has no abort case, and an
;; effect would put a wrapper function over a read that is already one.
;; The clock is a parameter for the same reason.
;;
;; Installing a handler needs (import (std effect)) for with-handler. A
;; macro is not a binding, so it cannot be re-exported from here.

(import (std effect))

;; `(warn "...")` - something is wrong, and carrying on is the default.
;;
;; Warn continues by default, but this is overridable.
;;
;; (log "...") - a line of diagnostics, and nobody's decision.
;;
;; Both are synchronous. A diagnostic that suspends is a promise answered here,
;; and acted on wherever it is awaited.
(defeffect Diagnostics
  (warn (-> string void))
  (log (-> string void))
  #:default ((warn (fun (s) (writeln (parameter-ref current-error-port) s)))
             (log (fun (s) (writeln (parameter-ref current-error-port) s)))))

;; (getenv "HOME") - the process environment, as an effect so that a test can
;; answer for it without writing to the real one.
;;
;; Cold: read at startup, not in a loop.
(defeffect Env
  (getenv (-> string (Option string)))
  #:default ((getenv get-environment-variable))
  #:cold (getenv))


;; ---------------------------------------------------------------------------
;; The clock
;; ---------------------------------------------------------------------------
;;
;; A `Clock` is a record of functions, so a program can make its own.
;; `monotonic-ms` and `monotonic-ns` read it here, and (std datetime)'s `now`
;; and `today` read the same one, so binding `current-clock` moves both.
;;
;; - `read-monotonic-ms`: milliseconds from an unspecified start. Never
;;   decreases; only differences mean anything.
;; - `read-monotonic-ns`: the same clock in nanoseconds, at the resolution
;;   the clock has. A clock made with only `#:monotonic-ms` reads it as its
;;   milliseconds times a million, so a fake clock gives the same exact
;;   elapsed time in both units.
;; - `read-utc-ticks`: the wall clock, in .NET ticks since 0001-01-01 UTC. May
;;   step back.
;; - `read-zone`: the zone (std datetime)'s `today` is in.
;; - `advance-ticks`: what (std datetime)'s `clock-advance!` calls, with a
;;   number of ticks. `None` for a clock that cannot be moved.
(type (: Clock (Record (: read-monotonic-ms (-> long))
                       (: read-monotonic-ns (-> long))
                       (: read-utc-ticks (-> long))
                       (: read-zone (-> System.TimeZoneInfo))
                       (: advance-ticks (Option (-> long void))))))

(import/extern
  (clock-ms (: BjolangRuntime.ClockMs (-> long)))
  (clock-ns (: BjolangRuntime.ClockNs (-> long)))
  (clr-utc-now (: System.DateTime.UtcNow #:get))
  (clr-local-zone (: System.TimeZoneInfo.Local #:get)))

(: (-> long))
(defun (system-utc-ticks) (.-Ticks clr-utc-now))

(: (-> System.TimeZoneInfo))
(defun (system-zone) clr-local-zone)

;; Each part not given is the system clock's, but for the nanoseconds: when
;; `#:monotonic-ns` is not given, they are read from the milliseconds, so that
;; a clock given only `#:monotonic-ms` agrees with itself.
(: (-> (#:monotonic-ms (-> long))
       (#:monotonic-ns (Option (-> long)))
       (#:utc-ticks (-> long))
       (#:zone (-> System.TimeZoneInfo))
       (#:advance (Option (-> long void)))
       Clock))
(defun (make-clock #:monotonic-ms clock-ms #:monotonic-ns None #:utc-ticks system-utc-ticks
                   #:zone system-zone #:advance None)
  (Clock (read-monotonic-ms monotonic-ms)
         (read-monotonic-ns (match monotonic-ns
                              ((Some ns) ns)
                              (None (fun () (* (monotonic-ms) 1000000L)))))
         (read-utc-ticks utc-ticks)
         (read-zone zone)
         (advance-ticks advance)))

;; The machine's clock, in the machine's zone.
(: Clock)
(def system-clock (make-clock #:monotonic-ns (Some clock-ns)))

;; Cold: nothing reads the time in a loop.
(: (Param Clock))
(def current-clock (make-cold-parameter system-clock))

;; Not called `now`: that is (std datetime)'s wall-clock `Instant`.
(: (-> (#:clock Clock) long))
(defun (monotonic-ms #:clock (parameter-ref current-clock))
  ((record-ref clock read-monotonic-ms)))

;; Nanoseconds on the same clock: for timing what takes less than a
;; millisecond.
(: (-> (#:clock Clock) long))
(defun (monotonic-ns #:clock (parameter-ref current-clock))
  ((record-ref clock read-monotonic-ns)))


;; ---------------------------------------------------------------------------
;; Iterating something other than the whole of a collection
;; ---------------------------------------------------------------------------
;;
;; `in-array`, `in-vec` and their reverses, `over-list`, `in-port`, `up-from`
;; and `down-from`. Included rather than imported: they are `Iterable`
;; implementations, and an implementation belongs with the trait — a module of
;; its own would have to import the prelude it is part of. What is above stays
;; here: the walk of a whole collection is what the collection itself is for.
;;
;; Here rather than beside those implementations because `in-port` names
;; `InputPort`, which the section above introduces.
(include "iterators.protobjo")


;; ---------------------------------------------------------------------------
;; Collection
;; ---------------------------------------------------------------------------
;;
;; Collection trait.
(def/trait (Collection %a)
  (type %content)
  (: length (-> %a int))
  (: empty? (-> %a bool)))

(impl (Collection (List %a))
  (type %content %a)
  (defun (length lst)
    (list-length lst))
  (defun (empty? lst)
    (list-empty? lst)))

(impl (Collection (Array %a))
  (type %content %a)
  (defun (length arr)
    (array-length arr))
  (defun (empty? arr)
    (= 0 (array-length arr))))

(impl (Collection (IArray %a))
  (type %content %a)
  (defun (length xs)
    (iarray-length xs))
  (defun (empty? xs)
    (iarray-empty? xs)))

(impl (Collection (Vec %a))
  (type %content %a)
  (defun (length vec)
    (vec-length vec))
  (defun (empty? vec)
    (vec-empty? vec)))

(impl (Collection (VecBuilder %a))
  (type %content %a)
  (defun (length vb)
    (vecbuilder-length vb))
  (defun (empty? vb)
    (= (vecbuilder-length vb) 0)))

(impl (Collection (TransientVec %a))
  (type %content %a)
  (defun (length t)
    (transientvec-length t))
  (defun (empty? t)
    (transientvec-empty? t)))

(impl (Collection (ListBuilder %a))
  (type %content %a)
  (defun (length lb)
    (listbuilder-length lb))
  (defun (empty? lb)
    (= (listbuilder-length lb) 0)))

;; Map collection.
(impl (Collection (Map %k %v))
  (type %content (Tuple %k %v))
  (defun (length m)
    (map-length m))
  (defun (empty? m)
    (map-empty? m)))

(impl (Collection (TransientMap %k %v))
  (type %content (%k . %v))
  (defun (length tm)
    (transientmap-length tm))
  (defun (empty? tm)
    (transientmap-empty? tm)))
(impl (Collection (MapBuilder %k %v))
  (type %content (%k . %v))
  (defun (length mb)
    (mapbuilder-length mb))
  (defun (empty? mb)
    (= (mapbuilder-length mb) 0)))

;; Range collection.
(impl (Collection Range)
  (type %content int)
  (defun (length r)
    (def lo (record-ref r lo))
    (def hi (record-ref r hi))
    (if (<= hi lo)
        0
        (range-length-of (- (cast long hi) (cast long lo)))))
  (defun (empty? r)
    (>= (record-ref r lo) (record-ref r hi))))

(impl (Collection RangeBy)
  (type %content int)
  (defun (length r)
    (def lo (record-ref r lo))
    (def hi (record-ref r hi))
    (if (<= hi lo)
        0
        (range-length-of (range-by-count lo hi (record-ref r by)))))
  (defun (empty? r)
    (>= (record-ref r lo) (record-ref r hi))))

;; A range from below zero to above it can hold more than `int-max` values,
;; which a length cannot give.
(: (-> long int))
(defun (range-length-of n)
  (when (> n (cast long int-max))
    (panic! #"length: the range holds ${n} values, more than an int can count."))
  (cast int n))

;; String collection.
(impl (Collection string)
  (type %content char)
  (defun (length s)
    (string-count s))
  (defun (empty? s)
    (string-empty? s)))

;; Seq collection.
(impl (Collection (Seq %a))
  (type %content %a)
  (defun (length s)
    (count s))
  (defun (empty? s)
    (seq-empty? s)))


;;----------------------------------------------------------------------------
;; Addable! trait
;; ---------------------------------------------------------------------------
;;
;; `add!` never fails. Afterwards the sink holds the element: a sequence has
;; it appended, a set has it as a member (which it may already have been),
;; and a map has the key mapped to the value, replacing an older one. An
;; operation that refuses what is already there is named `add-new` and is
;; not part of the trait.
(def/trait (Addable! %coll)
  (type %elem)
  (: add! (-> %coll %elem void))
  ;; Everything an `Iterable` holds, added in the order it yields. The
  ;; `(where ...)` is member-level: each call site brings its own source type,
  ;; with `#:elem` pinned to this trait's `%elem` — the element the source
  ;; yields *is* the element the sink takes, which no function-level clause
  ;; can say. The cursor is nobody's business, so it is pinned to a fresh
  ;; variable.
  (: add-all! (-> %coll %s void)
     (where (Iterable %s #:elem %elem #:cursor %k)))
  (defun (add-all! coll xs)
    (let go ((cur (iterable-start xs)))
      (unless (iterable-done? xs cur)
        (add! coll (iterable-current xs cur))
        (go (iterable-next xs cur))))))

(impl (Addable! StringBuilder)
  (type %elem Char)
  (defun (add! sb s)
    (stringbuilder-add! sb s)))

(impl (Addable! (VecBuilder %a))
  (type %elem %a)
  (defun (add! vb elem)
    (vecbuilder-add! vb elem)))

(impl (Addable! (TransientVec %a))
  (type %elem %a)
  (defun (add! t elem)
    (transientvec-add! t elem)))

(impl (Addable! (TransientMap %k %v))
  (type %elem (%k . %v))
  (defun (add! tm tup)
    (let (((k v) tup))
      (transientmap-set! tm k v))))

(impl (Addable! (MapBuilder %k %v))
  (type %elem (%k . %v))
  (defun (add! mb tup)
    (let (((k v) tup))
      (mapbuilder-add! mb k v))))

(impl (Addable! (ListBuilder %a))
  (type %elem %a)
  (defun (add! lb elem)
    (listbuilder-add! lb elem)))

;; The set and ordered-collection sinks live with their types, in (std hashset),
;; (std orderedset) and (std orderedmap): `Addable!` is this module's trait,
;; so a module that owns the type may implement it.

;;----------------------------------------------------------------------------
;; Addable — the persistent counterpart
;; ---------------------------------------------------------------------------
;;
;; The same shape without the `!`: `add` answers a new collection and touches
;; nothing. It never fails, as `add!` never does: a map-shaped collection
;; replaces a key already there, and a set already holding the element
;; answers an equal set. A caller who wants a duplicate key refused writes
;; `map-add-new`, which raises, or `map-try-add-new`, which answers a Result.
(def/trait (Addable %coll)
  (type %elem)
  (: add (-> %coll %elem %coll))
  (: add-all (-> %coll %s %coll)
     (where (Iterable %s #:elem %elem #:cursor %k)))
  ;; A mutable cell rather than a named-let accumulator, deliberately: the
  ;; cell is initialised from `coll`, so its type is the implementor's from
  ;; the first line, and `(add acc e)` resolves before the loop is even
  ;; checked. A let-bound accumulator is a fresh variable at the moment the
  ;; recursive call is inferred, which leaves `add`'s element projection
  ;; nothing to resolve against.
  (defun (add-all coll xs)
    (def/mutable acc coll)
    (let go ((cur (iterable-start xs)))
      (unless (iterable-done? xs cur)
        (set! acc (add acc (iterable-current xs cur)))
        (go (iterable-next xs cur))))
    acc))

(impl (Addable (Vec %a))
  (type %elem %a)
  (defun (add v elem)
    (vec-add v elem)))

(impl (Addable (Map %k %v))
  (type %elem (%k . %v))
  (defun (add m tup)
    (let (((k v) tup))
      (map-set m k v))))

;; As with the sinks, the set and ordered-collection impls live with their
;; types, in (std hashset), (std orderedset) and (std orderedmap).



;; ---------------------------------------------------------------------------
;; Control
;; ---------------------------------------------------------------------------
;;
;; Macros.
(import (std syntax-match))

;; Several forms as one.
(: (-> (List Syntax) Syntax Syntax))
(defun (one-form body whole)
  (match body
    (Nil (syntax-error whole "This clause has nothing to do."))
    ((Cons only Nil) only)
    (_ #'(let () ,@body))))

;; `(if-let ((name expr)) then else)`
;;
;; `name` bound to the `Some` value of `expr`, or `else` on a `None`.
(def/macro (if-let form inject compare)
  (syntax-match form
    ((_ (name value) then else)
     #'(match ,value ((Some ,name) ,then) (None ,else)))
    (bad (syntax-error bad "if-let takes one binding, a then and an else: (if-let (name expr) then else)"))))

;; `(when-let (name expr) body ...)`
;; The body run with `name` bound to a `Some` value, and nothing on `None`.
(def/macro (when-let form inject compare)
  (syntax-match form
    ((_ (name value) first body ...)
     ;; Evaluate value once.
     #'(let ((found ,value))
         (when (some? found)
           (let ((,name (option-ref found)))
             ,first
             ,@body))))
    (bad (syntax-error bad "when-let takes one binding and a body: (when-let (name expr) body ...)"))))

;; `(some-> value step ...)` — `->`, stopping at the first `None`.
;;
;; Threading option.
(def/macro (some-> form inject compare)
  (syntax-match form
    ((_ value step ...) (some-steps value step))
    (bad (syntax-error bad "some-> takes a value and steps: (some-> value (f &) (g &))"))))

(: (-> Syntax (List Syntax) Syntax))
(defun (some-steps value steps)
  (match steps
    ;; No steps.
    (Nil #'(Some ,value))
    ;; Last step.
    ((Cons step Nil) #'(-> ,value ,step))
    ((Cons step rest)
     ;; Step binding.
     #'(match (-> ,value ,step)
        (None None)
        ((Some it) ,(some-steps #'it rest))))))

;; `(try-> value step ...)` — `->`, stopping at the first `Err`.
;;
;; Threading result. Same as some-> but for Result.
(def/macro (try-> form inject compare)
  (syntax-match form
    ((_ value step ...) (try-steps value step))
    (bad (syntax-error bad "try-> takes a value and steps: (try-> value (f &) (g &))"))))

(: (-> Syntax (List Syntax) Syntax))
(defun (try-steps value steps)
  (match steps
    ;; No steps.
    (Nil #'(Ok ,value))
    ;; Last step.
    ((Cons step Nil) #'(-> ,value ,step))
    ((Cons step rest)
     ;; The error is rebuilt rather than passed along, because the two Results
     ;; differ in their Ok type.
     #'(match (-> ,value ,step)
        ((Err e) (Err e))
        ((Ok it) ,(try-steps #'it rest))))))

;; ---------------------------------------------------------------------------
;; Collector macros and the collectors they expand to
;; ---------------------------------------------------------------------------

;; Expands a collector macro's `(:acc [name] (head arg ...) [#:when cond])`.
;;
;; - `build` gets the name (a list of zero or one), the head's arguments and
;;   the condition, and returns the clauses.
;; - `build` returns None for arguments it does not take. That, and a clause
;;   of any other shape, is a syntax error with `usage`.
(: (-> Syntax string (-> (List Syntax) (List Syntax) (Option Syntax) (Option Syntax)) Syntax))
(defun (expand-acc-clause form usage build)
  (def built
    (syntax-match form
      ((_ (_ arg ...)) (build (list) arg None))
      ((_ (_ arg ...) #:when c) (build (list) arg (Some c)))
      ((_ name (_ arg ...)) (build (list name) arg None))
      ((_ name (_ arg ...) #:when c) (build (list name) arg (Some c)))
      (_ None)))
  (match built
    ((Some clauses) clauses)
    (None (syntax-error form usage))))

;; `(:acc ,@name ,collector [#:when ,cond])`.
(: (-> (List Syntax) Syntax (Option Syntax) Syntax))
(defun (acc-form name collector when)
  (match when
    ((Some c) #'(:acc ,@name ,collector #:when ,c))
    (None #'(:acc ,@name ,collector))))

;; `test`, or `(if ,cond ,test ,otherwise)` with a condition.
(: (-> (Option Syntax) Syntax Syntax Syntax))
(defun (acc-gated when test otherwise)
  (match when
    ((Some c) #'(if ,c ,test ,otherwise))
    (None test)))

;; `(:acc found (someing expr))`: the first `(Some v)` that `expr`, an
;; Option, evaluates to, which ends the loop, or `None` when the loop ends
;; first.
;;
;; - A `folding` of `expr`, and a `:finish` after it. Every round before the
;;   last gave `None`, so the fold needs no condition.
;; - `expr` is evaluated once a round. With `#:when cond`, a round `cond` does
;;   not hold for does not evaluate it.
(def/acc-clause (someing form inject compare)
  (expand-acc-clause form "someing takes one Option: (:acc found (someing (map-try-ref m k))), or {someing (map-try-ref m k) (:for k ks)}"
    (fun (name args when)
      (match args
        ((Cons expr Nil)
         (Some #'(begin (:let got ,(acc-gated when expr #'None))
                        (:acc ,@name (folding None got))
                        (:finish (some? got)))))
        (_ None)))))

;; `(:acc found (finding x))`: `(Some x)` for the first round that reaches it,
;; which ends the loop, or `None` when the loop ends first.
;;
;; - `(someing (Some x))`.
;; - With `#:when cond`, `cond` is evaluated once, and gates both: a round it
;;   does not hold for neither finds nor finishes.
(def/acc-clause (finding form inject compare)
  (expand-acc-clause form "finding takes one expression: (:acc found (finding x)), or {finding x (:for x xs)}"
    (fun (name args when)
      (match args
        ((Cons x Nil) (Some (acc-form name #'(someing (Some ,x)) when)))
        (_ None)))))

;; `(:acc all? (anding test))`: whether `test` holds in every round. The
;; first round it does not hold in ends the loop. #t for no rounds.
;;
;; - With `#:when cond`, a round `cond` does not hold for is not tested.
(def/acc-clause (anding form inject compare)
  (expand-acc-clause form "anding takes one test: (:acc all? (anding (even? x))), or {anding (even? x) (:for x xs)}"
    (fun (name args when)
      (match args
        ((Cons test Nil)
         (Some #'(begin (:let ok? ,(acc-gated when test #'#t))
                        (:acc ,@name (folding #t ok?))
                        (:finish (not ok?)))))
        (_ None)))))

;; `(:acc any? (oring test))`: whether `test` holds in some round. The first
;; round it holds in ends the loop. #f for no rounds.
;;
;; - With `#:when cond`, a round `cond` does not hold for is not tested.
(def/acc-clause (oring form inject compare)
  (expand-acc-clause form "oring takes one test: (:acc any? (oring (even? x))), or {oring (even? x) (:for x xs)}"
    (fun (name args when)
      (match args
        ((Cons test Nil)
         (Some #'(begin (:let ok? ,(acc-gated when test #'#f))
                        (:acc ,@name (folding #f ok?))
                        (:finish ok?))))
        (_ None)))))

;; `(:acc firsts (taking n x))`: a list of the first `n` values, in order. The
;; round that takes the `n`th ends the loop.
;;
;; - `n` is evaluated each round, before the value.
;; - With `#:when cond`, only the rounds `cond` holds for are taken.
;; - An `n` of 0 or less takes nothing, but still ends the loop only after
;;   its first round.
(def/acc-clause (taking form inject compare)
  (expand-acc-clause form "taking takes a count and an expression: (:acc firsts (taking 3 x)), or {taking 3 x (:for x xs)}"
    (fun (name args when)
      (def acc (match name ((Cons n Nil) n) (_ #'taken)))
      (match args
        ((Cons n (Cons x Nil))
         (Some #'(begin (:let limit ,n)
                        ,(acc-form (list acc) #'(listing ,x)
                                   (Some (acc-gated when #'(< (length ,acc) limit) #'#f)))
                        (:finish (>= (length ,acc) limit)))))
        (_ None)))))

;; `(:acc best (maximizing-by key x))`: `(Some x)` of the greatest `key`, or
;; `None` for no rounds. Of equal keys the first is kept.
;;
;; - `key` is evaluated once a round, and is compared by Ord.
;; - `maximizing/pairs` of `(Tuple key x)`.
(def/acc-clause (maximizing-by form inject compare)
  (expand-acc-clause form "maximizing-by takes a key and an expression: (:acc best (maximizing-by (score x) x))"
    (fun (name args when)
      (match args
        ((Cons key (Cons x Nil)) (Some (acc-form name #'(maximizing/pairs (Tuple ,key ,x)) when)))
        (_ None)))))

;; `(:acc best (minimizing-by key x))`: `(Some x)` of the least `key`, or
;; `None` for no rounds. Of equal keys the first is kept.
;;
;; - `key` is evaluated once a round, and is compared by Ord.
;; - `minimizing/pairs` of `(Tuple key x)`.
(def/acc-clause (minimizing-by form inject compare)
  (expand-acc-clause form "minimizing-by takes a key and an expression: (:acc best (minimizing-by (cost x) x))"
    (fun (name args when)
      (match args
        ((Cons key (Cons x Nil)) (Some (acc-form name #'(minimizing/pairs (Tuple ,key ,x)) when)))
        (_ None)))))

;; `(:acc groups (grouping key x))`: a map from each `key` to the list of its
;; `x`s, in the order they came.
;;
;; - `key` is evaluated once a round, and is compared by Eq.
;; - `grouping/pairs` of `(Tuple key x)`.
(def/acc-clause (grouping form inject compare)
  (expand-acc-clause form "grouping takes a key and an expression: (:acc groups (grouping (string-length w) w))"
    (fun (name args when)
      (match args
        ((Cons key (Cons x Nil)) (Some (acc-form name #'(grouping/pairs (Tuple ,key ,x)) when)))
        (_ None)))))

;; The greatest or least value by Ord, or None. Of equal ones the first is kept.
(type (: (Maximizing %a) (Union maximizing)))

(impl (Collector (Maximizing %a))
  (where (Ord %a))
  (type %elem %a)
  (type %acc (Option %a))
  (type %out (Option %a))
  (defun (collector-init c) None)
  (defun (collector-step c acc x)
    (match acc
      ((Some best) (if (greater? x best) (Some x) acc))
      (None (Some x))))
  (defun (collector-finish c acc) acc))

(type (: (Minimizing %a) (Union minimizing)))

(impl (Collector (Minimizing %a))
  (where (Ord %a))
  (type %elem %a)
  (type %acc (Option %a))
  (type %out (Option %a))
  (defun (collector-init c) None)
  (defun (collector-step c acc x)
    (match acc
      ((Some best) (if (less? x best) (Some x) acc))
      (None (Some x))))
  (defun (collector-finish c acc) acc))

;; Of `(Tuple key value)`s: the value of the greatest or least key by Ord, or
;; None. Of equal keys the first is kept.
(type (: (MaximizingPairs %k %v) (Union maximizing/pairs)))

(impl (Collector (MaximizingPairs %k %v))
  (where (Ord %k))
  (type %elem (Tuple %k %v))
  (type %acc (Option (Tuple %k %v)))
  (type %out (Option %v))
  (defun (collector-init c) None)
  (defun (collector-step c acc kv)
    (match acc
      ((Some best) (if (greater? (fst kv) (fst best)) (Some kv) acc))
      (None (Some kv))))
  (defun (collector-finish c acc)
    (match acc
      ((Some (Tuple _ v)) (Some v))
      (None None))))

(type (: (MinimizingPairs %k %v) (Union minimizing/pairs)))

(impl (Collector (MinimizingPairs %k %v))
  (where (Ord %k))
  (type %elem (Tuple %k %v))
  (type %acc (Option (Tuple %k %v)))
  (type %out (Option %v))
  (defun (collector-init c) None)
  (defun (collector-step c acc kv)
    (match acc
      ((Some best) (if (less? (fst kv) (fst best)) (Some kv) acc))
      (None (Some kv))))
  (defun (collector-finish c acc)
    (match acc
      ((Some (Tuple _ v)) (Some v))
      (None None))))

;; A map from each value to how many times it came.
(type (: (Tallying %a) (Union tallying)))

(impl (Collector (Tallying %a))
  (where (Eq %a))
  (type %elem %a)
  (type %acc (TransientMap %a int))
  (type %out (Map %a int))
  (defun (collector-init c) (transientmap-empty))
  (defun (collector-step c acc x)
    (transientmap-set! acc x (+ (transientmap-ref-or acc x 0) 1))
    acc)
  (defun (collector-finish c acc) (transientmap->map acc)))

;; Of `(Tuple key value)`s: a map from each key to the list of its values, in
;; the order they came.
(type (: (GroupingPairs %k %v) (Union grouping/pairs)))

(impl (Collector (GroupingPairs %k %v))
  (where (Eq %k))
  (type %elem (Tuple %k %v))
  (type %acc (TransientMap %k (ListBuilder %v)))
  (type %out (Map %k (List %v)))
  (defun (collector-init c) (transientmap-empty))
  (defun (collector-step c acc kv)
    (def (k v) kv)
    (match (transientmap-try-ref acc k)
      ((Some b) (listbuilder-add! b v))
      (None
       (def b (listbuilder-empty))
       (listbuilder-add! b v)
       (transientmap-set! acc k b)))
    acc)
  (defun (collector-finish c acc)
    (map-map-values listbuilder->list (transientmap->map acc))))

;; ---------------------------------------------------------------------------
;; Paths: get, try-get, set, update and set!
;; ---------------------------------------------------------------------------
;;
;; Path navigation for nested data structures:
;; - `(get target step ...)`: Reads a nested value along a path.
;; - `(set target step ... value)`: Returns a copy of `target` with the nested value
;;   replaced (leaving the original untouched).
;; - `(update target step ... f)`: Returns a copy of `target` with the nested value
;;   updated via `(f current-value)`.
;;
;; Steps are resolved purely by syntactic form before type checking:
;; - Field step (`.name`): Accesses record/struct fields or .NET members via `(.name x)`
;;   and writes via `record-set`.
;; - Key step (any other expression): Accesses keyed collections (`Refable`) via `ref`
;;   and writes (`Settable`) via `ref-set`.
;;
;; Semantics:
;; - `try-get`: Returns `(Some value)` or `None` if any key in the path is missing
;;   (evaluating keys lazily from left to right and stopping at the first `None`).
;; - `set` and `update`: Evaluate target, path keys (left-to-right), and value/fn
;;   exactly once. Missing intermediate keys raise an error; `set` on the final key
;;   adds it if absent on maps.
;; - `set!`: In-place mutation along the path via `ref-set!` (`Settable!`) or `record-set!`.

;; `(get target step ...)`, and `(get target)`, which is `(deref target)`.
(def/macro (get form inject compare)
  (syntax-match form
    ((_ target)
     #'(deref ,target))
    ((_ target first rest ...)
     (path-reads target (Cons first rest)))
    (bad (syntax-error bad "get needs a target: (get target step ...), where a step is .field or a key, or (get target) to read a Box or a Param."))))

;; `(try-get target step ...)`
(def/macro (try-get form inject compare)
  (syntax-match form
    ((_ target first rest ...)
     (path-tries target (Cons first rest)))
    (bad (syntax-error bad "try-get reads along a path and needs at least one step: (try-get target step ...), where a step is .field or a key."))))

;; `(set target step ... value)`
(def/macro (set form inject compare)
  (syntax-match form
    ((_ target first second rest ...)
     (path-write inject target (Cons first (Cons second rest)) #f))
    (bad (syntax-error bad "set needs a target, at least one step and a value: (set target step ... value)."))))

;; `(update target step ... f)`
(def/macro (update form inject compare)
  (syntax-match form
    ((_ target first second rest ...)
     (path-write inject target (Cons first (Cons second rest)) #t))
    (bad (syntax-error bad "update needs a target, at least one step and a function: (update target step ... f)."))))

;; True for a field step: an identifier written with a leading dot. `...` and a
;; lone `.` are punctuation, not identifiers, so they never get here.
(: (-> Syntax bool))
(defun (path-dotted? step)
  (match step
    ((SSym s)
     (let ((name (symbol->string s)))
       (and (> (string-length name) 1) (string-starts-with? name "."))))
    (_ #f)))

;; A `.-Name` step, which reads a .NET property.
(: (-> Syntax bool))
(defun (path-property? step)
  (match step
    ((SSym s) (string-starts-with? (symbol->string s) ".-"))
    (_ #f)))

;; Rejects a step that cannot be read: punctuation such as `.` or `...`.
(: (-> Syntax Syntax))
(defun (path-check step)
  (match step
    ((SPunct p)
     (syntax-error step (str "'" p "' is not a path step. A step is .field or a key expression.")))
    (_ step)))

;; One step's read of `acc`.
(: (-> Syntax Syntax Syntax))
(defun (path-read acc step)
  (syntax-at
    (if (path-dotted? (path-check step))
        #'(,step ,acc)
        #'(ref ,acc ,step))
    step))

(: (-> Syntax (List Syntax) Syntax))
(defun (path-reads acc steps)
  (match steps
    (Nil acc)
    ((Cons step rest) (path-reads (path-read acc step) rest))))

;; Every key read is a `try-ref`, and the first `None` is the answer. The last
;; key's Option is already the answer, so it is not taken apart and rebuilt.
(: (-> Syntax (List Syntax) Syntax))
(defun (path-tries acc steps)
  (match steps
    (Nil #'(Some ,acc))
    ((Cons step rest)
     ;; cond is not available here.
     (if (path-dotted? (path-check step))
         (path-tries (path-read acc step) rest)
         (let ((probe (syntax-at #'(try-ref ,acc ,step) step)))
           (match rest
             (Nil probe)
             (_ #'(match ,probe
                    (None None)
                    ((Some it) ,(path-tries #'it rest))))))))))

;; A name for the expansion's own temporaries. Built from a string so that each
;; depth gets a name of its own: hygiene gives every distinct name one fresh
;; spelling per expansion, so a single template name would shadow itself.
(: (-> string int Syntax))
(defun (path-temp prefix i)
  (SSym (string->symbol (str prefix (displayed i)))))

;; The field a `.name` step writes. A hygiene mark — `.name__37`, when the
;; step was written in another macro's template — is stripped, since a field
;; name is never renamed. Injected, so that this macro's hygiene leaves it
;; alone too.
(: (-> (-> Symbol Syntax) Syntax Syntax))
(defun (path-field inject step)
  (def name
    (match step
      ((SSym s)
       (def dotted (symbol->string s))
       (substring dotted (string-cursor-next dotted (string-cursor-start dotted)) (string-cursor-end dotted)))
      (_ "")))
  (def start (string-cursor-start name))
  (inject
    (string->symbol
      (match (string-search-backward name "__")
        ((Some mark)
         (if (and (string-cursor>? mark start)
                  (path-digits? name (string-cursor-next name (string-cursor-next name mark))))
             (substring name start mark)
             name))
        (None name)))))

;; Is everything in `s` from `c` on an ASCII digit, and is there anything?
(: (-> string StringCursor bool))
(defun (path-digits? s c)
  (and (not (string-cursor-end? s c))
       (let go ((c c))
         (if (string-cursor-end? s c)
             #t
             (let ((ch (string-cursor-ref s c)))
               (and (char>=? ch #\0) (char<=? ch #\9) (go (string-cursor-next s c))))))))

;; `set` and `update`. `items` is the steps followed by the value or function.
;;
;; Everything that is evaluated is bound first, in order — the target, each
;; key, then the value — and the copy is then built from the bound names, so
;; that nothing is evaluated twice or out of order.
(: (-> (-> Symbol Syntax) Syntax (List Syntax) bool Syntax))
(defun (path-write inject target items update?)
  (def reversed (list-reverse items))
  (def value (list-head reversed))
  (def steps (list-reverse (list-tail reversed)))
  (def named (path-key-names steps 1 #t))
  (def obj (path-temp "obj" 0))
  (def new (path-temp "new" 0))
  (def (: leaf (-> Syntax Syntax))
    (if update?
        (fun (old) #'(,new ,old))
        (fun (old) new)))
  (def body (path-body inject obj 0 named steps leaf))
  #'(let ((,obj ,target))
      ,(list-foldr (fun (binding inner) #'(let (,binding) ,inner))
                   #'(let ((,new ,value)) ,body)
                   (path-key-bindings named steps))))

;; The steps, each key replaced by the name it is bound to. A field step is
;; kept. When `copying?`, a .NET property is refused: `set` copies with
;; `record-set`, which has no way to copy a .NET object. `set!` copies nothing
;; and only reads the steps before its last, so it reads a property there.
(: (-> (List Syntax) int bool (List Syntax)))
(defun (path-key-names steps i copying?)
  (match steps
    (Nil Nil)
    ((Cons step rest)
     (if (path-dotted? (path-check step))
         (if (and copying? (path-property? step))
             (Cons (syntax-error step
                     (str (syntax->string step) " is a .NET property. set and update copy along the path with record-set, which only copies a Bjolang record or struct; set the property itself, or build the new value with get."))
                   Nil)
             (Cons step (path-key-names rest (+ i 1) copying?)))
         (Cons (path-temp "key" i) (path-key-names rest (+ i 1) copying?))))))

;; `(key-n expr)` for each key step, in order.
(: (-> (List Syntax) (List Syntax) (List Syntax)))
(defun (path-key-bindings named steps)
  (match named
    (Nil Nil)
    ((Cons name names)
     (def step (list-head steps))
     (def rest (path-key-bindings names (list-tail steps)))
     (if (path-dotted? step)
         rest
         (Cons #'(,name ,step) rest)))))

;; The copy of `obj` with the rest of the path written. `named` and `steps` run
;; side by side: `named` is what the code uses, and `steps` is where each step
;; was written.
(: (-> (-> Symbol Syntax) Syntax int (List Syntax) (List Syntax) (-> Syntax Syntax) Syntax))
(defun (path-body inject obj depth named steps leaf)
  (match named
    (Nil (syntax-error obj "internal error: a path ran out of steps."))
    ((Cons key names)
     (def step (list-head steps))
     (def rest (list-tail steps))
     (def inner (path-temp "obj" (+ depth 1)))
     (def field? (path-dotted? step))
     (def old (if field? #'(,step ,obj) #'(ref ,obj ,key)))
     (def value
       (match rest
         (Nil (leaf old))
         (_ #'(let ((,inner ,old))
                ,(path-body inject inner (+ depth 1) names rest leaf)))))
     (syntax-at
       (if field?
           #'(record-set ,obj (,(path-field inject step) ,value))
           #'(ref-set ,obj ,key ,value))
       step))))

;; `(set! target step ... value)`: the parser hands `set!` with three or more
;; operands here, and keeps `(set! name value)`, which writes a variable.
;;
;; Nothing is copied. The steps before the last are read, as `get` reads them,
;; and the last is written in place: a key with `ref-set!` (`Settable!`), a
;; field with `record-set!`, so the field has to be `#:mutable`. The target,
;; each key from left to right and then the value are each evaluated once,
;; before anything is read, as in `set`.
(def/hash-extend (set! form inject compare)
  (syntax-match form
    ((_ target first second rest ...)
     (path-write! inject target (Cons first (Cons second rest))))
    (bad (syntax-error bad "set! writes a variable, (set! name value), or along a path in place, (set! target step ... value), where a step is .field or a key."))))

(: (-> (-> Symbol Syntax) Syntax (List Syntax) Syntax))
(defun (path-write! inject target items)
  (def reversed (list-reverse items))
  (def value (list-head reversed))
  (def steps (list-reverse (list-tail reversed)))
  (def named (path-key-names steps 1 #f))
  (def obj (path-temp "obj" 0))
  (def new (path-temp "new" 0))
  #'(let ((,obj ,target))
      ,(list-foldr (fun (binding inner) #'(let (,binding) ,inner))
                   #'(let ((,new ,value)) ,(path-store! inject obj named steps new))
                   (path-key-bindings named steps))))

;; The reads of the steps before the last, then the write of the last. The
;; container a field is written in is bound to a name first, because
;; `record-set!` takes the name of a record and not an expression.
(: (-> (-> Symbol Syntax) Syntax (List Syntax) (List Syntax) Syntax Syntax))
(defun (path-store! inject acc named steps new)
  (match named
    (Nil (syntax-error acc "internal error: a path ran out of steps."))
    ((Cons key names)
     (def step (list-head steps))
     (def rest (list-tail steps))
     (def field? (path-dotted? step))
     (match rest
       (Nil
        ;; cond is not available here.
        (syntax-at
          (if (not field?)
              #'(ref-set! ,acc ,key ,new)
              (if (path-property? step)
                  (syntax-error step
                    (str (syntax->string step) " is a .NET property, and set! writes the last step with record-set!, which writes a field of a Bjolang record. Call the property's setter, (.set_Name obj value), instead."))
                  ;; The write carries the step's position, which is also
                  ;; what makes it the writing module's: `record-set!` may
                  ;; write only the fields of a record its own module declared.
                  (let ((dest (path-temp "dest" 0)))
                    #'(let ((,dest ,acc))
                        ,(syntax-at #'(record-set! ,dest (,(path-field inject step) ,new)) step)))))
          step))
       (_
        (def read (syntax-at (if field? #'(,step ,acc) #'(ref ,acc ,key)) step))
        (path-store! inject read names rest new))))))

;; ---------------------------------------------------------------------------
;; Hash macros
;; ---------------------------------------------------------------------------
;;
;; `#name(args...)` calls a hash macro, its own namespace beside the functions:
;; `#map` and `map` are two things. `(def/hash-extend (name form inject compare)
;; body)` writes one; like any macro it cannot be used in the module that
;; defines it, so nothing below uses these four.
;;
;; The parser hands an operator with three or more operands to the first three:
;; `(+ a b c)` is `#fl(+ a b c)` and `(< a b c)` is `#ch(< a b c)`.

;; `#fl(op a b c ...)` — `(op (op a b) c)`, folded left.
(def/hash-extend (fl form inject compare)
  (syntax-match form
    ((_ op a b rest ...) (fold-left-forms op #'(,op ,a ,b) rest))
    (bad (syntax-error bad "#fl takes an operator and at least two operands: #fl(+ a b c)."))))

(: (-> Syntax Syntax (List Syntax) Syntax))
(defun (fold-left-forms op acc rest)
  (match rest
    (Nil acc)
    ((Cons x more) (fold-left-forms op #'(,op ,acc ,x) more))))

;; `#fr(op a b c ...)` — `(op a (op b c))`, folded right.
(def/hash-extend (fr form inject compare)
  (syntax-match form
    ((_ op a b rest ...) (fold-right-forms op a (Cons b rest)))
    (bad (syntax-error bad "#fr takes an operator and at least two operands: #fr(Cons 1 2 Nil)."))))

(: (-> Syntax Syntax (List Syntax) Syntax))
(defun (fold-right-forms op x rest)
  (match rest
    (Nil x)
    ((Cons y more) #'(,op ,x ,(fold-right-forms op y more)))))

;; `#ch(op a b c ...)` — `(op a b)`, and `(op b c)`, and so on, each tested only
;; if the one before it held. A middle operand that is not an atom is bound
;; once, right before its first use. The operands are evaluated left to right:
;; with three or more, a first operand that is not an atom is bound before the
;; second is evaluated, and a name is read before the operand after it is.
;;
;; Three template names for the temporaries. `t` and `u` hold operands and are
;; taken in turn: hygiene gives a template name one fresh spelling per
;; expansion, so a single name would shadow itself and `(op t t)` would compare
;; a value to itself. `s` holds a name's value, and is never needed again once
;; the operand after the name is bound.
(def/hash-extend (ch form inject compare)
  (syntax-match form
    ((_ op a b) #'(,op ,a ,b))
    ((_ op a b rest ...)
     (if (syntax-atom? a)
         (chain-forms op a (syntax-constant? a) (Cons b rest) #t)
         #'(let ((u ,a)) ,(chain-forms op #'u #t (Cons b rest) #t))))
    (bad (syntax-error bad "#ch takes an operator and at least two operands: #ch(< a b c)."))))

;; `prev` is safe to read twice: the first operand, an atom, or a temporary.
;; `fixed` says whether reading it later gives what reading it now would: a
;; constant and a temporary do, and a name does not if the operand after it
;; sets it. So a name is bound before an operand that is not an atom is
;; evaluated. The last comparison needs no binding: a call evaluates its
;; arguments left to right.
;; `use-t` says which temporary is free; the other may be `prev`.
(: (-> Syntax Syntax bool (List Syntax) bool Syntax))
(defun (chain-forms op prev fixed rest use-t)
  (match rest
    (Nil prev)
    ((Cons x Nil) #'(,op ,prev ,x))
    ((Cons x more)
     (if (syntax-atom? x)
         #'(if (,op ,prev ,x) ,(chain-forms op x (syntax-constant? x) more use-t) #f)
         (if (not fixed)
             #'(let ((s ,prev)) ,(chain-forms op #'s #t rest use-t))
             (if use-t
                 #'(let ((t ,x)) (if (,op ,prev t) ,(chain-forms op #'t #t more #f) #f))
                 #'(let ((u ,x)) (if (,op ,prev u) ,(chain-forms op #'u #t more #t) #f))))))))

;; Cheap to evaluate twice: anything but a list.
(: (-> Syntax bool))
(defun (syntax-atom? s)
  (match s
    ((SList _) #f)
    (_ #t)))

;; The same each time it is evaluated: an atom that is not a name.
(: (-> Syntax bool))
(defun (syntax-constant? s)
  (match s
    ((SList _) #f)
    ((SSym _) #f)
    (_ #t)))

;; `#map((k v) ...)` — a `Map`. An entry is `(k v)`, `[k v]` or `(k . v)`.
(def/hash-extend (map form inject compare)
  (syntax-match form
    ((_) #'(map-empty))
    ((_ entry ...) #'(list->map ,(map-entries entry)))))

(: (-> (List Syntax) Syntax))
(defun (map-entries entries)
  (match entries
    (Nil #'Nil)
    ((Cons entry rest)
     (syntax-match entry
       (('vec-literal k v) #'(Cons (Tuple ,k ,v) ,(map-entries rest)))
       (('Tuple k v) #'(Cons (Tuple ,k ,v) ,(map-entries rest)))
       ((k v) #'(Cons (Tuple ,k ,v) ,(map-entries rest)))
       (bad (syntax-error bad "A map entry is (key value), [key value] or (key . value)."))))))

;; ---------------------------------------------------------------------------
;; Cancellation scopes
;; ---------------------------------------------------------------------------
;;
;; A scope owns the fibers started inside it. `(spawn ...)` and `(bjo ...)` put
;; a fiber into the current scope, and the scope does not return until every one
;; of those fibers has finished. So this
;;
;;     (with-cancel (cancel)
;;       (spawn (process a))
;;       (spawn (process b)))
;;
;; waits for both, and the work is over when the form returns.
;;
;; Stopping early is what the cancel thunk is for:
;;
;;     (with-cancel (cancel)
;;       (spawn (worker jobs answers))
;;       (def answer (sync (chan-recv answers)))
;;       (cancel (Requested "got it"))
;;       answer)
;;
;; The cost of waiting first rather than cancelling first: a scope whose children
;; never end, and which never calls `cancel`, hangs here. That is the intended
;; trade. A hang says where it is; a scope that returned while its children ran
;; on says nothing at all.
;;
;; Waiting is the whole of what the four forms do. In a bjoroutine the wait
;; suspends; in ordinary code it parks the thread, as the end of a `defun`
;; `main` does. `scope-form` writes both, with `by-colour`.

;; The shared expansion of `with-cancel`, `with-deadline` and `with-shield`.
;;
;; `opener` is the call that makes the scope, and it is the only difference
;; between the three: a linked scope, a linked scope with a timer, or an
;; unlinked one. Everything after that — install, run, close, restore — is the
;; same, which is the point of writing it once.
;;
;; The two nested `try`s are one mechanism each. The inner one turns a failure in
;; the body into a value, so that the close can be told about it and can cancel
;; the other children before waiting for them. The outer one restores the
;; dynamic binding however the form is left; a fired token left installed would
;; make every later scope look cancelled.
;;
;; `(raise failure)` puts the body's exception back with the stack it arrived
;; with. It is not reached when the close raised first, which is what happens
;; when children failed too — the close reports them together.
(def/macro (scope-form form inject compare)
  (syntax-match form
    ((_ opener (cancel) first body ...)
     #'(let ()
         (def scope ,opener)
         (def saved (scope-push! scope))
         (def ,cancel (scope-canceller scope))
         (try
             (match (try (let () ,first ,@body) #:catch (System.Exception))
               ((Ok answer)
                (by-colour (#:sync (scope-close!/blocking scope None))
                           (#:bjo (scope-close! scope None)))
                answer)
               ((Err failure)
                (by-colour (#:sync (scope-close!/blocking scope (Some failure)))
                           (#:bjo (scope-close! scope (Some failure))))
                (raise failure)))
           #:finally (dyn-restore! saved))))
    (bad (syntax-error bad "scope-form takes an opener, a cancel thunk name and a body. It is what with-scope, with-cancel, with-deadline and with-shield expand to; write one of those."))))

;; `(with-scope body ...)` - a scope with no cancel thunk.
;;
;; The plain form, and the one to put around resources. Everything the other
;; three do about fibers it does too: it waits for what was spawned in it, and
;; it releases what was owned in it, newest first.
;;
;; The cancel thunk is bound to a name the template made up, so nothing in the
;; body can reach it. A scope that wants to stop early says so with
;; `with-cancel`; a scope that only wants ownership says so with this.
(def/macro (with-scope form inject compare)
  (syntax-match form
    ((_ first body ...)
     #'(scope-form (scope-open! 0) (no-cancel) ,first ,@body))
    (bad (syntax-error bad "with-scope takes a body: (with-scope body ...). For a cancel thunk write with-cancel."))))

;; `(with-cancel (cancel) body ...)` — a scope that owns what it starts.
(def/macro (with-cancel form inject compare)
  (syntax-match form
    ((_ (cancel) first body ...)
     #'(scope-form (scope-open! 0) (,cancel) ,first ,@body))
    (bad (syntax-error bad "with-cancel names its cancel thunk and takes a body: (with-cancel (cancel) body ...)"))))

;; `(with-deadline duration body ...)` — the same scope, whose token also fires
;; on its own once `duration` has passed, with `Deadline`.
;;
;; The timer belongs to the scope rather than to a fiber racing it. A watcher
;; fiber cannot work here: the scope waits for its children before it fires its
;; token, and the watcher would be waiting for that token — so each would be
;; waiting for the other.
(def/macro (with-deadline form inject compare)
  (syntax-match form
    ((_ ms first body ...)
     #'(scope-form (deadline-open! (duration->timer-ms ,ms)) (fire) ,first ,@body))
    (bad (syntax-error bad "with-deadline takes a Duration and a body: (with-deadline (seconds 5) body ...)"))))

;; ---------------------------------------------------------------------------
;; The dynamic environment, for another thread
;; ---------------------------------------------------------------------------

;; Opaque: taken by `current-dynamic-environment`, used by
;; `call-with-dynamic-environment`, and nothing else.
(import/class
  (DynamicEnvironment (: Bjolang.Runtime.DynamicEnvironment)))

(import/extern
  (current-dynamic-environment (: Bjolang.Runtime.DynamicEnvironments.Current (-> DynamicEnvironment)))
  (call-with-dynamic-environment (: Bjolang.Runtime.DynamicEnvironments.Call (-> DynamicEnvironment (-> %a) %a))))

;; `(with-shield body ...)` and `(with-shield #:within ms body ...)` — a scope
;; whose token is *not* linked to its parent's.
;;
;; This is how cleanup runs after a cancellation. Every `sync` consults the
;; ambient token, so once a scope has been cancelled every `sync` in it raises —
;; including the ones in the code that is trying to tidy up. Inside a shield the
;; ambient token is a fresh one that has not fired, so a `sync` works again:
;;
;;     (with-shield #:within (seconds 5)
;;       (spawn (flush-buffer a))
;;       (spawn (flush-buffer b)))
;;
;; It is the same object as any other scope with one flag different, and that is
;; deliberate: `(current-cancel)` inside a shield also reads as "not cancelled",
;; so a hand-written `until-cancelled` agrees with what `sync` is doing rather
;; than contradicting it.
;;
;; The deadline is not optional and defaults to thirty seconds. A shield with no
;; deadline is unreachable from outside by construction, which would make the
;; program impossible to kill.
;;
;; A shield is a scope, so it waits for its own children like any other.
(def/macro (with-shield form inject compare)
  (syntax-match form
    ((_ #:within ms first body ...)
     #'(scope-form (shield-open! (duration->timer-ms ,ms)) (fire) ,first ,@body))
    ((_ first body ...)
     #'(with-shield #:within (seconds 30) ,first ,@body))
    (bad (syntax-error bad "with-shield takes a body and an optional deadline: (with-shield #:within (seconds 5) body ...)"))))

;; ---------------------------------------------------------------------------
;; Scope Resource Management
;; ---------------------------------------------------------------------------
;;
;; Scopes manage unmanaged resources (OS handles, file descriptors, network connections)
;; whose lifetimes must be strictly bound rather than left to garbage collection.
;;
;; Cleanup Lifecycle:
;; - Registered cleanup thunks run when the scope exits, after all spawned fibers
;;   have completed.
;; - Cleanup runs in LIFO order (newest first), ensuring wrappers (e.g., buffered ports)
;;   close before their underlying resources (e.g., streams).
;; - Cleanup thunks are synchronous and non-suspending. If a cleanup callback throws,
;;   the exception is recorded and remaining cleanup thunks continue to run.
;;
;; Scope Boundaries & Fibers:
;; - Fibers do NOT introduce their own scopes. A resource opened inside `(spawn ...)`
;;   or `(bjo ...)` belongs to the ambient enclosing scope and outlives the spawned fiber.
;; - For workloads that create many short-lived resources within a long-lived operation,
;;   use nested child scopes (`(with-scope ...)`) per work unit to reclaim resources promptly.
;; - A closing scope refuses new fibers (`spawn`), and a closed scope refuses new resource
;;   registrations (`own!`).

;; `(in-scope s body ...)` - run the body with `s` as the ambient scope.
;;
;; The way to own something in a scope other than the innermost one, and the
;; way back in from a .NET callback, which runs on a foreign thread with no
;; ambient scope at all:
;;
;;     (def home (current-scope))
;;     (on-event (fun (e) (in-scope home (handle e))))
;;
;; This installs a scope; it does not open or close one. Whoever opened `s` is
;; still the one that waits for it.
;;
;; A form and not a keyword argument on `spawn` and `own!`: Bjolang has no
;; rest-keywords, so a `#:scope` could not be forwarded through a wrapper, and
;; the two would drift apart.
(def/macro (in-scope form inject compare)
  (syntax-match form
    ((_ scope first body ...)
     #'(let ()
         (def saved (scope-push! ,scope))
         (try (let () ,first ,@body)
           #:finally (dyn-restore! saved))))
    (bad (syntax-error bad "in-scope takes a scope and a body: (in-scope s body ...)"))))

;; `(own-disposable x)` - register `.Dispose` on the ambient scope, and hand
;; `x` straight back.
;;
;; A form rather than a function so that `.Dispose` is resolved at the call
;; site, the way `with-open` resolves it. A function would need the whole
;; `System.IDisposable` constraint apparatus to say the same thing.
;;
;; `with-open` is the other spelling: that one for a resource whose lifetime is
;; one block, this one for a resource whose lifetime is the scope's.
(def/macro (own-disposable form inject compare)
  (syntax-match form
    ((_ x)
     ;; `unit` after the call, not the call itself: `.Dispose` is a void .NET
     ;; method, so it has no value for the thunk to return.
     #'(let ((it ,x))
         (ignore (own! (fun () (.Dispose it) unit)))
         it))
    (bad (syntax-error bad "own-disposable takes one expression: (own-disposable (StreamReader. path))"))))

;; ---------------------------------------------------------------------------
;; Durations
;; ---------------------------------------------------------------------------
;;
;; A length of time, which is what every timer takes. A number of milliseconds
;; is a number whose unit is in no type: `5000` is easy to misread, and a count
;; that overflows becomes a different deadline without a word. A Duration has
;; its unit, and making one past its range raises.
;;
;; In .NET ticks of 100 ns, within ±long-max ticks, about ±29 227 years. Leaving
;; out long's minimum gives every duration a negation, so each one prints and
;; reads back. (std datetime) adds the arithmetic, the calendar and the text.

(import/class
  (DurationSpan (: System.TimeSpan (-> long DurationSpan)))
  (DurationOverflow (: System.OverflowException (-> string DurationOverflow))))

(import/extern
  (clr-duration-span-ticks (: System.TimeSpan.Ticks (-> DurationSpan long) #:get)))

(type (: Duration #:opaque (Struct (: v DurationSpan))))

(: (-> %a))
(defun (duration-overflow)
  (raise (cast Exception (DurationOverflow. "a duration is outside ±29 227 years (±9 223 372 036 854 775 807 ticks)"))))

;; The duration of `t` ticks. Raises at long's minimum, the one tick count
;; with no negation.
(: (-> long Duration))
(defun (ticks->duration t)
  (when (= t -9223372036854775808L) (duration-overflow))
  (Duration (v (DurationSpan. t))))

(: (-> Duration long))
(defun (duration-ticks d) (clr-duration-span-ticks (struct-ref d v)))

;; n units of `unit` ticks, or a raise past the range of a Duration.
(: (-> long long Duration))
(defun (units->duration n unit)
  (def limit (/ 9223372036854775807L unit))
  (when (or (> n limit) (< n (- limit))) (duration-overflow))
  (Duration (v (DurationSpan. (* n unit)))))

(: (-> long Duration))
(defun (milliseconds n) (units->duration n 10000L))

(: (-> long Duration))
(defun (seconds n) (units->duration n 10000000L))

(: (-> long Duration))
(defun (minutes n) (units->duration n 600000000L))

(: (-> long Duration))
(defun (hours n) (units->duration n 36000000000L))

(impl (Ord Duration)
  (defun (compare a b) (compare (duration-ticks a) (duration-ticks b))))

;; ISO 8601: PT26H30M, PT0.5S, with one leading sign: -PT1H30M.
(impl (Written Duration)
  (defun (written d) (duration->iso (duration-ticks d)))
  (defun (write-into d b) (stringbuilder-add-string! b (written d))))

(: (-> long string))
(defun (duration->iso t)
  (if (= t 0L)
      "PT0S"
      (let ()
        (def a (if (< t 0L) (- t) t))
        (def h (/ a 36000000000L))
        (def m (% (/ a 600000000L) 60L))
        (def s (% (/ a 10000000L) 60L))
        (def f (% a 10000000L))
        (string-append
          (if (< t 0L) "-PT" "PT")
          (string-append
            (if (= h 0L) "" #"${h}H")
            (string-append
              (if (= m 0L) "" #"${m}M")
              (if (and (= s 0L) (= f 0L)) "" #"${s}${(tick-fraction f)}S")))))))

;; ".5" for half a second: up to seven digits, without trailing zeros, and
;; nothing for a whole second.
(: (-> long string))
(defun (tick-fraction f)
  (if (= f 0L)
      ""
      (let trim ((f f) (width 7))
        (if (= (% f 10L) 0L)
            (trim (/ f 10L) (- width 1))
            (let pad ((digits (displayed f)))
              (if (< (string-length digits) width)
                  (pad (string-append "0" digits))
                  (string-append "." digits)))))))

;; Milliseconds for a timer, which counts them in a long. A part of a
;; millisecond counts as a whole one, so that a positive duration does not
;; become zero, which a timer takes as a time already passed.
(: (-> Duration long))
(defun (duration->timer-ms d)
  (def t (duration-ticks d))
  (if (<= t 0L) 0L (+ (/ (- t 1L) 10000L) 1L)))

(import/extern
  (clr-timeout-ms (: BjolangRuntime.timeout (-> long (Event Unit)))))

;; An event that is ready `d` after each sync of it. Zero or less is ready at
;; once.
(: (-> Duration (Event Unit)))
(defun (timeout d) (clr-timeout-ms (duration->timer-ms d)))


;; ---------------------------------------------------------------------------
;; Select
;; ---------------------------------------------------------------------------

;; `(select branch ...)` - sync on the events of the branches, then run the
;; body of the branch whose event committed.
;;
;; - Branches:
;;   - `(event pattern body ...)`: `event` is any `(Event %a)`. When it
;;     commits, its value is matched against `pattern` and `body` runs.
;;   - `(#:timeout duration body ...)`: a branch on `(timeout duration)`.
;;   - `(#:else body ...)`: runs when no other branch can commit at once. The
;;     sync does not wait. It must be the last branch.
;; - Types: each event has its own value type. The bodies have one type, which
;;   is the type of the form.
;; - Order: the events are offered in the order written. When several can
;;   commit at once, the first one wins, as with `choose`.
;; - Bodies run after the sync, in the code that the form is in. In a
;;   bjoroutine a body can suspend. A branch that loses runs no body, and a
;;   `with-nack` in its event fires.
;; - Colour: in a bjoroutine the form suspends, as `sync`. In a `defun` it
;;   parks the thread, as `sync/blocking`.
;; - Errors: a pattern that can fail to match makes the form a non-exhaustive
;;   `match`, which is a compile error.
;;
;; Expansion: `wrap` tags each value with nested `Ok` and `Err`, one `choose`
;; takes the tagged events, and a `match` on the tag runs the body. The
;; compiler emits a `choose` of two to four `wrap`s at a `sync` as one
;; `Cml.WrapChooseOnce` event, so a warm sync of up to four branches
;; allocates nothing.
(def/macro (select form inject compare)
  (syntax-match form
    ((_) (syntax-error form "select takes branches: (select (event pattern body ...) ... (#:timeout duration body ...) (#:else body ...))"))
    ((_ (#:else body ...)) (syntax-error form "select needs a branch with an event, not only (#:else ...). Without one, write the body."))
    ((_ branch ...) (select-form (select-arms branch)))
    (bad (syntax-error bad "select takes branches: (select (event pattern body ...) ... (#:timeout duration body ...) (#:else body ...))"))))

;; One branch of a `select`: its event, its pattern and its body as one form.
(type (: SelectArm (Union (: Arm Syntax Syntax Syntax))))

;; The branches of a `select`, in the order written.
(: (-> (List Syntax) (List SelectArm)))
(defun (select-arms branches)
  (match branches
    (Nil Nil)
    ((Cons branch rest)
     (syntax-match branch
       ((#:else body ...)
        (match rest
          (Nil (list (Arm #'(always unit) (SSym (string->symbol "_")) (one-form body branch))))
          ((Cons after _)
           (list (select-arm-error after "(#:else ...) must be the last branch of a select: it runs only when no branch above it can commit at once.")))))
       ((#:timeout duration body ...)
        (Cons (Arm #'(timeout ,duration) (SSym (string->symbol "_")) (one-form body branch))
              (select-arms rest)))
       ((#:timeout)
        (list (select-arm-error branch "(#:timeout ...) is (#:timeout duration body ...).")))
       ((event pattern body ...)
        (match event
          ((SKey _)
           (list (select-arm-error event "A select branch starts with an event, #:timeout or #:else.")))
          (_ (Cons (Arm event pattern (one-form body branch)) (select-arms rest)))))
       (bad
        (list (select-arm-error bad "A select branch is (event pattern body ...), (#:timeout duration body ...) or (#:else body ...).")))))))

;; Raises the syntax error. Typed as an arm, for the lists above.
(: (-> Syntax string SelectArm))
(defun (select-arm-error at message)
  (def stx (syntax-error at message))
  (Arm stx stx stx))

;; The `match` on the sync of all the arms.
(: (-> (List SelectArm) Syntax))
(defun (select-form arms)
  (def last (- (list-length arms) 1))
  (def event
    (match arms
      ((Cons (Arm only _ _) Nil) only)
      (_ #'(choose ,@(select-wraps arms 0 last)))))
  #'(match (by-colour (#:sync (sync/blocking ,event)) (#:bjo (sync ,event)))
      ,@(select-clauses arms 0 last)))

;; Each arm's event, with its value tagged by `select-tag`.
(: (-> (List SelectArm) int int (List Syntax)))
(defun (select-wraps arms i last)
  (match arms
    (Nil Nil)
    ((Cons (Arm event _ _) rest)
     (Cons #'(wrap ,event (fun (v) ,(select-tag i last #'v)))
           (select-wraps rest (+ i 1) last)))))

;; Each arm's `match` clause: the arm's tag, and a `match` of the value on the
;; arm's pattern. The inner `match` is what a pattern that can fail is
;; reported on, so the message names the event's values, not the tags.
(: (-> (List SelectArm) int int (List Syntax)))
(defun (select-clauses arms i last)
  (match arms
    (Nil Nil)
    ((Cons (Arm _ pattern body) rest)
     (Cons #'(,(select-tag i last #'v) (match v (,pattern ,body)))
           (select-clauses rest (+ i 1) last)))))

;; The tag of arm `i` of `last + 1` arms around `x`, as an expression or a
;; pattern. Arm 0 is `(Ok x)`, arm 1 is `(Err (Ok x))`, and the last arm is
;; `x` inside `last` `Err`s. One arm has no tag.
(: (-> int int Syntax Syntax))
(defun (select-tag i last x)
  (if (= last 0)
      x
      (if (= i 0)
          #'(Ok ,x)
          #'(Err ,(select-tag (- i 1) (- last 1) x)))))


;; ---------------------------------------------------------------------------
;; Container formats
;; ---------------------------------------------------------------------------
;; A container is written with its elements written, and displayed with them
;; displayed, as Scheme's `write` and `display` do: `(written (list "a"))` is
;; `("a")` and `(displayed (list "a"))` is `(a)`. One function lays out each
;; kind of container, and is given the method that adds an element.

;; The elements of `xs` between `open` and `close`, with a space between two.
(: (-> string %s string StringBuilder (-> %a StringBuilder void) void)
   (where (Iterable %s #:elem %a)))
(defun (elements-print-into open xs close b each)
  (stringbuilder-add-string! b open)
  (loop (:with first? #t #f)
        (:for x xs)
        (:do (unless first? (stringbuilder-add-string! b " "))
             (each x b)))
  (stringbuilder-add-string! b close))

;; `(1 2 3)`. No quote, so a nested list is `((1) (2 3))` and the empty list
;; is `()`.
(impl (Written (List %a))
  (where (Written %a))
  (defun (write-into l b) (elements-print-into "(" l ")" b (fun (x b) (write-into x b)))))

(impl (Displayed (List %a))
  (where (Displayed %a))
  (defun (display-into l b) (elements-print-into "(" l ")" b (fun (x b) (display-into x b)))))

;; `[1 2 3]`, as the literal writes it.
(impl (Written (Vec %a))
  (where (Written %a))
  (defun (write-into v b) (elements-print-into "[" v "]" b (fun (x b) (write-into x b)))))

(impl (Displayed (Vec %a))
  (where (Displayed %a))
  (defun (display-into v b) (elements-print-into "[" v "]" b (fun (x b) (display-into x b)))))

;; `#[1 2 3]`, as the literal writes it.
(impl (Written (Array %a))
  (where (Written %a))
  (defun (write-into a b) (elements-print-into "#[" a "]" b (fun (x b) (write-into x b)))))

(impl (Displayed (Array %a))
  (where (Displayed %a))
  (defun (display-into a b) (elements-print-into "#[" a "]" b (fun (x b) (display-into x b)))))

;; `#iarray[1 2 3]`. There is no literal; the name says which collection it is.
(impl (Written (IArray %a))
  (where (Written %a))
  (defun (write-into a b) (elements-print-into "#iarray[" a "]" b (fun (x b) (write-into x b)))))

(impl (Displayed (IArray %a))
  (where (Displayed %a))
  (defun (display-into a b) (elements-print-into "#iarray[" a "]" b (fun (x b) (display-into x b)))))

;; A sequence is not walked: walking it could do anything, or never end.
(impl (Written (Seq %a))
  (defun (written s) "(Seq ...)")
  (defun (write-into s b) (stringbuilder-add-string! b "(Seq ...)")))

;; The entries of `xs` between `open` and `close`, each in parentheses, a key
;; and its value with a space between.
(: (-> string %s string StringBuilder (-> %k StringBuilder void) (-> %v StringBuilder void) void)
   (where (Iterable %s #:elem (Tuple %k %v))))
(defun (entries-print-into open xs close b each-key each-value)
  (stringbuilder-add-string! b open)
  (loop (:with first? #t #f)
        (:for entry xs)
        (:do (def (k v) entry)
             (unless first? (stringbuilder-add-string! b " "))
             (stringbuilder-add-string! b "(")
             (each-key k b)
             (stringbuilder-add-string! b " ")
             (each-value v b)
             (stringbuilder-add-string! b ")")))
  (stringbuilder-add-string! b close))

;; `#map((k v) ...)`, as the `#map` literal writes it.
(impl (Written (Map %k %v))
  (where (Written %k) (Written %v))
  (defun (write-into m b)
    (entries-print-into "#map(" m ")" b (fun (k b) (write-into k b)) (fun (v b) (write-into v b)))))

(impl (Displayed (Map %k %v))
  (where (Displayed %k) (Displayed %v))
  (defun (display-into m b)
    (entries-print-into "#map(" m ")" b (fun (k b) (display-into k b)) (fun (v b) (display-into v b)))))

;; A case of a union the runtime declares, in constructor syntax: `(Some 1)`
;; and `None`, `(Ok 1)` and `(Err "no")`.
(: (-> string %a StringBuilder (-> %a StringBuilder void) void))
(defun (case-print-into name x b each)
  (stringbuilder-add-string! b "(")
  (stringbuilder-add-string! b name)
  (stringbuilder-add-string! b " ")
  (each x b)
  (stringbuilder-add-string! b ")"))

(impl (Written (Option %a))
  (where (Written %a))
  (defun (write-into o b)
    (match o
      ((Some x) (case-print-into "Some" x b (fun (x b) (write-into x b))))
      (None (stringbuilder-add-string! b "None")))))

(impl (Displayed (Option %a))
  (where (Displayed %a))
  (defun (display-into o b)
    (match o
      ((Some x) (case-print-into "Some" x b (fun (x b) (display-into x b))))
      (None (stringbuilder-add-string! b "None")))))

(impl (Written (Result %e %a))
  (where (Written %e) (Written %a))
  (defun (write-into r b)
    (match r
      ((Ok x) (case-print-into "Ok" x b (fun (x b) (write-into x b))))
      ((Err e) (case-print-into "Err" e b (fun (e b) (write-into e b)))))))

(impl (Displayed (Result %e %a))
  (where (Displayed %e) (Displayed %a))
  (defun (display-into r b)
    (match r
      ((Ok x) (case-print-into "Ok" x b (fun (x b) (display-into x b))))
      ((Err e) (case-print-into "Err" e b (fun (e b) (display-into e b)))))))

;; `(Tuple 1 "a")`, as a tuple is built.
(impl (Written (Tuple %a %b))
  (where (Written %a) (Written %b))
  (defun (write-into t b)
    (def (x y) t)
    (stringbuilder-add-string! b "(Tuple ")
    (write-into x b)
    (stringbuilder-add-string! b " ")
    (write-into y b)
    (stringbuilder-add-string! b ")")))

(impl (Displayed (Tuple %a %b))
  (where (Displayed %a) (Displayed %b))
  (defun (display-into t b)
    (def (x y) t)
    (stringbuilder-add-string! b "(Tuple ")
    (display-into x b)
    (stringbuilder-add-string! b " ")
    (display-into y b)
    (stringbuilder-add-string! b ")")))

(impl (Written (Tuple %a %b %c))
  (where (Written %a) (Written %b) (Written %c))
  (defun (write-into t b)
    (def (Tuple x y z) t)
    (stringbuilder-add-string! b "(Tuple ")
    (write-into x b)
    (stringbuilder-add-string! b " ")
    (write-into y b)
    (stringbuilder-add-string! b " ")
    (write-into z b)
    (stringbuilder-add-string! b ")")))

(impl (Displayed (Tuple %a %b %c))
  (where (Displayed %a) (Displayed %b) (Displayed %c))
  (defun (display-into t b)
    (def (Tuple x y z) t)
    (stringbuilder-add-string! b "(Tuple ")
    (display-into x b)
    (stringbuilder-add-string! b " ")
    (display-into y b)
    (stringbuilder-add-string! b " ")
    (display-into z b)
    (stringbuilder-add-string! b ")")))

;; Four elements, as far as `Eq` and `Ord` go.
(impl (Written (Tuple %a %b %c %d))
  (where (Written %a) (Written %b) (Written %c) (Written %d))
  (defun (write-into t b)
    (def (Tuple w x y z) t)
    (stringbuilder-add-string! b "(Tuple ")
    (write-into w b)
    (stringbuilder-add-string! b " ")
    (write-into x b)
    (stringbuilder-add-string! b " ")
    (write-into y b)
    (stringbuilder-add-string! b " ")
    (write-into z b)
    (stringbuilder-add-string! b ")")))

(impl (Displayed (Tuple %a %b %c %d))
  (where (Displayed %a) (Displayed %b) (Displayed %c) (Displayed %d))
  (defun (display-into t b)
    (def (Tuple w x y z) t)
    (stringbuilder-add-string! b "(Tuple ")
    (display-into w b)
    (stringbuilder-add-string! b " ")
    (display-into x b)
    (stringbuilder-add-string! b " ")
    (display-into y b)
    (stringbuilder-add-string! b " ")
    (display-into z b)
    (stringbuilder-add-string! b ")")))

;; What `displayed` was called for these before it took every type.
(: (-> (List %a) string) (where (Displayed %a)))
(defun (list->str l) (displayed l))

(: (-> (Map %k %v) string) (where (Displayed %k) (Displayed %v)))
(defun (map->str m) (displayed m))

(: (-> (Tuple %k %v) string) (where (Displayed %k) (Displayed %v)))
(defun (map-entry->str entry)
  (def (k v) entry)
  (str (displayed k) " " (displayed v)))

(:deprecated list->str "use `displayed`")
(:deprecated map->str "use `displayed`")
(:deprecated map-entry->str "use `displayed` on the key and the value")

;; Sequential binding
;;
;; Sequential binding macros.

;; `(let* ((a 1) (b (+ a 1))) ...)` — each binding is in scope for the next.
(def/macro (let* form compare inject)
  (syntax-match form
    ((_ () body ...)
     #'(let () ,@body))
    ((_ (bind) body ...)
     #'(let (,bind) ,@body))
    ((_ (bind binds ...) body ...)
     #'(let (,bind)
         (let* (,@binds) ,@body)))))

;; parameterize* behavior.
(def/macro (parameterize* form compare inject)
  (syntax-match form
    ((_ () body ...)
     #'(let () ,@body))
    ((_ (bind) body ...)
     #'(parameterize (,bind) ,@body))
    ((_ (bind binds ...) body ...)
     #'(parameterize (,bind)
        (parameterize* (,@binds) ,@body)))))

;; ---------------------------------------------------------------------------
;; type/derive
;; ---------------------------------------------------------------------------

(: (-> string Syntax))
(defun (derive-sym name) (SSym (string->symbol name)))

;; Refuse invalid syntax.
(: (-> Syntax string (List Syntax)))
(defun (derive-fail form message)
  (ignore (syntax-error form message))
  Nil)

;; Head type variables.
(: (-> Syntax (List Syntax)))
(defun (derive-head-vars head)
  (match head
    ((SList items)
     (match items
       (Nil Nil)
       ((Cons _ vars) vars)))
    (_ Nil)))

;; Type name.
(: (-> Syntax Syntax))
(defun (derive-head-name head)
  (match head
    ((SList items)
     (match items
       ((Cons name _) name)
       (Nil head)))
    (_ head)))

;; Hash combine fold.
(: (-> (List Syntax) Syntax Syntax))
(defun (derive-hash-fold parts acc)
  (match parts
    (Nil acc)
    ((Cons p rest) (derive-hash-fold rest #'(hash-combine ,acc ,p)))))

;; --- Records and structs ---

;; Field names.
(: (-> (List Syntax) (List Syntax)))
(defun (derive-field-names fields)
  (list-map (fun (f)
              (syntax-match f
                ((_ name _ marker ...) name)
                (bad (syntax-error bad "a record field is written (: name type), optionally followed by #:mutable."))))
            fields))

;; Any mutable fields.
(: (-> (List Syntax) bool))
(defun (derive-any-mutable? fields)
  (def (Cons f rest) fields
    :leave-with #f)
  (if (syntax-match f
        ((_ _ _ marker markers ...) #t)
        (_ #f))
      #t
      (derive-any-mutable? rest)))

;; Eq record methods.
(: (-> Syntax (List Syntax) bool (List Syntax)))
(defun (derive-eq-record head names mutable?)
  (def equals
    #'(defun (= a b)
        (and ,@(list-map (fun (n) #'(= (record-ref a ,n) (record-ref b ,n))) names))))
  (def hashed
    (if mutable?
        #'(defun (eq-hash v) (unhashable ,(SStr (syntax->string (derive-head-name head)))))
        #'(defun (eq-hash v)
            ,(derive-hash-fold (list-map (fun (n) #'(eq-hash (record-ref v ,n))) names) #'17))))
  (Cons equals (Cons hashed Nil)))

;; --- Unions ---

;; A case is a bare name or `(: Name type ... markers ...)`.
(: (-> Syntax Syntax))
(defun (derive-case-name c)
  (def* ((SList items) c :leave-with c)
        ((Cons _ (Cons name _)) items :leave-with c))
  name)

;; The payload types, which end at the first marker: `#:tag` takes a name after
;; it, which is not a type.
(: (-> Syntax (List Syntax)))
(defun (derive-payload c)
  (def* ((SList items) c :leave-with Nil)
        ((Cons _ (Cons _ rest)) items :leave-with Nil))
  (derive-types-before-markers rest))

(: (-> (List Syntax) (List Syntax)))
(defun (derive-types-before-markers items)
  (match items
    ((Cons (SKey _) _) Nil)
    ((Cons t rest) (Cons t (derive-types-before-markers rest)))
    (Nil Nil)))

;; Binders.
(: (-> string int int (List Syntax)))
(defun (derive-binders side i n)
  (if (= i n)
      Nil
      (Cons (derive-sym (string-append side (int->string i)))
            (derive-binders side (+ i 1) n))))

(: (-> (List Syntax) (List Syntax) (List Syntax)))
(defun (derive-eq-pairs ls rs)
  (def* ((Cons l lrest) ls :leave-with Nil)
        ((Cons r rrest) rs :leave-with Nil))
  (Cons #'(= ,l ,r) (derive-eq-pairs lrest rrest)))

;; Union arm eq.
(: (-> Syntax Syntax))
(defun (derive-eq-union-arm c)
  (def name (derive-case-name c))
  (def n (list-length (derive-payload c)))
  (def ls (derive-binders "__derived-l" 0 n))
  (def rs (derive-binders "__derived-r" 0 n))
  #'((,name ,@ls)
     (match b
       ((,name ,@rs) (and ,@(derive-eq-pairs ls rs)))
       (,(derive-sym "_") #f))))

;; Hash union arms.
(: (-> (List Syntax) int (List Syntax)))
(defun (derive-hash-union-arms cases index)
  (def (Cons c rest) cases :leave-with Nil)
  (def name (derive-case-name c))
  (def ls (derive-binders "__derived-l" 0 (list-length (derive-payload c))))
  (def tag (SInt (int->string index)))
  (Cons #'((,name ,@ls)
           ,(derive-hash-fold (list-map (fun (l) #'(eq-hash ,l)) ls) tag))
        (derive-hash-union-arms rest (+ index 1))))

(: (-> (List Syntax) (List Syntax)))
(defun (derive-eq-union cases)
  (def equals
    #'(defun (= a b) (match a ,@(list-map derive-eq-union-arm cases))))
  (def hashed
    #'(defun (eq-hash v) (match v ,@(derive-hash-union-arms cases 0))))
  (Cons equals (Cons hashed Nil)))

;; --- Ordering ---

;; Ord chain.
(: (-> (List Syntax) int Syntax))
(defun (derive-ord-chain parts i)
  (def* ((Cons p rest) parts :leave-with #'0)
        (c (derive-sym (string-append "__derived-c" (int->string i)))))
  (if (list-empty? rest)
      p
      #'(let ((,c ,p))
          (if (= ,c 0) ,(derive-ord-chain rest (+ i 1)) ,c))))

;; Ord record.
(: (-> (List Syntax) (List Syntax)))
(defun (derive-ord-record names)
  (Cons #'(defun (compare a b)
            ,(derive-ord-chain
              (list-map (fun (n) #'(compare (record-ref a ,n) (record-ref b ,n))) names)
              0))
        Nil))

;; Wildcards.
(: (-> int (List Syntax)))
(defun (derive-wildcards n)
  (if (= n 0)
      Nil
      (Cons (derive-sym "_") (derive-wildcards (- n 1)))))

;; Index arms.
(: (-> (List Syntax) int (List Syntax)))
(defun (derive-ord-index-arms cases index)
  (def (Cons c rest) cases :leave-with Nil)
  (Cons #'((,(derive-case-name c) ,@(derive-wildcards (list-length (derive-payload c))))
           ,(SInt (int->string index)))
        (derive-ord-index-arms rest (+ index 1))))

(: (-> (List Syntax) (List Syntax) (List Syntax)))
(defun (derive-ord-pairs ls rs)
  (def* ((Cons l lrest) ls :leave-with Nil)
        ((Cons r rrest) rs :leave-with Nil))
  (Cons #'(compare ,l ,r) (derive-ord-pairs lrest rrest)))

;; Ord union arm.
(: (-> Syntax Syntax))
(defun (derive-ord-union-arm c)
  (def name (derive-case-name c))
  (def n (list-length (derive-payload c)))
  (def ls (derive-binders "__derived-l" 0 n))
  (def rs (derive-binders "__derived-r" 0 n))
  #'((,name ,@ls)
     (match b
       ((,name ,@rs) ,(derive-ord-chain (derive-ord-pairs ls rs) 0))
       (,(derive-sym "_") 0))))

;; Ord union.
(: (-> (List Syntax) (List Syntax)))
(defun (derive-ord-union cases)
  (def ia (derive-sym "__derived-ia"))
  (def ib (derive-sym "__derived-ib"))
  (def index-arms (derive-ord-index-arms cases 0))
  (Cons #'(defun (compare a b)
            (let ((,ia (match a ,@index-arms))
                  (,ib (match b ,@index-arms)))
              (if (= ,ia ,ib)
                  (match a ,@(list-map derive-ord-union-arm cases))
                  (compare ,ia ,ib))))
        Nil))

;; --- Putting one implementation together ---

;; Shape error.
(: (-> Syntax (List Syntax)))
(defun (derive-shape-error bad)
  (derive-fail bad "a type that derives has to be a Record, a Struct or a Union. An alias is a second spelling of a type rather than a type of its own — derive for the type it names."))

;; Eq methods.
(: (-> Syntax Syntax (List Syntax)))
(defun (derive-eq-methods head shape)
  (syntax-match shape
    (('Record fields ...) (derive-eq-record head (derive-field-names fields) (derive-any-mutable? fields)))
    (('Struct fields ...) (derive-eq-record head (derive-field-names fields) (derive-any-mutable? fields)))
    (('Union cases ...) (derive-eq-union cases))
    (('Enum cases ...) (derive-eq-union cases))
    (('Sum cases ...) (derive-eq-union cases))
    (bad (derive-shape-error bad))))

(: (-> Syntax (List Syntax)))
(defun (derive-ord-methods shape)
  (syntax-match shape
    (('Record fields ...) (derive-ord-record (derive-field-names fields)))
    (('Struct fields ...) (derive-ord-record (derive-field-names fields)))
    (('Union cases ...) (derive-ord-union cases))
    (('Enum cases ...) (derive-ord-union cases))
    (('Sum cases ...) (derive-ord-union cases))
    (bad (derive-shape-error bad))))

(: (-> Syntax Syntax Syntax (List Syntax)))
(defun (derive-methods trait head shape)
  (if (syntax-ident=? trait (derive-sym "Eq"))
      (derive-eq-methods head shape)
      (if (syntax-ident=? trait (derive-sym "Ord"))
          (derive-ord-methods shape)
          (derive-fail trait "the traits that can be derived are Eq and Ord."))))

;; Derive conditional implementation. A newtype's is the type it wraps,
;; whatever the trait.
(: (-> Syntax Syntax Syntax))
(defun (derive-impl trait typedef)
  (syntax-match typedef
    ((_ head ('Newtype underlying))
     #'(impl (,trait ,head) #:via ,underlying))
    ((_ head shape)
     #'(impl (,trait ,head)
        (where ,@(list-map (fun (v) #'(,trait ,v)) (derive-head-vars head)))
        ,@(derive-methods trait head shape)))
    (bad (syntax-error bad "a type declaration is written (: Name (Record ...))."))))

(: (-> (List Syntax) (List Syntax) (List Syntax)))
(defun (derive-impls traits typedefs)
  (match traits
    (Nil Nil)
    ((Cons t rest)
     (list-append (list-map (fun (td) (derive-impl t td)) typedefs)
                  (derive-impls rest typedefs)))))

(def/macro (type/derive form inject compare)
  (syntax-match form
    ((_ (trait ...) typedef ...)
     #'(begin
         (type ,@typedef)
         ,@(derive-impls trait typedef)))
    (bad (syntax-error bad "type/derive takes a list of traits and one or more type declarations, as in (type/derive (Eq) (: Point (Record (: x int))))."))))

;; -------------------------------------------------------------
;; Extra thingies for cml
;;

(: (-> (Promise %a) %a))
(defun (promise-join/sync p)
  (match (sync/blocking (promise-join p))
    ((Err e) (raise e))
    ((Ok v) v)))


;; Pattern matching thingies
;;
;; `Has` matches a `Refable`: the key is read with `try-ref`, and the pattern is
;; matched against the value when the key is there.
;; (match my-map
;; ((Has :my-key pat) (println #"The value of :my-key is ${pat}"))
;; (_ (panic! "it didn't have it")))
(def/pattern (Has form inject compare)
  (syntax-match form
    ((_ key p) #'(:view (try-ref & ,key) (Some ,p)))
    (bad (syntax-error bad "(Has key pattern) matches a Refable that holds key."))))

(def/pattern (Has* form inject compare)
  (syntax-match form
    ((_) (syntax-error form "(Has* (key pattern) ...) needs at least one key."))
    ((_ (key pat) ...)
     #'(and ,@{listing #'(Has ,k ,p) (:for k key) (:for p pat)}))
    (bad (syntax-error bad "(Has* (key pattern) ...) pairs each key with a pattern for its value."))))

;; ---------------------------------------------------------------------------
;; Taking, dropping and reversing a vec
;; ---------------------------------------------------------------------------
;;
;; A count past the end takes or drops the whole vec, as for a string; a
;; negative count is refused.

(: (-> (Vec %a) int (Vec %a)))
(defun (vec-take v n)
  (check-count "vec-take" n)
  (def len (vec-length v))
  (vec-slice v 0 (if (< n len) n len)))

(: (-> (Vec %a) int (Vec %a)))
(defun (vec-drop v n)
  (check-count "vec-drop" n)
  (def len (vec-length v))
  (def k (if (< n len) n len))
  (vec-slice v k (- len k)))

(: (-> (Vec %a) (Vec %a)))
(defun (vec-reverse v)
  {vecing x (:for x (in-reverse-vec v))})

;; The vec without its last element, or `None` when it is empty.
;; `vec-pop`, which raises on an empty vec, is a compiler builtin.
(: (-> (Vec %a) (Option (Vec %a))))
(defun (vec-try-pop v)
  (if (vec-empty? v) None (Some (vec-pop v))))

;; The vec without its first element, or `None` when it is empty.
(: (-> (Vec %a) (Option (Vec %a))))
(defun (vec-try-pop-first v)
  (if (vec-empty? v) None (Some (vec-pop-first v))))

;; ---------------------------------------------------------------------------
;; Membership
;; ---------------------------------------------------------------------------
;;
;; Whether a collection holds an element. Each compares with `=` and stops at
;; the first match. Written as loops: `oring` is defined in this file, and a
;; collector macro cannot be used where it is defined.

(: (-> (List %a) %a bool) (where (Eq %a)))
(defun (list-contains? xs x)
  (match xs
    (Nil #f)
    ((Cons y rest) (if (= y x) #t (list-contains? rest x)))))

(: (-> (Vec %a) %a bool) (where (Eq %a)))
(defun (vec-contains? v x)
  (let go ((c (vec-cursor v)))
    (if (vec-cursor-done? c)
        #f
        (if (= (vec-cursor-current c) x) #t (go (vec-cursor-next c))))))

(: (-> (Array %a) %a bool) (where (Eq %a)))
(defun (array-contains? a x)
  (def len (array-length a))
  (let go ((i 0))
    (if (= i len)
        #f
        (if (= (array-ref a i) x) #t (go (+ i 1))))))

(: (-> (IArray %a) %a bool) (where (Eq %a)))
(defun (iarray-contains? a x)
  (def len (iarray-length a))
  (let go ((i 0))
    (if (= i len)
        #f
        (if (= (iarray-ref a i) x) #t (go (+ i 1))))))

(: (-> (TransientVec %a) %a bool) (where (Eq %a)))
(defun (transientvec-contains? t x)
  (def len (transientvec-length t))
  (let go ((i 0))
    (if (= i len)
        #f
        (if (= (transientvec-ref t i) x) #t (go (+ i 1))))))

;; The elements in a new array, which the vec does not share. The runtime's,
;; which copies the leaves in one pass.
(import/extern
  (vec->array (: BjolangRuntime.vecsubgtarray (-> (Vec %a) (Array %a)))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/priority-queue.bjo — the persistent priority queue.
;;
;; A pairing heap. A cell holds an element, its priority, and a list of cells
;; that come out after it.
;;
;; Equal priorities come out in the order they were added: each cell has a
;; number from its queue's counter, and the number breaks a tie.
;;
;; Taking the least is O(log n) amortised, which holds when each queue is used
;; once. n adds leave a root with n children. Taking its least walks them
;; all; a queue kept and used again pays that each time. `priorityqueue-balance`
;; rebuilds a queue in O(n) as a tournament, so that the root has about log2(n)
;; children. A queue made from many entries at once is built that way.
;;
;; A cell's children are a chain of `child` and `sibling` fields, so a link
;; changes two fields. A transient is the same tree with an owner: a cell made
;; by the transient carries its owner and is changed in place, and any other
;; cell a link reaches is copied once. A persistent operation has an owner of
;; its own while it runs, so it copies a cell at most once too. Freezing hands the tree to
;; a persistent queue and gives the transient a new owner, so nothing the
;; queue holds is changed again. It does not balance: that would make it O(n).

(import (std prelude))

(:doc #:file "priority-queue.bjodoc")

(export
  PriorityQueue TransientPriorityQueue PriorityQueueBuilder
  ;; Named by functions an importer inlines. Opaque: nothing outside this
  ;; module can use them.
  PriorityQueueTree PriorityQueueCell PriorityQueueOwner
  priorityqueue-empty priorityqueue-by
  priorityqueue-length priorityqueue-empty? priorityqueue-peek
  priorityqueue-add priorityqueue-add-all priorityqueue-merge
  priorityqueue-pop priorityqueue-try-pop priorityqueue-remove-min
  priorityqueue-balance priorityqueue-balanced?
  list->priorityqueue priorityqueue->list priorityqueue->unordered-list
  priorityqueue->str
  transientpriorityqueue-empty transientpriorityqueue-by
  priorityqueue->transientpriorityqueue transientpriorityqueue->priorityqueue
  transientpriorityqueue-length transientpriorityqueue-empty? transientpriorityqueue-peek
  transientpriorityqueue-add! transientpriorityqueue-add-all!
  transientpriorityqueue-pop! transientpriorityqueue-try-pop!
  priorityqueuebuilder-empty priorityqueuebuilder-by
  priorityqueuebuilder-add! priorityqueuebuilder-length priorityqueuebuilder->priorityqueue)

;; ---------------------------------------------------------------------------
;; The tree
;; ---------------------------------------------------------------------------

;; What cells are owned by, compared with `eq?`. It has to have a field, but "transient" is never used
(type (: PriorityQueueOwner #:opaque (Record (: transient bool))))

(: (-> PriorityQueueOwner))
(defun (new-owner) (PriorityQueueOwner (transient #t)))

;; A cell's children are a chain: `child` is the first, and each child's
;; `sibling` the next. Linking two cells then changes two fields and
;; allocates nothing, when both are owned. `Tip` is the empty tree, and is
;; `null`; a cell is reached through its `Node`, which an operation keeps
;; when it changes the cell in place.
(type-rec
  (: (PriorityQueueTree %a %p) #:opaque
     (Union Tip (: Node (PriorityQueueCell %a %p))))
  (: (PriorityQueueCell %a %p) #:opaque
     (Record (: item %a)
             (: priority %p)
             (: seq long)
             (: child (PriorityQueueTree %a %p) #:mutable)
             (: sibling (PriorityQueueTree %a %p) #:mutable)
             (: owner PriorityQueueOwner))))

;; Whether `x` comes out before `y`: by priority, then by the order they were
;; added.
(: (-> (-> %p %p int) (PriorityQueueCell %a %p) (PriorityQueueCell %a %p) bool))
(defun (before? order x y)
  (def c (order (record-ref x priority) (record-ref y priority)))
  (or (< c 0) (and (= c 0) (< (record-ref x seq) (record-ref y seq)))))

(: (-> PriorityQueueOwner %a %p long (PriorityQueueTree %a %p)))
(defun (leaf owner x p seq)
  (Node (PriorityQueueCell (item x) (priority p) (seq seq) (child Tip) (sibling Tip) (owner owner))))

;; `t`, whose cell is `c`, with a new first child and next sibling. In place
;; when `owner` owns the cell; otherwise a copy that `owner` owns.
(: (-> PriorityQueueOwner (PriorityQueueTree %a %p) (PriorityQueueCell %a %p)
       (PriorityQueueTree %a %p) (PriorityQueueTree %a %p)
       (PriorityQueueTree %a %p)))
(defun (relinked owner t c first next)
  (if (eq? (record-ref c owner) owner)
      (begin
        (record-set! c (child first) (sibling next))
        t)
      (Node (PriorityQueueCell (item (record-ref c item)) (priority (record-ref c priority))
                               (seq (record-ref c seq)) (child first) (sibling next)
                               (owner owner)))))

;; Two cells as one heap whose next sibling is `next`: the one that comes out
;; later becomes the first child of the other.
(: (-> (-> %p %p int) PriorityQueueOwner
       (PriorityQueueTree %a %p) (PriorityQueueCell %a %p)
       (PriorityQueueTree %a %p) (PriorityQueueCell %a %p)
       (PriorityQueueTree %a %p)
       (PriorityQueueTree %a %p)))
(defun (link order owner a ca b cb next)
  (if (before? order cb ca)
      (adopt owner b cb a ca next)
      (adopt owner a ca b cb next)))

(: (-> PriorityQueueOwner
       (PriorityQueueTree %a %p) (PriorityQueueCell %a %p)
       (PriorityQueueTree %a %p) (PriorityQueueCell %a %p)
       (PriorityQueueTree %a %p)
       (PriorityQueueTree %a %p)))
(defun (adopt owner w cw l cl next)
  (relinked owner w cw
            (relinked owner l cl (record-ref cl child) (record-ref cw child))
            next))

(: (-> (PriorityQueueTree %a %p) (PriorityQueueCell %a %p)))
(defun (cell-of t)
  (match t
    ((Node c) c)
    (Tip (panic! "priority-queue: a cell of the empty tree"))))

;; `t` with no next sibling.
(: (-> PriorityQueueOwner (PriorityQueueTree %a %p) (PriorityQueueTree %a %p)))
(defun (alone owner t)
  (def (Node c) t :propagate)
  (match (record-ref c sibling)
    (Tip t)
    (_ (relinked owner t c (record-ref c child) Tip))))

;; Two roots, whose siblings are Tip, as one.
(: (-> (-> %p %p int) PriorityQueueOwner (PriorityQueueTree %a %p) (PriorityQueueTree %a %p)
       (PriorityQueueTree %a %p)))
(defun (merge order owner a b)
  (def (Node ca) a :leave-with b)
  (def (Node cb) b :leave-with a)
  (link order owner a ca b cb Tip))

;; A chain of children as one heap, in two passes. The first links
;; neighbours and chains the results through their siblings, last first; the
;; second links that chain from its head, which is the last pair to the
;; first. A link reads the next sibling before it changes it.
(: (-> (-> %p %p int) PriorityQueueOwner (PriorityQueueTree %a %p)
       (PriorityQueueTree %a %p)))
(defun (link-pairs order owner chain)
  (let pair ((t chain) (done Tip))
    (def (Node ca) t :leave-with (combine order owner done))
    (def b (record-ref ca sibling))
    (match b
      (Tip (combine order owner (relinked owner t ca (record-ref ca child) done)))
      ((Node cb)
       (def rest (record-ref cb sibling))
       (pair rest (link order owner t ca b cb done))))))

(: (-> (-> %p %p int) PriorityQueueOwner (PriorityQueueTree %a %p)
       (PriorityQueueTree %a %p)))
(defun (combine order owner done)
  (def (Node first) done :propagate)
  (let go ((acc done) (cacc first) (rest (record-ref first sibling)))
    (def (Node cr) rest :leave-with (alone owner acc))
    (def more (record-ref cr sibling))
    (def linked (link order owner rest cr acc cacc Tip))
    (go linked (cell-of linked) more)))

(: (-> (-> %p %p int) PriorityQueueOwner (PriorityQueueTree %a %p)
       (PriorityQueueTree %a %p)))
(defun (without-root order owner t)
  (def (Node c) t :leave-with Tip)
  (link-pairs order owner (record-ref c child)))

(: (-> (PriorityQueueTree %a %p) (Option (Tuple %a %p))))
(defun (root-entry t)
  (def (Node c) t :leave-with None)
  (Some (Tuple (record-ref c item) (record-ref c priority))))

;; One round of a tournament: neighbours linked, so the list is half as long.
(: (-> (-> %p %p int) PriorityQueueOwner (List (PriorityQueueTree %a %p))
       (List (PriorityQueueTree %a %p))))
(defun (link-round order owner ts)
  (let go ((rest ts) (out Nil))
    (match rest
      (Nil out)
      ((Cons t Nil) (Cons t out))
      ((Cons a (Cons b more))
       (go more (Cons (link order owner a (cell-of a) b (cell-of b) Tip) out))))))

;; Rounds until one tree is left. Round k makes roots with k children, so the
;; winner has about log2(n) children and the tree is about log2(n) deep. n-1
;; links in all.
(: (-> (-> %p %p int) PriorityQueueOwner (List (PriorityQueueTree %a %p))
       (PriorityQueueTree %a %p)))
(defun (tournament order owner ts)
  (match ts
    (Nil Tip)
    ((Cons t Nil) t)
    (_ (tournament order owner (link-round order owner ts)))))

;; The entries as a balanced tree, numbered from `first` in the order given.
(: (-> (-> %p %p int) PriorityQueueOwner (List (Tuple %a %p)) long
       (PriorityQueueTree %a %p)))
(defun (build order owner entries first)
  (def leaves
     (loop (:for rest entries)
           (:with seq first (+ seq 1L))
           (:let (x . p) rest)
           (:acc out (listing/reverse (leaf owner x p seq)))))
  (tournament order owner leaves))

;; Every cell, in no useful order. A worklist rather than recursion: a chain
;; of children can be as long as the queue.
(: (-> (PriorityQueueTree %a %p) (List (PriorityQueueCell %a %p))))
(defun (tree-cells t)
  (let walk ((todo (Cons t Nil)) (out Nil))
    (match todo
      (Nil out)
      ((Cons Tip rest) (walk rest out))
      ((Cons (Node c) rest)
       (walk (Cons (record-ref c child) (Cons (record-ref c sibling) rest)) (Cons c out))))))

;; ---------------------------------------------------------------------------
;; The persistent queue
;; ---------------------------------------------------------------------------

;; `next` is the number the next entry gets. `balanced` is #t when the tree
;; came out of a tournament and nothing has been done to it since.
(type (: (PriorityQueue %a %p) #:opaque
         (Record (: tree (PriorityQueueTree %a %p))
                 (: order (-> %p %p int))
                 (: size int)
                 (: next long)
                 (: balanced bool))))

(: (-> (-> %p %p int) (PriorityQueue %a %p)))
(defun (priorityqueue-by order)
  (PriorityQueue (tree Tip) (order order) (size 0) (next 0L) (balanced #t)))

(: (-> (PriorityQueue %a %p)) (where (Ord %p)))
(defun (priorityqueue-empty)
  (priorityqueue-by (fun (a b) (compare a b))))

(: (-> (PriorityQueue %a %p) int))
(defun (priorityqueue-length q) (record-ref q size))

(: (-> (PriorityQueue %a %p) bool))
(defun (priorityqueue-empty? q) (= (record-ref q size) 0))

(: (-> (PriorityQueue %a %p) (Option (Tuple %a %p))))
(defun (priorityqueue-peek q) (root-entry (record-ref q tree)))

(: (-> (PriorityQueue %a %p) bool))
(defun (priorityqueue-balanced? q) (record-ref q balanced))

(: (-> (PriorityQueue %a %p) (PriorityQueueTree %a %p) int long bool (PriorityQueue %a %p)))
(defun (changed q t size next balanced)
  (PriorityQueue (tree t) (order (record-ref q order)) (size size) (next next) (balanced balanced)))

(: (-> (PriorityQueue %a %p) %a %p (PriorityQueue %a %p)))
(defun (priorityqueue-add q x p)
  (def next (record-ref q next))
  (changed q
           (let ((owner (new-owner)))
             (merge (record-ref q order) owner (record-ref q tree) (leaf owner x p next)))
           (+ (record-ref q size) 1) (+ next 1L) #f))

;; The entries are built into a balanced tree of their own, with an owner of
;; their own so that the build links in place, and that tree is linked in.
(: (-> (PriorityQueue %a %p) (List (Tuple %a %p)) (PriorityQueue %a %p)))
(defun (priorityqueue-add-all q entries)
  (def order (record-ref q order))
  (def next (record-ref q next))
  (def n (list-length entries))
  (changed q
           (let ((owner (new-owner)))
             (merge order owner (record-ref q tree) (build order owner entries next)))
           (+ (record-ref q size) n) (+ next (cast long n))
           (or (= n 0) (and (priorityqueue-empty? q) (record-ref q balanced)))))

(: (-> (PriorityQueue %a %p) (PriorityQueue %a %p) (PriorityQueue %a %p)))
(defun (priorityqueue-merge a b)
  (cond
    ((priorityqueue-empty? b) a)
    ((priorityqueue-empty? a) (changed a (record-ref b tree) (record-ref b size)
                                       (record-ref b next) (record-ref b balanced)))
    (else
     (def na (record-ref a next))
     (def nb (record-ref b next))
     (changed a
              (merge (record-ref a order) (new-owner) (record-ref a tree) (record-ref b tree))
              (+ (record-ref a size) (record-ref b size)) (if (> na nb) na nb) #f))))

(: (-> (PriorityQueue %a %p) (PriorityQueue %a %p)))
(defun (priorityqueue-remove-min q)
  (match (record-ref q tree)
    (Tip q)
    (t (changed q (without-root (record-ref q order) (new-owner) t)
                (- (record-ref q size) 1) (record-ref q next) #f))))

(: (-> (PriorityQueue %a %p) (Option (Tuple %a %p (PriorityQueue %a %p)))))
(defun (priorityqueue-try-pop q)
  (match (record-ref q tree)
    (Tip None)
    ((Node c)
     (Some (Tuple (record-ref c item) (record-ref c priority) (priorityqueue-remove-min q))))))

(import/class
  (EmptyQueue (: System.InvalidOperationException (-> string EmptyQueue))))

;; The entry that comes out first, and the queue without it.
;; Raises InvalidOperationException when the queue is empty.
(: (-> (PriorityQueue %a %p) (Tuple %a %p (PriorityQueue %a %p))))
(defun (priorityqueue-pop q)
  (match (priorityqueue-try-pop q)
    ((Some entry) entry)
    (None (raise (cast Exception (EmptyQueue. "priorityqueue-pop: the queue is empty."))))))

;; O(n), and a field read on a queue that is balanced already. The cells are
;; made again, keeping their numbers: the old ones may be in other queues.
(: (-> (PriorityQueue %a %p) (PriorityQueue %a %p)))
(defun (priorityqueue-balance q)
  (if (record-ref q balanced)
      q
      (let ((owner (new-owner)))
        (changed q
                 (tournament (record-ref q order) owner
                             (list-map (fun (c) (leaf owner (record-ref c item) (record-ref c priority)
                                                      (record-ref c seq)))
                                       (tree-cells (record-ref q tree))))
                 (record-ref q size) (record-ref q next) #t))))

(: (-> (List (Tuple %a %p)) (PriorityQueue %a %p)) (where (Ord %p)))
(defun (list->priorityqueue entries)
  (priorityqueue-add-all (priorityqueue-empty) entries))

;; In the order they come out. Drained through a transient, which copies a
;; cell when it first wins a link and then adds to the copy, rather than
;; copying a cell for every link.
(: (-> (PriorityQueue %a %p) (List (Tuple %a %p))))
(defun (priorityqueue->list q)
  (def t (priorityqueue->transientpriorityqueue q))
  (let go ((out Nil))
    (match (transientpriorityqueue-try-pop! t)
      (None (list-reverse out))
      ((Some entry) (go (Cons entry out))))))

(: (-> (PriorityQueue %a %p) (List (Tuple %a %p))))
(defun (priorityqueue->unordered-list q)
  (list-map (fun (c) (Tuple (record-ref c item) (record-ref c priority)))
            (tree-cells (record-ref q tree))))

;; ---------------------------------------------------------------------------
;; The transient
;; ---------------------------------------------------------------------------

(type (: (TransientPriorityQueue %a %p) #:opaque
         (Record (: tree (PriorityQueueTree %a %p) #:mutable)
                 (: order (-> %p %p int))
                 (: size int #:mutable)
                 (: next long #:mutable)
                 (: owner PriorityQueueOwner #:mutable))))

(: (-> (PriorityQueue %a %p) (TransientPriorityQueue %a %p)))
(defun (priorityqueue->transientpriorityqueue q)
  (TransientPriorityQueue (tree (record-ref q tree)) (order (record-ref q order))
                          (size (record-ref q size)) (next (record-ref q next))
                          (owner (new-owner))))

(: (-> (-> %p %p int) (TransientPriorityQueue %a %p)))
(defun (transientpriorityqueue-by order)
  (priorityqueue->transientpriorityqueue (priorityqueue-by order)))

(: (-> (TransientPriorityQueue %a %p)) (where (Ord %p)))
(defun (transientpriorityqueue-empty)
  (priorityqueue->transientpriorityqueue (priorityqueue-empty)))

;; O(1). The transient goes on with a new owner, so its next change copies
;; what it reaches rather than changing the queue handed out.
(: (-> (TransientPriorityQueue %a %p) (PriorityQueue %a %p)))
(defun (transientpriorityqueue->priorityqueue t)
  (record-set! t (owner (new-owner)))
  (PriorityQueue (tree (record-ref t tree)) (order (record-ref t order))
                 (size (record-ref t size)) (next (record-ref t next)) (balanced #f)))

(: (-> (TransientPriorityQueue %a %p) int))
(defun (transientpriorityqueue-length t) (record-ref t size))

(: (-> (TransientPriorityQueue %a %p) bool))
(defun (transientpriorityqueue-empty? t) (= (record-ref t size) 0))

(: (-> (TransientPriorityQueue %a %p) (Option (Tuple %a %p))))
(defun (transientpriorityqueue-peek t) (root-entry (record-ref t tree)))

(: (-> (TransientPriorityQueue %a %p) %a %p void))
(defun (transientpriorityqueue-add! t x p)
  (def owner (record-ref t owner))
  (def next (record-ref t next))
  (record-set! t
               (tree (merge (record-ref t order) owner (record-ref t tree) (leaf owner x p next)))
               (size (+ (record-ref t size) 1))
               (next (+ next 1L))))

(: (-> (TransientPriorityQueue %a %p) (List (Tuple %a %p)) void))
(defun (transientpriorityqueue-add-all! t entries)
  (def order (record-ref t order))
  (def owner (record-ref t owner))
  (def next (record-ref t next))
  (def n (list-length entries))
  (record-set! t
               (tree (merge order owner (record-ref t tree) (build order owner entries next)))
               (size (+ (record-ref t size) n))
               (next (+ next (cast long n)))))

;; `None` leaves the transient as it was.
(: (-> (TransientPriorityQueue %a %p) (Option (Tuple %a %p))))
(defun (transientpriorityqueue-try-pop! t)
  (match (record-ref t tree)
    (Tip None)
    ((Node c)
     (record-set! t
                  (tree (link-pairs (record-ref t order) (record-ref t owner) (record-ref c child)))
                  (size (- (record-ref t size) 1)))
     (Some (Tuple (record-ref c item) (record-ref c priority))))))

;; The entry that comes out first, removed in place.
;; Raises InvalidOperationException when the transient is empty.
(: (-> (TransientPriorityQueue %a %p) (Tuple %a %p)))
(defun (transientpriorityqueue-pop! t)
  (match (transientpriorityqueue-try-pop! t)
    ((Some entry) entry)
    (None (raise (cast Exception (EmptyQueue. "transientpriorityqueue-pop!: the queue is empty."))))))

;; ---------------------------------------------------------------------------
;; The builder
;; ---------------------------------------------------------------------------

;; The entries, last first, and how many.
(type (: (PriorityQueueBuilder %a %p) #:opaque
         (Record (: entries (List (Tuple %a %p)) #:mutable)
                 (: count int #:mutable)
                 (: order (-> %p %p int)))))

(: (-> (-> %p %p int) (PriorityQueueBuilder %a %p)))
(defun (priorityqueuebuilder-by order)
  (PriorityQueueBuilder (entries Nil) (count 0) (order order)))

(: (-> (PriorityQueueBuilder %a %p)) (where (Ord %p)))
(defun (priorityqueuebuilder-empty)
  (priorityqueuebuilder-by (fun (a b) (compare a b))))

(: (-> (PriorityQueueBuilder %a %p) %a %p void))
(defun (priorityqueuebuilder-add! b x p)
  (record-set! b
               (entries (Cons (Tuple x p) (record-ref b entries)))
               (count (+ (record-ref b count) 1))))

(: (-> (PriorityQueueBuilder %a %p) int))
(defun (priorityqueuebuilder-length b) (record-ref b count))

;; O(n), and balanced. The builder keeps its entries, and may be added to and
;; built again.
(: (-> (PriorityQueueBuilder %a %p) (PriorityQueue %a %p)))
(defun (priorityqueuebuilder->priorityqueue b)
  (priorityqueue-add-all (priorityqueue-by (record-ref b order))
                         (list-reverse (record-ref b entries))))

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------

(impl (Collection (PriorityQueue %a %p))
  (type %content (Tuple %a %p))
  (defun (length q) (priorityqueue-length q))
  (defun (empty? q) (priorityqueue-empty? q)))

;; In the order the entries come out. The cursor is the rest of the queue, so
;; a walk leaves the queue itself alone. Each cursor is used once, so the
;; amortised bound holds.
(impl (Iterable (PriorityQueue %a %p))
  (type %elem (Tuple %a %p))
  (type %cursor (PriorityQueue %a %p))
  (defun (iterable-start q) q)
  (defun (iterable-done? q c) (priorityqueue-empty? c))
  (defun (iterable-current q c)
    (match (priorityqueue-peek c)
      ((Some entry) entry)
      (None (panic! "priority-queue: iterable-current past the end"))))
  (defun (iterable-next q c) (priorityqueue-remove-min c)))

;; Equal when the same entries come out in the same order. The shape of the
;; tree, and the numbers that break ties, are not compared.
(impl (Eq (PriorityQueue %a %p))
  (where (Eq %a) (Eq %p))
  (defun (= a b)
    (and (= (priorityqueue-length a) (priorityqueue-length b))
         (= (priorityqueue->list a) (priorityqueue->list b))))
  (defun (eq-hash q)
    (list-foldl (fun (entry acc) (hash-combine acc (eq-hash entry))) 17 (priorityqueue->list q))))

;; `#priorityqueue(element priority ...)`, in the order they come out. Each
;; element and priority is added by the function given for it.
(: (-> (PriorityQueue %a %p) StringBuilder (-> %a StringBuilder void) (-> %p StringBuilder void) void))
(defun (priorityqueue-print-into q b each-elem each-priority)
  (stringbuilder-add-string! b "#priorityqueue(")
  (loop (:with first? #t #f)
        (:for entry q)
        (:do (def (x p) entry)
             (unless first? (stringbuilder-add-string! b " "))
             (each-elem x b)
             (stringbuilder-add-string! b " ")
             (each-priority p b)))
  (stringbuilder-add-string! b ")"))

(impl (Written (PriorityQueue %a %p))
  (where (Written %a) (Written %p))
  (defun (write-into q b)
    (priorityqueue-print-into q b (fun (x b) (write-into x b)) (fun (p b) (write-into p b)))))

(impl (Displayed (PriorityQueue %a %p))
  (where (Displayed %a) (Displayed %p))
  (defun (display-into q b)
    (priorityqueue-print-into q b (fun (x b) (display-into x b)) (fun (p b) (display-into p b)))))

(: (-> (PriorityQueue %a %p) string) (where (Displayed %a) (Displayed %p)))
(defun (priorityqueue->str q) (displayed q))

(:deprecated priorityqueue->str "use `displayed`")
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/random.bjo — random numbers.
;;
;; The default source is ChaCha20 seeded with 256 bits from the OS CSPRNG, one
;; generator per thread, drawn from without a lock. Any user-supplied rng uses
;; a lock. Every function here takes `#:rng`, defaulting to
;; `(parameter-ref current-random)`, so a program that wants a different source
;; binds the parameter and changes no call site.
;;
;; By default, a thread get's it's own generator, so that reading from the
;; RandomSource is unconstrained and not risky. 
;;
;; **`current-random` holds no generator.** Its value is the *ambient* source,
;; which resolves to the calling thread's generator at each draw. That is what
;; makes the default safe on a bjoroutine: a fiber may resume on a different
;; thread than it suspended on, so a source read out of the parameter before a
;; suspension would otherwise go on drawing from the old thread's generator
;; while another fiber ran on that thread and drew from it too. 
;;
;; A source from `make-random` and its neighbours does name a generator, and two
;; fibers may hold it at once, so every operation on one takes its lock. The
;; lock is per *operation*: a shuffle is a contiguous slice of the stream rather
;; than one interleaved with another fiber's.
;;
;; **Locked is not reproducible.** Sharing a seeded source between fibers is
;; safe, but which fiber gets which value of the stream is the scheduler's
;; business. A run that has to replay wants `random-split` per fiber.
;;
;; **Seeded means reproducible, not unguessable.** `make-random` takes 256 bits
;; from the OS. `make-random/seed64` and `make-random/seed256` take what they
;; are given, and are for replaying a run. If you seed make-random with 256 random
;; bits yourself, it is safe.

(export
  RandomSource
  current-random
  ;; Making one
  make-random make-random/seed64 make-random/seed256 random-split
  ;; Drawing
  random-int random-double random-u32 random-u64
  random-bool random-chance?
  random-bytes! shuffle-array! shuffle-vec random-ref
  random-string random-string/chars
  ;; Binding one
  with-random-seed)

(import (std prelude))
(import (std syntax-match))

;; Every export is documented there; the module doc is too.
(:doc #:file "random.bjodoc")

(import/class
  (RandomSource      (: Bjolang.Runtime.RandomSource))
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

(import/extern
  (ambient-random     (: Bjolang.Runtime.RandomModule.Ambient     (-> RandomSource)))
  (make-random        (: Bjolang.Runtime.RandomModule.Fresh       (-> RandomSource)))
  (make-random/seed64 (: Bjolang.Runtime.RandomModule.FromSeed64  (-> ulong RandomSource)))
  (make-random/seed256
    (: Bjolang.Runtime.RandomModule.FromSeed256 (-> ulong ulong ulong ulong RandomSource)))
  (random-split       (: Bjolang.Runtime.RandomModule.Split       (-> RandomSource RandomSource)))

  (clr-random-range   (: Bjolang.Runtime.RandomModule.Range   (-> RandomSource int int int)))
  (clr-random-double  (: Bjolang.Runtime.RandomModule.Double  (-> RandomSource double)))
  (clr-random-u32     (: Bjolang.Runtime.RandomModule.U32     (-> RandomSource uint)))
  (clr-random-u64     (: Bjolang.Runtime.RandomModule.U64     (-> RandomSource ulong)))
  (clr-random-bool    (: Bjolang.Runtime.RandomModule.Bool    (-> RandomSource bool)))
  (clr-random-chance  (: Bjolang.Runtime.RandomModule.Chance  (-> RandomSource double bool)))
  (clr-random-fill    (: Bjolang.Runtime.RandomModule.Fill    (-> RandomSource (Array byte) void)))
  (clr-random-shuffle (: Bjolang.Runtime.RandomModule.Shuffle (-> RandomSource (Array %a) void)))
  (clr-random-string  (: Bjolang.Runtime.RandomModule.Str     (-> RandomSource int string))))

;; `make-parameter`. Takes a hot parameter.
;; The default ambient-random is null, meaning every read reads the threadlocal.
;; Parameterizing current-random to (ambient-random) is thus always safe.
(: (Param RandomSource))
(def current-random (make-parameter (ambient-random)))

;; ---------------------------------------------------------------------------
;; Drawing
;; ---------------------------------------------------------------------------

;; Inclusive at both ends. Lemire's method, so no modulo bias.
(: (-> int int (#:rng RandomSource) int))
(defun (random-int lo hi #:rng (parameter-ref current-random))
  (clr-random-range rng lo hi))

;; [0, 1), with 53 bits of randomness.
(: (-> (#:rng RandomSource) double))
(defun (random-double #:rng (parameter-ref current-random))
  (clr-random-double rng))

(: (-> (#:rng RandomSource) uint))
(defun (random-u32 #:rng (parameter-ref current-random))
  (clr-random-u32 rng))

(: (-> (#:rng RandomSource) ulong))
(defun (random-u64 #:rng (parameter-ref current-random))
  (clr-random-u64 rng))

;; One bit per call, taken 64 at a time.
(: (-> (#:rng RandomSource) bool))
(defun (random-bool #:rng (parameter-ref current-random))
  (clr-random-bool rng))

;; True with probability `p`. Outside [0, 1] it is the nearer of the two, and
;; costs no draw.
(: (-> double (#:rng RandomSource) bool))
(defun (random-chance? p #:rng (parameter-ref current-random))
  (clr-random-chance rng p))

;; Fills the array in place. Whole blocks go straight into it, so this is the
;; way to ask for many bytes at once.
(: (-> (Array byte) (#:rng RandomSource) void))
(defun (random-bytes! bytes #:rng (parameter-ref current-random))
  (clr-random-fill rng bytes))

;; Fisher-Yates, in place. Answers nothing: the array it was given is the
;; shuffled one.
(: (-> (Array %a) (#:rng RandomSource) void))
(defun (shuffle-array! a #:rng (parameter-ref current-random))
  (clr-random-shuffle rng a))

;; A `Vec` is persistent, so this answers a new one and leaves `v` alone.
;;
;; Walks the vec, and places it's elements into an array of the
;; same size randomly. 
(: (-> (Vec %a) (#:rng RandomSource) (Vec %a)))
(defun (shuffle-vec v #:rng (parameter-ref current-random))
  (cond
    ((< (vec-length v) 2) v)
    (else
      (def a (make-array (vec-length v)))
      (loop (:with i 0 (+ i 1))
            (:for x v)
            (:do (array-set! a i x)))
      (clr-random-shuffle rng a)
      (array->vec a))))

;; A string over `[A-Za-z0-9-_]`. The shim writes the UTF-8 bytes directly.
(: (-> int (#:rng RandomSource) string))
(defun (random-string length #:rng (parameter-ref current-random))
  (clr-random-string rng length))

;; ---------------------------------------------------------------------------
;; Over a collection
;; ---------------------------------------------------------------------------
;;
;; Written here rather than in the shim: each is an index and a draw, and
;; Bjolang's `char` is a `BjoChar`, which does not cross as a `char[]`.

;; An element of `v`. Randomly picked.
;; This could be a trait function, but currently we cannot constrain
;; Refable to integer keys.
(: (-> (Vec %a) (#:rng RandomSource) %a))
(defun (random-ref v #:rng (parameter-ref current-random))
  (when (= (vec-length v) 0)
    (raise (cast Exception (ArgumentException. "random-ref was given an empty vec."))))
  (vec-ref v (clr-random-range rng 0 (- (vec-length v) 1))))

;; A string over an alphabet of the caller's own.
(: (-> int (Vec char) (#:rng RandomSource) string))
(defun (random-string/chars length alphabet #:rng (parameter-ref current-random))
  (when (= (vec-length alphabet) 0)
    (raise (cast Exception (ArgumentException. "random-string/chars was given an empty alphabet."))))
  ;; The loop below counts up to `length`, so a negative one never ends.
  (when (< length 0)
    (raise (cast Exception (ArgumentException. "random-string/chars was given a negative length."))))
  (def sb (stringbuilder-empty))
  (def top (- (vec-length alphabet) 1))
  (let go ((i 0))
    (cond
      ((= i length) (stringbuilder->string sb))
      (else
        (stringbuilder-add! sb (vec-ref alphabet (clr-random-range rng 0 top)))
        (go (+ i 1))))))

;; ---------------------------------------------------------------------------
;; Binding a source
;; ---------------------------------------------------------------------------

;; `(with-random-seed 42UL body ...)` — the extent draws from a generator that
;; replays.
;;
;; One generator for the whole extent. Spawning a fiber here is apeshit bad
;; since fibers should NOT share state. It is at least safe — the source is
;; locked — but the order the fibers draw in is the scheduler's, so the run
;; stops replaying, which is the only reason to have seeded it. A fiber wants
;; `(random-split ...)` or a seed of its own.
(def/macro (with-random-seed form inject compare)
  (syntax-match form
    ((_ seed first body ...)
     #'(parameterize ((current-random (make-random/seed64 ,seed)))
         ,first
         ,@body))
    (bad (syntax-error bad "with-random-seed takes a seed and a body: (with-random-seed 42UL body ...)"))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/rational.bjo — exact fractions over bigint: the built-in `rational`,
;; with functions by name.

(import (std prelude)
        (std maths))

(:doc #:file "rational.bjodoc")

(export
  Rational
  ;; Construction and parts
  rational int->rational bigint->rational
  rational-num rational-den
  ;; Arithmetic
  rational+ rational- rational* rational/
  rational-negate rational-recip rational-abs rational-expt
  ;; Predicates
  rational-zero? rational-integer? rational-sign
  ;; Rounding to a bigint
  rational-floor rational-ceiling rational-truncate rational-round
  ;; Conversions
  rational->double rational->decimal
  rational->string string->rational)

(import/class
  (Exception (: System.Exception))
  (DivideByZeroException (: System.DivideByZeroException (-> string DivideByZeroException)))
  (OverflowException (: System.OverflowException (-> string OverflowException)))
  (BjoRational (: Bjolang.Runtime.BjoRational (-> bigint bigint BjoRational))))

(import/extern
  (clr-bit-length (: System.Numerics.BigInteger.GetBitLength (-> bigint long)))
  (clr-scale-b (: System.Math.ScaleB (-> double int double)))
  (clr-numerator (: Bjolang.Runtime.BjoRational.NumeratorOf (-> rational bigint)))
  (clr-denominator (: Bjolang.Runtime.BjoRational.DenominatorOf (-> rational bigint))))

;; The built-in: den > 0, gcd(num, den) = 1, zero is 0/1. Eq, Ord, Written and
;; Displayed are the prelude's.
(type (: Rational rational))

(: (-> string %a))
(defun (divide-by-zero what)
  (raise (cast Exception (DivideByZeroException. (str "(std rational): " what)))))

;; ---------------------------------------------------------------------------
;; Construction
;; ---------------------------------------------------------------------------

;; Normalizes sign and common factors.
(: (-> bigint bigint Rational))
(defun (rational n d)
  (when (zero? d)
    (divide-by-zero "the denominator is zero."))
  (BjoRational. n d))

(: (-> bigint Rational))
(defun (bigint->rational n) (BjoRational. n 1I))

(: (-> int Rational))
(defun (int->rational n) (bigint->rational (cast System.Numerics.BigInteger n)))

(: (-> Rational bigint))
(defun (rational-num x) (clr-numerator x))

(: (-> Rational bigint))
(defun (rational-den x) (clr-denominator x))

;; ---------------------------------------------------------------------------
;; Arithmetic
;; ---------------------------------------------------------------------------

(: (-> Rational Rational Rational))
(defun (rational+ a b) (+ a b))

(: (-> Rational Rational Rational))
(defun (rational- a b) (- a b))

(: (-> Rational Rational Rational))
(defun (rational* a b) (* a b))

;; Raises DivideByZeroException when b is zero.
(: (-> Rational Rational Rational))
(defun (rational/ a b) (rational* a (rational-recip b)))

(: (-> Rational Rational))
(defun (rational-negate x) (- x))

;; Raises DivideByZeroException for zero.
(: (-> Rational Rational))
(defun (rational-recip x)
  (when (rational-zero? x)
    (divide-by-zero "the reciprocal of zero."))
  (rational (rational-den x) (rational-num x)))

(: (-> Rational Rational))
(defun (rational-abs x) (abs x))

;; Any int exponent. A negative one takes the reciprocal; zero to a negative
;; power raises DivideByZeroException.
(: (-> Rational int Rational))
(defun (rational-expt x n)
  (if (< n 0)
      (rational-recip (rational-expt x (- 0 n)))
      (rational (expt (rational-num x) n) (expt (rational-den x) n))))

;; ---------------------------------------------------------------------------
;; Predicates
;; ---------------------------------------------------------------------------

(: (-> Rational bool))
(defun (rational-zero? x) (zero? x))

(: (-> Rational bool))
(defun (rational-integer? x) (= (rational-den x) 1I))

;; -1, 0 or 1.
(: (-> Rational int))
(defun (rational-sign x) (sign x))

;; ---------------------------------------------------------------------------
;; Rounding to a bigint
;; ---------------------------------------------------------------------------

(: (-> Rational bigint))
(defun (rational-floor x)
  (def n (rational-num x))
  (def d (rational-den x))
  (def q (/ n d))
  (if (< (% n d) 0) (- q 1I) q))

(: (-> Rational bigint))
(defun (rational-ceiling x)
  (- 0I (rational-floor (rational-negate x))))

(: (-> Rational bigint))
(defun (rational-truncate x)
  (/ (rational-num x) (rational-den x)))

;; Ties to even.
(: (-> Rational bigint))
(defun (rational-round x)
  (def f (rational-floor x))
  (def twice-frac (* 2I (- (rational-num x) (* f (rational-den x)))))
  (def c (compare twice-frac (rational-den x)))
  (if (or (> c 0) (and (= c 0) (odd? f))) (+ f 1I) f))

;; ---------------------------------------------------------------------------
;; Conversions
;; ---------------------------------------------------------------------------

(: (-> bigint int))
(defun (bit-length n) (cast System.Int32 (clr-bit-length n)))

;; q/d rounded to an integer, ties to even, given q = floor(x) and r its remainder.
(: (-> bigint bigint bigint bigint))
(defun (round-quotient q r d)
  (def c (compare (* 2I r) d))
  (if (or (> c 0) (and (= c 0) (odd? q))) (+ q 1I) q))

;; Nearest double, ties to even, subnormals included. Rounds once.
;; Too large for a double: infinity.
(: (-> Rational double))
(defun (rational->double x)
  (def n (abs (rational-num x)))
  (def d (rational-den x))
  (def mag
    (if (zero? n)
        0.0
        (let* ((e (- (bit-length n) (bit-length d)))  ; n/d in [2^(e-1), 2^(e+1))
               (k (max (- e 54) -1074))               ; exponent of q's last bit
               (num (if (< k 0) (shift-left n (- 0 k)) n))
               (den (if (< k 0) d (shift-left d k)))
               (q (/ num den))
               (r (% num den))
               (extra (max 0 (- (bit-length q) 53))))
          (if (= extra 0)
              (clr-scale-b (cast System.Double (round-quotient q r den)) k)
              (let* ((unit (shift-left 1I extra))
                     (low (% q unit))
                     (high (shift-right q extra))
                     (half (shift-right unit 1))
                     (c (compare low half))
                     (up (or (> c 0)
                             (and (= c 0) (not (zero? r)))
                             (and (= c 0) (zero? r) (odd? high)))))
                (clr-scale-b (cast System.Double (if up (+ high 1I) high)) (+ k extra)))))))
  (if (< (rational-num x) 0) (- 0.0 mag) mag))

(def decimal-limit (shift-left 1I 96))

;; The digits of q with a point before the last s of them.
(: (-> bigint int string))
(defun (point-digits q s)
  (def digits (string-pad-left (bigint->string q) (+ s 1) #:with #\0))
  (if (= s 0)
      digits
      (str (string-drop-right digits s) "." (string-take-right digits s))))

;; Nearest decimal, ties to even, at the largest scale that fits, with
;; trailing zeros removed. Rounds once.
;; Raises OverflowException when the integer part does not fit.
(: (-> Rational decimal))
(defun (rational->decimal x)
  (def n (abs (rational-num x)))
  (def d (rational-den x))
  (def (q s)
    (let down ((s 28))
      (def scaled (* n (expt 10I s)))
      (def q (round-quotient (/ scaled d) (% scaled d) d))
      (cond
        ((< q decimal-limit) (Tuple q s))
        ((= s 0) (raise (cast Exception (OverflowException.
                                         (str "(std rational): " (rational->string x)
                                              " is too large for a decimal.")))))
        (else (down (- s 1))))))
  (def (q* s*)
    (let strip ((q q) (s s))
      (if (and (> s 0) (zero? (% q 10I)))
          (strip (/ q 10I) (- s 1))
          (Tuple q s))))
  (string->decimal (str (if (< (rational-num x) 0) "-" "") (point-digits q* s*))))

;; "3", "-1/3".
(: (-> Rational string))
(defun (rational->string x)
  (if (rational-integer? x)
      (bigint->string (rational-num x))
      (str (bigint->string (rational-num x)) "/" (bigint->string (rational-den x)))))

;; ---------------------------------------------------------------------------
;; Parsing
;; ---------------------------------------------------------------------------

(: (-> char bool))
(defun (digit? c) (and (char>=? c #\0) (char<=? c #\9)))

(: (-> char bigint))
(defun (digit-value c) (cast System.Numerics.BigInteger (- (char->int c) 48)))

;; Digits from the front of cs: (Tuple value count rest).
(: (-> (List char) (Tuple bigint int (List char))))
(defun (read-digits cs)
  (let go ((cs cs) (v 0I) (k 0))
    (match cs
      ((Cons c rest) #:when (digit? c) (go rest (+ (* v 10I) (digit-value c)) (+ k 1)))
      (_ (Tuple v k cs)))))

;; An optional sign: (Tuple negative? rest).
(: (-> (List char) (Tuple bool (List char))))
(defun (read-sign cs)
  (match cs
    ((Cons c rest) #:when (char=? c #\-) (Tuple #t rest))
    ((Cons c rest) #:when (char=? c #\+) (Tuple #f rest))
    (_ (Tuple #f cs))))

;; sign? digits ("." digits)? (("e" | "E") sign? digits)?, at least one
;; mantissa digit, and nothing after.
(: (-> (List char) (Option Rational)))
(defun (read-decimal cs)
  (def (neg? cs1) (read-sign cs))
  (def (int-part int-count cs2) (read-digits cs1))
  (def (frac-part frac-count cs3)
    (match cs2
      ((Cons c rest) #:when (char=? c #\.) (read-digits rest))
      (_ (Tuple 0I 0 cs2))))
  (def exponent
    (match cs3
      (Nil (Some 0))
      ((Cons c rest) #:when (or (char=? c #\e) (char=? c #\E))
       (def (eneg? cs4) (read-sign rest))
       (def (ev ecount cs5) (read-digits cs4))
       (if (and (> ecount 0) (list-empty? cs5) (< ev 100000I))
           (Some (* (if eneg? -1 1) (cast System.Int32 ev)))
           None))
      (_ None)))
  (match exponent
    ((Some ex) #:when (> (+ int-count frac-count) 0)
     (def mantissa (rational (+ (* int-part (expt 10I frac-count)) frac-part)
                             (expt 10I frac-count)))
     (def scaled (rational* mantissa (rational-expt (bigint->rational 10I) ex)))
     (Some (if neg? (rational-negate scaled) scaled)))
    (_ None)))

;; "3", "-0.3048", "1e-3", "2.5E+10", "1|60", "1/3". The two sides of | or /
;; are decimals; no space anywhere. None for anything else, and for a zero
;; denominator.
(: (-> string (Option Rational)))
(defun (string->rational s)
  (def (a b)
    (match (string-split-once s "|")
      ((Some (Tuple x y)) (Tuple x (Some y)))
      (None (match (string-split-once s "/")
              ((Some (Tuple x y)) (Tuple x (Some y)))
              (None (Tuple s None))))))
  (match (Tuple (read-decimal (string->list a)) b)
    ((Tuple (Some x) None) (Some x))
    ((Tuple (Some x) (Some text))
     (match (read-decimal (string->list text))
       ((Some y) #:when (not (rational-zero? y)) (Some (rational/ x y)))
       (_ None)))
    (_ None)))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/run.bjo — run external programs.
(import (std prelude))
(import (std syntax-match))
(import (std stream))

;; Every export is documented there; the module doc is too.
(:doc #:file "run.bjodoc")

(export
  ;; The types. `Form` is a grammar. Its export also exports `Word`, and a
  ;; literal in another module is read by the grammar there.
  Word Form Cmd Filter Proc

  ;; Writing a form
  cmd

  ;; Running one, blocking
  run run/string run/strings run/status run/output

  ;; A running form
  proc-input proc-output wait wait/all kill

  ;; ...and from inside a bjoroutine
  Reader exited running run/channel by-lines run/stream

  ;; Scoped
  with-run)


;; ===========================================================================
;; Interop
;; ===========================================================================

(import/class
  (Process (: System.Diagnostics.Process))
  (ProcessStartInfo (: System.Diagnostics.ProcessStartInfo (-> ProcessStartInfo)))
  ;; Stream types for Process.
  (ClrStream (: System.IO.Stream))
  (StreamReader (: System.IO.StreamReader))
  ;; An input port over a stream, registered on no scope: a child's pipes are
  ;; `run`'s to close, not the ambient scope's.
  (PipePort (: Bjolang.Runtime.BjoInputPort (-> ClrStream PipePort)))
  (StreamWriter (: System.IO.StreamWriter))
  ;; An output port over a stream, registered on no scope, for the same
  ;; reason: a child's stdin and a redirect's file are `run`'s to close.
  (PortOver (: Bjolang.Runtime.BjoOutputPort (-> ClrStream bool PortOver)))
  (BufferMode (: Bjolang.Runtime.BufferMode))
  ;; An output port joined to an input port. See `BjolangRuntime/BjoProcess.cs`.
  (BjoPipe (: BjoPipe))
  (InvalidOperationException
   (: System.InvalidOperationException (-> string InvalidOperationException))))

(import/extern
  (settings-program!  (: System.Diagnostics.ProcessStartInfo.FileName
                         (-> ProcessStartInfo string void) #:set))
  (settings-redirect-input!  (: System.Diagnostics.ProcessStartInfo.RedirectStandardInput
                                (-> ProcessStartInfo bool void) #:set))
  (settings-redirect-output! (: System.Diagnostics.ProcessStartInfo.RedirectStandardOutput
                                (-> ProcessStartInfo bool void) #:set))
  (settings-redirect-errors! (: System.Diagnostics.ProcessStartInfo.RedirectStandardError
                                (-> ProcessStartInfo bool void) #:set))
  (settings-use-shell!       (: System.Diagnostics.ProcessStartInfo.UseShellExecute
                                (-> ProcessStartInfo bool void) #:set))
  ;; Add process arguments.
  (settings-argument  (: BjoProc.AddArgument
                         (-> ProcessStartInfo string ProcessStartInfo)))

  ;; Starts process.
  (process-start (: System.Diagnostics.Process.Start
                    (-> ProcessStartInfo Process)
                    #:exceptions (System.ComponentModel.Win32Exception
                                  System.InvalidOperationException)))
  ;; Awaits process exit.
  (process-await (: System.Diagnostics.Process.WaitForExitAsync (-> Process void) #:async))
  (process-code  (: System.Diagnostics.Process.ExitCode (-> Process int) #:get))
  (process-stdin (: System.Diagnostics.Process.StandardInput (-> Process StreamWriter) #:get))
  ;; The pipe under a child's input writer, written by nothing yet.
  (writer-bytes  (: System.IO.StreamWriter.BaseStream (-> StreamWriter ClrStream) #:get))
  (port-mode!    (: Bjolang.Runtime.BjoOutputPort.Mode (-> OutputPort BufferMode void) #:set))
  ;; The synchronous close. What `run` closes on a caller's behalf is a child's
  ;; stdin or a pipe's writer, both unbuffered, so there is nothing left to
  ;; write and no reason for the close to suspend.
  (close-unbuffered (: BjolangRuntime.CloseOutput (-> OutputPort void)))
  (unbuffered    (: Bjolang.Runtime.BufferMode.None BufferMode #:get))
  (process-stdout (: System.Diagnostics.Process.StandardOutput (-> Process StreamReader) #:get))
  (process-stderr (: System.Diagnostics.Process.StandardError (-> Process StreamReader) #:get))
  ;; The pipe under a child's output reader, read by nothing yet.
  (reader-bytes   (: System.IO.StreamReader.BaseStream (-> StreamReader ClrStream) #:get))
  ;; Killing something that has already exited is the ordinary case when a
  ;; later stage fails to start, so the raise is turned into a value here.
  (process-kill  (: System.Diagnostics.Process.Kill (-> Process void)
                    #:exceptions (System.InvalidOperationException
                                  System.NotSupportedException
                                  System.ComponentModel.Win32Exception)))

  (make-pipe    (: BjoPipe.Create (-> BjoPipe)))
  (pipe-input   (: BjoPipe.Reader (-> BjoPipe InputPort) #:get))
  (pipe-output  (: BjoPipe.Writer (-> BjoPipe OutputPort) #:get))

  ;; Not `double->string`, which is `.ToString()` and so follows the machine's
  ;; locale: on a Swedish one `(sleep 0.5)` would ask for `0,5`.
  (number->argument (: BjoProc.Argument (-> double string)))

  ;; Async pumps.
  (copy-stream  (: BjoProc.PumpAsync (-> InputPort OutputPort void) #:async))
  (copy-into    (: BjoProc.PumpIntoAsync (-> InputPort OutputPort void) #:async))
  (read-lines   (: BjoProc.ReadLinesAsync (-> InputPort (Array string)) #:async))
  (read-rest    (: Bjolang.Runtime.BjoInputPort.ReadToEndUtf8Async (-> InputPort string) #:async)))

;; A child's output as an input port over the pipe's own bytes, as Racket's
;; subprocess ports are: its text reads decode UTF-8, whatever encoding .NET
;; would have guessed the child writes.
(: (-> StreamReader InputPort))
(defun (output-port reader) (cast InputPort (PipePort. (reader-bytes reader))))

;; Nothing in the language builds an `Exception`, and every rejection below is
;; this module's own rather than .NET's, so they need a carrier.
(: (-> string Exception))
(defun (run-error message)
  (cast Exception (InvalidOperationException. message)))


;; ===========================================================================
;; What you write
;; ===========================================================================

;; A command written out, for when a program name is computed rather than
;; spelled: `,(cmd program "-l")`. An unquoted value has no literal shape and
;; is selected by its type, and `Cmd` is the only case carrying a record.
(type (: Cmd (Record (: program string) (: arguments (Vec string)))))

;; A filter stage.
(type (: Filter (-bjo-> InputPort OutputPort Unit)))

;; A form is one thing to run. You write it as a quoted literal, and this
;; grammar reads it. The keywords `pipe`, `from-file` and the four after them
;; name a case. The compiler checks the keyword, the number of items and each
;; item where you write the form. A list without one of these keywords is a
;; command, so `'(sort)` and `'(echo "x")` are commands. A keyword has
;; priority over a program name. To run a program with the name `pipe`, write
;; `"pipe"`.
(def/grammar Form
  (Form (:or (: FPipe      (pipe Form ...))
             (: FFrom      (from-file string Form))
             (: FInto      (into-file string Form))
             (: FAppend    (append-to-file string Form))
             (: FErrInto   (errors-into-file string Form))
             (: FErrAppend (errors-append-to-file string Form))
             (: FSym       symbol)         ;; a bare program name
             (: FStr       string)         ;; ...written as a string
             (: FArgv      (Vec string))  ;; ...or a computed argv
             (: FCmdValue  ,Cmd)           ;; a bare ,(cmd ...)
             (: FFilter    ,Filter)        ;; a bare ,filter
             (: FCmd       (Word ...+))))  ;; (echo "x")
  ;; The words of a command.
  (Word (:or (: WSym symbol) (: WStr string) (: WInt int) (: WDouble double)
             (: WArgv (Vec string)) (: WCmd ,Cmd) (: WFilter ,Filter)
             (: WForm (Word ...)))))

(: (-> string #:rest string Cmd))
(defun (cmd program #:rest arguments)
  (Cmd (program program)
       (arguments (loop (:for a arguments) (:acc out (vecing a))))))


;; ===========================================================================
;; What it parses to
;; ===========================================================================

;; Parsed AST node.
(type (: Redirection (Union RFrom RInto RAppend RErrInto RErrAppend)))

(type-rec
  (: Node (Union (: NCmd Cmd)
                 (: NFilter Filter)
                 (: NPipe (List Node))
                 (: NRedirect Redirect)))
  (: Redirect (Record (: how Redirection) (: path string) (: inner Node))))

(: (-> Word string))
(defun (word->text w)
  (match w
    ((WSym s) (symbol->string s))
    ((WStr s) (str "\"" s "\""))
    ((WInt n) (int->string n))
    ((WDouble x) (number->argument x))
    ((WArgv v) "a spliced vector")
    ((WCmd c) "an unquoted command")
    ((WFilter f) "an unquoted filter")
    ((WForm ws) "a nested form")))

(: (-> (Vec string) (Result Exception Node)))
(defun (command-of argv)
  (if (vec-empty? argv)
      (Err (run-error "a command with no program name"))
      (Ok (NCmd (Cmd (program (vec-ref argv 0))
                     (arguments (vec-slice argv 1 (- (vec-length argv) 1))))))))

;; One form, translated to a node. The grammar read each keyword where the
;; form was written. What is left is the shape of the pipeline and the words
;; of a command, which the grammar gives as a flat list.
(: (-> Form (Result Exception Node)))
(defun (form->node form)
  (match form
    ((FSym s) (command-of [(symbol->string s)]))
    ((FStr s) (command-of [s]))
    ((FArgv v) (command-of v))
    ((FCmdValue c) (Ok (NCmd c)))
    ((FFilter f) (Ok (NFilter f)))
    ((FCmd words) (parse-words words))
    ((FPipe forms) (parse-stages forms))
    ((FFrom path inner) (form->redirect RFrom path inner))
    ((FInto path inner) (form->redirect RInto path inner))
    ((FAppend path inner) (form->redirect RAppend path inner))
    ((FErrInto path inner) (form->redirect RErrInto path inner))
    ((FErrAppend path inner) (form->redirect RErrAppend path inner))))

(: (-> (List Form) (Result Exception Node)))
(defun (parse-stages forms)
  (match forms
    (Nil (Err (run-error "(pipe ...) needs at least one thing to run")))
    ((Cons only Nil) (form->node only))
    (_ (match (form->each forms)
         ((Err e) (Err e))
         ((Ok nodes) (Ok (NPipe nodes)))))))

(: (-> (List Form) (Result Exception (List Node))))
(defun (form->each forms)
  (match forms
    (Nil (Ok Nil))
    ((Cons form rest)
     (def (Ok node) (form->node form) :propagate)
     (def (Ok more) (form->each rest) :propagate)
     (Ok (Cons node more)))))

;; A redirection is its file name and the one form it wraps.
(: (-> Redirection string Form (Result Exception Node)))
(defun (form->redirect how path inner)
  (match (form->node inner)
    ((Err e) (Err e))
    ((Ok node) (Ok (NRedirect (Redirect (how how) (path path) (inner node)))))))

;; The words of one command, flattened. The first is the program and the rest
;; are its arguments, each one argument whatever it contains — nothing here
;; re-splits on whitespace.
(: (-> string))
(defun (tag-names)
  "pipe, from-file, into-file, append-to-file, errors-into-file, errors-append-to-file")

;; Adds a hint about the keywords to an error.
(: (-> (List Word) Exception Exception))
(defun (with-tag-hint ws original)
  (match ws
    ((Cons (WSym s) rest)
     (run-error (str (.-Message original)
                     " If `" (symbol->string s) "` was meant as a keyword, the keywords are: "
                     (tag-names) ".")))
    (_ original)))

(: (-> (List Word) (Result Exception Node)))
(defun (parse-command ws)
  (match (words->argv ws (vec-empty))
    ((Err e) (Err (with-tag-hint ws e)))
    ((Ok argv) (command-of argv))))

(: (-> (List Word) (Vec string) (Result Exception (Vec string))))
(defun (words->argv ws acc)
  (match ws
    (Nil (Ok acc))
    ((Cons w rest)
     (match w
       ((WSym s) (words->argv rest (vec-add acc (symbol->string s))))
       ((WStr s) (words->argv rest (vec-add acc s)))
       ((WInt n) (words->argv rest (vec-add acc (int->string n))))
       ((WDouble x) (words->argv rest (vec-add acc (number->argument x))))
       ((WArgv v) (words->argv rest (vec-append acc v)))
       (_ (Err (run-error
                (str "a command's words have to be plain — this one contains "
                     (word->text w)
                     ", which belongs in a (pipe ...) rather than among arguments."))))))))



;; A form written as a list is a command, or one thing that is a whole stage on
;; its own: `'(,my-filter)` and `'(,(cmd ...))`. The grammar read the keywords
;; before this runs, so a list's head is never one here — `'(sort)` and
;; `'(echo "x")` are commands, and `'(pipe ...)` never reaches this at all.
(: (-> (List Word) (Result Exception Node)))
(defun (parse-words ws)
  (match ws
    (Nil (Err (run-error "an empty form: there is nothing here to run")))
    ((Cons head rest)
     (match head
       ((WFilter f) (stage-alone (NFilter f) rest "a filter"))
       ((WCmd c) (stage-alone (NCmd c) rest "an unquoted ,(cmd ...)"))
       ((WForm ws)
        (Err (run-error
              "this form begins with another form. To run several things in turn, name it: (pipe (cat \"a\") (wc -l))")))
       ((WSym s) (parse-command ws))
       ((WStr s) (parse-command ws))
       ((WArgv v) (parse-command ws))
       (_ (Err (run-error
                (str "a program name has to be a word, and this form begins with "
                     (word->text head)))))))))

;; A form that is one thing already carries everything it needs, so a word
;; after it is a pipeline nobody named.
(: (-> Node (List Word) string (Result Exception Node)))
(defun (stage-alone node rest what)
  (match rest
    (Nil (Ok node))
    (_ (Err (run-error
             (str what
                  " is a whole stage on its own, so nothing may follow it here. To run several things in turn, name it: (pipe ...)"))))))


;; ===========================================================================
;; Running it
;; ===========================================================================

;; Started process state.
(type
  (: Proc #:opaque
     (Record (: input (Option OutputPort))
             (: output (Option InputPort))
             (: children (Vec Process))
             (: exits (Vec (Promise Unit)))
             (: waits (Vec (Promise Unit))))))

;; One node, wired but not yet connected to its neighbours. A command and a
;; filter differ only in where `sink` and `source` come from, which is what
;; makes the pipeline loop have no cases in it.
(type
  (: Wired #:opaque
     (Record (: sink (Option OutputPort))
             (: source (Option InputPort))
             (: children (Vec Process))
             (: exits (Vec (Promise Unit)))
             (: waits (Vec (Promise Unit))))))

;; Waits for child exit once.
(: (-> Process Unit))
(defbjo (awaiting-exit child)
  (process-await child)
  unit)

(: (-> Cmd (Option OutputPort) (Result Exception Wired)))
(defun (start-command c errors)
  (def settings (ProcessStartInfo.))
  (settings-program! settings (record-ref c program))
  (loop (:for arg (record-ref c arguments))
        (:do (ignore (settings-argument settings arg))))
  ;; `UseShellExecute` off is what makes redirection legal at all, and also
  ;; what makes the program name resolve against PATH rather than against a
  ;; desktop association.
  (settings-use-shell! settings #f)
  (settings-redirect-input! settings #t)
  (settings-redirect-output! settings #t)
  (when (some? errors)
    (settings-redirect-errors! settings #t))
  (match (process-start settings)
    ((Err e) (Err e))
    ((Ok child)
     (Ok (Wired (sink (Some (child-input child)))
                (source (Some (output-port (process-stdout child))))
                (children [child])
                (exits [(bjo (awaiting-exit child))])
                ;; Handle stderr copying.
                (waits (match errors
                         ((Some w) [(bjo (pump-into (output-port (process-stderr child)) w))])
                         (None (vec-empty)))))))))

;; Two pipes, because a filter has no ends of its own: one for what is written
;; to it and one for what it writes.
(: (-> Filter Wired))
(defun (start-filter f)
  (def in (make-pipe))
  (def out (make-pipe))
  (def done (bjo (f (pipe-input in) (pipe-output out))))
  (Wired (sink (Some (pipe-output in)))
         (source (Some (pipe-input out)))
         (children (vec-empty))
         (exits (vec-empty))
         (waits [(bjo (close-after done (pipe-output out)))])))

;; Closes and raises errors.
(: (-> (Promise Unit) OutputPort Unit))
(defbjo (close-after done out)
  (match (sync (promise-join done))
    ((Ok u) (close-output-port out) unit)
    ((Err e) (close-output-port out) (raise e))))

;; `PumpAsync` closes its target on exhaustion, which is the whole reason it is
;; a junction: an unclosed stdin leaves the process downstream waiting forever.
(: (-> InputPort OutputPort Unit))
(defbjo (pump from to)
  (copy-stream from to)
  unit)

;; The shared-destination pump. Closing is `close-when-all-done`'s job.
(: (-> InputPort OutputPort Unit))
(defbjo (pump-into from to)
  (copy-into from to)
  unit)

(: (-> (Vec (Promise Unit)) OutputPort Unit))
(defbjo (close-when-all-done ps out)
  ;; The failure is already carried by one of `ps`, which is in this form's
  ;; `waits` too, so it is not re-raised here — only the file is closed.
  (ignore (join-all ps))
  (close-output-port out)
  unit)

(: (-> (Vec Process) void))
(defun (kill-all children)
  (loop (:for child children)
        (:do (ignore (process-kill child)))))

;; Undoes a form that was wired and will not run, because a stage or a file
;; after it failed. Its processes are killed and its two ends closed: a filter
;; then reads to the end of its input and finishes, where it would otherwise
;; wait for input that never comes, and the scope that started it would wait
;; for it.
(: (-> Wired void))
(defun (abandon w)
  (kill-all (record-ref w children))
  (when-let (to (record-ref w sink)) (close-output-port to))
  (when-let (from (record-ref w source)) (close-input-port from)))

;; Connect wired stages front to back, pumping each one's output into the next
;; one's input.
(: (-> (List Node) (Option OutputPort) (Result Exception Wired)))
(defun (join-stages nodes errors)
  (match nodes
    (Nil (Err (run-error "a pipeline with no stages")))
    ((Cons first rest)
     (match (wire first errors)
       ((Err e) (Err e))
       ((Ok head) (join-onto head rest errors))))))

(: (-> Wired (List Node) (Option OutputPort) (Result Exception Wired)))
(defun (join-onto joined nodes errors)
  (match nodes
    (Nil (Ok joined))
    ((Cons node more)
     (match (wire node errors)
       ;; Whatever is already running would otherwise sit on a stdin nobody
       ;; will ever close.
       ((Err e) (abandon joined) (Err e))
       ((Ok w)
        (match (junction (record-ref joined source) (record-ref w sink))
          ((Err e)
           (abandon joined)
           (abandon w)
           (Err e))
          ((Ok link)
           (join-onto
            (record-set joined
                        (source (record-ref w source))
                        (children (vec-append (record-ref joined children) (record-ref w children)))
                        (exits (vec-append (record-ref joined exits) (record-ref w exits)))
                        (waits (vec-add (vec-append (record-ref joined waits) (record-ref w waits)) link)))
            more
            errors))))))))

;; Creates stream junction.
(: (-> (Option InputPort) (Option OutputPort) (Result Exception (Promise Unit))))
(defun (junction upstream downstream)
  (match upstream
    (None (Err (run-error
                "the stage before this one has its output redirected to a file, so there is nothing to pipe onwards")))
    ((Some from)
     (match downstream
       (None (Err (run-error
                   "this stage has its input redirected from a file, so nothing can be piped into it")))
       ((Some to) (Ok (bjo (pump from to))))))))

;; Through the prelude's `FS` effect, as every file is opened, so a handler
;; that answers for the disk answers for a redirect too.
(: (-> string bool (Result Exception OutputPort)))
(defun (open-for-writing path append)
  (match (open-write-stream path (if append Append Truncate))
    ((Err e) (Err e))
    ((Ok stream) (Ok (cast OutputPort (PortOver. stream #t))))))

;; A child's standard input as a port: unbuffered, as .NET's writer over it
;; was, because what reads the other end is another process and there is no
;; later moment at which we would know to flush. Closing it closes the pipe,
;; which is how the child sees the end of its input.
(: (-> Process OutputPort))
(defun (child-input child)
  (def port (cast OutputPort (PortOver. (writer-bytes (process-stdin child)) #t)))
  (port-mode! port unbuffered)
  port)

(: (-> Node (Option OutputPort) (Result Exception Wired)))
(defun (wire node errors)
  (match node
    ((NCmd c) (start-command c errors))
    ((NFilter f) (Ok (start-filter f)))
    ((NPipe nodes) (join-stages nodes errors))
    ((NRedirect r) (wire-redirect r errors))))

(: (-> Redirect (Option OutputPort) (Result Exception Wired)))
(defun (wire-redirect r errors)
  (def path (record-ref r path))
  (def inner (record-ref r inner))
  (match (record-ref r how)
    (RFrom
     (def (Ok file) (open-input-file path) :propagate)
     (redirect-input inner errors file))
    (RInto (redirect-output inner errors path #f))
    (RAppend (redirect-output inner errors path #t))
    (RErrInto
     (def (Ok file) (open-for-writing path #f) :propagate)
     (redirect-errors inner file))
    (RErrAppend
     (def (Ok file) (open-for-writing path #t) :propagate)
     (redirect-errors inner file))))

;; The file replaces the caller as the thing writing in, so `sink` goes away.
;; A form that does not start leaves the file closed.
(: (-> Node (Option OutputPort) InputPort (Result Exception Wired)))
(defun (redirect-input inner errors file)
  (match (wire inner errors)
    ((Err e) (close-input-port file) (Err e))
    ((Ok w)
     (match (record-ref w sink)
       (None (abandon w)
             (close-input-port file)
             (Err (run-error "this form already takes its input from somewhere, so it cannot also take it from a file")))
       ((Some to)
        (Ok (record-set w
                        (sink None)
                        (waits (vec-add (record-ref w waits) (bjo (pump file to)))))))))))

;; ...and the file replaces the caller as the thing reading out, so `source`
;; goes away. The file is opened once the form has started and has an output
;; to give it, since opening it to write truncates it: a form that does not
;; start leaves the file as it was.
(: (-> Node (Option OutputPort) string bool (Result Exception Wired)))
(defun (redirect-output inner errors path append)
  (def (Ok w) (wire inner errors) :propagate)
  (match (record-ref w source)
    (None (abandon w)
          (Err (run-error "this form already sends its output somewhere, so it cannot also send it to a file")))
    ((Some from)
     (match (open-for-writing path append)
       ((Err e) (abandon w) (Err e))
       ((Ok file)
        (Ok (record-set w
                        (source None)
                        (waits (vec-add (record-ref w waits) (bjo (pump from file)))))))))))

;; Standard error is not a stream this module wires between stages — every
;; command under here writes into the one file — so neither end changes.
;;
;; Unlike `redirect-output`, the file is opened before the form starts: it is
;; the error sink each command is started with. A form that does not start
;; leaves it closed, but already truncated.
(: (-> Node OutputPort (Result Exception Wired)))
(defun (redirect-errors inner file)
  ;; Every command under here writes to the one file from a fiber of its own;
  ;; a port takes each write whole, so they can share it as it is.
  (def shared file)

  (match (wire inner (Some shared))
    ((Err e) (close-output-port shared) (Err e))
    ((Ok w)
     (Ok (record-set w
                     (waits (vec-add (record-ref w waits)
                                     (bjo (close-when-all-done (record-ref w waits) shared)))))))))

;; Parses and runs form.
(: (-> Form (Result Exception Proc)))
(defun (run form)
  (def (Ok node) (form->node form) :propagate)
  (def (Ok w) (wire node None) :propagate)
  (Ok (Proc (input (record-ref w sink))
            (output (record-ref w source))
            (children (record-ref w children))
            (exits (record-ref w exits))
            (waits (record-ref w waits)))))

;; Where to write the form's input, unless a `from-file` took it. Closing it is
;; what ends the first stage.
(: (-> Proc (Option OutputPort)))
(defun (proc-input p) (record-ref p input))

;; Where to read the form's output, unless an `into-file` took it. It has to be
;; drained, or the last stage blocks writing and `wait` never returns.
(: (-> Proc (Option InputPort)))
(defun (proc-output p) (record-ref p output))

;; ---------------------------------------------------------------------------
;; Waiting, written once and seen from two sides
;; ---------------------------------------------------------------------------
;;
;; Checks if process finished.

(: (-> (Vec (Promise Unit)) (Result Exception Unit)))
(defbjo (join-all ps)
  (let go ((i 0))
    (if (= i (vec-length ps))
        (Ok unit)
        (match (sync (promise-join (vec-ref ps i)))
          ((Err e) (Err e))
          ((Ok u) (go (+ i 1)))))))

;; Awaits process completion.
(: (-> Proc (Result Exception int)))
(defbjo (waiting p)
  (def (Ok _) (join-all (record-ref p exits)) :propagate)
  (def (Ok _) (join-all (record-ref p waits)) :propagate)
  (def children (record-ref p children))
  (Ok (if (vec-empty? children)
          0
          (process-code (vec-ref children (- (vec-length children) 1))))))

(: (-> Proc (Result Exception (Vec int))))
(defbjo (waiting/all p)
  (def (Ok _) (join-all (record-ref p exits)) :propagate)
  (def (Ok _) (join-all (record-ref p waits)) :propagate)
  (Ok (vec-map (fun (c) (process-code c)) (record-ref p children))))

;; Gets blocking result.
(: blocking-result #:sync (-> (Promise (Result Exception %a)) (Result Exception %a)))
(defun (blocking-result p)
  (match (sync/blocking (promise-join p))
    ((Err e) (Err e))
    ((Ok answer) answer)))

;; Waits for last command exit.
(: (-> Proc (Result Exception int)))
(defun (wait p) (blocking-result (bjo (waiting p))))

;; Every command's exit code, in the order they were started. Filters
;; contribute nothing — their failures arrive as the `Err` instead.
(: (-> Proc (Result Exception (Vec int))))
(defun (wait/all p) (blocking-result (bjo (waiting/all p))))

;; Returns process exit event.
(: (-> Proc (Event (Result Exception int))))
(defun (exited p)
  (wrap (promise-join (bjo (waiting p)))
        (fun (r) (match r ((Err e) (Err e)) ((Ok answer) answer)))))

;; Stop everything. The pumps end on their own once the streams they are
;; copying do.
(: (-> Proc void))
(defun (kill p) (kill-all (record-ref p children)))

;; A background job added to a form after it has started, so that `wait` and
;; `exited` cover it too. `run/channel`'s reader is the one caller.
(: (-> Proc (Promise Unit) Proc))
(defun (with-wait p extra)
  (record-set p (waits (vec-add (record-ref p waits) extra))))


;; ===========================================================================
;; The whole thing at once
;; ===========================================================================
;;
;; --- Process runners ---

(: (-> Proc void))
(defun (close-input p)
  (match (proc-input p)
    ((Some w) (close-unbuffered w))
    (None unit)))

(: (-> Form (Result Exception string)))
(defun (run/string form)
  (match (run form)
    ((Err e) (Err e))
    ((Ok p)
     (close-input p)
     (blocking-result (bjo (collecting p))))))

;; Read the output, then wait. The order is what matters: a last stage blocked
;; on a full pipe never exits, so waiting first would wait forever.
(: (-> Proc (Result Exception string)))
(defbjo (collecting p)
  (let ((text (match (proc-output p)
                ((Some out) (read-rest out))
                (None ""))))
    (match (waiting p)
      ((Err e) (Err e))
      ((Ok code) (Ok text)))))

;; The output and the exit code, together.
;;
;; `run/string` answers the output and drops the code, which is enough for a
;; command that is expected to work. It is not enough for one whose failure is
;; an ordinary answer — `git show v1.0.0:manifest.bjodat` on a tag that has no
;; manifest prints nothing and exits non-zero, and that is not the same as a
;; tag whose manifest is empty. Both halves are here so that the caller does
;; not have to run the thing twice to get them.
(: (-> Form (Result Exception (Tuple int string))))
(defun (run/output form)
  (match (run form)
    ((Err e) (Err e))
    ((Ok p)
     (close-input p)
     (blocking-result (bjo (collecting/code p))))))

(: (-> Proc (Result Exception (Tuple int string))))
(defbjo (collecting/code p)
  (let ((text (match (proc-output p)
                ((Some out) (read-rest out))
                (None ""))))
    (match (waiting p)
      ((Err e) (Err e))
      ((Ok code) (Ok (Tuple code text))))))

(: (-> Form (Result Exception (Vec string))))
(defun (run/strings form)
  (match (run form)
    ((Err e) (Err e))
    ((Ok p)
     (close-input p)
     (blocking-result (bjo (collecting/lines p))))))

(: (-> Proc (Result Exception (Vec string))))
(defbjo (collecting/lines p)
  (let ((lines (match (proc-output p)
                 ((Some out) (loop (:for l (read-lines out)) (:acc v (vecing l))))
                 (None (vec-empty)))))
    (match (waiting p)
      ((Err e) (Err e))
      ((Ok code) (Ok lines)))))

;; Runs for status code.
(: (-> Form (Result Exception int)))
(defun (run/status form)
  (match (run form)
    ((Err e) (Err e))
    ((Ok p)
     (close-input p)
     (blocking-result (bjo (draining p))))))

(: (-> Proc (Result Exception int)))
(defbjo (draining p)
  (match (proc-output p)
    ((Some out) (ignore (read-rest out)))
    (None unit))
  (waiting p))


;; ===========================================================================
;; A form as a channel
;; ===========================================================================

;; Reader streams items to channel.
(type (: (Reader %a) (-bjo-> InputPort (Chan %a) Unit)))

;; Runs and streams to channel.
(: (-> Form (Reader %a) (Result Exception (Tuple (Chan %a) Proc))))
(defun (run/channel form read)
  (match (run form)
    ((Err e) (Err e))
    ((Ok p)
     (let ((items (make-chan)))
       (match (proc-output p)
         ;; Handles redirect output.
         (None (kill p)
               (Err (run-error
                     "this form sends its output to a file, so there is nothing left to read onto a channel")))
         ((Some out)
          (Ok (Tuple items
                     (with-wait p (bjo (read out items)))))))))))

;; The stock `Reader`: one item per line, `\r\n` and a missing last newline
;; handled by `read-line`'s default mode rather than by guessing. Each line is
;; sent as soon as it is read: a program whose output does not end, `tail -f`,
;; still delivers its lines, and the output is never held whole.
(: (Reader string))
(defbjo (by-lines out items)
  (let next ()
    (match (read-line/opt out)
      ((Some line)
       (sync (chan-send items line))
       (next))
      (None unit))))


;; ===========================================================================
;; A form as a stream
;; ===========================================================================

;; The items `reader` reads from a form's output, as a `Stream`. Each loop
;; over it runs the form afresh.
;;
;; - The form's input is closed at once. Each item is read when the loop asks
;;   for it.
;; - The exit code is dropped, as `run/strings` drops it.
;; - Lifecycle: the form is killed however the loop ends, early included.
;;   When the loop returns, its processes have exited. See `reaping`.
;; - Raises in the loop: when the form cannot start, its output goes to a
;;   file, a filter or a redirection fails, or `reader` raises.
(: (-> Form (-bjo-> InputPort %a) (Stream %a)))
(defun (run/stream form reader)
  (make-stream (bjoroutine (out) (stream-form form reader out))))

(: (-> Form (-bjo-> InputPort %a) (Emitter %a) void))
(defbjo (stream-form form reader out)
  (match (run form)
    ((Err e) (raise e))
    ((Ok p)
     (close-input p)
     (try
       (match (proc-output p)
         (None (raise (run-error "this form sends its output to a file, so there is nothing left to stream")))
         ((Some port)
          (let go ()
            (unless (port-eof? port)
              (emit! out (reader port))
              (go)))
          (match (waiting p)
            ((Err e) (raise e))
            ((Ok _) unit))))
       #:finally (reaping p)))))

;; The longest time `reaping` waits for a killed form's processes to exit.
(def reap-grace (seconds 5))

;; Awaits the exit of each of `children`.
(: (-> (Vec Process) Unit))
(defbjo (awaiting-exits children)
  (loop (:for child children) (:do (process-await child)))
  unit)

;; Kills the form, and waits until its processes have exited.
;;
;; - After this returns, the processes are dead and reaped, unless they took
;;   longer than `reap-grace` to exit.
;; - The wait runs in a shield, so it also works in a fiber that has been
;;   cancelled.
;; - Does not raise when the grace passes.
;; - It waits for the processes only, not for the form's pumps and filters.
;;   A pump can wait for a pipe that a grandchild process still holds open.
(: (-> Proc void))
(defbjo (reaping p)
  (kill p)
  ;; The shield's deadline is longer than `reap-grace`, so the timeout always
  ;; commits first and the shield never raises `Cancelled`. The inner scope
  ;; cancels the awaits that are still waiting when the grace passes.
  (with-shield #:within (seconds 6)
    (with-cancel (stop)
      (def exited (bjo (awaiting-exits (record-ref p children))))
      (ignore (sync (choose (wrap (promise-join exited) (fun (_) unit)) (timeout reap-grace))))
      (ignore (stop (Requested "the grace for the form to exit has passed"))))))


;; ===========================================================================
;; A form as one event
;; ===========================================================================

;; Stops on event.
(: (-> (Event Unit) Proc (Promise (Result Exception string)) Unit))
(defbjo (reap-on-loss nack p done)
  (when (sync (choose (wrap nack (fun (u) #t))
                      (wrap (promise-join done) (fun (r) #f))))
    (kill p))
  unit)

;; Wraps process in event.
(: (-> Form (Event (Result Exception string))))
(defun (running form)
  (with-nack
    (fun (nack)
      (match (run form)
        ((Err e) (always (Err e)))
        ((Ok p)
         (close-input p)
         (let ((done (bjo (collecting p))))
           (ignore (bjo (reap-on-loss nack p done)))
           (wrap (promise-join done)
                 (fun (r) (match r ((Err e) (Err e)) ((Ok answer) answer))))))))))


;; ===========================================================================
;; Scoped
;; ===========================================================================

;; Runs form scoped to body.
(def/macro (with-run form inject compare)
  (syntax-match form
    ((_ (name spec) first body ...)
     ;; `name` came from the input, so hygiene leaves it alone and the body sees
     ;; it. `started` is the template's and is renamed apart.
     #'(let ()
         (def started (run ,spec))
         (try
           (let ((,name started))
             ,first
             ,@body)
           #:finally (match started
                       ((Ok p) (kill p))
                       ((Err e) unit)))))
    (bad (syntax-error bad "with-run names a form and takes a body: (with-run (p '(ls \"-l\")) body ...)"))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/rx.bjo — regular expressions in s-expression notation.
;;
;;     #rx(seq :bos (+ :digit) (lit "-") ($ (** 4 4 :digit)) :eos)
;;
;; `#rx(...)` is read, checked and compiled to a pattern at compile time; a bad
;; pattern is a compile error pointing at the sub-form that is wrong. What the
;; expansion evaluates is `(rx-compile "<pattern>" "<names>")`.
;;
;; Anything that takes no argument is a keyword — `:bos`,
;; `:any`, `:digit`. Anything that takes one is a symbol heading a form —
;; `(seq ...)`, `(+ ...)`, `(in ...)`. A string or a character literal matches
;; itself: a literal sequence in pattern position, the set of its characters
;; inside `(in ...)`.
;;
;; The engine is BjoRx (`BjolangRuntime/BjoRx`): linear time, leftmost-first,
;; over the string's UTF-8 as it is. It has no backreferences, lookarounds,
;; atomic groups or conditionals. This to not tie (std rx)
;; to a backtracking engine for good, and nothing here can emit one.
;;
;; The pattern is BjoRx's tree as text (`BjoRx.Syntax.HirText`): literals,
;; classes as scalar ranges, sequence, alternation, repetition, groups and the
;; two anchors, nothing else. Classes are written out from the tables at the
;; bottom of this file, so `:digit` means the same thing on every runtime and
;; the meaning is readable here. The alphabet is the Unicode scalar, as it is
;; for `char` and the string cursors.

(export
  ;; The types.
  Rx RxMatch

  ;; Matching.
  rx-match rx-match? rx-search rx-search? rx-matches
  rx-replace rx-replace-with rx-split

  ;; Reading a match.
  rx-text rx-input rx-start rx-end
  rx-group rx-group-start rx-group-end rx-group/n rx-group-count

  ;; What `#rx(...)` expands into, and what a test reads back.
  rx-compile rx-pattern)

(import (std prelude))
(import (std syntax-match))

;; Every export is documented there; the module doc is too.
(:doc #:file "rx.bjodoc")

(import/class
  (Rx      (: Bjolang.Runtime.BjoRegex))
  (RxMatch (: Bjolang.Runtime.BjoMatch)))

(import/extern
  (clr-rx-compile   (: Bjolang.Runtime.BjoRegexModule.Compile          (-> string string Rx)))
  (clr-rx-pattern   (: Bjolang.Runtime.BjoRegexModule.Pattern          (-> Rx string)))
  (clr-rx-search    (: Bjolang.Runtime.BjoRegexModule.Search           (-> Rx string (Option RxMatch))))
  (clr-rx-search?   (: Bjolang.Runtime.BjoRegexModule.IsSearchMatch    (-> Rx string bool)))
  (clr-rx-whole     (: Bjolang.Runtime.BjoRegexModule.MatchWhole       (-> Rx string (Option RxMatch))))
  (clr-rx-whole?    (: Bjolang.Runtime.BjoRegexModule.IsWholeMatch     (-> Rx string bool)))
  (clr-rx-matches   (: Bjolang.Runtime.BjoRegexModule.Matches          (-> Rx string (Array RxMatch))))
  (clr-rx-replace   (: Bjolang.Runtime.BjoRegexModule.Replace          (-> Rx string string string)))
  (clr-rx-replace-with
   (: Bjolang.Runtime.BjoRegexModule.ReplaceWith      (-> Rx string (-> RxMatch string) string)))
  (clr-rx-split     (: Bjolang.Runtime.BjoRegexModule.Split            (-> Rx string (Array string))))
  (clr-rx-text      (: Bjolang.Runtime.BjoRegexModule.Text             (-> RxMatch string)))
  (clr-rx-input     (: Bjolang.Runtime.BjoRegexModule.Input            (-> RxMatch string)))
  (clr-rx-start     (: Bjolang.Runtime.BjoRegexModule.Start            (-> RxMatch StringCursor)))
  (clr-rx-end       (: Bjolang.Runtime.BjoRegexModule.End              (-> RxMatch StringCursor)))
  (clr-rx-count     (: Bjolang.Runtime.BjoRegexModule.Count            (-> RxMatch int)))
  (clr-rx-group-text  (: Bjolang.Runtime.BjoRegexModule.GroupText      (-> RxMatch int (Option string))))
  (clr-rx-group-start (: Bjolang.Runtime.BjoRegexModule.GroupStart     (-> RxMatch int (Option StringCursor))))
  (clr-rx-group-end   (: Bjolang.Runtime.BjoRegexModule.GroupEnd       (-> RxMatch int (Option StringCursor))))
  (clr-rx-index-of    (: Bjolang.Runtime.BjoRegexModule.MatchIndexOfName (-> RxMatch string int))))

;; ---------------------------------------------------------------------------
;; The runtime surface
;; ---------------------------------------------------------------------------
;;
;; Wrappers rather than the `import/extern` aliases directly, because a macro
;; template naming one of these resolves it to this module by the second
;; renaming rule, and there has to be a def of this module to resolve to.
;; TODO i guess.
;;  
;;
;; Compiles a pattern. `names` is the Bjolang name of each capture group in
;; order, comma-separated — a group `($ ...)` left unnamed contributes an empty
;; one. Commas cannot occur in a keyword, so it wont' be a part of a name.
(: (-> string string Rx))
(defun (rx-compile pattern names) (clr-rx-compile pattern names))

;; The pattern text this was compiled to. For testing purposes. 
(: (-> Rx string))
(defun (rx-pattern rx) (clr-rx-pattern rx))

;; The leftmost match anywhere in the string.
(: (-> Rx string (Option RxMatch)))
(defun (rx-search rx s) (clr-rx-search rx s))

(: (-> Rx string bool))
(defun (rx-search? rx s) (clr-rx-search? rx s))

;; The match against the whole string, anchored at both ends.
(: (-> Rx string (Option RxMatch)))
(defun (rx-match rx s) (clr-rx-whole rx s))

(: (-> Rx string bool))
(defun (rx-match? rx s) (clr-rx-whole? rx s))

;; Every non-overlapping match, left to right.
(: (-> Rx string (Vec RxMatch)))
(defun (rx-matches rx s) (array->vec (clr-rx-matches rx s)))

;; Every match replaced by `replacement`, which is literal text: there is no
;; `$1` substitution syntax. Use `rx-replace-with` to build the
;; replacement from the match.
(: (-> Rx string string string))
(defun (rx-replace rx s replacement) (clr-rx-replace rx s replacement))

(: (-> Rx string (-> RxMatch string) string))
(defun (rx-replace-with rx s f) (clr-rx-replace-with rx s f))

;; The pieces between the matches. An empty match returns []
(: (-> Rx string (Vec string)))
(defun (rx-split rx s) (array->vec (clr-rx-split rx s)))

(: (-> RxMatch string))
(defun (rx-text m) (clr-rx-text m))

(: (-> RxMatch string))
(defun (rx-input m) (clr-rx-input m))

(: (-> RxMatch StringCursor))
(defun (rx-start m) (clr-rx-start m))

(: (-> RxMatch StringCursor))
(defun (rx-end m) (clr-rx-end m))

(: (-> RxMatch int))
(defun (rx-group-count m) (clr-rx-count m))

;; The text of the group `(=> :name ...)` named, or `None` if that group took
;; no part in the match. An unknown name is `None` as well.
(: (-> RxMatch Keyword (Option string)))
(defun (rx-group m name) (clr-rx-group-text m (clr-rx-index-of m (keyword->string name))))

(: (-> RxMatch Keyword (Option StringCursor)))
(defun (rx-group-start m name) (clr-rx-group-start m (clr-rx-index-of m (keyword->string name))))

(: (-> RxMatch Keyword (Option StringCursor)))
(defun (rx-group-end m name) (clr-rx-group-end m (clr-rx-index-of m (keyword->string name))))

;; The text of the nth group, counted from zero in the order the groups open.
(: (-> RxMatch int (Option string)))
(defun (rx-group/n m n) (clr-rx-group-text m n))

;; ---------------------------------------------------------------------------
;; Character sets
;; ---------------------------------------------------------------------------
;;
;; A set is a sorted, disjoint list of inclusive scalar ranges. It never
;; contains a surrogate code point: `BjoChar` rejects those, so no set the
;; language can write holds one.

(type
  (: Cset (Record (: ranges (List (Tuple int int))))))

;; U+0000–U+D7FF and U+E000–U+10FFFF: every scalar.
(def universe (list (Tuple 0 55295) (Tuple 57344 1114111)))

(: (-> (Tuple int int) int))
(defun (range-lo r) (match r ((Tuple lo _) lo)))

(: (-> (Tuple int int) int))
(defun (range-hi r) (match r ((Tuple _ hi) hi)))

(: (-> int int int))
(defun (imin a b) (if (< a b) a b))

(: (-> int int int))
(defun (imax a b) (if (> a b) a b))

(: (-> (Tuple int int) (List (Tuple int int)) (List (Tuple int int))))
(defun (ranges-insert r sorted)
  (cond
    ((list-empty? sorted) (list r))
    ((<= (range-lo r) (range-lo (list-head sorted)))
     (Cons r sorted))
    (else (Cons (list-head sorted) (ranges-insert r (list-tail sorted))))))

;; Insertion sort: a set written by hand has a handful of ranges, and a table
;; read out of the hex below is already ascending, so this is the best case
;; either way.
(: (-> (List (Tuple int int)) (List (Tuple int int))))
(defun (ranges-sort rs)
  (match rs
    (Nil Nil)
    ((Cons r more) (ranges-insert r (ranges-sort more)))))

;; Adjacent as well as overlapping runs are merged, so that `[a-b][c-d]` with
;; `c = b + 1` comes out as one range and two sets built differently compare
;; the same.
(: (-> (List (Tuple int int)) (List (Tuple int int))))
(defun (ranges-merge rs)
  (match rs
    (Nil Nil)
    ((Cons r Nil) (list r))
    ((Cons a (Cons b more))
     (if (<= (range-lo b) (+ (range-hi a) 1))
         (ranges-merge (Cons (Tuple (range-lo a) (imax (range-hi a) (range-hi b))) more))
         (Cons a (ranges-merge (Cons b more)))))))

(: (-> (List (Tuple int int)) (List (Tuple int int))))
(defun (ranges-norm rs) (ranges-merge (ranges-sort rs)))

;; The parts of `lo`–`hi` that `rs` does not cover.
(: (-> int int (List (Tuple int int)) (List (Tuple int int))))
(defun (ranges-gaps lo hi rs)
  (match rs
    (Nil (if (<= lo hi) (list (Tuple lo hi)) Nil))
    ((Cons r more)
     (def before (if (< lo (range-lo r)) (list (Tuple lo (imin hi (- (range-lo r) 1)))) Nil))
     (list-append before (ranges-gaps (imax lo (+ (range-hi r) 1)) hi more)))))

(: (-> (List (Tuple int int)) (List (Tuple int int)) (List (Tuple int int))))
(defun (ranges-inter a b)
  (if (or (list-empty? a) (list-empty? b))
      Nil
      (let ((x (list-head a)) (y (list-head b)))
        (def lo (imax (range-lo x) (range-lo y)))
        (def hi (imin (range-hi x) (range-hi y)))
        (def rest (if (< (range-hi x) (range-hi y))
                      (ranges-inter (list-tail a) b)
                      (ranges-inter a (list-tail b))))
        (if (<= lo hi) (Cons (Tuple lo hi) rest) rest))))

;; Normalizing and clipping to the universe here is what lets every other
;; operation assume both. What does that even mean, Claude?
(: (-> (List (Tuple int int)) Cset))
(defun (cs-make rs)
  (Cset (ranges (ranges-inter (ranges-norm rs) universe))))

(: (-> Cset (List (Tuple int int))))
(defun (cs-ranges c) (record-ref c ranges))

(: (-> Cset Cset Cset))
(defun (cs-union a b)
  (cs-make (list-append (cs-ranges a) (cs-ranges b))))

(: (-> Cset Cset Cset))
(defun (cs-inter a b)
  (cs-make (ranges-inter (cs-ranges a) (cs-ranges b))))

(: (-> Cset Cset))
(defun (cs-complement a)
  (cs-make (ranges-gaps 0 1114111 (cs-ranges a))))

(: (-> Cset Cset Cset))
(defun (cs-diff a b) (cs-inter a (cs-complement b)))

(: (-> Cset bool))
(defun (cs-empty? c) (list-empty? (cs-ranges c)))

(: (-> int Cset))
(defun (cs-char n) (cs-make (list (Tuple n n))))

;; ---------------------------------------------------------------------------
;; Reading the tables
;; ---------------------------------------------------------------------------
;;
;; A table is one hex string, four digits per endpoint and eight per inclusive
;; range, rather than a list of literal tuples. Six hundred ranges written out
;; as forms is six hundred forms for the compiler to read on every build of
;; this module, and the answer is the same every time.

(: (-> char int))
(defun (hex-val c)
  (def n (char->int c))
  (cond ((and (>= n 48) (<= n 57)) (- n 48))
        ((and (>= n 65) (<= n 70)) (+ 10 (- n 65)))
        (else 0)))

(: (-> char char char char int))
(defun (hex4-val a b c d)
  (+ (* 4096 (hex-val a))
     (+ (* 256 (hex-val b))
        (+ (* 16 (hex-val c)) (hex-val d)))))

(: (-> (List char) (List (Tuple int int))))
(defun (hex-chunks cs)
  (match cs
    ((Cons a (Cons b (Cons c (Cons d (Cons e (Cons f (Cons g (Cons h rest))))))))
     (Cons (Tuple (hex4-val a b c d) (hex4-val e f g h)) (hex-chunks rest)))
    (_ Nil)))

(: (-> string (List (Tuple int int))))
(defun (hex-ranges text) (hex-chunks (string->list text)))

(: (-> string Cset))
(defun (table-set text) (cs-make (hex-ranges text)))

;; ---------------------------------------------------------------------------
;; The tree
;; ---------------------------------------------------------------------------

(type
  (: Rxn (Union
          RxEmpty
          (: RxLit string)
          (: RxSet Cset)
          (: RxSeq (List Rxn))
          (: RxAlt (List Rxn))
          ;; body, least, most (-1 for unbounded), greedy
          (: RxRep Rxn int int bool)
          ;; the name the user gave, "" for `($ ...)`
          (: RxGroup string Rxn)
          RxBos
          RxEos)))

;; ---------------------------------------------------------------------------
;; Emitting
;; ---------------------------------------------------------------------------
;;
;; The text of BjoRx's tree, one parenthesized form per node: `(l 61 62)` a
;; literal, `(c 30-39)` a class, `(s ...)` a sequence, `(a ...)` an
;; alternation, `(r least most g|l body)` a repetition, `(g n body)` a group,
;; `(A)` and `(z)` the anchors, `(e)` nothing. Scalars are hexadecimal.

(: (-> int string))
(defun (hex-digit-string n)
  (def d (% n 16))
  (char->string (if (< d 10) (int->char (+ 48 d)) (int->char (+ 55 d)))))

(: (-> int string))
(defun (hex n)
  (if (< n 16)
      (hex-digit-string n)
      (string-append (hex (/ n 16)) (hex-digit-string n))))

(: (-> (List string) string))
(defun (concat-strings parts) (list-foldl (fun (part acc) (string-append acc part)) "" parts))

(: (-> (List string) string string))
(defun (join-strings parts sep)
  (match parts
    (Nil "")
    ((Cons p Nil) p)
    ((Cons p more) (str p sep (join-strings more sep)))))

(: (-> (Tuple int int) string))
(defun (emit-range r)
  (if (= (range-lo r) (range-hi r))
      (str " " (hex (range-lo r)))
      (str " " (hex (range-lo r)) "-" (hex (range-hi r)))))

(: (-> Cset string))
(defun (emit-set c) (str "(c" (concat-strings (list-map emit-range (cs-ranges c))) ")"))

(: (-> string string))
(defun (emit-lit text)
  (str "(l" (concat-strings (list-map (fun (c) (str " " (hex (char->int c)))) (string->list text))) ")"))

(: (-> string (List Rxn) (Box (List string)) string))
(defun (emit-list head parts names)
  (match parts
    (Nil "(e)")
    ((Cons p Nil) (emit p names))
    (_ (str "(" head " " (join-strings (list-map (fun (p) (emit p names)) parts) " ") ")"))))

;; `names` accumulates the Bjolang name of each group, most recent first. A
;; group's number is one more than how many were open before it, so the count
;; is read before the body is walked and nested groups number after their
;; parent.
(: (-> Rxn (Box (List string)) string))
(defun (emit node names)
  (match node
    (RxEmpty "(e)")
    ((RxLit text) (emit-lit text))
    ((RxSet c) (emit-set c))
    ((RxSeq parts) (emit-list "s" parts names))
    ((RxAlt parts) (emit-list "a" parts names))
    ((RxRep body least most greedy)
     (str "(r " (int->string least) " " (if (< most 0) "-" (int->string most))
          (if greedy " g " " l ") (emit body names) ")"))
    ((RxGroup name body)
     (def n (+ 1 (list-length (box-ref names))))
     (box-set! names (Cons name (box-ref names)))
     (str "(g " (int->string n) " " (emit body names) ")"))
    (RxBos "(A)")
    (RxEos "(z)")))

;; ---------------------------------------------------------------------------
;; Reading a set
;; ---------------------------------------------------------------------------

(: (-> Syntax (Option string)))
(defun (head-name form)
  (match form
    ((SList items)
     (if (list-empty? items)
         None
         (match (list-head items)
           ((SSym s) (Some (symbol->string s)))
           (_ None))))
    (_ None)))

(: (-> Syntax (List Syntax)))
(defun (form-args form)
  (match form
    ((SList items) (if (list-empty? items) Nil (list-tail items)))
    (_ Nil)))

;; A character usable as a set member or a range endpoint.
(: (-> Syntax char int))
(defun (set-scalar form c)
  (ignore form)
  (char->int c))

;; `syntax-error` types as a `Syntax` and never returns, so a branch that owes
;; an `int` has to discard it.
(: (-> Syntax int))
(defun (ignore-int s) (ignore s) 0)

(: (-> Syntax Cset))
(defun (ignore-cset s) (ignore s) (cs-make Nil))

(: (-> Syntax (List Syntax) (List int)))
(defun (chars-of whole items)
  (match items
    (Nil Nil)
    ((Cons item more)
     (list-append
      (match item
        ((SChar c) (list (set-scalar item c)))
        ((SStr text) (list-map (fun (c) (set-scalar item c)) (string->list text)))
        (_ (list (ignore-int
                  (syntax-error item
                                "A range endpoint is a character or a string of them: (/ #\\a #\\z) or (/ \"az\").")))))
      (chars-of whole more)))))

;; `(/ #\a #\z #\0 #\9)` and `(/ "az09")` are the same set: the endpoints are
;; taken in pairs however they were written.
(: (-> Syntax (List int) Cset))
(defun (pairs->cset whole ns)
  (match ns
    (Nil (cs-make Nil))
    ((Cons lo (Cons hi more))
     (if (> lo hi)
         (ignore-cset
          (syntax-error whole "A range runs upwards, and this one does not."))
         (cs-union (cs-make (list (Tuple lo hi))) (pairs->cset whole more))))
    (_ (ignore-cset
        (syntax-error whole "A range takes its endpoints in pairs, and this one has an odd number.")))))

(: (-> (List Syntax) Cset))
(defun (union-of items)
  (list-foldl (fun (item acc) (cs-union acc (parse-set item))) (cs-make Nil) items))

(: (-> Syntax Cset))
(defun (parse-set form)
  (match form
    ((SChar c) (cs-char (set-scalar form c)))
    ((SStr text) (list-foldl (fun (c acc) (cs-union acc (cs-char (set-scalar form c))))
                             (cs-make Nil)
                             (string->list text)))
    ((SKey k)
     (match (named-class (keyword->string k))
       ((Some c) c)
       (None (ignore-cset
              (syntax-error form (str "There is no character set called "
                                      (keyword->string k) "."))))))
    ((SList _) (parse-set-form form))
    (_ (ignore-cset
        (syntax-error form
                      "A character set is a character, a string, a named set like :digit, or a form like (in ...), (/ ...), (~ ...), (- ...) or (& ...).")))))

(: (-> Syntax Cset))
(defun (parse-set-form form)
  (def args (form-args form))
  (match (head-name form)
    (None (ignore-cset (syntax-error form "A character set form is headed by a name.")))
    ((Some name)
     (cond
       ((= name "in") (union-of args))
       ((= name "or") (union-of args))
       ((= name "/") (pairs->cset form (chars-of form args)))
       ((= name "~") (cs-complement (union-of args)))
       ((= name "-") (set-difference form args))
       ((= name "&") (set-intersection form args))
       (else (ignore-cset
              (syntax-error form
                            (str "'" name "' is not a character set operation. Use (in ...), (/ ...), (~ ...), (- ...) or (& ...)."))))))))

(: (-> Syntax (List Syntax) Cset))
(defun (set-difference form args)
  (if (list-empty? args)
      (ignore-cset (syntax-error form "(- ...) takes a set to subtract from."))
      (cs-diff (parse-set (list-head args)) (union-of (list-tail args)))))

(: (-> Syntax (List Syntax) Cset))
(defun (set-intersection form args)
  (if (list-empty? args)
      (ignore-cset (syntax-error form "(& ...) takes at least one set."))
      (list-foldl (fun (item acc) (cs-inter acc (parse-set item)))
                  (parse-set (list-head args))
                  (list-tail args))))

;; A set reaching pattern position has to be able to match something, and an
;; empty one is a mistake that would otherwise silently never match.
(: (-> Syntax Cset Rxn))
(defun (set-node form c)
  (if (cs-empty? c)
      (ignore-node (syntax-error form "This character set is empty, so it can never match."))
      (RxSet c)))

(: (-> Syntax Rxn))
(defun (ignore-node s) (ignore s) RxEmpty)

;; ---------------------------------------------------------------------------
;; Reading a pattern
;; ---------------------------------------------------------------------------

(: (-> Syntax Rxn))
(defun (parse form)
  (match form
    ((SStr text) (RxLit text))
    ((SChar c) (RxLit (char->string c)))
    ((SKey k) (parse-atom form (keyword->string k)))
    ((SList _) (parse-form form))
    ((SSym s)
     (ignore-node
      (syntax-error form
                    (str "'" (symbol->string s) "' is a symbol, and an rx that takes no argument is a keyword. Did you mean :"
                         (symbol->string s) "?"))))
    ((SPunct _)
     (ignore-node
      (syntax-error form
                    "An rx is built at compile time, so it cannot splice a value in. Dynamically built patterns are not supported.")))
    (_ (ignore-node (syntax-error form "This is not an rx.")))))

(: (-> Syntax string Rxn))
(defun (parse-atom form name)
  (cond
    ((= name "bos") RxBos)
    ((= name "eos") RxEos)
    (else
     (match (named-class name)
       ((Some c) (set-node form c))
       (None
        (ignore-node
         (syntax-error form
                       (str "There is no rx called :" name
                            ". The ones taking no argument are :bos :eos :any :nonl and the named sets."))))))))

(: (-> (List Syntax) (List Rxn)))
(defun (parse-all items) (list-map parse items))

(: (-> Syntax Rxn))
(defun (parse-form form)
  (def args (form-args form))
  (match (head-name form)
    (None
     (ignore-node
      (syntax-error form
                    "An rx form is headed by a name: (seq ...), (or ...), (+ ...), (in ...) and so on.")))
    ((Some name)
     (cond
       ((= name "seq") (RxSeq (parse-all args)))
       ((= name "or") (RxAlt (parse-all args)))
       ((= name "lit") (RxLit (literal-text form args)))
       ((= name "*") (RxRep (one-arg form args) 0 -1 #t))
       ((= name "+") (RxRep (one-arg form args) 1 -1 #t))
       ((= name "?") (RxRep (one-arg form args) 0 1 #t))
       ((= name "*?") (RxRep (one-arg form args) 0 -1 #f))
       ((= name "+?") (RxRep (one-arg form args) 1 -1 #f))
       ((= name "??") (RxRep (one-arg form args) 0 1 #f))
       ((= name "**") (bounded form args))
       ((= name "=") (exactly form args))
       ((= name ">=") (at-least form args))
       ((= name "$") (RxGroup "" (RxSeq (parse-all args))))
       ((= name "=>") (named-group form args))
       ((= name "in") (set-node form (union-of args)))
       ((= name "/") (set-node form (pairs->cset form (chars-of form args))))
       ((= name "~") (set-node form (cs-complement (union-of args))))
       ((= name "-") (set-node form (set-difference form args)))
       ((= name "&") (set-node form (set-intersection form args)))
       (else
        (ignore-node
         (syntax-error form
                       (str "'" name "' is not an rx form. If you meant a set or an anchor, those are keywords: :"
                            name "."))))))))

(: (-> Syntax (List Syntax) Rxn))
(defun (one-arg form args)
  (match args
    ((Cons a Nil) (parse a))
    (_ (RxSeq (parse-all args)))))

(: (-> Syntax (List Syntax) string))
(defun (literal-text form args)
  (match args
    ((Cons a Nil)
     (match a
       ((SStr text) text)
       ((SChar c) (char->string c))
       (_ (ignore-string (syntax-error form "(lit ...) takes one string.")))))
    (_ (ignore-string (syntax-error form "(lit ...) takes one string.")))))

(: (-> Syntax string))
(defun (ignore-string s) (ignore s) "")

(: (-> Syntax Syntax int))
(defun (count-of whole s)
  (match s
    ((SInt text) (string->int text))
    (_ (ignore-int (syntax-error whole "A repetition count is a whole number.")))))

(: (-> Syntax (List Syntax) Rxn))
(defun (bounded form args)
  (match args
    ((Cons lo (Cons hi rest))
     (RxRep (one-arg form rest) (count-of form lo) (count-of form hi) #t))
    (_ (ignore-node (syntax-error form "(** least most rx) takes two counts and a pattern.")))))

(: (-> Syntax (List Syntax) Rxn))
(defun (exactly form args)
  (match args
    ((Cons n rest) (let ((k (count-of form n))) (RxRep (one-arg form rest) k k #t)))
    (_ (ignore-node (syntax-error form "(= n rx) takes a count and a pattern.")))))

(: (-> Syntax (List Syntax) Rxn))
(defun (at-least form args)
  (match args
    ((Cons n rest) (RxRep (one-arg form rest) (count-of form n) -1 #t))
    (_ (ignore-node (syntax-error form "(>= n rx) takes a count and a pattern.")))))

;; The name is a keyword for the same reason every other constant of this
;; sublanguage is one: it is a name used as data, not a binding, and a keyword
;; can never be mistaken for one.
(: (-> Syntax (List Syntax) Rxn))
(defun (named-group form args)
  (match args
    ((Cons n rest)
     (match n
       ((SKey k) (RxGroup (keyword->string k) (RxSeq (parse-all rest))))
       (_ (ignore-node
           (syntax-error form "(=> :name rx) names its group with a keyword.")))))
    (_ (ignore-node (syntax-error form "(=> :name rx) takes a name and a pattern.")))))

;; ---------------------------------------------------------------------------
;; The macros
;; ---------------------------------------------------------------------------

(: (-> (List string) string))
(defun (names-text names) (join-strings names ","))

;; `#rx(seq a b)` reads as `(#rx seq a b)`, so the arguments are the form's own
;; contents. One argument is the form itself, which is what makes `#rx(:digit)`
;; and `#rx((+ :digit))` work as well as `#rx(+ :digit)`.
;;
;; The compile is hoisted: both arguments are literals, so the regex is built
;; the first time the expression is reached and every later evaluation reuses
;; it. Without that, every evaluation looks the pattern up in `BjoRegex`'s
;; cache, which hashes and compares the whole pattern string, and an emitted
;; character class can be thousands of characters long. A pattern in a match
;; clause is evaluated once per match attempt, so it pays that every time.
(: (-> Syntax Syntax))
(defun (rx-expansion form)
  (def node (parse form))
  (def names (make-box Nil))
  (def pattern (emit node names))
  #'(please-hoist-i-promise-i-am-not-naughty
     (rx-compile ,(SStr pattern) ,(SStr (names-text (list-reverse (box-ref names)))))))

(def/hash-extend (rx form inject compare)
  (match (form-args form)
    (Nil (syntax-error form "#rx takes a pattern: #rx(seq :bos (+ :digit) :eos)."))
    ((Cons only Nil) (rx-expansion only))
    (items (rx-expansion (SList items)))))

;; `(Rx re p)` matches `re` against the whole string, `(RxIn re p)` searches for
;; it. Both hand the inner pattern an `(Option RxMatch)`, so a clause reads
;; `(Rx phone (Some m))`.
;;
;; A view covers nothing for exhaustiveness, so a `match` using either still
;; needs a clause that matches anything.
(def/pattern (Rx form inject compare)
  (syntax-match form
    ((_ re p) #'(:view (rx-match ,re &) ,p))
    (bad (syntax-error bad "(Rx regex pattern) matches regex against the whole string."))))

(def/pattern (RxIn form inject compare)
  (syntax-match form
    ((_ re p) #'(:view (rx-search ,re &) ,p))
    (bad (syntax-error bad "(RxIn regex pattern) searches the string for regex."))))

;; ---------------------------------------------------------------------------
;; The tables
;; ---------------------------------------------------------------------------
;;
;; Generated from .NET's `CharUnicodeInfo`, so that `:alpha` and
;; `char-alphabetic?` agree. This is to avoid walking the BMP on each build.

(def tbl-alpha
  (str
   "0041005A0061007A00AA00AA00B500B500BA00BA00C000D600D800F600F802C102C602D1"
   "02E002E402EC02EC02EE02EE0370037403760377037A037D037F037F038603860388038A"
   "038C038C038E03A103A303F503F70481048A052F05310556055905590560058805D005EA"
   "05EF05F20620064A066E066F067106D306D506D506E506E606EE06EF06FA06FC06FF06FF"
   "071007100712072F074D07A507B107B107CA07EA07F407F507FA07FA08000815081A081A"
   "0824082408280828084008580860086A087008870889088E08A008C909040939093D093D"
   "0950095009580961097109800985098C098F0990099309A809AA09B009B209B209B609B9"
   "09BD09BD09CE09CE09DC09DD09DF09E109F009F109FC09FC0A050A0A0A0F0A100A130A28"
   "0A2A0A300A320A330A350A360A380A390A590A5C0A5E0A5E0A720A740A850A8D0A8F0A91"
   "0A930AA80AAA0AB00AB20AB30AB50AB90ABD0ABD0AD00AD00AE00AE10AF90AF90B050B0C"
   "0B0F0B100B130B280B2A0B300B320B330B350B390B3D0B3D0B5C0B5D0B5F0B610B710B71"
   "0B830B830B850B8A0B8E0B900B920B950B990B9A0B9C0B9C0B9E0B9F0BA30BA40BA80BAA"
   "0BAE0BB90BD00BD00C050C0C0C0E0C100C120C280C2A0C390C3D0C3D0C580C5A0C5D0C5D"
   "0C600C610C800C800C850C8C0C8E0C900C920CA80CAA0CB30CB50CB90CBD0CBD0CDD0CDE"
   "0CE00CE10CF10CF20D040D0C0D0E0D100D120D3A0D3D0D3D0D4E0D4E0D540D560D5F0D61"
   "0D7A0D7F0D850D960D9A0DB10DB30DBB0DBD0DBD0DC00DC60E010E300E320E330E400E46"
   "0E810E820E840E840E860E8A0E8C0EA30EA50EA50EA70EB00EB20EB30EBD0EBD0EC00EC4"
   "0EC60EC60EDC0EDF0F000F000F400F470F490F6C0F880F8C1000102A103F103F10501055"
   "105A105D1061106110651066106E107010751081108E108E10A010C510C710C710CD10CD"
   "10D010FA10FC1248124A124D1250125612581258125A125D12601288128A128D129012B0"
   "12B212B512B812BE12C012C012C212C512C812D612D81310131213151318135A1380138F"
   "13A013F513F813FD1401166C166F167F1681169A16A016EA16F116F817001711171F1731"
   "174017511760176C176E1770178017B317D717D717DC17DC1820187818801884188718A8"
   "18AA18AA18B018F51900191E1950196D19701974198019AB19B019C91A001A161A201A54"
   "1AA71AA71B051B331B451B4C1B831BA01BAE1BAF1BBA1BE51C001C231C4D1C4F1C5A1C7D"
   "1C801C8A1C901CBA1CBD1CBF1CE91CEC1CEE1CF31CF51CF61CFA1CFA1D001DBF1E001F15"
   "1F181F1D1F201F451F481F4D1F501F571F591F591F5B1F5B1F5D1F5D1F5F1F7D1F801FB4"
   "1FB61FBC1FBE1FBE1FC21FC41FC61FCC1FD01FD31FD61FDB1FE01FEC1FF21FF41FF61FFC"
   "20712071207F207F2090209C2102210221072107210A2113211521152119211D21242124"
   "2126212621282128212A212D212F2139213C213F21452149214E214E218321842C002CE4"
   "2CEB2CEE2CF22CF32D002D252D272D272D2D2D2D2D302D672D6F2D6F2D802D962DA02DA6"
   "2DA82DAE2DB02DB62DB82DBE2DC02DC62DC82DCE2DD02DD62DD82DDE2E2F2E2F30053006"
   "30313035303B303C30413096309D309F30A130FA30FC30FF3105312F3131318E31A031BF"
   "31F031FF34004DBF4E00A48CA4D0A4FDA500A60CA610A61FA62AA62BA640A66EA67FA69D"
   "A6A0A6E5A717A71FA722A788A78BA7CDA7D0A7D1A7D3A7D3A7D5A7DCA7F2A801A803A805"
   "A807A80AA80CA822A840A873A882A8B3A8F2A8F7A8FBA8FBA8FDA8FEA90AA925A930A946"
   "A960A97CA984A9B2A9CFA9CFA9E0A9E4A9E6A9EFA9FAA9FEAA00AA28AA40AA42AA44AA4B"
   "AA60AA76AA7AAA7AAA7EAAAFAAB1AAB1AAB5AAB6AAB9AABDAAC0AAC0AAC2AAC2AADBAADD"
   "AAE0AAEAAAF2AAF4AB01AB06AB09AB0EAB11AB16AB20AB26AB28AB2EAB30AB5AAB5CAB69"
   "AB70ABE2AC00D7A3D7B0D7C6D7CBD7FBF900FA6DFA70FAD9FB00FB06FB13FB17FB1DFB1D"
   "FB1FFB28FB2AFB36FB38FB3CFB3EFB3EFB40FB41FB43FB44FB46FBB1FBD3FD3DFD50FD8F"
   "FD92FDC7FDF0FDFBFE70FE74FE76FEFCFF21FF3AFF41FF5AFF66FFBEFFC2FFC7FFCAFFCF"
   "FFD2FFD7FFDAFFDC"
   ))

(def tbl-digit
  (str
   "003000390660066906F006F907C007C90966096F09E609EF0A660A6F0AE60AEF0B660B6F"
   "0BE60BEF0C660C6F0CE60CEF0D660D6F0DE60DEF0E500E590ED00ED90F200F2910401049"
   "1090109917E017E9181018191946194F19D019D91A801A891A901A991B501B591BB01BB9"
   "1C401C491C501C59A620A629A8D0A8D9A900A909A9D0A9D9A9F0A9F9AA50AA59ABF0ABF9"
   "FF10FF19"
   ))

(def tbl-alnum
  (str
   "003000390041005A0061007A00AA00AA00B500B500BA00BA00C000D600D800F600F802C1"
   "02C602D102E002E402EC02EC02EE02EE0370037403760377037A037D037F037F03860386"
   "0388038A038C038C038E03A103A303F503F70481048A052F053105560559055905600588"
   "05D005EA05EF05F20620064A06600669066E066F067106D306D506D506E506E606EE06FC"
   "06FF06FF071007100712072F074D07A507B107B107C007EA07F407F507FA07FA08000815"
   "081A081A0824082408280828084008580860086A087008870889088E08A008C909040939"
   "093D093D09500950095809610966096F097109800985098C098F0990099309A809AA09B0"
   "09B209B209B609B909BD09BD09CE09CE09DC09DD09DF09E109E609F109FC09FC0A050A0A"
   "0A0F0A100A130A280A2A0A300A320A330A350A360A380A390A590A5C0A5E0A5E0A660A6F"
   "0A720A740A850A8D0A8F0A910A930AA80AAA0AB00AB20AB30AB50AB90ABD0ABD0AD00AD0"
   "0AE00AE10AE60AEF0AF90AF90B050B0C0B0F0B100B130B280B2A0B300B320B330B350B39"
   "0B3D0B3D0B5C0B5D0B5F0B610B660B6F0B710B710B830B830B850B8A0B8E0B900B920B95"
   "0B990B9A0B9C0B9C0B9E0B9F0BA30BA40BA80BAA0BAE0BB90BD00BD00BE60BEF0C050C0C"
   "0C0E0C100C120C280C2A0C390C3D0C3D0C580C5A0C5D0C5D0C600C610C660C6F0C800C80"
   "0C850C8C0C8E0C900C920CA80CAA0CB30CB50CB90CBD0CBD0CDD0CDE0CE00CE10CE60CEF"
   "0CF10CF20D040D0C0D0E0D100D120D3A0D3D0D3D0D4E0D4E0D540D560D5F0D610D660D6F"
   "0D7A0D7F0D850D960D9A0DB10DB30DBB0DBD0DBD0DC00DC60DE60DEF0E010E300E320E33"
   "0E400E460E500E590E810E820E840E840E860E8A0E8C0EA30EA50EA50EA70EB00EB20EB3"
   "0EBD0EBD0EC00EC40EC60EC60ED00ED90EDC0EDF0F000F000F200F290F400F470F490F6C"
   "0F880F8C1000102A103F104910501055105A105D1061106110651066106E107010751081"
   "108E108E1090109910A010C510C710C710CD10CD10D010FA10FC1248124A124D12501256"
   "12581258125A125D12601288128A128D129012B012B212B512B812BE12C012C012C212C5"
   "12C812D612D81310131213151318135A1380138F13A013F513F813FD1401166C166F167F"
   "1681169A16A016EA16F116F817001711171F1731174017511760176C176E1770178017B3"
   "17D717D717DC17DC17E017E9181018191820187818801884188718A818AA18AA18B018F5"
   "1900191E1946196D19701974198019AB19B019C919D019D91A001A161A201A541A801A89"
   "1A901A991AA71AA71B051B331B451B4C1B501B591B831BA01BAE1BE51C001C231C401C49"
   "1C4D1C7D1C801C8A1C901CBA1CBD1CBF1CE91CEC1CEE1CF31CF51CF61CFA1CFA1D001DBF"
   "1E001F151F181F1D1F201F451F481F4D1F501F571F591F591F5B1F5B1F5D1F5D1F5F1F7D"
   "1F801FB41FB61FBC1FBE1FBE1FC21FC41FC61FCC1FD01FD31FD61FDB1FE01FEC1FF21FF4"
   "1FF61FFC20712071207F207F2090209C2102210221072107210A2113211521152119211D"
   "212421242126212621282128212A212D212F2139213C213F21452149214E214E21832184"
   "2C002CE42CEB2CEE2CF22CF32D002D252D272D272D2D2D2D2D302D672D6F2D6F2D802D96"
   "2DA02DA62DA82DAE2DB02DB62DB82DBE2DC02DC62DC82DCE2DD02DD62DD82DDE2E2F2E2F"
   "3005300630313035303B303C30413096309D309F30A130FA30FC30FF3105312F3131318E"
   "31A031BF31F031FF34004DBF4E00A48CA4D0A4FDA500A60CA610A62BA640A66EA67FA69D"
   "A6A0A6E5A717A71FA722A788A78BA7CDA7D0A7D1A7D3A7D3A7D5A7DCA7F2A801A803A805"
   "A807A80AA80CA822A840A873A882A8B3A8D0A8D9A8F2A8F7A8FBA8FBA8FDA8FEA900A925"
   "A930A946A960A97CA984A9B2A9CFA9D9A9E0A9E4A9E6A9FEAA00AA28AA40AA42AA44AA4B"
   "AA50AA59AA60AA76AA7AAA7AAA7EAAAFAAB1AAB1AAB5AAB6AAB9AABDAAC0AAC0AAC2AAC2"
   "AADBAADDAAE0AAEAAAF2AAF4AB01AB06AB09AB0EAB11AB16AB20AB26AB28AB2EAB30AB5A"
   "AB5CAB69AB70ABE2ABF0ABF9AC00D7A3D7B0D7C6D7CBD7FBF900FA6DFA70FAD9FB00FB06"
   "FB13FB17FB1DFB1DFB1FFB28FB2AFB36FB38FB3CFB3EFB3EFB40FB41FB43FB44FB46FBB1"
   "FBD3FD3DFD50FD8FFD92FDC7FDF0FDFBFE70FE74FE76FEFCFF10FF19FF21FF3AFF41FF5A"
   "FF66FFBEFFC2FFC7FFCAFFCFFFD2FFD7FFDAFFDC"
   ))

(def tbl-upper
  (str
   "0041005A00C000D600D800DE0100010001020102010401040106010601080108010A010A"
   "010C010C010E010E0110011001120112011401140116011601180118011A011A011C011C"
   "011E011E0120012001220122012401240126012601280128012A012A012C012C012E012E"
   "0130013001320132013401340136013601390139013B013B013D013D013F013F01410141"
   "014301430145014501470147014A014A014C014C014E014E015001500152015201540154"
   "0156015601580158015A015A015C015C015E015E01600160016201620164016401660166"
   "01680168016A016A016C016C016E016E0170017001720172017401740176017601780179"
   "017B017B017D017D0181018201840184018601870189018B018E01910193019401960198"
   "019C019D019F01A001A201A201A401A401A601A701A901A901AC01AC01AE01AF01B101B3"
   "01B501B501B701B801BC01BC01C401C401C701C701CA01CA01CD01CD01CF01CF01D101D1"
   "01D301D301D501D501D701D701D901D901DB01DB01DE01DE01E001E001E201E201E401E4"
   "01E601E601E801E801EA01EA01EC01EC01EE01EE01F101F101F401F401F601F801FA01FA"
   "01FC01FC01FE01FE0200020002020202020402040206020602080208020A020A020C020C"
   "020E020E0210021002120212021402140216021602180218021A021A021C021C021E021E"
   "0220022002220222022402240226022602280228022A022A022C022C022E022E02300230"
   "02320232023A023B023D023E024102410243024602480248024A024A024C024C024E024E"
   "037003700372037203760376037F037F038603860388038A038C038C038E038F039103A1"
   "03A303AB03CF03CF03D203D403D803D803DA03DA03DC03DC03DE03DE03E003E003E203E2"
   "03E403E403E603E603E803E803EA03EA03EC03EC03EE03EE03F403F403F703F703F903FA"
   "03FD042F0460046004620462046404640466046604680468046A046A046C046C046E046E"
   "0470047004720472047404740476047604780478047A047A047C047C047E047E04800480"
   "048A048A048C048C048E048E0490049004920492049404940496049604980498049A049A"
   "049C049C049E049E04A004A004A204A204A404A404A604A604A804A804AA04AA04AC04AC"
   "04AE04AE04B004B004B204B204B404B404B604B604B804B804BA04BA04BC04BC04BE04BE"
   "04C004C104C304C304C504C504C704C704C904C904CB04CB04CD04CD04D004D004D204D2"
   "04D404D404D604D604D804D804DA04DA04DC04DC04DE04DE04E004E004E204E204E404E4"
   "04E604E604E804E804EA04EA04EC04EC04EE04EE04F004F004F204F204F404F404F604F6"
   "04F804F804FA04FA04FC04FC04FE04FE0500050005020502050405040506050605080508"
   "050A050A050C050C050E050E0510051005120512051405140516051605180518051A051A"
   "051C051C051E051E0520052005220522052405240526052605280528052A052A052C052C"
   "052E052E0531055610A010C510C710C710CD10CD13A013F51C891C891C901CBA1CBD1CBF"
   "1E001E001E021E021E041E041E061E061E081E081E0A1E0A1E0C1E0C1E0E1E0E1E101E10"
   "1E121E121E141E141E161E161E181E181E1A1E1A1E1C1E1C1E1E1E1E1E201E201E221E22"
   "1E241E241E261E261E281E281E2A1E2A1E2C1E2C1E2E1E2E1E301E301E321E321E341E34"
   "1E361E361E381E381E3A1E3A1E3C1E3C1E3E1E3E1E401E401E421E421E441E441E461E46"
   "1E481E481E4A1E4A1E4C1E4C1E4E1E4E1E501E501E521E521E541E541E561E561E581E58"
   "1E5A1E5A1E5C1E5C1E5E1E5E1E601E601E621E621E641E641E661E661E681E681E6A1E6A"
   "1E6C1E6C1E6E1E6E1E701E701E721E721E741E741E761E761E781E781E7A1E7A1E7C1E7C"
   "1E7E1E7E1E801E801E821E821E841E841E861E861E881E881E8A1E8A1E8C1E8C1E8E1E8E"
   "1E901E901E921E921E941E941E9E1E9E1EA01EA01EA21EA21EA41EA41EA61EA61EA81EA8"
   "1EAA1EAA1EAC1EAC1EAE1EAE1EB01EB01EB21EB21EB41EB41EB61EB61EB81EB81EBA1EBA"
   "1EBC1EBC1EBE1EBE1EC01EC01EC21EC21EC41EC41EC61EC61EC81EC81ECA1ECA1ECC1ECC"
   "1ECE1ECE1ED01ED01ED21ED21ED41ED41ED61ED61ED81ED81EDA1EDA1EDC1EDC1EDE1EDE"
   "1EE01EE01EE21EE21EE41EE41EE61EE61EE81EE81EEA1EEA1EEC1EEC1EEE1EEE1EF01EF0"
   "1EF21EF21EF41EF41EF61EF61EF81EF81EFA1EFA1EFC1EFC1EFE1EFE1F081F0F1F181F1D"
   "1F281F2F1F381F3F1F481F4D1F591F591F5B1F5B1F5D1F5D1F5F1F5F1F681F6F1FB81FBB"
   "1FC81FCB1FD81FDB1FE81FEC1FF81FFB2102210221072107210B210D2110211221152115"
   "2119211D212421242126212621282128212A212D21302133213E213F2145214521832183"
   "2C002C2F2C602C602C622C642C672C672C692C692C6B2C6B2C6D2C702C722C722C752C75"
   "2C7E2C802C822C822C842C842C862C862C882C882C8A2C8A2C8C2C8C2C8E2C8E2C902C90"
   "2C922C922C942C942C962C962C982C982C9A2C9A2C9C2C9C2C9E2C9E2CA02CA02CA22CA2"
   "2CA42CA42CA62CA62CA82CA82CAA2CAA2CAC2CAC2CAE2CAE2CB02CB02CB22CB22CB42CB4"
   "2CB62CB62CB82CB82CBA2CBA2CBC2CBC2CBE2CBE2CC02CC02CC22CC22CC42CC42CC62CC6"
   "2CC82CC82CCA2CCA2CCC2CCC2CCE2CCE2CD02CD02CD22CD22CD42CD42CD62CD62CD82CD8"
   "2CDA2CDA2CDC2CDC2CDE2CDE2CE02CE02CE22CE22CEB2CEB2CED2CED2CF22CF2A640A640"
   "A642A642A644A644A646A646A648A648A64AA64AA64CA64CA64EA64EA650A650A652A652"
   "A654A654A656A656A658A658A65AA65AA65CA65CA65EA65EA660A660A662A662A664A664"
   "A666A666A668A668A66AA66AA66CA66CA680A680A682A682A684A684A686A686A688A688"
   "A68AA68AA68CA68CA68EA68EA690A690A692A692A694A694A696A696A698A698A69AA69A"
   "A722A722A724A724A726A726A728A728A72AA72AA72CA72CA72EA72EA732A732A734A734"
   "A736A736A738A738A73AA73AA73CA73CA73EA73EA740A740A742A742A744A744A746A746"
   "A748A748A74AA74AA74CA74CA74EA74EA750A750A752A752A754A754A756A756A758A758"
   "A75AA75AA75CA75CA75EA75EA760A760A762A762A764A764A766A766A768A768A76AA76A"
   "A76CA76CA76EA76EA779A779A77BA77BA77DA77EA780A780A782A782A784A784A786A786"
   "A78BA78BA78DA78DA790A790A792A792A796A796A798A798A79AA79AA79CA79CA79EA79E"
   "A7A0A7A0A7A2A7A2A7A4A7A4A7A6A7A6A7A8A7A8A7AAA7AEA7B0A7B4A7B6A7B6A7B8A7B8"
   "A7BAA7BAA7BCA7BCA7BEA7BEA7C0A7C0A7C2A7C2A7C4A7C7A7C9A7C9A7CBA7CCA7D0A7D0"
   "A7D6A7D6A7D8A7D8A7DAA7DAA7DCA7DCA7F5A7F5FF21FF3A"
   ))

(def tbl-lower
  (str
   "0061007A00B500B500DF00F600F800FF0101010101030103010501050107010701090109"
   "010B010B010D010D010F010F0111011101130113011501150117011701190119011B011B"
   "011D011D011F011F0121012101230123012501250127012701290129012B012B012D012D"
   "012F012F01310131013301330135013501370138013A013A013C013C013E013E01400140"
   "01420142014401440146014601480149014B014B014D014D014F014F0151015101530153"
   "015501550157015701590159015B015B015D015D015F015F016101610163016301650165"
   "0167016701690169016B016B016D016D016F016F01710171017301730175017501770177"
   "017A017A017C017C017E0180018301830185018501880188018C018D0192019201950195"
   "0199019B019E019E01A101A101A301A301A501A501A801A801AA01AB01AD01AD01B001B0"
   "01B401B401B601B601B901BA01BD01BF01C601C601C901C901CC01CC01CE01CE01D001D0"
   "01D201D201D401D401D601D601D801D801DA01DA01DC01DD01DF01DF01E101E101E301E3"
   "01E501E501E701E701E901E901EB01EB01ED01ED01EF01F001F301F301F501F501F901F9"
   "01FB01FB01FD01FD01FF01FF0201020102030203020502050207020702090209020B020B"
   "020D020D020F020F0211021102130213021502150217021702190219021B021B021D021D"
   "021F021F0221022102230223022502250227022702290229022B022B022D022D022F022F"
   "0231023102330239023C023C023F0240024202420247024702490249024B024B024D024D"
   "024F0293029502AF037103710373037303770377037B037D0390039003AC03CE03D003D1"
   "03D503D703D903D903DB03DB03DD03DD03DF03DF03E103E103E303E303E503E503E703E7"
   "03E903E903EB03EB03ED03ED03EF03F303F503F503F803F803FB03FC0430045F04610461"
   "04630463046504650467046704690469046B046B046D046D046F046F0471047104730473"
   "047504750477047704790479047B047B047D047D047F047F04810481048B048B048D048D"
   "048F048F0491049104930493049504950497049704990499049B049B049D049D049F049F"
   "04A104A104A304A304A504A504A704A704A904A904AB04AB04AD04AD04AF04AF04B104B1"
   "04B304B304B504B504B704B704B904B904BB04BB04BD04BD04BF04BF04C204C204C404C4"
   "04C604C604C804C804CA04CA04CC04CC04CE04CF04D104D104D304D304D504D504D704D7"
   "04D904D904DB04DB04DD04DD04DF04DF04E104E104E304E304E504E504E704E704E904E9"
   "04EB04EB04ED04ED04EF04EF04F104F104F304F304F504F504F704F704F904F904FB04FB"
   "04FD04FD04FF04FF0501050105030503050505050507050705090509050B050B050D050D"
   "050F050F0511051105130513051505150517051705190519051B051B051D051D051F051F"
   "0521052105230523052505250527052705290529052B052B052D052D052F052F05600588"
   "10D010FA10FD10FF13F813FD1C801C881C8A1C8A1D001D2B1D6B1D771D791D9A1E011E01"
   "1E031E031E051E051E071E071E091E091E0B1E0B1E0D1E0D1E0F1E0F1E111E111E131E13"
   "1E151E151E171E171E191E191E1B1E1B1E1D1E1D1E1F1E1F1E211E211E231E231E251E25"
   "1E271E271E291E291E2B1E2B1E2D1E2D1E2F1E2F1E311E311E331E331E351E351E371E37"
   "1E391E391E3B1E3B1E3D1E3D1E3F1E3F1E411E411E431E431E451E451E471E471E491E49"
   "1E4B1E4B1E4D1E4D1E4F1E4F1E511E511E531E531E551E551E571E571E591E591E5B1E5B"
   "1E5D1E5D1E5F1E5F1E611E611E631E631E651E651E671E671E691E691E6B1E6B1E6D1E6D"
   "1E6F1E6F1E711E711E731E731E751E751E771E771E791E791E7B1E7B1E7D1E7D1E7F1E7F"
   "1E811E811E831E831E851E851E871E871E891E891E8B1E8B1E8D1E8D1E8F1E8F1E911E91"
   "1E931E931E951E9D1E9F1E9F1EA11EA11EA31EA31EA51EA51EA71EA71EA91EA91EAB1EAB"
   "1EAD1EAD1EAF1EAF1EB11EB11EB31EB31EB51EB51EB71EB71EB91EB91EBB1EBB1EBD1EBD"
   "1EBF1EBF1EC11EC11EC31EC31EC51EC51EC71EC71EC91EC91ECB1ECB1ECD1ECD1ECF1ECF"
   "1ED11ED11ED31ED31ED51ED51ED71ED71ED91ED91EDB1EDB1EDD1EDD1EDF1EDF1EE11EE1"
   "1EE31EE31EE51EE51EE71EE71EE91EE91EEB1EEB1EED1EED1EEF1EEF1EF11EF11EF31EF3"
   "1EF51EF51EF71EF71EF91EF91EFB1EFB1EFD1EFD1EFF1F071F101F151F201F271F301F37"
   "1F401F451F501F571F601F671F701F7D1F801F871F901F971FA01FA71FB01FB41FB61FB7"
   "1FBE1FBE1FC21FC41FC61FC71FD01FD31FD61FD71FE01FE71FF21FF41FF61FF7210A210A"
   "210E210F21132113212F212F2134213421392139213C213D21462149214E214E21842184"
   "2C302C5F2C612C612C652C662C682C682C6A2C6A2C6C2C6C2C712C712C732C742C762C7B"
   "2C812C812C832C832C852C852C872C872C892C892C8B2C8B2C8D2C8D2C8F2C8F2C912C91"
   "2C932C932C952C952C972C972C992C992C9B2C9B2C9D2C9D2C9F2C9F2CA12CA12CA32CA3"
   "2CA52CA52CA72CA72CA92CA92CAB2CAB2CAD2CAD2CAF2CAF2CB12CB12CB32CB32CB52CB5"
   "2CB72CB72CB92CB92CBB2CBB2CBD2CBD2CBF2CBF2CC12CC12CC32CC32CC52CC52CC72CC7"
   "2CC92CC92CCB2CCB2CCD2CCD2CCF2CCF2CD12CD12CD32CD32CD52CD52CD72CD72CD92CD9"
   "2CDB2CDB2CDD2CDD2CDF2CDF2CE12CE12CE32CE42CEC2CEC2CEE2CEE2CF32CF32D002D25"
   "2D272D272D2D2D2DA641A641A643A643A645A645A647A647A649A649A64BA64BA64DA64D"
   "A64FA64FA651A651A653A653A655A655A657A657A659A659A65BA65BA65DA65DA65FA65F"
   "A661A661A663A663A665A665A667A667A669A669A66BA66BA66DA66DA681A681A683A683"
   "A685A685A687A687A689A689A68BA68BA68DA68DA68FA68FA691A691A693A693A695A695"
   "A697A697A699A699A69BA69BA723A723A725A725A727A727A729A729A72BA72BA72DA72D"
   "A72FA731A733A733A735A735A737A737A739A739A73BA73BA73DA73DA73FA73FA741A741"
   "A743A743A745A745A747A747A749A749A74BA74BA74DA74DA74FA74FA751A751A753A753"
   "A755A755A757A757A759A759A75BA75BA75DA75DA75FA75FA761A761A763A763A765A765"
   "A767A767A769A769A76BA76BA76DA76DA76FA76FA771A778A77AA77AA77CA77CA77FA77F"
   "A781A781A783A783A785A785A787A787A78CA78CA78EA78EA791A791A793A795A797A797"
   "A799A799A79BA79BA79DA79DA79FA79FA7A1A7A1A7A3A7A3A7A5A7A5A7A7A7A7A7A9A7A9"
   "A7AFA7AFA7B5A7B5A7B7A7B7A7B9A7B9A7BBA7BBA7BDA7BDA7BFA7BFA7C1A7C1A7C3A7C3"
   "A7C8A7C8A7CAA7CAA7CDA7CDA7D1A7D1A7D3A7D3A7D5A7D5A7D7A7D7A7D9A7D9A7DBA7DB"
   "A7F6A7F6A7FAA7FAAB30AB5AAB60AB68AB70ABBFFB00FB06FB13FB17FF41FF5A"
   ))

(def tbl-space
  (str
   "0009000D002000200085008500A000A0168016802000200A20282029202F202F205F205F"
   "30003000"
   ))

(def tbl-word
  (str
   "003000390041005A005F005F0061007A00AA00AA00B500B500BA00BA00C000D600D800F6"
   "00F802C102C602D102E002E402EC02EC02EE02EE0300037403760377037A037D037F037F"
   "038603860388038A038C038C038E03A103A303F503F7048104830487048A052F05310556"
   "0559055905600588059105BD05BF05BF05C105C205C405C505C705C705D005EA05EF05F2"
   "0610061A06200669066E06D306D506DC06DF06E806EA06FC06FF06FF0710074A074D07B1"
   "07C007F507FA07FA07FD07FD0800082D0840085B0860086A087008870889088E089708E1"
   "08E309020904093A093C093D09410948094D094D095009630966096F097109810985098C"
   "098F0990099309A809AA09B009B209B209B609B909BC09BD09C109C409CD09CE09DC09DD"
   "09DF09E309E609F109FC09FC09FE09FE0A010A020A050A0A0A0F0A100A130A280A2A0A30"
   "0A320A330A350A360A380A390A3C0A3C0A410A420A470A480A4B0A4D0A510A510A590A5C"
   "0A5E0A5E0A660A750A810A820A850A8D0A8F0A910A930AA80AAA0AB00AB20AB30AB50AB9"
   "0ABC0ABD0AC10AC50AC70AC80ACD0ACD0AD00AD00AE00AE30AE60AEF0AF90AFF0B010B01"
   "0B050B0C0B0F0B100B130B280B2A0B300B320B330B350B390B3C0B3D0B3F0B3F0B410B44"
   "0B4D0B4D0B550B560B5C0B5D0B5F0B630B660B6F0B710B710B820B830B850B8A0B8E0B90"
   "0B920B950B990B9A0B9C0B9C0B9E0B9F0BA30BA40BA80BAA0BAE0BB90BC00BC00BCD0BCD"
   "0BD00BD00BE60BEF0C000C000C040C0C0C0E0C100C120C280C2A0C390C3C0C400C460C48"
   "0C4A0C4D0C550C560C580C5A0C5D0C5D0C600C630C660C6F0C800C810C850C8C0C8E0C90"
   "0C920CA80CAA0CB30CB50CB90CBC0CBD0CBF0CBF0CC60CC60CCC0CCD0CDD0CDE0CE00CE3"
   "0CE60CEF0CF10CF20D000D010D040D0C0D0E0D100D120D3D0D410D440D4D0D4E0D540D56"
   "0D5F0D630D660D6F0D7A0D7F0D810D810D850D960D9A0DB10DB30DBB0DBD0DBD0DC00DC6"
   "0DCA0DCA0DD20DD40DD60DD60DE60DEF0E010E3A0E400E4E0E500E590E810E820E840E84"
   "0E860E8A0E8C0EA30EA50EA50EA70EBD0EC00EC40EC60EC60EC80ECE0ED00ED90EDC0EDF"
   "0F000F000F180F190F200F290F350F350F370F370F390F390F400F470F490F6C0F710F7E"
   "0F800F840F860F970F990FBC0FC60FC61000102A102D1030103210371039103A103D1049"
   "105010551058106110651066106E108210851086108D108E10901099109D109D10A010C5"
   "10C710C710CD10CD10D010FA10FC1248124A124D1250125612581258125A125D12601288"
   "128A128D129012B012B212B512B812BE12C012C012C212C512C812D612D8131013121315"
   "1318135A135D135F1380138F13A013F513F813FD1401166C166F167F1681169A16A016EA"
   "16F116F817001714171F1733174017531760176C176E177017721773178017B517B717BD"
   "17C617C617C917D317D717D717DC17DD17E017E9180B180D180F181918201878188018AA"
   "18B018F51900191E1920192219271928193219321939193B1946196D19701974198019AB"
   "19B019C919D019D91A001A181A1B1A1B1A201A541A561A561A581A5E1A601A601A621A62"
   "1A651A6C1A731A7C1A7F1A891A901A991AA71AA71AB01ABD1ABF1ACE1B001B031B051B34"
   "1B361B3A1B3C1B3C1B421B421B451B4C1B501B591B6B1B731B801B811B831BA01BA21BA5"
   "1BA81BA91BAB1BE61BE81BE91BED1BED1BEF1BF11C001C231C2C1C331C361C371C401C49"
   "1C4D1C7D1C801C8A1C901CBA1CBD1CBF1CD01CD21CD41CE01CE21CF61CF81CFA1D001F15"
   "1F181F1D1F201F451F481F4D1F501F571F591F591F5B1F5B1F5D1F5D1F5F1F7D1F801FB4"
   "1FB61FBC1FBE1FBE1FC21FC41FC61FCC1FD01FD31FD61FDB1FE01FEC1FF21FF41FF61FFC"
   "203F20402054205420712071207F207F2090209C20D020DC20E120E120E520F021022102"
   "21072107210A2113211521152119211D212421242126212621282128212A212D212F2139"
   "213C213F21452149214E214E218321842C002CE42CEB2CF32D002D252D272D272D2D2D2D"
   "2D302D672D6F2D6F2D7F2D962DA02DA62DA82DAE2DB02DB62DB82DBE2DC02DC62DC82DCE"
   "2DD02DD62DD82DDE2DE02DFF2E2F2E2F30053006302A302D30313035303B303C30413096"
   "3099309A309D309F30A130FA30FC30FF3105312F3131318E31A031BF31F031FF34004DBF"
   "4E00A48CA4D0A4FDA500A60CA610A62BA640A66FA674A67DA67FA6E5A6F0A6F1A717A71F"
   "A722A788A78BA7CDA7D0A7D1A7D3A7D3A7D5A7DCA7F2A822A825A826A82CA82CA840A873"
   "A882A8B3A8C4A8C5A8D0A8D9A8E0A8F7A8FBA8FBA8FDA92DA930A951A960A97CA980A982"
   "A984A9B3A9B6A9B9A9BCA9BDA9CFA9D9A9E0A9FEAA00AA2EAA31AA32AA35AA36AA40AA4C"
   "AA50AA59AA60AA76AA7AAA7AAA7CAA7CAA7EAAC2AADBAADDAAE0AAEAAAECAAEDAAF2AAF4"
   "AAF6AAF6AB01AB06AB09AB0EAB11AB16AB20AB26AB28AB2EAB30AB5AAB5CAB69AB70ABE2"
   "ABE5ABE5ABE8ABE8ABEDABEDABF0ABF9AC00D7A3D7B0D7C6D7CBD7FBF900FA6DFA70FAD9"
   "FB00FB06FB13FB17FB1DFB28FB2AFB36FB38FB3CFB3EFB3EFB40FB41FB43FB44FB46FBB1"
   "FBD3FD3DFD50FD8FFD92FDC7FDF0FDFBFE00FE0FFE20FE2FFE33FE34FE4DFE4FFE70FE74"
   "FE76FEFCFF10FF19FF21FF3AFF3FFF3FFF41FF5AFF66FFBEFFC2FFC7FFCAFFCFFFD2FFD7"
   "FFDAFFDC"
   ))

(def tbl-control
  (str
   "0000001F007F009F"
   ))

;; ---------------------------------------------------------------------------
;; The named sets
;; ---------------------------------------------------------------------------

(def cs-alpha   (table-set tbl-alpha))
(def cs-digit   (table-set tbl-digit))
(def cs-alnum   (table-set tbl-alnum))
(def cs-upper   (table-set tbl-upper))
(def cs-lower   (table-set tbl-lower))
(def cs-space   (table-set tbl-space))
(def cs-word    (table-set tbl-word))
(def cs-control (table-set tbl-control))

(def cs-ascii     (cs-make (list (Tuple 0 127))))
(def cs-hex-digit (cs-make (list (Tuple 48 57) (Tuple 65 70) (Tuple 97 102))))

;; `:digit` is every Unicode decimal digit, which is what `numeric` means in
;; SRE and what `char-numeric?` answers. A program that wants the ten ASCII
;; ones usually means exactly that, so the narrowing has a name of its own
;; rather than being spelled `(& :digit :ascii)` at every use.
(def cs-ascii-digit (cs-inter cs-digit cs-ascii))
(def cs-ascii-alpha (cs-inter cs-alpha cs-ascii))
(def cs-ascii-alnum (cs-inter cs-alnum cs-ascii))

;; One scalar, whatever it is. `:nonl` is the same set without U+000A, which is
;; the only character either of them treats as a line break.
(def cs-any  (cs-make universe))
(def cs-nonl (cs-make (ranges-gaps 0 1114111 (list (Tuple 10 10)))))

(: (-> string (Option Cset)))
(defun (named-class name)
  (cond
    ((= name "any") (Some cs-any))
    ((= name "nonl") (Some cs-nonl))
    ((= name "alpha") (Some cs-alpha))
    ((= name "alphabetic") (Some cs-alpha))
    ((= name "digit") (Some cs-digit))
    ((= name "numeric") (Some cs-digit))
    ((= name "alnum") (Some cs-alnum))
    ((= name "alphanumeric") (Some cs-alnum))
    ((= name "upper") (Some cs-upper))
    ((= name "upper-case") (Some cs-upper))
    ((= name "lower") (Some cs-lower))
    ((= name "lower-case") (Some cs-lower))
    ((= name "space") (Some cs-space))
    ((= name "whitespace") (Some cs-space))
    ((= name "word") (Some cs-word))
    ((= name "control") (Some cs-control))
    ((= name "ascii") (Some cs-ascii))
    ((= name "hex-digit") (Some cs-hex-digit))
    ((= name "ascii-digit") (Some cs-ascii-digit))
    ((= name "ascii-alpha") (Some cs-ascii-alpha))
    ((= name "ascii-alnum") (Some cs-ascii-alnum))
    (else None)))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/scan.bjo — reading a line by a pattern, after OCaml's Scanf.
;;
;; `(scan line '(long "-" long))` reads two longs with a dash between them and
;; answers them as a tuple. The pattern is a quoted list, read when the macro
;; expands: the expansion is straight-line code over the line's bytes, one
;; step per item, with no pattern left to interpret at run time. The steps
;; are written #:inline, so each one is copied into the code that scans. A number step answers a value and the next position as a value
;; tuple, so a pattern of numbers allocates nothing. A text step answers only
;; the next position, and the value is a slice made after it, so text that
;; `(:skip ...)` drops allocates nothing either.
;;
;; The code is built in continuation-passing style: each item is given what to
;; do with its value and the position after it, and what to do when it does
;; not match. Where the rest of the pattern would be copied into two places —
;; after a repetition, an option or a choice — it is a local function instead,
;; a join point, so the expansion grows with the pattern and not faster.
(import (std prelude) (std syntax-match) (std interop))

(:doc #:file "scan.bjodoc")

(export scan-lit scan-byte scan-lit8 scan-ws scan-int scan-long scan-double scan-char scan-word
        scan-rest scan-chars scan-until scan-until-byte scan-slice scan-end? scan-fail scan-message)

;; ---------------------------------------------------------------------------
;; The steps
;; ---------------------------------------------------------------------------

;; What the expansion calls. They are exported, because a template reaches a
;; helper of its own module only through the module's exports. They are
;; #:inline, so an importer copies each step into the code that scans, and
;; the constant width that the macro gives is known there.
;;
;; A position is a byte offset into the line's UTF-8 bytes, from 0 to the
;; byte length. Each step takes the position it starts at and answers the
;; position after what it read, or -1 when the text there does not match. A
;; step that reads a number or a character answers it with the position, as
;; a tuple, so nothing is allocated. A step that reads text answers only the
;; position, and `scan-slice` makes the text when the pattern keeps it.
;;
;; A step that takes a width reads no more than that many characters, and
;; reads to the end of the line when the width is negative.
;;
;; A step never raises on text that does not match. The macro decides what a
;; failure does: `scan` calls `scan-fail`, and `try-scan` makes an `Err` with
;; `scan-message`.

(import/class
  (FormatException (: System.FormatException (-> string FormatException))))

;; The line's bytes are read-only, and a part of the line is made by
;; `SliceBytes`, which refuses an offset in the middle of a character.
(import/extern
  (line-bytes  (: BjoString.Utf8String.AsSpan (-> string (RoSpan byte))))
  (line-length (: BjoString.Utf8String.ByteLength (-> string int) #:get))
  (line-part   (: BjoString.Utf8String.SliceBytes (-> string int int string)))
  (bytes-part  (: Bjolang.Runtime.Spans.Slice (-> (RoSpan byte) int int (RoSpan byte))))
  (bytes-start-with? (: System.MemoryExtensions.StartsWith (-> (RoSpan byte) (RoSpan byte) bool)))
  (bytes-find  (: System.MemoryExtensions.IndexOf (-> (RoSpan byte) byte int)))
  (bytes-long  (: System.Buffers.Binary.BinaryPrimitives.ReadInt64LittleEndian (-> (RoSpan byte) long)))
  (bytes-double
   (: System.Buffers.Text.Utf8Parser.TryParse
      (-> (RoSpan byte) (out double) (out int) System.Char (Tuple bool double int)))))

;; Whether a byte is whitespace, as a space in a pattern matches it: the
;; ASCII blanks. One unsigned compare tests a range, as one branch.
(: (-> int bool))
(defun (blank? b) #:inline (or (= b 32) (<= (cast uint (- b 9)) 4u)))

(: (-> (RoSpan byte) int int))
(defun (byte-at b i) #:inline (cast int (ro-span-ref b i)))

;; The end of `width` characters from `pos`, or the end of the bytes.
(: (-> (RoSpan byte) int int int))
(defun (count-end b pos width)
  (def n (ro-span-length b))
  (let go ((i pos) (k 0))
    (if (and (< k width) (< i n))
        (let ((c (byte-at b i)))
          (go (+ i (cond ((< c 128) 1) ((>= c 240) 4) ((>= c 224) 3) (else 2))) (+ k 1)))
        i)))

;; The bytes of the line that a step with a width can read: all of them when
;; the width is negative. A loop that stops at their length tests each index
;; once, because .NET then knows it is in the span.
(: (-> string int int (RoSpan byte)))
(defun (within s pos width) #:inline
  (def b (line-bytes s))
  (if (< width 0) b (bytes-part b 0 (count-end b pos width))))

;; The text of `literal`, exactly.
(: (-> string int string int))
(defun (scan-lit s pos literal) #:inline
  (def b (line-bytes s))
  (def want (line-bytes literal))
  (if (bytes-start-with? (bytes-part b pos (ro-span-length b)) want)
      (+ pos (ro-span-length want))
      -1))

;; One ASCII byte, exactly. What a literal of one character compiles to.
(: (-> string int int int))
(defun (scan-byte s pos code) #:inline
  (def b (line-bytes s))
  (if (and (< pos (ro-span-length b)) (= (byte-at b pos) code)) (+ pos 1) -1))

;; A literal of 2 to 8 bytes, exactly. `bytes` holds the literal's bytes with
;; the first in the lowest byte, so that one 64-bit read and compare matches
;; it.
(: (-> string int long int int))
(defun (scan-lit8 s pos bytes length) #:inline
  (def b (line-bytes s))
  (def left (- (ro-span-length b) pos))
  (cond
    ((< left length) -1)
    ((< left 8) (lit-tail b pos bytes length))
    ((= (bitwise-and (bytes-long (bytes-part b pos (+ pos 8)))
                     (if (= length 8) -1L (- (shift-left 1L (* 8 length)) 1L)))
        bytes)
     (+ pos length))
    (else -1)))

;; `scan-lit8` with fewer than 8 bytes left in the line. Out of line, so that
;; each literal of a pattern does not copy the loop.
(: (-> (RoSpan byte) int long int int))
(defun (lit-tail b pos bytes length)
  (let go ((i 0) (acc 0L))
    (cond
      ((< i length)
       (go (+ i 1) (bitwise-ior acc (shift-left (cast long (ro-span-ref b (+ pos i))) (* 8 i)))))
      ((= acc bytes) (+ pos length))
      (else -1))))

;; Any amount of whitespace, none included. Never fails.
(: (-> string int int))
(defun (scan-ws s pos) #:inline
  (def b (line-bytes s))
  (def n (ro-span-length b))
  (let go ((i pos))
    (cond
      ((>= i n) i)
      ((blank? (byte-at b i)) (go (+ i 1)))
      (else i))))

;; The digits from `first` to `stop`, more than 18 of them, as a number of
;; at most `hi`, or `hi + 1` when the sign is a minus. Out of line, because
;; most of such digits are leading zeros or too many. The digits are added up
;; below zero, where the least long fits, and each one is checked.
(: (-> (RoSpan byte) int int bool long (Tuple long int)))
(defun (long-digits b first stop negative? hi)
  (def lo (if negative? (- (- 0L hi) 1L) (- 0L hi)))
  (let go ((i first) (acc 0L))
    (cond
      ((= i stop) (Tuple (if negative? acc (- 0L acc)) i))
      (else
       (def d (cast long (- (byte-at b i) 48)))
       (if (< acc (/ (+ lo d) 10L))
           (Tuple 0L -1)
           (go (+ i 1) (- (* acc 10L) d)))))))

;; An optional sign and one or more decimal digits from `pos`, as a number of
;; at most `hi`, or `hi + 1` when the sign is a minus, and the position after
;; them. The position is -1 when there is no digit or the number is too big.
;; Up to 18 digits cannot overflow a long, so the check comes after the loop.
(: (-> (RoSpan byte) int long (Tuple long int)))
(defun (read-digits b pos hi) #:inline
  (def* (end (ro-span-length b))
        (c (if (< pos end) (byte-at b pos) 0))
        (negative? (= c 45))
        (first (if (or negative? (= c 43)) (+ pos 1) pos)))
  (def (acc . stop)
    (let go ((i first) (acc 0L))
      (if (< i end)
          (let ((d (- (byte-at b i) 48)))
            (if (<= (cast uint d) 9u) (go (+ i 1) (+ (* acc 10L) (cast long d))) (Tuple acc i)))
          (Tuple acc i))))
  (cond
    ((= stop first) (Tuple 0L -1))
    ((> (- stop first) 18) (long-digits b first stop negative? hi))
    ((> (if negative? (- acc 1L) acc) hi) (Tuple 0L -1))
    (negative? (Tuple (- 0L acc) stop))
    (else (Tuple acc stop))))

;; An int: an optional sign and decimal digits. A number too big for an int
;; does not match.
(: (-> string int int (Tuple int int)))
(defun (scan-int s pos width) #:inline
  (def (v . q) (read-digits (within s pos width) pos 2147483647L))
  (Tuple (cast int v) q))

;; A long, as `scan-int` reads an int.
(: (-> string int int (Tuple long int)))
(defun (scan-long s pos width) #:inline
  (read-digits (within s pos width) pos 9223372036854775807L))

;; A double, as `string->double` reads one: `1.5`, `-2`, `3e8`.
(: (-> string int int (Tuple double int)))
(defun (scan-double s pos width) #:inline
  (def b (within s pos width))
  (let (((ok? v used) (bytes-double (bytes-part b pos (ro-span-length b)) (clr-default))))
    (if ok? (Tuple v (+ pos used)) (Tuple 0.0 -1))))

;; The low 6 bits of a continuation byte.
(: (-> (RoSpan byte) int int))
(defun (continued b i) #:inline (bitwise-and (byte-at b i) 63))

;; The character that starts with the byte `c` at `pos`, which is not ASCII,
;; and the position after it. The line is valid UTF-8.
(: (-> (RoSpan byte) int int (Tuple char int)))
(defun (decode-char b pos c)
  (cond
    ((>= c 240)
     (Tuple (int->char (bitwise-ior (bitwise-ior (shift-left (bitwise-and c 7) 18)
                                                     (shift-left (continued b (+ pos 1)) 12))
                                        (bitwise-ior (shift-left (continued b (+ pos 2)) 6)
                                                     (continued b (+ pos 3)))))
            (+ pos 4)))
    ((>= c 224)
     (Tuple (int->char (bitwise-ior (bitwise-ior (shift-left (bitwise-and c 15) 12)
                                                     (shift-left (continued b (+ pos 1)) 6))
                                        (continued b (+ pos 2))))
            (+ pos 3)))
    (else
     (Tuple (int->char (bitwise-ior (shift-left (bitwise-and c 31) 6) (continued b (+ pos 1))))
            (+ pos 2)))))

;; One character, whatever it is.
(: (-> string int (Tuple char int)))
(defun (scan-char s pos) #:inline
  (def b (line-bytes s))
  (if (>= pos (ro-span-length b))
      (Tuple (int->char 0) -1)
      (let ((c (byte-at b pos)))
        (if (< c 128) (Tuple (int->char c) (+ pos 1)) (decode-char b pos c)))))

;; One or more characters up to whitespace or the end.
(: (-> string int int int))
(defun (scan-word s pos width) #:inline
  (def b (within s pos width))
  (def end (ro-span-length b))
  (def stop
    (let go ((i pos))
      (cond
        ((>= i end) i)
        ((blank? (byte-at b i)) i)
        (else (go (+ i 1))))))
  (if (= stop pos) -1 stop))

;; The rest of the line, which may be empty. Never fails.
(: (-> string int int int))
(defun (scan-rest s pos width) #:inline
  (if (< width 0) (line-length s) (count-end (line-bytes s) pos width)))

;; Whether an ASCII set holds a byte. The set is a 128-bit mask in two
;; halves: bit c of `low` is character c, and bit c - 64 of `high` is
;; character c from 64. A byte from 128 up is in no set.
(: (-> int long long bool))
(defun (in-set? c low high) #:inline
  (if (< c 64)
      (not (= (bitwise-and (shift-right-logical low c) 1L) 0L))
      (and (< c 128) (not (= (bitwise-and (shift-right-logical high (- c 64)) 1L) 0L)))))

;; One or more characters of an ASCII set.
(: (-> string int long long int int))
(defun (scan-chars s pos low high width) #:inline
  (def b (within s pos width))
  (def end (ro-span-length b))
  (def stop
    (let go ((i pos))
      (cond
        ((>= i end) i)
        ((in-set? (byte-at b i) low high) (go (+ i 1)))
        (else i))))
  (if (= stop pos) -1 stop))

;; Any characters up to one of an ASCII set or the end, none included. Never
;; fails. A character that is not ASCII is in no set, so the end is never
;; inside one.
(: (-> string int long long int int))
(defun (scan-until s pos low high width) #:inline
  (def b (within s pos width))
  (def end (ro-span-length b))
  (let go ((i pos))
    (cond
      ((>= i end) i)
      ((in-set? (byte-at b i) low high) i)
      (else (go (+ i 1))))))

;; Any characters up to one ASCII byte or the end, none included. Never
;; fails. A vectorized search, which is what `scan-until` with one character
;; compiles to.
(: (-> string int int int int))
(defun (scan-until-byte s pos stop width) #:inline
  (def b (within s pos width))
  (def end (ro-span-length b))
  (def found (bytes-find (bytes-part b pos end) (cast byte stop)))
  (if (< found 0) end (+ pos found)))

;; The text from `from` to `to`, which shares the line's bytes. What a text
;; item's value is.
(: (-> string int int string))
(defun (scan-slice s from to) #:inline (line-part s from (- to from)))

;; Whether `pos` is the end of the line.
(: (-> string int bool))
(defun (scan-end? s pos) #:inline (= pos (line-length s)))

;; Why the line does not match: what was expected, at which column, and the
;; line. The column counts characters from 1.
(: (-> string int string string))
(defun (scan-message s pos expected)
  (def b (line-bytes s))
  (def column
    (let go ((i 0) (k 1))
      (if (< i pos)
          (go (+ i 1) (if (= (bitwise-and (byte-at b i) 192) 128) k (+ k 1)))
          k)))
  (str "scan: expected " expected " at column " (int->string column) " of \"" s "\""))

;; Raises a FormatException that says why the line does not match.
(: (-> string int string %a))
(defun (scan-fail s pos expected)
  (raise (cast Exception (FormatException. (scan-message s pos expected)))))

;; ---------------------------------------------------------------------------
;; The pattern
;; ---------------------------------------------------------------------------

;; One item of a pattern. An `(Option int)` is the item's width: the most
;; characters it reads. `ISet` is the characters of `(chars ...)`, or of
;; `(until ...)` when its flag is set, and the text an error says it
;; expected.
(type (: Item (Union (: ILit string)
                     IWs
                     (: IRead string (Option int))
                     (: ISet string bool string (Option int))
                     (: IGroup (List Item))
                     (: IRep Item (Option Item))
                     (: IOpt Item)
                     (: IOr (List Item))
                     (: ISkip Item))))

;; Whether the item reads a value, which the result then holds.
(: (-> Item bool))
(defun (valued? item)
  (match item
    ((ILit _) #f)
    (IWs #f)
    ((IRead _ _) #t)
    ((ISet _ _ _ _) #t)
    ((IGroup items) (any? valued? items))
    ((IRep _ _) #t)
    ((IOpt inner) (valued? inner))
    ((IOr alts) (valued? (list-head alts)))
    ((ISkip _) #f)))

;; What an error says the item was expected to be.
(: (-> Item string))
(defun (describe item)
  (match item
    ((ILit t) (str "\"" t "\""))
    (IWs "whitespace")
    ((IRead kind _)
     (case kind
       (("int") "an int")
       (("long") "a long")
       (("double") "a double")
       (("char") "a character")
       (("word") "a word")
       (else "the rest of the line")))
    ((ISet _ _ what _) what)
    ((IGroup items) (if (list-empty? items) "nothing" (describe (list-head items))))
    ((IRep inner _) (describe inner))
    ((IOpt inner) (describe inner))
    ((IOr alts) (str "one of " (string-join (list->vec (list-map describe alts)) ", ")))
    ((ISkip inner) (describe inner))))

(: (-> char bool))
(defun (space? c) (or (char=? c #\space) (char=? c #\tab)))

;; A string of a pattern: the text between its spaces, each exactly, and a
;; run of spaces as any amount of whitespace.
(: (-> string (List Item)))
(defun (string-items text)
  (defun (flush lit acc)
    (if (list-empty? lit) acc (Cons (ILit (list->string (list-reverse lit))) acc)))
  (defun (skip cs)
    (match cs
      ((Cons c rest) #:when (space? c) (skip rest))
      (_ cs)))
  (let go ((cs (string->list text)) (lit Nil) (acc Nil))
    (match cs
      (Nil (list-reverse (flush lit acc)))
      ((Cons c rest)
       (cond ((space? c)
              (go (skip rest) Nil (Cons IWs (flush lit acc))))
             (else (go rest (Cons c lit) acc)))))))

;; The bit of an ASCII character in a set's two halves.
(: (-> int (Tuple long long)))
(defun (char-bit code)
  (if (< code 64)
      (Tuple (shift-left 1L code) 0L)
      (Tuple 0L (shift-left 1L (- code 64)))))

;; The characters a `(chars ...)` keyword names.
(: (-> string (Option string)))
(defun (char-class name)
  (case name
    (("digit") (Some "0123456789"))
    (("lower") (Some "abcdefghijklmnopqrstuvwxyz"))
    (("upper") (Some "ABCDEFGHIJKLMNOPQRSTUVWXYZ"))
    (("alpha") (Some "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"))
    (("alnum") (Some "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"))
    (("hex") (Some "0123456789abcdefABCDEF"))
    (else None)))

;; A set of ASCII characters as the two halves `in-set?` tests a byte
;; against.
(: (-> string (Tuple long long)))
(defun (mask text)
  (string-fold
   (fun (c acc)
     (def (l . h) (char-bit (char->int c)))
     (Tuple (bitwise-ior (fst acc) l) (bitwise-ior (snd acc) h)))
   (Tuple 0L 0L) text))

;; The characters of a string of a set, with `a-z` as all the characters from
;; `a` to `z`. A `-` at the start or the end is itself.
(: (-> string string Syntax string))
(defun (with-ranges text form at)
  (defun (span from to)
    (let go ((c to) (acc Nil))
      (if (< c from) (list->string acc) (go (- c 1) (Cons (int->char c) acc)))))
  (let go ((cs (string->list text)) (acc ""))
    (match cs
      ((Cons a (Cons dash (Cons b rest))) #:when (char=? dash #\-)
       (def* (from (char->int a))
             (to (char->int b)))
       (when (> from to)
         (ignore (ret-error at (str form " has the range " (char->string a) "-" (char->string b) ", which goes backwards."))))
       (go rest (str acc (span from to))))
      ((Cons c rest) (go rest (str acc (char->string c))))
      (Nil acc))))

;; `(chars ".#")`, `(chars "a-z" :digit)` and the same for `(until ...)`:
;; the characters of the strings and the classes. `form` names the item in
;; errors.
(: (-> string bool (List Syntax) Syntax Item))
(defun (set-item form until? parts whole)
  (def text
    (fold
     (fun (part acc)
       (match part
         ((SStr s) (str acc (with-ranges s form part)))
         ((SKey k)
          (match (char-class (displayed k))
            ((Some cs) (str acc cs))
            (None (ret-error part (str form " knows the classes :digit, :lower, :upper, :alpha, :alnum and :hex, not :" (displayed k) ".")))))
         (_ (ret-error part (str form " takes strings of characters and class keywords, such as (chars \".#\"), (chars \"a-z\" \"_\") or (until \",\").")))))
     "" parts))
  (when (string-empty? text)
    (ignore (ret-error whole (str form " needs at least one character."))))
  (string-for-each
   (fun (c)
     (when (>= (char->int c) 128)
       (ignore (ret-error whole (str form " takes ASCII characters only, and " (char->string c) " is not one.")))))
   text)
  (def written
    (string-join
     (list->vec (list-map (fun (part)
                            (match part
                              ((SStr s) (str "\"" s "\""))
                              ((SKey k) (str ":" (displayed k)))
                              (_ "")))
                          parts))
     " "))
  (ISet text until? (if until? (str "text up to one of " written) (str "one of " written)) None))

;; A syntax error, as a value of any type: the transformer raises it, and the
;; expander reports it at the form it names.
(: (-> Syntax string %a))
(defun (ret-error at message)
  (ignore (syntax-error at message))
  (panic! "unreachable: syntax-error does not return"))

(: (-> Syntax string bool))
(defun (keyword? s name)
  (match s
    ((SKey k) (= (displayed k) name))
    (_ #f)))

(def items-are
  "The items are strings, int, long, double, char, word, rest, (chars ...), (until ...), a group (...), (:? ...), (:or ...) and (:skip ...).")

;; One item, without what follows it.
(: (-> Syntax Item))
(defun (parse-item s)
  (match s
    ((SStr text)
     (match (string-items text)
       ((Cons only Nil) only)
       (items (IGroup items))))
    ((SSym name)
     (def kind (symbol->string name))
     (if (any? #(= & kind) (list "int" "long" "double" "char" "word" "rest"))
         (IRead kind None)
         (ret-error s (str "'" kind "' is not an item of a scan pattern. " items-are))))
    ((SList (Cons head rest))
     (cond ((keyword? head "?")
            (IOpt (group-of (parse-items rest s))))
           ((keyword? head "or") (or-item rest s))
           ((keyword? head "skip")
            (ISkip (group-of (parse-items rest s))))
           ((named? head "chars") (set-item "(chars ...)" #f rest s))
           ((named? head "until") (set-item "(until ...)" #t rest s))
           (else (IGroup (parse-items (Cons head rest) s)))))
    ((SList Nil) (IGroup Nil))
    (_ (ret-error s (str "This is not an item of a scan pattern. " items-are)))))

(: (-> Syntax string bool))
(defun (named? s name)
  (match s
    ((SSym n) (= (symbol->string n) name))
    (_ #f)))

;; `item #:width n`: the item reads no more than n characters.
(: (-> Item int Syntax Item))
(defun (with-width item n at)
  (def message "#:width goes after int, long, double, word, rest, (chars ...) or (until ...).")
  (match item
    ((IRead _ (Some _)) (ret-error at "This item has a #:width already."))
    ((ISet _ _ _ (Some _)) (ret-error at "This item has a #:width already."))
    ((IRead kind None)
     (if (= kind "char") (ret-error at message) (IRead kind (Some n))))
    ((ISet text until? what None) (ISet text until? what (Some n)))
    (_ (ret-error at message))))

;; `(:or a b ...)`: each choice is one item, and they all read a value or
;; none does.
(: (-> (List Syntax) Syntax Item))
(defun (or-item choices whole)
  (def alts (list-map (fun (a) (group-of (parse-items (list a) a))) choices))
  (when (list-empty? alts)
    (ignore (ret-error whole "(:or ...) needs at least one choice.")))
  (when (not (all? #(= (valued? &) (valued? (list-head alts))) alts))
    (ignore (ret-error whole "(:or ...) needs every choice to read a value, or none of them.")))
  (IOr alts))

;; A list of items as one item.
(: (-> (List Item) Item))
(defun (group-of items)
  (match items
    ((Cons only Nil) only)
    (_ (IGroup items))))

;; The items of a list, with `item ...` and `item ... #:sep sep` read as
;; repetitions of the item before them, and `item #:width n` as the item
;; with a width.
(: (-> (List Syntax) Syntax (List Item)))
(defun (parse-items xs whole)
  (let go ((xs xs) (acc Nil))
    (match xs
      (Nil (list-reverse acc))
      ((Cons k (Cons (SInt digits) more)) #:when (and (keyword? k "width") (> (string->int digits) 0))
       (match acc
         (Nil (ret-error k "#:width gives a width to the item before it, and there is none."))
         ((Cons prev before) (go more (Cons (with-width prev (string->int digits) k) before)))))
      ((Cons k _) #:when (keyword? k "width")
       (ret-error k "#:width needs a number of characters after it, as in int #:width 4."))
      ((Cons (SPunct "...") rest)
       (match acc
         (Nil (ret-error whole "A '...' repeats the item before it, and there is none."))
         ((Cons prev before)
          (when (not (valued? prev))
            (ignore (ret-error whole "A '...' repeats an item that reads a value, and a string reads none. Put the string in a group with something that does, as in (\"(\" int \")\") ...")))
          (match rest
            ((Cons k (Cons sep more)) #:when (keyword? k "sep")
             (go more (Cons (IRep prev (Some (parse-item sep))) before)))
            ((Cons k Nil) #:when (keyword? k "sep")
             (ret-error whole "#:sep needs the separator after it, as in int ... #:sep \",\"."))
            (_ (go rest (Cons (IRep prev None) before)))))))
      ((Cons x rest) (go rest (Cons (parse-item x) acc))))))

;; ---------------------------------------------------------------------------
;; The expansion
;; ---------------------------------------------------------------------------

;; The line's name in the expansion, and a counter for the expansion's own
;; temporaries. Each temporary is built from a string, so that each one gets a
;; name of its own: hygiene gives every distinct name one fresh spelling per
;; expansion, and one template name used at every depth would shadow itself.
;; `keep` is false inside `(:skip ...)`: there an item gives no value, so a
;; text item makes no slice and a repetition no vec.
(type (: Ctx (Record (: line Syntax) (: counter (Array int)) (: keep bool))))

;; Whether the item gives a value here.
(: (-> Ctx Item bool))
(defun (holds? ctx item) (and (.keep ctx) (valued? item)))

(: (-> Ctx string Syntax))
(defun (fresh ctx prefix)
  (def n (ref (.counter ctx) 0))
  (array-set! (.counter ctx) 0 (+ n 1))
  (SSym (string->symbol (str "scan-" prefix (displayed n)))))

;; What an item does with its value and the position after it, and what it
;; does when it does not match.
(type (: OnValue (-> (Option Syntax) Syntax Syntax)))
(type (: OnMiss (-> Syntax string Syntax)))

;; A width as the steps take it: -1 for none.
(: (-> (Option int) Syntax))
(defun (width-literal width)
  (match width
    ((Some n) (SInt (int->string n)))
    (None (SInt "-1"))))

(: (-> int Syntax))
(defun (int-literal n) (SInt (int->string n)))

(: (-> long Syntax))
(defun (long-literal n) (SInt (str (long->string n) "L")))

;; Up to 8 byte values as one long, the first in the lowest byte, as
;; `scan-lit8` reads them.
(: (-> (List int) long))
(defun (packed codes)
  (list-foldr (fun (code acc) (bitwise-ior (shift-left acc 8) (cast System.Int64 code))) 0L codes))

;; Code for `item` at position `p`.
(: (-> Ctx Item Syntax OnValue OnMiss Syntax))
(defun (gen ctx item p ok fail)
  (def s (.line ctx))
  (match item
    ((ILit text)
     (def q (fresh ctx "q"))
     (def codes (list-map char->int (string->list text)))
     (def n (list-length codes))
     (def step
       (cond ((not (all? #(< & 128) codes)) #'(scan-lit ,s ,p ,(SStr text)))
             ((= n 1) #'(scan-byte ,s ,p ,(int-literal (list-head codes))))
             ((<= n 8) #'(scan-lit8 ,s ,p ,(long-literal (packed codes)) ,(int-literal n)))
             (else #'(scan-lit ,s ,p ,(SStr text)))))
     #'(let ((,q ,step))
         (if (< ,q 0) ,(fail p (describe item)) ,(ok None q))))
    (IWs
     (def q (fresh ctx "q"))
     #'(let ((,q (scan-ws ,s ,p))) ,(ok None q)))
    ((IRead kind width)
     (def w (width-literal width))
     (case kind
       (("int") (gen-number ctx item #'(scan-int ,s ,p ,w) p ok fail))
       (("long") (gen-number ctx item #'(scan-long ,s ,p ,w) p ok fail))
       (("double") (gen-number ctx item #'(scan-double ,s ,p ,w) p ok fail))
       (("char") (gen-number ctx item #'(scan-char ,s ,p) p ok fail))
       (("word") (gen-text ctx item #'(scan-word ,s ,p ,w) p ok fail))
       (else (gen-text ctx item #'(scan-rest ,s ,p ,w) p ok fail))))
    ((ISet text until? _ width)
     (def w (width-literal width))
     (def (low . high) (mask text))
     (def step
       (cond ((not until?) #'(scan-chars ,s ,p ,(long-literal low) ,(long-literal high) ,w))
             ((= (string-length text) 1)
              #'(scan-until-byte ,s ,p ,(int-literal (char->int (list-head (string->list text)))) ,w))
             (else #'(scan-until ,s ,p ,(long-literal low) ,(long-literal high) ,w))))
     (gen-text ctx item step p ok fail))
    ((IGroup items) (gen-seq ctx items p Nil ok fail))
    ((IOpt inner) (gen-opt ctx inner p ok))
    ((IOr alts) (gen-or ctx alts item p ok fail))
    ((IRep inner sep) (gen-rep ctx inner sep p ok))
    ((ISkip inner)
     (gen (Ctx (line s) (counter (.counter ctx)) (keep #f)) inner p
          (fun (_ q) (ok None q)) fail))))

;; A step that answers a value and the position after it.
(: (-> Ctx Item Syntax Syntax OnValue OnMiss Syntax))
(defun (gen-number ctx item step p ok fail)
  (def* (v (fresh ctx "v"))
        (q (fresh ctx "q")))
  #'(match ,step
      ((,v . ,q) (if (< ,q 0) ,(fail p (describe item)) ,(ok (if (.keep ctx) (Some v) None) q)))))

;; A step that answers the position after its text. The value is the slice
;; from `p` to there, made only where the value is used.
(: (-> Ctx Item Syntax Syntax OnValue OnMiss Syntax))
(defun (gen-text ctx item step p ok fail)
  (def* (s (.line ctx))
        (q (fresh ctx "q")))
  #'(let ((,q ,step))
      (if (< ,q 0)
          ,(fail p (describe item))
          ,(ok (if (.keep ctx) (Some #'(scan-slice ,s ,p ,q)) None) q))))

;; Code for items in order. A group's value is its one value, or a tuple of
;; its values, or none.
(: (-> Ctx (List Item) Syntax (List Syntax) OnValue OnMiss Syntax))
(defun (gen-seq ctx items p vals ok fail)
  (match items
    (Nil
     (match (list-reverse vals)
       (Nil (ok None p))
       ((Cons only Nil) (ok (Some only) p))
       (all (ok (Some #'(Tuple ,@all)) p))))
    ((Cons item rest)
     (gen ctx item p
          (fun (v q)
            (gen-seq ctx rest q (match v ((Some x) (Cons x vals)) (None vals)) ok fail))
          fail))))

;; A join point: a local function `k` that runs `ok` on its arguments, so
;; that several paths can continue with one copy of it. `valued?` says
;; whether it takes a value before the position.
(: (-> Ctx bool OnValue (-> (-> (Option Syntax) Syntax Syntax) Syntax) Syntax))
(defun (join ctx valued? ok body)
  (def* (k (fresh ctx "k"))
        (kv (fresh ctx "kv"))
        (kp (fresh ctx "kp")))
  (if valued?
      #'(let ()
          (defun (,k ,kv ,kp) ,(ok (Some kv) kp))
          ,(body (fun (v q) #'(,k ,(option-ref v) ,q))))
      #'(let ()
          (defun (,k ,kp) ,(ok None kp))
          ,(body (fun (v q) #'(,k ,q))))))

;; `(:? item)`: the item's value in a `Some`, or `None` and no text read.
(: (-> Ctx Item Syntax OnValue Syntax))
(defun (gen-opt ctx inner p ok)
  (def valued (holds? ctx inner))
  (join ctx valued ok
        (fun (k)
          (gen ctx inner p
               (fun (v q) (k (option-map (fun (x) #'(Some ,x)) v) q))
               (fun (_ _) (k (if valued (Some #'None) None) p))))))

;; `(:or a b ...)`: the first choice that matches, each tried at `p`.
(: (-> Ctx (List Item) Item Syntax OnValue OnMiss Syntax))
(defun (gen-or ctx alts whole p ok fail)
  (join ctx (valued? whole) ok
        (fun (k)
          (let next ((alts alts))
            (match alts
              (Nil (fail p (describe whole)))
              ((Cons a rest) (gen ctx a p k (fun (_ _) (next rest)))))))))

;; `item ...` and `item ... #:sep sep`: the items as a vec, as many as match,
;; none included. A separator not followed by an item is not read. The loop
;; is a named let, so it runs as a loop however many items there are. Where
;; the value is not kept, the loop only moves past the items.
(: (-> Ctx Item (Option Item) Syntax OnValue Syntax))
(defun (gen-rep ctx inner sep p ok)
  ;; The loop reads an item at `at`, and stops at `back` when the item does
  ;; not match: `back` is the position before the separator, so that a
  ;; separator not followed by an item is not read. The item and the
  ;; separator are each written once, so nested repetitions grow with the
  ;; pattern.
  (def* (keep (.keep ctx))
        (acc (fresh ctx "acc"))
        (back (fresh ctx "back"))
        (at (fresh ctx "at"))
        (more (fresh ctx "more")))
  (join ctx keep ok
        (fun (k)
          (defun (stop pos) (k (if keep (Some #'(vecbuilder->vec ,acc)) None) pos))
          ;; After the separator at `q`, or none: round again, or stop at `q`.
          (defun (next q)
            (match sep
              (None #'(,more ,q ,q))
              ((Some sep-item)
               (gen ctx sep-item q (fun (_ sq) #'(,more ,q ,sq)) (fun (_ _) (stop q))))))
          ;; After an item: add its value and go on from `q`. A round that
          ;; reads no text, as `rest` at the end does, ends the repetition
          ;; without its value, because it would match forever.
          (defun (add v q)
            #'(if (= ,q ,back)
                  ,(stop back)
                  ,(if keep
                       #'(let ()
                           (vecbuilder-add! ,acc ,(option-ref v))
                           ,(next q))
                       (next q))))
          (def loop
            #'(let ,more ((,back ,p) (,at ,p))
                ,(gen ctx inner at add (fun (_ _) (stop back)))))
          (if keep
              #'(let ((,acc (vecbuilder-empty/small))) ,loop)
              loop))))

;; The whole expansion: the items from position 0, then the end of the line.
(: (-> Syntax Syntax Syntax bool Syntax))
(defun (expand form line pattern raising?)
  (def quoted
    (match pattern
      ((SList (Cons (SSym q) items))
       (if (= (symbol->string q) "quoted-list") (Some items) None))
      (_ None)))
  (def items
    (match quoted
      ((Some items) (parse-items items pattern))
      (None (ret-error pattern "The pattern of a scan is a quoted list, such as '(int \",\" int)."))))
  (def ctx (Ctx (line (SSym (string->symbol "scan-line"))) (counter (make-array 1 #:fill 0)) (keep #t)))
  (def s (.line ctx))
  (defun (fail p expected)
    (if raising?
        #'(scan-fail ,s ,p ,(SStr expected))
        #'(Err (scan-message ,s ,p ,(SStr expected)))))
  (defun (finish v p)
    (def value (match v ((Some x) x) (None #'unit)))
    #'(if (scan-end? ,s ,p)
          ,(if raising? value #'(Ok ,value))
          ,(fail p "the end of the line")))
  #'(let ((,s ,line))
      ,(gen-seq ctx items #'0 Nil finish fail)))

;; `(scan line '(pattern ...))`: the values the pattern reads, or raises a
;; FormatException that says where the line does not match.
(def/macro (scan form inject compare)
  (syntax-match form
    ((_ line pattern) (expand form line pattern #t))
    (bad (syntax-error bad "scan takes a line and a quoted pattern, as in (scan line '(int \",\" int))."))))

;; `(try-scan line '(pattern ...))`: the same values in an `Ok`, or an `Err`
;; with the message `scan` would raise.
(def/macro (try-scan form inject compare)
  (syntax-match form
    ((_ line pattern) (expand form line pattern #f))
    (bad (syntax-error bad "try-scan takes a line and a quoted pattern, as in (try-scan line '(int \",\" int))."))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.


;; std/simd.bjo — vectors of numbers, a fixed number of lanes wide.
;;
;; A lane type is a .NET `Vector128<T>` or `Vector256<T>`: 128 or 256 bits of
;; numbers of one type. `Float64x4` is four doubles and `Int8x32` is 32 signed
;; bytes. An operation on one is one CPU instruction where the CPU has it.
;;
;; The operations are the methods of a few traits, which both widths
;; implement: `Simd` for what every element type has, `SimdFloat` and
;; `SimdInt` for what only floats or only integers have, `SimdBitwise` for
;; vectors and masks alike, and `SimdMask` for masks. So `simd-add` adds two
;; `Float64x2`s or two `Int32x8`s. What makes a vector — `simd256-load`,
;; `simd128-splat` — says its width, as there is no argument to take it from,
;; and so does a conversion, whose result is a vector of another type.
;;
;; A comparison gives a mask, `(Mask128 %a)` or `(Mask256 %a)`, which is a
;; type of its own: it can be combined, counted and selected with, but not
;; added up as numbers.
;;
;; The design follows OxCaml's SIMD library where that fits a language with
;; traits and unsigned types: typed masks, operations only where they mean
;; something, and conversions named for what they do with what does not fit.
;;
;; A 128-bit vector is fast on every x86-64 and ARM64 CPU. A 256-bit one needs
;; AVX2: elsewhere .NET emulates it, which is much slower than scalar code.
;; `simd256-accelerated?` says which. A Native AOT build has AVX2 only with
;; `(aot-cpu ...)`.
(import (std prelude)
        (std syntax-match)
        (std mutable vec))

(:doc #:file "simd.bjodoc")

(export
  Simd128 Simd256 Mask128 Mask256
  Float64x2 Float32x4 Int64x2 Int32x4 Int16x8 Int8x16
  UInt64x2 UInt32x4 UInt16x8 UInt8x16
  Float64x4 Float32x8 Int64x4 Int32x8 Int16x16 Int8x32
  UInt64x4 UInt32x8 UInt16x16 UInt8x32
  ;; The element types
  Lane IntLane FloatLane
  ;; What every vector has
  Simd
  simd-add simd-sub simd-mul simd-scale simd-neg simd-abs simd-min simd-max
  simd-eq simd-ne simd-lt simd-le simd-gt simd-ge simd-select
  simd-sum simd-dot simd-ref simd-with simd-store! simd-lanes
  ;; Bits, of vectors and masks
  SimdBitwise
  simd-and simd-or simd-xor simd-not simd-and-not
  ;; Masks
  SimdMask
  simd-any? simd-all? simd-count-true simd-first-true simd-mask-bits simd-mask->lanes
  ;; Floats only
  SimdFloat
  simd-div simd-sqrt simd-floor simd-ceiling simd-round simd-truncate simd-nan? simd-mul-add
  ;; Integers only
  SimdInt
  simd-shift-left simd-shift-right simd-shift-right-logical
  simd-add-saturating simd-sub-saturating simd-shuffle
  ;; Byte tables
  ByteLane SimdLookup simd-lookup
  ;; The same bits as another element type
  SimdBits
  simd-bits->int8 simd-bits->uint8 simd-bits->int16 simd-bits->uint16
  simd-bits->int32 simd-bits->uint32 simd-bits->int64 simd-bits->uint64
  simd-bits->float32 simd-bits->float64
  ;; Conversions
  SimdWiden SimdNarrow SimdIntToFloat SimdFloatToInt
  simd128-widen-lower simd128-widen-upper simd256-widen-lower simd256-widen-upper
  simd128-narrow-wrapping simd128-narrow-saturating
  simd256-narrow-wrapping simd256-narrow-saturating
  simd128-int->float simd256-int->float
  simd128-float->int-truncating simd256-float->int-truncating
  simd128-float->uint-truncating simd256-float->uint-truncating
  ;; Where vectors are loaded from
  SimdSource simd-source-length
  ;; Making one
  simd128-splat simd128-zero simd128-of simd128-load simd128-lanes simd128-whole
  simd256-splat simd256-zero simd256-of simd256-load simd256-lanes simd256-whole
  simd128-accelerated? simd256-accelerated?)

(import/class
  ((Simd128 %a) (: System.Runtime.Intrinsics.Vector128))
  ((Simd256 %a) (: System.Runtime.Intrinsics.Vector256)))

;; A comparison's answer: a vector of the same type, with every bit of a lane
;; set where the comparison holds and none where it does not, kept apart from
;; the vectors so that it is not mistaken for numbers. Only this module makes
;; one. The JIT keeps it in a register as it keeps the vector.
(type (: (Mask128 %a) #:opaque (Struct (: lanes (Simd128 %a)))))
(type (: (Mask256 %a) #:opaque (Struct (: lanes (Simd256 %a)))))

;; ---------------------------------------------------------------------------
;; The lane types
;; ---------------------------------------------------------------------------
;;
;; Named as OxCaml names them: the element's type and bits, and the lanes.

(type (: Float64x2 (Simd128 double)))
(type (: Float32x4 (Simd128 System.Single)))
(type (: Int64x2   (Simd128 long)))
(type (: Int32x4   (Simd128 int)))
(type (: Int16x8   (Simd128 short)))
(type (: Int8x16   (Simd128 System.SByte)))
(type (: UInt64x2  (Simd128 ulong)))
(type (: UInt32x4  (Simd128 uint)))
(type (: UInt16x8  (Simd128 ushort)))
(type (: UInt8x16  (Simd128 byte)))

(type (: Float64x4 (Simd256 double)))
(type (: Float32x8 (Simd256 System.Single)))
(type (: Int64x4   (Simd256 long)))
(type (: Int32x8   (Simd256 int)))
(type (: Int16x16  (Simd256 short)))
(type (: Int8x32   (Simd256 System.SByte)))
(type (: UInt64x4  (Simd256 ulong)))
(type (: UInt32x8  (Simd256 uint)))
(type (: UInt16x16 (Simd256 ushort)))
(type (: UInt8x32  (Simd256 byte)))

;; ---------------------------------------------------------------------------
;; The element types
;; ---------------------------------------------------------------------------
;;
;; .NET's vectors hold the primitive numbers below and nothing else: a vector
;; of another type is refused when it is made, at run time. `Lane` says which
;; types they are, so a vector of a char or a string is refused when it is
;; compiled. `IntLane` and `FloatLane` divide them, for the operations that
;; mean something for one kind only. None has methods, and nothing outside
;; this module implements them.

(def/trait (Lane %a))
(impl (Lane double))
(impl (Lane System.Single))
(impl (Lane long))
(impl (Lane int))
(impl (Lane short))
(impl (Lane System.SByte))
(impl (Lane ulong))
(impl (Lane uint))
(impl (Lane ushort))
(impl (Lane byte))

(def/trait (IntLane %a))
(impl (IntLane long))
(impl (IntLane int))
(impl (IntLane short))
(impl (IntLane System.SByte))
(impl (IntLane ulong))
(impl (IntLane uint))
(impl (IntLane ushort))
(impl (IntLane byte))

(def/trait (FloatLane %a))
(impl (FloatLane double))
(impl (FloatLane System.Single))

;; The element type `simd-lookup` takes its indices in.
(def/trait (ByteLane %a))
(impl (ByteLane byte))

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------

;; What every vector has. A comparison gives a `%mask`, which `simd-select`
;; takes: a lane from its second argument where the mask is set, and from its
;; third where it is not.
(def/trait (Simd %v)
  (type %elem)
  (type %mask)
  (: simd-add   (-> %v %v %v))
  (: simd-sub   (-> %v %v %v))
  (: simd-mul   (-> %v %v %v))
  (: simd-scale (-> %v %elem %v))
  (: simd-neg   (-> %v %v))
  (: simd-abs   (-> %v %v))
  (: simd-min   (-> %v %v %v))
  (: simd-max   (-> %v %v %v))
  (: simd-eq    (-> %v %v %mask))
  (: simd-ne    (-> %v %v %mask))
  (: simd-lt    (-> %v %v %mask))
  (: simd-le    (-> %v %v %mask))
  (: simd-gt    (-> %v %v %mask))
  (: simd-ge    (-> %v %v %mask))
  (: simd-select (-> %mask %v %v %v))
  (: simd-sum   (-> %v %elem))
  (: simd-dot   (-> %v %v %elem))
  (: simd-ref   (-> %v int %elem))
  (: simd-with  (-> %v int %elem %v))
  (: simd-store! (-> %v (Array %elem) int void))
  (: simd-lanes (-> %v int)))

;; The bitwise operations, which vectors and masks both have. Two masks
;; combine into a mask.
(def/trait (SimdBitwise %v)
  (: simd-and (-> %v %v %v))
  (: simd-or  (-> %v %v %v))
  (: simd-xor (-> %v %v %v))
  (: simd-not (-> %v %v))
  (: simd-and-not (-> %v %v %v)))

;; Reading a mask. A lane is set when every bit of it is, and the top bit says
;; it, so each reader is one move-mask instruction and an integer operation.
(def/trait (SimdMask %m)
  (type %vec)
  (: simd-any? (-> %m bool))
  (: simd-all? (-> %m bool))
  (: simd-count-true (-> %m int))
  (: simd-first-true (-> %m int))
  (: simd-mask-bits (-> %m int))
  (: simd-mask->lanes (-> %m %vec)))

;; What only float lanes have. Integer lanes divide one lane at a time, with
;; no instruction for it, and raise on a zero; that is not offered as a
;; vector operation.
(def/trait (SimdFloat %v)
  (type %mask)
  (: simd-div (-> %v %v %v))
  (: simd-sqrt (-> %v %v))
  (: simd-floor (-> %v %v))
  (: simd-ceiling (-> %v %v))
  (: simd-round (-> %v %v))
  (: simd-truncate (-> %v %v))
  (: simd-nan? (-> %v %mask))
  (: simd-mul-add (-> %v %v %v %v)))

;; What only integer lanes have. A shift of a float lane would shift the bits
;; of its encoding. A shuffle's indices are of the vector's own type, which
;; for a float vector would be another type; a float vector is shuffled as
;; the same bits read as integers, with `simd-bits->int64` and back.
(def/trait (SimdInt %v)
  (: simd-shift-left (-> %v int %v))
  (: simd-shift-right (-> %v int %v))
  (: simd-shift-right-logical (-> %v int %v))
  (: simd-add-saturating (-> %v %v %v))
  (: simd-sub-saturating (-> %v %v %v))
  (: simd-shuffle (-> %v %v %v)))

;; A lookup in a table of 16 bytes, one for each byte of the indices.
(def/trait (SimdLookup %v)
  (: simd-lookup (-> (Simd128 byte) %v %v)))

;; The same bits, read as lanes of another type. The lanes are as many as the
;; bits allow: an `Int64x2` is a `UInt8x16`.
(def/trait (SimdBits %v)
  (type %int8) (type %uint8) (type %int16) (type %uint16) (type %int32)
  (type %uint32) (type %int64) (type %uint64) (type %float32) (type %float64)
  (: simd-bits->int8 (-> %v %int8))
  (: simd-bits->uint8 (-> %v %uint8))
  (: simd-bits->int16 (-> %v %int16))
  (: simd-bits->uint16 (-> %v %uint16))
  (: simd-bits->int32 (-> %v %int32))
  (: simd-bits->uint32 (-> %v %uint32))
  (: simd-bits->int64 (-> %v %int64))
  (: simd-bits->uint64 (-> %v %uint64))
  (: simd-bits->float32 (-> %v %float32))
  (: simd-bits->float64 (-> %v %float64)))

;; ---------------------------------------------------------------------------
;; 128 bits
;; ---------------------------------------------------------------------------

(import/extern
  (v128-add (: System.Runtime.Intrinsics.Vector128.Add (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-sub (: System.Runtime.Intrinsics.Vector128.Subtract (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-mul (: System.Runtime.Intrinsics.Vector128.Multiply (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-div (: System.Runtime.Intrinsics.Vector128.Divide (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-scale (: System.Runtime.Intrinsics.Vector128.Multiply (-> (Simd128 %a) %a (Simd128 %a))))
  (v128-neg (: System.Runtime.Intrinsics.Vector128.Negate (-> (Simd128 %a) (Simd128 %a))))
  (v128-abs (: System.Runtime.Intrinsics.Vector128.Abs (-> (Simd128 %a) (Simd128 %a))))
  (v128-min (: System.Runtime.Intrinsics.Vector128.Min (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-max (: System.Runtime.Intrinsics.Vector128.Max (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-sqrt (: System.Runtime.Intrinsics.Vector128.Sqrt (-> (Simd128 %a) (Simd128 %a))))
  (v128-and (: System.Runtime.Intrinsics.Vector128.BitwiseAnd (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-or (: System.Runtime.Intrinsics.Vector128.BitwiseOr (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-xor (: System.Runtime.Intrinsics.Vector128.Xor (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-not (: System.Runtime.Intrinsics.Vector128.OnesComplement (-> (Simd128 %a) (Simd128 %a))))
  (v128-and-not (: System.Runtime.Intrinsics.Vector128.AndNot (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-eq (: System.Runtime.Intrinsics.Vector128.Equals (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-lt (: System.Runtime.Intrinsics.Vector128.LessThan (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-le (: System.Runtime.Intrinsics.Vector128.LessThanOrEqual (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-gt (: System.Runtime.Intrinsics.Vector128.GreaterThan (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-ge (: System.Runtime.Intrinsics.Vector128.GreaterThanOrEqual (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-nan? (: System.Runtime.Intrinsics.Vector128.IsNaN (-> (Simd128 %a) (Simd128 %a))))
  (v128-select (: System.Runtime.Intrinsics.Vector128.ConditionalSelect (-> (Simd128 %a) (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-sum (: System.Runtime.Intrinsics.Vector128.Sum (-> (Simd128 %a) %a)))
  (v128-dot (: System.Runtime.Intrinsics.Vector128.Dot (-> (Simd128 %a) (Simd128 %a) %a)))
  (v128-ref (: System.Runtime.Intrinsics.Vector128.GetElement (-> (Simd128 %a) int %a)))
  (v128-with (: System.Runtime.Intrinsics.Vector128.WithElement (-> (Simd128 %a) int %a (Simd128 %a))))
  (v128-store! (: System.Runtime.Intrinsics.Vector128.CopyTo (-> (Simd128 %a) (Array %a) int void)))
  (v128-add-saturating (: System.Runtime.Intrinsics.Vector128.AddSaturate (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-sub-saturating (: System.Runtime.Intrinsics.Vector128.SubtractSaturate (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-shuffle (: Bjolang.Runtime.BjoSimd.Shuffle128 (-> (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-lookup (: Bjolang.Runtime.BjoSimd.Lookup128 (-> (Simd128 byte) (Simd128 %a) (Simd128 %a))))
  (v128-as (: System.Runtime.Intrinsics.Vector128.As (-> (Simd128 %a) (Simd128 %b))))
  (v128-floor (: Bjolang.Runtime.BjoSimd.Floor128 (-> (Simd128 %a) (Simd128 %a))))
  (v128-ceiling (: Bjolang.Runtime.BjoSimd.Ceiling128 (-> (Simd128 %a) (Simd128 %a))))
  (v128-round (: Bjolang.Runtime.BjoSimd.Round128 (-> (Simd128 %a) (Simd128 %a))))
  (v128-truncate (: Bjolang.Runtime.BjoSimd.Truncate128 (-> (Simd128 %a) (Simd128 %a))))
  (v128-mul-add (: Bjolang.Runtime.BjoSimd.MulAdd128 (-> (Simd128 %a) (Simd128 %a) (Simd128 %a) (Simd128 %a))))
  (v128-lanes (: Bjolang.Runtime.BjoSimd.Lanes128 (-> (Simd128 %a) int)))
  (v128-any? (: Bjolang.Runtime.BjoSimd.Any128 (-> (Simd128 %a) bool)))
  (v128-all? (: Bjolang.Runtime.BjoSimd.All128 (-> (Simd128 %a) bool)))
  (v128-count-true (: Bjolang.Runtime.BjoSimd.CountTrue128 (-> (Simd128 %a) int)))
  (v128-first-true (: Bjolang.Runtime.BjoSimd.FirstTrue128 (-> (Simd128 %a) int)))
  (v128-splat (: System.Runtime.Intrinsics.Vector128.Create (-> %a (Simd128 %a))))
  (v128-zero (: Bjolang.Runtime.BjoSimd.Zero128 (-> (Simd128 %a))))
  (v128-mask-bits (: Bjolang.Runtime.BjoSimd.MaskBits128 (-> (Simd128 %a) int)))
  (v128-shift-left (: Bjolang.Runtime.BjoSimd.ShiftLeft128 (-> (Simd128 %a) int (Simd128 %a))))
  (v128-shift-right (: Bjolang.Runtime.BjoSimd.ShiftRight128 (-> (Simd128 %a) int (Simd128 %a))))
  (v128-shift-right-logical (: Bjolang.Runtime.BjoSimd.ShiftRightLogical128 (-> (Simd128 %a) int (Simd128 %a))))
  (v128-load-array (: System.Runtime.Intrinsics.Vector128.Create (-> (Array %a) int (Simd128 %a))))
  (v128-load-list (: Bjolang.Runtime.BjoSimd.LoadList128 (-> (MutableVec %a) int (Simd128 %a))))
  (v128-array-lanes (: Bjolang.Runtime.BjoSimd.ArrayLanes128 (-> (Array %a) int)))
  (v128-list-lanes (: Bjolang.Runtime.BjoSimd.ListLanes128 (-> (MutableVec %a) int)))
  (simd128-accelerated? (: System.Runtime.Intrinsics.Vector128.IsHardwareAccelerated #:get)))

(: (-> %a (Simd128 %a)) (where (Lane %a)))
(defun (simd128-splat x) (v128-splat x))

(: (-> (Simd128 %a)) (where (Lane %a)))
(defun (simd128-zero) (v128-zero))

;; The vector a mask holds, and the mask of a vector of a comparison's lanes.
(: (-> (Simd128 %a) (Mask128 %a)))
(defun (m128 v) (Mask128 (lanes v)))

(: (-> (Mask128 %a) (Simd128 %a)))
(defun (lanes128 m) (record-ref m lanes))

(impl (Simd (Simd128 %a)) (where (Lane %a))
  (type %elem %a)
  (type %mask (Mask128 %a))
  (defun (simd-add a b) (v128-add a b))
  (defun (simd-sub a b) (v128-sub a b))
  (defun (simd-mul a b) (v128-mul a b))
  (defun (simd-scale a b) (v128-scale a b))
  (defun (simd-neg a) (v128-neg a))
  (defun (simd-abs a) (v128-abs a))
  (defun (simd-min a b) (v128-min a b))
  (defun (simd-max a b) (v128-max a b))
  (defun (simd-eq a b) (m128 (v128-eq a b)))
  (defun (simd-ne a b) (m128 (v128-not (v128-eq a b))))
  (defun (simd-lt a b) (m128 (v128-lt a b)))
  (defun (simd-le a b) (m128 (v128-le a b)))
  (defun (simd-gt a b) (m128 (v128-gt a b)))
  (defun (simd-ge a b) (m128 (v128-ge a b)))
  (defun (simd-select m a b) (v128-select (lanes128 m) a b))
  (defun (simd-sum a) (v128-sum a))
  (defun (simd-dot a b) (v128-dot a b))
  (defun (simd-ref a i) (v128-ref a i))
  (defun (simd-with a i x) (v128-with a i x))
  (defun (simd-store! a b c) (v128-store! a b c))
  (defun (simd-lanes a) (v128-lanes a)))

(impl (SimdBitwise (Simd128 %a)) (where (Lane %a))
  (defun (simd-and a b) (v128-and a b))
  (defun (simd-or a b) (v128-or a b))
  (defun (simd-xor a b) (v128-xor a b))
  (defun (simd-not a) (v128-not a))
  (defun (simd-and-not a b) (v128-and-not a b)))

(impl (SimdBitwise (Mask128 %a)) (where (Lane %a))
  (defun (simd-and a b) (m128 (v128-and (lanes128 a) (lanes128 b))))
  (defun (simd-or a b) (m128 (v128-or (lanes128 a) (lanes128 b))))
  (defun (simd-xor a b) (m128 (v128-xor (lanes128 a) (lanes128 b))))
  (defun (simd-not a) (m128 (v128-not (lanes128 a))))
  (defun (simd-and-not a b) (m128 (v128-and-not (lanes128 a) (lanes128 b)))))

(impl (SimdMask (Mask128 %a)) (where (Lane %a))
  (type %vec (Simd128 %a))
  (defun (simd-any? m) (v128-any? (lanes128 m)))
  (defun (simd-all? m) (v128-all? (lanes128 m)))
  (defun (simd-count-true m) (v128-count-true (lanes128 m)))
  (defun (simd-first-true m) (v128-first-true (lanes128 m)))
  (defun (simd-mask-bits m) (v128-mask-bits (lanes128 m)))
  (defun (simd-mask->lanes m) (lanes128 m)))

(impl (SimdFloat (Simd128 %a)) (where (FloatLane %a))
  (type %mask (Mask128 %a))
  (defun (simd-div a b) (v128-div a b))
  (defun (simd-sqrt a) (v128-sqrt a))
  (defun (simd-floor a) (v128-floor a))
  (defun (simd-ceiling a) (v128-ceiling a))
  (defun (simd-round a) (v128-round a))
  (defun (simd-truncate a) (v128-truncate a))
  (defun (simd-nan? a) (m128 (v128-nan? a)))
  (defun (simd-mul-add a b c) (v128-mul-add a b c)))

(impl (SimdInt (Simd128 %a)) (where (IntLane %a))
  (defun (simd-shift-left a n) (v128-shift-left a n))
  (defun (simd-shift-right a n) (v128-shift-right a n))
  (defun (simd-shift-right-logical a n) (v128-shift-right-logical a n))
  (defun (simd-add-saturating a b) (v128-add-saturating a b))
  (defun (simd-sub-saturating a b) (v128-sub-saturating a b))
  (defun (simd-shuffle v idx) (v128-shuffle v idx)))

(impl (SimdLookup (Simd128 %a)) (where (ByteLane %a))
  (defun (simd-lookup table idx) (v128-lookup table idx)))

(impl (SimdBits (Simd128 %a)) (where (Lane %a))
  (type %int8 (Simd128 System.SByte))
  (type %uint8 (Simd128 byte))
  (type %int16 (Simd128 short))
  (type %uint16 (Simd128 ushort))
  (type %int32 (Simd128 int))
  (type %uint32 (Simd128 uint))
  (type %int64 (Simd128 long))
  (type %uint64 (Simd128 ulong))
  (type %float32 (Simd128 System.Single))
  (type %float64 (Simd128 double))
  (defun (simd-bits->int8 v) (v128-as v))
  (defun (simd-bits->uint8 v) (v128-as v))
  (defun (simd-bits->int16 v) (v128-as v))
  (defun (simd-bits->uint16 v) (v128-as v))
  (defun (simd-bits->int32 v) (v128-as v))
  (defun (simd-bits->uint32 v) (v128-as v))
  (defun (simd-bits->int64 v) (v128-as v))
  (defun (simd-bits->uint64 v) (v128-as v))
  (defun (simd-bits->float32 v) (v128-as v))
  (defun (simd-bits->float64 v) (v128-as v)))

;; ---------------------------------------------------------------------------
;; 256 bits
;; ---------------------------------------------------------------------------

(import/extern
  (v256-add (: System.Runtime.Intrinsics.Vector256.Add (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-sub (: System.Runtime.Intrinsics.Vector256.Subtract (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-mul (: System.Runtime.Intrinsics.Vector256.Multiply (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-div (: System.Runtime.Intrinsics.Vector256.Divide (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-scale (: System.Runtime.Intrinsics.Vector256.Multiply (-> (Simd256 %a) %a (Simd256 %a))))
  (v256-neg (: System.Runtime.Intrinsics.Vector256.Negate (-> (Simd256 %a) (Simd256 %a))))
  (v256-abs (: System.Runtime.Intrinsics.Vector256.Abs (-> (Simd256 %a) (Simd256 %a))))
  (v256-min (: System.Runtime.Intrinsics.Vector256.Min (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-max (: System.Runtime.Intrinsics.Vector256.Max (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-sqrt (: System.Runtime.Intrinsics.Vector256.Sqrt (-> (Simd256 %a) (Simd256 %a))))
  (v256-and (: System.Runtime.Intrinsics.Vector256.BitwiseAnd (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-or (: System.Runtime.Intrinsics.Vector256.BitwiseOr (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-xor (: System.Runtime.Intrinsics.Vector256.Xor (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-not (: System.Runtime.Intrinsics.Vector256.OnesComplement (-> (Simd256 %a) (Simd256 %a))))
  (v256-and-not (: System.Runtime.Intrinsics.Vector256.AndNot (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-eq (: System.Runtime.Intrinsics.Vector256.Equals (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-lt (: System.Runtime.Intrinsics.Vector256.LessThan (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-le (: System.Runtime.Intrinsics.Vector256.LessThanOrEqual (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-gt (: System.Runtime.Intrinsics.Vector256.GreaterThan (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-ge (: System.Runtime.Intrinsics.Vector256.GreaterThanOrEqual (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-nan? (: System.Runtime.Intrinsics.Vector256.IsNaN (-> (Simd256 %a) (Simd256 %a))))
  (v256-select (: System.Runtime.Intrinsics.Vector256.ConditionalSelect (-> (Simd256 %a) (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-sum (: System.Runtime.Intrinsics.Vector256.Sum (-> (Simd256 %a) %a)))
  (v256-dot (: System.Runtime.Intrinsics.Vector256.Dot (-> (Simd256 %a) (Simd256 %a) %a)))
  (v256-ref (: System.Runtime.Intrinsics.Vector256.GetElement (-> (Simd256 %a) int %a)))
  (v256-with (: System.Runtime.Intrinsics.Vector256.WithElement (-> (Simd256 %a) int %a (Simd256 %a))))
  (v256-store! (: System.Runtime.Intrinsics.Vector256.CopyTo (-> (Simd256 %a) (Array %a) int void)))
  (v256-add-saturating (: System.Runtime.Intrinsics.Vector256.AddSaturate (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-sub-saturating (: System.Runtime.Intrinsics.Vector256.SubtractSaturate (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-shuffle (: Bjolang.Runtime.BjoSimd.Shuffle256 (-> (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-lookup (: Bjolang.Runtime.BjoSimd.Lookup256 (-> (Simd128 byte) (Simd256 %a) (Simd256 %a))))
  (v256-as (: System.Runtime.Intrinsics.Vector256.As (-> (Simd256 %a) (Simd256 %b))))
  (v256-floor (: Bjolang.Runtime.BjoSimd.Floor256 (-> (Simd256 %a) (Simd256 %a))))
  (v256-ceiling (: Bjolang.Runtime.BjoSimd.Ceiling256 (-> (Simd256 %a) (Simd256 %a))))
  (v256-round (: Bjolang.Runtime.BjoSimd.Round256 (-> (Simd256 %a) (Simd256 %a))))
  (v256-truncate (: Bjolang.Runtime.BjoSimd.Truncate256 (-> (Simd256 %a) (Simd256 %a))))
  (v256-mul-add (: Bjolang.Runtime.BjoSimd.MulAdd256 (-> (Simd256 %a) (Simd256 %a) (Simd256 %a) (Simd256 %a))))
  (v256-lanes (: Bjolang.Runtime.BjoSimd.Lanes256 (-> (Simd256 %a) int)))
  (v256-any? (: Bjolang.Runtime.BjoSimd.Any256 (-> (Simd256 %a) bool)))
  (v256-all? (: Bjolang.Runtime.BjoSimd.All256 (-> (Simd256 %a) bool)))
  (v256-count-true (: Bjolang.Runtime.BjoSimd.CountTrue256 (-> (Simd256 %a) int)))
  (v256-first-true (: Bjolang.Runtime.BjoSimd.FirstTrue256 (-> (Simd256 %a) int)))
  (v256-splat (: System.Runtime.Intrinsics.Vector256.Create (-> %a (Simd256 %a))))
  (v256-zero (: Bjolang.Runtime.BjoSimd.Zero256 (-> (Simd256 %a))))
  (v256-mask-bits (: Bjolang.Runtime.BjoSimd.MaskBits256 (-> (Simd256 %a) int)))
  (v256-shift-left (: Bjolang.Runtime.BjoSimd.ShiftLeft256 (-> (Simd256 %a) int (Simd256 %a))))
  (v256-shift-right (: Bjolang.Runtime.BjoSimd.ShiftRight256 (-> (Simd256 %a) int (Simd256 %a))))
  (v256-shift-right-logical (: Bjolang.Runtime.BjoSimd.ShiftRightLogical256 (-> (Simd256 %a) int (Simd256 %a))))
  (v256-load-array (: System.Runtime.Intrinsics.Vector256.Create (-> (Array %a) int (Simd256 %a))))
  (v256-load-list (: Bjolang.Runtime.BjoSimd.LoadList256 (-> (MutableVec %a) int (Simd256 %a))))
  (v256-array-lanes (: Bjolang.Runtime.BjoSimd.ArrayLanes256 (-> (Array %a) int)))
  (v256-list-lanes (: Bjolang.Runtime.BjoSimd.ListLanes256 (-> (MutableVec %a) int)))
  (simd256-accelerated? (: System.Runtime.Intrinsics.Vector256.IsHardwareAccelerated #:get)))

(: (-> %a (Simd256 %a)) (where (Lane %a)))
(defun (simd256-splat x) (v256-splat x))

(: (-> (Simd256 %a)) (where (Lane %a)))
(defun (simd256-zero) (v256-zero))

;; The vector a mask holds, and the mask of a vector of a comparison's lanes.
(: (-> (Simd256 %a) (Mask256 %a)))
(defun (m256 v) (Mask256 (lanes v)))

(: (-> (Mask256 %a) (Simd256 %a)))
(defun (lanes256 m) (record-ref m lanes))

(impl (Simd (Simd256 %a)) (where (Lane %a))
  (type %elem %a)
  (type %mask (Mask256 %a))
  (defun (simd-add a b) (v256-add a b))
  (defun (simd-sub a b) (v256-sub a b))
  (defun (simd-mul a b) (v256-mul a b))
  (defun (simd-scale a b) (v256-scale a b))
  (defun (simd-neg a) (v256-neg a))
  (defun (simd-abs a) (v256-abs a))
  (defun (simd-min a b) (v256-min a b))
  (defun (simd-max a b) (v256-max a b))
  (defun (simd-eq a b) (m256 (v256-eq a b)))
  (defun (simd-ne a b) (m256 (v256-not (v256-eq a b))))
  (defun (simd-lt a b) (m256 (v256-lt a b)))
  (defun (simd-le a b) (m256 (v256-le a b)))
  (defun (simd-gt a b) (m256 (v256-gt a b)))
  (defun (simd-ge a b) (m256 (v256-ge a b)))
  (defun (simd-select m a b) (v256-select (lanes256 m) a b))
  (defun (simd-sum a) (v256-sum a))
  (defun (simd-dot a b) (v256-dot a b))
  (defun (simd-ref a i) (v256-ref a i))
  (defun (simd-with a i x) (v256-with a i x))
  (defun (simd-store! a b c) (v256-store! a b c))
  (defun (simd-lanes a) (v256-lanes a)))

(impl (SimdBitwise (Simd256 %a)) (where (Lane %a))
  (defun (simd-and a b) (v256-and a b))
  (defun (simd-or a b) (v256-or a b))
  (defun (simd-xor a b) (v256-xor a b))
  (defun (simd-not a) (v256-not a))
  (defun (simd-and-not a b) (v256-and-not a b)))

(impl (SimdBitwise (Mask256 %a)) (where (Lane %a))
  (defun (simd-and a b) (m256 (v256-and (lanes256 a) (lanes256 b))))
  (defun (simd-or a b) (m256 (v256-or (lanes256 a) (lanes256 b))))
  (defun (simd-xor a b) (m256 (v256-xor (lanes256 a) (lanes256 b))))
  (defun (simd-not a) (m256 (v256-not (lanes256 a))))
  (defun (simd-and-not a b) (m256 (v256-and-not (lanes256 a) (lanes256 b)))))

(impl (SimdMask (Mask256 %a)) (where (Lane %a))
  (type %vec (Simd256 %a))
  (defun (simd-any? m) (v256-any? (lanes256 m)))
  (defun (simd-all? m) (v256-all? (lanes256 m)))
  (defun (simd-count-true m) (v256-count-true (lanes256 m)))
  (defun (simd-first-true m) (v256-first-true (lanes256 m)))
  (defun (simd-mask-bits m) (v256-mask-bits (lanes256 m)))
  (defun (simd-mask->lanes m) (lanes256 m)))

(impl (SimdFloat (Simd256 %a)) (where (FloatLane %a))
  (type %mask (Mask256 %a))
  (defun (simd-div a b) (v256-div a b))
  (defun (simd-sqrt a) (v256-sqrt a))
  (defun (simd-floor a) (v256-floor a))
  (defun (simd-ceiling a) (v256-ceiling a))
  (defun (simd-round a) (v256-round a))
  (defun (simd-truncate a) (v256-truncate a))
  (defun (simd-nan? a) (m256 (v256-nan? a)))
  (defun (simd-mul-add a b c) (v256-mul-add a b c)))

(impl (SimdInt (Simd256 %a)) (where (IntLane %a))
  (defun (simd-shift-left a n) (v256-shift-left a n))
  (defun (simd-shift-right a n) (v256-shift-right a n))
  (defun (simd-shift-right-logical a n) (v256-shift-right-logical a n))
  (defun (simd-add-saturating a b) (v256-add-saturating a b))
  (defun (simd-sub-saturating a b) (v256-sub-saturating a b))
  (defun (simd-shuffle v idx) (v256-shuffle v idx)))

(impl (SimdLookup (Simd256 %a)) (where (ByteLane %a))
  (defun (simd-lookup table idx) (v256-lookup table idx)))

(impl (SimdBits (Simd256 %a)) (where (Lane %a))
  (type %int8 (Simd256 System.SByte))
  (type %uint8 (Simd256 byte))
  (type %int16 (Simd256 short))
  (type %uint16 (Simd256 ushort))
  (type %int32 (Simd256 int))
  (type %uint32 (Simd256 uint))
  (type %int64 (Simd256 long))
  (type %uint64 (Simd256 ulong))
  (type %float32 (Simd256 System.Single))
  (type %float64 (Simd256 double))
  (defun (simd-bits->int8 v) (v256-as v))
  (defun (simd-bits->uint8 v) (v256-as v))
  (defun (simd-bits->int16 v) (v256-as v))
  (defun (simd-bits->uint16 v) (v256-as v))
  (defun (simd-bits->int32 v) (v256-as v))
  (defun (simd-bits->uint32 v) (v256-as v))
  (defun (simd-bits->int64 v) (v256-as v))
  (defun (simd-bits->uint64 v) (v256-as v))
  (defun (simd-bits->float32 v) (v256-as v))
  (defun (simd-bits->float64 v) (v256-as v)))

;; ---------------------------------------------------------------------------
;; Conversions
;; ---------------------------------------------------------------------------
;;
;; A conversion gives a vector of another element type, which depends on the
;; one it is given. So these traits are over the element type, with an impl
;; for each, and a method for each width: the name says the width, as the
;; name of what makes a vector does.
;;
;; Each name says what happens to a value that does not fit. Widening keeps
;; every value: a signed lane is sign-extended and an unsigned one is
;; zero-extended, by its type. Narrowing takes two vectors, the lower half
;; first, and either wraps, keeping the low bits, or saturates, giving the
;; nearest value that fits.

;; The lower or the upper half of the lanes, each widened to twice the bits.
(def/trait (SimdWiden %a)
  (type %wide)
  (: simd128-widen-lower (-> (Simd128 %a) (Simd128 %wide)))
  (: simd128-widen-upper (-> (Simd128 %a) (Simd128 %wide)))
  (: simd256-widen-lower (-> (Simd256 %a) (Simd256 %wide)))
  (: simd256-widen-upper (-> (Simd256 %a) (Simd256 %wide))))

;; Two vectors into one of half the bits a lane.
(def/trait (SimdNarrow %a)
  (type %narrow)
  (: simd128-narrow-wrapping (-> (Simd128 %a) (Simd128 %a) (Simd128 %narrow)))
  (: simd128-narrow-saturating (-> (Simd128 %a) (Simd128 %a) (Simd128 %narrow)))
  (: simd256-narrow-wrapping (-> (Simd256 %a) (Simd256 %a) (Simd256 %narrow)))
  (: simd256-narrow-saturating (-> (Simd256 %a) (Simd256 %a) (Simd256 %narrow))))

;; Integer lanes to float lanes of the same bits, rounded to the nearest float
;; where the integer has more digits than the float holds.
(def/trait (SimdIntToFloat %a)
  (type %float)
  (: simd128-int->float (-> (Simd128 %a) (Simd128 %float)))
  (: simd256-int->float (-> (Simd256 %a) (Simd256 %float))))

;; Float lanes to integer lanes of the same bits, toward zero. A value past the
;; integer type's range gives its least or greatest value, and NaN gives 0.
(def/trait (SimdFloatToInt %a)
  (type %int)
  (type %uint)
  (: simd128-float->int-truncating (-> (Simd128 %a) (Simd128 %int)))
  (: simd256-float->int-truncating (-> (Simd256 %a) (Simd256 %int)))
  (: simd128-float->uint-truncating (-> (Simd128 %a) (Simd128 %uint)))
  (: simd256-float->uint-truncating (-> (Simd256 %a) (Simd256 %uint))))

(import/extern
  (widen128-lower-sb (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 System.SByte) (Simd128 short))))
  (widen128-upper-sb (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 System.SByte) (Simd128 short))))
  (widen256-lower-sb (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 System.SByte) (Simd256 short))))
  (widen256-upper-sb (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 System.SByte) (Simd256 short)))))
(impl (SimdWiden System.SByte)
  (type %wide short)
  (defun (simd128-widen-lower v) (widen128-lower-sb v))
  (defun (simd128-widen-upper v) (widen128-upper-sb v))
  (defun (simd256-widen-lower v) (widen256-lower-sb v))
  (defun (simd256-widen-upper v) (widen256-upper-sb v)))

(import/extern
  (widen128-lower-b (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 byte) (Simd128 ushort))))
  (widen128-upper-b (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 byte) (Simd128 ushort))))
  (widen256-lower-b (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 byte) (Simd256 ushort))))
  (widen256-upper-b (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 byte) (Simd256 ushort)))))
(impl (SimdWiden byte)
  (type %wide ushort)
  (defun (simd128-widen-lower v) (widen128-lower-b v))
  (defun (simd128-widen-upper v) (widen128-upper-b v))
  (defun (simd256-widen-lower v) (widen256-lower-b v))
  (defun (simd256-widen-upper v) (widen256-upper-b v)))

(import/extern
  (widen128-lower-s (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 short) (Simd128 int))))
  (widen128-upper-s (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 short) (Simd128 int))))
  (widen256-lower-s (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 short) (Simd256 int))))
  (widen256-upper-s (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 short) (Simd256 int)))))
(impl (SimdWiden short)
  (type %wide int)
  (defun (simd128-widen-lower v) (widen128-lower-s v))
  (defun (simd128-widen-upper v) (widen128-upper-s v))
  (defun (simd256-widen-lower v) (widen256-lower-s v))
  (defun (simd256-widen-upper v) (widen256-upper-s v)))

(import/extern
  (widen128-lower-us (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 ushort) (Simd128 uint))))
  (widen128-upper-us (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 ushort) (Simd128 uint))))
  (widen256-lower-us (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 ushort) (Simd256 uint))))
  (widen256-upper-us (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 ushort) (Simd256 uint)))))
(impl (SimdWiden ushort)
  (type %wide uint)
  (defun (simd128-widen-lower v) (widen128-lower-us v))
  (defun (simd128-widen-upper v) (widen128-upper-us v))
  (defun (simd256-widen-lower v) (widen256-lower-us v))
  (defun (simd256-widen-upper v) (widen256-upper-us v)))

(import/extern
  (widen128-lower-i (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 int) (Simd128 long))))
  (widen128-upper-i (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 int) (Simd128 long))))
  (widen256-lower-i (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 int) (Simd256 long))))
  (widen256-upper-i (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 int) (Simd256 long)))))
(impl (SimdWiden int)
  (type %wide long)
  (defun (simd128-widen-lower v) (widen128-lower-i v))
  (defun (simd128-widen-upper v) (widen128-upper-i v))
  (defun (simd256-widen-lower v) (widen256-lower-i v))
  (defun (simd256-widen-upper v) (widen256-upper-i v)))

(import/extern
  (widen128-lower-ui (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 uint) (Simd128 ulong))))
  (widen128-upper-ui (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 uint) (Simd128 ulong))))
  (widen256-lower-ui (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 uint) (Simd256 ulong))))
  (widen256-upper-ui (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 uint) (Simd256 ulong)))))
(impl (SimdWiden uint)
  (type %wide ulong)
  (defun (simd128-widen-lower v) (widen128-lower-ui v))
  (defun (simd128-widen-upper v) (widen128-upper-ui v))
  (defun (simd256-widen-lower v) (widen256-lower-ui v))
  (defun (simd256-widen-upper v) (widen256-upper-ui v)))

(import/extern
  (widen128-lower-f (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 System.Single) (Simd128 double))))
  (widen128-upper-f (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 System.Single) (Simd128 double))))
  (widen256-lower-f (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 System.Single) (Simd256 double))))
  (widen256-upper-f (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 System.Single) (Simd256 double)))))
(impl (SimdWiden System.Single)
  (type %wide double)
  (defun (simd128-widen-lower v) (widen128-lower-f v))
  (defun (simd128-widen-upper v) (widen128-upper-f v))
  (defun (simd256-widen-lower v) (widen256-lower-f v))
  (defun (simd256-widen-upper v) (widen256-upper-f v)))

(import/extern
  (narrow128-wrapping-s (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 short) (Simd128 short) (Simd128 System.SByte))))
  (narrow128-saturating-s (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 short) (Simd128 short) (Simd128 System.SByte))))
  (narrow256-wrapping-s (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 short) (Simd256 short) (Simd256 System.SByte))))
  (narrow256-saturating-s (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 short) (Simd256 short) (Simd256 System.SByte)))))
(impl (SimdNarrow short)
  (type %narrow System.SByte)
  (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-s a b))
  (defun (simd128-narrow-saturating a b) (narrow128-saturating-s a b))
  (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-s a b))
  (defun (simd256-narrow-saturating a b) (narrow256-saturating-s a b)))

(import/extern
  (narrow128-wrapping-us (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 ushort) (Simd128 ushort) (Simd128 byte))))
  (narrow128-saturating-us (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 ushort) (Simd128 ushort) (Simd128 byte))))
  (narrow256-wrapping-us (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 ushort) (Simd256 ushort) (Simd256 byte))))
  (narrow256-saturating-us (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 ushort) (Simd256 ushort) (Simd256 byte)))))
(impl (SimdNarrow ushort)
  (type %narrow byte)
  (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-us a b))
  (defun (simd128-narrow-saturating a b) (narrow128-saturating-us a b))
  (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-us a b))
  (defun (simd256-narrow-saturating a b) (narrow256-saturating-us a b)))

(import/extern
  (narrow128-wrapping-i (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 int) (Simd128 int) (Simd128 short))))
  (narrow128-saturating-i (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 int) (Simd128 int) (Simd128 short))))
  (narrow256-wrapping-i (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 int) (Simd256 int) (Simd256 short))))
  (narrow256-saturating-i (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 int) (Simd256 int) (Simd256 short)))))
(impl (SimdNarrow int)
  (type %narrow short)
  (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-i a b))
  (defun (simd128-narrow-saturating a b) (narrow128-saturating-i a b))
  (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-i a b))
  (defun (simd256-narrow-saturating a b) (narrow256-saturating-i a b)))

(import/extern
  (narrow128-wrapping-ui (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 uint) (Simd128 uint) (Simd128 ushort))))
  (narrow128-saturating-ui (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 uint) (Simd128 uint) (Simd128 ushort))))
  (narrow256-wrapping-ui (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 uint) (Simd256 uint) (Simd256 ushort))))
  (narrow256-saturating-ui (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 uint) (Simd256 uint) (Simd256 ushort)))))
(impl (SimdNarrow uint)
  (type %narrow ushort)
  (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-ui a b))
  (defun (simd128-narrow-saturating a b) (narrow128-saturating-ui a b))
  (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-ui a b))
  (defun (simd256-narrow-saturating a b) (narrow256-saturating-ui a b)))

(import/extern
  (narrow128-wrapping-l (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 long) (Simd128 long) (Simd128 int))))
  (narrow128-saturating-l (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 long) (Simd128 long) (Simd128 int))))
  (narrow256-wrapping-l (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 long) (Simd256 long) (Simd256 int))))
  (narrow256-saturating-l (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 long) (Simd256 long) (Simd256 int)))))
(impl (SimdNarrow long)
  (type %narrow int)
  (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-l a b))
  (defun (simd128-narrow-saturating a b) (narrow128-saturating-l a b))
  (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-l a b))
  (defun (simd256-narrow-saturating a b) (narrow256-saturating-l a b)))

(import/extern
  (narrow128-wrapping-ul (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 ulong) (Simd128 ulong) (Simd128 uint))))
  (narrow128-saturating-ul (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 ulong) (Simd128 ulong) (Simd128 uint))))
  (narrow256-wrapping-ul (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 ulong) (Simd256 ulong) (Simd256 uint))))
  (narrow256-saturating-ul (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 ulong) (Simd256 ulong) (Simd256 uint)))))
(impl (SimdNarrow ulong)
  (type %narrow uint)
  (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-ul a b))
  (defun (simd128-narrow-saturating a b) (narrow128-saturating-ul a b))
  (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-ul a b))
  (defun (simd256-narrow-saturating a b) (narrow256-saturating-ul a b)))

(import/extern
  (int->float128-i (: System.Runtime.Intrinsics.Vector128.ConvertToSingle (-> (Simd128 int) (Simd128 System.Single))))
  (int->float256-i (: System.Runtime.Intrinsics.Vector256.ConvertToSingle (-> (Simd256 int) (Simd256 System.Single)))))
(impl (SimdIntToFloat int)
  (type %float System.Single)
  (defun (simd128-int->float v) (int->float128-i v))
  (defun (simd256-int->float v) (int->float256-i v)))

(import/extern
  (int->float128-ui (: System.Runtime.Intrinsics.Vector128.ConvertToSingle (-> (Simd128 uint) (Simd128 System.Single))))
  (int->float256-ui (: System.Runtime.Intrinsics.Vector256.ConvertToSingle (-> (Simd256 uint) (Simd256 System.Single)))))
(impl (SimdIntToFloat uint)
  (type %float System.Single)
  (defun (simd128-int->float v) (int->float128-ui v))
  (defun (simd256-int->float v) (int->float256-ui v)))

(import/extern
  (int->float128-l (: System.Runtime.Intrinsics.Vector128.ConvertToDouble (-> (Simd128 long) (Simd128 double))))
  (int->float256-l (: System.Runtime.Intrinsics.Vector256.ConvertToDouble (-> (Simd256 long) (Simd256 double)))))
(impl (SimdIntToFloat long)
  (type %float double)
  (defun (simd128-int->float v) (int->float128-l v))
  (defun (simd256-int->float v) (int->float256-l v)))

(import/extern
  (int->float128-ul (: System.Runtime.Intrinsics.Vector128.ConvertToDouble (-> (Simd128 ulong) (Simd128 double))))
  (int->float256-ul (: System.Runtime.Intrinsics.Vector256.ConvertToDouble (-> (Simd256 ulong) (Simd256 double)))))
(impl (SimdIntToFloat ulong)
  (type %float double)
  (defun (simd128-int->float v) (int->float128-ul v))
  (defun (simd256-int->float v) (int->float256-ul v)))

(import/extern
  (float->int128-f (: System.Runtime.Intrinsics.Vector128.ConvertToInt32 (-> (Simd128 System.Single) (Simd128 int))))
  (float->uint128-f (: System.Runtime.Intrinsics.Vector128.ConvertToUInt32 (-> (Simd128 System.Single) (Simd128 uint))))
  (float->int256-f (: System.Runtime.Intrinsics.Vector256.ConvertToInt32 (-> (Simd256 System.Single) (Simd256 int))))
  (float->uint256-f (: System.Runtime.Intrinsics.Vector256.ConvertToUInt32 (-> (Simd256 System.Single) (Simd256 uint)))))
(impl (SimdFloatToInt System.Single)
  (type %int int)
  (type %uint uint)
  (defun (simd128-float->int-truncating v) (float->int128-f v))
  (defun (simd128-float->uint-truncating v) (float->uint128-f v))
  (defun (simd256-float->int-truncating v) (float->int256-f v))
  (defun (simd256-float->uint-truncating v) (float->uint256-f v)))

(import/extern
  (float->int128-d (: System.Runtime.Intrinsics.Vector128.ConvertToInt64 (-> (Simd128 double) (Simd128 long))))
  (float->uint128-d (: System.Runtime.Intrinsics.Vector128.ConvertToUInt64 (-> (Simd128 double) (Simd128 ulong))))
  (float->int256-d (: System.Runtime.Intrinsics.Vector256.ConvertToInt64 (-> (Simd256 double) (Simd256 long))))
  (float->uint256-d (: System.Runtime.Intrinsics.Vector256.ConvertToUInt64 (-> (Simd256 double) (Simd256 ulong)))))
(impl (SimdFloatToInt double)
  (type %int long)
  (type %uint ulong)
  (defun (simd128-float->int-truncating v) (float->int128-d v))
  (defun (simd128-float->uint-truncating v) (float->uint128-d v))
  (defun (simd256-float->int-truncating v) (float->int256-d v))
  (defun (simd256-float->uint-truncating v) (float->uint256-d v)))

;; ---------------------------------------------------------------------------
;; A vector of given lanes
;; ---------------------------------------------------------------------------
;;
;; `(simd128-of 1 2 3 4)`: a vector with these lanes, from lane 0. Each value
;; is written into a vector of zeros with `simd-with`. A lane not given is 0,
;; and a value past the last lane raises `ArgumentOutOfRangeException`. With
;; constants, the JIT folds the writes into one constant vector.

(: (-> Syntax int (List Syntax) Syntax))
(defun (lanes-of acc i xs)
  (match xs
    (Nil acc)
    ((Cons x rest)
     (lanes-of #'(simd-with ,acc ,(SInt (int->string i)) ,x) (+ i 1) rest))))

(def/macro (simd128-of form inject compare)
  (syntax-match form
    ((_ first rest ...) (lanes-of #'(simd128-zero) 0 (Cons first rest)))
    (bad (syntax-error bad "simd128-of takes the lanes, one or more: (simd128-of 1 2 3 4)."))))

(def/macro (simd256-of form inject compare)
  (syntax-match form
    ((_ first rest ...) (lanes-of #'(simd256-zero) 0 (Cons first rest)))
    (bad (syntax-error bad "simd256-of takes the lanes, one or more: (simd256-of 1 2 3 4)."))))

;; ---------------------------------------------------------------------------
;; Where vectors are loaded from
;; ---------------------------------------------------------------------------
;;
;; A source is a run of numbers in memory, one after the other: an array or a
;; `MutableVec`. Every load raises `ArgumentOutOfRangeException` when fewer
;; than a vector's lanes are left. A string is not a source: its positions
;; are cursors, and a lane of its bytes would be a byte index.

(def/trait (SimdSource %s)
  (type %elem)
  (: simd128-load (-> %s int (Simd128 %elem)))
  (: simd256-load (-> %s int (Simd256 %elem)))
  (: simd128-lanes (-> %s int))
  (: simd256-lanes (-> %s int))
  (: simd-source-length (-> %s int)))

(impl (SimdSource (Array %a)) (where (Lane %a))
  (type %elem %a)
  (defun (simd128-load a i) (v128-load-array a i))
  (defun (simd256-load a i) (v256-load-array a i))
  (defun (simd128-lanes a) (v128-array-lanes a))
  (defun (simd256-lanes a) (v256-array-lanes a))
  (defun (simd-source-length a) (array-length a)))

(impl (SimdSource (MutableVec %a)) (where (Lane %a))
  (type %elem %a)
  (defun (simd128-load v i) (v128-load-list v i))
  (defun (simd256-load v i) (v256-load-list v i))
  (defun (simd128-lanes v) (v128-list-lanes v))
  (defun (simd256-lanes v) (v256-list-lanes v))
  (defun (simd-source-length v) (mutablevec-length v)))

;; Where the elements after the last whole vector start.
(: (-> %s int) (where (SimdSource %s)))
(defun (simd128-whole s)
  (def w (simd128-lanes s))
  (* w (/ (simd-source-length s) w)))

(: (-> %s int) (where (SimdSource %s)))
(defun (simd256-whole s)
  (def w (simd256-lanes s))
  (* w (/ (simd-source-length s) w)))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/simpletest.bjo — basic test assertion module.

(export expect expect-true fail
        default-test-deadline overran?
        FakeFs fake-fs fake-fs-written fake-fs-contents fake-fs-set-modified!
        ;; The handlers, one per `FS` operation. Exported because the
        ;; `with-fake-fs` templates name them.
        fake-fs-reader fake-fs-writer fake-fs-info fake-fs-list
        fake-fs-create-directory fake-fs-delete-file fake-fs-delete-directory
        fake-fs-move)

(import (std prelude))
(import (std effect))
(import (std syntax-match))

(:doc #:file "simpletest.bjodoc")

;; A failure makes the program exit 1 even when `main` answers 0, so that a
;; test run is a command a shell or CI can check.
(import/extern
  (set-exit-code! (: System.Environment.ExitCode (-> int void) #:set)))

(: (-> string void))
(defun (failed! message)
  (println message)
  (set-exit-code! 1))

;; Asserts equality and prints result.
(: (-> string %a %a void) (where (Eq %a) (Written %a)))
(defun (expect label actual wanted)
  (if (= actual wanted)
      (println #"ok: ${label}")
      ;; Written, so that "1" and 1 cannot read the same.
      (failed! #"FAILURE: ${label} gave ${(written actual)} but wanted ${(written wanted)}")))

;; Asserts boolean and prints result.
(: (-> string bool void))
(defun (expect-true label ok)
  (if ok
      (println #"ok: ${label}")
      (failed! #"FAILURE: ${label}")))

;; Marks a failing branch.
(: (-> string void))
(defun (fail label)
  (failed! #"FAILURE: ${label}"))

;; ---------------------------------------------------------------------------
;; A test body, under a scope
;; ---------------------------------------------------------------------------
;;
;; `(with-test "name" body ...)` runs the body under a deadline and asserts that
;; it did not leave a resource behind:
;;
;;     (with-test "reads the fixture"
;;       (with-fake-fs (("a.txt" "hello"))
;;         (expect "read it" (call-with-input-file "a.txt" read-all) (Ok "hello"))))
;;
;; Four things come from the scope rather than from the harness. A test gets a
;; timeout, so a leaked fiber is a failure that names the test instead of a
;; suite that hangs. A fiber the body spawned is waited for, so an assertion
;; inside one cannot be missed. Resources the body opened are released. And
;; several failures in one body are reported together rather than only the
;; first.
;;
;; We check if the test left any resources open *before* the scope finishes. 
;; If we checked after, the scope would have already silently cleaned them up! 
;; We want tests to explicitly fail if they forget to close their files.
;;
;; You can disable this check using `#:open-ok` (useful if you are specifically 
;; testing that cleanup happens on scope-close).
;;
;; `#:within d`, a Duration, overrides the deadline.
;;
;; We catch `System.Exception` here to handle `Cancelled` exceptions caused by timeouts.
;;
;; Normally, catching and hiding a cancellation is a terrible idea in structured 
;; concurrency. But since this is the top-level test runner, we *want* to catch 
;; it. Doing so lets us report a clean failure with the test's name, instead 
;; of crashing and dumping an unhelpful stack trace.

(: (-> Duration))
(defun (default-test-deadline) (seconds 5))

(: (-> Syntax Syntax))
(defun (leak-check name)
  #'(let ((left (scope-owned-count (current-scope))))
      (unless (= left 0)
        (fail (string-append ,name (string-append " left " (string-append (int->string left) " resource(s) open")))))))

;; Whether the deadline was what ended the scope.
;;
;; A fiber the body leaked is *cancelled* by the deadline rather than failing,
;; and a cancelled child is not a failure - so the scope closes cleanly and the
;; test would otherwise pass, slowly. The token carries the reason, so the
;; scope is kept in a box and the reason read after it has closed.
(: (-> (Box (Option Scope)) bool))
(defun (overran? holder)
  (match (box-ref holder)
    (None #f)
    ((Some s)
     (match (cancel-reason (scope-token s))
       ((Some Deadline) #t)
       (_ #f)))))

(: (-> Syntax Syntax bool (List Syntax) Syntax))
(defun (test-expansion name ms check body)
  (let ((core (if check
                  #'(let () ,@body ,(leak-check name) unit)
                  #'(let () ,@body unit))))
    #'(let ((holder (make-box None)))
        (match (try (with-deadline ,ms (let () (box-set! holder (Some (current-scope))) ,core))
                 #:catch (System.Exception))
          ((Ok _)
           (when (overran? holder)
             (fail (string-append ,name " ran past its deadline: something it started never finished"))))
          ((Err e) (fail (string-append ,name (string-append " did not finish: " (.-Message e)))))))))

(def/macro (with-test form inject compare)
  (syntax-match form
    ((_ name #:within ms #:open-ok first body ...)
     (test-expansion name ms #f (cons first body)))
    ((_ name #:within ms first body ...)
     (test-expansion name ms #t (cons first body)))
    ((_ name #:open-ok first body ...)
     (test-expansion name #'(default-test-deadline) #f (cons first body)))
    ((_ name first body ...)
     (test-expansion name #'(default-test-deadline) #t (cons first body)))
    (bad (syntax-error bad "with-test takes a name and a body: (with-test \"name\" body ...), optionally #:within d, a Duration, and #:open-ok."))))

;; ---------------------------------------------------------------------------
;; A filesystem that is not one
;; ---------------------------------------------------------------------------
;;
;; `with-fake-fs` installs a handler for every operation of the prelude's `FS`
;; effect, so everything that reaches the disk through it - the text and byte
;; openers, `file-read-text` and its neighbours, `file-exists?`, `file-delete`,
;; `file-move` and the directory functions - answers out of a map instead:
;;
;;     (with-fake-fs (("notes.txt" "a b c")
;;                    ("empty.txt" ""))
;;       (expect "read it back" (call-with-input-file "notes.txt" read-all) (Ok "a b c")))
;;
;; A seeded path is a file, and every directory above one exists. A path
;; written inside it is readable afterwards. `with-fake-fs-of` installs an
;; existing `FakeFs`, so that `(fake-fs-written fs "out.txt")` can read back
;; what was written - which works after the port has been closed, because
;; `MemoryStream.ToArray` does.
;;
;; Paths are compared as text after one normalization: backslashes become
;; slashes, and a leading "./" and a trailing "/" are dropped. "" and "." are
;; the root. Nothing resolves "..", and a relative path is not made absolute, so
;; "a.txt" and "/tmp/a.txt" are different files.
;;
;; A failure answers the exception type the .NET call would raise, so a test
;; that matches on the error matches either way.
;;
;; The fake has a clock of its own, so that a test about modified times does
;; not depend on how fast it runs. It starts at 2000-01-01T00:00:00Z, which is
;; every seed's time, and each write opened and each directory created moves it
;; one second on and stamps the path: something written later is always newer.
;; A move keeps the times it moves, as a rename does on a disk.
;; `(fake-fs-set-modified! fs path ms)` sets one by hand.
;;
;; The openers answer a `MemoryStream`, and the ports over it are built and
;; owned by the same code that builds them over a file - so a fake port is
;; registered on the scope the open happened in, and a leak test over a fake
;; filesystem means what it means over a real one.
;;
;; `with-fake-fs` is a handler frame, not a scope: it does not wait for a fiber
;; spawned inside it. The fiber keeps the fake - a spawn takes the environment
;; as it stood at the spawn - but the body's assertions, and `with-test`'s leak
;; check, run before the fiber is done. So a fiber that reads or writes the fake
;; goes inside a `with-scope` of its own, inside `with-fake-fs`. No scope is
;; opened here, because one would hide from `with-test` what the body left open.

(import/class
  (ClrStream (: System.IO.Stream))
  (MemoryStream (: System.IO.MemoryStream (-> MemoryStream)))
  (Encoding (: System.Text.Encoding))
  (FileNotFound (: System.IO.FileNotFoundException (-> string FileNotFound)))
  (DirectoryNotFound (: System.IO.DirectoryNotFoundException (-> string DirectoryNotFound)))
  (IOException (: System.IO.IOException (-> string IOException)))
  (AccessDenied (: System.UnauthorizedAccessException (-> string AccessDenied))))

(import/extern
  (clr-utf8 (: System.Text.Encoding.UTF8 #:get))
  (stream-position! (: System.IO.Stream.Position (-> ClrStream long void) #:set)))

;; A seeded file holds its text. A written one holds the stream it was written
;; through, which is read when the file is: the writer may still be open, and a
;; read sees what has reached the stream so far, as a read of a real file sees
;; what has been flushed.
(type (: FakeEntry (Union (: FakeText string) (: FakeWritten MemoryStream) FakeDir)))

;; Mutable because a write has to be visible to a later read. `modified` holds
;; the time of every path something stamped; any other path has the clock's
;; start.
(type
  (: FakeFs #:opaque
     (Record (: entries (Map string FakeEntry) #:mutable)
             (: modified (Map string long) #:mutable)
             (: clock long #:mutable))))

;; 2000-01-01T00:00:00Z, in Unix milliseconds.
(: long)
(def fake-epoch-ms 946684800000L)

;; What follows `prefix` in `s`, which starts with it.
(: (-> string string string))
(defun (after-prefix s prefix)
  (let go ((c (string-cursor-start s)) (n (string-count prefix)))
    (if (= n 0)
        (substring s c (string-cursor-end s))
        (go (string-cursor-next s c) (- n 1)))))

(: (-> string string))
(defun (fake-path path)
  (let loop ((p (string-replace path "\\" "/")))
    (cond
      ((= p ".") "")
      ((string-starts-with? p "./") (loop (after-prefix p "./")))
      ((and (string-ends-with? p "/") (> (string-length p) 1))
       (loop (substring p (string-cursor-start p) (string-cursor-prev p (string-cursor-end p)))))
      (else p))))

(: (-> string string))
(defun (fake-parent path)
  (match (string-search-backward path "/")
    ((Some at) (substring path (string-cursor-start path) at))
    (None "")))

;; "" for the root, so that a child of the root is its own name.
(: (-> string string))
(defun (fake-prefix path)
  (if (string-empty? path) "" (str path "/")))

;; Every directory above `path` gets an entry of its own, so that it outlives
;; its contents: emptying a directory does not delete it.
(: (-> (Map string FakeEntry) string (Map string FakeEntry)))
(defun (fake-with-ancestors m path)
  (let loop ((m m) (dir (fake-parent path)))
    (if (or (string-empty? dir) (map-has-key? m dir))
        m
        (loop (map-set m dir FakeDir) (fake-parent dir)))))

(: (-> (List (Tuple string string)) FakeFs))
(defun (fake-fs seeds)
  (FakeFs (entries (list-foldl (fun (seed m)
                                 (def (raw text) seed)
                                 (def path (fake-path raw))
                                 (map-set (fake-with-ancestors m path) path (FakeText text)))
                               (map-empty)
                               seeds))
          (modified (map-empty))
          (clock fake-epoch-ms)))

(: (-> FakeFs (Map string FakeEntry)))
(defun (fake-entries fs) (record-ref fs entries))

(: (-> FakeFs (Map string FakeEntry) void))
(defun (fake-update! fs m) (record-set! fs (entries m)))

;; Moves the clock one second on, and gives `path` the new time.
(: (-> FakeFs string void))
(defun (fake-stamp! fs path)
  (def now (+ (record-ref fs clock) 1000L))
  (record-set! fs (clock now) (modified (map-set (record-ref fs modified) path now))))

(: (-> FakeFs string void))
(defun (fake-forget-time! fs path)
  (record-set! fs (modified (map-remove (record-ref fs modified) path))))

(: (-> FakeFs string long void))
(defun (fake-fs-set-modified! fs path ms)
  (record-set! fs (modified (map-set (record-ref fs modified) (fake-path path) ms))))

(: (-> (Array byte) string))
(defun (fake-bytes->text bytes) (.GetString (cast Encoding clr-utf8) bytes))

(: (-> string (Array byte)))
(defun (fake-text->bytes text) (.GetBytes (cast Encoding clr-utf8) text))

;; What a file entry holds. `None` for a directory.
(: (-> FakeEntry (Option string)))
(defun (fake-entry-text entry)
  (match entry
    ((FakeText text) (Some text))
    ((FakeWritten s) (Some (fake-bytes->text (.ToArray s))))
    (FakeDir None)))

;; What a file entry holds, as bytes. Reading and appending go through this
;; rather than through the text, so that bytes that are not UTF-8 come back as
;; they were written.
(: (-> FakeEntry (Option (Array byte))))
(defun (fake-entry-bytes entry)
  (match entry
    ((FakeText text) (Some (fake-text->bytes text)))
    ((FakeWritten s) (Some (.ToArray s)))
    (FakeDir None)))

(: (-> FakeFs string (Option (Array byte))))
(defun (fake-bytes-at fs path)
  (match (map-try-ref (fake-entries fs) path)
    ((Some entry) (fake-entry-bytes entry))
    (None None)))

;; In bytes, as a file on a disk would be.
(: (-> FakeEntry long))
(defun (fake-entry-size entry)
  (match (fake-entry-bytes entry)
    ((Some bytes) (cast System.Int64 (array-length bytes)))
    (None 0L)))

;; A path that is no entry of its own is still a directory when something is
;; below it.
(: (-> FakeFs string (Option FsKind)))
(defun (fake-kind fs path)
  (if (string-empty? path)
      (Some FsDirectory)
      (match (map-try-ref (fake-entries fs) path)
        ((Some FakeDir) (Some FsDirectory))
        ((Some _) (Some FsFile))
        (None (if (map-any? (fun (k v) (string-starts-with? k (fake-prefix path)))
                            (fake-entries fs))
                  (Some FsDirectory)
                  None)))))

(: (-> FakeFs string (Option FsInfo)))
(defun (fake-info fs path)
  (option-map (fun (k)
                (FsInfo (kind k)
                        (size (match (map-try-ref (fake-entries fs) path)
                                ((Some entry) (fake-entry-size entry))
                                (None 0L)))
                        (modified-ms (map-ref-or (record-ref fs modified) path fake-epoch-ms))
                        (link? #f)))
              (fake-kind fs path)))

(: (-> FakeFs string bool))
(defun (fake-dir? fs path)
  (match (fake-kind fs path)
    ((Some FsDirectory) #t)
    (_ #f)))

;; What a file holds, whether it was seeded or written.
(: (-> FakeFs string (Option string)))
(defun (fake-fs-contents fs path)
  (match (map-try-ref (fake-entries fs) (fake-path path))
    ((Some entry) (fake-entry-text entry))
    (None None)))

(: (-> (Array byte) MemoryStream))
(defun (fake-memory-stream bytes)
  (def s (MemoryStream.))
  (.Write s bytes 0 (array-length bytes))
  (stream-position! (cast ClrStream s) 0L)
  s)

(: (-> Exception (Result Exception %a)))
(defun (fake-error e) (Err e))

(: (-> string (Result Exception %a)))
(defun (not-found path)
  (fake-error (cast Exception (FileNotFound. #"the fake filesystem has no file ${path}"))))

(: (-> string (Result Exception %a)))
(defun (no-directory path)
  (fake-error (cast Exception (DirectoryNotFound. #"the fake filesystem has no directory ${path}"))))

(: (-> string (Result Exception %a)))
(defun (io-error message)
  (fake-error (cast Exception (IOException. message))))

(: (-> string (Result Exception %a)))
(defun (access-denied path)
  (fake-error (cast Exception (AccessDenied. #"the fake filesystem cannot do that to directory ${path}"))))

;; --- The handlers --------------------------------------------------------------

(: (-> FakeFs (-> string (Result Exception ClrStream))))
(defun (fake-fs-reader fs)
  (fun (raw)
    (def path (fake-path raw))
    (match (fake-kind fs path)
      ((Some FsDirectory) (access-denied path))
      ((Some FsFile) (match (fake-bytes-at fs path)
                       ((Some bytes) (Ok (cast ClrStream (fake-memory-stream bytes))))
                       (None (not-found path))))
      (None (if (fake-dir? fs (fake-parent path)) (not-found path) (no-directory path))))))

;; `Append` starts the new stream with what the path held.
(: (-> FakeFs (-> string WriteMode (Result Exception ClrStream))))
(defun (fake-fs-writer fs)
  (fun (raw mode)
    (def path (fake-path raw))
    (def kind (fake-kind fs path))
    (cond
      ((not (fake-dir? fs (fake-parent path))) (no-directory path))
      (else
       (match (Tuple mode kind)
         ((Tuple _ (Some FsDirectory)) (access-denied path))
         ((Tuple CreateNew (Some FsFile)) (io-error #"the fake filesystem already has ${path}"))
         (_
          (def s (match (Tuple mode (fake-bytes-at fs path))
                   ((Tuple Append (Some bytes)) (let ((m (fake-memory-stream bytes)))
                                                 (stream-position! (cast ClrStream m) (.-Length m))
                                                 m))
                   (_ (MemoryStream.))))
          (fake-update! fs (map-set (fake-entries fs) path (FakeWritten s)))
          (fake-stamp! fs path)
          (Ok (cast ClrStream s))))))))

(: (-> FakeFs (-> string (Option FsInfo))))
(defun (fake-fs-info fs)
  (fun (raw) (fake-info fs (fake-path raw))))

;; Whole paths, as `Directory.GetFileSystemEntries` gives them, sorted.
(: (-> FakeFs (-> string (Result Exception (Vec string)))))
(defun (fake-fs-list fs)
  (fun (raw)
    (def path (fake-path raw))
    (def prefix (fake-prefix path))
    (if (not (fake-dir? fs path))
        (no-directory path)
        (let ((children (map-fold (fun (k v found)
                                    (if (and (string-starts-with? k prefix) (not (= k path)))
                                        (let* ((rest (after-prefix k prefix))
                                               (child (match (string-search rest "/")
                                                        ((Some at) (substring rest (string-cursor-start rest) at))
                                                        (None rest))))
                                          (map-set found (str prefix child) #t))
                                        found))
                                  (map-empty)
                                  (fake-entries fs))))
          (Ok (list->vec (list-sort (loop (:for child (map-keys children))
                                          (:acc (listing child))))))))))

;; Every missing directory along the path, as `Directory.CreateDirectory`.
(: (-> FakeFs (-> string (Result Exception Unit))))
(defun (fake-fs-create-directory fs)
  (fun (raw)
    (def path (fake-path raw))
    (match (fake-kind fs path)
      ((Some FsFile) (io-error #"the fake filesystem has a file at ${path}"))
      ((Some FsDirectory) (Ok unit))
      (None (fake-update! fs (map-set (fake-with-ancestors (fake-entries fs) path) path FakeDir))
            (fake-stamp! fs path)
            (Ok unit)))))

;; A missing file is not an error, as with `File.Delete`.
(: (-> FakeFs (-> string (Result Exception Unit))))
(defun (fake-fs-delete-file fs)
  (fun (raw)
    (def path (fake-path raw))
    (match (fake-kind fs path)
      ((Some FsDirectory) (access-denied path))
      ((Some FsFile) (fake-update! fs (map-remove (fake-entries fs) path))
                     (fake-forget-time! fs path)
                     (Ok unit))
      (None (Ok unit)))))

(: (-> FakeFs (-> string (Result Exception Unit))))
(defun (fake-fs-delete-directory fs)
  (fun (raw)
    (def path (fake-path raw))
    (def prefix (fake-prefix path))
    (match (fake-kind fs path)
      ((Some FsFile) (io-error #"the fake filesystem has a file, not a directory, at ${path}"))
      (None (no-directory path))
      ((Some FsDirectory)
       (cond
         ((string-empty? path) (io-error "the fake filesystem's root cannot be deleted"))
         ((map-any? (fun (k v) (string-starts-with? k prefix)) (fake-entries fs))
          (io-error #"the fake directory ${path} is not empty"))
         (else (fake-update! fs (map-remove (fake-entries fs) path))
               (fake-forget-time! fs path)
               (Ok unit)))))))

;; `m` with `from`, and everything below it, renamed to `to`. Generic, because
;; the entries and their times move by the same renaming.
(: (-> (Map string %v) string string (Map string %v)))
(defun (fake-renamed m from to)
  (def from-prefix (fake-prefix from))
  (map-fold (fun (k v acc)
              (cond
                ((= k from) (map-set acc to v))
                ((string-starts-with? k from-prefix)
                 (map-set acc (str to "/" (after-prefix k from-prefix)) v))
                (else (map-set acc k v))))
            (map-empty)
            m))

;; A file, or a directory and everything below it. Fails if `to` exists,
;; unless `MoveReplace` asks to replace a file there with a file. The source is
;; checked first, as `File.Move` does.
(: (-> FakeFs (-> string string MoveMode (Result Exception Unit))))
(defun (fake-fs-move fs)
  (fun (raw-from raw-to mode)
    (def from (fake-path raw-from))
    (def to (fake-path raw-to))
    (def from-kind (fake-kind fs from))
    (def to-kind (fake-kind fs to))
    (def replacing (match (Tuple mode from-kind to-kind)
                     ((Tuple MoveReplace (Some FsFile) (Some FsFile)) #t)
                     (_ #f)))
    (cond
      ((none? from-kind) (not-found from))
      ((and (some? to-kind) (not replacing)) (io-error #"the fake filesystem already has ${to}"))
      ((not (fake-dir? fs (fake-parent to))) (no-directory to))
      (else
       (when replacing
         (fake-update! fs (map-remove (fake-entries fs) to))
         (fake-forget-time! fs to))
       (fake-update! fs (fake-renamed (fake-entries fs) from to))
       (record-set! fs (modified (fake-renamed (record-ref fs modified) from to)))
       (Ok unit)))))

;; What was written to `path`, or `None` if nothing was. A seeded file nobody
;; wrote is `None`.
(: (-> FakeFs string (Option string)))
(defun (fake-fs-written fs path)
  (match (map-try-ref (fake-entries fs) (fake-path path))
    ((Some (FakeWritten s)) (Some (fake-bytes->text (.ToArray s))))
    (_ None)))

;; --- The forms -----------------------------------------------------------------
;;
;; The operation names go through `inject`, so they arrive at `with-handler`
;; unrenamed. `with-handler` derives the cell it binds from the name it is
;; given, and a renamed `open-read-stream` would derive a cell nothing
;; declares. Any macro that writes a `with-handler` has to do this.

(: (-> (-> Symbol Syntax) string Syntax))
(defun (fake-op inject name) (inject (string->symbol name)))

;; `(with-fake-fs-of fs body ...)`: install `fs` for the body.
(def/macro (with-fake-fs-of form inject compare)
  (syntax-match form
    ((_ fs first body ...)
     #'(let ((the-fs ,fs))
         (with-handler ((,(fake-op inject "open-read-stream") (fake-fs-reader the-fs))
                        (,(fake-op inject "open-write-stream") (fake-fs-writer the-fs))
                        (,(fake-op inject "fs-info") (fake-fs-info the-fs))
                        (,(fake-op inject "fs-list") (fake-fs-list the-fs))
                        (,(fake-op inject "fs-create-directory") (fake-fs-create-directory the-fs))
                        (,(fake-op inject "fs-delete-file") (fake-fs-delete-file the-fs))
                        (,(fake-op inject "fs-delete-directory") (fake-fs-delete-directory the-fs))
                        (,(fake-op inject "fs-move") (fake-fs-move the-fs)))
           ,first ,@body)))
    (bad (syntax-error bad "with-fake-fs-of takes a FakeFs and a body: (with-fake-fs-of fs body ...)"))))

;; `(with-fake-fs ((path contents) ...) body ...)`: a fresh one, seeded.
(def/macro (with-fake-fs form inject compare)
  (syntax-match form
    ((_ ((path text) ...) first body ...)
     #'(with-fake-fs-of (fake-fs (list ,@{listing #'(Tuple ,p ,t) (:for p path) (:for t text)}))
         ,first ,@body))
    (bad (syntax-error bad "with-fake-fs takes a list of (path contents) pairs and a body: (with-fake-fs ((\"a.txt\" \"hello\")) body ...)"))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/stm.bjo — software transactional memory.
;;
;; A `(Loc T)` is a `Bjolang.Runtime.Loc<T>` and a `Tx` is a
;; `Bjolang.Runtime.Tx`: the log of one run of a transaction. The design is the
;; one of kcas (https://github.com/ocaml-multicore/kcas), whose licence is in
;; BjolangRuntime/BjoStm.cs. A transaction that retries is a CML event that is
;; not available yet; see `AtomicallyEvent` in the same file.
(import (std prelude))

(:doc #:file "stm.bjodoc")

(export
  Loc Tx
  make-loc loc-ref
  tx-ref tx-set! tx-update! tx-swap! tx-retry tx-or-else
  atomically-evt atomically try-atomically)

(import/class
  ((Loc %a) (: Bjolang.Runtime.Loc))
  (Tx       (: Bjolang.Runtime.Tx)))

(import/extern
  (make-loc       (: Bjolang.Runtime.StmModule.MakeLoc       (-> %a (Loc %a))))
  (loc-ref        (: Bjolang.Runtime.StmModule.Ref           (-> (Loc %a) %a)))
  (tx-ref         (: Bjolang.Runtime.StmModule.Get           (-> Tx (Loc %a) %a)))
  (tx-set!        (: Bjolang.Runtime.StmModule.Set           (-> Tx (Loc %a) %a void)))
  (tx-update!     (: Bjolang.Runtime.StmModule.Update        (-> Tx (Loc %a) (-> %a %a) %a)))
  (tx-swap!       (: Bjolang.Runtime.StmModule.Swap          (-> Tx (Loc %a) %a %a)))
  (tx-retry       (: Bjolang.Runtime.StmModule.Retry         (-> Tx %a)))
  (tx-or-else     (: Bjolang.Runtime.StmModule.OrElse        (-> Tx (-> Tx %a) (-> Tx %a) %a)))
  (atomically-evt (: Bjolang.Runtime.StmModule.AtomicallyEvt (-> (-> Tx %a) (Event %a))))
  (try-atomically (: Bjolang.Runtime.StmModule.TryAtomically (-> (-> Tx %a) (Option %a)))))

;; Run and commit the transaction, and wait while it retries. Defined by
;; syncing `atomically-evt`, in the colour of the caller.
(: (-> (-> Tx %a) %a))
(defbjouble (atomically f)
  (#:sync (sync/blocking (atomically-evt f)))
  (#:bjo  (sync (atomically-evt f))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/stm/counter.bjo — a counter that transactions add to without
;; conflicting with each other.
;;
;; The value is the sum of several locations, the stripes. An add writes only
;; the stripe of the CPU that the transaction runs on, so transactions on
;; different CPUs that add to the counter do not conflict. A read sums all
;; stripes, so it conflicts with every add. This is the Accumulator of
;; kcas_data (https://github.com/ocaml-multicore/kcas), which selects the
;; stripe by OCaml domain.
;;
;; A transaction can run again on another CPU, and then adds to another
;; stripe. That is correct: only the sum has a meaning.
(import (std prelude))
(import (std stm))

(:doc #:file "counter.bjodoc")

(export
  TCounter
  make-tcounter
  tcounter-add! tcounter-sub! tcounter-ref tcounter-set!)

(import/extern
  (processor-id    (: System.Threading.Thread.GetCurrentProcessorId (-> int)))
  (processor-count (: System.Environment.ProcessorCount int #:get)))

(type (: (TCounter %a) #:opaque
         (Record (: stripes (Vec (Loc %a))))))

;; One stripe for each CPU, rounded up to a power of two, so that the stripe of
;; a CPU is a mask of its id.
(: (-> int))
(defun (stripe-count)
  (let lp ((n 1))
    (if (>= n processor-count) n (lp (* n 2)))))

(: (-> %a (TCounter %a)) (where (Num %a)))
(defun (make-tcounter initial)
  (def zero (- initial initial))
  (TCounter (stripes (loop (:for i (range 0 (stripe-count)))
                           (:acc v (vecing (make-loc (if (= i 0) initial zero))))))))

(: (-> (TCounter %a) (Loc %a)))
(defun (own-stripe c)
  (def stripes (record-ref c stripes))
  (vec-ref stripes (bitwise-and (processor-id) (- (vec-length stripes) 1))))

(: (-> Tx (TCounter %a) %a void) (where (Num %a)))
(defun (tcounter-add! tx c n)
  (ignore (tx-update! tx (own-stripe c) #(+ & n))))

(: (-> Tx (TCounter %a) %a void) (where (Num %a)))
(defun (tcounter-sub! tx c n)
  (ignore (tx-update! tx (own-stripe c) #(- & n))))

(: (-> Tx (TCounter %a) %a) (where (Num %a)))
(defun (tcounter-ref tx c)
  (def stripes (record-ref c stripes))
  (loop (:for i (range 1 (vec-length stripes)))
        (:acc sum (folding (tx-ref tx (vec-ref stripes 0))
                           (+ sum (tx-ref tx (vec-ref stripes i)))))
        => sum))

(: (-> Tx (TCounter %a) %a void) (where (Num %a)))
(defun (tcounter-set! tx c value)
  (def stripes (record-ref c stripes))
  (def zero (- value value))
  (loop (:for i (range 0 (vec-length stripes)))
        (:do (tx-set! tx (vec-ref stripes i) (if (= i 0) value zero)))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/stm/map.bjo — a hash map for transactions.
;;
;; A hash array mapped trie in which each branch is a location of its own, as
;; in the StmMap of Haskell's stm-containers. A branch has a 32-bit bitmap and
;; one slot for each bit that is set. Level n uses bits 5n to 5n+4 of the hash
;; to select a slot: the key's `eq-hash`, mixed (see `mix`). A slot holds one
;; entry, a bucket of entries with the same hash, or the location of the next
;; branch.
;;
;; An operation reads the branches on the path to its key, and writes only the
;; branch where the slot changes. So two transactions that change different
;; keys conflict only if they change the same branch. A new key goes into the
;; root branch while its slot there is empty, so a small map is mostly the
;; root, and transactions that add keys to it conflict often. In a large map
;; the root slots point to branches, and the writes are spread over them.
;;
;; A branch that becomes empty is removed from its parent. A branch with one
;; entry left is not merged into its parent: that would write the parent.
;;
;; The size is a `TCounter`, whose stripes are selected by CPU. One location
;; would be written by every transaction that adds or removes a key, so all of
;; them would conflict. `tmap-length` reads all stripes.
(import (std prelude))
(import (std stm))
(import (std stm counter))

(:doc #:file "map.bjodoc")

(export
  TMap
  ;; Named by the constrained functions, whose bodies an importer
  ;; specialises. Opaque: nothing outside this module can use them.
  TMapSlot TMapBranch TMapChange
  make-tmap
  tmap-ref tmap-try-ref tmap-ref-or tmap-contains? tmap-has-key?
  tmap-set! tmap-remove! tmap-update!
  tmap-length tmap-empty? tmap-fold tmap->list)

(import/extern
  (popcount (: System.Int32.PopCount (-> int int))))

(import/class
  (KeyNotFound (: System.Collections.Generic.KeyNotFoundException
                  (-> string KeyNotFound))))

(type-rec
  (: (TMapSlot %k %v) #:opaque
     (Union (: Leaf int %k %v)
            (: Bucket int (List (Tuple %k %v)))
            (: Child (Loc (TMapBranch %k %v)))))
  (: (TMapBranch %k %v) #:opaque
     (Struct (: bitmap int) (: slots (Vec (TMapSlot %k %v))))))

;; What a change does to one slot.
(type (: (TMapChange %k %v) #:opaque
         (Union Same
                (: Put (TMapSlot %k %v) int)
                Drop)))

(type (: (TMap %k %v) #:opaque
         (Record (: root (Loc (TMapBranch %k %v)))
                 (: count (TCounter int)))))

;; ---------------------------------------------------------------------------
;; Branches
;; ---------------------------------------------------------------------------

(: (-> (TMapBranch %k %v)))
(defun (empty-branch) (TMapBranch (bitmap 0) (slots (vec-empty))))

;; The hash the map uses: `eq-hash` with every bit spread over all bits, by
;; the finaliser of MurmurHash3 (Austin Appleby, public domain). The `eq-hash`
;; of an int is the int, so keys that are near each other differ only in their
;; low bits. Without the mix, fibers that add such keys at the same time would
;; write the same branches. The mix is a bijection, so
;; keys with equal hashes still share a bucket.
(: (-> int int))
(defun (mix h0)
  (def h1 (* (bitwise-xor h0 (shift-right-logical h0 16)) -2048144789))
  (def h2 (* (bitwise-xor h1 (shift-right-logical h1 13)) -1028477387))
  (bitwise-xor h2 (shift-right-logical h2 16)))

;; The slot index of hash `h` at `level`: 0 to 31.
(: (-> int int int))
(defun (index-at h level)
  (bitwise-and (shift-right-logical h (* 5 level)) 31))

;; Where the slot for `bit` is in the vec: the number of set bits below it.
(: (-> int int int))
(defun (position bitmap bit)
  (popcount (bitwise-and bitmap (- bit 1))))

(: (-> (TMapSlot %k %v) int))
(defun (slot-hash slot)
  (match slot
    ((Leaf h _ _) h)
    ((Bucket h _) h)
    ((Child _) 0)))

(: (-> (TMapSlot %k %v) (TMapSlot %k %v) (Vec (TMapSlot %k %v))))
(defun (two-slots a b)
  (vec-add (vec-add (vec-empty) a) b))

;; A branch at `level` that holds `a` and `b`, whose hashes differ. While the
;; two hashes select the same slot, each level gets a branch of one slot.
(: (-> int (TMapSlot %k %v) (TMapSlot %k %v) (TMapBranch %k %v)))
(defun (split level a b)
  (def ia (index-at (slot-hash a) level))
  (def ib (index-at (slot-hash b) level))
  (if (= ia ib)
      (TMapBranch (bitmap (shift-left 1 ia))
                  (slots (vec-add (vec-empty) (Child (make-loc (split (+ level 1) a b))))))
      (TMapBranch (bitmap (bitwise-ior (shift-left 1 ia) (shift-left 1 ib)))
                  (slots (if (< ia ib) (two-slots a b) (two-slots b a))))))

;; ---------------------------------------------------------------------------
;; Buckets
;; ---------------------------------------------------------------------------

(: (-> %k (List (Tuple %k %v)) (Option %v)) (where (Eq %k)))
(defun (bucket-find k entries)
  (match entries
    (Nil None)
    ((Cons (Tuple k2 v) rest) (if (= k k2) (Some v) (bucket-find k rest)))))

(: (-> %k (List (Tuple %k %v)) (List (Tuple %k %v))) (where (Eq %k)))
(defun (bucket-without k entries)
  (match entries
    (Nil Nil)
    ((Cons (Tuple k2 v) rest)
     (if (= k k2) rest (Cons (Tuple k2 v) (bucket-without k rest))))))

;; ---------------------------------------------------------------------------
;; Lookup and change
;; ---------------------------------------------------------------------------

(: (-> Tx (Loc (TMapBranch %k %v)) int %k int (Option %v)) (where (Eq %k)))
(defun (find-in tx loc h k level)
  (def b (tx-ref tx loc))
  (def bitmap (struct-ref b bitmap))
  (def bit (shift-left 1 (index-at h level)))
  (if (= 0 (bitwise-and bitmap bit))
      None
      (match (vec-ref (struct-ref b slots) (position bitmap bit))
        ((Leaf h2 k2 v) (if (and (= h h2) (= k k2)) (Some v) None))
        ((Bucket h2 entries) (if (= h h2) (bucket-find k entries) None))
        ((Child sub) (find-in tx sub h k (+ level 1))))))

;; What `f` does to a slot that is not a branch, at `level`.
(: (-> (TMapSlot %k %v) int %k (-> (Option %v) (Option %v)) int (TMapChange %k %v))
   (where (Eq %k)))
(defun (change-slot slot h k f level)
  (match slot
    ((Leaf h2 k2 v2)
     (if (and (= h h2) (= k k2))
         (match (f (Some v2))
           (None Drop)
           ((Some v) (Put (Leaf h k v) 0)))
         (match (f None)
           (None Same)
           ((Some v)
            (if (= h h2)
                (Put (Bucket h (list (Tuple k v) (Tuple k2 v2))) 1)
                (Put (Child (make-loc (split (+ level 1) slot (Leaf h k v)))) 1))))))
    ((Bucket h2 entries)
     (if (not (= h h2))
         (match (f None)
           (None Same)
           ((Some v) (Put (Child (make-loc (split (+ level 1) slot (Leaf h k v)))) 1)))
         (let ((before (bucket-find k entries))
               (rest (bucket-without k entries)))
           (match (f before)
             (None
              (match before
                (None Same)
                ((Some _)
                 (match rest
                   ((Cons (Tuple k1 v1) Nil) (Put (Leaf h k1 v1) -1))
                   (_ (Put (Bucket h rest) -1))))))
             ((Some v)
              (Put (Bucket h (Cons (Tuple k v) rest))
                   (match before (None 1) ((Some _) 0))))))))
    ((Child _) Same)))

;; Applies `f` to the entry for `k` in the branch at `loc`. Returns the change
;; of the size: -1, 0 or 1.
(: (-> Tx (Loc (TMapBranch %k %v)) int %k (-> (Option %v) (Option %v)) int int)
   (where (Eq %k)))
(defun (change-in tx loc h k f level)
  (def b (tx-ref tx loc))
  (def bitmap (struct-ref b bitmap))
  (def slots (struct-ref b slots))
  (def bit (shift-left 1 (index-at h level)))
  (def pos (position bitmap bit))
  (if (= 0 (bitwise-and bitmap bit))
      (match (f None)
        (None 0)
        ((Some v)
         (tx-set! tx loc (TMapBranch (bitmap (bitwise-ior bitmap bit)) (slots (vec-insert slots pos (Leaf h k v)))))
         1))
      (match (vec-ref slots pos)
        ((Child sub)
         (def d (change-in tx sub h k f (+ level 1)))
         ;; A branch that is now empty is taken out of its parent.
         (when (and (= d -1) (= 0 (struct-ref (tx-ref tx sub) bitmap)))
           (tx-set! tx loc (TMapBranch (bitmap (bitwise-xor bitmap bit)) (slots (vec-remove-at slots pos)))))
         d)
        (slot
         (match (change-slot slot h k f level)
           (Same 0)
           ((Put s d)
            (tx-set! tx loc (TMapBranch (bitmap bitmap) (slots (vec-set slots pos s))))
            d)
           (Drop
            (tx-set! tx loc (TMapBranch (bitmap (bitwise-xor bitmap bit)) (slots (vec-remove-at slots pos))))
            -1))))))

(: (-> Tx (Loc (TMapBranch %k %v)) (-> %k %v %s %s) %s %s))
(defun (fold-in tx loc f acc)
  (loop (:for slot (in-vec (struct-ref (tx-ref tx loc) slots)))
        (:acc out (folding acc
                           (match slot
                             ((Leaf _ k v) (f k v out))
                             ((Bucket _ entries)
                              (list-foldl (fun (e a) (match e ((Tuple k v) (f k v a)))) out entries))
                             ((Child sub) (fold-in tx sub f out)))))
        => out))

;; ---------------------------------------------------------------------------
;; The map
;; ---------------------------------------------------------------------------

(: (-> (TMap %k %v)))
(defun (make-tmap)
  (TMap (root (make-loc (empty-branch)))
        (count (make-tcounter 0))))

(: (-> Tx (TMap %k %v) %k (Option %v)) (where (Eq %k)))
(defun (tmap-try-ref tx m k)
  (find-in tx (record-ref m root) (mix (eq-hash k)) k 0))

;; Not constrained, so that it is not specialised in the importer, where the
;; class `KeyNotFound` is not known.
(: (-> %v))
(defun (key-not-found)
  (raise (cast Exception (KeyNotFound. "tmap-ref: the key is not in the map"))))

(: (-> Tx (TMap %k %v) %k %v) (where (Eq %k)))
(defun (tmap-ref tx m k)
  (match (tmap-try-ref tx m k)
    ((Some v) v)
    (None (key-not-found))))

(: (-> Tx (TMap %k %v) %k %v %v) (where (Eq %k)))
(defun (tmap-ref-or tx m k default)
  (match (tmap-try-ref tx m k)
    ((Some v) v)
    (None default)))

(: (-> Tx (TMap %k %v) %k bool) (where (Eq %k)))
(defun (tmap-contains? tx m k)
  (match (tmap-try-ref tx m k)
    ((Some _) #t)
    (None #f)))

(: (-> Tx (TMap %k %v) %k bool) (where (Eq %k)))
(defun (tmap-has-key? tx m k) (tmap-contains? tx m k))

(: (-> Tx (TMap %k %v) %k (-> (Option %v) (Option %v)) void) (where (Eq %k)))
(defun (tmap-update! tx m k f)
  (def h (mix (eq-hash k)))
  (def d (change-in tx (record-ref m root) h k f 0))
  (when (not (= d 0))
    (tcounter-add! tx (record-ref m count) d)))

(: (-> Tx (TMap %k %v) %k %v void) (where (Eq %k)))
(defun (tmap-set! tx m k v)
  (tmap-update! tx m k (fun (_) (Some v))))

(: (-> Tx (TMap %k %v) %k void) (where (Eq %k)))
(defun (tmap-remove! tx m k)
  (tmap-update! tx m k (fun (_) None)))

(: (-> Tx (TMap %k %v) int))
(defun (tmap-length tx m)
  (tcounter-ref tx (record-ref m count)))

;; Reads only the root: a branch that becomes empty is removed, so the root is
;; empty when the map is.
(: (-> Tx (TMap %k %v) bool))
(defun (tmap-empty? tx m)
  (= 0 (struct-ref (tx-ref tx (record-ref m root)) bitmap)))

(: (-> Tx (-> %k %v %s %s) %s (TMap %k %v) %s))
(defun (tmap-fold tx f init m)
  (fold-in tx (record-ref m root) f init))

(: (-> Tx (TMap %k %v) (List (Tuple %k %v))))
(defun (tmap->list tx m)
  (tmap-fold tx (fun (k v acc) (Cons (Tuple k v) acc)) Nil m))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/stm/queue.bjo — a FIFO queue for transactions.
;;
;; Two locations: `front` holds the front of the queue in order, and `back`
;; holds the rear reversed. A push writes only `back` and a pop writes only
;; `front`, so a producer and a consumer conflict only when the front is
;; empty. Then a pop moves all of `back` to `front`, reversed. Each item is
;; moved once, so every operation is O(1) amortized.
;;
;; Each side keeps its count, so the length is O(1).
;;
;; A bounded queue counts its free places in two locations, as the TBQueue of
;; Haskell's stm package does. `write` is the free places that pushes use,
;; and `freed` is the places that pops have freed since. A push takes a place
;; from `write`, and reads `freed` only when `write` is 0. So pushes and pops
;; do not conflict on the capacity until the producers run out of places.
(import (std prelude))
(import (std stm))

(:doc #:file "queue.bjodoc")

(export
  TQueue
  make-queue make-bounded-queue
  ;; In a transaction.
  queue-push! queue-pop! queue-try-pop! queue-peek queue-try-peek
  queue-pop-all! queue-length queue-empty?
  ;; Outside one, as the channel operations are named.
  queue-send queue-recv queue-put queue-get)

(import/class
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

(type (: (Side %a) (Struct (: count int) (: items (List %a)))))

(type (: Room (Record (: write (Loc int)) (: freed (Loc int)))))

(type (: (TQueue %a) #:opaque
         (Record (: front (Loc (Side %a)))
                 (: back (Loc (Side %a)))
                 (: room (Option Room)))))

;; ---------------------------------------------------------------------------
;; Making one
;; ---------------------------------------------------------------------------

(: (-> (Side %a)))
(defun (empty-side) (Side (count 0) (items (list))))

(: (-> (TQueue %a)))
(defun (make-queue)
  (TQueue (front (make-loc (empty-side)))
          (back (make-loc (empty-side)))
          (room None)))

(: (-> int (TQueue %a)))
(defun (make-bounded-queue capacity)
  (when (< capacity 1)
    (raise (cast Exception (ArgumentException. "make-bounded-queue: the capacity must be at least 1"))))
  (TQueue (front (make-loc (empty-side)))
          (back (make-loc (empty-side)))
          (room (Some (Room (write (make-loc capacity)) (freed (make-loc 0)))))))

;; ---------------------------------------------------------------------------
;; Capacity
;; ---------------------------------------------------------------------------

;; Takes one free place, or retries when there is none.
(: (-> Tx (Option Room) void))
(defun (take-place! tx room)
  (match room
    (None unit)
    ((Some r)
     (def w (tx-ref tx (record-ref r write)))
     (if (> w 0)
         (tx-set! tx (record-ref r write) (- w 1))
         (let ((f (tx-ref tx (record-ref r freed))))
           (if (= f 0)
               (tx-retry tx)
               (begin
                 (tx-set! tx (record-ref r freed) 0)
                 (tx-set! tx (record-ref r write) (- f 1)))))))))

(: (-> Tx (Option Room) int void))
(defun (free-places! tx room n)
  (match room
    (None unit)
    ((Some r) (ignore (tx-update! tx (record-ref r freed) #(+ & n))))))

;; ---------------------------------------------------------------------------
;; In a transaction
;; ---------------------------------------------------------------------------

;; The front side, with the back moved to it when it is empty. Moves nothing
;; when both are empty, so that a pop of an empty queue writes nothing.
(: (-> Tx (TQueue %a) (Side %a)))
(defun (stocked-front tx q)
  (def f (tx-ref tx (record-ref q front)))
  (if (> (struct-ref f count) 0)
      f
      (let ((b (tx-ref tx (record-ref q back))))
        (if (= (struct-ref b count) 0)
            f
            (let ((moved (Side (count (struct-ref b count))
                               (items (list-reverse (struct-ref b items))))))
              (tx-set! tx (record-ref q back) (empty-side))
              (tx-set! tx (record-ref q front) moved)
              moved)))))

(: (-> Tx (TQueue %a) %a void))
(defun (queue-push! tx q x)
  (take-place! tx (record-ref q room))
  (def b (tx-ref tx (record-ref q back)))
  (tx-set! tx (record-ref q back)
           (Side (count (+ (struct-ref b count) 1))
                 (items (Cons x (struct-ref b items))))))

(: (-> Tx (TQueue %a) (Option %a)))
(defun (queue-try-pop! tx q)
  (def f (stocked-front tx q))
  (match (struct-ref f items)
    ((Cons x rest)
     (tx-set! tx (record-ref q front) (Side (count (- (struct-ref f count) 1)) (items rest)))
     (free-places! tx (record-ref q room) 1)
     (Some x))
    (Nil None)))

(: (-> Tx (TQueue %a) %a))
(defun (queue-pop! tx q)
  (match (queue-try-pop! tx q)
    ((Some x) x)
    (None (tx-retry tx))))

(: (-> Tx (TQueue %a) (Option %a)))
(defun (queue-try-peek tx q)
  (match (struct-ref (stocked-front tx q) items)
    ((Cons x _) (Some x))
    (Nil None)))

(: (-> Tx (TQueue %a) %a))
(defun (queue-peek tx q)
  (match (queue-try-peek tx q)
    ((Some x) x)
    (None (tx-retry tx))))

(: (-> Tx (TQueue %a) (List %a)))
(defun (queue-pop-all! tx q)
  (def f (tx-ref tx (record-ref q front)))
  (def b (tx-ref tx (record-ref q back)))
  (def n (+ (struct-ref f count) (struct-ref b count)))
  (if (= n 0)
      (list)
      (begin
        (tx-set! tx (record-ref q front) (empty-side))
        (tx-set! tx (record-ref q back) (empty-side))
        (free-places! tx (record-ref q room) n)
        (list-append (struct-ref f items) (list-reverse (struct-ref b items))))))

(: (-> Tx (TQueue %a) int))
(defun (queue-length tx q)
  (+ (struct-ref (tx-ref tx (record-ref q front)) count)
     (struct-ref (tx-ref tx (record-ref q back)) count)))

(: (-> Tx (TQueue %a) bool))
(defun (queue-empty? tx q) (= (queue-length tx q) 0))

;; ---------------------------------------------------------------------------
;; Outside a transaction
;; ---------------------------------------------------------------------------

(: (-> (TQueue %a) %a (Event Unit)))
(defun (queue-send q x)
  (atomically-evt (fun (tx) (queue-push! tx q x))))

(: (-> (TQueue %a) (Event %a)))
(defun (queue-recv q)
  (atomically-evt (fun (tx) (queue-pop! tx q))))

(: (-> (TQueue %a) %a void))
(defbjouble (queue-put q x)
  (#:sync (sync/blocking (queue-send q x)))
  (#:bjo  (sync (queue-send q x))))

(: (-> (TQueue %a) %a))
(defbjouble (queue-get q)
  (#:sync (sync/blocking (queue-recv q)))
  (#:bjo  (sync (queue-recv q))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

(import (std prelude))
(import (std syntax-match))
(import (std bench))

(import/class
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

(import/extern (get-memory-usage (: System.GC.GetAllocatedBytesForCurrentThread)))

(:doc #:file "stopwatch.bjodoc")

(export Stopwatch stopwatch stopwatch-start! stopwatch-stop! stopwatch-ns stopwatch-us stopwatch-ms
        ns->string get-memory-usage)

;; A stopwatch reads the prelude's `monotonic-ns` from `current-clock` rather
;; than holding a `System.Diagnostics.Stopwatch`, so that a test can bind a
;; clock and get an exact elapsed time instead of a real one.
;;
;; `started` is the reading in nanoseconds the watch was last started at, or -1
;; when it is not running; `elapsed` is the nanoseconds earlier runs added up
;; to. Both mutable, because a stopwatch is a thing that is started and stopped
;; rather than a value that is replaced. The other units are read from the
;; nanoseconds, so that a run under a millisecond is not lost.
(type
  (: Stopwatch (Record (: started long #:mutable)
                       (: elapsed long #:mutable))))

(: (-> Stopwatch))
(defun (stopwatch)
  (Stopwatch (started -1L) (elapsed 0L)))

(: (-> Stopwatch bool))
(defun (running? sw)
  (>= (record-ref sw started) 0L))

;; Starting a running watch does nothing, as restarting a .NET one does not.
(: (-> Stopwatch void))
(defun (stopwatch-start! sw)
  (unless (running? sw)
    (record-set! sw (started (monotonic-ns)))))

(: (-> Stopwatch void))
(defun (stopwatch-stop! sw)
  (when (running? sw)
    (record-set! sw
                 (elapsed (+ (record-ref sw elapsed) (- (monotonic-ns) (record-ref sw started))))
                 (started -1L))))

;; The reading while running includes the segment in progress, so a watch can be
;; read without stopping it.
(: (-> Stopwatch long))
(defun (stopwatch-ns sw)
  (if (running? sw)
      (+ (record-ref sw elapsed) (- (monotonic-ns) (record-ref sw started)))
      (record-ref sw elapsed)))

(: (-> Stopwatch long))
(defun (stopwatch-us sw)
  (/ (stopwatch-ns sw) 1000L))

(: (-> Stopwatch long))
(defun (stopwatch-ms sw)
  (/ (stopwatch-ns sw) 1000000L))

;; `(ref sw 'ms)` — the elapsed time under the name of a unit.
(impl (Refable Stopwatch)
  (type %index Symbol)
  (type %out long)
  (defun (ref sw sym)
    (case sym
      ('ns (stopwatch-ns sw))
      ('us (stopwatch-us sw))
      ('ms (stopwatch-ms sw))
      ('s (/ (stopwatch-ns sw) 1000000000L))
      (else
        (raise
          (cast Exception
                (ArgumentException.
                  (str "A stopwatch is read with 'ns, 'us, 'ms or 's, and this was '"
                       (displayed sym) "."))))))))

;; A duration in the largest unit that keeps it at 1 or more, with one decimal
;; under 100: "850 ns", "12.5 us", "3.2 ms", "1.5 s".
(: (-> long string))
(defun (ns->string ns)
  (def magnitude (abs ns))
  (cond
    ((< magnitude 1000L) (str (displayed ns) " ns"))
    ((< magnitude 1000000L) (scaled ns 1000L " us"))
    ((< magnitude 1000000000L) (scaled ns 1000000L " ms"))
    (else (scaled ns 1000000000L " s"))))

;; ns in units of `unit` nanoseconds: one decimal under 100, none above.
(: (-> long long string string))
(defun (scaled ns unit suffix)
  (def sign (if (< ns 0L) "-" ""))
  (def tenths (/ (* (abs ns) 10L) unit))
  (if (< tenths 1000L)
      (str sign (displayed (/ tenths 10L)) "." (displayed (% tenths 10L)) suffix)
      (str sign (displayed (/ tenths 10L)) suffix)))


;; `time-it` is `(std bench)`'s now, measured as a benchmark is: this one is
;; the same expansion, so that a program that imports this module for it goes
;; on finding it here. A macro is seen only where its own module is imported,
;; and what its expansion names has to be reachable from there, so those are
;; re-exported.
(def/macro (time-it form inject compare)
  (time-it-expansion form))

(re-export time-it-run time-it-fixed time-it-keep)

;; (std stream): loops over values that work which suspends produces.
;;
;; - A `Stream` is a recipe for a producer: a bjoroutine handed an `Emitter`.
;;   Holding one runs nothing. Each loop over it starts the producer in a
;;   fiber of its own, receives what it emits, and stops it when the loop
;;   ends, however the loop ends.
;; - `in-chan` reads a channel that something else fills, until an end event.
;;   The loop does not own the producer, and leaving early does not stop it.
;;
;; Both are drivers: `(:for x s)`. A loop in a bjoroutine suspends at each
;; receive. Anywhere else the receive parks the thread.
(import (std prelude))
(import (std syntax-match))

(:doc #:file "stream.bjodoc")

(export Stream Emitter StreamRun ChanReads Item
        make-stream emit! in-chan)

;; What the producer's wrapper sends: a value, the end, or what the
;; producer raised.
(type (: (Item %a) (Union (: Next %a) Done (: Failed Exception))))

;; Where a producer sends its values, and receives the loop's requests for
;; the next one.
(type (: (Emitter %a) #:opaque (Struct (: chan (Chan (Item %a))) (: demand (Chan Unit)))))

;; The producer's body. Each loop runs it afresh.
(type (: (Stream %a) #:opaque (Struct (: body (-bjo-> (Emitter %a) void)))))

;; A loop's cursor over a stream: the running producer's channels, the cancel
;; thunk that stops it, its promise, and the item the loop is at.
(type (: (StreamRun %a) #:opaque
         (Record (: chan (Chan (Item %a)))
                 (: demand (Chan Unit))
                 (: stop (-> CancelReason void))
                 (: producer (Promise Unit))
                 (: item (Item %a) #:mutable))))

;; A channel read until `end` commits.
(type (: (ChanReads %a %e) #:opaque (Struct (: chan (Chan %a)) (: end (Event %e)))))

;; A stream whose producer is `body`.
(: (-> (-bjo-> (Emitter %a) void) (Stream %a)))
(defun (make-stream body) (Stream (body body)))

;; Sends `v` to the loop, then waits until the loop asks for the next value.
;;
;; - The producer computes nothing the loop has not asked for.
;; - Raises `Cancelled` when the loop has ended, so the producer unwinds and
;;   its own releases run.
(: (-> (Emitter %a) %a void))
(defbjo (emit! out v) #:inline
  (sync (chan-send (.chan out) (Next v)))
  (sync (chan-recv (.demand out))))

;; The two halves of a receive: a bjoroutine's loop suspends, any other code
;; parks the thread.
(: (-> (Chan %a) %a))
(defbjouble (chan-take c)
  (#:sync (sync/blocking (chan-recv c)))
  (#:bjo (sync (chan-recv c))))

;; Asks the producer for its next item, and receives it.
(: (-> (Chan Unit) (Chan %a) %a))
(defbjouble (chan-ask demand c)
  (#:sync (sync/blocking (chan-send demand unit)) (sync/blocking (chan-recv c)))
  (#:bjo (sync (chan-send demand unit)) (sync (chan-recv c))))

(: (-> (Chan %a) (Event %e) (Option %a)))
(defbjouble (chan-take-until c end)
  (#:sync (sync/blocking (choose (wrap (chan-recv c) (fun (x) (Some x))) (wrap end (fun (_) None)))))
  (#:bjo (sync (choose (wrap (chan-recv c) (fun (x) (Some x))) (wrap end (fun (_) None))))))

;; Runs the producer's body, then sends the end: `Done`, or `Failed` with what
;; the body raised. A body that was cancelled sends nothing: the loop is gone.
;;
;; - The loop asked for the first item by starting the producer, and for each
;;   later one through `demand`, so the end is always asked for.
(: (-> (-bjo-> (Emitter %a) void) (Chan (Item %a)) (Chan Unit) CancelToken void))
(defbjo (run-producer body out demand token)
  (match (try (body (Emitter (chan out) (demand demand))) #:catch (System.Exception))
    ((Ok _) (sync (chan-send out Done)))
    ((Err e) (if (cancelled? token) (raise e) (sync (chan-send out (Failed e)))))))

;; Stores an item in the cursor. A `Failed` is raised here, in the loop.
(: (-> (StreamRun %a) (Item %a) (StreamRun %a)))
(defun (stream-set! run item)
  (match item
    ((Failed e) (raise e))
    (_ (record-set! run (item item)) run)))

;; The longest time the end of a loop waits for its producer to unwind.
(def producer-grace (seconds 5))

;; Waits until `p` lands or `d` has passed, whichever is first.
(: (-> (Promise %a) Duration void))
(defbjouble (await-within p d)
  (#:sync (ignore (sync/blocking (choose (wrap (promise-join p) (fun (_) unit)) (timeout d)))))
  (#:bjo (ignore (sync (choose (wrap (promise-join p) (fun (_) unit)) (timeout d))))))

;; Stops the producer, and waits until it has unwound.
;;
;; - After this returns, the producer's releases have run, unless it took
;;   longer than `producer-grace` to unwind.
;; - The wait runs in a shield, so it also works when the loop is left
;;   because its scope was cancelled.
;; - Does not raise when the grace passes. The producer then goes on
;;   unwinding, and the enclosing scope waits for it.
(: (-> (StreamRun %a) CancelReason void))
(defun (stop-producer run reason)
  ((.stop run) reason)
  ;; The shield's deadline is longer than `producer-grace`, so the timeout
  ;; always commits first and the shield never raises `Cancelled`.
  (with-shield #:within (seconds 6)
    (await-within (.producer run) producer-grace)))

;; A stream as a driver.
;;
;; - `iterable-start` starts the producer under a cancel token of its own,
;;   linked to the ambient one, and receives the first item.
;; - `iterable-next` asks for the next item, and receives it.
;; - `iterable-end` and `iterable-cleanup` stop it and wait until it has
;;   unwound: at a `:finish`, an `:abandon`, a driver beside it that ran out,
;;   an exception or a `ret`. See `stop-producer`.
(impl (Iterable (Stream %a))
  (type %elem %a)
  (type %cursor (StreamRun %a))
  (defun (iterable-start s)
    (def (Tuple stop token) (make-cancel))
    (link-cancel (parameter-ref current-cancel) token)
    (def c (make-chan))
    (def demand (make-chan))
    (def producer (parameterize ((current-cancel token)) (bjo (run-producer (.body s) c demand token))))
    (stream-set! (StreamRun (chan c) (demand demand) (stop stop) (producer producer) (item Done)) (chan-take c)))
  (defun (iterable-done? s run) (match (.item run) (Done #t) (_ #f)))
  (defun (iterable-current s run)
    (match (.item run)
      ((Next x) x)
      (_ (panic! "iterable-current of a stream that has ended"))))
  (defun (iterable-next s run) (stream-set! run (chan-ask (.demand run) (.chan run))))
  (defun (iterable-end s run) (stop-producer run (Requested "the loop over the stream ended")))
  (defun (iterable-cleanup s run) (stop-producer run (Requested "the loop over the stream was left"))))

;; The values received on `c` until `end` commits.
(: (-> (Chan %a) (Event %e) (ChanReads %a %e)))
(defun (in-chan c end) (ChanReads (chan c) (end end)))

(impl (Iterable (ChanReads %a %e))
  (type %elem %a)
  (type %cursor (Option %a))
  (defun (iterable-start s) (chan-take-until (.chan s) (.end s)))
  (defun (iterable-done? s c) (none? c))
  (defun (iterable-current s c) (option-ref c))
  (defun (iterable-next s c) (chan-take-until (.chan s) (.end s))))

;; `{streaming expr clause ...}`: a stream of `expr` for each round of a loop
;; of the clauses. The loop runs in the producer, so its clauses may suspend.
;;
;; - A loose `:when` arrives as a last `(:when ...)` clause.
(def/macro (streaming form inject compare)
  (syntax-match form
    ((_ expr clause ...)
     #'(make-stream (bjoroutine (out) (loop ,@clause (:do (emit! out ,expr))))))
    (bad (syntax-error bad "streaming takes an expression and loop clauses: {streaming (* x x) (:for x xs)}"))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/syntax-match.bjo
;;
;; Pattern matching for macro definitions.

;; Imports eq.
(import (std eq))

(import/extern
  (str-concat (: System.String.Concat)))

(:doc #:file "syntax-match.bjodoc")

(export all-of)

(: (-> string string string))
(defun (cat a b) (str-concat a b))

;; ---------------------------------------------------------------------------
;; What the expansion calls
;; ---------------------------------------------------------------------------

;; all-of accepts when f accepts all items, else None.
(: (-> (List %a) (-> %a (Option %b)) (Option (List %b))))
(defun (all-of items f)
  (match items
    (Nil (Some Nil))
    ((Cons item rest)
     (def (Some value) (f item) :propagate)
     (def (Some values) (all-of rest f) :propagate)
     (Some (Cons value values)))))

;; ---------------------------------------------------------------------------
;; Reading a pattern
;; ---------------------------------------------------------------------------

;; Checks for ellipsis.
(: (-> Syntax bool))
(defun (ellipsis? item)
  (match item
    ((SPunct text) (= text "..."))
    (_ #f)))

(: (-> Syntax bool))
(defun (wildcard? p)
  (match p
    ((SSym name) (= (symbol->string name) "_"))
    (_ #f)))

;; Checks if pattern cannot fail.
(: (-> Syntax bool))
(defun (irrefutable? p)
  (match p
    ((SSym _) #t)
    (_ #f)))

(: (-> Syntax Syntax bool))
(defun (same-name? a b)
  (match a
    ((SSym x)
     (match b
       ((SSym y) (= (symbol->string x) (symbol->string y)))
       (_ #f)))
    (_ #f)))

(: (-> (List Syntax) (List Syntax)))
(defun (init-of items)
  (match items
    (Nil Nil)
    ((Cons _ Nil) Nil)
    ((Cons item rest) (Cons item (init-of rest)))))

(: (-> (List Syntax) (Option Syntax)))
(defun (last-of items)
  (match items
    (Nil None)
    ((Cons item Nil) (Some item))
    ((Cons _ rest) (last-of rest))))

;; Private right fold.
(: (-> (-> %a %b %b) %b (List %a) %b))
(defun (fold-right f init xs)
  (let go ((rest (list-reverse xs)) (acc init))
    (if (list-empty? rest)
        acc
        (go (list-tail rest) (f (list-head rest) acc)))))

(: (-> (List Syntax) int))
(defun (count-ellipses items)
  (let go ((rest items) (n 0))
    (if (list-empty? rest)
        n
        (go (list-tail rest) (if (ellipsis? (list-head rest)) (+ n 1) n)))))

;; Extracts repeated pattern before trailing ellipsis.
(: (-> (List Syntax) Syntax (Option Syntax)))
(defun (repeated-of items whole)
  (def count (count-ellipses items))
  (if (= count 0)
      None
      (if (> count 1)
          (Some (syntax-error whole "A form repeats only its last pattern, so it may hold only one `...`."))
          (match (last-of items)
            (None None)
            ((Some final)
             (if (not (ellipsis? final))
                 (Some (syntax-error whole "`...` repeats the pattern before it, so it may only come last."))
                 (match (last-of (init-of items))
                   (None (Some (syntax-error whole "`...` repeats the pattern before it, and this form has none.")))
                   ((Some repeated) (Some repeated)))))))))

;; Items matched one for one.
(: (-> (List Syntax) Syntax (List Syntax)))
(defun (fixed-of items whole)
  (match (repeated-of items whole)
    (None items)
    ((Some _) (init-of (init-of items)))))

;; Every variable a pattern binds, in the order written.
(: (-> Syntax (List Syntax)))
(defun (pattern-vars p)
  (match p
    ((SSym _) (if (wildcard? p) Nil (list p)))
    ((SList items)
     (fold-right (fun (item found) (syntax-splice (pattern-vars item) found)) Nil items))
    (_ Nil)))

(: (-> Syntax (List Syntax) bool))
(defun (bound-already? var seen)
  (match seen
    (Nil #f)
    ((Cons other rest) (if (same-name? var other) #t (bound-already? var rest)))))

(: (-> Syntax bool))
(defun (uppercase-name? var)
  (match var
    ((SSym name)
     (def text (symbol->string name))
     (if (string-empty? text)
         #f
         (char-upper-case? (string-cursor-ref text (string-cursor-start text)))))
    (_ #f)))

;; Checked bound variables.
(: (-> Syntax (List Syntax)))
(defun (checked-vars p)
  (fold-right
    (fun (var seen)
      (if (uppercase-name? var)
          (Cons (syntax-error var "A pattern variable is lowercase; write 'name to match that identifier itself.") seen)
          (if (bound-already? var seen)
              (Cons (syntax-error var "This pattern binds the same name twice.") seen)
              (Cons var seen))))
    Nil
    (pattern-vars p)))

;; ---------------------------------------------------------------------------
;; Names the expansion binds
;; ---------------------------------------------------------------------------

;; Positional name.
(: (-> string string Syntax))
(defun (at-path prefix path)
  (SSym (string->symbol (cat prefix path))))

(: (-> string int string))
(defun (sub-path path index)
  (cat path (cat "-" (.ToString index))))

;; The path of the pattern a `...` repeats: the form's last item.
(: (-> (List Syntax) Syntax string string))
(defun (repeated-path items whole path)
  (sub-path path (list-length (fixed-of items whole))))

;; ---------------------------------------------------------------------------
;; What a pattern compiles to
;; ---------------------------------------------------------------------------

(: (-> Syntax string Syntax))
(defun (structural p path)
  (match p
    ((SSym _) p)
    ;; Bound not tested.
    ((SDatum _) (at-path "sm-lit" path))
    ((SInt text) #'(SInt ,(SStr text)))
    ((SStr _) #'(SStr ,p))
    ((SChar _) #'(SChar ,p))
    ((SKey _) #'(SKey ,p))
    ((SList items) #'(SList ,(cons-chain (fixed-of items p) path 0 (tail-of items p path))))
    (_ (syntax-error p "This is not a pattern."))))

;; Matches known length form.
(: (-> (List Syntax) string int Syntax Syntax))
(defun (cons-chain items path index tail)
  (match items
    (Nil tail)
    ((Cons item rest)
     #'(Cons ,(structural item (sub-path path index)) ,(cons-chain rest path (+ index 1) tail)))))

;; Ends cons chain.
(: (-> (List Syntax) Syntax string Syntax))
(defun (tail-of items whole path)
  (match (repeated-of items whole)
    (None #'Nil)
    ((Some repeated)
     (if (irrefutable? repeated) repeated (at-path "sm-rest" path)))))

(: (-> Syntax string (List Syntax)))
(defun (checks p path)
  (match p
    ((SDatum _) (list #'(syntax-ident=? ,(at-path "sm-lit" path) (SDatum ,p))))
    ((SList items)
     (syntax-splice (fixed-checks (fixed-of items p) path 0) (repeat-check items p path)))
    (_ Nil)))

(: (-> (List Syntax) string int (List Syntax)))
(defun (fixed-checks items path index)
  (match items
    (Nil Nil)
    ((Cons item rest)
     (syntax-splice (checks item (sub-path path index)) (fixed-checks rest path (+ index 1))))))

;; Checks repeated items match pattern.
(: (-> (List Syntax) Syntax string (List Syntax)))
(defun (repeat-check items whole path)
  (match (repeated-of items whole)
    (None Nil)
    ((Some repeated)
     (if (irrefutable? repeated)
         Nil
         (list #'(some? ,(fold-over items whole path repeated #'#t)))))))

;; Folds ellipsis over compound.
(: (-> (List Syntax) Syntax string Syntax Syntax Syntax))
(defun (fold-over items whole path repeated value)
  (def item (at-path "sm-item" path))
  (def inner (repeated-path items whole path))
  #'(all-of ,(at-path "sm-rest" path)
            (fun (,item)
              (match ,item
                (,(structural repeated inner) ,(answer (checks repeated inner) #'(Some ,value)))
                (_ None)))))

;; Item answer.
(: (-> (List Syntax) Syntax Syntax))
(defun (answer tests some)
  (if (list-empty? tests) some #'(if ,(conjunction tests) ,some None)))

(: (-> (List Syntax) Syntax))
(defun (conjunction tests)
  (match tests
    (Nil #'#t)
    ((Cons only Nil) only)
    (_ #'(and ,@tests))))

;; Variable value source.
(: (-> Syntax Syntax string (Option Syntax)))
(defun (value-of p var path)
  (match p
    ((SSym _) (if (same-name? p var) (Some p) None))
    ((SList items)
     (match (fixed-value (fixed-of items p) var path 0)
       ((Some found) (Some found))
       (None (repeated-value items p var path))))
    (_ None)))

(: (-> (List Syntax) Syntax string int (Option Syntax)))
(defun (fixed-value items var path index)
  (match items
    (Nil None)
    ((Cons item rest)
     (match (value-of item var (sub-path path index))
       ((Some found) (Some found))
       (None (fixed-value rest var path (+ index 1)))))))

(: (-> (List Syntax) Syntax Syntax string (Option Syntax)))
(defun (repeated-value items whole var path)
  (def (Some repeated) (repeated-of items whole) :propagate)
  (if (irrefutable? repeated)
      ;; The tail binder, bound by the pattern like any other variable.
      (if (same-name? repeated var) (Some repeated) None)
      (match (value-of repeated var (repeated-path items whole path))
        (None None)
        ((Some inner)
         ;; option-ref takes answer.
         (Some #'(option-ref ,(fold-over items whole path repeated inner)))))))

;; ---------------------------------------------------------------------------
;; A clause
;; ---------------------------------------------------------------------------

;; Clause bindings.
(: (-> Syntax (List Syntax)))
(defun (binds p)
  (fold-right
    (fun (var bindings)
      (match (value-of p var "")
        (None bindings)
        ((Some value)
         (if (same-name? value var) bindings (Cons #'(,var ,value) bindings)))))
    Nil
    (checked-vars p)))

(: (-> (List Syntax) Syntax))
(defun (single-body bodies)
  (match bodies
    ((Cons body Nil) body)
    (_ #'(let () ,@bodies))))

(: (-> Syntax Syntax))
(defun (compile-clause clause)
  (match clause
    ((SList (Cons p bodies))
     (if (list-empty? bodies)
         (syntax-error clause "A syntax-match clause is (pattern body ...), and this one has no body.")
         (let ((tests (checks p ""))
               (bindings (binds p))
               (body (single-body bodies)))
           (def whole (if (list-empty? bindings) body #'(let ,(SList bindings) ,body)))
           (if (list-empty? tests)
               #'(,(structural p "") ,whole)
               #'(,(structural p "") #:when ,(conjunction tests) ,whole)))))
    (_ (syntax-error clause "A syntax-match clause is (pattern body ...)."))))

;; True if last clause covers everything.
(: (-> (List Syntax) bool))
(defun (covered? clauses)
  (match (last-of clauses)
    (None #f)
    ((Some clause)
     (match clause
       ((SList (Cons p _)) (irrefutable? p))
       (_ #f)))))

(: (-> (List Syntax) (List Syntax)))
(defun (arms clauses)
  (syntax-splice
    ;; Built backwards from the reversed input, so it comes out in order.
    (let go ((rest (list-reverse clauses)) (acc Nil))
      (if (list-empty? rest)
          acc
          (go (list-tail rest) (cons (compile-clause (list-head rest)) acc))))
    (if (covered? clauses)
        Nil
        ;; Reports unmatched form.
        (list #'(sm-unmatched (syntax-error sm-unmatched "No clause of this syntax-match matched."))))))

;; ---------------------------------------------------------------------------
;; The macro
;; ---------------------------------------------------------------------------

;; syntax-match macro.
(def/macro (syntax-match form inject compare)
  (match form
    ((SList (Cons _ (Cons subject clauses)))
     (if (list-empty? clauses)
         (syntax-error form "syntax-match needs at least one clause.")
         #'(match ,subject ,@(arms clauses))))
    (_ (syntax-error form "syntax-match takes a subject and at least one clause: (syntax-match form (pattern body ...) ...)"))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/test.bjo — tests that are silent unless they fail.
;;
;;     (import (std prelude))
;;     (import (std test))
;;
;;     (deftest "parse adds"
;;       (check (= (parse "1+2") (Ok 3))))
;;
;;     (deftest "a bad number is an error"
;;       (check-match (parse "x") (Err _)))
;;
;;     (run-tests)
;;
;; `deftest` registers a test where it is written, and `(run-tests)` is the
;; file's `main`: it runs every test the module registered, in source order. A
;; test that passes prints nothing. A test that fails prints the test, where it
;; is, and every check in it that failed, with the values the check saw. The
;; run ends with one line, and exits 1 if anything failed.
;;
;; Arguments to the program choose tests: a test runs when its name contains
;; one of them. With none, every test runs. Arguments that start with `--` are
;; flags; see `parse-test-args`.
;;
;; Each test runs in a scope of its own, as `with-test` in `(std simpletest)`
;; does: under a deadline (`#:within d`, a Duration, 5 seconds by default), waiting for
;; the fibers it started, and failing if it left a resource open (unless
;; `#:open-ok`). A test that raises fails with the exception's message, and the
;; tests after it still run.
;;
;; Property tests are in `(std test prop)`, generators in `(std test gen)`,
;; and output snapshots in `(std test expect)`. They hook into a run through
;; `running-test`, `current-test-options`, `test-seed` and `at-run-end!`.

(export deftest run-tests
        check check-match check-raises check-fail check-approx check-output
        Test TestResult TestStatus registered-tests module-tests run-test run-registered-tests
        test-name test-module test-file test-line test-result-test test-result-failures
        test-result-status test-failed!
        ;; What the extensions of a run read
        TestOptions parse-test-args current-test-options running-test test-seed
        test-time-left next-test-index! take-test-output!
        mix-seed string-hash at-run-end! failures-into
        ;; Named by the templates of the macros above, so they have to be
        ;; reachable where those expand.
        register-test! check-failed! deftest-cases-row check-raises-message approx-close?)

(import (std prelude))
(import (std syntax-match))
(import (std random))

(:doc #:file "test.bjodoc")

;; A failure makes the program exit 1 even when `main` answers 0.
(import/extern
  (set-exit-code! (: System.Environment.ExitCode (-> int void) #:set)))

;; ---------------------------------------------------------------------------
;; Tests and their registry
;; ---------------------------------------------------------------------------

;; A registered test. `body` is the test's forms, run when the test runs.
;; `skip` and `todo` are the reasons given with `#:skip` and `#:todo`, and
;; empty when the test has none. `capture` is #t when the body uses
;; `expect-output`: its output then goes to a string port that
;; `take-test-output!` reads.
(type (: Test (Record (: name string)
                      (: module string)
                      (: file string)
                      (: line int)
                      (: within (Option Duration))
                      (: open-ok bool)
                      (: skip string)
                      (: todo string)
                      (: slow bool)
                      (: capture bool)
                      (: body (-> void)))))

(: (-> Test string))
(defun (test-name t) (record-ref t name))

(: (-> Test string))
(defun (test-module t) (record-ref t module))

(: (-> Test string))
(defun (test-file t) (record-ref t file))

(: (-> Test int))
(defun (test-line t) (record-ref t line))

;; Newest first. A box rather than a `def/mutable`, so that `register-test!`
;; can be inlined into the module that calls it.
(: (Box (List Test)))
(def registry (make-box (list)))

;; Adds a test. It answers a number only so that `deftest` can bind it at the
;; top level, which is what makes it run before `main`.
(: (-> string string string int (Option Duration) bool string string bool bool (-> void) int))
(defun (register-test! name module file line within open-ok skip todo slow capture body)
  (box-set! registry
            (cons (Test (name name) (module module) (file file) (line line)
                        (within within) (open-ok open-ok) (skip skip) (todo todo)
                        (slow slow) (capture capture) (body body))
                  (box-ref registry)))
  0)

;; Every test registered so far, in the order they were written.
(: (-> (List Test)))
(defun (registered-tests) (list-reverse (box-ref registry)))

;; The tests one module registered, by the key `(current-module)` answers
;; there, in the order they were written. Every module that imports this one
;; registers into the same list, and each runs only its own.
(: (-> string (List Test)))
(defun (module-tests module)
  (list-filter (fun (t) (= (record-ref t module) module)) (registered-tests)))

;; ---------------------------------------------------------------------------
;; The options of a run
;; ---------------------------------------------------------------------------

;; What the arguments of a test program asked for. `seed` is always set: from
;; `--seed`, from BJO_TEST_SEED, or from the OS. `runs` and `shrink-limit` are
;; 0 when not given, which means each `for-all`'s own default.
(type (: TestOptions (Record (: filters (Vec string))
                             (: seed ulong)
                             (: runs int)
                             (: shrink-limit int)
                             (: stats bool)
                             (: quick bool)
                             (: promote bool)
                             (: store bool)
                             (: junit (Option string)))))

;; A number in decimal, or in hex after 0x.
(: (-> string (Option ulong)))
(defun (parse-ulong s)
  (def chars (string->list s))
  (def hex? (match chars ((Cons #\0 (Cons (or #\x #\X) _)) #t) (_ #f)))
  (def digits (if hex? (list-tail (list-tail chars)) chars))
  (def base (if hex? 16UL 10UL))
  (if (list-empty? digits)
      None
      (let go ((left digits) (acc 0UL))
        (match left
          (Nil (Some acc))
          ((Cons ch more)
           (def d (char->int ch))
           (def v (cond ((and (>= d 48) (<= d 57)) (- d 48))
                        ((and hex? (>= d 97) (<= d 102)) (- d 87))
                        ((and hex? (>= d 65) (<= d 70)) (- d 55))
                        (else -1)))
           (if (< v 0)
               None
               (go more (+ (* acc base) (cast System.UInt64 v)))))))))

(: (-> string string (Result string int)))
(defun (parse-count flag s)
  (match (parse-ulong s)
    ((Some n) #:when (< n 1000000000UL) (Ok (cast System.Int32 n)))
    (_ (Err #"${flag} takes a number, and this is not one: ${s}"))))

;; The seed of a run that gives none: BJO_TEST_SEED, else 64 bits from the OS.
(: (-> (Result string ulong)))
(defun (fresh-seed)
  (match (get-environment-variable "BJO_TEST_SEED")
    ((Some s)
     (option->result (parse-ulong s) #"BJO_TEST_SEED is not a number: ${s}"))
    (None (Ok (random-u64 #:rng (make-random))))))

;; The arguments of a test program: flags, and name filters. Flags start with
;; `--`: `--seed n`, `--runs n`, `--shrink-limit n`, `--junit file`, and
;; `--stats`, `--quick`, `--promote`, `--no-store`.
(: (-> (Vec string) (Result string TestOptions)))
(defun (parse-test-args args)
  (def (Ok seed0) (fresh-seed) :propagate)
  (let go ((left (vec->list args)) (filters (list)) (seed seed0) (runs 0) (shrink 0)
           (stats #f) (quick #f) (promote #f) (store #t) (junit None))
    (defun (value-of flag more k)
      (match more
        ((Cons v after) (k v after))
        (Nil (Err #"${flag} needs a value."))))
    (match left
      (Nil
       (Ok (TestOptions (filters (list->vec (list-reverse filters))) (seed seed) (runs runs)
                        (shrink-limit shrink) (stats stats) (quick quick) (promote promote)
                        (store store) (junit junit))))
      ((Cons a more)
       (case a
         ("--seed"
          (value-of a more
            (fun (v after)
              (match (parse-ulong v)
                ((Some n) (go after filters n runs shrink stats quick promote store junit))
                (None (Err #"--seed takes a number, in decimal or 0x hex: ${v}"))))))
         ("--runs"
          (value-of a more
            (fun (v after)
              (match (parse-count a v)
                ((Ok n) (go after filters seed n shrink stats quick promote store junit))
                ((Err e) (Err e))))))
         ("--shrink-limit"
          (value-of a more
            (fun (v after)
              (match (parse-count a v)
                ((Ok n) (go after filters seed runs n stats quick promote store junit))
                ((Err e) (Err e))))))
         ("--junit"
          (value-of a more
            (fun (v after) (go after filters seed runs shrink stats quick promote store (Some v)))))
         ("--stats" (go more filters seed runs shrink #t quick promote store junit))
         ("--quick" (go more filters seed runs shrink stats #t promote store junit))
         ("--promote" (go more filters seed runs shrink stats quick #t store junit))
         ("--no-store" (go more filters seed runs shrink stats quick promote #f junit))
         (else
          (if (string-starts-with? a "--")
              (Err #"unknown flag ${a}. The flags are --seed, --runs, --shrink-limit, --junit, --stats, --quick, --promote and --no-store.")
              (go more (cons a filters) seed runs shrink stats quick promote store junit))))))))

;; The options of a run with no arguments, for tests run with `run-test`
;; outside `(run-tests)`.
(: (-> TestOptions))
(defun (default-options)
  (match (parse-test-args (vec-empty))
    ((Ok o) o)
    ((Err why) (panic! why))))

(: (Box TestOptions))
(def options-box (make-box (default-options)))

;; The options of the run in progress.
(: (-> TestOptions))
(defun (current-test-options) (box-ref options-box))

;; ---------------------------------------------------------------------------
;; Seeds
;; ---------------------------------------------------------------------------

;; FNV-1a over the string's characters. Not `GetHashCode`, which .NET
;; randomizes per process: a test's seed has to be the same in every run.
(: (-> string ulong))
(defun (string-hash s)
  (string-fold (fun (ch h)
                 (* (bitwise-xor h (cast System.UInt64 (char->int ch))) 1099511628211UL))
               14695981039346656037UL
               s))

;; SplitMix64's finalizer over a xor b: two seeds that differ in one bit give
;; unrelated results.
(: (-> ulong ulong ulong))
(defun (mix-seed a b)
  (def z0 (bitwise-xor a b))
  (def z1 (* (bitwise-xor z0 (shift-right z0 30)) 13787848793156543929UL))
  (def z2 (* (bitwise-xor z1 (shift-right z1 27)) 10723151780598845931UL))
  (bitwise-xor z2 (shift-right z2 31)))

;; The seed of one test: the run's seed and the test's name. So choosing
;; tests, or adding one, does not change what the others draw.
(: (-> Test ulong))
(defun (test-seed t)
  (mix-seed (record-ref (current-test-options) seed) (string-hash (record-ref t name))))

;; ---------------------------------------------------------------------------
;; Failures
;; ---------------------------------------------------------------------------

;; The failures of the test that is running, newest first, or `None` when no
;; test is running. A check outside a test reports at once.
(: (Box (Option (Box (List string)))))
(def current-failures (make-box None))

;; The test that is running, or `None`.
(: (Box (Option Test)))
(def running-box (make-box None))

(: (-> (Option Test)))
(defun (running-test) (box-ref running-box))

;; When the running test started, and its deadline in milliseconds.
(: (Box (Tuple long long)))
(def running-clock (make-box (Tuple 0L 0L)))

;; The milliseconds left before the running test's deadline, or a day when no
;; test runs. Shrinking stops early when this gets low.
(: (-> long))
(defun (test-time-left)
  (match (box-ref running-box)
    (None 86400000L)
    ((Some _)
     (match (box-ref running-clock)
       ((Tuple started within)
        (- (+ started within) (monotonic-ms)))))))

;; The port a capturing test writes to, and how far `take-test-output!` has
;; read it.
(: (Box (Option (Tuple OutputPort StringCursor))))
(def output-box (make-box None))

;; What the running test wrote since it started, or since the last call. None
;; when the test does not capture its output.
(: (-> (Option string)))
(defun (take-test-output!)
  (match (box-ref output-box)
    (None None)
    ((Some (Tuple port from))
     (def text (get-output-string port))
     (box-set! output-box (Some (Tuple port (string-cursor-end text))))
     (Some (substring text from (string-cursor-end text))))))

;; How many times `next-test-index!` was called with each key in the running
;; test. A `for-all` takes its position among those with the same bindings
;; from it, to find its stored cases.
(: (Box (Map string int)))
(def index-box (make-box (map-empty)))

(: (-> string int))
(defun (next-test-index! key)
  (def counts (box-ref index-box))
  (def i (map-ref-or counts key 0))
  (box-set! index-box (map-set counts key (+ i 1)))
  i)

;; Runs `body` with its failures going to `failures` instead of to the running
;; test, and puts the running test's back afterwards, also when `body` raises.
;; `for-all` runs each case this way, to decide which failures to keep.
(: (-> (Box (List string)) (-> %a) %a))
(defun (failures-into failures body)
  (def outer (box-ref current-failures))
  (box-set! current-failures (Some failures))
  (match (try (body) #:catch (System.Exception))
    ((Ok v) (box-set! current-failures outer) v)
    ((Err e) (box-set! current-failures outer) (raise e))))

;; A source file as a report names it: by its file name, which is enough to
;; find it and short enough to read.
(: (-> string string))
(defun (shown-file path)
  (match (path-filename path)
    ((Some name) name)
    (None path)))

;; Records a failure of the running test, or prints it and fails the program
;; when there is none.
(: (-> string void))
(defun (test-failed! message)
  (match (box-ref current-failures)
    ((Some failures) (box-set! failures (cons message (box-ref failures))))
    (None
     (println message)
     (set-exit-code! 1))))

;; The report of one failed check: where it is, what it said, and each value
;; it saw, as source text and value.
(: (-> string int string (List (Tuple string string)) void))
(defun (check-failed! file line source seen)
  (def width
    (loop (:for (Tuple src _) seen)
          (:acc w (folding 0 (let ((n (string-length src))) (if (> n w) n w))))))
  (def lines
    (loop (:for (Tuple src value) seen)
          (:acc out (listing #"    ${(string-pad-right src width)} => ${value}"))))
  (test-failed!
    (string-join (list->vec (cons #"  ${(shown-file file)}:${line}: ${source}" lines)) "\n")))

;; ---------------------------------------------------------------------------
;; Checks
;; ---------------------------------------------------------------------------

;; The source position of a form, as literals a template can splice.
(: (-> Syntax Syntax))
(defun (file-of form) (SStr (syntax-file form)))

(: (-> Syntax Syntax))
(defun (line-of form) (SInt (int->string (syntax-line form))))

(: (-> Syntax Syntax))
(defun (source-of form) (SStr (syntax->string form)))

;; A comparison, with both operands evaluated once, in order, and shown when
;; it does not hold.
(: (-> Syntax Syntax Syntax Syntax Syntax))
(defun (check-comparison form op a b)
  #'(let ((left ,a)
          (right ,b))
      (unless (,op left right)
        (check-failed! ,(file-of form) ,(line-of form) ,(source-of form)
                       (list (Tuple ,(source-of a) (displayed left))
                             (Tuple ,(source-of b) (displayed right)))))))

;; `(check expr)` passes when `expr` is #t. When it is a comparison — `=`, `<`,
;; `>`, `<=` or `>=` — a failure shows both operands' values; otherwise it says
;; that the expression was #f. A failed check does not stop the test.
(def/macro (check form inject compare)
  (syntax-match form
    ((_ ('= a b)) (check-comparison form #'= a b))
    ((_ ('< a b)) (check-comparison form #'< a b))
    ((_ ('> a b)) (check-comparison form #'> a b))
    ((_ ('<= a b)) (check-comparison form #'<= a b))
    ((_ ('>= a b)) (check-comparison form #'>= a b))
    ((_ expr)
     #'(unless ,expr
         (check-failed! ,(file-of form) ,(line-of form) ,(source-of form)
                        (list (Tuple ,(source-of expr) "#f")))))
    (bad (syntax-error bad "check takes one expression: (check (= (f x) 3))."))))

;; `(check-match expr pattern)` passes when the value matches the pattern. A
;; failure shows the value.
(def/macro (check-match form inject compare)
  (syntax-match form
    ((_ expr pattern)
     #'(let ((value ,expr))
         (match value
           (,pattern unit)
           (_ (check-failed! ,(file-of form) ,(line-of form) ,(source-of form)
                             (list (Tuple ,(source-of expr) (displayed value))))))))
    (bad (syntax-error bad "check-match takes an expression and a pattern: (check-match (parse \"x\") (Err _))."))))

;; The part of `check-raises` that `#:message` adds: the exception was of the
;; right type, and now its message has to contain `part`.
(: (-> string int string string string void))
(defun (check-raises-message file line source message part)
  (unless (string-contains? message part)
    (check-failed! file line source
                   (list (Tuple "message" message) (Tuple "should contain" part)))))

;; `(check-raises Type expr)` passes when evaluating `expr` raises a `Type`,
;; a .NET exception type. A failure shows what it answered instead. With
;; `#:message "part"`, the exception's message has to contain `part` too.
(def/macro (check-raises form inject compare)
  (syntax-match form
    ((_ type expr #:message part)
     #'(match (try ,expr #:catch (,type))
         ((Ok value)
          (check-failed! ,(file-of form) ,(line-of form) ,(source-of form)
                         (list (Tuple ,(source-of expr) (str "answered " (displayed value) " and raised nothing")))))
         ((Err e) (check-raises-message ,(file-of form) ,(line-of form) ,(source-of form) (.-Message e) ,part))))
    ((_ type expr)
     #'(match (try ,expr #:catch (,type))
         ((Ok value)
          (check-failed! ,(file-of form) ,(line-of form) ,(source-of form)
                         (list (Tuple ,(source-of expr) (str "answered " (displayed value) " and raised nothing")))))
         ((Err _) unit)))
    (bad (syntax-error bad "check-raises takes an exception type and an expression: (check-raises System.FormatException (parse-int \"x\")), optionally #:message \"part\" after them."))))

;; `(check-fail "why")` fails where it stands: for a branch a test must not
;; reach.
(def/macro (check-fail form inject compare)
  (syntax-match form
    ((_ why)
     #'(check-failed! ,(file-of form) ,(line-of form) ,(source-of form)
                      (list (Tuple "reason" ,why))))
    (bad (syntax-error bad "check-fail takes a reason: (check-fail \"this branch is unreachable\")."))))

;; Whether two doubles are within `eps` of each other: absolutely near 0, and
;; relative to the larger of the two away from it. Equal infinities are close;
;; NaN is close to nothing.
(: (-> double double double bool))
(defun (approx-close? a b eps)
  (or (= a b)
      (let* ((diff (abs (- a b)))
             (scale (max 1.0 (max (abs a) (abs b)))))
        (<= diff (* eps scale)))))

;; `(check-approx a b)` passes when two doubles are within 1e-9 of each other,
;; relative to their size when that is above 1. `#:eps e` changes the bound.
(: (-> Syntax Syntax Syntax Syntax Syntax))
(defun (check-approx-expansion form a b eps)
  #'(let ((left ,a) (right ,b))
      (unless (approx-close? left right ,eps)
        (check-failed! ,(file-of form) ,(line-of form) ,(source-of form)
                       (list (Tuple ,(source-of a) (displayed left))
                             (Tuple ,(source-of b) (displayed right)))))))

(def/macro (check-approx form inject compare)
  (syntax-match form
    ((_ a b #:eps eps) (check-approx-expansion form a b eps))
    ((_ a b) (check-approx-expansion form a b #'1e-9))
    (bad (syntax-error bad "check-approx takes two doubles: (check-approx (sqrt 2.0) 1.4142135623730951), optionally #:eps 1e-6 after them."))))

;; `(check-output expr "text")` passes when what `expr` writes to
;; `current-output-port` is the text.
(def/macro (check-output form inject compare)
  (syntax-match form
    ((_ expr text)
     #'(let* ((port (open-output-string))
              (ignored (parameterize ((current-output-port port)) ,expr unit))
              (written (get-output-string port))
              (wanted ,text))
         (unless (= written wanted)
           (check-failed! ,(file-of form) ,(line-of form) ,(source-of form)
                          (list (Tuple "it wrote" (displayed written))
                                (Tuple "wanted" (displayed wanted)))))))
    (bad (syntax-error bad "check-output takes an expression and a string: (check-output (display \"hi\") \"hi\")."))))

;; ---------------------------------------------------------------------------
;; deftest
;; ---------------------------------------------------------------------------

;; The deadline of a test that gives none.
(: Duration)
(def default-test-deadline (seconds 5))

;; What the options of one `deftest` came to. `cases` is the `#:cases` table:
;; the column names, then the rows.
(type (: DeftestOptions (Record (: within Syntax)
                                (: open-ok bool)
                                (: skip Syntax)
                                (: todo Syntax)
                                (: slow bool)
                                (: cases (Option (Tuple (List Syntax) (List Syntax))))
                                (: body (List Syntax)))))

(: (-> (List Syntax) DeftestOptions))
(defun (no-options body)
  (DeftestOptions (within #'None) (open-ok #f) (skip #'"") (todo #'"") (slow #f)
                  (cases None) (body body)))

;; Reads the options after a test's name, in any order, up to the body.
(: (-> Syntax (List Syntax) DeftestOptions))
(defun (deftest-options form items) (options-from form items (no-options (list))))

(: (-> Syntax (List Syntax) DeftestOptions DeftestOptions))
(defun (options-from form items o)
  (let ((go (fun (more o2) (options-from form more o2))))
    (syntax-match (SList items)
      ((#:within d more ...) (go more (record-set o (within #'(Some ,d)))))
      ((#:open-ok more ...) (go more (record-set o (open-ok #t))))
      ((#:skip why more ...) (go more (record-set o (skip why))))
      ((#:todo why more ...) (go more (record-set o (todo why))))
      ((#:slow more ...) (go more (record-set o (slow #t))))
      ((#:cases ((column ...) row ...) more ...) (go more (with-cases form column row o)))
      ((first more ...) (record-set o (body (cons first more))))
      (_ (ignore (syntax-error form "deftest takes a name and a body: (deftest \"name\" body ...), with options such as #:within (seconds 2), #:open-ok, #:skip \"why\", #:todo \"why\", #:slow and #:cases after the name."))
         o))))

;; `#:cases ((a b) row ...)`: the first list names the columns, and each row
;; after it gives them values.
(: (-> Syntax (List Syntax) (List Syntax) DeftestOptions DeftestOptions))
(defun (with-cases form columns rows o)
  (def width (list-length columns))
  (when (list-empty? rows)
    (ignore (syntax-error form "#:cases has no rows. It is the column names, then rows of as many values: #:cases ((input expected) (\"1\" 1) (\"2\" 2)).")))
  (loop (:for row rows)
        (:do (match row
               ((SList items)
                (unless (= (list-length items) width)
                  (ignore (syntax-error row #"This row of #:cases has ${(list-length items)} values, and there are ${width} columns."))))
               (_ (ignore (syntax-error row "A row of #:cases is a list of values, one for each column."))))))
  (record-set o (cases (Some (Tuple columns rows)))))

;; The name of one row of a `#:cases` test, and its values, for its report.
(: (-> int (List (Tuple string string)) string))
(defun (deftest-cases-row i seen)
  (string-join
    (list->vec (cons #"  case ${i}:"
                     (list-map (fun (p) (match p ((Tuple n v) #"    ${n} = ${v}"))) seen)))
    "\n"))

;; One row's body: the columns bound to the row's values. A column that the
;; body uses as the pattern of a `check-match` is spliced in instead, so a row
;; can hold a pattern such as `(Err _)`.
(: (-> Syntax int (List Syntax) Syntax (List Syntax) Syntax))
(defun (case-body form i columns row body)
  (def values (syntax-match row ((v ...) v) (_ (list))))
  (def pairs (zip-syntax columns values))
  (def pattern-columns (list-filter (fun (p) (used-as-pattern? (fst p) body)) pairs))
  (def value-columns (list-filter (fun (p) (not (used-as-pattern? (fst p) body))) pairs))
  (def spliced (list-map (fun (b) (splice-patterns pattern-columns b)) body))
  #'(let ((case-failures (make-box (list))))
      (failures-into case-failures
                     (fun ()
                       (let* ,(SList (list-map (fun (p) #'(,(fst p) ,(snd p))) value-columns))
                         ,@spliced
                         unit)))
      (unless (list-empty? (box-ref case-failures))
        (test-failed!
          (string-join
            (list->vec
              (cons (deftest-cases-row ,(SInt (int->string i))
                                       (list ,@(list-map (fun (p) #'(Tuple ,(SStr (syntax->string (fst p)))
                                                                           ,(SStr (syntax->string (snd p)))))
                                                         pairs)))
                    (list-reverse (box-ref case-failures))))
            "\n")))))


(: (-> (List Syntax) (List Syntax) (List (Tuple Syntax Syntax))))
(defun (zip-syntax xs ys)
  (match (Tuple xs ys)
    ((Tuple (Cons x xm) (Cons y ym)) (cons (Tuple x y) (zip-syntax xm ym)))
    (_ (list))))

;; Whether `name` stands in the pattern position of a `check-match` anywhere
;; in the forms.
(: (-> Syntax (List Syntax) bool))
(defun (used-as-pattern? name forms)
  (any? (fun (f)
          (or (match (check-match-pattern f)
                ((Some p) (syntax-ident=? p name))
                (None #f))
              (match f
                ((SList items) (used-as-pattern? name items))
                (_ #f))))
        forms))

;; The pattern of a `(check-match expr pattern)` whose pattern is a name.
(: (-> Syntax (Option Syntax)))
(defun (check-match-pattern f)
  (match f
    ((SList (Cons (SSym head) (Cons _ (Cons (SSym p) Nil))))
     (if (= (symbol->string head) "check-match") (Some (SSym p)) None))
    (_ None)))

;; The forms with each pattern column in a `check-match` replaced by its
;; row's value.
(: (-> (List (Tuple Syntax Syntax)) Syntax Syntax))
(defun (splice-patterns columns form)
  (match (check-match-pattern form)
    ((Some p)
     (match form
       ((SList (Cons head (Cons e _)))
        (match (list-filter (fun (c) (syntax-ident=? (fst c) p)) columns)
          ((Cons c _) (syntax-at (SList (list head (splice-patterns columns e) (snd c))) form))
          (Nil form)))
       (_ form)))
    (None
     (match form
       ((SList items) (syntax-at (SList (list-map (fun (x) (splice-patterns columns x)) items)) form))
       (_ form)))))

(: (-> Syntax Syntax DeftestOptions Syntax))
(defun (deftest-expansion form name o)
  (def body
    (match (record-ref o cases)
      (None (record-ref o body))
      ((Some (Tuple columns rows))
       (loop (:with i 1 (+ i 1))
             (:for row rows)
             (:acc out (listing (case-body form i columns row (record-ref o body))))))))
  #'(def registered
      (register-test! ,name (current-module) ,(file-of form) ,(line-of form)
                      ,(record-ref o within) ,(if (record-ref o open-ok) #'#t #'#f)
                      ,(record-ref o skip) ,(record-ref o todo) ,(if (record-ref o slow) #'#t #'#f)
                      ,(if (mentions? "expect-output" body) #'#t #'#f)
                      (fun () ,@body unit))))

;; Whether a name appears anywhere in the forms.
(: (-> string (List Syntax) bool))
(defun (mentions? name forms)
  (any? (fun (f)
          (match f
            ((SSym s) (= (symbol->string s) name))
            ((SList items) (mentions? name items))
            (_ #f)))
        forms))

;; `(deftest "name" body ...)`, with options after the name in any order:
;; `#:within d` (a Duration), `#:open-ok`, `#:skip "why"`, `#:todo "why"`, `#:slow` and
;; `#:cases ((column ...) row ...)`. Registers the test; `(run-tests)` runs it.
(def/macro (deftest form inject compare)
  (syntax-match form
    ((_ name first more ...)
     (deftest-expansion form name (deftest-options form (cons first more))))
    (bad (syntax-error bad "deftest takes a name and a body: (deftest \"name\" body ...)."))))

;; ---------------------------------------------------------------------------
;; Running
;; ---------------------------------------------------------------------------

;; How a test came out. `Todo` is a `#:todo` test that failed, as it was
;; expected to, and `Fixed` one that passed.
(type (: TestStatus (Union Passed Failed Skipped Todo Fixed)))

;; What running one test came to: its failures, in the order they happened.
(type (: TestResult (Record (: test Test)
                            (: status TestStatus)
                            (: failures (List string)))))

(: (-> TestResult Test))
(defun (test-result-test r) (record-ref r test))

(: (-> TestResult TestStatus))
(defun (test-result-status r) (record-ref r status))

(: (-> TestResult (List string)))
(defun (test-result-failures r) (record-ref r failures))

;; Whether the deadline was what ended the scope. A fiber the test leaked is
;; cancelled by the deadline rather than failing, so the scope closes cleanly
;; and the test would otherwise pass, slowly.
(: (-> (Box (Option Scope)) bool))
(defun (overran? holder)
  (match (box-ref holder)
    (None #f)
    ((Some s)
     (match (cancel-reason (scope-token s))
       ((Some Deadline) #t)
       (_ #f)))))

;; Runs one test in a scope of its own and answers what it came to. Prints
;; nothing. A `#:skip` test does not run.
(: (-> Test TestResult))
(defun (run-test t)
  (if (not (string-empty? (record-ref t skip)))
      (TestResult (test t) (status Skipped) (failures (list)))
      (run-test-body t)))

(: (-> Test TestResult))
(defun (run-test-body t)
  (def failures (make-box (list)))
  (def outer (box-ref current-failures))
  (def outer-test (box-ref running-box))
  (box-set! current-failures (Some failures))
  (box-set! running-box (Some t))
  (def holder (make-box None))
  (def body (record-ref t body))
  (def within (match (record-ref t within) ((Some d) d) (None default-test-deadline)))
  (def outer-clock (box-ref running-clock))
  (def outer-index (box-ref index-box))
  (box-set! running-clock (Tuple (monotonic-ms) (duration->timer-ms within)))
  (box-set! index-box (map-empty))
  (def outer-output (box-ref output-box))
  (def port (open-output-string))
  (box-set! output-box (if (record-ref t capture) (Some (Tuple port (string-cursor-start ""))) None))
  (match (try (with-deadline within
                (box-set! holder (Some (current-scope)))
                (if (record-ref t capture)
                    (parameterize ((current-output-port port)) (body))
                    (body))
                (unless (record-ref t open-ok)
                  (let ((left (scope-owned-count (current-scope))))
                    (unless (= left 0)
                      (test-failed! #"  it left ${left} resource(s) open")))))
         #:catch (System.Exception))
    ((Ok _)
     (when (overran? holder)
       (test-failed! #"  it ran past its deadline of ${within}: something it started never finished")))
    ((Err e) (test-failed! #"  it raised: ${(.-Message e)}")))
  (box-set! current-failures outer)
  (box-set! running-box outer-test)
  (box-set! running-clock outer-clock)
  (box-set! index-box outer-index)
  ;; What no `expect-output` took is printed, so that a test's prints are
  ;; not lost.
  (when (record-ref t capture)
    (match (take-test-output!)
      ((Some text) (unless (string-empty? text) (display text)))
      (None unit)))
  (box-set! output-box outer-output)
  (def failed (list-reverse (box-ref failures)))
  (def todo? (not (string-empty? (record-ref t todo))))
  (TestResult (test t)
              (status (cond ((and todo? (list-empty? failed)) Fixed)
                            (todo? Todo)
                            ((list-empty? failed) Passed)
                            (else Failed)))
              (failures failed)))

;; Whether a test is chosen by the filters: by a name that contains one of
;; them, or by there being none. `--quick` leaves out `#:slow` tests.
(: (-> TestOptions Test bool))
(defun (chosen? o t)
  (def filters (record-ref o filters))
  (and (not (and (record-ref o quick) (record-ref t slow)))
       (or (vec-empty? filters)
           (any? (fun (f) (string-contains? (record-ref t name) f)) filters))))

;; What extensions run when a run ends, oldest first: `(std test prop)` writes
;; its failure file, `(std test expect)` its promotions.
(: (Box (List (Tuple string (-> void)))))
(def run-end-hooks (make-box (list)))

(: (-> (Tuple string (-> void)) string))
(defun (hook-key h) (match h ((Tuple k _) k)))

;; Adds `hook` to what runs when `run-registered-tests` ends, once for each
;; `key`: a second hook with the same key is ignored.
(: (-> string (-> void) void))
(defun (at-run-end! key hook)
  (unless (any? (fun (h) (= (hook-key h) key)) (box-ref run-end-hooks))
    (box-set! run-end-hooks (cons (Tuple key hook) (box-ref run-end-hooks)))))

;; Text for an XML attribute or element.
(: (-> string string))
(defun (xml-escape s)
  (string-replace (string-replace (string-replace (string-replace s "&" "&amp;") "<" "&lt;")
                                  ">" "&gt;")
                  "\"" "&quot;"))

;; The results as JUnit XML, the format CI servers read.
(: (-> string (List TestResult) long string))
(defun (junit-xml module results ms)
  (def failed (list-length (list-filter (fun (r) (match (record-ref r status) (Failed #t) (_ #f))) results)))
  (def skipped (list-length (list-filter (fun (r) (match (record-ref r status) ((or Skipped Todo) #t) (_ #f))) results)))
  (def cases
    (list-map
      (fun (r)
        (def t (record-ref r test))
        (def head #"    <testcase name=\"${(xml-escape (record-ref t name))}\" classname=\"${(xml-escape module)}\" file=\"${(xml-escape (record-ref t file))}\" line=\"${(record-ref t line)}\"")
        (match (record-ref r status)
          (Failed
           (str head ">\n      <failure message=\"failed\">"
                (xml-escape (string-join (list->vec (record-ref r failures)) "\n"))
                "</failure>\n    </testcase>"))
          (Skipped (str head ">\n      <skipped message=\"" (xml-escape (record-ref t skip)) "\"/>\n    </testcase>"))
          (Todo (str head ">\n      <skipped message=\"todo: " (xml-escape (record-ref t todo)) "\"/>\n    </testcase>"))
          (_ (str head "/>"))))
      results))
  (str "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
       "<testsuites>\n"
       "  <testsuite name=\"" (xml-escape module) "\" tests=\"" (int->string (list-length results))
       "\" failures=\"" (int->string failed) "\" skipped=\"" (int->string skipped) "\" time=\""
       (double->string (/ (cast System.Double ms) 1000.0)) "\">\n"
       (string-join (list->vec cases) "\n")
       (if (list-empty? cases) "" "\n")
       "  </testsuite>\n</testsuites>\n"))

(: (-> TestStatus TestResult bool))
(defun (status-is? s r)
  (match (Tuple s (record-ref r status))
    ((Tuple Passed Passed) #t)
    ((Tuple Failed Failed) #t)
    ((Tuple Skipped Skipped) #t)
    ((Tuple Todo Todo) #t)
    ((Tuple Fixed Fixed) #t)
    (_ #f)))

;; The last line of a run.
(: (-> (List TestResult) long string))
(defun (summary-line results ms)
  (def total (list-length results))
  (def failed (list-length (list-filter (fun (r) (status-is? Failed r)) results)))
  (def skipped (list-length (list-filter (fun (r) (status-is? Skipped r)) results)))
  (def todo (list-length (list-filter (fun (r) (or (status-is? Todo r) (status-is? Fixed r))) results)))
  (def extra
    (str (if (> skipped 0) #", ${skipped} skipped" "")
         (if (> todo 0) #", ${todo} todo" "")))
  (if (= failed 0)
      #"${(- total (+ skipped todo))} test(s) passed${extra} in ${ms} ms."
      #"${failed} of ${total} test(s) failed${extra} in ${ms} ms."))

;; Runs the tests `module` registered that the arguments choose, prints each
;; failure as it happens and one line at the end, and answers the exit code: 0
;; when every test passed, 1 when one failed, 2 when the arguments are wrong.
(: (-> string (Vec string) int))
(defun (run-registered-tests module args)
  (match (parse-test-args args)
    ((Err why) (println #"test: ${why}") 2)
    ((Ok o) (run-with-options module o))))

(: (-> string TestOptions int))
(defun (run-with-options module o)
  (box-set! options-box o)
  (def started (monotonic-ms))
  (def tests (list-filter (fun (t) (chosen? o t)) (module-tests module)))
  (def results
    (loop (:for t tests)
          (:let result (run-test t))
          (:do (match (record-ref result status)
                 (Failed
                  (println #"FAIL ${(record-ref t name)} (${(shown-file (record-ref t file))}:${(record-ref t line)})")
                  (loop (:for f (record-ref result failures)) (:do (println f))))
                 (Fixed
                  (println #"FIXED ${(record-ref t name)} (${(shown-file (record-ref t file))}:${(record-ref t line)}): it passes, so its #:todo can go"))
                 (_ unit)))
          (:acc out (listing result))))
  (loop (:for h (list-reverse (box-ref run-end-hooks)))
        (:do (match h ((Tuple _ hook) (hook)))))
  (def ms (- (monotonic-ms) started))
  (match (record-ref o junit)
    ((Some path) (file-write-text path (junit-xml module results ms)))
    (None unit))
  (def failed (list-length (list-filter (fun (r) (status-is? Failed r)) results)))
  (if (list-empty? tests)
      (println (if (vec-empty? (record-ref o filters))
                   "There are no tests."
                   "No test's name contains any of the names given."))
      (println (summary-line results ms)))
  (if (= failed 0) 0 1))

;; `(run-tests)` — the file's `main`: runs every test this module registered.
;; A module that imports another with tests does not run those.
(def/macro (run-tests form inject compare)
  (syntax-match form
    ((_)
     ;; `main` is injected: a template's own names are renamed, and the
     ;; program's entry point has to be the name the compiler looks for.
     #'(begin
         (: ,(inject 'main) (-> (Vec string) int))
         (defun (,(inject 'main) args) (run-registered-tests (current-module) args))))
    (bad (syntax-error bad "run-tests takes nothing: (run-tests) at the end of a test file."))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/test/expect.bjo — output compared with a text in the test.
;;
;;     (deftest "printing a card"
;;       (print-value (Card (rank 12) (suit Hearts)))
;;       (expect-output """
;;         (Card (rank 12) (suit Hearts))
;;         """))
;;
;; A test whose body names `expect-output` writes its output to a string port
;; (see `take-test-output!` in `(std test)`). Each `expect-output` compares
;; what was written since the test started, or since the previous one, with
;; its text. With `--promote`, a text that differs is not a failure: the run
;; writes the output into the test file in its place, when it ends.

(export expect-output
        ;; Named by the template of the macro above, so it has to be
        ;; reachable where that expands.
        expect-output-check!)

(import (std prelude))
(import (std syntax-match))
(import (std test))

(:doc #:file "expect.bjodoc")

;; ---------------------------------------------------------------------------
;; Comparing
;; ---------------------------------------------------------------------------

;; Text as it is compared: its lines, without a carriage return or other
;; whitespace at their ends, and without empty lines at its start and end. A
;; raw string drops the line breaks next to its quotes, so this keeps a
;; printed final newline from being a difference.
(: (-> string (Vec string)))
(defun (compared-lines text)
  (def lines (vec->list (vec-map string-trim-right (string-split text "\n"))))
  (defun (drop-empty xs)
    (match xs
      ((Cons "" more) (drop-empty more))
      (_ xs)))
  (list->vec (list-reverse (drop-empty (list-reverse (drop-empty lines))))))

;; A line diff of two texts: a line only in `wanted` starts with -, one only
;; in `got` with +, and one in both with a space. The longest common
;; subsequence decides which lines are in both.
(: (-> (Vec string) (Vec string) (List string)))
(defun (diff-lines wanted got)
  (def n (vec-length wanted))
  (def m (vec-length got))
  (if (> (* n m) 1000000)
      ;; Too large to diff in reasonable time: both, in full.
      (list-append (list-map (fun (l) (str "-" l)) (vec->list wanted))
                   (list-map (fun (l) (str "+" l)) (vec->list got)))
      (let ((table (make-array (* (+ n 1) (+ m 1)))))
        (defun (at i j) (array-ref table (+ (* i (+ m 1)) j)))
        ;; table[i][j] is the length of the common subsequence of the lines
        ;; from i on and from j on.
        (loop (:for back-i (range 0 (+ n 1)))
              (:let i (- n back-i))
              (:do (loop (:for back-j (range 0 (+ m 1)))
                         (:let j (- m back-j))
                         (:do (array-set! table (+ (* i (+ m 1)) j)
                                          (cond
                                            ((or (= i n) (= j m)) 0)
                                            ((= (vec-ref wanted i) (vec-ref got j)) (+ 1 (at (+ i 1) (+ j 1))))
                                            (else (max (at (+ i 1) j) (at i (+ j 1))))))))))
        (let go ((i 0) (j 0) (out (list)))
          (cond
            ((and (= i n) (= j m)) (list-reverse out))
            ((= i n) (go i (+ j 1) (cons (str "+" (vec-ref got j)) out)))
            ((= j m) (go (+ i 1) j (cons (str "-" (vec-ref wanted i)) out)))
            ((= (vec-ref wanted i) (vec-ref got j)) (go (+ i 1) (+ j 1) (cons (str " " (vec-ref wanted i)) out)))
            ((>= (at (+ i 1) j) (at i (+ j 1))) (go (+ i 1) j (cons (str "-" (vec-ref wanted i)) out)))
            (else (go i (+ j 1) (cons (str "+" (vec-ref got j)) out))))))))

;; One `expect-output` whose text differed: where it is, the text it had when
;; the test ran, and what the test wrote instead. The old text is how the
;; promoter finds the form, and how it knows the file has not changed since.
(type (: Promotion (Record (: line int) (: wanted string) (: got string))))

;; The output a run promotes, by file, newest first.
(: (Box (List (Tuple string Promotion))))
(def promotions (make-box (list)))

(: (-> string string))
(defun (shown-name path)
  (match (path-filename path)
    ((Some name) name)
    (None path)))

;; What `expect-output` expands to.
(: (-> string int string void))
(defun (expect-output-check! file line wanted)
  (def where #"${(shown-name file)}:${line}")
  (match (take-test-output!)
    (None
     (test-failed! #"  ${where}: expect-output has no output to read. It reads the output of a test whose deftest body names it."))
    ((Some got)
     (def w (compared-lines wanted))
     (def g (compared-lines got))
     (unless (= w g)
       (if (record-ref (current-test-options) promote)
           (begin
             (box-set! promotions (cons (Tuple file (Promotion (line line) (wanted wanted) (got got)))
                                        (box-ref promotions)))
             (at-run-end! "std test expect: promote" promote-all!))
           (test-failed!
             (string-join
               (list->vec (cons #"  ${where}: the output is not the expected text (- expected, + written; --promote updates the text):"
                                (list-map (fun (l) (str "    " l)) (diff-lines w g))))
               "\n")))))))

;; `(expect-output text)` — the output since the test started, or since the
;; previous `expect-output`, is the text.
(def/macro (expect-output form inject compare)
  (syntax-match form
    ((_ text)
     #'(expect-output-check! ,(SStr (syntax-file form)) ,(SInt (int->string (syntax-line form))) ,text))
    (bad (syntax-error bad "expect-output takes a string: (expect-output \"\"\" ... \"\"\")."))))

;; ---------------------------------------------------------------------------
;; Promoting
;; ---------------------------------------------------------------------------

;; The index in `cs` of the first occurrence of `needle` at or after `from`.
(: (-> (Vec char) (Vec char) int (Option int)))
(defun (vec-search cs needle from)
  (def n (vec-length cs))
  (def k (vec-length needle))
  (let go ((i from))
    (cond
      ((> (+ i k) n) None)
      ((loop (:for j (range 0 k)) (:acc ok (folding #t (and ok (char=? (vec-ref cs (+ i j)) (vec-ref needle j))))))
       (Some i))
      (else (go (+ i 1))))))

;; The index where line `line` (from 1) starts.
(: (-> (Vec char) int int))
(defun (line-start cs line)
  (let go ((i 0) (l 1))
    (cond
      ((= l line) i)
      ((>= i (vec-length cs)) i)
      ((char=? (vec-ref cs i) #\newline) (go (+ i 1) (+ l 1)))
      (else (go (+ i 1) l)))))

;; How many quotes stand in a row from i.
(: (-> (Vec char) int int))
(defun (quote-run cs i)
  (let go ((j i))
    (if (and (< j (vec-length cs)) (char=? (vec-ref cs j) #\")) (go (+ j 1)) (- j i))))

;; Where the string literal that starts at i ends: the index after its
;; closing quotes. A run of three or more quotes is a raw string, closed by a
;; run as long.
(: (-> (Vec char) int (Option int)))
(defun (literal-end cs i)
  (def q (quote-run cs i))
  (def n (vec-length cs))
  (cond
    ((= q 0) None)
    ((= q 2) (Some (+ i 2)))
    ((= q 1)
     (let go ((j (+ i 1)))
       (cond
         ((>= j n) None)
         ((char=? (vec-ref cs j) #\\) (go (+ j 2)))
         ((char=? (vec-ref cs j) #\") (Some (+ j 1)))
         (else (go (+ j 1))))))
    (else
     (let go ((j (+ i q)))
       (cond
         ((>= j n) None)
         ((>= (quote-run cs j) q) (Some (+ j q)))
         (else (go (+ j 1))))))))

;; The longest run of quotes in a text.
(: (-> string int))
(defun (longest-quote-run text)
  (def cs (string->vec text))
  (let go ((i 0) (best 0))
    (if (>= i (vec-length cs))
        best
        (let ((q (quote-run cs i)))
          (if (> q 0) (go (+ i q) (max q best)) (go (+ i 1) best))))))

(: (-> string int string))
(defun (repeat-string s n)
  (string-join (list->vec (loop (:for _ (range 0 n)) (:acc out (listing s)))) ""))

;; The text as a literal for the test file. One line without a quote, a
;; backslash or a tab is a plain string. Anything else is a raw string whose
;; lines start at `margin` columns, with the closing quotes on a line of their
;; own at the margin; its quotes are one more than the longest run in the
;; text, and at least three.
(: (-> string int string))
(defun (raw-literal text margin)
  (def lines (compared-lines text))
  (cond
    ((vec-empty? lines) "\"\"")
    ((and (= (vec-length lines) 1)
          (not (string-contains? (vec-ref lines 0) "\""))
          (not (string-contains? (vec-ref lines 0) "\\"))
          (not (string-contains? (vec-ref lines 0) "\t")))
     (str "\"" (vec-ref lines 0) "\""))
    (else
      (let* ((q (repeat-string "\"" (max 3 (+ 1 (longest-quote-run text)))))
             (pad (repeat-string " " margin)))
        (str q "\n"
             (string-join (vec-map (fun (l) (if (string-empty? l) "" (str pad l))) lines) "\n")
             "\n" pad q)))))

;; What the string literal from i to end holds: a raw string without its
;; margin, or an escaped string with its escapes undone.
(: (-> (Vec char) int int string))
(defun (literal-text cs i end)
  (def q (quote-run cs i))
  (defun (chars from to) (list->string (vec->list (vec-slice cs from (- to from)))))
  (cond
    ((= q 2) "")
    ((= q 1) (unescape (chars (+ i 1) (- end 1))))
    (else
     (def inside (chars (+ i q) (- end q)))
     (if (string-starts-with? inside "\n")
         ;; Over several lines: the opening quotes end their line, and the
         ;; closing ones stand at the margin on a line of their own.
         (let* ((lines (vec->list (string-split inside "\n")))
                (body (list-tail lines))
                (margin (string-length (list-head (list-reverse body))))
                (text (list-reverse (list-tail (list-reverse body)))))
           (string-join (list->vec (list-map (fun (l) (drop-chars l margin)) text)) "\n"))
         inside))))

;; The string without its first n characters, or empty when it is shorter.
(: (-> string int string))
(defun (drop-chars s n)
  (let go ((left (string->list s)) (k n))
    (match left
      ((Cons _ more) #:when (> k 0) (go more (- k 1)))
      (_ (list->string left)))))

;; An escaped string's text: \n, \t, \r, \0, \\ and \" undone. Other escapes
;; are kept as written, which is enough to compare texts.
(: (-> string string))
(defun (unescape s)
  (let go ((left (string->list s)) (out (list)))
    (match left
      (Nil (list->string (list-reverse out)))
      ((Cons #\\ (Cons c more))
       (go more (cons (case c
                        (#\n #\newline)
                        (#\t #\tab)
                        (#\r #\return)
                        (#\0 (int->char 0))
                        (else c))
                      out)))
      ((Cons c more) (go more (cons c out))))))

;; The places on line `line` (from 1) where an `(expect-output` starts.
(: (-> (Vec char) int (List int)))
(defun (forms-on-line cs line)
  (def head (string->vec "(expect-output"))
  (def start (line-start cs line))
  (def stop (line-start cs (+ line 1)))
  (let go ((from start) (out (list)))
    (match (vec-search cs head from)
      ((Some at) (if (< at stop) (go (+ at 1) (cons at out)) (list-reverse out)))
      (None (list-reverse out)))))

;; Where the `expect-output` on `line` that still holds `wanted` is: the
;; start of the form, and the start and end of its literal. None when no form
;; there holds it, because the file changed since the test was compiled.
(: (-> (Vec char) int string (Option (Tuple int (Tuple int int)))))
(defun (find-form cs line wanted)
  (def head-length (string-length "(expect-output"))
  (loop (:for at (forms-on-line cs line))
        (:acc found
              (folding None
                       (match found
                         ((Some _) found)
                         (None
                          (match (literal-after cs (+ at head-length))
                            (None None)
                            ((Some (Tuple i end))
                             (if (= (compared-lines (literal-text cs i end)) (compared-lines wanted))
                                 (Some (Tuple at (Tuple i end)))
                                 None)))))))))

;; The text with the literal from i to end replaced by `got`, at the margin of
;; a form that starts at `at`.
(: (-> (Vec char) int int int int string string))
(defun (replace-literal cs line at i end got)
  (str (list->string (vec->list (vec-slice cs 0 i)))
       (raw-literal got (+ (- at (line-start cs line)) 2))
       (list->string (vec->list (vec-slice cs end (- (vec-length cs) end))))))

;; Where the literal after i starts and ends, past whitespace.
(: (-> (Vec char) int (Option (Tuple int int))))
(defun (literal-after cs i)
  (cond
    ((>= i (vec-length cs)) None)
    ((char-whitespace? (vec-ref cs i)) (literal-after cs (+ i 1)))
    (else
     (option-map (fun (end) (Tuple i end)) (literal-end cs i)))))

;; Writes every promotion into its file, from the last line to the first so
;; that a replacement does not move the lines of those still to come. Forms on
;; one line go from the right, for the same reason: a raw string over several
;; lines moves what follows it on its line.
(: (-> void))
(defun (promote-all!)
  (def by-file
    (list-foldl (fun (p m)
                  (match p
                    ((Tuple file entry) (map-set m file (cons entry (map-ref-or m file (list)))))))
                (map-empty)
                (box-ref promotions)))
  (map-for-each
    (fun (file entries)
      (match (promote-file (file-read-text file) file (list-sort-desc entries))
        ((Tuple new-text n)
         (when (> n 0)
           (file-write-text file new-text)
           (println #"promote: wrote ${n} text(s) into ${(shown-name file)}")))))
    by-file)
  (box-set! promotions (list)))

;; The text with the promotions written into it, and how many were. The
;; promotions come by line, the last first.
(: (-> string string (List Promotion) (Tuple string int)))
(defun (promote-file text file sorted)
  (match sorted
    (Nil (Tuple text 0))
    ((Cons p _)
     (def line (record-ref p line))
     (def here (list-filter (fun (q) (= (record-ref q line) line)) sorted))
     (def rest (list-filter (fun (q) (not (= (record-ref q line) line))) sorted))
     (def cs (string->vec text))
     ;; Each form found on the line, from the right.
     (def found
       (loop (:for q here)
             (:acc out (listing (Tuple q (find-form cs line (record-ref q wanted)))))))
     (loop (:for f found)
           (:do (match f
                  ((Tuple q None)
                   (println #"promote: the expect-output at ${(shown-name file)}:${line} no longer holds the text the test ran with, so it was left alone"))
                  (_ unit))))
     (def placed (sort-by-start-desc
                   (loop (:for f found)
                         (:when-let (Tuple q (Some place)) f)
                         (:acc out (listing (Tuple q place))))))
     (def new-text
       (list-foldl (fun (f t)
                     (match f
                       ((Tuple q (Tuple at (Tuple i end)))
                        (replace-literal (string->vec t) line at i end (record-ref q got)))))
                   text
                   placed))
     (match (promote-file new-text file rest)
       ((Tuple final n) (Tuple final (+ n (list-length placed))))))))

(: (-> (List (Tuple Promotion (Tuple int (Tuple int int))))
       (List (Tuple Promotion (Tuple int (Tuple int int))))))
(defun (sort-by-start-desc xs)
  (list-foldl insert-by-start (list) xs))

(: (-> (Tuple Promotion (Tuple int (Tuple int int)))
       (List (Tuple Promotion (Tuple int (Tuple int int))))
       (List (Tuple Promotion (Tuple int (Tuple int int))))))
(defun (insert-by-start e sorted)
  (match sorted
    (Nil (list e))
    ((Cons f more)
     (if (>= (start-of e) (start-of f)) (cons e sorted) (cons f (insert-by-start e more))))))

(: (-> (Tuple Promotion (Tuple int (Tuple int int))) int))
(defun (start-of e) (match e ((Tuple _ (Tuple at _)) at)))

;; The promotions by line, the last first. A form promoted twice, at the same
;; line with the same old text, keeps the first.
(: (-> (List Promotion) (List Promotion)))
(defun (list-sort-desc xs)
  (list-foldl insert-promotion (list) (list-reverse xs)))

(: (-> Promotion (List Promotion) (List Promotion)))
(defun (insert-promotion e sorted)
  (match sorted
    (Nil (list e))
    ((Cons f more)
     (cond
       ((and (= (record-ref e line) (record-ref f line)) (= (record-ref e wanted) (record-ref f wanted))) sorted)
       ((>= (record-ref e line) (record-ref f line)) (cons e sorted))
       (else (cons f (insert-promotion e more)))))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/test/gen.bjo — generators of test values.
;;
;; A generator does not read a RandomSource. It asks a `Choices` for numbers
;; with `draw`, and the `Choices` either takes them from a source and records
;; them, or replays a recorded vector. `(std test prop)` shrinks a failing
;; case by making the recorded vector smaller and replaying it, so no
;; generator needs a shrinker of its own.
;;
;; That works only when a smaller number gives a simpler value. Every
;; generator here keeps to that rule: 0 is the simplest choice at every draw.
;; A replayed vector that runs out answers 0 for every further draw, so a
;; shorter vector gives a simpler value too.

(export
  ;; The recorder
  Choices recording-choices replaying-choices
  draw choices-size choices-draws choices-discarded? discard!
  ;; Generators, and running one
  Gen make-gen gen-run gen-sample gen-samples
  ;; Combinators
  gen-pure gen-map gen-bind gen-let gen-such-that gen-one-of gen-frequency
  gen-element gen-sized gen-resize gen-recursive
  ;; Numbers
  gen-bool gen-int gen-int-range gen-long gen-long-range gen-ulong gen-double
  gen-bigint
  ;; Text
  gen-char gen-ascii-char gen-string gen-ascii-string gen-string-of
  ;; Shapes
  gen-unit gen-option gen-result gen-tuple2 gen-tuple3 gen-tuple4
  ;; Sequences
  gen-list gen-list-of-length gen-vec gen-array gen-shuffle
  ;; Maps and sets
  gen-map-of gen-hashset gen-orderedmap gen-orderedset
  ;; Defaults by type
  Arbitrary arbitrary)

(import (std prelude))
(import (std random))
(import (std syntax-match))
(import (std hashset))
(import (std orderedset))
(import (std orderedmap))

(import/class
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

(:doc #:file "gen.bjodoc")

;; ---------------------------------------------------------------------------
;; The recorder
;; ---------------------------------------------------------------------------

;; `source` is the source a recording draws from, `None` when replaying.
;; `replay` is the vector a replay reads, and `pos` how far it has read.
;; `recorded` holds every answer given, in both modes: after a replay it is
;; the part of the vector that was used, and the zeros answered after the
;; vector ran out. That is what a shrinker keeps.
(type
  (: Choices #:opaque
     (Record (: source (Option RandomSource))
             (: replay (Vec ulong))
             (: pos int #:mutable)
             (: recorded (VecBuilder ulong))
             (: size int #:mutable)
             (: discarded bool #:mutable))))

;; A generator that draws more than this many numbers is stopped, and its run
;; is discarded. A generator that never stops drawing would otherwise run
;; until the test's deadline.
(: int)
(def max-draws 10000)

(: (-> RandomSource int Choices))
(defun (recording-choices rng size)
  (Choices (source (Some rng)) (replay (vec-empty)) (pos 0)
           (recorded (vecbuilder-empty)) (size size) (discarded #f)))

(: (-> (Vec ulong) int Choices))
(defun (replaying-choices draws size)
  (Choices (source None) (replay draws) (pos 0)
           (recorded (vecbuilder-empty)) (size size) (discarded #f)))

(: (-> Choices int))
(defun (choices-size c) (record-ref c size))

(: (-> Choices (Vec ulong)))
(defun (choices-draws c) (vecbuilder->vec (record-ref c recorded)))

(: (-> Choices bool))
(defun (choices-discarded? c) (record-ref c discarded))

;; Marks the run as discarded and leaves it. The exception is only the way
;; out: whoever runs the generator looks at `choices-discarded?`, not at the
;; exception.
(: (-> Choices string %a))
(defun (discard! c why)
  (record-set! c (discarded #t))
  (raise (cast Exception (ArgumentException. (str "discarded: " why)))))

;; A number in [0, bound). A bound of 0 means every ulong.
(: (-> Choices ulong ulong))
(defun (draw c bound) (draw/forced c bound None))

;; `draw`, except that a recording records `forced` instead of drawing, when
;; it is given. A replay reads the vector as `draw` does. A generator uses it
;; to record a value it chose itself in the draws another way to that value
;; would make, so that a replay, and so shrinking, sees no difference.
(: (-> Choices ulong (Option ulong) ulong))
(defun (draw/forced c bound forced)
  (def recorded (record-ref c recorded))
  (when (>= (vecbuilder-length recorded) max-draws)
    (discard! c "the generator drew too many numbers"))
  (def raw
    (match (record-ref c source)
      ((Some rng)
       (match forced
         ((Some v) v)
         (None (random-u64 #:rng rng))))
      (None
       (def replay (record-ref c replay))
       (def pos (record-ref c pos))
       (record-set! c (pos (+ pos 1)))
       (if (< pos (vec-length replay)) (vec-ref replay pos) 0UL))))
  (def value (if (= bound 0UL) raw (% raw bound)))
  (vecbuilder-add! recorded value)
  value)

;; A number in [0, n) from the source of a recording, not recorded. Only a
;; choice that a forced draw records may be made this way. A replay answers 0.
(: (-> Choices int int))
(defun (unrecorded-below c n)
  (match (record-ref c source)
    ((Some rng) (cast System.Int32 (% (random-u64 #:rng rng) (cast System.UInt64 n))))
    (None 0)))

;; A number in [0, n), for an n above 0.
(: (-> Choices int int))
(defun (draw-below c n)
  (cast System.Int32 (draw c (cast System.UInt64 n))))

;; ---------------------------------------------------------------------------
;; Generators
;; ---------------------------------------------------------------------------

(type (: (Gen %a) #:opaque (Record (: run (-> Choices %a)))))

(: (-> (-> Choices %a) (Gen %a)))
(defun (make-gen f) (Gen (run f)))

(: (-> (Gen %a) Choices %a))
(defun (gen-run g c) ((record-ref g run) c))

;; One value, drawn from a fresh source unless `rng` is given.
(: (-> (Gen %a) (#:size int) (#:rng RandomSource) %a))
(defun (gen-sample g #:size 30 #:rng (make-random))
  (gen-run g (recording-choices rng size)))

;; `n` values, at sizes that grow from 1 to `size`, as a property's runs do.
(: (-> (Gen %a) int (#:size int) (#:rng RandomSource) (List %a)))
(defun (gen-samples g n #:size 100 #:rng (make-random))
  (loop (:for i (range 0 n))
        (:acc out (listing (gen-run g (recording-choices (random-split rng)
                                                         (size-of-run i n size)))))))

;; The size of run i of n: 1 at the first run, `top` at the last.
(: (-> int int int int))
(defun (size-of-run i n top)
  (if (<= n 1) top (+ 1 (/ (* i (- top 1)) (- n 1)))))

;; ---------------------------------------------------------------------------
;; Combinators
;; ---------------------------------------------------------------------------

(: (-> %a (Gen %a)))
(defun (gen-pure x) (make-gen (fun (c) x)))

(: (-> (-> %a %b) (Gen %a) (Gen %b)))
(defun (gen-map f g) (make-gen (fun (c) (f (gen-run g c)))))

(: (-> (Gen %a) (-> %a (Gen %b)) (Gen %b)))
(defun (gen-bind g f) (make-gen (fun (c) (gen-run (f (gen-run g c)) c))))

;; `(gen-let ((name gen) ...) body ...)` — a generator that draws each
;; binding in order, as `let*` binds, and answers the body. A later generator
;; may use an earlier value. `Gen` is not a `Monad`: `bind`'s callback may be a
;; bjoroutine, and a generator runs inside an ordinary function.
(def/macro (gen-let form inject compare)
  (syntax-match form
    ((_ ((name g) ...) first body ...)
     #'(make-gen
         (fun (choices)
           (let* ,(SList (list-map (fun (pair)
                                     (syntax-match pair
                                       ((n x) #'(,n (gen-run ,x choices)))
                                       (bad (syntax-error bad "a binding is (name generator)."))))
                                   (match form
                                     ((SList (Cons _ (Cons (SList bindings) _))) bindings)
                                     (_ (list)))))
             ,first
             ,@body))))
    (bad (syntax-error bad "gen-let takes bindings and a body: (gen-let ((x gen-int) (y (gen-int-range 0 x))) (Tuple x y))."))))

;; Retries until `pred` holds. After 100 tries the run is discarded, since a
;; generator that rarely satisfies `pred` tests little.
(: (-> (-> %a bool) (Gen %a) (Gen %a)))
(defun (gen-such-that pred g)
  (make-gen
    (fun (c)
      (let go ((tries 0))
        (if (= tries 100)
            (discard! c "gen-such-that found no value in 100 tries")
            (let ((x (gen-run g c)))
              (if (pred x) x (go (+ tries 1)))))))))

;; One of the generators. The first is the simplest.
(: (-> (List (Gen %a)) (Gen %a)))
(defun (gen-one-of gens)
  (when (list-empty? gens)
    (raise (cast Exception (ArgumentException. "gen-one-of was given no generators."))))
  (def v (list->vec gens))
  (make-gen (fun (c) (gen-run (vec-ref v (draw-below c (vec-length v))) c))))

;; One of the generators, by weight. The first is the simplest.
(: (-> (List (Tuple int (Gen %a))) (Gen %a)))
(defun (gen-frequency choices)
  (def total (loop (:for (Tuple w _) choices) (:acc n (summing w))))
  (when (any? (fun (p) (match p ((Tuple w _) (< w 0)))) choices)
    (raise (cast Exception (ArgumentException. "gen-frequency was given a weight below 0."))))
  (when (<= total 0)
    (raise (cast Exception (ArgumentException. "gen-frequency needs a weight above 0."))))
  (make-gen
    (fun (c)
      (let go ((left choices) (n (draw-below c total)))
        (match left
          ((Cons (Tuple w g) more) (if (< n w) (gen-run g c) (go more (- n w))))
          ;; Not reached: the draw is below the total of the weights, and
          ;; none of them is below 0.
          (Nil (raise (cast Exception (ArgumentException. "gen-frequency ran out of weights.")))))))))

;; One of the values. The first is the simplest.
(: (-> (List %a) (Gen %a)))
(defun (gen-element xs)
  (when (list-empty? xs)
    (raise (cast Exception (ArgumentException. "gen-element was given no values."))))
  (def v (list->vec xs))
  (make-gen (fun (c) (vec-ref v (draw-below c (vec-length v))))))

(: (-> (-> int (Gen %a)) (Gen %a)))
(defun (gen-sized f) (make-gen (fun (c) (gen-run (f (choices-size c)) c))))

;; `g` with the size `n`. The size goes back to what it was afterwards.
(: (-> int (Gen %a) (Gen %a)))
(defun (gen-resize n g)
  (make-gen
    (fun (c)
      (def outer (choices-size c))
      (record-set! c (size n))
      (def x (gen-run g c))
      (record-set! c (size outer))
      x)))

;; A generator of a recursive type. At size 0 it is `base`. Above, a draw
;; chooses `base` or `(step self)`, where `self` is this generator at half the
;; size. 0 chooses `base`, so a case shrinks toward it.
(: (-> (Gen %a) (-> (Gen %a) (Gen %a)) (Gen %a)))
(defun (gen-recursive base step)
  (make-gen
    (fun (c)
      (def n (choices-size c))
      (if (or (<= n 0) (= (draw c 4UL) 0UL))
          (gen-run base c)
          (gen-run (step (gen-resize (/ n 2) (gen-recursive base step))) c)))))

;; ---------------------------------------------------------------------------
;; Numbers
;; ---------------------------------------------------------------------------

(: (Gen bool))
(def gen-bool (make-gen (fun (c) (= (draw c 2UL) 1UL))))

;; A long in [lo, hi]. 0 gives the value nearest 0: lo when the range is all
;; above 0, hi when it is all below, and 0 itself otherwise. A range across 0
;; draws the sign first, positive at 0, then the distance from 0.
(: (-> Choices long long long))
(defun (draw-long-range c lo hi) (draw-long-range/forced c lo hi None))

;; The draw that records `target`, as `f` turns it into a draw.
(: (-> (Option long) (-> long long) (Option ulong)))
(defun (forced-by target f)
  (option-map (fun (t) (cast System.UInt64 (f t))) target))

;; `draw-long-range`, recording `target` when given: the draws it records are
;; those that give `target` when replayed.
(: (-> Choices long long (Option long) long))
(defun (draw-long-range/forced c lo hi target)
  (defun (forced f) (forced-by target f))
  (cond
    ((> lo hi)
     (raise (cast Exception (ArgumentException. #"An empty range: ${lo} to ${hi}."))))
    ((>= lo 0L)
     ;; hi - lo + 1 as a ulong, which is 0, meaning every ulong, only when
     ;; the range is all of them.
     (+ lo (cast System.Int64 (draw/forced c (+ (cast System.UInt64 (- hi lo)) 1UL)
                                           (forced (fun (t) (- t lo)))))))
    ((<= hi 0L)
     (- hi (cast System.Int64 (draw/forced c (+ (cast System.UInt64 (- hi lo)) 1UL)
                                           (forced (fun (t) (- hi t)))))))
    ((= (draw/forced c 2UL (forced (fun (t) (if (>= t 0L) 0L 1L)))) 0UL)
     (cast System.Int64 (draw/forced c (+ (cast System.UInt64 hi) 1UL) (forced (fun (t) t)))))
    (else
     ;; -(lo + 1) does not overflow, even at long-min.
     (- -1L (cast System.Int64 (draw/forced c (+ (cast System.UInt64 (- -1L lo)) 1UL)
                                            (forced (fun (t) (- -1L t)))))))))

(: (-> long long (Gen long)))
(defun (gen-long-range lo hi)
  (when (> lo hi)
    (raise (cast Exception (ArgumentException. #"gen-long-range: ${lo} is above ${hi}."))))
  (make-gen (fun (c) (draw-long-range c lo hi))))

(: (-> int int (Gen int)))
(defun (gen-int-range lo hi)
  (when (> lo hi)
    (raise (cast Exception (ArgumentException. #"gen-int-range: ${lo} is above ${hi}."))))
  (make-gen (fun (c) (cast System.Int32 (draw-long-range c (cast System.Int64 lo) (cast System.Int64 hi))))))

;; The distance from 0 that a value of size n reaches: n * n, so a size of 100
;; reaches 10000. A size at or below 0 reaches 0.
(: (-> int long))
(defun (reach n)
  (if (<= n 0) 0L (* (cast System.Int64 n) (cast System.Int64 n))))

;; An edge value: one of `edges`, drawn with `pad` more draws of 0 after it.
;; An edge has to cost as many draws as the longest other way to a value, or
;; shrinking, which prefers fewer draws, would turn a value such as 1000 into
;; an edge such as int-max.
(: (-> Choices (Vec %a) int %a))
(defun (draw-edge c edges pad)
  (def x (vec-ref edges (draw-below c (vec-length edges))))
  (loop (:for _ (range 0 pad)) (:do (ignore (draw c 1UL))))
  x)

;; A sized whole number, with the edges of its type now and then. The first
;; draw chooses: 0 to 7 a value within the size's reach, 8 any value of the
;; type, 9 an edge. An edge is recorded as kind 8 would record it, so the two
;; replay alike: shrinking turns the 9 into an 8 and keeps the value, and can
;; then make the value smaller.
(: (-> Choices long long (List long) long))
(defun (draw-sized-long c lo hi edges)
  (def kind (draw c 10UL))
  (cond
    ((< kind 8UL)
     (def r (reach (choices-size c)))
     (draw-long-range c (if (< (- 0L r) lo) lo (- 0L r)) (if (> r hi) hi r)))
    ((= kind 8UL) (draw-long-range c lo hi))
    (else
     (def v (list->vec edges))
     (draw-long-range/forced c lo hi (Some (vec-ref v (unrecorded-below c (vec-length v))))))))

(: (Gen long))
(def gen-long
  (make-gen
    (fun (c)
      (draw-sized-long c -9223372036854775808L 9223372036854775807L
                       (list 0L 1L -1L 9223372036854775807L -9223372036854775808L
                             2147483647L -2147483648L 4294967296L)))))

(: (Gen int))
(def gen-int
  (make-gen
    (fun (c)
      (cast System.Int32
            (draw-sized-long c -2147483648L 2147483647L
                             (list 0L 1L -1L 2147483647L -2147483648L 255L 256L 65535L))))))

;; Any ulong. 0 to 7 within the size's reach, 8 any, 9 an edge.
(: (Gen ulong))
(def gen-ulong
  (make-gen
    (fun (c)
      (def kind (draw c 10UL))
      (cond
        ((< kind 8UL) (draw c (+ (cast System.UInt64 (reach (choices-size c))) 1UL)))
        ((= kind 8UL) (draw c 0UL))
        (else
         ;; Recorded as kind 8 records a value: see draw-sized-long.
         (def v (list->vec (list 0UL 1UL 18446744073709551615UL 9223372036854775808UL 4294967296UL)))
         (draw/forced c 0UL (Some (vec-ref v (unrecorded-below c (vec-length v))))))))))

;; A bigint: a long most of the time, and a product of longs beyond the range
;; of a long otherwise.
(: (Gen bigint))
(def gen-bigint
  (make-gen
    (fun (c)
      (def low (cast System.Numerics.BigInteger (gen-run gen-long c)))
      (if (= (draw c 4UL) 3UL)
          (* low (+ (cast System.Numerics.BigInteger (gen-run gen-long c)) 9223372036854775808I))
          low))))

;; A double. Most draws give a whole part within the size's reach and a
;; fraction; 0 gives 0.0. An edge now and then: 0.0, ±1, the largest and
;; smallest values, the smallest above 0. `#:special` adds NaN and the
;; infinities to the edges.
(: (-> (#:special bool) (Gen double)))
(defun (gen-double #:special #f)
  (def edges
    (list->vec
      (list-append (list 0.0 1.0 -1.0 1.7976931348623157e308 -1.7976931348623157e308 5e-324 -0.0)
                   (if special (list (/ 0.0 0.0) (/ 1.0 0.0) (/ -1.0 0.0)) (list)))))
  (make-gen
    (fun (c)
      (def kind (draw c 10UL))
      (if (= kind 9UL)
          ;; Padded to the four draws of the other way: sign, distance and
          ;; fraction after the kind.
          (draw-edge c edges 2)
          (let* ((r (reach (choices-size c)))
                 (whole (draw-long-range c (- 0L r) r))
                 ;; 53 bits, so every fraction is a double exactly.
                 (frac (/ (cast System.Double (draw c 9007199254740992UL)) 9007199254740992.0)))
            (if (< whole 0L)
                (- (cast System.Double whole) frac)
                (+ (cast System.Double whole) frac)))))))

;; ---------------------------------------------------------------------------
;; Text
;; ---------------------------------------------------------------------------

;; Printable ASCII, from #\a: the 95 characters from space to ~, turned so
;; that 0 is a letter, the character a case most often shrinks to.
(: (-> Choices char))
(defun (draw-ascii c)
  (def d (cast System.Int32 (draw c 95UL)))
  (int->char (+ 32 (% (+ d 65) 95))))

(: (Gen char))
(def gen-ascii-char (make-gen draw-ascii))

;; Characters that find bugs in text code: the empty character, a combining
;; accent, a character outside the BMP (two UTF-16 units), a right-to-left
;; letter, the replacement character, a non-breaking space, a final sigma,
;; and a capital I with a dot, whose lower case is two characters.
(: (Vec char))
(def odd-chars
  (list->vec (list (int->char 0) (int->char 0x301) (int->char 0x1F600) (int->char 0x5D0)
                   (int->char 0xFFFD) (int->char 0xA0) (int->char 0x3C2) (int->char 0x130))))

;; Any Unicode scalar value. 0 to 7 printable ASCII, 8 one of the odd
;; characters above, 9 any scalar value.
(: (Gen char))
(def gen-char
  (make-gen
    (fun (c)
      (def kind (draw c 10UL))
      (cond
        ((< kind 8UL) (draw-ascii c))
        ((= kind 8UL) (vec-ref odd-chars (draw-below c (vec-length odd-chars))))
        (else
         ;; Every scalar value: 0 to 0x10FFFF without the surrogates.
         (def n (draw-below c 1112064))
         (int->char (if (< n 0xD800) n (+ n 2048))))))))

(: (-> (Gen char) (Gen string)))
(defun (gen-string-of g) (gen-map list->string (gen-list g)))

(: (Gen string))
(def gen-string (gen-string-of gen-char))

(: (Gen string))
(def gen-ascii-string (gen-string-of gen-ascii-char))

;; ---------------------------------------------------------------------------
;; Shapes
;; ---------------------------------------------------------------------------

(: (Gen void))
(def gen-unit (make-gen (fun (c) unit)))

;; None at 0, and a quarter of the time.
(: (-> (Gen %a) (Gen (Option %a))))
(defun (gen-option g)
  (make-gen (fun (c) (if (= (draw c 4UL) 0UL) None (Some (gen-run g c))))))

;; Ok at 0, and half of the time.
(: (-> (Gen %e) (Gen %a) (Gen (Result %e %a))))
(defun (gen-result ge ga)
  (make-gen (fun (c) (if (= (draw c 2UL) 0UL) (Ok (gen-run ga c)) (Err (gen-run ge c))))))

(: (-> (Gen %a) (Gen %b) (Gen (Tuple %a %b))))
(defun (gen-tuple2 ga gb)
  (make-gen (fun (c) (let* ((a (gen-run ga c)) (b (gen-run gb c))) (Tuple a b)))))

(: (-> (Gen %a) (Gen %b) (Gen %c) (Gen (Tuple %a %b %c))))
(defun (gen-tuple3 ga gb gc)
  (make-gen (fun (c) (let* ((a (gen-run ga c)) (b (gen-run gb c)) (x (gen-run gc c)))
                       (Tuple a b x)))))

(: (-> (Gen %a) (Gen %b) (Gen %c) (Gen %d) (Gen (Tuple %a %b %c %d))))
(defun (gen-tuple4 ga gb gc gd)
  (make-gen (fun (c) (let* ((a (gen-run ga c)) (b (gen-run gb c))
                            (x (gen-run gc c)) (d (gen-run gd c)))
                       (Tuple a b x d)))))

;; ---------------------------------------------------------------------------
;; Sequences
;; ---------------------------------------------------------------------------

;; The longest list a generator makes, whatever the size.
(: int)
(def max-length 1000)

;; A list whose length grows with the size: 1 + size / 5 on average. Before
;; each element a draw says whether there is one more, and 0 says no. So
;; deleting draws makes the list shorter, and setting one to 0 ends it there.
(: (-> (Gen %a) (Gen (List %a))))
(defun (gen-list g)
  (make-gen
    (fun (c)
      (def mean (cast System.UInt64 (+ 1 (/ (choices-size c) 5))))
      (def out (listbuilder-empty))
      (let go ((n 0))
        (if (and (< n max-length) (> (draw c (+ mean 1UL)) 0UL))
            (begin
              (listbuilder-add! out (gen-run g c))
              (go (+ n 1)))
            (listbuilder->list out))))))

(: (-> int (Gen %a) (Gen (List %a))))
(defun (gen-list-of-length n g)
  (make-gen
    (fun (c)
      (loop (:for _ (range 0 n))
            (:acc out (listing (gen-run g c)))))))

(: (-> (Gen %a) (Gen (Vec %a))))
(defun (gen-vec g) (gen-map list->vec (gen-list g)))

(: (-> (Gen %a) (Gen (Array %a))))
(defun (gen-array g)
  (gen-map (fun (xs)
             (def v (list->vec xs))
             (def a (make-array (vec-length v)))
             (loop (:with i 0 (+ i 1))
                   (:for x v)
                   (:do (array-set! a i x)))
             a)
           (gen-list g)))

;; A permutation of `xs`. Fisher-Yates from the front: element i swaps with
;; one at or after it, and a draw of 0 leaves it where it is. So the order
;; shrinks toward the order `xs` has.
(: (-> (List %a) (Gen (List %a))))
(defun (gen-shuffle xs)
  (make-gen
    (fun (c)
      (def v (list->vec xs))
      (def n (vec-length v))
      (def a (make-array n))
      (loop (:with i 0 (+ i 1))
            (:for x v)
            (:do (array-set! a i x)))
      (loop (:for i (range 0 n))
            (:do (let* ((j (+ i (draw-below c (- n i))))
                        (x (array-ref a i)))
                   (array-set! a i (array-ref a j))
                   (array-set! a j x))))
      (vec->list (array->vec a)))))

;; ---------------------------------------------------------------------------
;; Maps and sets
;; ---------------------------------------------------------------------------

;; Each is made from a list of entries, so it shrinks as the list does. A
;; later entry wins over an earlier one with the same key.

(: (-> (Gen %k) (Gen %v) (Gen (Map %k %v))) (where (Eq %k)))
(defun (gen-map-of gk gv) (gen-map list->map (gen-list (gen-tuple2 gk gv))))

(: (-> (Gen %a) (Gen (HashSet %a))) (where (Eq %a)))
(defun (gen-hashset g) (gen-map list->hashset (gen-list g)))

(: (-> (Gen %k) (Gen %v) (Gen (OrderedMap %k %v))) (where (Ord %k)))
(defun (gen-orderedmap gk gv) (gen-map list->orderedmap (gen-list (gen-tuple2 gk gv))))

(: (-> (Gen %a) (Gen (OrderedSet %a))) (where (Ord %a)))
(defun (gen-orderedset g) (gen-map list->orderedset (gen-list g)))

;; ---------------------------------------------------------------------------
;; Defaults by type
;; ---------------------------------------------------------------------------

;; The generator a type has when a property does not name one. An instance is
;; only a generator: shrinking needs nothing more.
(def/trait (Arbitrary %a)
  (: arbitrary (-> (Gen %a))))

(impl (Arbitrary bool) (defun (arbitrary) gen-bool))
(impl (Arbitrary int) (defun (arbitrary) gen-int))
(impl (Arbitrary long) (defun (arbitrary) gen-long))
(impl (Arbitrary ulong) (defun (arbitrary) gen-ulong))
(impl (Arbitrary bigint) (defun (arbitrary) gen-bigint))
(impl (Arbitrary double) (defun (arbitrary) (gen-double)))
(impl (Arbitrary char) (defun (arbitrary) gen-char))
(impl (Arbitrary string) (defun (arbitrary) gen-string))
(impl (Arbitrary void) (defun (arbitrary) gen-unit))

(impl (Arbitrary (Option %a)) (where (Arbitrary %a))
  (defun (arbitrary) (gen-option (arbitrary))))

(impl (Arbitrary (Result %e %a)) (where (Arbitrary %e) (Arbitrary %a))
  (defun (arbitrary) (gen-result (arbitrary) (arbitrary))))

(impl (Arbitrary (Tuple %a %b)) (where (Arbitrary %a) (Arbitrary %b))
  (defun (arbitrary) (gen-tuple2 (arbitrary) (arbitrary))))

(impl (Arbitrary (Tuple %a %b %c)) (where (Arbitrary %a) (Arbitrary %b) (Arbitrary %c))
  (defun (arbitrary) (gen-tuple3 (arbitrary) (arbitrary) (arbitrary))))

(impl (Arbitrary (List %a)) (where (Arbitrary %a))
  (defun (arbitrary) (gen-list (arbitrary))))

(impl (Arbitrary (Vec %a)) (where (Arbitrary %a))
  (defun (arbitrary) (gen-vec (arbitrary))))

(impl (Arbitrary (Map %k %v)) (where (Arbitrary %k) (Eq %k) (Arbitrary %v))
  (defun (arbitrary) (gen-map-of (arbitrary) (arbitrary))))

(impl (Arbitrary (HashSet %a)) (where (Arbitrary %a) (Eq %a))
  (defun (arbitrary) (gen-hashset (arbitrary))))

(impl (Arbitrary (OrderedMap %k %v)) (where (Arbitrary %k) (Ord %k) (Arbitrary %v))
  (defun (arbitrary) (gen-orderedmap (arbitrary) (arbitrary))))

(impl (Arbitrary (OrderedSet %a)) (where (Arbitrary %a) (Ord %a))
  (defun (arbitrary) (gen-orderedset (arbitrary))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/test/prop.bjo — property tests.
;;
;;     (deftest "reverse twice is the identity"
;;       (for-all ((: xs (List int)))
;;         (check (= (list-reverse (list-reverse xs)) xs))))
;;
;; `for-all` runs its body many times with generated values. When a run
;; fails, it shrinks the run: it makes the recorded draws of the generators
;; smaller and replays them, and keeps each change that still fails. What is
;; left is reported, and stored beside the test file in NAME.failed.bjodat,
;; where the next run of the test tries it first.

(export for-all label assume classify collect
        ;; Named by the templates of the macros above, so they have to be
        ;; reachable where those expand.
        run-for-all! with-label prop-draw prop-arbitrary)

(import (std prelude))
(import (std syntax-match))
(import (std random))
(import (std test))
(import (std test gen))
(import (text bjodat-core))

(import/class
  (InvalidOperationException (: System.InvalidOperationException (-> string InvalidOperationException))))

(:doc #:file "prop.bjodoc")


;; ---------------------------------------------------------------------------
;; Running one case
;; ---------------------------------------------------------------------------

;; What `for-all` writes: given the choices, it draws the bindings and answers
;; their values as text, for a report, and the body as a function.
(type (: Property (-> Choices (Tuple (List string) (-> void)))))

;; How one run came out. A failed run has its failures, as the test would
;; print them, the values of its bindings as text, and its signature: what
;; tells this failure apart from another (see `failure-signature`).
(type (: Outcome (Union Pass Discard (: Fail (List string) (List string) string))))

;; The choices of the run in progress, for `assume`. `None` outside a run.
(: (Box (Option Choices)))
(def choices-box (make-box None))

;; The labels of `classify` and `collect` in the run in progress, when the
;; run counts them. Only the random runs count them, not the replays of
;; shrinking.
(: (Box (Option (Box (List string)))))
(def labels-box (make-box None))

(: (-> (Gen %a) Choices %a))
(defun (prop-draw g c) (gen-run g c))

(: (-> Choices %a) (where (Arbitrary %a)))
(defun (prop-arbitrary c) (gen-run (arbitrary) c))

;; What a failure is, without the values it saw: the place of the first
;; check that failed, or the type of the exception. Shrinking keeps only cases
;; that fail as the first one did. A smaller case that fails some other way is
;; a different bug, and reporting it would hide the one that was found.
;;
;; A check's report starts with its place and source; a label puts its name
;; on a line above. A resource left open is one signature, whatever the count.
(: (-> (List string) string))
(defun (failure-signature failures)
  (match failures
    (Nil "")
    ((Cons first _)
     (def lines (vec->list (string-split first "\n")))
     (match lines
       ((Cons head (Cons next _))
        (if (string-starts-with? head "  in ") (str head "\n" next) head))
       ((Cons head Nil)
        (if (string-starts-with? head "  it left ") "a resource left open" head))
       (Nil "")))))

(: (-> Exception string))
(defun (exception-name e) (.-FullName (.GetType e)))

;; Runs the body of one case in a scope of its own: the scope waits for the
;; fibers the case started, so the next case does not share them. A case that
;; leaves a resource open fails, unless the test has `#:open-ok`.
(: (-> Property Choices bool Outcome))
(defun (run-case prop c open-ok)
  (def failures (make-box (list)))
  (def saved (box-ref choices-box))
  (box-set! choices-box (Some c))
  (def outcome
    (match (try (prop c) #:catch (System.Exception))
      ((Err e)
       (if (choices-discarded? c)
           Discard
           (Fail (list #"  the generator raised: ${(.-Message e)}") (list)
                 (str "the generator raised " (exception-name e)))))
      ((Ok (Tuple shown body))
       (match (try (with-scope
                     (failures-into failures body)
                     (unless open-ok
                       (let ((left (scope-owned-count (current-scope))))
                         (unless (= left 0)
                           (box-set! failures (cons #"  it left ${left} resource(s) open"
                                                    (box-ref failures)))))))
                   #:catch (System.Exception))
         ((Err e)
          (if (choices-discarded? c)
              Discard
              (Fail (list-reverse (cons #"  it raised: ${(.-Message e)}" (box-ref failures))) shown
                    (if (list-empty? (box-ref failures))
                        (str "it raised " (exception-name e))
                        (failure-signature (list-reverse (box-ref failures)))))))
         ((Ok _)
          (if (list-empty? (box-ref failures))
              Pass
              (Fail (list-reverse (box-ref failures)) shown
                    (failure-signature (list-reverse (box-ref failures))))))))))
  (box-set! choices-box saved)
  outcome)

;; ---------------------------------------------------------------------------
;; Shrinking
;; ---------------------------------------------------------------------------

;; The order shrinking makes a case smaller in: a shorter vector is smaller,
;; and of two as long, the one smaller at the first place they differ. Every
;; kept change makes the vector smaller in this order, so shrinking ends.
(: (-> (Vec ulong) (Vec ulong) bool))
(defun (shortlex< a b)
  (def n (vec-length a))
  (cond
    ((< n (vec-length b)) #t)
    ((> n (vec-length b)) #f)
    (else
     (let go ((i 0))
       (cond
         ((= i n) #f)
         ((< (vec-ref a i) (vec-ref b i)) #t)
         ((> (vec-ref a i) (vec-ref b i)) #f)
         (else (go (+ i 1))))))))

;; The state of one shrink: the smallest failing case so far and what it
;; failed with, and what the shrink has spent.
(type (: Shrink (Record (: prop Property)
                        (: size int)
                        (: open-ok bool)
                        (: signature string)
                        (: best (Vec ulong) #:mutable)
                        (: failures (List string) #:mutable)
                        (: shown (List string) #:mutable)
                        (: replays int #:mutable)
                        (: shrinks int #:mutable)
                        (: limit int)
                        (: reserve long)
                        (: cut-short bool #:mutable))))

;; Whether the shrink may replay once more: within its limit, and with more
;; than a fifth of the test's deadline left.
(: (-> Shrink bool))
(defun (may-replay? s)
  (def ok (and (< (record-ref s replays) (record-ref s limit))
               (> (test-time-left) (record-ref s reserve))))
  (unless ok (record-set! s (cut-short #t)))
  ok)

;; Replays `candidate`, and keeps it when it fails as the first case did and
;; is smaller than the best so far. What is kept is the part of the vector the
;; replay used.
(: (-> Shrink (Vec ulong) bool))
(defun (try! s candidate)
  (cond
    ((not (shortlex< candidate (record-ref s best))) #f)
    ((not (may-replay? s)) #f)
    (else
     (record-set! s (replays (+ (record-ref s replays) 1)))
     (def c (replaying-choices candidate (record-ref s size)))
     (match (run-case (record-ref s prop) c (record-ref s open-ok))
       ((Fail failures shown signature)
        (def used (choices-draws c))
        (if (and (= signature (record-ref s signature)) (shortlex< used (record-ref s best)))
            (begin
              (record-set! s (best used) (failures failures) (shown shown)
                           (shrinks (+ (record-ref s shrinks) 1)))
              #t)
            #f))
       (_ #f)))))

(: (-> (Vec ulong) int int (Vec ulong)))
(defun (vec-without v i k)
  (loop (:for j (range 0 (vec-length v)))
        (:when (or (< j i) (>= j (+ i k))))
        (:acc out (vecing (vec-ref v j)))))

(: (-> (Vec ulong) int int (Vec ulong)))
(defun (vec-zeroed v i k)
  (loop (:for j (range 0 (vec-length v)))
        (:acc out (vecing (if (and (>= j i) (< j (+ i k))) 0UL (vec-ref v j))))))

(: (-> (Vec ulong) int ulong (Vec ulong)))
(defun (vec-with v i x) (vec-set v i x))

(: (-> (Vec ulong) int (Vec ulong)))
(defun (vec-swapped v i)
  (vec-set (vec-set v i (vec-ref v (+ i 1))) (+ i 1) (vec-ref v i)))

;; Deletes blocks of k draws, from every place.
(: (-> Shrink int bool))
(defun (delete-blocks! s k)
  (let go ((i 0) (any #f))
    (if (> (+ i k) (vec-length (record-ref s best)))
        any
        (if (try! s (vec-without (record-ref s best) i k))
            (go i #t)
            (go (+ i 1) any)))))

;; Sets blocks of k draws to 0, from every place.
(: (-> Shrink int bool))
(defun (zero-blocks! s k)
  (let go ((i 0) (any #f))
    (if (> (+ i k) (vec-length (record-ref s best)))
        any
        (go (+ i 1) (or (try! s (vec-zeroed (record-ref s best) i k)) any)))))

;; Makes each draw smaller: 0 if that fails, else the smallest value that
;; fails, found by halving the distance between a value that passed and one
;; that failed. A property need not fail for every value above the smallest,
;; so this finds a small failing value, not always the smallest.
(: (-> Shrink bool))
(defun (minimize-draws! s)
  (let go ((i 0) (any #f))
    (if (>= i (vec-length (record-ref s best)))
        any
        (let ((v (vec-ref (record-ref s best) i)))
          (cond
            ((= v 0UL) (go (+ i 1) any))
            ((try! s (vec-with (record-ref s best) i 0UL)) (go (+ i 1) #t))
            (else (go (+ i 1) (or (search-down! s i 0UL v) any))))))))

;; Draw i fails at `hi` and passed at `lo`. Tries the middle until the two
;; meet, and answers whether it kept a smaller value.
(: (-> Shrink int ulong ulong bool))
(defun (search-down! s i lo hi)
  (if (or (<= (- hi lo) 1UL) (>= i (vec-length (record-ref s best))))
      #f
      (let ((mid (+ lo (/ (- hi lo) 2UL))))
        (if (try! s (vec-with (record-ref s best) i mid))
            (begin (ignore (search-down! s i lo mid)) #t)
            (search-down! s i mid hi)))))

;; The draws near i that a step pairs with draw i: up to 8 on each side.
(: (-> int int (List int)))
(defun (neighbours i n)
  (loop (:for j (range (if (< i 8) 0 (- i 8)) (if (> (+ i 9) n) n (+ i 9))))
        (:when (not (= j i)))
        (:acc out (listing j))))

;; Deletes a block of k draws and adds their sum to a draw near it. A list
;; whose elements must add up to something loses an element this way, which
;; deleting alone cannot do: the others have to grow to make up for it.
(: (-> Shrink int bool))
(defun (delete-and-shift! s k)
  (let go ((i 0) (any #f))
    (def best (record-ref s best))
    (if (> (+ i k) (vec-length best))
        any
        (let* ((removed (loop (:for j (range i (+ i k))) (:acc t (summing (vec-ref best j)))))
               (rest (vec-without best i k))
               (kept (any? (fun (j) (try! s (vec-with rest j (+ (vec-ref rest j) removed))))
                           (neighbours (if (< i (vec-length rest)) i (- (vec-length rest) 1))
                                       (vec-length rest)))))
          (if kept (go i #t) (go (+ i 1) any))))))

;; Moves value from a draw to a later one near it: the earlier draw gets
;; smaller, which makes the vector smaller, and the later one makes up for
;; it. Tries moving all of the earlier draw, then half of it.
(: (-> Shrink bool))
(defun (redistribute! s)
  (let go ((i 0) (any #f))
    (def best (record-ref s best))
    (if (>= i (vec-length best))
        any
        (let* ((v (vec-ref best i))
               (moved (and (> v 0UL)
                           (any? (fun (j)
                                   (and (> j i)
                                        (let ((b (record-ref s best)))
                                          (or (try! s (vec-with (vec-with b i 0UL) j (+ (vec-ref b j) v)))
                                              (try! s (vec-with (vec-with b i (- v (/ v 2UL))) j
                                                                (+ (vec-ref b j) (/ v 2UL))))))))
                                 (neighbours i (vec-length best))))))
          (go (+ i 1) (or moved any))))))

;; Swaps adjacent draws that are out of order.
(: (-> Shrink bool))
(defun (sort-pairs! s)
  (let go ((i 0) (any #f))
    (if (>= (+ i 1) (vec-length (record-ref s best)))
        any
        (let ((b (record-ref s best)))
          (if (and (> (vec-ref b i) (vec-ref b (+ i 1))) (try! s (vec-swapped b i)))
              (go (+ i 1) #t)
              (go (+ i 1) any))))))

;; Shrinks until a whole round of the steps changes nothing, or the shrink is
;; out of replays or time.
(: (-> Shrink void))
(defun (shrink! s)
  (let round ()
    (def deleted
      (loop (:for k (list 8 4 2 1))
            (:acc any (folding #f (let ((c (delete-blocks! s k))) (or c any))))))
    (def zeroed
      (loop (:for k (list 8 4 2 1))
            (:acc any (folding #f (let ((c (zero-blocks! s k))) (or c any))))))
    (def minimized (minimize-draws! s))
    (def sorted (sort-pairs! s))
    ;; The two steps that move value between draws cost many replays each,
    ;; so they run only when the cheap steps above found nothing.
    (def moved
      (if (or deleted zeroed minimized sorted)
          #f
          (let* ((a (delete-and-shift! s 2))
                 (b (delete-and-shift! s 1))
                 (c (redistribute! s)))
            (or a b c))))
    (def changed (or deleted zeroed minimized sorted moved))
    (when (and changed (not (record-ref s cut-short)))
      (round))))

;; ---------------------------------------------------------------------------
;; The failure file
;; ---------------------------------------------------------------------------

;; A stored case: which `for-all` it is for, the seed of the run that found
;; it, for a person to read, the size the draws were made at, and the draws.
;; In the file, a case is
;;
;;     (case (test "name") (bindings "((: xs (List int)))") (index 0)
;;           (seed "123") (size 37) (draws [1 0 5]))
;;
;; A `for-all` is found by its test, the source text of its bindings, and its
;; position among the `for-all`s of the test with the same bindings. Editing
;; the body keeps the case, since a case is most useful while the bug is being
;; fixed. Changing the generators drops it, since the draws no longer mean
;; what they meant. A `for-all` added to the test does not take over the cases
;; of another, unless its bindings are written the same.
;;
;; with a draw above long-max written as a ulong, 18446744073709551615UL, and
;; the others as plain numbers.
(type (: StoredCase (Record (: test string)
                            (: bindings string)
                            (: index int)
                            (: seed string)
                            (: size int)
                            (: draws (Vec ulong)))))

(: (-> string Bjodat Bjodat))
(defun (clause name value) (BjoList (list (BjoSym (string->symbol name)) value)))

(: (-> StoredCase Bjodat))
(defun (case->bjodat c)
  (BjoList
    (list (BjoSym (string->symbol "case"))
          (clause "test" (BjoStr (record-ref c test)))
          (clause "bindings" (BjoStr (record-ref c bindings)))
          (clause "index" (BjoInt (cast System.Int64 (record-ref c index))))
          (clause "seed" (BjoStr (record-ref c seed)))
          (clause "size" (BjoInt (cast System.Int64 (record-ref c size))))
          (clause "draws" (BjoVec (vec-map draw->bjodat (record-ref c draws)))))))

(: (-> ulong Bjodat))
(defun (draw->bjodat d)
  (if (<= d 9223372036854775807UL) (BjoInt (cast System.Int64 d)) (BjoULong d)))

;; The value of the clause `(name value)` in a form's clauses.
(: (-> string (List Bjodat) (Option Bjodat)))
(defun (clause-value name clauses)
  (match clauses
    (Nil None)
    ((Cons (BjoList (Cons (BjoSym s) (Cons v Nil))) more)
     (if (= (symbol->string s) name) (Some v) (clause-value name more)))
    ((Cons _ more) (clause-value name more))))

(: (-> Bjodat (Option StoredCase)))
(defun (bjodat->case d)
  (match d
    ((BjoList (Cons (BjoSym head) clauses))
     (match (Tuple (clause-value "test" clauses) (clause-value "bindings" clauses)
                   (clause-value "index" clauses) (clause-value "seed" clauses)
                   (clause-value "size" clauses) (clause-value "draws" clauses))
       ((Tuple (Some (BjoStr test)) (Some (BjoStr bindings)) (Some (BjoInt index)) (Some (BjoStr seed))
               (Some (BjoInt size)) (Some (BjoVec draws)))
        (if (= (symbol->string head) "case")
            (Some (StoredCase (test test) (bindings bindings) (index (cast System.Int32 index)) (seed seed)
                              (size (cast System.Int32 size))
                              (draws (vec-map (fun (x) (match x
                                                         ((BjoInt n) (cast System.UInt64 n))
                                                         ((BjoULong n) n)
                                                         (_ 0UL)))
                                              draws))))
            None))
       (_ None)))
    (_ None)))

;; The cases of a failure file: `(failed case ...)`. What does not read as a
;; case is left out.
(: (-> string (List StoredCase)))
(defun (parse-stored text)
  (match (parse-bjodat text)
    ((Ok (BjoList (Cons (BjoSym head) cases)))
     (if (= (symbol->string head) "failed")
         (loop (:for c cases)
               (:when-let (Some sc) (bjodat->case c))
               (:acc out (listing sc)))
         (list)))
    (_ (list))))

;; The cases of each file that was read, by its path: the text it had, and the
;; cases now.
(: (Box (Map string (Tuple string (List StoredCase)))))
(def store-box (make-box (map-empty)))

;; The failure file of a test file: beside it, NAME.failed.bjodat.
(: (-> string string))
(defun (store-path test-file)
  (if (string-ends-with? test-file ".bjo")
      (str (substring/slice-prefix test-file) ".failed.bjodat")
      (str test-file ".failed.bjodat")))

(: (-> string string))
(defun (substring/slice-prefix path)
  (list->string (list-reverse (list-tail (list-tail (list-tail (list-tail (list-reverse (string->list path)))))))))

(: (-> (List StoredCase) string))
(defun (stored-text cases)
  (if (list-empty? cases)
      ""
      (str "(failed\n"
           (string-join (list->vec (list-map (fun (c) (str "  " (bjodat->string (case->bjodat c)))) cases)) "\n")
           ")\n")))

;; The cases of a file, read on first use. A file that does not read is
;; treated as empty, and written over when the run changes it.
(: (-> string (List StoredCase)))
(defun (stored-cases path)
  (match (map-try-ref (box-ref store-box) path)
    ((Some (Tuple _ cases)) cases)
    (None
     (def text (if (file-exists? path) (file-read-text path) ""))
     (def cases (parse-stored text))
     (box-set! store-box (map-set (box-ref store-box) path (Tuple text cases)))
     (at-run-end! "std test prop: store" write-stores!)
     cases)))

(: (-> string (List StoredCase) void))
(defun (set-stored-cases! path cases)
  (ignore (stored-cases path))
  (match (map-try-ref (box-ref store-box) path)
    ((Some (Tuple text _)) (box-set! store-box (map-set (box-ref store-box) path (Tuple text cases))))
    (None unit)))

(: (-> string string int StoredCase bool))
(defun (case-for? test bindings index c)
  (and (= (record-ref c test) test)
       (= (record-ref c bindings) bindings)
       (= (record-ref c index) index)))

;; Writes each file whose cases changed, and deletes it when none is left. A
;; run without filters also drops the cases of tests that no longer exist.
(: (-> void))
(defun (write-stores!)
  (def o (current-test-options))
  (when (record-ref o store)
    (def names (list-map test-name (registered-tests)))
    (map-for-each
      (fun (path entry)
        (match entry
          ((Tuple text cases)
           (def kept
             (if (vec-empty? (record-ref o filters))
                 (list-filter (fun (c) (any? (fun (n) (= n (record-ref c test))) names)) cases)
                 cases))
           (def new-text (stored-text kept))
           (unless (= new-text text)
             (if (string-empty? new-text)
                 (when (file-exists? path) (file-delete path))
                 (file-write-text path new-text))))))
      (box-ref store-box))))

;; ---------------------------------------------------------------------------
;; for-all
;; ---------------------------------------------------------------------------

(: (-> (List string) (List string) (List string)))
(defun (shown-lines names shown)
  (match (Tuple names shown)
    ((Tuple (Cons n nm) (Cons v vm)) (cons #"    ${n} = ${v}" (shown-lines nm vm)))
    (_ (list))))

(: (-> string int string))
(defun (where-text file line)
  (def name (match (path-filename file) ((Some n) n) (None file)))
  #"${name}:${line}")

;; Shrinks a failing case and reports the smallest one found. Answers the
;; draws of that case, to store.
(: (-> Property int bool (List string) (Vec ulong) (List string) (List string)
       string string (Vec ulong)))
(defun (shrink-and-report! prop size open-ok names draws failures shown signature heading)
  (def o (current-test-options))
  (def limit (if (> (record-ref o shrink-limit) 0) (record-ref o shrink-limit) 2000))
  (def within (match (running-test) ((Some _) (test-time-left)) (None 86400000L)))
  (def s (Shrink (prop prop) (size size) (open-ok open-ok) (signature signature)
                 (best draws) (failures failures)
                 (shown shown) (replays 0) (shrinks 0) (limit limit)
                 (reserve (/ within 5L)) (cut-short #f)))
  (shrink! s)
  (def shrinks (record-ref s shrinks))
  (def how
    (cond
      ((record-ref s cut-short)
       #"  shrunk ${shrinks} time(s), then stopped, out of time or replays, at:")
      ((= shrinks 0) "  with:")
      (else #"  shrunk ${shrinks} time(s) to:")))
  (test-failed!
    (string-join
      (list->vec (list-append (cons heading (cons how (shown-lines names (record-ref s shown))))
                              (record-ref s failures)))
      "\n"))
  (record-ref s best))

(: (-> string int (Map string int) void))
(defun (print-stats where runs counts)
  (println #"for-all at ${where}: ${runs} run(s)")
  (loop (:for (Tuple label n) (list-sort-by-count (map->list counts)))
        (:do (println #"  ${(/ (* n 100) (if (= runs 0) 1 runs))}% ${label}"))))

(: (-> (List (Tuple string int)) (List (Tuple string int))))
(defun (list-sort-by-count xs)
  (defun (insert p sorted)
    (match sorted
      (Nil (list p))
      ((Cons q more)
       (if (> (count-of p) (count-of q)) (cons p sorted) (cons q (insert p more))))))
  (list-foldl insert (list) xs))

(: (-> (Tuple string int) int))
(defun (count-of p) (match p ((Tuple _ n) n)))

(: (-> (Map string int) (List string) (Map string int)))
(defun (count-labels counts labels)
  (list-foldl (fun (l m) (map-set m l (+ 1 (map-ref-or m l 0)))) counts labels))

;; Whether `sc` is one of the stored cases that still fail.
(: (-> (List (Tuple StoredCase (Tuple (List string) (Tuple (List string) string)))) StoredCase bool))
(defun (failing-case? failing sc)
  (any? (fun (f) (match f ((Tuple fc _) (eq? fc sc)))) failing))

;; What `for-all` expands to. `runs` is 0 when the form gives no `#:runs`.
(: (-> string int int string (List string) Property void))
(defun (run-for-all! file line runs bindings names prop)
  (def t
    (match (running-test)
      ((Some t) t)
      (None (raise (cast Exception (InvalidOperationException.
                                     #"for-all at ${(where-text file line)} runs only inside a deftest."))))))
  (def o (current-test-options))
  (def where (where-text file line))
  (def total (cond ((> runs 0) runs) ((> (record-ref o runs) 0) (record-ref o runs)) (else 100)))
  (def index (next-test-index! bindings))
  (def test (test-name t))
  (def open-ok (record-ref t open-ok))
  (def path (store-path (test-file t)))
  (def seed-text (displayed (record-ref o seed)))
  (def stored
    (if (record-ref o store)
        (list-filter (fun (c) (case-for? test bindings index c)) (stored-cases path))
        (list)))
  ;; Every stored case is tried first. Those that pass are dropped, and those
  ;; that still fail stay. The first that fails is shrunk again and reported,
  ;; and the random runs are skipped.
  (def failing
    (loop (:for c stored)
          (:let outcome (run-case prop (replaying-choices (record-ref c draws) (record-ref c size)) open-ok))
          (:when-let (Fail failures shown signature) outcome)
          (:acc out (listing (Tuple c (Tuple failures (Tuple shown signature)))))))
  (unless (= (list-length failing) (list-length stored))
    (set-stored-cases! path (list-filter (fun (sc) (or (not (case-for? test bindings index sc)) (failing-case? failing sc)))
                                         (stored-cases path))))
  (match failing
    ((Cons (Tuple c (Tuple failures (Tuple shown signature))) _)
     (def best (shrink-and-report! prop (record-ref c size) open-ok names (record-ref c draws) failures shown signature
                                   #"  for-all at ${where} failed on a case stored in ${(where-text path 1)} (seed ${(record-ref c seed)})"))
     ;; The case keeps its place in the file, so a run that changes nothing
     ;; writes nothing.
     (set-stored-cases! path (list-map (fun (sc) (if (eq? sc c) (record-set c (draws best)) sc))
                                       (stored-cases path))))
    (Nil (random-runs! t o where total bindings index names prop open-ok path seed-text))))

(: (-> Test TestOptions string int string int (List string) Property bool string string void))
(defun (random-runs! t o where total bindings index names prop open-ok path seed-text)
  (def root (make-random/seed64
              (mix-seed (test-seed t) (mix-seed (string-hash bindings) (cast System.UInt64 index)))))
  (def counts (make-box (map-empty)))
  (def labels (make-box (list)))
  (let go ((i 0) (ran 0) (discarded 0))
    (cond
      ((= ran total) (finish-stats o where ran (box-ref counts)))
      ((> discarded (* 10 total))
       (finish-stats o where ran (box-ref counts))
       (test-failed! #"  for-all at ${where} ran only ${ran} of ${total} run(s): ${discarded} were discarded, by assume or gen-such-that, so the property was hardly tested"))
      (else
       (def size (gen-size-of-run ran total))
       (def c (recording-choices (random-split root) size))
       (box-set! labels (list))
       (def saved-labels (box-ref labels-box))
       (box-set! labels-box (Some labels))
       (def outcome (run-case prop c open-ok))
       (box-set! labels-box saved-labels)
       (match outcome
         (Pass
          (box-set! counts (count-labels (box-ref counts) (box-ref labels)))
          (go (+ i 1) (+ ran 1) discarded))
         (Discard (go (+ i 1) ran (+ discarded 1)))
         ((Fail failures shown signature)
          (finish-stats o where (+ ran 1) (box-ref counts))
          (def draws (choices-draws c))
          (def heading #"  for-all at ${where} failed on run ${(+ ran 1)} of ${total} (seed ${seed-text})")
          ;; A case that passes when replayed depends on something other than
          ;; its draws, and shrinking it would chase noise.
          (match (run-case prop (replaying-choices draws size) open-ok)
            ((Fail _ _ _)
             (def best (shrink-and-report! prop size open-ok names draws failures shown signature heading))
             (when (record-ref o store)
               (set-stored-cases! path
                                  (list-append (stored-cases path)
                                               (list (StoredCase (test (test-name t)) (bindings bindings) (index index)
                                                                 (seed seed-text) (size size)
                                                                 (draws best)))))))
            (_
             (test-failed!
               (string-join
                 (list->vec (list-append (cons heading
                                               (cons "  and passed when run again with the same values, so it depends on something besides them:"
                                                     (shown-lines names shown)))
                                         failures))
                 "\n"))))))))))

(: (-> TestOptions string int (Map string int) void))
(defun (finish-stats o where ran counts)
  (when (and (record-ref o stats) (not (map-empty? counts)))
    (print-stats where ran counts)))

;; The size of run i of n: 1 at the first run, 100 at the last.
(: (-> int int int))
(defun (gen-size-of-run i n)
  (if (<= n 1) 100 (+ 1 (/ (* i 99) (- n 1)))))

;; ---------------------------------------------------------------------------
;; In a body
;; ---------------------------------------------------------------------------

;; `discard!` at void, which is what a statement position wants.
(: (-> Choices void))
(defun (discard-run! c) (discard! c "assume"))

;; Discards the run when `ok` is #f. A discarded run is not counted.
(: (-> bool void))
(defun (assume ok)
  (unless ok
    (match (box-ref choices-box)
      ((Some c) (discard-run! c))
      (None (raise (cast Exception (InvalidOperationException. "assume is for the body of a for-all.")))))))

;; Counts the run under `name` when `ok` is #t. `--stats` prints the counts.
(: (-> string bool void))
(defun (classify name ok)
  (when ok
    (match (box-ref labels-box)
      ((Some b) (box-set! b (cons name (box-ref b))))
      (None unit))))

;; Counts the run under the value as text. `--stats` prints the counts.
(: (-> %a void))
(defun (collect value) (classify (displayed value) #t))

;; Runs `body`, and puts `name` above each failure in it.
(: (-> string (-> void) void))
(defun (with-label name body)
  (def failures (make-box (list)))
  (match (try (failures-into failures body) #:catch (System.Exception))
    ((Ok _) unit)
    ((Err e) (box-set! failures (cons #"  it raised: ${(.-Message e)}" (box-ref failures)))))
  (loop (:for f (list-reverse (box-ref failures)))
        (:do (test-failed! #"  in ${name}:\n${f}"))))

;; `(label "name" body ...)` — a failure in the body is reported under the
;; name.
(def/macro (label form inject compare)
  (syntax-match form
    ((_ name first body ...) #'(with-label ,name (fun () ,first ,@body unit)))
    (bad (syntax-error bad "label takes a name and a body: (label \"after insert\" body ...)."))))

;; Whether a form is the bare `:` that a type declaration begins with.
(: (-> Syntax bool))
(defun (colon? s)
  (match s
    ((SList _) #f)
    (_ (= (syntax->string s) ":"))))

;; The definition of one binding of a `for-all`: `(name gen)`, or
;; `(: name Type)` for the type's `arbitrary`, declared as a `def` is.
(: (-> Syntax Syntax Syntax))
(defun (binding-def choices b)
  (match b
    ((SList (Cons colon (Cons name (Cons type Nil))))
     (cond
       ((not (colon? colon))
        (if (colon? name)
            (syntax-error b "A typed binding of for-all is declared as a def is: (: name Type), not (name : Type).")
            (syntax-error b "A binding of for-all is (name generator) or (: name Type).")))
       ;; The compiler would say only that a dictionary is missing.
       ((type-variable-in? type)
        (syntax-error type "for-all draws values of a type it knows, and this type has a type variable, such as %a, in it. Write a type such as (List int), or give a generator: (name gen)."))
       (else #'(def (: ,name ,type) (prop-arbitrary ,choices)))))
    ((SList (Cons name (Cons g Nil))) #'(def ,name (prop-draw ,g ,choices)))
    (_ (syntax-error b "A binding of for-all is (name generator) or (: name Type)."))))

;; Whether a type, as written, names a type variable: %a, alone or inside.
(: (-> Syntax bool))
(defun (type-variable-in? type)
  (match type
    ((SList items) (any? type-variable-in? items))
    (_ (let ((s (syntax->string type)))
         (or (string-starts-with? s "%") (string-starts-with? s "'"))))))

(: (-> Syntax Syntax))
(defun (binding-name b)
  (match b
    ((SList (Cons colon (Cons name (Cons _ Nil)))) (if (colon? colon) name colon))
    ((SList (Cons name _)) name)
    (_ b)))

(: (-> Syntax Syntax (List Syntax) (List Syntax) Syntax))
(defun (for-all-expansion form runs bindings body)
  (when (list-empty? bindings)
    (ignore (syntax-error form "for-all takes at least one binding: (for-all ((x gen-int)) body ...).")))
  (def names (list-map binding-name bindings))
  #'(run-for-all! ,(SStr (syntax-file form)) ,(SInt (int->string (syntax-line form))) ,runs
                  ,(SStr (syntax->string (SList bindings)))
                  (list ,@(list-map (fun (n) (SStr (syntax->string n))) names))
                  (fun (choices)
                    ,@(list-map (fun (b) (binding-def #'choices b)) bindings)
                    (Tuple (list ,@(list-map (fun (n) #'(displayed ,n)) names))
                           (fun () ,@body unit)))))

;; `(for-all ((name gen) ...) body ...)`, or with `#:runs n` before the
;; bindings. A binding is `(name generator)` or `(: name Type)`.
(def/macro (for-all form inject compare)
  (syntax-match form
    ((_ #:runs n (binding ...) first body ...)
     (for-all-expansion form n binding (cons first body)))
    ((_ (binding ...) first body ...)
     (for-all-expansion form #'0 binding (cons first body)))
    (bad (syntax-error bad "for-all takes bindings and a body: (for-all ((xs (gen-list gen-int))) (check ...)), optionally #:runs n before the bindings."))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/text.bjo — persistent text: a B+ tree of UTF-8 chunks. Inserting,
;; deleting, splitting and joining are O(log n) and share what they do not
;; change.
;;
;; - Positions are cursors, as in a string; no function takes or returns a
;;   byte offset. A cursor holds the byte offset internally, and a character
;;   index becomes a cursor in O(log n) through the character counts.
;; - A text of at most `chunk-max` bytes is one string and no tree.
;; - The tree's nodes are C# (`Bjolang.Runtime.TextTree`), because they hold their
;;   slots in inline arrays. Every operation is here, including those on one
;;   node.
;; - A node has up to 16 children and keeps each one's length in bytes and its
;;   number of newlines. The children of a leaves node (height 1) are chunks:
;;   strings of valid UTF-8 of at most `chunk-max` bytes. The children of a
;;   branch are nodes one level down.
;; - Every node other than a root has at least `min-kids` children. Chunks
;;   are kept at `chunk-min` bytes or more where an edit can merge them with a
;;   neighbour in the same node. This is not an invariant: reads do not
;;   depend on it.
;; - An edit copies the nodes on its path. A transient's owner lets it edit
;;   in place the nodes it copied. Owner 0 copies every node it edits.

(import (std prelude))
(import (std atomic))
(import (std unicode grapheme))

(:doc #:file "text.bjodoc")

(export
  Text TextWalk TextBuilder TransientText TextCursor TextBackward TextBackwardStep Texting TextingInto
  TextPiece textbuilder-add-piece!
  ;; Texts
  text-empty text string->text text->string text-length text-count text-empty? text-line-count
  text-substring text-split-at text-append text-insert text-insert-text text-delete text-replace-range
  ;; As a string has them
  text-take text-drop text-take-right text-drop-right
  text-contains? text-starts-with? text-ends-with?
  text-replace text-split text-split-once text-join
  text-search text-search-backward text-search-any text-find
  text-trim text-trim-left text-trim-right text-pad-left text-pad-right
  text-upcase text-downcase text-reverse
  text-fold text-fold-right text-for-each text->list text->vec
  ;; Lines
  text-line
  ;; Cursors
  text-cursor-start text-cursor-end text-cursor-start? text-cursor-end?
  text-cursor-ref text-cursor-next text-cursor-prev text-cursor-ref+next text-cursor-forward text-cursor-back
  text-cursor-at-line text-cursor-index text-cursor-line text-cursor-column text-cursor-line-start text-cursor-line-end
  text-cursor-search text-cursor-search-backward text-cursor-find text-cursor-find-backward
  text-cursor=? text-cursor<? text-cursor>? text-cursor<=? text-cursor>=?
  text-backward
  ;; Grapheme clusters
  text-cursor-next-grapheme text-cursor-prev-grapheme text-cursor-grapheme-boundary?
  text-grapheme-count text->graphemes
  ;; Chunks
  text-fold-chunks text-for-each-chunk text-chunk-count
  ;; Builders
  textbuilder-empty textbuilder-add-string! textbuilder-add! textbuilder-add-text! textbuilder-length textbuilder->text
  ;; Transients
  text->transienttext transienttext->text transienttext-insert! transienttext-delete!
  transienttext-replace-range! transienttext-append! transienttext-length transienttext-count
  ;; Checks, for tests
  text-check text-height)

(import/class
  (ArgumentException (: System.ArgumentException))
  (Utf8Builder (: BjoString.Utf8StringBuilder))
  (ArgumentOutOfRangeException (: System.ArgumentOutOfRangeException))
  (Exception (: System.Exception))
  (TextNode (: Bjolang.Runtime.TextTree.TextNode))
  (TextBranch (: Bjolang.Runtime.TextTree.TextBranch))
  (TextLeaves (: Bjolang.Runtime.TextTree.TextLeaves))
  (StringWalk (: Bjolang.Runtime.StringWalk)))

(import/class ((Lanes %a) (: System.Runtime.Intrinsics.Vector128)))
(import/class ((Bytes32 %a) (: System.Runtime.Intrinsics.Vector256)))

(import/extern
  (new-branch      (: Bjolang.Runtime.TextTree.TextNodes.NewBranch   (-> long TextBranch)))
  (new-leaves      (: Bjolang.Runtime.TextTree.TextNodes.NewLeaves   (-> long TextLeaves)))
  (copy-branch     (: Bjolang.Runtime.TextTree.TextNodes.CopyBranch  (-> TextBranch long TextBranch)))
  (copy-leaves     (: Bjolang.Runtime.TextTree.TextNodes.CopyLeaves  (-> TextLeaves long TextLeaves)))
  (as-branch       (: Bjolang.Runtime.TextTree.TextNodes.AsBranch    (-> TextNode TextBranch)))
  (as-leaves       (: Bjolang.Runtime.TextTree.TextNodes.AsLeaves    (-> TextNode TextLeaves)))
  (branch-node     (: Bjolang.Runtime.TextTree.TextNodes.BranchNode  (-> TextBranch TextNode)))
  (leaves-node     (: Bjolang.Runtime.TextTree.TextNodes.LeavesNode  (-> TextLeaves TextNode)))
  (node-owner      (: Bjolang.Runtime.TextTree.TextNodes.Owner       (-> TextNode long)))
  (node-count      (: Bjolang.Runtime.TextTree.TextNodes.Count       (-> TextNode int)))
  (set-count!      (: Bjolang.Runtime.TextTree.TextNodes.SetCount    (-> TextNode int void)))
  (slot-end        (: Bjolang.Runtime.TextTree.TextNodes.End         (-> TextNode int int)))
  (set-slot-end!   (: Bjolang.Runtime.TextTree.TextNodes.SetEnd      (-> TextNode int int void)))
  (slot-ends       (: Bjolang.Runtime.TextTree.TextNodes.Ends        (-> TextNode int (RoSpan int))))
  (slot-ends4      (: Bjolang.Runtime.TextTree.TextNodes.Ends4       (-> TextNode int (Lanes int))))
  (set-slot-ends4! (: Bjolang.Runtime.TextTree.TextNodes.SetEnds4    (-> TextNode int (Lanes int) void)))
  (slot-lines      (: Bjolang.Runtime.TextTree.TextNodes.Lines       (-> TextNode int int)))
  (set-slot-lines! (: Bjolang.Runtime.TextTree.TextNodes.SetLines    (-> TextNode int int void)))
  (slot-chars      (: Bjolang.Runtime.TextTree.TextNodes.Chars       (-> TextNode int int)))
  (set-slot-chars! (: Bjolang.Runtime.TextTree.TextNodes.SetChars    (-> TextNode int int void)))
  (kid             (: Bjolang.Runtime.TextTree.TextNodes.Kid         (-> TextBranch int TextNode)))
  (set-kid!        (: Bjolang.Runtime.TextTree.TextNodes.SetKid      (-> TextBranch int TextNode void)))
  (chunk-at        (: Bjolang.Runtime.TextTree.TextNodes.Chunk       (-> TextLeaves int string)))
  (set-chunk!      (: Bjolang.Runtime.TextTree.TextNodes.SetChunk    (-> TextLeaves int string void)))
  (string-walk         (: Bjolang.Runtime.StringWalks.Start   (-> string StringWalk)))
  (string-walk-done?   (: Bjolang.Runtime.StringWalks.Done    (-> StringWalk bool)))
  (string-walk-current (: Bjolang.Runtime.StringWalks.Current (-> StringWalk char)))
  (string-walk-next    (: Bjolang.Runtime.StringWalks.Next    (-> StringWalk StringWalk)))
  ;; One allocation of the exact size, for a chunk with text put in it.
  (concat2         (: BjoString.Utf8String.Concat (-> string string string)))
  (concat3         (: BjoString.Utf8String.Concat (-> string string string string)))
  (concat-all      (: BjoString.Utf8String.Concat (-> (RoSpan string) string)))
  ;; Checks the bytes and copies them.
  (utf8->string    (: BjoString.Utf8String.FromUtf8 (-> (RoSpan byte) string)))
  ;; Vectorized in .NET.
  (span-count      (: System.MemoryExtensions.Count             (-> (RoSpan byte) byte int)))
  (span-index-of   (: System.MemoryExtensions.IndexOf           (-> (RoSpan byte) byte int)))
  (span-find       (: System.MemoryExtensions.IndexOf           (-> (RoSpan byte) (RoSpan byte) int)))
  (span-equal?     (: System.MemoryExtensions.SequenceEqual     (-> (RoSpan byte) (RoSpan byte) bool)))
  (span-compare    (: System.MemoryExtensions.SequenceCompareTo (-> (RoSpan byte) (RoSpan byte) int)))
  (span-find-last  (: System.MemoryExtensions.LastIndexOf       (-> (RoSpan byte) (RoSpan byte) int)))
  (span-last-index-of (: System.MemoryExtensions.LastIndexOf    (-> (RoSpan byte) byte int)))
  (span-index-of-any (: System.MemoryExtensions.IndexOfAny      (-> (RoSpan byte) (RoSpan byte) int)))
  (span-first-outside (: System.MemoryExtensions.IndexOfAnyExceptInRange (-> (RoSpan int) int int int)))
  (pop-count32     (: System.Numerics.BitOperations.PopCount (-> uint int)))
  ;; 32 bytes at once.
  (bytes32-of      (: System.Runtime.Intrinsics.Vector256.Create (-> (RoSpan byte) (Bytes32 byte))))
  (bytes32-splat   (: System.Runtime.Intrinsics.Vector256.Create (-> byte (Bytes32 byte))))
  (bytes32-and     (: System.Runtime.Intrinsics.Vector256.BitwiseAnd (-> (Bytes32 byte) (Bytes32 byte) (Bytes32 byte))))
  (bytes32-eq      (: System.Runtime.Intrinsics.Vector256.Equals (-> (Bytes32 byte) (Bytes32 byte) (Bytes32 byte))))
  (bytes32-mask    (: System.Runtime.Intrinsics.Vector256.ExtractMostSignificantBits (-> (Bytes32 byte) uint)))
  ;; Four ints at once.
  (lanes-splat     (: System.Runtime.Intrinsics.Vector128.Create (-> int (Lanes int))))
  (lanes-of        (: System.Runtime.Intrinsics.Vector128.Create (-> int int int int (Lanes int))))
  (lanes-add       (: System.Runtime.Intrinsics.Vector128.Add (-> (Lanes int) (Lanes int) (Lanes int))))
  (lanes-and       (: System.Runtime.Intrinsics.Vector128.BitwiseAnd (-> (Lanes int) (Lanes int) (Lanes int))))
  (lanes-ge        (: System.Runtime.Intrinsics.Vector128.GreaterThanOrEqual (-> (Lanes int) (Lanes int) (Lanes int))))
  ;; The full 128-bit product, one mulx.
  (big-mul         (: System.Math.BigMul (-> ulong ulong uint128))))

(def cap 16)
(def min-kids 8)
(def chunk-max 1024)
(def chunk-min 256)
;; The longest text an edit puts into one chunk. A chunk that grows past
;; `chunk-max` then splits in two, each at most `chunk-max`.
(def small-insert 512)

(def newline (cast byte 10))

(: (-> string %a))
(defun (bad-argument why) (raise (cast Exception (ArgumentException. why))))

(: (-> string %a))
(defun (out-of-range why) (raise (cast Exception (ArgumentOutOfRangeException. "offset" why))))

;; ---------------------------------------------------------------------------
;; Chunks
;; ---------------------------------------------------------------------------

(: (-> string int))
(defun (newlines-in s) #:inline (span-count (.AsSpan s) newline))

;; The newlines in bytes `from` to `to` of `s`.
(: (-> string int int int))
(defun (newlines-between s from to)
  (if (= from to) 0 (span-count (ro-span-slice (.AsSpan s) #:from from #:to to) newline)))

;; The characters of `s`, counted with vectors.
(: (-> string int))
(defun (chars-in s) #:inline (if (<= (string-length s) 1) (string-length s) (.Count s)))

;; The characters in bytes `from` to `to` of `s`, boundaries of `s`.
;;
;; - Returns the bytes less those that continue a character (10xxxxxx),
;;   counted 32 at a time.
;; - Allocates nothing.
(: (-> string int int int))
(defun (chars-between s from to)
  (def sp (ro-span-slice (.AsSpan s) #:from from #:to to))
  (def len (- to from))
  (def blocks (* 32 (/ len 32)))
  (def top (bytes32-splat (cast byte 192)))
  (def cont (bytes32-splat (cast byte 128)))
  (def in-blocks
    (let go ((j 0) (n 0))
      (if (>= j blocks)
          n
          (begin
            (def v (bytes32-of (ro-span-slice sp #:from j #:to (+ j 32))))
            (go (+ j 32) (+ n (pop-count32 (bytes32-mask (bytes32-eq (bytes32-and v top) cont)))))))))
  (let step ((j blocks) (n in-blocks))
    (cond
      ((>= j len) (- len n))
      ((= (bitwise-and (cast int (ro-span-ref sp j)) 192) 128) (step (+ j 1) (+ n 1)))
      (else (step (+ j 1) n)))))

;; Whether byte `k` of `s` starts a character, or is its end.
(: (-> string int bool))
(defun (boundary? s k)
  (or (<= k 0) (>= k (string-length s))
      (not (= (bitwise-and (cast int (ro-span-ref (.AsSpan s) k)) 192) 128))))

;; The last character boundary at or before `k`.
(: (-> string int int))
(defun (boundary-before s k)
  (let go ((k k)) (if (boundary? s k) k (go (- k 1)))))

;; Bytes `from` to `to` of `s`, sharing its array.
(: (-> string int int string))
(defun (piece s from to) #:inline (.SliceBytes s from (- to from)))

;; ---------------------------------------------------------------------------
;; Nodes
;; ---------------------------------------------------------------------------

(def owners (make-atomic 0L))

(: (-> long))
(defun (owner-next) (atomic-increment! owners))

;; A node that holds nothing, the root of a text that is one string.
(def no-node (leaves-node (new-leaves 0L)))

(: (-> int long TextNode))
(defun (new-node h o)
  (if (= h 1) (leaves-node (new-leaves o)) (branch-node (new-branch o))))

;; `n` if `o` owns it, and otherwise a copy that `o` owns.
(: (-> int TextNode long TextNode))
(defun (editable h n o)
  (cond
    ((and (not (= o 0L)) (= (node-owner n) o)) n)
    ((= h 1) (leaves-node (copy-leaves (as-leaves n) o)))
    (else (branch-node (copy-branch (as-branch n) o)))))

;; The bytes before slot `i` of `n`.
(: (-> TextNode int int))
(defun (slot-base n i) #:inline (if (= i 0) 0 (slot-end n (- i 1))))

(: (-> TextNode int int))
(defun (slot-bytes n i) #:inline (- (slot-end n i) (slot-base n i)))

;; The slot numbers of 4 lanes from `from`.
(def lanes-0 (lanes-of 0 1 2 3))
(def lanes-4 (lanes-of 4 5 6 7))
(def lanes-8 (lanes-of 8 9 10 11))
(def lanes-12 (lanes-of 12 13 14 15))

;; Adds `d` to the 4 ends from slot `from` that are of slot `i` or after.
(: (-> TextNode int (Lanes int) (Lanes int) (Lanes int) void))
(defun (add-ends4! n from slots i d) #:inline
  (set-slot-ends4! n from (lanes-add (slot-ends4 n from) (lanes-and (lanes-ge slots i) d))))

;; Adds `d` to the ends of slot `i` and every slot after it, including those
;; past the count, 4 at a time with no branch.
(: (-> TextNode int int void))
(defun (add-slot-bytes! n i d)
  (def li (lanes-splat i))
  (def ld (lanes-splat d))
  (add-ends4! n 0 lanes-0 li ld)
  (add-ends4! n 4 lanes-4 li ld)
  (add-ends4! n 8 lanes-8 li ld)
  (add-ends4! n 12 lanes-12 li ld))

;; Slot `i` of `n` holds `b` bytes; the slots after it, up to the count, keep
;; theirs.
(: (-> TextNode int int void))
(defun (set-slot-bytes! n i b) #:inline
  (add-slot-bytes! n i (- b (slot-bytes n i))))

(: (-> TextNode int))
(defun (sum-bytes n)
  (def count (node-count n))
  (if (= count 0) 0 (slot-end n (- count 1))))

(: (-> TextNode int))
(defun (sum-lines n)
  (def count (node-count n))
  (let go ((i 0) (acc 0)) (if (< i count) (go (+ i 1) (+ acc (slot-lines n i))) acc)))

(: (-> TextNode int))
(defun (sum-chars n)
  (def count (node-count n))
  (let go ((i 0) (acc 0)) (if (< i count) (go (+ i 1) (+ acc (slot-chars n i))) acc)))

;; Slot `i` of the leaves node `n` holds `c`.
(: (-> TextNode int string void))
(defun (put-chunk! n i c)
  (set-chunk! (as-leaves n) i c)
  (set-slot-bytes! n i (string-length c))
  (set-slot-lines! n i (newlines-in c))
  (set-slot-chars! n i (chars-in c)))

;; Slot `i` of the branch `n` holds the node `k`.
(: (-> TextNode int TextNode void))
(defun (put-kid! n i k)
  (set-kid! (as-branch n) i k)
  (set-slot-bytes! n i (sum-bytes k))
  (set-slot-lines! n i (sum-lines k))
  (set-slot-chars! n i (sum-chars k)))

;; Copies slot `from` of `src` to slot `to` of `dst`, both of height `h`. The
;; end of the slot is that of `src` plus `d`.
(: (-> int TextNode int TextNode int int void))
(defun (copy-slot! h src from dst to d)
  (set-slot-end! dst to (+ (slot-end src from) d))
  (set-slot-lines! dst to (slot-lines src from))
  (set-slot-chars! dst to (slot-chars src from))
  (if (= h 1)
      (set-chunk! (as-leaves dst) to (chunk-at (as-leaves src) from))
      (set-kid! (as-branch dst) to (kid (as-branch src) from))))

;; Copies `n` slots of `src` from `from` to `dst` from `to`, adding `d` to
;; their ends. The ranges may overlap in one node.
(: (-> int TextNode int TextNode int int int void))
(defun (shift-slots! h src from dst to n d)
  (if (<= to from)
      (loop (:for i (range 0 n)) (:do (copy-slot! h src (+ from i) dst (+ to i) d)))
      (loop (:for i (down-from (- n 1) #:to 0)) (:do (copy-slot! h src (+ from i) dst (+ to i) d)))))

;; Copies `n` slots of `src` from `from` to `dst` from `to`. The ranges may
;; overlap in one node.
;;
;; - The moved slots keep their bytes, after the bytes before slot `to` of
;;   `dst`.
;; - The ends of the slots of `dst` after the moved ones are stale, until they
;;   are overwritten or past the count.
(: (-> int TextNode int TextNode int int void))
(defun (move-slots! h src from dst to n)
  (shift-slots! h src from dst to n (- (slot-base dst to) (slot-base src from))))

;; Empties slots `from` to `to`, so that they keep nothing alive.
(: (-> int TextNode int int void))
(defun (clear-slots! h n from to)
  (loop (:for i (range from to))
        (:do (if (= h 1)
                 (set-chunk! (as-leaves n) i "")
                 (set-kid! (as-branch n) i no-node)))))

;; Opens slot `i` of the editable node `n`, which has room, moving the slots
;; after it one to the right.
(: (-> int TextNode int void))
(defun (open-slot! h n i)
  (def count (node-count n))
  (move-slots! h n i n (+ i 1) (- count i))
  (set-count! n (+ count 1)))

;; Removes slot `i` of the editable node `n`.
(: (-> int TextNode int void))
(defun (remove-slot! h n i)
  (def count (node-count n))
  (move-slots! h n (+ i 1) n i (- count i 1))
  (clear-slots! h n (- count 1) count)
  (set-count! n (- count 1)))

;; A new node with slots `from` to `to` of `n`.
(: (-> int TextNode int int long TextNode))
(defun (node-part h n from to o)
  (def part (new-node h o))
  (move-slots! h n from part 0 (- to from))
  (set-count! part (- to from))
  part)

;; Moves the second half of the editable node `n` into a new node, and returns
;; that.
(: (-> int TextNode long TextNode))
(defun (split-half! h n o)
  (def count (node-count n))
  (def keep (/ count 2))
  (def right (node-part h n keep count o))
  (clear-slots! h n keep count)
  (set-count! n keep)
  right)

;; Moves every slot of `r` to the end of the editable node `l`, which has room.
(: (-> int TextNode TextNode void))
(defun (append-slots! h l r)
  (def cl (node-count l))
  (def cr (node-count r))
  (move-slots! h r 0 l cl cr)
  (set-count! l (+ cl cr)))

;; Moves slots between the editable neighbours `l` and `r` until each has
;; half of them. Moving slots to the front of `r` first shifts its slots right
;; by the bytes that come.
(: (-> int TextNode TextNode void))
(defun (even-out! h l r)
  (def cl (node-count l))
  (def cr (node-count r))
  (def target (/ (+ cl cr) 2))
  (cond
    ((> cl target)
     (def k (- cl target))
     (shift-slots! h r 0 r k cr (- (slot-end l (- cl 1)) (slot-base l target)))
     (move-slots! h l target r 0 k)
     (clear-slots! h l target cl)
     (set-count! l target)
     (set-count! r (+ cr k)))
    ((< cl target)
     (def k (- target cl))
     (move-slots! h r 0 l cl k)
     (move-slots! h r k r 0 (- cr k))
     (clear-slots! h r (- cr k) cr)
     (set-count! l target)
     (set-count! r (- cr k)))
    (else unit)))

;; As `slot-starting`, by comparing the ends in turn. For a step to the next
;; or previous chunk, whose slot is predictable: the loop is small enough for
;; .NET to inline into a per-character loop without taking its registers.
(: (-> TextNode int (Tuple int int)))
(defun (slot-scanning n at)
  (def last (- (node-count n) 1))
  (let go ((i 0) (base 0))
    (def e (slot-end n i))
    (if (and (< i last) (<= e at)) (go (+ i 1) e) (Tuple i (- at base)))))

;; The slot of `n` whose bytes hold offset `at`, and the offset in it. An
;; offset between two slots is the end of the first.
;;
;; - One vectorized search of the ends, with no branch on the slot it finds:
;;   a lookup at an arbitrary offset costs no misprediction at each level.
(: (-> TextNode int (Tuple int int)))
(defun (slot-ending n at)
  (def last (- (node-count n) 1))
  (def i (if (<= at 0) 0 (span-first-outside (slot-ends n last) 0 (- at 1))))
  (def j (if (< i 0) last i))
  (Tuple j (- at (slot-base n j))))

;; As `slot-ending`, but an offset between two slots is the start of the
;; second.
(: (-> TextNode int (Tuple int int)))
(defun (slot-starting n at)
  (def last (- (node-count n) 1))
  (def i (span-first-outside (slot-ends n last) 0 at))
  (def j (if (< i 0) last i))
  (Tuple j (- at (slot-base n j))))

;; ---------------------------------------------------------------------------
;; The text
;; ---------------------------------------------------------------------------

;; `small` is the text when `height` is 0, and `root` is unused. Otherwise
;; `root` is a node of that height and `small` is "".
(type (: Text #:opaque
         (Record (: root TextNode)
                 (: height int)
                 (: bytes int)
                 (: lines int)
                 (: chars int)
                 (: small string))))

(: (-> string Text))
(defun (small-text s)
  (Text (root no-node) (height 0) (bytes (string-length s)) (lines (newlines-in s)) (chars (chars-in s)) (small s)))

(def empty-text (small-text ""))

(: (-> Text))
(defun (text-empty) empty-text)

(: (-> Text int))
(defun (text-length r) (.bytes r))

(: (-> Text bool))
(defun (text-empty? r) (= (.bytes r) 0))

;; Lines are separated by "\n": one more than the newlines.
(: (-> Text int))
(defun (text-line-count r) (+ (.lines r) 1))

(: (-> Text int))
(defun (text-height r) (.height r))

;; ---------------------------------------------------------------------------
;; Chunks of a tree
;; ---------------------------------------------------------------------------

;; Folds `f` over the chunks of `n`, of height `h`, in order.
(: (-> (-?-> string %a %a) %a TextNode int %a))
(defun (fold-node-chunks f acc n h)
  (def count (node-count n))
  (if (= h 1)
      (let go ((i 0) (acc acc))
        (if (< i count) (go (+ i 1) (f (chunk-at (as-leaves n) i) acc)) acc))
      (let go ((i 0) (acc acc))
        (if (< i count) (go (+ i 1) (fold-node-chunks f acc (kid (as-branch n) i) (- h 1))) acc))))

(: (-> (-?-> string %a %a) %a Text %a))
(defun (text-fold-chunks f init r)
  (cond
    ((> (.height r) 0) (fold-node-chunks f init (.root r) (.height r)))
    ((= (.bytes r) 0) init)
    (else (f (.small r) init))))

(: (-> (-?-> string void) TextNode int void))
(defun (each-node-chunk f n h)
  (def count (node-count n))
  (if (= h 1)
      (loop (:for i (range 0 count)) (:do (f (chunk-at (as-leaves n) i))))
      (loop (:for i (range 0 count)) (:do (each-node-chunk f (kid (as-branch n) i) (- h 1))))))

(: (-> (-?-> string void) Text void))
(defun (text-for-each-chunk f r)
  (cond
    ((> (.height r) 0) (each-node-chunk f (.root r) (.height r)))
    ((> (.bytes r) 0) (f (.small r)))
    (else unit)))

(: (-> Text int))
(defun (text-chunk-count r) (text-fold-chunks (fun (_ n) (+ n 1)) 0 r))

;; The number of chunks under `n`.
(: (-> TextNode int int))
(defun (node-chunks n h)
  (def count (node-count n))
  (if (= h 1)
      count
      (let go ((i 0) (acc 0))
        (if (< i count) (go (+ i 1) (+ acc (node-chunks (kid (as-branch n) i) (- h 1)))) acc))))

;; Puts the chunks under `n` in `out` from `at`, and returns the slot after.
(: (-> TextNode int (Array string) int int))
(defun (gather-chunks! n h out at)
  (def count (node-count n))
  (if (= h 1)
      (begin
        (loop (:for i (range 0 count)) (:do (array-set! out (+ at i) (chunk-at (as-leaves n) i))))
        (+ at count))
      (let go ((i 0) (at at))
        (if (< i count) (go (+ i 1) (gather-chunks! (kid (as-branch n) i) (- h 1) out at)) at))))

;; The text under `n`, in one allocation of its size.
(: (-> TextNode int int string))
(defun (node->string n h bytes)
  (def (: out (Array string)) (make-array (node-chunks n h) #:fill ""))
  (ignore (gather-chunks! n h out 0))
  (concat-all (array->ro-span out)))

(: (-> Text string))
(defun (text->string r)
  (if (= (.height r) 0) (.small r) (node->string (.root r) (.height r) (.bytes r))))

;; The text of a tree, which may be empty (height 0), have a root with one
;; child, or hold no more than one chunk's worth.
(: (-> TextNode int Text))
(defun (tree->text n h)
  (cond
    ((or (= h 0) (= (node-count n) 0)) empty-text)
    ((and (> h 1) (= (node-count n) 1)) (tree->text (kid (as-branch n) 0) (- h 1)))
    (else
     (def bytes (sum-bytes n))
     (if (<= bytes chunk-max)
         (small-text (node->string n h bytes))
         (Text (root n) (height h) (bytes bytes) (lines (sum-lines n)) (chars (sum-chars n)) (small ""))))))

;; The tree of a text: its root and height, and height 0 for an empty one.
(: (-> Text long (Tuple TextNode int)))
(defun (tree-of r o)
  (cond
    ((> (.height r) 0) (Tuple (.root r) (.height r)))
    ((= (.bytes r) 0) (Tuple no-node 0))
    (else
     (def n (new-node 1 o))
     (set-count! n 1)
     (put-chunk! n 0 (.small r))
     (Tuple n 1))))

;; The chunk that holds byte `at`, and the offset it starts at. The end of the
;; text is in its last chunk.
(: (-> Text int (Tuple string int)))
(defun (chunk-of r at)
  (if (= (.height r) 0)
      (Tuple (.small r) 0)
      (let go ((n (.root r)) (h (.height r)) (k at))
        (def (Tuple i k2) (slot-starting n k))
        (if (= h 1)
            (Tuple (chunk-at (as-leaves n) i) (- at k2))
            (go (kid (as-branch n) i) (- h 1) k2)))))

;; As `chunk-of`, with `slot-scanning`: for the step to a neighbouring chunk.
(: (-> Text int (Tuple string int)))
(defun (chunk-scanning r at)
  (if (= (.height r) 0)
      (Tuple (.small r) 0)
      (let go ((n (.root r)) (h (.height r)) (k at))
        (def (Tuple i k2) (slot-scanning n k))
        (if (= h 1)
            (Tuple (chunk-at (as-leaves n) i) (- at k2))
            (go (kid (as-branch n) i) (- h 1) k2)))))

;; As `chunk-of`, with the chunk after it when the same leaves node holds
;; it, and "" otherwise.
(: (-> Text int (Tuple string int string)))
(defun (chunk-and-next r at)
  (if (= (.height r) 0)
      (Tuple (.small r) 0 "")
      (let go ((n (.root r)) (h (.height r)) (k at))
        (def (Tuple i k2) (slot-starting n k))
        (if (= h 1)
            (Tuple (chunk-at (as-leaves n) i) (- at k2)
                   (if (< (+ i 1) (node-count n)) (chunk-at (as-leaves n) (+ i 1)) ""))
            (go (kid (as-branch n) i) (- h 1) k2)))))

;; An iteration over the chunks of a text in order, either way, from the
;; chunk with a given byte. It keeps the path: `nodes` and `slots` hold the
;; node and slot at each level, the leaves node first. A step moves to the
;; next slot of the lowest node that has one, and down its first or last
;; children, so a step costs O(1) amortized and not a search from the root.
(type (: ChunkIter
         (Record (: nodes (Array TextNode))
                 (: slots (Array int))
                 (: height int)
                 (: chunk string #:mutable)
                 (: start int #:mutable))))

;; The iteration on the chunk that holds byte `at`, below the length of `r`.
(: (-> Text int ChunkIter))
(defun (chunk-iter r at)
  (def h (.height r))
  (if (= h 0)
      (ChunkIter (nodes (make-array 0 #:fill no-node)) (slots (make-array 0 #:fill 0)) (height 0)
                 (chunk (.small r)) (start 0))
      (begin
        (def nodes (make-array h #:fill no-node))
        (def slots (make-array h #:fill 0))
        (let go ((n (.root r)) (lvl (- h 1)) (k at) (start 0))
          (def (Tuple i k2) (slot-starting n k))
          (array-set! nodes lvl n)
          (array-set! slots lvl i)
          (if (= lvl 0)
              (ChunkIter (nodes nodes) (slots slots) (height h)
                         (chunk (chunk-at (as-leaves n) i)) (start (+ start (- k k2))))
              (go (kid (as-branch n) i) (- lvl 1) k2 (+ start (- k k2))))))))

;; Moves `it` to the next chunk, and returns #f when there is none.
(: (-> ChunkIter bool))
(defun (chunk-iter-next! it)
  (def nodes (.nodes it))
  (def slots (.slots it))
  (def h (.height it))
  (def next-start (+ (.start it) (string-length (.chunk it))))
  (let up ((lvl 0))
    (cond
      ((>= lvl h) #f)
      ((< (+ (array-ref slots lvl) 1) (node-count (array-ref nodes lvl)))
       (array-set! slots lvl (+ (array-ref slots lvl) 1))
       (let down ((l lvl))
         (when (> l 0)
           (array-set! nodes (- l 1) (kid (as-branch (array-ref nodes l)) (array-ref slots l)))
           (array-set! slots (- l 1) 0)
           (down (- l 1))))
       (record-set! it (chunk (chunk-at (as-leaves (array-ref nodes 0)) (array-ref slots 0))) (start next-start))
       #t)
      (else (up (+ lvl 1))))))

;; Moves `it` to the chunk before, and returns #f when there is none.
(: (-> ChunkIter bool))
(defun (chunk-iter-prev! it)
  (def nodes (.nodes it))
  (def slots (.slots it))
  (def h (.height it))
  (let up ((lvl 0))
    (cond
      ((>= lvl h) #f)
      ((> (array-ref slots lvl) 0)
       (array-set! slots lvl (- (array-ref slots lvl) 1))
       (let down ((l lvl))
         (when (> l 0)
           (def k (kid (as-branch (array-ref nodes l)) (array-ref slots l)))
           (array-set! nodes (- l 1) k)
           (array-set! slots (- l 1) (- (node-count k) 1))
           (down (- l 1))))
       (def c (chunk-at (as-leaves (array-ref nodes 0)) (array-ref slots 0)))
       (record-set! it (chunk c) (start (- (.start it) (string-length c))))
       #t)
      (else (up (+ lvl 1))))))

;; ---------------------------------------------------------------------------
;; Building
;; ---------------------------------------------------------------------------

;; Builds a text from strings added at its end.
;;
;; - `levels` holds the open node of each level, the leaves node at 0. A full
;;   node is closed, into the level above, when the next item comes.
;; - Text shorter than a chunk waits in `pending`. A longer string is cut into
;;   chunks that share its array.
;; - `textbuilder->text` closes the open nodes. The last node of a level, if
;;   it has fewer than `min-kids` children, is merged with or evened out with
;;   the one before it.
(type (: TextBuilder #:opaque
         (Record (: levels (Array TextNode))
                 (: top int #:mutable)
                 (: pending StringBuilder)
                 (: owner long #:mutable)
                 (: bytes int #:mutable))))

(: (-> TextBuilder))
(defun (textbuilder-empty)
  (TextBuilder (levels (make-array 24 #:fill no-node)) (top -1) (pending (Utf8Builder. chunk-max))
               (owner (owner-next)) (bytes 0)))

(: (-> TextBuilder int))
(defun (textbuilder-length b) (.bytes b))

;; The open node of level `lvl`, made if there is none.
(: (-> TextBuilder int TextNode))
(defun (open-node b lvl)
  (when (> lvl (.top b))
    (array-set! (.levels b) lvl (new-node (+ lvl 1) (.owner b)))
    (record-set! b (top lvl)))
  (array-ref (.levels b) lvl))

;; Closes the full open node of level `lvl` into the level above.
(: (-> TextBuilder int void))
(defun (close-level! b lvl)
  (def full (array-ref (.levels b) lvl))
  (array-set! (.levels b) lvl (new-node (+ lvl 1) (.owner b)))
  (add-node! b (+ lvl 1) full))

;; Adds `k`, a node of height `lvl`, to level `lvl`.
(: (-> TextBuilder int TextNode void))
(defun (add-node! b lvl k)
  (when (= (node-count (open-node b lvl)) cap) (close-level! b lvl))
  (def n (array-ref (.levels b) lvl))
  (def i (node-count n))
  (set-count! n (+ i 1))
  (put-kid! n i k))

(: (-> TextBuilder string void))
(defun (add-chunk! b c)
  (when (= (node-count (open-node b 0)) cap) (close-level! b 0))
  (def n (array-ref (.levels b) 0))
  (def i (node-count n))
  (set-count! n (+ i 1))
  (put-chunk! n i c))

(: (-> TextBuilder void))
(defun (flush-pending! b)
  (def p (.pending b))
  (when (> (stringbuilder-length p) 0)
    (def c (stringbuilder->string p))
    (stringbuilder-clear! p)
    (add-chunk! b c)))

(: (-> TextBuilder string void))
(defun (textbuilder-add-string! b s)
  (def len (string-length s))
  (def p (.pending b))
  (def pl (stringbuilder-length p))
  (record-set! b (bytes (+ (.bytes b) len)))
  (cond
    ((= len 0) unit)
    ((and (= pl 0) (>= len chunk-min) (<= len chunk-max)) (add-chunk! b s))
    ((<= (+ pl len) chunk-max)
     (stringbuilder-add-string! p s)
     (when (= (+ pl len) chunk-max) (flush-pending! b)))
    (else
     ;; The pending chunk filled up from the front of `s`, then whole chunks
     ;; that share `s`, then the rest pending.
     (def start
       (if (= pl 0)
           0
           (begin
             (def cut (boundary-before s (- chunk-max pl)))
             (stringbuilder-add-string! p (piece s 0 cut))
             (flush-pending! b)
             cut)))
     (let go ((at start))
       (if (> (- len at) chunk-max)
           (begin
             (def cut (boundary-before s (+ at chunk-max)))
             (add-chunk! b (piece s at cut))
             (go cut))
           (when (< at len)
             (if (>= (- len at) chunk-min)
                 (add-chunk! b (piece s at len))
                 (stringbuilder-add-string! p (piece s at len)))))))))

;; Adds `ch`, in the pending chunk, which goes to the tree first when `ch`
;; would take it past `chunk-max`.
(: (-> TextBuilder char void))
(defun (textbuilder-add! b ch)
  (def cp (char->int ch))
  (def w (cond ((< cp 128) 1) ((< cp 2048) 2) ((< cp 65536) 3) (else 4)))
  (def p (.pending b))
  (when (> (+ (stringbuilder-length p) w) chunk-max) (flush-pending! b))
  (stringbuilder-add! p ch)
  (record-set! b (bytes (+ (.bytes b) w))))

(: (-> TextBuilder Text void))
(defun (textbuilder-add-text! b r)
  (text-for-each-chunk (fun (c) (textbuilder-add-string! b c)) r))

;; The pending text as the last chunk. A short one is merged with the chunk
;; before it, or evened out with it.
(: (-> TextBuilder void))
(defun (flush-last! b)
  (def p (.pending b))
  (def pl (stringbuilder-length p))
  (when (> pl 0)
    (def c (stringbuilder->string p))
    (stringbuilder-clear! p)
    (def n (if (>= (.top b) 0) (array-ref (.levels b) 0) no-node))
    (def count (node-count n))
    (if (or (>= pl chunk-min) (= count 0))
        (add-chunk! b c)
        (begin
          (def last (chunk-at (as-leaves n) (- count 1)))
          (def t (string-append last c))
          (if (<= (string-length t) chunk-max)
              (put-chunk! n (- count 1) t)
              (begin
                (def cut (boundary-before t (/ (string-length t) 2)))
                (put-chunk! n (- count 1) (piece t 0 cut))
                (add-chunk! b (piece t cut (string-length t)))))))))

(: (-> TextBuilder Text))
(defun (textbuilder->text b)
  (flush-last! b)
  (def levels (.levels b))
  ;; Each level's open node goes into the level above, from the bottom. The
  ;; one before it is always there: a level above is only opened by closing
  ;; a node of this one.
  (let go ((lvl 0))
    (when (< lvl (.top b))
      (def x (array-ref levels lvl))
      (def cx (node-count x))
      (when (> cx 0)
        (def parent (array-ref levels (+ lvl 1)))
        (def cp (node-count parent))
        (if (and (< cx min-kids) (> cp 0))
            (begin
              (def h (+ lvl 1))
              (def y (editable h (kid (as-branch parent) (- cp 1)) (.owner b)))
              (if (<= (+ (node-count y) cx) cap)
                  (begin (append-slots! h y x) (put-kid! parent (- cp 1) y))
                  (begin
                    (even-out! h y x)
                    (put-kid! parent (- cp 1) y)
                    (add-node! b (+ lvl 1) x))))
            (add-node! b (+ lvl 1) x)))
      (go (+ lvl 1))))
  (def top (.top b))
  (def result (if (< top 0) empty-text (tree->text (array-ref levels top) (+ top 1))))
  ;; Start over, with an owner that does not own what was given away.
  (loop (:for i (range 0 (+ top 1))) (:do (array-set! levels i no-node)))
  (record-set! b (top -1) (owner (owner-next)) (bytes 0))
  result)

;; A text of the strings, in order.
(: (-> #:rest string Text))
(defun (text #:rest parts)
  (def b (textbuilder-empty))
  (loop (:for s parts) (:do (textbuilder-add-string! b s)))
  (textbuilder->text b))

(: (-> string Text))
(defun (string->text s)
  (if (<= (string-length s) chunk-max)
      (small-text s)
      (begin
        (def b (textbuilder-empty))
        (textbuilder-add-string! b s)
        (textbuilder->text b))))

;; ---------------------------------------------------------------------------
;; Joining
;; ---------------------------------------------------------------------------

;; A new node of height `h` with the children `a` and `b`.
(: (-> int TextNode TextNode long TextNode))
(defun (pair-node h a b o)
  (def n (new-node h o))
  (set-count! n 2)
  (put-kid! n 0 a)
  (put-kid! n 1 b)
  n)

;; Merges the chunks at slots `i - 1` and `i` of the editable leaves node `n`
;; when one is shorter than `chunk-min` and the two fit in one.
(: (-> int TextNode int void))
(defun (tidy-seam! h n i)
  (when (and (= h 1) (> i 0) (< i (node-count n)))
    (def a (slot-bytes n (- i 1)))
    (def b (slot-bytes n i))
    (when (and (or (< a chunk-min) (< b chunk-min)) (<= (+ a b) chunk-max))
      (def l (as-leaves n))
      (put-chunk! n (- i 1) (string-append (chunk-at l (- i 1)) (chunk-at l i)))
      (remove-slot! 1 n i))))

;; Puts `k` in slot `j` of the editable node `n`. A full `n` is split first;
;; the result is then the two halves in order and #t.
(: (-> TextNode int int TextNode long (Tuple TextNode TextNode bool)))
(defun (add-kid-at n h j k o)
  (if (< (node-count n) cap)
      (begin
        (open-slot! h n j)
        (put-kid! n j k)
        (Tuple n no-node #f))
      (begin
        (def right (split-half! h n o))
        (def keep (node-count n))
        (if (<= j keep)
            (begin (open-slot! h n j) (put-kid! n j k))
            (begin (open-slot! h right (- j keep)) (put-kid! right (- j keep) k)))
        (Tuple n right #t))))

;; Joins `b`, of height `hb`, to the right of `a`, of the greater height `ha`.
;; The result is a node of height `ha`, and a second one after it when the
;; first had no room.
(: (-> TextNode int TextNode int long (Tuple TextNode TextNode bool)))
(defun (join-right a ha b hb o)
  (def n (editable ha a o))
  (def last (- (node-count n) 1))
  (def y (kid (as-branch n) last))
  (if (= (- ha 1) hb)
      (begin
        (def cy (node-count y))
        (def cb (node-count b))
        (if (<= (+ cy cb) cap)
            (begin
              (def y2 (editable hb y o))
              (append-slots! hb y2 b)
              (tidy-seam! hb y2 cy)
              (put-kid! n last y2)
              (Tuple n no-node #f))
            (if (or (< cy min-kids) (< cb min-kids))
                (begin
                  (def y2 (editable hb y o))
                  (def b2 (editable hb b o))
                  (even-out! hb y2 b2)
                  (put-kid! n last y2)
                  (add-kid-at n ha (+ last 1) b2 o))
                (add-kid-at n ha (+ last 1) b o))))
      (begin
        (def (Tuple c extra split?) (join-right y (- ha 1) b hb o))
        (put-kid! n last c)
        (if split? (add-kid-at n ha (+ last 1) extra o) (Tuple n no-node #f)))))

;; Joins `a`, of height `ha`, to the left of `b`, of the greater height `hb`,
;; as `join-right` does.
(: (-> TextNode int TextNode int long (Tuple TextNode TextNode bool)))
(defun (join-left a ha b hb o)
  (def n (editable hb b o))
  (def y (kid (as-branch n) 0))
  (if (= (- hb 1) ha)
      (begin
        (def cy (node-count y))
        (def ca (node-count a))
        (if (<= (+ cy ca) cap)
            (begin
              (def a2 (editable ha a o))
              (append-slots! ha a2 y)
              (tidy-seam! ha a2 ca)
              (put-kid! n 0 a2)
              (Tuple n no-node #f))
            (if (or (< cy min-kids) (< ca min-kids))
                (begin
                  (def y2 (editable ha y o))
                  (def a2 (editable ha a o))
                  (even-out! ha a2 y2)
                  (put-kid! n 0 y2)
                  (add-kid-at n hb 0 a2 o))
                (add-kid-at n hb 0 a o))))
      (begin
        (def (Tuple first second split?) (join-left a ha y (- hb 1) o))
        (put-kid! n 0 first)
        (if split? (add-kid-at n hb 1 second o) (Tuple n no-node #f)))))

;; The tree of `a` followed by `b`. Either may be empty (height 0) or have a
;; root with fewer than `min-kids` children; every other node is full enough.
(: (-> TextNode int TextNode int long (Tuple TextNode int)))
(defun (join a ha b hb o)
  (cond
    ((= ha 0) (Tuple b hb))
    ((= hb 0) (Tuple a ha))
    ((= ha hb)
     (def ca (node-count a))
     (def cb (node-count b))
     (if (<= (+ ca cb) cap)
         (begin
           (def m (editable ha a o))
           (append-slots! ha m b)
           (tidy-seam! ha m ca)
           (Tuple m ha))
         (begin
           (def a2 (editable ha a o))
           (def b2 (editable ha b o))
           (when (or (< ca min-kids) (< cb min-kids)) (even-out! ha a2 b2))
           (Tuple (pair-node (+ ha 1) a2 b2 o) (+ ha 1)))))
    ((> ha hb)
     (def (Tuple n extra split?) (join-right a ha b hb o))
     (if split? (Tuple (pair-node (+ ha 1) n extra o) (+ ha 1)) (Tuple n ha)))
    (else
     (def (Tuple first second split?) (join-left a ha b hb o))
     (if split? (Tuple (pair-node (+ hb 1) first second o) (+ hb 1)) (Tuple first hb)))))

(: (-> Text Text long Text))
(defun (concat-with a b o)
  (cond
    ((= (.bytes a) 0) b)
    ((= (.bytes b) 0) a)
    ((and (= (.height b) 0) (<= (.bytes b) small-insert)) (insert-with a (.bytes a) (.small b) o))
    ((and (= (.height a) 0) (<= (.bytes a) small-insert)) (insert-with b 0 (.small a) o))
    (else
     (def (Tuple na ha) (tree-of a o))
     (def (Tuple nb hb) (tree-of b o))
     (def (Tuple n h) (join na ha nb hb o))
     (tree->text n h))))

(: (-> Text Text Text))
(defun (text-append a b) (concat-with a b (owner-next)))

;; ---------------------------------------------------------------------------
;; Splitting
;; ---------------------------------------------------------------------------

(: (-> string int string void))
(defun (check-boundary s k who)
  (unless (boundary? s k) (inside-character who)))

;; Raises that an offset is inside a character. Not inlined, as
;; `cursor-at-edge`.
(: (-> string void))
(defun (inside-character who) #:no-inline
  (bad-argument (str who ": the offset is in the middle of a character.")))

;; Merges the chunk at slot `i` of the editable leaves node `n` with a
;; neighbour when it is shorter than `chunk-min`: into one chunk if they fit,
;; and otherwise into two of about the same length.
(: (-> TextNode int void))
(defun (merge-small-chunk! n i)
  (def count (node-count n))
  (when (and (> count 1) (< (slot-bytes n i) chunk-min))
    (def a (if (< (+ i 1) count) i (- i 1)))
    (def l (as-leaves n))
    (def t (string-append (chunk-at l a) (chunk-at l (+ a 1))))
    (def tl (string-length t))
    (if (<= tl chunk-max)
        (begin (put-chunk! n a t) (remove-slot! 1 n (+ a 1)))
        (begin
          (def cut (boundary-before t (/ tl 2)))
          (put-chunk! n a (piece t 0 cut))
          (put-chunk! n (+ a 1) (piece t cut tl))))))

;; The trees of the bytes of `n`, of height `h`, before and after `at`, where
;; 0 < `at` < the bytes of `n`.
(: (-> TextNode int int string long (Tuple TextNode int TextNode int)))
(defun (split-tree n h at who o)
  (def (Tuple i k) (slot-ending n at))
  (def count (node-count n))
  (if (= h 1)
      (begin
        (def c (chunk-at (as-leaves n) i))
        (def cl (string-length c))
        (check-boundary c k who)
        (def left (node-part 1 n 0 i o))
        (set-count! left (+ i 1))
        (put-chunk! left i (piece c 0 k))
        (merge-small-chunk! left i)
        (def rest (- count i 1))
        (def right
          (if (< k cl)
              (begin
                (def r (new-node 1 o))
                (set-count! r 1)
                (put-chunk! r 0 (piece c k cl))
                (move-slots! 1 n (+ i 1) r 1 rest)
                (set-count! r (+ rest 1))
                (merge-small-chunk! r 0)
                r)
              (node-part 1 n (+ i 1) count o)))
        (Tuple left 1 right (if (= (node-count right) 0) 0 1)))
      (if (= k (slot-bytes n i))
          (Tuple (node-part h n 0 (+ i 1) o) h
                 (node-part h n (+ i 1) count o) (if (= (+ i 1) count) 0 h))
          (begin
            (def (Tuple ln lh rn rh) (split-tree (kid (as-branch n) i) (- h 1) k who o))
            (def (Tuple l2 lh2) (join (node-part h n 0 i o) (if (= i 0) 0 h) ln lh o))
            (def (Tuple r2 rh2) (join rn rh (node-part h n (+ i 1) count o) (if (= (+ i 1) count) 0 h) o))
            (Tuple l2 lh2 r2 rh2)))))

(: (-> Text int string long (Tuple Text Text)))
(defun (split-with r at who o)
  (check-offset r at who)
  (cond
    ((= at 0) (Tuple empty-text r))
    ((= at (.bytes r)) (Tuple r empty-text))
    ((= (.height r) 0)
     (def s (.small r))
     (check-boundary s at who)
     (Tuple (small-text (piece s 0 at)) (small-text (piece s at (.bytes r)))))
    (else
     (def (Tuple ln lh rn rh) (split-tree (.root r) (.height r) at who o))
     (Tuple (tree->text ln lh) (tree->text rn rh)))))

(: (-> Text int string void))
(defun (check-offset r at who)
  (when (or (< at 0) (> at (.bytes r))) (offset-outside who at (.bytes r))))

;; Raises that `at` is outside a text of `bytes` bytes. Not inlined, as
;; `cursor-at-edge`.
(: (-> string int int void))
(defun (offset-outside who at bytes) #:no-inline
  (out-of-range #"${who}: the offset ${at} is outside the text, of ${bytes} bytes."))

(: (-> Text TextCursor (Tuple Text Text)))
(defun (text-split-at r c) (split-with r (.at c) "text-split-at" (owner-next)))

;; Bytes `from` to `to` of `r` as a text that shares its chunks.
(: (-> Text int int Text))
(defun (slice r from to)
  (check-range r from to "text-substring")
  (def o (owner-next))
  (def (Tuple _ tail) (split-with r from "text-substring" o))
  (def (Tuple mid _) (split-with tail (- to from) "text-substring" o))
  mid)

;; Bytes `from` to `to` of `r` as a text: one string when they fit in a chunk,
;; one leaves node of the pieces of the chunks when they are in at most `cap`
;; chunks, and a slice that shares the chunks otherwise.
(: (-> Text int int Text))
(defun (range->text r from to)
  (cond
    ((<= (- to from) chunk-max) (small-text (substring-string r from to)))
    ((or (> (- to from) (* cap chunk-min)) (= (.height r) 0)) (slice r from to))
    (else
     (check-range r from to "text-substring")
     (def it (chunk-iter r from))
     (def n (new-node 1 0L))
     (let go ((i 0))
       (def c (.chunk it))
       (def start (.start it))
       (def e (min to (+ start (string-length c))))
       (set-count! n (+ i 1))
       (put-chunk! n i (piece c (- (max from start) start) (- e start)))
       (cond
         ((>= e to) (tree->text n 1))
         ((or (= (+ i 1) cap) (not (chunk-iter-next! it))) (slice r from to))
         (else (go (+ i 1))))))))

(: (-> Text int int string void))
(defun (check-range r from to who)
  (check-offset r from who)
  (check-offset r to who)
  (when (> from to) (backward-range who)))

;; Raises that a range ends before it starts. Not inlined, as
;; `cursor-at-edge`.
(: (-> string void))
(defun (backward-range who) #:no-inline
  (out-of-range (str who ": the end is before the start.")))

;; The characters of `r` from cursor `a` to cursor `b`, as a text.
(: (-> Text TextCursor TextCursor Text))
(defun (text-substring r a b)
  (when (> (.at a) (.at b)) (backward-range "text-substring"))
  (range->text r (.at a) (.at b)))

;; ---------------------------------------------------------------------------
;; Inserting
;; ---------------------------------------------------------------------------

;; Puts `s`, of `sl` newlines, `sc` characters and at most `small-insert`
;; bytes, at byte `at` of the editable node `n`, which has room for one more
;; slot.
(: (-> TextNode int int string int int long void))
(defun (insert-into! n h at s sl sc o)
  (def slen (string-length s))
  (def (Tuple i k) (slot-ending n at))
  (if (= h 1)
      (begin
        (def l (as-leaves n))
        (def c (chunk-at l i))
        (check-boundary c k "text-insert")
        (def cl (string-length c))
        (def t (concat3 (piece c 0 k) s (piece c k cl)))
        (if (<= (+ cl slen) chunk-max)
            (begin
              (set-chunk! l i t)
              (set-slot-bytes! n i (+ cl slen))
              (set-slot-lines! n i (+ (slot-lines n i) sl))
              (set-slot-chars! n i (+ (slot-chars n i) sc)))
            (begin
              (def tl (string-length t))
              (def cut (boundary-before t (/ tl 2)))
              (open-slot! 1 n (+ i 1))
              (put-chunk! n i (piece t 0 cut))
              (put-chunk! n (+ i 1) (piece t cut tl)))))
      (begin
        (def b (as-branch n))
        (def child (editable (- h 1) (kid b i) o))
        (set-kid! b i child)
        (if (= (node-count child) cap)
            (begin
              (def right (split-half! (- h 1) child o))
              (open-slot! h n (+ i 1))
              (put-kid! n i child)
              (put-kid! n (+ i 1) right)
              (def lb (slot-bytes n i))
              (if (<= k lb)
                  (into-kid! n h i child k s sl sc o)
                  (into-kid! n h (+ i 1) right (- k lb) s sl sc o)))
            (into-kid! n h i child k s sl sc o)))))

(: (-> TextNode int int TextNode int string int int long void))
(defun (into-kid! n h i child k s sl sc o)
  (insert-into! child (- h 1) k s sl sc o)
  (add-slot-bytes! n i (string-length s))
  (set-slot-lines! n i (+ (slot-lines n i) sl))
  (set-slot-chars! n i (+ (slot-chars n i) sc)))

(: (-> Text int string long Text))
(defun (insert-with r at s o)
  (check-offset r at "text-insert")
  (def slen (string-length s))
  (cond
    ((= slen 0) r)
    ((= (.height r) 0)
     (def s0 (.small r))
     (def len0 (.bytes r))
     (check-boundary s0 at "text-insert")
     (if (<= (+ len0 slen) chunk-max)
         (small-text (concat3 (piece s0 0 at) s (piece s0 at len0)))
         (text (piece s0 0 at) s (piece s0 at len0))))
    ((<= slen small-insert)
     (def h (.height r))
     (def n (editable h (.root r) o))
     (def sl (newlines-in s))
     (def sc (chars-in s))
     (def (Tuple root height)
       (if (= (node-count n) cap)
           (begin
             (def right (split-half! h n o))
             (Tuple (pair-node (+ h 1) n right o) (+ h 1)))
           (Tuple n h)))
     (insert-into! root height at s sl sc o)
     (Text (root root) (height height) (bytes (+ (.bytes r) slen)) (lines (+ (.lines r) sl)) (chars (+ (.chars r) sc))
           (small "")))
    (else
     (def (Tuple l rr) (split-with r at "text-insert" o))
     (concat-with (concat-with l (string->text s) o) rr o))))

(: (-> Text TextCursor string Text))
(defun (text-insert r c s) (insert-with r (.at c) s (owner-next)))

(: (-> Text TextCursor Text Text))
(defun (text-insert-text r c x)
  (def o (owner-next))
  (def (Tuple l rr) (split-with r (.at c) "text-insert-text" o))
  (concat-with (concat-with l x o) rr o))

;; ---------------------------------------------------------------------------
;; Deleting
;; ---------------------------------------------------------------------------

;; Merges child `i` of the editable branch `n`, of height `h`, which has fewer
;; than `min-kids` children, with a neighbour, or evens the two out.
(: (-> TextNode int int long void))
(defun (fix-underflow! n h i o)
  (def count (node-count n))
  (when (> count 1)
    (def a (if (< (+ i 1) count) i (- i 1)))
    (def b (as-branch n))
    (def x (editable (- h 1) (kid b a) o))
    (def y (editable (- h 1) (kid b (+ a 1)) o))
    (def cx (node-count x))
    (if (<= (+ cx (node-count y)) cap)
        (begin
          (append-slots! (- h 1) x y)
          (tidy-seam! (- h 1) x cx)
          (put-kid! n a x)
          (remove-slot! h n (+ a 1)))
        (begin
          (even-out! (- h 1) x y)
          (put-kid! n a x)
          (put-kid! n (+ a 1) y)))))

;; Deletes bytes `at` to `at + len` of the editable node `n`, all in one
;; chunk, which hold `dl` newlines and `dc` characters.
(: (-> TextNode int int int int int long void))
(defun (delete-into! n h at len dl dc o)
  (def (Tuple i k) (slot-starting n at))
  (if (= h 1)
      (begin
        (def l (as-leaves n))
        (def c (chunk-at l i))
        (def cl (string-length c))
        (check-boundary c k "text-delete")
        (check-boundary c (+ k len) "text-delete")
        (if (= len cl)
            (remove-slot! 1 n i)
            (begin
              (set-chunk! l i (concat3 (piece c 0 k) "" (piece c (+ k len) cl)))
              (set-slot-bytes! n i (- cl len))
              (set-slot-lines! n i (- (slot-lines n i) dl))
              (set-slot-chars! n i (- (slot-chars n i) dc))
              (merge-small-chunk! n i))))
      (begin
        (def b (as-branch n))
        (def child (editable (- h 1) (kid b i) o))
        (set-kid! b i child)
        (delete-into! child (- h 1) k len dl dc o)
        (add-slot-bytes! n i (- len))
        (set-slot-lines! n i (- (slot-lines n i) dl))
        (set-slot-chars! n i (- (slot-chars n i) dc))
        (when (< (node-count child) min-kids) (fix-underflow! n h i o)))))

(: (-> Text int int long Text))
(defun (delete-with r from to o)
  (check-range r from to "text-delete")
  (def len (- to from))
  (cond
    ((= len 0) r)
    ((= (.height r) 0)
     (def s (.small r))
     (check-boundary s from "text-delete")
     (check-boundary s to "text-delete")
     (small-text (concat3 (piece s 0 from) "" (piece s to (.bytes r)))))
    (else
     (def (Tuple c start) (chunk-of r from))
     (if (<= to (+ start (string-length c)))
         (begin
           (def h (.height r))
           (def n (editable h (.root r) o))
           (def k (- from start))
           (check-boundary c k "text-delete")
           (check-boundary c (+ k len) "text-delete")
           (def dl (newlines-between c k (+ k len)))
           (def dc (chars-between c k (+ k len)))
           (delete-into! n h from len dl dc o)
           (tree->text-sized n h (- (.bytes r) len) (- (.lines r) dl) (- (.chars r) dc)))
         (begin
           (def (Tuple l tail) (split-with r from "text-delete" o))
           (def (Tuple _ rr) (split-with tail len "text-delete" o))
           (concat-with l rr o))))))

;; `tree->text` for a root whose bytes, newlines and characters are known.
(: (-> TextNode int int int int Text))
(defun (tree->text-sized n h bytes lines chars)
  (if (or (<= bytes chunk-max) (and (> h 1) (= (node-count n) 1)))
      (tree->text n h)
      (Text (root n) (height h) (bytes bytes) (lines lines) (chars chars) (small ""))))

(: (-> Text TextCursor TextCursor Text))
(defun (text-delete r a b) (delete-with r (.at a) (.at b) (owner-next)))

(: (-> Text TextCursor TextCursor string Text))
(defun (text-replace-range r a b s)
  (def o (owner-next))
  (insert-with (delete-with r (.at a) (.at b) o) (.at a) s o))

;; ---------------------------------------------------------------------------
;; Characters and text
;; ---------------------------------------------------------------------------

;; The number of bytes of the character whose first byte is `b`.
(: (-> int int))
(defun (char-width b)
  (cond ((< b 128) 1) ((< b 224) 2) ((< b 240) 3) (else 4)))

(: (-> (RoSpan byte) int int))
(defun (byte-at sp i) #:inline (cast int (ro-span-ref sp i)))

;; The character that starts at byte `k` of `c`.
(: (-> string int char))
(defun (decode-at c k)
  (def sp (.AsSpan c))
  (def b0 (byte-at sp k))
  (int->char
   (cond
     ((< b0 128) b0)
     ((< b0 224) (bitwise-ior (shift-left (bitwise-and b0 31) 6) (bitwise-and (byte-at sp (+ k 1)) 63)))
     ((< b0 240) (bitwise-ior (shift-left (bitwise-and b0 15) 12)
                              (bitwise-ior (shift-left (bitwise-and (byte-at sp (+ k 1)) 63) 6)
                                           (bitwise-and (byte-at sp (+ k 2)) 63))))
     (else (bitwise-ior (shift-left (bitwise-and b0 7) 18)
                        (bitwise-ior (shift-left (bitwise-and (byte-at sp (+ k 1)) 63) 12)
                                     (bitwise-ior (shift-left (bitwise-and (byte-at sp (+ k 2)) 63) 6)
                                                  (bitwise-and (byte-at sp (+ k 3)) 63))))))))

;; Bytes `from` to `to` as a string of their own.
(: (-> Text int int string))
(defun (substring-string r from to)
  (check-range r from to "text-substring")
  (cond
    ((= from to) "")
    ((= (.height r) 0)
     (check-boundary (.small r) from "text-substring")
     (check-boundary (.small r) to "text-substring")
     (string-copy (piece (.small r) from to)))
    (else
     ;; The pieces of the chunks from `from` to `to`, joined in one
     ;; allocation of their size.
     (def (Tuple c0 s0 after) (chunk-and-next r from))
     (check-boundary c0 (- from s0) "text-substring")
     (def e0 (+ s0 (string-length c0)))
     (if (<= to e0)
         (begin
           (check-boundary c0 (- to s0) "text-substring")
           (string-copy (piece c0 (- from s0) (- to s0))))
         (begin
           (def first (piece c0 (- from s0) (string-length c0)))
           (def c1 (if (= (string-length after) 0)
                       (begin (def (Tuple c _) (chunk-of r e0)) c)
                       after))
           (def s1 e0)
           (if (<= to (+ s1 (string-length c1)))
               (begin
                 (check-boundary c1 (- to s1) "text-substring")
                 (concat2 first (piece c1 0 (- to s1))))
               (begin
                 ;; The chunks the range touches, counted on a second iteration.
                 (def it (chunk-iter r e0))
                 (def ci (chunk-iter r e0))
                 (def count (let go ((n 2))
                              (if (>= (+ (.start ci) (string-length (.chunk ci))) to)
                                  n
                                  (begin (ignore (chunk-iter-next! ci)) (go (+ n 1))))))
                 (def (: parts (Array string)) (make-array count #:fill ""))
                 (array-set! parts 0 first)
                 (let fill ((k 1))
                   (def c (.chunk it))
                   (def st (.start it))
                   (if (= k (- count 1))
                       (begin
                         (check-boundary c (- to st) "text-substring")
                         (array-set! parts k (piece c 0 (- to st))))
                       (begin
                         (array-set! parts k c)
                         (ignore (chunk-iter-next! it))
                         (fill (+ k 1)))))
                 (concat-all (array->ro-span parts)))))))))

(: (-> Text int))
(defun (text-count r) (.chars r))

;; ---------------------------------------------------------------------------
;; Lines
;; ---------------------------------------------------------------------------

;; The offset after the `line`-th newline of `c`, counting from 1.
(: (-> string int int))
(defun (nth-newline-end c line)
  (def sp (.AsSpan c))
  (let go ((from 0) (left line))
    (def j (+ from (span-index-of (ro-span-slice sp #:from from) newline)))
    (if (= left 1) (+ j 1) (go (+ j 1) (- left 1)))))

;; The offset after the `line`-th newline of `n`, counting from 1.
(: (-> TextNode int int int))
(defun (line-start-in n h line)
  (let go ((i 0) (line line))
    (def l (slot-lines n i))
    (cond
      ((> line l) (go (+ i 1) (- line l)))
      ((= h 1) (+ (slot-base n i) (nth-newline-end (chunk-at (as-leaves n) i) line)))
      (else (+ (slot-base n i) (line-start-in (kid (as-branch n) i) (- h 1) line))))))

(: (-> Text int string void))
(defun (check-line r n who)
  (when (or (< n 0) (> n (.lines r)))
    (out-of-range #"${who}: there is no line ${n} in a text of ${(+ (.lines r) 1)} lines.")))

;; The offset of the first byte of line `n`, counting from 0.
(: (-> Text int int))
(defun (line-start r n)
  (check-line r n "text-line")
  (cond
    ((= n 0) 0)
    ((= (.height r) 0) (nth-newline-end (.small r) n))
    (else (line-start-in (.root r) (.height r) n))))

;; The offset of the newline that ends line `n`, or the end of the text for
;; the last line.
(: (-> Text int int))
(defun (line-end r n)
  (check-line r n "text-line")
  (if (= n (.lines r)) (.bytes r) (- (line-start r (+ n 1)) 1)))

;; The line that byte `at` is on: the newlines before it.
(: (-> Text int int))
(defun (line-of r at)
  (if (= (.height r) 0)
      (newlines-between (.small r) 0 at)
      (let go ((n (.root r)) (h (.height r)) (at at) (acc 0))
        (def (Tuple i k) (slot-starting n at))
        (def before (let sum ((j 0) (acc acc)) (if (< j i) (sum (+ j 1) (+ acc (slot-lines n j))) acc)))
        (if (= h 1)
            (+ before (newlines-between (chunk-at (as-leaves n) i) 0 k))
            (go (kid (as-branch n) i) (- h 1) k before)))))

;; Line `n`, without its newline.
(: (-> Text int Text))
(defun (text-line r n)
  (range->text r (line-start r n) (line-end r n)))

;; ---------------------------------------------------------------------------
;; Searching
;; ---------------------------------------------------------------------------

;; Whether the bytes of `r` from `at` are those of `s`.
(: (-> Text int string bool))
(defun (matches-at? r at s)
  (def m (string-length s))
  (def ss (.AsSpan s))
  (and (<= (+ at m) (.bytes r))
       (let go ((p at))
         (if (>= p (+ at m))
             #t
             (begin
               (def (Tuple c start) (chunk-of r p))
               (def e (min (+ start (string-length c)) (+ at m)))
               (and (span-equal? (ro-span-slice (.AsSpan c) #:from (- p start) #:to (- e start))
                                 (ro-span-slice ss #:from (- p at) #:to (- e at)))
                    (go e)))))))

;; The offset of the first `s` at or after `at`, or -1, searched from the
;; chunks of `it`, which is on the chunk that holds `at` or one before it.
;; Leaves `it` on the chunk where the search stopped, at or before the chunk
;; of the match's end, so the next search can continue from it.
(: (-> Text ChunkIter string int int))
(defun (index-from! r it s at)
  (def m (string-length s))
  (def bytes (.bytes r))
  (def ss (.AsSpan s))
  (def first (cast byte (ro-span-ref ss 0)))
  (let skip ()
    (when (and (<= (+ (.start it) (string-length (.chunk it))) at) (chunk-iter-next! it)) (skip)))
  (let go ((at at))
    (if (> (+ at m) bytes)
        -1
        (begin
          (def c (.chunk it))
          (def start (.start it))
          (def cl (string-length c))
          (def sp (.AsSpan c))
          (def k (- at start))
          (def j (span-find (ro-span-slice sp #:from k) ss))
          (cond
            ((>= j 0) (+ at j))
            ((not (chunk-iter-next! it)) -1)
            (else
             ;; A match that starts in this chunk and ends in the next one,
             ;; or past it.
             (def nc (.chunk it))
             (let straddle ((p (max k (- cl (- m 1)))))
               (def q (if (< p cl) (span-index-of (ro-span-slice sp #:from p) first) -1))
               (if (< q 0)
                   (go (+ start cl))
                   (begin
                     (def pp (+ p q))
                     (def head (- cl pp))
                     (if (and (span-equal? (ro-span-slice sp #:from pp) (ro-span-slice ss #:from 0 #:to head))
                              (if (<= (- m head) (string-length nc))
                                  (span-equal? (ro-span-slice (.AsSpan nc) #:from 0 #:to (- m head))
                                               (ro-span-slice ss #:from head))
                                  (matches-at? r (+ start pp) s)))
                         (+ start pp)
                         (straddle (+ pp 1))))))))))))

;; The offset of the first `s` at or after `#:from`.
(: (-> Text string (#:from int) (Option int)))
(defun (index-of r s #:from 0)
  (def m (string-length s))
  (cond
    ((= m 0) (Some from))
    ((= (.height r) 0)
     (def j (span-find (ro-span-slice (.AsSpan (.small r)) #:from from) (.AsSpan s)))
     (if (>= j 0) (Some (+ from j)) None))
    ((> (+ from m) (.bytes r)) None)
    (else
     (def k (index-from! r (chunk-iter r from) s from))
     (if (>= k 0) (Some k) None))))

(: (-> Text string bool))
(defun (text-contains? r s) (some? (index-of r s)))

(: (-> Text string bool))
(defun (text-starts-with? r s) (matches-at? r 0 s))

(: (-> Text string bool))
(defun (text-ends-with? r s)
  (def at (- (.bytes r) (string-length s)))
  (and (>= at 0) (matches-at? r at s)))

;; The offset of the last `s` that ends at or before `#:to`.
;;
;; - Each chunk, last first, is searched for the matches that start in it and
;;   end in a later one, and then for those inside it.
(: (-> Text string (#:to int) (Option int)))
(defun (last-index-of r s #:to (text-length r))
  (def m (string-length s))
  (cond
    ((= m 0) (Some to))
    ((> m to) None)
    ((= (.height r) 0)
     (def j (span-find-last (ro-span-slice (.AsSpan (.small r)) #:from 0 #:to to) (.AsSpan s)))
     (if (>= j 0) (Some j) None))
    (else
     (def first (cast byte (ro-span-ref (.AsSpan s) 0)))
     (def it (chunk-iter r (- to 1)))
     (def ss (.AsSpan s))
     ;; `after` is the chunk after this one, which the last step left.
     (let go ((after ""))
       (def c (.chunk it))
       (def start (.start it))
       (def cl (string-length c))
       (def sp (.AsSpan c))
       ;; A match from byte p of the chunk crosses its end when p > cl - m,
       ;; and ends in time when start + p + m <= to.
       (def lo (max 0 (+ (- cl m) 1)))
       (def straddler
         (let back ((hi (min (- cl 1) (- (- to start) m))))
           (if (< hi lo)
               None
               (begin
                 (def q (span-last-index-of (ro-span-slice sp #:from lo #:to (+ hi 1)) first))
                 (def pp (+ lo q))
                 (def head (- cl pp))
                 (cond
                   ((< q 0) None)
                   ((and (span-equal? (ro-span-slice sp #:from pp) (ro-span-slice ss #:from 0 #:to head))
                         (if (<= (- m head) (string-length after))
                             (span-equal? (ro-span-slice (.AsSpan after) #:from 0 #:to (- m head))
                                          (ro-span-slice ss #:from head))
                             (matches-at? r (+ start pp) s)))
                    (Some (+ start pp)))
                   (else (back (- pp 1))))))))
       (match straddler
         ((Some k) (Some k))
         (None
          (def e (min cl (- to start)))
          (def j (if (>= e m) (span-find-last (ro-span-slice sp #:from 0 #:to e) (.AsSpan s)) -1))
          (cond
            ((>= j 0) (Some (+ start j)))
            ((= start 0) None)
            (else (ignore (chunk-iter-prev! it)) (go c)))))))))

;; ---------------------------------------------------------------------------
;; Equality, order and hashing
;; ---------------------------------------------------------------------------

(: (-> (RoSpan byte) (RoSpan byte) bool int))
(defun (compare-spans x y order?) #:inline
  (cond (order? (span-compare x y)) ((span-equal? x y) 0) (else 1)))

;; Compares the bytes of `a` and `b` from 0 to `len`, which both have, part by
;; part, where a part lies in one chunk of each. Returns 0 if they are all
;; equal. Otherwise the order of the first part that differs when `order?`,
;; and 1 when not.
(: (-> Text Text int bool int))
(defun (compare-parts a b len order?)
  (if (= len 0)
      0
      (begin
        (def ia (chunk-iter a 0))
        (def ib (chunk-iter b 0))
        (let go ((p 0))
          (if (>= p len)
              0
              (begin
                (def ca (.chunk ia))
                (def cb (.chunk ib))
                (def sa (.start ia))
                (def sb (.start ib))
                (def ea (+ sa (string-length ca)))
                (def eb (+ sb (string-length cb)))
                (def e (min len (min ea eb)))
                (def d (compare-spans (ro-span-slice (.AsSpan ca) #:from (- p sa) #:to (- e sa))
                                      (ro-span-slice (.AsSpan cb) #:from (- p sb) #:to (- e sb))
                                      order?))
                (cond
                  ((not (= d 0)) d)
                  ((>= e len) 0)
                  (else
                   (when (= e ea) (ignore (chunk-iter-next! ia)))
                   (when (= e eb) (ignore (chunk-iter-next! ib)))
                   (go e)))))))))

(: (-> Text Text bool))
(defun (text=? a b)
  (and (= (.bytes a) (.bytes b))
       (= (.lines a) (.lines b))
       (if (and (= (.height a) 0) (= (.height b) 0))
           (= (.small a) (.small b))
           (= (compare-parts a b (.bytes a) #f) 0))))

(: (-> Text Text int))
(defun (text-compare a b)
  (def d (compare-parts a b (min (.bytes a) (.bytes b)) #t))
  (cond
    ((< d 0) -1)
    ((> d 0) 1)
    (else (compare (.bytes a) (.bytes b)))))

;; A hash of the bytes in order, whatever chunks hold them: the folded
;; multiply of wyhash, as the string hash does.
;;
;; - The bytes are read as 16-byte blocks of two words. Block k of the text
;;   goes to lane k mod 4, and the four lanes do not wait for each other.
;; - A block that spans two chunks is carried over in `w0` and `w1`, with `n`
;;   of its bytes, and `lane` is the lane of the next block.
(type (: HashState (Struct (: s0 ulong) (: s1 ulong) (: s2 ulong) (: s3 ulong)
                           (: w0 ulong) (: w1 ulong) (: n int) (: lane int))))

(def hash-p0 (cast ulong 11562461410679940143UL))
(def hash-p1 (cast ulong 16646288086500911323UL))
(def hash-p2 (cast ulong 10285213230658275043UL))
(def hash-p3 (cast ulong 6384245875588680899UL))

;; The high and low words of a * b, xored.
(: (-> ulong ulong ulong))
(defun (fold-mul a b) #:inline
  (def p (big-mul a b))
  (bitwise-xor (cast ulong (shift-right p 64)) (cast ulong p)))

(: (-> HashState ulong ulong HashState))
(defun (hash-feed st a b)
  (def l (.lane st))
  (def st2
    (cond
      ((= l 0) (record-set st (s0 (fold-mul (bitwise-xor a hash-p0) (bitwise-xor b (.s0 st))))))
      ((= l 1) (record-set st (s1 (fold-mul (bitwise-xor a hash-p1) (bitwise-xor b (.s1 st))))))
      ((= l 2) (record-set st (s2 (fold-mul (bitwise-xor a hash-p2) (bitwise-xor b (.s2 st))))))
      (else (record-set st (s3 (fold-mul (bitwise-xor a hash-p3) (bitwise-xor b (.s3 st))))))))
  (record-set st2 (lane (bitwise-and (+ l 1) 3))))

;; Bytes `from` to `to` of `sp`, at most 8, as the low bytes of a word, the
;; first lowest.
(: (-> (RoSpan byte) int int ulong))
(defun (word-of sp from to)
  (let go ((k (- to 1)) (w (cast ulong 0)))
    (if (< k from) w (go (- k 1) (bitwise-ior (shift-left w 8) (cast ulong (ro-span-ref sp k)))))))

;; Bytes `from` to `to` of `sp`, which fit in the carried block, put in it.
(: (-> HashState (RoSpan byte) int int HashState))
(defun (carry-bytes st sp from to)
  (def n (.n st))
  (def k (- to from))
  ;; The bytes that go to w0, from its byte n, and those after, to w1.
  (def into0 (max 0 (min k (- 8 n))))
  (def w0 (bitwise-ior (.w0 st) (shift-left (word-of sp from (+ from into0)) (* 8 (min n 8)))))
  (def w1 (if (= into0 k)
              (.w1 st)
              (bitwise-ior (.w1 st) (shift-left (word-of sp (+ from into0) to) (* 8 (max 0 (- n 8)))))))
  (record-set st (w0 w0) (w1 w1) (n (+ n k))))

(: (-> string HashState HashState))
(defun (hash-chunk c st0)
  (def sp (.AsSpan c))
  (def len (string-length c))
  (def/mutable st st0)
  (def/mutable i 0)
  ;; The carried block, made whole and fed.
  (when (> (.n st) 0)
    (def take (min len (- 16 (.n st))))
    (set! st (carry-bytes st sp 0 take))
    (set! i take)
    (when (= (.n st) 16)
      (set! st (hash-feed (record-set st (w0 (cast ulong 0)) (w1 (cast ulong 0)) (n 0)) (.w0 st) (.w1 st)))))
  (def blocks (/ (- len i) 16))
  (when (> blocks 0)
    (def (: ws (RoSpan ulong)) (ro-span-> (ro-span-slice sp #:from i #:to (+ i (* 16 blocks)))))
    (def/mutable k 0)
    ;; Single blocks up to lane 0, then four at a time, then the rest.
    (loop (:while (and (> (.lane st) 0) (< k blocks)))
          (:do (set! st (hash-feed st (ro-span-ref ws (* 2 k)) (ro-span-ref ws (+ (* 2 k) 1))))
               (set! k (+ k 1))))
    (def/mutable s0 (.s0 st))
    (def/mutable s1 (.s1 st))
    (def/mutable s2 (.s2 st))
    (def/mutable s3 (.s3 st))
    (loop (:while (<= (+ k 4) blocks))
          (:do (def b (ro-span-slice ws #:from (* 2 k) #:to (+ (* 2 k) 8)))
               (set! s0 (fold-mul (bitwise-xor (ro-span-ref b 0) hash-p0) (bitwise-xor (ro-span-ref b 1) s0)))
               (set! s1 (fold-mul (bitwise-xor (ro-span-ref b 2) hash-p1) (bitwise-xor (ro-span-ref b 3) s1)))
               (set! s2 (fold-mul (bitwise-xor (ro-span-ref b 4) hash-p2) (bitwise-xor (ro-span-ref b 5) s2)))
               (set! s3 (fold-mul (bitwise-xor (ro-span-ref b 6) hash-p3) (bitwise-xor (ro-span-ref b 7) s3)))
               (set! k (+ k 4))))
    (set! st (record-set st (s0 s0) (s1 s1) (s2 s2) (s3 s3)))
    (loop (:while (< k blocks))
          (:do (set! st (hash-feed st (ro-span-ref ws (* 2 k)) (ro-span-ref ws (+ (* 2 k) 1))))
               (set! k (+ k 1))))
    (set! i (+ i (* 16 blocks))))
  (if (< i len) (carry-bytes st sp i len) st))

(: (-> Text int))
(defun (text-hash r)
  (def zero (cast ulong 0))
  (def seed (cast ulong 0x9E3779B97F4A7C15UL))
  (def st (text-fold-chunks hash-chunk
                            (HashState (s0 seed) (s1 (bitwise-xor seed hash-p1)) (s2 (bitwise-xor seed hash-p2))
                                       (s3 (bitwise-xor seed hash-p3)) (w0 zero) (w1 zero) (n 0) (lane 0))
                            r))
  (def lanes (bitwise-xor (bitwise-xor (.s0 st) (.s1 st)) (bitwise-xor (.s2 st) (.s3 st))))
  (def h (fold-mul (bitwise-xor (.w0 st) hash-p0)
                   (bitwise-xor (.w1 st) (bitwise-xor lanes (cast ulong (+ (* 16 (.n st)) (.bytes r)))))))
  (def fin (fold-mul (bitwise-xor h hash-p1) (bitwise-xor (cast ulong (.lane st)) hash-p2)))
  (cast int (bitwise-xor fin (shift-right-logical fin 32))))

(impl (Eq Text)
  (defun (= a b) (text=? a b))
  (defun (eq-hash r) (text-hash r)))

(impl (Ord Text)
  (defun (compare a b) (text-compare a b)))

(impl (Written Text)
  (defun (write-into r b) (write-into (text->string r) b)))

(impl (Displayed Text)
  (defun (display-into r b) (text-for-each-chunk (fun (c) (stringbuilder-add-string! b c)) r))
  (defun (displayed r) (text->string r)))

;; ---------------------------------------------------------------------------
;; Iteration
;; ---------------------------------------------------------------------------

;; A walk over the characters of a text, a struct: a walk over one chunk, and
;; the offset of the next. A step off the end of a chunk moves on to the
;; next one, so that only the last chunk's walk is ever done.
(type (: TextWalk #:opaque
         (Struct (: text Text)
                 (: walk StringWalk)
                 (: next int))))

(: (-> Text TextWalk))
(defun (text-walk r)
  (if (= (.height r) 0)
      (TextWalk (text r) (walk (string-walk (.small r))) (next (.bytes r)))
      (text-walk-at r 0)))

;; The walk over the chunk that starts at `at`.
(: (-> Text int TextWalk))
(defun (text-walk-at r at)
  (def (Tuple c _) (chunk-scanning r at))
  (TextWalk (text r) (walk (string-walk c)) (next (+ at (string-length c)))))

(: (-> TextWalk TextWalk))
(defun (text-walk-next w)
  (def s (string-walk-next (.walk w)))
  (if (and (string-walk-done? s) (< (.next w) (.bytes (.text w))))
      (text-walk-at (.text w) (.next w))
      (record-set w (walk s))))

(impl (Iterable Text)
  (type %elem char)
  (type %cursor TextWalk)
  (defun (iterable-start r) (text-walk r))
  (defun (iterable-done? r w) (string-walk-done? (.walk w)))
  (defun (iterable-current r w) (string-walk-current (.walk w)))
  (defun (iterable-next r w) (text-walk-next w))
  (defun (count r) (.chars r)))

(impl (Collection Text)
  (type %content char)
  (defun (length r) (.chars r))
  (defun (empty? r) (text-empty? r)))

(impl (Refable Text)
  (type %index TextCursor)
  (type %out char)
  (defun (ref r c) (text-cursor-ref r c))
  (defun (try-ref r c) (if (at-end? c) None (Some (char-at c)))))

(impl (Cursor Text)
  (type %elem char)
  (type %cursor TextCursor)
  (defun (cursor-start r) (text-cursor-start r))
  (defun (cursor-end r) (text-cursor-end r))
  (defun (cursor-end? r c) (at-end? c))
  (defun (cursor-ref r c) (text-cursor-ref r c))
  (defun (cursor-next r c) (text-cursor-next r c))
  (defun (cursor-prev r c) (text-cursor-prev r c)))

;; ---------------------------------------------------------------------------
;; As a string has them
;; ---------------------------------------------------------------------------

(: (-> string int void))
(defun (check-count who n)
  (when (< n 0) (bad-argument #"${who}: the count ${n} is negative.")))


;; The bytes of the first `n` characters of `c`, or all of it when it has
;; fewer.
;;
;; - Counts the characters of 32 bytes at a time, as `chars-between` does,
;;   and scans the bytes of the block that holds the start of character `n`.
;; - Allocates nothing.
(: (-> string int int))
(defun (bytes-of-first c n)
  (def sp (.AsSpan c))
  (def blocks (* 32 (/ (string-length c) 32)))
  (def top (bytes32-splat (cast byte 192)))
  (def cont (bytes32-splat (cast byte 128)))
  (let go ((j 0) (n n))
    (if (>= j blocks)
        (nth-start c j n)
        (begin
          (def v (bytes32-of (ro-span-slice sp #:from j #:to (+ j 32))))
          (def k (- 32 (pop-count32 (bytes32-mask (bytes32-eq (bytes32-and v top) cont)))))
          (if (< n k) (nth-start c j n) (go (+ j 32) (- n k)))))))

;; The byte from `k` on where the `n`-th character after `k` that starts there
;; or later starts, counting from 0, or the length of `c` when there are fewer.
(: (-> string int int int))
(defun (nth-start c k n)
  (def sp (.AsSpan c))
  (def len (string-length c))
  (let step ((k k) (n n))
    (cond
      ((>= k len) len)
      ((= (bitwise-and (byte-at sp k) 192) 128) (step (+ k 1) n))
      ((= n 0) k)
      (else (step (+ k 1) (- n 1))))))

;; The characters in the first `k` bytes of `c`, which has `cc` characters.
;; Counts from the nearer end: a chunk with as many characters as bytes is
;; ASCII, and needs no count.
(: (-> string int int int))
(defun (chars-of-first c cc k)
  (def cl (string-length c))
  (cond
    ((= cc cl) k)
    ((> (* 2 k) cl) (- cc (chars-between c k cl)))
    (else (chars-between c 0 k))))

;; The offset after the first `n` characters of `r`, or its end when it has
;; fewer.
(: (-> Text int int))
(defun (offset-after-chars r n) (byte-of-char r (min n (.chars r))))

;; The offset before the last `n` characters of `r`, or 0 when it has fewer.
(: (-> Text int int))
(defun (offset-before-last-chars r n) (byte-of-char r (max 0 (- (.chars r) n))))

;; Adds bytes `from` to `to` of `r` to `b`, from the chunks of `it`, which is on
;; the chunk that holds `from` or one before it. Leaves `it` on the chunk that
;; holds `to`, or the last.
(: (-> TextBuilder Text ChunkIter int int void))
(defun (add-range! b r it from to)
  (let skip ()
    (when (and (<= (+ (.start it) (string-length (.chunk it))) from) (chunk-iter-next! it)) (skip)))
  (let go ()
    (def c (.chunk it))
    (def start (.start it))
    (def e (min to (+ start (string-length c))))
    (when (< from e)
      (textbuilder-add-string! b (piece c (- (max from start) start) (- e start))))
    (when (and (< e to) (chunk-iter-next! it)) (go))))

;; The first `n` characters of `r`, or all of it when it has fewer.
(: (-> Text int Text))
(defun (text-take r n)
  (check-count "text-take" n)
  (def (Tuple l _) (split-with r (offset-after-chars r n) "text-take" (owner-next)))
  l)

;; `r` without its first `n` characters.
(: (-> Text int Text))
(defun (text-drop r n)
  (check-count "text-drop" n)
  (def (Tuple _ rr) (split-with r (offset-after-chars r n) "text-drop" (owner-next)))
  rr)

;; The last `n` characters of `r`, or all of it when it has fewer.
(: (-> Text int Text))
(defun (text-take-right r n)
  (check-count "text-take-right" n)
  (def (Tuple _ rr) (split-with r (offset-before-last-chars r n) "text-take-right" (owner-next)))
  rr)

;; `r` without its last `n` characters.
(: (-> Text int Text))
(defun (text-drop-right r n)
  (check-count "text-drop-right" n)
  (def (Tuple l _) (split-with r (offset-before-last-chars r n) "text-drop-right" (owner-next)))
  l)

;; Bytes `from` to `to` of `r` as a text, from the chunks of `it`, which is on
;; the chunk that holds `from` or one before it. A range inside one chunk is
;; a string that shares the chunk's bytes; a longer one is copied when it is
;; at most `chunk-max` bytes, and a slice that shares the chunks otherwise.
(: (-> Text ChunkIter int int Text))
(defun (range-from! r it from to)
  (let skip ()
    (when (and (<= (+ (.start it) (string-length (.chunk it))) from) (chunk-iter-next! it)) (skip)))
  (def c (.chunk it))
  (def start (.start it))
  (cond
    ((<= to (+ start (string-length c))) (small-text (piece c (- from start) (- to start))))
    (else (range->text r from to))))

;; `r` with every `old` replaced by `new`, left to right without overlaps.
;;
;; - Searches through one chunk iteration, and adds the text between the
;;   matches through another, sharing the arrays of the chunks it keeps whole.
;; - Raises when `old` is empty.
(: (-> Text string string Text))
(defun (text-replace r old new)
  (when (string-empty? old) (bad-argument "text-replace: the string to replace is empty."))
  (def m (string-length old))
  (cond
    ((< (.bytes r) m) r)
    ((= (.height r) 0)
     (if (string-contains? (.small r) old) (string->text (string-replace (.small r) old new)) r))
    (else
     (def si (chunk-iter r 0))
     (def first (index-from! r si old 0))
     (if (< first 0)
         r
         (begin
           (def b (textbuilder-empty))
           (def it (chunk-iter r 0))
           (let go ((from 0) (at first))
             (add-range! b r it from at)
             (textbuilder-add-string! b new)
             (def next (+ at m))
             (def k (index-from! r si old next))
             (if (>= k 0)
                 (go next k)
                 (add-range! b r it next (.bytes r))))
           (textbuilder->text b))))))

;; The fields of `r` between the occurrences of `sep`, empty ones kept: one
;; more than the occurrences. An empty `sep` gives `r` as the one field. A
;; field inside one chunk shares the chunk's bytes.
(: (-> Text string (Vec Text)))
(defun (text-split r sep)
  (def m (string-length sep))
  (cond
    ((= m 0) (vec-add (vec-empty) r))
    ((= (.height r) 0) (vec-map small-text (string-split (.small r) sep)))
    (else
     (def si (chunk-iter r 0))
     (def ft (chunk-iter r 0))
     (def out (vecbuilder-empty))
     (let go ((from 0))
       (def k (index-from! r si sep from))
       (if (>= k 0)
           (begin (vecbuilder-add! out (range-from! r ft from k)) (go (+ k m)))
           (vecbuilder-add! out (range-from! r ft from (.bytes r)))))
     (vecbuilder->vec out))))

;; The text before the first `sep` and the text after it, or None.
(: (-> Text string (Option (Tuple Text Text))))
(defun (text-split-once r sep)
  (match (index-of r sep)
    ((Some k)
     (def o (owner-next))
     (def (Tuple l rest) (split-with r k "text-split-once" o))
     (def (Tuple _ after) (split-with rest (string-length sep) "text-split-once" o))
     (Some (Tuple l after)))
    (None None)))

;; The texts of `parts` in order, with `sep` between each two.
(: (-> (Vec Text) string Text))
(defun (text-join parts sep)
  (def b (textbuilder-empty))
  (loop (:with first? #t #f)
        (:for part parts)
        (:do (unless first? (textbuilder-add-string! b sep))
             (textbuilder-add-text! b part)))
  (textbuilder->text b))

;; The cursor on the first `needle` in `r`. An empty needle is at the start.
(: (-> Text string (Option TextCursor)))
(defun (text-search r needle) (text-cursor-search r (text-cursor-start r) needle))

;; The cursor on the last `needle` in `r`. An empty needle is at the end.
(: (-> Text string (Option TextCursor)))
(defun (text-search-backward r needle) (text-cursor-search-backward r (text-cursor-end r) needle))

;; The cursor on the first character of `r` that `pred` holds for.
(: (-> (-?-> char bool) Text (Option TextCursor)))
(defun (text-find pred r) (text-cursor-find pred r (text-cursor-start r)))

;; The cursor on the first character at or after `start` that is in `chars`.
;;
;; - A set of ASCII characters is searched for in each chunk's bytes at once.
;; - An empty set is never found.
(: (-> Text string TextCursor (Option TextCursor)))
(defun (text-search-any r chars start)
  (cond
    ((or (string-empty? chars) (at-end? start)) None)
    ((= (string-length chars) (string-count chars))
     (def set (.AsSpan chars))
     (def it (chunk-iter r (.at start)))
     (let go ((from (- (.at start) (.start it))))
       (def k (span-index-of-any (ro-span-slice (.AsSpan (.chunk it)) #:from from) set))
       (cond
         ((>= k 0) (Some (cursor-at-byte r (+ (.start it) from k))))
         ((chunk-iter-next! it) (go 0))
         (else None))))
    (else
     (def set (loop (:for ch chars) (:acc (arraying ch))))
     (def n (array-length set))
     (text-cursor-find (fun (ch) (let go ((i 0)) (and (< i n) (or (char=? (array-ref set i) ch) (go (+ i 1)))))) r start))))

;; Whether `ch` is not white space.
(: (-> char bool))
(defun (not-space? ch) (not (char-whitespace? ch)))

;; `r` without the white space at its start.
(: (-> Text Text))
(defun (text-trim-left r)
  (match (text-find not-space? r)
    ((Some c) (if (at-start? c) r (range->text r (.at c) (.bytes r))))
    (None empty-text)))

;; `r` without the white space at its end.
(: (-> Text Text))
(defun (text-trim-right r)
  (match (text-cursor-find-backward not-space? r (text-cursor-end r))
    ((Some c)
     (def e (step-next c))
     (if (at-end? e) r (range->text r 0 (.at e))))
    (None empty-text)))

;; `r` without the white space at both ends.
(: (-> Text Text))
(defun (text-trim r) (text-trim-right (text-trim-left r)))

;; `n` copies of `ch`.
(: (-> char int string))
(defun (repeated-char ch n)
  (def b (stringbuilder-empty))
  (loop (:for _ (range 0 n)) (:do (stringbuilder-add! b ch)))
  (stringbuilder->string b))

;; `r` with `#:with` before it until it is `width` characters long.
(: (-> Text int (#:with char) Text))
(defun (text-pad-left r width #:with #\space)
  (def k (- width (.chars r)))
  (if (<= k 0) r (insert-with r 0 (repeated-char with k) (owner-next))))

;; `r` with `#:with` after it until it is `width` characters long.
(: (-> Text int (#:with char) Text))
(defun (text-pad-right r width #:with #\space)
  (def k (- width (.chars r)))
  (if (<= k 0) r (insert-with r (.bytes r) (repeated-char with k) (owner-next))))

;; `r` with each character uppercased in the invariant culture, chunk by chunk.
(: (-> Text Text))
(defun (text-upcase r)
  (def b (textbuilder-empty))
  (text-for-each-chunk (fun (c) (textbuilder-add-string! b (.ToUpperInvariant c))) r)
  (textbuilder->text b))

;; `r` with each character lowercased in the invariant culture, chunk by chunk.
(: (-> Text Text))
(defun (text-downcase r)
  (def b (textbuilder-empty))
  (text-for-each-chunk (fun (c) (textbuilder-add-string! b (.ToLowerInvariant c))) r)
  (textbuilder->text b))

;; The characters of `r` in reverse order: its chunks from the last, each
;; reversed.
(: (-> Text Text))
(defun (text-reverse r)
  (if (= (.height r) 0)
      (small-text (.Reverse (.small r)))
      (begin
        (def b (textbuilder-empty))
        (def it (chunk-iter r (- (.bytes r) 1)))
        (let go ()
          (textbuilder-add-string! b (.Reverse (.chunk it)))
          (when (chunk-iter-prev! it) (go)))
        (textbuilder->text b))))

;; Folds `f` over the characters of `r`, first to last.
(: (-> (-?-> char %a %a) %a Text %a))
(defun (text-fold f init r)
  (loop (:for ch r) (:acc acc (folding init (f ch acc))) => acc))

;; Folds `f` over the characters of `r`, last to first.
(: (-> (-?-> char %a %a) %a Text %a))
(defun (text-fold-right f init r)
  (loop (:for ch (text-backward r (text-cursor-end r))) (:acc acc (folding init (f ch acc))) => acc))

(: (-> (-?-> char void) Text void))
(defun (text-for-each f r) (loop (:for ch r) (:do (f ch))))

(: (-> Text (List char)))
(defun (text->list r) (text-fold-right (fun (ch acc) (Cons ch acc)) Nil r))

(: (-> Text (Vec char)))
(defun (text->vec r) (loop (:for ch r) (:acc (vecing ch))))

;; ---------------------------------------------------------------------------
;; Collecting and adding
;; ---------------------------------------------------------------------------

;; What a builder takes a piece of text as: a string or a character.
(def/trait (TextPiece %a)
  (: textbuilder-add-piece! (-> TextBuilder %a void)))

(impl (TextPiece string)
  (defun (textbuilder-add-piece! b s) (textbuilder-add-string! b s)))

(impl (TextPiece char)
  (defun (textbuilder-add-piece! b ch) (textbuilder-add! b ch)))

;; The collector that gathers characters or strings into a text, through a
;; builder, as `stringing` gathers characters into a string.
(type (: (Texting %a) (Union texting)))

(impl (Collector (Texting %a))
  (where (TextPiece %a))
  (type %elem %a)
  (type %acc TextBuilder)
  (type %out Text)
  (defun (collector-init c) (textbuilder-empty))
  (defun (collector-step c acc x) (textbuilder-add-piece! acc x) acc)
  (defun (collector-finish c acc) (textbuilder->text acc)))

;; `(texting x #:into t)`: the builder starts with the chunks of `t`.
(type (: (TextingInto %a) (Struct (: seed Text))))

(impl (Collector (TextingInto %a))
  (where (TextPiece %a))
  (type %elem %a)
  (type %acc TextBuilder)
  (type %out Text)
  (defun (collector-init c)
    (def b (textbuilder-empty))
    (textbuilder-add-text! b (.seed c))
    b)
  (defun (collector-step c acc x) (textbuilder-add-piece! acc x) acc)
  (defun (collector-finish c acc) (textbuilder->text acc)))

(impl (CollectorInto (Texting %a))
  (type %seed Text)
  (type %into (TextingInto %a))
  (defun (collector-into c seed) (TextingInto (seed seed))))

(impl (Addable Text)
  (type %elem string)
  (defun (add r s) (insert-with r (.bytes r) s (owner-next))))

(impl (Addable! TextBuilder)
  (type %elem char)
  (defun (add! b ch) (textbuilder-add! b ch)))

(impl (Addable! TransientText)
  (type %elem string)
  (defun (add! t s) (transienttext-append! t s)))

;; ---------------------------------------------------------------------------
;; Cursors
;; ---------------------------------------------------------------------------

;; A position in a text, a struct that holds the chunk it is in.
;;
;; - `at` is a byte offset on a character boundary. `chunk` starts at byte
;;   `start` of the text, ends at byte `end`, and holds the character at `at`.
;; - At the end of the text, `chunk` is "" and `start` and `end` are `at`.
;; - A step inside the chunk reads its bytes only; a step out of it finds the
;;   next chunk from the root.
(type (: TextCursor #:opaque
         (Struct (: text Text)
                 (: chunk string)
                 (: start int)
                 (: end int)
                 (: at int))))

(: (-> Text TextCursor))
(defun (end-cursor r)
  (TextCursor (text r) (chunk "") (start (.bytes r)) (end (.bytes r)) (at (.bytes r))))

;; The cursor at `at`, a boundary below the length of `r`, found by
;; `chunk-of`.
;;
;; - Not inlined: inlined into a loop that starts with it, the descent takes
;;   the room .NET would inline the loop's steps with.
(: (-> Text int TextCursor))
(defun (cursor-in r at) #:no-inline
  (def (Tuple c start) (chunk-of r at))
  (TextCursor (text r) (chunk c) (start start) (end (+ start (string-length c))) (at at)))

;; As `cursor-in`, for a step into the next or previous chunk.
(: (-> Text int TextCursor))
(defun (cursor-stepped r at)
  (def (Tuple c start) (chunk-scanning r at))
  (TextCursor (text r) (chunk c) (start start) (end (+ start (string-length c))) (at at)))

;; The cursor at byte `at`, a character boundary of `r`.
(: (-> Text int TextCursor))
(defun (cursor-at-byte r at)
  (if (= at (.bytes r)) (end-cursor r) (cursor-in r at)))

;; The characters before slot `i` of `n`.
(: (-> TextNode int int))
(defun (chars-before-slot n i)
  (let go ((j 0) (acc 0)) (if (< j i) (go (+ j 1) (+ acc (slot-chars n j))) acc)))

;; The cursor on character `n` of `r`, where 0 <= `n` <= its characters.
;;
;; - Descends by the character counts of the nodes, then steps through one
;;   chunk; a chunk with as many characters as bytes is ASCII, and needs no
;;   steps.
(: (-> Text int TextCursor))
(defun (cursor-of-char r n) #:no-inline
  (cond
    ((>= n (.chars r)) (end-cursor r))
    ((= (.height r) 0)
     (def c (.small r))
     (def at (if (= (.chars r) (.bytes r)) n (bytes-of-first c n)))
     (TextCursor (text r) (chunk c) (start 0) (end (string-length c)) (at at)))
    (else
     (let go ((nd (.root r)) (h (.height r)) (n n) (start 0))
       (def last (- (node-count nd) 1))
       (def (Tuple i k)
         (let scan ((i 0) (k n))
           (def ci (slot-chars nd i))
           (if (and (< i last) (>= k ci)) (scan (+ i 1) (- k ci)) (Tuple i k))))
       (def s2 (+ start (slot-base nd i)))
       (if (= h 1)
           (begin
             (def c (chunk-at (as-leaves nd) i))
             (def cl (string-length c))
             (def off (if (= (slot-chars nd i) cl) k (bytes-of-first c k)))
             (TextCursor (text r) (chunk c) (start s2) (end (+ s2 cl)) (at (+ s2 off))))
           (go (kid (as-branch nd) i) (- h 1) k s2))))))

;; The byte offset of character `n` of `r`, where 0 <= `n` <= its characters.
(: (-> Text int int))
(defun (byte-of-char r n) (.at (cursor-of-char r n)))

;; The characters before byte `at`, a boundary of `r`.
(: (-> Text int int))
(defun (chars-before r at)
  (cond
    ((= at 0) 0)
    ((= at (.bytes r)) (.chars r))
    ((= (.height r) 0) (chars-of-first (.small r) (.chars r) at))
    (else
     (let go ((n (.root r)) (h (.height r)) (at at) (acc 0))
       (def (Tuple i k) (slot-starting n at))
       (def before (+ acc (chars-before-slot n i)))
       (if (= h 1)
           (+ before (chars-of-first (chunk-at (as-leaves n) i) (slot-chars n i) k))
           (go (kid (as-branch n) i) (- h 1) k before))))))

(: (-> Text TextCursor))
(defun (text-cursor-start r)
  (if (= (.bytes r) 0) (end-cursor r) (cursor-in r 0)))

(: (-> Text TextCursor))
(defun (text-cursor-end r) (end-cursor r))

(: (-> TextCursor bool))
(defun (at-end? c) (= (.at c) (.bytes (.text c))))

(: (-> TextCursor bool))
(defun (at-start? c) (= (.at c) 0))

;; Whether `c`, a cursor of `r`, is at its end.
(: (-> Text TextCursor bool))
(defun (text-cursor-end? r c) (at-end? c))

;; Whether `c`, a cursor of `r`, is at its start.
(: (-> Text TextCursor bool))
(defun (text-cursor-start? r c) (at-start? c))

(: (-> TextCursor string void))
(defun (check-not-end c who)
  (when (at-end? c) (cursor-at-edge who "end")))

(: (-> TextCursor string void))
(defun (check-not-start c who)
  (when (at-start? c) (cursor-at-edge who "start")))

;; Raises that the cursor is at the `edge` of the text. Not inlined, so that
;; the message is not built in the steps that check.
(: (-> string string void))
(defun (cursor-at-edge who edge) #:no-inline
  (out-of-range #"${who}: the cursor is at the ${edge} of the text."))

;; The character at `c`, which is not at the end.
(: (-> TextCursor char))
(defun (char-at c) #:inline (decode-at (.chunk c) (- (.at c) (.start c))))

;; The cursor `to`, after the character at `c`.
(: (-> TextCursor int TextCursor))
(defun (moved-to c to)
  (cond
    ((< to (.end c)) (record-set c (at to)))
    ((< to (.bytes (.text c))) (cursor-stepped (.text c) to))
    (else (end-cursor (.text c)))))

;; The cursor after the character at `c`, which is not at the end.
(: (-> TextCursor TextCursor))
(defun (step-next c)
  (moved-to c (+ (.at c) (char-width (byte-at (.AsSpan (.chunk c)) (- (.at c) (.start c)))))))

;; The character at `c`, which is not at the end, and the cursor after it,
;; from one read of the chunk.
(: (-> TextCursor (Tuple char TextCursor)))
(defun (ref+next c)
  (def sp (.AsSpan (.chunk c)))
  (def k (- (.at c) (.start c)))
  (def b0 (byte-at sp k))
  (def ch (if (< b0 128) (int->char b0) (decode-at (.chunk c) k)))
  (Tuple ch (moved-to c (+ (.at c) (char-width b0)))))

;; The cursor on the character before `c`, which is not at the start.
(: (-> TextCursor TextCursor))
(defun (step-prev c)
  (def p (if (> (.at c) (.start c)) c (cursor-stepped (.text c) (- (.at c) 1))))
  (def sp (.AsSpan (.chunk p)))
  (let go ((k (- (- (.at c) 1) (.start p))))
    (if (and (> k 0) (= (bitwise-and (byte-at sp k) 192) 128))
        (go (- k 1))
        (record-set p (at (+ (.start p) k))))))

;; The character at `c`, a cursor of `r`.
(: (-> Text TextCursor char))
(defun (text-cursor-ref r c)
  (check-not-end c "text-cursor-ref")
  (char-at c))

(: (-> Text TextCursor TextCursor))
(defun (text-cursor-next r c)
  (check-not-end c "text-cursor-next")
  (step-next c))

(: (-> Text TextCursor TextCursor))
(defun (text-cursor-prev r c)
  (check-not-start c "text-cursor-prev")
  (step-prev c))

(: (-> Text TextCursor (Tuple char TextCursor)))
(defun (text-cursor-ref+next r c)
  (check-not-end c "text-cursor-ref+next")
  (ref+next c))

;; Raises that a count is negative. Not inlined, as `cursor-at-edge`.
(: (-> string int void))
(defun (negative-count who n) #:no-inline
  (bad-argument #"${who}: the count ${n} is negative."))

;; Raises that there are fewer than `n` characters on the `side` of the
;; cursor. Not inlined, as `cursor-at-edge`.
(: (-> string int string void))
(defun (too-few who n side) #:no-inline
  (out-of-range #"${who}: there are fewer than ${n} characters ${side} the cursor."))

;; The steps a cursor takes one character at a time. A longer move finds its
;; character from the root, by the character counts.
(def few-steps 16)

;; The cursor `n` characters after `c`, a cursor of `r`.
(: (-> Text TextCursor int TextCursor))
(defun (text-cursor-forward r c n)
  (when (< n 0) (negative-count "text-cursor-forward" n))
  (if (<= n few-steps)
      (let go ((p c) (k n))
        (cond
          ((= k 0) p)
          ((at-end? p) (too-few "text-cursor-forward" n "after") p)
          (else (go (step-next p) (- k 1)))))
      (begin
        (def t (.text c))
        (def target (+ (chars-before t (.at c)) n))
        (when (> target (.chars t)) (too-few "text-cursor-forward" n "after"))
        (cursor-of-char t target))))

;; The cursor `n` characters before `c`, a cursor of `r`.
(: (-> Text TextCursor int TextCursor))
(defun (text-cursor-back r c n)
  (when (< n 0) (negative-count "text-cursor-back" n))
  (if (<= n few-steps)
      (let go ((p c) (k n))
        (cond
          ((= k 0) p)
          ((at-start? p) (too-few "text-cursor-back" n "before") p)
          (else (go (step-prev p) (- k 1)))))
      (begin
        (def t (.text c))
        (def target (- (chars-before t (.at c)) n))
        (when (< target 0) (too-few "text-cursor-back" n "before"))
        (cursor-of-char t target))))

;; The cursor `column` characters into line `n`, or at the line's end when
;; it is shorter.
(: (-> Text int (#:column int) TextCursor))
(defun (text-cursor-at-line r n #:column 0)
  (def from (line-start r n))
  (def e (line-end r n))
  (let go ((p (cursor-at-byte r from)) (k column))
    (if (or (<= k 0) (>= (.at p) e)) p (go (step-next p) (- k 1)))))

;; The line of `c`, a cursor of `r`, counting from 0.
(: (-> Text TextCursor int))
(defun (text-cursor-line r c) (line-of (.text c) (.at c)))

(: (-> Text TextCursor TextCursor))
(defun (text-cursor-line-start r c)
  (def t (.text c))
  (cursor-at-byte t (line-start t (line-of t (.at c)))))

(: (-> Text TextCursor TextCursor))
(defun (text-cursor-line-end r c)
  (def t (.text c))
  (cursor-at-byte t (line-end t (line-of t (.at c)))))

;; The characters before cursor `c` of `r`: its index, from 0 to
;; `text-count`. O(log n) time.
(: (-> Text TextCursor int))
(defun (text-cursor-index r c) (chars-before (.text c) (.at c)))

;; The characters between the start of the cursor's line and the cursor.
(: (-> Text TextCursor int))
(defun (text-cursor-column r c)
  (def t (.text c))
  (def from (line-start t (line-of t (.at c))))
  (- (chars-before t (.at c)) (chars-before t from)))

;; The first `s` that starts at or after `c`, a cursor of `r`.
(: (-> Text TextCursor string (Option TextCursor)))
(defun (text-cursor-search r c s)
  (def t (.text c))
  (option-map (fun (k) (cursor-at-byte t k)) (index-of t s #:from (.at c))))

;; The last `s` that ends at or before `c`, a cursor of `r`.
(: (-> Text TextCursor string (Option TextCursor)))
(defun (text-cursor-search-backward r c s)
  (def t (.text c))
  (option-map (fun (k) (cursor-at-byte t k)) (last-index-of t s #:to (.at c))))

;; The first character at or after `c`, a cursor of `r`, that `pred` holds
;; for.
(: (-> (-?-> char bool) Text TextCursor (Option TextCursor)))
(defun (text-cursor-find pred r c)
  (let go ((p c))
    (cond
      ((at-end? p) None)
      (else
       (def (Tuple ch n) (ref+next p))
       (if (pred ch) (Some p) (go n))))))

;; The last character before `c`, a cursor of `r`, that `pred` holds for.
(: (-> (-?-> char bool) Text TextCursor (Option TextCursor)))
(defun (text-cursor-find-backward pred r c)
  (let go ((p c))
    (if (at-start? p)
        None
        (begin
          (def q (step-prev p))
          (if (pred (char-at q)) (Some q) (go q))))))

;; Comparisons of two cursors of one text.
(: (-> TextCursor TextCursor bool))
(defun (text-cursor=? a b) (= (.at a) (.at b)))
(: (-> TextCursor TextCursor bool))
(defun (text-cursor<? a b) (< (.at a) (.at b)))
(: (-> TextCursor TextCursor bool))
(defun (text-cursor>? a b) (> (.at a) (.at b)))
(: (-> TextCursor TextCursor bool))
(defun (text-cursor<=? a b) (<= (.at a) (.at b)))
(: (-> TextCursor TextCursor bool))
(defun (text-cursor>=? a b) (>= (.at a) (.at b)))

(impl (Eq TextCursor)
  (defun (= a b) (= (.at a) (.at b)))
  (defun (eq-hash c) (eq-hash (.at c))))

(impl (Ord TextCursor)
  (defun (compare a b) (compare (.at a) (.at b))))

;; The characters from the cursor to the end.
(impl (Iterable TextCursor)
  (type %elem char)
  (type %cursor TextCursor)
  (defun (iterable-start c) c)
  (defun (iterable-done? c p) (at-end? p))
  (defun (iterable-current c p) (char-at p))
  (defun (iterable-next c p) (step-next p)))

;; The characters before a cursor, last first.
(type (: TextBackward #:opaque (Struct (: from TextCursor))))

;; A step of a backward loop: the cursor on its character, or done.
(type (: TextBackwardStep #:opaque (Struct (: on TextCursor) (: done? bool))))

;; The characters before `c`, a cursor of `r`, last first.
(: (-> Text TextCursor TextBackward))
(defun (text-backward r c) (TextBackward (from c)))

(: (-> TextCursor TextBackwardStep))
(defun (backward-from c)
  (if (at-start? c)
      (TextBackwardStep (on c) (done? #t))
      (TextBackwardStep (on (step-prev c)) (done? #f))))

(impl (Iterable TextBackward)
  (type %elem char)
  (type %cursor TextBackwardStep)
  (defun (iterable-start b) (backward-from (.from b)))
  (defun (iterable-done? b s) (.done? s))
  (defun (iterable-current b s) (char-at (.on s)))
  (defun (iterable-next b s) (backward-from (.on s))))

;; ---------------------------------------------------------------------------
;; Grapheme clusters
;; ---------------------------------------------------------------------------

;; The front end of (std unicode grapheme) for texts. Its cursors must be on
;; a cluster boundary, as the string front end's are.

;; The cursor after the cluster that starts at `c`, a cluster boundary of `r`.
;; - Between two bytes of printable ASCII in the chunk, the next byte.
(: (-> Text TextCursor TextCursor))
(defun (text-cursor-next-grapheme r c)
  (check-not-end c "text-cursor-next-grapheme")
  (def sp (.AsSpan (.chunk c)))
  (def k (- (.at c) (.start c)))
  (if (and (< (+ k 1) (ro-span-length sp)) (grapheme-plain-break? (byte-at sp k) (byte-at sp (+ k 1))))
      (record-set c (at (+ (.at c) 1)))
      (next-grapheme-by-rules r c)))

(: (-> Text TextCursor TextCursor))
(defun (next-grapheme-by-rules r c)
  (def (Tuple ch0 c1) (text-cursor-ref+next r c))
  (let go ((st (grapheme-start ch0)) (c c1))
    (if (at-end? c)
        c
        (begin
          (def (Tuple ch nxt) (text-cursor-ref+next r c))
          (def st2 (grapheme-step st ch))
          (if (grapheme-break? st2) c (go st2 nxt))))))

;; The backward state `w` once known, reading back from `k`, the cursor of
;; the last character given.
(: (-> Text int TextCursor int))
(defun (text-back-resolve r w k)
  (let more ((w w) (k k))
    (cond
      ((not (grapheme-back-wants? w)) w)
      ((at-start? k) (grapheme-back-sot w))
      (else
       (begin
         (def k2 (step-prev k))
         (more (grapheme-back-more w (text-cursor-ref r k2)) k2))))))

;; The start of the cluster that ends at `c`, a cluster boundary of `r`.
;; - After two bytes of printable ASCII in the chunk, the byte before.
(: (-> Text TextCursor TextCursor))
(defun (text-cursor-prev-grapheme r c)
  (check-not-start c "text-cursor-prev-grapheme")
  (def sp (.AsSpan (.chunk c)))
  (def k (- (.at c) (.start c)))
  (if (and (>= k 2) (grapheme-plain-break? (byte-at sp (- k 2)) (byte-at sp (- k 1))))
      (record-set c (at (- (.at c) 1)))
      (prev-grapheme-by-rules r c)))

(: (-> Text TextCursor TextCursor))
(defun (prev-grapheme-by-rules r c)
  (def c1 (step-prev c))
  (def b1 (text-cursor-ref r c1))
  (def w0 (if (and (grapheme-back-needs-after? b1) (not (at-end? c)))
              (grapheme-back-start-before (text-cursor-ref r c))
              (grapheme-back-start)))
  ;; `bc` is the cursor of the earliest character of the cluster so far, `b`.
  (let go ((w w0) (bc c1) (b b1))
    (if (at-start? bc)
        bc
        (begin
          (def ac (step-prev bc))
          (def a (text-cursor-ref r ac))
          (def w1 (grapheme-back-pair w a b))
          (def w2 (if (grapheme-back-wants? w1) (text-back-resolve r w1 ac) w1))
          (if (grapheme-back-break? w2) bc (go w2 ac a))))))

;; Whether `c` is on a cluster boundary of `r`.
(: (-> Text TextCursor bool))
(defun (text-cursor-grapheme-boundary? r c)
  (if (or (at-start? c) (at-end? c))
      #t
      (begin
        (def ac (step-prev c))
        (grapheme-back-break?
          (text-back-resolve r (grapheme-back-pair (grapheme-back-start) (text-cursor-ref r ac) (text-cursor-ref r c)) ac)))))

;; The clusters of `r`.
(: (-> Text int))
(defun (text-grapheme-count r)
  (def c0 (text-cursor-start r))
  (if (at-end? c0)
      0
      (begin
        (def (Tuple ch0 c1) (text-cursor-ref+next r c0))
        (let go ((st (grapheme-start ch0)) (c c1) (n 1))
          (if (at-end? c)
              n
              (begin
                (def (Tuple ch nxt) (text-cursor-ref+next r c))
                (def st2 (grapheme-step st ch))
                (go st2 nxt (if (grapheme-break? st2) (+ n 1) n))))))))

;; The clusters of `r`, in order, as texts.
(: (-> Text (Vec Text)))
(defun (text->graphemes r)
  (let go ((c (text-cursor-start r)) (acc '()))
    (if (at-end? c)
        (list->vec (list-reverse acc))
        (begin
          (def d (text-cursor-next-grapheme r c))
          (go d (cons (text-substring r c d) acc))))))

;; ---------------------------------------------------------------------------
;; Transients
;; ---------------------------------------------------------------------------

;; A text that is edited in place: its edits copy only the nodes it does not
;; own yet. `transienttext->text` returns the text it holds and takes a new
;; owner, so that the text does not change after.
;;
;; - Positions are character indices, from 0 to the characters it holds.
;; - The hot chunk: the chunk of the last edit, held in `buf` as bytes 0 to
;;   `hot-len`, with `hot-chars` characters, the first of which is character
;;   `hot-first` of the text. `path` holds the nodes down to it, the leaves
;;   node first, and `slots` the slot taken in each. `hot-len` is -1 when no
;;   chunk is hot.
;; - `mark-char` and `mark-byte` are a character of the hot chunk and its byte
;;   in `buf`, where the last edit in it ended: the next edit steps from there
;;   to its own byte, and a chunk with as many bytes as characters needs no
;;   steps.
;; - An edit inside the hot chunk moves bytes in `buf` and adds to the newline
;;   and character counts on the path, and allocates nothing. The byte counts
;;   on the path stay those of the old string until `cool!`. It must leave the
;;   chunk at most `chunk-max` bytes, and not empty unless the text is one
;;   string.
;; - While a chunk is hot, `bytes`, `lines` and `chars` are the text's, not
;;   those in `text`, and when `dirty?` its slot holds an old string. `cool!`
;;   puts the chunk back; every other operation cools first.
;; - An edit outside the hot chunk makes its own chunk hot only when it is
;;   within `hot-reach` characters of `last`, where the edit before it ended.
;;   Edits far apart edit the tree, which costs less than copying each chunk
;;   in and out for one edit.
(def hot-reach 64)

(type (: TransientText #:opaque
         (Record (: text Text #:mutable)
                 (: owner long #:mutable)
                 (: buf (Array byte))
                 (: path (Array TextNode))
                 (: slots (Array int))
                 (: hot-len int #:mutable)
                 (: hot-chars int #:mutable)
                 (: hot-first int #:mutable)
                 (: mark-char int #:mutable)
                 (: mark-byte int #:mutable)
                 (: dirty? bool #:mutable)
                 (: bytes int #:mutable)
                 (: lines int #:mutable)
                 (: chars int #:mutable)
                 (: last int #:mutable))))

(: (-> Text TransientText))
(defun (text->transienttext r)
  (TransientText (text r) (owner (owner-next))
                 (buf (make-array chunk-max #:fill (cast byte 0)))
                 (path (make-array 24 #:fill no-node))
                 (slots (make-array 24 #:fill 0))
                 (hot-len -1) (hot-chars 0) (hot-first 0) (mark-char 0) (mark-byte 0)
                 (dirty? #f) (bytes 0) (lines 0) (chars 0) (last 0)))

;; Copies the bytes of `s` into `buf` from `at`.
(: (-> string (Array byte) int void))
(defun (copy-into! s buf at) #:inline
  (.CopyTo (.AsSpan s) (span-slice (array->span buf) #:from at)))

;; Whether byte `k` of `buf` continues a character.
(: (-> (Array byte) int bool))
(defun (continues? buf k) #:inline
  (= (bitwise-and (cast int (array-ref buf k)) 192) 128))

;; Makes the chunk with byte `at` hot, when no chunk is: the chunk that ends
;; at `at` when `left?`, and otherwise the one that starts there.
;;
;; - Copies the nodes on the path that the transient does not own.
;; - Returns #f, and leaves no chunk hot, when the chunk does not have `room`
;;   bytes free before `chunk-max`.
(: (-> TransientText int bool int bool))
(defun (heat! t at left? room)
  (def r (.text t))
  (def h (.height r))
  (def o (.owner t))
  (def buf (.buf t))
  (if (= h 0)
      (begin
        (def len (.bytes r))
        (if (> (+ len room) chunk-max)
            #f
            (begin
              (copy-into! (.small r) buf 0)
              (record-set! t (hot-len len) (hot-chars (.chars r)) (hot-first 0) (mark-char 0) (mark-byte 0)
                           (dirty? #f) (bytes len) (lines (.lines r)) (chars (.chars r)))
              #t)))
      (begin
        (def path (.path t))
        (def slots (.slots t))
        (def root (editable h (.root r) o))
        (unless (= (node-owner (.root r)) o)
          (record-set! t (text (Text (root root) (height h) (bytes (.bytes r)) (lines (.lines r)) (chars (.chars r)) (small "")))))
        (let go ((n root) (lvl (- h 1)) (at at) (first 0))
          (def (Tuple i k) (if left? (slot-ending n at) (slot-starting n at)))
          (def f (+ first (chars-before-slot n i)))
          (array-set! path lvl n)
          (array-set! slots lvl i)
          (if (= lvl 0)
              (begin
                (def c (chunk-at (as-leaves n) i))
                (def len (string-length c))
                (if (> (+ len room) chunk-max)
                    #f
                    (begin
                      (copy-into! c buf 0)
                      (record-set! t (hot-len len) (hot-chars (slot-chars n i)) (hot-first f) (mark-char 0) (mark-byte 0)
                                   (dirty? #f) (bytes (.bytes r)) (lines (.lines r)) (chars (.chars r)))
                      #t)))
              (begin
                (def b (as-branch n))
                (def child (editable lvl (kid b i) o))
                (set-kid! b i child)
                (go child (- lvl 1) k f)))))))

;; Adds `dl` newlines and `dc` characters to the slots on the path to the hot
;; chunk.
(: (-> TransientText int int void))
(defun (adjust-path-counts! t dl dc)
  (def path (.path t))
  (def slots (.slots t))
  (loop (:for lvl (range 0 (.height (.text t))))
        (:do (def n (array-ref path lvl))
             (def i (array-ref slots lvl))
             (set-slot-lines! n i (+ (slot-lines n i) dl))
             (set-slot-chars! n i (+ (slot-chars n i) dc)))))

;; Adds `db` bytes to the slots on the path to the hot chunk.
(: (-> TransientText int void))
(defun (adjust-path-bytes! t db)
  (def path (.path t))
  (def slots (.slots t))
  (loop (:for lvl (range 0 (.height (.text t))))
        (:do (add-slot-bytes! (array-ref path lvl) (array-ref slots lvl) db))))

;; Whether an edit at character `n` is near enough the last one to make its
;; chunk hot.
(: (-> TransientText int bool))
(defun (near? t n) #:inline
  (<= (abs (- n (.last t))) hot-reach))

;; The byte in `buf` that `k` characters after byte `b` start at.
(: (-> (Array byte) int int int))
(defun (buf-forward buf b k)
  (let go ((b b) (k k))
    (if (= k 0) b (go (+ b (char-width (cast int (array-ref buf b)))) (- k 1)))))

;; The byte in `buf` that `k` characters before byte `b` start at.
(: (-> (Array byte) int int int))
(defun (buf-back buf b k)
  (let go ((b b) (k k))
    (if (= k 0)
        b
        (go (let back ((p (- b 1))) (if (continues? buf p) (back (- p 1)) p)) (- k 1)))))

;; The byte in `buf` where character `kc` of the hot chunk starts, stepped
;; from the mark.
(: (-> TransientText int int))
(defun (hot-byte t kc)
  (def m (.mark-char t))
  (cond
    ((= (.hot-chars t) (.hot-len t)) kc)
    ((>= kc m) (buf-forward (.buf t) (.mark-byte t) (- kc m)))
    (else (buf-back (.buf t) (.mark-byte t) (- m kc)))))

;; Puts `s`, of `sc` characters, at character `n` when that is in the hot
;; chunk and `s` fits in it, and returns whether it did.
(: (-> TransientText int string int bool))
(defun (hot-insert! t n s sc)
  (def len (.hot-len t))
  (def kc (- n (.hot-first t)))
  (def slen (string-length s))
  (if (and (>= len 0) (>= kc 0) (<= kc (.hot-chars t)) (<= (+ len slen) chunk-max))
      (begin
        (def buf (.buf t))
        (def k (hot-byte t kc))
        (array-copy! buf k buf (+ k slen) (- len k))
        (copy-into! s buf k)
        (def sl (newlines-in s))
        (adjust-path-counts! t sl sc)
        (record-set! t (hot-len (+ len slen)) (hot-chars (+ (.hot-chars t) sc))
                     (mark-char (+ kc sc)) (mark-byte (+ k slen)) (dirty? #t)
                     (bytes (+ (.bytes t) slen)) (lines (+ (.lines t) sl)) (chars (+ (.chars t) sc)))
        #t)
      #f))

;; Deletes characters `from` to `to`, where `from` < `to`, when they are in
;; the hot chunk and leave some of it in a tree, and returns whether it did.
(: (-> TransientText int int bool))
(defun (hot-delete! t from to)
  (def len (.hot-len t))
  (def hc (.hot-chars t))
  (def kc (- from (.hot-first t)))
  (def ec (- to (.hot-first t)))
  (if (and (>= len 0) (>= kc 0) (<= ec hc) (or (< (- ec kc) hc) (= (.height (.text t)) 0)))
      (begin
        (def buf (.buf t))
        (def dc (- ec kc))
        (def k (hot-byte t kc))
        (def e (if (= hc len) (+ k dc) (buf-forward buf k dc)))
        (def dl (span-count (ro-span-slice (array->ro-span buf) #:from k #:to e) newline))
        (array-copy! buf e buf k (- len e))
        (adjust-path-counts! t (- dl) (- dc))
        (record-set! t (hot-len (- len (- e k))) (hot-chars (- hc dc)) (mark-char kc) (mark-byte k) (dirty? #t)
                     (bytes (- (.bytes t) (- e k))) (lines (- (.lines t) dl)) (chars (- (.chars t) dc)))
        #t)
      #f))

;; Puts the hot chunk back in its slot, if there is one.
;;
;; - A chunk left shorter than `chunk-min` is merged with a neighbour, and
;;   nodes left with fewer than `min-kids` children on the path then too, as
;;   a delete does.
(: (-> TransientText void))
(defun (cool! t)
  (when (and (>= (.hot-len t) 0) (.dirty? t))
    (def r (.text t))
    (def h (.height r))
    (def len (.hot-len t))
    (def c (utf8->string (ro-span-slice (array->ro-span (.buf t)) #:from 0 #:to len)))
    (if (= h 0)
        (record-set! t (text (small-text c)))
        (begin
          (def path (.path t))
          (def slots (.slots t))
          (def o (.owner t))
          (def leaves (array-ref path 0))
          (adjust-path-bytes! t (- len (slot-bytes leaves (array-ref slots 0))))
          (set-chunk! (as-leaves leaves) (array-ref slots 0) c)
          (when (< len chunk-min)
            (merge-small-chunk! leaves (array-ref slots 0))
            (loop (:for lvl (range 1 h))
                  (:do (when (< (node-count (array-ref path (- lvl 1))) min-kids)
                         (fix-underflow! (array-ref path lvl) (+ lvl 1) (array-ref slots lvl) o)))))
          (record-set! t (text (tree->text-sized (.root r) h (.bytes t) (.lines t) (.chars t)))))))
  (record-set! t (hot-len -1) (dirty? #f)))

(: (-> TransientText Text))
(defun (transienttext->text t)
  (cool! t)
  (record-set! t (owner (owner-next)))
  (.text t))

;; The bytes of the text it holds.
(: (-> TransientText int))
(defun (transienttext-length t)
  (if (>= (.hot-len t) 0) (.bytes t) (.bytes (.text t))))

;; The characters of the text it holds.
(: (-> TransientText int))
(defun (transienttext-count t)
  (if (>= (.hot-len t) 0) (.chars t) (.chars (.text t))))

;; Raises that character index `n` is outside a text of `chars` characters.
;; Not inlined, as `cursor-at-edge`.
(: (-> string int int void))
(defun (index-outside who n chars) #:no-inline
  (out-of-range #"${who}: the index ${n} is outside the text, of ${chars} characters."))

(: (-> Text int string void))
(defun (check-index r n who)
  (when (or (< n 0) (> n (.chars r))) (index-outside who n (.chars r))))

;; Puts `s` before character `n`.
(: (-> TransientText int string void))
(defun (transienttext-insert! t n s)
  (def slen (string-length s))
  (def sc (chars-in s))
  (unless (hot-insert! t n s sc)
    (cool! t)
    (check-index (.text t) n "transienttext-insert!")
    (def at (byte-of-char (.text t) n))
    (unless (and (> slen 0) (<= slen small-insert) (near? t n) (heat! t at #t slen) (hot-insert! t n s sc))
      (cool! t)
      (record-set! t (text (insert-with (.text t) at s (.owner t))))))
  (record-set! t (last (+ n sc))))

(: (-> TransientText string void))
(defun (transienttext-append! t s)
  (transienttext-insert! t (transienttext-count t) s))

;; Deletes characters `from` to `to`.
(: (-> TransientText int int void))
(defun (transienttext-delete! t from to)
  (unless (and (< from to) (hot-delete! t from to))
    (cool! t)
    (def r (.text t))
    (check-index r from "transienttext-delete!")
    (check-index r to "transienttext-delete!")
    (when (> from to) (backward-range "transienttext-delete!"))
    (when (< from to)
      (def a (byte-of-char r from))
      (unless (and (near? t from) (heat! t a #f 0) (hot-delete! t from to))
        (cool! t)
        (def r2 (.text t))
        (record-set! t (text (delete-with r2 a (byte-of-char r2 to) (.owner t)))))))
  (record-set! t (last from)))

;; Replaces characters `from` to `to` with `s`.
(: (-> TransientText int int string void))
(defun (transienttext-replace-range! t from to s)
  (transienttext-delete! t from to)
  (transienttext-insert! t from s))

;; ---------------------------------------------------------------------------
;; Checks
;; ---------------------------------------------------------------------------

;; What is wrong with the tree under `n`, of height `h`, if anything: the
;; first problem found.
(: (-> TextNode int bool (Option string)))
(defun (check-node n h root?)
  (def count (node-count n))
  (cond
    ((> count cap) (Some #"a node of height ${h} has ${count} children"))
    ((and (not root?) (< count min-kids)) (Some #"a node of height ${h} has ${count} children, fewer than ${min-kids}"))
    ((= count 0) (Some #"a node of height ${h} is empty"))
    (else
     (let go ((i 0))
       (if (>= i count)
           None
           (if (= h 1)
               (begin
                 (def c (chunk-at (as-leaves n) i))
                 (cond
                   ((= (string-length c) 0) (Some "an empty chunk"))
                   ((> (string-length c) chunk-max) (Some #"a chunk of ${(string-length c)} bytes"))
                   ((not (= (slot-bytes n i) (string-length c))) (Some "a chunk's byte count is wrong"))
                   ((not (= (slot-lines n i) (newlines-in c))) (Some "a chunk's newline count is wrong"))
                   ((not (= (slot-chars n i) (chars-in c))) (Some "a chunk's character count is wrong"))
                   (else (go (+ i 1)))))
               (begin
                 (def k (kid (as-branch n) i))
                 (cond
                   ((not (= (slot-bytes n i) (sum-bytes k))) (Some #"a node of height ${h} has a wrong byte count"))
                   ((not (= (slot-lines n i) (sum-lines k))) (Some #"a node of height ${h} has a wrong newline count"))
                   ((not (= (slot-chars n i) (sum-chars k))) (Some #"a node of height ${h} has a wrong character count"))
                   (else
                    (match (check-node k (- h 1) #f)
                      ((Some why) (Some why))
                      (None (go (+ i 1)))))))))))))

;; What is wrong with `r`, if anything. For tests.
(: (-> Text (Option string)))
(defun (text-check r)
  (cond
    ((= (.height r) 0)
     (cond
       ((> (.bytes r) chunk-max) (Some "a text of one string is longer than a chunk"))
       ((not (= (.bytes r) (string-length (.small r)))) (Some "the byte count is wrong"))
       ((not (= (.lines r) (newlines-in (.small r)))) (Some "the newline count is wrong"))
       ((not (= (.chars r) (chars-in (.small r)))) (Some "the character count is wrong"))
       (else None)))
    ((not (= (.bytes r) (sum-bytes (.root r)))) (Some "the byte count is wrong"))
    ((not (= (.lines r) (sum-lines (.root r)))) (Some "the newline count is wrong"))
    ((not (= (.chars r) (sum-chars (.root r)))) (Some "the character count is wrong"))
    ((<= (.bytes r) chunk-max) (Some "a tree holds no more than one chunk"))
    ((and (> (.height r) 1) (< (node-count (.root r)) 2)) (Some "a root branch has one child"))
    (else (check-node (.root r) (.height r) #t))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/unicode/casefold.bjo — full Unicode case folding (CaseFolding.txt).
;;
;; - The fold is the default, culture-independent one: the mappings of status
;;   C and F. The Turkic mappings (status T) are not used.
;; - The table is generated by gen_casefold.py into casefold-table.protobjo,
;;   and decoded once when the module is initialized.
;;
;; Cost:
;; - ASCII is folded without the table. Other characters take a binary
;;   search over the 1,606 mappings.
;; - A string that has nothing to fold is returned as it is, with no copy.

(export
  casefold-unicode-version
  string-foldcase)

(import (std prelude))

(:doc #:file "casefold.bjodoc")

(include "casefold-table.protobjo")

;; ---------------------------------------------------------------------------
;; The table
;; ---------------------------------------------------------------------------

(: string)
(def casefold-unicode-version unicode-version)

;; The value of a hex digit, as a byte of ASCII.
(: (-> byte int))
(defun (hex-value b)
  (def x (cast int b))
  (if (< x 97) (- x 48) (- x 87)))

;; The bytes of `text` that are not white space: the generated text is
;; indented, and every character it holds is '0' or above.
(: (-> string (Array byte)))
(defun (table-bytes text)
  (def src (string->bytes text))
  (def n (loop (:for b src) (:when (>= (cast int b) 48)) (:acc (summing 1))))
  (def (: out (Array byte)) (make-array n))
  (let go ((i 0) (j 0))
    (cond
      ((>= i (array-length src)) out)
      ((< (cast int (array-ref src i)) 48) (go (+ i 1) j))
      (else (array-set! out j (array-ref src i)) (go (+ i 1) (+ j 1))))))

(: (Array byte))
(def fold-digits (table-bytes fold-text))

;; The value of the `n` hex digits at `i` in `fold-digits`.
(: (-> int int int))
(defun (hex-at i n)
  (let go ((k 0) (v 0))
    (if (= k n)
        v
        (go (+ k 1) (+ (* 16 v) (hex-value (array-ref fold-digits (+ i k))))))))

;; The code points that fold, in order.
(: (Array int))
(def fold-keys (make-array fold-count))

;; Mapping `i` folds to `fold-targets` from `(array-ref fold-starts i)` up to
;; `(array-ref fold-starts (+ i 1))`.
(: (Array int))
(def fold-starts (make-array (+ fold-count 1)))

;; What the code points fold to. Filling it also fills `fold-keys` and
;; `fold-starts`.
(: (Array int))
(def fold-targets
  (begin
    (def (: out (Array int)) (make-array (/ (- (array-length fold-digits) (* 7 fold-count)) 6)))
    (let go ((m 0) (i 0) (t 0))
      (array-set! fold-starts m t)
      (when (< m fold-count)
        (array-set! fold-keys m (hex-at i 6))
        (def n (hex-at (+ i 6) 1))
        (loop (:for k (range 0 n))
              (:do (array-set! out (+ t k) (hex-at (+ i 7 (* 6 k)) 6))))
        (go (+ m 1) (+ i 7 (* 6 n)) (+ t n))))
    out))

;; The index of the mapping of `cp`, or -1 if `cp` does not fold.
(: (-> int int))
(defun (fold-index cp)
  (let go ((lo 0) (hi fold-count))
    (if (>= lo hi)
        -1
        (let* ((mid (shift-right (+ lo hi) 1))
               (k (array-ref fold-keys mid)))
          (cond
            ((< cp k) (go lo mid))
            ((> cp k) (go (+ mid 1) hi))
            (else mid))))))

(: (-> int bool))
(defun (ascii-upper? cp) #:inline (and (>= cp 65) (<= cp 90)))

;; True if `ch` folds to something other than itself.
(: (-> char bool))
(defun (folds? ch) #:inline
  (def cp (char->int ch))
  (if (< cp 128) (ascii-upper? cp) (>= (fold-index cp) 0)))

;; Adds the fold of `ch` to `sb`.
(: (-> StringBuilder char void))
(defun (add-folded! sb ch)
  (def cp (char->int ch))
  (if (< cp 128)
      (stringbuilder-add-byte! sb (if (ascii-upper? cp) (+ cp 32) cp))
      (let ((i (fold-index cp)))
        (if (< i 0)
            (stringbuilder-add! sb ch)
            (loop (:for k (range (array-ref fold-starts i) (array-ref fold-starts (+ i 1))))
                  (:do (stringbuilder-add! sb (int->char (array-ref fold-targets k)))))))))

;; ---------------------------------------------------------------------------
;; Strings
;; ---------------------------------------------------------------------------

;; The fold of `s` from the cursor `from` on, after the part before it, which
;; has nothing to fold.
(: (-> string StringCursor string))
(defun (fold-from s from)
  (def sb (stringbuilder-empty))
  (stringbuilder-add-substring! sb s (string-cursor-start s) from)
  (let go ((c from))
    (if (string-cursor-end? s c)
        (stringbuilder->string sb)
        (begin
          (def (Tuple ch next) (string-cursor-ref+next s c))
          (add-folded! sb ch)
          (go next)))))

(: (-> string string))
(defun (string-foldcase s)
  (let scan ((c (string-cursor-start s)))
    (if (string-cursor-end? s c)
        s
        (begin
          (def (Tuple ch next) (string-cursor-ref+next s c))
          (if (folds? ch)
              (fold-from s c)
              (scan next))))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/unicode/grapheme.bjo — extended grapheme clusters (UAX #29).
;;
;; - The rules are two state machines over characters, which know nothing of
;;   the container: a forward step, and a backward one that decides the
;;   boundary between two characters and asks for the characters before them
;;   when a rule needs them. A front end drives them with its own cursors.
;; - This module has the front end for strings. (std text) has the one for
;;   texts.
;; - The property table is generated by gen_grapheme.py into
;;   grapheme-table.protobjo.
;;
;; Cost:
;; - A step forward reads each character of the cluster once.
;; - A step back reads the cluster, and, for the three rules with context
;;   (GB9c, GB11, GB12/13), the run of characters the rule looks back over.
;;   A run of regional indicators is counted once: from a boundary followed by
;;   one, the parity is known, so a loop back over a text takes O(n) time.

(export
  grapheme-unicode-version
  ;; The forward state machine
  grapheme-plain-break?
  grapheme-start grapheme-step grapheme-break?
  ;; The backward state machine
  grapheme-back-start grapheme-back-needs-after? grapheme-back-start-before grapheme-back-pair
  grapheme-back-wants? grapheme-back-more grapheme-back-sot grapheme-back-break?
  ;; Strings
  string-cursor-next-grapheme string-cursor-prev-grapheme string-cursor-grapheme-boundary?
  string-grapheme-count string->graphemes)

(import (std prelude))

(:doc #:file "grapheme.bjodoc")

(import/class
  (ArgumentOutOfRangeException (: System.ArgumentOutOfRangeException))
  (Exception (: System.Exception)))

(include "grapheme-table.protobjo")

;; ---------------------------------------------------------------------------
;; The property table
;; ---------------------------------------------------------------------------

(: string)
(def grapheme-unicode-version unicode-version)

;; The value of a hex digit, as a byte of ASCII.
(: (-> byte int))
(defun (hex-value b)
  (def x (cast int b))
  (if (< x 97) (- x 48) (- x 87)))

;; The bytes of `text` that are not white space: the generated texts are
;; indented, and every character they hold is '0' or above.
(: (-> string (Array byte)))
(defun (table-bytes text)
  (def src (string->bytes text))
  (def n (loop (:for b src) (:when (>= (cast int b) 48)) (:acc (summing 1))))
  (def (: out (Array byte)) (make-array n))
  (let go ((i 0) (j 0))
    (cond
      ((>= i (array-length src)) out)
      ((< (cast int (array-ref src i)) 48) (go (+ i 1) j))
      (else (array-set! out j (array-ref src i)) (go (+ i 1) (+ j 1))))))

;; Bytes written as two hex digits each.
(: (-> string (Array byte)))
(defun (hex-bytes text)
  (def digits (table-bytes text))
  (def (: out (Array byte)) (make-array (/ (array-length digits) 2)))
  (loop (:for i (range 0 (array-length out)))
        (:do (array-set! out i (cast byte (+ (* 16 (hex-value (array-ref digits (* 2 i))))
                                            (hex-value (array-ref digits (+ (* 2 i) 1))))))))
  out)

;; The block of each 256 code points.
(: (Array byte))
(def stage1 (hex-bytes stage1-text))

;; The property bytes of the blocks, one a code point.
(: (Array byte))
(def stage2
  (begin
    (def kinds (hex-bytes kinds-text))
    (def codes (table-bytes stage2-text))
    (def (: out (Array byte)) (make-array (array-length codes)))
    (loop (:for i (range 0 (array-length codes)))
          (:do (array-set! out i (array-ref kinds (- (cast int (array-ref codes i)) 48)))))
    out))

;; The property byte of a character:
;; - bits 0-3, Grapheme_Cluster_Break (the gcb- constants);
;; - bit 4, Extended_Pictographic;
;; - bits 5-6, Indic_Conjunct_Break (the incb- constants).
(: (-> char int))
(defun (props-of ch) #:inline
  (def cp (char->int ch))
  (cast int (array-ref stage2 (+ (shift-left (cast int (array-ref stage1 (shift-right cp 8))) 8)
                                 (bitwise-and cp 255)))))

(def gcb-cr 1)
(def gcb-lf 2)
(def gcb-control 3)
(def gcb-extend 4)
(def gcb-zwj 5)
(def gcb-ri 6)
(def gcb-prepend 7)
(def gcb-spacing-mark 8)
(def gcb-l 9)
(def gcb-v 10)
(def gcb-t 11)
(def gcb-lv 12)
(def gcb-lvt 13)

(def incb-linker 1)
(def incb-consonant 2)
(def incb-extend 3)

(: (-> int int))
(defun (gcb p) #:inline (bitwise-and p 15))

(: (-> int bool))
(defun (pictographic? p) #:inline (not (= (bitwise-and p 16) 0)))

(: (-> int int))
(defun (incb p) #:inline (bitwise-and (shift-right p 5) 3))

;; Whether a code point is printable ASCII: GCB=Other, and none of the
;; other properties.
(: (-> int bool))
(defun (printable-ascii? cp) #:inline (and (>= cp 32) (< cp 127)))

;; Whether the rules put a boundary between code points `a` and `b` with no
;; context: both are printable ASCII. A front end can test bytes with it.
(: (-> int int bool))
(defun (grapheme-plain-break? a b) #:inline (and (printable-ascii? a) (printable-ascii? b)))

;; ---------------------------------------------------------------------------
;; The rules
;; ---------------------------------------------------------------------------

;; What the rules say of the boundary between two characters, by their
;; properties alone.
(def rule-join 0)
(def rule-break 1)
;; GB9c: a Linker, then InCB=Extend characters, then a Consonant.
(def rule-conjunct 2)
;; GB11: a pictograph, then Extend characters and a ZWJ, then a pictograph.
(def rule-emoji 3)
;; GB12/13: an odd number of regional indicators before the second.
(def rule-flag 4)

;; The rule for the boundary between characters of properties `pa` and `pb`:
;; join, break, or which rule with context decides it. UAX #29 3.1.1, in
;; order.
(: (-> int int int))
(defun (pair-rule pa pb)
  (def a (gcb pa))
  (def b (gcb pb))
  (cond
    ;; GB3, GB4, GB5
    ((and (= a gcb-cr) (= b gcb-lf)) rule-join)
    ((or (= a gcb-cr) (= a gcb-lf) (= a gcb-control)) rule-break)
    ((or (= b gcb-cr) (= b gcb-lf) (= b gcb-control)) rule-break)
    ;; GB6, GB7, GB8
    ((and (= a gcb-l) (or (= b gcb-l) (= b gcb-v) (= b gcb-lv) (= b gcb-lvt))) rule-join)
    ((and (or (= a gcb-lv) (= a gcb-v)) (or (= b gcb-v) (= b gcb-t))) rule-join)
    ((and (or (= a gcb-lvt) (= a gcb-t)) (= b gcb-t)) rule-join)
    ;; GB9, GB9a, GB9b
    ((or (= b gcb-extend) (= b gcb-zwj) (= b gcb-spacing-mark)) rule-join)
    ((= a gcb-prepend) rule-join)
    ;; GB9c, GB11, GB12/13
    ((and (= (incb pb) incb-consonant) (or (= (incb pa) incb-linker) (= (incb pa) incb-extend))) rule-conjunct)
    ((and (= a gcb-zwj) (pictographic? pb)) rule-emoji)
    ((and (= a gcb-ri) (= b gcb-ri)) rule-flag)
    ;; GB999
    (else rule-break)))

;; ---------------------------------------------------------------------------
;; Forward
;; ---------------------------------------------------------------------------

;; A forward state is an int:
;; - bits 0-6, the properties of the last character;
;; - bit 7, the text ends in a pictograph and Extend characters (GB11);
;; - bit 8, it ends in those and a ZWJ (GB11);
;; - bit 9, it ends in an odd number of regional indicators (GB12/13);
;; - bit 10, it ends in a Linker and InCB=Extend characters (GB9c);
;; - bit 11, there is a boundary before the last character.
(def fwd-pict 128)
(def fwd-pict-zwj 256)
(def fwd-odd-ri 512)
(def fwd-linker 1024)
(def fwd-break 2048)

(: (-> int int bool))
(defun (has? st bit) #:inline (not (= (bitwise-and st bit) 0)))

;; The state after a character of properties `p`, from the state `st` before
;; it.
(: (-> int int bool int))
(defun (state-after p st brk)
  (def g (gcb p))
  (def i (incb p))
  (bitwise-ior
    p
    (if (or (pictographic? p) (and (= g gcb-extend) (has? st fwd-pict))) fwd-pict 0)
    (if (and (= g gcb-zwj) (has? st fwd-pict)) fwd-pict-zwj 0)
    (if (and (= g gcb-ri) (not (and (= (gcb st) gcb-ri) (has? st fwd-odd-ri)))) fwd-odd-ri 0)
    (if (or (= i incb-linker) (and (= i incb-extend) (has? st fwd-linker))) fwd-linker 0)
    (if brk fwd-break 0)))

;; The state after the first character of a text, or of a cluster.
(: (-> char int))
(defun (grapheme-start ch) (state-after (props-of ch) 0 #t))

;; The state after `ch`, from the state after the character before it.
;; - Before printable ASCII, the rules break unless the character before is
;;   Prepend (GB9b), and the state after it is that of Other.
(: (-> int char int))
(defun (grapheme-step st ch) #:inline
  (if (printable-ascii? (char->int ch))
      (if (= (gcb st) gcb-prepend) 0 fwd-break)
      (grapheme-step-table st ch)))

(: (-> int char int))
(defun (grapheme-step-table st ch)
  (def p (props-of ch))
  (def r (pair-rule (bitwise-and st 127) p))
  (state-after p st
    (cond
      ((= r rule-join) #f)
      ((= r rule-break) #t)
      ((= r rule-conjunct) (not (has? st fwd-linker)))
      ((= r rule-emoji) (not (has? st fwd-pict-zwj)))
      (else (not (has? st fwd-odd-ri))))))

;; Whether there is a boundary before the last character of a state.
(: (-> int bool))
(defun (grapheme-break? st) #:inline (has? st fwd-break))

;; ---------------------------------------------------------------------------
;; Backward
;; ---------------------------------------------------------------------------

;; A backward state is an int:
;; - bits 0-1, the answer: 0 join, 1 break, 2 not known yet;
;; - bits 2-4, while not known, the rule that is looking back;
;; - bit 5, the parity of the regional indicators that end at the first
;;   character given so far is known; bit 6, it is odd;
;; - bit 7, while counting regional indicators, an odd number so far.
(def back-join 0)
(def back-break 1)
(def back-wants 2)
(def back-known 32)
(def back-odd 64)
(def back-count-odd 128)

;; The answer `answer`, where the regional indicators that end at the first
;; character given number `odd`: #t odd, #f even, or unknown with `known` #f.
(: (-> int bool bool int))
(defun (back-answer answer known odd)
  (bitwise-ior answer (if known back-known 0) (if odd back-odd 0)))

;; The backward state of a cursor at the end of a text, or of one where
;; nothing is known of what follows.
(: (-> int))
(defun (grapheme-back-start) 0)

;; Whether the backward state of a cursor needs the character after it, given
;; `b`, the character before it: only when `b` is a regional indicator, as
;; the parity of those that end at `b` matters only then.
(: (-> char bool))
(defun (grapheme-back-needs-after? b) #:inline
  (and (not (printable-ascii? (char->int b))) (= (gcb (props-of b)) gcb-ri)))

;; The backward state of a cursor on a cluster boundary, before `ch`.
;; - A boundary before a regional indicator has an even number of them
;;   before it, which saves counting them.
(: (-> char int))
(defun (grapheme-back-start-before ch)
  (if (and (not (printable-ascii? (char->int ch))) (= (gcb (props-of ch)) gcb-ri)) back-known 0))

;; The state for the boundary between `a` and `b`, where `b` is the first
;; character given before, or the one before the cursor.
;; - Known at once, unless a rule needs the characters before `a`: then
;;   grapheme-back-wants? is #t.
;; - Between two characters of printable ASCII, a break, and no regional
;;   indicators end at `a`.
(: (-> int char char int))
(defun (grapheme-back-pair w a b) #:inline
  (if (grapheme-plain-break? (char->int a) (char->int b))
      (bitwise-ior back-break back-known)
      (grapheme-back-pair-table w a b)))

(: (-> int char char int))
(defun (grapheme-back-pair-table w a b)
  (def pa (props-of a))
  (def pb (props-of b))
  (def r (pair-rule pa pb))
  (def a-ri (= (gcb pa) gcb-ri))
  ;; The parity at `a`, from that at `b`, when it follows without counting.
  (def known (or (not a-ri) (and (= (gcb pb) gcb-ri) (has? w back-known))))
  (def odd (and a-ri (= (gcb pb) gcb-ri) (not (has? w back-odd))))
  (cond
    ((= r rule-join) (back-answer back-join known odd))
    ((= r rule-break) (back-answer back-break known odd))
    ((and (= r rule-conjunct) (= (incb pa) incb-linker)) (back-answer back-join known odd))
    ((and (= r rule-flag) known) (back-answer (if odd back-join back-break) known odd))
    ;; Count the indicators before `a`, starting with `a`.
    ((= r rule-flag) (bitwise-ior back-wants (shift-left r 2) back-count-odd))
    (else (bitwise-ior back-wants (shift-left r 2)))))

;; Whether the state needs the character before the last one given.
(: (-> int bool))
(defun (grapheme-back-wants? w) #:inline (= (bitwise-and w 3) back-wants))

;; The state after `ch`, the character before the last one given.
(: (-> int char int))
(defun (grapheme-back-more w ch)
  (def p (props-of ch))
  (def r (bitwise-and (shift-right w 2) 7))
  (cond
    ((= r rule-conjunct)
     (cond
       ((= (incb p) incb-extend) w)
       ((= (incb p) incb-linker) (back-answer back-join #t #f))
       (else (back-answer back-break #t #f))))
    ((= r rule-emoji)
     (cond
       ((= (gcb p) gcb-extend) w)
       ((pictographic? p) (back-answer back-join #t #f))
       (else (back-answer back-break #t #f))))
    ((= (gcb p) gcb-ri) (bitwise-xor w back-count-odd))
    (else (grapheme-back-sot w))))

;; The state when there is no character before the last one given.
(: (-> int int))
(defun (grapheme-back-sot w)
  (def r (bitwise-and (shift-right w 2) 7))
  (if (= r rule-flag)
      (let ((odd (has? w back-count-odd)))
        (back-answer (if odd back-join back-break) #t odd))
      (back-answer back-break #t #f)))

;; Whether the state, once known, is a boundary.
(: (-> int bool))
(defun (grapheme-back-break? w) #:inline (= (bitwise-and w 3) back-break))

;; ---------------------------------------------------------------------------
;; Strings
;; ---------------------------------------------------------------------------

(: (-> string %a))
(defun (out-of-range why) #:no-inline
  (raise (cast Exception (ArgumentOutOfRangeException. "c" why))))

;; The cursor after the cluster that starts at `c`, a cluster boundary of `s`.
(: (-> string StringCursor StringCursor))
(defun (string-cursor-next-grapheme s c)
  (when (string-cursor-end? s c) (out-of-range "string-cursor-next-grapheme: the cursor is at the end."))
  (def (Tuple ch0 c1) (string-cursor-ref+next s c))
  (let go ((st (grapheme-start ch0)) (c c1))
    (if (string-cursor-end? s c)
        c
        (begin
          (def (Tuple ch nxt) (string-cursor-ref+next s c))
          (def st2 (grapheme-step st ch))
          (if (grapheme-break? st2) c (go st2 nxt))))))

;; The state `w` once known, reading back from `k`, the cursor of the last
;; character given.
(: (-> string StringCursor int StringCursor int))
(defun (string-back-resolve s start w k)
  (let more ((w w) (k k))
    (cond
      ((not (grapheme-back-wants? w)) w)
      ((string-cursor=? k start) (grapheme-back-sot w))
      (else
       (begin
         (def k2 (string-cursor-prev s k))
         (more (grapheme-back-more w (string-cursor-ref s k2)) k2))))))

;; The start of the cluster that ends at `c`, a cluster boundary of `s`.
(: (-> string StringCursor StringCursor))
(defun (string-cursor-prev-grapheme s c)
  (def start (string-cursor-start s))
  (when (string-cursor=? c start) (out-of-range "string-cursor-prev-grapheme: the cursor is at the start."))
  (def c1 (string-cursor-prev s c))
  (def b1 (string-cursor-ref s c1))
  (def w0 (if (and (grapheme-back-needs-after? b1) (not (string-cursor-end? s c)))
              (grapheme-back-start-before (string-cursor-ref s c))
              (grapheme-back-start)))
  ;; `bc` is the cursor of the earliest character of the cluster so far, `b`.
  (let go ((w w0) (bc c1) (b b1))
    (if (string-cursor=? bc start)
        bc
        (begin
          (def ac (string-cursor-prev s bc))
          (def a (string-cursor-ref s ac))
          (def w1 (grapheme-back-pair w a b))
          (def w2 (if (grapheme-back-wants? w1) (string-back-resolve s start w1 ac) w1))
          (if (grapheme-back-break? w2) bc (go w2 ac a))))))

;; Whether `c` is on a cluster boundary of `s`.
(: (-> string StringCursor bool))
(defun (string-cursor-grapheme-boundary? s c)
  (def start (string-cursor-start s))
  (if (or (string-cursor=? c start) (string-cursor-end? s c))
      #t
      (begin
        (def ac (string-cursor-prev s c))
        (grapheme-back-break?
          (string-back-resolve s start (grapheme-back-pair (grapheme-back-start) (string-cursor-ref s ac) (string-cursor-ref s c)) ac)))))

;; The clusters of `s`.
(: (-> string int))
(defun (string-grapheme-count s)
  (def c0 (string-cursor-start s))
  (if (string-cursor-end? s c0)
      0
      (begin
        (def (Tuple ch0 c1) (string-cursor-ref+next s c0))
        (let go ((st (grapheme-start ch0)) (c c1) (n 1))
          (if (string-cursor-end? s c)
              n
              (begin
                (def (Tuple ch nxt) (string-cursor-ref+next s c))
                (def st2 (grapheme-step st ch))
                (go st2 nxt (if (grapheme-break? st2) (+ n 1) n))))))))

;; The clusters of `s`, in order.
(: (-> string (Vec string)))
(defun (string->graphemes s)
  (let go ((c (string-cursor-start s)) (acc '()))
    (if (string-cursor-end? s c)
        (list->vec (list-reverse acc))
        (begin
          (def d (string-cursor-next-grapheme s c))
          (go d (cons (substring s c d) acc))))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/units.bjo — units of measure: unit tables, the unit expression parser,
;; and measurements at run time.

(import (std prelude) (std maths) (std rational) (std orderedmap))

(:doc #:file "units.bjodoc")

(export
  ;; Factors
  Factor factor factor-rational factor-pi factor-one factor-one?
  factor* factor/ factor-expt factor->double
  ;; Atoms and resolved unit expressions
  Atom atom atom-name atom-dims atom-factor
  Resolved resolved-coeff resolved-term resolved-atoms
  resolved-dims resolved-factor
  ;; Tables
  UnitTable PrefixUse
  empty-unit-table def-dimension def-unit def-prefix def-preferred
  unit-table-names standard-units
  ;; Resolving
  resolve-unit conformable-units
  ;; Measurements
  Measurement measurement measurement-value measurement-dims
  string->measurement measurement+ measurement- measurement* measurement/
  measurement-scale measurement-expt measurement-sqrt conformable?
  measurement-in measurement->string measurement->string/mixed
  convert-query dims->string
  ;; Temperatures
  Celsius Fahrenheit celsius->kelvin kelvin->celsius
  fahrenheit->kelvin kelvin->fahrenheit
  ;; Logarithmic
  dBW dBm dBV dBSPL dBW->power dBm->power dBV->voltage dBSPL->pressure)

(import/class
  (Exception (: System.Exception))
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

(import/extern
  (clr-invariant (: System.Globalization.CultureInfo.InvariantCulture #:get)))

(: (-> string %a))
(defun (argument-error what)
  (raise (cast Exception (ArgumentException. (str "(std units): " what)))))

;; A Result's value, or ArgumentException with its message.
(: (-> (Result string %a) %a))
(defun (ok-or-raise r)
  (match r
    ((Ok v) v)
    ((Err e) (argument-error e))))

(: (-> int bigint))
(defun (int->bigint n) (cast System.Numerics.BigInteger n))

(def rational-one (rational 1 1))

(: (-> (Tuple %a %b) %a))
(defun (fst t) (match t ((Tuple a _) a)))

(: (-> (Tuple %a %b) %b))
(defun (snd t) (match t ((Tuple _ b) b)))

;; ---------------------------------------------------------------------------
;; Factors
;; ---------------------------------------------------------------------------

;; rat × pi^pi-power.
(type (: Factor #:opaque (Struct (: rat Rational) (: pi-power int))))

(: (-> Rational int Factor))
(defun (factor r k) (Factor (rat r) (pi-power k)))

(: (-> Factor Rational))
(defun (factor-rational f) (struct-ref f rat))

(: (-> Factor int))
(defun (factor-pi f) (struct-ref f pi-power))

(: factor-one Factor)
(def factor-one (factor rational-one 0))

(: (-> Factor bool))
(defun (factor-one? f) (= f factor-one))

(: (-> Factor Factor Factor))
(defun (factor* a b)
  (factor (rational* (factor-rational a) (factor-rational b)) (+ (factor-pi a) (factor-pi b))))

(: (-> Factor Factor Factor))
(defun (factor/ a b)
  (factor (rational/ (factor-rational a) (factor-rational b)) (- (factor-pi a) (factor-pi b))))

(: (-> Factor double))
(defun (factor->double f)
  (* (rational->double (factor-rational f)) (pow pi (cast System.Double (factor-pi f)))))

(: (-> bigint int))
(defun (bit-count n)
  (let go ((n n) (k 0))
    (if (zero? n) k (go (shift-right n 1) (+ k 1)))))

;; The exact b-th root of n >= 0, or None.
(: (-> bigint int (Option bigint)))
(defun (exact-root n b)
  (if (or (< n 2I) (= b 1))
      (Some n)
      (let* ((bb (int->bigint b))
             (start (shift-left 1I (+ (/ (bit-count n) b) 1)))
             (r (let go ((x start))
                  (def y (/ (+ (* (- bb 1I) x) (/ n (expt x (- b 1)))) bb))
                  (if (>= y x) x (go y)))))
        (if (= (expt r b) n) (Some r) None))))

;; f^q, or None when the result is irrational, or q is negative and f zero.
(: (-> Factor Rational (Option Factor)))
(defun (factor-expt f q)
  (def p (cast System.Int32 (rational-num q)))
  (def s (cast System.Int32 (rational-den q)))
  (def k (* (factor-pi f) p))
  (def n (rational-num (factor-rational f)))
  (def d (rational-den (factor-rational f)))
  (cond
    ((not (= 0 (% k s))) None)
    ((and (zero? n) (< p 0)) None)
    ((and (< n 0I) (even? s)) None)
    (else
     (match (Tuple (exact-root (abs n) s) (exact-root d s))
       ((Tuple (Some a) (Some b))
        (Some (factor (rational-expt (rational (if (< n 0I) (- 0I a) a) b) p) (/ k s))))
       (_ None)))))

;; ---------------------------------------------------------------------------
;; Exponent maps: dimensions and unit terms
;; ---------------------------------------------------------------------------
;; A map from a name to a nonzero rational exponent. Empty is 1.

(: (-> (OrderedMap string Rational) (OrderedMap string Rational) (OrderedMap string Rational)))
(defun (exps* a b)
  (orderedmap-filter (fun (k v) (not (rational-zero? v)))
                     (orderedmap-merge-with (fun (k x y) (rational+ x y)) a b)))

(: (-> (OrderedMap string Rational) Rational (OrderedMap string Rational)))
(defun (exps-scale m q)
  (if (rational-zero? q)
      (orderedmap-empty)
      (orderedmap-map-values (fun (v) (rational* v q)) m)))

(: (-> (OrderedMap string Rational) (OrderedMap string Rational)))
(defun (exps-invert m) (exps-scale m (rational -1 1)))

(: (-> string Rational string))
(defun (exp-part name q)
  (cond
    ((= q rational-one) name)
    ((rational-integer? q) (str name "^" (rational->string q)))
    (else (str name "^(" (rational->string q) ")"))))

;; "kg m^2/(s^3 A)", "1/s", "1" for the empty map.
(: (-> (OrderedMap string Rational) string))
(defun (exps->string m)
  (def entries (orderedmap->list m))
  (def pos (list-filter (fun (e) (> (rational-sign (snd e)) 0)) entries))
  (def neg (list-filter (fun (e) (< (rational-sign (snd e)) 0)) entries))
  (: (-> (List (Tuple string Rational)) bool string))
  (defun (parts es negate?)
    (string-join (list->vec (list-map (fun (e) (exp-part (fst e) (if negate? (rational-negate (snd e)) (snd e)))) es))
                 " "))
  (def numer (if (list-empty? pos) "1" (parts pos #f)))
  (cond
    ((list-empty? neg) numer)
    ((= (list-length neg) 1) (str numer "/" (parts neg #t)))
    (else (str numer "/(" (parts neg #t) ")"))))

;; A dimension as text: "length/time", "mass/(length time^2)", "dimensionless".
(: (-> (OrderedMap string Rational) string))
(defun (dims->string d)
  (if (orderedmap-empty? d) "dimensionless" (exps->string d)))

;; ---------------------------------------------------------------------------
;; Atoms and resolved expressions
;; ---------------------------------------------------------------------------

;; A named unit with its dimension and its factor to the coherent unit of
;; that dimension.
(type (: Atom #:opaque (Struct (: name string)
                               (: dims (OrderedMap string Rational))
                               (: factor Factor))))

(: (-> string (OrderedMap string Rational) Factor Atom))
(defun (atom name dims f) (Atom (name name) (dims dims) (factor f)))

(: (-> Atom string))
(defun (atom-name a) (struct-ref a name))

(: (-> Atom (OrderedMap string Rational)))
(defun (atom-dims a) (struct-ref a dims))

(: (-> Atom Factor))
(defun (atom-factor a) (struct-ref a factor))

;; coeff × the product of atoms to their exponents. atoms holds every atom
;; the term names.
(type (: Resolved #:opaque (Struct (: coeff Factor)
                                   (: term (OrderedMap string Rational))
                                   (: atoms (OrderedMap string Atom)))))

(: (-> Resolved Factor))
(defun (resolved-coeff r) (struct-ref r coeff))

(: (-> Resolved (OrderedMap string Rational)))
(defun (resolved-term r) (struct-ref r term))

(: (-> Resolved (OrderedMap string Atom)))
(defun (resolved-atoms r) (struct-ref r atoms))

(: (-> Factor Resolved))
(defun (resolved-number f) (Resolved (coeff f) (term (orderedmap-empty)) (atoms (orderedmap-empty))))

(: (-> Atom Resolved))
(defun (resolved-atom a)
  (Resolved (coeff factor-one)
            (term (orderedmap-set (orderedmap-empty) (atom-name a) rational-one))
            (atoms (orderedmap-set (orderedmap-empty) (atom-name a) a))))

(: (-> Resolved Resolved Resolved))
(defun (resolved* a b)
  (Resolved (coeff (factor* (resolved-coeff a) (resolved-coeff b)))
            (term (exps* (resolved-term a) (resolved-term b)))
            (atoms (orderedmap-merge-with (fun (k x y) x) (resolved-atoms a) (resolved-atoms b)))))

;; a/b, or None when b's coefficient is zero.
(: (-> Resolved Resolved (Option Resolved)))
(defun (resolved/ a b)
  (option-map (fun (inv)
                (resolved* a (Resolved (coeff inv) (term (exps-invert (resolved-term b))) (atoms (resolved-atoms b)))))
              (factor-expt (resolved-coeff b) (rational -1 1))))

;; r^q, or None when the coefficient's power is irrational.
(: (-> Resolved Rational (Option Resolved)))
(defun (resolved-expt r q)
  (option-map (fun (c) (Resolved (coeff c) (term (exps-scale (resolved-term r) q)) (atoms (resolved-atoms r))))
              (factor-expt (resolved-coeff r) q)))

(: (-> Resolved (OrderedMap string Rational)))
(defun (resolved-dims r)
  (orderedmap-fold (fun (name q acc) (exps* acc (exps-scale (atom-dims (orderedmap-ref (resolved-atoms r) name)) q)))
                   (orderedmap-empty)
                   (resolved-term r)))

;; The exact factor from r to the coherent unit of its dimension.
(: (-> Resolved (Result string Factor)))
(defun (resolved-factor r)
  (orderedmap-fold
   (fun (name q acc)
     (match acc
       ((Ok f)
        (match (factor-expt (atom-factor (orderedmap-ref (resolved-atoms r) name)) q)
          ((Some g) (Ok (factor* f g)))
          (None (Err (str "the factor of " (exp-part name q) " is irrational")))))
       (e e)))
   (Ok (resolved-coeff r))
   (resolved-term r)))

;; The factor as a double, rational powers included.
(: (-> Resolved double))
(defun (resolved-double r)
  (orderedmap-fold (fun (name q acc)
                     (* acc (pow (factor->double (atom-factor (orderedmap-ref (resolved-atoms r) name)))
                                 (rational->double q))))
                   (factor->double (resolved-coeff r))
                   (resolved-term r)))

;; ---------------------------------------------------------------------------
;; Tables
;; ---------------------------------------------------------------------------

;; Which prefixes printing may put on a preferred unit.
(type (: PrefixUse (Union NoPrefix SmallPrefixes AllPrefixes)))

(type (: UnitDef (Union (: UnitAtom Atom) (: UnitAlias Resolved))))

(type (: UnitTable #:opaque
         (Record (: dimensions (OrderedMap string string))
                 (: units (OrderedMap string UnitDef))
                 (: prefixes (OrderedMap string Rational))
                 (: preferred (OrderedMap string (Tuple string PrefixUse))))))

(: empty-unit-table UnitTable)
(def empty-unit-table
  (UnitTable (dimensions (orderedmap-empty)) (units (orderedmap-empty))
             (prefixes (orderedmap-empty)) (preferred (orderedmap-empty))))

(: (-> UnitTable (List string)))
(defun (unit-table-names t)
  (orderedmap-fold (fun (k v acc) (Cons k acc)) Nil (record-ref t units)))

(: (-> char bool))
(defun (name-start? c)
  (or (char-alphabetic? c) (char=? c #\_) (char=? c #\%)))

(: (-> char bool))
(defun (name-char? c)
  (or (name-start? c) (char-numeric? c)))

(: (-> string bool))
(defun (valid-name? s)
  (match (string->list s)
    ((Cons c rest) (and (name-start? c) (list-all? name-char? rest) (not (= s "pi"))))
    (Nil #f)))

(: (-> (-> char bool) (List char) bool))
(defun (list-all? ok? cs)
  (match cs
    (Nil #t)
    ((Cons c rest) (and (ok? c) (list-all? ok? rest)))))

(: (-> UnitTable string (Result string UnitTable)))
(defun (check-new-name t name)
  (cond
    ((not (valid-name? name)) (Err (str "'" name "' is not a valid unit name")))
    ((orderedmap-has-key? (record-ref t units) name) (Err (str "the unit '" name "' is already defined")))
    (else (Ok t))))

;; A base dimension and its coherent unit: (def-dimension t "length" "m").
(: (-> UnitTable string string (Result string UnitTable)))
(defun (def-dimension t dim unit)
  (def (Ok _) (check-new-name t unit) :propagate)
  (if (orderedmap-has-key? (record-ref t dimensions) dim)
      (Err (str "the dimension '" dim "' is already defined"))
      (Ok (record-set t
            (dimensions (orderedmap-set (record-ref t dimensions) dim unit))
            (units (orderedmap-set (record-ref t units) unit
                                   (UnitAtom (atom unit (orderedmap-set (orderedmap-empty) dim rational-one) factor-one))))))))

;; A unit defined by a unit expression: (def-unit t "ft" "12 in").
;; A definition with coefficient 1 makes an alias: "N" is "kg m/s^2". Any
;; other makes a unit of its own.
(: (-> UnitTable string string (Result string UnitTable)))
(defun (def-unit t name definition)
  (def (Ok _) (check-new-name t name) :propagate)
  (def (Ok r) (result-map-err (fun (e) (str name ": " e)) (resolve-unit t definition)) :propagate)
  (def (Ok f) (result-map-err (fun (e) (str name ": " e)) (resolved-factor r)) :propagate)
  (def entry
    (if (factor-one? (resolved-coeff r))
        (UnitAlias r)
        (UnitAtom (atom name (resolved-dims r) f))))
  (Ok (record-set t (units (orderedmap-set (record-ref t units) name entry)))))

;; A prefix and its value: (def-prefix t "k" "1000"), (def-prefix t "m" "1|1000").
(: (-> UnitTable string string (Result string UnitTable)))
(defun (def-prefix t name value)
  (match (string->rational value)
    ((Some q) #:when (valid-name? name)
     (Ok (record-set t (prefixes (orderedmap-set (record-ref t prefixes) name q)))))
    ((Some q) (Err (str "'" name "' is not a valid prefix")))
    (None (Err (str "the value of the prefix '" name "' is not a number: " value)))))

;; The unit printing uses for its dimension, and which prefixes it may take.
(: (-> UnitTable string PrefixUse (Result string UnitTable)))
(defun (def-preferred t unit use)
  (def (Ok r) (resolve-unit t unit) :propagate)
  (Ok (record-set t (preferred (orderedmap-set (record-ref t preferred)
                                               (exps->string (resolved-dims r))
                                               (Tuple unit use))))))

;; ---------------------------------------------------------------------------
;; Lookup
;; ---------------------------------------------------------------------------

(: (-> UnitDef Resolved))
(defun (def->resolved d)
  (match d
    ((UnitAtom a) (resolved-atom a))
    ((UnitAlias r) r)))

;; name as base and a trailing power: "m2" is (Tuple "m" 2).
(: (-> string (Option (Tuple string int))))
(defun (split-power name)
  (def cs (list-reverse (string->list name)))
  (let go ((cs cs) (digits Nil))
    (match cs
      ((Cons c rest) #:when (char-numeric? c) (go rest (Cons c digits)))
      ((Cons c rest) #:when (not (list-empty? digits))
       (Some (Tuple (list->string (list-reverse cs)) (string->int (list->string digits)))))
      (_ None))))

;; The longest prefix p with name = p ++ rest, rest a unit of the table.
(: (-> UnitTable string (Option (Tuple Rational string))))
(defun (find-prefix t name)
  (orderedmap-fold
   (fun (p q best)
     (if (and (string-starts-with? name p) (< (string-length p) (string-length name)))
         (let ((rest (string-drop name (string-length p))))
           (if (and (orderedmap-has-key? (record-ref t units) rest)
                    (match best
                      ((Some (Tuple _ r)) (< (string-length rest) (string-length r)))
                      (None #t)))
               (Some (Tuple q rest))
               best))
         best))
   None
   (record-ref t prefixes)))

;; A unit by name: the exact name, then a trailing power ("m2"), then a
;; prefix on a unit of the table ("km"). A prefixed unit is a new atom.
(: (-> UnitTable string (Option Resolved)))
(defun (lookup t name)
  (match (orderedmap-try-ref (record-ref t units) name)
    ((Some d) (Some (def->resolved d)))
    (None
     (match (split-power name)
       ((Some (Tuple base n)) #:when (> n 0)
        (match (lookup t base)
          ((Some r) (resolved-expt r (int->rational n)))
          (None None)))
       (_
        (match (find-prefix t name)
          ((Some (Tuple q rest))
           (def r (def->resolved (orderedmap-ref (record-ref t units) rest)))
           (match (resolved-factor r)
             ((Ok f) (Some (resolved-atom (atom name (resolved-dims r) (factor* (factor q 0) f)))))
             ((Err _) None)))
          (None None)))))))

(: (-> char char int))
(defun (char-cost a b) (if (char=? a b) 0 1))

;; Levenshtein distance.
(: (-> string string int))
(defun (edit-distance a b)
  (def bcs (string->list b))
  (def first-row (let go ((j (list-length bcs)) (acc Nil))
                   (if (< j 0) acc (go (- j 1) (Cons j acc)))))
  (: (-> char (List int) (List int)))
  (defun (next-row ca prev)
    (match prev
      ((Cons p0 _)
       (let go ((bs bcs) (prev prev) (left (+ p0 1)) (acc (Cons (+ p0 1) Nil)))
         (match (Tuple bs prev)
           ((Tuple (Cons cb more) (Cons diag (Cons up rest)))
            (def v (min (+ up 1) (min (+ left 1) (+ diag (char-cost ca cb)))))
            (go more (Cons up rest) v (Cons v acc)))
           (_ (list-reverse acc)))))
      (Nil Nil)))
  (def last-row (list-foldl next-row first-row (string->list a)))
  (match (list-reverse last-row)
    ((Cons d _) d)
    (Nil 0)))

(: (-> UnitTable string string))
(defun (unknown-unit t name)
  (def limit (max 1 (/ (string-length name) 3)))
  (def best
    (list-foldl (fun (candidate best)
                  (def d (edit-distance name candidate))
                  (match best
                    ((Some (Tuple bd _)) #:when (<= bd d) best)
                    (_ (if (<= d limit) (Some (Tuple d candidate)) best))))
                None
                (list-reverse (unit-table-names t))))
  (match best
    ((Some (Tuple _ c)) (str "unknown unit '" name "'. Did you mean '" c "'?"))
    (None (str "unknown unit '" name "'"))))

;; ---------------------------------------------------------------------------
;; The unit expression grammar
;; ---------------------------------------------------------------------------
;; Tightest first:
;;   a|b    a number divided by a number, inside the number token
;;   ^      a power with a rational exponent: m^2, Hz^(-1/2), s^-1
;;   juxtaposition, a product: kg m
;;   * /    left to right: kg m/s^2 is kg·m/s², m/s kg is m/(s·kg)
;;   + -    measurements only
;; A trailing number on a name is a power: m2. pi is the number.

(type (: Token (Union (: TNum Rational) (: TName string) (: TOp char))))

(type (: Expr (Union (: ENum Rational)
                     EPi
                     (: EUnit string)
                     (: EMul Expr Expr)
                     (: EDiv Expr Expr)
                     (: EPow Expr Rational)
                     (: ENeg Expr)
                     (: EAdd Expr Expr)
                     (: ESub Expr Expr)
                     (: ECall string Expr))))

(: (-> char bool))
(defun (number-char? c) (or (char-numeric? c) (char=? c #\.)))

;; Chars of a number from the front of cs: digits and points, an exponent
;; when digits follow the e, and a |-quotient.
(: (-> (List char) (Tuple (List char) (List char))))
(defun (take-number cs)
  (: (-> (List char) (List char) (Tuple (List char) (List char))))
  (defun (mantissa cs acc)
    (match cs
      ((Cons c rest) #:when (number-char? c) (mantissa rest (Cons c acc)))
      ((Cons e rest) #:when (or (char=? e #\e) (char=? e #\E))
       (match rest
         ((Cons s (Cons d more)) #:when (and (or (char=? s #\-) (char=? s #\+)) (char-numeric? d))
          (digits more (Cons d (Cons s (Cons e acc)))))
         ((Cons d more) #:when (char-numeric? d) (digits more (Cons d (Cons e acc))))
         (_ (Tuple acc cs))))
      (_ (Tuple acc cs))))
  (: (-> (List char) (List char) (Tuple (List char) (List char))))
  (defun (digits cs acc)
    (match cs
      ((Cons c rest) #:when (char-numeric? c) (digits rest (Cons c acc)))
      (_ (Tuple acc cs))))
  (def (acc rest) (mantissa cs Nil))
  (match rest
    ((Cons bar (Cons c more)) #:when (and (char=? bar #\|) (number-char? c))
     (def (acc2 rest2) (mantissa (Cons c more) (Cons bar acc)))
     (Tuple (list-reverse acc2) rest2))
    (_ (Tuple (list-reverse acc) rest))))

(: (-> (List char) (Tuple (List char) (List char))))
(defun (take-name cs)
  (let go ((cs cs) (acc Nil))
    (match cs
      ((Cons c rest) #:when (name-char? c) (go rest (Cons c acc)))
      (_ (Tuple (list-reverse acc) cs)))))

(: (-> string (Result string (List Token))))
(defun (tokenize text)
  (let go ((cs (string->list text)) (acc Nil))
    (match cs
      (Nil (Ok (list-reverse acc)))
      ((Cons c rest) #:when (char-whitespace? c) (go rest acc))
      ((Cons #\* (Cons #\* rest)) (go rest (Cons (TOp #\^) acc)))
      ((Cons c rest) #:when (number-char? c)
       (def (chars more) (take-number cs))
       (match (string->rational (list->string chars))
         ((Some q) (go more (Cons (TNum q) acc)))
         (None (Err (str "'" (list->string chars) "' is not a number")))))
      ((Cons c rest) #:when (name-start? c)
       (def (chars more) (take-name cs))
       (go more (Cons (TName (list->string chars)) acc)))
      ((Cons c rest) #:when (string-contains? "*/^+-()" (char->string c))
       (go rest (Cons (TOp c) acc)))
      ((Cons c rest) (Err (str "unexpected '" (char->string c) "'"))))))

(: (-> (List Token) bool))
(defun (starts-factor? ts)
  (match ts
    ((Cons (TNum _) _) #t)
    ((Cons (TName _) _) #t)
    ((Cons (TOp #\() _) #t)
    (_ #f)))

(: (-> (List Token) string))
(defun (describe-next ts)
  (match ts
    (Nil "the end")
    ((Cons (TNum q) _) (str "'" (rational->string q) "'"))
    ((Cons (TName n) _) (str "'" n "'"))
    ((Cons (TOp c) _) (str "'" (char->string c) "'"))))

(: (-> (List Token) (Result string (Tuple Expr (List Token)))))
(defun (parse-sum ts)
  (def (Ok (Tuple first rest)) (parse-product ts) :propagate)
  (let go ((e first) (ts rest))
    (match ts
      ((Cons (TOp #\+) more)
       (def (Ok (Tuple r after)) (parse-product more) :propagate)
       (go (EAdd e r) after))
      ((Cons (TOp #\-) more)
       (def (Ok (Tuple r after)) (parse-product more) :propagate)
       (go (ESub e r) after))
      (_ (Ok (Tuple e ts))))))

(: (-> (List Token) (Result string (Tuple Expr (List Token)))))
(defun (parse-product ts)
  (def (Ok (Tuple first rest)) (parse-juxt ts) :propagate)
  (let go ((e first) (ts rest))
    (match ts
      ((Cons (TOp #\*) more)
       (def (Ok (Tuple r after)) (parse-juxt more) :propagate)
       (go (EMul e r) after))
      ((Cons (TOp #\/) more)
       (def (Ok (Tuple r after)) (parse-juxt more) :propagate)
       (go (EDiv e r) after))
      (_ (Ok (Tuple e ts))))))

(: (-> (List Token) (Result string (Tuple Expr (List Token)))))
(defun (parse-juxt ts)
  (def (Ok (Tuple first rest)) (parse-unary ts) :propagate)
  (let go ((e first) (ts rest))
    (if (starts-factor? ts)
        (let ()
          (def (Ok (Tuple r after)) (parse-power ts) :propagate)
          (go (EMul e r) after))
        (Ok (Tuple e ts)))))

(: (-> (List Token) (Result string (Tuple Expr (List Token)))))
(defun (parse-unary ts)
  (match ts
    ((Cons (TOp #\-) more)
     (def (Ok (Tuple e after)) (parse-unary more) :propagate)
     (Ok (Tuple (ENeg e) after)))
    (_ (parse-power ts))))

(: (-> (List Token) (Result string (Tuple Expr (List Token)))))
(defun (parse-power ts)
  (def (Ok (Tuple base rest)) (parse-atom ts) :propagate)
  (match rest
    ((Cons (TOp #\^) more)
     (def (Ok (Tuple q after)) (parse-exponent more) :propagate)
     (Ok (Tuple (EPow base q) after)))
    (_ (Ok (Tuple base rest)))))

(: (-> (List Token) (Result string (Tuple Rational (List Token)))))
(defun (parse-signed ts)
  (match ts
    ((Cons (TOp #\-) (Cons (TNum q) rest)) (Ok (Tuple (rational-negate q) rest)))
    ((Cons (TOp #\+) (Cons (TNum q) rest)) (Ok (Tuple q rest)))
    ((Cons (TNum q) rest) (Ok (Tuple q rest)))
    (_ (Err (str "expected an exponent, got " (describe-next ts))))))

;; 2, -1, 0.5, (1/2), (-1|2).
(: (-> (List Token) (Result string (Tuple Rational (List Token)))))
(defun (parse-exponent ts)
  (match ts
    ((Cons (TOp #\() more)
     (def (Ok (Tuple q rest)) (parse-signed more) :propagate)
     (match rest
       ((Cons (TOp #\)) after) (Ok (Tuple q after)))
       ((Cons (TOp #\/) (Cons (TNum d) (Cons (TOp #\)) after))) #:when (not (rational-zero? d))
        (Ok (Tuple (rational/ q d) after)))
       (_ (Err (str "expected ')' in an exponent, got " (describe-next rest))))))
    (_ (parse-signed ts))))

(: (-> (List Token) (Result string (Tuple Expr (List Token)))))
(defun (parse-atom ts)
  (match ts
    ((Cons (TNum q) rest) (Ok (Tuple (ENum q) rest)))
    ((Cons (TName "pi") rest) (Ok (Tuple EPi rest)))
    ((Cons (TName f) (Cons (TOp #\() rest)) #:when (or (= f "sqrt") (= f "cbrt"))
     (def (Ok (Tuple e after)) (parse-sum rest) :propagate)
     (match after
       ((Cons (TOp #\)) more) (Ok (Tuple (ECall f e) more)))
       (_ (Err (str "expected ')', got " (describe-next after))))))
    ((Cons (TName n) rest) (Ok (Tuple (EUnit n) rest)))
    ((Cons (TOp #\() rest)
     (def (Ok (Tuple e after)) (parse-sum rest) :propagate)
     (match after
       ((Cons (TOp #\)) more) (Ok (Tuple e more)))
       (_ (Err (str "expected ')', got " (describe-next after))))))
    (_ (Err (str "expected a number or a unit, got " (describe-next ts))))))

(: (-> string (Result string Expr)))
(defun (parse-expression text)
  (def (Ok ts) (tokenize text) :propagate)
  (def (Ok (Tuple e rest)) (parse-sum ts) :propagate)
  (match rest
    (Nil (Ok e))
    (_ (Err (str "unexpected " (describe-next rest))))))

;; ---------------------------------------------------------------------------
;; Resolving: exact
;; ---------------------------------------------------------------------------

(: (-> UnitTable Expr (Result string Resolved)))
(defun (resolve-expr t e)
  (match e
    ((ENum q) (Ok (resolved-number (factor q 0))))
    (EPi (Ok (resolved-number (factor rational-one 1))))
    ((EUnit n)
     (match (lookup t n)
       ((Some r) (Ok r))
       (None (Err (unknown-unit t n)))))
    ((EMul a b)
     (def (Ok x) (resolve-expr t a) :propagate)
     (def (Ok y) (resolve-expr t b) :propagate)
     (Ok (resolved* x y)))
    ((EDiv a b)
     (def (Ok x) (resolve-expr t a) :propagate)
     (def (Ok y) (resolve-expr t b) :propagate)
     (option->result (resolved/ x y) "division by zero"))
    ((EPow a q)
     (def (Ok x) (resolve-expr t a) :propagate)
     (match (resolved-expt x q)
       ((Some r) (Ok r))
       (None (Err (str "a power " (exp-part "x" q) " with an irrational factor")))))
    ((ENeg a)
     (def (Ok x) (resolve-expr t a) :propagate)
     (Ok (resolved* (resolved-number (factor (rational -1 1) 0)) x)))
    ((EAdd _ _) (Err "+ is not allowed in a unit expression"))
    ((ESub _ _) (Err "- is not allowed in a unit expression"))
    ((ECall f _) (Err (str f " is not allowed in a unit expression")))))

;; A unit expression against a table: "km/h", "1|60 deg", "kg m^2/s^2".
(: (-> UnitTable string (Result string Resolved)))
(defun (resolve-unit t text)
  (def (Ok e) (parse-expression text) :propagate)
  (resolve-expr t e))

;; The names in the table whose dimension is that of text: GNU's ?.
(: (-> UnitTable string (Result string (List string))))
(defun (conformable-units t text)
  (def (Ok r) (resolve-unit t text) :propagate)
  (def d (resolved-dims r))
  (Ok (list-filter (fun (n) (match (lookup t n)
                              ((Some u) (= (resolved-dims u) d))
                              (None #f)))
                   (list-reverse (unit-table-names t)))))

;; ---------------------------------------------------------------------------
;; The declarations: the standard table, for types and literals
;; ---------------------------------------------------------------------------

;; The same table as `standard-definitions` below, which the run-time
;; functions use. A change to one is a change to the other.

(def/dimension length #:base m)
(def/dimension mass #:base kg)
(def/dimension time #:base s)
(def/dimension current #:base A)
(def/dimension temperature #:base K)
(def/dimension amount #:base mol)
(def/dimension luminosity #:base cd)
(def/dimension information #:base b #:carrier long)

(def/prefix Q "1e30")
(def/prefix R "1e27")
(def/prefix Y "1e24")
(def/prefix Z "1e21")
(def/prefix E "1e18")
(def/prefix P "1e15")
(def/prefix T "1e12")
(def/prefix G "1e9")
(def/prefix M "1e6")
(def/prefix k "1e3")
(def/prefix h "1e2")
(def/prefix da "1e1")
(def/prefix d "1e-1")
(def/prefix c "1e-2")
(def/prefix m "1e-3")
(def/prefix u "1e-6")
(def/prefix µ "1e-6")
(def/prefix n "1e-9")
(def/prefix p "1e-12")
(def/prefix f "1e-15")
(def/prefix a "1e-18")
(def/prefix z "1e-21")
(def/prefix y "1e-24")
(def/prefix r "1e-27")
(def/prefix q "1e-30")
(def/prefix Ki "1024")
(def/prefix ki "1024")
(def/prefix Mi "1048576")
(def/prefix Gi "1073741824")
(def/prefix Ti "1099511627776")
(def/prefix Pi "1125899906842624")
(def/prefix Ei "1152921504606846976")
(def/prefix Zi "1180591620717411303424")
(def/prefix Yi "1208925819614629174706176")

(def/unit meter "1 m")
(def/unit meters "1 m")
(def/unit metre "1 m")
(def/unit metres "1 m")
(def/unit sec "1 s")
(def/unit second "1 s")
(def/unit seconds "1 s")
(def/unit bit "1 b")
(def/unit bits "1 b")
(def/unit amp "1 A")
(def/unit Amp "1 A")
(def/unit ampere "1 A")
(def/unit c "299792458 m/s")
(def/unit gn "9.80665 m/s^2")
(def/unit min "60 s")
(def/unit minute "1 min")
(def/unit minutes "1 min")
(def/unit h "60 min")
(def/unit hr "1 h")
(def/unit hour "1 h")
(def/unit hours "1 h")
(def/unit d "24 h")
(def/unit day "1 d")
(def/unit days "1 d")
(def/unit wk "7 d")
(def/unit week "1 wk")
(def/unit weeks "1 wk")
(def/unit fortnight "14 d")
(def/unit fortnights "1 fortnight")
(def/unit yr "31557600 s")
(def/unit year "1 yr")
(def/unit years "1 yr")
(def/unit in "0.0254 m")
(def/unit inch "1 in")
(def/unit inches "1 in")
(def/unit ft "12 in")
(def/unit foot "1 ft")
(def/unit feet "1 ft")
(def/unit yd "3 ft")
(def/unit yard "1 yd")
(def/unit yards "1 yd")
(def/unit mi "5280 ft")
(def/unit mile "1 mi")
(def/unit miles "1 mi")
(def/unit nmi "1852 m")
(def/unit fath "6 ft")
(def/unit fathom "1 fath")
(def/unit fathoms "1 fath")
(def/unit fur "660 ft")
(def/unit furlong "1 fur")
(def/unit furlongs "1 fur")
(def/unit au "149597870700 m")
(def/unit ly "c yr")
(def/unit lightyear "1 ly")
(def/unit lightyears "1 ly")
(def/unit pc "648000 au/pi")
(def/unit parsec "1 pc")
(def/unit parsecs "1 pc")
(def/unit ha "10000 m2")
(def/unit hectare "1 ha")
(def/unit hectares "1 ha")
(def/unit are "100 m2")
(def/unit ares "1 are")
(def/unit acre "4046.8564224 m2")
(def/unit acres "1 acre")
(def/unit l "0.001 m3")
(def/unit L "1 l")
(def/unit litre "1 l")
(def/unit litres "1 l")
(def/unit liter "1 l")
(def/unit liters "1 l")
(def/unit gal "3.785411784 l")
(def/unit gallon "1 gal")
(def/unit gallons "1 gal")
(def/unit qt "1|4 gal")
(def/unit quart "1 qt")
(def/unit quarts "1 qt")
(def/unit pt "1|2 qt")
(def/unit pint "1 pt")
(def/unit pints "1 pt")
(def/unit fl_oz "1|16 pt")
(def/unit fluid_ounce "1 fl_oz")
(def/unit fluid_ounces "1 fl_oz")
(def/unit fl_dr "1|8 fl_oz")
(def/unit gi "4 fl_oz")
(def/unit tsp "5 ml")
(def/unit teaspoon "1 tsp")
(def/unit teaspoons "1 tsp")
(def/unit tbsp "15 ml")
(def/unit tablespoon "1 tbsp")
(def/unit tablespoons "1 tbsp")
(def/unit cup "250 ml")
(def/unit cups "1 cup")
(def/unit us_tsp "4.92892159375 ml")
(def/unit us_tbsp "3 us_tsp")
(def/unit us_cup "16 us_tbsp")
(def/unit us_cups "1 us_cup")
(def/unit g "0.001 kg")
(def/unit gram "1 g")
(def/unit grams "1 g")
(def/unit t "1000 kg")
(def/unit ton "1 t")
(def/unit tons "1 t")
(def/unit tonne "1 t")
(def/unit tonnes "1 t")
(def/unit u "1.66053906660e-27 kg")
(def/unit amu "1 u")
(def/unit slug "14.5939029 kg")
(def/unit lb "0.45359237 kg")
(def/unit pound "1 lb")
(def/unit pounds "1 lb")
(def/unit oz "1|16 lb")
(def/unit ounce "1 oz")
(def/unit ounces "1 oz")
(def/unit gr "1|7000 lb")
(def/unit grain "1 gr")
(def/unit grains "1 gr")
(def/unit st "14 lb")
(def/unit stone "1 st")
(def/unit short_ton "2000 lb")
(def/unit short_tons "1 short_ton")
(def/unit long_ton "2240 lb")
(def/unit long_tons "1 long_ton")
(def/unit N "kg m/s^2")
(def/unit newton "1 N")
(def/unit newtons "1 N")
(def/unit lbf "lb gn")
(def/unit kgf "kg gn")
(def/unit dyn "1e-5 N")
(def/unit Pa "N/m^2")
(def/unit pascal "1 Pa")
(def/unit bar "100000 Pa")
(def/unit atm "101325 Pa")
(def/unit atmosphere "1 atm")
(def/unit atmospheres "1 atm")
(def/unit psi "lbf/in^2")
(def/unit mHg "133322.387415 Pa")
(def/unit mhg "1 mHg")
(def/unit mH2O "9806.65 Pa")
(def/unit mh2o "1 mH2O")
(def/unit J "N m")
(def/unit joule "1 J")
(def/unit joules "1 J")
(def/unit Nm "1 J")
(def/unit W "J/s")
(def/unit watt "1 W")
(def/unit watts "1 W")
(def/unit Watt "1 W")
(def/unit Watts "1 W")
(def/unit Wh "W h")
(def/unit hp "745.6998715822702 W")
(def/unit horsepower "1 hp")
(def/unit cal "4.184 J")
(def/unit calorie "1 cal")
(def/unit calories "1 cal")
(def/unit BTU "1055.05585262 J")
(def/unit eV "1.602176634e-19 J")
(def/unit erg "1e-7 J")
(def/unit C "A s")
(def/unit coulomb "1 C")
(def/unit V "W/A")
(def/unit volt "1 V")
(def/unit volts "1 V")
(def/unit ohm "V/A")
(def/unit ohms "1 ohm")
(def/unit S "1/ohm")
(def/unit siemens "1 S")
(def/unit F "C/V")
(def/unit farad "1 F")
(def/unit H "V s/A")
(def/unit henry "1 H")
(def/unit Wb "V s")
(def/unit weber "1 Wb")
(def/unit T "Wb/m^2")
(def/unit tesla "1 T")
(def/unit T_tesla "1 T")
(def/unit gauss "1e-4 T")
(def/unit Mx "1e-8 Wb")
(def/unit maxwell "1 Mx")
(def/unit Oe "1000/(4 pi) * A/m")
(def/unit oersted "1 Oe")
(def/unit poise "0.1 Pa s")
(def/unit St "1e-4 m^2/s")
(def/unit stokes "1 St")
(def/unit B "8 b")
(def/unit byte "1 B")
(def/unit bytes "1 B")
(def/unit bps "b/s")
(def/unit Bps "B/s")
(def/unit sr "1")
(def/unit steradian "1 sr")
(def/unit lm "cd sr")
(def/unit lumen "1 lm")
(def/unit lx "lm/m^2")
(def/unit lux "1 lx")
(def/unit Hz "1/s")
(def/unit hertz "1 Hz")
(def/unit Hertz "1 Hz")
(def/unit hz "1 Hz")
(def/unit Bq "1/s")
(def/unit becquerel "1 Bq")
(def/unit Ci "3.7e10 Bq")
(def/unit curie "1 Ci")
(def/unit Gy "J/kg")
(def/unit gray "1 Gy")
(def/unit Sv "J/kg")
(def/unit sievert "1 Sv")
(def/unit rem "0.01 Sv")
(def/unit rad_dose "0.01 Gy")
(def/unit R "2.58e-4 C/kg")
(def/unit roentgen "1 R")
(def/unit degR "5|9 K")
(def/unit rankine "1 degR")
(def/unit rad "1")
(def/unit radian "1 rad")
(def/unit radians "1 rad")
(def/unit deg "pi/180")
(def/unit degree "1 deg")
(def/unit degrees "1 deg")
(def/unit grad "pi/200")
(def/unit rev "2 pi")
(def/unit revolution "1 rev")
(def/unit revolutions "1 rev")
(def/unit rpm "rev/min")
(def/unit arcmin "1|60 deg")
(def/unit arcsec "1|3600 deg")
(def/unit % "0.01")
(def/unit percent "1 %")
(def/unit ppm "1e-6")
(def/unit ppb "1e-9")
(def/unit ppt "1e-12")
(def/unit planck "6.62607015e-34 J s")
(def/unit hbar "planck/(2 pi)")
(def/unit kph "km/h")
(def/unit mph "mi/h")
(def/unit kn "nmi/h")
(def/unit knot "1 kn")
(def/unit knots "1 kn")

;; ---------------------------------------------------------------------------
;; The standard table
;; ---------------------------------------------------------------------------

(def si-prefixes
  (list (list "Q" "1e30") (list "R" "1e27") (list "Y" "1e24") (list "Z" "1e21") (list "E" "1e18") (list "P" "1e15")
    (list "T" "1e12") (list "G" "1e9") (list "M" "1e6") (list "k" "1e3") (list "h" "1e2") (list "da" "1e1")
    (list "d" "1e-1") (list "c" "1e-2") (list "m" "1e-3") (list "u" "1e-6") (list "µ" "1e-6") (list "n" "1e-9")
    (list "p" "1e-12") (list "f" "1e-15") (list "a" "1e-18") (list "z" "1e-21") (list "y" "1e-24")
    (list "r" "1e-27") (list "q" "1e-30")
    (list "Ki" "1024") (list "ki" "1024") (list "Mi" "1048576") (list "Gi" "1073741824")
    (list "Ti" "1099511627776") (list "Pi" "1125899906842624") (list "Ei" "1152921504606846976")
    (list "Zi" "1180591620717411303424") (list "Yi" "1208925819614629174706176")))

(def si-dimensions
  (list (list "length" "m") (list "mass" "kg") (list "time" "s") (list "current" "A") (list "temperature" "K")
    (list "amount" "mol") (list "luminosity" "cd") (list "information" "b")))

;; (name definition alias ...), in order: a definition uses only names above it.
(def standard-definitions
  (list ;; Aliases of the base units
    (list "m" "m" "meter" "meters" "metre" "metres")
    (list "s" "s" "sec" "second" "seconds")
    (list "b" "b" "bit" "bits")
    (list "A" "A" "amp" "Amp" "ampere")
    ;; Constants
    (list "c" "299792458 m/s")
    (list "gn" "9.80665 m/s^2")
    ;; Time
    (list "min" "60 s" "minute" "minutes")
    (list "h" "60 min" "hr" "hour" "hours")
    (list "d" "24 h" "day" "days")
    (list "wk" "7 d" "week" "weeks")
    (list "fortnight" "14 d" "fortnights")
    (list "yr" "31557600 s" "year" "years")
    ;; Length
    (list "in" "0.0254 m" "inch" "inches")
    (list "ft" "12 in" "foot" "feet")
    (list "yd" "3 ft" "yard" "yards")
    (list "mi" "5280 ft" "mile" "miles")
    (list "nmi" "1852 m")
    (list "fath" "6 ft" "fathom" "fathoms")
    (list "fur" "660 ft" "furlong" "furlongs")
    (list "au" "149597870700 m")
    (list "ly" "c yr" "lightyear" "lightyears")
    (list "pc" "648000 au/pi" "parsec" "parsecs")
    ;; Area
    (list "ha" "10000 m2" "hectare" "hectares")
    (list "are" "100 m2" "ares")
    (list "acre" "4046.8564224 m2" "acres")
    ;; Volume
    (list "l" "0.001 m3" "L" "litre" "litres" "liter" "liters")
    (list "gal" "3.785411784 l" "gallon" "gallons")
    (list "qt" "1|4 gal" "quart" "quarts")
    (list "pt" "1|2 qt" "pint" "pints")
    (list "fl_oz" "1|16 pt" "fluid_ounce" "fluid_ounces")
    (list "fl_dr" "1|8 fl_oz")
    (list "gi" "4 fl_oz")
    (list "tsp" "5 ml" "teaspoon" "teaspoons")
    (list "tbsp" "15 ml" "tablespoon" "tablespoons")
    (list "cup" "250 ml" "cups")
    (list "us_tsp" "4.92892159375 ml")
    (list "us_tbsp" "3 us_tsp")
    (list "us_cup" "16 us_tbsp" "us_cups")
    ;; Mass
    (list "g" "0.001 kg" "gram" "grams")
    (list "t" "1000 kg" "ton" "tons" "tonne" "tonnes")
    (list "u" "1.66053906660e-27 kg" "amu")
    (list "slug" "14.5939029 kg")
    (list "lb" "0.45359237 kg" "pound" "pounds")
    (list "oz" "1|16 lb" "ounce" "ounces")
    (list "gr" "1|7000 lb" "grain" "grains")
    (list "st" "14 lb" "stone")
    (list "short_ton" "2000 lb" "short_tons")
    (list "long_ton" "2240 lb" "long_tons")
    ;; Force and pressure
    (list "N" "kg m/s^2" "newton" "newtons")
    (list "lbf" "lb gn")
    (list "kgf" "kg gn")
    (list "dyn" "1e-5 N")
    (list "Pa" "N/m^2" "pascal")
    (list "bar" "100000 Pa")
    (list "atm" "101325 Pa" "atmosphere" "atmospheres")
    (list "psi" "lbf/in^2")
    (list "mHg" "133322.387415 Pa" "mhg")
    (list "mH2O" "9806.65 Pa" "mh2o")
    ;; Energy and power
    (list "J" "N m" "joule" "joules" "Nm")
    (list "W" "J/s" "watt" "watts" "Watt" "Watts")
    (list "Wh" "W h")
    (list "hp" "745.6998715822702 W" "horsepower")
    (list "cal" "4.184 J" "calorie" "calories")
    (list "BTU" "1055.05585262 J")
    (list "eV" "1.602176634e-19 J")
    (list "erg" "1e-7 J")
    ;; Electricity and magnetism
    (list "C" "A s" "coulomb")
    (list "V" "W/A" "volt" "volts")
    (list "ohm" "V/A" "ohms")
    (list "S" "1/ohm" "siemens")
    (list "F" "C/V" "farad")
    (list "H" "V s/A" "henry")
    (list "Wb" "V s" "weber")
    (list "T" "Wb/m^2" "tesla" "T_tesla")
    (list "gauss" "1e-4 T")
    (list "Mx" "1e-8 Wb" "maxwell")
    (list "Oe" "1000/(4 pi) * A/m" "oersted")
    ;; Viscosity
    (list "poise" "0.1 Pa s")
    (list "St" "1e-4 m^2/s" "stokes")
    ;; Information
    (list "B" "8 b" "byte" "bytes")
    (list "bps" "b/s")
    (list "Bps" "B/s")
    ;; Photometry
    (list "sr" "1" "steradian")
    (list "lm" "cd sr" "lumen")
    (list "lx" "lm/m^2" "lux")
    ;; Frequency and radioactivity
    (list "Hz" "1/s" "hertz" "Hertz" "hz")
    (list "Bq" "1/s" "becquerel")
    (list "Ci" "3.7e10 Bq" "curie")
    (list "Gy" "J/kg" "gray")
    (list "Sv" "J/kg" "sievert")
    (list "rem" "0.01 Sv")
    (list "rad_dose" "0.01 Gy")
    (list "R" "2.58e-4 C/kg" "roentgen")
    ;; Temperature differences
    (list "degR" "5|9 K" "rankine")
    ;; Angles
    (list "rad" "1" "radian" "radians")
    (list "deg" "pi/180" "degree" "degrees")
    (list "grad" "pi/200")
    (list "rev" "2 pi" "revolution" "revolutions")
    (list "rpm" "rev/min")
    (list "arcmin" "1|60 deg")
    (list "arcsec" "1|3600 deg")
    ;; Ratios
    (list "%" "0.01" "percent")
    (list "ppm" "1e-6")
    (list "ppb" "1e-9")
    (list "ppt" "1e-12")
    ;; Physical constants
    (list "planck" "6.62607015e-34 J s")
    (list "hbar" "planck/(2 pi)")
    ;; Rates
    (list "kph" "km/h")
    (list "mph" "mi/h")
    (list "kn" "nmi/h" "knot" "knots")))

(def standard-preferred
  (list (Tuple "m" AllPrefixes) (Tuple "g" AllPrefixes) (Tuple "s" SmallPrefixes)
        (Tuple "A" AllPrefixes) (Tuple "K" SmallPrefixes) (Tuple "mol" AllPrefixes)
        (Tuple "cd" AllPrefixes) (Tuple "b" AllPrefixes)
        (Tuple "N" AllPrefixes) (Tuple "Pa" AllPrefixes) (Tuple "J" AllPrefixes)
        (Tuple "W" AllPrefixes) (Tuple "C" AllPrefixes) (Tuple "V" AllPrefixes)
        (Tuple "ohm" AllPrefixes) (Tuple "S" AllPrefixes) (Tuple "F" AllPrefixes)
        (Tuple "H" AllPrefixes) (Tuple "Wb" AllPrefixes) (Tuple "T" AllPrefixes)
        (Tuple "lx" AllPrefixes) (Tuple "Hz" AllPrefixes) (Tuple "Gy" AllPrefixes)))

(: (-> UnitTable (List string) UnitTable))
(defun (add-definition t entry)
  (match entry
    ((Cons name (Cons definition aliases))
     (list-foldl (fun (alias t) (ok-or-raise (def-unit t alias (str "1 " name))))
                 (if (orderedmap-has-key? (record-ref t units) name)
                     t
                     (ok-or-raise (def-unit t name definition)))
                 aliases))
    (_ (argument-error "a definition needs a name and a definition"))))

(: standard-units UnitTable)
(def standard-units
  (let* ((t (list-foldl (fun (d t) (ok-or-raise (def-dimension t (first-of-2 d) (second-of-2 d))))
                        empty-unit-table si-dimensions))
         (t (list-foldl (fun (p t) (ok-or-raise (def-prefix t (first-of-2 p) (second-of-2 p))))
                        t si-prefixes))
         (t (list-foldl (fun (e t) (add-definition t e)) t standard-definitions)))
    (list-foldl (fun (p t) (ok-or-raise (def-preferred t (fst p) (snd p))))
                t standard-preferred)))

(: (-> (List string) string))
(defun (first-of-2 xs)
  (match xs
    ((Cons a _) a)
    (Nil "")))

(: (-> (List string) string))
(defun (second-of-2 xs)
  (match xs
    ((Cons _ (Cons b _)) b)
    (_ "")))

;; ---------------------------------------------------------------------------
;; Measurements
;; ---------------------------------------------------------------------------

;; A value in the coherent unit of its dimension.
(type (: Measurement (Struct (: value double) (: dims (OrderedMap string Rational)))))

(: (-> double (OrderedMap string Rational) Measurement))
(defun (measurement v d) (Measurement (value v) (dims d)))

(: (-> Measurement double))
(defun (measurement-value m) (struct-ref m value))

(: (-> Measurement (OrderedMap string Rational)))
(defun (measurement-dims m) (struct-ref m dims))

(def no-dims (orderedmap-empty))

(: (-> string Measurement Measurement (Result string Measurement)))
(defun (same-dims what a b)
  (if (= (measurement-dims a) (measurement-dims b))
      (Ok a)
      (Err (str "cannot " what " " (dims->string (measurement-dims a))
                " and " (dims->string (measurement-dims b))))))

(: (-> UnitTable Expr (Result string Measurement)))
(defun (eval-measure t e)
  (match e
    ((ENum q) (Ok (measurement (rational->double q) no-dims)))
    (EPi (Ok (measurement pi no-dims)))
    ((EUnit n)
     (match (lookup t n)
       ((Some r) (Ok (measurement (resolved-double r) (resolved-dims r))))
       (None (Err (unknown-unit t n)))))
    ((EMul a b)
     (def (Ok x) (eval-measure t a) :propagate)
     (def (Ok y) (eval-measure t b) :propagate)
     (Ok (measurement* x y)))
    ((EDiv a b)
     (def (Ok x) (eval-measure t a) :propagate)
     (def (Ok y) (eval-measure t b) :propagate)
     (Ok (measurement/ x y)))
    ((EPow a q)
     (def (Ok x) (eval-measure t a) :propagate)
     (Ok (measurement-expt x q)))
    ((ENeg a)
     (def (Ok x) (eval-measure t a) :propagate)
     (Ok (measurement-scale -1.0 x)))
    ((EAdd a b)
     (def (Ok x) (eval-measure t a) :propagate)
     (def (Ok y) (eval-measure t b) :propagate)
     (def (Ok _) (same-dims "add" x y) :propagate)
     (Ok (measurement (+ (measurement-value x) (measurement-value y)) (measurement-dims x))))
    ((ESub a b)
     (def (Ok x) (eval-measure t a) :propagate)
     (def (Ok y) (eval-measure t b) :propagate)
     (def (Ok _) (same-dims "subtract" x y) :propagate)
     (Ok (measurement (- (measurement-value x) (measurement-value y)) (measurement-dims x))))
    ((ECall f a)
     (def (Ok x) (eval-measure t a) :propagate)
     (Ok (measurement-expt x (if (= f "sqrt") (rational 1 2) (rational 1 3)))))))

;; "6000 m + 50 ft", "0.5 * 1.225 kg/m^3 * (120 km/h)^2", "sqrt(2 m^2)".
(: (-> string (#:table UnitTable) (Result string Measurement)))
(defun (string->measurement text #:table standard-units)
  (def (Ok e) (parse-expression text) :propagate)
  (eval-measure table e))

(: (-> string Measurement Measurement Measurement))
(defun (checked what a b)
  (match (same-dims what a b)
    ((Ok _) a)
    ((Err e) (argument-error e))))

;; Raises ArgumentException when the dimensions differ.
(: (-> Measurement Measurement Measurement))
(defun (measurement+ a b)
  (ignore (checked "add" a b))
  (measurement (+ (measurement-value a) (measurement-value b)) (measurement-dims a)))

;; Raises ArgumentException when the dimensions differ.
(: (-> Measurement Measurement Measurement))
(defun (measurement- a b)
  (ignore (checked "subtract" a b))
  (measurement (- (measurement-value a) (measurement-value b)) (measurement-dims a)))

(: (-> Measurement Measurement Measurement))
(defun (measurement* a b)
  (measurement (* (measurement-value a) (measurement-value b))
               (exps* (measurement-dims a) (measurement-dims b))))

(: (-> Measurement Measurement Measurement))
(defun (measurement/ a b)
  (measurement (/ (measurement-value a) (measurement-value b))
               (exps* (measurement-dims a) (exps-invert (measurement-dims b)))))

(: (-> double Measurement Measurement))
(defun (measurement-scale k m)
  (measurement (* k (measurement-value m)) (measurement-dims m)))

(: (-> Measurement Rational Measurement))
(defun (measurement-expt m q)
  (measurement (pow (measurement-value m) (rational->double q))
               (exps-scale (measurement-dims m) q)))

(: (-> Measurement Measurement))
(defun (measurement-sqrt m) (measurement-expt m (rational 1 2)))

(: (-> Measurement Measurement bool))
(defun (conformable? a b) (= (measurement-dims a) (measurement-dims b)))

;; m as a number of target: (measurement-in m "ft"), (measurement-in m "12 mi/h").
(: (-> Measurement string (#:table UnitTable) (Result string double)))
(defun (measurement-in m target #:table standard-units)
  (def (Ok u) (string->measurement target #:table table) :propagate)
  (if (conformable? m u)
      (Ok (/ (measurement-value m) (measurement-value u)))
      (Err (str "cannot convert " (dims->string (measurement-dims m))
                " to " target ", which is " (dims->string (measurement-dims u))))))

;; ---------------------------------------------------------------------------
;; Printing
;; ---------------------------------------------------------------------------

;; 7 significant digits, invariant culture.
(: (-> double string))
(defun (format-number x)
  (string-replace (.ToString x "G7" clr-invariant) "−" "-"))

(def print-prefixes
  (list "Q" "R" "Y" "Z" "E" "P" "T" "G" "M" "k" "m" "u" "n" "p" "f" "a" "z" "y" "r" "q"))

;; The prefix for x in a unit: the largest power of 1000 at most |x|, or the
;; smallest when |x| is below all.
(: (-> UnitTable double PrefixUse (Tuple string double)))
(defun (choose-prefix t x use)
  (def a (abs x))
  (def candidates
    (match use
      (NoPrefix Nil)
      (_ (list-filter (fun (p) (match use
                                 (SmallPrefixes (< (snd p) 1.0))
                                 (_ #t)))
                      (list-map (fun (p) (Tuple p (rational->double (orderedmap-ref (record-ref t prefixes) p))))
                                (list-filter (fun (p) (orderedmap-has-key? (record-ref t prefixes) p)) print-prefixes))))))
  (def all (Cons (Tuple "" 1.0) candidates))
  (def fitting (list-filter (fun (p) (<= (snd p) a)) all))
  (cond
    ((= a 0.0) (Tuple "" 1.0))
    ((list-empty? fitting)
     (list-foldl (fun (p best) (if (< (snd p) (snd best)) p best)) (Tuple "" 1.0) all))
    (else
     (list-foldl (fun (p best) (if (> (snd p) (snd best)) p best)) (Tuple "" 0.0) fitting))))

;; In the dimension's preferred unit with a prefix by magnitude, or else in
;; the coherent unit written with the base units: "3.6 MJ", "33.33333 m/s".
(: (-> Measurement (#:table UnitTable) string))
(defun (measurement->string m #:table standard-units)
  (def d (measurement-dims m))
  (def v (measurement-value m))
  (cond
    ((orderedmap-empty? d) (format-number v))
    (:match (Some (Tuple unit use)) (orderedmap-try-ref (record-ref table preferred) (exps->string d))
     (def u (match (lookup table unit)
              ((Some r) (resolved-double r))
              (None 1.0)))
     (def x (/ v u))
     (def (p pv) (choose-prefix table x use))
     (str (format-number (/ x pv)) " " p unit))
    (else
     (def names (orderedmap-fold (fun (dim q acc)
                                   (orderedmap-set acc (orderedmap-ref-or (record-ref table dimensions) dim dim) q))
                                 (orderedmap-empty) d))
     (str (format-number v) " " (exps->string names)))))

;; Mixed output: (measurement->string/mixed m (list "h" "min")) is "5 h 25 min".
;; Each unit but the last takes a whole number, the last the rest.
(: (-> Measurement (List string) (#:table UnitTable) (Result string string)))
(defun (measurement->string/mixed m units #:table standard-units)
  (def (Ok targets)
    (list-foldl (fun (u acc)
                  (match acc
                    ((Ok xs)
                     (match (string->measurement u #:table table)
                       ((Ok v) #:when (conformable? v m) (Ok (Cons (Tuple u (measurement-value v)) xs)))
                       ((Ok v) (Err (str "cannot convert " (dims->string (measurement-dims m)) " to " u)))
                       ((Err e) (Err e))))
                    (e e)))
                (Ok Nil) units)
    :propagate)
  (let go ((ts (list-reverse targets)) (remaining (measurement-value m)) (parts Nil))
    (match ts
      (Nil (Ok (string-join (list->vec (list-reverse parts)) " ")))
      ((Cons (Tuple u scale) Nil)
       (def x (/ remaining scale))
       (go Nil 0.0 (if (or (> (abs x) 1e-9) (list-empty? parts))
                       (Cons (str (format-number x) " " u) parts)
                       parts)))
      ((Cons (Tuple u scale) more)
       (def whole (truncate (round-to (/ remaining scale) 9)))
       (go more (- remaining (* whole scale))
           (if (= whole 0.0) parts (Cons (str (format-number whole) " " u) parts)))))))

(import/extern
  (clr-round-digits (: System.Math.Round (-> double int double))))

(: (-> double int double))
(defun (round-to x digits) (clr-round-digits x digits))

;; A query as Umits takes it: "12 h in s", "65 mi/(12 mi/h) in h,min",
;; "6 ft, 2 in in m", or an expression alone, printed by measurement->string.
;; A single target of another dimension divides: "65 mi in 12 mi/h" is
;; "19500 s".
(: (-> string (#:table UnitTable) (Result string string)))
(defun (convert-query query #:table standard-units)
  (def (lhs rhs)
    (match (string-search-backward query " in ")
      ((Some at) (Tuple (substring query (string-cursor-start query) at)
                        (Some (substring query (string-cursor-forward query at 4) (string-cursor-end query)))))
      (None (Tuple query None))))
  (def (Ok m) (string->measurement (string-replace lhs "," " + ") #:table table) :propagate)
  (match rhs
    (None (Ok (measurement->string m #:table table)))
    ((Some targets)
     (def units (list-map string-trim (vec->list (string-split targets ","))))
     (match units
       ((Cons u Nil)
        (def (Ok target) (string->measurement u #:table table) :propagate)
        (if (conformable? m target)
            (Ok (str (format-number (/ (measurement-value m) (measurement-value target))) " " u))
            (Ok (measurement->string (measurement/ m target) #:table table))))
       (_ (measurement->string/mixed m units #:table table))))))

(impl (Written Measurement)
  (defun (written m) (str "(string->measurement \"" (measurement->string m) "\")"))
  (defun (write-into m b) (stringbuilder-add-string! b (written m))))

(impl (Displayed Measurement)
  (defun (displayed m) (measurement->string m))
  (defun (display-into m b) (stringbuilder-add-string! b (measurement->string m))))

;; ---------------------------------------------------------------------------
;; Temperatures
;; ---------------------------------------------------------------------------
;; Absolute temperatures. Differences are measured in K or degR.

(type (: Celsius (Newtype double)))
(type (: Fahrenheit (Newtype double)))

(: (-> Celsius Measurement))
(defun (celsius->kelvin c)
  (def (Celsius x) c)
  (measurement (+ x 273.15) (measurement-dims (ok-or-raise (string->measurement "K")))))

;; Raises ArgumentException when k is not a temperature.
(: (-> Measurement Celsius))
(defun (kelvin->celsius k)
  (Celsius (- (ok-or-raise (measurement-in k "K")) 273.15)))

(: (-> Fahrenheit Measurement))
(defun (fahrenheit->kelvin f)
  (def (Fahrenheit x) f)
  (measurement (* (+ x 459.67) (/ 5.0 9.0)) (measurement-dims (ok-or-raise (string->measurement "K")))))

;; Raises ArgumentException when k is not a temperature.
(: (-> Measurement Fahrenheit))
(defun (kelvin->fahrenheit k)
  (Fahrenheit (- (* (ok-or-raise (measurement-in k "K")) (/ 9.0 5.0)) 459.67)))

(impl (Written Celsius)
  (defun (written c) (def (Celsius x) c) (str "(Celsius " (written x) ")"))
  (defun (write-into c b) (stringbuilder-add-string! b (written c))))

(impl (Displayed Celsius)
  (defun (displayed c) (def (Celsius x) c) (str (format-number x) " °C"))
  (defun (display-into c b) (stringbuilder-add-string! b (displayed c))))

(impl (Written Fahrenheit)
  (defun (written f) (def (Fahrenheit x) f) (str "(Fahrenheit " (written x) ")"))
  (defun (write-into f b) (stringbuilder-add-string! b (written f))))

(impl (Displayed Fahrenheit)
  (defun (displayed f) (def (Fahrenheit x) f) (str (format-number x) " °F"))
  (defun (display-into f b) (stringbuilder-add-string! b (displayed f))))

;; ---------------------------------------------------------------------------
;; Logarithmic
;; ---------------------------------------------------------------------------
;; Each raises ArgumentException when the measurement has the wrong dimension.

(: (-> double Measurement string double))
(defun (decibels k m reference)
  (* k (log10 (ok-or-raise (measurement-in m reference)))))

(: (-> double double string Measurement))
(defun (from-decibels k x reference)
  (measurement-scale (pow 10.0 (/ x k)) (ok-or-raise (string->measurement reference))))

(: (-> Measurement double))
(defun (dBW p) (decibels 10.0 p "1 W"))

(: (-> Measurement double))
(defun (dBm p) (decibels 10.0 p "1 mW"))

(: (-> Measurement double))
(defun (dBV v) (decibels 20.0 v "1 V"))

(: (-> Measurement double))
(defun (dBSPL p) (decibels 20.0 p "20 uPa"))

(: (-> double Measurement))
(defun (dBW->power x) (from-decibels 10.0 x "1 W"))

(: (-> double Measurement))
(defun (dBm->power x) (from-decibels 10.0 x "1 mW"))

(: (-> double Measurement))
(defun (dBV->voltage x) (from-decibels 20.0 x "1 V"))

(: (-> double Measurement))
(defun (dBSPL->pressure x) (from-decibels 20.0 x "20 uPa"))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/url.bjo — URLs: parse, write, resolve, and percent-encoding.
;;
;; The grammar is RFC 3986's, read strictly. A URL holds each part as it is
;; written, percent-encoding included, so that a URL read and written again is
;; the same text. The decoded forms are functions over it.

(import (std prelude))

;; Every export is documented there; the module doc is too.
(:doc #:file "url.bjodoc")

(export
  ;; The value.
  Url Authority
  url-scheme url-authority url-path url-query url-fragment
  authority-userinfo authority-host authority-port
  url-host url-port url-absolute?

  ;; To and from readable forms
  parse-url url->string

  ;; A reference against a base.
  url-resolve remove-dot-segments

  ;; The decoded parts.
  url-segments url-params
  query->params params->query param-ref param-ref-all

  ;; Percent-encoding, one function for each part.
  encode-segment encode-query-part encode-fragment encode-userinfo encode-form
  percent-decode form-decode

  ;; A URL written as a quoted literal. `Link` is a grammar. Its export also
  ;; exports UrlParts, PathParts, QueryParam, UrlText and UrlPiece.
  Link make-url link

  ;; The checked parts of a literal, and the checks that make them.
  Scheme Host Port scheme host port scheme->string host->string port->int)

;; ===========================================================================
;; Interop
;; ===========================================================================

(import/class
  (IPAddress (: System.Net.IPAddress))
  (IdnMapping (: System.Globalization.IdnMapping (-> IdnMapping))))

(import/extern
  (clr-parse-ip (: System.Net.IPAddress.Parse (-> string IPAddress)
                   #:exceptions (System.FormatException)))
  (clr-idn-ascii (: System.Globalization.IdnMapping.GetAscii (-> IdnMapping string string)
                    #:exceptions (System.ArgumentException))))

;; ===========================================================================
;; The value
;; ===========================================================================

;; `//userinfo@host:port`. The user info and the host are as written: the host
;; of an IP literal keeps its brackets, `[::1]`.
(type (: Authority (Record (: userinfo (Option string))
                           (: host string)
                           (: port (Option int)))))

;; A URL, or a relative reference, as RFC 3986 calls one that has no scheme.
;; Each part is as written, percent-encoding included, and without the
;; punctuation that separates it: no `:` after the scheme, no `?` before the
;; query, no `#` before the fragment. `None` is a part that is not there,
;; which is not the same as one that is empty: `http://h/?` has a query, `""`.
(type (: Url (Record (: scheme (Option string))
                     (: authority (Option Authority))
                     (: path string)
                     (: query (Option string))
                     (: fragment (Option string)))))

(: (-> Url (Option string)))
(defun (url-scheme u) (record-ref u scheme))

(: (-> Url (Option Authority)))
(defun (url-authority u) (record-ref u authority))

(: (-> Url string))
(defun (url-path u) (record-ref u path))

(: (-> Url (Option string)))
(defun (url-query u) (record-ref u query))

(: (-> Url (Option string)))
(defun (url-fragment u) (record-ref u fragment))

(: (-> Authority (Option string)))
(defun (authority-userinfo a) (record-ref a userinfo))

(: (-> Authority string))
(defun (authority-host a) (record-ref a host))

(: (-> Authority (Option int)))
(defun (authority-port a) (record-ref a port))

(: (-> Url (Option string)))
(defun (url-host u)
  (option-map (fun (a) (record-ref a host)) (record-ref u authority)))

(: (-> Url (Option int)))
(defun (url-port u)
  (match (record-ref u authority)
    ((Some a) (record-ref a port))
    (None None)))

;; Has a scheme: `https://example.com/`, `mailto:a@b`, and not `/a` or `//h/a`.
(: (-> Url bool))
(defun (url-absolute? u) (some? (record-ref u scheme)))

;; ===========================================================================
;; Bytes and their classes
;; ===========================================================================

(: (-> (Array byte) int int))
(defun (byte-at bytes i) (cast int (array-ref bytes i)))

(: (-> int bool))
(defun (alpha? b) (or (and (>= b 65) (<= b 90)) (and (>= b 97) (<= b 122))))

(: (-> int bool))
(defun (digit? b) (and (>= b 48) (<= b 57)))

(: (-> int int))
(defun (hex-value b)
  (cond
    ((digit? b) (- b 48))
    ((and (>= b 65) (<= b 70)) (- b 55))
    ((and (>= b 97) (<= b 102)) (- b 87))
    (else -1)))

;; ALPHA DIGIT - . _ ~
(: (-> int bool))
(defun (unreserved? b)
  (or (alpha? b) (digit? b) (= b 45) (= b 46) (= b 95) (= b 126)))

;; ! $ & ' ( ) * + , ; =
(: (-> int bool))
(defun (sub-delim? b)
  (case b
    ((33 36 38 39 40 41 42 43 44 59 61) #t)
    (else #f)))

;; What a path segment holds as it is: unreserved, sub-delims, `:` and `@`.
(: (-> int bool))
(defun (pchar? b) (or (unreserved? b) (sub-delim? b) (= b 58) (= b 64)))

(: (-> int bool))
(defun (path-char? b) (or (pchar? b) (= b 47)))

;; A query and a fragment hold `/` and `?` too.
(: (-> int bool))
(defun (query-char? b) (or (pchar? b) (= b 47) (= b 63)))

(: (-> int bool))
(defun (userinfo-char? b) (or (unreserved? b) (sub-delim? b) (= b 58)))

(: (-> int bool))
(defun (reg-name-char? b) (or (unreserved? b) (sub-delim? b)))

(: (-> int bool))
(defun (scheme-char? b) (or (alpha? b) (digit? b) (= b 43) (= b 45) (= b 46)))

;; ===========================================================================
;; Percent-encoding
;; ===========================================================================

(: (Array byte))
(def hex-bytes (string->bytes "0123456789ABCDEF"))

(: (-> StringBuilder int void))
(defun (add-escape! sb b)
  (stringbuilder-add-byte! sb 37)
  (stringbuilder-add-byte! sb (byte-at hex-bytes (/ b 16)))
  (stringbuilder-add-byte! sb (byte-at hex-bytes (% b 16))))

;; `s` as UTF-8, with each byte that `keep?` refuses written `%XX`. With `plus`,
;; a space is `+`, as a form writes it.
(: (-> (-> int bool) bool string string))
(defun (encode-with keep? plus s)
  (def bytes (string->bytes s))
  (def sb (stringbuilder-empty))
  (loop (:for i (range 0 (array-length bytes)))
        (:do (let ((b (byte-at bytes i)))
               (cond
                 ((keep? b) (stringbuilder-add-byte! sb b))
                 ((and plus (= b 32)) (stringbuilder-add-byte! sb 43))
                 (else (add-escape! sb b))))))
  (stringbuilder->string sb))

;; One segment of a path. A `/` is encoded, so it cannot begin a segment.
(: (-> string string))
(defun (encode-segment s) (encode-with pchar? #f s))

;; A key or a value of a query. `&`, `=`, `+`, `;` and `#` are encoded, so a
;; value cannot begin another parameter or end the query, and a space is
;; `%20`, which every server reads as a space.
(: (-> int bool))
(defun (query-part-char? b)
  (or (unreserved? b)
      (case b
        ((33 36 39 40 41 42 44 58 64 47 63) #t)
        (else #f))))

(: (-> string string))
(defun (encode-query-part s) (encode-with query-part-char? #f s))

(: (-> string string))
(defun (encode-fragment s) (encode-with query-char? #f s))

;; A user name or a password. `:` and `@` are encoded.
(: (-> string string))
(defun (encode-userinfo s) (encode-with reg-name-char? #f s))

;; As `application/x-www-form-urlencoded` writes it, which is what a browser
;; sends for a form: letters, digits and `* - . _` as they are, a space as
;; `+`, and every other byte `%XX`.
(: (-> int bool))
(defun (form-char? b)
  (or (alpha? b) (digit? b) (= b 42) (= b 45) (= b 46) (= b 95)))

(: (-> string string))
(defun (encode-form s) (encode-with form-char? #t s))

;; The text that `s` encodes. With `plus`, a `+` is a space. Refused: a `%`
;; that two hex digits do not follow, and bytes that are not UTF-8.
(: (-> bool string (Result string string)))
(defun (decode-with plus s)
  (def bytes (string->bytes s))
  (def n (array-length bytes))
  (def sb (stringbuilder-empty))
  (let go ((i 0))
    (cond
      ((>= i n)
       (result-map-err (fun (_) "the percent-encoded bytes are not UTF-8")
                       (try (stringbuilder->string sb) #:catch (System.InvalidOperationException))))
      (else
       (def b (byte-at bytes i))
       (cond
         ((= b 37)
          (def hi (if (< (+ i 1) n) (hex-value (byte-at bytes (+ i 1))) -1))
          (def lo (if (< (+ i 2) n) (hex-value (byte-at bytes (+ i 2))) -1))
          (if (and (>= hi 0) (>= lo 0))
              (begin (stringbuilder-add-byte! sb (+ (* hi 16) lo)) (go (+ i 3)))
              (Err (str "column " (int->string (+ i 1))
                        ": a '%' begins an escape of two hex digits"))))
         ((and plus (= b 43)) (stringbuilder-add-byte! sb 32) (go (+ i 1)))
         (else (stringbuilder-add-byte! sb b) (go (+ i 1))))))))

(: (-> string (Result string string)))
(defun (percent-decode s) (decode-with #f s))

(: (-> string (Result string string)))
(defun (form-decode s) (decode-with #t s))

;; ===========================================================================
;; Reading a URL
;; ===========================================================================

(: (-> (Array byte) int int string))
(defun (slice bytes from to) (bytes->string bytes #:start from #:end to))

;; The first index from `i` before `end` whose byte is one of `stops`, or `end`.
(: (-> (Array byte) int int (-> int bool) int))
(defun (find-any bytes i end stop?)
  (cond
    ((>= i end) end)
    ((stop? (byte-at bytes i)) i)
    (else (find-any bytes (+ i 1) end stop?))))

;; What a refusal calls the byte at `i`.
(: (-> int string))
(defun (what-byte b)
  (cond
    ((= b 32) "a space")
    ((>= b 128) "a character that is not ASCII")
    ((< b 32) "a control character")
    (else (str "'" (list->string (list (int->char b))) "'"))))

(: (-> int string (Result string %a)))
(defun (refuse-at i what) (Err (str "column " (int->string (+ i 1)) ": " what)))

;; Every byte from `i` to `end` is one `ok?` allows, or a `%XX`.
(: (-> (Array byte) int int (-> int bool) string (Result string bool)))
(defun (check-part bytes i end ok? part)
  (if (>= i end)
      (Ok #t)
      (let ((b (byte-at bytes i)))
        (cond
          ((= b 37)
           (if (and (< (+ i 2) end)
                    (>= (hex-value (byte-at bytes (+ i 1))) 0)
                    (>= (hex-value (byte-at bytes (+ i 2))) 0))
               (check-part bytes (+ i 3) end ok? part)
               (refuse-at i "a '%' begins an escape of two hex digits")))
          ((ok? b) (check-part bytes (+ i 1) end ok? part))
          (else
           (refuse-at i (str (what-byte b) " is not allowed in the " part
                             ", and has to be percent-encoded")))))))

;; Every byte from `i` to `end` is a hex digit.
(: (-> (Array byte) int int bool))
(defun (all-hex? bytes i end)
  (cond
    ((>= i end) #t)
    ((< (hex-value (byte-at bytes i)) 0) #f)
    (else (all-hex? bytes (+ i 1) end))))

;; The digits of a port, `None` for none.
(: (-> (Array byte) int int (Result string (Option int))))
(defun (read-port bytes i end)
  (let go ((k i) (n 0))
    (cond
      ((>= k end) (if (= k i) (Ok None) (Ok (Some n))))
      ((not (digit? (byte-at bytes k)))
       (refuse-at k (str "a port is digits, and this is " (what-byte (byte-at bytes k)))))
      ((> (+ (* n 10) (- (byte-at bytes k) 48)) 65535)
       (refuse-at i "a port is at most 65535"))
      (else (go (+ k 1) (+ (* n 10) (- (byte-at bytes k) 48)))))))

;; `[...]`: an IPv6 address, or `v1.x`, the IPvFuture of RFC 3986. A zone, the
;; `%` of `fe80::1%eth0`, is not part of a URL's address.
(: (-> (Array byte) int int (Result string bool)))
(defun (check-ip-literal bytes i end)
  (def first (if (< i end) (byte-at bytes i) -1))
  (cond
    ((or (= first 118) (= first 86))
     (def dot (find-any bytes (+ i 1) end (fun (b) (= b 46))))
     (cond
       ((or (= dot (+ i 1)) (>= dot end) (not (all-hex? bytes (+ i 1) dot)))
        (refuse-at i "an IPvFuture address is v, hex digits, '.' and then its text"))
       ((= (+ dot 1) end) (refuse-at dot "an IPvFuture address has text after its '.'"))
       (else (check-part bytes (+ dot 1) end userinfo-char? "address"))))
    ((< (find-any bytes i end (fun (b) (= b 37))) end)
     (refuse-at (find-any bytes i end (fun (b) (= b 37))) "an IPv6 address in a URL has no zone"))
    (else
     (match (clr-parse-ip (slice bytes i end))
       ((Ok _) (Ok #t))
       ((Err _) (refuse-at i "this is not an IPv6 address"))))))

;; `userinfo@host:port`, from `i` to `end`.
(: (-> (Array byte) int int (Result string Authority)))
(defun (read-authority bytes i end)
  (def at (find-any bytes i end (fun (b) (= b 64))))
  (def host-start (if (< at end) (+ at 1) i))
  (def (Ok userinfo)
    (if (< at end)
        (result-map (fun (_) (Some (slice bytes i at)))
                    (check-part bytes i at userinfo-char? "user info"))
        (Ok None))
    :propagate)
  (def (Ok host-end)
    (cond
      ((and (< host-start end) (= (byte-at bytes host-start) 91))
       (def close (find-any bytes host-start end (fun (b) (= b 93))))
       (cond
         ((>= close end) (refuse-at host-start "a '[' begins an IP address, and there is no ']'"))
         (else
          (def (Ok _) (check-ip-literal bytes (+ host-start 1) close) :propagate)
          (if (or (= (+ close 1) end) (= (byte-at bytes (+ close 1)) 58))
              (Ok (+ close 1))
              (refuse-at (+ close 1) "after ']' comes ':' and the port, or the end of the host")))))
      (else
       (def colon (find-any bytes host-start end (fun (b) (= b 58))))
       (result-map (fun (_) colon)
                   (check-part bytes host-start colon reg-name-char? "host"))))
    :propagate)
  (def (Ok port)
    (if (< host-end end) (read-port bytes (+ host-end 1) end) (Ok None))
    :propagate)
  (Ok (Authority (userinfo userinfo) (host (slice bytes host-start host-end)) (port port))))

;; A URL, or a relative reference, read by RFC 3986. What the grammar does not
;; allow is refused, with the column where it is: a space, a character that
;; is not ASCII, a `%` without two hex digits, a port that is not a number.
;; Such text has to be percent-encoded first.
(: (-> string (Result string Url)))
(defun (parse-url s)
  (def bytes (string->bytes s))
  (def n (array-length bytes))
  ;; A scheme is a letter and scheme characters, and then a ':'.
  (def scheme-end
    (if (and (> n 0) (alpha? (byte-at bytes 0)))
        (let ((k (find-any bytes 1 n (fun (b) (not (scheme-char? b))))))
          (if (and (< k n) (= (byte-at bytes k) 58)) k -1))
        -1))
  (def scheme (if (>= scheme-end 0) (Some (slice bytes 0 scheme-end)) None))
  (def after-scheme (if (>= scheme-end 0) (+ scheme-end 1) 0))
  (def has-authority?
    (and (< (+ after-scheme 1) n)
         (= (byte-at bytes after-scheme) 47)
         (= (byte-at bytes (+ after-scheme 1)) 47)))
  (def authority-end
    (if has-authority?
        (find-any bytes (+ after-scheme 2) n (fun (b) (or (= b 47) (= b 63) (= b 35))))
        after-scheme))
  (def (Ok authority)
    (if has-authority?
        (result-map Some (read-authority bytes (+ after-scheme 2) authority-end))
        (Ok None))
    :propagate)
  (def path-end (find-any bytes authority-end n (fun (b) (or (= b 63) (= b 35)))))
  (def (Ok _) (check-part bytes authority-end path-end path-char? "path") :propagate)
  ;; `a:b` is the scheme `a`. A relative reference whose first segment has a
  ;; ':' would read as one, so RFC 3986 does not allow it: `./a:b` does.
  (def first-slash (find-any bytes authority-end path-end (fun (b) (= b 47))))
  (def first-colon (find-any bytes authority-end first-slash (fun (b) (= b 58))))
  (def (Ok _)
    (if (and (none? scheme) (not has-authority?) (< first-colon first-slash))
        (refuse-at first-colon
                   "a relative reference cannot have a ':' in its first segment; write './' before it")
        (Ok #t))
    :propagate)
  (def has-query? (and (< path-end n) (= (byte-at bytes path-end) 63)))
  (def query-end
    (if has-query? (find-any bytes (+ path-end 1) n (fun (b) (= b 35))) path-end))
  (def (Ok query)
    (if has-query?
        (result-map (fun (_) (Some (slice bytes (+ path-end 1) query-end)))
                    (check-part bytes (+ path-end 1) query-end query-char? "query"))
        (Ok None))
    :propagate)
  (def (Ok fragment)
    (if (< query-end n)
        (result-map (fun (_) (Some (slice bytes (+ query-end 1) n)))
                    (check-part bytes (+ query-end 1) n query-char? "fragment"))
        (Ok None))
    :propagate)
  (Ok (Url (scheme scheme)
           (authority authority)
           (path (slice bytes authority-end path-end))
           (query query)
           (fragment fragment))))

;; ===========================================================================
;; Writing a URL
;; ===========================================================================

(: (-> Url string))
(defun (url->string u)
  (def sb (stringbuilder-empty))
  (match (record-ref u scheme)
    ((Some s) (stringbuilder-add-string! sb s) (stringbuilder-add! sb #\:))
    (None unit))
  (match (record-ref u authority)
    ((Some a)
     (stringbuilder-add-string! sb "//")
     (match (record-ref a userinfo)
       ((Some ui) (stringbuilder-add-string! sb ui) (stringbuilder-add! sb #\@))
       (None unit))
     (stringbuilder-add-string! sb (record-ref a host))
     (match (record-ref a port)
       ((Some p) (stringbuilder-add! sb #\:) (stringbuilder-add-string! sb (int->string p)))
       (None unit)))
    (None unit))
  (stringbuilder-add-string! sb (record-ref u path))
  (match (record-ref u query)
    ((Some q) (stringbuilder-add! sb #\?) (stringbuilder-add-string! sb q))
    (None unit))
  (match (record-ref u fragment)
    ((Some f) (stringbuilder-add! sb #\#) (stringbuilder-add-string! sb f))
    (None unit))
  (stringbuilder->string sb))

;; ===========================================================================
;; Resolving a reference
;; ===========================================================================

;; RFC 3986 5.2.4: `.` and `..` segments taken out of a path. A `..` above the
;; root is dropped, so `/a/../../b` is `/b`. A path that ended in `.` or `..`
;; ends in `/`.
(: (-> string string))
(defun (remove-dot-segments path)
  (def absolute? (string-starts-with? path "/"))
  (def parts (vec->list (string-split path "/")))
  (def segments (if absolute? (list-tail parts) parts))
  (def kept
    (let go ((rest segments) (out (list)))
      (match rest
        (Nil (list-reverse out))
        ((Cons s more)
         (def last? (list-empty? more))
         (cond
           ((= s ".") (go more (if last? (cons "" out) out)))
           ((= s "..")
            (def popped (if (list-empty? out) out (list-tail out)))
            (go more (if last? (cons "" popped) popped)))
           (else (go more (cons s out))))))))
  (str (if absolute? "/" "") (string-join (list->vec kept) "/")))

;; RFC 3986 5.2.3: a relative path against the base's.
(: (-> Url string string))
(defun (merge-paths base path)
  (def base-path (record-ref base path))
  (cond
    ((and (some? (record-ref base authority)) (string-empty? base-path)) (str "/" path))
    (else
     (def bytes (string->bytes base-path))
     (def last-slash
       (let go ((k (- (array-length bytes) 1)))
         (cond ((< k 0) -1)
               ((= (byte-at bytes k) 47) k)
               (else (go (- k 1))))))
     (if (< last-slash 0)
         path
         (str (slice bytes 0 (+ last-slash 1)) path)))))

;; `ref` read where `base` is, as a browser reads a link on a page: RFC 3986
;; 5.2.2. A reference with a scheme stands for itself. `base` should be
;; absolute, as the RFC asks.
(: (-> Url Url Url))
(defun (url-resolve base ref)
  (def r-path (record-ref ref path))
  (cond
    ((some? (record-ref ref scheme))
     (record-set ref (path (remove-dot-segments r-path))))
    ((some? (record-ref ref authority))
     (record-set ref (scheme (record-ref base scheme)) (path (remove-dot-segments r-path))))
    ((string-empty? r-path)
     (Url (scheme (record-ref base scheme))
          (authority (record-ref base authority))
          (path (record-ref base path))
          (query (match (record-ref ref query)
                   ((Some q) (Some q))
                   (None (record-ref base query))))
          (fragment (record-ref ref fragment))))
    (else
     (Url (scheme (record-ref base scheme))
          (authority (record-ref base authority))
          (path (remove-dot-segments
                 (if (string-starts-with? r-path "/") r-path (merge-paths base r-path))))
          (query (record-ref ref query))
          (fragment (record-ref ref fragment))))))

;; ===========================================================================
;; The decoded parts
;; ===========================================================================

;; The path's segments, decoded. `/a/b%2Fc/` is `("a" "b/c" "")`: the leading
;; `/` begins the first segment, a `/` that ends the path leaves an empty one,
;; and an encoded `/` is part of its segment. The empty path has none.
(: (-> Url (Result string (List string))))
(defun (url-segments u)
  (def path (record-ref u path))
  (cond
    ((string-empty? path) (Ok (list)))
    (else
     (def parts (vec->list (string-split path "/")))
     (def raw (if (string-starts-with? path "/") (list-tail parts) parts))
     (result-map list-reverse
                 (list-foldl (fun (seg acc)
                               (def (Ok done) acc :propagate)
                               (result-map (fun (d) (Cons d done)) (percent-decode seg)))
                             (Ok (list))
                             raw)))))

;; `a=1&b=two+words&b` as pairs, in order, decoded, with `+` read as a space as
;; a form writes it. A parameter with no `=` has the value `""`, and an empty
;; one, `a=1&&b=2`, is no parameter. A key that comes twice is there twice.
(: (-> string (Result string (List (Tuple string string)))))
(defun (query->params q)
  (def pieces (list-filter (fun (p) (not (string-empty? p))) (vec->list (string-split q "&"))))
  (result-map list-reverse
              (list-foldl (fun (piece acc)
                            (def (Ok done) acc :propagate)
                            (def bytes (string->bytes piece))
                            (def n (array-length bytes))
                            (def eq (find-any bytes 0 n (fun (b) (= b 61))))
                            (def (Ok k) (form-decode (slice bytes 0 eq)) :propagate)
                            (def (Ok v)
                              (if (< eq n) (form-decode (slice bytes (+ eq 1) n)) (Ok ""))
                              :propagate)
                            (Ok (Cons (Tuple k v) done)))
                          (Ok (list))
                          pieces)))

;; The query's parameters, as `query->params` reads them. No query has none.
(: (-> Url (Result string (List (Tuple string string)))))
(defun (url-params u)
  (match (record-ref u query)
    ((Some q) (query->params q))
    (None (Ok (list)))))

;; Pairs written as a query, without the `?`. Each key and value is encoded
;; with `encode-query-part`, so it cannot begin another parameter.
(: (-> (List (Tuple string string)) string))
(defun (params->query params)
  (string-join
   (list->vec
    (list-map (fun (p)
                (match p
                  ((Tuple k v) (str (encode-query-part k) "=" (encode-query-part v)))))
              params))
   "&"))

;; The first value of `key`.
(: (-> (List (Tuple string string)) string (Option string)))
(defun (param-ref params key)
  (match params
    (Nil None)
    ((Cons (Tuple k v) rest) (if (= k key) (Some v) (param-ref rest key)))))

;; Every value of `key`, in order.
(: (-> (List (Tuple string string)) string (List string)))
(defun (param-ref-all params key)
  (match params
    (Nil (list))
    ((Cons (Tuple k v) rest)
     (if (= k key) (Cons v (param-ref-all rest key)) (param-ref-all rest key)))))

;; ===========================================================================
;; A URL written as a quoted literal
;; ===========================================================================
;;
;;     '(url https "example.com" (port 8443) (path "users" ,name)
;;           (query (page 2) (tag ,t)) (fragment "top"))
;;     '(path "users" ,id "edit" (query (q ,text)))
;;
;; The grammar `Link` reads the literal. Each value is encoded for the part it
;; is in: a segment as a segment, a query key or value as one. So a `/` in a
;; computed segment does not make two segments, and a `&` in a computed value
;; does not begin a parameter. The compiler checks the keywords, the order and
;; the number of each part where you write the literal. It also runs the
;; checks `scheme`, `host` and `port` on a literal scheme, host and port.

;; A scheme that `scheme` accepted, such as `https`.
(type (: Scheme #:opaque (Record (: name string))))

;; A host that `host` accepted, in the form a URL writes it.
(type (: Host #:opaque (Record (: name string))))

;; A port that `port` accepted: from 0 to 65535.
(type (: Port #:opaque (Record (: number int))))

;; A letter, then letters, digits, `+`, `-` and `.`.
(: (-> Symbol (Result string Scheme)))
(defun (scheme s)
  (def text (symbol->string s))
  (def bytes (string->bytes text))
  (def n (array-length bytes))
  (if (and (> n 0) (alpha? (byte-at bytes 0))
           (= (find-any bytes 1 n (fun (b) (not (scheme-char? b)))) n))
      (Ok (Scheme (name text)))
      (Err (str "'" text "' is not a scheme"))))

;; A host as a URL writes it. An IPv6 address gets its brackets. A name that
;; is not ASCII becomes the IDNA form (`bücher.example` is
;; `xn--bcher-kva.example`). Text that is not a name or an address is an
;; error.
(: (-> string (Result string Host)))
(defun (host h)
  (def bytes (string->bytes h))
  (def n (array-length bytes))
  (cond
    ((and (> n 0) (= (byte-at bytes 0) 91))
     (match (if (= (byte-at bytes (- n 1)) 93)
                (check-ip-literal bytes 1 (- n 1))
                (Err "it has no ']'"))
       ((Ok _) (Ok (Host (name h))))
       ((Err e) (Err (str "'" h "' is not a host: " e)))))
    ((< (find-any bytes 0 n (fun (b) (= b 58))) n)
     (match (check-ip-literal bytes 0 n)
       ((Ok _) (Ok (Host (name (str "[" h "]")))))
       ((Err e) (Err (str "'" h "' is not a host: " e)))))
    ((= (find-any bytes 0 n (fun (b) (not (reg-name-char? b)))) n) (Ok (Host (name h))))
    ((< (find-any bytes 0 n (fun (b) (>= b 128))) n)
     (match (clr-idn-ascii (IdnMapping.) h)
       ((Ok ascii) (host ascii))
       ((Err _) (Err (str "'" h "' is not a host name")))))
    (else (Err (str "'" h "' is not a host name")))))

(: (-> int (Result string Port)))
(defun (port n)
  (if (and (>= n 0) (<= n 65535))
      (Ok (Port (number n)))
      (Err (str "a port is from 0 to 65535, and this is " (int->string n)))))

(: (-> Scheme string))
(defun (scheme->string s) (record-ref s name))

(: (-> Host string))
(defun (host->string h) (record-ref h name))

(: (-> Port int))
(defun (port->int p) (record-ref p number))

(def/grammar Link
  ;; A URL with a scheme and a host, or a path on this site.
  (Link (:or (: LinkUrl  (url Scheme Host UrlParts))
             (: LinkPath (path PathParts))))

  ;; The parts of a URL after the host, in any order, each at most once.
  (UrlParts (:each (: port     (:? (port Port)))
                   (: user     (:? (user UrlText)))
                   (: path     (:? (path UrlPiece ...)))
                   (: query    (:? (query QueryParam ...)))
                   (: fragment (:? (fragment UrlText)))))

  ;; The segments of a path on this site, then its query and its fragment.
  (PathParts (: segments UrlPiece ...)
             (:each (: query    (:? (query QueryParam ...)))
                    (: fragment (:? (fragment UrlText)))))

  ;; One parameter of a query: `(key value)`, `(key)` with no value, or a
  ;; computed pair, `,(Tuple k v)`.
  (QueryParam (:or (: ParamForm (UrlText (:? UrlPiece)))
                   (: ParamPair ,(Tuple string string))))

  ;; One text: a string, a symbol as its name, or an int.
  (UrlText (:or (: TextStr string) (: TextSym symbol) (: TextInt int)))

  ;; One text, or a computed `(Vec string)`: several segments, or several
  ;; values of one key.
  (UrlPiece (:or (: PieceOne UrlText) (: PieceMany ,(Vec string))))

  (Scheme (symbol #:check scheme))
  (Host (string #:check host))
  (Port (int #:check port)))

(: (-> UrlText string))
(defun (text-of t)
  (match t
    ((TextStr s) s)
    ((TextSym s) (symbol->string s))
    ((TextInt n) (int->string n))))

(: (-> UrlPiece (List string)))
(defun (piece-texts p)
  (match p
    ((PieceOne t) (list (text-of t)))
    ((PieceMany v) (vec->list v))))

(: (-> (List UrlPiece) (List string)))
(defun (pieces-texts ps)
  (list-foldr (fun (p acc) (list-append (piece-texts p) acc)) (list) ps))

;; `key=value` texts, encoded, for one parameter.
(: (-> QueryParam (List string)))
(defun (param-texts p)
  (match p
    ((ParamPair (Tuple k v)) (list (str (encode-query-part k) "=" (encode-query-part v))))
    ((ParamForm key None) (list (encode-query-part (text-of key))))
    ((ParamForm key (Some value))
     (def k (encode-query-part (text-of key)))
     (list-map (fun (v) (str k "=" (encode-query-part v))) (piece-texts value)))))

(: (-> (List QueryParam) string))
(defun (query-text params)
  (string-join (list->vec (list-foldr (fun (p acc) (list-append (param-texts p) acc))
                                      (list) params))
               "&"))

(: (-> (List string) string))
(defun (path-text segs)
  (if (list-empty? segs)
      "/"
      (string-join (list->vec (list-map (fun (s) (str "/" (encode-segment s))) segs)) "")))

(: (-> (Option UrlText) (Option string)))
(defun (fragment-text f) (option-map (fun (t) (encode-fragment (text-of t))) f))

;; The `Url` that a literal stands for. The grammar and the checks refused
;; each bad literal, so this does not fail.
(: (-> Link Url))
(defun (make-url l)
  (match l
    ((LinkPath parts)
     (Url (scheme None) (authority None)
          (path (path-text (pieces-texts (record-ref parts segments))))
          (query (option-map query-text (record-ref parts query)))
          (fragment (fragment-text (record-ref parts fragment)))))
    ((LinkUrl s h parts)
     (Url (scheme (Some (scheme->string s)))
          (authority (Some (Authority
                            (userinfo (option-map (fun (t) (encode-userinfo (text-of t)))
                                                  (record-ref parts user)))
                            (host (host->string h))
                            (port (option-map port->int (record-ref parts port))))))
          (path (match (record-ref parts path)
                  ((Some pieces) (path-text (pieces-texts pieces)))
                  (None "")))
          (query (option-map query-text (record-ref parts query)))
          (fragment (fragment-text (record-ref parts fragment)))))))

;; The text of a literal: `(url->string (make-url l))`.
(: (-> Link string))
(defun (link l) (url->string (make-url l)))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; std/uuid.bjo — uuids, as RFC 9562 specifies them.
;;
;; A `Uuid` is 16 bytes, held as two `ulong`s in network byte order. Two uuids
;; compare as their bytes compare, which is also how their text compares and
;; how Postgres orders its `uuid` type. A version 7 uuid starts with the time,
;; so version 7 uuids sort by the time they were made.
;;
;; The random bits come from `(std random)`, and the time of a version 7 uuid
;; from `current-clock`. Both are parameters, so a test can fix them.

(export
  Uuid
  ;; Making one
  uuid-v4 uuid-v7 uuid-v5 uuid-v3
  uuid-nil uuid-max
  uuid-ns-dns uuid-ns-url uuid-ns-oid uuid-ns-x500
  ;; Text and bytes
  uuid->string string->uuid uuid-string?
  uuid->bytes bytes->uuid
  ;; What is in one
  uuid-version uuid-v7-time
  ;; .NET
  uuid->guid guid->uuid)

(import (std prelude))
(import (only (std random) current-random random-u64))
(import (only (std datetime) Instant now instant->unix-milliseconds unix-milliseconds->instant))
(import (only (std crypto hash) HashAlgorithm make-hasher hasher-add! hasher-add-string! hasher-finish!))
(import (only (std atomic) Atomic make-atomic atomic-update!))

(:doc #:file "uuid.bjodoc")

;; The constructor that takes the fields in the order of the bytes:
;; `(uint, ushort, ushort, byte x 8)` is bytes 0-3, 4-5, 6-7 and 8-15,
;; big-endian.
(import/class
  (Guid (: System.Guid (-> uint ushort ushort byte byte byte byte byte byte byte byte Guid))))

(import/extern
  ;; With `#t`: the 16 bytes in the order of the text.
  (clr-guid-bytes (: System.Guid.ToByteArray (-> Guid bool (Array byte)))))

;; `hi` is the first 8 bytes, `lo` the last 8. Opaque, so every `Uuid` is one
;; that a function here made. `type/derive` does not take `#:opaque`, so `Eq`
;; and `Ord` are written out: `hi` first, then `lo`, which is the order of the
;; bytes.
(type (: Uuid #:opaque (Struct (: hi ulong) (: lo ulong))))

(impl (Eq Uuid)
  (defun (= a b) (and (= (struct-ref a hi) (struct-ref b hi)) (= (struct-ref a lo) (struct-ref b lo))))
  (defun (eq-hash u) (bitwise-xor (eq-hash (struct-ref u hi)) (* 31 (eq-hash (struct-ref u lo))))))

(impl (Ord Uuid)
  (defun (compare a b)
    (def by-hi (compare (struct-ref a hi) (struct-ref b hi)))
    (if (= by-hi 0) (compare (struct-ref a lo) (struct-ref b lo)) by-hi)))

(: (-> Uuid ulong))
(defun (hi-of u) (struct-ref u hi))

(: (-> Uuid ulong))
(defun (lo-of u) (struct-ref u lo))

;; ---------------------------------------------------------------------------
;; Bits and bytes
;; ---------------------------------------------------------------------------

;; Everything in `hi` except the 4 version bits, bits 12 to 15.
(: ulong)
(def version-mask 0xFFFFFFFFFFFF0FFF)

;; Everything in `lo` except the 2 variant bits at the top.
(: ulong)
(def variant-mask 0x3FFFFFFFFFFFFFFF)

;; The variant of RFC 9562: the top two bits of `lo` are 10.
(: ulong)
(def rfc-variant 0x8000000000000000)

(: (-> ulong int ulong))
(defun (with-version hi version)
  (bitwise-ior (bitwise-and hi version-mask) (shift-left (cast ulong version) 12)))

(: (-> ulong ulong))
(defun (with-variant lo) (bitwise-ior (bitwise-and lo variant-mask) rfc-variant))

;; The 8 bytes from `start`, big-endian.
(: (-> (Array byte) int ulong))
(defun (read-u64 bytes start)
  (let go ((i start) (v 0UL))
    (if (= i (+ start 8))
        v
        (go (+ i 1) (bitwise-ior (shift-left v 8) (cast ulong (array-ref bytes i)))))))

;; Writes `v` into the 8 bytes from `start`, big-endian.
(: (-> (Array byte) int ulong void))
(defun (write-u64! bytes start v)
  (loop (:for i (range 0 8))
        (:do (array-set! bytes (+ start i) (cast byte (shift-right-logical v (* 8 (- 7 i))))))))

;; ---------------------------------------------------------------------------
;; Making one
;; ---------------------------------------------------------------------------

;; 122 random bits.
(: (-> (#:rng RandomSource) Uuid))
(defun (uuid-v4 #:rng (parameter-ref current-random))
  (def r1 (random-u64 #:rng rng))
  (def r2 (random-u64 #:rng rng))
  (Uuid (hi (with-version r1 4)) (lo (with-variant r2))))

;; A version 7 uuid without its version and variant bits: the 48 bits of the
;; time and then the 74 random bits, as one 122-bit number. `high` is its top
;; 58 bits and `low` its bottom 64.
(type (: V7Bits (Struct (: high ulong) (: low ulong))))

(: (-> V7Bits V7Bits bool))
(defun (v7-greater? a b)
  (or (> (struct-ref a high) (struct-ref b high))
      (and (= (struct-ref a high) (struct-ref b high))
           (> (struct-ref a low) (struct-ref b low)))))

;; `b` plus 1.
(: (-> V7Bits V7Bits))
(defun (v7-next b)
  (def bottom (+ (struct-ref b low) 1UL))
  (V7Bits (high (if (= bottom 0UL) (+ (struct-ref b high) 1UL) (struct-ref b high)))
          (low bottom)))

;; The last version 7 value that this process made.
(: (Atomic V7Bits))
(def last-v7 (make-atomic (V7Bits (high 0UL) (low 0UL))))

;; The Unix time in milliseconds, then 74 random bits. The uuids that this
;; process makes always increase, also when two are made in one millisecond
;; and when the clock steps back.
;;
;; - A value that is not greater than the last one is the last one plus 1, as
;;   RFC 9562 section 6.2 allows.
;; - `last-v7` changes in one atomic step, so this holds across threads.
(: (-> (#:clock Clock) (#:rng RandomSource) Uuid))
(defun (uuid-v7 #:clock (parameter-ref current-clock) #:rng (parameter-ref current-random))
  (def ms (bitwise-and (cast ulong (instant->unix-milliseconds (now #:clock clock))) 0xFFFFFFFFFFFF))
  (def r1 (random-u64 #:rng rng))
  (def r2 (random-u64 #:rng rng))
  (def candidate (V7Bits (high (bitwise-ior (shift-left ms 10) (bitwise-and r1 0x3FF)))
                         (low r2)))
  (def chosen
    (atomic-update! last-v7 (fun (last) (if (v7-greater? candidate last) candidate (v7-next last)))))
  (def top (struct-ref chosen high))
  (def bottom (struct-ref chosen low))
  ;; 48 bits of time, the version, then the top 12 random bits.
  (def rand-a (bitwise-and (bitwise-ior (shift-left top 2) (shift-right-logical bottom 62)) 0xFFF))
  (Uuid (hi (bitwise-ior (shift-left (shift-right-logical top 10) 16) (bitwise-ior 0x7000 rand-a)))
        (lo (with-variant bottom))))

;; A digest of the namespace's 16 bytes and the name's UTF-8 bytes. The uuid
;; is the first 16 bytes of the digest, with the version and the variant set.
;; SHA-1 and MD5 digests are 20 and 16 bytes.
(: (-> HashAlgorithm int Uuid string Uuid))
(defun (name-based algorithm version namespace name)
  (def h (make-hasher #:algorithm algorithm))
  (hasher-add! h (uuid->bytes namespace))
  (hasher-add-string! h name)
  (def digest (hasher-finish! h))
  (Uuid (hi (with-version (read-u64 digest 0) version))
        (lo (with-variant (read-u64 digest 8)))))

;; The same namespace and name always give the same uuid. SHA-1.
(: (-> Uuid string Uuid))
(defun (uuid-v5 namespace name) (name-based Sha1 5 namespace name))

;; As `uuid-v5`, with MD5. For a system that already uses version 3.
(: (-> Uuid string Uuid))
(defun (uuid-v3 namespace name) (name-based Md5 3 namespace name))

(: Uuid)
(def uuid-nil (Uuid (hi 0UL) (lo 0UL)))

(: Uuid)
(def uuid-max (Uuid (hi 18446744073709551615UL) (lo 18446744073709551615UL)))

;; The namespaces of RFC 9562, section 6.6.
(: (-> string Uuid))
(defun (known text)
  (match (string->uuid text)
    ((Some u) u)
    (None (panic! (str "(std uuid): not a uuid: " text)))))

(: Uuid)
(def uuid-ns-dns (known "6ba7b810-9dad-11d1-80b4-00c04fd430c8"))

(: Uuid)
(def uuid-ns-url (known "6ba7b811-9dad-11d1-80b4-00c04fd430c8"))

(: Uuid)
(def uuid-ns-oid (known "6ba7b812-9dad-11d1-80b4-00c04fd430c8"))

(: Uuid)
(def uuid-ns-x500 (known "6ba7b814-9dad-11d1-80b4-00c04fd430c8"))

;; ---------------------------------------------------------------------------
;; Text and bytes
;; ---------------------------------------------------------------------------

(: (Array byte))
(def lower-digits (string->bytes "0123456789abcdef"))

(: (Array byte))
(def upper-digits (string->bytes "0123456789ABCDEF"))

;; Whether the 32 hex digits have a hyphen before digit `i`.
(: (-> int bool))
(defun (hyphen-before? i) (or (= i 8) (= i 12) (= i 16) (= i 20)))

;; 8-4-4-4-12 hex digits, lower case unless `#:upper #t`.
(: (-> Uuid (#:upper bool) string))
(defun (uuid->string u #:upper #f)
  (def digits (if upper upper-digits lower-digits))
  (def sb (stringbuilder-empty))
  (loop (:for i (range 0 32))
        (:do (when (hyphen-before? i) (stringbuilder-add-byte! sb 45))
             (let ((word (if (< i 16) (hi-of u) (lo-of u)))
                   (shift (* 4 (- 15 (% i 16)))))
               (stringbuilder-add-byte!
                 sb (cast int (array-ref digits (cast int (bitwise-and (shift-right-logical word shift) 15UL))))))))
  (stringbuilder->string sb))

;; 0 to 15 for an ASCII hex digit in either case, -1 for any other byte.
(: (-> byte int))
(defun (hex-value b)
  (def c (cast int b))
  (cond
    ((and (>= c 48) (<= c 57)) (- c 48))
    ((and (>= c 65) (<= c 70)) (- c 55))
    ((and (>= c 97) (<= c 102)) (- c 87))
    (else -1)))

;; Only the 36-character form, in upper or lower case: no braces, no
;; `urn:uuid:`, and no form without hyphens, so that one uuid has one text in
;; each case.
(: (-> string (Option Uuid)))
(defun (string->uuid text)
  (def bytes (string->bytes text))
  (if (not (= (array-length bytes) 36))
      None
      ;; `i` indexes the text, `digit` counts the hex digits read.
      (let go ((i 0) (digit 0) (h 0UL) (l 0UL))
        (cond
          ((= i 36) (Some (Uuid (hi h) (lo l))))
          ((or (= i 8) (= i 13) (= i 18) (= i 23))
           (if (= (array-ref bytes i) 45uy) (go (+ i 1) digit h l) None))
          (else
           (def v (hex-value (array-ref bytes i)))
           (cond
             ((< v 0) None)
             ((< digit 16) (go (+ i 1) (+ digit 1) (bitwise-ior (shift-left h 4) (cast ulong v)) l))
             (else (go (+ i 1) (+ digit 1) h (bitwise-ior (shift-left l 4) (cast ulong v))))))))))

(: (-> string bool))
(defun (uuid-string? text) (some? (string->uuid text)))

;; 16 bytes, in network byte order.
(: (-> Uuid (Array byte)))
(defun (uuid->bytes u)
  (def (: bytes (Array byte)) (make-array 16))
  (write-u64! bytes 0 (hi-of u))
  (write-u64! bytes 8 (lo-of u))
  bytes)

;; `None` unless there are exactly 16 bytes.
(: (-> (Array byte) (Option Uuid)))
(defun (bytes->uuid bytes)
  (if (= (array-length bytes) 16)
      (Some (Uuid (hi (read-u64 bytes 0)) (lo (read-u64 bytes 8))))
      None))

(impl (Written Uuid)
  (defun (written u) (uuid->string u))
  (defun (write-into u b) (stringbuilder-add-string! b (written u))))

;; ---------------------------------------------------------------------------
;; What is in one
;; ---------------------------------------------------------------------------

;; The 4 version bits: 4 for `uuid-v4`, 7 for `uuid-v7`. 0 for `uuid-nil` and
;; 15 for `uuid-max`.
(: (-> Uuid int))
(defun (uuid-version u)
  (cast int (bitwise-and (shift-right-logical (hi-of u) 12) 15UL)))

;; When a version 7 uuid was made, to the millisecond. `None` for any other
;; version.
(: (-> Uuid (Option Instant)))
(defun (uuid-v7-time u)
  (if (= (uuid-version u) 7)
      (unix-milliseconds->instant (cast long (shift-right-logical (hi-of u) 16)))
      None))

;; ---------------------------------------------------------------------------
;; .NET
;; ---------------------------------------------------------------------------

;; A `System.Guid` with the same 16 bytes, for a .NET library such as Npgsql.
;; A Guid keeps its first three fields in the machine's byte order, and its
;; `CompareTo` does not compare the bytes in order. Compare the `Uuid`.
(: (-> Uuid Guid))
(defun (uuid->guid u)
  (def b (uuid->bytes u))
  (def hi (hi-of u))
  (Guid. (cast uint (shift-right-logical hi 32))
         (cast ushort (shift-right-logical hi 16))
         (cast ushort hi)
         (array-ref b 8) (array-ref b 9) (array-ref b 10) (array-ref b 11)
         (array-ref b 12) (array-ref b 13) (array-ref b 14) (array-ref b 15)))

(: (-> Guid Uuid))
(defun (guid->uuid g)
  (def b (clr-guid-bytes g #t))
  (Uuid (hi (read-u64 b 0)) (lo (read-u64 b 8))))
;; text/bjodat-core.bjo
;;
;; The bjodat value, the reader and the writer. The reader reads the document's
;; UTF-8 bytes.
;;
;; **This is the machinery, not the interface.** `(text bjodat)` is what a
;; program imports: it declares a record and generates a decoder that reads the
;; text straight into it, with no `Bjodat` value in between. What is here is
;; what that decoder is built out of, and what a program reaches for directly
;; only when the shape of its data is not known until run time.
;;
;; A program that declares records with `def/bjodat-type` does not need this
;; module: `(text bjodat)` re-exports `Bjodat`, `Reader` and the one-pass
;; driver, so one import covers the generator, the traits and the writer. What
;; is left here, and the reason to import it by name, is the *tree* —
;; `parse-bjodat`, `bjodat-ref` and the entry accessors.
;;
;; It used to be imported alongside `(text bjodat)` always, and the comment
;; here said a facade could not spare you the second line. It can; I was wrong
;; about what `re-export` could be made to do. See `re-export` in
;; `Docs/MODULES.org`.
;;
;; Source's shapes, read as data: a bare identifier is a Symbol rather than a
;; reference, and `{...}` is a map rather than a comprehension. A file is a
;; stream of values.
;;
;; A map is held as a *flat* vec of alternating key and value, in the order it
;; was written. Not a trie, for two reasons that a benchmark settled: building
;; one is about seventy percent of what reading a keyed document costs, and a
;; record decoded from it reads every field once and then throws it away. A
;; linear scan of reference comparisons over a handful of interned keywords is
;; cheaper than hashing them. Keeping the order is what lets a value be written
;; back out looking like what was read.

(export
  Bjodat
  bjodat-max-depth
  read-bjodat parse-bjodat
  read-bjodat-all parse-bjodat-all
  bjodat->string
  bjodat-entry-count bjodat-entry-key bjodat-entry-value bjodat-ref
  ;; What a generated one-pass decoder drives. Not an API for people: see the
  ;; section it is defined in.
  Reader
  open-map! next-key! end-entry! key=? field-value skip-value!
  ;; The same, for a record written as a tagged form: `(tag (field value) ...)`.
  open-form! next-field! end-clause! clause-value skip-clause! clause-values!
  read-one-pass parse-one-pass read-one-pass-all parse-one-pass-all)

(import (std prelude))
(import (only (std ports) read-bytes))
(import (std maths))
(import (std simd))

;; Every export is documented there; the module doc is too.
(:doc #:file "bjodat-core.bjodoc")

(import/extern
  (clr-to-long   (: Bjolang.Runtime.BjoNum.ParseLong))
  (clr-to-double (: Bjolang.Runtime.BjoNum.ParseDouble)))

(import/extern
  (clr-nan?               (: System.Double.IsNaN              (-> double bool)))
  (clr-positive-infinity? (: System.Double.IsPositiveInfinity (-> double bool)))
  (clr-negative-infinity? (: System.Double.IsNegativeInfinity (-> double bool))))

;; Hex and binary, which have no reader of their own.
(import/extern
  (clr-to-ulong (: System.UInt64.Parse (-> string ulong)))
  (clr-radix-ulong (: System.Convert.ToUInt64 (-> string int ulong))))

;; ---------------------------------------------------------------------------
;; The value
;; ---------------------------------------------------------------------------

(type
  (: Bjodat (Union
             (: BjoBool bool)
             (: BjoInt long)
             (: BjoFloat double)
             ;; Written with the suffixes of Bjolang's own literals: 12UL,
             ;; 1.50M, 123I.
             (: BjoULong ulong)
             (: BjoDecimal decimal)
             (: BjoBigInt bigint)
             (: BjoStr string)
             (: BjoChar char)
             (: BjoSym Symbol)
             (: BjoKey Keyword)
             (: BjoList (List Bjodat))
             (: BjoVec (Vec Bjodat))
             ;; key, value, key, value — an even length, in written order.
             (: BjoMap (Vec Bjodat)))))

(: int)
(def bjodat-max-depth 200)

;; --- what a map is, for anyone holding one ---------------------------------

(: (-> (Vec Bjodat) int))
(defun (bjodat-entry-count entries) (/ (vec-length entries) 2))

(: (-> (Vec Bjodat) int Bjodat))
(defun (bjodat-entry-key entries i) (vec-ref entries (* i 2)))

(: (-> (Vec Bjodat) int Bjodat))
(defun (bjodat-entry-value entries i) (vec-ref entries (+ (* i 2) 1)))

;; A keyword key, found by reference. Written order, so the first wins.
(: (-> (Vec Bjodat) Keyword (Option Bjodat)))
(defun (bjodat-ref entries k)
  (def n (vec-length entries))
  (let go ((i 0))
    (if (>= i n)
        None
        (match (vec-ref entries i)
          ((BjoKey kk) (if (eq? kk k) (Some (vec-ref entries (+ i 1))) (go (+ i 2))))
          (_ (go (+ i 2)))))))

;; ---------------------------------------------------------------------------
;; The name cache
;; ---------------------------------------------------------------------------
;; A document repeats its names. A manifest of twenty thousand records writes
;; the same five keys twenty thousand times, and interning one costs a string
;; built off the scratch buffer and then a dictionary probe — so almost every
;; one of those allocations is of a string that already exists.
;;
;; This holds the last name seen in each slot and compares the buffer against it
;; without building a string, which is what `stringbuilder=?` is for. A hit
;; costs that comparison and nothing else. A miss costs what interning
;; always cost, plus the failed comparison.
;;
;; Direct-mapped rather than associative: a probe is one comparison, and two
;; names colliding costs a miss rather than a wrong answer. Sized for the number
;; of distinct names a *record* has, not the number of records.
;;
;; This was a class in `BjolangRuntime/BjoText.cs` — a hundred lines of C# that
;; existed for one library. It is here instead because a library in the standard
;; library should not need a file in the runtime; what the runtime owes it is
;; primitives, and `stringbuilder-byte-ref` and `stringbuilder=?` are the two it
;; was missing.

(: int)
(def name-slots 64)

;; Empty slots hold `""`. No name of any other length can equal it, and a name
;; that *is* empty answers `""`, which is the right answer anyway — so there is
;; no empty case to test for. `make-array` would leave nulls, which
;; `string-length` would not survive.
(: (-> (Array string)))
(defun (new-name-cache)
  (def a (make-array name-slots))
  (loop (:for i (range 0 name-slots))
        (:do (array-set! a i "")))
  a)

;; The first, middle and last unit and the length: three reads whatever the
;; length, and enough to part the names of one record. With the first unit and
;; the length alone, `name` and `note` shared a slot, and so did `name` and
;; `pinned`, and each made the other a string every time.
(: (-> StringBuilder int int))
(defun (name-slot sb n)
  (if (= n 0)
      0
      (bitwise-and (+ (* (+ (* (+ (* (stringbuilder-byte-ref sb 0) 31)
                                  (stringbuilder-byte-ref sb (shift-right n 1)))
                               31)
                            (stringbuilder-byte-ref sb (- n 1)))
                         31)
                      n)
                   (- name-slots 1))))

;; The buffer's contents as a string, reusing the one already handed out for
;; those same units.
;;
;; This stops short of answering the interned `Keyword` itself, which would save
;; the probe inside `string->keyword` as well. Nothing here can: interning is
;; the runtime's, and a name read from a port is not known until it is read.
(: (-> (Array string) StringBuilder string))
(defun (cached-name cache sb)
  (def n (stringbuilder-length sb))
  (def i (name-slot sb n))
  (def name (array-ref cache i))
  (cond
    ((stringbuilder=? sb name) name)
    (else
     (def s (stringbuilder->string sb))
     (array-set! cache i s)
     s)))

;; ---------------------------------------------------------------------------
;; The reader
;; ---------------------------------------------------------------------------
;; The reader works on bytes: a chunk of the document in `bytes`, the one it is
;; looking at at `next`, and `limit` where the chunk ends. Looking is an array
;; load and stepping is an increment. A chunk comes from the port with
;; `read-bytes` (see `ensure!`); the `parse-` readers hand over the string's
;; own bytes as the one chunk there is, and no port.
;;
;; All the format's punctuation is ASCII. A character of several bytes - in a
;; string, a symbol, a keyword or a character literal - is checked to be UTF-8
;; and copied whole; one that is not UTF-8 is refused. A column counts
;; characters: only a byte that starts one moves it.
;;
;; The byte is handled as a number, so that the end of the document is a
;; value, `spent`, rather than a second case.

(: int)
(def spent -1)

;; How much a read from the port asks for: the first one what `read-some`
;; gives a file, each one after it twice the one before, up to `read-most`.
;; Each read costs a sync on the port, a new array and the reader's slow path
;; at the chunk's end, so a big document is read in big chunks. `read-bytes`
;; makes the port's buffer as big as what it asks for, so a small one is
;; still read in a small one.
(: int)
(def read-first 16384)

(: int)
(def read-most 262144)

(type
  (: Reader (Record (: port (Option InputPort) #:mutable)
                    ;; How many bytes the next read from the port asks for.
                    (: ask int #:mutable)
                    (: bytes (Array byte) #:mutable)
                    (: next int #:mutable)
                    (: limit int #:mutable)
                    (: row int #:mutable)
                    (: col int #:mutable)
                    ;; One buffer for every string, symbol and number.
                    (: scratch StringBuilder)
                    ;; One array for every list, vec and map in the document.
                    (: stack (Array Bjodat) #:mutable)
                    (: top int #:mutable)
                    ;; One cache for every name in the document.
                    (: names (Array string))
                    ;; The keywords read, by the slot of their name: the name
                    ;; in `keys`, and the value made for it in `key-values`.
                    ;; See `cached-keyword`.
                    (: keys (Array string))
                    (: key-values (Array Bjodat)))))

;; A reader over `bytes`, and then whatever `port` has after them.
(: (-> (Option InputPort) (Array byte) Reader))
(defun (reader port bytes)
  (Reader (port port)
          (ask read-first)
          (bytes bytes)
          (next 0)
          (limit (array-length bytes))
          (row 1)
          (col 1)
          (scratch (stringbuilder-empty))
          (stack (make-array 16))
          (top 0)
          (names (new-name-cache))
          (keys (new-name-cache))
          (key-values (make-array name-slots))))

(: (-> Reader Bjodat void))
(defun (push! rd v)
  (def a (record-ref rd stack))
  (def n (record-ref rd top))
  (cond
    ((< n (array-length a)) (array-set! a n v))
    (else
     (def bigger (array-resize a (* n 2)))
     (array-set! bigger n v)
     (record-set! rd (stack bigger))))
  (record-set! rd (top (+ n 1))))

(: (-> Reader int (Vec Bjodat)))
(defun (harvest! rd mark)
  (def v (subarray->vec (record-ref rd stack) mark (- (record-ref rd top) mark)))
  (record-set! rd (top mark))
  v)

(: (-> Reader StringBuilder))
(defun (buffer rd)
  (def sb (record-ref rd scratch))
  (stringbuilder-clear! sb)
  sb)

;; `count` bytes of `old` from `from`, then `more`: what is left of one chunk
;; and the next. Almost always nothing is left, and `more` is the answer.
(: (-> (Array byte) int int (Array byte) (Array byte)))
(defun (join-bytes old from count more)
  (if (= count 0)
      more
      (let ((joined (make-array (+ count (array-length more)))))
        (array-copy! old from joined 0 count)
        (array-copy! more 0 joined count (array-length more))
        joined)))

;; Whether there are at least `n` bytes from `next` on, reading from the port
;; until there are or it ends. Bytes not yet looked at are kept, so a
;; character cut in two by the end of a chunk is whole afterwards.
(: (-> Reader int bool))
(defun (ensure! rd n)
  (let go ()
    (def have (- (record-ref rd limit) (record-ref rd next)))
    (cond
      ((>= have n) #t)
      (else
       (match (record-ref rd port)
         (None #f)
         ((Some p)
          (def ask (record-ref rd ask))
          (match (read-bytes p (if (> n ask) n ask))
            (None
             (record-set! rd (port None))
             #f)
            ((Some more)
             (record-set! rd
                          (bytes (join-bytes (record-ref rd bytes) (record-ref rd next) have more))
                          (next 0)
                          (limit (+ have (array-length more)))
                          (ask (if (< ask read-most) (* ask 2) read-most)))
             (go)))))))))

;; The byte the reader is looking at, or `spent` at the end of the document.
(: (-> Reader int))
(defun (held-code rd)
  (def i (record-ref rd next))
  (cond
    ((< i (record-ref rd limit)) (cast int (array-ref (record-ref rd bytes) i)))
    ((ensure! rd 1) (cast int (array-ref (record-ref rd bytes) (record-ref rd next))))
    (else spent)))

(: (-> Reader bool))
(defun (at-end? rd) (= (held-code rd) spent))

;; Anything that is not ASCII answers #\nul, because every character this is
;; asked about is one of the format's own punctuation.
(: (-> Reader char))
(defun (peek rd)
  (def c (held-code rd))
  (if (and (>= c 0) (< c 128)) (int->char c) #\nul))

;; Moves past the byte being looked at, which is a character of its own: every
;; caller has just looked at it and found it ASCII. A character of several
;; bytes is taken by `take-character!`.
(: (-> Reader void))
(defun (step! rd)
  (record-set! rd
               (next (+ (record-ref rd next) 1))
               (col (+ (record-ref rd col) 1))))

(: (-> Reader void))
(defun (newline! rd)
  (record-set! rd
               (next (+ (record-ref rd next) 1))
               (row (+ (record-ref rd row) 1))
               (col 1)))

;; --- characters of several bytes --------------------------------------------

(: string)
(def not-utf-8 "a byte that is not UTF-8")

;; How many bytes a character starting with `lead` has, by RFC 3629; 0 for a
;; byte no character starts with: a continuation byte, C0 and C1 (which could
;; only start an overlong form), and F5 and up.
(: (-> int int))
(defun (utf8-length lead)
  (cond
    ((< lead 128) 1)
    ((< lead 194) 0)
    ((< lead 224) 2)
    ((< lead 240) 3)
    ((< lead 245) 4)
    (else 0)))

(: (-> int bool))
(defun (continuation? b) (= (bitwise-and b 192) 128))

;; How many bytes the character the reader is looking at has, when they are
;; UTF-8; 0 when they are not, or the document ends inside them. The second
;; byte is narrower after E0 and F0, which would otherwise start an overlong
;; form, after ED, which would start a surrogate, and after F4, which would
;; go past U+10FFFF.
(: (-> Reader int))
(defun (sequence-length rd)
  (def lead (held-code rd))
  (def n (utf8-length lead))
  (cond
    ((<= n 1) n)
    ((not (ensure! rd n)) 0)
    (else (sequence-length-at (record-ref rd bytes) (record-ref rd next) (record-ref rd limit)))))

;; The same for the character at `i` of `b`, whose bytes end at `limit`: 0
;; also when the character runs past `limit`, where the reader may have more
;; of it to fetch.
(: (-> (Array byte) int int int))
(defun (sequence-length-at b i limit)
  (def lead (cast int (array-ref b i)))
  (def n (utf8-length lead))
  (cond
    ((<= n 1) n)
    ((> (+ i n) limit) 0)
    (else
     (def second (cast int (array-ref b (+ i 1))))
     (def low (cond ((= lead 224) 160) ((= lead 240) 144) (else 128)))
     (def high (cond ((= lead 237) 159) ((= lead 244) 143) (else 191)))
     (if (and (>= second low)
              (<= second high)
              (let check ((k 2))
                (cond
                  ((>= k n) #t)
                  ((continuation? (cast int (array-ref b (+ i k)))) (check (+ k 1)))
                  (else #f))))
         n
         0))))

;; Copies the `n` bytes of the character being looked at into `sb`, and moves
;; past them: one column.
(: (-> Reader StringBuilder int void))
(defun (take-character! rd sb n)
  (def b (record-ref rd bytes))
  (def i (record-ref rd next))
  (loop (:for k (range 0 n))
        (:do (stringbuilder-add-byte! sb (cast int (array-ref b (+ i k))))))
  (record-set! rd
               (next (+ i n))
               (col (+ (record-ref rd col) 1))))

;; The character being looked at, of `n` bytes known to be UTF-8.
(: (-> Reader int char))
(defun (held-character rd n)
  (def i (record-ref rd next))
  (vec-ref (string->vec (bytes->string (record-ref rd bytes) #:start i #:end (+ i n))) 0))

;; The character being looked at, `c`, copied into `sb` and stepped past: an
;; ASCII byte as it is, a character of several bytes once it is known to be
;; UTF-8. False, having taken nothing, when it is not.
(: (-> Reader StringBuilder int bool))
(defun (add-held! rd sb c)
  (cond
    ((< c 128)
     (stringbuilder-add-unit! sb c)
     (step! rd)
     #t)
    (else
     (def n (sequence-length rd))
     (cond
       ((= n 0) #f)
       (else
        (take-character! rd sb n)
        #t)))))

;; --- character classes ------------------------------------------------------

;; A `case` rather than a chain of `=`: it is a C# switch, which Roslyn turns
;; into a jump table, and this is asked about every character of the document.
(: (-> int bool))
(defun (space-code? c)
  (case c
    ((32 10 9 13) #t)
    (else #f)))

;; What ends a name: whitespace, the end of the port (`spent`, -1), or one of
;; the delimiters the lexer refuses inside a symbol — ( ) [ ] { } , : " ; '
;;
;; One case rather than three calls and fifteen comparisons, because
;; `take-name!` asks this per character of every symbol, keyword and number.
;;
;; A `.` and a `/` are deliberately absent: they are symbol characters, which
;; is what makes `net8.0` and `bjolang/http` one symbol each.
(: (-> int bool))
(defun (symbol-code? c)
  (case c
    ((-1 32 10 9 13 40 41 91 93 123 125 44 58 34 59 39) #f)
    (else #t)))

(: (-> int bool))
(defun (digit-code? c) (and (>= c 48) (<= c 57)))

;; White space from `start`, 32 or 16 bytes at a time, to the first byte that
;; is not white space or the last whole block before `limit`: where it ends,
;; the line and column there, and whether a byte that is not white space was
;; found. The line moves by the newlines; after one, the column is counted
;; from it. 256 bits where the CPU has them, and 128 where it has not, which
;; is every ARM64 CPU.
(: (-> (Array byte) int int int int (Tuple int int int bool)))
(defun (scan-space b start limit row col)
  (if simd256-accelerated?
      (scan-space/256 b start limit row col)
      (scan-space/128 b start limit row col)))

;; The line and column after `n` bytes of white space whose newlines are the
;; set bits of `nls`.
(: (-> int int int int (Tuple int int)))
(defun (after-space nls n row col)
  (if (= nls 0)
      (Tuple row (+ col n))
      ;; The bytes after the last newline are the column, less one.
      (Tuple (+ row (popcount nls))
             (- n (- 31 (leading-zeros nls))))))

(: (-> (Array byte) int int int int (Tuple int int int bool)))
(defun (scan-space/256 b start limit row col)
  (let go ((i start) (row row) (col col))
    (if (> (+ i 32) limit)
        (Tuple i row col #f)
        (let ()
          (def v (simd256-load b i))
          (def nl (simd-eq v (simd256-splat 10uy)))
          (def spaces (simd-mask-bits (simd-or (simd-or nl (simd-eq v (simd256-splat 32uy)))
                                               (simd-or (simd-eq v (simd256-splat 9uy))
                                                        (simd-eq v (simd256-splat 13uy))))))
          (def nls (simd-mask-bits nl))
          (if (= spaces -1)
              (let ()
                (def (Tuple r c) (after-space nls 32 row col))
                (go (+ i 32) r c))
              (let ()
                (def t (trailing-zeros (bitwise-not spaces)))
                (def (Tuple r c) (after-space (bitwise-and nls (- (shift-left 1 t) 1)) t row col))
                (Tuple (+ i t) r c #t)))))))

(: (-> (Array byte) int int int int (Tuple int int int bool)))
(defun (scan-space/128 b start limit row col)
  (let go ((i start) (row row) (col col))
    (if (> (+ i 16) limit)
        (Tuple i row col #f)
        (let ()
          (def v (simd128-load b i))
          (def nl (simd-eq v (simd128-splat 10uy)))
          (def spaces (simd-mask-bits (simd-or (simd-or nl (simd-eq v (simd128-splat 32uy)))
                                               (simd-or (simd-eq v (simd128-splat 9uy))
                                                        (simd-eq v (simd128-splat 13uy))))))
          (def nls (simd-mask-bits nl))
          (if (= spaces 65535)
              (let ()
                (def (Tuple r c) (after-space nls 16 row col))
                (go (+ i 16) r c))
              (let ()
                (def t (trailing-zeros (bitwise-xor spaces 65535)))
                (def (Tuple r c) (after-space (bitwise-and nls (- (shift-left 1 t) 1)) t row col))
                (Tuple (+ i t) r c #t)))))))

;; White space a chunk at a time: the chunk is scanned with `scan-space` and
;; then a byte at a time, and the reader is moved once for it. A chunk that
;; ends in white space is followed by the next.
(: (-> Reader void))
(defun (skip-space-run! rd)
  (def b (record-ref rd bytes))
  (def limit (record-ref rd limit))
  (def (Tuple fast-end fast-row fast-col stopped)
    (scan-space b (record-ref rd next) limit (record-ref rd row) (record-ref rd col)))
  (def (Tuple end row col)
    (let go ((i fast-end) (row fast-row) (col fast-col))
      (if (or stopped (>= i limit))
          (Tuple i row col)
          (let ((c (cast int (array-ref b i))))
            (cond
              ((= c 10) (go (+ i 1) (+ row 1) 1))
              ((or (= c 32) (= c 9) (= c 13)) (go (+ i 1) row (+ col 1)))
              (else (Tuple i row col)))))))
  (record-set! rd (next end) (row row) (col col))
  (when (and (>= end limit) (ensure! rd 1))
    (skip-space-run! rd)))

;; Whitespace, and `;` to the end of the line.
(: (-> Reader void))
(defun (skip-space! rd)
  (let go ()
    (def c (held-code rd))
    (cond
     ((space-code? c)
      (skip-space-run! rd)
      (go))
     ((= c 59)
      (let skip ()
        (def k (held-code rd))
        (when (and (not (= k spent)) (not (= k 10)))
          (step! rd)
          (skip)))
      (go))
     (else unit))))

;; ---------------------------------------------------------------------------
;; Refusals
;; ---------------------------------------------------------------------------

(: (-> Reader string string))
(defun (at rd what)
  (str "line " (int->string (record-ref rd row))
       ", column " (int->string (record-ref rd col))
       ": " what))

(: (-> Reader string))
(defun (found rd)
  (def c (held-code rd))
  (cond
    ((= c spent) "the document ended")
    ((< c 128) (str "'" (list->string (list (int->char c))) "'"))
    (else
     (def n (sequence-length rd))
     (if (= n 0)
         not-utf-8
         (str "'" (list->string (list (held-character rd n))) "'")))))

;; The refusal of a string or a name with bytes in it that are not UTF-8.
(: (-> Reader string))
(defun (not-utf-8-here rd) (at rd "text has to be UTF-8, and this byte is not"))

(: (-> Reader string string))
(defun (too-deep rd what)
  (at rd (str "this " what " nests deeper than " (int->string bjodat-max-depth)
              ", which is as far as reading goes")))

;; ---------------------------------------------------------------------------
;; Strings
;; ---------------------------------------------------------------------------

(: (-> char int))
(defun (hex-digit c)
  (cond
    ((and (char>=? c #\0) (char<=? c #\9)) (- (char->int c) 48))
    ((and (char>=? c #\a) (char<=? c #\f)) (- (char->int c) 87))
    ((and (char>=? c #\A) (char<=? c #\F)) (- (char->int c) 55))
    (else -1)))

;; The escapes of a Bjolang string, read by the rules `readEscape` in the
;; compiler's Lexer.fs reads them by:
;;
;; - `\n`, `\t`, `\r`, `\"` and `\\`.
;; - `\u{H}`: the character of 1 to 6 hex digits, a Unicode scalar value.
;; - A backslash that ends its line, spaces and tabs after it aside, continues
;;   the string on the next line. That line's spaces and tabs up to
;;   `text-col`, the column the string's text starts at, stand for nothing.
;; - Any other escape is refused, at its backslash.
(: (-> Reader StringBuilder int (Result string bool)))
(defun (read-escape! rd sb text-col)
  (def row (record-ref rd row))
  (def col (record-ref rd col))
  (def refuse (fun (what) (Err (str "line " (int->string row) ", column " (int->string col) ": " what))))
  (step! rd)
  (def c (held-code rd))
  (def add (fun (code) (stringbuilder-add-byte! sb code) (step! rd) (Ok #t)))
  (cond
    ((= c spent) (Err (at rd "the document ended inside a string")))
    ((= c 110) (add 10))
    ((= c 116) (add 9))
    ((= c 114) (add 13))
    ((= c 34) (add 34))
    ((= c 92) (add 92))
    ((= c 117) (read-code-point! rd sb refuse))
    ((or (= c 32) (= c 9) (= c 10) (= c 13)) (read-continuation! rd refuse text-col))
    (else
     (refuse (str (if (< c 128) (str "\\" (list->string (list (int->char c)))) (str "a backslash before " (found rd)))
                  " is not an escape. A string's escapes are \\n, \\t, \\r, \\\", \\\\ and \\u{...}. "
                  "For a backslash, write \\\\, or write the text as a raw string")))))

;; `\u{H}`, at its `u`.
(: (-> Reader StringBuilder (-> string (Result string bool)) (Result string bool)))
(defun (read-code-point! rd sb refuse)
  (def shape "\\u is written \\u{...}, with 1 to 6 hex digits between the braces, as in \\u{E9} or \\u{1F600}")
  (step! rd)
  (cond
    ((not (= (held-code rd) 123)) (refuse shape))
    (else
     (step! rd)
     (let go ((digits "") (n 0))
       (def d (hex-digit (peek rd)))
       (cond
         ((and (= (held-code rd) 125) (> (string-length digits) 0))
          (step! rd)
          (cond
            ((or (> n 1114111) (and (>= n 55296) (<= n 57343)))
             (refuse (str "\\u{" digits "} is not a Unicode scalar value. A surrogate, D800 to DFFF, "
                          "is half of a UTF-16 pair, and nothing is above 10FFFF")))
            (else (stringbuilder-add! sb (int->char n)) (Ok #t))))
         ((and (>= d 0) (< (string-length digits) 6))
          (def digit (peek rd))
          (step! rd)
          (go (str digits (list->string (list digit))) (+ (* n 16) d)))
         (else (refuse shape)))))))

;; A backslash before a space, a tab or a line break, at that character: the
;; end of the line, and the margin of the next, stand for nothing.
(: (-> Reader (-> string (Result string bool)) int (Result string bool)))
(defun (read-continuation! rd refuse text-col)
  (let blanks ()
    (def c (held-code rd))
    (when (or (= c 32) (= c 9))
      (step! rd)
      (blanks)))
  (def broke
    (cond
      ((= (held-code rd) 10) (newline! rd) #t)
      ((= (held-code rd) 13)
       (step! rd)
       (cond
         ((= (held-code rd) 10) (newline! rd) #t)
         (else #f)))
      (else #f)))
  (cond
    ((not broke)
     (refuse (str "a backslash before a space is an escape only at the end of a line, where it "
                  "continues the string on the next. For a backslash, write \\\\")))
    (else
     (let margin ()
       (def c (held-code rd))
       (when (and (< (record-ref rd col) text-col) (or (= c 32) (= c 9)))
         (step! rd)
         (margin)))
     (Ok #t))))

;; A string, raw or escaped. `""` is the empty string, and a third quote after
;; it opens a raw one.
(: (-> Reader (Result string string)))
(defun (read-text rd)
  (step! rd)
  (cond
    ((not (= (held-code rd) 34)) (read-escaped-text rd))
    (else
     (step! rd)
     (if (= (held-code rd) 34)
         (read-raw-text rd)
         (Ok "")))))

;; ---------------------------------------------------------------------------
;; Raw strings
;; ---------------------------------------------------------------------------
;; `"""..."""`, read by the rules `readRawString` in the compiler's Lexer.fs
;; reads it by, which Docs/Syntax.org describes. The two have to agree, and
;; TestFiles/263_raw_strings.bjo reads one file with both to show that they do.

;; Steps over a run of quotes, and answers how long it was.
(: (-> Reader int))
(defun (skip-quotes! rd)
  (let go ((k 0))
    (cond
      ((= (held-code rd) 34) (step! rd) (go (+ k 1)))
      (else k))))

(: (-> StringBuilder int void))
(defun (add-quotes! sb k)
  (loop (:for _ (range 0 k))
        (:do (stringbuilder-add-byte! sb 34))))

(: (-> Reader int int string))
(defun (too-many-quotes rd n k)
  (at rd (str "a raw string opened with " (int->string n) " quotes has " (int->string k)
              " in a row. Open and close it with more quotes than it holds in a row")))

;; The held quote is the third of the opening ones.
(: (-> Reader (Result string string)))
(defun (read-raw-text rd)
  (def n (+ 2 (skip-quotes! rd)))
  (def sb (buffer rd))
  (let spaces ()
    (def c (held-code rd))
    (when (or (= c 32) (= c 9))
      (stringbuilder-add-unit! sb c)
      (step! rd)
      (spaces)))
  (cond
    ((= (held-code rd) 10) (newline! rd) (read-raw-lines rd n))
    ((= (held-code rd) 13)
     (step! rd)
     (cond
       ((= (held-code rd) 10) (newline! rd) (read-raw-lines rd n))
       (else (stringbuilder-add-byte! sb 13) (read-raw-line rd n sb))))
    (else (read-raw-line rd n sb))))

;; A raw string on the line of its opening quotes, which ends on that line.
(: (-> Reader int StringBuilder (Result string string)))
(defun (read-raw-line rd n sb)
  (let go ()
    (def c (held-code rd))
    (cond
      ((= c spent) (Err (at rd "the document ended inside a string")))
      ((= c 10)
       (Err (at rd (str "a raw string that begins on the line of its opening quotes ends on that line. "
                        "One over several lines begins on the line after them"))))
      ((= c 34)
       (def k (skip-quotes! rd))
       (cond
         ((= k n) (Ok (stringbuilder->string sb)))
         ((> k n) (Err (too-many-quotes rd n k)))
         (else (add-quotes! sb k) (go))))
      ((add-held! rd sb c) (go))
      (else (Err (not-utf-8-here rd))))))

;; A raw string over several lines. Each line is kept as it was written, with
;; the row it is on, until the closing quotes say what the margin is.
(: (-> Reader int (Result string string)))
(defun (read-raw-lines rd n)
  (let next-line ((lines Nil))
    (def row (record-ref rd row))
    (def sb (stringbuilder-empty))
    (let spaces ()
      (def c (held-code rd))
      (when (or (= c 32) (= c 9))
        (stringbuilder-add-unit! sb c)
        (step! rd)
        (spaces)))
    (def lead (skip-quotes! rd))
    (cond
      ((= lead n) (strip-margin (stringbuilder->string sb) (list-reverse lines)))
      ((> lead n) (Err (too-many-quotes rd n lead)))
      (else
       (add-quotes! sb lead)
       (let go ()
         (def c (held-code rd))
         (cond
           ((= c spent)
            (Err (at rd (str "the document ended inside a raw string, which closes with "
                             (int->string n) " quotes on a line of their own"))))
           ((= c 10)
            (newline! rd)
            (next-line (Cons (Tuple row (stringbuilder->string sb)) lines)))
           ((= c 13)
            (step! rd)
            (cond
              ((= (held-code rd) 10)
               (newline! rd)
               (next-line (Cons (Tuple row (stringbuilder->string sb)) lines)))
              (else (stringbuilder-add-byte! sb 13) (go))))
           ((= c 34)
            (def k (skip-quotes! rd))
            (cond
              ((> k n) (Err (too-many-quotes rd n k)))
              ((= k n)
               (Err (at rd (str "a raw string closes only on a line of its own. For " (int->string n)
                                " quotes in the text, open and close it with more"))))
              (else (add-quotes! sb k) (go))))
           ((add-held! rd sb c) (go))
           (else (Err (not-utf-8-here rd)))))))))

(: (-> string bool))
(defun (blank? s)
  (none? (string-find #(not (or (char=? &1 #\space) (char=? &1 #\tab))) s)))

;; What follows `prefix` in `s`, which starts with it.
(: (-> string string string))
(defun (after-prefix s prefix)
  (let go ((c (string-cursor-start s)) (n (string-count prefix)))
    (if (= n 0)
        (substring s c (string-cursor-end s))
        (go (string-cursor-next s c) (- n 1)))))

;; The margin is the whitespace before the closing quotes. It comes off every
;; line, and a line with text that does not begin with it is refused, since
;; cutting it anyway would cut text.
(: (-> string (List (Tuple int string)) (Result string string)))
(defun (strip-margin margin lines)
  (def out (stringbuilder-empty))
  (let go ((left lines) (first #t))
    (match left
      (Nil (Ok (stringbuilder->string out)))
      ((Cons (Tuple row s) more)
       (unless first (stringbuilder-add-byte! out 10))
       (cond
         ((string-starts-with? s margin)
          (stringbuilder-add-string! out (after-prefix s margin))
          (go more #f))
         ((blank? s) (go more #f))
         (else
          (Err (str "line " (int->string row) ", column 1: this line of a raw string does not "
                    "begin with its margin. The whitespace before the closing quotes is removed "
                    "from every line, so every line with text in it begins with that whitespace"))))))))

;; Where a run of a string from `start` stops, 32 or 16 bytes at a time: at
;; the first `"`, `\` or newline, or at the last whole block before `limit`.
;; Also how many characters are before it, which is the bytes that are not a
;; continuation byte (10xxxxxx), whether one of them is 128 or more, and
;; whether a stop was found. The count is the characters' count only when the
;; bytes are UTF-8, which `take-run!` checks when one is 128 or more.
(: (-> (Array byte) int int (Tuple int int bool bool)))
(defun (scan-run b start limit)
  (if simd256-accelerated?
      (scan-run/256 b start limit)
      (scan-run/128 b start limit)))

(: (-> (Array byte) int int (Tuple int int bool bool)))
(defun (scan-run/256 b start limit)
  (let go ((i start) (chars 0) (high #f))
    (if (> (+ i 32) limit)
        (Tuple i chars high #f)
        (let ()
          (def v (simd256-load b i))
          (def stops (simd-mask-bits (simd-or (simd-or (simd-eq v (simd256-splat 34uy))
                                                       (simd-eq v (simd256-splat 92uy)))
                                              (simd-eq v (simd256-splat 10uy)))))
          (def conts (simd-mask-bits (simd-eq (simd-and v (simd256-splat 192uy)) (simd256-splat 128uy))))
          (def highs (simd-mask-bits (simd-ge v (simd256-splat 128uy))))
          (if (= stops 0)
              (go (+ i 32) (+ chars (- 32 (popcount conts))) (or high (not (= highs 0))))
              (let ()
                ;; The lanes before the first stop.
                (def t (trailing-zeros stops))
                (def before (- (shift-left 1 t) 1))
                (Tuple (+ i t)
                       (+ chars (- t (popcount (bitwise-and conts before))))
                       (or high (not (= (bitwise-and highs before) 0)))
                       #t)))))))

(: (-> (Array byte) int int (Tuple int int bool bool)))
(defun (scan-run/128 b start limit)
  (let go ((i start) (chars 0) (high #f))
    (if (> (+ i 16) limit)
        (Tuple i chars high #f)
        (let ()
          (def v (simd128-load b i))
          (def stops (simd-mask-bits (simd-or (simd-or (simd-eq v (simd128-splat 34uy))
                                                       (simd-eq v (simd128-splat 92uy)))
                                              (simd-eq v (simd128-splat 10uy)))))
          (def conts (simd-mask-bits (simd-eq (simd-and v (simd128-splat 192uy)) (simd128-splat 128uy))))
          (def highs (simd-mask-bits (simd-ge v (simd128-splat 128uy))))
          (if (= stops 0)
              (go (+ i 16) (+ chars (- 16 (popcount conts))) (or high (not (= highs 0))))
              (let ()
                (def t (trailing-zeros stops))
                (def before (- (shift-left 1 t) 1))
                (Tuple (+ i t)
                       (+ chars (- t (popcount (bitwise-and conts before))))
                       (or high (not (= (bitwise-and highs before) 0)))
                       #t)))))))

;; Copies the run of a string that needs no attention, from where the reader
;; is to the first `"`, `\`, newline, byte that is not UTF-8 or end of the
;; chunk, into `sb` in one piece, and moves past it. Whatever stopped the run
;; is left for `read-escaped-text` to look at.
;;
;; The run is found a block at a time by `scan-run`, and the bytes left
;; before `limit` one at a time. A run with a byte of 128 or more is checked
;; as UTF-8 in one call. A run that is not UTF-8, or that the end of the chunk
;; cuts inside a character, is read again by `take-run/checked!`, which stops
;; before the byte at fault, so that the error and its column are where they
;; were.
(: (-> Reader StringBuilder void))
(defun (take-run! rd sb)
  (def b (record-ref rd bytes))
  (def start (record-ref rd next))
  (def limit (record-ref rd limit))
  (def (Tuple fast-end fast-chars fast-high stopped) (scan-run b start limit))
  (def (Tuple end chars high)
    (if stopped
        (Tuple fast-end fast-chars fast-high)
        (let go ((i fast-end) (chars fast-chars) (high fast-high))
          (if (>= i limit)
              (Tuple i chars high)
              (let ((c (cast int (array-ref b i))))
                (cond
                  ((or (= c 34) (= c 92) (= c 10)) (Tuple i chars high))
                  ((< c 128) (go (+ i 1) (+ chars 1) high))
                  ((continuation? c) (go (+ i 1) chars #t))
                  (else (go (+ i 1) (+ chars 1) #t))))))))
  (cond
    ((and high (not (bytes-utf-8? b start end))) (take-run/checked! rd sb))
    ((> end start)
     (stringbuilder-add-bytes! sb b start end)
     (record-set! rd
                  (next end)
                  (col (+ (record-ref rd col) chars))))
    (else unit)))

;; `take-run!` a byte at a time, each character checked as UTF-8: the run
;; stops before a byte that is not, and before a character the end of the
;; chunk cuts.
(: (-> Reader StringBuilder void))
(defun (take-run/checked! rd sb)
  (def b (record-ref rd bytes))
  (def start (record-ref rd next))
  (def limit (record-ref rd limit))
  (def (Tuple end chars)
    (let go ((i start) (chars 0))
      (if (>= i limit)
          (Tuple i chars)
          (let ((c (cast int (array-ref b i))))
            (cond
              ((or (= c 34) (= c 92) (= c 10)) (Tuple i chars))
              ((< c 128) (go (+ i 1) (+ chars 1)))
              (else
               (let ((n (sequence-length-at b i limit)))
                 (if (= n 0) (Tuple i chars) (go (+ i n) (+ chars 1))))))))))
  (when (> end start)
    (stringbuilder-add-bytes! sb b start end)
    (record-set! rd
                 (next end)
                 (col (+ (record-ref rd col) chars)))))

;; An escaped string, after its opening quote. The bytes between the ones
;; that need a look are copied in runs by `take-run!`.
(: (-> Reader (Result string string)))
(defun (read-escaped-text rd)
  (def sb (buffer rd))
  ;; The column after the opening quote, which a continued line lines up with.
  (def text-col (record-ref rd col))
  (let go ()
    (take-run! rd sb)
    (cond
     ((at-end? rd) (Err (at rd "the document ended inside a string")))
     (else
      (def c (held-code rd))
      (cond
       ((= c 34) (step! rd) (Ok (stringbuilder->string sb)))
       ((= c 92) (match (read-escape! rd sb text-col)
                   ((Err e) (Err e))
                   ((Ok _) (go))))
       ((= c 10)
        (stringbuilder-add-unit! sb c)
        (newline! rd)
        (go))
       ((add-held! rd sb c) (go))
       (else (Err (not-utf-8-here rd))))))))

;; ---------------------------------------------------------------------------
;; Numbers
;; ---------------------------------------------------------------------------

(: (-> Reader StringBuilder int))
(defun (take-digits! rd sb)
  (let go ((n 0))
    (if (digit-code? (held-code rd))
        (begin
          (stringbuilder-add-unit! sb (held-code rd))
          (step! rd)
          (go (+ n 1)))
        n)))

;; The largest magnitude a long holds, and the one a negative long holds.
(: ulong)
(def long-max-magnitude 9223372036854775807UL)

(: ulong)
(def long-min-magnitude 9223372036854775808UL)

;; A digit of `radix`, or -1.
(: (-> int char int))
(defun (radix-digit radix c)
  (def d (hex-digit c))
  (if (< d radix) d -1))

;; `0x` and `0b`. The `0` has been stepped over and the marker is held.
;;
;; The digits are a magnitude, read as a ulong. `Convert.ToInt64` reads 16 hex
;; digits as a bit pattern, so `0xFFFFFFFFFFFFFFFF` was -1. Here a magnitude
;; above `long-max-magnitude` does not fit in a long, except as the magnitude
;; of `-0x8000000000000000`.
(: (-> Reader bool int (Result string Bjodat)))
(defun (read-radix rd neg radix)
  (step! rd)
  (def sb (buffer rd))
  (let go ((n 0))
    (if (and (not (at-end? rd)) (>= (radix-digit radix (peek rd)) 0))
        (begin (stringbuilder-add-unit! sb (held-code rd)) (step! rd) (go (+ n 1)))
        (cond
          ((= n 0)
           (Err (at rd (str "this number has no digits after its radix, and there is "
                            (found rd)))))
          ((and (= radix 2) (digit-code? (held-code rd)))
           (Err (at rd (str "a binary number has the digits 0 and 1, and this is "
                            (found rd)))))
          (else
           (def unsigned? (ulong-suffix? rd))
           (match (try (clr-radix-ulong (stringbuilder->string sb) radix)
                       #:catch (System.OverflowException))
             ((Err _)
              (Err (at rd (if unsigned?
                              "this number does not fit in a ulong"
                              "this number does not fit in a long"))))
             ((Ok v)
              (cond
                ((and unsigned? neg) (Err (at rd "a ulong is not negative")))
                (unsigned? (Ok (BjoULong v)))
                ((and neg (= v long-min-magnitude)) (Ok (BjoInt (- -9223372036854775807L 1L))))
                ((> v long-max-magnitude) (Err (at rd "this number does not fit in a long")))
                (neg (Ok (BjoInt (- 0L (cast long v)))))
                (else (Ok (BjoInt (cast long v))))))))))))

;; Steps over `UL` when it is held. A `U` without the `L` is refused by what
;; reads next, as any other character after a number is.
(: (-> Reader bool))
(defun (ulong-suffix? rd)
  (cond
    ((and (= (held-code rd) 85) (ensure! rd 2)
          (= (cast int (array-ref (record-ref rd bytes) (+ (record-ref rd next) 1))) 76))
     (step! rd)
     (step! rd)
     #t)
    (else #f)))

(: (-> Reader StringBuilder (Result string bool)))
(defun (scan-fraction! rd sb)
  (if (not (= (held-code rd) 46))
      (Ok #f)
      (begin
        (stringbuilder-add-byte! sb 46)
        (step! rd)
        (if (= (take-digits! rd sb) 0)
            (Err (at rd (str "a number needs a digit after its decimal point, and this is "
                             (found rd))))
            (Ok #t)))))

(: (-> Reader StringBuilder (Result string bool)))
(defun (scan-exponent! rd sb)
  (def c (held-code rd))
  (cond
    ((not (or (= c 101) (= c 69))) (Ok #f))
    (else
     (stringbuilder-add-byte! sb 101)
     (step! rd)
     ;; `s` is bound rather than read three times, and it is not only the
     ;; calls: with a name, `(or (= s 43) (= s 45))` is emitted as a
     ;; ternary, and with the calls inlined the operands stop being simple
     ;; and it becomes an if/else over a temporary, to keep the short
     ;; circuit in the right order.
     (def s (held-code rd))
     (when (or (= s 43) (= s 45))
       (stringbuilder-add-unit! sb s)
       (step! rd))
     (if (= (take-digits! rd sb) 0)
         (Err (at rd (str "a number needs a digit after its exponent, and this is "
                          (found rd))))
         (Ok #t)))))

;; A digit is held. The sign, if there was one, has been stepped over already —
;; which is what tells `-` the symbol apart from `-1` the number, since deciding
;; needs a character of lookahead the reader does not have until it has moved.
;;
;; A number ends at whitespace, at a delimiter or at the end, as source
;; tokenises it. `1abc`, `1.2.3` and `0x1g` are refused. Before, each was read
;; as two values.
(: (-> Reader bool (Result string Bjodat)))
(defun (read-magnitude rd neg)
  (def (Ok v) (read-number rd neg) :propagate)
  (if (symbol-code? (held-code rd))
      (Err (at rd (str "a number ends at a space or a delimiter, and this is " (found rd))))
      (Ok v)))

(: (-> Reader bool (Result string Bjodat)))
(defun (read-number rd neg)
  (cond
    ((= (held-code rd) 48)
     (step! rd)
     (def c (held-code rd))
     (cond
       ((or (= c 120) (= c 88)) (read-radix rd neg 16))
       ((or (= c 98) (= c 66)) (read-radix rd neg 2))
       (else
        (def sb (signed-buffer rd neg))
        (stringbuilder-add-byte! sb 48)
        (finish-number rd neg sb))))
    (else (finish-number rd neg (signed-buffer rd neg)))))

;; The buffer, with the `-` in it for a negative number. The sign is parsed
;; with the digits: `-9223372036854775808` fits in a long and its magnitude
;; does not, and `-0.0` is a zero with a sign, which subtracting from 0.0 loses.
(: (-> Reader bool StringBuilder))
(defun (signed-buffer rd neg)
  (def sb (buffer rd))
  (when neg (stringbuilder-add-byte! sb 45))
  sb)

(: (-> Reader bool StringBuilder (Result string Bjodat)))
(defun (finish-number rd neg sb)
  (ignore (take-digits! rd sb))
  (def (Ok frac?) (scan-fraction! rd sb) :propagate)
  (def (Ok exp?) (scan-exponent! rd sb) :propagate)
  (cond
    ;; 1.50M: a decimal keeps the scale it was written with.
    ((= (held-code rd) 77)
     (step! rd)
     (if exp?
         (Err (at rd "a decimal is written without an exponent"))
         (match (try (string->decimal (stringbuilder->string sb))
                     #:catch (System.OverflowException))
           ((Ok d) (Ok (BjoDecimal d)))
           ((Err _) (Err (at rd "this number does not fit in a decimal"))))))
    ((= (held-code rd) 73)
     (step! rd)
     (if (or frac? exp?)
         (Err (at rd "a bigint is a whole number: it has no decimal point and no exponent"))
         (Ok (BjoBigInt (string->bigint (stringbuilder->string sb))))))
    ((ulong-suffix? rd)
     (cond
       ((or frac? exp?) (Err (at rd "a ulong is a whole number: it has no decimal point and no exponent")))
       (neg (Err (at rd "a ulong is not negative")))
       (else
        (match (try (clr-to-ulong (stringbuilder->string sb)) #:catch (System.OverflowException))
          ((Ok n) (Ok (BjoULong n)))
          ((Err _) (Err (at rd "this number does not fit in a ulong")))))))
    ((or frac? exp?) (Ok (BjoFloat (clr-to-double sb))))
    ;; Wider than a long is still a number.
    (else
     (match (try (clr-to-long sb) #:catch (System.OverflowException))
       ((Ok n) (Ok (BjoInt n)))
       ((Err _) (Ok (BjoFloat (clr-to-double sb))))))))

;; ---------------------------------------------------------------------------
;; Symbols, keywords, and what a `#` begins
;; ---------------------------------------------------------------------------

(: (-> Reader StringBuilder int))
;; A byte that is not UTF-8 ends the name, and is refused by what reads next.
(defun (take-name! rd sb)
  (let go ((n 0))
    (def c (held-code rd))
    (if (and (symbol-code? c) (add-held! rd sb c))
        (go (+ n 1))
        n)))

;; NaN and the infinities, spelled as Scheme spells them. They read as names
;; do, so a name is checked for them before it becomes a symbol.
(: (-> string (Option double)))
(defun (special-double name)
  (cond
    ((= name "+inf.0") (Some (/ 1.0 0.0)))
    ((= name "-inf.0") (Some (/ -1.0 0.0)))
    ((or (= name "+nan.0") (= name "-nan.0")) (Some (/ 0.0 0.0)))
    (else None)))

;; A name as a symbol, unless it is one of the doubles above.
(: (-> Reader StringBuilder Bjodat))
(defun (name-value rd sb)
  (def name (cached-name (record-ref rd names) sb))
  (match (special-double name)
    ((Some d) (BjoFloat d))
    (None (BjoSym (string->symbol name)))))

(: (-> Reader (Result string Bjodat)))
(defun (read-symbol rd)
  (def sb (buffer rd))
  (if (= (take-name! rd sb) 0)
      (Err (at rd (str (found rd) " does not begin a value")))
      (Ok (name-value rd sb))))

;; `:name`. The colon is held.
(: (-> Reader (Result string Bjodat)))
(defun (read-keyword rd)
  (step! rd)
  (def sb (buffer rd))
  (if (= (take-name! rd sb) 0)
      (Err (at rd (str "a keyword needs a name after its colon, and this is "
                       (found rd))))
      (Ok (cached-keyword rd sb))))

;; The keyword in the buffer, as a value. A map repeats its keys, and a key read
;; before is the value already made for it: no string, no lookup in the table
;; of interned keywords, and no allocation. `string->keyword` makes a .NET
;; string of the name and looks that up, each time it is called. Two keywords
;; that share a slot take turns in it and are made each time.
(: (-> Reader StringBuilder Bjodat))
(defun (cached-keyword rd sb)
  (def keys (record-ref rd keys))
  (def i (name-slot sb (stringbuilder-length sb)))
  (cond
    ((stringbuilder=? sb (array-ref keys i)) (array-ref (record-ref rd key-values) i))
    (else
     (def name (stringbuilder->string sb))
     (def v (BjoKey (string->keyword name)))
     (array-set! keys i name)
     (array-set! (record-ref rd key-values) i v)
     v)))

;; The characters source spells with a word.
(: (-> string int))
(defun (named-char name)
  (cond
    ((= name "newline") 10)
    ((= name "space") 32)
    ((= name "tab") 9)
    ((= name "return") 13)
    ((= name "nul") 0)
    (else -1)))

;; `#\c`, `#\newline`. The backslash is held.
(: (-> Reader (Result string Bjodat)))
(defun (read-char-literal rd)
  (step! rd)
  (cond
    ((at-end? rd) (Err (at rd "the document ended inside a character")))
    ((= (sequence-length rd) 0) (Err (not-utf-8-here rd)))
    (else
     (def lead (held-code rd))
     (def first (if (< lead 128) (int->char lead) (held-character rd (sequence-length rd))))
     (def sb (buffer rd))
     ;; `#\` and a line feed is the line feed, and the next line begins.
     (if (= lead 10)
         (begin (stringbuilder-add-unit! sb lead) (newline! rd))
         (ignore (add-held! rd sb lead)))
     ;; A name is a run of symbol characters, as in the compiler's lexer. After
     ;; `#\(`, `#\ ` or `#\` and a line feed the literal is that one
     ;; character, and what follows is the next value.
     ;; `n` is bound rather than inlined into the test: a name compares
     ;; straight into the `if`, where the call needs a temporary for its
     ;; result and then a second one for the comparison.
     (def n (if (symbol-code? lead) (take-name! rd sb) 0))
     (cond
       ((= n 0) (Ok (BjoChar first)))
       (else
        (def name (stringbuilder->string sb))
        (def code (named-char name))
        (if (< code 0)
            (Err (at rd (str "#\\" name " is not a character")))
            (Ok (BjoChar (int->char code)))))))))

;; `#t`, `#f`, `#\c`, `#:keyword`. The `#` is held.
(: (-> Reader (Result string Bjodat)))
(defun (read-hash rd)
  (step! rd)
  (cond
    ((at-end? rd) (Err (at rd "the document ended after '#'")))
    (else
     (def c (peek rd))
     (cond
       ((char=? c #\t) (step! rd) (Ok (BjoBool #t)))
       ((char=? c #\f) (step! rd) (Ok (BjoBool #f)))
       ((char=? c #\\) (read-char-literal rd))
       ((char=? c #\:) (read-keyword rd))
       (else (Err (at rd (str "'#' begins #t, #f, #\\c or #:keyword, and this is "
                              (found rd)))))))))

;; ---------------------------------------------------------------------------
;; Lists, vecs and maps
;; ---------------------------------------------------------------------------

(: (-> Reader int int char string (Result string (Vec Bjodat))))
(defun (read-sequence rd depth closer-code closer what)
  (step! rd)
  (def mark (record-ref rd top))
  (let go ()
    (skip-space! rd)
    (cond
      ((at-end? rd)
       (Err (at rd (str "the document ended inside a " what))))
      ((= (held-code rd) closer-code)
       (step! rd)
       (Ok (harvest! rd mark)))
      (else
       (match (read-value rd (+ depth 1))
         ((Err e) (Err e))
         ((Ok v) (push! rd v) (go)))))))

;; A key is any value but a list. Refusing one is what makes `{(` unambiguous:
;; a `(` where a key would go means the map is written in the parenthesised
;; form, and nothing else.
(: (-> Reader int (Result string Bjodat)))
(defun (read-key rd depth)
  (match (read-value rd depth)
    ((Err e) (Err e))
    ((Ok (BjoList _)) (Err (at rd "a list is not a key")))
    ((Ok v) (Ok v))))

;; `{(k v) (k v)}`. The opening brace has been stepped over and a `(` is held.
(: (-> Reader int int (Result string Bjodat)))
(defun (read-paired-map rd depth mark)
  (let go ()
    (skip-space! rd)
    (cond
      ((at-end? rd) (Err (at rd "the document ended inside a map")))
      ((= (held-code rd) 125)
       (step! rd)
       (Ok (BjoMap (harvest! rd mark))))
      ((not (= (held-code rd) 40))
       (Err (at rd (str "this map is written (key value), so its entries all are, "
                        "and here there is " (found rd)))))
      (else
       (step! rd)
       (def (Ok k) (read-key rd (+ depth 1)) :propagate)
       (push! rd k)
       (def (Ok v) (read-value rd (+ depth 1)) :propagate)
       (push! rd v)
       (skip-space! rd)
       (if (= (held-code rd) 41)
           (begin (step! rd) (go))
           (Err (at rd (str "an entry holds a key and a value, and then there is "
                            (found rd)))))))))

;; `{k v k v}`. The opening brace has been stepped over before reaching this fun.
(: (-> Reader int int (Result string Bjodat)))
(defun (read-flat-map rd depth mark)
  (let go ()
    (skip-space! rd)
    (cond
      ((at-end? rd) (Err (at rd "the document ended inside a map")))
      ((= (held-code rd) 125)
       (step! rd)
       (Ok (BjoMap (harvest! rd mark))))
      ((= (held-code rd) 40)
       (Err (at rd "this map is written flat, so a '(' here is an entry of the other form")))
      (else
       (def (Ok k) (read-key rd (+ depth 1)) :propagate)
       (push! rd k)
       (skip-space! rd)
       (if (or (at-end? rd) (= (held-code rd) 125))
           (Err (at rd "this map has a key with no value"))
           (match (read-value rd (+ depth 1))
             ((Err e) (Err e))
             ((Ok v) (push! rd v) (go))))))))

;; Which form a map is written in is decided by the first thing inside it, and
;; then every entry has to agree.
(: (-> Reader int (Result string Bjodat)))
(defun (read-map rd depth)
  (step! rd)
  (def mark (record-ref rd top))
  (skip-space! rd)
  (cond
    ((at-end? rd) (Err (at rd "the document ended inside a map")))
    ((= (held-code rd) 125)
     (step! rd)
     (Ok (BjoMap (harvest! rd mark))))
    ((= (held-code rd) 40) (read-paired-map rd depth mark))
    (else (read-flat-map rd depth mark))))

;; ---------------------------------------------------------------------------
;; A value
;; ---------------------------------------------------------------------------

(: (-> Reader int (Result string Bjodat)))
(defun (read-value rd depth)
  (skip-space! rd)
  (cond
    ((> depth bjodat-max-depth) (Err (too-deep rd "value")))
    ((at-end? rd) (Err (at rd "the document ended where a value was expected")))
    (else
     (def c (peek rd))
     (cond
       ((char=? c #\()
        (match (read-sequence rd depth 41 #\) "list")
          ((Err e) (Err e))
          ((Ok v) (Ok (BjoList (vec->list v))))))
       ((char=? c #\[)
        (match (read-sequence rd depth 93 #\] "vec")
          ((Err e) (Err e))
          ((Ok v) (Ok (BjoVec v)))))
       ((char=? c #\{) (read-map rd depth))
       ((char=? c #\")
        (match (read-text rd)
          ((Err e) (Err e))
          ((Ok s) (Ok (BjoStr s)))))
       ((char=? c #\#) (read-hash rd))
       ((char=? c #\:) (read-keyword rd))
       ((char=? c #\)) (Err (at rd "')' closes nothing here")))
       ((char=? c #\]) (Err (at rd "']' closes nothing here")))
       ((char=? c #\}) (Err (at rd "'}' closes nothing here")))
       ((char=? c #\,) (Err (at rd "a comma is not a separator here")))
       ((char=? c #\') (Err (at rd "a symbol in bjodat is written without a quote")))
       ((digit-code? (held-code rd)) (read-magnitude rd #f))
       ;; `-` is a symbol of its own, and the start of a number when a digit
       ;; follows it. Stepping over it first is what makes the difference
       ;; visible; if no digit came, what was stepped over is the first
       ;; character of a symbol.
       ((char=? c #\-)
        (step! rd)
        (cond
          ((digit-code? (held-code rd)) (read-magnitude rd #t))
          (else
           (def sb (buffer rd))
           (stringbuilder-add-byte! sb 45)
           (ignore (take-name! rd sb))
           (Ok (name-value rd sb)))))
       (else (read-symbol rd))))))

;; ---------------------------------------------------------------------------
;; Reading a document
;; ---------------------------------------------------------------------------

;; A byte order mark, EF BB BF, is skipped and is not a column.
(: (-> Reader void))
(defun (skip-bom! rd)
  (when (and (ensure! rd 3)
             (let ((b (record-ref rd bytes)) (i (record-ref rd next)))
               (and (= (cast int (array-ref b i)) 239)
                    (= (cast int (array-ref b (+ i 1))) 187)
                    (= (cast int (array-ref b (+ i 2))) 191))))
    (record-set! rd (next (+ (record-ref rd next) 3)))))

(: (Array byte))
(def no-bytes (make-array 0))

;; A reader for a port, and one for a string's own bytes.
(: (-> InputPort Reader))
(defun (port-reader port) (reader (Some port) no-bytes))

(: (-> string Reader))
(defun (string-reader text) (reader None (string->bytes text)))

;; Exactly one value, and then the end.
(: (-> InputPort (Result string Bjodat)))
(defun (read-bjodat port) (one-value (port-reader port)))

(: (-> Reader (Result string Bjodat)))
(defun (one-value rd)
  (skip-bom! rd)
  (match (read-value rd 1)
    ((Err e) (Err e))
    ((Ok v)
     (skip-space! rd)
     (if (at-end? rd)
         (Ok v)
         (Err (at rd (str "the value ended here, and then there is " (found rd))))))))

(: (-> string (Result string Bjodat)))
(defun (parse-bjodat text) (one-value (string-reader text)))

;; Every value in the file, in order. A file is a stream, so an empty one is an
;; empty list rather than a refusal.
(: (-> InputPort (Result string (List Bjodat))))
(defun (read-bjodat-all port) (all-values (port-reader port)))

(: (-> Reader (Result string (List Bjodat))))
(defun (all-values rd)
  (skip-bom! rd)
  ;; Built front to back, so a value is one cons and there is no reverse.
  (def out (listbuilder-empty))
  (let go ()
    (skip-space! rd)
    (if (at-end? rd)
        (Ok (listbuilder->list out))
        (match (read-value rd 1)
          ((Err e) (Err e))
          ((Ok v) (listbuilder-add! out v) (go))))))

(: (-> string (Result string (List Bjodat))))
(defun (parse-bjodat-all text) (all-values (string-reader text)))

;; ---------------------------------------------------------------------------
;; Reading a record in one pass
;; ---------------------------------------------------------------------------
;; What `(text bjodat-codec)`'s generated decoders drive, so that a record can be
;; read without a `Bjodat` tree standing between the text and the fields.
;;
;; The tree is not free. A map costs, per entry, a name string built off the
;; buffer, an interned `Keyword`, a `BjoKey` node and two slots in a vec — and
;; then the decoder scans that vec once per field, and the whole of it is thrown
;; away the moment the record exists. A generated decoder knows its field names
;; when it is *compiled*, so it can compare the key against them in the buffer
;; and build none of it.
;;
;; The caller drives the loop rather than handing in a table, because the caller
;; is generated code that already knows what it is looking for, and a table
;; would be the thing this is trying not to build. So these are small and
;; awkward on purpose: they are not an API for people.
;;
;; What is *not* here is a typed value reader. A field's value still goes
;; through `read-value` and the field type's own `bjodat->` instance, which is
;; what keeps every instance — including one a user wrote — working, and every
;; error message identical to the tree decoder's. The entry is what this saves,
;; not the value.

;; Steps over `{`, and answers whether this map's entries are parenthesised.
;; Which form it is in is decided by the first thing inside it, exactly as
;; `read-map` decides it, and then every entry has to agree.
(: (-> Reader string (Result string bool)))
(defun (open-map! rd owner)
  (skip-space! rd)
  (cond
    ((at-end? rd)
     (Err (at rd (str owner ": the document ended where a map was expected"))))
    ((not (= (held-code rd) 123))
     (Err (at rd (str owner ": expected a map, and here there is " (found rd)))))
    (else
     (step! rd)
     (skip-space! rd)
     (Ok (= (held-code rd) 40)))))

;; A key's name, taken into the buffer without being interned.
;;
;; A key that is not a keyword is read and dropped, and leaves the buffer empty
;; — which no field name can equal, so it takes the same path an unknown key
;; does. That matches the tree decoder, where `bjodat-ref` only ever matches a
;; `BjoKey` and a key of any other shape is simply never found.
;;
;; A key written `#:name` is the same keyword as `:name`, as `read-keyword`
;; reads it, so it takes the same path.
(: (-> Reader string (Result string bool)))
(defun (take-key! rd owner)
  (skip-space! rd)
  (when (and (= (held-code rd) 35) (ensure! rd 2)
             (= (cast int (array-ref (record-ref rd bytes) (+ (record-ref rd next) 1))) 58))
    (step! rd))
  (cond
    ((= (held-code rd) 58)
     (step! rd)
     (def sb (buffer rd))
     (if (= (take-name! rd sb) 0)
         (Err (at rd (str owner ": a keyword needs a name after its colon, and this is "
                          (found rd))))
         (Ok #t)))
    (else
     (match (read-key rd 1)
       ((Err e) (Err e))
       ((Ok _)
        (stringbuilder-clear! (record-ref rd scratch))
        (Ok #t))))))

;; `#t` when a key is in the buffer and the reader stands at its value; `#f`
;; when the map closed.
(: (-> Reader string bool (Result string bool)))
(defun (next-key! rd owner paired)
  (skip-space! rd)
  (cond
    ((at-end? rd) (Err (at rd (str owner ": the document ended inside a map"))))
    ((= (held-code rd) 125) (step! rd) (Ok #f))
    ((and paired (not (= (held-code rd) 40)))
     (Err (at rd (str owner ": this map is written (key value), so its entries all are, "
                      "and here there is " (found rd)))))
    ((and (not paired) (= (held-code rd) 40))
     (Err (at rd (str owner ": this map is written flat, so a '(' here is an entry of the "
                      "other form"))))
    (else
     (when paired (step! rd))
     (match (take-key! rd owner)
       ((Err e) (Err e))
       ((Ok _)
        (skip-space! rd)
        (if (or (at-end? rd) (= (held-code rd) 125))
            (Err (at rd (str owner ": this map has a key with no value")))
            (Ok #t)))))))

;; The `)` of a parenthesised entry. Nothing at all in the flat form.
(: (-> Reader string bool (Result string bool)))
(defun (end-entry! rd owner paired)
  (if (not paired)
      (Ok #t)
      (begin
        (skip-space! rd)
        (if (= (held-code rd) 41)
            (begin (step! rd) (Ok #t))
            (Err (at rd (str owner ": an entry holds a key and a value, and then there is "
                             (found rd))))))))

;; Is the key in the buffer this name? The comparison the whole thing is for:
;; no string is built, and nothing is interned.
(: (-> Reader string bool))
(defun (key=? rd name)
  (def sb (record-ref rd scratch))
  (stringbuilder=? sb name))

;; A field's value, as a `Bjodat` for its own instance to convert.
(: (-> Reader (Result string Bjodat)))
(defun (field-value rd) (read-value rd 1))

;; An unknown key's value, read and dropped. It has to be *read*: a value is not
;; a token, and finding where it ends means parsing it.
(: (-> Reader (Result string bool)))
(defun (skip-value! rd)
  (match (read-value rd 1)
    ((Err e) (Err e))
    ((Ok _) (Ok #t))))

;; ---------------------------------------------------------------------------
;; Reading a record written as a tagged form
;; ---------------------------------------------------------------------------
;; `(package (name bjolang/http) (version "1.2.3"))` — the head symbol names the
;; type, and each clause is a field.
;;
;; This is the *parenthesised map* with two differences: a `(` opens it rather
;; than a `{`, and it carries a tag. So the loop a generated decoder runs is the
;; one `next-key!` and `end-entry!` already drive, and what is new here is only
;; the opener, the clause name being a bare symbol rather than a keyword, and
;; what an unknown clause costs to get past.
;;
;; **The shape never picks the type.** A `BjoList` headed by a symbol is a
;; tagged record where a tagged record is expected and a union case where a
;; union is, exactly as a `{...}` is a record in one field and a lookup table in
;; the next. The reader commits to neither and there is nothing here that could:
;; these functions are called by a decoder that already knows what it wants.

;; Steps over `(` and the head symbol, and refuses a head that is not `tag`.
(: (-> Reader string string (Result string bool)))
(defun (open-form! rd owner tag)
  (skip-space! rd)
  (cond
    ((at-end? rd)
     (Err (at rd (str owner ": the document ended where (" tag " ...) was expected"))))
    ((not (= (held-code rd) 40))
     (Err (at rd (str owner ": expected (" tag " ...), and here there is " (found rd)))))
    (else
     (step! rd)
     (skip-space! rd)
     (def sb (buffer rd))
     (cond
       ((= (take-name! rd sb) 0)
        (Err (at rd (str owner ": a form begins with the name of its type, and this is "
                         (found rd)))))
       ;; The comparison the whole thing is for: the tag is matched against
       ;; the buffer, so a form costs no string and interns nothing for its
       ;; own name any more than it does for a field's.
       ((key=? rd tag) (Ok #t))
       ;; Only here, on the way to refusing, is a string built.
       (else
        (Err (at rd (str owner ": expected (" tag " ...), and this is ("
                         (stringbuilder->string sb) " ...)"))))))))

;; `#t` when a clause's field name sits in the buffer and the reader stands at
;; its first value; `#f` when the form closed.
(: (-> Reader string (Result string bool)))
(defun (next-field! rd owner)
  (skip-space! rd)
  (cond
    ((at-end? rd) (Err (at rd (str owner ": the document ended inside a form"))))
    ((= (held-code rd) 41) (step! rd) (Ok #f))
    ((not (= (held-code rd) 40))
     (Err (at rd (str owner ": a clause is written (name value), and here there is "
                      (found rd)))))
    (else
     (step! rd)
     (skip-space! rd)
     ;; Where the head begins, which is where a refusal points.
     (def row (record-ref rd row))
     (def col (record-ref rd col))
     (if (field-name! rd)
         (Ok #t)
         ;; The tree decoder's sentence, `bjodat-clause-shape`'s, so that the
         ;; two decoders refuse the same document in the same words.
         (begin
           (record-set! rd (row row) (col col))
           (Err (at rd (str owner ": a clause begins with a field name"))))))))

;; Takes the name of a clause into the buffer, and answers whether it is one: a
;; name that `read-value` reads as a symbol. `#t`, `123`, `-1` and `+inf.0` are
;; runs of symbol characters too, and they are not symbols. The tree decoder
;; refuses a clause that does not begin with a symbol, and so does this.
(: (-> Reader bool))
(defun (field-name! rd)
  (def c (held-code rd))
  (def sb (buffer rd))
  (cond
    ((or (= c 35) (digit-code? c)) #f)
    ((and (= c 45) (ensure! rd 2)
          (digit-code? (cast int (array-ref (record-ref rd bytes) (+ (record-ref rd next) 1)))))
     #f)
    ((= (take-name! rd sb) 0) #f)
    ((or (= c 43) (= c 45))
     (not (or (stringbuilder=? sb "+inf.0") (stringbuilder=? sb "-inf.0")
              (stringbuilder=? sb "+nan.0") (stringbuilder=? sb "-nan.0"))))
    (else #t)))

;; The `)` that closes a clause holding one value.
;;
;; The owner and the field name arrive apart and are joined only on the way to
;; refusing. Joined by the caller they would be one string built per field per
;; record — which is the cost this whole path exists to avoid — and dropped they
;; would leave a message that does not say which clause was wrong. The tree
;; decoder's `bjodat-one-value` says the same sentence under the same name.
(: (-> Reader string string (Result string bool)))
(defun (end-clause! rd owner name)
  (skip-space! rd)
  (if (= (held-code rd) 41)
      (begin (step! rd) (Ok #t))
      (Err (at rd (str owner "." name ": a clause holds a field name and one value")))))

;; The value of a clause that holds one. A clause with no value, `(name)`, is
;; refused in `bjodat-one-value`'s words: `read-value` would refuse its `)` as
;; closing nothing.
(: (-> Reader string string (Result string Bjodat)))
(defun (clause-value rd owner name)
  (skip-space! rd)
  (if (= (held-code rd) 41)
      (Err (at rd (str owner "." name ": a clause holds a field name and one value")))
      (read-value rd 1)))

;; An unknown clause, read and dropped: the name has been taken, the values have
;; not. **Any arity**, and that is the point of it. A flat map's unknown key is
;; one value and done, but a clause may hold none, one or five, and a reader
;; that insisted on one would make every new clause a breaking change for every
;; older tool. This is what keeps a manifest forward-compatible.
(: (-> Reader (Result string bool)))
(defun (skip-clause! rd)
  (let go ()
    (skip-space! rd)
    (cond
      ((at-end? rd) (Err (at rd "the document ended inside a clause")))
      ((= (held-code rd) 41) (step! rd) (Ok #t))
      (else
       (match (read-value rd 1)
         ((Err e) (Err e))
         ((Ok _) (go)))))))

;; Every value of a clause, in written order, for a `#:rest` field that takes
;; them all. The closing `)` is stepped over.
;;
;; The values arrive as `Bjodat` and are converted by the element type's own
;; instance, exactly as the single-value case does through `field-value`. That
;; is not laziness: `read-value` carries the depth budget, so a manifest nesting
;; `(package (depends (package (depends ...))))` five hundred deep is refused by
;; `bjodat-max-depth` rather than by the stack. A `#:rest` that recursed
;; straight back into a generated decoder would have no budget at all.
(: (-> Reader (Result string (List Bjodat))))
(defun (clause-values! rd)
  (def out (listbuilder-empty))
  (let go ()
    (skip-space! rd)
    (cond
      ((at-end? rd) (Err (at rd "the document ended inside a clause")))
      ((= (held-code rd) 41) (step! rd) (Ok (listbuilder->list out)))
      (else
       (match (read-value rd 1)
         ((Err e) (Err e))
         ((Ok v) (listbuilder-add! out v) (go)))))))

;; One record, and then the end, for a decoder generated by `def/bjodat-type`.
;;
;; The decoder is declared `-?->` rather than `->`, and has to be: a written
;; arrow outranks the default, so a plain one would pin the generated decoder to
;; its *sync* copy — and then a bjoroutine reading a socket would call the copy
;; that parks the thread. The blocking lint says so in as many words, which is
;; how this was found.
(: (-> InputPort (-?-> Reader (Result string %a)) (Result string %a)))
(defun (read-one-pass port decode) (one-record (port-reader port) decode))

(: (-> Reader (-?-> Reader (Result string %a)) (Result string %a)))
(defun (one-record rd decode)
  (skip-bom! rd)
  (match (decode rd)
    ((Err e) (Err e))
    ((Ok v)
     (skip-space! rd)
     (if (at-end? rd)
         (Ok v)
         (Err (at rd (str "the value ended here, and then there is " (found rd))))))))

(: (-> string (-?-> Reader (Result string %a)) (Result string %a)))
(defun (parse-one-pass text decode) (one-record (string-reader text) decode))

;; A `[...]` of them, which is what a manifest's list of dependencies is.
;;
;; A `List` rather than a `Vec` because it is built by consing and turned round
;; once, where a vec would be an RRB append per element. A caller that wants a
;; vec can say so in one call; a caller that folds over it once — which is most
;; of them — should not pay for a tree it does not index into.
(: (-> InputPort (-?-> Reader (Result string %a)) (Result string (List %a))))
(defun (read-one-pass-all port decode) (all-records (port-reader port) decode))

(: (-> Reader (-?-> Reader (Result string %a)) (Result string (List %a))))
(defun (all-records rd decode)
  (skip-bom! rd)
  (skip-space! rd)
  (if (not (= (held-code rd) 91))
      (Err (at rd (str "expected a vec of values, and here there is " (found rd))))
      (begin
        (step! rd)
        ;; Built front to back, so a record is one cons and there is no reverse.
        (def out (listbuilder-empty))
        (let go ()
          (skip-space! rd)
          (cond
            ((at-end? rd) (Err (at rd "the document ended inside a vec")))
            ((= (held-code rd) 93) (step! rd) (Ok (listbuilder->list out)))
            (else
             (match (decode rd)
               ((Err e) (Err e))
               ((Ok v) (listbuilder-add! out v) (go)))))))))

(: (-> string (-?-> Reader (Result string %a)) (Result string (List %a))))
(defun (parse-one-pass-all text decode) (all-records (string-reader text) decode))

;; ---------------------------------------------------------------------------
;; Writing
;; ---------------------------------------------------------------------------
;; The canonical spelling: maps flat, in the order they are held.

(: (-> StringBuilder string void))
(defun (escape-into! sb s)
  (stringbuilder-add! sb #\")
  (loop (:for c s)
        (:do (let* ((n (char->int c)))
               (cond
                 ((= n 34) (stringbuilder-add-string! sb "\\\""))
                 ((= n 92) (stringbuilder-add-string! sb "\\\\"))
                 ((= n 10) (stringbuilder-add-string! sb "\\n"))
                 ((= n 9) (stringbuilder-add-string! sb "\\t"))
                 ((= n 13) (stringbuilder-add-string! sb "\\r"))
                 (else (stringbuilder-add! sb c))))))
  (stringbuilder-add! sb #\"))

(: (-> StringBuilder char void))
(defun (char-into! sb c)
  (def n (char->int c))
  (stringbuilder-add-string! sb "#\\")
  (cond
    ((= n 10) (stringbuilder-add-string! sb "newline"))
    ((= n 32) (stringbuilder-add-string! sb "space"))
    ((= n 9) (stringbuilder-add-string! sb "tab"))
    ((= n 13) (stringbuilder-add-string! sb "return"))
    ((= n 0) (stringbuilder-add-string! sb "nul"))
    (else (stringbuilder-add! sb c))))

;; A double with a fraction or an exponent, so that it reads back as a
;; `BjoFloat`: .NET writes 1.0 as "1", which the reader takes for an integer.
;; "1E+20" already has its exponent. NaN and the infinities are written as
;; Scheme writes them, which the reader reads back.
(: (-> double string))
(defun (double-text d)
  (cond
    ((clr-nan? d) "+nan.0")
    ((clr-positive-infinity? d) "+inf.0")
    ((clr-negative-infinity? d) "-inf.0")
    (else
     (def s (double->string d))
     (if (or (string-contains? s ".") (string-contains? s "E") (string-contains? s "e"))
         s
         (str s ".0")))))

(: (-> StringBuilder Bjodat void))
(defun (write-into! sb v)
  (match v
    ((BjoBool b) (stringbuilder-add-string! sb (if b "#t" "#f")))
    ((BjoInt n) (stringbuilder-add-string! sb (long->string n)))
    ((BjoFloat d) (stringbuilder-add-string! sb (double-text d)))
    ((BjoULong n) (stringbuilder-add-string! sb (str (displayed n) "UL")))
    ((BjoDecimal d) (stringbuilder-add-string! sb (str (decimal->string d) "M")))
    ((BjoBigInt n) (stringbuilder-add-string! sb (str (bigint->string n) "I")))
    ((BjoStr s) (escape-into! sb s))
    ((BjoChar c) (char-into! sb c))
    ((BjoSym s) (stringbuilder-add-string! sb (symbol->string s)))
    ((BjoKey k)
     (stringbuilder-add! sb #\:)
     (stringbuilder-add-string! sb (keyword->string k)))
    ((BjoList xs)
     (stringbuilder-add! sb #\()
     (loop (:with first #t #f)
           (:for x xs)
           (:do (unless first (stringbuilder-add! sb #\space))
                (write-into! sb x)))
     (stringbuilder-add! sb #\)))
    ((BjoVec xs)
     (stringbuilder-add! sb #\[)
     (loop (:with first #t #f)
           (:for x xs)
           (:do (unless first (stringbuilder-add! sb #\space))
                (write-into! sb x)))
     (stringbuilder-add! sb #\]))
    ((BjoMap entries)
     (stringbuilder-add! sb #\{)
     (loop (:with first #t #f)
           (:for x entries)
           (:do (unless first (stringbuilder-add! sb #\space))
                (write-into! sb x)))
     (stringbuilder-add! sb #\}))))

(: (-> Bjodat string))
(defun (bjodat->string v)
  (def sb (stringbuilder-empty))
  (write-into! sb v)
  (stringbuilder->string sb))
;; text/bjodat.bjo
;;
;; The bjodat data format: declare a record, get a reader and a writer.
;;
;;     (import (text bjodat))
;;
;;     (def/bjodat-type
;;       (: Dep (Record (: name string)
;;                      (: version string #:default "*")
;;                      (: pinned bool #:optional))))
;;
;;     (bjodat-parse-Dep "{:name \"http\" :version \"1.2\"}")   ;; -> (Ok Dep)
;;     (Dep->bjodat d)                                          ;; -> Bjodat
;;
;; `bjodat-parse-<Name>` reads the text **straight into the record**. There is
;; no intermediate tree: the key is compared against the field names in the
;; reader's own buffer, so a record costs no key string, no interned `Keyword`,
;; no node and no map. That is the interface to reach for, and it is about a
;; fifth faster than going through a tree.
;;
;; The tree is still here and still useful — for data whose shape is not known
;; until run time, and as the thing a field's own decoder converts. It lives in
;; `(text bjodat-core)`, and the names this module re-exports below are enough
;; that most programs never need to name it.
;;
;; One import is enough. That was not true when this module was split out, and
;; the reason it was not is worth recording rather than leaving as a silence: I
;; believed a type could not be republished through a facade, wrote that in
;; three places, and was wrong. `re-export` takes a type now — it publishes the
;; declaring module's declaration under a spelling of this module's, so there
;; is one `Bjodat` and not two — and a union comes with its cases, so a
;; hand-written `->bjodat` instance can name `BjoInt` here as readily as it
;; could over there. See `re-export` in `Docs/MODULES.org`.
;;
;; A record is a map whose keys are keywords naming its fields. The keys are
;; emitted as keyword *literals* at expansion time, so decoding a field through
;; a tree compares interned references and never interns anything at run time.
;;
;; There is no `null`. If a field is allowed to be missing, mark it with `#:optional` 
;; (which is just shorthand for `#:default None`). Since this format maps directly 
;; to Bjolang values, missing fields are strictly represented as `None`. If an 
;; `(Option %a)` must be explicitly written out in the file, `None` is encoded as 
;; an empty list `()`.

(export
  ;; The generator. Everything else here exists to serve what it writes.
  def/bjodat-type
  ;; The traits, for a type of your own that the generator did not declare.
  ->bjodat bjodat->
  ;; What generated field code calls. Exported because a macro template's names
  ;; resolve against this module's scope.
  bjodat-map bjodat-field bjodat-field/default
  bjodat-read-field bjodat-need-field
  bjodat-key bjodat-entries
  ;; And the same, for a record written as a tagged form.
  bjodat-clauses bjodat-clause-field bjodat-clause-field/default
  bjodat-clause-rest bjodat-clause-rest/default
  bjodat-read-clause bjodat-read-rest bjodat-need-clause
  bjodat-clause bjodat-clause-many bjodat-clause-values bjodat-form-value)

(import (std prelude))
(import (std syntax-match))
(import (text bjodat-core))

;; Every export is documented there, but the re-exports, whose docs are in
;; bjodat-core; the module doc is too.
(:doc #:file "bjodat.bjodoc")

;; What `def/bjodat-type` writes has to resolve where it lands, and where it
;; lands is somebody else's module. So everything the template names travels
;; from here, whether or not this module wrote it.
;;
;; The two types are the reason the split used to cost every consumer a second
;; import: a generated signature reads `(-> Name Bjodat)` and `(-> Reader ...)`.
;; They are *this* module's spellings of `bjodat-core`'s declarations, not
;; copies of them — the key is still `bjodat_core`'s, so a value built through
;; either name is one type. `Bjodat`'s cases come with it, which is what lets a
;; hand-written instance match `BjoInt` without naming the other module.
;;
;; The rest is the one-pass driver the generated decoder calls, and the entry
;; points that drive a generated `bjodat-read-<Name>`. `bjodat->string` is here
;; because writing a record back out is the other half of the round trip, and
;; nothing else about the tree is: a program that wants to read data of unknown
;; shape imports `(text bjodat-core)` and gets the reader, the writer and the
;; accessors together.
;;
;; What does *not* travel is an implementation. A re-export publishes a
;; spelling and the declaration behind it, and an impl reaches a consumer
;; through the module that wrote it — see `re-export` in `Docs/MODULES.org`.
;; `bjodat-core` writes none, so there is nothing here to miss.
(re-export
  Bjodat Reader
  open-map! next-key! end-entry! key=? skip-value!
  open-form! next-field! skip-clause!
  read-one-pass parse-one-pass read-one-pass-all parse-one-pass-all
  bjodat->string)

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------

(def/trait (->bjodat %a)
  (: ->bjodat (-> %a Bjodat)))

(def/trait (bjodat-> %a)
  (: bjodat-> (-> Bjodat (Result string %a))))

;; ---------------------------------------------------------------------------
;; Scalars
;; ---------------------------------------------------------------------------

(impl (->bjodat Bjodat) (defun (->bjodat v) v))
(impl (->bjodat long)   (defun (->bjodat n) (BjoInt n)))
(impl (->bjodat int)    (defun (->bjodat n) (BjoInt (cast System.Int64 n))))
(impl (->bjodat double) (defun (->bjodat n) (BjoFloat n)))
(impl (->bjodat ulong)  (defun (->bjodat n) (BjoULong n)))
(impl (->bjodat decimal) (defun (->bjodat n) (BjoDecimal n)))
(impl (->bjodat bigint) (defun (->bjodat n) (BjoBigInt n)))
(impl (->bjodat bool)   (defun (->bjodat b) (BjoBool b)))
(impl (->bjodat string) (defun (->bjodat s) (BjoStr s)))
(impl (->bjodat char)   (defun (->bjodat c) (BjoChar c)))
(impl (->bjodat Symbol) (defun (->bjodat s) (BjoSym s)))
(impl (->bjodat Keyword) (defun (->bjodat k) (BjoKey k)))

(impl (bjodat-> Bjodat)
  (defun (bjodat-> v) (Ok v)))

(impl (bjodat-> long)
  (defun (bjodat-> v)
    (match v
      ((BjoInt n) (Ok n))
      (_ (Err "expected an integer")))))

;; A cast would keep the low 32 bits, and a number out of range would read as
;; a different one.
(impl (bjodat-> int)
  (defun (bjodat-> v)
    (match v
      ((BjoInt n)
       (if (and (>= n -2147483648L) (<= n 2147483647L))
           (Ok (cast System.Int32 n))
           (Err "expected an integer that fits in an int")))
      (_ (Err "expected an integer")))))

(impl (bjodat-> double)
  (defun (bjodat-> v)
    (match v
      ((BjoFloat d) (Ok d))
      ;; An integer is a perfectly good double.
      ((BjoInt n) (Ok (cast System.Double n)))
      (_ (Err "expected a number")))))

;; A plain integer that is not negative is a ulong too, so a file need not
;; write the suffix.
(impl (bjodat-> ulong)
  (defun (bjodat-> v)
    (match v
      ((BjoULong n) (Ok n))
      ((BjoInt n) (if (>= n 0L) (Ok (cast System.UInt64 n)) (Err "expected an integer that is not negative")))
      (_ (Err "expected an integer")))))

;; An integer is a decimal exactly. A double is not: 0.1 has no exact binary
;; form, so it is refused rather than rounded.
(impl (bjodat-> decimal)
  (defun (bjodat-> v)
    (match v
      ((BjoDecimal d) (Ok d))
      ((BjoInt n) (Ok (cast System.Decimal n)))
      (_ (Err "expected a decimal, such as 1.50M")))))

(impl (bjodat-> bigint)
  (defun (bjodat-> v)
    (match v
      ((BjoBigInt n) (Ok n))
      ((BjoInt n) (Ok (cast System.Numerics.BigInteger n)))
      ((BjoULong n) (Ok (cast System.Numerics.BigInteger n)))
      (_ (Err "expected an integer")))))

(impl (bjodat-> bool)
  (defun (bjodat-> v)
    (match v
      ((BjoBool b) (Ok b))
      (_ (Err "expected #t or #f")))))

(impl (bjodat-> string)
  (defun (bjodat-> v)
    (match v
      ((BjoStr s) (Ok s))
      (_ (Err "expected a string")))))

(impl (bjodat-> char)
  (defun (bjodat-> v)
    (match v
      ((BjoChar c) (Ok c))
      (_ (Err "expected a character")))))

;; A symbol and a keyword are different values, and a field asking for one does
;; not take the other: that distinction is the reason both are in the format.
(impl (bjodat-> Symbol)
  (defun (bjodat-> v)
    (match v
      ((BjoSym s) (Ok s))
      (_ (Err "expected a symbol")))))

(impl (bjodat-> Keyword)
  (defun (bjodat-> v)
    (match v
      ((BjoKey k) (Ok k))
      (_ (Err "expected a keyword")))))

;; ---------------------------------------------------------------------------
;; Absence
;; ---------------------------------------------------------------------------

(impl (->bjodat (Option %a))
  (where (->bjodat %a))
  (defun (->bjodat o)
    (def (Some x) o :leave-with (BjoList Nil))
    (->bjodat x)))

(impl (bjodat-> (Option %a))
  (where (bjodat-> %a))
  (defun (bjodat-> v)
    (match v
      ((BjoList Nil) (Ok None))
      (_ (match (bjodat-> v)
           ((Err e) (Err e))
           ((Ok x) (Ok (Some x))))))))

;; ---------------------------------------------------------------------------
;; Sequences
;; ---------------------------------------------------------------------------

(impl (->bjodat (Vec %a))
  (where (->bjodat %a))
  (defun (->bjodat v)
    (BjoVec (vec-map (fun (x) (->bjodat x)) v))))

(impl (bjodat-> (Vec %a))
  (where (bjodat-> %a))
  (defun (bjodat-> v)
    (match v
      ;; Into one builder, which the vec takes at the end, and not a
      ;; `vec-add` per element, which made a vec for each. The first element
      ;; that does not decode ends the walk.
      ((BjoVec items)
       (def b (vecbuilder-empty/small))
       (let go ((c (vec-cursor items)))
         (if (vec-cursor-done? c)
             (Ok (vecbuilder->vec b))
             (match (bjodat-> (vec-cursor-current c))
               ((Err e) (Err e))
               ((Ok x)
                (vecbuilder-add! b x)
                (go (vec-cursor-next c)))))))
      (_ (Err "expected a vec")))))

;; A list is the other sequence, and it is a different shape on the page: a
;; `(...)` rather than a `[...]`. A field takes the one its type names.
(impl (->bjodat (List %a))
  (where (->bjodat %a))
  (defun (->bjodat xs)
    (BjoList (list-map (fun (x) (->bjodat x)) xs))))

(impl (bjodat-> (List %a))
  (where (bjodat-> %a))
  (defun (bjodat-> v)
    (def (BjoList items) v :leave-with (Err "expected a list"))
    (def (Ok b)(list-foldl (fun (el acc)
                             (def* ((Ok b) acc :propagate)
                                   ((Ok x) (bjodat-> el) :propagate))
                             (Ok (Cons x b)))
                           (Ok Nil)
                           items) :propagate)
    (Ok (list-reverse b))))

;; ---------------------------------------------------------------------------
;; What generated field code calls
;; ---------------------------------------------------------------------------

(: (-> string Bjodat (Result string (Vec Bjodat))))
(defun (bjodat-map owner v)
  (match v
    ((BjoMap entries) (Ok entries))
    (_ (Err (str owner ": expected a map")))))

;; The owner and the key are prefixed onto whatever the value's own decoder
;; said, so a wrong type reads "Package.version: expected a string" rather than
;; naming neither.
(: (-> string (Vec Bjodat) Keyword (Result string %a)) (where (bjodat-> %a)))
(defun (bjodat-field owner entries key)
  (match (bjodat-ref entries key)
    (None (Err (str owner ": the map has no :" (keyword->string key) " entry")))
    ((Some v)
     (result-map-err (fun (e) (str owner "." (keyword->string key) ": " e)) (bjodat-> v)))))

;; Absent is the default. Unlike JSON there is no null to mean it as well.
(: (-> string (Vec Bjodat) Keyword %a (Result string %a)) (where (bjodat-> %a)))
(defun (bjodat-field/default owner entries key fallback)
  (match (bjodat-ref entries key)
    (None (Ok fallback))
    ((Some v)
     (result-map-err (fun (e) (str owner "." (keyword->string key) ": " e)) (bjodat-> v)))))

;; --- and what a one-pass decoder calls --------------------------------------
;;
;; Wait, if this is a "one-pass" decoder, why are we still reading values into 
;; intermediate `Bjodat` trees? This is deliberate! It allows us to reuse all the 
;; existing `bjodat->` decoders (including custom ones written by users) and keeps 
;; error messages perfectly consistent. 
;; 
;; The performance savings of "one-pass" come entirely from the keys: we avoid 
;; allocating the map, interning the keywords, creating `BjoKey` nodes, and doing 
;; map lookups.

;; --- the two cases a generated encoder builds, as functions -----------------
;;
;; `(BjoKey k)` written directly into the template would not compile in the
;; importing module. A union's *cases* do not cross a second module boundary:
;; this module can publish the `Bjodat` type it imported, but not `BjoKey`, and
;; the encoder is spliced into the module that wrote `def/bjodat-type`. A
;; function is an ordinary binding and re-exports cleanly, so these two are what
;; make one import enough.
;;
;; Writing an instance by hand for a type of your own still wants the cases, and
;; that is what `(import (text bjodat-core))` is for. It is the rarer thing and
;; it is the one that should say so.

(: (-> Keyword Bjodat))
(defun (bjodat-key k) (BjoKey k))

;; The entries were pushed in reverse, and are turned round once here.
(: (-> (List Bjodat) Bjodat))
(defun (bjodat-entries acc) (BjoMap (list->vec (list-reverse acc))))

(: (-> string Keyword Reader (Result string %a)) (where (bjodat-> %a)))
(defun (bjodat-read-field owner key rd)
  (match (field-value rd)
    ((Err e) (Err e))
    ((Ok v)
     (result-map-err (fun (e) (str owner "." (keyword->string key) ": " e)) (bjodat-> v)))))

;; A field that was never met. The message is `bjodat-field`'s, word for word,
;; so which decoder read the document is not something an error report reveals.
(: (-> (Option %a) string Keyword (Result string %a)))
(defun (bjodat-need-field slot owner key)
  (match slot
    ((Some x) (Ok x))
    (None (Err (str owner ": the map has no :" (keyword->string key) " entry")))))

;; ---------------------------------------------------------------------------
;; The same, for a record written as a tagged form
;; ---------------------------------------------------------------------------
;; `(package (name bjolang/http) (version "1.2.3"))`. A clause names a field and
;; holds its value, and the head symbol names the type.
;;
;; The clauses are scanned where a map's entries are scanned, and for the same
;; reason: a record has a handful of fields, and walking a handful beats
;; building a trie to throw away. What is deliberately *not* done is flattening
;; the clauses into a keyword map first and reusing `bjodat-ref`. That was the
;; shorter implementation and it would have interned a `Keyword` per field —
;; which is the one cost the tagged form has no reason to pay, since its names
;; arrive as symbols that are interned already.

;; Every clause has to be `(name ...)`. Checked up front rather than as each
;; field is looked for, because the one-pass reader refuses a malformed clause
;; whether or not any field wanted it, and two decoders that disagree about the
;; same document are worse than either rule.
(: (-> string (List Bjodat) (Result string (List Bjodat))))
(defun (bjodat-clause-shape owner cs)
  (let go ((rest cs))
    (match rest
      (Nil (Ok cs))
      ((Cons c more)
       (match c
         ((BjoList (Cons (BjoSym _) _)) (go more))
         ;; `next-field!`'s sentence, so that the two decoders refuse the same
         ;; document in the same words.
         (_ (Err (str owner ": a clause begins with a field name"))))))))

;; The clauses of `(tag (name value) ...)`, with the tag checked.
(: (-> string string Bjodat (Result string (List Bjodat))))
(defun (bjodat-clauses owner tag v)
  (match v
    ((BjoList (Cons (BjoSym h) cs))
     (if (= (symbol->string h) tag)
         (bjodat-clause-shape owner cs)
         (Err (str owner ": expected (" tag " ...), and this is ("
                   (symbol->string h) " ...)"))))
    (_ (Err (str owner ": expected (" tag " ...)")))))

;; The values of the first clause named `name`. **The first wins**, which is
;; what `bjodat-ref` does for a map and what the one-pass decoder does for a
;; repeated clause, so a document read any of the three ways answers the same.
(: (-> (List Bjodat) string (Option (List Bjodat))))
(defun (bjodat-clause-ref cs name)
  (let go ((rest cs))
    (def* ((Cons c more) rest :leave-with None)
          ((BjoList (Cons (BjoSym h) vs)) c :leave-with (go more)))
    (if (= (symbol->string h) name)
        (Some vs)
        (go more))))

;; A clause holding exactly one value, which is every field but a `#:rest` one.
(: (-> string string (List Bjodat) (Result string %a)) (where (bjodat-> %a)))
(defun (bjodat-one-value owner name vs)
  (match vs
    ((Cons v Nil)
     (result-map-err (fun (e) (str owner "." name ": " e)) (bjodat-> v)))
    (_ (Err (str owner "." name ": a clause holds a field name and one value")))))

;; Every value of a clause, for a `#:rest` field.
(: (-> string string (List Bjodat) (Result string (List %a))) (where (bjodat-> %a)))
(defun (bjodat-all-values owner name vs)
  (result-map list-reverse
              (list-foldl (fun (el acc)
                            (def (Ok b) acc :propagate)
                            (match (bjodat-> el)
                              ((Err e) (Err (str owner "." name ": " e)))
                              ((Ok x) (Ok (Cons x b)))))
                          (Ok Nil)
                          vs)))

(: (-> string (List Bjodat) string (Result string %a)) (where (bjodat-> %a)))
(defun (bjodat-clause-field owner cs name)
  (match (bjodat-clause-ref cs name)
    (None (Err (str owner ": there is no (" name " ...) clause")))
    ((Some vs) (bjodat-one-value owner name vs))))

(: (-> string (List Bjodat) string %a (Result string %a)) (where (bjodat-> %a)))
(defun (bjodat-clause-field/default owner cs name fallback)
  (match (bjodat-clause-ref cs name)
    (None (Ok fallback))
    ((Some vs) (bjodat-one-value owner name vs))))

(: (-> string (List Bjodat) string (Result string (List %a))) (where (bjodat-> %a)))
(defun (bjodat-clause-rest owner cs name)
  (match (bjodat-clause-ref cs name)
    (None (Err (str owner ": there is no (" name " ...) clause")))
    ((Some vs) (bjodat-all-values owner name vs))))

(: (-> string (List Bjodat) string (List %a) (Result string (List %a))) (where (bjodat-> %a)))
(defun (bjodat-clause-rest/default owner cs name fallback)
  (match (bjodat-clause-ref cs name)
    (None (Ok fallback))
    ((Some vs) (bjodat-all-values owner name vs))))

;; --- and what a one-pass decoder calls --------------------------------------
;;
;; Both of these leave the clause fully consumed, closing paren and all, so the
;; generated loop has one shape whether the field took one value or every value.

(: (-> string string Reader (Result string %a)) (where (bjodat-> %a)))
(defun (bjodat-read-clause owner name rd)
  (def* ((Ok v) (clause-value rd owner name) :propagate)
        ((Ok x) (bjodat-> v) :leave ((Err e) (Err (str owner "." name ": " e))))
        ((Ok y) (end-clause! rd owner name) :propagate))
  (Ok x))


(: (-> string string Reader (Result string (List %a))) (where (bjodat-> %a)))
(defun (bjodat-read-rest owner name rd)
  (match (clause-values! rd)
    ((Err e) (Err e))
    ((Ok vs) (bjodat-all-values owner name vs))))

;; A clause that was never met. The message is `bjodat-clause-field`'s, word for
;; word, so which decoder read the document is not something a report reveals.
(: (-> (Option %a) string string (Result string %a)))
(defun (bjodat-need-clause slot owner name)
  (match slot
    ((Some x) (Ok x))
    (None (Err (str owner ": there is no (" name " ...) clause")))))

;; --- the three shapes a generated encoder builds, as functions --------------
;;
;; Here for the reason `bjodat-key` and `bjodat-entries` are: a union's *cases*
;; do not cross a second module boundary, and the encoder is spliced into
;; whichever module wrote `def/bjodat-type`.

(: (-> string Bjodat Bjodat))
(defun (bjodat-clause name v)
  (BjoList (Cons (BjoSym (string->symbol name)) (Cons v Nil))))

(: (-> string (List Bjodat) Bjodat))
(defun (bjodat-clause-many name vs)
  (BjoList (Cons (BjoSym (string->symbol name)) vs)))

(: (-> (List %a) (List Bjodat)) (where (->bjodat %a)))
(defun (bjodat-clause-values xs)
  (list-map (fun (x) (->bjodat x)) xs))

;; The clauses were pushed in reverse, and are turned round once here.
(: (-> string (List Bjodat) Bjodat))
(defun (bjodat-form-value tag acc)
  (BjoList (Cons (BjoSym (string->symbol tag)) (list-reverse acc))))

;; ---------------------------------------------------------------------------
;; The generator
;; ---------------------------------------------------------------------------
;; A runtime macro. Field types are never inspected: what a field encodes as is
;; settled later by instance resolution, and only the markers are read here.

(: (-> string Syntax))
(defun (jb-sym name) (SSym (string->symbol name)))

(: (-> Syntax Syntax))
(defun (jb-str s) (SStr (syntax->string s)))

;; `(Box %a)` names the type `Box`; a bare `Package` names itself.
(: (-> Syntax Syntax))
(defun (jb-head-name head)
  (match head
    ((SList parts) (list-head parts))
    (_ head)))

(: (-> Syntax (List Syntax)))
(defun (jb-head-vars head)
  (match head
    ((SList parts) (list-tail parts))
    (_ Nil)))

(: (-> (-> Symbol Syntax) Syntax string (List Syntax)))
(defun (jb-where inject head trait)
  (def vars (jb-head-vars head))
  (if (list-empty? vars)
      Nil
      (list #'(,(inject 'where)
               ,@(list-map (fun (v) #'(,(inject (string->symbol trait)) ,v)) vars)))))

;; --- markers ---------------------------------------------------------------

(: (-> Syntax string bool))
(defun (jb-marker=? m name) (= (syntax->string m) name))

;; The key this field is written under. `#:key :other-name` renames it; without
;; one the key is the field's own name. Built here, at expansion time, so the
;; generated code holds a keyword literal.
;;
;; Note: We generate a keyword literal directly in the decoder for every field. 
;; You might worry this is slow (e.g. hashing and interning the string every time), 
;; and historically, this macro used to generate a static variable to cache it. 
;; 
;; We don't do that anymore because the compiler's `Codegen` phase is now smart 
;; enough to automatically hoist all keyword literals into static fields behind 
;; the scenes. It's just as fast, and the code is cleaner.
(: (-> Syntax (List Syntax) Syntax))
(defun (jb-key name markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) (SKey (string->keyword (syntax->string name))))
      ((jb-marker=? (list-head ms) "#:key") (list-head (list-tail ms)))
      (else (go (list-tail ms))))))

;; The fallback expression, or None if the field is required.
(: (-> (List Syntax) (Option Syntax)))
(defun (jb-fallback markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) None)
      (:def m (list-head ms))
      ((jb-marker=? m "#:optional") (Some (jb-sym "None")))
      ((jb-marker=? m "#:default") (Some (list-head (list-tail ms))))
      (else (go (list-tail ms))))))

(: (-> (List Syntax) bool))
(defun (jb-optional? markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) #f)
      ((jb-marker=? (list-head ms) "#:optional") #t)
      (else (go (list-tail ms))))))

;; `#:rest` — the field takes *every* value of its clause rather than one, so
;; `(depends (package ...) (package ...))` fills a `(List Dep)`.
;;
;; This is the answer to the question a tagged list otherwise leaves open, which
;; is which clauses may repeat. It is answered per *field*, by the type, because
;; that is the only place that knows: `version` holds one and `depends` holds
;; many, and nothing about the syntax could tell them apart.
;;
;; A field is marked, not a clause repeated. `(depends a) (depends b)` does not
;; accumulate — the first clause wins, as the first entry under a key does.
;; Accumulating would have meant one rule for maps and another for forms.
(: (-> (List Syntax) bool))
(defun (jb-rest? markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) #f)
      ((jb-marker=? (list-head ms) "#:rest") #t)
      (else (go (list-tail ms))))))

;; `(Record #:tag package (: name ...) ...)` — the marker, if it is there, and
;; the fields with it taken off.
;;
;; There is no default. `Package` spelled `package` is a casing guess, and which
;; tag a type is written under belongs in the file's vocabulary rather than in a
;; rule about capital letters.
(: (-> (List Syntax) (Option Syntax)))
(defun (jb-tag body)
  (cond
    ((list-empty? body) None)
    ((jb-marker=? (list-head body) "#:tag")
     (Some (jb-str (list-head (list-tail body)))))
    (else None)))

(: (-> (List Syntax) (List Syntax)))
(defun (jb-fields body)
  (if (some? (jb-tag body))
      (list-tail (list-tail body))
      body))

;; What the record declaration keeps: everything that is not ours.
(: (-> (List Syntax) (List Syntax)))
(defun (jb-keep markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) Nil)
      (:def m (list-head ms))
      ((jb-marker=? m "#:default") (go (list-tail (list-tail ms))))
      ((jb-marker=? m "#:key") (go (list-tail (list-tail ms))))
      ((jb-marker=? m "#:optional") (go (list-tail ms)))
      ((jb-marker=? m "#:rest") (go (list-tail ms)))
      (else (Cons m (go (list-tail ms)))))))

(: string)
(def jb-shape
  "a field is written (: name type), then any of #:mutable, #:optional, #:default <expr>, #:key :name, #:rest.")

(: (-> Syntax string (List Syntax)))
(defun (jb-fail-list form message)
  (ignore (syntax-error form message))
  Nil)

(: (-> Syntax Syntax))
(defun (jb-field-name f)
  (syntax-match f
    ((_ name _ marker ...) name)
    (bad (syntax-error bad jb-shape))))

(: (-> Syntax (List Syntax)))
(defun (jb-field-markers f)
  (syntax-match f
    ((_ _ _ marker ...) marker)
    (bad (jb-fail-list bad jb-shape))))

(: (-> Syntax Syntax))
(defun (jb-strip f)
  (syntax-match f
    ((colon name ty marker ...) #'(,colon ,name ,ty ,@(jb-keep marker)))
    (bad (syntax-error bad jb-shape))))

;; --- encode ----------------------------------------------------------------
;;
;; The entries are built up in reverse and turned round once, so a map is one
;; vec and the order is the order the fields were declared in. An optional
;; field that is None is not written at all.

(: (-> (List Syntax) Syntax Syntax))
(defun (jb-enc-chain fields acc)
  (cond
    ((list-empty? fields) #'(bjodat-entries ,acc))
    (else
     (def f (list-head fields))
     (def name (jb-field-name f))
     (def markers (jb-field-markers f))
     (def key (jb-key name markers))
     (def push #'(Cons (->bjodat (record-ref v ,name))
                       (Cons (bjodat-key ,key) ,acc)))
     (def step (if (jb-optional? markers)
                   #'(if (some? (record-ref v ,name)) ,push ,acc)
                   push))
     #'(let ((,acc ,step))
         ,(jb-enc-chain (list-tail fields) acc)))))

(: (-> (List Syntax) Syntax))
(defun (jb-encode fields)
  (def acc (jb-sym "__jb-acc"))
  #'(let ((,acc Nil))
      ,(jb-enc-chain fields acc)))

;; The same, as a tagged form. An optional field that is `None` is not written,
;; and neither is a `#:rest` field that is empty *and* has a fallback to come
;; back as — `(authors)` is noise, and a field with no fallback has to be
;; written or the document would not read back.

(: (-> (List Syntax) Syntax Syntax Syntax))
(defun (jb-form-enc-chain fields tag acc)
  (cond
    ((list-empty? fields) #'(bjodat-form-value ,tag ,acc))
    (else
     (def f (list-head fields))
     (def name (jb-field-name f))
     (def markers (jb-field-markers f))
     (def nm (jb-key-name name markers))
     (def push (if (jb-rest? markers)
                   #'(Cons (bjodat-clause-many
                            ,nm (bjodat-clause-values (record-ref v ,name)))
                           ,acc)
                   #'(Cons (bjodat-clause ,nm (->bjodat (record-ref v ,name)))
                           ,acc)))
     (def step (cond
                 ((jb-optional? markers)
                  #'(if (some? (record-ref v ,name)) ,push ,acc))
                 ((and (jb-rest? markers) (some? (jb-fallback markers)))
                  #'(if (list-empty? (record-ref v ,name)) ,acc ,push))
                 (else push)))
     #'(let ((,acc ,step))
         ,(jb-form-enc-chain (list-tail fields) tag acc)))))

(: (-> (List Syntax) Syntax Syntax))
(defun (jb-form-encode fields tag)
  (def acc (jb-sym "__jb-acc"))
  #'(let ((,acc Nil))
      ,(jb-form-enc-chain fields tag acc)))

;; --- decode ----------------------------------------------------------------
;;
;; One `match` per field wrapping the next, with the constructor innermost.

(: (-> Syntax Syntax (List Syntax) int Syntax Syntax))
(defun (jb-chain owner entries fields i ctor)
  (cond
    ((list-empty? fields) ctor)
    (else
     (def f (list-head fields))
     (def v (jb-sym (str "__jb-v" (int->string i))))
     (def e (jb-sym (str "__jb-e" (int->string i))))
     (def markers (jb-field-markers f))
     (def key (jb-key (jb-field-name f) markers))
     (def get (match (jb-fallback markers)
                (None #'(bjodat-field ,owner ,entries ,key))
                ((Some d) #'(bjodat-field/default ,owner ,entries ,key ,d))))
     #'(match ,get
         ((Err ,e) (Err ,e))
         ((Ok ,v) ,(jb-chain owner entries (list-tail fields) (+ i 1) ctor))))))

(: (-> Syntax (List Syntax) Syntax))
(defun (jb-ctor head fields)
  #'(Ok (,(jb-head-name head)
         ,@(let go ((fs fields) (i 0))
             (if (list-empty? fs)
                 Nil
                 (Cons #'(,(jb-field-name (list-head fs))
                          ,(jb-sym (str "__jb-v" (int->string i))))
                       (go (list-tail fs) (+ i 1))))))))

(: (-> Syntax (List Syntax) Syntax))
(defun (jb-decode head fields)
  (def owner (jb-str (jb-head-name head)))
  (def entries (jb-sym "__jb-entries"))
  (def e (jb-sym "__jb-e"))
  #'(match (bjodat-map ,owner v)
      ((Err ,e) (Err ,e))
      ((Ok ,entries) ,(jb-chain owner entries fields 0 (jb-ctor head fields)))))

;; The same, over the clauses of a tagged form. `jb-ctor` is shared, so both
;; decoders end in one constructor call written once.

(: (-> Syntax Syntax (List Syntax) int Syntax Syntax))
(defun (jb-form-chain owner cs fields i ctor)
  (cond
    ((list-empty? fields) ctor)
    (else
     (def f (list-head fields))
     (def v (jb-var i))
     (def e (jb-sym "__jb-e"))
     (def markers (jb-field-markers f))
     (def nm (jb-key-name (jb-field-name f) markers))
     (def get (if (jb-rest? markers)
                  (match (jb-fallback markers)
                    (None #'(bjodat-clause-rest ,owner ,cs ,nm))
                    ((Some d) #'(bjodat-clause-rest/default ,owner ,cs ,nm ,d)))
                  (match (jb-fallback markers)
                    (None #'(bjodat-clause-field ,owner ,cs ,nm))
                    ((Some d) #'(bjodat-clause-field/default ,owner ,cs ,nm ,d)))))
     #'(match ,get
         ((Err ,e) (Err ,e))
         ((Ok ,v) ,(jb-form-chain owner cs (list-tail fields) (+ i 1) ctor))))))

(: (-> Syntax Syntax (List Syntax) Syntax))
(defun (jb-form-decode head tag fields)
  (def owner (jb-str (jb-head-name head)))
  (def cs (jb-sym "__jb-clauses"))
  (def e (jb-sym "__jb-e"))
  #'(match (bjodat-clauses ,owner ,tag v)
      ((Err ,e) (Err ,e))
      ((Ok ,cs) ,(jb-form-chain owner cs fields 0 (jb-ctor head fields)))))

;; --- decode in one pass -----------------------------------------------------
;;
;; The same record, read straight off the reader with no `Bjodat` map in
;; between. The shape is a loop carrying one slot per field:
;;
;;     (let go ((s0 None) (s1 None))
;;       (match (next-key! rd owner paired)
;;         ((Ok #f) <every slot answered for, then the constructor>)
;;         ((Ok _)  (cond ((key=? rd "name") ... (go (Some x) s1))
;;                        ((key=? rd "git")  ... (go s0 (Some x)))
;;                        (else <skip the value> (go s0 s1))))))
;;
;; We use an `(Option %a)` variable to track each field during the loop, rather 
;; than a separate value and a boolean 'seen' flag. Since `Option` is a lightweight 
;; struct, this adds no extra memory overhead and elegantly tracks both "has it 
;; been seen?" and "what is the value?" simultaneously.
;;
;; We compare incoming keys directly against the expected string in the reader's 
;; raw buffer using `key=?`. This is the core optimization of the one-pass 
;; decoder: we never allocate memory or intern strings for keys we encounter in 
;; the file. (The standard tree decoder had to intern every single key just to 
;; look it up in the map).

(: (-> int Syntax))
(defun (jb-slot i) (jb-sym (str "__jb-s" (int->string i))))

(: (-> int Syntax))
(defun (jb-var i) (jb-sym (str "__jb-v" (int->string i))))

;; The key's name as a string, for `key=?`. `#:key :other-name` renames it here
;; exactly as it renames the keyword literal, so the two cannot drift.
(: (-> Syntax (List Syntax) Syntax))
(defun (jb-key-name name markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) (SStr (syntax->string name)))
      ((jb-marker=? (list-head ms) "#:key")
       (match (list-head (list-tail ms))
         ((SKey k) (SStr (keyword->string k)))
         (other (SStr (syntax->string other)))))
      (else (go (list-tail ms))))))

(: (-> (List Syntax) int (List Syntax)))
(defun (jb-slot-inits fields i)
  (if (list-empty? fields)
      Nil
      (Cons #'(,(jb-slot i) None)
            (jb-slot-inits (list-tail fields) (+ i 1)))))

;; The slots a recursive call passes on, unchanged.
(: (-> (List Syntax) int (List Syntax)))
(defun (jb-slot-args fields i)
  (if (list-empty? fields)
      Nil
      (Cons (jb-slot i) (jb-slot-args (list-tail fields) (+ i 1)))))

;; The same, with slot `hit` replaced by the value just read.
(: (-> (List Syntax) int int Syntax (List Syntax)))
(defun (jb-slot-args/hit fields i hit v)
  (match fields
    (Nil Nil)
    (_ (Cons (if (= i hit) v (jb-slot i))
             (jb-slot-args/hit (list-tail fields) (+ i 1) hit v)))))

(: (-> Syntax Syntax Syntax (List Syntax) (List Syntax) int (List Syntax)))
(defun (jb-clauses owner paired go all fields i)
  (cond
    ((list-empty? fields) Nil)
    (else
     (def f (list-head fields))
     (def markers (jb-field-markers f))
     (def name (jb-field-name f))
     (def x (jb-sym "__jb-x"))
     (def e (jb-sym "__jb-e"))
     ;; The *first* entry under a key wins, which is what `bjodat-ref` does
     ;; for the tree decoder: it answers the first match and stops. So a key
     ;; whose slot is filled is not this field's any more. It goes to the
     ;; `else` branch, which skips its value as it skips an unknown key's.
     ;; Decoded, a later value of the wrong type was refused here and not by
     ;; the tree decoder, which never looks at it.
     (Cons #'((and (none? ,(jb-slot i)) (key=? rd ,(jb-key-name name markers)))
              (match (bjodat-read-field ,owner ,(jb-key name markers) rd)
                ((Err ,e) (Err ,e))
                ((Ok ,x)
                 (match (end-entry! rd ,owner ,paired)
                   ((Err ,e) (Err ,e))
                   ((Ok _)
                    (,go ,@(jb-slot-args/hit all 0 i #'(Some ,x))))))))
           (jb-clauses owner paired go all (list-tail fields) (+ i 1))))))

;; Every slot answered for, innermost first, ending in the constructor. A
;; required field that was never met is the one refusal; a field with a fallback
;; takes it here rather than at the entry, because absence is only known once
;; the map has closed.
(: (-> Syntax Syntax (List Syntax) (List Syntax) int Syntax))
(defun (jb-settle head owner all fields i)
  (cond
    ((list-empty? fields) (jb-ctor head all))
    (else
     (def f (list-head fields))
     (def markers (jb-field-markers f))
     (def key (jb-key (jb-field-name f) markers))
     (def e (jb-sym "__jb-e"))
     (def p (jb-sym "__jb-p"))
     (def rest (jb-settle head owner all (list-tail fields) (+ i 1)))
     (match (jb-fallback markers)
       (None #'(match (bjodat-need-field ,(jb-slot i) ,owner ,key)
                 ((Err ,e) (Err ,e))
                 ((Ok ,(jb-var i)) ,rest)))
       ((Some d) #'(let ((,(jb-var i) (match ,(jb-slot i)
                                        ((Some ,p) ,p)
                                        (None ,d))))
                     ,rest))))))

(: (-> Syntax (List Syntax) Syntax))
(defun (jb-one-pass head fields)
  (def owner (jb-str (jb-head-name head)))
  (def paired (jb-sym "__jb-paired"))
  (def go (jb-sym "__jb-go"))
  (def e (jb-sym "__jb-e"))
  #'(match (open-map! rd ,owner)
      ((Err ,e) (Err ,e))
      ((Ok ,paired)
       (let ,go (,@(jb-slot-inits fields 0))
         (match (next-key! rd ,owner ,paired)
           ((Err ,e) (Err ,e))
           ((Ok #f) ,(jb-settle head owner fields fields 0))
           ((Ok _)
            (cond
              ,@(jb-clauses owner paired go fields fields 0)
              (else
               (match (skip-value! rd)
                 ((Err ,e) (Err ,e))
                 ((Ok _)
                  (match (end-entry! rd ,owner ,paired)
                    ((Err ,e) (Err ,e))
                    ((Ok _) (,go ,@(jb-slot-args fields 0))))))))))))))

;; --- reading a tagged form in one pass --------------------------------------
;;
;; The same loop, over `(package (name "x") (version "1.2"))`. Three things
;; differ from the map above and nothing else does: `open-form!` checks the tag
;; where `open-map!` only found a brace, a clause name is a bare symbol rather
;; than a keyword, and an unknown clause costs `skip-clause!` rather than
;; `skip-value!` because it may hold any number of values.
;;
;; The closing `)` of a clause is consumed by whichever of `bjodat-read-clause`
;; and `bjodat-read-rest` read it, which is why there is no `end-entry!` here to
;; match the map decoder's. That keeps the two branches one shape.

(: (-> Syntax Syntax (List Syntax) (List Syntax) int (List Syntax)))
(defun (jb-form-clauses owner go all fields i)
  (cond
    ((list-empty? fields) Nil)
    (else
     (def f (list-head fields))
     (def markers (jb-field-markers f))
     (def nm (jb-key-name (jb-field-name f) markers))
     (def x (jb-sym "__jb-x"))
     (def e (jb-sym "__jb-e"))
     (def read (if (jb-rest? markers)
                   #'(bjodat-read-rest ,owner ,nm rd)
                #'(bjodat-read-clause ,owner ,nm rd)))
     ;; The first clause under a name wins, as the first entry under a key
     ;; does. A later one goes to the `else` branch and is skipped as an
     ;; unknown clause is, at any arity, as the tree decoder ignores it.
     (Cons #'((and (none? ,(jb-slot i)) (key=? rd ,nm))
              (match ,read
                ((Err ,e) (Err ,e))
                ((Ok ,x)
                 (,go ,@(jb-slot-args/hit all 0 i #'(Some ,x))))))
           (jb-form-clauses owner go all (list-tail fields) (+ i 1))))))

(: (-> Syntax Syntax (List Syntax) (List Syntax) int Syntax))
(defun (jb-form-settle head owner all fields i)
  (cond
    ((list-empty? fields) (jb-ctor head all))
    (else
     (def f (list-head fields))
     (def markers (jb-field-markers f))
     (def nm (jb-key-name (jb-field-name f) markers))
     (def e (jb-sym "__jb-e"))
     (def p (jb-sym "__jb-p"))
     (def rest (jb-form-settle head owner all (list-tail fields) (+ i 1)))
     (match (jb-fallback markers)
       (None #'(match (bjodat-need-clause ,(jb-slot i) ,owner ,nm)
                 ((Err ,e) (Err ,e))
                 ((Ok ,(jb-var i)) ,rest)))
       ((Some d) #'(let ((,(jb-var i) (match ,(jb-slot i)
                                        ((Some ,p) ,p)
                                        (None ,d))))
                     ,rest))))))

(: (-> Syntax Syntax (List Syntax) Syntax))
(defun (jb-form-one-pass head tag fields)
  (def owner (jb-str (jb-head-name head)))
  (def go (jb-sym "__jb-go"))
  (def e (jb-sym "__jb-e"))
  #'(match (open-form! rd ,owner ,tag)
      ((Err ,e) (Err ,e))
      ((Ok _)
       (let ,go (,@(jb-slot-inits fields 0))
         (match (next-field! rd ,owner)
           ((Err ,e) (Err ,e))
           ((Ok #f) ,(jb-form-settle head owner fields fields 0))
           ((Ok _)
            (cond
              ,@(jb-form-clauses owner go fields fields 0)
              (else
               (match (skip-clause! rd)
                 ((Err ,e) (Err ,e))
                 ((Ok _) (,go ,@(jb-slot-args fields 0))))))))))))

;; --- the form ---------------------------------------------------------------

(: (-> (-> Symbol Syntax) Syntax Syntax (List Syntax) Syntax))
(defun (jb-expand inject colon head body)
  (def tag (jb-tag body))
  (def fields (jb-fields body))
  (def name (syntax->string (jb-head-name head)))
  ;; Which of the two spellings this record is written in is settled
  ;; once, here, and the three generated bodies agree by construction.
  ;; A type has one encoding: a value that read as `{...}` writes back as
  ;; `{...}`, and one that read as `(tag ...)` writes back as that.
  (def encode (match tag
                (None (jb-encode fields))
                ((Some t) (jb-form-encode fields t))))
  (def decode (match tag
                (None (jb-decode head fields))
                ((Some t) (jb-form-decode head t fields))))
  (def one-pass (match tag
                  (None (jb-one-pass head fields))
                  ((Some t) (jb-form-one-pass head t fields))))
  (def enc (inject (string->symbol (str name "->bjodat"))))
  (def dec (inject (string->symbol (str "bjodat->" name))))
  (def rdr (inject (string->symbol (str "bjodat-read-" name))))
  (def prs (inject (string->symbol (str "bjodat-parse-" name))))
  #'(begin
      (type (,colon ,head (Record ,@(list-map jb-strip fields))))

      (impl (->bjodat ,head)
        ,@(jb-where inject head "->bjodat")
        (defun (->bjodat v) ,encode))

      (impl (bjodat-> ,head)
        ,@(jb-where inject head "bjodat->")
        (defun (bjodat-> v) ,decode))

      ;; The named pair. `bjodat->` alone would be ambiguous at a call site;
      ;; this signature is what fixes the instance, once per type.
      (: ,enc (-> ,head ,(inject 'Bjodat)) ,@(jb-where inject head "->bjodat"))
      (defun (,enc v) (->bjodat v))
      (: ,dec (-> ,(inject 'Bjodat) (Result string ,head)) ,@(jb-where inject head "bjodat->"))
      (defun (,dec v) (bjodat-> v))

      ;; And the one-pass pair, which reads the text itself rather than a
      ;; map someone else built. Same record, same messages; no tree.
      (: ,rdr (-> ,(inject 'Reader) (Result string ,head)) ,@(jb-where inject head "bjodat->"))
      (defun (,rdr rd) ,one-pass)
      (: ,prs (-> string (Result string ,head)) ,@(jb-where inject head "bjodat->"))
      ;; The reader is passed by *name*, not wrapped in a `(fun (rd) ...)`.
      ;; A lambda body is a member of its own and is sealed to the sync copy
      ;; of whatever it calls, so the wrapped form made the suspending copy
      ;; of this function park the thread — which the blocking lint said in
      ;; as many words. A bare name carries an effect variable instead, and
      ;; `parse-one-pass` declares its parameter `-?->`, so the caller's
      ;; colour picks the copy.
      (defun (,prs text) (parse-one-pass text ,rdr))))

(def/macro (def/bjodat-type form inject compare)
  (syntax-match form
    ((_ typedef)
     (syntax-match typedef
       ((colon head ('Record field ...)) (jb-expand inject colon head field))
       (bad (syntax-error bad "def/bjodat-type takes a record: (def/bjodat-type (: Name (Record (: field type) ...)))"))))
    (bad (syntax-error bad "def/bjodat-type takes exactly one record declaration."))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; text/csv-codec.bjo — records to CSV rows and back.
;;
;; Two traits for a field, a trait for a record, and a macro that writes the
;; record's instance, as (text json-codec) has them for JSON. The readers and
;; writers here are built on (text csv), and an error is its CsvError.
;;
;; A record's columns are found by name in the header row, so the order of
;; the columns does not matter and columns the record does not have are not
;; read. Without a header, the fields are the columns in order.
;;
;; Its reference is its docs, in csv-codec.bjodoc.

(export
  ;; Fields
  ->csv-field csv-field->
  ;; Records
  CsvRecord csv-columns csv-record-> ->csv-record
  CsvColumns CsvColumn
  ;; What generated code calls. Exported because a macro template's names
  ;; resolve against this module's exports.
  csv-columns-of csv-required csv-optional csv-decode-field csv-decode-field/default csv-row-of
  ;; The generator
  def/csv-type
  ;; Reading and writing records
  csv-fold-records csv-for-each-record parse-csv-records
  csv-records->string write-csv-records)

(import (std prelude))
(import (std syntax-match))
(import (text csv))

;; Every export is documented there; the module doc is too.
(:doc #:file "csv-codec.bjodoc")

;; ===========================================================================
;; Fields
;; ===========================================================================

(def/trait (->csv-field %a)
  (: ->csv-field (-> %a string)))

;; The error is what is wrong with the text, without where: the reader adds
;; the line, the row and the column.
(def/trait (csv-field-> %a)
  (: csv-field-> (-> string (Result string %a))))

(impl (->csv-field string) (defun (->csv-field s) s))
(impl (->csv-field int) (defun (->csv-field n) (int->string n)))
(impl (->csv-field long) (defun (->csv-field n) (long->string n)))
(impl (->csv-field double) (defun (->csv-field n) (double->string n)))
(impl (->csv-field decimal) (defun (->csv-field n) (decimal->string n)))
(impl (->csv-field bool) (defun (->csv-field b) (if b "true" "false")))

;; `parse` raises on text that is not a number, as the prelude's string->int
;; does. The text is quoted in the error: "12,5" is not a decimal.
(: (-> string string (-> string %a) (Result string %a)))
(defun (parsed s what parse)
  (result-map-err (fun (_) (str (written s) " is not " what))
                  (try (parse s) #:catch (System.FormatException System.OverflowException))))

(impl (csv-field-> string) (defun (csv-field-> s) (Ok s)))
(impl (csv-field-> int) (defun (csv-field-> s) (parsed s "an int" string->int)))
(impl (csv-field-> long) (defun (csv-field-> s) (parsed s "a long" string->long)))
(impl (csv-field-> double) (defun (csv-field-> s) (parsed s "a number" string->double)))
(impl (csv-field-> decimal) (defun (csv-field-> s) (parsed s "a decimal" string->decimal)))

;; true or false, in any case, with space around it or not.
(impl (csv-field-> bool)
  (defun (csv-field-> s)
    (def t (string-downcase (string-trim s)))
    (cond
      ((= t "true") (Ok #t))
      ((= t "false") (Ok #f))
      (else (Err (str (written s) " is not true or false"))))))

;; An empty field is None, and is written for None.
(impl (->csv-field (Option %a))
  (where (->csv-field %a))
  (defun (->csv-field o)
    (match o
      ((Some x) (->csv-field x))
      (None ""))))

(impl (csv-field-> (Option %a))
  (where (csv-field-> %a))
  (defun (csv-field-> s)
    (if (string-empty? s)
        (Ok None)
        (match (csv-field-> s)
          ((Ok v) (Ok (Some v)))
          ((Err e) (Err e))))))

;; ===========================================================================
;; Records
;; ===========================================================================

;; A column a record reads: its name in the header, and whether the header
;; has to have it.
(type (: CsvColumn (Record (: name string) (: required bool))))

;; A record's columns, one for each field, in the order of the fields. `%a`
;; is the record: it says whose columns they are, and nothing else.
(type (: (CsvColumns %a) (Newtype (IArray CsvColumn))))

;; The error of csv-record-> is the field it is about, counted from 0 in the
;; record's order, and what is wrong.
(def/trait (CsvRecord %a)
  (: csv-columns (-> (CsvColumns %a)))
  ;; `at` holds, for each field, where its column is in the row: -1 for a
  ;; column the header does not have.
  (: csv-record-> (-> (IArray int) (IArray string) (Result (Tuple int string) %a)))
  (: ->csv-record (-> %a (IArray string))))

;; ---------------------------------------------------------------------------
;; What generated code calls
;; ---------------------------------------------------------------------------

(: (-> string CsvColumn))
(defun (csv-required name) (CsvColumn (name name) (required #t)))

(: (-> string CsvColumn))
(defun (csv-optional name) (CsvColumn (name name) (required #f)))

(: (-> (List CsvColumn) (CsvColumns %a)))
(defun (csv-columns-of cols) (CsvColumns (list->iarray cols)))

(: (-> (List string) (IArray string)))
(defun (csv-row-of fields) (list->iarray fields))

;; Field `k`, whose column is at `pos` in the row.
(: (-> (IArray int) int (IArray string) (Result (Tuple int string) %a)) (where (csv-field-> %a)))
(defun (csv-decode-field at k row)
  (def pos (iarray-ref at k))
  (if (>= pos (iarray-length row))
      (Err (Tuple k "the row ends before this column"))
      (result-map-err (fun (e) (Tuple k e)) (csv-field-> (iarray-ref row pos)))))

;; The same, with `fallback` when the header has no such column, or the row
;; ends before it, or the field is empty.
(: (-> (IArray int) int (IArray string) %a (Result (Tuple int string) %a)) (where (csv-field-> %a)))
(defun (csv-decode-field/default at k row fallback)
  (def pos (iarray-ref at k))
  (cond
    ((or (< pos 0) (>= pos (iarray-length row))) (Ok fallback))
    (:def s (iarray-ref row pos))
    ((string-empty? s) (Ok fallback))
    (else
     (result-map-err (fun (e) (Tuple k e)) (csv-field-> s)))))

;; ===========================================================================
;; Reading
;; ===========================================================================

;; Where each column is in the row without a header: the fields in order.
(: (-> (IArray CsvColumn) (IArray int)))
(defun (in-order cols)
  (list->iarray (loop (:for i (range 0 (iarray-length cols))) (:acc (listing i)))))

;; Where each column is in the row: the first column in the header with its
;; name, or -1 for one the header need not have.
(: (-> (IArray CsvColumn) (IArray string) (Result string (IArray int))))
(defun (layout cols names)
  (def found (transientmap-empty))
  (loop (:for (Tuple i name) (enumerate names))
        (:do (unless (transientmap-contains? found name)
               (transientmap-set! found name i))))
  (let go ((k 0) (out (listbuilder-empty)))
    (cond
      ((= k (iarray-length cols)) (Ok (list->iarray (listbuilder->list out))))
      (:def col (iarray-ref cols k))
      (:match (Some i) (transientmap-try-ref found (record-ref col name))
       (listbuilder-add! out i)
       (go (+ k 1) out))
      ((record-ref col required)
       (Err (str "the header has no column " (written (record-ref col name)))))
      (else
       (listbuilder-add! out -1)
       (go (+ k 1) out)))))

;; A field that does not decode, as a CsvError: at the line its row starts
;; on, in the field the column is in, and named by the column.
(: (-> CsvReader (IArray CsvColumn) (IArray int) int string CsvError))
(defun (field-error rd cols at k message)
  (CsvError (message (str (record-ref (iarray-ref cols k) name) ": " message))
            (line (csv-row-line rd))
            (row (csv-row-number rd))
            (field (+ (iarray-ref at k) 1))))

;; With a header, the first row is it, and an input with no rows has no
;; records. An error in the header is about all of it, so its field is 0.
(: (-> CsvReader bool %seed (-> %a %seed %seed) (Result CsvError %seed)) (where (CsvRecord %a)))
(defun (fold-records rd header? seed f)
  (: columns (CsvColumns %a))
  (def columns (csv-columns))
  (def (CsvColumns cols) columns)
  (def placed
    (cond
      ((not header?) (Ok (Some (in-order cols))))
      (else
       (match (csv-next! rd)
         ((Err e) (Err e))
         ((Ok None) (Ok None))
         ((Ok (Some names))
          (match (layout cols names)
            ((Ok at) (Ok (Some at)))
            ((Err message)
             (Err (CsvError (message message) (line (csv-row-line rd)) (row 1) (field 0))))))))))
  (match placed
    ((Err e) (Err e))
    ((Ok None) (Ok seed))
    ((Ok (Some at))
     (let loop ((seed seed))
       (match (csv-next! rd)
         ((Err e) (Err e))
         ((Ok None) (Ok seed))
         ((Ok (Some row))
          (match (csv-record-> at row)
            ((Ok v) (loop (f v seed)))
            ((Err (Tuple k message)) (Err (field-error rd cols at k message))))))))))

;; Each record and the seed so far give the next seed. The records before an
;; error have been folded when the error is answered.
(: (-> %s %seed (-> %a %seed %seed)
       (#:header? bool) (#:separator char) (#:quote-char (Option char)) (#:strict? bool)
       (Result CsvError %seed))
   (where (CsvSource %s) (CsvRecord %a)))
(defun (csv-fold-records source seed f
                         #:header? #t #:separator #\, #:quote-char (Some #\") #:strict? #t)
  (call-with-csv-reader source
                        (fun (rd) (fold-records rd header? seed f))
                        #:separator separator #:quote-char quote-char #:strict? strict?))

(: (-> %s (-> %a void)
       (#:header? bool) (#:separator char) (#:quote-char (Option char)) (#:strict? bool)
       (Result CsvError Unit))
   (where (CsvSource %s) (CsvRecord %a)))
(defun (csv-for-each-record source f
                            #:header? #t #:separator #\, #:quote-char (Some #\") #:strict? #t)
  (call-with-csv-reader source
                        (fun (rd) (fold-records rd header? unit (fun (v u) (f v) u)))
                        #:separator separator #:quote-char quote-char #:strict? strict?))

(: (-> %s
       (#:header? bool) (#:separator char) (#:quote-char (Option char)) (#:strict? bool)
       (Result CsvError (Vec %a)))
   (where (CsvSource %s) (CsvRecord %a)))
(defun (parse-csv-records source
                          #:header? #t #:separator #\, #:quote-char (Some #\") #:strict? #t)
  (def out (transientvec-empty))
  (match (call-with-csv-reader source
                               (fun (rd) (fold-records rd header? unit (fun (v u) (transientvec-add! out v) u)))
                               #:separator separator #:quote-char quote-char #:strict? strict?)
    ((Ok _) (Ok (transientvec->vec out)))
    ((Err e) (Err e))))

;; ===========================================================================
;; Writing
;; ===========================================================================

;; The column names, in the order of the fields.
(: (-> (CsvColumns %a) (IArray string)))
(defun (column-names columns)
  (def (CsvColumns cols) columns)
  (iarray-map (fun (c) (record-ref c name)) cols))

;; The header, then a row for each record, each ended by a line end.
(: (-> %rs
       (#:header? bool) (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool)
       (#:crlf? bool)
       string)
   (where (CsvRecord %a) (Iterable %rs #:elem %a)))
(defun (csv-records->string records
                            #:header? #t #:separator #\, #:quote-char (Some #\") #:quote-all? #f
                            #:crlf? #f)
  (: columns (CsvColumns %a))
  (def columns (csv-columns))
  (def head (if header?
                (csv->string (list (column-names columns))
                             #:separator separator #:quote-char quote-char #:quote-all? quote-all?
                             #:crlf? crlf?)
                ""))
  (str head
       (csv->string (map (fun (r) (->csv-record r)) records)
                    #:separator separator #:quote-char quote-char #:quote-all? quote-all?
                    #:crlf? crlf?)))

(: (-> OutputPort %rs
       (#:header? bool) (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool)
       (#:crlf? bool)
       void)
   (where (CsvRecord %a) (Iterable %rs #:elem %a)))
(defun (write-csv-records port records
                          #:header? #t #:separator #\, #:quote-char (Some #\") #:quote-all? #f
                          #:crlf? #f)
  (: columns (CsvColumns %a))
  (def columns (csv-columns))
  (when header?
    (write-csv-row port (column-names columns)
                   #:separator separator #:quote-char quote-char #:quote-all? quote-all?
                   #:crlf? crlf?))
  (write-csv port (map (fun (r) (->csv-record r)) records)
             #:separator separator #:quote-char quote-char #:quote-all? quote-all?
             #:crlf? crlf?))

;; ===========================================================================
;; The generator
;; ===========================================================================
;; A runtime macro, as def/json-type is. Field types are never inspected:
;; how a field reads and writes is settled later by instance resolution, and
;; only the markers are read here.

(: (-> string Syntax))
(defun (ct-sym name) (SSym (string->symbol name)))

(: (-> Syntax Syntax))
(defun (ct-str s) (SStr (syntax->string s)))

(: (-> int Syntax))
(defun (ct-int n) (SInt (int->string n)))

;; `(Box %a)` names the type `Box`; a bare `Point` names itself.
(: (-> Syntax Syntax))
(defun (ct-head-name head)
  (match head
    ((SList parts) (list-head parts))
    (_ head)))

(: (-> Syntax (List Syntax)))
(defun (ct-head-vars head)
  (match head
    ((SList parts) (list-tail parts))
    (_ Nil)))

;; Each type parameter has to read and write as a field.
(: (-> (-> Symbol Syntax) Syntax (List Syntax)))
(defun (ct-where inject head)
  (def vars (ct-head-vars head))
  (if (list-empty? vars)
      Nil
      (list #'(,(inject 'where)
               ,@(list-append
                   (list-map (fun (v) #'(,(inject (string->symbol "csv-field->")) ,v)) vars)
                   (list-map (fun (v) #'(,(inject (string->symbol "->csv-field")) ,v)) vars))))))

;; --- fields ----------------------------------------------------------------

(: (-> Syntax string bool))
(defun (ct-marker=? m name) (= (syntax->string m) name))

;; The column's name: `#:csv "Name"`, or the field's own.
(: (-> Syntax (List Syntax) Syntax))
(defun (ct-column name markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) (ct-str name))
      ((ct-marker=? (list-head ms) "#:csv") (list-head (list-tail ms)))
      (else (go (list-tail ms))))))

;; The fallback expression, or None if the field is required. `#:optional` is
;; `#:default None`.
(: (-> (List Syntax) (Option Syntax)))
(defun (ct-fallback markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) None)
      (else
       (def m (list-head ms))
       (cond
         ((ct-marker=? m "#:optional") (Some (ct-sym "None")))
         ((ct-marker=? m "#:default") (Some (list-head (list-tail ms))))
         (else (go (list-tail ms))))))))

;; The markers the record type keeps: all but #:csv, #:default and #:optional.
(: (-> (List Syntax) (List Syntax)))
(defun (ct-keep markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) Nil)
      (else
       (def m (list-head ms))
       (cond
         ((ct-marker=? m "#:default") (go (list-tail (list-tail ms))))
         ((ct-marker=? m "#:csv") (go (list-tail (list-tail ms))))
         ((ct-marker=? m "#:optional") (go (list-tail ms)))
         (else (Cons m (go (list-tail ms)))))))))

(: string)
(def ct-shape
  "a field is written (: name type), then any of #:mutable, #:optional, #:default <expr>, #:csv \"column\".")

;; `syntax-error` has the type Syntax, so this is how a list is answered.
(: (-> Syntax string (List Syntax)))
(defun (ct-fail-list form message)
  (ignore (syntax-error form message))
  Nil)

(: (-> Syntax Syntax))
(defun (ct-field-name f)
  (syntax-match f
    ((_ name _ marker ...) name)
    (bad (syntax-error bad ct-shape))))

(: (-> Syntax (List Syntax)))
(defun (ct-field-markers f)
  (syntax-match f
    ((_ _ _ marker ...) marker)
    (bad (ct-fail-list bad ct-shape))))

(: (-> Syntax Syntax))
(defun (ct-strip f)
  (syntax-match f
    ((colon name ty marker ...) #'(,colon ,name ,ty ,@(ct-keep marker)))
    (bad (syntax-error bad ct-shape))))

;; --- columns, encode, decode ------------------------------------------------

;; A column is required unless the field has a fallback.
(: (-> (List Syntax) Syntax))
(defun (ct-columns fields)
  #'(csv-columns-of
      (list ,@(list-map (fun (f)
                          (def column (ct-column (ct-field-name f) (ct-field-markers f)))
                          (match (ct-fallback (ct-field-markers f))
                            (None #'(csv-required ,column))
                            ((Some _) #'(csv-optional ,column))))
                        fields))))

(: (-> (List Syntax) Syntax))
(defun (ct-encode fields)
  #'(csv-row-of
      (list ,@(list-map (fun (f) #'(->csv-field (record-ref v ,(ct-field-name f)))) fields))))

;; One `match` per field wrapping the next, with the constructor innermost.
(: (-> (List Syntax) int Syntax Syntax))
(defun (ct-chain fields i ctor)
  (cond
    ((list-empty? fields) ctor)
    (else
     (def f (list-head fields))
     (def v (ct-sym (str "__ct-v" (int->string i))))
     (def e (ct-sym (str "__ct-e" (int->string i))))
     (def get (match (ct-fallback (ct-field-markers f))
                (None #'(csv-decode-field at ,(ct-int i) row))
                ((Some d) #'(csv-decode-field/default at ,(ct-int i) row ,d))))
     #'(match ,get
         ((Err ,e) (Err ,e))
         ((Ok ,v) ,(ct-chain (list-tail fields) (+ i 1) ctor))))))

(: (-> Syntax (List Syntax) Syntax))
(defun (ct-ctor head fields)
  #'(Ok (,(ct-head-name head)
         ,@(let go ((fs fields) (i 0))
             (if (list-empty? fs)
                 Nil
                 (Cons #'(,(ct-field-name (list-head fs)) ,(ct-sym (str "__ct-v" (int->string i))))
                       (go (list-tail fs) (+ i 1))))))))

;; --- the form ---------------------------------------------------------------

(: (-> (-> Symbol Syntax) Syntax Syntax (List Syntax) Syntax))
(defun (ct-expand inject colon head fields)
  #'(begin
      (type (,colon ,head (Record ,@(list-map ct-strip fields))))

      (impl (CsvRecord ,head)
        ,@(ct-where inject head)
        (defun (csv-columns) ,(ct-columns fields))
        (defun (csv-record-> at row) ,(ct-chain fields 0 (ct-ctor head fields)))
        (defun (->csv-record v) ,(ct-encode fields)))))

(def/macro (def/csv-type form inject compare)
  (syntax-match form
    ((_ typedef)
     (syntax-match typedef
       ((colon head ('Record field ...)) (ct-expand inject colon head field))
       (bad (syntax-error bad "def/csv-type takes a record: (def/csv-type (: Name (Record (: field type) ...)))"))))
    (bad (syntax-error bad "def/csv-type takes exactly one record declaration."))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; text/csv.bjo — the module (text csv): RFC 4180, read and written.
;;
;; Three layers for reading, each built on the one below:
;;
;;   1. csv-next!: a pull reader, one row at a time, as OCaml's Csv.next
;;   2. csv-fold and csv-for-each: every row, never more than one held
;;   3. parse-csv: the whole document, as a vec of rows
;;
;; The reader works on bytes, as (text json) does: a chunk of the input in
;; `bytes`, the next byte at `next`, and the end of the chunk at `limit`. The
;; separator, the quote, CR and LF are all ASCII, and in UTF-8 an ASCII byte
;; is never part of a longer character. So the scan looks at bytes only, and a
;; field is checked to be UTF-8 once, when it becomes a string.
;;
;; A field is always whole in one chunk. When the scan reaches the end of the
;; chunk, the bytes from the start of the field are kept, the next chunk is
;; added after them, and the scan continues where it stopped.
;;
;; Its reference is its docs, in csv.bjodoc.
;;
;; A row is an IArray: the reader keeps one array for the fields of the row
;; it is reading, and copies them out once, into the row's own array.
;;
;; Each chunk is checked to be UTF-8 once, as it arrives, by making a
;; Utf8Chunk of it. A field is then taken from the chunk and not checked
;; again; the chunk itself makes sure that its offsets are between
;; characters.
;;
;; Open:
;; - The scan is a byte loop. A SIMD search for the four special bytes, as
;;   (text json) skips white space, would be faster for long fields.
;; - A header row, and rows decoded to records by column name, are still to
;;   come.

(export
  ;; Errors
  CsvError
  ;; Sources
  CsvInput CsvSource csv-input CsvFile csv-file
  ;; Pulling rows
  CsvReader call-with-csv-reader csv-next! csv-row-line csv-row-number
  ;; Folds
  csv-fold csv-for-each
  ;; The whole document
  parse-csv
  ;; Rows of bytes, cut into pieces
  csv-row-end csv-cut-rows
  ;; Writing
  csv-row->string csv->string write-csv-row write-csv)

(import (std prelude))
(import (only (std ports) read-bytes))
(import (std maths))
(import (std simd))

(import/class
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

;; Every export is documented there; the module doc is too.
(:doc #:file "csv.bjodoc")

;; An ArgumentException, which is what a programming error raises here.
(: (-> string %a))
(defun (argument-error message)
  (raise (cast Exception (ArgumentException. (str "(text csv): " message)))))

;; ===========================================================================
;; Errors
;; ===========================================================================

;; `row` counts the rows read, from 1, and blank lines are not rows. `field`
;; counts from 1 in that row.
(type (: CsvError (Record (: message string) (: line int) (: row int) (: field int))))

(impl (Displayed CsvError)
  (defun (displayed e)
    (str "line " (int->string (record-ref e line))
         ", row " (int->string (record-ref e row))
         ", field " (int->string (record-ref e field))
         ": " (record-ref e message)))
  (defun (display-into e b) (stringbuilder-add-string! b (displayed e))))

;; ===========================================================================
;; Options
;; ===========================================================================

;; The end of the input, as a byte the reader looks at.
(: int)
(def spent -1)

;; The quote byte when there is no quoting. It is not `spent`, so that the end
;; of the input does not start a quoted field.
(: int)
(def no-quote -2)

(type (: Options (Record (: sep-code int) (: quote-code int) (: strict bool))))

(: (-> char (Option char) bool Options))
(defun (options separator quote-char strict)
  (def sep (char->int separator))
  (def q (match quote-char ((Some c) (char->int c)) (None no-quote)))
  (when (or (>= sep 128) (= sep 10) (= sep 13))
    (argument-error "#:separator has to be an ASCII character that does not end a line"))
  (when (or (>= q 128) (= q 10) (= q 13) (= q sep))
    (argument-error
     "#:quote-char has to be an ASCII character that does not end a line and is not the separator"))
  (Options (sep-code sep) (quote-code q) (strict strict)))

;; ===========================================================================
;; The reader
;; ===========================================================================

;; How much a read from the port asks for: 16 KiB first, and each read after
;; it twice the one before, up to 64 KiB. Each chunk is copied into a
;; Utf8Chunk, and an array over 85 000 bytes goes on .NET's large object
;; heap, which is slow to allocate and freed only by a full collection.
(: int)
(def read-first 16384)

(: int)
(def read-most 65536)

;; `chunk` holds a checked copy of `bytes`, from 0 to `limit`; see
;; check-chunk!.
;; `line` is the line the reader is on, from 1. `rows` is the number of rows
;; read. `started` is set once the byte order mark check is done. After an
;; error, `failed` holds it, and every later read gives it again.
(type (: CsvReader #:opaque (Record (: port (Option InputPort) #:mutable)
                                    (: ask int #:mutable)
                                    (: bytes (Array byte) #:mutable)
                                    (: next int #:mutable)
                                    (: limit int #:mutable)
                                    (: chunk Utf8Chunk #:mutable)
                                    (: line int #:mutable)
                                    ;; The line the row being read, or last read, starts on.
                                    (: row-line int #:mutable)
                                    (: rows int #:mutable)
                                    (: options Options)
                                    ;; One buffer for every field with a doubled quote.
                                    (: scratch StringBuilder)
                                    ;; The fields of the row being read, and how many.
                                    (: fields (Array string) #:mutable)
                                    (: count int #:mutable)
                                    (: started bool #:mutable)
                                    (: failed (Option CsvError) #:mutable))))

(: (Array byte))
(def no-bytes (make-array 0))

(: Utf8Chunk)
(def no-chunk (bytes->utf8-chunk no-bytes))

;; A reader over `bytes`, and then whatever `port` has after them.
(: (-> (Option InputPort) (Array byte) Options CsvReader))
(defun (make-reader port bytes opts)
  (def rd (CsvReader (port port)
                     (ask read-first)
                     (bytes bytes)
                     (next 0)
                     (limit (array-length bytes))
                     (chunk no-chunk)
                     (line 1)
                     (row-line 1)
                     (rows 0)
                     (options opts)
                     (scratch (stringbuilder-empty))
                     (fields (make-array 16))
                     (count 0)
                     (started #f)
                     (failed None)))
  (check-chunk! rd)
  rd)

(: (-> CsvReader int int string CsvError))
(defun (error-at rd field line message)
  (CsvError (message message) (line line) (row (+ (record-ref rd rows) 1)) (field field)))

(: (-> CsvReader int string CsvError))
(defun (csv-error rd field message)
  (error-at rd field (record-ref rd line) message))

;; ---------------------------------------------------------------------------
;; Chunks
;; ---------------------------------------------------------------------------

;; `count` bytes of `old` from `from`, then `more`: what is left of one chunk
;; and the next.
(: (-> (Array byte) int int (Array byte) (Array byte)))
(defun (join-bytes old from count more)
  (if (= count 0)
      more
      (let ((joined (make-array (+ count (array-length more)))))
        (array-copy! old from joined 0 count)
        (array-copy! more 0 joined count (array-length more))
        joined)))

;; A Utf8Chunk of the bytes the reader has now, so that its offsets are the
;; reader's. A field before its valid end is taken from it without being
;; checked again. One after it is checked by itself, so that the error can
;; say which field it is.
(: (-> CsvReader void))
(defun (check-chunk! rd)
  (set! rd .chunk (bytes->utf8-chunk (get rd .bytes) #:end (get rd .limit))))

;; Whether there are at least `n` bytes from `next` on. Reads from the port
;; until there are or the port ends. The bytes from `next` on are kept, and
;; `next` becomes 0.
(: (-> CsvReader int bool))
(defun (ensure! rd n)
  (let go ()
    (def have (- (get rd .limit) (get rd .next)))
    (cond
      ((>= have n) #t)
      (else
       (def (Some p) (get rd .port) :leave-with #f)
       (def ask (get rd .ask))
       (match (read-bytes p (if (> n ask) n ask))
         (None
          (set! rd .port None)
          #f)
         ((Some more)
          (record-set! rd
                       (bytes (join-bytes (get rd .bytes) (get rd .next) have more))
                       (next 0)
                       (limit (+ have (length more)))
                       (ask (if (< ask read-most) (* ask 2) read-most)))
          (check-chunk! rd)
          (go)))))))

;; At least one byte more than the chunk has now, with the bytes from `next`
;; on kept. #f when the input has no more. A scan at index `i` continues at
;; `(+ next (- i start))` afterwards, where `start` was `next` before the call.
(: (-> CsvReader bool))
(defun (more! rd)
  (ensure! rd (+ (- (get rd .limit) (get rd .next)) 1)))

;; The byte at `next`, or `spent` at the end of the input.
(: (-> CsvReader int))
(defun (held-code rd)
  (def i (get rd .next))
  (cond
    ((< i (get rd .limit)) (cast int (get (get rd .bytes) i)))
    ((ensure! rd 1) (cast int (get (get rd .bytes) (get rd .next))))
    (else spent)))

(: (-> CsvReader void))
(defun (step! rd)
  (set! rd .next (+ (get rd .next) 1)))

;; Past one line end: LF, CR LF, or CR alone.
(: (-> CsvReader void))
(defun (end-line! rd)
  (def c (held-code rd))
  (step! rd)
  (when (and (= c 13) (= (held-code rd) 10))
    (step! rd))
  (record-set! rd (line (+ (record-ref rd line) 1))))

;; Whether the reader is at a byte order mark: EF BB BF.
(: (-> CsvReader bool))
(defun (at-mark? rd)
  (and (ensure! rd 3)
       (let ((b (record-ref rd bytes)) (i (record-ref rd next)))
         (and (= (cast int (array-ref b i)) 239)
              (= (cast int (array-ref b (+ i 1))) 187)
              (= (cast int (array-ref b (+ i 2))) 191)))))

;; The byte at `next`, for an error message.
(: (-> CsvReader string))
(defun (found rd)
  (def c (held-code rd))
  (if (and (>= c 32) (< c 127))
      (str "'" (list->string (list (int->char c))) "'")
      (str "the byte " (int->string c))))

;; ---------------------------------------------------------------------------
;; Fields
;; ---------------------------------------------------------------------------

(: string)
(def not-utf-8 "the field has bytes that are not UTF-8")

;; Whether the bytes from `start` to `end` of the chunk are UTF-8.
(: (-> CsvReader (Array byte) int int bool))
(defun (utf-8? rd b start end)
  (or (<= end (utf8-chunk-valid-end (record-ref rd chunk))) (bytes-utf-8? b start end)))

;; The bytes from `start` to `end` as a string, or None when they are not
;; UTF-8. A field starts and ends next to an ASCII byte or at the end of the
;; chunk, so both offsets are between characters.
(: (-> CsvReader (Array byte) int int (Option string)))
(defun (text-of rd b start end)
  (def c (record-ref rd chunk))
  (cond
    ((= start end) (Some ""))
    ((<= end (utf8-chunk-valid-end c)) (Some (utf8-chunk-substring c start end)))
    ((bytes-utf-8? b start end) (Some (bytes->string b #:start start #:end end)))
    (else None)))

(: (-> CsvReader int (Array byte) int int (Result CsvError string)))
(defun (field-string rd field b start end)
  (match (text-of rd b start end)
    ((Some s) (Ok s))
    (None (Err (csv-error rd field not-utf-8)))))

;; The first index from `from` that holds the separator, the quote, CR or LF,
;; or `limit` if there is none.
;;
;; 32 or 16 bytes at a time, as (text json) skips white space: each block is
;; compared with the four bytes at once, and the first match is the lowest set
;; bit of the mask. The bytes after the last whole block are looked at one by
;; one. Without a quote, `q` is not a byte, and the separator takes its lane.
(: (-> (Array byte) int int int int int))
(defun (plain-end b from limit sep q)
  (def qb (if (< q 0) sep q))
  (if simd256-accelerated?
      (plain-end/256 b from limit sep qb)
      (plain-end/128 b from limit sep qb)))

(: (-> (Array byte) int int int int int))
(defun (plain-end/256 b from limit sep q)
  (def vs (simd256-splat (cast byte sep)))
  (def vq (simd256-splat (cast byte q)))
  (def lf (simd256-splat 10uy))
  (def cr (simd256-splat 13uy))
  (let go ((i from))
    (cond
      ((> (+ i 32) limit) (plain-end/bytes b i limit sep q))
      (:def v (simd256-load b i))
      (:def m (simd-mask-bits (simd-or (simd-or (simd-eq v vs) (simd-eq v vq))
                                       (simd-or (simd-eq v lf) (simd-eq v cr)))))
      ((= m 0) (go (+ i 32)))
      (else (+ i (trailing-zeros m))))))

(: (-> (Array byte) int int int int int))
(defun (plain-end/128 b from limit sep q)
  (def vs (simd128-splat (cast byte sep)))
  (def vq (simd128-splat (cast byte q)))
  (def lf (simd128-splat 10uy))
  (def cr (simd128-splat 13uy))
  (let go ((i from))
    (cond
      ((> (+ i 16) limit) (plain-end/bytes b i limit sep q))
      (:def v (simd128-load b i))
      (:def m (simd-mask-bits (simd-or (simd-or (simd-eq v vs) (simd-eq v vq))
                                       (simd-or (simd-eq v lf) (simd-eq v cr)))))
      ((= m 0) (go (+ i 16)))
      (else (+ i (trailing-zeros m))))))

(: (-> (Array byte) int int int int int))
(defun (plain-end/bytes b from limit sep q)
  (let go ((i from))
    (cond
      ((>= i limit) i)
      (:def c (cast int (array-ref b i)))
      ((or (= c sep) (= c 10) (= c 13) (= c q)) i)
      (else (go (+ i 1))))))

;; A field that does not start with the quote. It ends before the separator,
;; a line end or the end of the input. When strict, a quote in it is an error;
;; else the quote is part of the field.
(: (-> CsvReader int (Result CsvError string)))
(defun (read-plain rd field)
  (def* (opts (.options rd))
        (sep  (.sep-code opts))
        (q    (.quote-code opts)))
  (let go ((from (get rd .next)))
    (def* (b     (.bytes rd))
          (start (.next rd))
          (limit (.limit rd))
          (i     (plain-end b from limit sep q)))
    (cond
      ((and (< i limit) (= (cast int (array-ref b i)) q))
       (if (get opts .strict)
           (Err (csv-error rd field "a quote inside a field that does not start with one"))
           (go (+ i 1))))
      ((< i limit)
       (set! rd .next i)
       (field-string rd field b start i))
      ((more! rd) (go (+ (get rd .next) (- i start))))
      (else
       (set! rd .next i)
       (field-string rd field b start i)))))

;; The first index from `from` that holds the quote, CR or LF, or `limit` if
;; there is none. A block at a time, as plain-end; `q` is always a byte here,
;; since a field is quoted only when there is a quote.
(: (-> (Array byte) int int int int))
(defun (quoted-end b from limit q)
  (if simd256-accelerated?
      (quoted-end/256 b from limit q)
      (quoted-end/128 b from limit q)))

(: (-> (Array byte) int int int int))
(defun (quoted-end/256 b from limit q)
  (def vq (simd256-splat (cast byte q)))
  (def lf (simd256-splat 10uy))
  (def cr (simd256-splat 13uy))
  (let go ((i from))
    (cond
      ((> (+ i 32) limit) (quoted-end/bytes b i limit q))
      (:def v (simd256-load b i))
      (:def m (simd-mask-bits (simd-or (simd-eq v vq) (simd-or (simd-eq v lf) (simd-eq v cr)))))
      ((= m 0) (go (+ i 32)))
      (else (+ i (trailing-zeros m))))))

(: (-> (Array byte) int int int int))
(defun (quoted-end/128 b from limit q)
  (def vq (simd128-splat (cast byte q)))
  (def lf (simd128-splat 10uy))
  (def cr (simd128-splat 13uy))
  (let go ((i from))
    (cond
      ((> (+ i 16) limit) (quoted-end/bytes b i limit q))
      (:def v (simd128-load b i))
      (:def m (simd-mask-bits (simd-or (simd-eq v vq) (simd-or (simd-eq v lf) (simd-eq v cr)))))
      ((= m 0) (go (+ i 16)))
      (else (+ i (trailing-zeros m))))))

(: (-> (Array byte) int int int int))
(defun (quoted-end/bytes b from limit q)
  (let go ((i from))
    (cond
      ((>= i limit) i)
      (:def c (cast int (array-ref b i)))
      ((or (= c q) (= c 10) (= c 13)) i)
      (else (go (+ i 1))))))

;; The text between the quotes, from `start` to `end`. With `escaped`, each
;; quote in it is the first of a doubled pair, and the second is dropped.
(: (-> CsvReader int (Array byte) int int bool (Result CsvError string)))
(defun (quoted-string rd field b start end escaped)
  (def q (get rd .options .quote-code))
  (cond
    ((not escaped) (field-string rd field b start end))
    ((not (utf-8? rd b start end)) (Err (csv-error rd field not-utf-8)))
    (else
     (def sb (record-ref rd scratch))
     (stringbuilder-clear! sb)
     (let go ((from start))
       (def i (quoted-end b from end q))
       (cond
         ;; A line end inside the quotes is part of the text.
         ((and (< i end) (not (= (cast int (array-ref b i)) q)))
          (stringbuilder-add-bytes! sb b from (+ i 1))
          (go (+ i 1)))
         ((< i end)
          (stringbuilder-add-bytes! sb b from (+ i 1))
          (go (+ i 2)))
         (else
          (stringbuilder-add-bytes! sb b from end)
          (Ok (stringbuilder->string sb))))))))

;; After the closing quote: the separator, a line end or the end of the input.
;; When not strict, anything else up to one of those is added to the field.
(: (-> CsvReader int string (Result CsvError string)))
(defun (after-quote rd field content)
  (def opts (record-ref rd options))
  (def c (held-code rd))
  (cond
    ((or (= c (record-ref opts sep-code)) (= c 10) (= c 13) (= c spent)) (Ok content))
    ((record-ref opts strict)
     (Err (csv-error rd field (str "a quoted field ends at its closing quote, and then there is "
                                   (found rd)))))
    (else
     (match (read-plain rd field)
       ((Ok tail) (Ok (str content tail)))
       ((Err e) (Err e))))))

;; A field that starts with the quote, at `next`. `from` is where the search
;; for the closing quote continues, `escaped` whether a doubled quote was
;; passed, and `lines` how many line ends are inside the quotes.
;;
;; A quote or a CR is only known after the byte that follows it is seen, so
;; when that byte is not in the chunk, the chunk is made longer first.
(: (-> CsvReader int (Result CsvError string)))
(defun (read-quoted rd field)
  (def* (opts       (.options rd))
        (q          (.quote-code opts))
        (first-line (.line rd)))
  (let go ((from (+ (.next rd) 1)) (escaped #f) (lines 0))
    (def* (b     (.bytes rd))
          (start (.next rd))
          (limit (.limit rd))
          (i     (quoted-end b from limit q))
          (c     (if (< i limit)
                     (cast int (array-ref b i))
                     spent))
          (after (if (< (+ i 1) limit)
                     (cast int (array-ref b (+ i 1)))
                     spent)))
    (cond
      ((and (or (= c spent) (and (or (= c q) (= c 13)) (= after spent)))
            (more! rd))
       (go (+ (.next rd) (- i start)) escaped lines))
      ((= c 10) (go (+ i 1) escaped (+ lines 1)))
      ((= c 13) (go (if (= after 10) (+ i 2) (+ i 1)) escaped (+ lines 1)))
      ((and (= c q) (= after q)) (go (+ i 2) #t lines))
      ((= c q)
       (record-set! rd (next (+ i 1)) (line (+ (.line rd) lines)))
       (match (quoted-string rd field b (+ start 1) i escaped)
         ((Ok content) (after-quote rd field content))
         ((Err e) (Err e))))
      ;; The input ended inside the quotes.
      ((.strict opts)
       (Err (error-at rd field first-line "the input ends inside a quoted field that starts on this line")))
      (else
       (record-set! rd (next i) (line (+ (.line rd) lines)))
       (quoted-string rd field b (+ start 1) i escaped)))))

(: (-> CsvReader int (Result CsvError string)))
(defun (read-field rd field)
  (if (= (held-code rd) (get rd .options .quote-code))
      (read-quoted rd field)
      (read-plain rd field)))

;; ---------------------------------------------------------------------------
;; Rows
;; ---------------------------------------------------------------------------

;; The next row, or None at the end of the input. Blank lines are skipped. A
;; line end after the last row is not an empty row.
(: (-> CsvReader (Result CsvError (Option (IArray string)))))
(defun (read-row rd)
  (unless (record-ref rd started)
    (record-set! rd (started #t))
    (when (at-mark? rd)
      (record-set! rd (next (+ (record-ref rd next) 3)))))
  (def sep (get rd .options .sep-code))
  (let skip ()
    (def c (held-code rd))
    (cond
      ((= c spent) (Ok None))
      ((or (= c 10) (= c 13))
       (end-line! rd)
       (skip))
      (else
       (record-set! rd (row-line (record-ref rd line)))
       (def q (record-ref (record-ref rd options) quote-code))
       (let go ((field 1))
         (def* (b     (.bytes rd))
               (start (.next rd))
               (limit (.limit rd))
               (i (plain-end b start limit sep q)))
         ;; At the end of the chunk, `c` is the quote, so that read-field
         ;; reads the field and the chunk after it.
         (def c (if (< i limit) (cast int (array-ref b i)) q))
         (cond
           ;; The usual field: no quote, and its end in the chunk. It is read
           ;; here, without read-field.
           ((not (= c q))
            (match (text-of rd b start i)
              (None (Err (csv-error rd field not-utf-8)))
              ((Some s)
               (push-field! rd s)
               (cond
                 ((= c sep)
                  (record-set! rd (next (+ i 1)))
                  (go (+ field 1)))
                 (else
                  (record-set! rd (next i))
                  (end-row! rd))))))
           (else
            (match (read-field rd field)
              ((Err e) (Err e))
              ((Ok s)
               (push-field! rd s)
               ;; A field ends only at the separator, a line end or the end
               ;; of the input.
               (cond
                 ((= (held-code rd) sep)
                  (step! rd)
                  (go (+ field 1)))
                 (else (end-row! rd))))))))))))

;; At the end of a row: past its line end, if it has one, and the row.
(: (-> CsvReader (Result CsvError (Option (IArray string)))))
(defun (end-row! rd)
  (unless (= (held-code rd) spent) (end-line! rd))
  (record-set! rd (rows (+ (record-ref rd rows) 1)))
  (Ok (Some (take-row! rd))))

;; Adds a field to the row being read, growing the array when it is full.
(: (-> CsvReader string void))
(defun (push-field! rd s)
  (def a (.fields rd))
  (def n (.count rd))
  (cond
    ((< n (array-length a)) (array-set! a n s))
    (else
     (def bigger (array-resize a (* n 2)))
     (array-set! bigger n s)
     (record-set! rd (fields bigger))))
  (record-set! rd (count (+ n 1))))

;; The row read, as an IArray of its own: one copy of the fields, and no
;; object but the array. The array is used again for the next.
(: (-> CsvReader (IArray string)))
(defun (take-row! rd)
  (def row (subarray->iarray (record-ref rd fields) 0 (record-ref rd count)))
  (record-set! rd (count 0))
  row)

;; The next row, or None at the end of the input. After an error, the same
;; error again.
(: (-> CsvReader (Result CsvError (Option (IArray string)))))
(defun (csv-next! rd)
  (match (record-ref rd failed)
    ((Some e) (Err e))
    (None
     (match (read-row rd)
       ((Err e)
        (record-set! rd (failed (Some e)))
        (Err e))
       (r r)))))

;; The line the row last read starts on, from 1: where a row's own errors,
;; such as a field that does not decode, are.
(: (-> CsvReader int))
(defun (csv-row-line rd) (record-ref rd row-line))

;; How many rows have been read, the last one included. Blank lines are not
;; rows.
(: (-> CsvReader int))
(defun (csv-row-number rd) (record-ref rd rows))

;; ===========================================================================
;; Sources
;; ===========================================================================

;; Declared before the trait that names it.
(type (: CsvInput #:opaque (Union (: CsvInText string)
                                  (: CsvInPort InputPort)
                                  (: CsvInFile string))))

;; What can be read. A type of your own gets an implementation by handing
;; csv-input something that already has one.
(def/trait (CsvSource %s)
  (: csv-input (-> %s CsvInput)))

;; A path, which the reader opens and closes. A string on its own is CSV.
(type (: CsvFile #:opaque (Record (: path string))))

(: (-> string CsvFile))
(defun (csv-file path) (CsvFile (path path)))

(impl (CsvSource string)
  (defun (csv-input s) (CsvInText s)))

(impl (CsvSource InputPort)
  (defun (csv-input p) (CsvInPort p)))

(impl (CsvSource CsvFile)
  (defun (csv-input f) (CsvInFile (record-ref f path))))

;; A port that was handed over stays open. A file is opened here, through the
;; FS effect, and closed here; a file that does not open raises.
(: (-> CsvInput Options (-> CsvReader %a) %a))
(defun (with-reader input opts proc)
  (match input
    ((CsvInText text) (proc (make-reader None (string->bytes text) opts)))
    ((CsvInPort p) (proc (make-reader (Some p) no-bytes opts)))
    ((CsvInFile path)
     (match (call-with-input-file path (fun (p) (proc (make-reader (Some p) no-bytes opts))))
       ((Ok a) a)
       ((Err e) (raise e))))))

;; The reader is for proc's use, and its file is closed when proc returns.
(: (-> %s (-> CsvReader %a)
       (#:separator char) (#:quote-char (Option char)) (#:strict? bool)
       %a)
   (where (CsvSource %s)))
(defun (call-with-csv-reader source proc #:separator #\, #:quote-char (Some #\") #:strict? #t)
  (with-reader (csv-input source) (options separator quote-char strict?) proc))

;; ===========================================================================
;; Folds
;; ===========================================================================

(: (-> CsvReader %seed (-> (IArray string) %seed %seed) (Result CsvError %seed)))
(defun (fold-rows rd seed f)
  (let loop ((seed seed))
    (match (csv-next! rd)
      ((Err e) (Err e))
      ((Ok None) (Ok seed))
      ((Ok (Some row)) (loop (f row seed))))))

;; Each row and the seed so far give the next seed. The rows before an error
;; have been folded when the error is answered.
(: (-> %s %seed (-> (IArray string) %seed %seed)
       (#:separator char) (#:quote-char (Option char)) (#:strict? bool)
       (Result CsvError %seed))
   (where (CsvSource %s)))
(defun (csv-fold source seed f #:separator #\, #:quote-char (Some #\") #:strict? #t)
  (with-reader (csv-input source) (options separator quote-char strict?)
    (fun (rd) (fold-rows rd seed f))))

(: (-> %s (-> (IArray string) void)
       (#:separator char) (#:quote-char (Option char)) (#:strict? bool)
       (Result CsvError Unit))
   (where (CsvSource %s)))
(defun (csv-for-each source f #:separator #\, #:quote-char (Some #\") #:strict? #t)
  (with-reader (csv-input source) (options separator quote-char strict?)
    (fun (rd) (fold-rows rd unit (fun (row u) (f row) u)))))

;; ===========================================================================
;; The whole document
;; ===========================================================================

(: (-> %s
       (#:separator char) (#:quote-char (Option char)) (#:strict? bool)
       (Result CsvError (Vec (IArray string))))
   (where (CsvSource %s)))
(defun (parse-csv source #:separator #\, #:quote-char (Some #\") #:strict? #t)
  (with-reader (csv-input source) (options separator quote-char strict?)
    (fun (rd)
      (def rows (transientvec-empty))
      (match (fold-rows rd unit (fun (row u) (transientvec-add! rows row) u))
        ((Ok _) (Ok (transientvec->vec rows)))
        ((Err e) (Err e))))))

;; ===========================================================================
;; Rows of bytes, cut into pieces
;; ===========================================================================
;;
;; For a reader that reads the pieces of a large text on several threads. A
;; cut is just after a line feed that is not inside quotes. Whether a line
;; feed is inside quotes is the parity of the quotes before it, since a quote
;; in a quoted field is written as two quotes. So the quotes of each piece
;; are counted with one vectorized call, and only the bytes from each cut's
;; target to the next line feed are looked at one by one.

(import/extern
  (count-byte (: System.MemoryExtensions.Count (-> (RoSpan byte) byte int)))
  (index-of-byte (: System.MemoryExtensions.IndexOf (-> (RoSpan byte) byte int)))
  (index-of-either (: System.MemoryExtensions.IndexOfAny (-> (RoSpan byte) byte byte int))))

;; The quote's byte, or -1 without quoting.
(: (-> (Option char) int))
(defun (quote-byte quote-char)
  (match quote-char
    (None -1)
    ((Some c)
     (def q (char->int c))
     (when (or (>= q 128) (= q 10) (= q 13))
       (argument-error "#:quote-char has to be an ASCII character that does not end a line"))
     q)))

;; The index just after the first line feed from `from` that is not inside
;; quotes, when `from` is inside quotes or not; None when there is none.
(: (-> (Array byte) int int bool (Option int)))
(defun (row-end-from b from q inside)
  (let go ((i from) (inside inside))
    (def rest (array->ro-span b #:from i))
    (def j (if (< q 0) (index-of-byte rest 10uy) (index-of-either rest 10uy (cast byte q))))
    (cond
      ((< j 0) None)
      ((not (= (array-ref b (+ i j)) 10uy)) (go (+ i j 1) (not inside)))
      (inside (go (+ i j 1) inside))
      (else (Some (+ i j 1))))))

;; The number of quotes from `from` to `to`.
(: (-> (Array byte) int int int int))
(defun (quotes-in b from to q)
  (if (< q 0) 0 (count-byte (array->ro-span b #:from from #:to to) (cast byte q))))

;; The index just after the line feed that ends the row starting at `from`, a
;; line feed inside quotes being part of the row; None when no line feed ends
;; it, as for the last row of a text without a final line feed.
(: (-> (Array byte) int (#:quote-char (Option char)) (Option int)))
(defun (csv-row-end bytes from #:quote-char (Some #\"))
  (when (or (< from 0) (> from (array-length bytes)))
    (argument-error "csv-row-end: the index is not in the bytes"))
  (row-end-from bytes from (quote-byte quote-char) #f))

;; The bytes from `#:from`, a row's start, to the end cut into at most
;; `parts` pieces of whole rows of about one size: the index where each
;; piece starts, and the end. No piece is empty. None when the quotes do not
;; close.
(: (-> (Array byte) int (#:from int) (#:quote-char (Option char)) (Option (Array int))))
(defun (csv-cut-rows bytes parts #:from 0 #:quote-char (Some #\"))
  (def n (array-length bytes))
  (when (< parts 1) (argument-error "csv-cut-rows: the number of pieces has to be 1 or more"))
  (when (or (< from 0) (> from n)) (argument-error "csv-cut-rows: #:from is not in the bytes"))
  (def q (quote-byte quote-char))
  ;; Where each piece would start if all were of one size, and the end.
  (def targets
    {arraying (+ from (cast int (quotient (* (cast long (- n from)) (cast long k)) (cast long parts))))
              (:for k (range 0 (+ parts 1)))})
  (def counts {arraying (quotes-in bytes (array-ref targets k) (array-ref targets (+ k 1)) q)
                        (:for k (range 0 parts))})
  (if (odd? (loop (:for c counts) (:acc s (folding 0 (+ s c)))))
      None
      (Some (cuts-at bytes q targets counts from n))))

;; The cuts: from the byte before each target, the end of the row it is in,
;; so that a target just after a line feed is a cut. The quotes before the
;; target are the sum of the counts before it; the byte before the target,
;; when it is a quote, is not before that byte.
(: (-> (Array byte) int (Array int) (Array int) int int (Array int)))
(defun (cuts-at b q targets counts from n)
  (def parts (array-length counts))
  (def out (: (make-array (+ parts 1)) (Array int)))
  (array-set! out 0 from)
  (let go ((k 1) (before (array-ref counts 0)) (found 1))
    (if (= k parts)
        (finish-cuts out found n)
        (match (let* ((target (array-ref targets k))
                      (start (max from (- target 1)))
                      (quote-before? (and (< start target) (>= q 0) (= (cast int (array-ref b start)) q))))
                 (row-end-from b start q (odd? (if quote-before? (- before 1) before))))
          (None (finish-cuts out found n))
          ((Some c)
           (def last (array-ref out (- found 1)))
           (if (and (> c last) (< c n))
               (begin
                 (array-set! out found c)
                 (go (+ k 1) (+ before (array-ref counts k)) (+ found 1)))
               (go (+ k 1) (+ before (array-ref counts k)) found)))))))

;; The `found` cuts and then the end, when the last cut is not the end.
(: (-> (Array int) int int (Array int)))
(defun (finish-cuts out found n)
  (if (< (array-ref out (- found 1)) n)
      (begin
        (array-set! out found n)
        (array-resize out (+ found 1)))
      (array-resize out found)))

;; ===========================================================================
;; The writer
;; ===========================================================================
;;
;; A field gets quotes only when it holds the separator, the quote, CR or LF,
;; or when it is the only field of its row and empty: an empty line is not a
;; row when it is read back, and `""` is.

;; `specials` holds the characters that make a field need quotes.
(type (: Writer (Record (: separator char)
                        (: quote-char (Option char))
                        (: specials string)
                        (: quote-all bool)
                        (: newline string))))

(: (-> char (Option char) bool bool Writer))
(defun (writer separator quote-char quote-all crlf)
  (ignore (options separator quote-char #t))
  (def specials (match quote-char
                  ((Some q) (list->string (list separator q #\return #\newline)))
                  (None (list->string (list separator #\return #\newline)))))
  (Writer (separator separator)
          (quote-char quote-char)
          (specials specials)
          (quote-all quote-all)
          (newline (if crlf "\r\n" "\n"))))

;; `s` between quotes, each quote in it doubled.
(: (-> StringBuilder string char void))
(defun (add-quoted! sb s q)
  (def qs (list->string (list q)))
  (stringbuilder-add! sb q)
  (let go ((from (string-cursor-start s)))
    (match (string-search-any s qs from)
      (None (stringbuilder-add-substring! sb s from (string-cursor-end s)))
      ((Some at)
       (def after (string-cursor-next s at))
       (stringbuilder-add-substring! sb s from after)
       (stringbuilder-add! sb q)
       (go after))))
  (stringbuilder-add! sb q))

(: (-> StringBuilder Writer string void))
(defun (add-field! sb w s)
  (def needs (or (record-ref w quote-all)
                 (some? (string-search-any s (record-ref w specials) (string-cursor-start s)))))
  (cond
    ((not needs) (stringbuilder-add-string! sb s))
    (:match (Some q) (record-ref w quote-char) (add-quoted! sb s q))
    ((string-empty? s) unit)
    (else
     (argument-error
      (str "the field " (written s) " needs quotes, and there is no #:quote-char")))))

;; Without the line end. The fields are walked once, so the row can be any
;; Iterable; a row of one empty field is known only after it.
(: (-> StringBuilder Writer %row void) (where (Iterable %row #:elem string)))
(defun (add-row! sb w row)
  (def mark (stringbuilder-length sb))
  (def n (loop (:for s row)
               (:with first? #t #f)
               (:do (unless first? (stringbuilder-add! sb (record-ref w separator)))
                    (add-field! sb w s))
               (:acc (counting s))))
  (when (and (= n 1) (= (stringbuilder-length sb) mark))
    (match (record-ref w quote-char)
      ((Some q)
       (stringbuilder-add! sb q)
       (stringbuilder-add! sb q))
      (None unit))))

;; One row as CSV text, without a line end.
(: (-> %row (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool) string)
   (where (Iterable %row #:elem string)))
(defun (csv-row->string row #:separator #\, #:quote-char (Some #\") #:quote-all? #f)
  (def sb (stringbuilder-empty))
  (add-row! sb (writer separator quote-char quote-all? #f) row)
  (stringbuilder->string sb))

;; Every row, each ended by a line end.
(: (-> %rows (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool) (#:crlf? bool)
       string)
   (where (Iterable %rows #:elem %row) (Iterable %row #:elem string)))
(defun (csv->string rows #:separator #\, #:quote-char (Some #\") #:quote-all? #f #:crlf? #f)
  (def w (writer separator quote-char quote-all? crlf?))
  (def sb (stringbuilder-empty))
  (loop (:for row rows)
        (:do (add-row! sb w row)
             (stringbuilder-add-string! sb (record-ref w newline))))
  (stringbuilder->string sb))

;; One row and its line end.
(: (-> OutputPort %row (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool)
       (#:crlf? bool)
       void)
   (where (Iterable %row #:elem string)))
(defun (write-csv-row port row #:separator #\, #:quote-char (Some #\") #:quote-all? #f #:crlf? #f)
  (def w (writer separator quote-char quote-all? crlf?))
  (def sb (stringbuilder-empty))
  (add-row! sb w row)
  (stringbuilder-add-string! sb (record-ref w newline))
  (write-string port (stringbuilder->string sb)))

;; Text is written to the port each time this much has been made, so a long
;; document is never one string.
(: int)
(def write-chunk 65536)

;; Every row, each ended by a line end. The rows are read once, in order, so
;; a Seq is written as it is made.
(: (-> OutputPort %rows (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool)
       (#:crlf? bool)
       void)
   (where (Iterable %rows #:elem %row) (Iterable %row #:elem string)))
(defun (write-csv port rows #:separator #\, #:quote-char (Some #\") #:quote-all? #f #:crlf? #f)
  (def w (writer separator quote-char quote-all? crlf?))
  (def sb (stringbuilder-empty))
  (loop (:for row rows)
        (:do (add-row! sb w row)
             (stringbuilder-add-string! sb (record-ref w newline))
             (when (>= (stringbuilder-length sb) write-chunk)
               (write-string port (stringbuilder->string sb))
               (stringbuilder-clear! sb))))
  (when (> (stringbuilder-length sb) 0)
    (write-string port (stringbuilder->string sb))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; text/encodings.bjo
;;
;; Every character encoding .NET knows, as a value to hand to
;; `bytes->string`, `string->bytes` and `reencode-input-port`, and finding
;; one by name or code page.
;;
;; The names are .NET's, lower-cased: `utf-8`, `windows-1252`, `dos-862`,
;; `shift_jis`. Two kinds differ. .NET's `utf-16` and `utf-32` are
;; little-endian and are `utf-16le` and `utf-32le` here, next to `utf-16be`
;; and `utf-32be`. Where .NET gives one name to two code pages, the less
;; common one is `cp` and its number: `euc-jp` is 51932 and `cp20932` the
;; other, `iso-2022-jp` is 50220 and `cp50222` the other.
;;
;; Each value is a property of `Bjolang.Runtime.Encodings` that makes its
;; encoding the first time it is read, so importing the module makes none of
;; them and a program pays only for the ones it uses.

(export
  encoding-named encoding-of-code-page encoding-name
  asmo-708 big5 cp1025 cp20932 cp50222 cp866 cp875 csiso2022jp dos-720
  dos-862 euc-cn euc-jp euc-kr gb18030 gb2312 hz-gb-2312 ibm-thai ibm00858
  ibm00924 ibm01047 ibm01140 ibm01141 ibm01142 ibm01143 ibm01144 ibm01145
  ibm01146 ibm01147 ibm01148 ibm01149 ibm037 ibm1026 ibm273 ibm277 ibm278
  ibm280 ibm284 ibm285 ibm290 ibm297 ibm420 ibm423 ibm424 ibm437 ibm500
  ibm737 ibm775 ibm850 ibm852 ibm855 ibm857 ibm860 ibm861 ibm863 ibm864
  ibm865 ibm869 ibm870 ibm871 ibm880 ibm905 iso-2022-jp iso-2022-kr
  iso-8859-1 iso-8859-13 iso-8859-15 iso-8859-2 iso-8859-3 iso-8859-4
  iso-8859-5 iso-8859-6 iso-8859-7 iso-8859-8 iso-8859-8-i iso-8859-9 johab
  koi8-r koi8-u ks_c_5601-1987 macintosh shift_jis us-ascii utf-16be
  utf-16le utf-32be utf-32le utf-8 windows-1250 windows-1251 windows-1252
  windows-1253 windows-1254 windows-1255 windows-1256 windows-1257
  windows-1258 windows-874 x-chinese-cns x-chinese-eten x-cp20001 x-cp20003
  x-cp20004 x-cp20005 x-cp20261 x-cp20269 x-cp20936 x-cp20949 x-cp50227
  x-ebcdic-koreanextended x-europa x-ia5 x-ia5-german x-ia5-norwegian
  x-ia5-swedish x-iscii-as x-iscii-be x-iscii-de x-iscii-gu x-iscii-ka
  x-iscii-ma x-iscii-or x-iscii-pa x-iscii-ta x-iscii-te x-mac-arabic
  x-mac-ce x-mac-chinesesimp x-mac-chinesetrad x-mac-croatian x-mac-cyrillic
  x-mac-greek x-mac-hebrew x-mac-icelandic x-mac-japanese x-mac-korean
  x-mac-romanian x-mac-thai x-mac-turkish x-mac-ukrainian)

(import (std prelude))

(:doc #:file "encodings.bjodoc")

(import/extern
  (clr-named        (: Bjolang.Runtime.Encodings.Named (-> string (Option Encoding))))
  (clr-of-code-page (: Bjolang.Runtime.Encodings.OfCodePage (-> int (Option Encoding))))
  (clr-name-of      (: Bjolang.Runtime.Encodings.NameOf (-> Encoding string))))

(: (-> string (Option Encoding)))
(defun (encoding-named name) (clr-named name))

(: (-> int (Option Encoding)))
(defun (encoding-of-code-page code-page) (clr-of-code-page code-page))

(: (-> Encoding string))
(defun (encoding-name encoding) (clr-name-of encoding))

;; The encodings.
(import/extern
  (asmo-708                (: Bjolang.Runtime.Encodings.Cp708 #:get))
  (big5                    (: Bjolang.Runtime.Encodings.Cp950 #:get))
  (cp1025                  (: Bjolang.Runtime.Encodings.Cp21025 #:get))
  (cp20932                 (: Bjolang.Runtime.Encodings.Cp20932 #:get))
  (cp50222                 (: Bjolang.Runtime.Encodings.Cp50222 #:get))
  (cp866                   (: Bjolang.Runtime.Encodings.Cp866 #:get))
  (cp875                   (: Bjolang.Runtime.Encodings.Cp875 #:get))
  (csiso2022jp             (: Bjolang.Runtime.Encodings.Cp50221 #:get))
  (dos-720                 (: Bjolang.Runtime.Encodings.Cp720 #:get))
  (dos-862                 (: Bjolang.Runtime.Encodings.Cp862 #:get))
  (euc-cn                  (: Bjolang.Runtime.Encodings.Cp51936 #:get))
  (euc-jp                  (: Bjolang.Runtime.Encodings.Cp51932 #:get))
  (euc-kr                  (: Bjolang.Runtime.Encodings.Cp51949 #:get))
  (gb18030                 (: Bjolang.Runtime.Encodings.Cp54936 #:get))
  (gb2312                  (: Bjolang.Runtime.Encodings.Cp936 #:get))
  (hz-gb-2312              (: Bjolang.Runtime.Encodings.Cp52936 #:get))
  (ibm-thai                (: Bjolang.Runtime.Encodings.Cp20838 #:get))
  (ibm00858                (: Bjolang.Runtime.Encodings.Cp858 #:get))
  (ibm00924                (: Bjolang.Runtime.Encodings.Cp20924 #:get))
  (ibm01047                (: Bjolang.Runtime.Encodings.Cp1047 #:get))
  (ibm01140                (: Bjolang.Runtime.Encodings.Cp1140 #:get))
  (ibm01141                (: Bjolang.Runtime.Encodings.Cp1141 #:get))
  (ibm01142                (: Bjolang.Runtime.Encodings.Cp1142 #:get))
  (ibm01143                (: Bjolang.Runtime.Encodings.Cp1143 #:get))
  (ibm01144                (: Bjolang.Runtime.Encodings.Cp1144 #:get))
  (ibm01145                (: Bjolang.Runtime.Encodings.Cp1145 #:get))
  (ibm01146                (: Bjolang.Runtime.Encodings.Cp1146 #:get))
  (ibm01147                (: Bjolang.Runtime.Encodings.Cp1147 #:get))
  (ibm01148                (: Bjolang.Runtime.Encodings.Cp1148 #:get))
  (ibm01149                (: Bjolang.Runtime.Encodings.Cp1149 #:get))
  (ibm037                  (: Bjolang.Runtime.Encodings.Cp37 #:get))
  (ibm1026                 (: Bjolang.Runtime.Encodings.Cp1026 #:get))
  (ibm273                  (: Bjolang.Runtime.Encodings.Cp20273 #:get))
  (ibm277                  (: Bjolang.Runtime.Encodings.Cp20277 #:get))
  (ibm278                  (: Bjolang.Runtime.Encodings.Cp20278 #:get))
  (ibm280                  (: Bjolang.Runtime.Encodings.Cp20280 #:get))
  (ibm284                  (: Bjolang.Runtime.Encodings.Cp20284 #:get))
  (ibm285                  (: Bjolang.Runtime.Encodings.Cp20285 #:get))
  (ibm290                  (: Bjolang.Runtime.Encodings.Cp20290 #:get))
  (ibm297                  (: Bjolang.Runtime.Encodings.Cp20297 #:get))
  (ibm420                  (: Bjolang.Runtime.Encodings.Cp20420 #:get))
  (ibm423                  (: Bjolang.Runtime.Encodings.Cp20423 #:get))
  (ibm424                  (: Bjolang.Runtime.Encodings.Cp20424 #:get))
  (ibm437                  (: Bjolang.Runtime.Encodings.Cp437 #:get))
  (ibm500                  (: Bjolang.Runtime.Encodings.Cp500 #:get))
  (ibm737                  (: Bjolang.Runtime.Encodings.Cp737 #:get))
  (ibm775                  (: Bjolang.Runtime.Encodings.Cp775 #:get))
  (ibm850                  (: Bjolang.Runtime.Encodings.Cp850 #:get))
  (ibm852                  (: Bjolang.Runtime.Encodings.Cp852 #:get))
  (ibm855                  (: Bjolang.Runtime.Encodings.Cp855 #:get))
  (ibm857                  (: Bjolang.Runtime.Encodings.Cp857 #:get))
  (ibm860                  (: Bjolang.Runtime.Encodings.Cp860 #:get))
  (ibm861                  (: Bjolang.Runtime.Encodings.Cp861 #:get))
  (ibm863                  (: Bjolang.Runtime.Encodings.Cp863 #:get))
  (ibm864                  (: Bjolang.Runtime.Encodings.Cp864 #:get))
  (ibm865                  (: Bjolang.Runtime.Encodings.Cp865 #:get))
  (ibm869                  (: Bjolang.Runtime.Encodings.Cp869 #:get))
  (ibm870                  (: Bjolang.Runtime.Encodings.Cp870 #:get))
  (ibm871                  (: Bjolang.Runtime.Encodings.Cp20871 #:get))
  (ibm880                  (: Bjolang.Runtime.Encodings.Cp20880 #:get))
  (ibm905                  (: Bjolang.Runtime.Encodings.Cp20905 #:get))
  (iso-2022-jp             (: Bjolang.Runtime.Encodings.Cp50220 #:get))
  (iso-2022-kr             (: Bjolang.Runtime.Encodings.Cp50225 #:get))
  (iso-8859-1              (: Bjolang.Runtime.Encodings.Cp28591 #:get))
  (iso-8859-13             (: Bjolang.Runtime.Encodings.Cp28603 #:get))
  (iso-8859-15             (: Bjolang.Runtime.Encodings.Cp28605 #:get))
  (iso-8859-2              (: Bjolang.Runtime.Encodings.Cp28592 #:get))
  (iso-8859-3              (: Bjolang.Runtime.Encodings.Cp28593 #:get))
  (iso-8859-4              (: Bjolang.Runtime.Encodings.Cp28594 #:get))
  (iso-8859-5              (: Bjolang.Runtime.Encodings.Cp28595 #:get))
  (iso-8859-6              (: Bjolang.Runtime.Encodings.Cp28596 #:get))
  (iso-8859-7              (: Bjolang.Runtime.Encodings.Cp28597 #:get))
  (iso-8859-8              (: Bjolang.Runtime.Encodings.Cp28598 #:get))
  (iso-8859-8-i            (: Bjolang.Runtime.Encodings.Cp38598 #:get))
  (iso-8859-9              (: Bjolang.Runtime.Encodings.Cp28599 #:get))
  (johab                   (: Bjolang.Runtime.Encodings.Cp1361 #:get))
  (koi8-r                  (: Bjolang.Runtime.Encodings.Cp20866 #:get))
  (koi8-u                  (: Bjolang.Runtime.Encodings.Cp21866 #:get))
  (ks_c_5601-1987          (: Bjolang.Runtime.Encodings.Cp949 #:get))
  (macintosh               (: Bjolang.Runtime.Encodings.Cp10000 #:get))
  (shift_jis               (: Bjolang.Runtime.Encodings.Cp932 #:get))
  (us-ascii                (: Bjolang.Runtime.Encodings.Cp20127 #:get))
  (utf-16be                (: Bjolang.Runtime.Encodings.Cp1201 #:get))
  (utf-16le                (: Bjolang.Runtime.Encodings.Cp1200 #:get))
  (utf-32be                (: Bjolang.Runtime.Encodings.Cp12001 #:get))
  (utf-32le                (: Bjolang.Runtime.Encodings.Cp12000 #:get))
  (utf-8                   (: Bjolang.Runtime.Encodings.Cp65001 #:get))
  (windows-1250            (: Bjolang.Runtime.Encodings.Cp1250 #:get))
  (windows-1251            (: Bjolang.Runtime.Encodings.Cp1251 #:get))
  (windows-1252            (: Bjolang.Runtime.Encodings.Cp1252 #:get))
  (windows-1253            (: Bjolang.Runtime.Encodings.Cp1253 #:get))
  (windows-1254            (: Bjolang.Runtime.Encodings.Cp1254 #:get))
  (windows-1255            (: Bjolang.Runtime.Encodings.Cp1255 #:get))
  (windows-1256            (: Bjolang.Runtime.Encodings.Cp1256 #:get))
  (windows-1257            (: Bjolang.Runtime.Encodings.Cp1257 #:get))
  (windows-1258            (: Bjolang.Runtime.Encodings.Cp1258 #:get))
  (windows-874             (: Bjolang.Runtime.Encodings.Cp874 #:get))
  (x-chinese-cns           (: Bjolang.Runtime.Encodings.Cp20000 #:get))
  (x-chinese-eten          (: Bjolang.Runtime.Encodings.Cp20002 #:get))
  (x-cp20001               (: Bjolang.Runtime.Encodings.Cp20001 #:get))
  (x-cp20003               (: Bjolang.Runtime.Encodings.Cp20003 #:get))
  (x-cp20004               (: Bjolang.Runtime.Encodings.Cp20004 #:get))
  (x-cp20005               (: Bjolang.Runtime.Encodings.Cp20005 #:get))
  (x-cp20261               (: Bjolang.Runtime.Encodings.Cp20261 #:get))
  (x-cp20269               (: Bjolang.Runtime.Encodings.Cp20269 #:get))
  (x-cp20936               (: Bjolang.Runtime.Encodings.Cp20936 #:get))
  (x-cp20949               (: Bjolang.Runtime.Encodings.Cp20949 #:get))
  (x-cp50227               (: Bjolang.Runtime.Encodings.Cp50227 #:get))
  (x-ebcdic-koreanextended (: Bjolang.Runtime.Encodings.Cp20833 #:get))
  (x-europa                (: Bjolang.Runtime.Encodings.Cp29001 #:get))
  (x-ia5                   (: Bjolang.Runtime.Encodings.Cp20105 #:get))
  (x-ia5-german            (: Bjolang.Runtime.Encodings.Cp20106 #:get))
  (x-ia5-norwegian         (: Bjolang.Runtime.Encodings.Cp20108 #:get))
  (x-ia5-swedish           (: Bjolang.Runtime.Encodings.Cp20107 #:get))
  (x-iscii-as              (: Bjolang.Runtime.Encodings.Cp57006 #:get))
  (x-iscii-be              (: Bjolang.Runtime.Encodings.Cp57003 #:get))
  (x-iscii-de              (: Bjolang.Runtime.Encodings.Cp57002 #:get))
  (x-iscii-gu              (: Bjolang.Runtime.Encodings.Cp57010 #:get))
  (x-iscii-ka              (: Bjolang.Runtime.Encodings.Cp57008 #:get))
  (x-iscii-ma              (: Bjolang.Runtime.Encodings.Cp57009 #:get))
  (x-iscii-or              (: Bjolang.Runtime.Encodings.Cp57007 #:get))
  (x-iscii-pa              (: Bjolang.Runtime.Encodings.Cp57011 #:get))
  (x-iscii-ta              (: Bjolang.Runtime.Encodings.Cp57004 #:get))
  (x-iscii-te              (: Bjolang.Runtime.Encodings.Cp57005 #:get))
  (x-mac-arabic            (: Bjolang.Runtime.Encodings.Cp10004 #:get))
  (x-mac-ce                (: Bjolang.Runtime.Encodings.Cp10029 #:get))
  (x-mac-chinesesimp       (: Bjolang.Runtime.Encodings.Cp10008 #:get))
  (x-mac-chinesetrad       (: Bjolang.Runtime.Encodings.Cp10002 #:get))
  (x-mac-croatian          (: Bjolang.Runtime.Encodings.Cp10082 #:get))
  (x-mac-cyrillic          (: Bjolang.Runtime.Encodings.Cp10007 #:get))
  (x-mac-greek             (: Bjolang.Runtime.Encodings.Cp10006 #:get))
  (x-mac-hebrew            (: Bjolang.Runtime.Encodings.Cp10005 #:get))
  (x-mac-icelandic         (: Bjolang.Runtime.Encodings.Cp10079 #:get))
  (x-mac-japanese          (: Bjolang.Runtime.Encodings.Cp10001 #:get))
  (x-mac-korean            (: Bjolang.Runtime.Encodings.Cp10003 #:get))
  (x-mac-romanian          (: Bjolang.Runtime.Encodings.Cp10010 #:get))
  (x-mac-thai              (: Bjolang.Runtime.Encodings.Cp10021 #:get))
  (x-mac-turkish           (: Bjolang.Runtime.Encodings.Cp10081 #:get))
  (x-mac-ukrainian         (: Bjolang.Runtime.Encodings.Cp10017 #:get)))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; text/json-codec.bjo — records to JSON and back.
;;
;; Two traits and a macro that writes the instances for a record.
;;
;; Nullable types are Options, keys that may be absent just specify a default
;; value

(export
  ;; The traits
  ->json json->
  ;; What generated field code calls. Exported because a macro template's names
  ;; resolve against this module's exports.
  json-object json-field json-field/default
  ;; The generator
  def/json-type)

(import (std prelude))
(import (std syntax-match))
(import (text json))

;; Every export is documented there; the module doc is too.
(:doc #:file "json-codec.bjodoc")

;; ---------------------------------------------------------------------------
;; The traits
;; ---------------------------------------------------------------------------

(def/trait (->json %a)
  (: ->json (-> %a Json)))


(def/trait (json-> %a)
  (: json-> (-> Json (Result string %a))))

;; ---------------------------------------------------------------------------
;; Scalars
;; ---------------------------------------------------------------------------

(impl (->json Json)   (defun (->json j) j))
(impl (->json long)   (defun (->json n) (JsonInt n)))
(impl (->json int)    (defun (->json n) (JsonInt (cast System.Int64 n))))
(impl (->json double) (defun (->json n) (JsonFloat n)))
(impl (->json bool)   (defun (->json b) (JsonBool b)))
(impl (->json string) (defun (->json s) (JsonStr s)))

(impl (json-> Json)
  (defun (json-> j) (Ok j)))

(impl (json-> long)
  (defun (json-> j)
    (match j
      ((JsonInt n) (Ok n))
      (_ (Err "expected an integer")))))

(impl (json-> int)
  (defun (json-> j)
    (match j
      ;; A JsonInt is a long; one outside an int's range is refused rather
      ;; than wrapped by the cast.
      ((JsonInt n)
       (if (and (>= n -2147483648L) (<= n 2147483647L))
           (Ok (cast System.Int32 n))
           (Err "expected an integer that fits in an int")))
      (_ (Err "expected an integer")))))


(impl (json-> double)
  (defun (json-> j)
    (match j
      ((JsonFloat d) (Ok d))
      ;; an int is a valid double.
      ((JsonInt n) (Ok (cast System.Double n)))
      (_ (Err "expected a number")))))

(impl (json-> bool)
  (defun (json-> j)
    (match j
      ((JsonBool b) (Ok b))
      (_ (Err "expected a boolean")))))

(impl (json-> string)
  (defun (json-> j)
    (match j
      ((JsonStr s) (Ok s))
      (_ (Err "expected a string")))))

;; ---------------------------------------------------------------------------
;; Nullability
;; ---------------------------------------------------------------------------

(impl (->json (Option %a))
  (where (->json %a))
  (defun (->json o)
    (match o
      ((Some x) (->json x))
      (None JsonNull))))

(impl (json-> (Option %a))
  (where (json-> %a))
  (defun (json-> j)
    (match j
      (JsonNull (Ok None))
      (_ (match (json-> j)
           ((Err e) (Err e))
           ((Ok v) (Ok (Some v))))))))

(impl (->json (Vec %a))
  (where (->json %a))
  (defun (->json v)
    (JsonArr (vec->iarray (vec-map (fun (x) (->json x)) v)))))

(impl (json-> (Vec %a))
  (where (json-> %a))
  (defun (json-> j)
    (match j
      ((JsonArr items)
       (fold (fun (el acc)
               (def (Ok b) acc :propagate)
               (def (Ok v) (json-> el) :propagate)
               (Ok (vec-add b v)))
             (Ok (vec-empty))
             items))
      (_ (Err "expected an array")))))

;; A JSON object with values all of one type.
(impl (->json (Map string %a))
  (where (->json %a))
  (defun (->json m)
    (JsonObj (map-map-values (fun (v) (->json v)) m))))

(impl (json-> (Map string %a))
  (where (json-> %a))
  (defun (json-> j)
    (match j
      ((JsonObj m)
       (map-fold (fun (k v acc)
                   (match acc
                     ((Err e) (Err e))
                     ((Ok b)
                      (match (json-> v)
                        ((Err e) (Err (str "\"" k "\": " e)))
                        ((Ok d) (Ok (map-set b k d)))))))
                 (Ok (map-empty))
                 m))
      ;; In order, so that a key written twice keeps its last value here too.
      ((JsonFields fields)
       (fold (fun (p acc)
               (def (Tuple k v) p)
               (match acc
                 ((Err e) (Err e))
                 ((Ok b)
                  (match (json-> v)
                    ((Err e) (Err (str "\"" k "\": " e)))
                    ((Ok d) (Ok (map-set b k d)))))))
             (Ok (map-empty))
             fields))
      (_ (Err "expected an object")))))

;; ---------------------------------------------------------------------------
;; What generated field code calls
;; ---------------------------------------------------------------------------

;; The object itself, a JsonObj or a JsonFields: json-field looks in either.
(: (-> string Json (Result string Json)))
(defun (json-object owner j)
  (match j
    ((JsonObj _) (Ok j))
    ((JsonFields _) (Ok j))
    (_ (Err (str owner ": expected an object")))))

;; The owner and key are prefixed onto whatever the value's own decoder said,
;; so a wrong type reads "Point.x: expected an integer" rather than naming
;; neither. Paths down the tree are not provided
(: (-> string Json string (Result string %a)) (where (json-> %a)))
(defun (json-field owner m key)
  (match (json-object-ref m key)
    (None (Err (str owner ": the object has no \"" key "\" field")))
    ((Some j)
     (result-map-err (fun (e) (str owner "." key ": " e)) (json-> j)))))

;; Absent and null ar the same in the sense that a nullable has the default value None
(: (-> string Json string %a (Result string %a)) (where (json-> %a)))
(defun (json-field/default owner m key fallback)
  (match (json-object-ref m key)
    (None (Ok fallback))
    ((Some JsonNull) (Ok fallback))
    ((Some j)
     (result-map-err (fun (e) (str owner "." key ": " e)) (json-> j)))))

;; ---------------------------------------------------------------------------
;; The generator
;; ---------------------------------------------------------------------------
;; This is a runtime macro. Field types are never inspected: what a field encodes as is
;; settled later by instance resolution, and only the markers are read here.

(: (-> string Syntax))
(defun (jt-sym name) (SSym (string->symbol name)))

(: (-> Syntax Syntax))
(defun (jt-str s) (SStr (syntax->string s)))

;; `(Box %a)` names the type `Box`; a bare `Point` names itself.
(: (-> Syntax Syntax))
(defun (jt-head-name head)
  (match head
    ((SList parts) (list-head parts))
    (_ head)))

(: (-> Syntax (List Syntax)))
(defun (jt-head-vars head)
  (match head
    ((SList parts) (list-tail parts))
    (_ Nil)))


(: (-> (-> Symbol Syntax) Syntax (List Syntax)))
(defun (jt-where-enc inject head)
  (def vars (jt-head-vars head))
  (if (list-empty? vars)
      Nil
      (list #'(,(inject 'where)
               ,@(list-map (fun (v) #'(,(inject (string->symbol "->json")) ,v)) vars)))))

(: (-> (-> Symbol Syntax) Syntax (List Syntax)))
(defun (jt-where-dec inject head)
  (def vars (jt-head-vars head))
  (if (list-empty? vars)
      Nil
      (list #'(,(inject 'where)
               ,@(list-map (fun (v) #'(,(inject (string->symbol "json->")) ,v)) vars)))))

;; --- fields ----------------------------------------------------------------

(: (-> Syntax string bool))
(defun (jt-marker=? m name) (= (syntax->string m) name))

;; The JSON key: `#:json "name"` means it uses the "name" as the key for the field
;; instead of the defined name
(: (-> Syntax (List Syntax) Syntax))
(defun (jt-key name markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) (jt-str name))
      ((jt-marker=? (list-head ms) "#:json") (list-head (list-tail ms)))
      (else (go (list-tail ms))))))

;; The fallback expression, or None if the field is required. `#:optional` is
;; `#:default None`.
(: (-> (List Syntax) (Option Syntax)))
(defun (jt-fallback markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) None)
      (else
       (def m (list-head ms))
       (cond
         ((jt-marker=? m "#:optional") (Some (jt-sym "None")))
         ((jt-marker=? m "#:default") (Some (list-head (list-tail ms))))
         (else (go (list-tail ms))))))))

;; strip json markers (default, json, optional)
(: (-> (List Syntax) (List Syntax)))
(defun (jt-keep markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) Nil)
      (else
       (def m (list-head ms))
       (cond
         ((jt-marker=? m "#:default") (go (list-tail (list-tail ms))))
         ((jt-marker=? m "#:json") (go (list-tail (list-tail ms))))
         ((jt-marker=? m "#:optional") (go (list-tail ms)))
         (else (Cons m (go (list-tail ms)))))))))

(: string)
(def jt-shape
  "a field is written (: name type), then any of #:mutable, #:optional, #:default <expr>, #:json \"key\".")

;; `syntax-error` has the type Syntax, so this is how we work around it.
(: (-> Syntax string (List Syntax)))
(defun (jt-fail-list form message)
  (ignore (syntax-error form message))
  Nil)

(: (-> Syntax Syntax))
(defun (jt-field-name f)
  (syntax-match f
    ((_ name _ marker ...) name)
    (bad (syntax-error bad jt-shape))))

(: (-> Syntax (List Syntax)))
(defun (jt-field-markers f)
  (syntax-match f
    ((_ _ _ marker ...) marker)
    (bad (jt-fail-list bad jt-shape))))

(: (-> Syntax Syntax))
(defun (jt-strip f)
  (syntax-match f
    ((colon name ty marker ...) #'(,colon ,name ,ty ,@(jt-keep marker)))
    (bad (syntax-error bad jt-shape))))

;; --- encode ----------------------------------------------------------------

;; An optional field is dropped when it is None; everything else is written.
(: (-> Syntax Syntax Syntax))
(defun (jt-put buf f)
  (def name (jt-field-name f))
  (def markers (jt-field-markers f))
  (def write #'(transientmap-set! ,buf ,(jt-key name markers) (->json (record-ref v ,name))))
  (if (jt-optional? markers)
      #'(when (some? (record-ref v ,name)) ,write)
      write))

(: (-> (List Syntax) bool))
(defun (jt-optional? markers)
  (let go ((ms markers))
    (cond
      ((list-empty? ms) #f)
      ((jt-marker=? (list-head ms) "#:optional") #t)
      (else (go (list-tail ms))))))

(: (-> (List Syntax) Syntax))
(defun (jt-encode fields)
  (def buf (jt-sym "__jt-buf"))
  #'(let ((,buf (transientmap-empty)))
      ,@(list-map (fun (f) (jt-put buf f)) fields)
      (JsonObj (transientmap->map ,buf))))

;; --- decode ----------------------------------------------------------------

;; One `match` per field wrapping the next, with the constructor innermost.
;; Verbose. Macrology.
(: (-> Syntax Syntax (List Syntax) int Syntax Syntax))
(defun (jt-chain owner m fields i ctor)
  (cond
    ((list-empty? fields) ctor)
    (else
     (def f (list-head fields))
     (def v (jt-sym (str "__jt-v" (int->string i))))
     (def e (jt-sym (str "__jt-e" (int->string i))))
     (def name (jt-field-name f))
     (def markers (jt-field-markers f))
     (def get (match (jt-fallback markers)
                (None #'(json-field ,owner ,m ,(jt-key name markers)))
                ((Some d) #'(json-field/default ,owner ,m ,(jt-key name markers) ,d))))
     #'(match ,get
         ((Err ,e) (Err ,e))
         ((Ok ,v) ,(jt-chain owner m (list-tail fields) (+ i 1) ctor))))))

(: (-> Syntax (List Syntax) Syntax))
(defun (jt-ctor head fields)
  #'(Ok (,(jt-head-name head)
         ,@(let go ((fs fields) (i 0))
             (if (list-empty? fs)
                 Nil
                 (Cons #'(,(jt-field-name (list-head fs)) ,(jt-sym (str "__jt-v" (int->string i))))
                       (go (list-tail fs) (+ i 1))))))))

(: (-> Syntax (List Syntax) Syntax))
(defun (jt-decode head fields)
  (def owner (jt-str (jt-head-name head)))
  (def m (jt-sym "__jt-m"))
  (def e (jt-sym "__jt-e"))
  #'(match (json-object ,owner j)
      ((Err ,e) (Err ,e))
      ((Ok ,m) ,(jt-chain owner m fields 0 (jt-ctor head fields)))))

;; --- the form ---------------------------------------------------------------

(: (-> (-> Symbol Syntax) Syntax Syntax (List Syntax) Syntax))
(defun (jt-expand inject colon head fields)
  (def name (syntax->string (jt-head-name head)))
  (def enc (inject (string->symbol (str name "->json"))))
  (def dec (inject (string->symbol (str "json->" name))))
  #'(begin
      (type (,colon ,head (Record ,@(list-map jt-strip fields))))

      (impl (->json ,head)
        ,@(jt-where-enc inject head)
        (defun (->json v) ,(jt-encode fields)))

      (impl (json-> ,head)
        ,@(jt-where-dec inject head)
        (defun (json-> j) ,(jt-decode head fields)))

      ;; The named pair. `json->` alone would be ambiguous at a call site;
      ;; this signature is what fixes the instance, once per type.
      (: ,enc (-> ,head Json) ,@(jt-where-enc inject head))
      (defun (,enc v) (->json v))
      (: ,dec (-> Json (Result string ,head)) ,@(jt-where-dec inject head))
      (defun (,dec j) (json-> j))))

(def/macro (def/json-type form inject compare)
  (syntax-match form
    ((_ typedef)
     (syntax-match typedef
       ((colon head ('Record field ...)) (jt-expand inject colon head field))
       (bad (syntax-error bad "def/json-type takes a record: (def/json-type (: Name (Record (: field type) ...)))"))))
    (bad (syntax-error bad "def/json-type takes exactly one record declaration."))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; text/json.bjo
;;
;; RFC 8259, read off the document's UTF-8 bytes.

(export
  Json JsonObjects
  json-max-depth
  read-json parse-json
  json-object-ref
  json->string write-json)

(import (std prelude))
(import (only (std ports) read-bytes))
(import (std maths))
(import (std simd))

;; Every export is documented there; the module doc is too.
(:doc #:file "json.bjodoc")

;; TODO: use bjolangs builtins: bjolangs builtins uses the invariant culture now.
(import/extern
  (clr-to-long   (: Bjolang.Runtime.BjoNum.ParseLong))
  (clr-to-double (: Bjolang.Runtime.BjoNum.ParseDouble))
  ;; The same parsers, on the bytes in the reader's buffer: a number that is
  ;; whole in the chunk is parsed where it is, without a copy.
  (span->long    (: BjoString.Utf8Number.ParseLong   (-> (RoSpan byte) long)))
  (span->double  (: BjoString.Utf8Number.ParseDouble (-> (RoSpan byte) double))))

(type
  (: Json (Union
           JsonNull
           (: JsonBool bool)
           (: JsonInt long)
           (: JsonFloat double)
           (: JsonStr string)
           ;; An `IArray`: a parsed array is read and not changed, and an
           ;; `IArray` is its elements' array and no object of its own.
           (: JsonArr (IArray Json))
           (: JsonObj (Map string Json))
           ;; An object read with `#:objects 'fields`: its keys and values in
           ;; the order the document has them, each key once.
           (: JsonFields (IArray (Tuple string Json))))))

;; How a reader stores an object: as a Map, or as its fields in an IArray,
;; which is quicker to build and keeps their order.
(def/grammar JsonObjects
  (:or (: ObjectsAsMap 'map) (: ObjectsAsFields 'fields)))

;; The value of `key` in an object of either kind, or None when `j` is not an
;; object or has no such key. In a JsonFields this is a walk over the fields.
(: (-> Json string (Option Json)))
(defun (json-object-ref j key)
  (match j
    ((JsonObj m) (map-try-ref m key))
    ((JsonFields fields)
     (def n (iarray-length fields))
     (let go ((i 0))
       (cond
         ((= i n) None)
         (:def (Tuple k v) (iarray-ref fields i))
         ((= k key) (Some v))
         (else (go (+ i 1))))))
    (_ None)))

;; Max stack depth. This should probably be a parameter.
(: int)
(def json-max-depth 1000)

;; ---------------------------------------------------------------------------
;; The reader
;; ---------------------------------------------------------------------------
;; The reader works on bytes: a chunk of the document in `bytes`, the one it is
;; looking at at `next`, and `limit` where the chunk ends. Looking is an array
;; load and stepping is an increment. A chunk comes from the port with
;; `read-bytes` (see `ensure!`); `parse-json` hands over the string's own bytes as the one
;; chunk there is, and no port.
;;
;; Everything JSON says outside a string is ASCII, so outside a string a byte
;; is a character. Inside one, a character of several bytes is checked to be
;; UTF-8 and copied as it is; one that is not UTF-8 is refused. A column
;; counts characters: only a byte that starts one moves it.
;;
;; The byte is handled as a number, so that the end of the document is a
;; value, `spent`, rather than a second case.

(: int)
(def spent -1)

;; How much a read from the port asks for: the first one what `read-some`
;; gives a file, each one after it twice the one before, up to `read-most`.
;; Each read costs a sync on the port, a new array and a pass of the reader's
;; slow path at the chunk's end, so a big document is read in big chunks.
;; `read-bytes` makes the port's buffer as big as what it asks for, so a small
;; one is still read in a small one.
(: int)
(def read-first 16384)

(: int)
(def read-most 262144)

(type
  (: Reader (Record (: port (Option InputPort) #:mutable)
                    ;; How many bytes the next read from the port asks for.
                    (: ask int #:mutable)
                    (: bytes (Array byte) #:mutable)
                    (: next int #:mutable)
                    (: limit int #:mutable)
                    (: row int #:mutable)
                    (: col int #:mutable)
                    ;; One buffer for every string and every number in the
                    ;; document.
                    (: scratch StringBuilder)
                    ;; One array for every array in the document, and how much
                    ;; of it is in use.
                    (: stack (Array Json) #:mutable)
                    (: top int #:mutable)
                    ;; The keys last read, by slot; empty until the first
                    ;; object. See `cached-key`.
                    (: keys (Array string) #:mutable)
                    ;; Whether objects are read as JsonFields.
                    (: as-fields bool)
                    ;; One array for every JsonFields' pairs, as `stack` is
                    ;; for arrays.
                    (: pairs (Array (Tuple string Json)) #:mutable)
                    (: ptop int #:mutable)
                    ;; A hash table of pair indices, for the check for a
                    ;; repeated key; see repeated-key/hashed?. A slot is in
                    ;; use when its stamp is the check's, so the table is never
                    ;; cleared.
                    (: slot-stamps (Array int) #:mutable)
                    (: slot-pairs (Array int) #:mutable)
                    (: stamp int #:mutable))))

;; What a reader's key cache is until the first key.
(: (Array string))
(def no-keys (make-array 0))

;; What a reader's pair stack and hash table are until the first JsonFields
;; that needs them.
(: (Array (Tuple string Json)))
(def no-pairs (make-array 0))

(: (Array int))
(def no-slots (make-array 0))

;; A reader over `bytes`, and then whatever `port` has after them.
(: (-> (Option InputPort) (Array byte) JsonObjects Reader))
(defun (reader port bytes objects)
  (Reader (port port)
          (as-fields (match objects (ObjectsAsMap #f) (ObjectsAsFields #t)))
          (pairs no-pairs)
          (ptop 0)
          (slot-stamps no-slots)
          (slot-pairs no-slots)
          (stamp 0)
          (ask read-first)
          (bytes bytes)
          (next 0)
          (limit (array-length bytes))
          (row 1)
          (col 1)
          (scratch (stringbuilder-empty))
          (stack (make-array 16))
          (top 0)
          (keys no-keys)))

;; ---------------------------------------------------------------------------
;; The key cache
;; ---------------------------------------------------------------------------
;; An array of objects repeats its keys: twenty thousand records write the same
;; five keys twenty thousand times. This holds the last key read in each slot
;; and compares the buffer with it, without making a string, which is what
;; `stringbuilder=?` is for. A key read before costs that comparison and no
;; allocation, and the objects share one string for it. `(text bjodat-core)`
;; caches its names the same way, with the same slots.
;;
;; 64 slots, because an object may have many keys: a tweet has forty. Keys that
;; share a slot take turns in it and are made each time, as without a cache.

(: int)
(def key-slots 64)

;; Empty slots hold `""`, which no key but the empty one equals, and that one
;; is `""` anyway.
(: (-> (Array string)))
(defun (new-key-cache)
  (def a (make-array key-slots))
  (loop (:for i (range 0 key-slots))
        (:do (array-set! a i "")))
  a)

;; The first, middle and last byte and the length: three reads whatever the
;; length, and enough to part the keys of one object. With the first byte and
;; the length alone, `name` and `note` shared a slot, and so did `name` and
;; `pinned`, and each made the other a string every time.
(: (-> StringBuilder int int))
(defun (key-slot sb n)
  (if (= n 0)
      0
      (bitwise-and (+ (* (+ (* (+ (* (stringbuilder-byte-ref sb 0) 31)
                                  (stringbuilder-byte-ref sb (shift-right n 1)))
                               31)
                            (stringbuilder-byte-ref sb (- n 1)))
                         31)
                      n)
                   (- key-slots 1))))

;; The buffer as a string, the one already made for the same key if there is
;; one.
(: (-> Reader StringBuilder string))
(defun (cached-key rd sb)
  (when (= (array-length (record-ref rd keys)) 0)
    (record-set! rd (keys (new-key-cache))))
  (def cache (record-ref rd keys))
  (def i (key-slot sb (stringbuilder-length sb)))
  (def known (array-ref cache i))
  (cond
    ((stringbuilder=? sb known) known)
    (else
     (def s (stringbuilder->string sb))
     (array-set! cache i s)
     s)))

;; Pushes a new element into the array or grows it 
(: (-> Reader Json void))
(defun (push! rd v)
  (def a (record-ref rd stack))
  (def n (record-ref rd top))
  (cond
    ((< n (array-length a)) (array-set! a n v))
    (else
     (def bigger (array-resize a (* n 2)))
     (array-set! bigger n v)
     (record-set! rd (stack bigger))))
  (record-set! rd (top (+ n 1))))

;; Harvests the array, by copying it out of the stack
(: (-> Reader int (IArray Json)))
(defun (harvest! rd mark)
  (def v (subarray->iarray (record-ref rd stack) mark (- (record-ref rd top) mark)))
  (record-set! rd (top mark))
  v)

;; The buffer, emptied and ready.
(: (-> Reader StringBuilder))
(defun (buffer rd)
  (def sb (record-ref rd scratch))
  (stringbuilder-clear! sb)
  sb)

;; `count` bytes of `old` from `from`, then `more`: what is left of one chunk
;; and the next. Almost always nothing is left, and `more` is the answer.
(: (-> (Array byte) int int (Array byte) (Array byte)))
(defun (join-bytes old from count more)
  (if (= count 0)
      more
      (let ((joined (make-array (+ count (array-length more)))))
        (array-copy! old from joined 0 count)
        (array-copy! more 0 joined count (array-length more))
        joined)))

;; Whether there are at least `n` bytes from `next` on, reading from the port
;; until there are or it ends. Bytes not yet looked at are kept, so a
;; character cut in two by the end of a chunk is whole afterwards.
(: (-> Reader int bool))
(defun (ensure! rd n)
  (let go ()
    (def have (- (record-ref rd limit) (record-ref rd next)))
    (cond
      ((>= have n) #t)
      (else
       (def (Some p) (get rd .port) :leave-with #f)
       (def ask (record-ref rd ask))
       (match (read-bytes p (if (> n ask) n ask))
         (None
          (record-set! rd (port None))
          #f)
         ((Some more)
          (record-set! rd
                       (bytes (join-bytes (record-ref rd bytes) (record-ref rd next) have more))
                       (next 0)
                       (limit (+ have (array-length more)))
                       (ask (if (< ask read-most) (* ask 2) read-most)))
          (go)))))))

;; The byte the reader is looking at, or `spent` at the end of the document.
(: (-> Reader int))
(defun (held-code rd)
  (def i (record-ref rd next))
  (cond
    ((< i (record-ref rd limit)) (cast int (array-ref (record-ref rd bytes) i)))
    ((ensure! rd 1) (cast int (array-ref (record-ref rd bytes) (record-ref rd next))))
    (else spent)))

(: (-> Reader bool))
(defun (at-end? rd)
  (= (held-code rd) spent))

;; Every character this is asked about is a json char meaning it is ascii.
;; Anything else — the end of the document, the first byte of a letter from a
;; language — answers #\nul.
(: (-> Reader char))
(defun (peek rd)
  (def c (held-code rd))
  (if (and (>= c 0) (< c 128)) (int->char c) #\nul))

;; Moves past the byte being looked at, which is a character of its own.
;;
;; It does not look at what it is stepping over, because every caller just has:
;; a digit, an ASCII character inside a string, a bracket. A character of
;; several bytes is taken by `take-character!`.
(: (-> Reader void))
(defun (step! rd)
  (record-set! rd
               (next (+ (record-ref rd next) 1))
               (col (+ (record-ref rd col) 1))))


(: (-> int bool))
(defun (space? c) (or (= c 32) (= c 10) (= c 9) (= c 13)))

;; Moves past the white space. Usually there is none, and that costs one look
;; at the next byte.
(: (-> Reader void))
(defun (skip-space! rd)
  (when (space? (held-code rd))
    (skip-space-run! rd)))

;; White space from `start`, 32 or 16 bytes at a time, to the first byte that
;; is not white space or the last whole block before `limit`: where it ends,
;; the line and column there, and whether a byte that is not white space was
;; found. The line moves by the newlines; after one, the column is counted
;; from it. 256 bits where the CPU has them, as `scan-run` has.
(: (-> (Array byte) int int int int (Tuple int int int bool)))
(defun (scan-space b start limit row col)
  (if simd256-accelerated?
      (scan-space/256 b start limit row col)
      (scan-space/128 b start limit row col)))

;; The line and column after `n` bytes of white space whose newlines are the
;; set bits of `nls`.
(: (-> int int int int (Tuple int int)))
(defun (after-space nls n row col)
  (if (= nls 0)
      (Tuple row (+ col n))
      ;; The bytes after the last newline are the column, less one.
      (Tuple (+ row (popcount nls))
             (- n (- 31 (leading-zeros nls))))))

(: (-> (Array byte) int int int int (Tuple int int int bool)))
(defun (scan-space/256 b start limit row col)
  (let go ((i start) (row row) (col col))
    (cond
      ((> (+ i 32) limit)
        (Tuple i row col #f))
      (:def v (simd256-load b i))
      (:def nl (simd-eq v (simd256-splat 10uy)))
      (:def spaces (simd-mask-bits (simd-or (simd-or nl (simd-eq v (simd256-splat 32uy)))
                                            (simd-or (simd-eq v (simd256-splat 9uy))
                                                     (simd-eq v (simd256-splat 13uy))))))
      (:def nls (simd-mask-bits nl))
      ((= spaces -1)
         (def (Tuple r c) (after-space nls 32 row col))
         (go (+ i 32) r c))
      (else
       (def t (trailing-zeros (bitwise-not spaces)))
       (def (Tuple r c) (after-space (bitwise-and nls (- (shift-left 1 t) 1)) t row col))
       (Tuple (+ i t) r c #t)))))

(: (-> (Array byte) int int int int (Tuple int int int bool)))
(defun (scan-space/128 b start limit row col)
  (let go ((i start) (row row) (col col))
    (cond 
      ((> (+ i 16) limit) (Tuple i row col #f))
      (:def v (simd128-load b i))
      (:def nl (simd-eq v (simd128-splat 10uy)))
      (:def spaces (simd-mask-bits (simd-or (simd-or nl (simd-eq v (simd128-splat 32uy)))
                                            (simd-or (simd-eq v (simd128-splat 9uy))
                                                     (simd-eq v (simd128-splat 13uy))))))
      (:def nls (simd-mask-bits nl))
      ((= spaces 65535)
        (def (Tuple r c) (after-space nls 16 row col))
        (go (+ i 16) r c))
      (else
       (def t (trailing-zeros (bitwise-xor spaces 65535)))
       (def (Tuple r c) (after-space (bitwise-and nls (- (shift-left 1 t) 1)) t row col))
       (Tuple (+ i t) r c #t)))))

;; White space a chunk at a time: the chunk is scanned with a local index, and
;; the reader is moved once for it. A chunk that ends in white space is
;; followed by the next.
(: (-> Reader void))
(defun (skip-space-run! rd)
  (def b (record-ref rd bytes))
  (def limit (record-ref rd limit))
  (def (Tuple fast-end fast-row fast-col stopped)
    (scan-space b (record-ref rd next) limit (record-ref rd row) (record-ref rd col)))
  (def (Tuple end row col)
    (let go ((i fast-end) (row fast-row) (col fast-col))
      (cond
        ((or stopped (>= i limit))
          (Tuple i row col))
        (:def c (cast int (array-ref b i)))
        ((= c 10) (go (+ i 1) (+ row 1) 1))
        ((or (= c 32) (= c 9) (= c 13)) (go (+ i 1) row (+ col 1)))
        (else (Tuple i row col)))))
  (record-set! rd (next end) (row row) (col col))
  (when (and (>= end limit) (ensure! rd 1))
    (skip-space-run! rd)))



;; ---------------------------------------------------------------------------
;; Characters of several bytes
;; ---------------------------------------------------------------------------

(: string)
(def not-utf-8 "a byte that is not UTF-8")

;; How many bytes a character starting with `lead` has, by RFC 3629; 0 for a
;; byte no character starts with: a continuation byte, C0 and C1 (which could
;; only start an overlong form), and F5 and up.
(: (-> int int))
(defun (utf8-length lead)
  (cond
    ((< lead 128) 1)
    ((< lead 194) 0)
    ((< lead 224) 2)
    ((< lead 240) 3)
    ((< lead 245) 4)
    (else 0)))

(: (-> int bool))
(defun (continuation? b) (= (bitwise-and b 192) 128))

;; How many bytes the character the reader is looking at has, when they are
;; UTF-8; 0 when they are not, or the document ends inside them. The second
;; byte is narrower after E0 and F0, which would otherwise start an overlong
;; form, after ED, which would start a surrogate, and after F4, which would
;; go past U+10FFFF.
(: (-> Reader int))
(defun (sequence-length rd)
  (def lead (held-code rd))
  (def n (utf8-length lead))
  (cond
    ((<= n 1) n)
    ((not (ensure! rd n)) 0)
    (else (sequence-length-at (record-ref rd bytes) (record-ref rd next) (record-ref rd limit)))))

;; The same for the character at `i` of `b`, whose bytes end at `limit`: 0
;; also when the character runs past `limit`, where the reader may have more
;; of it to fetch.
(: (-> (Array byte) int int int))
(defun (sequence-length-at b i limit)
  (def lead (cast int (array-ref b i)))
  (def n (utf8-length lead))
  (cond
    ((<= n 1) n)
    ((> (+ i n) limit) 0)
     (:def second (cast int (array-ref b (+ i 1))))
     (:def low (cond ((= lead 224) 160) ((= lead 240) 144) (else 128)))
     (:def high (cond ((= lead 237) 159) ((= lead 244) 143) (else 191)))
     ((and (>= second low)
           (<= second high)
           (let check ((k 2))
             (cond
               ((>= k n) #t)
               ((continuation? (cast int (array-ref b (+ i k)))) (check (+ k 1)))
               (else #f))))
      n)
     (else 0)))

;; Copies the `n` bytes of the character being looked at into `sb`, and moves
;; past them: one column.
(: (-> Reader StringBuilder int void))
(defun (take-character! rd sb n)
  (def b (record-ref rd bytes))
  (def i (record-ref rd next))
  (loop (:for k (range 0 n))
        (:do (stringbuilder-add-byte! sb (cast int (array-ref b (+ i k))))))
  (record-set! rd
               (next (+ i n))
               (col (+ (record-ref rd col) 1))))

;; ---------------------------------------------------------------------------
;; Refusals
;; ---------------------------------------------------------------------------
;; Functions to raise errors

(: (-> Reader string string))
(defun (at rd what)
  (str "line " (int->string (record-ref rd row))
       ", column " (int->string (record-ref rd col))
       ": " what))

(: (-> Reader string))
(defun (found rd)
  (def c (held-code rd))
  (cond
    ((= c spent) "the document ended")
    ((< c 128) (str "'" (list->string (list (int->char c))) "'"))
    (else
     (def n (sequence-length rd))
     (def i (record-ref rd next))
     (if (= n 0)
         not-utf-8
         (str "'" (bytes->string (record-ref rd bytes) #:start i #:end (+ i n)) "'")))))

(: (-> Reader string string))
(defun (too-deep rd what)
  (at rd (str "this " what " nests deeper than " (int->string json-max-depth)
              ", which is as far as reading goes")))

;; ---------------------------------------------------------------------------
;; Strings
;; ---------------------------------------------------------------------------

;; The Bjolang lexer has no \b or \f of its own.
(: string)
(def escaped-backspace (list->string (list (int->char 8))))

(: string)
(def escaped-formfeed (list->string (list (int->char 12))))

(: (-> char int))
(defun (hex-digit c)
  (cond
    ((and (char>=? c #\0) (char<=? c #\9)) (- (char->int c) 48))
    ((and (char>=? c #\a) (char<=? c #\f)) (- (char->int c) 87))
    ((and (char>=? c #\A) (char<=? c #\F)) (- (char->int c) 55))
    (else -1)))

(: (-> int bool))
(defun (high-surrogate? n) (and (>= n 55296) (<= n 56319)))

(: (-> int bool))
(defun (low-surrogate? n) (and (>= n 56320) (<= n 57343)))

;; The scalar a surrogate pair stands for.
(: (-> int int int))
(defun (scalar-of hi lo)
  (+ 65536 (* (- hi 55296) 1024) (- lo 56320)))

(: (-> Reader (Result string int)))
(defun (read-hex4 rd)
  (let go ((n 0) (left 4))
    (cond
      ((= left 0) (Ok n))
      ((at-end? rd) (Err (at rd "the document ended inside a \\u escape")))
      (:def d (hex-digit (peek rd)))
      ((< d 0) (Err (at rd (str "a \\u escape takes four hex digits, and this is " (found rd)))))
      (else
       (step! rd)
       (go (+ (* n 16) d) (- left 1))))))

;; The pair is one character. A Bjolang char is a Unicode scalar... Not a match made in heaven
(: (-> Reader int (Result string string)))
(defun (read-low-surrogate rd hi)
  (def incomplete "a high surrogate has to be followed by \\u and its low surrogate")
  (cond
   ((or (at-end? rd) (not (char=? (peek rd) #\\))) (Err (at rd incomplete)))
   (:do (step! rd))
   ((or (at-end? rd) (not (char=? (peek rd) #\u))) (Err (at rd incomplete)))
   (:do (step! rd))
   (:def (Ok lo) (read-hex4 rd)
     :propagate)
   ((low-surrogate? lo) (Ok (list->string (list (int->char (scalar-of hi lo))))))
   (else (Err (at rd incomplete)))))

(: (-> Reader (Result string string)))
(defun (read-unicode rd)
  (def (Ok n) (read-hex4 rd)
    :propagate)
  (cond
    ((low-surrogate? n) (Err (at rd "a low surrogate with no high surrogate before it")))
    ((high-surrogate? n) (read-low-surrogate rd n))
    (else (Ok (list->string (list (int->char n)))))))

(: (-> Reader (Result string string)))
(defun (read-escape rd)
  (if (at-end? rd)
      (Err (at rd "the document ended after a backslash"))
      (case (peek rd)
        ((#\") (step! rd) (Ok "\""))
        ((#\\) (step! rd) (Ok "\\"))
        ((#\/) (step! rd) (Ok "/"))
        ((#\b) (step! rd) (Ok escaped-backspace))
        ((#\f) (step! rd) (Ok escaped-formfeed))
        ((#\n) (step! rd) (Ok "\n"))
        ((#\r) (step! rd) (Ok "\r"))
        ((#\t) (step! rd) (Ok "\t"))
        ((#\u) (step! rd) (read-unicode rd))
        (else
         (Err (at rd (str "\\" (list->string (list (peek rd))) " is not an escape")))))))

;; Where a run from `start` stops, 32 or 16 bytes at a time: at the first `"`, `\`
;; or control character, or at the last whole block before `limit`.
;; Also how many characters are before it, which is the bytes that are not a
;; continuation byte (10xxxxxx), whether one of them is 128 or more, and
;; whether a stop was found. The count is the characters' count only when the
;; bytes are UTF-8, which `take-run!` checks when one is 128 or more.
;;
;; 256 bits where the CPU has them, and 128 where it has not, which is every
;; ARM64 CPU: there a 256-bit vector is two 128-bit ones done in software.
(: (-> (Array byte) int int (Tuple int int bool bool)))
(defun (scan-run b start limit)
  (if simd256-accelerated?
      (scan-run/256 b start limit)
      (scan-run/128 b start limit)))

(: (-> (Array byte) int int (Tuple int int bool bool)))
(defun (scan-run/128 b start limit)
  (let go ((i start) (chars 0) (high #f))
    (if (> (+ i 16) limit)
        (Tuple i chars high #f)
        (let ()
          (def v (simd128-load b i))
          (def stops (simd-mask-bits (simd-or (simd-or (simd-eq v (simd128-splat 34uy))
                                                       (simd-eq v (simd128-splat 92uy)))
                                              (simd-lt v (simd128-splat 32uy)))))
          (def conts (simd-mask-bits (simd-eq (simd-and v (simd128-splat 192uy)) (simd128-splat 128uy))))
          (def highs (simd-mask-bits (simd-ge v (simd128-splat 128uy))))
          (if (= stops 0)
              (go (+ i 16) (+ chars (- 16 (popcount conts))) (or high (not (= highs 0))))
              (let ()
                (def t (trailing-zeros stops))
                (def before (- (shift-left 1 t) 1))
                (Tuple (+ i t)
                       (+ chars (- t (popcount (bitwise-and conts before))))
                       (or high (not (= (bitwise-and highs before) 0)))
                       #t)))))))

(: (-> (Array byte) int int (Tuple int int bool bool)))
(defun (scan-run/256 b start limit)
  (let go ((i start) (chars 0) (high #f))
    (if (> (+ i 32) limit)
        (Tuple i chars high #f)
        (let ()
          (def v (simd256-load b i))
          (def stops (simd-mask-bits (simd-or (simd-or (simd-eq v (simd256-splat 34uy))
                                                       (simd-eq v (simd256-splat 92uy)))
                                              (simd-lt v (simd256-splat 32uy)))))
          (def conts (simd-mask-bits (simd-eq (simd-and v (simd256-splat 192uy)) (simd256-splat 128uy))))
          (def highs (simd-mask-bits (simd-ge v (simd256-splat 128uy))))
          (if (= stops 0)
              (go (+ i 32) (+ chars (- 32 (popcount conts))) (or high (not (= highs 0))))
              (let ()
                ;; The lanes before the first stop.
                (def t (trailing-zeros stops))
                (def before (- (shift-left 1 t) 1))
                (Tuple (+ i t)
                       (+ chars (- t (popcount (bitwise-and conts before))))
                       (or high (not (= (bitwise-and highs before) 0)))
                       #t)))))))

;; Copies the run of characters that need no attention, from where the reader
;; is to the first `"`, `\`, control character, byte that is not UTF-8 or end
;; of the chunk, into `sb` in one piece, and moves past it. Whatever stopped
;; the run is left for `read-text` to look at.
;;
;; The run is found a block at a time by `scan-run`, and the bytes left
;; before `limit` one at a time. A run with a byte of 128 or more is checked
;; as UTF-8 in one call. A run that is not UTF-8, or that the end of the chunk
;; cuts inside a character, is read again by `take-run/checked!`, which stops
;; before the byte at fault, so that the error and its column are where they
;; were.
(: (-> Reader StringBuilder void))
(defun (take-run! rd sb)
  (def b (record-ref rd bytes))
  (def start (record-ref rd next))
  (def limit (record-ref rd limit))
  (def (Tuple fast-end fast-chars fast-high stopped) (scan-run b start limit))
  (def (Tuple end chars high)
    (if stopped
        (Tuple fast-end fast-chars fast-high)
        (let go ((i fast-end) (chars fast-chars) (high fast-high))
          (if (>= i limit)
              (Tuple i chars high)
              (let ((c (cast int (array-ref b i))))
                (cond
                  ((or (= c 34) (= c 92) (< c 32)) (Tuple i chars high))
                  ((< c 128) (go (+ i 1) (+ chars 1) high))
                  ((continuation? c) (go (+ i 1) chars #t))
                  (else (go (+ i 1) (+ chars 1) #t))))))))
  (cond
    ((and high (not (bytes-utf-8? b start end))) (take-run/checked! rd sb))
    ((> end start)
     (stringbuilder-add-bytes! sb b start end)
     (record-set! rd
                  (next end)
                  (col (+ (record-ref rd col) chars))))
    (else unit)))

;; `take-run!` a byte at a time, each character checked as UTF-8: the run
;; stops before a byte that is not, and before a character the end of the
;; chunk cuts.
(: (-> Reader StringBuilder void))
(defun (take-run/checked! rd sb)
  (def b (record-ref rd bytes))
  (def start (record-ref rd next))
  (def limit (record-ref rd limit))
  (def (Tuple end chars)
    (let go ((i start) (chars 0))
      (if (>= i limit)
          (Tuple i chars)
          (let ((c (cast int (array-ref b i))))
            (cond
              ((or (= c 34) (= c 92) (< c 32)) (Tuple i chars))
              ((< c 128) (go (+ i 1) (+ chars 1)))
              (else
               (let ((n (sequence-length-at b i limit)))
                 (if (= n 0) (Tuple i chars) (go (+ i n) (+ chars 1))))))))))
  (when (> end start)
    (stringbuilder-add-bytes! sb b start end)
    (record-set! rd
                 (next end)
                 (col (+ (record-ref rd col) chars)))))

;; Reads a json string. An object's key goes through the key cache.
(: (-> Reader bool (Result string string)))
(defun (read-text rd key?)
  (step! rd)
  (def sb (buffer rd))
  (let go ()
    (take-run! rd sb)
    (def c (held-code rd))
    (cond
     ((= c spent) (Err (at rd "the document ended inside a string")))
     ((= c 34) (step! rd) (Ok (if key? (cached-key rd sb) (stringbuilder->string sb))))
     ((= c 92)
      (step! rd)
      (def (Ok piece) (read-escape rd)
        :propagate)
      (stringbuilder-add-string! sb piece)
      (go))
     ((< c 32)
      (Err (at rd "a control character in a string has to be written as an escape")))
     ((< c 128)
      (stringbuilder-add-unit! sb c)
      (step! rd)
      (go))
     ;; The first byte of a character of several, all copied once they are
     ;; known to be UTF-8.
     (:def n (sequence-length rd))
     ((= n 0) (Err (at rd "a string has to be UTF-8, and this byte is not")))
     (else
      (take-character! rd sb n)
      (go)))))

;; ---------------------------------------------------------------------------
;; Numbers
;; ---------------------------------------------------------------------------

(: (-> char bool))
(defun (digit? c) (and (char>=? c #\0) (char<=? c #\9)))


(: (-> Reader StringBuilder int))
;; Reads a digit into the string builder and advances until
;; whatever is in the reader is no longer a digit. Returns the number of digits taken
(defun (take-digits! rd sb)
  (let go ((n 0))
    (def c (held-code rd))
    ;; ascii codes for 0-9
    (cond
      ((and (>= c 48) (<= c 57))
       (stringbuilder-add-unit! sb c)
       (step! rd)
       (go (+ n 1)))
      (else n))))

;; Answers whether what was scanned has to be read as a double.
(: (-> Reader StringBuilder (Result string bool)))
(defun (scan-exponent rd sb)
  (cond
    ((and (not (at-end? rd)) (or (char=? (peek rd) #\e) (char=? (peek rd) #\E)))
     (stringbuilder-add! sb #\e)
     (step! rd)
     (when (and (not (at-end? rd)) (or (char=? (peek rd) #\+) (char=? (peek rd) #\-)))
       (stringbuilder-add! sb (peek rd))
       (step! rd))
     (if (= (take-digits! rd sb) 0)
         (Err (at rd (str "a number needs a digit after its exponent, and this is " (found rd))))
         (Ok #t)))
    (else (Ok #f))))

(: (-> Reader StringBuilder (Result string bool)))
(defun (scan-fraction rd sb)
  (cond
    ((and (not (at-end? rd)) (char=? (peek rd) #\.))
     (stringbuilder-add! sb #\.)
     (step! rd)
     (if (= (take-digits! rd sb) 0)
         (Err (at rd (str "a number needs a digit after its decimal point, and this is "
                          (found rd))))
         (match (scan-exponent rd sb)
           ((Err e) (Err e))
           ((Ok _) (Ok #t)))))
    (else (scan-exponent rd sb))))

(: (-> Reader StringBuilder (Result string bool)))
(defun (scan-number rd sb)
  (when (and (not (at-end? rd)) (char=? (peek rd) #\-))
    (stringbuilder-add! sb #\-)
    (step! rd))
  (cond
    ((or (at-end? rd) (not (digit? (peek rd))))
     (Err (at rd (str "a number needs a digit here, and this is " (found rd)))))

    ;; Check whether a zero is a lone zero or if it is a number with a leading zero
    ;; which is not allowed.
    ((char=? (peek rd) #\0)
     (stringbuilder-add! sb #\0)
     (step! rd)
     (if (and (not (at-end? rd)) (digit? (peek rd)))
         (Err (at rd "a number may not have a leading zero"))
         (scan-fraction rd sb)))
    (else
     (ignore (take-digits! rd sb))
     (scan-fraction rd sb))))

;; A number byte by byte, fetching more of the document as it goes: for a
;; number that the end of a chunk cuts.
(: (-> Reader (Result string Json)))
(defun (read-number/slow rd)
  (def sb (buffer rd))
  (def (Ok fractional?) (scan-number rd sb)
    :propagate)
  (if fractional?
      (Ok (JsonFloat (clr-to-double sb)))
      ;; Wider than a long is still a number, and a double is what every
      ;; other reader answers with.
      (match (try (clr-to-long sb) #:catch (System.OverflowException))
        ((Ok n) (Ok (JsonInt n)))
        ((Err _) (Ok (JsonFloat (clr-to-double sb)))))))

;; ---------------------------------------------------------------------------
;; Numbers off the chunk
;; ---------------------------------------------------------------------------
;; A number is read off the chunk's bytes with a local index, and the reader
;; moves past it once at the end, rather than once a byte. A number is ASCII,
;; so the column moves by as many bytes as the number has.
;;
;; A number that reaches the end of the chunk, when the port may have more,
;; could be longer than the chunk shows. That one is read by
;; `read-number/slow`, which fetches the rest.

;; What a scan of a number found, with a position: the end of the number, or
;; for a refusal the byte that is wrong. A tuple of two ints rather than a
;; union, because a tuple is a value and a union case with a field is an
;; allocation for every number in the document.
(type (: NumberScan (Tuple int int)))

(: int)
(def num-int 0)
(: int)
(def num-frac 1)
(: int)
(def num-no-digit 2)
(: int)
(def num-leading-zero 3)
(: int)
(def num-no-fraction-digit 4)
(: int)
(def num-no-exponent-digit 5)

(: (-> (Array byte) int int bool))
(defun (digit-at? b i limit)
  (and (< i limit)
       (let ((c (cast int (array-ref b i))))
         (and (>= c 48) (<= c 57)))))

(: (-> (Array byte) int int int bool))
(defun (byte-at? b i limit code)
  (and (< i limit) (= (cast int (array-ref b i)) code)))

;; The position after the digits from `i`.
(: (-> (Array byte) int int int))
(defun (digits-end b i limit)
  (let go ((i i))
    (if (digit-at? b i limit) (go (+ i 1)) i)))

;; The exponent, if there is one, from `i`, after a number that is fractional
;; when `frac?` is.
(: (-> (Array byte) int int bool NumberScan))
(defun (scan-exponent-at b i limit frac?)
  (cond
    ((or (byte-at? b i limit 101) (byte-at? b i limit 69))
     (def signed (if (or (byte-at? b (+ i 1) limit 43) (byte-at? b (+ i 1) limit 45)) (+ i 2) (+ i 1)))
     (def end (digits-end b signed limit))
     (if (= end signed) (Tuple num-no-exponent-digit end) (Tuple num-frac end)))
    (frac? (Tuple num-frac i))
    (else (Tuple num-int i))))

;; The fraction, if there is one, from `i`.
(: (-> (Array byte) int int NumberScan))
(defun (scan-fraction-at b i limit)
  (cond
    ((byte-at? b i limit 46)
     (def end (digits-end b (+ i 1) limit))
     (if (= end (+ i 1))
         (Tuple num-no-fraction-digit end)
         (scan-exponent-at b end limit #t)))
    (else (scan-exponent-at b i limit #f))))

;; The number from `pos`, whose digits start at `first`, by RFC 8259.
(: (-> (Array byte) int int int NumberScan))
(defun (scan-number-at b pos first limit)
  (cond
    ((not (digit-at? b first limit)) (Tuple num-no-digit first))
    ((byte-at? b first limit 48)
     (if (digit-at? b (+ first 1) limit)
         (Tuple num-leading-zero (+ first 1))
         (scan-fraction-at b (+ first 1) limit)))
    (else (scan-fraction-at b (digits-end b first limit) limit))))

;; Up to 18 digits cannot overflow a long, so they are added up as they are.
;; Longer ones go through the parser, which answers a double when the number
;; is too wide for a long.
(: (-> Reader (Array byte) int int int Json))
(defun (integer-of rd b start first end)
  (cond
    ((<= (- end first) 18)
     (def n (let go ((i first) (acc 0L))
              (if (< i end)
                  (go (+ i 1) (+ (* acc 10L) (cast System.Int64 (- (cast int (array-ref b i)) 48))))
                  acc)))
     (JsonInt (if (= first start) n (- 0L n))))
    (else
     (match (try (long-at b start end) #:catch (System.OverflowException))
       ((Ok n) (JsonInt n))
       ((Err _) (JsonFloat (double-at b start end)))))))

;; The number from `start` to `end` of the reader's bytes, parsed where it is.
;; In functions of their own, so that the span is not a local of the readers
;; that call them.
(: (-> (Array byte) int int long))
(defun (long-at b start end) (span->long (array->ro-span b #:from start #:to end)))

(: (-> (Array byte) int int double))
(defun (double-at b start end) (span->double (array->ro-span b #:from start #:to end)))

(: (-> Reader (Result string Json)))
(defun (read-number rd)
  (def* (b (record-ref rd bytes))
        (start (record-ref rd next))
        (limit (record-ref rd limit))
        (first (if (byte-at? b start limit 45) (+ start 1) start))
        ((status . end) (scan-number-at b start first limit))
        (more? (match (record-ref rd port) (None #f) (_ #t))))
  (cond
    ((and more? (>= end limit)) (read-number/slow rd))
    (else
     (record-set! rd
                  (next end)
                  (col (+ (record-ref rd col) (- end start))))
     (cond
       ((= status num-int) (Ok (integer-of rd b start first end)))
       ((= status num-frac) (Ok (JsonFloat (double-at b start end))))
       ((= status num-no-digit)
        (Err (at rd (str "a number needs a digit here, and this is " (found rd)))))
       ((= status num-leading-zero)
        (Err (at rd "a number may not have a leading zero")))
       ((= status num-no-fraction-digit)
        (Err (at rd (str "a number needs a digit after its decimal point, and this is "
                         (found rd)))))
       (else
        (Err (at rd (str "a number needs a digit after its exponent, and this is "
                         (found rd)))))))))

;; ---------------------------------------------------------------------------
;; Values
;; ---------------------------------------------------------------------------

;; only one json null, true or false value.
(: Json)
(def json-true (JsonBool #t))

(: Json)
(def json-false (JsonBool #f))

(: Json)
(def json-null JsonNull)

(: (-> Reader string Json (Result string Json)))
(defun (read-word rd word value)
  (let go ((k (string-cursor-start word)))
    (cond
     ((string-cursor-end? word k)
      (Ok value))
     ((or (at-end? rd) (not (char=? (peek rd) (string-cursor-ref word k))))
      (Err (at rd (str "'" word "' is spelled out to here, and then there is " (found rd)))))
     (else
      (step! rd)
      (go (string-cursor-next word k))))))

(: (-> Reader int (Result string Json)))
(defun (read-array rd depth)
  (cond
   ((> depth json-max-depth) (Err (too-deep rd "array")))
   (:do
    (step! rd)
    (skip-space! rd))
   (:def mark (record-ref rd top))
   ((at-end? rd) (Err (at rd "the document ended inside an array")))
   ((char=? (peek rd) #\]) (step! rd) (Ok (JsonArr (harvest! rd mark))))
   (else
    (let go ()
      (def (Ok v) (read-value rd (+ depth 1))
        :propagate)
      (push! rd v)
      (skip-space! rd)
      (cond
        ((at-end? rd) (Err (at rd "the document ended inside an array")))
        ((char=? (peek rd) #\,) (step! rd) (go))
        ((char=? (peek rd) #\]) (step! rd) (Ok (JsonArr (harvest! rd mark))))
        (else
         (Err (at rd (str "an array wants ',' or ']' here, and this is "
                          (found rd))))))))))

;; ---------------------------------------------------------------------------
;; Objects as fields
;; ---------------------------------------------------------------------------
;; An object's pairs go on the reader's pair stack, as an array's elements go
;; on `stack`, and are copied out once into the object's own IArray. A key
;; written twice is found when the object closes: by comparing every two
;; keys of a small object, and through a hash table of a bigger one, which
;; the reader keeps, so that the check allocates nothing. Only an object that
;; does repeat a key is built again.

;; Pushes a pair onto the pair stack, growing it.
(: (-> Reader (Tuple string Json) void))
(defun (push-pair! rd p)
  (def a (record-ref rd pairs))
  (def n (record-ref rd ptop))
  (cond
    ((< n (array-length a)) (array-set! a n p))
    (else
     (def bigger (array-resize a (if (< n 8) 16 (* n 2))))
     (array-set! bigger n p)
     (record-set! rd (pairs bigger))))
  (record-set! rd (ptop (+ n 1))))

;; Up to this many keys, every two are compared.
(: int)
(def few-keys 8)

;; Whether a key is in the pairs from `from` to `to` twice.
(: (-> Reader (Array (Tuple string Json)) int int bool))
(defun (repeated-key? rd a from to)
  (if (<= (- to from) few-keys)
      (let outer ((i from))
        (cond
          ((>= i to) #f)
          ((let inner ((j (+ i 1)))
             (cond
               ((>= j to) #f)
               ((= (fst (array-ref a i)) (fst (array-ref a j))) #t)
               (else (inner (+ j 1)))))
           #t)
          (else (outer (+ i 1)))))
      (repeated-key/hashed? rd a from to)))

;; The table at least twice as big as `n`, and a power of two.
(: (-> Reader int void))
(defun (fit-table! rd n)
  (def size (array-length (record-ref rd slot-stamps)))
  (when (< size (* 2 n))
    (def bigger (let grow ((s 64)) (if (< s (* 2 n)) (grow (* s 2)) s)))
    (record-set! rd (slot-stamps (make-array bigger)) (slot-pairs (make-array bigger)) (stamp 0))))

;; Open addressing over the key's hash. Keys are compared only when their
;; slots collide.
(: (-> Reader (Array (Tuple string Json)) int int bool))
(defun (repeated-key/hashed? rd a from to)
  (fit-table! rd (- to from))
  (def stamps (record-ref rd slot-stamps))
  (def index (record-ref rd slot-pairs))
  (def mask (- (array-length stamps) 1))
  (def stamp (+ (record-ref rd stamp) 1))
  (record-set! rd (stamp stamp))
  (let go ((i from))
    (cond
      ((>= i to) #f)
      (else
       (def k (fst (array-ref a i)))
       (let probe ((s (bitwise-and (eq-hash k) mask)))
         (cond
           ((not (= (array-ref stamps s) stamp))
            (array-set! stamps s stamp)
            (array-set! index s i)
            (go (+ i 1)))
           ((= (fst (array-ref a (array-ref index s))) k) #t)
           (else (probe (bitwise-and (+ s 1) mask)))))))))

;; The pairs from `from` to `to` with each key once: where it was last, with
;; its last value, as a Map keeps the last value.
(: (-> (Array (Tuple string Json)) int int (IArray (Tuple string Json))))
(defun (last-of-each a from to)
  (def out (listbuilder-empty))
  (loop (:for i (range from to))
        (:do (def k (fst (array-ref a i)))
             (unless (let later ((j (+ i 1)))
                       (cond ((>= j to) #f)
                             ((= (fst (array-ref a j)) k) #t)
                             (else (later (+ j 1)))))
               (listbuilder-add! out (array-ref a i)))))
  (list->iarray (listbuilder->list out)))

;; The object's pairs, copied out of the stack.
(: (-> Reader int (IArray (Tuple string Json))))
(defun (harvest-pairs! rd mark)
  (def a (record-ref rd pairs))
  (def top (record-ref rd ptop))
  (def v (if (repeated-key? rd a mark top)
             (last-of-each a mark top)
             (subarray->iarray a mark (- top mark))))
  (record-set! rd (ptop mark))
  v)

;; Reads an object as a JsonFields. On a duplicate key, the last value wins.
(: (-> Reader int (Result string Json)))
(defun (read-object/fields rd depth)
  (cond
   ((> depth json-max-depth) (Err (too-deep rd "object")))
   (:do
    (step! rd)
    (skip-space! rd))
   (:def mark (record-ref rd ptop))
   ((at-end? rd) (Err (at rd "the document ended inside an object")))
   ((char=? (peek rd) #\}) (step! rd) (Ok (JsonFields (harvest-pairs! rd mark))))
   (else
    (let go ()
      (skip-space! rd)
      (cond
        ((at-end? rd) (Err (at rd "the document ended inside an object")))
        ((not (char=? (peek rd) #\"))
         (Err (at rd (str "an object's key has to be a string, and this is " (found rd)))))
        (:def (Ok key) (read-text rd #t)
              :propagate)
        (:do (skip-space! rd))
        ((or (at-end? rd) (not (char=? (peek rd) #\:)))
         (Err (at rd (str "an object wants ':' after a key, and this is "
                          (found rd)))))
        (:do (step! rd))
        (:def (Ok v) (read-value rd (+ depth 1))
              :propagate)
        (:do (push-pair! rd (Tuple key v))
             (skip-space! rd))
        ((at-end? rd) (Err (at rd "the document ended inside an object")))
        ((char=? (peek rd) #\,) (step! rd) (go))
        ((char=? (peek rd) #\})
         (step! rd)
         (Ok (JsonFields (harvest-pairs! rd mark))))
        (else
         (Err (at rd (str "an object wants ',' or '}' here, and this is "
                          (found rd))))))))))

;; Reads an object as `#:objects` says.
(: (-> Reader int (Result string Json)))
(defun (read-object rd depth)
  (if (record-ref rd as-fields)
      (read-object/fields rd depth)
      (read-object/map rd depth)))

;; Reads an object as a JsonObj. On a duplicate key, the last value wins.
(: (-> Reader int (Result string Json)))
(defun (read-object/map rd depth)
  (cond
   ((> depth json-max-depth) (Err (too-deep rd "object")))
   (:do 
    (step! rd)
    (skip-space! rd))
   (:def b (transientmap-empty))
   ((at-end? rd) (Err (at rd "the document ended inside an object")))
   ((char=? (peek rd) #\}) (step! rd) (Ok (JsonObj (transientmap->map b))))
   (else
    (let go ()
      (skip-space! rd)
      (cond
        ((at-end? rd) (Err (at rd "the document ended inside an object")))
        ((not (char=? (peek rd) #\"))
         (Err (at rd (str "an object's key has to be a string, and this is " (found rd)))))
        (:def (Ok key) (read-text rd #t)
              :propagate)
        (:do (skip-space! rd))
        ((or (at-end? rd) (not (char=? (peek rd) #\:)))
         (Err (at rd (str "an object wants ':' after a key, and this is "
                          (found rd)))))
        (:do (step! rd))
        (:def (Ok v) (read-value rd (+ depth 1))
              :propagate)
        (:do (transientmap-set! b key v)
             (skip-space! rd))
        ((at-end? rd) (Err (at rd "the document ended inside an object")))
        ((char=? (peek rd) #\,) (step! rd) (go))
        ((char=? (peek rd) #\})
         (step! rd)
         (Ok (JsonObj (transientmap->map b))))
        (else
         (Err (at rd (str "an object wants ',' or '}' here, and this is "
                          (found rd))))))))))

(: (-> Reader int (Result string Json)))
(defun (read-value rd depth)
  (skip-space! rd)
  (if (at-end? rd)
      (Err (at rd "the document ended where a value was expected"))
      (case (peek rd)
        ((#\{) (read-object rd depth))
        ((#\[) (read-array rd depth))
        ((#\") (match (read-text rd #f)
                 ((Err e) (Err e))
                 ((Ok s) (Ok (JsonStr s)))))
        ((#\t) (read-word rd "true" json-true))
        ((#\f) (read-word rd "false" json-false))
        ((#\n) (read-word rd "null" json-null))
        ((#\- #\0 #\1 #\2 #\3 #\4 #\5 #\6 #\7 #\8 #\9) (read-number rd))
        (else (Err (at rd (str (found rd) " does not begin a value")))))))

;; ---------------------------------------------------------------------------
;; Reading a document
;; ---------------------------------------------------------------------------
;; Whether the reader is looking at a byte order mark: EF BB BF.
(: (-> Reader bool))
(defun (at-mark? rd)
  (and (ensure! rd 3)
       (let ((b (record-ref rd bytes)) (i (record-ref rd next)))
         (and (= (cast int (array-ref b i)) 239)
              (= (cast int (array-ref b (+ i 1))) 187)
              (= (cast int (array-ref b (+ i 2))) 191)))))

(: (-> Reader (Result string Json)))
(defun (read-document rd)
  ;; A byte order mark ahead of the value is skipped, as RFC 8259 lets a
  ;; parser do, and does not count as a column.
  (when (at-mark? rd)
    (record-set! rd (next (+ (record-ref rd next) 3))))
  (def (Ok v) (read-value rd 1)
    :propagate)
  (skip-space! rd)
  (if (at-end? rd)
      (Ok v)
      (Err (at rd (str "the value ended here, and then there is " (found rd))))))

(: (Array byte))
(def no-bytes (make-array 0))

(: (-> InputPort (#:objects JsonObjects) (Result string Json)))
(defun (read-json port #:objects 'map)
  (read-document (reader (Some port) no-bytes objects)))

;; The string's bytes are the document: no port in between.
(: (-> string (#:objects JsonObjects) (Result string Json)))
(defun (parse-json text #:objects 'map)
  (read-document (reader None (string->bytes text) objects)))

;; ---------------------------------------------------------------------------
;; The writer
;; ---------------------------------------------------------------------------
;; One `StringBuilder` for the whole document, which `write-json` sends to
;; its port and empties each time it is full. The keys of a JsonObj are
;; written in sorted order: a `Map` does not keep the order they were read in,
;; and sorted keys make the same value give the same text in every run. A
;; JsonFields has an order of its own, and is written in it.

(import/class
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

(import/extern
  (clr-finite? (: System.Double.IsFinite (-> double bool))))

(: (Vec char))
(def hex-digits (string->vec "0123456789abcdef"))

(: (-> StringBuilder int void))
(defun (add-hex4! sb n)
  (stringbuilder-add-string! sb "\\u")
  (loop (:for shift (list 12 8 4 0))
        (:do (stringbuilder-add! sb (vec-ref hex-digits (bitwise-and (shift-right n shift) 15))))))

;; One character as JSON writes it in a string: an escape for `"`, `\` and the
;; control characters, as RFC 8259 requires, and with `ascii` for every
;; character outside ASCII too, as one `\u` escape or, outside the BMP, as a
;; surrogate pair of them.
(: (-> StringBuilder char bool void))
(defun (add-char! sb c ascii)
  (def n (char->int c))
  (cond
    ((= n 34) (stringbuilder-add-string! sb "\\\""))
    ((= n 92) (stringbuilder-add-string! sb "\\\\"))
    ((= n 10) (stringbuilder-add-string! sb "\\n"))
    ((= n 13) (stringbuilder-add-string! sb "\\r"))
    ((= n 9) (stringbuilder-add-string! sb "\\t"))
    ((= n 8) (stringbuilder-add-string! sb "\\b"))
    ((= n 12) (stringbuilder-add-string! sb "\\f"))
    ((< n 32) (add-hex4! sb n))
    ((and ascii (> n 126) (< n 65536)) (add-hex4! sb n))
    ((and ascii (>= n 65536))
     (def v (- n 65536))
     (add-hex4! sb (+ 55296 (shift-right v 10)))
     (add-hex4! sb (+ 56320 (bitwise-and v 1023))))
    (else (stringbuilder-add! sb c))))

;; The characters a string has to have escaped: `"`, `\` and the control
;; characters. All ASCII, so `string-search-any` finds them with a byte search.
(: string)
(def must-escape
  (str "\"\\" (list->string (loop (:for n (range 0 32)) (:acc (listing (int->char n)))))))

;; A string, quoted. The runs between the characters that have to be escaped
;; are copied whole. With `ascii` every character outside ASCII has to be
;; escaped too, so each character is looked at.
(: (-> StringBuilder string bool void))
(defun (add-string! sb s ascii)
  (stringbuilder-add! sb #\")
  (if ascii
      (string-for-each (fun (c) (add-char! sb c ascii)) s)
      (let go ((from (string-cursor-start s)))
        (match (string-search-any s must-escape from)
          (None (stringbuilder-add-substring! sb s from (string-cursor-end s)))
          ((Some at)
           (stringbuilder-add-substring! sb s from at)
           (add-char! sb (string-cursor-ref s at) ascii)
           (go (string-cursor-next s at))))))
  (stringbuilder-add! sb #\"))

;; The shortest text that reads back as the same double, with `.0` added when
;; it has neither a point nor an exponent, so that it reads back as a
;; `JsonFloat` and not a `JsonInt`.
(: (-> StringBuilder double void))
(defun (add-float! sb d)
  (unless (clr-finite? d)
    (raise (cast Exception (ArgumentException. (str "JSON has no number for " (double->string d))))))
  (def from (stringbuilder-length sb))
  (stringbuilder-add-double! sb d)
  ;; The bytes just written: `.` is 46, `E` 69 and `e` 101.
  (def fraction?
    (let go ((i from))
      (cond ((= i (stringbuilder-length sb)) #f)
            ((let ((b (stringbuilder-byte-ref sb i))) (or (= b 46) (= b 69) (= b 101))) #t)
            (else (go (+ i 1))))))
  (unless fraction?
    (stringbuilder-add-string! sb ".0")))

;; An object's entries in an array, in the order of their keys. An insertion
;; sort, in place: an object has a few keys, and a sort that allocates as it
;; merges costs more than the comparisons for them. One with many keys is
;; sorted by `vec-sort-by`, which takes n log n comparisons.
(: (-> (Map string Json) (Array (Tuple string Json))))
(defun (sorted-entries fields)
  (def n (map-length fields))
  (if (> n 16)
      (vec->array (vec-sort-by (fun (a b) (compare (fst a) (fst b))) (map->vec fields)))
      (let ()
        (def entries (make-array n))
        (loop (:for e (map->seq fields))
              (:for i (range 0 n))
              (:do (array-set! entries i e)))
        (loop (:for i (range 1 n))
              (:do (def e (array-ref entries i))
                   (let shift ((j (- i 1)))
                     (if (and (>= j 0) (> (compare (fst (array-ref entries j)) (fst e)) 0))
                         (let ()
                           (array-set! entries (+ j 1) (array-ref entries j))
                           (shift (- j 1)))
                         (array-set! entries (+ j 1) e)))))
        entries)))

(: (-> StringBuilder int int void))
(defun (add-newline! sb indent level)
  (when (> indent 0)
    (stringbuilder-add! sb #\newline)
    (loop (:for _ (range 0 (* indent level)))
          (:do (stringbuilder-add! sb #\space)))))

;; How full the builder may get before `write-json` sends it to its port.
;; Below the builder's chunk size, so the builder stays one buffer.
(: int)
(def json-write-part 32768)

;; Hands the builder to `flush` once it holds `json-write-part` bytes. Called
;; between two elements or two fields.
(: (-> StringBuilder (-?-> StringBuilder void) void))
(defun (maybe-flush! sb flush) #:inline
  (when (>= (stringbuilder-length sb) json-write-part)
    (flush sb)))

;; `j` added to `sb`. `flush` is given the builder when it is full, and
;; empties it; `json->string` gives one that does nothing.
(: (-> StringBuilder Json int bool int (-?-> StringBuilder void) void))
(defun (add-value! sb j indent ascii level flush)
  (match j
    (JsonNull (stringbuilder-add-string! sb "null"))
    ((JsonBool b) (stringbuilder-add-string! sb (if b "true" "false")))
    ((JsonInt n) (stringbuilder-add-long! sb n))
    ((JsonFloat d) (add-float! sb d))
    ((JsonStr s) (add-string! sb s ascii))
    ((JsonArr items)
     (cond
       ((iarray-empty? items) (stringbuilder-add-string! sb "[]"))
       (else
        (stringbuilder-add! sb #\[)
        (loop (:for item items)
              (:with first? #t #f)
              (:do (unless first? (stringbuilder-add! sb #\,))
                   (add-newline! sb indent (+ level 1))
                   (add-value! sb item indent ascii (+ level 1) flush)
                   (maybe-flush! sb flush)))
        (add-newline! sb indent level)
        (stringbuilder-add! sb #\]))))
    ((JsonObj fields)
     (cond
       ((map-empty? fields) (stringbuilder-add-string! sb "{}"))
       (else
        ;; The entries, sorted by key once: each is written with the value it
        ;; already holds, and no key is looked up again.
        (stringbuilder-add! sb #\{)
        (loop (:for (Tuple k v) (sorted-entries fields))
              (:with first? #t #f)
              (:do (add-field! sb k v first? indent ascii level flush)))
        (add-newline! sb indent level)
        (stringbuilder-add! sb #\}))))
    ;; In their own order, which for a document read is the document's. The
    ;; same loop as a JsonObj's, written again so that each walks a type it
    ;; knows: a function generic in the collection would have a suspending
    ;; copy, which is not specialised.
    ((JsonFields fields)
     (cond
       ((iarray-empty? fields) (stringbuilder-add-string! sb "{}"))
       (else
        (stringbuilder-add! sb #\{)
        (loop (:for (Tuple k v) fields)
              (:with first? #t #f)
              (:do (add-field! sb k v first? indent ascii level flush)))
        (add-newline! sb indent level)
        (stringbuilder-add! sb #\}))))))

;; One field of an object at `level`, and the comma before it unless it is
;; the first.
(: (-> StringBuilder string Json bool int bool int (-?-> StringBuilder void) void))
(defun (add-field! sb k v first? indent ascii level flush) #:inline
  (unless first? (stringbuilder-add! sb #\,))
  (add-newline! sb indent (+ level 1))
  (add-string! sb k ascii)
  (stringbuilder-add-string! sb (if (> indent 0) ": " ":"))
  (add-value! sb v indent ascii (+ level 1) flush)
  (maybe-flush! sb flush))

;; JSON text for a value. Compact unless `#:indent` is more than 0, which puts
;; each element and field on a line of its own, indented that many spaces a
;; level. With `#:ascii #t`, the text is ASCII.
(: (-> Json (#:indent int) (#:ascii bool) string))
(defun (json->string j #:indent 0 #:ascii #f)
  (def sb (stringbuilder-empty))
  (add-value! sb j indent ascii 0 (fun (_) unit))
  (stringbuilder->string sb))

(import/extern
  (clr-write-builder       (: Bjolang.Runtime.BjoOutputPort.WriteBuilder
                              (-> OutputPort StringBuilder void) #:blocking))
  (clr-write-builder-await (: Bjolang.Runtime.BjoOutputPort.WriteBuilderAsync
                              (-> OutputPort StringBuilder void) #:async)))

;; A builder's bytes to `p`, and the builder emptied.
(: (-> OutputPort StringBuilder void))
(defbjouble (write-builder! p sb)
  (#:sync (clr-write-builder p sb) (stringbuilder-clear! sb))
  (#:bjo  (clr-write-builder-await p sb) (stringbuilder-clear! sb)))

;; The text `json->string` gives, written to `port`. It goes out in parts of
;; about `json-write-part` bytes as it is made, so no string of the whole
;; text is made. From a bjoroutine, each part's write suspends.
(: (-> OutputPort Json (#:indent int) (#:ascii bool) void))
(defun (write-json port j #:indent 0 #:ascii #f)
  (def sb (stringbuilder-empty))
  ;; A local function, not a `fun`, so that it takes the colour of the copy
  ;; it is in: in the suspending copy of `write-json`, its write suspends.
  (defun (flush b) (write-builder! port b))
  (add-value! sb j indent ascii 0 flush)
  (write-builder! port sb))

(impl (Written Json)
  (defun (written j) (json->string j))
  (defun (write-into j b) (stringbuilder-add-string! b (written j))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; text/xml.bjo — the module (text xml): names, nodes and documents.
;;
;; The tree is opaque, and its constructors enforce what every tree obeys,
;; whoever built it: no two text children side by side, no empty text child,
;; no duplicate attribute, and no namespace declaration among the attributes.
;; The reader, the writer and the modules still to come — matching, paths,
;; decoding, HTML — use only what is exported here, so the views below are
;; the whole of the access there is.
;;
;; A tree can be built by hand as deep as memory allows, and a stack overflow
;; cannot be caught in .NET. So nothing here recurses per nesting level: every
;; walk keeps its own stack, a list of the child lists still to visit.
;;
;; Its reference is its docs, in xml.bjodoc.
;;
;; Open:
;; - The printed form does not read back. SXML literals and patterns for this
;;   tree are (text xml match)'s, still to come.
;; - Only an element carries a position; a text, a comment or a PI has none,
;;   so a decoder can point no closer than the element around it.

(export
  ;; Names
  XmlName xml-name xml-name-local xml-name-ns xml-namespace
  ;; Small types
  XmlPos XmlAttrs Doctype
  ;; Nodes
  Node xml-element xml-text xml-comment xml-pi
  element? text? comment? pi?
  element-name element-attrs element-children element-pos element-view
  text-value comment-value pi-target pi-data
  node-case
  ;; Helpers
  attr-ref element-children-only text-content descendants node=?
  ;; Documents
  Document xml-document document-root document-doctype document-prolog document-epilog
  document=?)

(import (std prelude))

(import/class
  (XName (: System.Xml.Linq.XName))
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

(import/extern
  (clr-xname (: System.Xml.Linq.XName.Get (-> string string XName)))
  (clr-xname-local (: System.Xml.Linq.XName.LocalName (-> XName string) #:get))
  (clr-xname-ns (: System.Xml.Linq.XName.NamespaceName (-> XName string) #:get))
  (clr-xname-hash (: System.Xml.Linq.XName.GetHashCode (-> XName int)))
  (clr-xname-clark (: System.Xml.Linq.XName.ToString (-> XName string)))
  (clr-verify-ncname (: System.Xml.XmlConvert.VerifyNCName (-> string string)
                        #:exceptions (System.Xml.XmlException System.ArgumentNullException))))

;; Every export is documented there; the module doc is too.
(:doc #:file "xml.bjodoc")

;; An ArgumentException, which is what a programming error raises here. Never
;; an XmlException: the reader treats those as the document's errors.
(: (-> string %a))
(defun (argument-error message)
  (raise (cast Exception (ArgumentException. (str "(text xml): " message)))))

;; ===========================================================================
;; Names
;; ===========================================================================

;; An XName, which .NET interns, so two names are equal exactly when they are
;; the same object. No prefix: a prefix belongs to the text a name was read
;; from or is written to, not to the name.
(type (: XmlName #:opaque (Struct (: v XName))))

(impl (Eq XmlName)
  (defun (= a b) (eq? (struct-ref a v) (struct-ref b v)))
  (defun (eq-hash n) (clr-xname-hash (struct-ref n v))))

;; Clark notation: {uri}local, or local alone in no namespace.
(impl (Written XmlName)
  (defun (written n) (clr-xname-clark (struct-ref n v)))
  (defun (write-into n b) (stringbuilder-add-string! b (written n))))

(: string)
(def xml-namespace "http://www.w3.org/XML/1998/namespace")

;; What xmlns and xmlns:p are in. Nothing is ever named in it.
(: string)
(def xmlns-namespace "http://www.w3.org/2000/xmlns/")

(: (-> string (#:ns string) XmlName))
(defun (xml-name local #:ns "")
  (match (clr-verify-ncname local)
    ((Ok _) (XmlName (v (clr-xname local ns))))
    ((Err e) (argument-error (str "xml-name: " (.-Message e))))))

(: (-> XmlName string))
(defun (xml-name-local n) (clr-xname-local (struct-ref n v)))

;; "" when the name is in no namespace.
(: (-> XmlName string))
(defun (xml-name-ns n) (clr-xname-ns (struct-ref n v)))

;; ===========================================================================
;; Small types
;; ===========================================================================

;; As .NET reports it: 1-based, and the column of the name after the '<'.
(type (: XmlPos (Struct (: line int) (: column int))))

;; In document order.
(type (: XmlAttrs (Alias (List (Tuple XmlName string)))))

;; Kept so that a document can be written back with it. Nothing validates
;; against it.
(type (: Doctype (Record (: name string)
                         (: public-id (Option string))
                         (: system-id (Option string)))))

;; ===========================================================================
;; Nodes
;; ===========================================================================

;; The case names are long because an importer receives them along with the
;; type, although only this module may use them.
(type-rec
  (: Node #:opaque (Union (: XmlNodeElement XmlName XmlAttrs (List Node) (Option XmlPos))
                          (: XmlNodeText string)
                          (: XmlNodeComment string)
                          (: XmlNodePI string string))))

(: (-> Node string))
(defun (variant-name n)
  (match n
    ((XmlNodeElement _ _ _ _) "an element")
    ((XmlNodeText _) "a text")
    ((XmlNodeComment _) "a comment")
    ((XmlNodePI _ _) "a processing instruction")))

(: (-> string Node string %a))
(defun (wrong-variant who n wanted)
  (argument-error (str who ": the node is " (variant-name n) ", not " wanted)))

;; ---- Attributes -----------------------------------------------------------

;; Above this many attributes, duplicates are found with a map rather than by
;; comparing every pair.
(: int)
(def few-attributes 16)

(: (-> XmlName bool))
(defun (namespace-declaration? name)
  (or (= (xml-name-ns name) xmlns-namespace)
      (and (= (xml-name-ns name) "") (= (xml-name-local name) "xmlns"))))

(: (-> XmlName XmlAttrs (Option string)))
(defun (attr-value name attrs)
  (let loop ((xs attrs))
    (match xs
      (Nil None)
      ((Cons (Tuple n value) rest) (if (= n name) (Some value) (loop rest))))))

(: (-> XmlName void))
(defun (check-attr-name name)
  (when (namespace-declaration? name)
    (argument-error
      (str "xml-element: " (displayed name) " is a namespace declaration, which is not an attribute. "
           "The writer declares namespaces itself."))))

(: (-> XmlName %a))
(defun (duplicate-attr name)
  (argument-error (str "xml-element: the attribute " (displayed name) " is given twice")))

(: (-> XmlAttrs void))
(defun (check-few-attrs attrs)
  (let loop ((xs attrs))
    (match xs
      (Nil unit)
      ((Cons (Tuple name _) rest)
       (check-attr-name name)
       (when (some? (attr-value name rest)) (duplicate-attr name))
       (loop rest)))))

(: (-> XmlAttrs void))
(defun (check-many-attrs attrs)
  (def seen (transientmap-empty))
  (let loop ((xs attrs))
    (match xs
      (Nil unit)
      ((Cons (Tuple name _) rest)
       (check-attr-name name)
       (when (transientmap-contains? seen name) (duplicate-attr name))
       (transientmap-set! seen name #t)
       (loop rest)))))

(: (-> XmlAttrs void))
(defun (check-attrs attrs)
  (if (> (list-length attrs) few-attributes)
      (check-many-attrs attrs)
      (check-few-attrs attrs)))

;; ---- Children -------------------------------------------------------------

;; Whether the children already obey the invariants, in which case the list
;; is kept as it is. The parser's always do.
(: (-> (List Node) bool))
(defun (children-normal? kids)
  (let loop ((xs kids) (after-text #f))
    (match xs
      (Nil #t)
      ((Cons (XmlNodeText s) rest)
       (if (or after-text (string-empty? s)) #f (loop rest #t)))
      ((Cons _ rest) (loop rest #f)))))

(: (-> (List Node) (List Node)))
(defun (normalize-children kids)
  (def out (listbuilder-empty))
  (def pending (stringbuilder-empty))
  (defun (flush)
    (when (> (stringbuilder-length pending) 0)
      (listbuilder-add! out (XmlNodeText (stringbuilder->string pending)))
      (stringbuilder-clear! pending)))
  (let loop ((xs kids))
    (match xs
      (Nil
       (flush)
       (listbuilder->list out))
      ((Cons (XmlNodeText s) rest)
       (stringbuilder-add-string! pending s)
       (loop rest))
      ((Cons n rest)
       (flush)
       (listbuilder-add! out n)
       (loop rest)))))

;; ---- Construction ---------------------------------------------------------

;; Merges adjacent text children and drops empty ones. Raises on a duplicate
;; attribute or a namespace declaration.
(: (-> XmlName XmlAttrs (List Node) (#:pos (Option XmlPos)) Node))
(defun (xml-element name attrs children #:pos None)
  (check-attrs attrs)
  (XmlNodeElement name attrs (if (children-normal? children) children (normalize-children children)) pos))

;; Empty text is allowed on its own; an element drops it.
(: (-> string Node))
(defun (xml-text s) (XmlNodeText s))

;; Not checked here: a comment with "--" in it is refused when it is written.
(: (-> string Node))
(defun (xml-comment s) (XmlNodeComment s))

;; Target, then data. Checked when it is written, as a comment is.
(: (-> string string Node))
(defun (xml-pi target data) (XmlNodePI target data))

;; ---- Views ----------------------------------------------------------------

(: (-> Node bool))
(defun (element? n) (match n ((XmlNodeElement _ _ _ _) #t) (_ #f)))

(: (-> Node bool))
(defun (text? n) (match n ((XmlNodeText _) #t) (_ #f)))

(: (-> Node bool))
(defun (comment? n) (match n ((XmlNodeComment _) #t) (_ #f)))

(: (-> Node bool))
(defun (pi? n) (match n ((XmlNodePI _ _) #t) (_ #f)))

(: (-> Node XmlName))
(defun (element-name n)
  (match n
    ((XmlNodeElement name _ _ _) name)
    (_ (wrong-variant "element-name" n "an element"))))

(: (-> Node XmlAttrs))
(defun (element-attrs n)
  (match n
    ((XmlNodeElement _ attrs _ _) attrs)
    (_ (wrong-variant "element-attrs" n "an element"))))

(: (-> Node (List Node)))
(defun (element-children n)
  (match n
    ((XmlNodeElement _ _ kids _) kids)
    (_ (wrong-variant "element-children" n "an element"))))

(: (-> Node (Option XmlPos)))
(defun (element-pos n)
  (match n
    ((XmlNodeElement _ _ _ pos) pos)
    (_ (wrong-variant "element-pos" n "an element"))))

;; The whole element in one call. Option is a struct and Tuple a value tuple,
;; so this allocates nothing.
(: (-> Node (Option (Tuple XmlName XmlAttrs (List Node)))))
(defun (element-view n)
  (match n
    ((XmlNodeElement name attrs kids _) (Some (Tuple name attrs kids)))
    (_ None)))

(: (-> Node string))
(defun (text-value n)
  (match n
    ((XmlNodeText s) s)
    (_ (wrong-variant "text-value" n "a text"))))

(: (-> Node string))
(defun (comment-value n)
  (match n
    ((XmlNodeComment s) s)
    (_ (wrong-variant "comment-value" n "a comment"))))

(: (-> Node string))
(defun (pi-target n)
  (match n
    ((XmlNodePI target _) target)
    (_ (wrong-variant "pi-target" n "a processing instruction"))))

(: (-> Node string))
(defun (pi-data n)
  (match n
    ((XmlNodePI _ data) data)
    (_ (wrong-variant "pi-data" n "a processing instruction"))))

(: (-> Node %r))
(defun (no-clause n)
  (argument-error (str "node-case: no clause for " (variant-name n) ", and no #:else")))

;; #:else is declared first because the other defaults call it.
(: (-> Node
       (#:else (-> Node %r))
       (#:element (-> XmlName XmlAttrs (List Node) (Option XmlPos) %r))
       (#:text (-> string %r))
       (#:comment (-> string %r))
       (#:pi (-> string string %r))
       %r))
(defun (node-case n
                  #:else (fun (m) (no-clause m))
                  #:element (fun (name attrs kids pos) (else n))
                  #:text (fun (s) (else n))
                  #:comment (fun (s) (else n))
                  #:pi (fun (target data) (else n)))
  (match n
    ((XmlNodeElement name attrs kids pos) (element name attrs kids pos))
    ((XmlNodeText s) (text s))
    ((XmlNodeComment s) (comment s))
    ((XmlNodePI target data) (pi target data))))

;; ===========================================================================
;; Helpers
;; ===========================================================================

(: (-> Node XmlName (Option string)))
(defun (attr-ref n name)
  (match n
    ((XmlNodeElement _ attrs _ _) (attr-value name attrs))
    (_ None)))

(: (-> Node (List Node)))
(defun (element-children-only n)
  (match n
    ((XmlNodeElement _ _ kids _) (list-filter element? kids))
    (_ Nil)))

;; The text of every text node below, in document order. A text node's own
;; value; "" for a comment or a processing instruction.
(: (-> Node string))
(defun (text-content n)
  (match n
    ((XmlNodeText s) s)
    ((XmlNodeElement _ _ kids _)
     (def sb (stringbuilder-empty))
     (let loop ((stack (list kids)))
       (match stack
         (Nil (stringbuilder->string sb))
         ((Cons Nil rest) (loop rest))
         ((Cons (Cons (XmlNodeText s) more) rest)
          (stringbuilder-add-string! sb s)
          (loop (Cons more rest)))
         ((Cons (Cons (XmlNodeElement _ _ inner _) more) rest)
          (loop (Cons inner (Cons more rest))))
         ((Cons (Cons _ more) rest)
          (loop (Cons more rest))))))
    (_ "")))

;; Every node below, in document order, not the node itself.
(: (-> Node (Seq Node)))
(defun (descendants n)
  (def top (match n ((XmlNodeElement _ _ kids _) kids) (_ Nil)))
  (seq
    (let loop ((stack (list top)))
      (match stack
        (Nil unit)
        ((Cons Nil rest) (loop rest))
        ((Cons (Cons k more) rest)
         (yield k)
         (match k
           ((XmlNodeElement _ _ inner _) (loop (Cons inner (Cons more rest))))
           (_ (loop (Cons more rest)))))))))

;; ---- Equality -------------------------------------------------------------

;; Pair by pair, which is the usual case when the lists are equal at all.
(: (-> XmlAttrs XmlAttrs bool))
(defun (attrs-in-order=? xs ys)
  (let loop ((xs xs) (ys ys))
    (match (Tuple xs ys)
      ((Tuple Nil Nil) #t)
      ((Tuple (Cons (Tuple n1 v1) r1) (Cons (Tuple n2 v2) r2))
       (if (and (= n1 n2) (= v1 v2)) (loop r1 r2) #f))
      (_ #f))))

;; Both lists are free of duplicates, so the same length and every attribute
;; of one found in the other is the same set.
(: (-> XmlAttrs XmlAttrs bool))
(defun (attrs-found? xs ys)
  (let loop ((xs xs))
    (match xs
      (Nil #t)
      ((Cons (Tuple name value) rest)
       (if (= (attr-value name ys) (Some value)) (loop rest) #f)))))

(: (-> XmlAttrs XmlAttrs bool))
(defun (attrs-mapped? xs ys)
  (def m (transientmap-empty))
  (let fill ((ys ys))
    (match ys
      (Nil unit)
      ((Cons (Tuple name value) rest)
       (transientmap-set! m name value)
       (fill rest))))
  (let loop ((xs xs))
    (match xs
      (Nil #t)
      ((Cons (Tuple name value) rest)
       (if (= (transientmap-try-ref m name) (Some value)) (loop rest) #f)))))

;; In any order, as XML has them.
(: (-> XmlAttrs XmlAttrs bool))
(defun (attrs=? xs ys)
  (cond
    ((attrs-in-order=? xs ys) #t)
    (:def n (list-length xs))
    ((not (= n (list-length ys))) #f)
    ((> n few-attributes) (attrs-mapped? xs ys))
    (else (attrs-found? xs ys))))

;; Structure only: positions are not compared, and attributes in any order.
;; The stack holds the pairs of child lists still to compare.
(: (-> Node Node bool))
(defun (node=? a b)
  (let loop ((stack (list (Tuple (list a) (list b)))))
    (match stack
      (Nil #t)
      ((Cons (Tuple Nil Nil) rest) (loop rest))
      ((Cons (Tuple (Cons x xs) (Cons y ys)) rest)
       (match (Tuple x y)
         ((Tuple (XmlNodeElement n1 a1 k1 _) (XmlNodeElement n2 a2 k2 _))
          (if (and (= n1 n2) (attrs=? a1 a2))
              (loop (Cons (Tuple k1 k2) (Cons (Tuple xs ys) rest)))
              #f))
         ((Tuple (XmlNodeText s1) (XmlNodeText s2))
          (if (= s1 s2) (loop (Cons (Tuple xs ys) rest)) #f))
         ((Tuple (XmlNodeComment s1) (XmlNodeComment s2))
          (if (= s1 s2) (loop (Cons (Tuple xs ys) rest)) #f))
         ((Tuple (XmlNodePI t1 d1) (XmlNodePI t2 d2))
          (if (and (= t1 t2) (= d1 d2)) (loop (Cons (Tuple xs ys) rest)) #f))
         (_ #f)))
      (_ #f))))

;; Summed, so that the order of the attributes does not change it.
(: (-> XmlAttrs int))
(defun (attrs-hash attrs)
  (list-foldl (fun (a h) (def (Tuple name value) a) (+ h (hash-combine (eq-hash name) (eq-hash value))))
              0
              attrs))

;; Agrees with node=?: positions and attribute order leave it alone. The end
;; of each child list is hashed too, so that nesting is part of it.
(: (-> Node int))
(defun (node-hash n)
  (let loop ((stack (list (list n))) (h 17))
    (match stack
      (Nil h)
      ((Cons Nil rest) (loop rest (hash-combine h 1)))
      ((Cons (Cons k more) rest)
       (match k
         ((XmlNodeElement name attrs kids _)
          (loop (Cons kids (Cons more rest))
                (hash-combine (hash-combine (hash-combine h 2) (eq-hash name)) (attrs-hash attrs))))
         ((XmlNodeText s) (loop (Cons more rest) (hash-combine (hash-combine h 3) (eq-hash s))))
         ((XmlNodeComment s) (loop (Cons more rest) (hash-combine (hash-combine h 4) (eq-hash s))))
         ((XmlNodePI target data)
          (loop (Cons more rest)
                (hash-combine (hash-combine (hash-combine h 5) (eq-hash target)) (eq-hash data)))))))))

;; Without this, = on a Node would be the C# record's own Equals, which
;; recurses per nesting level and compares positions.
(impl (Eq Node)
  (defun (= a b) (node=? a b))
  (defun (eq-hash n) (node-hash n)))

;; ===========================================================================
;; Documents
;; ===========================================================================

(type (: Document #:opaque (Record (: doctype (Option Doctype))
                                   (: prolog (List Node))
                                   (: root Node)
                                   (: epilog (List Node)))))

(: (-> string (List Node) void))
(defun (check-misc where nodes)
  (list-for-each
    (fun (n)
      (unless (or (comment? n) (pi? n))
        (argument-error
          (str "xml-document: the " where " may hold comments and processing instructions, and this is "
               (variant-name n)))))
    nodes))

;; Raises if the root is not an element, or the prolog or epilog holds
;; anything but comments and processing instructions.
(: (-> Node (#:doctype (Option Doctype)) (#:prolog (List Node)) (#:epilog (List Node))
       Document))
(defun (xml-document root #:doctype None #:prolog Nil #:epilog Nil)
  (unless (element? root)
    (argument-error (str "xml-document: the root has to be an element, and this is " (variant-name root))))
  (check-misc "prolog" prolog)
  (check-misc "epilog" epilog)
  (Document (doctype doctype) (prolog prolog) (root root) (epilog epilog)))

(: (-> Document Node))
(defun (document-root d) (record-ref d root))

(: (-> Document (Option Doctype)))
(defun (document-doctype d) (record-ref d doctype))

(: (-> Document (List Node)))
(defun (document-prolog d) (record-ref d prolog))

(: (-> Document (List Node)))
(defun (document-epilog d) (record-ref d epilog))

(: (-> (List Node) (List Node) bool))
(defun (nodes=? xs ys)
  (match (Tuple xs ys)
    ((Tuple Nil Nil) #t)
    ((Tuple (Cons x xr) (Cons y yr)) (if (node=? x y) (nodes=? xr yr) #f))
    (_ #f)))

(: (-> Document Document bool))
(defun (document=? a b)
  (and (= (record-ref a doctype) (record-ref b doctype))
       (nodes=? (record-ref a prolog) (record-ref b prolog))
       (node=? (record-ref a root) (record-ref b root))
       (nodes=? (record-ref a epilog) (record-ref b epilog))))

(impl (Eq Document)
  (defun (= a b) (document=? a b))
  (defun (eq-hash d)
    (list-foldl (fun (n h) (hash-combine h (node-hash n)))
                (node-hash (record-ref d root))
                (list-append (record-ref d prolog) (record-ref d epilog)))))

;; ===========================================================================
;; The printer
;; ===========================================================================
;;
;; SXML's look, for the REPL and for debugging:
;;
;;     (p (@ (class "x")) "hi" (b "!"))
;;     (*COMMENT* " c ")  (*PI* target "data")
;;     (*TOP* (*DOCTYPE* html SYSTEM "a.dtd") ... (root ...))
;;
;; A name in a namespace is written in Clark notation, {uri}local. Nothing
;; promises yet that this reads back.

(: (-> StringBuilder string void))
(defun (add-quoted! sb s)
  (stringbuilder-add! sb #\")
  (loop (:for c s)
        (:do (case c
               ((#\") (stringbuilder-add-string! sb "\\\""))
               ((#\\) (stringbuilder-add-string! sb "\\\\"))
               ((#\newline) (stringbuilder-add-string! sb "\\n"))
               ((#\tab) (stringbuilder-add-string! sb "\\t"))
               (else (if (= (char->int c) 13)
                         (stringbuilder-add-string! sb "\\r")
                         (stringbuilder-add! sb c))))))
  (stringbuilder-add! sb #\"))

;; A node still to print, or text that closes one.
(type (: PrintItem (Union (: PrintNode Node) (: PrintText string))))

(: (-> StringBuilder Node void))
(defun (print-node! sb n)
  (let loop ((items (list (PrintNode n))))
    (match items
      (Nil unit)
      ((Cons (PrintText s) rest)
       (stringbuilder-add-string! sb s)
       (loop rest))
      ((Cons (PrintNode (XmlNodeText s)) rest)
       (add-quoted! sb s)
       (loop rest))
      ((Cons (PrintNode (XmlNodeComment s)) rest)
       (stringbuilder-add-string! sb "(*COMMENT* ")
       (add-quoted! sb s)
       (stringbuilder-add! sb #\))
       (loop rest))
      ((Cons (PrintNode (XmlNodePI target data)) rest)
       (stringbuilder-add-string! sb (str "(*PI* " target " "))
       (add-quoted! sb data)
       (stringbuilder-add! sb #\))
       (loop rest))
      ((Cons (PrintNode (XmlNodeElement name attrs kids _)) rest)
       (stringbuilder-add! sb #\()
       (write-into name sb)
       (unless (list-empty? attrs)
         (stringbuilder-add-string! sb " (@")
         ;; todo: rewrite as loop. 
         (let each ((as attrs))
           (match as
             (Nil unit)
             ((Cons (Tuple an value) more)
              (stringbuilder-add-string! sb " (")
              (write-into an sb)
              (stringbuilder-add! sb #\space)
              (add-quoted! sb value)
              (stringbuilder-add! sb #\))
              (each more))))
         (stringbuilder-add! sb #\)))
       (loop (list-foldr (fun (k acc) (Cons (PrintText " ") (Cons (PrintNode k) acc)))
                         (Cons (PrintText ")") rest)
                         kids))))))

(impl (Written Node)
  (defun (write-into n b) (print-node! b n)))

(: (-> StringBuilder Doctype void))
(defun (print-doctype! sb dt)
  (stringbuilder-add-string! sb (str "(*DOCTYPE* " (get dt .name)))
  (match (Tuple (get dt .public-id) (get dt .system-id))
    ((Tuple (Some pub) (Some sys))
     (stringbuilder-add-string! sb " PUBLIC ")
     (add-quoted! sb pub)
     (stringbuilder-add! sb #\space)
     (add-quoted! sb sys))
    ((Tuple (Some pub) None)
     (stringbuilder-add-string! sb " PUBLIC ")
     (add-quoted! sb pub))
    ((Tuple None (Some sys))
     (stringbuilder-add-string! sb " SYSTEM ")
     (add-quoted! sb sys))
    ((Tuple None None) unit))
  (stringbuilder-add! sb #\)))

(impl (Written Document)
  (defun (write-into d sb)
    (stringbuilder-add-string! sb "(*TOP*")
    (match (get d .doctype)
      ((Some dt)
       (stringbuilder-add! sb #\space)
       (print-doctype! sb dt))
      (None unit))
    (list-for-each (fun (n)
                     (stringbuilder-add! sb #\space)
                     (print-node! sb n))
                   (list-append (get d .prolog) (Cons (get d .root) (get d .epilog))))
    (stringbuilder-add! sb #\))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; text/xml/read.bjo — the module (text xml read).
;;
;; Four layers, each built on the one below:
;;
;;   1. signals: a pull stream, as OCaml's Xmlm has it
;;   2. xml-fold: Kiselyov's SSAX fold, generic in its seed
;;   3. xml-tree-fold: a tree of the caller's own type
;;   4. parse-xml: a Document; and xml-for-each, one element at a time
;;
;; Only the signal layer calls XmlReader, so its quirks stay there: an empty
;; element's missing end tag, text in several pieces, namespace declarations
;; among the attributes. It is also where XmlException is caught and becomes an
;; XmlError, around the reader calls and nothing else, so an exception a
;; handler raises passes through as it was.
;;
;; A fold that fails at a node has already run its handlers for every node
;; before it. That is what streaming means.
;;
;; Its reference is its docs, in read.bjodoc.
;;
;; Open:
;; - #:dtd 'parse refuses a DOCTYPE that names an external DTD, because
;;   XmlResolver.ThrowingResolver refuses to open it. So an XHTML document
;;   reads only under 'ignore, where XmlReader does not report the DOCTYPE at
;;   all and the Document has no doctype. .NET 10's default resolver would skip
;;   an external DTD silently instead, and expand an external entity to
;;   nothing, which keeps the doctype but quietly changes content.
;; - A DOCTYPE refused and a limit reached have line and column 0: .NET throws
;;   them without a position, and has reset the reader's own by then.
;; - xml-skip-element! does not check what it skips against #:max-depth.
;;   XmlReader's own node stack grows with that depth, bounded only by
;;   #:max-characters.
;; - #:whitespace 'drop drops whitespace-only text inside xml:space="preserve"
;;   as well.
;; - A try compiles to a closure and a delegate, so each signal costs two small
;;   allocations. A try emitted as a statement would remove them.
;; - Every element and attribute read makes its name again: a VerifyNCName and
;;   an XName.Get. XmlReader hands out the same string objects for a name it
;;   has seen, so a small cache keyed on them would save both.
;; - Reading is synchronous. An async variant over XmlReader.ReadAsync belongs
;;   in the signal layer, where the reader is, and nowhere above it.

(export
  ;; Options
  DtdMode WhitespaceMode
  ;; Errors
  XmlError XmlErrorKind
  ;; Sources
  XmlInput XmlSource xml-input XmlFile xml-file
  ;; Signals
  XmlSignal XmlSignals call-with-xml-signals xml-next! xml-skip-element!
  ;; Folds
  xml-fold xml-tree-fold
  ;; Documents
  parse-xml xml-for-each)

(import (std prelude))
(import (text xml))

(import/class
  (XmlReader (: System.Xml.XmlReader))
  (XmlReaderSettings (: System.Xml.XmlReaderSettings (-> XmlReaderSettings)))
  (XmlResolver (: System.Xml.XmlResolver))
  (DtdProcessing (: System.Xml.DtdProcessing))
  (XmlNodeType (: System.Xml.XmlNodeType))
  (IXmlLineInfo (: System.Xml.IXmlLineInfo))
  (XmlException (: System.Xml.XmlException))
  (Stream (: System.IO.Stream))
  (TextReader (: System.IO.TextReader))
  (Object (: System.Object))
  (InvalidOperationException (: System.InvalidOperationException (-> string InvalidOperationException)))
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

(import/extern
  ;; For a string, read as text: there is no encoding to declare.
  (clr-reader-over-text (: System.Xml.XmlReader.Create (-> TextReader XmlReaderSettings XmlReader)))
  (clr-reader-over-stream (: System.Xml.XmlReader.Create (-> Stream XmlReaderSettings XmlReader)))
  (clr-settings-dtd! (: System.Xml.XmlReaderSettings.DtdProcessing (-> XmlReaderSettings DtdProcessing void) #:set))
  (clr-settings-resolver! (: System.Xml.XmlReaderSettings.XmlResolver (-> XmlReaderSettings XmlResolver void) #:set))
  (clr-settings-entity-characters! (: System.Xml.XmlReaderSettings.MaxCharactersFromEntities
                                      (-> XmlReaderSettings long void) #:set))
  (clr-settings-document-characters! (: System.Xml.XmlReaderSettings.MaxCharactersInDocument
                                        (-> XmlReaderSettings long void) #:set))
  (clr-settings-close-input! (: System.Xml.XmlReaderSettings.CloseInput (-> XmlReaderSettings bool void) #:set))
  (clr-settings-ignore-comments! (: System.Xml.XmlReaderSettings.IgnoreComments
                                    (-> XmlReaderSettings bool void) #:set))
  (clr-throwing-resolver (: System.Xml.XmlResolver.ThrowingResolver #:get))
  (clr-dtd-prohibit (: System.Xml.DtdProcessing.Prohibit #:get))
  (clr-dtd-ignore (: System.Xml.DtdProcessing.Ignore #:get))
  (clr-dtd-parse (: System.Xml.DtdProcessing.Parse #:get))
  (clr-read (: System.Xml.XmlReader.Read (-> XmlReader bool)))
  (clr-skip (: System.Xml.XmlReader.Skip (-> XmlReader void)))
  (clr-node-type (: System.Xml.XmlReader.NodeType (-> XmlReader XmlNodeType) #:get))
  (clr-local-name (: System.Xml.XmlReader.LocalName (-> XmlReader string) #:get))
  (clr-namespace (: System.Xml.XmlReader.NamespaceURI (-> XmlReader string) #:get))
  (clr-name (: System.Xml.XmlReader.Name (-> XmlReader string) #:get))
  (clr-value (: System.Xml.XmlReader.Value (-> XmlReader string) #:get))
  (clr-depth (: System.Xml.XmlReader.Depth (-> XmlReader int) #:get))
  (clr-empty-element? (: System.Xml.XmlReader.IsEmptyElement (-> XmlReader bool) #:get))
  (clr-next-attribute (: System.Xml.XmlReader.MoveToNextAttribute (-> XmlReader bool)))
  (clr-to-element (: System.Xml.XmlReader.MoveToElement (-> XmlReader bool)))
  ;; Kept a .NET string, whose null says the attribute is absent.
  (clr-attribute (: System.Xml.XmlReader.GetAttribute (-> XmlReader string System.String)))
  (clr-has-line-info (: System.Xml.IXmlLineInfo.HasLineInfo (-> IXmlLineInfo bool)))
  (clr-line-number (: System.Xml.IXmlLineInfo.LineNumber (-> IXmlLineInfo int) #:get))
  (clr-line-position (: System.Xml.IXmlLineInfo.LinePosition (-> IXmlLineInfo int) #:get))
  (clr-exception-line (: System.Xml.XmlException.LineNumber (-> XmlException int) #:get))
  (clr-exception-column (: System.Xml.XmlException.LinePosition (-> XmlException int) #:get))
  (clr-inner-exception (: System.Exception.InnerException (-> Exception Exception) #:get))
  (clr-port-stream (: Bjolang.Runtime.BjoInputPort.AsStream (-> InputPort Stream))))

;; Every export is documented there; the module doc is too.
(:doc #:file "read.bjodoc")

;; ===========================================================================
;; Options
;; ===========================================================================

;; 'prohibit refuses a DOCTYPE. 'ignore skips it, and XmlReader then does not
;; report it at all, so the document has no doctype. 'parse reads it, never
;; resolving anything external, and expands at most 10 000 000 characters of
;; entities.
(def/grammar DtdMode
  (:or (: DtdProhibit 'prohibit) (: DtdIgnore 'ignore) (: DtdParse 'parse)))

;; 'drop leaves out text that is only whitespace, which includes the space in
;; <b>x</b> <i>y</i>.
(def/grammar WhitespaceMode
  (:or (: KeepWhitespace 'keep) (: DropWhitespace 'drop)))

(type (: Options (Record (: dtd DtdMode)
                         (: max-depth int)
                         (: max-characters long)
                         (: drop-whitespace bool)
                         (: comments bool))))

(: long)
(def entity-characters 10000000L)

(: (-> DtdMode int long WhitespaceMode bool Options))
(defun (options dtd max-depth max-characters whitespace comments)
  (when (< max-depth 1)
    (raise (cast Exception (ArgumentException. "(text xml read): #:max-depth has to be at least 1"))))
  (when (< max-characters 0L)
    (raise (cast Exception (ArgumentException. "(text xml read): #:max-characters cannot be negative"))))
  (Options (dtd dtd)
           (max-depth max-depth)
           (max-characters max-characters)
           (drop-whitespace (match whitespace (KeepWhitespace #f) (DropWhitespace #t)))
           (comments comments)))

(: (-> Options XmlReaderSettings))
(defun (reader-settings opts)
  (def settings (XmlReaderSettings.))
  (clr-settings-close-input! settings #f)
  (clr-settings-ignore-comments! settings (not (record-ref opts comments)))
  (clr-settings-document-characters! settings (record-ref opts max-characters))
  (clr-settings-resolver! settings clr-throwing-resolver)
  (match (record-ref opts dtd)
    (DtdProhibit (clr-settings-dtd! settings clr-dtd-prohibit))
    (DtdIgnore (clr-settings-dtd! settings clr-dtd-ignore))
    (DtdParse
     (clr-settings-dtd! settings clr-dtd-parse)
     (clr-settings-entity-characters! settings entity-characters)))
  settings)

;; ===========================================================================
;; Errors
;; ===========================================================================

(def/grammar XmlErrorKind
  (:or (: Malformed 'malformed)
       (: DtdProhibited 'dtd-prohibited)
       (: TooDeep 'too-deep)
       (: LimitExceeded 'limit-exceeded)))

;; line and column are 0 when neither the exception nor the reader knew them.
(type (: XmlError (Record (: kind XmlErrorKind) (: message string) (: line int) (: column int))))

(: (-> XmlErrorKind string))
(defun (kind-name k)
  (match k
    (Malformed "malformed")
    (DtdProhibited "dtd-prohibited")
    (TooDeep "too-deep")
    (LimitExceeded "limit-exceeded")))

(impl (Displayed XmlError)
  (defun (displayed e)
    (str (kind-name (record-ref e kind))
         " at line " (int->string (record-ref e line))
         ", column " (int->string (record-ref e column))
         ": " (record-ref e message)))
  (defun (display-into e b) (stringbuilder-add-string! b (displayed e))))

(: (-> XmlErrorKind string (Option XmlPos) XmlError))
(defun (xml-error kind message pos)
  (match pos
    ((Some at) (XmlError (kind kind) (message message) (line (struct-ref at line)) (column (struct-ref at column))))
    (None (XmlError (kind kind) (message message) (line 0) (column 0)))))

;; The message .NET gives for `text` read with `settings`, or "" if it reads.
(: (-> string XmlReaderSettings string))
(defun (reference-message text settings)
  (match (try (with-open ((r (clr-reader-over-text (cast TextReader (open-input-string text)) settings)))
                (let loop () (when (clr-read r) (loop))))
              #:catch (System.Xml.XmlException))
    ((Ok _) "")
    ((Err e) (.-Message e))))

;; A DOCTYPE, a document longer than its limit, and entities longer than
;; theirs are XmlExceptions like any other, told apart only by their message
;; and thrown with no position. So their messages are compared with the ones
;; .NET gives for three tiny documents, made to fail the same way. Exactly, and
;; not by searching for a word: a malformed document can quote any word back.
(: (-> string))
(defun (dtd-prohibited-message)
  (def settings (XmlReaderSettings.))
  (clr-settings-dtd! settings clr-dtd-prohibit)
  (reference-message "<!DOCTYPE a><a/>" settings))

(: (-> string))
(defun (document-limit-message)
  (def settings (XmlReaderSettings.))
  (clr-settings-document-characters! settings 2L)
  (reference-message "<a>0123456789</a>" settings))

(: (-> string))
(defun (entity-limit-message)
  (def settings (XmlReaderSettings.))
  (clr-settings-dtd! settings clr-dtd-parse)
  (clr-settings-resolver! settings clr-throwing-resolver)
  (clr-settings-entity-characters! settings 2L)
  (reference-message "<!DOCTYPE a [<!ENTITY e '0123456789'>]><a>&e;</a>" settings))

;; ThrowingResolver's refusal arrives wrapped in the reader's own exception.
(: (-> XmlException bool))
(defun (refused-resource? e)
  (match (cast Object (clr-inner-exception (cast Exception e)))
    ((:is System.Xml.XmlException _) #t)
    (_ #f)))

(: (-> DtdMode XmlException XmlErrorKind))
(defun (exception-kind dtd e)
  (def message (.-Message e))
  (cond
    ((= message (dtd-prohibited-message)) DtdProhibited)
    ((= message (document-limit-message)) LimitExceeded)
    ((= message (entity-limit-message)) LimitExceeded)
    ((and (match dtd (DtdParse #t) (_ #f)) (refused-resource? e)) DtdProhibited)
    (else Malformed)))

(: (-> (Option IXmlLineInfo) (Option XmlPos)))
(defun (line-info-pos info)
  (match info
    ((Some li)
     (if (clr-has-line-info li)
         (Some (XmlPos (line (clr-line-number li)) (column (clr-line-position li))))
         None))
    (None None)))

;; Where the exception says. A DOCTYPE refused, a limit reached and a resource
;; refused are thrown with no position, and by then the reader has reset its
;; own to 0 as well, so those have none.
(: (-> DtdMode XmlException XmlError))
(defun (exception-error dtd e)
  (def line (clr-exception-line e))
  (xml-error (exception-kind dtd e)
             (.-Message e)
             (if (> line 0)
                 (Some (XmlPos (line line) (column (clr-exception-column e))))
                 None)))

;; ===========================================================================
;; Sources
;; ===========================================================================

;; Declared before the trait that names it.
(type (: XmlInput #:opaque (Union (: InText string)
                                  (: InStream Stream)
                                  (: InFile string))))

;; What can be read. A type of your own gets an implementation by handing
;; xml-input something that already has one.
(def/trait (XmlSource %s)
  (: xml-input (-> %s XmlInput)))

;; A path, which the reader opens and closes. A string on its own is XML.
(type (: XmlFile #:opaque (Record (: path string))))

(: (-> string XmlFile))
(defun (xml-file path) (XmlFile (path path)))

(impl (XmlSource string)
  (defun (xml-input s) (InText s)))

;; Bytes, whose encoding XmlReader works out: a byte order mark, or the
;; encoding the XML declaration names, or UTF-8.
(impl (XmlSource Stream)
  (defun (xml-input s) (InStream s)))

;; A port is bytes too, read the same way. Through the port's own stream view,
;; so bytes it has already buffered are not lost.
(impl (XmlSource InputPort)
  (defun (xml-input p) (InStream (clr-port-stream p))))

(impl (XmlSource XmlFile)
  (defun (xml-input f) (InFile (record-ref f path))))

;; ===========================================================================
;; 1. Signals
;; ===========================================================================

(type (: XmlSignal (Union (: XmlStart XmlName XmlAttrs (Option XmlPos))
                          XmlEnd
                          (: XmlData string)
                          (: XmlComment string)
                          (: XmlPI string string)
                          (: XmlDoctype Doctype))))

;; `held` means the reader stands on a node that has not been reported yet,
;; because merging text had to read one node past it. `owed-end` is the end of
;; an empty element, which XmlReader never reports.
(type (: XmlSignals #:opaque (Record (: reader (Option XmlReader))
                                     (: line-info (Option IXmlLineInfo))
                                     (: options Options)
                                     (: held bool #:mutable)
                                     (: owed-end bool #:mutable)
                                     (: after-start bool #:mutable)
                                     (: finished bool #:mutable)
                                     (: failed (Option XmlError) #:mutable))))

(: (-> (Option XmlReader) Options (Option XmlError) XmlSignals))
(defun (signals reader opts failed)
  (XmlSignals (reader reader)
              (line-info (match reader
                           ((Some r) (match (cast Object r)
                                       ((:is System.Xml.IXmlLineInfo li) (Some li))
                                       (_ None)))
                           (None None)))
              (options opts)
              (held #f)
              (owed-end #f)
              (after-start #f)
              (finished #f)
              (failed failed)))

;; XmlNodeType, as numbers.
(: int)
(def element-node 1)
(: int)
(def text-node 3)
(: int)
(def cdata-node 4)
(: int)
(def pi-node 7)
(: int)
(def comment-node 8)
(: int)
(def doctype-node 10)
(: int)
(def whitespace-node 13)
(: int)
(def significant-whitespace-node 14)
(: int)
(def end-element-node 15)

(: (-> XmlReader int))
(defun (node-code r) (cast int (clr-node-type r)))

(: (-> int bool))
(defun (text-code? code)
  (or (= code text-node) (= code cdata-node) (= code whitespace-node) (= code significant-whitespace-node)))

(: string)
(def xmlns-namespace "http://www.w3.org/2000/xmlns/")

;; Space, tab, line feed and carriage return: XML's whitespace, and no other.
(: (-> string bool))
(defun (whitespace-only? s)
  (string-fold (fun (c all) (and all (let ((n (char->int c))) (or (= n 32) (= n 9) (= n 10) (= n 13)))))
               #t
               s))

(: (-> XmlReader string (Option string)))
(defun (attribute-opt r name)
  (match (cast Object (clr-attribute r name))
    ((:is System.String s) (Some (clr-string->string s)))
    (_ None)))

;; In document order, without the namespace declarations, and back on the
;; element afterwards.
(: (-> XmlReader XmlAttrs))
(defun (read-attrs r)
  (def out (listbuilder-empty))
  (let loop ()
    (when (clr-next-attribute r)
      (def ns (clr-namespace r))
      (unless (= ns xmlns-namespace)
        (listbuilder-add! out (Tuple (xml-name (clr-local-name r) #:ns ns) (clr-value r))))
      (loop)))
  (ignore (clr-to-element r))
  (listbuilder->list out))

(: (-> XmlSignals XmlReader (Result XmlError (Option XmlSignal))))
(defun (start-element s r)
  (def pos (line-info-pos (record-ref s line-info)))
  (def max-depth (record-ref (record-ref s options) max-depth))
  (cond
    ((>= (clr-depth r) max-depth)
     (Err (xml-error TooDeep
                     (str "the elements nest deeper than #:max-depth, " (int->string max-depth))
                     pos)))
    (else
     ;; Before the attributes: on an attribute, IsEmptyElement is false.
     (def empty (clr-empty-element? r))
     (def name (xml-name (clr-local-name r) #:ns (clr-namespace r)))
     (def attrs (read-attrs r))
     (record-set! s (owed-end empty) (after-start #t))
     (Ok (Some (XmlStart name attrs pos))))))

;; Text, CDATA and whitespace, from the current node to the next node that is
;; none of them, as one string. The reader is left on that node.
(: (-> XmlSignals XmlReader string))
(defun (read-data s r)
  (def first (clr-value r))
  (cond
    ((not (clr-read r)) (record-set! s (finished #t)) first)
    ((not (text-code? (node-code r))) (record-set! s (held #t)) first)
    (else
     (def sb (stringbuilder-empty))
     (stringbuilder-add-string! sb first)
     (let loop ()
       (stringbuilder-add-string! sb (clr-value r))
       (cond
         ((not (clr-read r)) (record-set! s (finished #t)))
         ((text-code? (node-code r)) (loop))
         (else (record-set! s (held #t)))))
     (stringbuilder->string sb))))

(: (-> XmlSignals XmlReader (Result XmlError (Option XmlSignal))))
(defun (read-signal s r)
  (let loop ()
    (cond
      ((and (not (record-ref s held)) (not (clr-read r)))
       (record-set! s (finished #t))
       (Ok None))
      (:do (record-set! s (held #f)))
      (:def code (node-code r))
      ((= code element-node) (start-element s r))
      ((= code end-element-node) (Ok (Some XmlEnd)))
      ;; Outside the root there is only whitespace, and it is not data.
      ((and (text-code? code) (= (clr-depth r) 0)) (loop))
      ((text-code? code)
       (def data (read-data s r))
       (if (and (record-ref (record-ref s options) drop-whitespace) (whitespace-only? data))
           (loop)
           (Ok (Some (XmlData data)))))
      ((= code comment-node) (Ok (Some (XmlComment (clr-value r)))))
      ((= code pi-node) (Ok (Some (XmlPI (clr-name r) (clr-value r)))))
      ((= code doctype-node)
       (Ok (Some (XmlDoctype (Doctype (name (clr-name r))
                                      (public-id (attribute-opt r "PUBLIC"))
                                      (system-id (attribute-opt r "SYSTEM")))))))
      ;; The XML declaration.
      (else (loop)))))

(: (-> XmlSignals XmlError (Result XmlError %a)))
(defun (fail s e)
  (record-set! s (failed (Some e)))
  (Err e))

(: (-> XmlSignals Exception XmlError))
(defun (caught s e)
  (match (cast Object e)
    ((:is System.Xml.XmlException x)
     (exception-error (record-ref (record-ref s options) dtd) x))
    (_ (raise e))))

;; The next signal, or None at the end of the document. After an error, the
;; same error again.
(: (-> XmlSignals (Result XmlError (Option XmlSignal))))
(defun (xml-next! s)
  (cond
    (:match (Some e) (record-ref s failed) (Err e))
    ((record-ref s owed-end)
     (record-set! s (owed-end #f) (after-start #f))
     (Ok (Some XmlEnd)))
    (:do (record-set! s (after-start #f)))
    ((record-ref s finished) (Ok None))
    (:match (Some r) (record-ref s reader)
     (match (try (read-signal s r) #:catch (System.Xml.XmlException))
       ((Ok (Err e)) (fail s e))
       ((Ok signal) signal)
       ((Err e) (fail s (caught s e)))))
    (else (Ok None))))

;; Right after an XmlStart: skips the rest of that element, its XmlEnd
;; included. Anywhere else it raises. The skipped part is not checked against
;; #:max-depth.
(: (-> XmlSignals (Result XmlError Unit)))
(defun (xml-skip-element! s)
  (cond
    (:match (Some e) (record-ref s failed) (Err e))
    ((not (record-ref s after-start))
     (raise (cast Exception (InvalidOperationException.
                              "(text xml read): xml-skip-element! is for right after an XmlStart"))))
    (:do (record-set! s (after-start #f)))
    ((record-ref s owed-end) (record-set! s (owed-end #f)) (Ok unit))
    (:match (Some r) (record-ref s reader)
     (match (try (let ()
                   (clr-skip r)
                   (if (= (node-code r) 0)
                       (record-set! s (finished #t))
                       (record-set! s (held #t))))
                 #:catch (System.Xml.XmlException))
       ((Ok _) (Ok unit))
       ((Err e) (fail s (caught s e)))))
    (else (Ok unit))))

;; XmlReader.Create reads ahead, so it can fail too. Then the first xml-next!
;; answers with the error.
(: (-> (-> XmlReaderSettings XmlReader) Options (-> XmlSignals %a) %a))
(defun (run-signals create opts proc)
  (def settings (reader-settings opts))
  (def s (match (try (create settings) #:catch (System.Xml.XmlException))
           ((Ok r) (signals (Some r) opts None))
           ((Err e) (match (cast Object e)
                      ((:is System.Xml.XmlException x)
                       (signals None opts (Some (exception-error (record-ref opts dtd) x))))
                      (_ (raise e))))))
  (try (proc s)
    #:finally (match (record-ref s reader)
                ((Some r) (.Dispose r))
                (None unit))))

;; A port or stream that was handed over stays open. A file is opened here,
;; through the FS effect, and closed here.
(: (-> XmlInput Options (-> XmlSignals %a) %a))
(defun (with-signals input opts proc)
  (match input
    ((InText text)
     (def port (open-input-string text))
     (run-signals (fun (settings) (clr-reader-over-text (cast TextReader port) settings)) opts proc))
    ((InStream stream)
     (run-signals (fun (settings) (clr-reader-over-stream stream settings)) opts proc))
    ((InFile path)
     (match (open-read-stream path)
       ((Ok opened)
        (with-open ((stream opened))
          (run-signals (fun (settings) (clr-reader-over-stream stream settings)) opts proc)))
       ((Err e) (raise e))))))

;; The XmlSignals is for proc's use, and closed when proc returns.
(: (-> %s (-> XmlSignals %a)
       (#:dtd DtdMode) (#:max-depth int) (#:max-characters long)
       (#:whitespace WhitespaceMode) (#:comments? bool)
       %a)
   (where (XmlSource %s)))
(defun (call-with-xml-signals source proc
                              #:dtd 'prohibit #:max-depth 512 #:max-characters 0L
                              #:whitespace 'keep #:comments? #f)
  (with-signals (xml-input source) (options dtd max-depth max-characters whitespace comments?) proc))

;; ===========================================================================
;; 2. The SSAX fold
;; ===========================================================================

;; An element still open: what finish is called with when it closes.
(type (: (Frame %seed) (Record (: name XmlName)
                               (: attrs XmlAttrs)
                               (: pos (Option XmlPos))
                               (: parent %seed))))

;; One iteration per signal, as a loop rather than a call: a flat document of a
;; million elements is a million iterations. The open elements are the list.
(: (-> XmlSignals %seed
       (-> XmlName XmlAttrs (Option XmlPos) %seed %seed)
       (-> XmlName XmlAttrs (Option XmlPos) %seed %seed %seed)
       (-> string %seed %seed)
       (-> string %seed %seed)
       (-> string string %seed %seed)
       (-> Doctype %seed %seed)
       (Result XmlError %seed)))
(defun (fold-signals sig seed new-level finish text comment pi doctype)
  (let loop ((seed seed) (open Nil))
    (match (xml-next! sig)
      ((Err e) (Err e))
      ((Ok None) (Ok seed))
      ((Ok (Some (XmlStart name attrs pos)))
       (loop (new-level name attrs pos seed)
             (Cons (Frame (name name) (attrs attrs) (pos pos) (parent seed)) open)))
      ((Ok (Some XmlEnd))
       (match open
         ((Cons f rest)
          (loop (finish (record-ref f name) (record-ref f attrs) (record-ref f pos) (record-ref f parent) seed)
                rest))
         (Nil (Ok seed))))
      ((Ok (Some (XmlData s))) (loop (text s seed) open))
      ((Ok (Some (XmlComment s))) (loop (comment s seed) open))
      ((Ok (Some (XmlPI target data))) (loop (pi target data seed) open))
      ((Ok (Some (XmlDoctype dt))) (loop (doctype dt seed) open)))))

;; Kiselyov's SSAX: the seed is threaded through the document. Descending into
;; an element, new-level turns the seed into the element's own; leaving it,
;; finish gets the seed from before the element and the element's final seed,
;; and answers the seed to carry on with. Every handler passes the seed
;; through unless given.
(: (-> %s %seed
       (#:new-level (-> XmlName XmlAttrs (Option XmlPos) %seed %seed))
       (#:finish (-> XmlName XmlAttrs (Option XmlPos) %seed %seed %seed))
       (#:text (-> string %seed %seed))
       (#:comment (-> string %seed %seed))
       (#:pi (-> string string %seed %seed))
       (#:doctype (-> Doctype %seed %seed))
       (#:dtd DtdMode) (#:max-depth int) (#:max-characters long)
       (#:whitespace WhitespaceMode)
       (Result XmlError %seed))
   (where (XmlSource %s)))
(defun (xml-fold source seed
                 #:new-level (fun (name attrs pos seed) seed)
                 #:finish (fun (name attrs pos parent seed) seed)
                 #:text (fun (s seed) seed)
                 #:comment (fun (s seed) seed)
                 #:pi (fun (target data seed) seed)
                 #:doctype (fun (dt seed) seed)
                 #:dtd 'prohibit #:max-depth 512 #:max-characters 0L #:whitespace 'keep)
  (with-signals (xml-input source) (options dtd max-depth max-characters whitespace #t)
    (fun (sig) (fold-signals sig seed new-level finish text comment pi doctype))))

;; ===========================================================================
;; 3. The tree fold
;; ===========================================================================

;; The seed is the children so far, last first, and whether this is the level
;; outside the root, where only the root is kept.
(: (-> (Option %a) (Tuple (List %a) bool) (Tuple (List %a) bool)))
(defun (keep-child value seed)
  (def (Tuple kids top) seed)
  (match value
    ((Some a) #:when (not top) (Tuple (Cons a kids) top))
    (_ seed)))

(: (-> XmlInput
       (-> XmlName XmlAttrs (Option XmlPos) (List %a) %a)
       (-> string %a)
       (-> string (Option %a))
       (-> string string (Option %a))
       Options
       (Result XmlError %a)))
(defun (tree-fold input element text comment pi opts)
  (def folded
    (with-signals input opts
      (fun (sig)
        (fold-signals sig (Tuple Nil #t)
                      (fun (name attrs pos seed) (Tuple Nil #f))
                      (fun (name attrs pos parent seed)
                        (def (Tuple kids _) seed)
                        (def (Tuple siblings top) parent)
                        (Tuple (Cons (element name attrs pos (list-reverse kids)) siblings) top))
                      (fun (s seed)
                        (def (Tuple kids top) seed)
                        (Tuple (Cons (text s) kids) top))
                      (fun (s seed) (keep-child (comment s) seed))
                      (fun (target data seed) (keep-child (pi target data) seed))
                      (fun (dt seed) seed)))))
  (match folded
    ((Err e) (Err e))
    ((Ok (Tuple (Cons root _) _)) (Ok root))
    ((Ok _) (Err (xml-error Malformed "the document has no root element" None)))))

;; A tree of the caller's own type, as OCaml's Xmlm builds with input_tree:
;; element gets an element's name, attributes, position and folded children,
;; and text a piece of text. A comment or PI handler answers None to leave the
;; node out, which is what the defaults do. The answer is the root element's;
;; what is outside the root is not folded.
(: (-> %s
       (-> XmlName XmlAttrs (Option XmlPos) (List %a) %a)
       (-> string %a)
       (#:comment (-> string (Option %a)))
       (#:pi (-> string string (Option %a)))
       (#:dtd DtdMode) (#:max-depth int) (#:max-characters long)
       (#:whitespace WhitespaceMode)
       (Result XmlError %a))
   (where (XmlSource %s)))
(defun (xml-tree-fold source element text
                      #:comment (fun (s) None)
                      #:pi (fun (target data) None)
                      #:dtd 'prohibit #:max-depth 512 #:max-characters 0L #:whitespace 'keep)
  (tree-fold (xml-input source) element text comment pi
             (options dtd max-depth max-characters whitespace #t)))

;; ===========================================================================
;; 4. The whole document
;; ===========================================================================

;; Outside the root: what the Document is made of, last first.
(type (: DocumentParts (Record (: doctype (Option Doctype))
                               (: prolog (List Node))
                               (: root (Option Node))
                               (: epilog (List Node)))))

(type (: DocumentSeed (Union (: AtTop DocumentParts)
                             (: InElement (List Node)))))

(: (-> Node DocumentSeed DocumentSeed))
(defun (add-node n seed)
  (match seed
    ((InElement kids) (InElement (Cons n kids)))
    ((AtTop parts)
     (match (record-ref parts root)
       (None (AtTop (record-set parts (prolog (Cons n (record-ref parts prolog))))))
       ((Some _) (AtTop (record-set parts (epilog (Cons n (record-ref parts epilog))))))))))

(: (-> XmlName XmlAttrs (Option XmlPos) DocumentSeed DocumentSeed DocumentSeed))
(defun (finish-element name attrs pos parent seed)
  (def kids (match seed ((InElement kids) (list-reverse kids)) ((AtTop _) Nil)))
  (def element (xml-element name attrs kids #:pos pos))
  (match parent
    ((InElement siblings) (InElement (Cons element siblings)))
    ((AtTop parts) (AtTop (record-set parts (root (Some element)))))))

(: (-> DocumentSeed (Result XmlError Document)))
(defun (document-of seed)
  (match seed
    ((AtTop (DocumentParts doctype prolog (root (Some root)) epilog))
     (Ok (xml-document root
                       #:doctype doctype
                       #:prolog (list-reverse prolog)
                       #:epilog (list-reverse epilog))))
    (_ (Err (xml-error Malformed "the document has no root element" None)))))

(: (-> XmlInput Options (Result XmlError Document)))
(defun (parse-document input opts)
  (def start (AtTop (DocumentParts (doctype None) (prolog Nil) (root None) (epilog Nil))))
  (def folded
    (with-signals input opts
      (fun (sig)
        (fold-signals sig start
                      (fun (name attrs pos seed) (InElement Nil))
                      finish-element
                      (fun (s seed) (add-node (xml-text s) seed))
                      (fun (s seed) (add-node (xml-comment s) seed))
                      (fun (target data seed) (add-node (xml-pi target data) seed))
                      (fun (dt seed)
                        (match seed
                          ((AtTop parts) (AtTop (record-set parts (doctype (Some dt)))))
                          (_ seed)))))))
  (match folded
    ((Err e) (Err e))
    ((Ok seed) (document-of seed))))

;; Processing instructions are always kept, and comments when #:comments?
;; says so, in the prolog and epilog as in the tree.
(: (-> %s
       (#:dtd DtdMode) (#:max-depth int) (#:max-characters long)
       (#:whitespace WhitespaceMode) (#:comments? bool)
       (Result XmlError Document))
   (where (XmlSource %s)))
(defun (parse-xml source
                  #:dtd 'prohibit #:max-depth 512 #:max-characters 0L
                  #:whitespace 'keep #:comments? #f)
  (parse-document (xml-input source) (options dtd max-depth max-characters whitespace comments?)))

;; ===========================================================================
;; 5. One element at a time
;; ===========================================================================

;; An element still open while one is being built, and its children so far,
;; last first.
(type (: Building (Record (: name XmlName)
                          (: attrs XmlAttrs)
                          (: pos (Option XmlPos))
                          (: kids (List Node)))))

(: (-> Node (List Building) (List Building)))
(defun (add-child n open)
  (match open
    ((Cons top rest) (Cons (record-set top (kids (Cons n (record-ref top kids)))) rest))
    (Nil Nil)))

;; The rest of the element whose XmlStart was just read, as a Node. The open
;; elements are the list, so this is a loop however deep the element goes.
(: (-> XmlSignals XmlName XmlAttrs (Option XmlPos) (Result XmlError Node)))
(defun (build-element sig name attrs pos)
  (let loop ((open (list (Building (name name) (attrs attrs) (pos pos) (kids Nil)))))
    (match (xml-next! sig)
      ((Err e) (Err e))
      ((Ok None) (Err (xml-error Malformed "the document ended inside an element" None)))
      ((Ok (Some (XmlStart n a p)))
       (loop (Cons (Building (name n) (attrs a) (pos p) (kids Nil)) open)))
      ((Ok (Some XmlEnd))
       (match open
         ((Cons top rest)
          (def element (xml-element (record-ref top name) (record-ref top attrs)
                                    (list-reverse (record-ref top kids))
                                    #:pos (record-ref top pos)))
          (match rest
            (Nil (Ok element))
            (_ (loop (add-child element rest)))))
         (Nil (Err (xml-error Malformed "an end with no start" None)))))
      ((Ok (Some (XmlData s))) (loop (add-child (xml-text s) open)))
      ((Ok (Some (XmlComment s))) (loop (add-child (xml-comment s) open)))
      ((Ok (Some (XmlPI target data))) (loop (add-child (xml-pi target data) open)))
      ((Ok (Some (XmlDoctype _))) (loop open)))))

(: (-> XmlSignals XmlName (-> Node void) (Result XmlError Unit)))
(defun (for-each-element sig name f)
  (let loop ()
    (match (xml-next! sig)
      ((Err e) (Err e))
      ((Ok None) (Ok unit))
      ((Ok (Some (XmlStart n attrs pos)))
       (cond
         ((not (= n name)) (loop))
         (:def (Ok element) (build-element sig n attrs pos) :propagate)
         (else
          (f element)
          (loop))))
      ((Ok (Some _)) (loop)))))

;; Burst mode: a Node for each outermost element with this name, handed to f
;; and then dropped, so what is held is one such element and never the whole
;; document. An element with this name inside one is part of it, and not
;; visited on its own.
(: (-> %s XmlName (-> Node void)
       (#:dtd DtdMode) (#:max-depth int) (#:max-characters long)
       (#:whitespace WhitespaceMode) (#:comments? bool)
       (Result XmlError Unit))
   (where (XmlSource %s)))
(defun (xml-for-each source name f
                     #:dtd 'prohibit #:max-depth 512 #:max-characters 0L
                     #:whitespace 'keep #:comments? #f)
  (with-signals (xml-input source) (options dtd max-depth max-characters whitespace comments?)
    (fun (sig) (for-each-element sig name f))))
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; As a special exception to the Mozilla Public License, version 2.0, if you
;; compile your application source code and portions of this software are
;; embedded into the generated object code or executable form as a normal
;; consequence of the compilation process (such as inline functions,
;; templates, generics, or macros), you may redistribute such embedded portions
;; in such object code or executable form without complying with the source code
;; availability requirements or notice obligations of Section 3 of the MPL 2.0.

;; text/xml/write.bjo — the module (text xml write).
;;
;; System.Xml.XmlWriter does the escaping. What it does not do is refuse all
;; content it cannot write: a comment with "--" in it, PI data with "?>" in it
;; and a PI named xml are silently changed or dropped rather than refused, so
;; those are checked here first and are an XmlWriteError like the rest.
;;
;; A name carries no prefix, so the prefixes are chosen here: one already in
;; scope for the namespace, then one the caller hinted at, then a default
;; namespace declaration for an element, or a generated ns1, ns2 ... for an
;; attribute, which the default namespace does not apply to. XmlWriter writes
;; the declarations the chosen prefixes need.
;;
;; The tree is walked with a stack of its own, and indentation is written by
;; hand: see `indents?`.
;;
;; Its reference is its docs, in write.bjodoc.
;;
;; Open:
;; - The doctype is written with WriteRaw, after checking its parts, because
;;   WriteDocType takes a null for an id that is not there and there is no
;;   null to pass it.
;; - A doctype's internal subset is not kept by the reader, so it is not
;;   written back. Its entities were expanded while reading, but an attribute
;;   default it declared is then an ordinary attribute of every element it
;;   applied to.
;; - What was written to a port before an error stays written, and nothing
;;   says how far it got.
;; - #:prefixes may give one prefix twice. The first one is used, and the
;;   namespace of the second gets a generated prefix where the two meet.

(export
  XmlWriteError
  node->xml node->xml-string
  document->xml document->xml-string)

(import (std prelude))
(import (text xml))

(import/class
  (XmlWriter (: System.Xml.XmlWriter))
  ;; What `XmlWriter.Create` takes, and what an output port also is.
  (TextWriter (: System.IO.TextWriter))
  (XmlWriterSettings (: System.Xml.XmlWriterSettings (-> XmlWriterSettings)))
  (ConformanceLevel (: System.Xml.ConformanceLevel))
  (ClrBuilder (: System.Text.StringBuilder (-> ClrBuilder)))
  (ArgumentException (: System.ArgumentException (-> string ArgumentException))))

(import/extern
  (clr-writer-over-text (: System.Xml.XmlWriter.Create (-> TextWriter XmlWriterSettings XmlWriter)))
  (clr-writer-over-builder (: System.Xml.XmlWriter.Create (-> ClrBuilder XmlWriterSettings XmlWriter)))
  (clr-settings-omit-declaration! (: System.Xml.XmlWriterSettings.OmitXmlDeclaration
                                     (-> XmlWriterSettings bool void) #:set))
  (clr-settings-conformance! (: System.Xml.XmlWriterSettings.ConformanceLevel
                                (-> XmlWriterSettings ConformanceLevel void) #:set))
  (clr-settings-close-output! (: System.Xml.XmlWriterSettings.CloseOutput (-> XmlWriterSettings bool void) #:set))
  (clr-settings-end-on-close! (: System.Xml.XmlWriterSettings.WriteEndDocumentOnClose
                                 (-> XmlWriterSettings bool void) #:set))
  (clr-conformance-document (: System.Xml.ConformanceLevel.Document #:get))
  (clr-conformance-fragment (: System.Xml.ConformanceLevel.Fragment #:get))
  (clr-start-element (: System.Xml.XmlWriter.WriteStartElement (-> XmlWriter string string string void)))
  (clr-end-element (: System.Xml.XmlWriter.WriteEndElement (-> XmlWriter void)))
  (clr-attribute (: System.Xml.XmlWriter.WriteAttributeString (-> XmlWriter string string string string void)))
  (clr-string (: System.Xml.XmlWriter.WriteString (-> XmlWriter string void)))
  (clr-comment (: System.Xml.XmlWriter.WriteComment (-> XmlWriter string void)))
  (clr-pi (: System.Xml.XmlWriter.WriteProcessingInstruction (-> XmlWriter string string void)))
  (clr-whitespace (: System.Xml.XmlWriter.WriteWhitespace (-> XmlWriter string void)))
  (clr-raw (: System.Xml.XmlWriter.WriteRaw (-> XmlWriter string void)))
  (clr-flush (: System.Xml.XmlWriter.Flush (-> XmlWriter void)))
  (clr-verify-ncname (: System.Xml.XmlConvert.VerifyNCName (-> string string)
                        #:exceptions (System.Xml.XmlException System.ArgumentNullException)))
  (clr-verify-name (: System.Xml.XmlConvert.VerifyName (-> string string)
                      #:exceptions (System.Xml.XmlException System.ArgumentNullException)))
  (clr-verify-public-id (: System.Xml.XmlConvert.VerifyPublicId (-> string string)
                           #:exceptions (System.Xml.XmlException System.ArgumentNullException)))
  (clr-verify-xml-chars (: System.Xml.XmlConvert.VerifyXmlChars (-> string string)
                           #:exceptions (System.Xml.XmlException System.ArgumentNullException))))

;; Every export is documented there; the module doc is too.
(:doc #:file "write.bjodoc")

;; ===========================================================================
;; Errors
;; ===========================================================================

(type (: XmlWriteError (Record (: message string))))

(impl (Displayed XmlWriteError)
  (defun (displayed e) (record-ref e message))
  (defun (display-into e b) (stringbuilder-add-string! b (record-ref e message))))

(: (-> string (Result XmlWriteError %a)))
(defun (write-error message)
  (Err (XmlWriteError (message message))))

(: (Result XmlWriteError Unit))
(def done (Ok unit))

;; ===========================================================================
;; Content XmlWriter would change rather than refuse
;; ===========================================================================

(: (-> string (Option string)))
(defun (comment-problem text)
  (cond
    ((string-contains? text "--") (Some "a comment may not contain \"--\""))
    ((string-ends-with? text "-") (Some "a comment may not end in \"-\""))
    (else None)))

(: (-> string string (Option string)))
(defun (pi-problem target data)
  (cond
    ((err? (clr-verify-ncname target))
     (Some (str "\"" target "\" is not a processing instruction target")))
    ((= (string-downcase target) "xml")
     (Some "a processing instruction may not be named xml; the XML declaration is #:declaration?"))
    ((string-contains? data "?>") (Some "a processing instruction's data may not contain \"?>\""))
    (else None)))

;; ===========================================================================
;; Namespaces
;; ===========================================================================

(: string)
(def xmlns-namespace "http://www.w3.org/2000/xmlns/")

;; The declarations in scope, innermost first, as (prefix . uri): one list per
;; open element that declared anything, so that looking a prefix up costs the
;; number of declaring elements around it rather than the depth. The default
;; namespace is the prefix "".
(type (: Scope (Alias (List (List (Tuple string string))))))

;; Prefix to URI, as #:prefixes gives them.
(type (: Hints (Alias (List (Tuple string string)))))

(: (-> (List (Tuple string string)) string (Option string)))
(defun (bound-in frame prefix)
  (match frame
    (Nil None)
    ((Cons (Tuple p u) rest) (if (= p prefix) (Some u) (bound-in rest prefix)))))

;; What a prefix means here. "xml" always means the xml namespace, and the
;; default namespace is none until something declares one.
(: (-> Scope string (Option string)))
(defun (scope-uri scope prefix)
  (if (= prefix "xml")
      (Some xml-namespace)
      (let loop ((frames scope))
        (match frames
          (Nil (if (= prefix "") (Some "") None))
          ((Cons frame rest)
           (match (bound-in frame prefix)
             ((Some u) (Some u))
             (None (loop rest))))))))

;; A prefix that means `uri` here, and is not hidden by a nearer declaration of
;; the same prefix. The default namespace only when `default-ok`.
(: (-> Scope string bool (Option string)))
(defun (scope-prefix scope uri default-ok)
  (let frames ((fs scope))
    (match fs
      (Nil None)
      ((Cons frame rest)
       (let bindings ((bs frame))
         (match bs
           (Nil (frames rest))
           ((Cons (Tuple p u) more)
            (if (and (= u uri)
                     (or default-ok (not (string-empty? p)))
                     (= (scope-uri scope p) (Some uri)))
                (Some p)
                (bindings more)))))))))

(: (-> Hints string (Option string)))
(defun (hint-prefix hints uri)
  (match hints
    (Nil None)
    ((Cons (Tuple p u) rest) (if (= u uri) (Some p) (hint-prefix rest uri)))))

(: (-> Hints string bool))
(defun (hinted? hints prefix)
  (match hints
    (Nil #f)
    ((Cons (Tuple p _) rest) (or (= p prefix) (hinted? rest prefix)))))

;; A prefix that nothing here and nothing hinted uses: ns1, ns2 and so on.
(: (-> Scope Hints string))
(defun (fresh-prefix scope hints)
  (let loop ((n 1))
    (def p (str "ns" (int->string n)))
    (if (or (some? (scope-uri scope p)) (hinted? hints p))
        (loop (+ n 1))
        p)))

;; An element's prefix, and the declaration it needs, if any. An element in no
;; namespace inside a default namespace needs xmlns="".
(: (-> Scope Hints string (Tuple string (List (Tuple string string)))))
(defun (element-prefix scope hints uri)
  (cond
    ((string-empty? uri)
     (Tuple "" (if (= (scope-uri scope "") (Some "")) Nil (list (Tuple "" "")))))
    (:match (Some p) (scope-prefix scope uri #t) (Tuple p Nil))
    (:match (Some p) (hint-prefix hints uri) (Tuple p (list (Tuple p uri))))
    (else (Tuple "" (list (Tuple "" uri))))))

;; An attribute's prefix, given the declarations the element has made so far.
;; A hinted prefix this element already uses for another namespace cannot be
;; declared again on it, so a generated one is used instead.
(: (-> Scope (List (Tuple string string)) Hints string
       (Tuple string (List (Tuple string string)))))
(defun (attribute-prefix scope frame hints uri)
  (def here (Cons frame scope))
  (cond
    ((string-empty? uri) (Tuple "" frame))
    ((= uri xml-namespace) (Tuple "xml" frame))
    (:match (Some p) (scope-prefix here uri #f) (Tuple p frame))
    (:match (Some p) (hint-prefix hints uri) #:when (none? (bound-in frame p))
     (Tuple p (Cons (Tuple p uri) frame)))
    (else
     (def p (fresh-prefix here hints))
     (Tuple p (Cons (Tuple p uri) frame)))))

;; Every namespace a name in the tree is in.
(: (-> Node (TransientMap string bool)))
(defun (used-namespaces root)
  (def used (transientmap-empty))
  (let loop ((stack (list (list root))))
    (match stack
      (Nil used)
      ((Cons Nil rest) (loop rest))
      ((Cons (Cons n more) rest)
       (match (element-view n)
         ((Some (Tuple name attrs kids))
          (transientmap-set! used (xml-name-ns name) #t)
          (list-for-each (fun (a) (def (Tuple an _) a) (transientmap-set! used (xml-name-ns an) #t)) attrs)
          (loop (Cons kids (Cons more rest))))
         (None (loop (Cons more rest))))))))

;; A bad hint is a programming error, as a bad name is.
(: (-> Hints void))
(defun (check-hints hints)
  (list-for-each
    (fun (h)
      (def (Tuple p u) h)
      (when (or (err? (clr-verify-ncname p)) (= p "xml") (= p "xmlns"))
        (raise (cast Exception (ArgumentException. (str "(text xml write): #:prefixes: \"" p "\" cannot be a prefix")))))
      (when (or (string-empty? u) (= u xml-namespace) (= u xmlns-namespace))
        (raise (cast Exception (ArgumentException. (str "(text xml write): #:prefixes: \"" p "\" cannot name \"" u "\""))))))
    hints))

;; ===========================================================================
;; The walk
;; ===========================================================================

;; A node to write, or an element to close. `newline` is whether an indent
;; goes before it, and `mixed` whether its subtree may have whitespace added.
;; An element closing says whether it put declarations in scope, which are
;; then taken out again.
(type (: Step (Union (: Visit Node int bool bool)
                     (: Leave int bool bool))))

(: XmlName)
(def xml-space (xml-name "space" #:ns xml-namespace))

(: (-> (List Node) bool))
(defun (has-text? kids)
  (match kids
    (Nil #f)
    ((Cons k rest) (or (text? k) (has-text? rest)))))

;; Whether an element's children go on lines of their own. Never inside a
;; subtree where whitespace would be content: an element with text among its
;; children, what is below one, and xml:space="preserve".
(: (-> Node (List Node) bool bool))
(defun (indents? n kids mixed)
  (and (not mixed)
       (not (list-empty? kids))
       (not (has-text? kids))
       (not (= (attr-ref n xml-space) (Some "preserve")))))

(: (-> int string))
(defun (newline-indent depth)
  (str "\n" (string-pad-left "" (* 2 depth))))

;; The start tag and the attributes, and the declarations it made. The root
;; also declares the hinted prefixes for the namespaces the tree uses.
(: (-> XmlWriter Scope Hints Hints XmlName XmlAttrs (List (Tuple string string))))
(defun (write-start w scope hints root-hints name attrs)
  (def uri (xml-name-ns name))
  (def (Tuple prefix declared) (element-prefix scope hints uri))
  (clr-start-element w prefix (xml-name-local name) uri)
  (def with-hints
    (list-foldl (fun (h frame)
                  (def (Tuple p u) h)
                  (cond
                    ((some? (bound-in frame p)) frame)
                    (else
                     (clr-attribute w "xmlns" p xmlns-namespace u)
                     (Cons (Tuple p u) frame))))
                declared
                root-hints))
  (list-foldl (fun (a frame)
                (def (Tuple an value) a)
                (def auri (xml-name-ns an))
                (def (Tuple ap frame2) (attribute-prefix scope frame hints auri))
                (clr-attribute w ap (xml-name-local an) auri value)
                frame2)
              with-hints
              attrs))

(: (-> XmlWriter Node bool Hints (Result XmlWriteError Unit)))
(defun (write-tree w root indent hints)
  (def root-hints
    (if (list-empty? hints)
        Nil
        (let ((used (used-namespaces root)))
          (list-filter (fun (h) (def (Tuple _ u) h) (transientmap-contains? used u)) hints))))
  (let loop ((steps (list (Visit root 0 #f (not indent)))) (scope Nil) (first #t))
    (match steps
      (Nil done)
      ((Cons (Leave depth newline declared) rest)
       (when newline (clr-whitespace w (newline-indent depth)))
       (clr-end-element w)
       (loop rest (match scope ((Cons _ outer) #:when declared outer) (_ scope)) #f))
      ((Cons (Visit n depth newline mixed) rest)
       (when newline (clr-whitespace w (newline-indent depth)))
       (cond
         ((text? n)
          (clr-string w (text-value n))
          (loop rest scope #f))
         ((comment? n)
          (match (comment-problem (comment-value n))
            ((Some problem) (write-error problem))
            (None (clr-comment w (comment-value n)) (loop rest scope #f))))
         ((pi? n)
          (match (pi-problem (pi-target n) (pi-data n))
            ((Some problem) (write-error problem))
            (None (clr-pi w (pi-target n) (pi-data n)) (loop rest scope #f))))
         (else
          (def kids (element-children n))
          (def frame (write-start w scope hints (if first root-hints Nil) (element-name n) (element-attrs n)))
          (def inside (indents? n kids mixed))
          (def declared (not (list-empty? frame)))
          (loop (list-append (list-map (fun (k) (Visit k (+ depth 1) inside (not inside))) kids)
                             (Cons (Leave depth inside declared) rest))
                (if declared (Cons frame scope) scope)
                #f)))))))

;; ===========================================================================
;; Documents
;; ===========================================================================

;; Written raw, because WriteDocType takes null for an id that is not there and
;; there is no null to pass. Nothing is escaped: each part is checked instead,
;; and the system id quoted with whichever quote it does not contain.
(: (-> Doctype (Result XmlWriteError string)))
(defun (doctype-markup dt)
  (def name (record-ref dt name))
  (def system-literal
    (match (record-ref dt system-id)
      (None (Ok None))
      ((Some sys)
       (cond
         ((err? (clr-verify-xml-chars sys)) (write-error "the doctype's system id has a character XML does not allow"))
         ((not (string-contains? sys "\"")) (Ok (Some (str "\"" sys "\""))))
         ((not (string-contains? sys "'")) (Ok (Some (str "'" sys "'"))))
         (else (write-error "the doctype's system id has both kinds of quote in it"))))))
  (cond
    ((err? (clr-verify-name name)) (write-error (str "\"" name "\" is not a doctype name")))
    (:def (Ok sys) system-literal :propagate)
    (else
     (match (Tuple (record-ref dt public-id) sys)
       ((Tuple None None) (Ok (str "<!DOCTYPE " name ">")))
       ((Tuple None (Some s)) (Ok (str "<!DOCTYPE " name " SYSTEM " s ">")))
       ((Tuple (Some pub) (Some s))
        (if (err? (clr-verify-public-id pub))
            (write-error "the doctype's public id has a character a public id may not")
            (Ok (str "<!DOCTYPE " name " PUBLIC \"" pub "\" " s ">"))))
       ((Tuple (Some _) None) (write-error "a doctype with a public id needs a system id too"))))))

(type (: TopItem (Union (: TopDoctype Doctype) (: TopNode Node))))

(: (-> Document (List TopItem)))
(defun (top-items doc)
  (def nodes (list-map TopNode (list-append (document-prolog doc)
                                            (Cons (document-root doc) (document-epilog doc)))))
  (match (document-doctype doc)
    ((Some dt) (Cons (TopDoctype dt) nodes))
    (None nodes)))

(: (-> XmlWriter Document bool Hints bool (Result XmlWriteError Unit)))
(defun (write-document w doc indent hints declaration)
  (when declaration (clr-pi w "xml" "version=\"1.0\""))
  (let loop ((items (top-items doc)) (first (not declaration)))
    (match items
      (Nil done)
      ((Cons item rest)
       (when (and indent (not first)) (clr-whitespace w "\n"))
       (def (Ok _)
         (match item
           ((TopDoctype dt)
            (def (Ok markup) (doctype-markup dt) :propagate)
            (clr-raw w markup)
            done)
           ((TopNode n) (write-tree w n indent (if (element? n) hints Nil))))
         :propagate)
       (loop rest #f)))))

;; ===========================================================================
;; Running a writer
;; ===========================================================================

(: (-> bool bool XmlWriterSettings))
(defun (writer-settings document declaration)
  (def settings (XmlWriterSettings.))
  (clr-settings-omit-declaration! settings (not declaration))
  (clr-settings-conformance! settings (if document clr-conformance-document clr-conformance-fragment))
  ;; The port is the caller's, and a write that failed half-way is not ended
  ;; with end tags nobody asked for.
  (clr-settings-close-output! settings #f)
  (clr-settings-end-on-close! settings #f)
  settings)

;; What XmlWriter refuses is an ArgumentException, an XmlException or an
;; InvalidOperationException. Anything else, an IOException from the port say,
;; passes through. What was written before the error stays written.
(: (-> XmlWriter (-> XmlWriter (Result XmlWriteError Unit)) (Result XmlWriteError Unit)))
(defun (run-writer writer body)
  (match (try (with-open ((w writer))
                (def result (body w))
                (when (ok? result) (clr-flush w))
                result)
              #:catch (System.ArgumentException System.Xml.XmlException System.InvalidOperationException))
    ((Ok result) result)
    ((Err e) (write-error (.-Message e)))))

(: (-> Node OutputPort (#:indent? bool) (#:prefixes (List (Tuple string string)))
       (Result XmlWriteError Unit)))
(defun (node->xml node port #:indent? #f #:prefixes Nil)
  (check-hints prefixes)
  (run-writer (clr-writer-over-text (cast TextWriter port) (writer-settings #f #f))
              (fun (w) (write-tree w node indent? prefixes))))

(: (-> Node (#:indent? bool) (#:prefixes (List (Tuple string string)))
       (Result XmlWriteError string)))
(defun (node->xml-string node #:indent? #f #:prefixes Nil)
  (check-hints prefixes)
  (def sb (ClrBuilder.))
  (match (run-writer (clr-writer-over-builder sb (writer-settings #f #f))
                     (fun (w) (write-tree w node indent? prefixes)))
    ((Ok _) (Ok (.ToString sb)))
    ((Err e) (Err e))))

;; #:declaration? writes <?xml version="1.0"?>, with no encoding: which one the
;; text ends up in is the port's business.
(: (-> Document OutputPort (#:indent? bool) (#:prefixes (List (Tuple string string)))
       (#:declaration? bool)
       (Result XmlWriteError Unit)))
(defun (document->xml doc port #:indent? #f #:prefixes Nil #:declaration? #t)
  (check-hints prefixes)
  (run-writer (clr-writer-over-text (cast TextWriter port) (writer-settings #t declaration?))
              (fun (w) (write-document w doc indent? prefixes declaration?))))

(: (-> Document (#:indent? bool) (#:prefixes (List (Tuple string string)))
       (#:declaration? bool)
       (Result XmlWriteError string)))
(defun (document->xml-string doc #:indent? #f #:prefixes Nil #:declaration? #t)
  (check-hints prefixes)
  (def sb (ClrBuilder.))
  (match (run-writer (clr-writer-over-builder sb (writer-settings #t declaration?))
                     (fun (w) (write-document w doc indent? prefixes declaration?)))
    ((Ok _) (Ok (.ToString sb)))
    ((Err e) (Err e))))
