commit 53a00029076f9a8820dc2e0d180b75e6340b30ae9564e485c747abf56db51908 Author: Linus Björnstam Date: Tue Nov 17 11:14:39 2026 +0100 keyscore: a scorer, optimizer and GUI for keyboard layouts Scores layouts for ISO Swedish with the angle mod on Swedish and English text, by Michael Capewell's rules plus row motion, triples, pinky load and finger distance. - src/main.bjo: the report. src/optimize.bjo: simulated annealing. - src/gui.bjo: a bjogui window for tuning the numbers, with help on every number, presets and saved layouts. - corpus/: the texts, and fetch.py for the Wikipedia ones. Co-Authored-By: Claude Opus 5.5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ca84534 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.bjo/ +publish/ +*.dll +*.exe +*.pdb +*.bjobuild +*.runtimeconfig.json +*.deps.json diff --git a/corpus/bjolang.txt b/corpus/bjolang.txt new file mode 100644 index 0000000..33b0964 --- /dev/null +++ b/corpus/bjolang.txt @@ -0,0 +1,55690 @@ +;; bjo bench — benchmark files built, and run: as they are, from one menu over +;; several, or two versions of one against each other. +;; +;; What `bjo.bjo` owns, building a file and starting a program that has the +;; console, comes in as a `BenchTools`, so that this file does not import that +;; one and that one stays the command line and the compiler processes. +;; +;; A benchmark program is `(std bench)`'s, and what it does by itself — the +;; menu, the table, `--save` and `--compare` — is not repeated here. What is +;; here is what one program cannot do on its own: a menu over several files, +;; and an A/B run, which drives two programs over `--serve` and alternates +;; their samples. + +(export BenchProgram BenchTools cmd-bench split-bench-args) + +(import (std prelude)) +(import (std bench)) +(import (std bench store)) +(import (std run)) +(import (std random)) +(import (text bjodat-core)) +(import "project.bjo") + +(import/extern + (clr-file-name (: System.IO.Path.GetFileName (-> string string))) + (clr-temp-path (: System.IO.Path.GetTempPath (-> string))) + (clr-full-path (: System.IO.Path.GetFullPath (-> string string))) + (clr-relative-path (: System.IO.Path.GetRelativePath (-> string string string))) + (clr-change-extension (: System.IO.Path.ChangeExtension (-> string string string))) + (clr-tick-count (: System.Environment.TickCount64 (-> long) #:get)) + (clr-process-id (: System.Environment.ProcessId (-> int) #:get)) + (console-input-redirected? (: System.Console.IsInputRedirected (-> bool) #:get)) + (console-output-redirected? (: System.Console.IsOutputRedirected (-> bool) #:get)) + (error-message (: System.Exception.Message (-> Exception string) #:get))) + +;; A benchmark file built, ready to run. +(type (: BenchProgram (Record (: source string) (: exe string)))) + +;; What `bjo.bjo` lends: building a file (an exit code when it fails, said +;; already), running a program on the console, and finding a source file. +(type (: BenchTools (Record (: build (-> string (Result int BenchProgram))) + (: exec (-> string (Vec string) int)) + (: resolve (-> string (Option string)))))) + +(: (-> BenchProgram string)) +(defun (program-exe p) (record-ref p exe)) + +(: (-> BenchProgram string)) +(defun (program-source p) (record-ref p source)) + +(: (-> string string)) +(defun (file-name-of path) (clr-file-name path)) + +(: (-> void)) +(defun (flush!) (flush-port (parameter-ref current-output-port))) + +;; --------------------------------------------------------------------------- +;; The arguments +;; --------------------------------------------------------------------------- + +;; The flags of a benchmark program, and of an A/B run, that take a value, so +;; that the value is not taken for a file or a name. +(def bench-value-flags + (list "--save" "--compare" "--params" "--min-samples" "--max-time" "--alpha" + "--threshold" "--fail-slower" "--cases" "--alloc" + "--against" "-N" "--processes" "--seed" "--against-warmup" "--against-sample" + "--against-build")) + +(: (-> string bool)) +(defun (value-flag? a) (any? (fun (f) (= f a)) bench-value-flags)) + +;; The benchmark files among the arguments, and what goes to the programs. +(: (-> (-> string (Option string)) (List string) (Tuple (List string) (List string)))) +(defun (split-bench-args resolve args) + (let go ((left args) (files (list)) (passed (list))) + (match left + (Nil (Tuple (list-reverse files) (list-reverse passed))) + ((Cons a more) + (match (Tuple (value-flag? a) more) + ((Tuple #t (Cons v after)) (go after files (cons v (cons a passed)))) + (_ + (match (if (string-ends-with? a ".bjo") (resolve a) None) + ((Some file) (go more (cons file files) passed)) + (None (go more files (cons a passed)))))))))) + +;; Whether the arguments leave the choice to a menu: nothing named, and no +;; flag that says what to run or that the program is not run for a person. +(: (-> (List string) bool)) +(defun (bench-menu-args? passed) + (let go ((left passed)) + (match left + (Nil #t) + ((Cons a more) + (cond + ((value-flag? a) (go (match more ((Cons _ after) after) (Nil Nil)))) + ((or (not (string-starts-with? a "-")) + (any? (fun (f) (= f a)) (list "--all" "--list" "--serve"))) + #f) + (else (go more))))))) + +(: (-> bool)) +(defun (bench-at-terminal?) + (or (= (get-environment-variable "BJO_BENCH_MENU") (Some "1")) + (and (not console-input-redirected?) (not console-output-redirected?) + (none? (get-environment-variable "CI"))))) + +(: (-> (List string) string bool)) +(defun (has-flag? args flag) (any? (fun (a) (= a flag)) args)) + +;; --------------------------------------------------------------------------- +;; The command +;; --------------------------------------------------------------------------- + +;; `bjo bench`: each benchmark file built in Release and run with the +;; arguments that are not files. One file is run as it is: at a terminal it +;; asks what to run itself. Several files, at a terminal and with nothing +;; named, get one menu of all their benchmarks, numbered across the files +;; under a heading for each; each file is then run with `--cases` and the +;; cases of the choice that are its own. Otherwise each file is run in turn +;; with the arguments, a table each. With `--against REF`, the one file is +;; run against itself as it is at `REF`. +(: cmd-bench #:sync (-> BenchTools (List string) (List string) int)) +(defun (cmd-bench tools files passed) + (with-return ret + (when (has-flag? passed "--against") + (ret (match files + ((Cons file Nil) (cmd-against tools file passed)) + (_ (fail "bjo bench: --against compares one benchmark file; name one."))))) + (def programs + (loop (:for file files) + (:let built ((record-ref tools build) file)) + (:do (match built ((Err status) (ret status)) (_ unit))) + (:when-let (Ok p) built) + (:acc out (listing p)))) + (def exec (record-ref tools exec)) + (match programs + ((Cons p Nil) (exec "dotnet" (list->vec (cons (program-exe p) passed)))) + (_ + (when (or (has-flag? passed "--save") (has-flag? passed "--compare")) + (ret (fail "bjo bench: --save and --compare take one benchmark file; name one."))) + (if (and (bench-menu-args? passed) (bench-at-terminal?)) + (bench-files-menu tools programs passed) + (let () + (def statuses + (loop (:for p programs) + (:do (println "") + (println #"bjo bench: ${(file-name-of (program-source p))}") + (flush!)) + (:acc out (listing (exec "dotnet" (list->vec (cons (program-exe p) passed))))))) + (if (any? (fun (s) (not (= s 0))) statuses) 1 0))))))) + +;; --------------------------------------------------------------------------- +;; What a program has +;; --------------------------------------------------------------------------- + +;; One listed case: its number, its benchmark's name, labels, group, `once` +;; and skip. +(type (: Listed (Record (: index int) + (: name string) + (: labels (List (Tuple string string))) + (: group string) + (: once int) + (: skip string)))) + +(: (-> Listed string)) +(defun (listed-name l) (record-ref l name)) + +(: (-> Listed (List (Tuple string string)))) +(defun (listed-labels l) (record-ref l labels)) + +(: (-> Listed int)) +(defun (listed-index l) (record-ref l index)) + +(: (-> Listed string)) +(defun (listed-skip l) (record-ref l skip)) + +;; What a case is known by on both sides: its name and its parameters. +(: (-> Listed string)) +(defun (listed-key l) + (string-join (list->vec (cons (listed-name l) + (list-map (fun (t) (match t ((Tuple p v) (str p "=" v)))) (listed-labels l)))) + " ")) + +(: (-> (List Bjodat) string (Option Bjodat))) +(defun (clause-of clauses name) + (match clauses + ((Cons (BjoList (Cons (BjoSym s) (Cons v Nil))) more) + (if (= (symbol->string s) name) (Some v) (clause-of more name))) + ((Cons _ more) (clause-of more name)) + (Nil None))) + +(: (-> Bjodat (Option Listed))) +(defun (listed-of d) + (match d + ((BjoList (Cons (BjoSym _) (Cons (BjoInt i) clauses))) + (def (Some (BjoStr name)) (clause-of clauses "name") :leave-with None) + (def labels + (match (find (fun (c) (match c ((BjoList (Cons (BjoSym s) _)) (= (symbol->string s) "labels")) (_ #f))) + clauses) + ((Some (BjoList (Cons _ pairs))) + (loop (:for p pairs) + (:acc (listing (match p + ((BjoList (Cons (BjoStr a) (Cons (BjoStr b) Nil))) (Tuple a b)) + (_ (Tuple "" ""))))))) + (_ (list)))) + (Some (Listed (index (cast System.Int32 i)) (name name) (labels labels) + (group (match (clause-of clauses "group") ((Some (BjoStr g)) g) (_ ""))) + (once (match (clause-of clauses "once") ((Some (BjoInt n)) (cast System.Int32 n)) (_ 0))) + (skip (match (clause-of clauses "skip") ((Some (BjoStr s)) s) (_ "")))))) + (_ None))) + +;; The cases of a `(cases ...)` reply. +(: (-> Bjodat (Option (List Listed)))) +(defun (listed-cases-of reply) + (match reply + ((BjoList (Cons (BjoSym head) cases)) + (if (= (symbol->string head) "cases") + (Some (loop (:for c cases) (:when-let (Some l) (listed-of c)) (:acc ls (listing l)))) + None)) + (_ None))) + +;; What a benchmark program has, from its `--serve` `(list)`, asked once. +(: listed-cases #:sync (-> BenchProgram (List string) (Result string (List Listed)))) +(defun (listed-cases p passed) + (def commands (str (program-exe p) ".serve-input")) + (file-write-text commands "(list)\n(quit)\n") + (def answer (run/output '(from-file ,commands (dotnet [,(program-exe p) "--serve" ,@(list->vec passed)])))) + (file-delete commands) + (match answer + ((Ok (Tuple 0 text)) + (match (parse-bjodat (string-trim text)) + ((Ok reply) + (match (listed-cases-of reply) + ((Some ls) (Ok ls)) + (None (Err #"bjo bench: ${(program-source p)} did not list its benchmarks: ${(string-trim text)}")))) + (_ (Err #"bjo bench: ${(program-source p)} did not list its benchmarks: ${(string-trim text)}")))) + (_ (Err #"bjo bench: ${(program-source p)} could not be asked for its benchmarks.")))) + +;; --------------------------------------------------------------------------- +;; One menu over several files +;; --------------------------------------------------------------------------- + +(: (-> Bench string)) +(defun (bench-name b) (record-ref b name)) + +(: (-> Bench string)) +(defun (bench-module b) (record-ref b module)) + +(: (-> string (List Listed) (List Listed))) +(defun (take-while-name name xs) + (match xs + ((Cons x more) (if (= (listed-name x) name) (cons x (take-while-name name more)) (list))) + (Nil (list)))) + +(: (-> int (List %a) (List %a))) +(defun (drop-count n xs) + (match xs + ((Cons _ more) #:when (> n 0) (drop-count (- n 1) more)) + (_ xs))) + +;; The listed cases of one file as benchmarks, for `(std bench)`'s menu: one +;; per run of cases of one name, under a heading of the file and its group. +;; `module` is the program, and a case's labels find its number again. +(: (-> BenchProgram (List Listed) (List Bench))) +(defun (listed-benches p listed) + (def shown (file-name-of (program-source p))) + (let go ((left listed) (acc (list))) + (match left + (Nil (list-reverse acc)) + ((Cons first _) + (def same (take-while-name (listed-name first) left)) + (def rest (drop-count (list-length same) left)) + (def group (record-ref first group)) + (def cases (list-map (fun (l) (BenchCase (labels (listed-labels l)) + (prepare (fun () (panic! "bjo bench: a listed case is run by its program"))))) + same)) + (go rest + (cons (Bench (name (listed-name first)) (module (program-exe p)) (file (program-source p)) + (line 0) (group (if (string-empty? group) shown (str shown ": " group))) + (skip (listed-skip first)) (once (record-ref first once)) (alloc "") + (param-names (list-map (fun (t) (match t ((Tuple n _) n))) (listed-labels first))) + (cases (fun () cases))) + acc)))))) + +(: bench-files-menu #:sync (-> BenchTools (List BenchProgram) (List string) int)) +(defun (bench-files-menu tools programs passed) + (with-return ret + (def per-file + (loop (:for p programs) + (:let listed (listed-cases p passed)) + (:do (match listed ((Err why) (ret (fail why))) (_ unit))) + (:when-let (Ok ls) listed) + (:acc out (listing (Tuple p ls))))) + (def benches + (fold (fun (pl acc) (match pl ((Tuple p ls) (list-append acc (listed-benches p ls))))) (list) per-file)) + (: (-> Bench BenchCase int)) + (defun (number-of b c) + (match (find (fun (pl) (match pl ((Tuple p _) (= (program-exe p) (bench-module b))))) per-file) + ((Some (Tuple _ ls)) + (match (find (fun (l) (and (= (listed-name l) (bench-name b)) + (= (listed-labels l) (case-labels c)))) + ls) + ((Some l) (listed-index l)) + (None -1))) + (None -1))) + (: (-> (List Selection) void)) + (defun (run-choice chosen) + (loop (:for pl per-file) + (:let (Tuple p _) pl) + (:let numbers + (fold (fun (s acc) + (if (= (bench-module (sel-bench s)) (program-exe p)) + (list-append acc (list-map (fun (c) (number-of (sel-bench s) c)) (sel-cases s))) + acc)) + (list) chosen)) + (:do (unless (list-empty? numbers) + (println "") + (println #"bjo bench: ${(file-name-of (program-source p))}") + (flush!) + (ignore ((record-ref tools exec) + "dotnet" + (list->vec (cons (program-exe p) + (cons "--cases" + (cons (string-join (list->vec (list-map int->string numbers)) ",") + passed)))))))))) + (def title (string-join (list->vec (list-map (fun (p) (file-name-of (program-source p))) programs)) ", ")) + (menu-loop (menu-lines title benches) benches + (fun () (read-line/opt (parameter-ref current-input-port))) + run-choice) + 0)) + +;; --------------------------------------------------------------------------- +;; A/B: --against REF +;; --------------------------------------------------------------------------- + +;; What `--against` and the flags beside it ask for. Warm-up is seconds and +;; the sample target milliseconds, as the flags are given. +(type (: Against (Record (: ref string) + (: rounds int) + (: processes int) + (: seed ulong) + (: warmup-ms long) + (: sample-ms long) + (: build (Option string))))) + +(: (-> string (Option int))) +(defun (parse-positive s) + (match (try (string->int s) #:catch (System.Exception)) + ((Ok n) (if (> n 0) (Some n) None)) + (_ None))) + +(: (-> string (Option double))) +(defun (parse-number s) + (match (try (string->double s) #:catch (System.Exception)) + ((Ok x) (if (> x 0.0) (Some x) None)) + (_ None))) + +;; The A/B flags read, and the arguments without them, which go on to the +;; programs and to `(std bench)`'s own reading of `--alpha` and the rest. +(: (-> (List string) (Result string (Tuple Against (List string))))) +(defun (against-options passed) + (let go ((left passed) + (a (Against (ref "") (rounds 20) (processes 1) + (seed (cast System.UInt64 (+ clr-tick-count (cast System.Int64 clr-process-id)))) + (warmup-ms 1000L) (sample-ms 10L) (build None))) + (rest (list))) + (defun (number-flag flag v after (: k (-> int Against))) + (match (parse-positive v) + ((Some n) (go after (k n) rest)) + (None (Err #"bjo bench: ${flag} takes a positive whole number: ${v}")))) + (match left + (Nil (if (string-empty? (record-ref a ref)) + (Err "bjo bench: --against needs a git ref or a path.") + (Ok (Tuple a (list-reverse rest))))) + ((Cons "--against" (Cons v after)) (go after (record-set a (ref v)) rest)) + ((Cons "-N" (Cons v after)) (number-flag "-N" v after (fun (n) (record-set a (rounds n))))) + ((Cons "--processes" (Cons v after)) (number-flag "--processes" v after (fun (n) (record-set a (processes n))))) + ((Cons "--seed" (Cons v after)) + (match (try (cast System.UInt64 (string->bigint v)) #:catch (System.Exception)) + ((Ok s) (go after (record-set a (seed s)) rest)) + (_ (Err #"bjo bench: --seed takes a whole number: ${v}")))) + ((Cons "--against-warmup" (Cons v after)) + (match (parse-number v) + ((Some s) (go after (record-set a (warmup-ms (cast System.Int64 (* s 1000.0)))) rest)) + (None (Err #"bjo bench: --against-warmup takes seconds: ${v}")))) + ((Cons "--against-sample" (Cons v after)) + (match (parse-number v) + ((Some ms) (go after (record-set a (sample-ms (max 1L (cast System.Int64 ms)))) rest)) + (None (Err #"bjo bench: --against-sample takes milliseconds: ${v}")))) + ((Cons "--against-build" (Cons v after)) (go after (record-set a (build (Some v))) rest)) + ((Cons a1 more) (go more a (cons a1 rest)))))) + +;; Runs git and answers its output, trimmed; the reason when it fails. +(: git-out #:sync (-> (List string) (Result string string))) +(defun (git-out args) + (match (run/output '(git [,@(list->vec args)])) + ((Ok (Tuple 0 out)) (Ok (string-trim out))) + ((Ok (Tuple _ _)) (Err (str "git " (string-join (list->vec args) " ") " failed"))) + ((Err e) (Err (str "could not run git: " (error-message e)))))) + +(import/extern + (clr-directory-name (: System.IO.Path.GetDirectoryName (-> string string)))) + +(: (-> string string)) +(defun (file-directory file) (clr-directory-name (clr-full-path file))) + +;; The other side's file: a path is taken as it is, a `.bjo` file or a +;; checkout holding this file at the same place in it; anything else is a git +;; ref, checked out in a temporary worktree, which is answered too so that it +;; can be removed. +(type (: OtherSide (Record (: file string) + (: worktree (Option (Tuple string string)))))) + +(: other-side #:sync (-> string string (Result string OtherSide))) +(defun (other-side file ref) + (cond + ((and (string-ends-with? ref ".bjo") (file-exists? ref)) + (Ok (OtherSide (file ref) (worktree None)))) + ((directory-exists? ref) + (match (git-out (list "-C" (file-directory file) "rev-parse" "--show-toplevel")) + ((Ok top) + (Ok (OtherSide (file (path-combine ref (clr-relative-path top (clr-full-path file)))) (worktree None)))) + ((Err _) + (Ok (OtherSide (file (path-combine ref (clr-file-name file))) (worktree None)))))) + (else + (match (git-out (list "-C" (file-directory file) "rev-parse" "--show-toplevel")) + ((Err why) (Err #"bjo bench: --against ${ref}: not a path, and ${file} is not in a git repository (${why}).")) + ((Ok top) + (def tmp (path-combine (clr-temp-path) #"bjo-against-${clr-process-id}-${clr-tick-count}")) + (match (git-out (list "-C" top "worktree" "add" "--detach" "--quiet" tmp ref)) + ((Err why) (Err #"bjo bench: --against ${ref}: ${why}; is it a commit, a branch or a tag?")) + ((Ok _) + (Ok (OtherSide (file (path-combine tmp (clr-relative-path top (clr-full-path file)))) + (worktree (Some (Tuple top tmp)))))))))))) + +(: remove-worktree #:sync (-> OtherSide void)) +(defun (remove-worktree side) + (match (record-ref side worktree) + ((Some (Tuple top tmp)) (ignore (git-out (list "-C" top "worktree" "remove" "--force" tmp)))) + (None unit))) + +;; The other side built: by `bjo`, or by `--against-build CMD` run in its +;; file's directory, after which its program is where `bjo` would put it. +(: build-other #:sync (-> BenchTools Against string (Result int BenchProgram))) +(defun (build-other tools a file) + (match (record-ref a build) + (None ((record-ref tools build) file)) + ((Some command) + (println #"bjo bench: building ${file} with: ${command}") + (flush!) + (def status ((record-ref tools exec) "sh" ["-c" (str "cd " (shell-quote (file-directory file)) " && " command)])) + (def exe (clr-change-extension file ".exe")) + (cond + ((not (= status 0)) (Err (fail #"bjo bench: --against-build failed, with exit code ${status}."))) + ((not (file-exists? exe)) (Err (fail #"bjo bench: --against-build left no ${exe}."))) + (else (Ok (BenchProgram (source file) (exe exe)))))))) + +(: (-> string string)) +(defun (shell-quote s) (str "'" (string-replace s "'" "'\\''") "'")) + +;; --- Serving ----------------------------------------------------------------- + +;; A benchmark program running with `--serve`: a command written, a reply read. +(type (: Server (Record (: proc Proc) (: input OutputPort) (: output InputPort) (: what string)))) + +(: start-server #:sync (-> BenchProgram (List string) string (Result string Server))) +(defun (start-server p passed what) + (match (run '(dotnet [,(program-exe p) "--serve" ,@(list->vec passed)])) + ((Err e) (Err #"bjo bench: could not start ${what}: ${(error-message e)}")) + ((Ok proc) + (match (Tuple (proc-input proc) (proc-output proc)) + ((Tuple (Some in) (Some out)) (Ok (Server (proc proc) (input in) (output out) (what what)))) + (_ (Err #"bjo bench: ${what} has no pipes to talk to it over.")))))) + +;; One command and its reply. A reply that is `(error ...)`, or no reply, is +;; an `Err`. +(: ask #:sync (-> Server string (Result string Bjodat))) +(defun (ask s command) + (write-string (record-ref s input) (str command "\n")) + (flush-port (record-ref s input)) + (match (read-line/opt (record-ref s output)) + (None (Err #"bjo bench: ${(record-ref s what)} stopped answering, after ${command}")) + ((Some line) + (match (parse-bjodat line) + ((Ok (BjoList (Cons (BjoSym head) (Cons (BjoStr why) Nil)))) + (if (= (symbol->string head) "error") + (Err #"bjo bench: ${(record-ref s what)}: ${why}") + (Err #"bjo bench: ${(record-ref s what)} answered ${line}"))) + ((Ok reply) (Ok reply)) + ((Err why) (Err #"bjo bench: ${(record-ref s what)} answered what is not bjodat: ${line}")))))) + +(: stop-server #:sync (-> Server void)) +(defun (stop-server s) + (write-string (record-ref s input) "(quit)\n") + (flush-port (record-ref s input)) + (ignore (wait (record-ref s proc)))) + +;; The value after keyword `name` in a reply. +(: (-> Bjodat string (Option Bjodat))) +(defun (reply-arg reply name) + (match reply + ((BjoList items) + (let go ((left items)) + (match left + ((Cons (BjoKey k) (Cons v more)) (if (= (keyword->string k) name) (Some v) (go more))) + ((Cons _ more) (go more)) + (Nil None)))) + (_ None))) + +(: (-> Bjodat string long)) +(defun (reply-long reply name) + (match (reply-arg reply name) ((Some (BjoInt n)) n) (_ -1L))) + +;; Prepares case `i` on a server, and answers its batch size and whether it +;; got steady while it warmed up. +(: prepare-on #:sync (-> Server int Against (Result string (Tuple long bool)))) +(defun (prepare-on s i a) + (match (ask s #"(prepare ${i} :warmup ${(record-ref a warmup-ms)} :sample ${(record-ref a sample-ms)})") + ((Ok reply) + (Ok (Tuple (reply-long reply "k") + (match (reply-arg reply "steady") ((Some (BjoBool b)) b) (_ #t))))) + ((Err why) (Err why)))) + +;; One sample of `k` calls of case `i`, as nanoseconds per call. +(: sample-on #:sync (-> Server int long (Result string double))) +(defun (sample-on s i k) + (match (ask s #"(sample ${i} :k ${k})") + ((Ok reply) (Ok (/ (cast System.Double (reply-long reply "ns")) (cast System.Double k)))) + ((Err why) (Err why)))) + +;; --- The rounds ---------------------------------------------------------------- + +;; A case both sides have: its number on each. +(type (: Shared (Record (: listed Listed) (: a int) (: b int)))) + +(: (-> Shared Listed)) +(defun (shared-listed s) (record-ref s listed)) + +;; Whether a filter chooses a case: its name contains one, or it is one of +;; its parameters as the report shows them. +(: (-> (List string) Listed bool)) +(defun (chosen-by? names l) + (or (list-empty? names) + (any? (fun (f) (or (string-contains? (listed-name l) f) + (any? (fun (t) (match t ((Tuple p v) (= (str p "=" v) f)))) (listed-labels l)))) + names))) + +;; What the programs are given as they are started: the flags of +;; `(std bench)` that decide what a case is, `--params` today. +(: (-> (List string) (List string))) +(defun (serve-flags rest) + (let go ((left rest) (acc (list))) + (match left + ((Cons "--params" (Cons v more)) (go more (cons v (cons "--params" acc)))) + ((Cons _ more) (go more acc)) + (Nil (list-reverse acc))))) + +;; The names among the arguments: what is not a flag or a flag's value. +(: (-> (List string) (List string))) +(defun (bench-names rest) + (let go ((left rest) (acc (list))) + (match left + ((Cons a more) + (cond + ((value-flag? a) (go (match more ((Cons _ after) after) (Nil Nil)) acc)) + ((string-starts-with? a "-") (go more acc)) + (else (go more (cons a acc))))) + (Nil (list-reverse acc))))) + +;; How many of `rounds` fall to process pair `j` of `pairs`: as even as can +;; be, the first ones taking what is left over. +(: (-> int int int int)) +(defun (rounds-of rounds pairs j) + (+ (/ rounds pairs) (if (< j (% rounds pairs)) 1 0))) + +;; One process pair: both started, every shared case prepared on both with +;; the smaller batch size, and its rounds taken, the order of the sides +;; alternating and the cases shuffled each round. Answers, per shared case, +;; A's and B's per-call times in round order. +(: run-pair #:sync (-> BenchProgram BenchProgram (List Shared) Against int (List string) RandomSource + (Box (List string)) + (Result string (Vec (Tuple (Vec double) (Vec double)))))) +(defun (run-pair pa pb shared a j flags rng unsteady) + (with-return ret + (def (Ok sa) (start-server pa flags (str "A (" (record-ref a ref) ")")) :leave ((Err why) (Err why))) + (def (Ok sb) (start-server pb flags "B (this code)") :leave ((Err why) (stop-server sa) (Err why))) + (defun (leave (: why string)) : (Result string (Vec (Tuple (Vec double) (Vec double)))) + (stop-server sa) + (stop-server sb) + (Err why)) + (def n (list-length shared)) + (def live (bench-at-terminal?)) + (def ks + (loop (:for s shared) + (:with i 0 (+ i 1)) + (:do (when live + (print (str "\r[pair " (int->string (+ j 1)) "/" (int->string (record-ref a processes)) + "] preparing " (int->string (+ i 1)) "/" (int->string n) ": " + (listed-key (shared-listed s)) "\u{1B}[K")) + (flush!))) + (:let ka (prepare-on sa (record-ref s a) a)) + (:let kb (prepare-on sb (record-ref s b) a)) + ;; The smaller batch on both sides, so that a sample of each is + ;; the same number of calls. + (:let k (match (Tuple ka kb) + ((Tuple (Ok (Tuple x steady-a)) (Ok (Tuple y steady-b))) + (unless steady-a (box-set! unsteady (cons (str (listed-key (shared-listed s)) " on A") (box-ref unsteady)))) + (unless steady-b (box-set! unsteady (cons (str (listed-key (shared-listed s)) " on B") (box-ref unsteady)))) + (Ok (min x y))) + ((Tuple (Err why) _) (Err why)) + ((Tuple _ (Err why)) (Err why)))) + (:do (match k ((Err why) (ret (leave why))) (_ unit))) + (:acc out (vecing (match k ((Ok x) x) (_ 1L)))))) + (def rounds (rounds-of (record-ref a rounds) (record-ref a processes) j)) + (def as (make-array n #:fill (vec-empty))) + (def bs (make-array n #:fill (vec-empty))) + (def order (list->vec (index-list n))) + (loop (:for r (up-from 0 #:to rounds)) + (:do (when live + (print (str "\r[pair " (int->string (+ j 1)) "/" (int->string (record-ref a processes)) + "] round " (int->string (+ r 1)) "/" (int->string rounds) "\u{1B}[K")) + (flush!))) + (:subloop (:for i (shuffle-vec order #:rng rng)) + (:let s (list-ref shared i)) + (:let k (vec-ref ks i)) + ;; A first in even rounds, B first in odd ones. + (:let first-a (= (% (+ r j) 2) 0)) + (:let x1 (if first-a (sample-on sa (record-ref s a) k) (sample-on sb (record-ref s b) k))) + (:let x2 (if first-a (sample-on sb (record-ref s b) k) (sample-on sa (record-ref s a) k))) + (:do (match (Tuple x1 x2) + ((Tuple (Ok t1) (Ok t2)) + (array-set! as i (vec-add (array-ref as i) (if first-a t1 t2))) + (array-set! bs i (vec-add (array-ref bs i) (if first-a t2 t1)))) + ((Tuple (Err why) _) (ret (leave why))) + ((Tuple _ (Err why)) (ret (leave why))))))) + (when live (print "\r\u{1B}[K") (flush!)) + (stop-server sa) + (stop-server sb) + (Ok {vecing (Tuple (array-ref as i) (array-ref bs i)) (:for i (range 0 n))}))) + +(: (-> int (List int))) +(defun (index-list n) {listing i (:for i (range 0 n))}) + +(: cmd-against #:sync (-> BenchTools string (List string) int)) +(defun (cmd-against tools file passed) + (with-return ret + (def (Ok (Tuple a rest)) (against-options passed) :leave ((Err why) (fail why))) + (def (Ok o) (parse-bench-args (list->vec (list-filter (fun (x) (not (= x "--serve"))) rest))) + :leave ((Err why) (fail #"bjo bench: ${why}"))) + (when (some? (record-ref o compare)) + (ret (fail "bjo bench: --against and --compare are two comparisons; ask for one."))) + (def (Ok pb) ((record-ref tools build) file) :leave ((Err status) status)) + (def (Ok side) (other-side file (record-ref a ref)) :leave ((Err why) (fail why))) + (def code (against-with tools a o rest pb side)) + (remove-worktree side) + code)) + +(: against-with #:sync (-> BenchTools Against BenchOptions (List string) BenchProgram OtherSide int)) +(defun (against-with tools a o rest pb side) + (with-return ret + (def ref (record-ref a ref)) + (unless (file-exists? (record-ref side file)) + (ret (fail #"bjo bench: ${ref} has no ${(record-ref side file)}: the benchmark file is not there."))) + (def (Ok pa) (build-other tools a (record-ref side file)) :leave ((Err status) status)) + (def flags (serve-flags rest)) + (def (Ok listed-a) (listed-cases pa flags) :leave ((Err why) (fail why))) + (def (Ok listed-b) (listed-cases pb flags) :leave ((Err why) (fail why))) + (def names (bench-names rest)) + (defun (runnable (: l Listed)) : bool (and (string-empty? (listed-skip l)) (chosen-by? names l))) + (def wanted-a (list-filter runnable listed-a)) + (def wanted-b (list-filter runnable listed-b)) + (def shared + (loop (:for l wanted-b) + (:when-let (Some other) (find (fun (x) (= (listed-key x) (listed-key l))) wanted-a)) + (:acc out (listing (Shared (listed l) (a (listed-index other)) (b (listed-index l))))))) + (def only-a (loop (:for l wanted-a) + (:acc out (listing (listed-key l)) + #:when (none? (find (fun (x) (= (listed-key x) (listed-key l))) wanted-b))))) + (def only-b (loop (:for l wanted-b) + (:acc out (listing (listed-key l)) + #:when (none? (find (fun (x) (= (listed-key x) (listed-key l))) wanted-a))))) + (when (list-empty? shared) + (ret (fail "bjo bench: the two sides have no benchmark in common to compare."))) + (def rng (make-random/seed64 (record-ref a seed))) + (def pairs (record-ref a processes)) + ;; The cases that did not get steady while warming up, and on which side. + (: (Box (List string))) + (def unsteady (make-box (list))) + (def taken + (loop (:for j (up-from 0 #:to pairs)) + (:let got (run-pair pa pb shared a j flags rng unsteady)) + (:do (match got ((Err why) (ret (fail why))) (_ unit))) + (:when-let (Ok v) got) + (:acc out (listing v)))) + (def cases + (loop (:for s shared) + (:with i 0 (+ i 1)) + (:acc out (listing (PairedCase (name (listed-name (shared-listed s))) + (labels (listed-labels (shared-listed s))) + (a (list->vec (list-map (fun (v) (match (vec-ref v i) ((Tuple x _) x))) taken))) + (b (list->vec (list-map (fun (v) (match (vec-ref v i) ((Tuple _ y) y))) taken)))))))) + (def result (compare-paired cases only-a only-b (record-ref o alpha) (record-ref o threshold))) + (loop (:for l (header-lines (list-length shared))) (:do (println l))) + (println (str "bench: A/B against " ref ", " (int->string (record-ref a rounds)) " rounds, " + (if (= pairs 1) + "paired (Wilcoxon signed-rank); " + #"${pairs} process pairs (t-test on their log ratios); ") + (comparisons-text (list-length shared)))) + (when (= pairs 1) + (println "bench: one process per side: a difference between processes is not in the p-values (--processes P measures it)")) + (println (str "bench: seed " (displayed (record-ref a seed)) " (--seed repeats the order of the cases)")) + (unless (list-empty? (box-ref unsteady)) + (println (str "bench: warning: not steady after warming up, so the JIT may have changed them during the rounds: " + (string-join (list->vec (distinct-strings (list-reverse (box-ref unsteady)))) ", ") + ". --against-warmup S gives them longer."))) + (def shown (if (file-exists? ref) (file-name-of ref) ref)) + (print-comparison-table result o shown "now" #"A (${shown})" #f) + (match (record-ref o save) + ((Some path) (save-sides path ref cases)) + (None unit)) + (if (fails-slower? result o) 1 0))) + +(: (-> (List string) (List string))) +(defun (distinct-strings xs) + (fold (fun (x acc) (if (any? (fun (y) (= y x)) acc) acc (list-append acc (list x)))) (list) xs)) + +;; `--save FILE` of an A/B run: this code's samples to FILE, the other side's +;; beside it, `FILE` with `.against` before its extension. Every process +;; pair's samples, one after another. +(: (-> string string (List PairedCase) void)) +(defun (save-sides path ref cases) + (def other (clr-change-extension path (str ".against" (path-extension-of path)))) + (defun (side (: pick (-> PairedCase (Vec (Vec double))))) : (List SavedCase) + (list-map (fun (c) (SavedCase (name (record-ref c name)) (labels (record-ref c labels)) (k 0L) (once #f) + (bytes 0.0) (samples (vec-fold (fun (v acc) (vec-append acc v)) (vec-empty) (pick c))))) + cases)) + (write-saved-run! path (this-run (side (fun (c) (record-ref c b))))) + (write-saved-run! other (this-run (side (fun (c) (record-ref c a))))) + (println #"bench: saved this code's samples to ${path}, and ${ref}'s to ${other}")) + +(: (-> string string)) +(defun (path-extension-of path) + (match (string-search-backward path ".") + ((Some at) (substring path at (string-cursor-end path))) + (None ""))) +;; bjo — the command that runs Bjolang. +;; +;; The commands all end in the compiler: `run`, `build`, `check`, `repl`, and +;; `init` and `fetch`, which prepare what the others compile. What is here +;; rather than in the compiler is what a compiler cannot do for itself — decide +;; whether to start one at all, find the project a command was run inside, +;; fetch what its manifest names, and hand the console to what it starts. +;; +;; A *project* is a directory with a `manifest.bjodat`. Inside one, every +;; command compiles against the project's `.bjo/roots`, which says which +;; directory each package name means; outside one, every command behaves as it +;; did before there were projects. What a project is and how its dependencies +;; are resolved is in `project.bjo`, `resolve.bjo`, `git.bjo` and `lock.bjo` — +;; this file is the command line and the compiler processes, and stays that. +;; +;; `bjo` does not compile dependencies one at a time. The compiler builds an +;; imported module itself, in dependency order, so what `bjo` has to do is make +;; the modules findable — the roots file — and then compile the one entry. +;; +;; Staleness is *read*, not worked out. A build writes `.bjobuild` +;; naming the mode it produced, the artefact, the compiler, and every source and +;; module it read; see `Build.writeBuildRecord`. So the question "is there +;; anything to do?" is a timestamp comparison over a list, and nothing here has +;; to know what an `include` is or where a module path resolves to. Those are +;; the compiler's to know. A driver that worked them out from the source +;; itself could disagree with the compiler about what a file depends on. +;; +;; Children inherit this process's console, which is why `(std run)` is not used +;; to run them: it redirects stdin and stdout in order to read them, and the +;; output of a program run under `bjo` belongs to whoever ran it. The REPL needs +;; a terminal on both ends for the same reason. The one child whose output `bjo` +;; reads is a benchmark program asked for its list, which nobody watches. +;; +;; Everything that waits for a child is `#:sync`, down from `dispatch`. Waiting +;; is what this command does and it has one thread to do it on, so a suspending +;; copy of any of it would be a body nothing here ever calls — reported, rightly, +;; as parking a thread that is nobody's to yield. +;; +;; A command line is a sequence of refusals before it is a command, so most of +;; what follows is written as guards: `def` binds the case that carries on and +;; says what the body produces when it does not, and `with-return` is what a +;; failure that ends the command leaves through. The alternative is the same +;; decisions as nesting, indented once per thing that can go wrong. +;; +;; Where the failing case has nothing to say for itself — two `Option`s in a row +;; and no answer but `None` — it is a `(do ...)` block instead, and neither +;; failure is written down at all. + +(import (std prelude)) +;; The manifest types, for the fields this file reads out of a `Package`: +;; `record-ref` needs the declaration, and an import of `project.bjo` alone +;; does not bring it. `(text bjodat)` comes with them — the manifest's own +;; instances are written against its traits. +(import (text bjodat)) +(import "manifest.bjo") +(import "project.bjo") +(import "resolve.bjo") +(import "publish.bjo") +(import "bench.bjo") +(import "format.bjo") +(import "diff.bjo") + +;; --------------------------------------------------------------------------- +;; Interop +;; --------------------------------------------------------------------------- + +(import/class + (Process (: System.Diagnostics.Process)) + (ProcessStartInfo (: System.Diagnostics.ProcessStartInfo (-> ProcessStartInfo))) + (DateTime (: System.DateTime))) + +(import/extern + (settings-program! (: System.Diagnostics.ProcessStartInfo.FileName + (-> ProcessStartInfo string void) #:set)) + (settings-use-shell! (: System.Diagnostics.ProcessStartInfo.UseShellExecute + (-> ProcessStartInfo bool void) #:set)) + (settings-argument (: BjoProc.AddArgument + (-> ProcessStartInfo string ProcessStartInfo))) + (process-start (: System.Diagnostics.Process.Start (-> ProcessStartInfo Process) + #:exceptions (System.ComponentModel.Win32Exception + System.InvalidOperationException))) + ;; The synchronous wait, and deliberately: `bjo` has nothing to do until the + ;; child is done, and the thread it parks is the process's own. + (process-wait (: System.Diagnostics.Process.WaitForExit (-> Process void) #:blocking)) + (process-code (: System.Diagnostics.Process.ExitCode (-> Process int) #:get)) + (error-message (: System.Exception.Message (-> Exception string) #:get)) + (clr-mtime (: System.IO.File.GetLastWriteTimeUtc (-> string DateTime))) + (clr-ticks (: System.DateTime.Ticks (-> DateTime long) #:get)) + (string->long (: System.Int64.TryParse (-> string (out long) (Option long)))) + (clr-change-extension (: System.IO.Path.ChangeExtension (-> string string string))) + ;; Where this executable sits, for an installation that was not told where it + ;; is. See `root`. + (clr-base-dir (: System.AppContext.BaseDirectory #:get))) + + +;; --------------------------------------------------------------------------- +;; Starting something, and waiting for it +;; --------------------------------------------------------------------------- + +;; Runs `program` and answers its exit code. +;; +;; No redirection anywhere: the child writes to the terminal `bjo` was started +;; from, reads from the same stdin, and is not a pipe stage. `UseShellExecute` +;; off is what makes the program name resolve against PATH. +(: exec #:sync (-> string (Vec string) int)) +(defun (exec program arguments) + (def settings (ProcessStartInfo.)) + (settings-program! settings program) + (settings-use-shell! settings #f) + (loop (:for a arguments) + (:do (ignore (settings-argument settings a)))) + (match (process-start settings) + ((Err e) + (println #"bjo: could not start ${program}: ${(error-message e)}") + ;; What a shell answers for a command it could not run. + 127) + ((Ok child) + (process-wait child) + (process-code child)))) + + +;; --------------------------------------------------------------------------- +;; Timestamps +;; --------------------------------------------------------------------------- + +;; A file's last write, in ticks. A file that is not there is as old as anything +;; can be, so a dependency that has been deleted is not newer than the artefact +;; and does not force a rebuild that would fail for the same reason. +(: (-> string long)) +(defun (mtime path) + (if (file-exists? path) (clr-ticks (clr-mtime path)) 0L)) + +(: (-> string string bool)) +(defun (newer? a b) (> (mtime a) (mtime b))) + + +;; --------------------------------------------------------------------------- +;; The installation +;; --------------------------------------------------------------------------- + +;; The directory the compiler and its sources live in. +;; +;; `BJO_ROOT` is what the launcher exports. Without it, this executable was +;; started some other way, and the one thing it knows about itself is where it +;; sits: a build puts the binary beside its source, so `bjo.exe` is in +;; `/bjo`. +(: string) +(def root + (match (get-environment-variable "BJO_ROOT") + ((Some dir) dir) + (None (path-absolute (path-combine clr-base-dir ".."))))) + +;; The compiler. `build_compiler.sh` is the one place that knows where it is, +;; and the launcher exports its answer as `BJOLANG_COMPILER`. The fallback is +;; that same answer written down, for a `bjo.exe` started without the launcher. +(: string) +(def compiler-dll + (match (get-environment-variable "BJOLANG_COMPILER") + ((Some path) path) + (None (path-combine root "bin" "compiler" "Bjolang.dll")))) + + +;; --------------------------------------------------------------------------- +;; The compiler binary +;; --------------------------------------------------------------------------- + +;; Is there a compiler to use at all? +(: (-> bool)) +(defun (have-compiler?) + (cond + ((file-exists? compiler-dll) #t) + (else + (println "bjo: there is no compiler built here yet. Run 'bjo rebjo'.") + #f))) + + +;; --------------------------------------------------------------------------- +;; The build record +;; --------------------------------------------------------------------------- + +;; What a build wrote down about itself. `compiler`, `source` and `dep` lines +;; are read into one list: they are all files the artefact must be newer than, +;; and nothing here has a reason to tell them apart. +(type + (: BuildRecord (Record (: mode string) + (: output string) + (: inputs (List string)) + ;; The compiler that built it. Its timestamp is also + ;; in `inputs`; this is compared as a path, because a + ;; different compiler can be older than the artefact, + ;; and the timestamp check would then call the + ;; artefact current. + (: compiler string) + ;; The .NET shared frameworks the file's package + ;; declared when it was built. Not a file, so not an + ;; input: a declaration that is *removed* deletes the + ;; declarations file, and a file that is not there is + ;; newer than nothing. + (: frameworks (List string)) + ;; The declarations file this was built against, or "" + ;; where there was none. It is an input as well — a + ;; rewrite is a change the timestamps can see — and + ;; this is for the change they cannot: the file + ;; appearing, or going away with the last declaration + ;; in the whole graph. + (: declaration-file string) + ;; The `--nuget` directory it was built with, or "". + ;; Compared for the same reason as `declaration-file`: + ;; packages removed from the manifest leave no file + ;; that a timestamp could show. + (: nuget string) + ;; For a program: its runtime settings, as + ;; `name=value`, sorted. Compared for the same reason + ;; as `nuget`: they are in the manifest, and a change + ;; to them changes no file that the build read. + (: runtime (List string)) + ;; For a program built with `--aot`: its native + ;; executable. "" otherwise. + (: aot string) + ;; With `--aot-cpu`: the CPU it was built for. "" + ;; otherwise. + (: aot-cpu string) + ;; For a program: what its resolver loads, as the + ;; `link` and `native` lines name them. What + ;; `bjo publish` copies. + (: links (List (Tuple string string))) + (: natives (List (Tuple string string))) + ;; The artefact's last write when the record was + ;; written, 0 where it does not say. A record whose + ;; artefact was written since is another build's. + (: written long) + ;; For a library, the digest of what an importer is + ;; compiled against, or "". + (: interface string) + ;; Each linked module's digest as this build saw it, as + ;; `(digest . path)`. + (: interface-of (List (Tuple string string))) + ;; The `source` lines alone. + (: sources (List string))))) + +(: (-> string string)) +(defun (record-file file) (clr-change-extension file ".bjobuild")) + +;; A line is `key value`, and the value may contain spaces — a path is allowed +;; them — so it is everything after the first one rather than a field of a +;; split. A line with no space at all is no line, which is the `None` the bind +;; short-circuits to. +(: (-> string (Option (Tuple string string)))) +(defun (split-line line) + (option-map (fun (space) + (Tuple (substring line (string-cursor-start line) space) + (substring line (string-cursor-next line space) (string-cursor-end line)))) + (string-find #(char=? & #\space) line))) + +;; A `link` or `native` value: a name, then a path. A value with no space in it +;; is a name with no path, which a publish reports as a file that is not there. +(: (-> string (Tuple string string))) +(defun (pair-of value) + (match (split-line value) + ((Some pair) pair) + (None (Tuple value "")))) + +;; The keys that name a file the artefact must be newer than. What they are +;; called apart is nothing this has to know. +;; `roots` is an input like any other: it says which directory each package name +;; means, so a build made against a different one is a build against different +;; code — and the `dep` lines here are the files the old one chose. +(: (-> string bool)) +(defun (input-key? key) + (case key + (("source" "dep" "compiler" "roots" "frameworks" "nuget-list") #t) + (else #f))) + +;; `None` when there is no record, or when what is there names no artefact — +;; which is a record written by a newer `bjo` or a truncated one, and either way +;; not something to decide against. +(: (-> string (Option BuildRecord))) +(defun (read-record file) + (with-return ret + (def path (record-file file)) + (unless (file-exists? path) (ret None)) + + (loop (:for line (string-split (file-read-text path) "\n")) + ;; A line with no key at all is not one. `:when-let` skips it, and an + ;; accumulator that is not stepped keeps what it had. + (:when-let (Some (Tuple key value)) (split-line (string-trim line))) + (:acc mode (folding "" (if (= key "mode") value mode))) + (:acc artefact (folding "" (if (= key "output") value artefact))) + (:acc compiler (folding "" (if (= key "compiler") value compiler))) + ;; A key this `bjo` does not know is a record from a later one, and is + ;; accumulated nowhere: the three fields here are what a decision is + ;; made of, and they are all present. + (:acc inputs (listing value) #:when (input-key? key)) + (:acc frameworks (listing value) #:when (= key "framework")) + (:acc declarations (folding "" (if (= key "frameworks") value declarations))) + (:acc nuget (folding "" (if (= key "nuget") value nuget))) + (:acc runtime (listing value) #:when (= key "runtime")) + (:acc aot (folding "" (if (= key "aot") value aot))) + (:acc aot-cpu (folding "" (if (= key "aot-cpu") value aot-cpu))) + ;; `link ` and `native `: the value splits + ;; at its first space the way a line does, and a path may hold more. + (:acc links (listing (pair-of value)) #:when (= key "link")) + (:acc natives (listing (pair-of value)) #:when (= key "native")) + (:acc written (folding 0L (if (= key "written") + (match (string->long value) ((Some n) n) (None 0L)) + written))) + (:acc interface (folding "" (if (= key "interface") value interface))) + (:acc interface-of (listing (pair-of value)) #:when (= key "interface-of")) + (:acc sources (listing value) #:when (= key "source")) + => (if (= artefact "") + None + (Some (BuildRecord (mode mode) + (output artefact) + (inputs inputs) + (compiler compiler) + (frameworks (list-sort frameworks)) + (declaration-file declarations) + (nuget nuget) + (runtime (list-sort runtime)) + (aot aot) + (aot-cpu aot-cpu) + (links links) + (natives natives) + (written written) + (interface interface) + (interface-of interface-of) + (sources sources))))))) + +;; --------------------------------------------------------------------------- +;; Compiling +;; --------------------------------------------------------------------------- + +;; The source a name refers to. A bare `foo` is `foo.bjo` when that exists, +;; which is what makes `bjo run examples/vec` work. +(: (-> string (Option string))) +(defun (resolve-source name) + (cond + ((file-exists? name) (Some name)) + ((file-exists? (str name ".bjo")) (Some (str name ".bjo"))) + (else None))) + +;; `--roots ` where there is a project, and nothing where there is not. +;; The compiler records the file as an input of what it builds, so a build made +;; against other packages is stale by the same comparison every other change is. +;; What a compile is made against: where the packages are, and which of them +;; declared a .NET shared framework. Carried together because every command +;; that passes one passes the other, and two things a caller has to keep in +;; step are two things a caller can get out of step. +;; +;; Both are `None` outside a project, and the second is `None` inside one where +;; no package declared anything — which is the ordinary case, and is what keeps +;; such a project's command line exactly what it was before frameworks existed. +;; +;; `nuget` is the directory of the restored NuGet packages, and `None` unless +;; the manifest names some. +;; +;; `runtime` is the root manifest's runtime settings, as `name=value`, which a +;; program is built with, `aot` whether it is built as a Native AOT +;; executable too, and `aot-cpu` the CPU that executable is for, or "". Empty, +;; #f and "" outside a project. +(type (: Against (Record (: roots (Option string)) + (: frameworks (Option string)) + (: nuget (Option string)) + (: runtime (List string)) + (: aot bool) + (: aot-cpu string)))) + +(: Against) +(def nothing-against (Against (roots None) (frameworks None) (nuget None) (runtime Nil) (aot #f) (aot-cpu ""))) + +(: (-> Prepared Against)) +(defun (prepared-against ready) + (Against (roots (Some (prepared-roots-file ready))) + (frameworks (prepared-frameworks-file ready)) + (nuget (prepared-nuget-dir ready)) + (runtime (prepared-runtime ready)) + (aot (prepared-aot ready)) + (aot-cpu (prepared-aot-cpu ready)))) + +(: (-> string (Option string) (List string))) +(defun (flag-for flag path) + (match path + ((Some p) (Cons flag (Cons p Nil))) + (None Nil))) + +;; `--runtime name=value` for each runtime setting. +(: (-> (List string) (List string))) +(defun (runtime-flags settings) + (match settings + (Nil Nil) + ((Cons setting rest) (Cons "--runtime" (Cons setting (runtime-flags rest)))))) + +;; `--aot`, and `--aot-cpu` with it when the manifest names a CPU. +(: (-> Against (List string))) +(defun (aot-flags a) + (cond ((not (record-ref a aot)) Nil) + ((= (record-ref a aot-cpu) "") (list "--aot")) + (else (list "--aot" "--aot-cpu" (record-ref a aot-cpu))))) + +;; The compiler records both files as inputs of what it builds, so a build made +;; against other packages — or against another set of declarations — is stale by +;; the same comparison every other change is. +(: (-> Against (List string))) +(defun (against-flags a) + ;; TODO list-append*. + (list-append (flag-for "--roots" (record-ref a roots)) + (list-append (flag-for "--frameworks" (record-ref a frameworks)) + (list-append (flag-for "--nuget" (record-ref a nuget)) + (list-append (aot-flags a) + (runtime-flags (record-ref a runtime))))))) + +;; What the frameworks file says the package holding `file` declares. +;; +;; The longest directory that contains the file wins, which is the rule the +;; compiler uses to decide the same question — and has to be, because a +;; dependency fetched into `.bjo/pkg` lives *under* the root project. +(: (-> Against string (List string))) +(defun (declared-for against file) + (def (Some path) (get against .frameworks) :leave-with Nil) + (cond + ((not (file-exists? path)) Nil) + (:def full (path-absolute file)) + (:def lines (string-split (file-read-text path) "\n")) + ;; Two passes rather than one: the package a file belongs to is the + ;; longest directory containing it, and which that is cannot be known + ;; until every line has been seen. + (:def owner + (loop (:for line lines) + (:when-let (Some (Tuple dir name)) (split-tab (string-trim line))) + (:acc out (folding "" (if (and (string-starts-with? full (str dir "/")) + (> (string-length dir) (string-length out))) + dir + out))))) + ((= owner "") Nil) + (else (list-sort + (loop (:for line lines) + (:when-let (Some (Tuple dir name)) (split-tab (string-trim line))) + (:acc out (listing name) #:when (= dir owner))))))) + +;; A frameworks-file line: a directory, a tab, a framework name. +(: (-> string (Option (Tuple string string)))) +(defun (split-tab line) + (def (Some tab) (string-find #(char=? & #\tab) line) :propagate) + (Some (Tuple (substring line (string-cursor-start line) tab) + (substring line (string-cursor-next line tab) + (string-cursor-end line))))) + +;; Is there anything to do before the artefact can be used? +(: (-> string string Against bool)) +(defun (stale? file wanted against) + ;; No record at all is *stale*, not current: there is nothing that says what + ;; the artefact was built from, so the only safe answer is to build it. `#f` + ;; here makes `ensure-built` serve a record that is not there, and fail with + ;; an exit code and no message. + (def (Some rec) (read-record file) :leave-with #t) + (or (built-otherwise? rec wanted against file) + (any? #(newer? & (record-ref rec output)) (record-ref rec inputs)))) + +;; Whether the record describes a build other than the one asked for, whatever +;; the timestamps say. +(: (-> BuildRecord string Against string bool)) +(defun (built-otherwise? rec wanted against file) + (def artefact (record-ref rec output)) + (or (not (= (record-ref rec mode) wanted)) + (not (file-exists? artefact)) + ;; Built by another compiler than the one this would build it with, or by + ;; one the record does not name. + (= (record-ref rec compiler) "") + (not (= (path-absolute (record-ref rec compiler)) (path-absolute compiler-dll))) + ;; A changed declaration is a changed build: what a module may *name* + ;; follows from it, so one built under another set has to be built again + ;; — and told, if it now names something it may not. + (not (= (record-ref rec frameworks) (declared-for against file))) + ;; And the same for the file itself coming or going. The last + ;; declaration in a graph being taken out deletes it, and a file that is + ;; not there is newer than nothing — while the dependency that used to + ;; declare it still has to be built again, which is the compiler's own + ;; comparison and only runs if this one sends it there. + (not (= (record-ref rec declaration-file) + (match (record-ref against frameworks) + ((Some path) path) + (None "")))) + ;; Packages added to or removed from the manifest. + (not (= (record-ref rec nuget) + (match (record-ref against nuget) + ((Some dir) (path-absolute dir)) + (None "")))) + ;; Runtime settings changed in the manifest. Only a program has them: a + ;; library has no runtimeconfig, and its record no `runtime` lines. + (and (string-ends-with? artefact ".exe") + (not (= (record-ref rec runtime) (record-ref against runtime)))) + ;; `(aot ...)` changed in the manifest, or the native executable is gone. + (and (string-ends-with? artefact ".exe") + (not (= (not (= (record-ref rec aot) "")) (record-ref against aot)))) + ;; `(aot-cpu ...)` changed, for a program that is built as AOT. + (and (string-ends-with? artefact ".exe") + (record-ref against aot) + (not (= (record-ref rec aot-cpu) (record-ref against aot-cpu)))) + (and (record-ref against aot) + (not (file-exists? (record-ref rec aot)))))) + +;; A linked module whose own record vouches for the interface `entry` says the +;; build saw: the module and the sources it was built from. Nothing otherwise. +(: (-> (Tuple string string) (List string))) +(defun (interface-kept entry) + (def (Tuple digest dll) entry) + (def (Some dep) (read-record dll) :leave-with Nil) + (if (and (= (record-ref dep written) (mtime dll)) + (= (record-ref dep interface) digest)) + (cons dll (record-ref dep sources)) + Nil)) + +;; Early cutoff, for an artefact whose imports have just been built: current if +;; everything newer than it is a linked module whose interface is the one it was +;; built against, or a source such a module was built from. The compiler decides +;; a library this way; this is the same for the entry it is never asked about. +(: (-> string string Against bool)) +(defun (current-by-interface? file wanted against) + (def* ((Some rec) (read-record file) :leave-with #f) + (artefact (record-ref rec output))) + (def excused + (loop (:for entry (record-ref rec interface-of)) + (:acc out (folding Nil (list-append (interface-kept entry) out))) + => out)) + (and (not (built-otherwise? rec wanted against file)) + (file-exists? artefact) + (= (record-ref rec written) (mtime artefact)) + (all? (fun (input) (or (not (newer? input artefact)) (any? #(= & input) excused))) + (record-ref rec inputs)))) + + +(: compile #:sync (-> string bool bool Against int)) +(defun (compile file debug? lib? roots) + (def flags (list-append (if lib? (Cons "--lib" Nil) Nil) + (list-append (if debug? (Cons "-d" Nil) Nil) + (against-flags roots)))) + (exec "dotnet" (list->vec (Cons compiler-dll (list-append flags (Cons file Nil)))))) + +;; Modules and everything they import, built in parallel, each as soon as what +;; it imports is built: the compiler's `--build-graph`. With `deps-only?` the +;; files themselves are left out, which is what a program's entry needs before +;; it is compiled as a program. +(: compile-graph #:sync (-> (List string) bool Against int)) +(defun (compile-graph files deps-only? roots) + (def flags (list-append (Cons "--build-graph" (Cons "--quiet" (if deps-only? (Cons "--deps-only" Nil) Nil))) + (against-flags roots))) + (exec "dotnet" (list->vec (Cons compiler-dll (list-append flags files))))) + +;; Every module of a library, in one compiler process, one after another. +;; +;; `--if-stale` leaves out what is current, and `--batch` pays the compiler's +;; half-second start once rather than once per module. What a debug build of a +;; library goes through: a graph build's workers build without `-d`. +(: compile-batch #:sync (-> (List string) bool Against int)) +(defun (compile-batch files debug? roots) + (def flags (list-append (Cons "--batch" (Cons "--if-stale" Nil)) + (list-append (if debug? (Cons "-d" Nil) Nil) + (against-flags roots)))) + (exec "dotnet" (list->vec (cons compiler-dll (list-append flags files))))) + +;; The artefact as it already stands, for a file that needs no building. +;; +;; Only reached when `stale?` has answered from a record, so there is one to +;; read; `None` is the file having gone away between the two reads. +(: (-> string (Result int (Tuple BuildRecord bool)))) +(defun (record-as-built file) + (match (read-record file) + ((Some rec) (Ok (rec . #f))) + ;; Only reachable when `stale?` answered from a record that has gone away + ;; since, and the exit code alone would be a command that failed without + ;; saying anything at all. + (None (Err (fail #"bjo: the build record for ${file} went away while it was being read."))))) + +;; Whether what `file` imports should be built before it, by a graph build: when +;; nothing says what it was built from, or something other than the file itself +;; changed. An edit to the entry alone is the common case, and it compiles +;; straight away; the entry's own compile would build a stale import too, one +;; after another, which is what the graph saves. +(: (-> string bool)) +(defun (imports-first? file) + (def* ((Some rec) (read-record file) :leave-with #t) + (artefact (record-ref rec output)) + (self (path-absolute file))) + (or (not (file-exists? artefact)) + (any? #(and (not (= (path-absolute &) self)) (newer? & artefact)) (record-ref rec inputs)))) + +;; Compiles `file` if it needs it, and answers the record the build left — which +;; is where the artefact's name comes from. A file with no `main` is a library +;; whether or not `--lib` was passed, so the compiler is the only one that knows +;; whether it produced a `.exe` or a `.dll`. +;; +;; `Err` carries an exit code, so a failed compile leaves through here with the +;; compiler's own status rather than a made-up one. The `bool` says whether this +;; call is what built it. +(: ensure-built #:sync (-> string bool bool bool Against (Result int (Tuple BuildRecord bool)))) +(defun (ensure-built file debug? lib? force? roots) + (with-return ret + (unless (have-compiler?) (ret (Err 1))) + + ;; A debug build is never served from the record: the dumps `-d` asks for + ;; are produced by compiling, so a cached one has nothing to show. + (def wanted (if debug? "debug" "release")) + (unless (or force? debug? (stale? file wanted roots)) + (ret (record-as-built file))) + + (when (imports-first? file) + (let ((imports (compile-graph (Cons file Nil) #t roots))) + (unless (= imports 0) (ret (Err imports)))) + ;; What they changed may be nothing the entry was compiled against. + (unless (or force? debug? (not (current-by-interface? file wanted roots))) + (ret (record-as-built file)))) + + (def status (compile file debug? lib? roots)) + (unless (= status 0) (ret (Err status))) + + ;; The compiler said it succeeded and left no record. It writes one after + ;; every build that did, so this is a compiler too old for this `bjo`. + (def (Some built) (read-record file) + :leave (None (println #"bjo: ${file} compiled, but left no build record.") + (Err 1))) + + (Ok (Tuple built #t)))) + + +;; --------------------------------------------------------------------------- +;; The commands +;; --------------------------------------------------------------------------- + +;; Starts a built program with `arguments`: its Native AOT executable when the +;; record names one that is there, and the `.exe` under `dotnet` otherwise. +(: run-built #:sync (-> BuildRecord (List string) int)) +(defun (run-built rec arguments) + (def native (record-ref rec aot)) + (if (and (not (= native "")) (file-exists? native)) + (exec native (list->vec arguments)) + (exec "dotnet" (list->vec (Cons (record-ref rec output) arguments))))) + +;; Everything after the source file belongs to the program, which is the only +;; reading that lets a program have flags of its own. +(: cmd-run #:sync (-> string (List string) bool bool Against int)) +(defun (cmd-run file program-args debug? force? roots) + (match (ensure-built file debug? #f force? roots) + ((Err status) status) + ((Ok (Tuple rec _)) #:when (string-ends-with? (record-ref rec output) ".dll") + (println #"bjo: ${file} has no main, so it builds a library: ${(record-ref rec output)}") + (println "bjo: there is nothing to run. Use 'bjo build' for a library.") + 1) + ((Ok (Tuple rec _)) + (run-built rec program-args)))) + +(: cmd-build #:sync (-> string bool bool bool Against int)) +(defun (cmd-build file debug? lib? force? roots) + (match (ensure-built file debug? lib? force? roots) + ((Err status) status) + ((Ok (Tuple rec built?)) + (def what (if built? "Built" "Up to date")) + (println #"${what}: ${(record-ref rec output)}") + 0))) + +;; Nothing is written, so no record is read and none is left. A dependency is +;; still built — checking a module against an import means reading that import's +;; metadata, and the metadata is in the `.dll` — but that is the compiler's +;; business rather than this one's. +;; +;; Many files go to one compiler process: a cold one costs about half a second +;; before it does anything, and `--batch` pays that once instead of per file. +(: cmd-check #:sync (-> (List string) Against int)) +(defun (cmd-check files roots) + (cond + ((not (have-compiler?)) 1) + (else + (def batch + (match files + ((Cons _ (Cons _ _)) (Cons "--batch" Nil)) + (_ Nil))) + (exec "dotnet" + (list->vec #fr(cons compiler-dll "--check" + (list-append (against-flags roots) + (list-append batch files)))))))) + +;; `bjo test`: builds each test file and runs it, with `filters` as its +;; arguments, so that a test file ending in `(run-tests)` runs the tests whose +;; names contain one of them. A test file prints its failures and one line, +;; so this adds nothing for a single file; for several, one line at the end +;; says how many failed. Answers 1 if any file failed to build or to pass. +;; +;; `named?` says that the files were named on the command line. A file found +;; under tests/ that has no main is a module that the tests import, such as a +;; fixture, and is not run. A named file without a main is an error. +(: cmd-test #:sync (-> (List string) bool (List string) bool bool bool int)) +(defun (cmd-test files named? filters debug? force? locked?) + (defun (destructure-roots file) + (def (Ok roots) (roots-for-file file locked?) + :leave ((Err why) (Some (fail why)))) + (match (ensure-built file debug? #f force? roots) + ((Err status) (Some status)) + ((Ok (Tuple rec _)) #:when (string-ends-with? (record-ref rec output) ".dll") + (if named? + (Some (fail #"bjo test: ${file} has no main. End a test file with (run-tests).")) + None)) + ((Ok (Tuple rec _)) + (Some (exec "dotnet" (list->vec (Cons (record-ref rec output) filters))))))) + (def statuses + (loop (:for file files) + (:let status (destructure-roots file)) + (:when-let (Some code) status) + (:do (unless (= code 0) (println #"bjo test: ${file} failed."))) + (:acc out (listing code)))) + (def failed (length (list-filter #(not (= & 0)) statuses))) + (when (> (length statuses) 1) + (println (if (= failed 0) + #"bjo test: ${(length statuses)} files, all passed." + #"bjo test: ${failed} of ${(length statuses)} files failed."))) + (if (= failed 0) 0 1)) + +;; `bjo bench` is `bench.bjo`'s. What it needs from here is lent to it: a +;; benchmark file built in Release, whatever `-d` says; a program run on the +;; console; and a source file found. +(: build-bench #:sync (-> string bool bool (Result int BenchProgram))) +(defun (build-bench file force? locked?) + (match (roots-for-file file locked?) + ((Err why) (Err (fail why))) + ((Ok roots) + (match (ensure-built file #f #f force? roots) + ((Err status) (Err status)) + ((Ok (Tuple rec _)) #:when (string-ends-with? (record-ref rec output) ".dll") + (Err (fail #"bjo bench: ${file} has no main. End a benchmark file with (run-benches)."))) + ((Ok (Tuple rec _)) (Ok (BenchProgram (source file) (exe (record-ref rec output))))))))) + +(: (-> bool bool BenchTools)) +(defun (bench-tools force? locked?) + (BenchTools (build (fun (file) (build-bench file force? locked?))) + (exec exec) + (resolve resolve-source))) + +;; The compiler's REPL edits its own lines at a terminal, with completion and +;; history, so it is run as it is. Under a wrapper that edits lines itself, +;; such as `rlwrap`, set BJOLANG_REPL_PLAIN to have it read line by line. +(: cmd-repl #:sync (-> Against int)) +(defun (cmd-repl roots) + (def flags (against-flags roots)) + (if (not (have-compiler?)) + 1 + (exec "dotnet" (list->vec (Cons compiler-dll + (list-append flags (Cons "--repl" Nil))))))) + + +;; --------------------------------------------------------------------------- +;; Projects +;; --------------------------------------------------------------------------- + +;; The flags a command was given, and what was left after them. Declared here +;; rather than beside the parser below it, because the project commands take +;; one and a signature cannot name a type that has not been declared yet. +(type + (: Cmdline (Record (: debug bool) + (: force bool) + (: lib bool) + (: locked bool) + ;; `-o DIR`: where `publish` writes. Only `publish` takes it. + (: output (Option string)) + ;; `--whole-stdlib`: `publish` copies all of the standard + ;; library, sources and documentation included. + (: whole-stdlib bool) + (: rest (List string))))) + +;; The directory a file sits in, for the walk upward that finds a project. +(: (-> string string)) +(defun (file-directory file) + (match (parent-directory (path-absolute file)) + ((Some dir) dir) + (None (path-absolute file)))) + +;; The project a directory is inside, if it is inside one. +(: project-around #:sync (-> string (Result string (Option Project)))) +(defun (project-around where) + (match (find-project-dir where) + (None (Ok None)) + ((Some dir) + (def (Ok project) (project-at dir) :propagate) + (Ok (Some project))))) + +;; The project a command that named no file was run in. Not finding one is an +;; error here, because the command had nothing else to work on. +(: project-here #:sync (-> string (Result string Project))) +(defun (project-here command) + (def (Ok (Some project)) (project-around (current-directory)) + :leave ((Ok None) + (Err (str #"bjo ${command}: this is not a project — there is no ${manifest-name} " + "here or in any directory above. Name a file, or make a project with 'bjo init'."))) + ((Err why) (Err why))) + (Ok project)) + +;; The roots a command compiles against, from where it was run. +;; +;; `None` outside a project, which is every command behaving as it did before +;; there were projects. Inside one, the project is resolved and fetched first: +;; a file in a project imports the project's packages whether it is reached as +;; `src/main.bjo` or as the entry. +(: roots-near #:sync (-> string bool (Result string Against))) +(defun (roots-near where locked?) + (def (Ok (Some project)) (project-around where) + :leave ((Ok None) (Ok nothing-against)) + ((Err why) (Err why))) + (def (Ok ready) (prepare project locked?) :propagate) + (Ok (prepared-against ready))) + +(: roots-for-file #:sync (-> string bool (Result string Against))) +(defun (roots-for-file file locked?) + (roots-near (file-directory file) locked?)) + +;; Builds a project, and answers the entry's build record where there is one. +;; +;; The entry decides the shape. A program is one compile of one file: the +;; compiler follows its imports and builds what they name, in order. A library +;; has no entry to follow, so every module under `src/` is named instead. +;; +;; `say?` is off for `run`, which has the program's own output to show and +;; nothing to add to it. +(: build-project #:sync (-> Project Cmdline bool (Result int (Option BuildRecord)))) +(defun (build-project project line say?) + (with-return ret + (def (Ok ready) (prepare project (record-ref line locked)) + :leave ((Err why) (Err (fail why)))) + (def roots (prepared-against ready)) + + (unless (project-has-entry? project) + (ret (build-library project line roots say?))) + + (match (ensure-built (project-entry project) + (record-ref line debug) #f (record-ref line force) roots) + ((Err status) (Err status)) + ((Ok (Tuple rec built?)) + (when say? + (println (str (if built? "Built" "Up to date") ": " (record-ref rec output)))) + (Ok (Some rec)))))) + +(: (-> Project string)) +(defun (package-name-of project) + (name->string (record-ref (project-package project) name))) + +(: build-library #:sync (-> Project Cmdline Against bool (Result int (Option BuildRecord)))) +(defun (build-library project line roots say?) + (with-return ret + (def sources (sources-under (project-src project))) + (when (empty? sources) + (ret (Err (fail #"bjo build: there are no modules under ${(project-src project)}.")))) + + (def status (if (record-ref line debug) + (compile-batch sources #t roots) + (compile-graph sources #f roots))) + (unless (= status 0) (ret (Err status))) + + (when say? + (def n (length sources)) + (println (str (package-name-of project) " is a library: " + (int->string n) (if (= n 1) " module" " modules") + " under src/, all current."))) + (Ok None))) + +(: build-and-say #:sync (-> Project Cmdline int)) +(defun (build-and-say project line) + (match (build-project project line #t) + ((Err status) status) + ((Ok _) 0))) + +;; `bjo run` on a project: build it, then start the entry's artefact. +(: run-project #:sync (-> Project Cmdline (List string) int)) +(defun (run-project project line arguments) + (with-return ret + (unless (project-has-entry? project) + (ret (fail (str "bjo run: " (package-name-of project) " is a library — there is no " + (record-ref (project-package project) entry) + " to run. 'bjo build' builds it.")))) + + (match (build-project project line #f) + ((Err status) status) + ;; A project with an entry always answers with its record. + ((Ok None) 1) + ((Ok (Some rec)) + (def out (record-ref rec output)) + (if (string-ends-with? out ".dll") + (fail (str "bjo run: " (project-entry project) " has no main, so it builds a library: " out)) + (run-built rec arguments)))))) + +;; `bjo check` with no file, inside a project: everything the project holds. +(: check-project #:sync (-> Project Cmdline int)) +(defun (check-project project line) + (with-return ret + (def (Ok ready) (prepare project (record-ref line locked)) + :leave ((Err why) (fail why))) + (def sources (sources-under (project-src project))) + (def files (if (and (project-has-entry? project) + (not (any? #(= & (project-entry project)) sources))) + (Cons (project-entry project) sources) + sources)) + (when (empty? files) + (ret (fail #"bjo check: there are no modules under ${(project-src project)}."))) + (cmd-check files (prepared-against ready)))) + + +;; --------------------------------------------------------------------------- +;; Publishing +;; --------------------------------------------------------------------------- + +;; The runtime the compiler links programs against, where the compiler finds +;; it (`Paths.runtimeDir`): the compiler's own directory, whose copies are +;; precompiled, or the Release build of BjolangRuntime when BJOLANG_NO_R2R +;; turns the precompiled code off. +(: string) +(def runtime-dir + (let ((precompiled (path-combine root "bin" "compiler")) + (off? (match (get-environment-variable "BJOLANG_NO_R2R") + ((Some v) (not (or (= v "") (= v "0")))) + (None #f)))) + (if (and (not off?) (file-exists? (path-combine precompiled "BjolangRuntime.dll"))) + precompiled + (path-combine root "BjolangRuntime" "bin" "Release" target-framework)))) + +;; The packages a project was built from, with the resources their manifests +;; declare. The project itself is the one whose files go under `app/`. +(: (-> Project Prepared (List PublishedPackage))) +(defun (published-packages project ready) + (def own-name (record-ref (project-package project) name)) + (loop (:for r (prepared-roots ready)) + (:when-let (Some dir) (root-package-dir r)) + (:let resources (match (root-package r) + ((Some package) (record-ref package resources)) + (None Nil))) + (:acc out (listing (PublishedPackage (name (root-name r)) + (dir dir) + (resources resources) + (own (= (root-name r) own-name))))) + => out)) + +;; Publishes a program that has been built, and says where to. +(: publish-built #:sync (-> BuildRecord (List PublishedPackage) Cmdline string int)) +(defun (publish-built rec packages line out) + (def program (record-ref rec output)) + (if (string-ends-with? program ".dll") + (fail (str "bjo publish: " program " has no main, so it is a library. Only a program can be published.")) + (match (publish (Publication (program program) + (native-program (record-ref rec aot)) + (links (record-ref rec links)) + (natives (record-ref rec natives)) + (packages packages) + (lib (lib-dir)) + (whole-stdlib (record-ref line whole-stdlib)) + (runtime runtime-dir) + (out out))) + ((Err why) (fail why)) + ((Ok n) + (def dir (path-absolute out)) + (def (Some name) (path-filename program) + :default program) + ;; The copies of the program's modules, precompiled in place. The + ;; builds stay IL, so that a rebuild after an edit runs no crossgen2. + (def status (exec "dotnet" (list->vec (list compiler-dll "--precompile-published" dir)))) + (unless (= status 0) + (println #"bjo publish: precompiling the published modules failed (exit code ${(int->string status)}). They are left as IL.")) + (println #"Published ${program} to ${dir}: ${n} files.") + (def (Some native) (path-filename (record-ref rec aot)) + :default "") + (if (= native "") + (println #"Run it with: dotnet ${(path-combine dir name)}") + (println #"Run it with: ${(path-combine dir native)}")) + 0)))) + +;; `bjo publish` in a project: a release build of its entry, then the copy. +(: publish-project #:sync (-> Project Cmdline int)) +(defun (publish-project project line) + (with-return ret + (unless (project-has-entry? project) + (ret (fail (str "bjo publish: " (package-name-of project) " is a library — there is no " + (record-ref (project-package project) entry) " to publish.")))) + (def (Ok ready) (prepare project (record-ref line locked)) + :leave ((Err why) (fail why))) + (match (ensure-built (project-entry project) #f #f (record-ref line force) (prepared-against ready)) + ((Err status) status) + ((Ok (Tuple rec _)) + (publish-built rec + (published-packages project ready) + line + (match (record-ref line output) + ((Some dir) dir) + (None (path-combine (project-dir project) "publish")))))))) + +;; `bjo publish file.bjo`. A file inside a project is built against the +;; project's packages, and carries their resources, as `bjo build` would build +;; it; one outside a project has neither. +(: publish-file #:sync (-> string Cmdline int)) +(defun (publish-file file line) + (with-return ret + (def (Ok around) (project-around (file-directory file)) + :leave ((Err why) (fail why))) + (def context + (match around + (None (Ok None)) + ((Some project) + (match (prepare project (record-ref line locked)) + ((Ok ready) (Ok (Some (Tuple project ready)))) + ((Err why) (Err why)))))) + (def (Ok prepared) context :leave ((Err why) (fail why))) + (def roots (match prepared + ((Some (Tuple _ ready)) (prepared-against ready)) + (None nothing-against))) + (def packages (match prepared + ((Some (Tuple project ready)) (published-packages project ready)) + (None Nil))) + (match (ensure-built file #f #f (record-ref line force) roots) + ((Err status) status) + ((Ok (Tuple rec _)) + (publish-built rec + packages + line + (match (record-ref line output) + ((Some dir) dir) + (None (path-combine (file-directory file) "publish")))))))) + + +;; --------------------------------------------------------------------------- +;; The command line +;; --------------------------------------------------------------------------- + +;; Flags come before the source file, and `--` ends them for a file whose name +;; begins with a dash. The first thing that is not a flag ends them too, and +;; everything from there on is positional — so `bjo run prog.bjo -d` passes `-d` +;; to the program rather than to the compiler. +;; +;; With `pass-unknown?`, a flag bjo does not know ends bjo's flags instead of +;; being an error: `bjo test --seed 7` gives `--seed 7` to the test files. +(: (-> (List string) bool (Result string Cmdline))) +(defun (parse-flags argv pass-unknown?) + (let go ((left argv) (debug #f) (force #f) (lib #f) (locked #f) (output None) (whole-stdlib #f)) + (defun (done rest) + (Cmdline (debug debug) (force force) (lib lib) (locked locked) + (output output) (whole-stdlib whole-stdlib) (rest rest))) + (match left + (Nil (Ok (done Nil))) + ((Cons a more) + (case a + ("--" (Ok (done more))) + (("-d" "--debug") (go more #t force lib locked output whole-stdlib)) + (("-c" "--compile") (go more debug #t lib locked output whole-stdlib)) + ("--lib" (go more debug force #t locked output whole-stdlib)) + ("--locked" (go more debug force lib #t output whole-stdlib)) + ("--whole-stdlib" (go more debug force lib locked output #t)) + (("-o" "--output") + (match more + ((Cons dir after) (go after debug force lib locked (Some dir) whole-stdlib)) + (Nil (Err "bjo: -o needs a directory")))) + (else + (if (and (string-starts-with? a "-") (not pass-unknown?)) + (Err #"bjo: unknown flag '${a}'") + (Ok (done left))))))))) + +;; `bjo fmt`: each file laid out as bjo-mode lays it out (see `format.bjo`), +;; and written back if that changed it, or with `check?` only named. Answers 1 +;; when a file could not be laid out, or, with `check?`, when one is not. +(: (-> (List string) bool bool int)) +(defun (cmd-fmt files check? diff?) + (def failures (list-foldl (fun (file n) (+ n (fmt-file file check? diff?))) 0 files)) + (if (> failures 0) 1 0)) + +;; Lays out one file, and answers 1 when that fails the command: the file +;; could not be laid out, or, with `check?`, it is not. A file that is laid out +;; already is not mentioned. With `diff?` nothing is written: what would change +;; is printed as a unified diff, which `patch -p0` takes, and so what is said +;; about a line goes to standard error, out of the diff's way. +(: (-> string bool bool int)) +(defun (fmt-file file check? diff?) + (def source (file-read-text file)) + (def shown (as-seen-from-here file)) + (match (format-source source) + ((Err why) (println #"bjo fmt: ${shown}: ${why}") 1) + ((Ok (Tuple text _)) #:when (= text source) 0) + ((Ok (Tuple text long)) #:when diff? + (display (unified-diff shown shown source text)) + (loop (:for at long) + (:do (writeln (parameter-ref current-error-port) (noted shown at)))) + (if check? 1 0)) + ((Ok (Tuple _ long)) #:when check? + (println #"${shown}: not formatted") + (list-for-each #(println (noted shown &)) long) + 1) + ((Ok (Tuple text long)) + (file-write-text file text) + (println #"formatted ${shown}") + (list-for-each #(println (noted shown &)) long) + 0))) + +;; What `format-source` says about a line of `file`, as a line of output. +(: (-> string (Tuple int string) string)) +(defun (noted file at) + (def (Tuple line note) at) + #"${file}:${(int->string line)}: ${note}") + +;; A path under the current directory without it, as a person would type it. +(: (-> string string)) +(defun (as-seen-from-here path) + (def cwd (current-directory)) + (def here (if (string-ends-with? cwd "/") cwd (str cwd "/"))) + (if (string-starts-with? path here) (string-drop path (string-length here)) path)) + +;; `bjo fmt -`: standard input laid out to standard output, for an editor. What +;; is wrong goes to standard error, and nothing to standard output. +(: (-> bool int)) +(defun (fmt-stdin diff?) + (def input (get current-input-port)) + (def lines + (let go ((acc Nil)) + (match (read-line/opt input) + ((Some line) (go (Cons line acc))) + (None (list-reverse acc))))) + (def source (str (string-join (list->vec lines) "\n") "\n")) + (match (format-source source) + ((Ok (Tuple text long)) + (display (if diff? (unified-diff "-" "-" source text) text)) + (loop (:for at long) + (:do (writeln (parameter-ref current-error-port) (noted "-" at)))) + 0) + ((Err why) + (writeln (parameter-ref current-error-port) #"bjo fmt: ${why}") + 1))) + +;; The files `bjo fmt` lays out: those named, a directory's named, or with +;; nothing named this project's src/, tests/ and benches/. +(: (-> (List string) (Result string (List string)))) +(defun (fmt-targets named) + (match named + (Nil + (def (Ok project) (project-here "fmt") + :propagate) + ;; TODO: this is not how to build a list :( + (Ok (loop (:for dir (list "src" "tests" "benches")) + (:acc out (folding Nil (list-append out (sources-under (path-combine (project-dir project) dir))))) + => out))) + (_ + (defun (do-it name out) + (def (Ok so-far) out :propagate) + (cond + ((directory-exists? name) (Ok (list-append so-far (sources-under name)))) + ((file-exists? name) (Ok (list-append so-far (list name)))) + (else (Err #"bjo fmt: no such file or directory '${name}'.")))) + + (loop (:for name named) + (:acc out (folding (Ok Nil) (do-it name out))) + => out)))) + +(: (-> void)) +(defun (usage) + (println "bjo - the Bjolang command") + (println "") + (println "Usage:") + (println " bjo init [name] [--lib] Make a project here. The name defaults to") + (println " this directory's; --lib makes a library.") + (println " bjo fetch [--locked] Fetch what the manifest names. Compiles nothing.") + (println " bjo run [-d] [-c] [args...] Build this project and run it.") + (println " bjo run . [args...] The same, when the program takes arguments.") + (println " bjo run [-d] [-c] [args...] Compile one file if it needs it, then run it.") + (println " Arguments after the file go to the program.") + (println " bjo build [-d] [-c] [--locked] Build this project.") + (println " bjo build [-d] [-c] [--lib] Compile one file if it needs it, and stop.") + (println " bjo check [...] Report errors without building anything.") + (println " With no file, every module of this project.") + (println " bjo test [-d] [-c] [files...] [names...]") + (println " Build and run test files: those named, or") + (println " every one under tests/. A name that is not") + (println " a file runs only the tests that contain it.") + (println " bjo bench [-c] [files...] [names...] [flags...]") + (println " Build benchmark files in Release and run them:") + (println " those named, or every one under benches/.") + (println " At a terminal with nothing named, a menu asks") + (println " what to run. The flags are the benchmark") + (println " programs': --all, --quick, --save FILE, ...") + (println " bjo publish [-c] [-o dir] [] Build a program for release, and copy it with") + (println " everything it loads into one directory that") + (println " runs from anywhere. Default: publish/ beside it.") + (println " bjo fmt [--check] [--diff] [files or dirs...]") + (println " Lay source out as bjo-mode does: those named,") + (println " or every file under src/, tests/ and benches/.") + (println " --check writes nothing and names what is not") + (println " laid out; --diff writes nothing and prints the") + (println " changes as a diff. bjo fmt - reads standard input.") + (println " bjo repl Start a REPL.") + (println " bjo rebjo Rebuild the compiler and bjo itself.") + (println " bjo help Show this message.") + (println "") + (println "Flags:") + (println " -d, --debug Unoptimized, for a debugger.") + (println " Always recompiles, since the dumps are made by compiling.") + (println " -c, --compile Compile even when nothing has changed.") + (println " --lib Build a .dll. A file with no main builds one anyway.") + (println " --locked Refuse to change bjo.lock or packages.lock.json: any difference") + (println " from them is an error. For a build that must be the one the") + (println " locks describe.") + (println " -o Where publish writes. An existing directory is replaced only") + (println " when an earlier publish made it.") + (println " --whole-stdlib For publish: copy all of the standard library, sources and") + (println " documentation included, for a program that reads Bjolang's") + (println " own modules at run time. Otherwise only what it links.") + (println " -q, --quiet Before the command: say nothing about building bjo itself") + (println " unless that fails. For a tool reading what bjo writes.") + (println " -- End of flags, for a file whose name begins with a dash.") + (println "") + (println "A project is a directory with a manifest.bjodat in it, found by looking") + (println "upward from where you stand. Its modules are in src/: src/lexer.bjo of a") + (println "package called (myapp) is the module (myapp lexer). Outside a project every") + (println "command works on the file it is given, as it always has.") + (println "") + (println "Nothing is rebuilt when the artefact is newer than everything the last") + (println "build read: the compiler records that, and bjo compares timestamps.") + (println "") + (println "The compiler is not one of the things bjo keeps current: it is not asked") + (println "whether it is behind its own sources, on the way to anything. Editing it") + (println "is 'bjo rebjo', which builds it and this command — and that makes every") + (println "artefact stale the way any other edit does.")) + +;; Every name resolved, or the first that names nothing. Resolved here rather +;; than left to the compiler, so that a misspelling is reported as one by the +;; command that was given it. +(: (-> string (List string) (Result string (List string)))) +(defun (resolve-all command names) + (with-return ret + (Ok (loop (:for name names) + (:let resolved (resolve-source name)) + ;; The one exit a loop cannot take by itself: not the end of the + ;; loop but the end of the command. `ret` is a statement here + ;; because that is the only place it may stand. + (:do (unless (some? resolved) + (ret (Err #"bjo ${command}: no such file '${name}'.")))) + (:when-let (Some file) resolved) + (:acc out (listing file)) + => out)))) + +(: dispatch #:sync (-> string Cmdline int)) +(defun (dispatch command line) + (def* (debug? (record-ref line debug)) + (force? (record-ref line force)) + (lib? (record-ref line lib)) + (locked? (record-ref line locked)) + (rest (record-ref line rest))) + + (case command + ("init" + (with-return ret + (when (or debug? force? locked?) + (ret (fail "bjo init: writes files and compiles nothing, so it takes no build flags."))) + (cmd-init rest lib?))) + + ;; Resolve, fetch and write the roots file, and stop there. What `build` + ;; does first, for a checkout that is about to be built offline or by + ;; something that is not `bjo`. + ("fetch" + (with-return ret + (when (or debug? force? lib?) + (ret (fail "bjo fetch: compiles nothing, so it takes no build flags."))) + (unless (empty? rest) (ret (fail "bjo fetch: takes no arguments."))) + (def (Ok project) (project-here "fetch") :leave ((Err why) (fail why))) + (def (Ok ready) (prepare project locked?) :leave ((Err why) (fail why))) + (println (str (package-name-of project) ": " + (int->string (prepared-dependencies ready)) " dependencies ready.")) + 0)) + + ("run" + (with-return ret + (when lib? + (ret (fail "bjo run: --lib builds a library, and a library does not run."))) + ;; No file names this project. So does `.`, which exists only so that + ;; arguments can follow it: everything after the file belongs to the + ;; program, and a project with no file named has nothing to put them + ;; after. + (def (Cons name args) rest + :leave (Nil (match (project-here "run") + ((Err why) (fail why)) + ((Ok project) (run-project project line Nil))))) + (when (= name ".") + (ret (match (project-here "run") + ((Err why) (fail why)) + ((Ok project) (run-project project line args))))) + (def (Some file) (resolve-source name) + :leave-with (fail #"bjo run: no such file '${name}'.")) + (def (Ok roots) (roots-for-file file locked?) :leave ((Err why) (fail why))) + (cmd-run file args debug? force? roots))) + + ("build" + (with-return ret + (def (Cons name more) rest + :leave (Nil (match (project-here "build") + ((Err why) (fail why)) + ((Ok project) (build-and-say project line))))) + (when (= name ".") + (ret (cond + ((not (empty? more)) + (fail "bjo build: '.' is the whole project, so nothing may follow it.")) + (else (match (project-here "build") + ((Err why) (fail why)) + ((Ok project) (build-and-say project line))))))) + (unless (empty? more) (ret (fail "bjo build: takes one source file."))) + (def (Some file) (resolve-source name) + :leave-with (fail #"bjo build: no such file '${name}'.")) + (def (Ok roots) (roots-for-file file locked?) :leave ((Err why) (fail why))) + (cmd-build file debug? lib? force? roots))) + + ("check" + (with-return ret + (when (or debug? force? lib?) + (ret (fail "bjo check: builds nothing, so it takes no build flags."))) + (when (empty? rest) + (ret (match (project-here "check") + ((Err why) (fail why)) + ((Ok project) (check-project project line))))) + (def (Ok resolved) (resolve-all "check" rest) + :leave ((Err message) (fail message))) + (def (Cons first _) resolved + :leave-with (fail "bjo check: no source file. Try 'bjo help'.")) + (def (Ok roots) (roots-for-file first locked?) :leave ((Err why) (fail why))) + (cmd-check resolved roots))) + + ;; The test files named, or every one under the project's tests/. An + ;; argument that is not a file is a filter on test names. + ("test" + (with-return ret + (when lib? (ret (fail "bjo test: a test file is a program, so it takes no --lib."))) + (def named (list-filter #(some? (resolve-source &)) rest)) + (def filters (list-filter #(none? (resolve-source &)) rest)) + (def found + (if (empty? named) + (match (project-here "test") + ((Err why) (Err why)) + ((Ok project) (Ok (sources-under (path-combine (project-dir project) "tests"))))) + (Ok (loop (:for name named) + (:when-let (Some file) (resolve-source name)) + (:acc out (listing file)))))) + (def (Ok files) found :leave ((Err why) (fail why))) + (when (empty? files) + (ret (fail "bjo test: there are no test files. Put them under tests/, or name them."))) + (cmd-test files (not (empty? named)) filters debug? force? locked?))) + + ;; The benchmark files named, or every one under the project's benches/. + ;; Always a Release build: a Debug one measures code the JIT did not + ;; optimise. + ("bench" + (with-return ret + (when lib? (ret (fail "bjo bench: a benchmark file is a program, so it takes no --lib."))) + (when debug? (println "bjo bench: -d is ignored: benchmarks are always built in Release.")) + (def (Tuple named passed) (split-bench-args resolve-source rest)) + (def found + (if (empty? named) + (match (project-here "bench") + ((Err why) (Err why)) + ((Ok project) (Ok (sources-under (path-combine (project-dir project) "benches"))))) + (Ok named))) + (def (Ok files) found :leave ((Err why) (fail why))) + (when (empty? files) + (ret (fail "bjo bench: there are no benchmark files. Put them under benches/, or name them."))) + (cmd-bench (bench-tools force? locked?) files passed))) + + ;; A release build of a program, and a copy of everything it loads in one + ;; directory that can be moved anywhere. See `publish.bjo`. + ("publish" + (with-return ret + (when (or debug? lib?) + (ret (fail "bjo publish: publishes a release build of a program, so it takes no -d or --lib."))) + (cond + ((or (empty? rest) (= rest (list "."))) + (match (project-here "publish") + ((Err why) (fail why)) + ((Ok project) (publish-project project line)))) + ((= (length rest) 1) + (def (Cons name _) rest :leave-with 1) + (def (Some file) (resolve-source name) + :leave-with (fail #"bjo publish: no such file '${name}'.")) + (publish-file file line)) + (else (fail "bjo publish: takes one source file, or none for this project."))))) + + ("repl" + (with-return ret + (when (or debug? force? lib?) (ret (fail "bjo repl: takes no build flags."))) + (unless (empty? rest) (ret (fail "bjo repl: takes no arguments."))) + ;; Inside a project the prompt sees the project's packages, which is + ;; what makes `(import (myapp lexer))` work there. + (def (Ok roots) (roots-near (current-directory) locked?) + :leave ((Err why) (fail why))) + (cmd-repl roots))) + + ;; Reached only when this binary was started directly. The launcher takes + ;; `rebjo` before it gets here, and has to: the command rebuilds `bjo.exe`, + ;; which is the file this process is running from. + ("rebjo" + (fail "bjo rebjo: run it through the bjo launcher — it rebuilds this binary, which is running.")) + + ;; Source laid out as bjo-mode lays it out. See `format.bjo`. + ("fmt" + (with-return ret + (when (or debug? force? lib?) + (ret (fail "bjo fmt: builds nothing, so it takes no build flags."))) + (def check? (not (empty? (list-filter #(= & "--check") rest)))) + (def diff? (not (empty? (list-filter #(= & "--diff") rest)))) + (def named (list-filter #(not (or (= & "--check") (= & "--diff"))) rest)) + (when (= named (list "-")) + (ret (if check? + (fail "bjo fmt: - writes what it lays out, so it takes no --check.") + (fmt-stdin diff?)))) + (match (list-filter #(string-starts-with? & "-") named) + ((Cons flag _) (ret (fail #"bjo fmt: unknown flag '${flag}'."))) + (Nil (ignore 0))) + (def (Ok files) (fmt-targets named) :leave ((Err why) (fail why))) + (cmd-fmt files check? diff?))) + + (("help" "-h" "--help") (usage) 0) + + (else (fail #"bjo: '${command}' is not a command. Try 'bjo help'.")))) + +(defun (main args) + ;; Before any git process is started. `(std run)` redirects a child's stdin + ;; in order to write to it, so a repository that wants a password would read + ;; from a pipe nobody writes to and wait there forever. With this set, git + ;; fails instead, and says what it wanted. + (set-environment-variable! "GIT_TERMINAL_PROMPT" "0") + + ;; The command comes first and is never a flag, so the flags of the command + ;; are what follows it. + (def (Cons command rest) (vec->list args) + :leave (Nil (usage) 1)) + (match (parse-flags rest (or (= command "test") (= command "bench") (= command "fmt"))) + ((Err message) (fail message)) + ((Ok line) #:when (and (some? (record-ref line output)) (not (= command "publish"))) + (fail #"bjo ${command}: -o is for publish, which writes a directory; ${command} does not.")) + ((Ok line) (dispatch command line)))) +;; bjo/diff.bjo — a unified diff of two texts, line by line, for +;; `bjo fmt --diff`. +;; +;; Made for what `bjo fmt` does to a file: it changes lines in place, joins a +;; line of closing brackets to the one before it, and drops blank lines. So +;; the lines are walked together, and where they part the next place two lines +;; in a row agree again is looked for a few lines ahead; where there is none, +;; the lines are taken as changed one for one, which is what a re-indented +;; block is. That is not always the shortest diff, but it is always a true one, +;; and it costs nothing on a long file. + +(export unified-diff) + +(import (std prelude)) + +;; What becomes of a line: kept, at its index in the old text and in the new; +;; or removed or added, at the index in the old and the new where it is. +(type (: Op (Union (: Same int int) (: Del int int) (: Add int int)))) + +;; How far ahead a place where the texts agree again is looked for. +(: reach int) +(def reach 40) + +;; Lines of context around a change. +(: context int) +(def context 3) + +(: lines (-> string (Vec string))) +(defun (lines s) + (def parts (string-split s "\n")) + (def n (vec-length parts)) + (if (and (> n 0) (string-empty? (vec-ref parts (- n 1)))) (vec-take parts (- n 1)) parts)) + +;; Whether old line `i` and new line `j` agree, and the lines after them too, +;; or one of the texts ends there: one equal line alone, a blank one or a `))`, +;; agrees anywhere. +(: agree? (-> (Vec string) (Vec string) int int bool)) +(defun (agree? a b i j) + (def n (vec-length a)) + (def m (vec-length b)) + (cond + ((and (= i n) (= j m)) #t) + ((or (>= i n) (>= j m)) #f) + ((not (= (vec-ref a i) (vec-ref b j))) #f) + ((or (= (+ i 1) n) (= (+ j 1) m)) #t) + (else (= (vec-ref a (+ i 1)) (vec-ref b (+ j 1)))))) + +;; How many old lines to remove and new ones to add from `i` and `j` on for the +;; texts to agree again, the fewest first, if they do within `reach`. +(: resync (-> (Vec string) (Vec string) int int (Option (Tuple int int)))) +(defun (resync a b i j) + (let by-distance ((d 1)) + (if (> d reach) + None + (let by-split ((x 0)) + (cond + ((> x d) (by-distance (+ d 1))) + ((agree? a b (+ i x) (+ j (- d x))) (Some (Tuple x (- d x)))) + (else (by-split (+ x 1)))))))) + +;; `count` ops made by `make` from `from` on, onto `ops`, the last first. +(: ops-onto (-> (List Op) int int (-> int Op) (List Op))) +(defun (ops-onto ops from count make) + (let go ((k 0) (ops ops)) + (if (>= k count) ops (go (+ k 1) (Cons (make (+ from k)) ops))))) + +(: edit-script (-> (Vec string) (Vec string) (Vec Op))) +(defun (edit-script a b) + (def n (vec-length a)) + (def m (vec-length b)) + (let go ((i 0) (j 0) (ops Nil)) + (cond + ((and (>= i n) (>= j m)) (list->vec (list-reverse ops))) + ((>= i n) (go i (+ j 1) (Cons (Add i j) ops))) + ((>= j m) (go (+ i 1) j (Cons (Del i j) ops))) + ((= (vec-ref a i) (vec-ref b j)) (go (+ i 1) (+ j 1) (Cons (Same i j) ops))) + (else + (match (resync a b i j) + ((Some (Tuple x y)) + (def removed (ops-onto ops i x #(Del & j))) + (go (+ i x) (+ j y) (ops-onto removed j y #(Add (+ i x) &)))) + (None (go (+ i 1) (+ j 1) (Cons (Add (+ i 1) j) (Cons (Del i j) ops))))))))) + +(: changed? (-> Op bool)) +(defun (changed? op) + (match op + ((Same _ _) #f) + (_ #t))) + +;; The old and the new index an op is at. +(: at-of (-> Op (Tuple int int))) +(defun (at-of op) + (match op + ((Same i j) (Tuple i j)) + ((Del i j) (Tuple i j)) + ((Add i j) (Tuple i j)))) + +;; The hunks: each as the first and the last op in it, changes no more than +;; twice the context apart sharing one. +(: hunks (-> (Vec Op) (List (Tuple int int)))) +(defun (hunks ops) + (def n (vec-length ops)) + (def changes (loop (:for k (range 0 n)) + (:acc out (listing k) #:when (changed? (vec-ref ops k))) + => out)) + (defun (close-hunk first last) (Tuple (max0 (- first context)) (min-of (- n 1) (+ last context)))) + (match changes + (Nil Nil) + ((Cons start rest) + (let go ((rest rest) (first start) (last start) (out Nil)) + (match rest + (Nil (list-reverse (Cons (close-hunk first last) out))) + ((Cons k more) + (if (<= (- k last) (* 2 context)) + (go more first k out) + (go more k k (Cons (close-hunk first last) out))))))))) + +(: max0 (-> int int)) +(defun (max0 x) (if (< x 0) 0 x)) + +(: min-of (-> int int int)) +(defun (min-of x y) (if (< x y) x y)) + +;; `@@ -start,count +start,count @@`, where a side with no lines starts at the +;; line before. +(: hunk-header (-> (Vec Op) int int string)) +(defun (hunk-header ops first last) + (def (Tuple i j) (at-of (vec-ref ops first))) + (def olds (loop (:for k (range first (+ last 1))) + (:acc c (counting k) #:when (match (vec-ref ops k) ((Add _ _) #f) (_ #t))) + => c)) + (def news (loop (:for k (range first (+ last 1))) + (:acc c (counting k) #:when (match (vec-ref ops k) ((Del _ _) #f) (_ #t))) + => c)) + (defun (side start count) #"${(int->string (if (= count 0) start (+ start 1)))},${(int->string count)}") + #"@@ -${(side i olds)} +${(side j news)} @@") + +;; The lines of one hunk: a run of changes as its removed lines and then its +;; added ones, as a unified diff writes it. +(: hunk-lines (-> (Vec string) (Vec string) (Vec Op) int int (List string))) +(defun (hunk-lines a b ops first last) + ;; `out`, `dels` and `adds` are each the last first, so a run of changes goes + ;; onto `out` as its additions and then its removals. + (let go ((k first) (dels Nil) (adds Nil) (out Nil)) + (defun (flushed) (list-append adds (list-append dels out))) + (if (> k last) + (list-reverse (flushed)) + (match (vec-ref ops k) + ((Same i _) (go (+ k 1) Nil Nil (Cons (str " " (vec-ref a i)) (flushed)))) + ((Del i _) (go (+ k 1) (Cons (str "-" (vec-ref a i)) dels) adds out)) + ((Add _ j) (go (+ k 1) dels (Cons (str "+" (vec-ref b j)) adds) out)))))) + +;; The changes from `old` to `new` as a unified diff, the old text named `from` +;; and the new `to`, or "" when they are the same. +(: unified-diff (-> string string string string string)) +(defun (unified-diff from to old new) + (def a (lines old)) + (def b (lines new)) + (def ops (edit-script a b)) + (def found (hunks ops)) + (cond + ((= old new) "") + ((empty? found) + ;; The same lines, so what differs is the end of the file. + #"--- ${from}\n+++ ${to}\n(the file ends in a newline, and it did not)\n") + (else + (def body + (loop (:for h found) + (:acc out (folding Nil + (match h + ((Tuple first last) + (list-append out + (Cons (hunk-header ops first last) + (hunk-lines a b ops first last))))))) + => out)) + (str #"--- ${from}\n+++ ${to}\n" (string-join (list->vec body) "\n") "\n")))) +;; bjo/format.bjo — `bjo fmt`: Bjolang source laid out as bjomode.el lays it +;; out. +;; +;; It re-indents and tidies, keeps every line break the source has but those +;; before a closing bracket, and adds the few that make code read the same +;; everywhere: +;; +;; - every line is indented as `indent-region` in `bjo-mode` indents it; +;; - a line of closing brackets goes up to the end of the line before it, +;; Lisp style, unless that line ends in a comment; +;; - a `def`'s failure part starts a line, as does each form of a `cond`, +;; `begin`, `seq` or `try` over several lines, and each binding of a `let` +;; whose bindings are over several lines (see "Reshaping"); +;; - a line wider than `wide` is broken in its lists, and a string that runs +;; past the limit is split with a line continuation after the last space +;; that fits (see "Breaking long lines"); +;; - trailing whitespace goes, as do spaces just inside a bracket; a tab +;; becomes spaces, or `\t` in a string; +;; - three or more blank lines in a row become two, and the file ends in one +;; newline. +;; +;; The indentation follows Emacs's own algorithm rather than an idea of it, +;; because the two have to agree: `calculate-lisp-indent` for the column a +;; line's form lines up under, `lisp-indent-specform` and `lisp-indent-defform` +;; for a form with a body, the rules `bjo-indent-function` adds, and even the +;; column `lisp-indent-region` keeps for the following lines at the same depth. +;; The comments below say which of them each part is. +;; +;; A string's lines are text, not code. Its continued lines move with it, and +;; a long one is split, but what it stands for is never changed. What comes +;; out is checked against what went in: the same tokens in the same order, a +;; string compared by what it stands for, or the file is left alone. + +(export format-source format-with wide) + +(import (std prelude)) + +;; --------------------------------------------------------------------------- +;; Tokens + +(type (: Tok (Union + (: TOpen string) + (: TClose string) + ;; A symbol, a number, a keyword, a character, or a prefix such + ;; as ' or #' — anything up to a space, a bracket, a quote or a + ;; semicolon. + (: TAtom string) + ;; A string of any kind, as written, over several lines too. + (: TStr string) + (: TComment string) + (: TSpace string) + TNewline))) + +(: text-between (-> (Vec char) int int string)) +(defun (text-between cs from to) + (def sb (stringbuilder-empty)) + (let go ((i from)) + (when (< i to) + (stringbuilder-add! sb (vec-ref cs i)) + (go (+ i 1)))) + (stringbuilder->string sb)) + +(: char-at (-> (Vec char) int char)) +(defun (char-at cs i) + (if (< i (vec-length cs)) (vec-ref cs i) #\nul)) + +(: opening? (-> char bool)) +(defun (opening? c) (or (= c #\() (= c #\[) (= c #\{))) + +(: closing? (-> char bool)) +(defun (closing? c) (or (= c #\)) (= c #\]) (= c #\}))) + +(: line-break? (-> (Vec char) int bool)) +(defun (line-break? cs i) + (or (= (char-at cs i) #\newline) + (and (= (char-at cs i) #\return) (= (char-at cs (+ i 1)) #\newline)))) + +(: blank-char? (-> (Vec char) int bool)) +(defun (blank-char? cs i) + (and (< i (vec-length cs)) (char-whitespace? (vec-ref cs i)) (not (line-break? cs i)))) + +;; What the compiler's lexer counts as part of a name: a character literal's +;; name, `#\space`, is a run of them. +(: symbol-char? (-> char bool)) +(defun (symbol-char? c) + (not (or (char-whitespace? c) (opening? c) (closing? c) + (= c #\,) (= c #\:) (= c #\") (= c #\;) (= c #\')))) + +(: atom-end (-> (Vec char) int int)) +(defun (atom-end cs i) + (def c (char-at cs i)) + (if (or (>= i (vec-length cs)) + (char-whitespace? c) (opening? c) (closing? c) (= c #\") (= c #\;)) + i + (atom-end cs (+ i 1)))) + +(: blank-end (-> (Vec char) int int)) +(defun (blank-end cs i) + (if (blank-char? cs i) (blank-end cs (+ i 1)) i)) + +(: line-end (-> (Vec char) int int)) +(defun (line-end cs i) + (if (or (>= i (vec-length cs)) (line-break? cs i)) i (line-end cs (+ i 1)))) + +(: after-break (-> (Vec char) int int)) +(defun (after-break cs i) + (if (= (char-at cs i) #\return) (+ i 2) (+ i 1))) + +(: quotes-at (-> (Vec char) int int)) +(defun (quotes-at cs i) + (let go ((p i)) + (if (= (char-at cs p) #\") (go (+ p 1)) (- p i)))) + +(: line-of (-> (Vec char) int int)) +(defun (line-of cs i) + (let go ((k 0) (line 1)) + (cond + ((>= k i) line) + ((= (vec-ref cs k) #\newline) (go (+ k 1) (+ line 1))) + (else (go (+ k 1) line))))) + +(: unterminated (-> (Vec char) int string (Result string int))) +(defun (unterminated cs i what) + (Err #"line ${(int->string (line-of cs i))}: ${what} that does not end")) + +;; Past the end of the string at `i`, which is one of the three kinds the +;; lexer reads: plain, raw on one line, or raw over several lines, which ends +;; on the first line whose text begins with as many quotes as opened it. +(: string-end (-> (Vec char) int (Result string int))) +(defun (string-end cs i) + (def n (vec-length cs)) + (def k (quotes-at cs i)) + (def opened (+ i k)) + (def w (blank-end cs opened)) + (cond + ((and (>= k 3) (>= w n)) (unterminated cs i "a raw string")) + ((and (>= k 3) (line-break? cs w)) + (let lines ((p (after-break cs w))) + (def start (blank-end cs p)) + (def e (line-end cs p)) + (cond + ((>= p n) (unterminated cs i "a raw string")) + ((= (quotes-at cs start) k) (Ok (+ start k))) + ((>= e n) (unterminated cs i "a raw string")) + (else (lines (after-break cs e)))))) + ((>= k 3) + (let one ((p opened)) + (def run (quotes-at cs p)) + (cond + ((or (>= p n) (line-break? cs p)) (unterminated cs i "a raw string")) + ((>= run k) (Ok (+ p k))) + ((> run 0) (one (+ p run))) + (else (one (+ p 1)))))) + (else + (let plain ((p (+ i 1))) + (cond + ((>= p n) (unterminated cs i "a string")) + ((= (vec-ref cs p) #\\) (plain (+ p 2))) + ((= (vec-ref cs p) #\") (Ok (+ p 1))) + (else (plain (+ p 1)))))))) + +;; Past the end of the `#"..."` at `i`. A `${...}` hole is code, delimited by +;; brace depth, with strings and `#\{` inside it skipped whole, as the lexer +;; does. +(: interpolated-end (-> (Vec char) int (Result string int))) +(defun (interpolated-end cs i) + (def n (vec-length cs)) + (let outside ((p (+ i 2))) + (cond + ((>= p n) (unterminated cs i "an interpolated string")) + ((= (vec-ref cs p) #\") (Ok (+ p 1))) + ((= (vec-ref cs p) #\\) (outside (+ p 2))) + ((and (= (vec-ref cs p) #\$) (= (char-at cs (+ p 1)) #\{)) + (let hole ((q (+ p 2)) (depth 1)) + (def c (char-at cs q)) + (cond + ((>= q n) (unterminated cs i "an interpolated string")) + ((= c #\") + (match (string-end cs q) + ((Ok e) (hole e depth)) + ((Err why) (Err why)))) + ((and (= c #\#) (= (char-at cs (+ q 1)) #\\)) (hole (+ q 3) depth)) + ((= c #\{) (hole (+ q 1) (+ depth 1))) + ((and (= c #\}) (= depth 1)) (outside (+ q 1))) + ((= c #\}) (hole (+ q 1) (- depth 1))) + (else (hole (+ q 1) depth))))) + (else (outside (+ p 1)))))) + +;; Past the end of the character literal at `i`: a name, `#\space`, or any one +;; character, `#\(` and `#\;` included. +(: char-literal-end (-> (Vec char) int int)) +(defun (char-literal-end cs i) + (def start (+ i 2)) + (def name-end + (let go ((p start)) + (if (and (< p (vec-length cs)) (symbol-char? (vec-ref cs p))) (go (+ p 1)) p))) + (cond + ((> (- name-end start) 1) name-end) + ((< start (vec-length cs)) (+ start 1)) + (else start))) + +;; The token at `i`, and where the next one starts. +(: token-at (-> (Vec char) int (Result string (Tuple Tok int)))) +(defun (token-at cs i) + (def c (vec-ref cs i)) + (defun (to e) (Tuple (TAtom (text-between cs i e)) e)) + (cond + ((line-break? cs i) (Ok (Tuple TNewline (after-break cs i)))) + ((blank-char? cs i) + (def e (blank-end cs i)) + (Ok (Tuple (TSpace (text-between cs i e)) e))) + ((= c #\;) + (def e (line-end cs i)) + (Ok (Tuple (TComment (string-trim-right (text-between cs i e))) e))) + ((opening? c) (Ok (Tuple (TOpen (char->string c)) (+ i 1)))) + ((closing? c) (Ok (Tuple (TClose (char->string c)) (+ i 1)))) + ((= c #\") + (def (Ok e) (string-end cs i) + :leave ((Err why) (Err why))) + (Ok (Tuple (TStr (without-continued-indent (text-between cs i e) (column-at cs i))) e))) + ((and (= c #\#) (= (char-at cs (+ i 1)) #\")) + (def (Ok e) (interpolated-end cs i) + :leave ((Err why) (Err why))) + (Ok (Tuple (TStr (without-continued-indent (text-between cs i e) (column-at cs i))) e))) + ((and (= c #\#) (= (char-at cs (+ i 1)) #\\)) (Ok (to (char-literal-end cs i)))) + (else (Ok (to (atom-end cs i)))))) + +(: tokenize (-> (Vec char) (Result string (List Tok)))) +(defun (tokenize cs) + (let go ((i 0) (out Nil)) + (if (>= i (vec-length cs)) + (Ok (list-reverse out)) + (match (token-at cs i) + ((Err why) (Err why)) + ((Ok (Tuple tok next)) (go next (Cons tok out))))))) + +;; The tokens that make the program: everything but the spacing, and a string +;; with its tabs written `\t`, as the layout writes them, and its continued +;; lines joined, since the layout may split a long one. A comment's words are +;; compared, its tabs being spaces once laid out. +(: significant (-> (List Tok) (List string))) +(defun (significant toks) + (loop (:for t toks) + (:acc out (listing (match t + ((TStr s) (escape-tabs (joined-continuations s))) + ((TComment s) (string-replace (string-replace s "\t" "") " " "")) + (_ (tok-text t)))) + #:when (meaningful? t)) + => out)) + +(: meaningful? (-> Tok bool)) +(defun (meaningful? t) + (match t + ((TSpace _) #f) + (TNewline #f) + (_ #t))) + +(: tok-text (-> Tok string)) +(defun (tok-text t) + (match t + ((TOpen s) s) + ((TClose s) s) + ((TAtom s) s) + ((TStr s) s) + ((TComment s) s) + ((TSpace s) s) + (TNewline "\n"))) + +;; --------------------------------------------------------------------------- +;; Reshaping: the line breaks the layout adds +;; +;; Each rule says where a line has to break, and what it adds is a line break, +;; so that the tokens are what they were: +;; +;; - a `def`'s failure part, `:leave`, `:leave-with`, `:default` or +;; `:propagate`, starts a line of its own, so that the way out stands out; +;; - in a `cond`, a `begin`, a `seq` or a `try` over several lines, each form +;; after the head starts a line, but a keyword argument's value, which +;; stays on the keyword's line: `#:finally (restore)`; +;; - in a `let`'s bindings over several lines, each binding starts a line. +;; +;; A tab in a string, which is otherwise left as it is, is written `\t`. + +;; The line breaks in a token. +(: breaks-in (-> Tok int)) +(defun (breaks-in t) + (match t + (TNewline 1) + ((TStr s) (snd (advance s 0))) + (_ 0))) + +;; For each token, the index of the token that closes it if it opens a list, +;; and how many line breaks come before it. +(: closers-and-lines (-> (Vec Tok) (Tuple (Vec int) (Vec int)))) +(defun (closers-and-lines v) + (def n (vec-length v)) + (def closes (vecbuilder-empty)) + (def lines (vecbuilder-empty)) + (let fill ((k 0) (line 0)) + (when (< k n) + (vecbuilder-add! closes -1) + (vecbuilder-add! lines line) + (fill (+ k 1) (+ line (breaks-in (vec-ref v k)))))) + (let go ((k 0) (open Nil)) + (when (< k n) + (match (vec-ref v k) + ((TOpen _) (go (+ k 1) (Cons k open))) + ((TClose _) + (match open + ((Cons o outer) + (vecbuilder-set! closes o k) + (go (+ k 1) outer)) + (Nil (go (+ k 1) open)))) + (_ (go (+ k 1) open))))) + (Tuple (vecbuilder->vec closes) (vecbuilder->vec lines))) + +;; A list being reshaped: its bracket, its head, what its second form is if it +;; is an atom, its last form so far, how many forms it has so far, whether it +;; spans lines, and whether it is the bindings of a `let`, the forms of which +;; go a line each. +(type (: Shape (Record (: bracket string) + (: head string) + (: second string) + (: last string) + (: count int) + (: spans bool) + (: bindings bool)))) + +;; Whether a form written `raw` is a keyword argument, `#:catch`, which the +;; form after it belongs with: the two stay on one line. +(: keyword-arg? (-> string bool)) +(defun (keyword-arg? raw) + (and (string-starts-with? raw "#:") (> (string-length raw) 2))) + +(: body-only? (-> string bool)) +(defun (body-only? head) + (match head + ((or "cond" "begin" "seq" "try") #t) + (_ #f))) + +;; Whether `s` is a `def` clause, of a `def`, a `:def` or a `def*`, in which a +;; failure part may be. +(: clause? (-> Shape (Option Shape) bool)) +(defun (clause? s outer) + (or (= (record-ref s head) "def") + (= (record-ref s head) ":def") + (and (= (record-ref s bracket) "(") + (not (= (record-ref s head) "def*")) + (match outer + ((Some o) (= (record-ref o head) "def*")) + (None #f))))) + +;; Whether a list opening as the next form of `parent` is that form's bindings. +(: opens-bindings? (-> Shape string bool)) +(defun (opens-bindings? parent bracket) + (def head (record-ref parent head)) + (def named? + (and (= head "let") + (match (string->list (record-ref parent second)) + ((Cons c _) (char-alphabetic? c)) + (Nil #f)))) + (and (= bracket "(") + (binding-form? head) + (= (record-ref parent count) (if named? 2 1)))) + +;; `s` with one more form, written `raw`. +(: with-form (-> Shape string Shape)) +(defun (with-form s raw) + (def n (record-ref s count)) + (Shape (bracket (record-ref s bracket)) + (head (if (= n 0) raw (record-ref s head))) + (second (if (= n 1) raw (record-ref s second))) + (last raw) + (count (+ n 1)) + (spans (record-ref s spans)) + (bindings (record-ref s bindings)))) + +;; Whether the form at position `n` of `s`, written `raw`, has to start a line. +;; One after a keyword argument stays with it. +(: breaks-before? (-> Shape (Option Shape) int string bool)) +(defun (breaks-before? s outer n raw) + (or (and (failure-keyword? raw) (clause? s outer)) + (and (>= n 1) + (record-ref s spans) + (or (body-only? (record-ref s head)) (record-ref s bindings)) + (not (keyword-arg? (record-ref s last)))))) + +;; The tokens with the line breaks the rules ask for, and their strings' tabs +;; written `\t`. +(: reshape (-> (List Tok) (List Tok))) +(defun (reshape toks) + (def v (list->vec toks)) + (def n (vec-length v)) + (def (Tuple closes lines) (closers-and-lines v)) + (defun (spans? k) + (def c (vec-ref closes k)) + (and (>= c 0) (> (vec-ref lines c) (vec-ref lines k)))) + ;; `shapes` are the lists open, the innermost first; `fresh?` whether + ;; nothing but space is on the line yet; `prefixed?` whether a prefix + ;; began the form this token belongs to. + (let go ((k 0) (shapes Nil) (fresh? #t) (prefixed? #f) (out Nil)) + (if (>= k n) + (list-reverse out) + (let ((t (vec-ref v k))) + (def rest (if (< (+ k 1) n) (list (vec-ref v (+ k 1))) Nil)) + (def starts? (and (starts-form? t) (not prefixed?))) + (def raw + (match t + ((TStr _) "\"") + (_ (tok-text t)))) + (def break? + (and starts? + (not fresh?) + (match shapes + ((Cons s outer) + (breaks-before? s (match outer ((Cons o _) (Some o)) (Nil None)) (record-ref s count) raw)) + (Nil #f)))) + (def counted + (match shapes + ((Cons s outer) #:when starts? (Cons (with-form s raw) outer)) + (_ shapes))) + (def shapes-after + (match t + ((TOpen b) + (Cons (Shape (bracket b) (head "") (second "") (last "") (count 0) (spans (spans? k)) + (bindings (match shapes + ((Cons s _) (opens-bindings? s b)) + (Nil #f)))) + counted)) + ((TClose _) + (match counted + ((Cons _ outer) outer) + (Nil Nil))) + (_ counted))) + (def written + (match t + ((TStr s) (TStr (escape-tabs s))) + (_ t))) + (def out-after (Cons written (if break? (Cons TNewline out) out))) + (go (+ k 1) + shapes-after + (match t + (TNewline #t) + ((TSpace _) fresh?) + (_ #f)) + (prefix-of-next? t rest) + out-after))))) + +;; The tokens reshaped until no rule asks for more. A line break one rule adds +;; can make a list span lines, which another rule asks a line break of: a +;; failure part inside a `begin` on one line. The rules only add line breaks, +;; so this ends. +(: reshaped (-> (List Tok) (List Tok))) +(defun (reshaped toks) + (def again (reshape toks)) + (if (= (list-length again) (list-length toks)) again (reshaped again))) + +;; A string's tabs written `\t`, as its value is unchanged by: outside the +;; holes of an interpolated string, and never in a raw one, which has no +;; escapes. A tab after a backslash is kept: the continuations are joined +;; already, so the lexer rejects it, and fmt does not make it mean something. +(: escape-tabs (-> string string)) +(defun (escape-tabs s) + (cond + ((string-starts-with? s "\"\"\"") s) + ((not (string-contains? s "\t")) s) + (else + (def cs (string->vec s)) + (def n (vec-length cs)) + (def sb (stringbuilder-empty)) + (defun (copy from to) (stringbuilder-add-string! sb (text-between cs from to))) + (let go ((p 0)) + (when (< p n) + (def c (vec-ref cs p)) + (cond + ((= c #\\) + (copy p (+ p 2)) + (go (+ p 2))) + ((= c #\tab) + (stringbuilder-add-string! sb "\\t") + (go (+ p 1))) + ((and (= c #\$) (= (char-at cs (+ p 1)) #\{) (string-starts-with? s "#\"")) + (def past (hole-end cs (+ p 2))) + (copy p past) + (go past)) + (else + (stringbuilder-add! sb c) + (go (+ p 1)))))) + (stringbuilder->string sb)))) + +;; --------------------------------------------------------------------------- +;; Line continuations +;; +;; A backslash that ends a line in a string stands for nothing, and neither +;; does the next line's indentation up to the column the string's text starts +;; at; indentation beyond that is text. What a continued line says therefore +;; depends on where the string starts, which the layout moves. So a string is +;; read without the indentation a continuation drops, and written with as much +;; as where it is written drops: the text it stands for stays the same. +;; +;; A raw string has no escapes. In the holes of an interpolated string nothing +;; is changed: a string in a hole that continues lines is left as it is. + +;; The column of `i` as the compiler's lexer counts it: characters since the +;; line began, a tab one. +(: column-at (-> (Vec char) int int)) +(defun (column-at cs i) + (let go ((k i)) + (if (or (= k 0) (= (vec-ref cs (- k 1)) #\newline)) (- i k) (go (- k 1))))) + +;; The column the text of the string `s` written at `col` starts at. +(: text-column (-> string int int)) +(defun (text-column s col) + (if (string-starts-with? s "#\"") (+ col 2) (+ col 1))) + +;; Whether the backslash at `p` is a line continuation, and if so where the +;; line break after it starts. +(: continuation-at (-> (Vec char) int (Option int))) +(defun (continuation-at cs p) + (let go ((k (+ p 1))) + (cond + ((line-break? cs k) (Some k)) + ((or (= (char-at cs k) #\space) (= (char-at cs k) #\tab)) (go (+ k 1))) + (else None)))) + +;; The string `s`, written with its text from `text-col` on, each continuation +;; as `\` and the line break alone, and from each continued line the +;; indentation it drops: what it stands for wherever it is written. Indentation +;; it keeps is kept. +(: without-continued-indent (-> string int string)) +(defun (without-continued-indent s col) + (def text-col (text-column s col)) + (rewrite-continuations s #t (fun (sb cs at) + (let skip ((j at)) + (if (and (< (- j at) text-col) + (or (= (char-at cs j) #\space) (= (char-at cs j) #\tab))) + (skip (+ j 1)) + j))))) + +;; The string `s`, as `without-continued-indent` reads it, written at `col`: +;; each continued line indented to where its text starts. +(: with-continued-indent (-> string int string)) +(defun (with-continued-indent s col) + (def text-col (text-column s col)) + (rewrite-continuations s #t (fun (sb cs at) + (unless (or (>= at (vec-length cs)) (line-break? cs at)) + (stringbuilder-add-string! sb (spaces text-col))) + at))) + +;; `s` with each continuation written as `\` and its line break, and after it +;; what `after` writes, given where the continued line starts; it answers where +;; to go on from. Without `keep?` a continuation's `\` and line break are left +;; out, which joins the lines. A raw string is as it is. +(: rewrite-continuations (-> string bool (-> StringBuilder (Vec char) int int) string)) +(defun (rewrite-continuations s keep? after) + (cond + ((string-starts-with? s "\"\"\"") s) + ((not (string-contains? s "\\")) s) + (else + (def cs (string->vec s)) + (def n (vec-length cs)) + (def sb (stringbuilder-empty)) + (def interpolated? (string-starts-with? s "#\"")) + (let go ((p 0)) + (when (< p n) + (def c (vec-ref cs p)) + (cond + ((= c #\\) + (match (continuation-at cs p) + ((Some k) + (def past (if (= (char-at cs k) #\return) (+ k 2) (+ k 1))) + (when keep? + (stringbuilder-add! sb #\\) + (stringbuilder-add-string! sb (text-between cs k past))) + (go (after sb cs past))) + (None + (stringbuilder-add-string! sb (text-between cs p (+ p 2))) + (go (+ p 2))))) + ((and interpolated? (= c #\$) (= (char-at cs (+ p 1)) #\{)) + (def past (hole-end cs (+ p 2))) + (stringbuilder-add-string! sb (text-between cs p past)) + (go past)) + (else + (stringbuilder-add! sb c) + (go (+ p 1)))))) + (stringbuilder->string sb)))) + +;; The string `s`, as `without-continued-indent` reads it, with its continued +;; lines joined: what it stands for, written without continuations, which a +;; string split by the layout is compared by. +(: joined-continuations (-> string string)) +(defun (joined-continuations s) + (rewrite-continuations s #f (fun (sb cs at) at))) + +;; A token as it is written at `col`: a string with its continued lines +;; indented to where its text then starts. +(: tok-written (-> Tok int string)) +(defun (tok-written t col) + (match t + ((TStr s) (with-continued-indent s col)) + (_ (tok-text t)))) + +;; Past the `}` closing the hole whose code starts at `q`, as +;; `interpolated-end` reads one. +(: hole-end (-> (Vec char) int int)) +(defun (hole-end cs q) + (let hole ((q q) (depth 1)) + (def c (char-at cs q)) + (cond + ((>= q (vec-length cs)) q) + ((= c #\") + (match (string-end cs q) + ((Ok e) (hole e depth)) + ((Err _) (vec-length cs)))) + ((and (= c #\#) (= (char-at cs (+ q 1)) #\\)) (hole (+ q 3) depth)) + ((= c #\{) (hole (+ q 1) (+ depth 1))) + ((and (= c #\}) (= depth 1)) (+ q 1)) + ((= c #\}) (hole (+ q 1) (- depth 1))) + (else (hole (+ q 1) depth))))) + +;; --------------------------------------------------------------------------- +;; Lines + +;; A line's tokens, its own indentation taken off, and that indentation as it +;; was, which a `;;;` comment keeps. +(type (: Line (Record (: toks (List Tok)) (: indent string)))) + +(: lines-of (-> (List Tok) (List Line))) +(defun (lines-of toks) + (let go ((toks toks) (here Nil) (out Nil)) + (match toks + (Nil (list-reverse (Cons (tidy (list-reverse here)) out))) + ((Cons TNewline rest) (go rest Nil (Cons (tidy (list-reverse here)) out))) + ((Cons t rest) (go rest (Cons t here) out))))) + +(: space? (-> Tok bool)) +(defun (space? t) + (match t + ((TSpace _) #t) + (_ #f))) + +(: comment? (-> Tok bool)) +(defun (comment? t) + (match t + ((TComment _) #t) + (_ #f))) + +(: close? (-> Tok bool)) +(defun (close? t) + (match t + ((TClose _) #t) + (_ #f))) + +;; A line without the spaces at its ends and just inside its brackets. +(: tidy (-> (List Tok) Line)) +(defun (tidy toks) + (def indent + (match toks + ((Cons (TSpace s) _) s) + (_ ""))) + (def trimmed (list-reverse (drop-spaces (list-reverse (drop-spaces toks))))) + (Line (toks (inside-brackets trimmed)) (indent indent))) + +(: drop-spaces (-> (List Tok) (List Tok))) +(defun (drop-spaces toks) + (match toks + ((Cons t rest) #:when (space? t) (drop-spaces rest)) + (_ toks))) + +(: inside-brackets (-> (List Tok) (List Tok))) +(defun (inside-brackets toks) + (match toks + ((Cons (TOpen o) (Cons s rest)) #:when (and (space? s) (not (starts-comment? rest))) + (inside-brackets (Cons (TOpen o) rest))) + ((Cons s (Cons (TClose c) rest)) #:when (space? s) + (inside-brackets (Cons (TClose c) rest))) + ((Cons t rest) (Cons t (inside-brackets rest))) + (Nil Nil))) + +(: starts-comment? (-> (List Tok) bool)) +(defun (starts-comment? toks) + (match toks + ((Cons t _) (comment? t)) + (Nil #f))) + +(: blank? (-> Line bool)) +(defun (blank? l) (empty? (record-ref l toks))) + +(: drop-closers (-> (List Tok) (List Tok))) +(defun (drop-closers toks) + (match toks + ((Cons t rest) #:when (close? t) (drop-closers rest)) + (_ toks))) + +;; Closing brackets alone on a line, perhaps with a comment after them. +(: closers-only? (-> Line bool)) +(defun (closers-only? l) + (def toks (record-ref l toks)) + (and (match toks + ((Cons t _) (close? t)) + (Nil #f)) + (match (drop-closers toks) + (Nil #t) + ((Cons t Nil) (comment? t)) + ((Cons s (Cons t Nil)) (and (space? s) (comment? t))) + (_ #f)))) + +(: ends-in-comment? (-> Line bool)) +(defun (ends-in-comment? l) + (match (list-reverse (record-ref l toks)) + ((Cons t _) (comment? t)) + (Nil #f))) + +(: drop-blank-lines (-> (List Line) (List Line))) +(defun (drop-blank-lines lines) + (match lines + ((Cons l rest) #:when (blank? l) (drop-blank-lines rest)) + (_ lines))) + +;; Lines of closing brackets go up to the end of the code before them, the +;; blank lines between going with them, unless that code ends in a comment. +(: join-closers (-> (List Line) (List Line))) +(defun (join-closers lines) + ;; `done` is the lines so far, the last first. + (let go ((lines lines) (done Nil)) + (match lines + (Nil (list-reverse done)) + ((Cons l rest) + (match (drop-blank-lines done) + ((Cons before earlier) #:when (and (closers-only? l) (not (ends-in-comment? before))) + (def joined (list-append (record-ref before toks) (record-ref l toks))) + (go rest (Cons (record-set before (toks joined)) earlier))) + (_ (go rest (Cons l done)))))))) + +;; No blank lines at the ends, and at most two in a row anywhere else. +(: squeeze-blank-lines (-> (List Line) (List Line))) +(defun (squeeze-blank-lines lines) + (def trimmed (list-reverse (drop-blank-lines (list-reverse (drop-blank-lines lines))))) + (let go ((lines trimmed) (run 0) (out Nil)) + (match lines + (Nil (list-reverse out)) + ((Cons l rest) #:when (blank? l) + (if (>= run 2) (go rest run out) (go rest (+ run 1) (Cons l out)))) + ((Cons l rest) (go rest 0 (Cons l out)))))) + +;; --------------------------------------------------------------------------- +;; Indentation + +;; A form inside a list, as the indentation reads it: where it starts, a prefix +;; such as ' or #' included; the physical line it starts on; the column of the +;; first form that starts on that line; its text if it is an atom, or its +;; opening bracket; and for a list the column of that bracket. +(type (: Elem (Record (: col int) + (: line int) + (: line-first int) + (: raw string) + (: is-list bool) + (: bracket int)))) + +;; An open list: its bracket, the bracket's column and line, the forms in it so +;; far, the last first, and the first two of them. +(type (: Frame (Record (: bracket string) + (: col int) + (: line int) + (: elems (List Elem)) + (: count int) + (: first (Option Elem)) + (: second (Option Elem))))) + +;; A column, and whether `lisp-indent-region` keeps it for the following lines +;; at the same depth, which it does for a plain number and not for the +;; (COLUMN START) a rule answers when the next line may differ. +(type (: Indent (Record (: col int) (: keep bool)))) + +(: indent (-> int bool Indent)) +(defun (indent col keep) (Indent (col col) (keep keep))) + +(: with-elem (-> Frame Elem Frame)) +(defun (with-elem f e) + (def n (record-ref f count)) + (Frame (bracket (record-ref f bracket)) + (col (record-ref f col)) + (line (record-ref f line)) + (elems (Cons e (record-ref f elems))) + (count (+ n 1)) + (first (if (= n 0) (Some e) (record-ref f first))) + (second (if (= n 1) (Some e) (record-ref f second))))) + +(: raw-of (-> (Option Elem) string)) +(defun (raw-of e) + (match e + ((Some x) (record-ref x raw)) + (None ""))) + +(: last-of (-> Frame Elem Elem)) +(defun (last-of f head) + (match (record-ref f elems) + ((Cons e _) e) + (Nil head))) + +;; How a form with a body indents, from `bjo-indent-forms` in bjomode.el: how +;; many forms come before the body, or a rule of its own. +(type (: Rule (Union (: Body int) DefForm NamedLet DefStar))) + +(: rule-of (-> string (Option Rule))) +(defun (rule-of head) + (match head + ((or "seq" "letrec" "export" "re-export" "import" "import/extern" "import/class" + "type" "type-rec" "bjo" "begin" "spawn" "spawn/daemon" "spawn/detached" + "with-shield" "cond" "time-it" "try") + (Some (Body 0))) + ;; A definition: everything after its head is the body, whatever else is on + ;; the head's line. + ((or "defun" "defbjo" "defbjouble" "impl" "impl/extern" ":doc") (Some DefForm)) + ((or "def/mutable" "def/trait" "def/macro" "def/pattern" "type/derive" "when" + "unless" "match" "case" "with-open" "parameterize" "parameterize*" "fun" "bjoroutine" + "let*" "syntax-match" "when-let" "if-let" "with-cancel" "with-deadline" + "with-response" "with-run" "with-return") + (Some (Body 1))) + ("let/mono" (Some (Body 2))) + ("if" (Some (Body 3))) + ("def" (Some DefForm)) + ("let" (Some NamedLet)) + ("def*" (Some DefStar)) + (_ None))) + +;; The characters Emacs reads as a prefix of the form after them. +(: prefix-char? (-> char bool)) +(defun (prefix-char? c) + (or (= c #\') (= c #\#) (= c #\,) (= c #\@) (= c #\`))) + +(: list-drop-while (-> (-> %a bool) (List %a) (List %a))) +(defun (list-drop-while drop? xs) + (match xs + ((Cons x rest) #:when (drop? x) (list-drop-while drop? rest)) + (_ xs))) + +(: without-prefix (-> string string)) +(defun (without-prefix s) + (list->string (list-drop-while prefix-char? (string->list s)))) + +(: prefix-length (-> string int)) +(defun (prefix-length s) + (- (string-length s) (string-length (without-prefix s)))) + +(: prefix-only? (-> string bool)) +(defun (prefix-only? s) + (and (not (string-empty? s)) (string-empty? (without-prefix s)))) + +;; Whether a form written `raw` begins with a word or a symbol character once +;; its prefix is past: a symbol or a number, not a list, a string or a +;; character literal. +(: symbolic? (-> string bool)) +(defun (symbolic? raw) + (match (string->list (without-prefix raw)) + ((Cons c _) (not (or (opening? c) (= c #\") (= c #\\) (= c #\;)))) + (Nil #f))) + +;; `calculate-lisp-indent`'s own column, before any rule: under the first form +;; when it is a list, under the second when the forms so far are all on the +;; first one's line, and otherwise under the first form on the line of the +;; last of them. +(: normal-indent (-> Frame Elem int)) +(defun (normal-indent f head) + (def last (last-of f head)) + (cond + ((record-ref head is-list) (record-ref head bracket)) + ((= (record-ref last line) (record-ref head line)) + (match (record-ref f second) + ((Some s) (record-ref s col)) + (None (record-ref head col)))) + (else (record-ref last line-first)))) + +;; `lisp-indent-specform`: the first two of a form's `n` distinguished forms +;; four in, a third under them, and the body two in. +(: specform (-> int Frame int Indent)) +(defun (specform n f normal) + (def args (- (record-ref f count) 1)) + (def left (- n args)) + (def body (+ (record-ref f col) 2)) + (cond + ((> left 0) (indent (if (<= args 1) (+ (record-ref f col) 4) normal) #f)) + ((or (and (= n 0) (= args 0)) (and (= left 0) (<= body normal))) (indent body #t)) + (else (indent normal #t)))) + +;; `lisp-indent-defform`: a body two in, when the form before it is on the +;; line of the bracket. +(: defform (-> Frame Elem (Option Indent))) +(defun (defform f head) + (if (= (record-ref (last-of f head) line) (record-ref f line)) + (Some (indent (+ (record-ref f col) 2) #t)) + None)) + +;; Whether a `let` is a named one: a name after `let` on its line. +(: named-let? (-> Frame Elem bool)) +(defun (named-let? f head) + (match (record-ref f second) + ((Some s) #:when (= (record-ref s line) (record-ref head line)) + (match (string->list (record-ref s raw)) + ((Cons c _) (char-alphabetic? c)) + (Nil #f))) + (_ #f))) + +;; `bjo-indent-function`: what the form's head says, if it says anything. +(: by-head (-> Frame Elem int (Option Indent))) +(defun (by-head f head normal) + (def raw (record-ref head raw)) + (def name (without-prefix raw)) + (def head-col (+ (record-ref head col) (prefix-length raw))) + (cond + ((string-starts-with? raw "#:") (Some (specform 0 f normal))) + ;; A list or a string as the head: the rest under the form after it on its + ;; line, or under the head when nothing follows it there. A guard is part + ;; of the pattern, so after `#:when` and its guard the form is the one + ;; after those. + ((not (symbolic? raw)) + (def forms (list-reverse (record-ref f elems))) + (def after + (match forms + ((Cons _ (Cons w (Cons _ (Cons x _)))) #:when (= (record-ref w raw) "#:when") (Some x)) + ((Cons _ (Cons w _)) #:when (= (record-ref w raw) "#:when") None) + ((Cons _ (Cons x _)) (Some x)) + (_ None))) + (Some (match after + ((Some s) #:when (= (record-ref s line) (record-ref head line)) (indent (record-ref s col) #t)) + (_ (indent (record-ref head col) #t))))) + ((= name ":match") (Some (indent head-col #t))) + (else + (match (rule-of name) + ((Some DefForm) (defform f head)) + ((Some (Body n)) (Some (specform n f normal))) + ((Some NamedLet) (Some (specform (if (named-let? f head) 2 1) f normal))) + ((Some DefStar) None) + ;; A head that begins with `def` defines something and one that begins + ;; with `with-` runs a body in something; both have a body. + (None + (if (and (> (string-length name) 3) + (or (string-starts-with? name "def") (string-starts-with? name "with-"))) + (defform f head) + None)))))) + +;; A line that starts with a keyword goes under a keyword that starts the line +;; of the form before it, when that is not the list's first form. +(: keyword-aligned (-> Frame Elem int string int)) +(defun (keyword-aligned f head normal here) + (def last (last-of f head)) + (def on-its-line + (list-filter #(= (record-ref & line) (record-ref last line)) (record-ref f elems))) + (match (list-reverse on-its-line) + ((Cons s _) #:when (and (string-starts-with? here ":") + (not (and (= (record-ref s col) (record-ref head col)) + (= (record-ref s line) (record-ref head line)))) + (string-starts-with? (record-ref s raw) ":")) + (record-ref s col)) + (_ normal))) + +;; Where `def`'s failure part may be: the forms after a `def`'s head, or all +;; of a parenthesised clause of a `def*`. `None` for any other list. +(: clause-forms (-> Frame (Option Frame) (Option (List Elem)))) +(defun (clause-forms f outer) + (def forms (list-reverse (record-ref f elems))) + (def head (raw-of (record-ref f first))) + (def in-def-star? + (match outer + ((Some o) (= (raw-of (record-ref o first)) "def*")) + (None #f))) + (cond + ((or (= head "def") (= head ":def")) + (Some (match forms + ((Cons _ rest) rest) + (Nil Nil)))) + ((and in-def-star? (= (record-ref f bracket) "(") (not (= head "def*"))) (Some forms)) + (else None))) + +(: failure-keyword? (-> string bool)) +(defun (failure-keyword? s) + (match (without-prefix s) + ((or ":leave-with" ":leave" ":propagate" ":default") #t) + (_ #f))) + +;; `bjo--failure-indent`: a failure keyword two in from the clause's bracket, +;; and the arms after `:leave` under the first of them. +(: failure-indent (-> Frame (Option Frame) string bool (Option Indent))) +(defun (failure-indent f outer here here-form?) + (match (clause-forms f outer) + ((Some forms) #:when (>= (+ (list-length forms) (if here-form? 1 0)) 2) + (cond + ((and here-form? (failure-keyword? here)) (Some (indent (+ (record-ref f col) 2) #f))) + (else + (match (list-drop-while #(not (= (without-prefix (record-ref & raw)) ":leave")) forms) + ((Cons _ (Cons arm _)) (Some (indent (record-ref arm col) #f))) + ((Cons leave Nil) (Some (indent (record-ref leave col) #f))) + (Nil None))))) + (_ None))) + +;; A form whose result a `=>` introduces, which goes two in, as a body does. +(: loop-form? (-> Frame bool)) +(defun (loop-form? f) + (match (raw-of (record-ref f first)) + ((or "loop" ":loop" "subloop" ":subloop" "seql") #t) + (_ #f))) + +(: binding-form? (-> string bool)) +(defun (binding-form? head) + (match head + ((or "let" "let*" "letrec" "with-open" "parameterize" "parameterize*") #t) + (_ #f))) + +;; `bjo--binding-list-p`: whether `f` is the bindings of a `let` or a form like +;; it, `outer`: the list after the head, or after the name of a named `let`. +(: binding-list? (-> Frame (Option Frame) bool)) +(defun (binding-list? f outer) + (match outer + ((Some o) + (def head (raw-of (record-ref o first))) + (def named? + (and (= head "let") + (match (string->list (raw-of (record-ref o second))) + ((Cons c _) (char-alphabetic? c)) + (Nil #f)))) + ;; `f` is open inside `o`, so it is the last of `o`'s forms so far. + (and (= (record-ref f bracket) "(") + (binding-form? head) + (= (record-ref o count) (if named? 3 2)))) + (None #f))) + +;; Whether `bjo--calculate-indent` stops the column being kept: in a `def` +;; clause and in a `loop` or a `seql`, where the next line may indent +;; differently. +(: no-reuse? (-> Frame (Option Frame) bool)) +(defun (no-reuse? f outer) + (or (loop-form? f) + (match (clause-forms f outer) + ((Some forms) (not (empty? forms))) + (None #f)))) + +;; The column for a line inside `f`, whose first token is `here` and starts a +;; form if `here-form?`. +(: indent-in (-> Frame (Option Frame) string bool Indent)) +(defun (indent-in f outer here here-form?) + (def calculated + (match (record-ref f first) + (None (indent (+ (record-ref f col) 1) #t)) + ;; A vec or an array is data, and a `let`'s bindings are a list of them: + ;; every element goes under the first, whatever it is. The rules below + ;; are for code. + ((Some head) #:when (or (= (record-ref f bracket) "[") (binding-list? f outer)) + (indent (record-ref head col) #t)) + ((Some head) + (def normal (normal-indent f head)) + (match (by-head f head normal) + ((Some i) i) + (None (indent (keyword-aligned f head normal here) #t)))))) + (match (failure-indent f outer here here-form?) + ((Some i) i) + (None + (cond + ((and (loop-form? f) (= here "=>")) (indent (+ (record-ref f col) 2) #f)) + ((and (record-ref calculated keep) (no-reuse? f outer)) (indent (record-ref calculated col) #f)) + (else calculated))))) + +;; --------------------------------------------------------------------------- +;; Writing it out + +;; What is known while the lines are written: the lists open, the innermost +;; first; the column kept at each depth, the innermost first, one more than +;; there are lists open, and how many lists were open when they were kept; +;; the physical line being written; and what is to be said about the lines so +;; far, each with its line, the last first. +(type (: State (Record (: frames (List Frame)) + (: kept (List (Option int))) + (: kept-depth int) + (: line int) + (: notes (List (Tuple int string)))))) + +;; The width past which a line the layout made wider is pointed out: an +;; indenter only moves text right, and a line it moves too far wants breaking +;; by hand. Racket's style has the same limit. +(: wide int) +(def wide 102) + +;; What is to be said about a line whose tokens are `toks` as it is written: +;; that it has a tab in a raw string, which no escape can write. How wide it +;; is is said once it is broken; see `width-notes`. +(: notes-on (-> (List Tok) (List string))) +(defun (notes-on toks) + (if (any? (fun (t) + (match t + ((TStr s) (and (string-starts-with? s "\"\"\"") (string-contains? s "\t"))) + (_ #f))) + toks) + (list "a tab in a raw string, which has no escape for it; write the text another way") + Nil)) + +;; The lines of `result` wider than `wide`, each with its width as a note, but +;; one that is part of a line of `source` as wide already: a string, a comment +;; or a table nothing here breaks, which a person has to, and has been told of. +(: width-notes (-> string string (List (Tuple int string)))) +(defun (width-notes source result) + (def wide-before + (loop (:for l (string-split source "\n")) + (:acc out (listing (string-trim l)) #:when (> (columns l) wide)) + => out)) + (def lines (string-split result "\n")) + (loop (:for i (range 0 (vec-length lines))) + (:acc out (listing (Tuple (+ i 1) + #"${(int->string (columns (vec-ref lines i)))} columns, more than ${(int->string wide)}; break the line")) + #:when (let ((l (vec-ref lines i))) + (and (> (columns l) wide) + (not (any? #(string-contains? & (string-trim l)) wide-before))))) + => out)) + +(: spaces (-> int string)) +(defun (spaces n) + (def sb (stringbuilder-empty)) + (let go ((i 0)) + (when (< i n) + (stringbuilder-add! sb #\space) + (go (+ i 1)))) + (stringbuilder->string sb)) + +;; The column after `s` written at `col`, and how many line breaks it has. +(: advance (-> string int (Tuple int int))) +(defun (advance s col) + (string-fold (fun (c at) + (match at + ((Tuple column breaks) + (if (= c #\newline) (Tuple 0 (+ breaks 1)) (Tuple (+ column 1) breaks))))) + (Tuple col 0) + s)) + +(: expand-tabs (-> string int string)) +(defun (expand-tabs s col) + (def sb (stringbuilder-empty)) + (ignore + (string-fold (fun (c at) + (cond + ((= c #\tab) + (def to (* 8 (+ 1 (/ at 8)))) + (stringbuilder-add-string! sb (spaces (- to at))) + to) + (else + (stringbuilder-add! sb c) + (+ at 1)))) + col + s)) + (stringbuilder->string sb)) + +;; The kept columns adjusted from `from` lists open to `depth`, as +;; `lisp-indent-calc-next` adjusts its stack: one dropped per list closed +;; since, a fresh one per list opened. +(: kept-at (-> (List (Option int)) int int (List (Option int)))) +(defun (kept-at kept from depth) + (cond + ((> from depth) + (kept-at (match kept + ((Cons _ rest) rest) + (Nil Nil)) + (- from 1) + depth)) + ((< from depth) (kept-at (Cons None kept) (+ from 1) depth)) + (else kept))) + +;; The text of a line's first token, and whether it starts a form. +(: first-of-line (-> (List Tok) (Tuple string bool))) +(defun (first-of-line toks) + (match toks + ((Cons (TAtom s) _) (Tuple s #t)) + ((Cons (TOpen s) _) (Tuple s #t)) + ((Cons (TStr s) _) (Tuple s #t)) + ((Cons (TComment s) _) (Tuple s #f)) + ((Cons (TClose s) _) (Tuple s #f)) + (_ (Tuple "" #f)))) + +;; One line, indented, and the state after it. +(: write-line (-> StringBuilder State Line State)) +(defun (write-line out st l) + (def toks (record-ref l toks)) + (def frames (record-ref st frames)) + (def depth (list-length frames)) + (def kept (kept-at (record-ref st kept) (record-ref st kept-depth) depth)) + (def (Tuple here here-form?) (first-of-line toks)) + (def (Tuple col keep) + (match kept + ((Cons (Some c) _) (Tuple c #f)) + (_ + (match frames + (Nil (Tuple 0 #t)) + ((Cons f rest) + (def outer + (match rest + ((Cons o _) (Some o)) + (Nil None))) + (def i (indent-in f outer here here-form?)) + (Tuple (record-ref i col) (record-ref i keep))))))) + (def kept-now + (match kept + ((Cons None rest) #:when keep (Cons (Some col) rest)) + (_ kept))) + ;; A `;;;` comment stays where it was, as `lisp-indent-line` leaves it. + (def lead (if (string-starts-with? here ";;;") (record-ref l indent) (spaces col))) + (def buffer (stringbuilder-empty)) + (unless (empty? toks) (stringbuilder-add-string! buffer lead)) + (def (Tuple frames-after line-after) + (write-toks buffer toks frames (string-length lead) (record-ref st line))) + (stringbuilder-add-string! out (stringbuilder->string buffer)) + (stringbuilder-add! out #\newline) + (State (frames frames-after) + (kept kept-now) + (kept-depth depth) + (line (+ line-after 1)) + (notes (list-foldl (fun (note acc) (Cons (Tuple (record-ref st line) note) acc)) + (record-ref st notes) + (notes-on toks))))) + +;; Whether the token is a prefix, ' or #' or #, written right before the form +;; it belongs to. +(: prefix-of-next? (-> Tok (List Tok) bool)) +(defun (prefix-of-next? t rest) + (match t + ((TAtom s) #:when (prefix-only? s) + (match rest + ((Cons (TOpen _) _) #t) + ((Cons (TStr _) _) #t) + ((Cons (TAtom _) _) #t) + (_ #f))) + (_ #f))) + +(: starts-form? (-> Tok bool)) +(defun (starts-form? t) + (match t + ((TAtom _) #t) + ((TOpen _) #t) + ((TStr _) #t) + (_ #f))) + +;; The forms the token adds to the lists open, and the list it opens or closes. +(: after-tok (-> Tok (List Frame) Elem (List Frame))) +(defun (after-tok t frames e) + (def added + (match frames + ((Cons f outer) (if (starts-form? t) (Cons (with-elem f e) outer) frames)) + (Nil Nil))) + (match t + ((TOpen s) + (Cons (Frame (bracket s) (col (record-ref e bracket)) (line (record-ref e line)) + (elems Nil) (count 0) (first None) (second None)) + added)) + ((TClose _) + (match frames + ((Cons _ outer) outer) + (Nil Nil))) + (_ added))) + +;; The tokens of one line, written from `col` on physical line `line`, and the +;; lists open after them and the physical line they end on. +(: write-toks (-> StringBuilder (List Tok) (List Frame) int int (Tuple (List Frame) int))) +(defun (write-toks out toks frames col line) + ;; `first` is the column of the first form on the physical line, once there + ;; is one; `prefix` that of a prefix waiting for the form it belongs to. + (let go ((toks toks) (frames frames) (col col) (line line) (first None) (prefix None)) + (match toks + (Nil (Tuple frames line)) + ((Cons t rest) + (def written + (match t + ((TSpace s) (expand-tabs s col)) + ((TComment s) (expand-tabs s col)) + (_ (tok-written t col)))) + (stringbuilder-add-string! out written) + (def (Tuple next-col breaks) (advance written col)) + (def waits? (prefix-of-next? t rest)) + (def form-col + (match prefix + ((Some p) p) + (None col))) + (def line-first + (match first + ((Some c) c) + (None form-col))) + (def e (Elem (col form-col) + (line line) + (line-first line-first) + (raw (match t + ((TStr _) "\"") + (_ (tok-text t)))) + (is-list (match t + ((TOpen _) #t) + (_ #f))) + (bracket col))) + (def frames-after (if waits? frames (after-tok t frames e))) + (def first-after + (cond + ((> breaks 0) None) + ((and (or waits? (starts-form? t)) (none? first)) (Some form-col)) + (else first))) + (def prefix-after + (cond + (waits? (Some form-col)) + (else None))) + (go rest frames-after next-col (+ line breaks) first-after prefix-after))))) + +;; --------------------------------------------------------------------------- +;; Brackets + +(: partner (-> string string)) +(defun (partner open) + (match open + ("(" ")") + ("[" "]") + (_ "}"))) + +;; Whether every bracket closes the one opened last: indentation means nothing +;; where they do not, and Emacs, asked to indent such a file, never finishes. +(: balanced (-> (List Tok) (Result string Unit))) +(defun (balanced toks) + ;; `open` holds each bracket still open and its line, the innermost first. + (let go ((toks toks) (open Nil) (line 1)) + (match toks + (Nil + (match open + (Nil (Ok unit)) + ((Cons (Tuple b at) _) + (Err #"line ${(int->string at)}: a ${b} that is never closed")))) + ((Cons t rest) + (match t + (TNewline (go rest open (+ line 1))) + ((TStr s) (go rest open (+ line (snd (advance s 0))))) + ((TOpen b) (go rest (Cons (Tuple b line) open) line)) + ((TClose c) + (match open + (Nil (Err #"line ${(int->string line)}: a ${c} that closes nothing")) + ((Cons (Tuple b at) outer) + (if (= (partner b) c) + (go rest outer line) + (Err #"line ${(int->string line)}: a ${c} where the ${b} of line ${(int->string at)} wants a ${(partner b)}"))))) + (_ (go rest open line))))))) + +;; --------------------------------------------------------------------------- +;; Breaking long lines +;; +;; A line wider than `wide` is broken in a list on it that runs past the limit, +;; the outermost that has somewhere to break, by putting its forms on lines of +;; their own the way the indentation then lays them out: a call's arguments +;; after the first under it, a body's forms two in, an `if`'s arms four in, a +;; clause's forms after its pattern. A call whose arguments are all small, and +;; a vec, are filled instead: broken only before the form that runs past the +;; limit. A list runs past the limit when it does with the closing brackets +;; right after it, so a line too wide only for the brackets that end it is +;; broken in its last list. A line that continues a list opened on a line +;; before is filled in that list. What none of that can take, a call's first +;; argument moving down gives a line more, in the call where that moves the +;; most left. A line none of that can break, a string that runs past the limit +;; is split on, with a line continuation after the last space that fits. Then +;; the text is laid out again and looked at again, until nothing more can be +;; broken. +;; +;; A keyword argument's value stays on the keyword's line, and a clause's +;; `#:when` and guard on the line of its pattern. +;; +;; A macro is left as it is: what its forms are is its own business. Those of +;; the prelude are named here; a head that begins with `with-` is one too. So +;; is everything in an `import/extern` or an `import/class`, which are tables +;; written to be read in columns. + +(: macro-head? (-> string bool)) +(defun (macro-head? head) + (or (string-starts-with? head "with-") + (match head + ((or "cond" "get" "if-let" "in-scope" "let*" "own-disposable" "parameterize*" "scope-form" + "set" "some->" "try->" "try-get" "type/derive" "update" "when-let") + #t) + (_ #f)))) + +(: table-head? (-> string bool)) +(defun (table-head? head) + (match head + ((or "import/extern" "import/class") #t) + (_ #f))) + +;; How the forms of a list may start lines of their own. +(type (: Breaking (Union + ;; Each from this position on. + (: OneALine int) + ;; Only the one that runs past the limit, from this position on. + (: Filled int) + ;; A call: filled if its arguments are all small, else one a line. + (: Call int)))) + +;; How the forms of a list headed `head`, written `bracket`, may start lines of +;; their own, or `None` for a list to leave as it is. +(: breaking-of (-> string string (Option Breaking))) +(defun (breaking-of bracket head) + (def name (without-prefix head)) + (cond + ;; A vec and an export list are data, filled. + ((= bracket "[") (Some (Filled 1))) + ((or (= name "export") (= name "re-export")) (Some (Filled 1))) + ((= bracket "{") None) + ((or (macro-head? name) (table-head? name)) None) + ((not (symbolic? head)) (Some (OneALine 1))) + ((= name "if") (Some (OneALine 2))) + (else + (match (rule-of name) + ((Some (Body n)) (Some (OneALine (+ n 1)))) + ((Some DefStar) (Some (OneALine 1))) + ((Some _) (Some (OneALine 2))) + (None + (if (and (> (string-length name) 3) (string-starts-with? name "def")) + (Some (OneALine 2)) + (Some (Call 2)))))))) + +;; A form of a list on the line looked at: its token, its position in the +;; list, the columns it starts and ends at, the closing brackets right after +;; it included; whether it is small: an atom, a string, or a list on one line +;; with no list in it; and whether it is glued to the form before it, which a +;; line break never comes between: the value of a keyword argument, and a +;; clause's `#:when`, which is part of its pattern. +(type (: Form (Record (: tok int) (: at int) (: from int) (: to int) (: small bool) (: glued bool)))) + +;; What the breaking reads of the laid-out tokens: each list's closing token, +;; each token's line and column, whether it is inside a table, and whether it +;; is inside a doc, whose strings may not continue lines: a published doc is +;; bjodat, which has no columns to go by. +(type (: Placed (Record (: toks (Vec Tok)) + (: closes (Vec int)) + (: lines (Vec int)) + (: cols (Vec int)) + (: tabled (Vec bool)) + (: docked (Vec bool))))) + +(: placed (-> (List Tok) Placed)) +(defun (placed toks) + (def v (list->vec toks)) + (def (Tuple closes _) (closers-and-lines v)) + (def lines (vecbuilder-empty)) + (def cols (vecbuilder-empty)) + (def tabled (vecbuilder-empty)) + (def docked (vecbuilder-empty)) + ;; `open` holds for each list open whether it is, or is in, a table, and + ;; whether it is, or is in, a doc. + (let go ((k 0) (line 0) (col 0) (open Nil)) + (when (< k (vec-length v)) + (def t (vec-ref v k)) + (def (Tuple in-table in-doc) (match open ((Cons b _) b) (Nil (Tuple #f #f)))) + (vecbuilder-add! lines line) + (vecbuilder-add! cols col) + (vecbuilder-add! tabled in-table) + (vecbuilder-add! docked in-doc) + (def (Tuple after breaks) (advance (tok-written t col) col)) + (def open-after + (match t + ((TOpen _) + (def head (match (vec-ref v (+ k 1)) ((TAtom s) s) (_ ""))) + (Cons (Tuple (or in-table (table-head? head)) (or in-doc (= head ":doc"))) open)) + ((TClose _) (match open ((Cons _ outer) outer) (Nil Nil))) + (_ open))) + (go (+ k 1) (+ line breaks) after open-after))) + (Placed (toks v) (closes closes) (lines (vecbuilder->vec lines)) + (cols (vecbuilder->vec cols)) (tabled (vecbuilder->vec tabled)) + (docked (vecbuilder->vec docked)))) + +(: tok-at (-> Placed int Tok)) +(defun (tok-at p k) (vec-ref (record-ref p toks) k)) + +(: opens? (-> Tok bool)) +(defun (opens? t) + (match t + ((TOpen _) #t) + (_ #f))) + +;; The column after the form starting at `j`, and after the closing brackets +;; right after it, which go where it goes. +(: form-end (-> Placed int int)) +(defun (form-end p j) + (closers-end p (if (opens? (tok-at p j)) (vec-ref (record-ref p closes) j) j))) + +;; The column after token `k` and the closing brackets right after it. +(: closers-end (-> Placed int int)) +(defun (closers-end p k) + (def n (vec-length (record-ref p toks))) + (let go ((k k)) + (if (and (< (+ k 1) n) (close? (tok-at p (+ k 1)))) + (go (+ k 1)) + (fst (advance (tok-written (tok-at p k) (vec-ref (record-ref p cols) k)) (vec-ref (record-ref p cols) k)))))) + +(: small-form? (-> Placed int bool)) +(defun (small-form? p j) + (def close (vec-ref (record-ref p closes) j)) + (cond + ((not (opens? (tok-at p j))) #t) + ((not (= (vec-ref (record-ref p lines) close) (vec-ref (record-ref p lines) j))) #f) + (else (not (any? #(opens? (tok-at p &)) (range (+ j 1) close)))))) + +;; The forms of the list opening at `k`, each with its position, from the +;; first; those that start on line `ln`, unless `ln` is -1. +(: forms-of (-> Placed int int (List Form))) +(defun (forms-of p k ln) + (def close (vec-ref (record-ref p closes) k)) + (def n (vec-length (record-ref p toks))) + (def head-list? (and (< (+ k 1) n) (opens? (tok-at p (+ k 1))))) + ;; `prev` is the text of the form before, if it is an atom. + (let go ((j (+ k 1)) (at 0) (prev "") (out Nil)) + (cond + ((or (< close 0) (>= j close)) (list-reverse out)) + ((not (starts-form? (tok-at p j))) (go (+ j 1) at prev out)) + (else + ;; A prefix is the start of the form after it. + (def body (if (and (< (+ j 1) n) (prefix-of-next? (tok-at p j) (list (tok-at p (+ j 1))))) (+ j 1) j)) + (def next (if (opens? (tok-at p body)) (+ (vec-ref (record-ref p closes) body) 1) (+ body 1))) + (def here? (or (= ln -1) (= (vec-ref (record-ref p lines) j) ln))) + (def text (if (opens? (tok-at p body)) "" (tok-text (tok-at p j)))) + (def form (Form (tok j) (at at) (from (vec-ref (record-ref p cols) j)) (to (form-end p body)) + (small (small-form? p body)) + (glued (or (keyword-arg? prev) (and (= at 1) head-list? (= text "#:when")))))) + (go next (+ at 1) text (if here? (Cons form out) out)))))) + +(: head-text (-> Placed (List Form) string)) +(defun (head-text p forms) + (match forms + ((Cons f _) #:when (= (record-ref f at) 0) (tok-text (tok-at p (record-ref f tok)))) + (_ ""))) + +;; The first of `forms` that runs past the limit, to break before, or, if it is +;; glued to the one before it, the first of them it is glued to. +(: first-past (-> (List Form) (List int))) +(defun (first-past forms) + (def v (list->vec forms)) + (def n (vec-length v)) + (defun (past? i) (> (record-ref (vec-ref v i) to) wide)) + (defun (glued? i) (record-ref (vec-ref v i) glued)) + (defun (back i) + (cond + ((not (glued? i)) (list (record-ref (vec-ref v i) tok))) + ((> i 0) (back (- i 1))) + (else Nil))) + (let find ((i 0)) + (cond + ((>= i n) Nil) + ((past? i) (back i)) + (else (find (+ i 1)))))) + +(: unglued (-> (List Form) (List int))) +(defun (unglued forms) + (list-map #(record-ref & tok) (list-filter #(not (record-ref & glued)) forms))) + +;; Where to break line `ln` in the list opening at `k` on it. +(: breaks-in-list (-> Placed int int (List int))) +(defun (breaks-in-list p k ln) + (def all (forms-of p k -1)) + (def here (forms-of p k ln)) + (def bracket (tok-text (tok-at p k))) + (defun (from-on n) + (list-filter #(and (>= (record-ref & at) n) (> (record-ref & from) (+ (vec-ref (record-ref p cols) k) 1))) + here)) + (match (breaking-of bracket (head-text p all)) + (None Nil) + ((Some (OneALine n)) (unglued (from-on n))) + ((Some (Filled n)) (first-past (from-on n))) + ((Some (Call n)) + (if (all? #(record-ref & small) (list-filter #(>= (record-ref & at) 1) all)) + (first-past (from-on n)) + (unglued (from-on n)))))) + +;; Whether the list opening at `k` runs past the limit on line `ln`, the +;; closing brackets right after it included: a line too wide only for the +;; brackets that end it is broken in the last list on it. +(: runs-past? (-> Placed int int bool)) +(defun (runs-past? p k ln) + (def c (vec-ref (record-ref p closes) k)) + (or (not (= (vec-ref (record-ref p lines) c) ln)) + (> (closers-end p c) wide))) + +;; The list a line starts inside of: the innermost open at its first token. +(: enclosing (-> Placed int (Option int))) +(defun (enclosing p first) + (let go ((k 0) (open Nil)) + (cond + ((>= k first) + (match open + ((Cons o _) (Some o)) + (Nil None))) + ((opens? (tok-at p k)) (go (+ k 1) (Cons k open))) + ((close? (tok-at p k)) + (go (+ k 1) + (match open + ((Cons _ outer) outer) + (Nil Nil)))) + (else (go (+ k 1) open))))) + +;; A line that continues a list opened before it is filled in that list. +(: breaks-continuing (-> Placed int int (List int))) +(defun (breaks-continuing p first ln) + (match (enclosing p first) + ((Some k) #:when (not (vec-ref (record-ref p tabled) k)) + (match (breaking-of (tok-text (tok-at p k)) (head-text p (forms-of p k -1))) + (None Nil) + ((Some _) + (first-past (list-filter #(> (record-ref & tok) first) (forms-of p k ln)))))) + (_ Nil))) + +;; The first argument of a call of `lists` that has it on the line of its head, +;; moved down, when nothing else can be broken: the list the line continues, +;; which moves the whole line left, or one on the line, which moves what runs +;; past the limit left. Either moves it left by as much as the argument is to +;; the right of the call's bracket, so the call where that is the most is +;; taken, the first of those that gain as much. +(: first-argument-down (-> Placed (List int) (List int))) +(defun (first-argument-down p lists) + (defun (call? k forms) + (match (breaking-of (tok-text (tok-at p k)) (head-text p forms)) + ((Some (Call _)) #t) + (_ #f))) + ;; The argument of the call opening at `k` to move down, and how far left + ;; that moves what comes after it. + (defun (candidate k) + (def forms (forms-of p k (vec-ref (record-ref p lines) k))) + (match forms + ((Cons h (Cons arg _)) #:when (and (= (record-ref h at) 0) + (= (record-ref arg at) 1) + (not (vec-ref (record-ref p tabled) k)) + (call? k forms)) + (Some (Tuple (record-ref arg tok) (- (record-ref arg from) (+ (vec-ref (record-ref p cols) k) 1))))) + (_ None))) + (def best + (list-foldl (fun (k acc) + (match (Tuple (candidate k) acc) + ((Tuple (Some (Tuple tok gain)) (Some (Tuple _ most))) #:when (> gain most) + (Some (Tuple tok gain))) + ((Tuple (Some c) None) (Some c)) + (_ acc))) + None + lists)) + (match best + ((Some (Tuple tok gain)) #:when (> gain 0) (list tok)) + (_ Nil))) + +;; A string that line `ln` has some of, split there with a line continuation, +;; as its token's index and new text; `None` when there is none to split. A raw +;; string has no continuations, a doc's strings may not have them, and a table +;; is left as it is. +(: split-for (-> Placed int (Option (Tuple int string)))) +(defun (split-for p ln) + (def n (vec-length (record-ref p toks))) + (let find ((k 0)) + (cond + ((or (>= k n) (> (vec-ref (record-ref p lines) k) ln)) None) + (else + (match (tok-at p k) + ((TStr s) #:when (not (or (vec-ref (record-ref p tabled) k) + (vec-ref (record-ref p docked) k) + (string-starts-with? s "\"\"\""))) + ;; The closing brackets right after it go where its end goes. + (def closers (- (closers-end p k) (closers-end-alone p k))) + (match (split-string s (vec-ref (record-ref p cols) k) (- ln (vec-ref (record-ref p lines) k)) closers) + ((Some text) (Some (Tuple k text))) + (None (find (+ k 1))))) + (_ (find (+ k 1)))))))) + +;; The string `s`, written at `col`, split on its physical line `nth` with a +;; line continuation: after the last space on it that leaves room for the +;; backslash within `wide`, with text before it on the line and a word after +;; it. Never in an escape or an interpolation's hole. What it stands for is +;; the same: the space stays before the backslash, and the continued line +;; starts where the text does. `None` when the line has no such space. +(: split-string (-> string int int int (Option string))) +(defun (split-string s col nth closers) + (def written (with-continued-indent s col)) + (def text-col (text-column s col)) + (def cs (string->vec written)) + (def n (vec-length cs)) + (def interpolated? (string-starts-with? s "#\"")) + (def opening (if interpolated? 2 1)) + ;; `seen` is whether the line has had text, the opening quote aside, and + ;; `reach` the column the string reaches on line `nth`: one that does not run + ;; past the limit there, with the `closers` columns of brackets after it if + ;; it ends there, is not what makes the line too wide, and is left whole. + (let go ((p opening) (line 0) (column (+ col opening)) (best None) (seen #f) (reach 0)) + (def c (char-at cs p)) + (def reach-now (if (= line nth) column reach)) + (cond + ((or (>= p n) (> line nth)) + (match best + ((Some at) #:when (> (if (and (>= p n) (= line nth)) (+ reach-now closers) reach-now) wide) + (Some (without-continued-indent + (str (text-between cs 0 (+ at 1)) "\\\n" (spaces text-col) (text-between cs (+ at 1) n)) + col))) + (_ None))) + ((= c #\newline) (go (+ p 1) (+ line 1) 0 best #f reach-now)) + ;; A continuation's line break is the next character's to deal with. + ((and (= c #\\) (= (char-at cs (+ p 1)) #\newline)) (go (+ p 1) line (+ column 1) best #t reach-now)) + ((= c #\\) (go (+ p 2) line (+ column 2) best #t reach-now)) + ((and interpolated? (= c #\$) (= (char-at cs (+ p 1)) #\{)) + (def past (hole-end cs (+ p 2))) + (def (Tuple after breaks) (advance (text-between cs p past) column)) + (go past (+ line breaks) after best #t reach-now)) + ((= c #\space) + (def next (char-at cs (+ p 1))) + (def fits? (<= (+ column 2) wide)) + (def word-after? (and (< (+ p 2) n) (not (or (= next #\space) (= next #\newline) (= next #\"))))) + (go (+ p 1) line (+ column 1) + (if (and (= line nth) seen fits? word-after?) (Some p) best) + seen + reach-now)) + (else (go (+ p 1) line (+ column 1) best #t reach-now))))) + +;; The column after token `k`, the closing brackets after it not counted. +(: closers-end-alone (-> Placed int int)) +(defun (closers-end-alone p k) + (def col (vec-ref (record-ref p cols) k)) + (fst (advance (tok-written (tok-at p k) col) col))) + +;; `toks` with the strings `splits` names, by index, given their new text. +(: with-splits (-> (List Tok) (List (Tuple int string)) (List Tok))) +(defun (with-splits toks splits) + (if (empty? splits) + toks + (let go ((toks toks) (k 0) (out Nil)) + (match toks + (Nil (list-reverse out)) + ((Cons t rest) + (def now + (match (list-filter #(= (fst &) k) splits) + ((Cons (Tuple _ text) _) (TStr text)) + (Nil t))) + (go rest (+ k 1) (Cons now out))))))) + +;; The tokens to put a line break before, to break line `ln`; and, when there +;; are none, those that move a first argument down, which is the last resort: +;; a string that runs past the limit is split before it is moved (see +;; `broken`). +(: breaks-for (-> Placed int (Tuple (List int) (List int)))) +(defun (breaks-for p ln) + (def n (vec-length (record-ref p toks))) + (def on-line + (loop (:for k (range 0 n)) + (:acc out (listing k) #:when (= (vec-ref (record-ref p lines) k) ln)) + => out)) + (def crossing + (list-filter #(and (opens? (tok-at p &)) (not (vec-ref (record-ref p tabled) &)) (runs-past? p & ln)) + on-line)) + (def first (match (list-filter #(starts-form? (tok-at p &)) on-line) ((Cons f _) f) (Nil -1))) + (def around + (match (if (< first 0) None (enclosing p first)) + ((Some k) (list k)) + (None Nil))) + ;; The least column a form on the line can be moved to: in the list the + ;; line continues, or past where the line starts. + (def least + (match around + ((Cons k _) + (def a (vec-ref (record-ref p cols) k)) + (def b (vec-ref (record-ref p cols) first)) + (+ (if (< a b) a b) 1)) + (Nil 1))) + ;; A string or comment too wide to fit even moved there keeps the line wide + ;; whatever is broken. One that can not be split leaves the line as it is. + ;; One that can is split once the line's lists are broken (see `split-for`), + ;; and moving it left, by filling or by moving a first argument down, would + ;; only split it later. + (def too-wide + (list-filter #(and (not (opens? (tok-at p &))) (> (+ least (columns (tok-text (tok-at p &)))) wide)) + on-line)) + (def splits-instead? (not (empty? too-wide))) + (cond + ((not (all? #(splittable? p &) too-wide)) (Tuple Nil Nil)) + (else + (match (list-filter #(not (empty? &)) (list-map #(breaks-in-list p & ln) crossing)) + ((Cons found _) (Tuple found Nil)) + (Nil #:when splits-instead? (Tuple Nil Nil)) + (Nil + (match (if (< first 0) Nil (breaks-continuing p first ln)) + (Nil (Tuple Nil (first-argument-down p (list-append around crossing)))) + (found (Tuple found Nil)))))))) + +;; Whether the token at `k` is a string `split-for` may split: not a raw one, +;; not one in a doc or a table, and with a space in it. +(: splittable? (-> Placed int bool)) +(defun (splittable? p k) + (match (tok-at p k) + ((TStr s) (not (or (string-starts-with? s "\"\"\"") + (vec-ref (record-ref p tabled) k) + (vec-ref (record-ref p docked) k) + (not (string-contains? s " "))))) + (_ #f))) + +;; The laid-out text with its long lines broken, as far as they can be. +(: broken (-> (Tuple string (List (Tuple int string))) (Tuple string (List (Tuple int string))))) +(defun (broken laid) + (let again ((laid laid) (rounds 0)) + (def (Tuple text _) laid) + (match (tokenize (string->vec text)) + ((Err _) laid) + ((Ok toks) + (def p (placed toks)) + (def widths (vec-map columns (string-split text "\n"))) + (def long + (loop (:for ln (range 0 (vec-length widths))) + (:acc out (listing ln) #:when (> (vec-ref widths ln) wide)) + => out)) + ;; For each long line the line breaks that break it; when there are + ;; none, a string that runs past the limit split with a line + ;; continuation; and when there is none of those, a first argument + ;; moved down. + (def (Tuple wanted splits) + (list-foldl (fun (ln acc) + (def (Tuple breaks split) acc) + (match (breaks-for p ln) + ((Tuple Nil last-resort) + (match (split-for p ln) + ((Some s) (Tuple breaks (Cons s split))) + (None (Tuple (list-append breaks last-resort) split)))) + ((Tuple found _) (Tuple (list-append breaks found) split)))) + (Tuple Nil Nil) + long)) + (if (or (and (empty? wanted) (empty? splits)) (>= rounds 40)) + laid + (again (layout (with-breaks-before (with-splits toks splits) wanted)) (+ rounds 1))))))) + +(: with-breaks-before (-> (List Tok) (List int) (List Tok))) +(defun (with-breaks-before toks at) + (let go ((toks toks) (k 0) (out Nil)) + (match toks + (Nil (list-reverse out)) + ((Cons t rest) + (go rest (+ k 1) (Cons t (if (any? #(= & k) at) (Cons TNewline out) out))))))) + +;; How many columns `s` takes, in characters: `string-length` counts the bytes +;; of its UTF-8. +(: columns (-> string int)) +(defun (columns s) (fst (advance s 0))) + +;; --------------------------------------------------------------------------- +;; The whole of it + +;; `source` laid out, with what is to be said about its lines, each with its +;; line number in what is laid out (see `notes-on`); or why it cannot be: it +;; does not read, or, which would be a bug here, laying it out would change +;; its tokens. +(: format-source (-> string (Result string (Tuple string (List (Tuple int string)))))) +(defun (format-source source) (format-with source #t)) + +;; `format-source`, its long lines broken if `break?`. +(: format-with (-> string bool (Result string (Tuple string (List (Tuple int string)))))) +(defun (format-with source break?) + (def cs (string->vec source)) + (def (Ok toks) (tokenize cs) + :leave ((Err why) (Err why))) + (def (Ok _) (balanced toks) + :leave ((Err why) (Err why))) + (def (Tuple result notes) (if break? (broken (layout toks)) (layout toks))) + (def (Ok again) (tokenize (string->vec result)) + :leave ((Err why) (Err #"formatting made it unreadable: ${why}"))) + (if (= (significant again) (significant toks)) + (Ok (Tuple result (list-sort (list-append notes (width-notes source result))))) + (Err "formatting would change its tokens, which is a bug in bjo fmt"))) + +;; The tokens laid out, and what is to be said about their lines. +(: layout (-> (List Tok) (Tuple string (List (Tuple int string))))) +(defun (layout toks) + (def lines (squeeze-blank-lines (join-closers (lines-of (reshaped toks))))) + (def out (stringbuilder-empty)) + (def done + (list-foldl (fun (l st) (write-line out st l)) + (State (frames Nil) (kept (list None)) (kept-depth 0) (line 1) (notes Nil)) + lines)) + (Tuple (stringbuilder->string out) (list-reverse (record-ref done notes)))) +;; bjo/git.bjo — the git a project is fetched with. +;; +;; One bare clone per repository URL, under `.bjo/git/`. Bare, because nothing +;; a resolve asks for needs a working tree: the tags, a manifest at a tag and a +;; tag's commit are all read out of the object database. A version is only ever +;; checked out when it has been chosen, and then into `.bjo/pkg/` rather than +;; into the clone. +;; +;; **The network is touched as little as possible.** A clone is made once and +;; fetched only when this run needs something it has not got: a lower bound +;; with no tag that satisfies it, or a commit the lock names and the clone does +;; not hold. A project whose dependencies are all here builds with the network +;; unplugged. +;; +;; Every command is run through `(std run)` rather than through `bjo`'s own +;; `exec`, because the output is read rather than shown. That redirects stdin, +;; which is why `bjo` sets `GIT_TERMINAL_PROMPT=0` before any of this runs: a +;; repository asking for a password would otherwise read from a pipe nobody +;; writes to and wait there for ever. + +(export + Tool tool-for git-on-path? tool-said + clone-directory ensure-clone fetch-tags + version-tags version->tag tag-commit commit-present? manifest-at + package-directory materialise) + +(import (std prelude)) +(import (std run)) +(import (text bjodat)) +(import "manifest.bjo") +(import "project.bjo") + +(import/extern + (error-message (: System.Exception.Message (-> Exception string) #:get)) + ;; `git archive --format=zip` and this are the whole of a checkout: git + ;; writes the tree, the runtime unpacks it. No third program — `tar` is not + ;; everywhere — and no working tree, index or HEAD added to the bare clone. + (zip-extract (: System.IO.Compression.ZipFile.ExtractToDirectory (-> string string void) + #:exceptions (System.IO.IOException + System.UnauthorizedAccessException + System.NotSupportedException))) + ;; The last step of a checkout, and the reason there is one: a rename is + ;; atomic enough that a directory which exists is a directory that is whole. + (directory-move (: System.IO.Directory.Move (-> string string void) + #:exceptions (System.IO.IOException + System.UnauthorizedAccessException)))) + +;; Where git's standard error goes. One scratch file, overwritten per command: +;; it is read only to quote git in a message about a command that failed. +(type (: Tool (Record (: errors string)))) + +(: tool-for #:sync (-> string Tool)) +(defun (tool-for tmp) + (directory-create tmp) + (Tool (errors (path-combine tmp "git-stderr.txt")))) + +(: (-> Tool string)) +(defun (tool-said g) + (if (file-exists? (record-ref g errors)) + (string-trim (file-read-text (record-ref g errors))) + "")) + +;; Is there a git to run at all? Asked before the first command, so that a +;; machine without one is told what is missing rather than what failed. +(: (-> bool)) +(defun (git-on-path?) (some? (which "git"))) + +;; Runs git, and answers its exit code and its output. A non-zero code is an +;; ordinary answer here — "this tag has no manifest" is one — so it is not an +;; `Err`. `Err` is git not starting at all. +(: git-run #:sync (-> Tool (Vec string) (Result string (Tuple int string)))) +(defun (git-run g arguments) + (result-map-err (fun (e) (str "bjo: could not run git: " (error-message e))) + (run/output '(errors-into-file ,(record-ref g errors) (git ,arguments))))) + +;; A command that has to work, with what git said when it did not. +(: git-must #:sync (-> Tool (Vec string) string (Result string string))) +(defun (git-must g arguments what) + (def (Ok (Tuple code out)) (git-run g arguments) :propagate) + (if (= code 0) + (Ok out) + (Err (str "bjo: " what " failed.\n " (tool-said g))))) + + +;; --------------------------------------------------------------------------- +;; The bare clone +;; --------------------------------------------------------------------------- + +;; One directory per URL, named after it. Every character that is not a letter +;; or a digit becomes `_`, so the name is a name on every filesystem and two +;; URLs that differ anywhere differ here. +(: (-> string string)) +(defun (url->directory-name url) + (loop (:for c url) + (:acc out (stringing (if (or (char-alphabetic? c) (char-numeric? c)) c #\_))) + => out)) + +(: (-> string string string)) +(defun (clone-directory dot-bjo url) + (path-combine dot-bjo "git" (url->directory-name url))) + +;; The clone for a URL, made the first time it is wanted. +;; +;; A fresh clone brings every tag with it, so nothing fetches after one. +(: ensure-clone #:sync (-> Tool string string (List Symbol) (Result string string))) +(defun (ensure-clone g dot-bjo url name) + (def dir (clone-directory dot-bjo url)) + (cond + ((directory-exists? dir) (Ok dir)) + ((not (git-on-path?)) + (Err (str "bjo: git is not on PATH; it is needed to fetch " + (name->string name) " from " url "."))) + (else + (directory-create (path-combine dot-bjo "git")) + (def (Ok cloned) (git-must g ["clone" "--bare" "--quiet" url dir] + (str "cloning " url)) + :propagate) + (Ok dir)))) + +;; New tags, for when this run needs something the clone has not got. +(: fetch-tags #:sync (-> Tool string (Result string Unit))) +(defun (fetch-tags g dir) + (def (Ok fetched) + (git-must g ["--git-dir" dir "fetch" "--tags" "--force" "--quiet"] + (str "fetching tags into " dir)) + :propagate) + (Ok unit)) + + +;; --------------------------------------------------------------------------- +;; Versions, as tags +;; --------------------------------------------------------------------------- + +(: (-> Version string)) +(defun (version->tag v) (str "v" (version->string v))) + +;; A tag as a version, if it is one. +;; +;; Exactly `vX.Y.Z`: the version is written back out and compared with what was +;; read, so `v1.2` and `v1.2.3-rc1` are not versions and are passed over. A tag +;; that is not a version is a name for something else — a branch point, a demo, +;; a talk — and a resolver that guessed at them would install one. +(: (-> string (Option Version))) +(defun (tag->version tag) + (if (not (string-starts-with? tag "v")) + None + (let ((text (substring tag + (string-cursor-next tag (string-cursor-start tag)) + (string-cursor-end tag)))) + (match (parse-version text) + ((Err why) None) + ((Ok v) (if (= (version->string v) text) (Some v) None)))))) + +;; Every version the clone holds, oldest first. +(: version-tags #:sync (-> Tool string (Result string (List Version)))) +(defun (version-tags g dir) + (def (Ok out) (git-must g ["--git-dir" dir "tag" "--list"] (str "reading the tags of " dir)) + :propagate) + (Ok (list-sort-versions + (loop (:for line (string-split out "\n")) + (:let v (tag->version (string-trim line))) + (:when-let (Some version) v) + (:acc found (listing version)) + => found)))) + +;; Insertion sort, oldest first. A repository has tens of tags, not thousands, +;; and `Version` is a record with no ordering of its own. +(: (-> (List Version) (List Version))) +(defun (list-sort-versions versions) + (match versions + (Nil Nil) + ((Cons v rest) (insert-version v (list-sort-versions rest))))) + +(: (-> Version (List Version) (List Version))) +(defun (insert-version v sorted) + (match sorted + (Nil (Cons v Nil)) + ((Cons head rest) + (if (<= (version-compare v head) 0) + (Cons v sorted) + (Cons head (insert-version v rest)))))) + +;; The commit a version's tag points at. `^{commit}` unwraps an annotated tag, +;; whose own object is not the commit. +(: tag-commit #:sync (-> Tool string Version (Result string string))) +(defun (tag-commit g dir v) + (def (Ok out) + (git-must g ["--git-dir" dir "rev-parse" (str (version->tag v) "^{commit}")] + (str "reading the commit of " (version->tag v))) + :propagate) + (Ok (string-trim out))) + +;; Is a commit in this clone? Asked about a commit the lock names, which a +;; clone made after it was written may not have. +(: commit-present? #:sync (-> Tool string string bool)) +(defun (commit-present? g dir commit) + (match (git-run g ["--git-dir" dir "cat-file" "-e" (str commit "^{commit}")]) + ((Err why) #f) + ((Ok (Tuple code out)) (= code 0)))) + +;; A version's manifest, read straight out of the object database. +;; +;; Nothing is checked out for this: a resolve reads one file per version, and +;; most of the versions it reads are not the ones it ends up choosing. +(: manifest-at #:sync (-> Tool string Version (Result string string))) +(defun (manifest-at g dir v) + (def (Ok (Tuple code out)) + (git-run g ["--git-dir" dir "show" (str (version->tag v) ":" manifest-name)]) + :propagate) + (if (= code 0) + (Ok out) + (Err (str "bjo: " (version->tag v) " in " dir " has no " manifest-name + ", so there is no package there.\n " (tool-said g))))) + + +;; --------------------------------------------------------------------------- +;; Putting a version on disk +;; --------------------------------------------------------------------------- + +;; `.bjo/pkg/bjorsec@1.2.3`, or `.bjo/pkg/my.app@1.2.3` for a name of several +;; segments. One directory per version, so two projects' dependencies never +;; share one and a version that is no longer wanted can be deleted by name. +(: (-> string (List Symbol) Version string)) +(defun (package-directory dot-bjo name v) + (path-combine dot-bjo "pkg" + (str (string-join (list->vec (list-map symbol->string name)) ".") + "@" (version->string v)))) + +;; What a materialised directory says it holds. +(: (-> string string)) +(defun (commit-marker dir) (path-combine dir ".bjo-commit")) + +(: (-> string string)) +(defun (marked-commit dir) + (if (file-exists? (commit-marker dir)) + (string-trim (file-read-text (commit-marker dir))) + "")) + +;; Puts a commit's tree at `target`. +;; +;; A directory already there whose `.bjo-commit` matches is used as it stands; +;; one that does not match is deleted and replaced, because a tag that moved +;; must not leave the old tree under the new version's name. The new tree is +;; unpacked beside it and renamed into place, so a build never reads half of a +;; checkout. +(: materialise #:sync (-> Tool string string string string (Result string Unit))) +(defun (materialise g clone commit tmp target) + (with-return ret + (when (and (directory-exists? target) (= (marked-commit target) commit)) + (ret (Ok unit))) + + (directory-create tmp) + (def archive (path-combine tmp (str commit ".zip"))) + (def unpacked (path-combine tmp (str "unpack-" commit))) + + (when (directory-exists? unpacked) (directory-delete-tree unpacked)) + (when (file-exists? archive) (file-delete archive)) + + (def (Ok archived) + (git-must g ["--git-dir" clone "archive" "--format=zip" "-o" archive commit] + (str "writing an archive of " commit)) + :propagate) + + (def (Ok unzipped) (zip-extract archive unpacked) + :leave ((Err e) (Err (str "bjo: could not unpack " archive ": " (error-message e))))) + + (file-write-text (commit-marker unpacked) commit) + (file-delete archive) + + (when (directory-exists? target) (directory-delete-tree target)) + (directory-create (path-combine target "..")) + + (def (Ok moved) (directory-move unpacked target) + :leave ((Err e) (Err (str "bjo: could not put " commit " at " target ": " + (error-message e))))) + (Ok unit))) +;; bjo/lock.bjo — `bjo.lock`: the commit each git dependency resolved to. +;; +;; (lock +;; (locked +;; (package (name (bjorsec)) (version "1.2.3") +;; (source (git (url "https://github.com/you/bjorsec"))) +;; (commit "a1b2c3d4e5f6...")))) +;; +;; One entry per git package, sorted by name, so that a diff shows what +;; changed and not what was reordered. Path dependencies are not here: there is +;; nothing to pin about a directory on your own disk. +;; +;; An entry is written `(package ...)` under one `locked` clause, the way a +;; manifest writes `(depends (package ...) (package ...))`: a clause holds its +;; values, and a clause that repeated with the fields of a record spread +;; through it is not something the reader can express. It is the same record +;; either way — named, versioned, sourced, and pinned to a commit. +;; +;; The commit is the whole of the pinning. A version is a tag, a tag is a name +;; somebody can move, and a commit is the content — so what this file protects +;; against is a `v1.2.3` that is not the `v1.2.3` the last build read. There is +;; no separate hash of the tree: the commit already is one. + +(export + Locked Lock + locked-name locked-version locked-source locked-commit + read-lock lock-text write-lock lock-entry-for lock-changes) + +(import (std prelude)) +(import (text bjodat)) +(import "manifest.bjo") +(import "project.bjo") + +(def/bjodat-type + (: Locked (Record #:tag package + (: name (List Symbol)) + (: version Version) + (: source Source) + (: commit string)))) + +(def/bjodat-type + (: Lock (Record #:tag lock + (: locked (List Locked) #:rest #:default Nil)))) + +(: (-> Locked (List Symbol))) +(defun (locked-name l) (record-ref l name)) + +(: (-> Locked Version)) +(defun (locked-version l) (record-ref l version)) + +(: (-> Locked Source)) +(defun (locked-source l) (record-ref l source)) + +(: (-> Locked string)) +(defun (locked-commit l) (record-ref l commit)) + +;; What the last resolve wrote, or nothing at all. +;; +;; A missing file is an empty lock rather than an error: the first build of a +;; checkout that has never been built has no lock, and writing one is what it +;; is about to do. +(: (-> string (Result string (List Locked)))) +(defun (read-lock file) + (if (not (file-exists? file)) + (Ok Nil) + (match (bjodat-parse-Lock (file-read-text file)) + ((Err why) + (Err (str "bjo: " file ": " why + "\n Delete it to resolve from the manifests again."))) + ((Ok lock) (Ok (record-ref lock locked)))))) + +;; A source as a manifest writes it. The writer is the one in `(text bjodat)`, +;; so what comes out here is what a manifest may say. +(: (-> Source string)) +(defun (source-text source) (bjodat->string (->bjodat source))) + +(: (-> Locked string)) +(defun (locked-text entry) + (str " (package (name " (name->string (locked-name entry)) ")" + " (version \"" (version->string (locked-version entry)) "\")\n" + " (source " (source-text (locked-source entry)) ")\n" + " (commit \"" (locked-commit entry) "\"))")) + +;; The whole file. Sorted by name: the entry begins with it, so sorting the +;; entries by their text sorts by name. +(: (-> (List Locked) string)) +(defun (lock-text entries) + (if (empty? entries) + "(lock)\n" + (str "(lock\n (locked\n" + (string-join (list->vec (list-sort (list-map locked-text entries))) "\n") + "))\n"))) + +(: write-lock #:sync (-> string (List Locked) void)) +(defun (write-lock file entries) + (file-write-text file (lock-text entries))) + +(: (-> (List Locked) (List Symbol) (Option Locked))) +(defun (lock-entry-for entries name) + (find #(= (name->string (locked-name &)) (name->string name)) entries)) + +;; What changed between two locks, one line each, sorted. +;; +;; With minimal version selection this is empty unless a manifest changed: the +;; same manifests choose the same versions however many times they are read. +(: (-> (List Locked) (List Locked) (List string))) +(defun (lock-changes before after) + (def added-or-changed + (loop (:for entry after) + (:let was (lock-entry-for before (locked-name entry))) + (:let line + (match was + (None (Some (str "added " (name->string (locked-name entry)) " " + (version->string (locked-version entry))))) + ((Some old) + (if (= (version->string (locked-version old)) + (version->string (locked-version entry))) + None + (Some (str (name->string (locked-name entry)) " " + (version->string (locked-version entry)) + " (was " (version->string (locked-version old)) ")")))))) + (:when-let (Some text) line) + (:acc out (listing text)) + => out)) + (def removed + (loop (:for entry before) + (:when (not (some? (lock-entry-for after (locked-name entry))))) + (:acc out (listing (str "removed " (name->string (locked-name entry)) " " + (version->string (locked-version entry))))) + => out)) + (list-sort (list-append added-or-changed removed))) +;; bjo/manifest.bjo +;; +;; What a `manifest.bjodat` says, as types. +;; +;; (package +;; (name (bjolang http)) +;; (version "1.2.3") +;; (depends +;; (package (name (bjolang cml)) (version (version-at-least "2.0.1"))))) +;; +;; A manifest is a *tagged form*: the head symbol names the type and each clause +;; names a field. That is `#:tag` in `def/bjodat-type`, and everything below is +;; an ordinary use of it — there is no manifest-shaped special case anywhere in +;; `(text bjodat)`, and there should not be. See the docs of `def/bjodat-type`. +;; +;; Two things here are decisions rather than mechanism, and both come from the +;; language rather than from the format: +;; +;; **A module name is a parenthesised list of symbols.** `(std run)`, +;; `(text json)`, `(bjolang http)` — the same spelling `import` uses, so a name +;; in a manifest and a name in a source file are one thing written one way. +;; That makes a package name a `(List Symbol)`, which bjodat already reads: a +;; `(...)` is a list and a bare identifier is a symbol, so nothing had to be +;; added to the format to carry it. A string would have been a second spelling +;; of the same value and a thing to parse twice. +;; +;; **A version constraint is a tagged form of its own.** `(version-at-least +;; "1.2")` rather than `">=1.2"`, because a constraint written inside a string +;; is a small language smuggled past the reader — it needs its own parser, its +;; own errors, and its own position information that the reader already has. +;; Written as a form it is read by the reader, refused by the reader, and +;; located by the reader. + +(export + ;; Versions + Version version version->string parse-version version-compare + ;; Constraints. A union's cases travel with it, so `AtLeast` and its + ;; neighbours are not named here — naming one asks for a signature it has no + ;; way to have. + VersionReq version-satisfies? + ;; Where a package comes from + Git Local Registry Source + ;; The manifest + Dep Package + NuGetPackage nuget-id nuget-version nuget-precompile + RuntimeSetting runtime-settings + module-name->string + ;; What `def/bjodat-type` generated for the two records people read. + bjodat-parse-Package bjodat-read-Package bjodat->Package Package->bjodat + bjodat-parse-Dep bjodat-read-Dep bjodat->Dep Dep->bjodat) + +(import (std prelude)) +(import (text bjodat)) + +(import/extern + (string->long (: System.Int64.TryParse (-> string (out long) (Option long))))) + +;; --------------------------------------------------------------------------- +;; A module name +;; --------------------------------------------------------------------------- +;; `(List Symbol)`, with no alias over it. An alias would read the same and buy +;; nothing: `(List Symbol)` already has both instances, so `(name (std run))` +;; decodes with no code here at all. This is only how to show one in a message. + +(: (-> (List Symbol) string)) +(defun (module-name->string m) + (def sb (stringbuilder-empty)) + (stringbuilder-add! sb #\() + (loop (:with first #t #f) + (:for s m) + (:do (unless first (stringbuilder-add! sb #\space)) + (stringbuilder-add-string! sb (symbol->string s)))) + (stringbuilder-add! sb #\)) + (stringbuilder->string sb)) + +;; --------------------------------------------------------------------------- +;; A version +;; --------------------------------------------------------------------------- +;; Three numbers. `"1.2"` is `1.2.0` and `"1"` is `1.0.0`, so a manifest may +;; write as much as it means and no more. + +(type (: Version (Record (: major int) (: minor int) (: patch int)))) + +(: (-> int int int Version)) +(defun (version ma mi pa) (Version (major ma) (minor mi) (patch pa))) + +(: (-> Version string)) +(defun (version->string v) + (str (int->string (record-ref v major)) "." + (int->string (record-ref v minor)) "." + (int->string (record-ref v patch)))) + +(: (-> string string)) +(defun (version-shape s) + (str "\"" s "\" is not a version: a version is written \"1.2.3\"")) + +(: (-> (List int) string (Result string Version))) +(defun (finish-version out s) + (match (list-reverse out) + ((Cons a Nil) (Ok (version a 0 0))) + ((Cons a (Cons b Nil)) (Ok (version a b 0))) + ((Cons a (Cons b (Cons c Nil))) (Ok (version a b c))) + (_ (Err (version-shape s))))) + +;; Read by hand rather than by splitting on `.` and converting each piece: the +;; split would allocate a list of strings to throw away, and the conversion +;; would have to refuse a piece like `"1x"` all over again. +(: (-> string (Result string Version))) +(defun (parse-version s) + (if (string-empty? s) + (Err (version-shape s)) + (let go ((c (string-cursor-start s)) (acc 0) (digits 0) (out Nil)) + (if (string-cursor-end? s c) + (if (= digits 0) + (Err (version-shape s)) + (finish-version (Cons acc out) s)) + (let ((ch (string-cursor-ref s c)) + (next (string-cursor-next s c))) + (cond + ((and (char>=? ch #\0) (char<=? ch #\9)) + (go next (+ (* acc 10) (- (char->int ch) 48)) (+ digits 1) out)) + ((char=? ch #\.) + (if (= digits 0) + (Err (version-shape s)) + (go next 0 0 (Cons acc out)))) + (else (Err (version-shape s))))))))) + +;; Negative when `a` is the older, zero when they are the same version. +(: (-> Version Version int)) +(defun (version-compare a b) + (def d (- (record-ref a major) (record-ref b major))) + (if (not (= d 0)) + d + (let ((e (- (record-ref a minor) (record-ref b minor)))) + (if (not (= e 0)) + e + (- (record-ref a patch) (record-ref b patch)))))) + +;; A version is written as a string, because it is one value and not a record. +(impl (->bjodat Version) + (defun (->bjodat v) (BjoStr (version->string v)))) + +(impl (bjodat-> Version) + (defun (bjodat-> v) + (match v + ((BjoStr s) (parse-version s)) + (_ (Err "expected a version, such as \"1.2.3\""))))) + +;; --------------------------------------------------------------------------- +;; A version constraint +;; --------------------------------------------------------------------------- +;; `(version-at-least "1.2")`, `(version-at-most "2.0")`, +;; `(version-between "1.2" "2.0")`. +;; +;; **All three are inclusive.** `version-between` is exactly `version-at-least` +;; and `version-at-most` together, which is what its name says and the only +;; rule that does not need a footnote. A half-open range would read the same +;; and mean something the spelling does not admit to. + +(type + (: VersionReq (Union (: AtLeast Version) + (: AtMost Version) + (: Between Version Version)))) + +(: (-> Version VersionReq bool)) +(defun (version-satisfies? v req) + (match req + ((AtLeast lo) (>= (version-compare v lo) 0)) + ((AtMost hi) (<= (version-compare v hi) 0)) + ((Between lo hi) (and (>= (version-compare v lo) 0) + (<= (version-compare v hi) 0))))) + +(: string) +(def req-shape + (str "expected a version constraint: (version-at-least \"1.2\"), " + "(version-at-most \"2.0\") or (version-between \"1.2\" \"2.0\")")) + +(: (-> (List Bjodat) string (Result string Version))) +(defun (one-version args tag) + (match args + ((Cons a Nil) (bjodat-> a)) + (_ (Err (str "(" tag " ...) takes one version"))))) + +;; The dispatch is a `cond` over the head symbol, written out rather than held +;; in a table. A table would have to be built at run time and looked up per +;; value, and there are three cases known when this is compiled. +(impl (bjodat-> VersionReq) + (defun (bjodat-> v) + (match v + ((BjoList (Cons (BjoSym h) args)) + (let ((tag (symbol->string h))) + (cond + ((= tag "version-at-least") + (match (one-version args tag) + ((Err e) (Err e)) + ((Ok a) (Ok (AtLeast a))))) + ((= tag "version-at-most") + (match (one-version args tag) + ((Err e) (Err e)) + ((Ok a) (Ok (AtMost a))))) + ((= tag "version-between") + (match args + ((Cons lo (Cons hi Nil)) + (match (bjodat-> lo) + ((Err e) (Err e)) + ((Ok l) (match (bjodat-> hi) + ((Err e) (Err e)) + ((Ok h2) (Ok (Between l h2))))))) + (_ (Err "(version-between ...) takes two versions")))) + (else (Err (str "(" tag " ...) is not a version constraint: " req-shape)))))) + (_ (Err req-shape))))) + +(impl (->bjodat VersionReq) + (defun (->bjodat r) + (match r + ((AtLeast a) + (BjoList (Cons (BjoSym (string->symbol "version-at-least")) + (Cons (->bjodat a) Nil)))) + ((AtMost a) + (BjoList (Cons (BjoSym (string->symbol "version-at-most")) + (Cons (->bjodat a) Nil)))) + ((Between lo hi) + (BjoList (Cons (BjoSym (string->symbol "version-between")) + (Cons (->bjodat lo) (Cons (->bjodat hi) Nil)))))))) + +;; --------------------------------------------------------------------------- +;; Where a package comes from +;; --------------------------------------------------------------------------- +;; Three tagged records, and a union that picks between them by their tags. + +(def/bjodat-type + (: Git (Record #:tag git + (: url string) + (: commit (Option string) #:optional) + (: branch (Option Symbol) #:optional)))) + +(def/bjodat-type + (: Local (Record #:tag path (: dir string)))) + +(def/bjodat-type + (: Registry (Record #:tag registry + (: host string #:default "https://packages.bjolang.org")))) + +(type (: Source (Union (: FromGit Git) (: FromPath Local) (: FromRegistry Registry)))) + +;; This is the dispatch a manifest wants, and it is worth saying what it is not: +;; it is not the reader deciding what `(git ...)` means. The reader hands over a +;; list headed by a symbol and nothing more; the *expected type* — `Source`, +;; here — is what turns that into a case. The same list under a field expecting +;; a `(List Symbol)` is a module name, and neither reading is privileged. +(impl (bjodat-> Source) + (defun (bjodat-> v) + (match v + ((BjoList (Cons (BjoSym h) _)) + (let ((tag (symbol->string h))) + (cond + ((= tag "git") + (match (bjodat->Git v) ((Err e) (Err e)) ((Ok g) (Ok (FromGit g))))) + ((= tag "path") + (match (bjodat->Local v) ((Err e) (Err e)) ((Ok p) (Ok (FromPath p))))) + ((= tag "registry") + (match (bjodat->Registry v) ((Err e) (Err e)) ((Ok r) (Ok (FromRegistry r))))) + (else (Err (str "(" tag " ...) is not a source: write (git (url \"...\")), " + "(path (dir \"...\")) or (registry)")))))) + (_ (Err "expected a source, such as (git (url \"https://...\"))"))))) + +(impl (->bjodat Source) + (defun (->bjodat s) + (match s + ((FromGit g) (->bjodat g)) + ((FromPath p) (->bjodat p)) + ((FromRegistry r) (->bjodat r))))) + +;; --------------------------------------------------------------------------- +;; The manifest +;; --------------------------------------------------------------------------- +;; A dependency is written `(package ...)`, under the same tag as the manifest +;; itself, because that is what it is: a package, named and constrained rather +;; than described. Two types sharing one tag is not a collision — which record a +;; form decodes into is the field's to say, and `depends` says `Dep` exactly as +;; the file's root says `Package`. + +(def/bjodat-type + (: Dep (Record #:tag package + (: name (List Symbol)) + (: version (Option VersionReq) #:optional) + (: source (Option Source) #:optional)))) + +;; A NuGet package: `(nuget (id "Npgsql") (version "9.0.3"))`. +;; +;; The version is NuGet's, passed to `PackageReference` as written: "9.0.3" +;; means 9.0.3 or newer, "[9.0.3]" exactly that, and ranges and floating +;; versions mean what NuGet says they mean. It is a string rather than a +;; `Version` because bjo never reads it. +;; +;; `(precompile #f)` keeps the package's assemblies as IL: they are not +;; precompiled with ReadyToRun. Absent, the package is precompiled. The root +;; project's entry for an id wins over a dependency's, so a root can turn it +;; back on with `(precompile #t)`. See `bjo/nuget.bjo`. +(def/bjodat-type + (: NuGetPackage (Record #:tag nuget + (: id string) + (: version string) + (: precompile (Option bool) #:optional)))) + +(: (-> NuGetPackage string)) +(defun (nuget-id p) (record-ref p id)) + +(: (-> NuGetPackage string)) +(defun (nuget-version p) (record-ref p version)) + +;; What the entry says about precompiling: `None` when it says nothing. +(: (-> NuGetPackage (Option bool))) +(defun (nuget-precompile p) (record-ref p precompile)) + +;; One .NET runtime setting of the program, `(gc-server #t)`, as the compiler's +;; `--runtime` takes it: `name=value`. The names are those of +;; `Build.runtimeSettings`, and the two lists below must stay the same as it. +;; They are checked here too, so that a mistake is reported against the +;; manifest and not against a compiler flag. +(type (: RuntimeSetting (Record (: name string) (: value string)))) + +(def flag-settings + (list "gc-concurrent" "gc-dynamic-adaptation" "gc-server" "invariant-globalization" + "tiered-compilation" "tiered-pgo")) +(def count-settings (list "gc-heap-count")) +(def millis-settings (list "call-counting-delay")) + +(: string) +(def known-settings + (string-join (list->vec (list-sort (list-append flag-settings (list-append count-settings millis-settings)))) + ", ")) + +(impl (bjodat-> RuntimeSetting) + (defun (bjodat-> v) + (match v + ((BjoList (Cons (BjoSym h) (Cons value Nil))) + (let ((name (symbol->string h))) + (cond + ((any? #(= & name) flag-settings) + (match value + ((BjoBool b) (Ok (RuntimeSetting (name name) (value (if b "true" "false"))))) + (_ (Err (str "(" name " ...) is #t or #f"))))) + ((any? #(= & name) count-settings) + (match value + ((BjoInt n) #:when (>= n 1L) (Ok (RuntimeSetting (name name) (value (long->string n))))) + (_ (Err (str "(" name " ...) is a whole number from 1"))))) + ((any? #(= & name) millis-settings) + (match value + ((BjoInt n) #:when (>= n 0L) (Ok (RuntimeSetting (name name) (value (long->string n))))) + (_ (Err (str "(" name " ...) is a number of milliseconds from 0"))))) + (else (Err (str "(" name " ...) is not a runtime setting. The settings are: " + known-settings)))))) + (_ (Err "a runtime setting is (name value), such as (gc-server #t)"))))) + +(impl (->bjodat RuntimeSetting) + (defun (->bjodat r) + (def value (record-ref r value)) + (BjoList (list (BjoSym (string->symbol (record-ref r name))) + (case value + ("true" (BjoBool #t)) + ("false" (BjoBool #f)) + (else (BjoInt (match (string->long value) ((Some n) n) (None 0L))))))))) + +;; The settings as the compiler's `--runtime` takes them, `name=value`, sorted +;; by name: also the form of the `runtime` lines of a program's build record. +(: (-> (List RuntimeSetting) (List string))) +(defun (runtime-settings settings) + (list-sort (list-map #(str (record-ref & name) "=" (record-ref & value)) settings))) + +(def/bjodat-type + (: Package (Record #:tag package + (: name (List Symbol)) + (: version Version) + (: description (Option string) #:optional) + (: license (Option string) #:optional) + (: authors (Vec string) #:default (vec-empty)) + (: entry string #:default "src/main.bjo") + (: targets (Vec Symbol) #:default (vec-empty)) + ;; The .NET shared frameworks this package's modules may + ;; name types out of: `(frameworks "Microsoft.AspNetCore.App")`. + ;; + ;; Optional, and allowed in any package, root or + ;; dependency. `Microsoft.NETCore.App` is always there and + ;; must not be listed — saying it would suggest that not + ;; saying it left it out. A package without the field + ;; behaves exactly as it always has. + ;; + ;; `#:rest` for the same reason `depends` has it: the + ;; clause holds as many names as it holds, and a `[...]` + ;; around them would be a bracket carrying no information. + (: frameworks (List string) #:rest #:default Nil) + ;; Files and directories the package's modules read at + ;; run time, relative to the package directory: + ;; `(resources "grammars")`. `bjo publish` copies them, + ;; keeping their place relative to the package's modules, + ;; so code that finds them from its own `.dll` still does. + (: resources (List string) #:rest #:default Nil) + ;; NuGet packages the package's modules may use through + ;; `import/extern` and `import/class`. Allowed in any + ;; package, root or dependency. + (: packages (List NuGetPackage) #:rest #:default Nil) + ;; Directories of C# sources, relative to the package + ;; directory: `(csharp "glue")`. bjo compiles them into + ;; one assembly per package, against the package's NuGet + ;; packages and the Bjolang runtime, and its modules use + ;; it through `import/extern` and `import/class`. Allowed + ;; in any package, root or dependency. + (: csharp (List string) #:rest #:default Nil) + ;; The .NET runtime settings of the program that the + ;; project builds: `(runtime (gc-server #t))`. Only the + ;; root project's count: they are settings of a process, + ;; and a dependency is not one. + (: runtime (List RuntimeSetting) #:rest #:default Nil) + ;; `(aot #t)`: the program is also built as a Native AOT + ;; executable, and `bjo run` runs that one. Only the root + ;; project's counts, as with `runtime`. Optional rather + ;; than defaulted, so that a manifest written back without + ;; it does not gain an `(aot #f)`. + (: aot (Option bool) #:optional) + ;; `(aot-cpu "native")`: the CPU the Native AOT executable + ;; is built for, passed as `--aot-cpu`. `native` is the + ;; machine that builds; `x86-64-v3` is any CPU with AVX2. + ;; Without it the executable runs on any x86-64 CPU and + ;; uses 128-bit vectors. Only `(aot #t)` reads it. + (: aot-cpu (Option string) #:optional) + (: source (Option Source) #:optional) + ;; `#:rest`: the clause holds as many packages as it holds, + ;; rather than one vec of them. This is the field that made + ;; `#:rest` worth having — `(depends (package ...) (package + ;; ...))` is how a person writes it, and a `[...]` around + ;; them would be a bracket that carries no information. + (: depends (List Dep) #:rest #:default Nil) + (: dev-depends (List Dep) #:rest #:default Nil)))) +;; bjo/nuget.bjo — the NuGet packages a project's manifest names. +;; +;; (package +;; (name (shop)) +;; (version "0.1.0") +;; (packages (nuget (id "Npgsql") (version "9.0.3")))) +;; +;; The packages are resolved by the .NET SDK rather than here. Which files of a +;; package a program compiles against and which it runs with depends on the +;; target framework, on the shared frameworks the program also uses (ASP.NET +;; already contains some packages, and its copy has to win), on `_._` +;; placeholders and on runtime-specific assets. MSBuild already knows all of +;; that, so `bjo` generates a small project in `.bjo/nuget/root/`, runs the +;; SDK's own restore and reference resolution on it, and has a target of its own +;; write the results into three files beside it: +;; +;; compile.txt the package assemblies the C# compile references +;; runtime.txt the package assemblies the program loads, and the type +;; checker reflects over +;; native.txt runtime-specific assets, for every runtime identifier: one +;; line each of package, asset type, RID and path. The compiler +;; picks the ones for the machine it runs on. +;; +;; A dependency that declares packages gets a generated project of its own in +;; `.bjo/nuget/packages/`, which the root project references. NuGet then +;; resolves versions across all of them, and the root's own declaration of a +;; package wins over a dependency's. +;; +;; A package whose manifest says `(csharp "glue")` has C# sources, its glue. +;; Each such package gets a generated project of its own in +;; `.bjo/nuget/glue/`, which compiles them into one assembly against the +;; package's NuGet packages and the Bjolang runtime's assemblies. The root +;; project references it, so the one restore resolves its packages along with +;; everything else's, and the lists name the assembly it builds. The glue is +;; built after the restore, by `dotnet build`. +;; +;; The compiler is given the directory with `--nuget`. The lists are written +;; only when their contents change, because the compiler records them as inputs +;; of every module it builds against them. +;; +;; Starting MSBuild costs a couple of seconds, so it is skipped when nothing it +;; depends on has changed. A stamp file holds a hash of the generated project — +;; which spells out the packages, the frameworks and the target framework — and +;; of `packages.lock.json`, and the lists have to name files that are still +;; there: a cleared NuGet cache is a restore to redo. A project that names no +;; packages starts no `dotnet` at all. + +(export restore-packages) + +(import (std prelude)) +(import (std run)) +(import (text bjodat)) +(import "manifest.bjo") +(import "project.bjo") + +(import/class (Encoding (: System.Text.Encoding))) + +(import/extern + (error-message (: System.Exception.Message (-> Exception string) #:get)) + (utf8 (: System.Text.Encoding.UTF8 #:get)) + (encode (: System.Text.Encoding.GetBytes (-> Encoding string (Array byte)))) + (sha256 (: System.Security.Cryptography.SHA256.HashData (-> (Array byte) (Array byte)))) + (hex (: System.Convert.ToHexString (-> (Array byte) string))) + (index-of (: System.String.IndexOf (-> string string int))) + (substring-from (: System.String.Substring (-> string int string))) + (substring-range (: System.String.Substring (-> string int int string)))) + + +;; --------------------------------------------------------------------------- +;; The generated project +;; --------------------------------------------------------------------------- + +(: (-> string string)) +(defun (xml-escape s) + (string-replace + (string-replace (string-replace (string-replace s "&" "&") "<" "<") ">" ">") + "\"" """)) + +;; The target that writes the three lists. +;; +;; A package's compile assets are the `ReferencePath` items with a +;; `NuGetPackageId`. The shared frameworks' reference assemblies carry one too +;; (`Microsoft.NETCore.App.Ref`), so those are told apart by their +;; `FrameworkReferenceName`. Runtime assets are the `.dll`s among +;; `ReferenceCopyLocalPaths` that go in the output directory itself: that +;; leaves out symbols, documentation, satellite resource assemblies and +;; runtime-specific files, which all go in a subdirectory or are not `.dll`s. +;; Both filter on `NuGetPackageId` as well, which leaves out the generated +;; projects of dependencies: those are never built, and have nothing to offer. +;; +;; Runtime-specific assets are `RuntimeTargetsCopyLocalItems`, one per runtime +;; identifier a package has files for, with the RID and whether it is `native` +;; or `runtime` (managed) as metadata. `NativeCopyLocalItems` are native files +;; that are the same everywhere, and have no RID. +;; +;; `glue-dlls` are the assemblies the glue projects build. They go in both +;; lists by name, since the restore does not build them: they are built after +;; it, and are there before anything reads the lists. +(: (-> (List string) string)) +(defun (resolve-target glue-dlls) + (str " \n" + " \n" + " <_BjoCompile Include=\"@(ReferencePath)\" Condition=\"'%(ReferencePath.NuGetPackageId)' != '' and '%(ReferencePath.FrameworkReferenceName)' == ''\" />\n" + " <_BjoRuntime Include=\"@(ReferenceCopyLocalPaths)\" Condition=\"'%(ReferenceCopyLocalPaths.NuGetPackageId)' != '' and '%(ReferenceCopyLocalPaths.Extension)' == '.dll' and '%(ReferenceCopyLocalPaths.DestinationSubDirectory)' == ''\" />\n" + (string-join (list->vec (list-map #(str " <_BjoCompile Include=\"" (xml-escape &) "\" />\n" + " <_BjoRuntime Include=\"" (xml-escape &) "\" />\n") + glue-dlls)) + "") + " <_BjoNative Include=\"@(NativeCopyLocalItems);@(RuntimeTargetsCopyLocalItems)\" />\n" + " \n" + " '%(FullPath)')\" Overwrite=\"true\" WriteOnlyWhenDifferent=\"true\" />\n" + " '%(FullPath)')\" Overwrite=\"true\" WriteOnlyWhenDifferent=\"true\" />\n" + " '%(NuGetPackageId) %(AssetType) %(RuntimeIdentifier) %(FullPath)')\" Overwrite=\"true\" WriteOnlyWhenDifferent=\"true\" />\n" + " '%(NuGetPackageId) %(FullPath)')\" Overwrite=\"true\" WriteOnlyWhenDifferent=\"true\" />\n" + " \n")) + +(: (-> (List NuGetPackage) (List string) string)) +(defun (item-lines packages frameworks) + (def package-lines + (list-map #(str " \n") + packages)) + (def framework-lines + (list-map #(str " \n") frameworks)) + (str (string-join (list->vec package-lines) "") + (string-join (list->vec framework-lines) ""))) + +;; `Root.csproj`. A library with no sources; `CopyLocalLockFileAssemblies` is +;; what puts a library's package runtime assets in `ReferenceCopyLocalPaths`. +;; The shared frameworks are referenced so that the SDK's conflict resolution +;; drops a package assembly the framework already provides. `references` are +;; the generated projects of the dependencies that declare packages, and of +;; the glue; `glue-dlls` are what the glue projects build. +(: (-> (List NuGetPackage) (List string) (List string) (List string) string string)) +(defun (csproj-text packages frameworks references glue-dlls lock-file) + (def reference-lines + (list-map #(str " \n") references)) + (str "\n" + "\n" + " \n" + " " target-framework "\n" + " Library\n" + " false\n" + " true\n" + " true\n" + " " (xml-escape lock-file) "\n" + " \n" + " \n" + (item-lines packages frameworks) + (string-join (list->vec reference-lines) "") + " \n" + (resolve-target glue-dlls) + "\n")) + +;; The generated project of a dependency: its packages and frameworks, which +;; reach the root project through its `ProjectReference`. Never built — the +;; restore is run with `BuildProjectReferences=false`. +(: (-> string (List NuGetPackage) (List string) string)) +(defun (dependency-csproj-text manifest packages frameworks) + (str "\n" + "\n" + " \n" + " " target-framework "\n" + " Library\n" + " false\n" + " \n" + " \n" + (item-lines packages frameworks) + " \n" + "\n")) + +;; Writes `path` unless it already says `text`. +(: write-if-changed #:sync (-> string string void)) +(defun (write-if-changed path text) + (unless (and (file-exists? path) (= (file-read-text path) text)) + (file-write-text path text))) + + +;; --------------------------------------------------------------------------- +;; The glue +;; --------------------------------------------------------------------------- + +;; A package of the build with C# sources: its name, its directory, and what +;; its manifest says. +(type (: Glue (Record (: name (List Symbol)) (: dir string) (: package Package)))) + +;; The packages of the build that have glue, the project itself included. +(: (-> (List Root) (List Glue))) +(defun (glues-of roots) + (loop (:for r roots) + (:when-let (Some dir) (root-package-dir r)) + (:when-let (Some package) (root-package r)) + (:when (not (empty? (record-ref package csharp)))) + (:acc out (listing (Glue (name (root-name r)) (dir (path-absolute dir)) (package package)))) + => out)) + +(: (-> Glue string)) +(defun (glue-slug g) + (string-join (list->vec (list-map symbol->string (record-ref g name))) ".")) + +;; The assembly's name, which is also its file's. +(: (-> Glue string)) +(defun (glue-assembly g) (str "BjoGlue." (glue-slug g))) + +(: (-> Project Glue string)) +(defun (glue-project-dir project g) + (path-combine (project-dot-bjo project) "nuget" "glue" (glue-slug g))) + +(: (-> Project Glue string)) +(defun (glue-csproj-file project g) + (path-combine (glue-project-dir project g) (str (glue-assembly g) ".csproj"))) + +;; Where the glue project puts its assembly: a fixed place, so that the lists +;; can name it before it is built. +(: (-> Project Glue string)) +(defun (glue-dll project g) + (path-combine (glue-project-dir project g) "out" (str (glue-assembly g) ".dll"))) + +;; The source directories, as the manifest names them, made absolute. +(: (-> Glue (List string))) +(defun (glue-source-dirs g) + (list-map #(path-absolute (path-combine (record-ref g dir) &)) (record-ref (record-ref g package) csharp))) + +;; The Bjolang runtime's assemblies, which glue may use: `Option` and +;; `Result` from BjolangRuntime, `string` from BjoString, the collections. +;; Those the compiler's directory has. +(: (List string)) +(def runtime-assembly-names + (list "BjolangRuntime" "BjoString" "SchemeList" "BjoRx")) + +(: (-> (List string))) +(defun (runtime-assemblies) + (loop (:for name runtime-assembly-names) + (:let path (path-combine (compiler-dir) (str name ".dll"))) + (:acc out (listing path) #:when (file-exists? path)) + => out)) + +;; The glue's project. It compiles every `.cs` file under its source +;; directories except those under `bin` and `obj`, against its package's NuGet +;; packages and frameworks and the runtime's assemblies. Those are not copied +;; next to it (`Private` false): a program loads them anyway. +(: (-> Glue string)) +(defun (glue-csproj-text g) + (def package (record-ref g package)) + (def compile-lines + (list-map #(str " \n") + (glue-source-dirs g))) + (def runtime-lines + (list-map #(str " " + "" (xml-escape &) "false\n") + (runtime-assemblies))) + (str "\n" + "\n" + " \n" + " " target-framework "\n" + " Library\n" + " " (xml-escape (glue-assembly g)) "\n" + " false\n" + " enable\n" + " disable\n" + " true\n" + " $(MSBuildProjectDirectory)/out/\n" + " false\n" + " false\n" + " \n" + " \n" + (string-join (list->vec compile-lines) "") + (item-lines (record-ref package packages) (record-ref package frameworks)) + (string-join (list->vec runtime-lines) "") + " \n" + "\n")) + +;; What the glue's build reads that its project file does not say: every C# +;; source and its text, and when each runtime assembly was written. A change +;; to either is a build to redo. +(: (-> Glue string)) +(defun (glue-fingerprint g) + (def sources + (loop (:for dir (glue-source-dirs g)) + (:when (directory-exists? dir)) + (:for file (directory-walk dir #:into? (fun (d) (not (any? #(= (path-filename d) (Some &)) + (list "bin" "obj")))))) + (:when (string-ends-with? file ".cs")) + (:acc out (listing (str file "\n" (file-read-text file)))) + => (list-sort out))) + (def runtime + (list-map #(str & " " (match (file-info &) + ((Some info) (long->string (record-ref info modified-ms))) + (None "absent"))) + (runtime-assemblies))) + (string-join (list->vec (list-append sources runtime)) "\n")) + +;; The C# compiler's errors in `output`, each once: `file(line,col): error +;; CS1002: ; expected`, without the project MSBuild appends. +(: (-> string (List string))) +(defun (compile-errors output) + (def found + (loop (:for line (string-split output "\n")) + (:let at (index-of line ": error ")) + (:when (>= at 0)) + (:let end (index-of line " [")) + (:acc out (listing (string-trim (if (> end at) (substring-range line 0 end) line)))) + => out)) + (loop (:for m found) + (:acc out (folding Nil (if (any? #(= & m) out) out (Cons m out)))) + => (list-reverse out))) + +;; Builds one glue project, after the restore that resolved its packages. +(: build-glue #:sync (-> Project Glue (Result string Unit))) +(defun (build-glue project g) + (def dir (glue-project-dir project g)) + (def errors (path-combine dir "build-stderr.txt")) + (def arguments + (list->vec + (list "build" (glue-csproj-file project g) "-c" "Release" "--no-restore" "-nologo" + "-p:ImportDirectoryBuildProps=false" + "-p:ImportDirectoryBuildTargets=false" + "-p:ImportDirectoryPackagesProps=false"))) + (match (run/output '(errors-into-file ,errors (dotnet ,arguments))) + ((Err e) (Err (str "bjo: could not run dotnet: " (error-message e)))) + ((Ok (Tuple 0 _)) (Ok unit)) + ((Ok (Tuple _ out)) + (def said (string-trim (str out "\n" (if (file-exists? errors) (file-read-text errors) "")))) + (def name (name->string (record-ref g name))) + (Err (match (compile-errors said) + (Nil (str "bjo: building the C# glue of " name " failed. dotnet said:\n" said)) + (found (str "bjo: the C# glue of " name " does not compile:\n" + (string-join (list->vec (list-map #(str " " &) found)) "\n")))))))) + + +;; --------------------------------------------------------------------------- +;; Skipping the restore +;; --------------------------------------------------------------------------- + +(: (-> string string string)) +(defun (stamp-of csproj lock-file) + (def lock (if (file-exists? lock-file) (file-read-text lock-file) "")) + (hex (sha256 (encode utf8 (str csproj "\n--- packages.lock.json\n" lock))))) + +;; The paths in a list file, or `None` when there is no such file. +(: (-> string (Option (List string)))) +(defun (listed file) + (if (not (file-exists? file)) + None + (Some (loop (:for line (string-split (file-read-text file) "\n")) + (:let path (string-trim line)) + (:acc out (listing path) #:when (not (= path ""))) + => out)))) + +(: (-> string bool)) +(defun (list-current? file) + (match (listed file) + (None #f) + ((Some paths) (all? file-exists? paths)))) + +;; The same for `native.txt`, whose path is the last of four fields. +(: (-> string bool)) +(defun (native-current? file) + (match (listed file) + (None #f) + ((Some lines) + (all? (fun (line) + (def fields (string-split line "\t")) + (and (= (vec-length fields) 4) (file-exists? (vec-ref fields 3)))) + lines)))) + +;; Is the last restore still the answer? +(: (-> string string bool)) +(defun (restore-current? dir stamp) + (def stamp-file (path-combine dir "stamp")) + (and (file-exists? stamp-file) + (= (string-trim (file-read-text stamp-file)) stamp) + (list-current? (path-combine dir "compile.txt")) + (list-current? (path-combine dir "runtime.txt")) + (native-current? (path-combine dir "native.txt")) + ;; Written by a target added later: a restore without it is redone. + (file-exists? (path-combine dir "packages.txt")))) + + +;; --------------------------------------------------------------------------- +;; Running MSBuild +;; --------------------------------------------------------------------------- + +;; Runs the restore and the resolve target, and answers the exit code and +;; everything MSBuild said. +;; +;; The `Directory.*` imports are turned off because a project is often inside a +;; repository that has them, and they would apply to the generated project: a +;; `Directory.Packages.props` turns on central package management, which +;; refuses a `Version` on a `PackageReference`, and a `Directory.Build.props` +;; can change the target framework. `NuGet.config` and `global.json` are still +;; found by walking up from the generated project, which reaches the user's. +(: run-msbuild #:sync (-> string bool (Result string (Tuple int string)))) +(defun (run-msbuild dir locked?) + (def errors (path-combine dir "msbuild-stderr.txt")) + (def arguments + (list->vec + (list-append + (list "msbuild" (path-combine dir "Root.csproj") "-restore" "-t:BjoResolve" "-nologo" + "-p:BuildProjectReferences=false" + "-p:ImportDirectoryBuildProps=false" + "-p:ImportDirectoryBuildTargets=false" + "-p:ImportDirectoryPackagesProps=false") + (if locked? (list "-p:RestoreLockedMode=true") Nil)))) + (set-environment-variable! "DOTNET_NOLOGO" "1") + (match (run/output '(errors-into-file ,errors (dotnet ,arguments))) + ((Err e) (Err (str "bjo: could not run dotnet: " (error-message e)))) + ((Ok (Tuple code out)) + (def said (if (file-exists? errors) (file-read-text errors) "")) + (Ok (Tuple code (string-trim (str out "\n" said))))))) + +;; Every distinct `NUxxxx: message` MSBuild reported with `code`, from the code +;; to the end of the line. +(: (-> string string (List string))) +(defun (messages-with output code) + (def found + (loop (:for line (string-split output "\n")) + (:let at (index-of line (str code ": "))) + (:acc out (listing (string-trim (substring-from line at))) #:when (>= at 0)) + => out)) + (loop (:for m found) + (:acc out (folding Nil (if (any? #(= & m) out) out (Cons m out)))) + => (list-reverse out))) + +;; What a failed restore is reported as. A NuGet error is quoted with its code +;; and a hint about what usually causes it; anything else is MSBuild's own +;; output, which is the only account there is. +(: (-> string string string (Option string))) +(defun (explain output code hint) + (match (messages-with output code) + (Nil None) + (found (Some (str (string-join (list->vec (list-map #(str " " &) found)) "\n") + "\n hint: " hint))))) + +(: (-> string bool string string)) +(defun (failure-message output locked? lock-file) + (def not-found "Check the package name and version, and the NuGet sources in your NuGet.config.") + (def explained + (loop (:for answer (list (explain output "NU1101" not-found) + (explain output "NU1102" not-found) + (explain output "NU1301" + "The first restore of a package needs network access, or a local feed in NuGet.config.") + (explain output "NU1004" + (if locked? + (str "Run 'bjo build' without --locked to update " lock-file ".") + (str lock-file " does not match the manifest."))))) + (:when-let (Some text) answer) + (:acc out (listing text)) + => out)) + (if (empty? explained) + (str "bjo: restoring the NuGet packages failed. MSBuild said:\n" output) + (str "bjo: restoring the NuGet packages failed:\n" + (string-join (list->vec explained) "\n")))) + +;; --------------------------------------------------------------------------- +;; ReadyToRun copies +;; --------------------------------------------------------------------------- +;; +;; After a restore, the package assemblies are precompiled with ReadyToRun by +;; the compiler's `--precompile-packages`, into a cache outside the NuGet +;; cache. `Build.fs` describes the cache and the files in the restore's +;; directory. +;; +;; - Each package is precompiled unless an entry for its id says +;; `(precompile #f)`. +;; - The root project's entry wins. Without one, a dependency's `#f` turns it +;; off. +;; - A package that is not declared (a transitive one) is opted out by +;; declaring it with `(precompile #f)`. +;; - `BJOLANG_NO_R2R` turns it all off. + +;; What the entries in `packages` for the id `key` (lowercase) say. +(: (-> (List NuGetPackage) string (List bool))) +(defun (said-about packages key) + (loop (:for p packages) + (:when (= (string-downcase (nuget-id p)) key)) + (:when-let (Some b) (nuget-precompile p)) + (:acc out (listing b)) + => out)) + +;; Whether the package `id` is precompiled. +(: (-> (List NuGetPackage) (List Declaring) string bool)) +(defun (precompiled? own deps id) + (def key (string-downcase id)) + (match (said-about own key) + ((Cons b _) b) + (Nil (not (any? #(any? not (said-about (declared-by &) key)) deps))))) + +;; The NuGet packages a dependency declares. +(: (-> Declaring (List NuGetPackage))) +(defun (declared-by d) (record-ref (record-ref d package) packages)) + +;; The lines of `r2r-inputs.txt`: `r2r` or `il`, a tab, and the path, for each +;; package assembly in `packages.txt`. The glue's assemblies have no package +;; id there, and are left out. +(: (-> string (List NuGetPackage) (List Declaring) string)) +(defun (precompile-inputs dir own deps) + (match (listed (path-combine dir "packages.txt")) + (None "") + ((Some lines) + (string-join + (list->vec + (loop (:for line lines) + (:let fields (string-split line "\t")) + (:when (and (= (vec-length fields) 2) (not (= (vec-ref fields 0) "")))) + (:acc out (listing (str (if (precompiled? own deps (vec-ref fields 0)) "r2r" "il") + "\t" (vec-ref fields 1)))) + => out)) + "\n")))) + +;; Whether every copy `r2r.txt` names is still there. +(: (-> string bool)) +(defun (copies-current? file) + (match (listed file) + (None #f) + ((Some lines) + (all? (fun (line) + (def fields (string-split line "\t")) + (and (= (vec-length fields) 2) (file-exists? (vec-ref fields 1)))) + lines)))) + +;; Fills the cache for the restore in `dir`, unless the last run had the same +;; inputs and its copies are all still there. +(: precompile-packages #:sync (-> string (List NuGetPackage) (List Declaring) (Result string Unit))) +(defun (precompile-packages dir own deps) + (with-return ret + (match (get-environment-variable "BJOLANG_NO_R2R") + ((Some v) (unless (or (= v "") (= v "0")) (ret (Ok unit)))) + (None unit)) + (def inputs (precompile-inputs dir own deps)) + (def inputs-file (path-combine dir "r2r-inputs.txt")) + (when (and (file-exists? inputs-file) + (= (file-read-text inputs-file) inputs) + (copies-current? (path-combine dir "r2r.txt"))) + (ret (Ok unit))) + ;; The compiler reads the inputs from the file. It is written again after + ;; a success, so that a failed run is tried again next time. + (def pending (path-combine dir "r2r-inputs.pending")) + (file-write-text pending inputs) + (def arguments (list->vec (list (path-combine (compiler-dir) "Bjolang.dll") "--precompile-packages" dir pending))) + (match (run/output '(dotnet ,arguments)) + ((Err e) (Err (str "bjo: could not run the compiler: " (error-message e)))) + ((Ok (Tuple 0 out)) + (unless (= (string-trim out) "") (println (string-trim out))) + (file-write-text inputs-file inputs) + (Ok unit)) + ((Ok (Tuple code out)) + (Err (str "bjo: precompiling the NuGet packages failed (exit code " (int->string code) "):\n" + (string-trim out))))))) + +;; --------------------------------------------------------------------------- +;; restore-packages +;; --------------------------------------------------------------------------- + +;; A package of the build that declares NuGet packages: its manifest, and what +;; the manifest says. +(type (: Declaring (Record (: name (List Symbol)) (: manifest string) (: package Package)))) + +;; The dependencies that declare packages. The project being built is not one +;; of them: its packages go in the root project itself. +(: (-> Project (List Root) (List Declaring))) +(defun (declaring-dependencies project roots) + (def own (path-absolute (project-dir project))) + (loop (:for r roots) + (:when-let (Some dir) (root-package-dir r)) + (:when (not (= (path-absolute dir) own))) + (:when-let (Some package) (root-package r)) + (:when (not (empty? (record-ref package packages)))) + (:acc out (listing (Declaring (name (root-name r)) + (manifest (path-combine dir manifest-name)) + (package package)))) + => out)) + +;; Where a dependency's generated project goes. The file name becomes the +;; project's name in the lock file, and is prefixed so that it cannot be taken +;; for a NuGet package of the same name. +(: (-> Project (List Symbol) string)) +(defun (dependency-csproj-file project name) + (def slug (string-join (list->vec (list-map symbol->string name)) ".")) + (path-combine (project-dot-bjo project) "nuget" "packages" slug + (str "BjoPackage." slug ".csproj"))) + +(: (-> (List NuGetPackage) (Option string))) +(defun (duplicate-id packages) + ;; NuGet ids are case-insensitive, so `npgsql` and `Npgsql` are one. + (def ids (list-sort (list-map #(string-downcase (nuget-id &)) packages))) + (let go ((left ids)) + (match left + ((Cons a (Cons b more)) (if (= a b) (Some a) (go (Cons b more)))) + (_ None)))) + +(: (-> string Package (Option string))) +(defun (duplicate-in manifest package) + (option-map (fun (id) (str "bjo: " manifest ": the NuGet package " id + " is listed twice under (packages ...).")) + (duplicate-id (record-ref package packages)))) + +;; Restores the NuGet packages of the build, and answers the directory to hand +;; the compiler with `--nuget` — or `None` when no manifest names any. +;; +;; `roots` are every package of the build: a dependency's packages are the +;; build's packages too. +(: restore-packages #:sync (-> Project (List Root) bool (Result string (Option string)))) +(defun (restore-packages project roots locked?) + (with-return ret + (def package (project-package project)) + (def packages (record-ref package packages)) + (def deps (declaring-dependencies project roots)) + (def glues (glues-of roots)) + (when (and (empty? packages) (empty? deps) (empty? glues)) (ret (Ok None))) + + (def own-manifest (path-combine (project-dir project) manifest-name)) + (def duplicates + (loop (:for message (Cons (duplicate-in own-manifest package) + (list-map #(duplicate-in (record-ref & manifest) (record-ref & package)) + deps))) + (:when-let (Some text) message) + (:acc out (listing text)) + => out)) + (match duplicates + ((Cons text _) (ret (Err text))) + (Nil unit)) + + (def dir (project-nuget-dir project)) + (def lock-file (project-nuget-lock-file project)) + (def dep-files (list-map #(dependency-csproj-file project (record-ref & name)) deps)) + (def dep-texts + (list-map #(dependency-csproj-text (record-ref & manifest) + (record-ref (record-ref & package) packages) + (record-ref (record-ref & package) frameworks)) + deps)) + (def glue-files (list-map #(glue-csproj-file project &) glues)) + (def glue-texts (list-map glue-csproj-text glues)) + (def glue-dlls (list-sort (list-map #(glue-dll project &) glues))) + (def csproj (csproj-text packages (record-ref package frameworks) + (list-sort (list-append dep-files glue-files)) glue-dlls lock-file)) + ;; Everything the restore and the glue's builds read that `bjo` wrote, and + ;; the glue's sources, in an order that does not depend on the order of the + ;; roots. + (def generated + (string-join (list->vec (Cons csproj + (list-sort (list-append + (loop (:for file dep-files) + (:for text dep-texts) + (:acc out (listing (str file "\n" text))) + => out) + (loop (:for g glues) + (:for file glue-files) + (:for text glue-texts) + (:acc out (listing (str file "\n" text "\n" + (glue-fingerprint g)))) + => out))))) + "\n")) + + (when (restore-current? dir (stamp-of generated lock-file)) + (ret (match (precompile-packages dir packages deps) + ((Err e) (Err e)) + ((Ok _) (Ok (Some dir)))))) + + (unless (some? (which "dotnet")) + (ret (Err (str "bjo: the build " + (if (empty? glues) "names NuGet packages" "has C# glue") + ", and " + (if (empty? glues) "restoring them" "compiling it") + " needs the .NET SDK's 'dotnet' command, which is not on PATH.")))) + + (directory-create dir) + (write-if-changed (path-combine dir "Root.csproj") csproj) + (loop (:for file (list-append dep-files glue-files)) + (:for text (list-append dep-texts glue-texts)) + (:do (match (parent-directory file) + ((Some parent) (directory-create parent)) + (None unit)) + (write-if-changed file text))) + (def (Ok (Tuple code output)) (run-msbuild dir locked?) :propagate) + (unless (= code 0) + (ret (Err (failure-message output locked? lock-file)))) + + ;; After the restore, which resolved the glue's packages. + (loop (:for g glues) + (:do (match (build-glue project g) + ((Err e) (ret (Err e))) + ((Ok _) unit)))) + + ;; A rebuilt glue assembly may have changed what it offers, and the + ;; modules that use it are compiled against what it had. The lists name + ;; it and not its contents, so they are written again: the compiler + ;; counts them as inputs of every module, which are then stale. + (unless (empty? glues) + (loop (:for list-file (list "compile.txt" "runtime.txt")) + (:let path (path-combine dir list-file)) + (:when (file-exists? path)) + (:do (file-write-text path (file-read-text path))))) + + ;; After the restore, which may have written the lock file, and after the + ;; glue is built: a build that failed is tried again next time. + (file-write-text (path-combine dir "stamp") (stamp-of generated lock-file)) + (def (Ok _) (precompile-packages dir packages deps) :propagate) + (Ok (Some dir)))) +;; bjo/project.bjo — a project on disk, and where it is. +;; +;; A project is a directory with a `manifest.bjodat` in it. It is found by +;; walking upward from where the user stood, so that every command works +;; anywhere inside the tree rather than only at its top — the same rule git +;; uses for `.git`, and for the same reason. +;; +;; The layout is fixed: +;; +;; manifest.bjodat what the package is called, and what it needs +;; bjo.lock the commit each git dependency resolved to +;; src/ the package's modules: src/lexer.bjo is (myapp lexer) +;; tests/ +;; .bjo/ roots, clones, fetched packages. Never committed. +;; +;; `src/` is not a setting. A module's name is its path below the package +;; directory, so a second layout would be a second answer to "which file is +;; (myapp lexer)" — and the answer has to be the same for a package the author +;; wrote and one that was fetched, which nothing in the fetched copy can +;; configure after the fact. + +(export + ;; A project + Project project-dir project-package + manifest-name find-project-dir project-at read-package + project-src project-dot-bjo project-tmp project-roots-file project-lock-file + project-entry project-has-entry? + + ;; Package names + reserved-name? parse-package-name name->string + + ;; The installed standard library + lib-dir + + ;; The roots file + Root root-name root-dir write-roots root-package-dir root-package + project-frameworks-file write-frameworks + + ;; NuGet + project-nuget-dir project-nuget-lock-file target-framework compiler-dir + + ;; Sources + sources-under + + ;; Files, made once + write-new parent-directory + + ;; Saying no + fail + + ;; Finding a program + which + + ;; The command + cmd-init) + +(import (std prelude)) +;; The traits the manifest's own instances are written against. A module that +;; decodes a `Package` names them whether or not it says so, because the +;; generated reader calls them. +(import (text bjodat)) +(import "manifest.bjo") + +(import/extern + (clr-directory-name (: System.IO.Path.GetDirectoryName (-> string string))) + (clr-null-or-empty? (: System.String.IsNullOrEmpty (-> string bool))) + ;; Where this executable sits, for the `lib` search below. `bjo.bjo` reads + ;; the same thing for `BJO_ROOT`. + (clr-base-dir (: System.AppContext.BaseDirectory #:get))) + + +;; --------------------------------------------------------------------------- +;; Saying no +;; --------------------------------------------------------------------------- + +;; Prints a message and answers what a failed command answers, so that a guard +;; is `(ret (fail "..."))` — one expression, and the `when` it stands in has a +;; value to give rather than a dropped one. +(: (-> string int)) +(defun (fail message) + (println message) + 1) + + +;; --------------------------------------------------------------------------- +;; Paths +;; --------------------------------------------------------------------------- + +;; The directory above this one, or `None` at the root of the filesystem, where +;; `GetDirectoryName` answers null rather than looping. +(: (-> string (Option string))) +(defun (parent-directory dir) + (def up (clr-directory-name (path-absolute dir))) + (if (clr-null-or-empty? up) None (Some up))) + +;; A program on PATH, if there is one. Two ways to have no answer — no PATH at +;; all, and nothing on it by that name — and both of them propagate. +(: (-> string (Option string))) +(defun (which program) + (def* ((Some path) (get-environment-variable "PATH") :propagate) + ((Some dir) (find #(file-exists? (path-combine & program)) + (string-split path ":")) :propagate)) + (Some (path-combine dir program))) + +(: string) +(def manifest-name "manifest.bjodat") + +;; The project a directory is in, if it is in one. +;; +;; Upward rather than downward: a project is the tree under the manifest, and a +;; command run three directories down is still run inside it. +(: (-> string (Option string))) +(defun (find-project-dir start) + (let go ((dir (path-absolute start))) + (if (file-exists? (path-combine dir manifest-name)) + (Some dir) + (match (parent-directory dir) + ((Some up) (go up)) + (None None))))) + + +;; --------------------------------------------------------------------------- +;; The manifest +;; --------------------------------------------------------------------------- + +;; A project: the directory, and what its manifest says. +(type (: Project (Record (: dir string) (: package Package)))) + +(: (-> Project string)) +(defun (project-dir p) (record-ref p dir)) + +(: (-> Project Package)) +(defun (project-package p) (record-ref p package)) + +(: (-> Project string)) +(defun (project-src p) (path-combine (project-dir p) "src")) + +(: (-> Project string)) +(defun (project-dot-bjo p) (path-combine (project-dir p) ".bjo")) + +(: (-> Project string)) +(defun (project-tmp p) (path-combine (project-dot-bjo p) "tmp")) + +(: (-> Project string)) +(defun (project-roots-file p) (path-combine (project-dot-bjo p) "roots")) + +;; Which package declared which .NET shared framework. Beside the roots file, +;; because it answers the same kind of question about the same set of packages. +(: (-> Project string)) +(defun (project-frameworks-file p) (path-combine (project-dot-bjo p) "frameworks")) + +;; The generated project the root package's NuGet packages are restored with. +;; `root/` because a later dependency with packages gets a directory of its own. +(: (-> Project string)) +(defun (project-nuget-dir p) (path-combine (project-dot-bjo p) "nuget" "root")) + +;; The NuGet lock file. At the project root, beside the manifest, and committed. +(: (-> Project string)) +(defun (project-nuget-lock-file p) (path-combine (project-dir p) "packages.lock.json")) + +;; The target framework the compiler is built for and compiles programs for. +;; The compiler's own directory is named after it, and a restore has to resolve +;; packages for it. +(: string) +(def target-framework "net10.0") + +;; The compiler's directory, where the Bjolang runtime's assemblies are: a +;; package's C# glue is compiled against them. The launcher exports the +;; compiler's path as `BJOLANG_COMPILER`; without it, the compiler is where +;; `build_compiler.sh` puts it. +(: (-> string)) +(defun (compiler-dir) + (match (get-environment-variable "BJOLANG_COMPILER") + ((Some path) (option-ref-or (parent-directory (path-absolute path)) ".")) + (None + (path-combine (match (get-environment-variable "BJO_ROOT") + ((Some dir) dir) + (None (path-absolute (path-combine clr-base-dir "..")))) + "bin" "compiler")))) + +(: (-> Project string)) +(defun (project-lock-file p) (path-combine (project-dir p) "bjo.lock")) + +;; What the manifest says to build. Relative to the manifest, so that a project +;; means the same thing wherever it is checked out. +(: (-> Project string)) +(defun (project-entry p) + (path-combine (project-dir p) (record-ref (project-package p) entry))) + +;; A package with no entry file is a library. Read from the disk rather than +;; from a flag in the manifest: `entry` has a default, and a library would +;; otherwise have to say twice that it has no program. +(: (-> Project bool)) +(defun (project-has-entry? p) (file-exists? (project-entry p))) + +;; The manifest at `file`, read into the types in `manifest.bjo`. +;; +;; The file name is in every message: a resolve reads the manifests of +;; dependencies too, and "expected a version" says nothing about which one. +(: (-> string (Result string Package))) +(defun (read-package file) + (if (not (file-exists? file)) + (Err #"bjo: no manifest at ${file}.") + (result-map-err (fun (why) #"bjo: ${file}: ${why}") + (bjodat-parse-Package (file-read-text file))))) + +;; The project a directory holds, manifest read and nothing else checked yet. +(: (-> string (Result string Project))) +(defun (project-at dir) + (def (Ok package) (read-package (path-combine dir manifest-name)) :propagate) + (Ok (Project (dir (path-absolute dir)) (package package)))) + + +;; --------------------------------------------------------------------------- +;; Package names +;; --------------------------------------------------------------------------- + +;; `(myapp lexer)`, for a message. +(: (-> (List Symbol) string)) +(defun (name->string name) (module-name->string name)) + +;; The installed `lib`, found the way the compiler finds it: an override first, +;; then the tree the binary is in, then the working directory, and a candidate +;; only counts if `std` is under it. The two have to agree, because a name this +;; one misses is a name the compiler refuses to give a directory to. +(: (-> (List string))) +(defun (lib-candidates) + (def from-env + (list-append + (match (get-environment-variable "BJOLANG_LIB") + ((Some dir) (Cons (path-absolute dir) Nil)) + (None Nil)) + (match (get-environment-variable "BJOLANG_HOME") + ((Some dir) (Cons (path-absolute (path-combine dir "lib")) Nil)) + (None Nil)))) + (list-append from-env + (list-append (ancestor-libs clr-base-dir) + (ancestor-libs (current-directory))))) + +;; `/lib` for `dir` and every directory above it, deepest first. +(: (-> string (List string))) +(defun (ancestor-libs start) + (let go ((dir (path-absolute start))) + (Cons (path-combine dir "lib") + (match (parent-directory dir) + ((Some up) (go up)) + (None Nil))))) + +(: (-> (Option string))) +(defun (lib-dir) + (find #(directory-exists? (path-combine & "std")) (lib-candidates))) + +;; A name whose first segment is a directory of `lib`. +;; +;; Refused rather than shadowed: the compiler gives the standard library its +;; roots whether or not a roots file was written, and refuses a roots file that +;; names one of them. A package called `(std)` would therefore build here and +;; fail at the compiler, one step later than the manifest that asked for it. +(: (-> (List Symbol) bool)) +(defun (reserved-name? name) + (match name + (Nil #f) + ((Cons first _) + (match (lib-dir) + (None #f) + ((Some lib) + (def head (symbol->string first)) + (any? #(match (path-filename &) + ((Some dir-name) (= dir-name head)) + (None #f)) + (directory-subdirectories lib))))))) + +;; The characters a name segment may be written with. +;; +;; A package name is read as symbols by the manifest reader and written as a +;; module path by an import, so a segment that needs quoting anywhere is a +;; segment that is spelled differently in two places. `init` is the only place +;; a name is made out of a string — a directory called `my.app` has no name +;; here, and has to be given one. +(: (-> char bool)) +(defun (name-char? c) + (or (and (char>=? c #\a) (char<=? c #\z)) + (and (char>=? c #\A) (char<=? c #\Z)) + (and (char>=? c #\0) (char<=? c #\9)) + (char=? c #\-) + (char=? c #\_))) + +(: (-> string bool)) +(defun (valid-segment? s) + (and (not (= s "")) + (let ((first (string-cursor-ref s (string-cursor-start s)))) + (not (and (char>=? first #\0) (char<=? first #\9)))) + (all? name-char? (string->list s)))) + +;; The name a command line wrote: `myapp`, or `(my app)` for several segments. +(: (-> string (Result string (List Symbol)))) +(defun (parse-package-name written) + (def trimmed (string-trim written)) + (def inner (if (and (string-starts-with? trimmed "(") (string-ends-with? trimmed ")")) + (string-trim (substring trimmed + (string-cursor-next trimmed (string-cursor-start trimmed)) + (string-cursor-prev trimmed (string-cursor-end trimmed)))) + trimmed)) + (def segments (loop (:for part (string-split inner " ")) + (:acc out (listing part) #:when (not (= part ""))) + => out)) + (cond + ((empty? segments) (Err #"bjo init: '${written}' is not a package name.")) + ((not (all? valid-segment? segments)) + (Err (str #"bjo init: '${written}' is not a package name. " + "A name is one or more words of letters, digits, '-' and '_', " + "written (my app) when there are several."))) + (else (Ok (list-map string->symbol segments))))) + + +;; --------------------------------------------------------------------------- +;; The roots file +;; --------------------------------------------------------------------------- + +;; One package, and the directory its modules are in. +(type (: Root (Record (: name (List Symbol)) (: dir string)))) + +(: (-> Root (List Symbol))) +(defun (root-name r) (record-ref r name)) + +(: (-> Root string)) +(defun (root-dir r) (record-ref r dir)) + +;; A directory as a Bjolang string literal. +(: (-> string string)) +(defun (quoted s) + (str "\"" (string-replace (string-replace s "\\" "\\\\") "\"" "\\\"") "\"")) + +;; The roots file, sorted, so that a diff of two builds shows what changed +;; rather than what was reordered. The line begins with the name, so sorting +;; the lines sorts by name. +(: (-> (List Root) string)) +(defun (roots-text roots) + (def lines (list-sort (list-map #(str " (root " (name->string (root-name &)) + " " (quoted (root-dir &)) ")") + roots))) + (str "(roots\n" (string-join (list->vec lines) "\n") ")\n")) + +;; Writes the roots file, and answers whether it had to. +;; +;; **Only when the contents change.** The compiler records the roots file as an +;; input of every build made against it, so rewriting the same bytes with a new +;; timestamp makes every artefact in the project stale. +(: write-roots #:sync (-> string (List Root) bool)) +(defun (write-roots path roots) + (def text (roots-text roots)) + (def current (if (file-exists? path) (file-read-text path) "")) + (cond + ((= current text) #f) + (else + (match (parent-directory path) + ((Some dir) (directory-create dir)) + (None unit)) + (file-write-text path text) + #t))) + + +;; --------------------------------------------------------------------------- +;; The frameworks file +;; --------------------------------------------------------------------------- +;; +;; Which package declared which .NET shared framework, for the compiler's +;; `--frameworks`. One line per pair, two TAB-separated fields: the package's +;; own directory — the one its `manifest.bjodat` is in, not its `src` — and the +;; framework's name. +;; +;; A package's directory rather than its module root, because that is what the +;; compiler compares a source file against, and because a dependency fetched +;; into `.bjo/pkg` sits *under* the root project: the innermost directory +;; containing the file is the package the file belongs to, and it has to be the +;; package rather than a directory inside it. +;; +;; The declarations are read from each root's own manifest rather than carried +;; out of the resolver: a root's directory is `/src`, its manifest is +;; beside it, and reading a handful of small files is cheaper than a second +;; answer to "what did this package say" that could disagree with the first. + +;; The package directory a root's modules live under. +(: (-> Root (Option string))) +(defun (root-package-dir r) + (parent-directory (root-dir r))) + +;; The manifest of the package a root belongs to, or `None` when it has none. +;; +;; A manifest that does not read is also `None`. It is not this function's to +;; report: whatever is about to build against it says so, with the reason. +(: (-> Root (Option Package))) +(defun (root-package r) + (def* ((Some dir) (root-package-dir r) :propagate) + (manifest-file (path-combine dir manifest-name))) + (if (not (file-exists? manifest-file)) + None + (result->option (bjodat-parse-Package (file-read-text manifest-file))))) + +;; What one root's package declares, or nothing when it has no manifest to say +;; it in — which is what a root without one means. +(: (-> Root (List string))) +(defun (root-frameworks r) + (match (root-package r) + (None Nil) + ((Some package) (record-ref package frameworks)))) + +;; The file's text, sorted for the same reason the roots file is: a diff shows +;; what changed rather than what was reordered. +(: (-> (List Root) string)) +(defun (frameworks-text roots) + (def lines + (loop (:for r roots) + (:acc out (folding Nil + (match (root-package-dir r) + (None out) + ((Some dir) + (loop (:for name (root-frameworks r)) + (:acc inner (folding out (Cons (str dir "\t" name) inner))) + => inner))))) + => out)) + (if (empty? lines) + "" + (str (string-join (list->vec (list-sort lines)) "\n") "\n"))) + +;; Writes the frameworks file, and answers whether the build has one at all. +;; +;; No package declaring anything means no file and no flag, so a project that +;; uses no framework is compiled by exactly the command line it was before. +;; Written only when the contents change, for the reason `write-roots` gives: +;; the compiler records it as an input of every module built against it. +(: write-frameworks #:sync (-> string (List Root) bool)) +(defun (write-frameworks path roots) + (def text (frameworks-text roots)) + (def current (if (file-exists? path) (file-read-text path) "")) + (cond + ((= text "") + (when (file-exists? path) (file-delete path)) + #f) + ((= current text) #t) + (else + (match (parent-directory path) + ((Some dir) (directory-create dir)) + (None unit)) + (file-write-text path text) + #t))) + + +;; --------------------------------------------------------------------------- +;; Sources +;; --------------------------------------------------------------------------- + +;; Every `.bjo` under a directory, at any depth, sorted. +;; +;; A library has no entry to start the compiler from, so this is the list the +;; batch is given. Sorted only so that a failure is reported in the same order +;; twice; the compiler builds imports on demand and does not need the order. +(: (-> string (List string))) +(defun (sources-under dir) + (list-sort (collect-sources dir))) + +(: (-> string (List string))) +(defun (collect-sources dir) + (if (not (directory-exists? dir)) + Nil + (let ((here (loop (:for file (directory-files dir)) + (:acc out (listing file) #:when (string-ends-with? file ".bjo")) + => out))) + (list-append here + (loop (:for sub (directory-subdirectories dir)) + (:acc out (folding Nil (list-append out (collect-sources sub)))) + => out))))) + + +;; --------------------------------------------------------------------------- +;; init +;; --------------------------------------------------------------------------- + +;; Writes a file that was not there, and says so. A file that is there is left +;; alone: `init` in a directory someone has already written in is a command +;; that adds what is missing, not one that replaces what is not. +(: write-new #:sync (-> string string bool)) +(defun (write-new path contents) + (cond + ((file-exists? path) #f) + (else + (match (parent-directory path) + ((Some dir) (directory-create dir)) + (None unit)) + (file-write-text path contents) + #t))) + +(: (-> (List Symbol) string)) +(defun (manifest-template name) + (str "(package\n" + " (name " (name->string name) ")\n" + " (version \"0.1.0\")\n" + "\n" + " ;; What this package needs. One (package ...) clause each:\n" + " ;;\n" + " ;; (depends\n" + " ;; (package (name (bjorsec))\n" + " ;; (version (version-at-least \"0.1\"))\n" + " ;; (source (git (url \"https://github.com/bjoli/bjorsec\")))))\n" + " ;;\n" + " ;; A package on disk, instead of one to fetch:\n" + " ;;\n" + " ;; (depends\n" + " ;; (package (name (bjorsec)) (source (path (dir \"../bjorsec\")))))\n" + " )\n")) + +;; The name without its parentheses, for prose. `name->string` is how an +;; import spells a name, which is not how a greeting does. +(: (-> (List Symbol) string)) +(defun (plain-name name) + (string-join (list->vec (list-map symbol->string name)) " ")) + +(: (-> (List Symbol) string)) +(defun (main-template name) + (str ";; " (name->string name) " — the program.\n" + "\n" + "(import (std prelude))\n" + "\n" + "(defun (main)\n" + " (println \"hello from " (plain-name name) "\")\n" + " 0)\n")) + +(: (-> (List Symbol) string)) +(defun (core-template name) + (str ";; " (name->string name) " — imported as " (name->string (list-append name (Cons (string->symbol "core") Nil))) ".\n" + "\n" + "(import (std prelude))\n" + "(export greet)\n" + "\n" + "(: greet (-> string string))\n" + "(defun (greet who) (str \"hello, \" who))\n")) + +;; `.bjo` holds everything fetched and everything generated, `publish` is where +;; `bjo publish` writes by default, and the rest is what the compiler writes +;; beside a source: one artefact, its debug symbols, the two files the runtime +;; needs to start an executable, and the build record. +(: string) +(def gitignore-template + (str ".bjo/\n" + "publish/\n" + "*.dll\n" + "*.exe\n" + "*.pdb\n" + "*.bjobuild\n" + "*.runtimeconfig.json\n" + "*.deps.json\n")) + +;; Makes a project in the current directory. +;; +;; The name defaults to the directory's, because that is what the user already +;; chose when they made it. It is refused rather than mangled when it is not a +;; name: `bjo init my-app` is one word to type, and a guess at what `my.app` +;; was meant to be is a package name nobody wrote. +(: (-> (List string) bool int)) +(defun (cmd-init arguments lib?) + (with-return ret + (def here (path-absolute (current-directory))) + + (when (file-exists? (path-combine here manifest-name)) + (ret (fail #"bjo init: there is already a ${manifest-name} here."))) + + ;; `--lib` is read here as well as among the leading flags: flag parsing + ;; stops at the first thing that is not one, so `bjo init mylib --lib` + ;; would otherwise be a name and a second argument. + (def* ((None) (find #(and (string-starts-with? & "-") (not (= & "--lib"))) arguments) + :leave ((Some flag) (fail #"bjo init: unknown flag '${flag}'."))) + (library? (or lib? (any? #(= & "--lib") arguments))) + (names (loop (:for a arguments) + (:acc out (listing a) #:when (not (string-starts-with? a "-"))) + => out))) + + (def written + (match names + (Nil (match (path-filename here) + ((Some dir-name) dir-name) + (None ""))) + ((Cons only Nil) only) + (_ ""))) + + (when (= written "") + (ret (fail "bjo init: takes one name, or none to use the directory's."))) + + (def (Ok name) (parse-package-name written) + :leave ((Err why) (fail why))) + + (when (reserved-name? name) + (ret (fail (str #"bjo init: ${(name->string name)} is part of the standard library, " + "so a package cannot be called that. Pick another name.")))) + + (ignore (write-new (path-combine here manifest-name) (manifest-template name))) + (ignore (write-new (path-combine here ".gitignore") gitignore-template)) + (directory-create (path-combine here "tests")) + + (if library? + (ignore (write-new (path-combine here "src" "core.bjo") (core-template name))) + (ignore (write-new (path-combine here "src" "main.bjo") (main-template name)))) + + (println #"Created ${(name->string name)} in ${here}") + 0)) +;; bjo/publish.bjo — a program and everything it loads, in one directory. +;; +;; A built program loads its modules, the runtime and its NuGet packages from +;; wherever they were when it was built, by absolute path. A published one +;; carries copies of them and loads only those: the compiler's resolver reads +;; `.bjolinks` beside the program, when there is one, and loads +;; nothing that the table does not name. +;; +;; The layout keeps each file's place relative to its neighbours, because some +;; code finds files from its own `.dll` — a package's resources, or the +;; standard library's `lib/` from `(janitor modules)`: +;; +;; /.exe and its runtimeconfig, deps and pdb +;; / its Native AOT executable, when it has one +;; /.bjolinks assembly and native library -> path +;; /lib/... standard library modules, as under lib/ +;; /runtime/... the Bjolang runtime assemblies +;; /app/... the project's own modules and resources +;; /packages//... each dependency, as under its directory +;; /external/... NuGet packages, as below the NuGet +;; cache; anything else below the deepest +;; directory it shares with the rest + +(export PublishedPackage Publication publish published-marker) + +(import (std prelude)) + +(import/extern + (clr-relative-path (: System.IO.Path.GetRelativePath (-> string string string))) + (clr-path-rooted? (: System.IO.Path.IsPathRooted (-> string bool))) + (clr-directory-of (: System.IO.Path.GetDirectoryName (-> string string))) + (clr-file-name (: System.IO.Path.GetFileName (-> string string))) + (clr-file-stem (: System.IO.Path.GetFileNameWithoutExtension (-> string string))) + (clr-change-ext (: System.IO.Path.ChangeExtension (-> string string string))) + (clr-file-copy (: System.IO.File.Copy (-> string string bool void))) + (error-message (: System.Exception.Message (-> Exception string) #:get))) + +;; A package the program was built from: its name, the directory its manifest +;; is in, the resources that manifest declares, and whether it is the project +;; being published, whose files go under `app/`. +(type (: PublishedPackage (Record (: name (List Symbol)) + (: dir string) + (: resources (List string)) + (: own bool)))) + +;; What to publish. `links` and `natives` are the build record's `link` and +;; `native` lines: a name or key, and the file's path. `native-program` is the +;; program's Native AOT executable, or "" when it has none. +;; +;; `whole-stdlib` copies all of the standard library, sources and documentation +;; included, rather than only the modules the program links. A program that +;; reads Bjolang's own modules at run time, such as one that shows their +;; documentation, needs that. +(type (: Publication (Record (: program string) + (: native-program string) + (: links (List (Tuple string string))) + (: natives (List (Tuple string string))) + (: packages (List PublishedPackage)) + (: lib (Option string)) + (: whole-stdlib bool) + (: runtime string) + (: out string)))) + +;; Written into every output directory. An existing directory is only replaced +;; when it has this file, so a mistyped `-o` cannot delete something else. +(: string) +(def published-marker ".bjopublish") + +;; --------------------------------------------------------------------------- +;; Where a file goes +;; --------------------------------------------------------------------------- + +;; `path` relative to `dir`, or `None` when it is not below it. +(: (-> string string (Option string))) +(defun (below dir path) + (def rel (clr-relative-path (path-absolute dir) (path-absolute path))) + (if (or (= rel ".") (string-starts-with? rel "..") (clr-path-rooted? rel)) + None + (Some rel))) + +(: (-> PublishedPackage string)) +(defun (package-prefix p) + (if (record-ref p own) + "app" + (path-combine "packages" + (string-join (list->vec (list-map symbol->string (record-ref p name))) ".")))) + +;; The package whose directory holds `path`, and the path relative to it. The +;; deepest directory wins, because fetched dependencies live in `.bjo/pkg` +;; inside the project's own directory. +(: (-> (List PublishedPackage) string (Option (Tuple PublishedPackage string)))) +(defun (package-of packages path) + (loop (:for p packages) + (:when-let (Some rel) (below (record-ref p dir) path)) + (:acc best (folding None + (match best + ((Some (Tuple q _)) #:when (> (string-length (record-ref q dir)) + (string-length (record-ref p dir))) + best) + (_ (Some (Tuple p rel)))))) + => best)) + +;; Where `path` goes in the output, when it is under the standard library, +;; the runtime or a package. `None` for anything else. +(: (-> Publication string (Option string))) +(defun (known-destination p path) + (def in-lib (match (record-ref p lib) + ((Some lib) (below lib path)) + (None None))) + (match in-lib + ((Some rel) (Some (path-combine "lib" rel))) + (None + (match (below (record-ref p runtime) path) + ((Some rel) (Some (path-combine "runtime" rel))) + (None + (match (package-of (record-ref p packages) path) + ((Some (Tuple package rel)) (Some (path-combine (package-prefix package) rel))) + (None None))))))) + +;; The NuGet cache, where `dotnet` puts packages. +(: (-> string)) +(defun (nuget-packages-dir) + (match (get-environment-variable "NUGET_PACKAGES") + ((Some dir) #:when (not (= dir "")) dir) + (_ (path-combine (option-ref-or (get-environment-variable "HOME") "/") ".nuget" "packages")))) + +;; The path of a NuGet package's file below its store, or `None` for a file +;; that is in neither store: +;; - the NuGet cache +;; - the compiler's ReadyToRun cache, `.../bjolang/r2r///`, which has the same layout. See `Build.precompilePackages`. +;; A package's assemblies are in the second and its native libraries in the +;; first, so both are laid out as the NuGet cache is. +(: (-> string (Option string))) +(defun (package-store-path path) + (match (below (nuget-packages-dir) path) + ((Some rel) (Some rel)) + (None + (let go ((parts (vec->list (string-split (path-absolute path) "/")))) + (match parts + ((Cons "bjolang" (Cons "r2r" (Cons _ (Cons _ (Cons first rest))))) + (Some (string-join (list->vec (Cons first rest)) "/"))) + ((Cons _ more) (go more)) + (Nil None)))))) + +;; The longest list both start with. +(: (-> (List string) (List string) (List string))) +(defun (common-prefix a b) + (match (Tuple a b) + ((Tuple (Cons x xs) (Cons y ys)) #:when (= x y) (Cons x (common-prefix xs ys))) + (_ Nil))) + +;; The deepest directory that holds every one of `paths`. +(: (-> (List string) string)) +(defun (common-directory paths) + (match (list-map (fun (path) (vec->list (string-split (clr-directory-of (path-absolute path)) "/"))) + paths) + (Nil "/") + ((Cons first rest) + (def dir (string-join (list->vec (list-foldl common-prefix first rest)) "/")) + (if (= dir "") "/" dir)))) + +;; --------------------------------------------------------------------------- +;; Copying +;; --------------------------------------------------------------------------- + +;; Copies `from` to `rel` under `out`, making the directories on the way. +(: copy-to! #:sync (-> string string string void)) +(defun (copy-to! out rel from) + (def to (path-combine out rel)) + (directory-create (clr-directory-of to)) + (file-copy from to #:mode Truncate)) + +;; Copies a native executable. Through .NET's `File.Copy`, which keeps the file's +;; mode on Unix: a copy made by `file-copy` could not be run. +(: copy-executable! #:sync (-> string string string void)) +(defun (copy-executable! out rel from) + (def to (path-combine out rel)) + (directory-create (clr-directory-of to)) + (clr-file-copy from to #t)) + +;; Copies an assembly, and its `.pdb` when there is one, so that a stack trace +;; from the published program still names the source. +(: copy-assembly! #:sync (-> string string string void)) +(defun (copy-assembly! out rel from) + (copy-to! out rel from) + (def pdb (clr-change-ext from ".pdb")) + (when (file-exists? pdb) + (copy-to! out (clr-change-ext rel ".pdb") pdb))) + +;; Copies a directory and everything in it, leaving out the files `skip?` +;; answers true for. +(: copy-tree! #:sync (-> string string (-> string bool) void)) +(defun (copy-tree! from to skip?) + (directory-create to) + (loop (:for f (directory-files from)) + (:when (not (skip? f))) + (:do (file-copy f (path-combine to (clr-file-name f)) #:mode Truncate))) + (loop (:for d (directory-subdirectories from)) + (:do (copy-tree! d (path-combine to (clr-file-name d)) skip?)))) + +;; The standard library is copied with its sources and documentation, but not +;; its build records, which name this machine's paths and which nothing reads +;; in a published program. +(: (-> string bool)) +(defun (build-record? f) (string-ends-with? f ".bjobuild")) + +(: (-> string bool)) +(defun (nothing-skipped f) #f) + +;; Copies a resource, a file or a directory, to `rel` under `out`. +(: copy-resource! #:sync (-> string string string void)) +(defun (copy-resource! out rel from) + (if (directory-exists? from) + (copy-tree! from (path-combine out rel) nothing-skipped) + (copy-to! out rel from))) + +;; How many files are under `dir`, at any depth. +(: count-files #:sync (-> string int)) +(defun (count-files dir) + (+ (length (directory-files dir)) + (loop (:for d (directory-subdirectories dir)) + (:acc n (summing (count-files d))) + => n))) + +;; Empties the output directory, or makes it. A directory that is there and +;; holds files is replaced only when an earlier publish made it. Answers why +;; it refused, or `None` when the directory is ready. +(: prepare-output! #:sync (-> string (Option string))) +(defun (prepare-output! out) + (cond + ((not (directory-exists? out)) + (directory-create out) + None) + ((file-exists? (path-combine out published-marker)) + (directory-delete-tree out) + (directory-create out) + None) + ((and (empty? (directory-files out)) (empty? (directory-subdirectories out))) + None) + (else + (Some (str "bjo publish: " out " is not empty, and bjo publish did not make it. " + "Give an empty or new directory with -o."))))) + +;; Why a declared resource cannot be published, or nothing when it can. +(: (-> PublishedPackage string (Option string))) +(defun (resource-problem package resource) + (def shown (str "the resource \"" resource "\" of " (record-ref package dir))) + (def from (path-combine (record-ref package dir) resource)) + (cond + ((or (clr-path-rooted? resource) + (any? #(= & "..") (string-split (string-replace resource "\\" "/") "/"))) + (Some (str "bjo publish: " shown " is not a path inside the package. " + "A resource is written relative to the package's directory."))) + ((not (or (file-exists? from) (directory-exists? from))) + (Some (str "bjo publish: " shown " is not there."))) + (else None))) + +;; Every resource problem, across all the packages. +(: (-> (List PublishedPackage) (List string))) +(defun (resource-problems packages) + (loop (:for package packages) + (:subloop) + (:for resource (record-ref package resources)) + (:when-let (Some why) (resource-problem package resource)) + (:acc out (listing why)) + => out)) + +;; --------------------------------------------------------------------------- +;; Publishing +;; --------------------------------------------------------------------------- + +;; Lays out the program in `out`, and answers how many files the output holds. +;; +;; Every resource is checked before anything is written, so that a manifest +;; naming one that is not there leaves the output as it was. +(: publish #:sync (-> Publication (Result string int))) +(defun (publish p) + (with-return ret + (when (empty? (record-ref p links)) + (ret (Err (str "bjo publish: the build record of " (record-ref p program) + " names nothing the program loads. Rebuild it with -c and publish again.")))) + + (match (resource-problems (record-ref p packages)) + ((Cons why _) (ret (Err why))) + (Nil unit)) + + (def out (path-absolute (record-ref p out))) + (match (prepare-output! out) + ((Some why) (ret (Err why))) + (None unit)) + + (def files (list-append (list-map (fun (link) (match link ((Tuple name path) (Tuple "link" name path)))) + (record-ref p links)) + (list-map (fun (native) (match native ((Tuple key path) (Tuple "native" key path)))) + (record-ref p natives)))) + (def external-root + (common-directory (loop (:for (Tuple _ _ path) files) + (:acc paths (listing path) #:when (and (not (some? (known-destination p path))) + (not (some? (package-store-path path))))) + => paths))) + (defun (destination path) + (match (known-destination p path) + ((Some rel) rel) + (None (path-combine "external" + (match (package-store-path path) + ((Some rel) rel) + (None (match (below external-root path) + ((Some rel) rel) + (None (clr-file-name path))))))))) + + (def program (record-ref p program)) + + (def copied + (try + (let () + (loop (:for companion (list program + (clr-change-ext program ".runtimeconfig.json") + (clr-change-ext program ".deps.json") + (clr-change-ext program ".pdb"))) + (:when (file-exists? companion)) + (:do (copy-to! out (clr-file-name companion) companion))) + (def native (record-ref p native-program)) + (when (and (not (= native "")) (file-exists? native)) + (copy-executable! out (clr-file-name native) native)) + (match (record-ref p lib) + ((Some lib) #:when (record-ref p whole-stdlib) + (copy-tree! lib (path-combine out "lib") build-record?)) + (_ unit)) + (loop (:for (Tuple _ _ path) files) + (:do (copy-assembly! out (destination path) path))) + (loop (:for package (record-ref p packages)) + (:subloop) + (:for resource (record-ref package resources)) + (:do (copy-resource! out + (path-combine (package-prefix package) resource) + (path-combine (record-ref package dir) resource)))) + ;; Relative to the program, with `/`, which .NET accepts on every + ;; platform, so that the table means the same wherever the directory + ;; is copied to. + (def table + (loop (:for (Tuple kind name path) files) + (:acc lines (listing (str kind "\t" name "\t" (string-replace (destination path) "\\" "/")))) + => lines)) + (file-write-text (path-combine out (str (clr-file-stem program) ".bjolinks")) + (str (string-join (list->vec table) "\n") "\n")) + (file-write-text (path-combine out published-marker) + "Made by bjo publish. Publishing again replaces this directory.\n") + (count-files out)) + #:catch (System.Exception))) + + (result-map-err (fun (e) (str "bjo publish: " (error-message e))) copied))) +;; bjo/resolve.bjo — what a project builds against. +;; +;; One entry point, `prepare`: it turns a project's manifest into the list of +;; packages the compiler is given, fetches whatever that needs, and writes +;; `.bjo/roots`. Every command that compiles anything inside a project goes +;; through it, so that `bjo run` and `bjo check` see the same packages. +;; +;; The walk +;; -------- +;; A manifest names dependencies; each of those has a manifest of its own that +;; names more. The walk reads them breadth first, and a package is read once. +;; +;; A dependency may be written with no `source`, which means "somebody else in +;; this graph says where this comes from". So the walk cannot simply follow +;; edges: it keeps the dependencies whose source is still unknown in a waiting +;; list and retries them whenever it has read another manifest. A whole pass +;; that satisfies none of them is the end — and then a name with no source +;; anywhere is an error naming it. +;; +;; Two rules decide a source when there are several: +;; +;; - the root manifest wins. That is how a user overrides a source a +;; dependency chose: write the name in your own manifest with the source +;; you want, and every manifest that asked for that name gets yours. +;; - otherwise every source given for one name must be the same, compared as +;; URLs with a trailing `/` and `.git` removed. Two different ones is an +;; error listing both and which manifest gave each, because picking either +;; silently would build code the other manifest never asked for. + +(export + Prepared prepared-roots-file prepared-frameworks-file prepared-nuget-dir + prepared-runtime + prepared-aot prepared-aot-cpu + prepared-roots prepared-dependencies + prepare) + +(import (std prelude)) +(import (text bjodat)) +(import "manifest.bjo") +(import "project.bjo") +(import "lock.bjo") +(import "git.bjo") +(import "nuget.bjo") + +;; What a project turned out to need: the roots file to hand the compiler, and +;; the packages it names. +;; `frameworks-file` is `None` when no package in the graph declared a .NET +;; shared framework, which is the ordinary case: then no file is written and no +;; flag is passed, and the build is exactly the build it was before. +;; `nuget-dir` is `None` the same way, when the manifest names no NuGet +;; packages. +;; `runtime` is the root manifest's runtime settings, as `name=value`, and +;; `aot` whether it asks for a Native AOT build, and `aot-cpu` the CPU that +;; build is for, or "" for the baseline. +(type (: Prepared (Record (: roots-file string) + (: frameworks-file (Option string)) + (: nuget-dir (Option string)) + (: runtime (List string)) + (: aot bool) + (: aot-cpu string) + (: roots (List Root))))) + +;; A manifest that has been read and whose dependencies have not been walked. +;; +;; `path-ok?` travels with it: a `(path ...)` source points at a directory on +;; the author's disk, which is a fact about a checkout rather than about a +;; package, so it may only be written by a manifest that was itself reached +;; through path sources. +(type (: Task (Record (: package Package) + (: dir string) + (: from string) + (: path-ok? bool) + (: dev? bool)))) + +;; A dependency that has been seen and cannot be placed yet, because nothing +;; has said where it comes from. +(type (: Want (Record (: dep Dep) + (: want-from string) + (: want-dir string) + (: want-path-ok? bool)))) + +;; A source, and the manifest that gave it. +(type (: Given (Record (: given Source) (: given-from string) (: given-root? bool)))) + +;; Somebody's constraint on a package's version, for the check after the +;; versions have been chosen. +(type (: Need (Record (: need-name (List Symbol)) + (: need-req (Option VersionReq)) + (: need-from string)))) + +;; A git package that has been chosen: everything the lock and the roots file +;; need to say about it. +(type (: Chosen (Record (: chosen-name (List Symbol)) + (: chosen-version Version) + (: chosen-source Source) + (: chosen-clone string)))) + +;; The walk's state. Boxes rather than a threaded record: every step reads what +;; every earlier step decided, and a name is placed once however many manifests +;; asked for it. +;; +;; A path package is placed as it is found — there is one directory and nothing +;; to choose. A git package is not: every requirement on it makes a *node*, a +;; version that some manifest asked for, and which of them is used is only +;; known when the walk is over. See `chosen` below. +(type (: Walk (Record (: sources (Box (Map string Given))) + (: placed (Box (Map string Root))) + (: needs (Box (List Need))) + (: nodes (Box (Map string (List Version)))) + (: names (Box (Map string (List Symbol)))) + (: clones (Box (Map string string))) + (: origins (Box (Map string Source))) + (: walked (Box (Map string bool))) + (: fetched (Box (Map string bool))) + (: own (List Symbol)) + (: own-from string) + (: tool Tool) + (: dot-bjo string) + (: was-locked (List Locked))))) + +(: (-> Prepared string)) +(defun (prepared-roots-file p) (record-ref p roots-file)) + +(: (-> Prepared (Option string))) +(defun (prepared-frameworks-file p) (record-ref p frameworks-file)) + +(: (-> Prepared (Option string))) +(defun (prepared-nuget-dir p) (record-ref p nuget-dir)) + +(: (-> Prepared (List string))) +(defun (prepared-runtime p) (record-ref p runtime)) + +(: (-> Prepared bool)) +(defun (prepared-aot p) (record-ref p aot)) + +(: (-> Prepared string)) +(defun (prepared-aot-cpu p) (record-ref p aot-cpu)) + +(: (-> Prepared (List Root))) +(defun (prepared-roots p) (record-ref p roots)) + +;; How many packages other than the project itself. +(: (-> Prepared int)) +(defun (prepared-dependencies p) (- (length (prepared-roots p)) 1)) + + +;; --------------------------------------------------------------------------- +;; Names and sources, as text +;; --------------------------------------------------------------------------- + +;; A package name is the key everywhere here: two names are the same name when +;; they are written the same, and `(a b)` is a string a map can hold. +(: (-> (List Symbol) string)) +(defun (key-of name) (name->string name)) + +(: (-> Source string)) +(defun (source-shown s) + (match s + ((FromGit g) (str "(git (url \"" (record-ref g url) "\"))")) + ((FromPath p) (str "(path (dir \"" (record-ref p dir) "\"))")) + ((FromRegistry r) (str "(registry (host \"" (record-ref r host) "\"))")))) + +(: (-> string string string)) +(defun (without-suffix s suffix) + (if (not (string-ends-with? s suffix)) + s + (let go ((c (string-cursor-end s)) (left (string-count suffix))) + (if (= left 0) + (substring s (string-cursor-start s) c) + (go (string-cursor-prev s c) (- left 1)))))) + +;; What two sources are compared as. A repository is the same repository under +;; `…/x`, `…/x/` and `…/x.git`, and a manifest that spells it either way means +;; the same place. +(: (-> Source string)) +(defun (source-key s) + (match s + ((FromGit g) (str "git " (without-suffix (without-suffix (record-ref g url) "/") ".git"))) + ((FromPath p) (str "path " (record-ref p dir))) + ((FromRegistry r) (str "registry " (record-ref r host))))) + + +;; --------------------------------------------------------------------------- +;; Reading a dependency's manifest +;; --------------------------------------------------------------------------- + +;; The manifest of a package that was found on disk, checked against the name +;; it was asked for. +;; +;; A directory that answers to another name is refused rather than used: the +;; roots file would map the name that was asked for onto these modules, and +;; every module in it would then compile under a name its own author never +;; wrote. +(: (-> (List Symbol) string string (Result string Package))) +(defun (package-in-directory wanted dir asked-by) + (def manifest-file (path-combine dir manifest-name)) + (cond + ((not (directory-exists? dir)) + (Err (str "bjo: " asked-by " wants " (name->string wanted) " from " dir + ", which is not a directory."))) + ((not (file-exists? manifest-file)) + (Err (str "bjo: " asked-by " wants " (name->string wanted) " from " dir + ", which has no " manifest-name "."))) + (else + (def (Ok package) (read-package manifest-file) :propagate) + (if (not (= (key-of (record-ref package name)) (key-of wanted))) + (Err (str "bjo: " asked-by " wants " (name->string wanted) ", but " + manifest-file " says the package there is " + (name->string (record-ref package name)) ".")) + (Ok package))))) + + +;; --------------------------------------------------------------------------- +;; The walk +;; --------------------------------------------------------------------------- + +;; Records the source a manifest gave for a name. +;; +;; The root's wins and is never overwritten; anyone else's has to agree with +;; what is already there. +(: (-> Walk (List Symbol) Source string bool (Result string Unit))) +(defun (give-source! walk name source from root?) + (def k (key-of name)) + (def given (Given (given source) (given-from from) (given-root? root?))) + (match (map-try-ref (box-ref (record-ref walk sources)) k) + (None + (box-set! (record-ref walk sources) + (map-set (box-ref (record-ref walk sources)) k given)) + (Ok unit)) + ((Some already) + (cond + ;; The root said where this comes from, so nothing else is consulted. + ((record-ref already given-root?) (Ok unit)) + (root? + (box-set! (record-ref walk sources) + (map-set (box-ref (record-ref walk sources)) k given)) + (Ok unit)) + ((= (source-key (record-ref already given)) (source-key source)) (Ok unit)) + (else + (Err (str "bjo: " (name->string name) " is given two different sources.\n" + " " (source-shown (record-ref already given)) + " in " (record-ref already given-from) "\n" + " " (source-shown source) " in " from "\n" + " Name it in " (record-ref walk own-from) + " with the source you want: the root manifest's wins everywhere."))))))) + +;; What a dependency clause has to satisfy before anything is fetched for it. +(: (-> Walk Dep string (Result string Unit))) +(defun (check-dep walk dep from) + (def name (record-ref dep name)) + (cond + ((empty? name) (Err (str "bjo: " from ": a dependency with no name."))) + ((reserved-name? name) + (Err (str "bjo: " from ": " (name->string name) + " is part of the standard library, so it cannot be a dependency." + " It is there already."))) + ((= (key-of name) (key-of (record-ref walk own))) + (Err (str "bjo: " from ": " (name->string name) + " is this project itself, so it cannot be one of its dependencies."))) + (else (Ok unit)))) + +;; Every dependency of one manifest: its clauses checked, its sources recorded, +;; and one `Want` each for the step that places them. +(: (-> Walk Task (Result string (List Want)))) +(defun (dependencies-of walk task) + (with-return ret + (def package (record-ref task package)) + (def from (record-ref task from)) + (def clauses (if (record-ref task dev?) + (list-append (record-ref package depends) + (record-ref package dev-depends)) + (record-ref package depends))) + (Ok (loop (:for dep clauses) + (:do + (match (check-dep walk dep from) + ((Err why) (ret (Err why))) + ((Ok u) unit)) + (box-set! (record-ref walk needs) + (Cons (Need (need-name (record-ref dep name)) + (need-req (record-ref dep version)) + (need-from from)) + (box-ref (record-ref walk needs)))) + (match (record-ref dep source) + (None unit) + ((Some source) + (match (give-source! walk (record-ref dep name) source from + (record-ref task dev?)) + ((Err why) (ret (Err why))) + ((Ok u) unit))))) + (:acc out (listing (Want (dep dep) + (want-from from) + (want-dir (record-ref task dir)) + (want-path-ok? (record-ref task path-ok?))))) + => out)))) + +;; Places one dependency, now that its source is known: the package's directory +;; is decided, its manifest read, and a task made of it. +;; +;; `None` when the package is already placed. A name is one directory in the +;; roots file, so the second manifest that asks for it adds a requirement and +;; nothing else. +(: place #:sync (-> Walk Want Given (Result string (Option Task)))) +(defun (place walk want given) + (with-return ret + (def dep (record-ref want dep)) + (def name (record-ref dep name)) + (def k (key-of name)) + (def asked-by (record-ref want want-from)) + + (when (some? (map-try-ref (box-ref (record-ref walk placed)) k)) + (ret (Ok None))) + + (match (record-ref given given) + ((FromRegistry r) + (Err (str "bjo: " asked-by " wants " (name->string name) + " from a registry, and there are none yet." + " Use a git source, or a path source for a package on this disk."))) + + ((FromPath local) + ;; Relative to the manifest that named it, never to the working + ;; directory: the same manifest has to mean the same directory whoever + ;; runs `bjo` and from wherever. + (def dir (path-absolute (path-combine (record-ref want want-dir) (record-ref local dir)))) + (unless (record-ref want want-path-ok?) + (ret (Err (str "bjo: " asked-by " gives " (name->string name) + " a path source, " (source-shown (record-ref given given)) + ". A fetched package may not: the directory is on its author's" + " disk and says nothing about yours." + " Give " (name->string name) " a source in " + (record-ref walk own-from) " instead.")))) + (def (Ok package) (package-in-directory name dir asked-by) :propagate) + (def src (path-combine dir "src")) + (unless (directory-exists? src) + (ret (Err (str "bjo: " (name->string name) " at " dir " has no src/." + " A package keeps its modules in src/, beside its manifest.")))) + (box-set! (record-ref walk placed) + (map-set (box-ref (record-ref walk placed)) k + (Root (name name) (dir src)))) + (Ok (Some (Task (package package) + (dir dir) + (from (path-combine dir manifest-name)) + (path-ok? #t) + (dev? #f))))) + + ((FromGit source) + (def (Ok made) (place-git walk want source) :propagate) + (Ok made))))) + +;; A git dependency: the node this requirement makes, and the manifest at that +;; version. +;; +;; The manifest is read out of the bare clone, so a version that is considered +;; and then superseded costs a `git show` and nothing else — no checkout, no +;; directory, no entry in the lock. +(: place-git #:sync (-> Walk Want Git (Result string (Option Task)))) +(defun (place-git walk want source) + (with-return ret + (def dep (record-ref want dep)) + (def name (record-ref dep name)) + (def k (key-of name)) + (def asked-by (record-ref want want-from)) + (def url (record-ref source url)) + + ;; Refused rather than ignored: a manifest that pins a commit is asking for + ;; something this does not do, and building the tag instead would be a + ;; different program from the one it asked for. + (when (some? (record-ref source commit)) + (ret (Err (str "bjo: " asked-by " pins " (name->string name) + " to a commit, which bjo does not do yet." + " Use a version tag, or a path source for a copy on this disk.")))) + (when (some? (record-ref source branch)) + (ret (Err (str "bjo: " asked-by " pins " (name->string name) + " to a branch, which bjo does not do yet." + " Use a version tag, or a path source for a copy on this disk.")))) + + (def (Ok clone) (ensure-clone (record-ref walk tool) (record-ref walk dot-bjo) url name) + :propagate) + (remember! walk k name (FromGit source) clone) + + (def (Ok required) (requirement-of walk dep asked-by clone) :propagate) + (def (Ok version) (node-version walk clone name (lower-bound required) asked-by) :propagate) + (add-node! walk k version) + + ;; Every node is walked once. Two manifests asking for the same version ask + ;; for the same manifest, and its requirements are already in hand. + (def node-key (str k "@" (version->string version))) + (when (some? (map-try-ref (box-ref (record-ref walk walked)) node-key)) + (ret (Ok None))) + (box-set! (record-ref walk walked) + (map-set (box-ref (record-ref walk walked)) node-key #t)) + + (def (Ok text) (manifest-at (record-ref walk tool) clone version) :propagate) + (def shown (str url " " (version->tag version))) + (def (Ok package) (package-at-tag name text shown) :propagate) + + (Ok (Some (Task (package package) + (dir clone) + (from shown) + (path-ok? #f) + (dev? #f)))))) + +;; A manifest read at a tag, checked against the name it was asked for. +;; +;; A version whose manifest says another name is refused: the roots file would +;; file these modules under the name that was asked for, and every one of them +;; would then compile under a name its author never wrote. +(: (-> (List Symbol) string string (Result string Package))) +(defun (package-at-tag wanted text shown) + (match (bjodat-parse-Package text) + ((Err why) (Err (str "bjo: " manifest-name " at " shown ": " why))) + ((Ok package) + (if (not (= (key-of (record-ref package name)) (key-of wanted))) + (Err (str "bjo: " shown " was asked for as " (name->string wanted) + ", but its " manifest-name " says the package there is " + (name->string (record-ref package name)) ".")) + (Ok package))))) + +;; What a name is called, where it comes from and where its clone is. Written +;; down when the first requirement on it is seen, and read again when the +;; chosen version is materialised and locked. +(: (-> Walk string (List Symbol) Source string Unit)) +(defun (remember! walk k name source clone) + (box-set! (record-ref walk names) (map-set (box-ref (record-ref walk names)) k name)) + (box-set! (record-ref walk origins) (map-set (box-ref (record-ref walk origins)) k source)) + (box-set! (record-ref walk clones) (map-set (box-ref (record-ref walk clones)) k clone)) + unit) + +(: (-> Walk string Version Unit)) +(defun (add-node! walk k version) + (def had (match (map-try-ref (box-ref (record-ref walk nodes)) k) + ((Some versions) versions) + (None Nil))) + (box-set! (record-ref walk nodes) + (map-set (box-ref (record-ref walk nodes)) k (Cons version had))) + unit) + +;; A git dependency has to say which versions it will take. +;; +;; There is no default. "Whatever is newest" is a manifest that means something +;; different every week, and "whatever is oldest" is one that never gets a +;; fix — so the manifest says, and the tags that exist are listed here because +;; the answer is one of them. +(: requirement-of #:sync (-> Walk Dep string string (Result string VersionReq))) +(defun (requirement-of walk dep asked-by clone) + (match (record-ref dep version) + ((Some req) (Ok req)) + (None + (def (Ok tags) (version-tags (record-ref walk tool) clone) :propagate) + (Err (str "bjo: " asked-by " asks for " (name->string (record-ref dep name)) + " from git without saying which versions it takes.\n" + " Write one, as in (version (version-at-least \"0.1.0\")).\n" + " The versions that repository has: " (versions-shown tags)))))) + +(: (-> (List Version) string)) +(defun (versions-shown versions) + (if (empty? versions) + "none — it has no vX.Y.Z tags at all." + (string-join (list->vec (list-map version->tag versions)) ", "))) + +;; The version a requirement asks for, as minimal version selection reads it: +;; the bottom of the range, and `0.0.0` where there is no bottom. +(: (-> VersionReq Version)) +(defun (lower-bound req) + (match req + ((AtLeast lo) lo) + ((Between lo hi) lo) + ((AtMost hi) (version 0 0 0)))) + +;; The node a requirement makes: the *lowest* version that satisfies it. +;; +;; Not the newest. A manifest that says `(version-at-least "1.2")` says it +;; needs 1.2, not that it has been tested against everything after it — so the +;; build uses 1.2 unless something else in the graph asks for more. That is the +;; whole of minimal version selection, and it is why two builds of the same +;; manifests a year apart use the same code. +(: node-version #:sync (-> Walk string (List Symbol) Version string (Result string Version))) +(defun (node-version walk clone name lower asked-by) + (def (Ok tags) (version-tags (record-ref walk tool) clone) :propagate) + (match (find #(>= (version-compare & lower) 0) tags) + ((Some found) (Ok found)) + (None + ;; Only now: a lower bound nothing here satisfies is the one question the + ;; clone cannot answer out of what it already has. + (def (Ok fetched?) (fetch-once walk clone) :propagate) + (def (Ok after) (version-tags (record-ref walk tool) clone) :propagate) + (match (find #(>= (version-compare & lower) 0) after) + ((Some found) (Ok found)) + (None + (Err (str "bjo: " asked-by " asks for " (name->string name) + " at least " (version->string lower) + ", and that repository has no such version.\n" + " The versions it has: " (versions-shown after)))))))) + +;; Fetches a clone once per run. A resolve asks several questions of one +;; repository, and a fetch that answered the first answers the rest. +(: fetch-once #:sync (-> Walk string (Result string bool))) +(defun (fetch-once walk clone) + (cond + ((some? (map-try-ref (box-ref (record-ref walk fetched)) clone)) (Ok #f)) + (else + (box-set! (record-ref walk fetched) + (map-set (box-ref (record-ref walk fetched)) clone #t)) + (def (Ok got) (fetch-tags (record-ref walk tool) clone) :propagate) + (Ok #t)))) + +;; The dependencies that can be placed now, and the ones that still cannot. +(: (-> Walk (List Want) (Tuple (List Want) (List Want)))) +(defun (ready-and-waiting walk wants) + (def known (box-ref (record-ref walk sources))) + (Tuple (list-filter #(some? (map-try-ref known (key-of (record-ref (record-ref & dep) name)))) wants) + (list-filter #(not (some? (map-try-ref known (key-of (record-ref (record-ref & dep) name))))) wants))) + +;; A name nobody said where to get. +(: (-> Walk Want string)) +(defun (no-source-message walk want) + (def name (record-ref (record-ref want dep) name)) + (str "bjo: " (record-ref want want-from) " wants " (name->string name) + " and no manifest says where it comes from." + " Give it a source: (source (git (url \"https://…\"))) to fetch it," + " or (source (path (dir \"../" (name->string name) "\"))) for a copy on this disk.")) + +;; Places every dependency that can be placed, and answers the tasks they made. +(: place-all #:sync (-> Walk (List Want) (Result string (List Task)))) +(defun (place-all walk wants) + (match wants + (Nil (Ok Nil)) + ((Cons want rest) + (def known (box-ref (record-ref walk sources))) + (def (Ok made) + (match (map-try-ref known (key-of (record-ref (record-ref want dep) name))) + ;; Not this pass. It stays in the waiting list, which is the caller's. + (None (Ok None)) + ((Some given) (place walk want given))) + :propagate) + (def (Ok more) (place-all walk rest) :propagate) + (Ok (match made + ((Some task) (Cons task more)) + (None more)))))) + +;; The whole walk: read a manifest, place what it names, repeat. +;; +;; `waiting` holds the dependencies with no source yet. When the queue empties, +;; the ones that have a source by now are placed and the walk goes on; a pass +;; that places none of them ends it, and whatever is still waiting is an error. +(: walk-graph #:sync (-> Walk (List Task) (List Want) (Result string Unit))) +(defun (walk-graph walk tasks waiting) + (match tasks + ((Cons task rest) + (def (Ok found) (dependencies-of walk task) :propagate) + (def (Tuple ready still) (ready-and-waiting walk (list-append waiting found))) + (def (Ok made) (place-all walk ready) :propagate) + (walk-graph walk (list-append rest made) still)) + (Nil + (if (empty? waiting) + (Ok unit) + (let ((split (ready-and-waiting walk waiting))) + (match split + ((Tuple Nil (Cons first _)) (Err (no-source-message walk first))) + ((Tuple ready still) + (def (Ok made) (place-all walk ready) :propagate) + (walk-graph walk made still)))))))) + + +;; --------------------------------------------------------------------------- +;; The root package +;; --------------------------------------------------------------------------- + +;; The root package's own entry in the roots file. +;; +;; `src/` has to exist: the compiler refuses a roots file naming a directory +;; that is not there, and refusing it here says which package and which +;; directory instead of naming a file the user never wrote. +(: (-> Project (Result string Root))) +(defun (root-package-root project) + (def name (record-ref (project-package project) name)) + (def src (project-src project)) + (cond + ((empty? name) + (Err (str "bjo: " (path-combine (project-dir project) manifest-name) + ": the package has no name."))) + ((reserved-name? name) + (Err (str "bjo: " (path-combine (project-dir project) manifest-name) ": " + (name->string name) + " is part of the standard library, so a package cannot be called that."))) + ((not (directory-exists? src)) + (Err (str "bjo: " (name->string name) " has no " src "." + " A package keeps its modules in src/, beside its manifest."))) + (else (Ok (Root (name name) (dir src)))))) + + +;; --------------------------------------------------------------------------- +;; Choosing versions +;; --------------------------------------------------------------------------- + +;; The version each git package ends up at: the highest of its nodes. +;; +;; No search and no backtracking. Every node is a version some manifest asked +;; for, so the highest of them is the lowest version that satisfies all of +;; them — and since every reachable node counts, including versions that were +;; later superseded, adding a manifest to the graph can raise a version but +;; never lower one. +;; +;; A worked example. The app needs A at least 1.0 and B at least 1.0; B 1.0 +;; needs A at least 1.1; A has tags v1.0.0, v1.1.0 and v1.2.0. +;; +;; the app's (version-at-least "1.0") on A -> node A 1.0.0 +;; the app's (version-at-least "1.0") on B -> node B 1.0.0 +;; B 1.0.0's (version-at-least "1.1") on A -> node A 1.1.0 +;; +;; A's nodes are 1.0.0 and 1.1.0, so A is 1.1.0 — not 1.2.0, which nobody +;; asked for, and not 1.0.0, which B would not work with. Remove B and the +;; next build uses A 1.0.0 again. +(: (-> Walk (List Chosen))) +(defun (choose walk) + (map-fold (fun (k versions found) (Cons (chosen-of walk k versions) found)) + Nil + (box-ref (record-ref walk nodes)))) + +(: (-> Walk string (List Version) Chosen)) +(defun (chosen-of walk k versions) + (Chosen (chosen-name (map-ref (box-ref (record-ref walk names)) k)) + (chosen-version (highest versions)) + (chosen-source (map-ref (box-ref (record-ref walk origins)) k)) + (chosen-clone (map-ref (box-ref (record-ref walk clones)) k)))) + +;; `list-foldl` hands the item first and the accumulator second. +(: (-> (List Version) Version)) +(defun (highest versions) + (list-foldl (fun (v best) (if (> (version-compare v best) 0) v best)) + (version 0 0 0) + versions)) + +;; A constraint as a manifest writes it. +(: (-> (Option VersionReq) string)) +(defun (req-shown req) + (match req + (None "(no version)") + ((Some r) (bjodat->string (->bjodat r))))) + +;; Every upper bound, checked against the version that was chosen. +;; +;; An upper bound is a *check*, never a choice: it cannot make the resolver +;; take a lower version, because a version that is too low for one manifest is +;; a build that does not work at all, while a version that is too high for +;; another is a build the author of that one has not tried. So the graph +;; decides and the bounds say whether the answer is acceptable. +(: (-> Walk Chosen (Result string Unit))) +(defun (check-bounds walk c) + (def name (record-ref c chosen-name)) + (def picked (record-ref c chosen-version)) + (def mine (list-filter #(= (key-of (record-ref & need-name)) (key-of name)) + (box-ref (record-ref walk needs)))) + (def broken (list-filter (fun (n) (match (record-ref n need-req) + (None #f) + ((Some req) (not (version-satisfies? picked req))))) + mine)) + (if (empty? broken) + (Ok unit) + (Err (str "bjo: " (name->string name) " " (version->string picked) + " does not satisfy everything asked of it.\n" + " What each manifest asks for:\n" + (string-join (list->vec (list-sort (list-map #(need-line picked &) mine))) "\n") + "\n The highest version anybody asks for is the one that is used, and an" + " upper bound never lowers it.\n" + " Raise the bound, or lower what asks for more.")))) + +(: (-> Version Need string)) +(defun (need-line picked n) + (def satisfied? + (match (record-ref n need-req) + (None #t) + ((Some req) (version-satisfies? picked req)))) + (str " " (req-shown (record-ref n need-req)) " in " (record-ref n need-from) + (if satisfied? "" (str " <- " (version->string picked) " breaks this")))) + +(: (-> Walk (List Chosen) (Result string Unit))) +(defun (check-every-bound walk chosen) + (match chosen + (Nil (Ok unit)) + ((Cons c rest) + (def (Ok held) (check-bounds walk c) :propagate) + (check-every-bound walk rest)))) + + +;; --------------------------------------------------------------------------- +;; The lock +;; --------------------------------------------------------------------------- + +(: (-> (List Symbol) Version string string string string)) +(defun (moved-tag-message name v locked-at now lock-file) + (str "bjo: " (name->string name) " " (version->tag v) + " no longer points at the commit " lock-file " names.\n" + " locked " locked-at "\n" + " now " now "\n" + " A version is a tag, and a tag can be moved. Nothing was built:" + " the same version naming different code is what the lock exists to catch.\n" + " If the move was meant, remove that entry from " lock-file " and fetch again.")) + +;; What the lock should say about one chosen package. +;; +;; The check that matters is here: a package whose version has not changed must +;; still be the commit the lock names. +(: lock-entry #:sync (-> Walk Chosen string (Result string Locked))) +(defun (lock-entry walk c lock-file) + (def name (record-ref c chosen-name)) + (def picked (record-ref c chosen-version)) + (def clone (record-ref c chosen-clone)) + (def was (lock-entry-for (record-ref walk was-locked) name)) + (def same-version? + (match was + (None #f) + ((Some old) (= (version->string (locked-version old)) (version->string picked))))) + + ;; A commit the lock names and this clone has not got is the one thing worth + ;; going to the network for: it existed when the lock was written. + (def (Ok fetched-for-lock?) + (match was + ((Some old) (if (and same-version? + (not (commit-present? (record-ref walk tool) clone (locked-commit old)))) + (fetch-once walk clone) + (Ok #f))) + (None (Ok #f))) + :propagate) + + (def (Ok commit) (tag-commit (record-ref walk tool) clone picked) :propagate) + + (def was-commit (match was ((Some old) (locked-commit old)) (None ""))) + (if (and same-version? (not (= was-commit commit))) + (Err (moved-tag-message name picked was-commit commit lock-file)) + (Ok (Locked (name name) (version picked) + (source (record-ref c chosen-source)) + (commit commit))))) + +(: lock-entries #:sync (-> Walk (List Chosen) string (Result string (List Locked)))) +(defun (lock-entries walk chosen lock-file) + (match chosen + (Nil (Ok Nil)) + ((Cons c rest) + (def (Ok entry) (lock-entry walk c lock-file) :propagate) + (def (Ok more) (lock-entries walk rest lock-file) :propagate) + (Ok (Cons entry more))))) + + +;; --------------------------------------------------------------------------- +;; Putting the chosen versions on disk +;; --------------------------------------------------------------------------- + +;; One directory per chosen version, and its `src/` in the roots file. +(: materialise-all #:sync (-> Walk (List Chosen) (List Locked) (Result string (List Root)))) +(defun (materialise-all walk chosen entries) + (match chosen + (Nil (Ok Nil)) + ((Cons c rest) + (def name (record-ref c chosen-name)) + (def picked (record-ref c chosen-version)) + (def (Some entry) (lock-entry-for entries name) + :leave-with (Err (str "bjo: " (name->string name) " was chosen and not locked."))) + (def where (package-directory (record-ref walk dot-bjo) name picked)) + (def (Ok put) (materialise (record-ref walk tool) + (record-ref c chosen-clone) + (locked-commit entry) + (path-combine (record-ref walk dot-bjo) "tmp") + where) + :propagate) + (def src (path-combine where "src")) + (def (Ok more) (materialise-all walk rest entries) :propagate) + (if (not (directory-exists? src)) + (Err (str "bjo: " (name->string name) " " (version->string picked) + " has no src/, so there are no modules in it." + " A package keeps its modules in src/, beside its manifest.")) + (Ok (Cons (Root (name name) (dir src)) more)))))) + + +;; --------------------------------------------------------------------------- +;; prepare +;; --------------------------------------------------------------------------- + +;; Resolves the project, fetches what that needs, and writes the roots file. +;; +;; `locked?` is `--locked`: any difference from `bjo.lock` is an error rather +;; than an update, which is what a build that must be the one the lock +;; describes wants. +(: prepare #:sync (-> Project bool (Result string Prepared))) +(defun (prepare project locked?) + (def (Ok own) (root-package-root project) :propagate) + (def lock-file (project-lock-file project)) + (def (Ok was-locked) (read-lock lock-file) :propagate) + + (def walk (Walk (sources (make-box (map-empty))) + (placed (make-box (map-empty))) + (needs (make-box Nil)) + (nodes (make-box (map-empty))) + (names (make-box (map-empty))) + (clones (make-box (map-empty))) + (origins (make-box (map-empty))) + (walked (make-box (map-empty))) + (fetched (make-box (map-empty))) + (own (record-ref (project-package project) name)) + (own-from (path-combine (project-dir project) manifest-name)) + (tool (tool-for (project-tmp project))) + (dot-bjo (project-dot-bjo project)) + (was-locked was-locked))) + + ;; The root task is the only one whose `dev-depends` count — a dependency's + ;; are what its author needs in order to test it, and building it here runs + ;; no tests — and the only one allowed to say `(path ...)`, which its own + ;; path dependencies inherit: a directory beside yours is as much yours as + ;; your own. + (def first-task (Task (package (project-package project)) + (dir (project-dir project)) + (from (path-combine (project-dir project) manifest-name)) + (path-ok? #t) + (dev? #t))) + + (def (Ok walked) (walk-graph walk (Cons first-task Nil) Nil) :propagate) + + (def chosen (choose walk)) + (def (Ok bounds-held) (check-every-bound walk chosen) :propagate) + (def (Ok entries) (lock-entries walk chosen lock-file) :propagate) + (def (Ok settled) (settle-lock project locked? was-locked entries) :propagate) + (def (Ok fetched-roots) (materialise-all walk chosen entries) :propagate) + + (def path-roots (loop (:for entry (map-values (box-ref (record-ref walk placed)))) + (:acc out (listing entry)) + => out)) + (def roots (Cons own (list-append path-roots fetched-roots))) + (def roots-path (project-roots-file project)) + (ignore (write-roots roots-path roots)) + + ;; Beside the roots file, and written by the same rule: only when its + ;; contents change, because the compiler records it as an input of every + ;; module built against it. + (def frameworks-path (project-frameworks-file project)) + (def declared? (write-frameworks frameworks-path roots)) + + (def (Ok nuget-dir) (restore-packages project roots locked?) :propagate) + + (Ok (Prepared (roots-file roots-path) + (frameworks-file (if declared? (Some frameworks-path) None)) + (nuget-dir nuget-dir) + (runtime (runtime-settings (record-ref (project-package project) runtime))) + (aot (match (record-ref (project-package project) aot) + ((Some aot?) aot?) + (None #f))) + (aot-cpu (match (record-ref (project-package project) aot-cpu) + ((Some cpu) cpu) + (None ""))) + (roots roots)))) + +;; Writes the lock if it has to, and says what changed. +;; +;; With minimal version selection there is nothing to write unless a manifest +;; changed: the same manifests choose the same versions every time. So a line +;; printed here is a real change, and `--locked` refusing one is a build that +;; would not have been the build the lock describes. +(: settle-lock #:sync (-> Project bool (List Locked) (List Locked) (Result string Unit))) +(defun (settle-lock project locked? before after) + (def changes (lock-changes before after)) + (def lock-file (project-lock-file project)) + (cond + ((empty? changes) (Ok unit)) + (locked? + (Err (str "bjo: --locked, and the manifests no longer agree with " lock-file ":\n" + (string-join (list->vec (list-map #(str " " &) changes)) "\n") + "\n Run 'bjo fetch' without --locked to bring the lock up to date."))) + (else + (write-lock lock-file after) + (println (str "Updated " lock-file ":")) + (loop (:for line changes) + (:do (println (str " " line)))) + (Ok unit)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; janitor/docs.bjo — a module's published docs, as records. +;; +;; The compiler publishes a module's `(:doc ...)` forms in its assembly, under +;; the metadata key `BjolangDocs` (Docs/Documentation.org, "Where the +;; documentation goes"). This reads them back without loading the assembly, so +;; a tool can read the docs of a module it also links, or of many modules, +;; without keeping them loaded. +;; +;; A clause this version does not know is kept as `DocOther` rather than +;; refused: docs written for a later compiler still read. + +(export + Doc DocSubject DocKind DocClause + parse-docs assembly-docs module-docs + doc-summary doc-named module-doc doc-subject->string) + +(import (std prelude)) +(import (text bjodat-core)) +(import (janitor modules)) + +(import/extern + (clr-read-metadata (: Bjolang.Runtime.BjoAssemblyMetadata.Read (-> string string string) + #:exceptions (System.IO.IOException + System.UnauthorizedAccessException + System.BadImageFormatException)))) + +(:doc #:module + (summary "A module's published docs, read back as records.") + (see module-docs assembly-docs Doc) + (reference """ + Each (:doc ...) form a module was compiled with is a Doc. Its clauses are + as written, in the order written, with what the compiler added taken out + into the record: the kind of definition, and for a function or a value its + signature, and for a defun its head as written. + """)) + +;; --------------------------------------------------------------------------- +;; The records +;; --------------------------------------------------------------------------- + +(:doc DocSubject + (summary "What a doc is about.") + (case SubjectName "A definition, by its name.") + (case SubjectReader "A reader extension: (:doc name #:reader ...).") + (case SubjectModule "The module itself: (:doc #:module ...).")) +(type (: DocSubject (Union (: SubjectName Symbol) + (: SubjectReader Symbol) + SubjectModule))) + +(:doc DocKind + (summary "What kind of definition a doc is about, as the compiler found it.") + (case KindOther "A kind this version does not know, as published.")) +(type (: DocKind (Union KindFunction KindValue KindRecord KindUnion KindAlias + KindTrait KindMacro KindReader KindModule + (: KindOther Symbol)))) + +(:doc DocClause + (summary "One clause of a doc, as written.") + (case DocArg "A positional parameter's name and text.") + (case DocKey "A keyword parameter's name, without #:, and text.") + (case DocRaises "The exception's type as written, and when.") + (case DocTparam "The type parameter as written, %a, and its text.") + (case DocForm "How a macro is written, as source text: (with-x name body ...).") + (case DocOther "A clause this version does not know: its head and the rest.")) +(type (: DocClause (Union + (: DocSummary string) + (: DocArg Symbol string) + (: DocRest Symbol string) + (: DocKey Symbol string) + (: DocReturns string) + (: DocRaises Symbol string) + (: DocSee (List Symbol)) + (: DocExample string) + (: DocReference string) + (: DocField Symbol string) + (: DocCase Symbol string) + (: DocTparam Symbol string) + (: DocForm string) + (: DocLiteral (List Symbol)) + (: DocOther Symbol (List Bjodat))))) + +(:doc Doc + (summary "One published doc.") + (field subject "What it is about.") + (field kind "What kind of definition that is.") + (field signature "A function's or a value's type, as source writes it, when it has one.") + (field definition "A defun's head as written, defaults and all: (random-int lo hi #:rng r).") + (field clauses "Every other clause, in the order written.")) +(type (: Doc (Record (: subject DocSubject) + (: kind DocKind) + (: signature (Option string)) + (: definition (Option string)) + (: clauses (List DocClause))))) + +;; --------------------------------------------------------------------------- +;; Reading +;; --------------------------------------------------------------------------- + +;; A message names the form, so a published doc that does not read says where. +(: (-> string Bjodat (Result string %a))) +(defun (malformed what form) + (Err #"${what}: ${(bjodat->string form)}")) + +(: (-> Symbol DocKind)) +(defun (kind-named s) + (match (symbol->string s) + ("function" KindFunction) + ("value" KindValue) + ("record" KindRecord) + ("union" KindUnion) + ("alias" KindAlias) + ("trait" KindTrait) + ("macro" KindMacro) + ("reader" KindReader) + ("module" KindModule) + (_ (KindOther s)))) + +(: (-> (List Bjodat) (Option (List Symbol)))) +(defun (symbol-list xs) + (match xs + (Nil (Some Nil)) + ((Cons (BjoSym s) rest) + (option-map (fun (more) (Cons s more)) (symbol-list rest))) + (_ None))) + +;; A clause by its head. `#:rng` and `rng` are the same key. +(: (-> Bjodat (Result string DocClause))) +(defun (read-clause c) + (match c + ((BjoList (Cons (BjoSym head) args)) + (match (Tuple (symbol->string head) args) + ((Tuple "summary" (Cons (BjoStr t) Nil)) (Ok (DocSummary t))) + ((Tuple "arg" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocArg n t))) + ((Tuple "rest" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocRest n t))) + ((Tuple "key" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocKey n t))) + ((Tuple "key" (Cons (BjoKey k) (Cons (BjoStr t) Nil))) + (Ok (DocKey (string->symbol (keyword->string k)) t))) + ((Tuple "returns" (Cons (BjoStr t) Nil)) (Ok (DocReturns t))) + ((Tuple "raises" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocRaises n t))) + ((Tuple "example" (Cons (BjoStr t) Nil)) (Ok (DocExample t))) + ((Tuple "reference" (Cons (BjoStr t) Nil)) (Ok (DocReference t))) + ((Tuple "field" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocField n t))) + ((Tuple "case" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocCase n t))) + ((Tuple "tparam" (Cons (BjoSym n) (Cons (BjoStr t) Nil))) (Ok (DocTparam n t))) + ((Tuple "form" (Cons (BjoStr t) Nil)) (Ok (DocForm t))) + ;; As compilers before the string published it. + ((Tuple "form" (Cons (BjoList shape) Nil)) (Ok (DocForm (bjodat->string (BjoList shape))))) + ((Tuple "see" names) + (match (symbol-list names) + ((Some ns) (Ok (DocSee ns))) + (None (malformed "a see clause names symbols" c)))) + ((Tuple "literal" names) + (match (symbol-list names) + ((Some ns) (Ok (DocLiteral ns))) + (None (malformed "a literal clause names symbols" c)))) + ((Tuple (or "summary" "arg" "rest" "key" "returns" "raises" "example" + "reference" "field" "case" "tparam" "form") _) + (malformed "a clause of the wrong shape" c)) + (_ (Ok (DocOther head args))))) + (_ (malformed "a clause is a list headed by a symbol" c)))) + +;; The doc being built: the record's parts, before its clauses are reversed. +(type (: Partial (Record (: kind DocKind) + (: signature (Option string)) + (: definition (Option string)) + (: clauses (List DocClause))))) + +;; The symbol a list is headed by, or "" for anything else. +(: (-> Bjodat string)) +(defun (head-of b) + (match b + ((BjoList (Cons (BjoSym h) _)) (symbol->string h)) + (_ ""))) + +;; The added clauses go into the record, the rest into its clauses. +(: (-> Partial (List Bjodat) (Result string Partial))) +(defun (read-clauses p cs) + (match cs + (Nil (Ok p)) + ((Cons c rest) + (match (Tuple (head-of c) c) + ((Tuple "kind" (BjoList (Cons _ (Cons (BjoSym k) Nil)))) + (read-clauses (record-set p (kind (kind-named k))) rest)) + ((Tuple "signature" (BjoList (Cons _ (Cons (BjoStr s) Nil)))) + (read-clauses (record-set p (signature (Some s))) rest)) + ((Tuple "definition" (BjoList (Cons _ (Cons (BjoStr s) Nil)))) + (read-clauses (record-set p (definition (Some s))) rest)) + (_ + (def (Ok clause) (read-clause c) :propagate) + (read-clauses (record-set p (clauses (Cons clause (record-ref p clauses)))) rest)))))) + +;; `(:doc name ...)`, `(:doc name #:reader ...)` or `(:doc #:module ...)`. +(: (-> Bjodat (Result string Doc))) +(defun (read-doc form) + (def start (Partial (kind (KindOther (string->symbol "unknown"))) + (signature None) (definition None) (clauses Nil))) + (def subject+clauses + (match form + ((BjoList (Cons (BjoKey d) after)) + (if (not (= (keyword->string d) "doc")) + None + (match after + ((Cons (BjoKey m) cs) + (if (= (keyword->string m) "module") (Some (Tuple SubjectModule cs)) None)) + ((Cons (BjoSym n) (Cons (BjoKey r) cs)) + (if (= (keyword->string r) "reader") (Some (Tuple (SubjectReader n) cs)) None)) + ((Cons (BjoSym n) cs) (Some (Tuple (SubjectName n) cs))) + (_ None)))) + (_ None))) + (match subject+clauses + (None (malformed "a doc is (:doc name clause ...)" form)) + ((Some (Tuple subject cs)) + (def (Ok p) (read-clauses start cs) :propagate) + (Ok (Doc (subject subject) + (kind (record-ref p kind)) + (signature (record-ref p signature)) + (definition (record-ref p definition)) + (clauses (list-reverse (record-ref p clauses)))))))) + +(: (-> (List Bjodat) (Result string (List Doc)))) +(defun (read-docs forms) + (match forms + (Nil (Ok Nil)) + ((Cons f rest) + (def (Ok doc) (read-doc f) :propagate) + (def (Ok more) (read-docs rest) :propagate) + (Ok (Cons doc more))))) + +(:doc parse-docs + (summary "Published docs, from the text the compiler wrote.") + (arg text "A bjodat vec of (:doc ...) forms, or \"\" for a module with none.") + (returns "The docs, in the order the module wrote them, or why the text does not read.") + (see assembly-docs)) +(: (-> string (Result string (Vec Doc)))) +(defun (parse-docs text) + (if (= (string-trim text) "") + (Ok []) + (match (parse-bjodat text) + ((Err why) (Err why)) + ((Ok (BjoVec forms)) + (def (Ok docs) (read-docs (vec->list forms)) :propagate) + (Ok (list->vec docs))) + ((Ok other) (malformed "published docs are a vec" other))))) + +(:doc assembly-docs + (summary "The docs published in an assembly, read without loading it.") + (arg path "The module's dll, or a program's exe.") + (returns "Its docs, none when it has none, or why the file could not be read.") + (see module-docs parse-docs)) +(: (-> string (Result string (Vec Doc)))) +(defun (assembly-docs path) + (if (not (file-exists? path)) + (Err #"no such file: ${path}") + (match (clr-read-metadata path "BjolangDocs") + ((Err e) (Err #"${path}: ${(.-Message e)}")) + ((Ok text) + (result-map-err (fun (why) #"${path}: ${why}") (parse-docs text)))))) + +(:doc module-docs + (summary "The docs of a module, by name.") + (arg roots "The packages to look in: (standard-roots), and a project's from read-roots-file.") + (arg name "The module: (std random).") + (returns "Its docs, or why there are none to read.") + (see assembly-docs find-module-assembly standard-roots read-roots-file)) +(: (-> (List PackageRoot) (List Symbol) (Result string (Vec Doc)))) +(defun (module-docs roots name) + (def (Ok dll) (find-module-assembly roots name) :propagate) + (assembly-docs dll)) + +;; --------------------------------------------------------------------------- +;; Asking +;; --------------------------------------------------------------------------- + +(:doc doc-summary + (summary "A doc's one-line summary.") + (arg doc "The doc.") + (returns "Its summary, or \"\" for one published without.")) +(: (-> Doc string)) +(defun (doc-summary doc) + (loop (:for c (record-ref doc clauses)) + (:acc found (folding "" + (match c + ((DocSummary s) (if (= found "") s found)) + (_ found)))))) + +(:doc doc-named + (summary "The doc of a definition, by its name.") + (arg docs "A module's docs.") + (arg name "The definition's name. A reader extension's is written with its #: #pair.") + (returns "The doc, or None when the module has none by that name.") + (see module-doc)) +(: (-> (Vec Doc) string (Option Doc))) +(defun (doc-named docs name) + (find #(match (record-ref & subject) + ((SubjectName n) (= (symbol->string n) name)) + ((SubjectReader n) (= (str "#" (symbol->string n)) name)) + (SubjectModule #f)) + docs)) + +(:doc module-doc + (summary "The doc of the module itself: its (:doc #:module ...).") + (arg docs "A module's docs.") + (returns "The doc, or None when the module has none.") + (see doc-named)) +(: (-> (Vec Doc) (Option Doc))) +(defun (module-doc docs) + (find #(match (record-ref & subject) + (SubjectModule #t) + (_ #f)) + docs)) + +(:doc doc-subject->string + (summary "What a doc is about, as source writes it.") + (arg subject "The subject.") + (returns "random-int, #pair for a reader extension, or #:module.")) +(: (-> DocSubject string)) +(defun (doc-subject->string subject) + (match subject + ((SubjectName n) (symbol->string n)) + ((SubjectReader n) (str "#" (symbol->string n))) + (SubjectModule "#:module"))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; janitor/modules.bjo — where a module name leads, found the way the compiler +;; finds it. +;; +;; `janitor` holds what tools need from the compiler's side of things. The +;; rules here are `Paths.fs`'s, written again: the installed `lib` is the first +;; candidate with a `std` in it, each directory of it is a package, a roots +;; file adds more, and the longest package name that is a proper prefix of a +;; module name owns it. A module's dll sits beside its source. + +(export + PackageRoot + module-name->string + lib-directory standard-roots read-roots-file + find-module-source find-module-assembly) + +(import (std prelude)) +(import (text bjodat-core)) + +(import/extern + (clr-base-dir (: System.AppContext.BaseDirectory #:get)) + (clr-directory-name (: System.IO.Path.GetDirectoryName (-> string string))) + (clr-change-ext (: System.IO.Path.ChangeExtension (-> string string string))) + (clr-null-or-empty? (: System.String.IsNullOrEmpty (-> string bool)))) + +(import/class + (ClrType (: System.Type)) + (ClrAssembly (: System.Reflection.Assembly))) + +(import/extern + ;; Untyped: written (-> object ClrType), a use says object and object do + ;; not match. + (clr-type-of (: System.Object.GetType)) + (clr-assembly-of (: System.Type.Assembly (-> ClrType ClrAssembly) #:get)) + (clr-location (: System.Reflection.Assembly.Location (-> ClrAssembly string) #:get))) + +;; A value of a type this module defines, whose assembly is this module's. +(type (: OwnMarker (Record (: unused int)))) +(: OwnMarker) +(def own-marker (OwnMarker (unused 0))) + +(:doc #:module + (summary "Where a module name leads: its package, its source and its dll.") + (see find-module-assembly read-roots-file) + (reference """ + The rules are the compiler's. The installed lib is the first candidate + that has a std in it: $BJOLANG_LIB, $BJOLANG_HOME/lib, then a lib in the + directory of the running program or any directory above it, then the same + from the working directory. Each directory of lib is a package of the + standard library. A roots file, which bjo writes as .bjo/roots, adds the + packages of a project. The longest package name that is a proper prefix of + a module name owns the module, and the rest of the name is a path below + the package's directory. + """)) + +(:doc PackageRoot + (summary "A package: its name, and the directory its modules are in.") + (field name "As an import spells it: (std), (bjolang http).") + (field directory "Absolute.") + (field standard "Whether it is part of the installed standard library.")) +(type (: PackageRoot (Record (: name (List Symbol)) + (: directory string) + (: standard bool)))) + +(:doc module-name->string + (summary "A module name as source writes it.") + (arg name "The name.") + (returns "(std random), for (std random).")) +(: (-> (List Symbol) string)) +(defun (module-name->string name) + (str "(" (string-join (list->vec (list-map symbol->string name)) " ") ")")) + +;; --------------------------------------------------------------------------- +;; The installed library +;; --------------------------------------------------------------------------- + +;; `None` at the root of the filesystem, where `GetDirectoryName` answers null. +(: (-> string (Option string))) +(defun (parent-of dir) + (def up (clr-directory-name (path-absolute dir))) + (if (clr-null-or-empty? up) None (Some up))) + +;; `/lib` for `dir` and every directory above it, deepest first. +(: (-> string (List string))) +(defun (ancestor-libs start) + (let go ((dir (path-absolute start))) + (Cons (path-combine dir "lib") + (match (parent-of dir) + ((Some up) (go up)) + (None Nil))))) + +(: (-> string string (List string))) +(defun (env-dir var below) + (match (get-environment-variable var) + ((Some dir) (Cons (path-absolute (if (= below "") dir (path-combine dir below))) Nil)) + (None Nil))) + +;; The lib this module was loaded from: lib/janitor/modules.dll. A program run +;; from outside the installation finds no lib above itself or its working +;; directory, but it does link this. +(: (-> (List string))) +(defun (own-lib) + (def own (clr-location (clr-assembly-of (clr-type-of own-marker)))) + (if (clr-null-or-empty? own) + Nil + (match (parent-of own) + ((Some janitor) (match (parent-of janitor) + ((Some lib) (Cons lib Nil)) + (None Nil))) + (None Nil)))) + +(:doc lib-directory + (summary "The installed lib directory, or None when there is none to find.") + (returns "The first candidate with a std directory in it, as the compiler picks it.") + (see standard-roots) + (reference """ + The candidates are the compiler's, with one before them: the lib this + module was loaded from, which a program run from anywhere has. + """)) +(: (-> (Option string))) +(defun (lib-directory) + (find #(directory-exists? (path-combine & "std")) + (list-append (env-dir "BJOLANG_LIB" "") + (list-append (env-dir "BJOLANG_HOME" "lib") + (list-append (own-lib) + (list-append (ancestor-libs clr-base-dir) + (ancestor-libs (current-directory)))))))) + +(:doc standard-roots + (summary "The standard library's packages: one per directory of lib.") + (returns "The packages, or none when there is no lib to find.") + (see lib-directory)) +(: (-> (List PackageRoot))) +(defun (standard-roots) + (match (lib-directory) + (None Nil) + ((Some lib) + (loop (:for dir (directory-subdirectories lib)) + (:acc out (listing (PackageRoot (name (Cons (string->symbol (option-ref-or (path-filename dir) "")) Nil)) + (directory (path-absolute dir)) + (standard #t)))) + => out)))) + +;; --------------------------------------------------------------------------- +;; A roots file +;; --------------------------------------------------------------------------- + +;; `(root (pkg name) "dir")`, the directory relative to the file's. +(: (-> string Bjodat (Result string PackageRoot))) +(defun (root-entry base entry) + (match entry + ((BjoList (Cons (BjoSym head) (Cons (BjoList parts) (Cons (BjoStr dir) Nil)))) + (if (not (= (symbol->string head) "root")) + (Err #"expected (root (name ...) \"directory\"), not ${(bjodat->string entry)}") + (match (symbols-of parts) + (None (Err #"a package name is a list of symbols: ${(bjodat->string entry)}")) + ((Some Nil) (Err "a package name has at least one segment")) + ((Some name) + (Ok (PackageRoot (name name) + (directory (path-absolute (path-combine base dir))) + (standard #f))))))) + (_ (Err #"expected (root (name ...) \"directory\"), not ${(bjodat->string entry)}")))) + +(: (-> (List Bjodat) (Option (List Symbol)))) +(defun (symbols-of parts) + (match parts + (Nil (Some Nil)) + ((Cons (BjoSym s) rest) + (option-map (fun (more) (Cons s more)) (symbols-of rest))) + (_ None))) + +(: (-> string (List Bjodat) (Result string (List PackageRoot)))) +(defun (root-entries base entries) + (match entries + (Nil (Ok Nil)) + ((Cons e rest) + (def (Ok root) (root-entry base e) :propagate) + (def (Ok more) (root-entries base rest) :propagate) + (Ok (Cons root more))))) + +(:doc read-roots-file + (summary "The packages a roots file names, as bjo writes one in .bjo/roots.") + (arg file "The roots file: (roots (root (name ...) \"directory\") ...).") + (returns "Its packages, or why the file could not be read. A relative directory is relative to the file.") + (see standard-roots)) +(: (-> string (Result string (List PackageRoot)))) +(defun (read-roots-file file) + (if (not (file-exists? file)) + (Err #"no roots file at ${file}") + (match (parse-bjodat (file-read-text file)) + ((Err why) (Err #"${file}: ${why}")) + ((Ok (BjoList (Cons (BjoSym head) entries))) + (if (= (symbol->string head) "roots") + (result-map-err (fun (why) #"${file}: ${why}") + (root-entries (option-ref-or (parent-of file) ".") entries)) + (Err #"${file}: expected one (roots ...) form"))) + ((Ok _) (Err #"${file}: expected one (roots ...) form"))))) + +;; --------------------------------------------------------------------------- +;; Resolving +;; --------------------------------------------------------------------------- + +;; `prefix` is a proper prefix of `name`: shorter, and equal as far as it goes. +(: (-> (List Symbol) (List Symbol) bool)) +(defun (proper-prefix? prefix name) + (match (Tuple prefix name) + ((Tuple Nil (Cons _ _)) #t) + ((Tuple (Cons p ps) (Cons n ns)) (and (= p n) (proper-prefix? ps ns))) + (_ #f))) + +(: (-> int (List Symbol) (List Symbol))) +(defun (drop-segments n name) + (match name + ((Cons _ rest) (if (> n 0) (drop-segments (- n 1) rest) name)) + (Nil Nil))) + +(:doc find-module-source + (summary "The .bjo a module name names, whether or not it exists.") + (arg roots "The packages to look in, standard ones included.") + (arg name "The module: (std random).") + (returns "The source path, or None when no package owns the name.") + (see find-module-assembly standard-roots)) +(: (-> (List PackageRoot) (List Symbol) (Option string))) +(defun (find-module-source roots name) + (def owner + (loop (:for r roots) + (:acc best + (folding None + (if (and (proper-prefix? (record-ref r name) name) + (match best + (None #t) + ((Some b) (> (list-length (record-ref r name)) + (list-length (record-ref b name)))))) + (Some r) + best))))) + (match owner + (None None) + ((Some r) + (def below (list-map symbol->string (drop-segments (list-length (record-ref r name)) name))) + (Some (path-absolute (str (path-combine (record-ref r directory) (string-join (list->vec below) "/")) ".bjo")))))) + +(:doc find-module-assembly + (summary "The compiled dll of a module.") + (arg roots "The packages to look in, standard ones included.") + (arg name "The module: (std random).") + (returns "The dll beside the module's source, or why there is none.") + (see find-module-source)) +(: (-> (List PackageRoot) (List Symbol) (Result string string))) +(defun (find-module-assembly roots name) + (match (find-module-source roots name) + (None (Err #"no package provides ${(module-name->string name)}")) + ((Some source) + (def dll (clr-change-ext source ".dll")) + (cond + ((file-exists? dll) (Ok dll)) + ((file-exists? source) (Err #"${(module-name->string name)} is not compiled: no ${dll}")) + (else (Err #"no module ${(module-name->string name)}: no ${source}")))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/array.bjo — whole-array operations on numbers. +;; +;; Each operation is one loop in the runtime (`BjoArray.cs`). The loop uses +;; the CPU's vector instructions when the element is a primitive number, so +;; one instruction does 4 doubles or 8 ints with AVX2. Other numbers, such as +;; `BigInteger`, go through an ordinary loop. +;; +;; Every function that needs arithmetic asks for `Num`, whose CLR constraint is +;; `INumber`. That is what the runtime's kernels ask for, so the C# call is +;; legal in a generic function too. +(import (std prelude)) + +(:doc #:file "array.bjodoc") + +(export + array-sum array-dot array-min array-max + array-add! array-scale! + array-count array-index-of + array-fill!) + +(import/extern + (clr-sum (: Bjolang.Runtime.BjoArray.Sum (-> (Array %a) %a))) + (clr-dot (: Bjolang.Runtime.BjoArray.Dot (-> (Array %a) (Array %a) %a))) + (clr-min (: Bjolang.Runtime.BjoArray.Min (-> (Array %a) %a))) + (clr-max (: Bjolang.Runtime.BjoArray.Max (-> (Array %a) %a))) + (clr-add! (: Bjolang.Runtime.BjoArray.AddInPlace (-> (Array %a) (Array %a) void))) + (clr-scale! (: Bjolang.Runtime.BjoArray.ScaleInPlace (-> (Array %a) %a void))) + (clr-count (: Bjolang.Runtime.BjoArray.Count (-> (Array %a) %a int))) + (clr-index-of (: Bjolang.Runtime.BjoArray.IndexOf (-> (Array %a) %a int))) + (clr-fill! (: Bjolang.Runtime.BjoArray.Fill (-> (Array %a) %a void)))) + +;; --------------------------------------------------------------------------- +;; Reductions +;; --------------------------------------------------------------------------- + +(: (-> (Array %a) %a) (where (Num %a))) +(defun (array-sum a) (clr-sum a)) + +(: (-> (Array %a) (Array %a) %a) (where (Num %a))) +(defun (array-dot a b) (clr-dot a b)) + +(: (-> (Array %a) %a) (where (Num %a))) +(defun (array-min a) (clr-min a)) + +(: (-> (Array %a) %a) (where (Num %a))) +(defun (array-max a) (clr-max a)) + +;; --------------------------------------------------------------------------- +;; In place +;; --------------------------------------------------------------------------- + +(: (-> (Array %a) (Array %a) void) (where (Num %a))) +(defun (array-add! dst src) (clr-add! dst src)) + +(: (-> (Array %a) %a void) (where (Num %a))) +(defun (array-scale! a k) (clr-scale! a k)) + +;; Any element type: `Array.Fill` needs no arithmetic. +(: (-> (Array %a) %a void)) +(defun (array-fill! a x) (clr-fill! a x)) + +;; --------------------------------------------------------------------------- +;; Search +;; --------------------------------------------------------------------------- + +;; Equality is the number's `==`, as `=` is. `Eq` is not asked for: the kernel +;; cannot call a Bjolang implementation. +(: (-> (Array %a) %a int) (where (Num %a))) +(defun (array-count a x) (clr-count a x)) + +;; -1 when no element matches, as `mutablevec-index-of` answers. +(: (-> (Array %a) %a int) (where (Num %a))) +(defun (array-index-of a x) (clr-index-of a x)) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/atomic.bjo — a cell that threads can read and write without a lock. +;; +;; An `(Atomic T)` is a `Bjolang.Runtime.Atomic`. The runtime chooses the +;; class for T when the atomic is made: one class for each of int, uint, long +;; and ulong, `AtomicBox` for other reference types and +;; `AtomicStructBox` for other value types. See BjolangRuntime/BjoAtomic.cs. +;; +;; The integer operations are the methods of the trait `AtomicInteger`. A call +;; at a known integer type is inlined, and becomes a direct call to +;; `Interlocked`. +(import (std prelude)) + +(:doc #:file "atomic.bjodoc") + +(export + Atomic + make-atomic atomic-ref atomic-set! atomic-swap! atomic-compare-and-swap! + atomic-update! + AtomicInteger atomic-increment! atomic-decrement! atomic-add! atomic-sub! + atomic-array-increment!) + +(import/class + ((Atomic %a) (: Bjolang.Runtime.Atomic))) + +;; --------------------------------------------------------------------------- +;; Any T +;; --------------------------------------------------------------------------- + +(import/extern + (make-atomic (: Bjolang.Runtime.AtomicModule.Make (-> %a (Atomic %a)))) + (atomic-ref (: Bjolang.Runtime.AtomicModule.Ref (-> (Atomic %a) %a))) + (atomic-set! (: Bjolang.Runtime.AtomicModule.Set (-> (Atomic %a) %a void))) + (atomic-swap! (: Bjolang.Runtime.AtomicModule.Swap (-> (Atomic %a) %a %a))) + (atomic-compare-and-swap! (: Bjolang.Runtime.AtomicModule.CompareAndSwap (-> (Atomic %a) %a %a %a))) + (atomic-update! (: Bjolang.Runtime.AtomicModule.Update (-> (Atomic %a) (-> %a %a) %a)))) + +;; --------------------------------------------------------------------------- +;; Integers +;; --------------------------------------------------------------------------- + +;; TODO: make this trait closed once Bjolang supports closed traits. +(def/trait (AtomicInteger %a) + (: atomic-increment! (-> (Atomic %a) %a)) + (: atomic-decrement! (-> (Atomic %a) %a)) + (: atomic-add! (-> (Atomic %a) %a %a)) + (: atomic-sub! (-> (Atomic %a) %a %a))) + +(import/extern + (int-increment! (: Bjolang.Runtime.AtomicModule.Increment (-> (Atomic int) int))) + (int-decrement! (: Bjolang.Runtime.AtomicModule.Decrement (-> (Atomic int) int))) + (int-add! (: Bjolang.Runtime.AtomicModule.Add (-> (Atomic int) int int))) + (int-sub! (: Bjolang.Runtime.AtomicModule.Sub (-> (Atomic int) int int))) + (uint-increment! (: Bjolang.Runtime.AtomicModule.Increment (-> (Atomic uint) uint))) + (uint-decrement! (: Bjolang.Runtime.AtomicModule.Decrement (-> (Atomic uint) uint))) + (uint-add! (: Bjolang.Runtime.AtomicModule.Add (-> (Atomic uint) uint uint))) + (uint-sub! (: Bjolang.Runtime.AtomicModule.Sub (-> (Atomic uint) uint uint))) + (long-increment! (: Bjolang.Runtime.AtomicModule.Increment (-> (Atomic long) long))) + (long-decrement! (: Bjolang.Runtime.AtomicModule.Decrement (-> (Atomic long) long))) + (long-add! (: Bjolang.Runtime.AtomicModule.Add (-> (Atomic long) long long))) + (long-sub! (: Bjolang.Runtime.AtomicModule.Sub (-> (Atomic long) long long))) + (ulong-increment! (: Bjolang.Runtime.AtomicModule.Increment (-> (Atomic ulong) ulong))) + (ulong-decrement! (: Bjolang.Runtime.AtomicModule.Decrement (-> (Atomic ulong) ulong))) + (ulong-add! (: Bjolang.Runtime.AtomicModule.Add (-> (Atomic ulong) ulong ulong))) + (ulong-sub! (: Bjolang.Runtime.AtomicModule.Sub (-> (Atomic ulong) ulong ulong)))) + +(impl (AtomicInteger int) + (defun (atomic-increment! a) (int-increment! a)) + (defun (atomic-decrement! a) (int-decrement! a)) + (defun (atomic-add! a n) (int-add! a n)) + (defun (atomic-sub! a n) (int-sub! a n))) + +(impl (AtomicInteger uint) + (defun (atomic-increment! a) (uint-increment! a)) + (defun (atomic-decrement! a) (uint-decrement! a)) + (defun (atomic-add! a n) (uint-add! a n)) + (defun (atomic-sub! a n) (uint-sub! a n))) + +(impl (AtomicInteger long) + (defun (atomic-increment! a) (long-increment! a)) + (defun (atomic-decrement! a) (long-decrement! a)) + (defun (atomic-add! a n) (long-add! a n)) + (defun (atomic-sub! a n) (long-sub! a n))) + +(impl (AtomicInteger ulong) + (defun (atomic-increment! a) (ulong-increment! a)) + (defun (atomic-decrement! a) (ulong-decrement! a)) + (defun (atomic-add! a n) (ulong-add! a n)) + (defun (atomic-sub! a n) (ulong-sub! a n))) + +;; --------------------------------------------------------------------------- +;; Elements of an array +;; --------------------------------------------------------------------------- + +;; Element `i` of the array plus 1, as one atomic step; the new value. For +;; counts kept in an array that several threads add to. +(import/extern + (atomic-array-increment! (: Bjolang.Runtime.AtomicModule.IncrementAt (-> (Array int) int int)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/bench.bjo — benchmarks. +;; +;; (import (std prelude)) +;; (import (std bench)) +;; +;; (def/bench "list-sort" +;; #:params ((n [1000 100000])) +;; #:setup ((xs (random-list n))) +;; (list-sort xs)) +;; +;; (run-benches) +;; +;; `def/bench` registers a benchmark where it is written, and `(run-benches)` +;; is the file's `main`, as `deftest` and `(run-tests)` are. A run warms each +;; benchmark up until its times are steady, calibrates how many calls make a +;; sample, takes samples until the median is known well enough, and prints a +;; table: the median per call, its spread, the minimum, allocation, and +;; collections. See `Docs/Benchmark-design.org`, which this follows. +;; +;; The statistics are `(std bench stats)`; the runtime's part, the sink and +;; the collector's counters, is `BjoBench`. + +(export def/bench run-benches time-it + Bench BenchCase CaseRun BenchOptions Measured Settings BenchParam bench-param-read + registered-benches module-benches parse-bench-args measure-case bench-params-set! + bench-settings time-text bytes-text + Selection parse-choice bench-menu menu-loop menu-lines run-selection run-and-report run-with-options bench-serve + print-comparison-table comparisons-text fails-slower? header-lines header-facts this-run + sel-bench sel-cases case-labels + ;; Named by the templates of the macros, so they have to be reachable + ;; where those expand. + register-bench! bench-keep! bench-param-values bench-now bench-allocated + run-registered-benches time-it-run time-it-fixed time-it-expansion time-it-keep) + +;; Every export is documented there; the module doc is too. +(:doc #:file "bench.bjodoc") + +(import (std prelude)) +(import (std maths)) +(import (std syntax-match)) +(import (std bench stats)) +(import (std fmt)) +(import (std bench store)) +(import (std random)) +(import (text bjodat-core)) + +(import/extern + (bench-keep! (: Bjolang.Runtime.BjoBench.Consume (-> %a void))) + (collect! (: Bjolang.Runtime.BjoBench.Collect (-> void))) + (gc-count (: Bjolang.Runtime.BjoBench.Collections (-> int int))) + (allocated-thread (: Bjolang.Runtime.BjoBench.AllocatedOnThread (-> long))) + (allocated-process (: Bjolang.Runtime.BjoBench.AllocatedInProcess (-> long))) + (cpu-model (: Bjolang.Runtime.BjoBench.CpuModel (-> string))) + (core-count (: Bjolang.Runtime.BjoBench.Cores (-> int))) + (os-name (: Bjolang.Runtime.BjoBench.OperatingSystemName (-> string))) + (arch-name (: Bjolang.Runtime.BjoBench.Architecture (-> string))) + (dotnet-version (: Bjolang.Runtime.BjoBench.DotnetVersion (-> string))) + (server-gc? (: Bjolang.Runtime.BjoBench.ServerGc (-> bool))) + (governor (: Bjolang.Runtime.BjoBench.Governor (-> string))) + (debugger? (: Bjolang.Runtime.BjoBench.DebuggerAttached (-> bool))) + (debug-build? (: Bjolang.Runtime.BjoBench.DebugBuild (-> bool))) + (output-terminal? (: Bjolang.Runtime.BjoBench.OutputIsTerminal (-> bool))) + (input-terminal? (: Bjolang.Runtime.BjoBench.InputIsTerminal (-> bool))) + (machine-key (: Bjolang.Runtime.BjoBench.MachineKey (-> string))) + (now-utc (: Bjolang.Runtime.BjoBench.NowUtc (-> string))) + (set-exit-code! (: System.Environment.ExitCode (-> int void) #:set))) + +;; --------------------------------------------------------------------------- +;; Small helpers +;; --------------------------------------------------------------------------- + +(: (-> string (Option int))) +(defun (parse-int s) + (match (try (string->int s) #:catch (System.Exception)) + ((Ok n) (Some n)) + (_ None))) + +(: (-> string (Option long))) +(defun (parse-long s) + (match (try (cast System.Int64 (string->bigint s)) #:catch (System.Exception)) + ((Ok n) (Some n)) + (_ None))) + +(: (-> string (Option double))) +(defun (parse-double s) + (match (try (string->double s) #:catch (System.Exception)) + ((Ok n) (Some n)) + (_ None))) + +;; The text before and after the first `sep`, when it is there and has text +;; before it. +(: (-> string string (Option (Tuple string string)))) +(defun (split-once s sep) + (match (string-search s sep) + ((Some at) #:when (not (string-cursor=? at (string-cursor-start s))) + (Some (Tuple (substring s (string-cursor-start s) at) + (substring s (skip-chars s at (string-count sep)) (string-cursor-end s))))) + (_ None))) + +(: (-> string StringCursor int StringCursor)) +(defun (skip-chars s at n) + (if (= n 0) at (skip-chars s (string-cursor-next s at) (- n 1)))) + +(: (-> (List %a) (List %b) (List (Tuple %a %b)))) +(defun (list-zip xs ys) + (match (Tuple xs ys) + ((Tuple (Cons x xm) (Cons y ym)) (cons (Tuple x y) (list-zip xm ym))) + (_ (list)))) + +(: (-> int (List int))) +(defun (index-list n) {listing i (:for i (range 0 n))}) + +;; The clock, read as a stopwatch reads it, so that a test can bind +;; `current-clock` and get exact numbers. +(: (-> long)) +(defun (bench-now) (monotonic-ns)) + +;; The sink as an ordinary function, for `time-it`: `(std stopwatch)` +;; re-exports what its expansion names, and a foreign import cannot be. +(: (-> %a void)) +(defun (time-it-keep x) (bench-keep! x)) + +;; What `alloc/op` counts when a benchmark does not say: "thread" or +;; "process". `--alloc` sets it. +(: (Box string)) +(def alloc-default (make-box "thread")) + +;; A benchmark's `#:alloc`, or the run's default for "". +(: (-> string string)) +(defun (alloc-counted mode) + (if (string-empty? mode) (box-ref alloc-default) mode)) + +;; The bytes allocated so far, as `mode` counts them: +;; +;; - "thread": this thread's, exact, and blind to the work a body hands to +;; fibers or the pool. +;; - "process": every thread's, also the JIT's background compiles and +;; timers, so noisier. The read is precise, and not free. +;; - "": the run's default. +(: (-> string long)) +(defun (bench-allocated mode) + (if (= (alloc-counted mode) "process") (allocated-process) (allocated-thread))) + +;; --------------------------------------------------------------------------- +;; Benchmarks and their registry +;; --------------------------------------------------------------------------- + +;; What running one case comes to: `batch` runs the body `k` times; `call`, +;; for a `#:setup-every` benchmark, runs the setup and then one timed call, and +;; answers the call's nanoseconds and the bytes it allocated, the setup's +;; left out of both. +(type (: CaseRun (Record (: batch (-> long void)) + (: call (Option (-> (Tuple long long)))) + ;; The benchmark's `#:alloc`: "thread", "process", or + ;; "" for the run's default. + (: alloc string)))) + +;; One row: a benchmark at one combination of its parameters. `labels` are the +;; parameters' names and values as the report shows them; `prepare` runs the +;; setup and answers what to run. +(type (: BenchCase (Record (: labels (List (Tuple string string))) + (: prepare (-> CaseRun))))) + +;; A registered benchmark. `once` is the number of cold runs of a `#:once` +;; benchmark, and 0 for a calibrated one. `cases` makes its rows; it is a +;; function so that the parameters' values are read when the run starts, +;; after the flags that may replace them. +(type (: Bench (Record (: name string) + (: module string) + (: file string) + (: line int) + (: group string) + (: skip string) + (: once int) + ;; Its `#:alloc`, or "". + (: alloc string) + (: param-names (List string)) + (: cases (-> (List BenchCase)))))) + +;; Newest first, as `(std test)`'s registry. +(: (Box (List Bench))) +(def bench-registry (make-box (list))) + +(: (-> string string string int string string int string (List string) (-> (List BenchCase)) int)) +(defun (register-bench! name module file line group skip once alloc param-names cases) + (box-set! bench-registry + (cons (Bench (name name) (module module) (file file) (line line) (group group) + (skip skip) (once once) (alloc alloc) (param-names param-names) (cases cases)) + (box-ref bench-registry))) + 0) + +(: (-> (List Bench))) +(defun (registered-benches) (list-reverse (box-ref bench-registry))) + +(: (-> string (List Bench))) +(defun (module-benches module) + (list-filter (fun (b) (= (record-ref b module) module)) (registered-benches))) + +;; --------------------------------------------------------------------------- +;; Parameters +;; --------------------------------------------------------------------------- + +;; `--params n=10,100` replaces a parameter's values for the run, as text, +;; read in the parameter's own type. +(: (Box (Map string (Vec string)))) +(def param-overrides (make-box (map-empty))) + +;; How a parameter's value is read from `--params`. +(def/trait (BenchParam %a) + (: bench-param-read (-> string (Option %a)))) + +(impl (BenchParam int) (defun (bench-param-read s) (parse-int s))) +(impl (BenchParam long) (defun (bench-param-read s) (parse-long s))) +(impl (BenchParam double) (defun (bench-param-read s) (parse-double s))) +(impl (BenchParam string) (defun (bench-param-read s) (Some s))) +(impl (BenchParam Symbol) (defun (bench-param-read s) (Some (string->symbol (string-trim-quote s))))) +(impl (BenchParam bool) + (defun (bench-param-read s) + (case s + (("#t" "true") (Some #t)) + (("#f" "false") (Some #f)) + (else None)))) + +;; A symbol may be written 'sorted or sorted. +(: (-> string string)) +(defun (string-trim-quote s) + (if (string-starts-with? s "'") + (substring s (string-cursor-next s (string-cursor-start s)) (string-cursor-end s)) + s)) + +;; Replaces parameters' values for the benchmarks run after it, as +;; `--params` does. +(: (-> (Map string (Vec string)) void)) +(defun (bench-params-set! overrides) (box-set! param-overrides overrides)) + +;; The values of parameter `name`: its own, or those `--params` gave. +(: (-> string (Vec %a) (Vec %a)) (where (BenchParam %a))) +(defun (bench-param-values name values) + (match (map-try-ref (box-ref param-overrides) name) + (None values) + ((Some texts) + (vec-map (fun (t) + (match (bench-param-read t) + ((Some v) v) + (None (panic! #"--params: ${name}=${t} is not a value of the parameter's type.")))) + texts)))) + +;; --------------------------------------------------------------------------- +;; def/bench +;; --------------------------------------------------------------------------- + +;; What the options of one `def/bench` came to. The setups are bindings, +;; `(name expr)`, made in order as `let*` makes them. +(type (: BenchSyntax (Record (: setup (List Syntax)) + (: setup-every (List Syntax)) + (: params (List (Tuple Syntax Syntax))) + (: once Syntax) + (: group Syntax) + (: skip Syntax) + (: alloc Syntax) + (: body (List Syntax))))) + +(: (-> BenchSyntax)) +(defun (no-bench-options) + (BenchSyntax (setup (list)) (setup-every (list)) (params (list)) (once #'0) (group #'"") + (skip #'"") (alloc #'"") (body (list)))) + +(: (-> Syntax (List Syntax) BenchSyntax BenchSyntax)) +(defun (bench-options form items o) + (let ((go (fun (more o2) (bench-options form more o2)))) + (syntax-match (SList items) + ((#:setup ((name e) ...) more ...) + (go more (record-set o (setup (zip-bindings name e))))) + ((#:setup bad more ...) + (ignore (syntax-error bad "#:setup takes bindings, made once before the benchmark runs: #:setup ((deck (full-deck))). The body then names them.")) + o) + ((#:setup-every ((name e) ...) more ...) + (go more (record-set o (setup-every (zip-bindings name e))))) + ((#:setup-every bad more ...) + (ignore (syntax-error bad "#:setup-every takes bindings, made again before every call: #:setup-every ((cards (vec->list deck))).")) + o) + ((#:params ((name values) ...) more ...) + (go more (record-set o (params (zip-syntax name values))))) + ((#:once n more ...) (go more (record-set o (once n)))) + ((#:group g more ...) (go more (record-set o (group g)))) + ((#:skip why more ...) (go more (record-set o (skip why)))) + ((#:alloc mode more ...) + (case (syntax->string mode) + ("'thread" (go more (record-set o (alloc #'"thread")))) + ("'process" (go more (record-set o (alloc #'"process")))) + (else + (ignore (syntax-error mode "#:alloc takes 'thread or 'process: what alloc/op counts, this thread's allocation or every thread's.")) + o))) + ((first more ...) (record-set o (body (cons first more)))) + (_ (ignore (syntax-error form "def/bench takes a name and a body: (def/bench \"name\" body ...), with options such as #:setup, #:setup-every, #:params, #:once, #:group, #:skip and #:alloc after the name.")) + o)))) + +(: (-> (List Syntax) (List Syntax) (List (Tuple Syntax Syntax)))) +(defun (zip-syntax xs ys) + (match (Tuple xs ys) + ((Tuple (Cons x xm) (Cons y ym)) (cons (Tuple x y) (zip-syntax xm ym))) + (_ (list)))) + +(: (-> (List Syntax) (List Syntax) (List Syntax))) +(defun (zip-bindings names values) + (list-map (fun (p) (match p ((Tuple n v) #'(,n ,v)))) (zip-syntax names values))) + +;; `prepare` of one case: the setup's bindings, made once, and the runs. The +;; parameters are bound already, by the loop the cases are made in, so the +;; setup and the body name them as they are. +(: (-> Syntax BenchSyntax Syntax)) +(defun (prepare-expansion form o) + (def body #'(begin ,@(record-ref o body))) + (def setup (SList (record-ref o setup))) + (def alloc (record-ref o alloc)) + (match (record-ref o setup-every) + (Nil + #'(fun () + (let* ,setup + (CaseRun (batch (fun (k) + (let lp ((i 0L)) + (when (< i k) + (bench-keep! ,body) + (lp (+ i 1L)))))) + (call None) + (alloc ,alloc))))) + (every-bindings + ;; Made again before every call, outside the time and the allocation: + ;; each call is timed on its own. + (def every (SList every-bindings)) + #'(fun () + (let* ,setup + (CaseRun (batch (fun (k) + (let lp ((i 0L)) + (when (< i k) + (let* ,every (bench-keep! ,body)) + (lp (+ i 1L)))))) + (call (Some (fun () + (let* ,every + (let* ((b0 (bench-allocated ,alloc)) + (t0 (bench-now))) + (bench-keep! ,body) + (let ((t1 (bench-now))) + (Tuple (- t1 t0) (- (bench-allocated ,alloc) b0)))))))) + (alloc ,alloc))))))) + +;; The cases: one per combination of the parameters' values, the last varying +;; fastest. +(: (-> Syntax BenchSyntax Syntax)) +(defun (cases-expansion form o) + (def params (record-ref o params)) + (def names (list-map (fun (p) (match p ((Tuple n _) n))) params)) + (def case-form + #'(BenchCase (labels (list ,@(list-map (fun (n) #'(Tuple ,(SStr (syntax->string n)) (displayed ,n))) names))) + (prepare ,(prepare-expansion form o)))) + (if (list-empty? params) + #'(fun () (list ,case-form)) + (let () + (def fors + (list-map (fun (p) + (match p + ((Tuple n vs) #'(:for ,n (bench-param-values ,(SStr (syntax->string n)) ,vs))))) + params)) + ;; `(:subloop)` between the drivers, so that each is a level of its own + ;; and they nest rather than walk in step. + (def clauses (interleave-subloop fors)) + ;; One unnamed accumulator is the loop's value: a `=>` written in a + ;; template would be renamed by hygiene, and no longer read as one. + #'(fun () + (loop ,@clauses + (:acc (listing ,case-form))))))) + +(: (-> (List Syntax) (List Syntax))) +(defun (interleave-subloop fors) + (match fors + (Nil (list)) + ((Cons f Nil) (list f)) + ((Cons f more) (cons f (cons #'(:subloop) (interleave-subloop more)))))) + +(: (-> Syntax Syntax BenchSyntax Syntax)) +(defun (def-bench-expansion form name o) + (when (list-empty? (record-ref o body)) + (ignore (syntax-error form "def/bench has no body: (def/bench \"name\" body ...)."))) + (def param-names (list-map (fun (p) (match p ((Tuple n _) (SStr (syntax->string n))))) (record-ref o params))) + #'(def registered + (register-bench! ,name (current-module) ,(SStr (syntax-file form)) ,(SInt (int->string (syntax-line form))) + ,(record-ref o group) ,(record-ref o skip) ,(record-ref o once) ,(record-ref o alloc) + (list ,@param-names) + ,(cases-expansion form o)))) + +;; `(def/bench "name" body ...)`, with options after the name in any order: +;; `#:setup ((name e) ...)`, `#:setup-every ((name e) ...)`, +;; `#:params ((p values) ...)`, `#:once n`, `#:group "g"`, `#:skip "why"` and +;; `#:alloc 'process`. +;; The body is forms, which the parameters and the setups' names are in +;; scope for. Registers the benchmark; `(run-benches)` runs it. +(def/macro (def/bench form inject compare) + (syntax-match form + ((_ name first more ...) + (def-bench-expansion form name (bench-options form (cons first more) (no-bench-options)))) + (bad (syntax-error bad "def/bench takes a name and a body: (def/bench \"name\" body ...).")))) + +;; --------------------------------------------------------------------------- +;; The options of a run +;; --------------------------------------------------------------------------- + +;; What the arguments of a benchmark program asked for. `scale` multiplies the +;; time targets: 0.5 for `--quick`, 4 for `--long`. +(type (: BenchOptions (Record (: filters (Vec string)) + (: all bool) + (: list bool) + (: scale double) + (: min-samples int) + (: max-time-ns long) + (: detail bool) + (: params (Map string (Vec string))) + ;; Saving and comparing. + (: save (Option string)) + (: compare (Option string)) + (: alpha double) + ;; A fraction: 0.02 for 2%. + (: threshold double) + (: p-raw bool) + (: fail-slower (Option double)) + ;; Driven over standard input and output. + (: serve bool) + ;; Exactly these cases, numbered as `--serve`'s + ;; `(list)` numbers them: what `bjo bench` asks + ;; for after a menu over several files. + (: cases (Option (Vec int))) + ;; How time grows with a numeric parameter. + (: scaling bool) + ;; The cases of each `#:group` compared, in + ;; `rounds` interleaved rounds shuffled from + ;; `seed`. + (: compare-within bool) + (: rounds int) + (: seed (Option ulong)) + ;; What alloc/op counts where a benchmark does + ;; not say: "thread" or "process". + (: alloc string)))) + +;; `2`, `2%` or `0.5` as a fraction of one: a percentage. +(: (-> string (Option double))) +(defun (parse-percent s) + (def digits (if (string-ends-with? s "%") + (substring s (string-cursor-start s) (string-cursor-prev s (string-cursor-end s))) + s)) + (match (parse-double digits) + ((Some x) #:when (>= x 0.0) (Some (/ x 100.0))) + (_ None))) + +(: (-> (Vec string) (Result string BenchOptions))) +(defun (parse-bench-args args) + (let go ((left (vec->list args)) (o (BenchOptions (filters (vec-empty)) (all #f) (list #f) (scale 1.0) + (min-samples 20) (max-time-ns 0L) (detail #f) + (params (map-empty)) + (save None) (compare None) (alpha 0.05) + (threshold 0.02) (p-raw #f) (fail-slower None) (serve #f) + (cases None) (scaling #f) (compare-within #f) + (rounds 20) (seed None) (alloc "thread")))) + (defun (value-of flag more k) + (match more + ((Cons v after) (k v after)) + (Nil (Err #"${flag} needs a value.")))) + (match left + (Nil (Ok o)) + ((Cons a more) + (case a + ("--all" (go more (record-set o (all #t)))) + ("--list" (go more (record-set o (list #t)))) + ("--quick" (go more (record-set o (scale 0.5)))) + ("--long" (go more (record-set o (scale 4.0)))) + ("--detail" (go more (record-set o (detail #t)))) + ("--p-raw" (go more (record-set o (p-raw #t)))) + ("--serve" (go more (record-set o (serve #t)))) + ("--scaling" (go more (record-set o (scaling #t)))) + ("--compare-within" (go more (record-set o (compare-within #t)))) + ("-N" + (value-of a more + (fun (v after) + (match (parse-int v) + ((Some n) #:when (> n 1) (go after (record-set o (rounds n)))) + (_ (Err #"-N takes a number of rounds above 1: ${v}")))))) + ("--seed" + (value-of a more + (fun (v after) + (match (parse-long v) + ((Some n) (go after (record-set o (seed (Some (cast System.UInt64 n)))))) + (_ (Err #"--seed takes a whole number: ${v}")))))) + ("--cases" + (value-of a more + (fun (v after) + (match (all-ints (string-split v ",")) + ((Some is) (go after (record-set o (cases (Some is))))) + (None (Err #"--cases takes case numbers, 0,3,4: ${v}")))))) + ("--alloc" + (value-of a more + (fun (v after) + (if (or (= v "thread") (= v "process")) + (go after (record-set o (alloc v))) + (Err #"--alloc takes thread or process: ${v}"))))) + ("--save" (value-of a more (fun (v after) (go after (record-set o (save (Some v))))))) + ("--compare" (value-of a more (fun (v after) (go after (record-set o (compare (Some v))))))) + ("--alpha" + (value-of a more + (fun (v after) + (match (parse-double v) + ((Some p) #:when (and (> p 0.0) (< p 1.0)) (go after (record-set o (alpha p)))) + (_ (Err #"--alpha takes a level between 0 and 1: ${v}")))))) + ("--threshold" + (value-of a more + (fun (v after) + (match (parse-percent v) + ((Some t) (go after (record-set o (threshold t)))) + (_ (Err #"--threshold takes a percentage: ${v}")))))) + ("--fail-slower" + (value-of a more + (fun (v after) + (match (parse-percent v) + ((Some t) (go after (record-set o (fail-slower (Some t))))) + (_ (Err #"--fail-slower takes a percentage: ${v}")))))) + ("--min-samples" + (value-of a more + (fun (v after) + (match (parse-int v) + ((Some n) #:when (> n 1) (go after (record-set o (min-samples n)))) + (_ (Err #"--min-samples takes a number above 1: ${v}")))))) + ("--max-time" + (value-of a more + (fun (v after) + (match (parse-double v) + ((Some s) #:when (> s 0.0) + (go after (record-set o (max-time-ns (cast System.Int64 (* s 1e9)))))) + (_ (Err #"--max-time takes seconds: ${v}")))))) + ("--params" + (value-of a more + (fun (v after) + (match (parse-param-override v) + ((Ok (Tuple name values)) + (go after (record-set o (params (map-set (record-ref o params) name values))))) + ((Err e) (Err e)))))) + (else + (if (string-starts-with? a "--") + (Err #"unknown flag ${a}. The flags are --all, --list, --quick, --long, --detail, --min-samples N, --max-time S, --params P=V,..., --save FILE, --compare FILE, --alpha P, --threshold PCT, --p-raw, --fail-slower PCT, --serve, --cases I,J,..., --scaling, --compare-within, -N ROUNDS, --seed N and --alloc thread|process") + (go more (record-set o (filters (vec-add (record-ref o filters) a))))))))))) + +;; Every text a number, or none of them. +(: (-> (Vec string) (Option (Vec int)))) +(defun (all-ints texts) + (def parsed (vec-map parse-int texts)) + (if (vec-iter some? parsed) + (Some (vec-map (fun (x) (match x ((Some n) n) (None 0))) parsed)) + None)) + +;; `n=10,100` as `n` and its values. +(: (-> string (Result string (Tuple string (Vec string))))) +(defun (parse-param-override s) + (match (split-once s "=") + ((Some (Tuple name values)) (Ok (Tuple name (string-split values ",")))) + (None (Err #"--params takes NAME=VALUE,VALUE...: ${s}")))) + +;; The targets of a run, in nanoseconds, from its options. +(type (: Settings (Record (: sample-ns long) + (: time-ns long) + (: warmup-ns long) + (: max-ns long) + (: min-samples int) + (: max-samples int)))) + +(: (-> BenchOptions Settings)) +(defun (bench-settings o) + (def scale (record-ref o scale)) + (def time-ns (cast System.Int64 (* 2e9 scale))) + (Settings (sample-ns 10000000L) + (time-ns time-ns) + (warmup-ns (cast System.Int64 (* 1e9 scale))) + (max-ns (if (> (record-ref o max-time-ns) 0L) (record-ref o max-time-ns) (* 3L time-ns))) + (min-samples (record-ref o min-samples)) + (max-samples (max 100 (record-ref o min-samples))))) + +;; --------------------------------------------------------------------------- +;; Measuring +;; --------------------------------------------------------------------------- + +;; What measuring one case came to. `samples` are nanoseconds per call, in the +;; order they were taken; for a `#:once` benchmark, one per cold run. +;; `calls` is every call the samples timed, which the collections and the +;; allocation are per. +(type (: Measured (Record (: samples (Vec double)) + (: k long) + (: calls long) + (: bytes long) + (: gc0 long) + (: gc1 long) + (: gc2 long) + (: steady bool) + (: once bool) + (: per-call bool)))) + +;; Where measuring says how far it has come: a phase, such as "warming up" or +;; "sampling 14/20 · median 6.21 ms". Nothing, unless the runner sets it: at a +;; terminal it rewrites one line. +(: (Box (-> string void))) +(def progress-box (make-box (fun (what) unit))) + +(: (-> string void)) +(defun (progress what) ((box-ref progress-box) what)) + +;; The nanoseconds of one batch of `k` calls. +(: (-> CaseRun long long)) +(defun (time-batch run k) + (def t0 (bench-now)) + ((record-ref run batch) k) + (- (bench-now) t0)) + +;; One sample of `k` calls, after a collection: its nanoseconds, bytes and +;; collections. +(type (: Sample (Record (: ns long) (: bytes long) (: gc0 long) (: gc1 long) (: gc2 long)))) + +(: (-> CaseRun long Sample)) +(defun (take-sample run k) + (collect!) + (def g0 (gc-count 0)) + (def g1 (gc-count 1)) + (def g2 (gc-count 2)) + (def alloc (record-ref run alloc)) + (def b0 (bench-allocated alloc)) + (def (Tuple ns bytes) + (match (record-ref run call) + ;; `#:setup-every`: each call on its own, its setup outside the time + ;; and the allocation. + ((Some call) + (let go ((i 0L) (total 0L) (bytes 0L)) + (if (< i k) + (match (call) ((Tuple t b) (go (+ i 1L) (+ total t) (+ bytes b)))) + (Tuple total bytes)))) + (None (let ((t (time-batch run k))) (Tuple t (- (bench-allocated alloc) b0)))))) + (Sample (ns ns) (bytes bytes) + (gc0 (cast System.Int64 (- (gc-count 0) g0))) + (gc1 (cast System.Int64 (- (gc-count 1) g1))) + (gc2 (cast System.Int64 (- (gc-count 2) g2))))) + +(: (-> long long double)) +(defun (per-call-ns ns k) (/ (cast System.Double ns) (cast System.Double k))) + +;; The batch size whose batch takes `target` nanoseconds: start at 1, and +;; multiply by what the last batch says is missing, at most by 10. +(: (-> CaseRun long long)) +(defun (calibrate run target) + (let go ((k 1L)) + (def t (time-batch run k)) + (cond + ((>= t target) k) + ((> k 1000000000000L) k) + (else + (def factor + (if (<= t 0L) + 10L + (min 10L (max 2L (cast System.Int64 (ceiling (/ (* 1.1 (cast System.Double target)) (cast System.Double t)))))))) + (go (* k factor)))))) + +;; Runs batches of `k` until the medians of the last two windows of five +;; batches agree within 2%, or the warm-up has had `limit` nanoseconds. +;; Answers whether it got steady. +(: (-> CaseRun long long bool)) +(defun (warm-up run k limit) + (def started (bench-now)) + (let go ((times (vec-empty))) + (def t (cast System.Double (time-batch run k))) + (def now-times (vec-add times t)) + (def n (vec-length now-times)) + (cond + ;; Not before half the limit: the JIT recompiles a hot method only + ;; once no new method has needed compiling for 100 ms, and then after + ;; 30 more calls, so a body can be steady at its first tier for a + ;; while before it gets faster. + ((and (>= n 10) + (>= (- (bench-now) started) (/ limit 2L)) + (let ((recent (median (vec-slice now-times (- n 5) 5))) + (before (median (vec-slice now-times (- n 10) 5)))) + (and (> before 0.0) (< (abs (- recent before)) (* 0.02 before))))) + #t) + ((>= (- (bench-now) started) limit) #f) + (else (go now-times))))) + +;; The half-width of a 95% interval of the median of sorted samples, from +;; the order statistics around it, as a fraction of the median. +(: (-> (Vec double) double)) +(defun (median-precision samples) + (def n (vec-length samples)) + (if (< n 5) + 1.0 + (let () + (def half (* 0.98 (sqrt (cast System.Double n)))) + (def lo (quantile samples (max 0.0 (- 0.5 (/ half (cast System.Double n)))))) + (def hi (quantile samples (min 1.0 (+ 0.5 (/ half (cast System.Double n)))))) + (def m (median samples)) + (if (<= m 0.0) 0.0 (/ (- hi lo) (* 2.0 m)))))) + +;; Measures one case, a calibrated one or `once` cold runs of one call. +(: (-> BenchCase int Settings Measured)) +(defun (measure-case c once s) + (def run ((record-ref c prepare))) + (if (> once 0) + (measure-once run once) + (measure-calibrated run s))) + +(: (-> CaseRun int Measured)) +(defun (measure-once run n) + (def taken + (loop (:for i (up-from 0 #:to n)) + (:do (progress #"cold run ${(+ i 1)}/${n}")) + (:acc out (vecing (take-sample run 1L))) + => out)) + (summed taken 1L #t #t (record-ref run call))) + +;; Calibrates to samples of `sample-ns` and warms up for at most `warmup-ns`: +;; the batch size, and whether the times got steady. +(: (-> CaseRun long long (Tuple long bool))) +(defun (prepare-run run sample-ns warmup-ns) + (progress "calibrating") + (def k0 (calibrate run sample-ns)) + (progress "warming up") + (def steady (warm-up run k0 warmup-ns)) + ;; The JIT may have made the body much faster while it warmed up. + (def k (let ((t (time-batch run k0))) + (if (and (> t 0L) (< (* 2L t) sample-ns)) + (calibrate-from run k0 sample-ns) + k0))) + (Tuple k steady)) + +(: (-> CaseRun Settings Measured)) +(defun (measure-calibrated run s) + ;; A `#:setup-every` call is timed on its own, so a sample is k such calls, + ;; and calibration counts them the same way. + (def (Tuple k steady) (prepare-run run (record-ref s sample-ns) (record-ref s warmup-ns))) + (def taken + (let go ((samples (vec-empty)) (spent 0L)) + (def one (take-sample run k)) + (def now (vec-add samples one)) + (def spent2 (+ spent (record-ref one ns))) + (def n (vec-length now)) + (def times (vec-map (fun (x) (per-call-ns (record-ref x ns) k)) now)) + ;; What is left, at the time the last sample took: the samples still + ;; owed to the minimum, or the time still owed to the target. + (def owed-ns (max (* (cast System.Int64 (max 0 (- (record-ref s min-samples) n))) (record-ref one ns)) + (- (record-ref s time-ns) spent2))) + (progress (str "sampling " (int->string n) "/" (int->string (record-ref s min-samples)) + " median " (time-text (median times)) + (if (> owed-ns 0L) (str " ~" (time-text (cast System.Double owed-ns)) " left") ""))) + (cond + ((>= n (record-ref s max-samples)) now) + ((>= spent2 (record-ref s max-ns)) now) + ((or (< n (record-ref s min-samples)) (< spent2 (record-ref s time-ns))) (go now spent2)) + ;; Enough of both: more only while the median is not known to ±1%. + ((> (median-precision times) 0.01) (go now spent2)) + (else now)))) + (summed taken k steady #f (record-ref run call))) + +(: (-> CaseRun long long long)) +(defun (calibrate-from run k target) + (let go ((k k)) + (def t (time-batch run k)) + (if (or (>= t target) (> k 1000000000000L)) + k + (go (* k (if (<= t 0L) 10L (min 10L (max 2L (cast System.Int64 (ceiling (/ (* 1.1 (cast System.Double target)) (cast System.Double t)))))))))))) + +(: (-> (Vec Sample) long bool bool (Option (-> (Tuple long long))) Measured)) +(defun (summed taken k steady once call) + (Measured (samples (vec-map (fun (x) (per-call-ns (record-ref x ns) k)) taken)) + (k k) + (calls (* k (cast System.Int64 (vec-length taken)))) + (bytes (vec-fold (fun (x acc) (+ acc (record-ref x bytes))) 0L taken)) + (gc0 (vec-fold (fun (x acc) (+ acc (record-ref x gc0))) 0L taken)) + (gc1 (vec-fold (fun (x acc) (+ acc (record-ref x gc1))) 0L taken)) + (gc2 (vec-fold (fun (x acc) (+ acc (record-ref x gc2))) 0L taken)) + (steady steady) + (once once) + (per-call (some? call)))) + +;; --------------------------------------------------------------------------- +;; Reporting +;; --------------------------------------------------------------------------- + +;; `v` with `digits` digits after the point. +(: (-> double int string)) +(defun (fixed v digits) + (def scale (pow 10.0 (cast System.Double digits))) + (def n (cast System.Int64 (round (* (abs v) scale)))) + (def whole (/ n (cast System.Int64 scale))) + (def frac (- n (* whole (cast System.Int64 scale)))) + (def sign (if (and (< v 0.0) (> n 0L)) "-" "")) + (if (= digits 0) + (str sign (long->string whole)) + (str sign (long->string whole) "." (string-pad-left (long->string frac) digits #:with #\0)))) + +;; Three significant digits. +(: (-> double string)) +(defun (three v) + (cond ((>= (abs v) 100.0) (fixed v 0)) + ((>= (abs v) 10.0) (fixed v 1)) + (else (fixed v 2)))) + +;; Nanoseconds, in the unit that reads best. +(: (-> double string)) +(defun (time-text ns) + (cond ((>= ns 1e9) (str (three (/ ns 1e9)) " s")) + ((>= ns 1e6) (str (three (/ ns 1e6)) " ms")) + ((>= ns 1e3) (str (three (/ ns 1e3)) " µs")) + (else (str (three ns) " ns")))) + +(: (-> double string)) +(defun (bytes-text b) + (cond ((>= b 1073741824.0) (str (three (/ b 1073741824.0)) " GB")) + ((>= b 1048576.0) (str (three (/ b 1048576.0)) " MB")) + ((>= b 1024.0) (str (three (/ b 1024.0)) " KB")) + (else (str (fixed b 0) " B")))) + +;; One row of the report: the cells, and the marks after them. +(type (: Row (Record (: name string) + (: labels (List (Tuple string string))) + (: cells (List string)) + (: notes (List string)) + (: extra (List string))))) + +;; The samples outside Tukey's fences and also more than 2% from the median. +;; On a quiet machine the fences of a tight benchmark are a fraction of a +;; percent apart, and a sample past them is no news. +(: (-> (Vec double) double int)) +(defun (far-outliers xs med) + (def q1 (quantile xs 0.25)) + (def q3 (quantile xs 0.75)) + (def fence (max (* 1.5 (- q3 q1)) (* 0.02 med))) + (loop (:for x xs) + (:acc (counting x) #:when (or (< x (- q1 fence)) (> x (+ q3 fence)))))) + +;; --------------------------------------------------------------------------- +;; Pictures of samples, for --detail +;; --------------------------------------------------------------------------- + +(def spark-levels "▁▂▃▄▅▆▇█") + +;; The level of `x` between `lo` and `hi`, as one of the eight blocks. +(: (-> double double double string)) +(defun (spark-char x lo hi) + (def i (if (<= hi lo) + 3 + (min 7 (max 0 (cast System.Int32 (floor (* 8.0 (/ (- x lo) (- hi lo))))))))) + (substring spark-levels (string-cursor-at spark-levels i) (string-cursor-at spark-levels (+ i 1)))) + +(: (-> string int StringCursor)) +(defun (string-cursor-at s i) + (let go ((c (string-cursor-start s)) (n i)) + (if (or (= n 0) (string-cursor-end? s c)) c (go (string-cursor-next s c) (- n 1))))) + +;; The samples in the order they were taken, at most `width` of them: more +;; are averaged in runs. What shows a drift, such as a body that slows down +;; as its heap grows. +(: (-> (Vec double) int string)) +(defun (sparkline xs width) + (def n (vec-length xs)) + (def shown + (if (<= n width) + xs + {vecing (mean (vec-slice xs (/ (* i n) width) (max 1 (- (/ (* (+ i 1) n) width) (/ (* i n) width))))) + (:for i (range 0 width))})) + (def lo (vec-fold min (vec-ref shown 0) shown)) + (def hi (vec-fold max (vec-ref shown 0) shown)) + (string-join (vec-map (fun (x) (spark-char x lo hi)) shown) "")) + +;; How many samples fall in each of `bins` equal steps from the lowest to the +;; highest, a block each, a space for none. What shows two values rather +;; than a spread around one. +(: (-> (Vec double) int string)) +(defun (histogram xs bins) + (def lo (vec-fold min (vec-ref xs 0) xs)) + (def hi (vec-fold max (vec-ref xs 0) xs)) + (def counts (make-array bins #:fill 0)) + (loop (:for x xs) + (:let i (if (<= hi lo) 0 (min (- bins 1) (cast System.Int32 (floor (* (cast System.Double bins) (/ (- x lo) (- hi lo)))))))) + (:do (array-set! counts i (+ 1 (array-ref counts i))))) + (def most (cast System.Double (loop (:for i (up-from 0 #:to bins)) (:acc m (folding 0 (max m (array-ref counts i)))) => m))) + (string-join {vecing (if (= (array-ref counts i) 0) + " " + (spark-char (cast System.Double (array-ref counts i)) 0.0 most)) + (:for i (range 0 bins))} + "")) + +;; Collections per 1000 calls, or per call for a run of single calls. +(: (-> Measured string)) +(defun (gc-text m) + (def calls (cast System.Double (record-ref m calls))) + (def per (if (record-ref m once) 1.0 1000.0)) + (defun (one (: n long)) : string + (if (= n 0L) "0" (three (* per (/ (cast System.Double n) calls))))) + (if (= 0L (+ (record-ref m gc0) (+ (record-ref m gc1) (record-ref m gc2)))) + "-" + (str (one (record-ref m gc0)) "/" (one (record-ref m gc1)) "/" (one (record-ref m gc2))))) + +(: (-> string (List (Tuple string string)) Measured bool Row)) +(defun (measured-row name labels m detail) + (def xs (record-ref m samples)) + (def med (median xs)) + (def spread (if (> med 0.0) (* 100.0 (/ (mad xs) med)) 0.0)) + (def lowest (vec-fold min (vec-ref xs 0) xs)) + (def outliers (far-outliers xs med)) + (def calls (cast System.Double (record-ref m calls))) + (def notes + (list-filter (fun (n) (not (string-empty? n))) + (list (if (and (not (record-ref m once)) (> spread 3.0)) "noisy" "") + (if (and (not (record-ref m once)) (not (record-ref m steady))) "unsteady" "") + (if (and (not (record-ref m once)) (> outliers 0)) + (str (int->string outliers) (if (= outliers 1) " outlier" " outliers")) "") + (if (and (record-ref m per-call) (< med 1000.0)) "setup-every: below resolution" "")))) + (def extra + (if (record-ref m once) + (list (str "(" (string-join (vec-map time-text xs) ", ") ")")) + (if detail + (list (str "mean " (time-text (mean xs)) ", sd " (time-text (std-dev xs)) + ", q1 " (time-text (quantile xs 0.25)) ", q3 " (time-text (quantile xs 0.75)) + ", " (int->string (vec-length xs)) " samples of " (long->string (record-ref m k)) " calls") + (str "in order " (sparkline xs 60)) + (str "histogram " (time-text lowest) " " (histogram xs 24) " " + (time-text (vec-fold max (vec-ref xs 0) xs)))) + (list)))) + (Row (name name) (labels labels) + (cells (list (time-text med) + (if (record-ref m once) "-" (str (fixed spread 1) "%")) + (time-text lowest) + (bytes-text (/ (cast System.Double (record-ref m bytes)) calls)) + (gc-text m))) + (notes notes) + (extra extra))) + +(: (-> int (List string))) +(defun (header-lines count) + (def facts + (str "bench: " (int->string count) (if (= count 1) " benchmark, " " benchmarks, ") + (if (debug-build?) "Debug build, " "Release build, ") + (dotnet-version) ", " (arch-name) ", " (int->string (core-count)) " cores, gcServer=" + (if (server-gc?) "true" "false"))) + (def gov (governor)) + (list-filter (fun (l) (not (string-empty? l))) + (list facts + (str "bench: " (cpu-model) ", " (os-name)) + (if (debug-build?) "bench: warning: a Debug build measures code the JIT did not optimize" "") + (if (debugger?) "bench: warning: a debugger is attached" "") + (if (and (not (string-empty? gov)) (not (= gov "performance"))) + #"bench: warning: CPU frequency governor is '${gov}'; results will be noisy" "")))) + +;; What alloc/op counts: a line when it is every thread's, by `--alloc` or by +;; a benchmark's `#:alloc`. The default, this thread's, says nothing. +(: (-> (List Selection) (List string))) +(defun (alloc-lines chosen) + (def default (box-ref alloc-default)) + (def others + (loop (:for p chosen) + (:let b (sel-bench p)) + (:when (not (= (alloc-counted (record-ref b alloc)) default))) + (:acc (listing (record-ref b name))))) + (def what (fun (mode) (if (= mode "process") "every thread of the process" "the measuring thread"))) + (def names (string-join (list->vec others) ", ")) + (cond + ((list-empty? others) + (if (= default "process") (list #"bench: alloc/op counts ${(what default)}") (list))) + (else + (list #"bench: alloc/op counts ${(what default)}, and ${(what (if (= default "process") "thread" "process"))} for ${names} (#:alloc)")))) + +;; What a run is asked to measure: benchmarks, each with the cases chosen. +(type (: Selection (Record (: bench Bench) (: cases (List BenchCase))))) + +(: (-> Selection Bench)) +(defun (sel-bench p) (record-ref p bench)) + +(: (-> Selection (List BenchCase))) +(defun (sel-cases p) (record-ref p cases)) + +;; The layout of a table: the parameters it has columns for, and the widths +;; and alignments of its columns, from `(std fmt)`. Fixed before the first +;; row, so that a row can be printed the moment it is measured and a long +;; run read while it goes. +(type (: Layout (Record (: params (List string)) + (: heads (List string)) + (: table TableLayout)))) + +;; The cells after the parameters are numbers, right-aligned, of a width +;; known in advance; a longer one pushes its row out rather than the column. +;; After them come the notes, unaligned. +(def number-heads (list "median" "±" "min" "alloc/op" "gc/1k 0/1/2")) +(def number-widths (list 9 6 9 9 11)) + +(: string) +(def table-sep " ") + +;; The label of parameter `p` among a case's, or nothing. +(: (-> (List (Tuple string string)) string string)) +(defun (label-of labels p) + (match (find (fun (l) (= (fst l) p)) labels) + ((Some l) (snd l)) + (None ""))) + +(: (-> (List string) (Vec Block))) +(defun (blocks cells) (list->vec (list-map BStr cells))) + +(: (-> (List Selection) bool Layout)) +(defun (layout-for chosen scaling) + (def params (selection-params chosen)) + (def heads (if scaling (list-append number-heads (list "scaling")) number-heads)) + (def head-widths (if scaling (list-append number-widths (list 14)) number-widths)) + ;; The name and parameter columns are as wide as their widest cell. + (def label-rows + (cons (blocks (cons "name" params)) + (fold (fun (p acc) + (list-append acc + (list-map (fun (c) + (blocks (cons (record-ref (sel-bench p) name) + (list-map (fun (n) (label-of (case-labels c) n)) + params)))) + (sel-cases p)))) + (list) chosen))) + (def widths + (list->vec (list-append (vec->list (table-widths (list->vec label-rows))) + (list-map (fun (p) (match p ((Tuple h w) (max w (string-count h))))) + (list-zip heads head-widths))))) + (def aligns + (list->vec (list-append (list-map (fun (n) AlignLeft) (cons "name" params)) + (list-map (fun (h) AlignRight) heads)))) + (Layout (params params) (heads heads) (table (table-layout widths #:sep table-sep #:align aligns)))) + +(: (-> Layout void)) +(defun (print-head l) + (println "") + (show [(table-row (record-ref l table) + (blocks (list-append (cons "name" (record-ref l params)) (record-ref l heads))))])) + +(: (-> Layout Row void)) +(defun (print-row l r) + (def notes (record-ref r notes)) + (def cells + (list-append (cons (record-ref r name) + (list-map (fun (p) (label-of (record-ref r labels) p)) (record-ref l params))) + (list-append (record-ref r cells) + (if (list-empty? notes) (list) (list (string-join (list->vec notes) ", ")))))) + (show [(table-row (record-ref l table) (blocks cells))]) + ;; Extra lines start under the first column after the name. + (def indent (+ (vec-ref (record-ref (record-ref l table) widths) 0) (string-count table-sep))) + (loop (:for e (record-ref r extra)) + (:do (println (str (string-pad-right "" indent) e))))) + + +;; --------------------------------------------------------------------------- +;; Running +;; --------------------------------------------------------------------------- + +;; The parameters of what is chosen, each once, in the order first met. +(: (-> (List Selection) (List string))) +(defun (selection-params chosen) + (fold (fun (p acc) + (list-append acc (list-filter (fun (n) (not (any? (fun (m) (= m n)) acc))) + (record-ref (sel-bench p) param-names)))) + (list) chosen)) + +(: (-> (List Selection) int)) +(defun (case-count chosen) (fold (fun (p n) (+ n (list-length (sel-cases p)))) 0 chosen)) + +;; Whether a filter chooses a benchmark's case: its name, or its parameters as +;; the report shows them, `n=1000`. +(: (-> (Vec string) Bench BenchCase bool)) +(defun (chosen? filters b c) + (or (vec-empty? filters) + (any? (fun (f) + (or (string-contains? (record-ref b name) f) + (any? (fun (l) (= (str (fst l) "=" (snd l)) f)) (case-labels c)))) + filters))) + +;; The benchmarks the filters choose, with their cases. +(: (-> (List Bench) (Vec string) (List Selection))) +(defun (filtered benches filters) + (loop (:for b benches) + (:let picked (list-filter (fun (c) (chosen? filters b c)) ((record-ref b cases)))) + (:when (not (list-empty? picked))) + (:acc (listing (Selection (bench b) (cases picked)))))) + +;; Whether there is someone to show a progress line to: a terminal, and not +;; a comint buffer, which would show every rewrite as a line of its own. +(: (-> bool)) +(defun (progress-line?) + (and (output-terminal?) + (none? (get-environment-variable "INSIDE_EMACS")) + (none? (get-environment-variable "CI")))) + +;; Measures what is chosen and prints the table, a row as each case is done. +;; Answers what was measured, as a saved run holds it; a skipped case is not +;; in it. +(: (-> (List Selection) BenchOptions (List SavedCase))) +(defun (run-selection chosen o) + (def s (bench-settings o)) + (def count (case-count chosen)) + (loop (:for l (header-lines count)) (:do (println l))) + (loop (:for l (alloc-lines chosen)) (:do (println l))) + (def layout (layout-for chosen (record-ref o scaling))) + (print-head layout) + (flush-port (parameter-ref current-output-port)) + (def live (progress-line?)) + (def/mutable done 0) + (def/mutable measured (list)) + ;; For --scaling: the benchmark, parameter value and median of the row + ;; before. + (: (Box (Option (Tuple string (Tuple double double))))) + (def previous (make-box None)) + (loop (:for p chosen) + (:let b (sel-bench p)) + (:subloop (:for c (sel-cases p)) + (:do (set! done (+ done 1))) + (:let prefix + (str "[" (int->string done) "/" (int->string count) "] " (record-ref b name) + (string-join (list->vec (list-map (fun (l) (str " " (fst l) "=" (snd l))) + (case-labels c))) + "") + " ")) + (:do (when live + (box-set! progress-box + (fun (what) + (print (str "\r" prefix what "\u{1B}[K")) + (flush-port (parameter-ref current-output-port)))))) + (:let m (if (string-empty? (record-ref b skip)) + (Some (measure-case c (record-ref b once) s)) + None)) + (:let row + (match m + ((Some m) + (def r (measured-row (record-ref b name) (case-labels c) m (record-ref o detail))) + (if (record-ref o scaling) + (record-set r (cells (list-append (record-ref r cells) + (list (scaling-text previous b c (median (record-ref m samples))))))) + r)) + (None + (Row (name (record-ref b name)) (labels (case-labels c)) + (cells (list)) + (notes (list (str "skipped: " (record-ref b skip)))) (extra (list)))))) + (:do (when live (print "\r\u{1B}[K")) + (print-row layout row) + (flush-port (parameter-ref current-output-port)) + (match m + ((Some m) (set! measured (cons (saved-case (record-ref b name) (case-labels c) m) measured))) + (None unit))))) + (box-set! progress-box (fun (what) unit)) + (list-reverse measured)) + +(: (-> string (List (Tuple string string)) Measured SavedCase)) +(defun (saved-case name labels m) + (SavedCase (name name) (labels labels) (k (record-ref m k)) (once (record-ref m once)) + (bytes (/ (cast System.Double (record-ref m bytes)) (cast System.Double (max 1L (record-ref m calls))))) + (samples (record-ref m samples)))) + +;; `×98.2 for ×100`: how the median grew from the row before, against how +;; the benchmark's one numeric parameter did. Nothing for the first row of a +;; benchmark, or one whose parameters are not one number. +(: (-> (Box (Option (Tuple string (Tuple double double)))) Bench BenchCase double string)) +(defun (scaling-text previous b c med) + (def value + (match (case-labels c) + ((Cons (Tuple _ v) Nil) (parse-double v)) + (_ None))) + (match value + (None (box-set! previous None) "") + ((Some x) + (def text + (match (box-ref previous) + ((Some (Tuple name (Tuple x0 m0))) #:when (= name (record-ref b name)) + (if (and (> x0 0.0) (> m0 0.0)) + (str "×" (three (/ med m0)) " for ×" (three (/ x x0))) + "")) + (_ ""))) + (box-set! previous (Some (Tuple (record-ref b name) (Tuple x med)))) + text))) + +;; The facts of this machine and build, as a saved run keeps them. +(: (-> (List (Tuple string string)))) +(defun (header-facts) + (list (Tuple "build" (if (debug-build?) "Debug" "Release")) + (Tuple "dotnet" (dotnet-version)) + (Tuple "arch" (arch-name)) + (Tuple "cores" (int->string (core-count))) + (Tuple "gc-server" (if (server-gc?) "true" "false")) + (Tuple "cpu" (cpu-model)) + (Tuple "os" (os-name)) + (Tuple "governor" (governor)))) + +(: (-> (List SavedCase) SavedRun)) +(defun (this-run cases) + (SavedRun (saved (now-utc)) (machine (machine-key)) (header (header-facts)) (cases cases))) + +;; `+8.9%`: a change, signed, to a tenth of a percent. +(: (-> double string)) +(defun (change-text x) + (def t (fixed (* 100.0 x) 1)) + (str (if (string-starts-with? t "-") "" "+") t "%")) + +;; A percentage as it was given: `2%`, `1.5%`. +(: (-> double string)) +(defun (percent-text x) + (def v (* 100.0 x)) + (str (if (= v (round v)) (fixed v 0) (fixed v 1)) "%")) + +(: (-> double string)) +(defun (p-text p) + (cond ((< p 0.001) "<0.001") + ((< p 0.1) (fixed p 3)) + (else (fixed p 2)))) + +(: (-> Verdict double string)) +(defun (verdict-text v threshold) + (match v + (Faster "faster") + (Slower "SLOWER") + (SmallChange #"no change (< ${(percent-text threshold)})") + (NoChange "no change"))) + +(: (-> Comparison (List string) BenchOptions bool bool bool (List string))) +(defun (comparison-row c params o other spread? ratio?) + (def labels (record-ref c labels)) + (list-append + (cons (record-ref c name) (list-map (fun (p) (label-of labels p)) params)) + (list-append + (list (time-text (record-ref c was)) + (time-text (record-ref c now)) + (change-text (record-ref c change)) + (str (change-text (record-ref c lo)) " .. " (change-text (record-ref c hi))) + (p-text (record-ref c p))) + (list-append + (list-append + (if ratio? (list (str "×" (fixed (+ 1.0 (record-ref c change)) 2))) (list)) + (if (record-ref o p-raw) (list (p-text (record-ref c p-raw))) (list))) + (list-append + (if spread? (list (str (fixed (* 100.0 (record-ref c spread)) 1) "%")) (list)) + (list (str (verdict-text (record-ref c verdict) (record-ref o threshold)) + (if other " (other machine)" "")))))))) + +(: (-> Comparison (List (Tuple string string)))) +(defun (comparison-labels c) (record-ref c labels)) + +(: (-> Comparison double)) +(defun (comparison-spread c) (record-ref c spread)) + +;; The comparison with a saved run, after the run's own table. +(: (-> (Option string) SavedRun CompareResult BenchOptions void)) +(defun (print-comparison file saved result o) + (def path (match file ((Some p) p) (None ""))) + (def n (list-length (record-ref result comparisons))) + (println "") + (println (str "bench: compared with " path ", saved " (record-ref saved saved) + "; unpaired (Mann-Whitney U); " (comparisons-text n))) + (println "bench: one process each: a difference between processes is not in the p-values") + (print-comparison-table result o "was" "now" path #f)) + +;; `3 comparisons, Holm-corrected`. +(: (-> int string)) +(defun (comparisons-text n) + (str (int->string n) (if (= n 1) " comparison" " comparisons") (if (> n 1) ", Holm-corrected" ""))) + +;; A comparison's warnings, its table and what one side has alone, under the +;; header its caller printed: `was` and `now` name the two medians' columns, +;; and `other-side` is what the side `was` is from is called, for a case +;; only it has. +(: (-> CompareResult BenchOptions string string string bool void)) +(defun (print-comparison-table result o was-head now-head other-side ratio?) + (def comparisons (record-ref result comparisons)) + (def other (record-ref result other-machine)) + (def n (list-length comparisons)) + (when other + (println "bench: warning: the saved run is from another machine: the numbers are shown, and no verdict is to be believed")) + (loop (:for w (record-ref result warnings)) + (:do (println (str "bench: warning: " w)))) + (when (> n 0) + (def params + (fold (fun (c acc) + (list-append acc (list-filter (fun (p) (not (has-string? acc p))) + (list-map fst (comparison-labels c))))) + (list) comparisons)) + (def spread? (any? (fun (c) (>= (comparison-spread c) 0.0)) comparisons)) + (def heads + (list-append (cons "name" params) + (list-append (list was-head now-head "change" "95% interval" "p") + (list-append (list-append (if ratio? (list "ratio") (list)) + (if (record-ref o p-raw) (list "p raw") (list))) + (list-append (if spread? (list "between processes") (list)) + (list "verdict")))))) + (def numbers (+ 5 (if ratio? 1 0) (if (record-ref o p-raw) 1 0) (if spread? 1 0))) + (def aligns + (list->vec (list-append (list-map (fun (h) AlignLeft) (cons "name" params)) + (list-append (list-map (fun (i) AlignRight) (index-list numbers)) + (list AlignLeft))))) + (println "") + (show [(tabular/rows (list->vec (cons (blocks heads) + (list-map (fun (c) (blocks (comparison-row c params o other spread? ratio?))) + comparisons))) + #:sep table-sep #:align aligns)])) + (unless (list-empty? (record-ref result only-now)) + (println (str "bench: only in this run: " (string-join (list->vec (record-ref result only-now)) ", ")))) + (unless (list-empty? (record-ref result only-saved)) + (println (str "bench: only in " other-side ": " (string-join (list->vec (record-ref result only-saved)) ", ")))) + (match (record-ref o fail-slower) + ((Some _) #:when other + (println "bench: --fail-slower does not apply: the saved run is from another machine")) + (_ unit))) + +;; Whether `--fail-slower PCT` fails the run: something SLOWER by more than +;; PCT, against a run from this machine. +(: (-> CompareResult BenchOptions bool)) +(defun (fails-slower? result o) + (match (record-ref o fail-slower) + ((Some limit) + (and (not (record-ref result other-machine)) + (any? (fun (c) (and (match (record-ref c verdict) (Slower #t) (_ #f)) + (> (record-ref c change) limit))) + (record-ref result comparisons)))) + (None #f))) + +;; Runs what is chosen, then saves and compares as the options say. Answers +;; the exit code: 1 when `--fail-slower` is exceeded, 2 when the file to +;; compare with cannot be used, 0 otherwise. +(: (-> (List Selection) BenchOptions int)) +(defun (run-and-report chosen o) + ;; A comparison file that cannot be read is known before anything is run. + (def baseline + (option-map read-saved-run (record-ref o compare))) + (match baseline + ((Some (Err why)) (println #"bench: --compare: ${why}") 2) + (_ + (def run (this-run (run-selection chosen o))) + (match (record-ref o save) + ((Some path) + (match (try (write-saved-run! path run) #:catch (System.Exception)) + ((Ok _) (println #"bench: saved to ${path}")) + (_ (println #"bench: --save: ${path} could not be written")))) + (None unit)) + (when (record-ref o compare-within) (compare-within chosen o)) + (match baseline + ((Some (Ok saved)) + (def result (compare-runs saved run (record-ref o alpha) (record-ref o threshold))) + (print-comparison (record-ref o compare) saved result o) + (if (fails-slower? result o) 1 0)) + (_ 0))))) + +;; --------------------------------------------------------------------------- +;; --compare-within: the cases of a group against its first, in rounds +;; --------------------------------------------------------------------------- + +;; The groups of what is chosen, in the order first met, each with its cases +;; that are not skipped: a benchmark's cases one after another. +(: (-> (List Selection) (List (Tuple string (List (Tuple Bench BenchCase)))))) +(defun (chosen-groups chosen) + (fold (fun (p acc) + (def b (sel-bench p)) + (def g (record-ref b group)) + (def cases (list-map (fun (c) (Tuple b c)) (sel-cases p))) + (cond + ((or (string-empty? g) (not (string-empty? (record-ref b skip)))) acc) + ((has-group? acc g) + (list-map (fun (e) (if (= (fst-of e) g) (Tuple g (list-append (snd-of e) cases)) e)) acc)) + (else (list-append acc (list (Tuple g cases)))))) + (list) chosen)) + +(: (-> (List (Tuple string (List (Tuple Bench BenchCase)))) string bool)) +(defun (has-group? groups g) (any? (fun (e) (= (fst-of e) g)) groups)) + +(: (-> (Tuple string (List (Tuple Bench BenchCase))) string)) +(defun (fst-of e) (match e ((Tuple a _) a))) + +(: (-> (Tuple string (List (Tuple Bench BenchCase))) (List (Tuple Bench BenchCase)))) +(defun (snd-of e) (match e ((Tuple _ b) b))) + +;; `sort n=1000`: a case as a comparison names it. +(: (-> Bench BenchCase string)) +(defun (case-key-of b c) + (string-join (list->vec (cons (record-ref b name) + (list-map (fun (l) (str (fst l) "=" (snd l))) (case-labels c)))) + " ")) + +;; Every group of two cases or more, measured again in `-N` rounds in this +;; process: each case prepared, and each round a sample of every case, in an +;; order shuffled from `--seed`. Each case is compared with the group's first +;; one, paired by round, as A/B pairs its sides; there is one process, so a +;; difference between processes is not in it to hide. +(: (-> (List Selection) BenchOptions void)) +(defun (compare-within chosen o) + (def s (bench-settings o)) + (def seed (match (record-ref o seed) ((Some n) n) (None (cast System.UInt64 (bench-now))))) + (def rng (make-random/seed64 seed)) + (def rounds (record-ref o rounds)) + (def live (progress-line?)) + (loop (:for g (chosen-groups chosen)) + (:let (Tuple name members) g) + (:when (>= (list-length members) 2)) + (:let cases (list->vec members)) + (:let n (vec-length cases)) + (:let runs + (vec-map (fun (bc) + (match bc + ((Tuple b c) + (def run ((record-ref c prepare))) + (def k (if (> (record-ref b once) 0) + 1L + (match (prepare-run run (record-ref s sample-ns) (record-ref s warmup-ns)) + ((Tuple k _) k)))) + (Tuple run k)))) + cases)) + (:let taken (make-array n #:fill (vec-empty))) + (:do (loop (:for r (up-from 0 #:to rounds)) + (:do (when live + (print (str "\r[" name "] round " (int->string (+ r 1)) "/" (int->string rounds) "\u{1B}[K")) + (flush-port (parameter-ref current-output-port)))) + (:subloop (:for i (shuffle-vec (list->vec (index-list n)) #:rng rng)) + (:let (Tuple run k) (vec-ref runs i)) + (:let x (take-sample run k)) + (:do (array-set! taken i (vec-add (array-ref taken i) + (per-call-ns (record-ref x ns) k)))))) + (when live (print "\r\u{1B}[K"))) + (:let (Tuple b0 c0) (vec-ref cases 0)) + (:let base (case-key-of b0 c0)) + (:let paired + (loop (:for i (up-from 1 #:to n)) + (:let (Tuple b c) (vec-ref cases i)) + (:acc out (listing (PairedCase (name (record-ref b name)) (labels (case-labels c)) + (a [(array-ref taken 0)]) (b [(array-ref taken i)])))))) + (:let result (compare-paired paired (list) (list) (record-ref o alpha) (record-ref o threshold))) + (:do (println "") + (println (str "bench: within " name ", against " base ": " (int->string rounds) + " rounds in this process, paired (Wilcoxon signed-rank); " + (comparisons-text (list-length paired)))) + (println (str "bench: seed " (displayed seed) " (--seed repeats the order of the cases)")) + (print-comparison-table result o base "this" base #t)))) + +;; --------------------------------------------------------------------------- +;; The menu +;; --------------------------------------------------------------------------- + +;; The menu's lines: the benchmarks numbered from 0, under their groups, each +;; with its parameters and their values. +(: (-> string (List Bench) (List string))) +(defun (menu-lines file benches) + (def/mutable last-group "") + (def lines + (loop (:for b benches) + (:with i 0 (+ i 1)) + (:let group (record-ref b group)) + (:let heading + (if (and (not (string-empty? group)) (not (= group last-group))) + (list (str " " group)) + (list))) + (:do (set! last-group group)) + (:let indent (if (string-empty? group) " " " ")) + (:let params (param-summary b)) + (:acc out (folding (list) (list-append out (list-append heading + (list (string-trim-right + (str indent "#" (int->string i) " " + (string-pad-right (record-ref b name) 18) + params))))))) + => out)) + (list-append (cons (str "Benchmarks in " file ":") (cons "" lines)) + (list "" + "Choose by number (0 2-4), by name (sort), with parameters (2 n=1000)," + "or * for all. Nothing quits, or repeats the last choice once there is one."))) + +;; `n: 10 1000 order: sorted random`. +(: (-> (List string) string bool)) +(defun (has-string? xs s) (any? (fun (x) (= x s)) xs)) + +(: (-> BenchCase (List (Tuple string string)))) +(defun (case-labels c) (record-ref c labels)) + +(: (-> Bench string)) +(defun (param-summary b) + (: (List BenchCase)) + (def cases ((record-ref b cases))) + (: (-> string (List string))) + (defun (values-of n) + (fold (fun (c acc) + (match (find (fun (l) (= (fst l) n)) (case-labels c)) + ((Some l) (if (has-string? acc (snd l)) acc (list-append acc (list (snd l))))) + (None acc))) + (list) cases)) + (string-join + (list->vec + (list-map (fun (n) (str n ": " (string-join (list->vec (values-of n)) " "))) + (record-ref b param-names))) + " ")) + +;; A menu choice: numbers, ranges of them, names, `*`, and after a benchmark, +;; `p=v,w` to narrow its cases to those values of parameter `p`. +(: (-> string (List Bench) (Result string (List Selection)))) +(defun (parse-choice text benches) + (def tokens (list-filter (fun (t) (not (string-empty? t))) (vec->list (string-split (string-trim text) " ")))) + (defun (all-of (: b Bench)) : Selection (Selection (bench b) (cases ((record-ref b cases))))) + (let go ((left tokens) (acc (list)) (last (list))) + ;; `acc` is what is chosen so far, `last` the selections the most recent + ;; token made, which a `p=v` narrows. + (match left + (Nil (Ok (list-append acc last))) + ((Cons t more) + (cond + ((= t "*") (go more (list-append acc last) (list-map all-of benches))) + ((some? (split-once t "=")) + (def (Some (Tuple p vs)) (split-once t "=") :leave-with (Err t)) + (def wanted (vec->list (string-split vs ","))) + (if (list-empty? last) + (Err #"${t} narrows the benchmark before it, and there is none.") + (go more acc + (list-map (fun (sel) + (Selection (bench (sel-bench sel)) + (cases (list-filter (fun (c) + (any? (fun (l) (and (= (fst l) p) + (any? (fun (w) (= w (snd l))) wanted))) + (case-labels c))) + (sel-cases sel))))) + last)))) + (else + (match (choice-indices t (list-length benches)) + ((Some is) + (go more (list-append acc last) + (list-map (fun (i) (all-of (list-ref benches i))) is))) + (None + (def named (list-filter (fun (b) (string-contains? (record-ref b name) t)) benches)) + (cond + ((some? (parse-int t)) + (Err #"There is no benchmark ${t}: they are numbered 0 to ${(- (list-length benches) 1)}.")) + ((list-empty? named) + (Err #"Nothing is called ${t}, and it is not a number of one.")) + (else + (go more (list-append acc last) (list-map all-of named)))))))))))) + +;; `3` or `2-4`, as indices below `n`. +(: (-> string int (Option (List int)))) +(defun (choice-indices t n) + (match (split-once t "-") + ((Some (Tuple a b)) + (match (Tuple (parse-int a) (parse-int b)) + ((Tuple (Some lo) (Some hi)) #:when (and (<= 0 lo) (<= lo hi) (< hi n)) + (Some {listing i (:for i (range lo (+ hi 1)))})) + (_ None))) + (None + (match (parse-int t) + ((Some i) #:when (and (<= 0 i) (< i n)) (Some (list i))) + (_ None))))) + +;; The menu: ask, run, and ask again, until nothing is chosen with nothing to +;; repeat, `q`, or the end of the input. +(: (-> string (List Bench) BenchOptions (-> (Option string)) void)) +(defun (bench-menu file benches o read) + (menu-loop (menu-lines file benches) benches read + (fun (chosen) (ignore (run-and-report chosen o))))) + +;; The menu's asking, with the lines to show first and what to do with a +;; choice: `bench-menu` runs it here, and `bjo bench`, over several files, +;; runs each file with the cases that fall in it. +(: (-> (List string) (List Bench) (-> (Option string)) (-> (List Selection) void) void)) +(defun (menu-loop lines benches read run) + (loop (:for l lines) (:do (println l))) + (let ask ((previous None)) + (print "bench> ") + (flush-port (parameter-ref current-output-port)) + (match (read) + (None (println "")) + ((Some line) + (def text (string-trim line)) + (cond + ((= text "q") unit) + ((string-empty? text) + (match previous + (None unit) + ((Some chosen) (run chosen) (ask previous)))) + (else + (match (parse-choice text benches) + ((Err why) (println why) (ask previous)) + ((Ok chosen) + (if (= (case-count chosen) 0) + (let () (println "That chooses nothing.") (ask previous)) + (let () (run chosen) (ask (Some chosen)))))))))))) + +;; Whether to ask what to run: with no names and no `--all`, at a terminal, +;; and not in CI. `BJO_BENCH_MENU=1` asks anyway, which is how it is tested. +(: (-> BenchOptions bool)) +(defun (menu? o) + (and (vec-empty? (record-ref o filters)) + (not (record-ref o all)) + (none? (record-ref o cases)) + (or (= (get-environment-variable "BJO_BENCH_MENU") (Some "1")) + (and (input-terminal?) (output-terminal?) + (none? (get-environment-variable "CI")))))) + +;; --------------------------------------------------------------------------- +;; --serve: driven by another program +;; --------------------------------------------------------------------------- +;; +;; A command a line on standard input, a reply a line on standard output, +;; both bjodat. Cases are numbered from 0 in the order `(list)` gives them. +;; +;; (list) → (cases (case 0 (name "sort") (labels ("n" "10")) (once 0) (skip "")) ...) +;; (prepare 3 :warmup 1000 :sample 10) → (prepared 3 :k 4096 :steady #t) +;; (sample 3 :k 4096) → (sample 3 :ns 41213 :bytes 0 :gc (0 0 0)) +;; (quit) → nothing; the program ends +;; +;; Keywords are bjodat's, `:k`; `#:k` reads as the same keyword. `:warmup` +;; and `:sample` are milliseconds, 1000 and 10 unless given. A sample's `:ns` +;; is the whole sample's, `k` calls. Anything wrong is `(error "why")`, and +;; the next command is read. `prepare` +;; runs the case's setup; a case is sampled only once it is prepared. + +(: (-> string Bjodat)) +(defun (sym s) (BjoSym (string->symbol s))) + +(: (-> string Bjodat)) +(defun (kw s) (BjoKey (string->keyword s))) + +(: (-> string (List Bjodat) Bjodat)) +(defun (form head xs) (BjoList (cons (sym head) xs))) + +(: (-> int Bjodat)) +(defun (long-of n) (BjoInt (cast System.Int64 n))) + +;; The value after keyword `name` in a command's arguments. +(: (-> (List Bjodat) string (Option Bjodat))) +(defun (keyword-arg args name) + (match args + ((Cons (BjoKey k) (Cons v more)) (if (= (keyword->string k) name) (Some v) (keyword-arg more name))) + ((Cons _ more) (keyword-arg more name)) + (Nil None))) + +(: (-> (List Bjodat) string long (Option long))) +(defun (long-arg args name default) + (match (keyword-arg args name) + (None (Some default)) + ((Some (BjoInt n)) (Some n)) + (_ None))) + +(: (-> string Bjodat)) +(defun (serve-error why) (form "error" (list (BjoStr why)))) + +(: (-> int Bench BenchCase Bjodat)) +(defun (case-listing i b c) + (form "case" (list (long-of i) + (form "name" (list (BjoStr (record-ref b name)))) + (form "labels" (list-map (fun (l) (BjoList (list (BjoStr (fst l)) (BjoStr (snd l))))) + (case-labels c))) + (form "group" (list (BjoStr (record-ref b group)))) + (form "once" (list (long-of (record-ref b once)))) + (form "skip" (list (BjoStr (record-ref b skip))))))) + +;; Answers one command. `runs` holds the prepared cases. +(: (-> Bjodat (Vec (Tuple Bench BenchCase)) (Array (Option CaseRun)) Bjodat)) +(defun (serve-reply command cases runs) + (def n (vec-length cases)) + (defun (case-at (: i long)) : (Option (Tuple Bench BenchCase)) + (if (and (>= i 0L) (< i (cast System.Int64 n))) (Some (vec-ref cases (cast System.Int32 i))) None)) + (match command + ((BjoList (Cons (BjoSym head) args)) + (case (symbol->string head) + ("list" + (form "cases" (loop (:for p cases) (:with i 0 (+ i 1)) + (:acc (listing (match p ((Tuple b c) (case-listing i b c)))))))) + ("prepare" + (match (Tuple args (long-arg args "warmup" 1000L) (long-arg args "sample" 10L)) + ((Tuple (Cons (BjoInt i) _) (Some warmup-ms) (Some sample-ms)) + (match (case-at i) + (None (serve-error #"there is no case ${i}")) + ((Some (Tuple b c)) + (if (not (string-empty? (record-ref b skip))) + (serve-error #"case ${i} is skipped: ${(record-ref b skip)}") + (match (try (let () + (def run ((record-ref c prepare))) + (array-set! runs (cast System.Int32 i) (Some run)) + (if (> (record-ref b once) 0) + (Tuple 1L #t) + (prepare-run run (* sample-ms 1000000L) (* warmup-ms 1000000L)))) + #:catch (System.Exception)) + ((Ok (Tuple k steady)) + (form "prepared" (list (BjoInt i) (kw "k") (BjoInt k) (kw "steady") (BjoBool steady)))) + ((Err e) (serve-error #"case ${i} failed to prepare: ${(.-Message e)}"))))))) + (_ (serve-error "prepare takes a case number, and :warmup and :sample in milliseconds")))) + ("sample" + (match (Tuple args (long-arg args "k" 1L)) + ((Tuple (Cons (BjoInt i) _) (Some k)) + (match (if (some? (case-at i)) (array-ref runs (cast System.Int32 i)) None) + (None (serve-error #"case ${i} is not prepared")) + ((Some run) + (match (try (take-sample run k) #:catch (System.Exception)) + ((Ok x) + (form "sample" (list (BjoInt i) (kw "ns") (BjoInt (record-ref x ns)) + (kw "bytes") (BjoInt (record-ref x bytes)) + (kw "gc") (BjoList (list (BjoInt (record-ref x gc0)) (BjoInt (record-ref x gc1)) + (BjoInt (record-ref x gc2))))))) + ((Err e) (serve-error #"case ${i} failed: ${(.-Message e)}")))))) + (_ (serve-error "sample takes a case number, and :k")))) + (else (serve-error #"unknown command ${(symbol->string head)}: the commands are list, prepare, sample and quit")))) + (_ (serve-error "a command is a form: (list), (prepare i ...), (sample i ...) or (quit)")))) + +(: (-> Bench (List BenchCase))) +(defun (cases-of b) ((record-ref b cases))) + +;; Whether a command is `(quit)`. +(: (-> Bjodat bool)) +(defun (quit? command) + (match command + ((BjoList (Cons (BjoSym head) _)) (= (symbol->string head) "quit")) + (_ #f))) + +;; Reads commands until `(quit)` or the end of the input. +(: (-> (List Bench) (-> (Option string)) void)) +(defun (bench-serve benches read) + (def cases + (list->vec (fold (fun (b acc) (list-append acc (list-map (fun (c) (Tuple b c)) (cases-of b)))) + (list) benches))) + (def runs (make-array (vec-length cases) #:fill None)) + (let go () + (match (read) + (None unit) + ((Some line) + (match (parse-bjodat line) + ((Ok command) + (unless (quit? command) + (println (bjodat->string (serve-reply command cases runs))) + (flush-port (parameter-ref current-output-port)) + (go))) + ((Err why) + (println (bjodat->string (serve-error why))) + (flush-port (parameter-ref current-output-port)) + (go))))))) + +(: (-> string (Vec string) int)) +(defun (run-registered-benches module args) + (match (parse-bench-args args) + ((Err why) (println #"bench: ${why}") 2) + ((Ok o) (run-with-options module o)))) + +(: (-> string BenchOptions int)) +(defun (run-with-options module o) + (bench-params-set! (record-ref o params)) + (box-set! alloc-default (record-ref o alloc)) + (def benches (module-benches module)) + (cond + ((record-ref o serve) + (bench-serve benches (fun () (read-line/opt (parameter-ref current-input-port)))) + 0) + ((record-ref o list) + (loop (:for b benches) + (:do (println (str (record-ref b name) + (if (list-empty? (record-ref b param-names)) + "" + (str " " (string-join (list->vec (record-ref b param-names)) " "))))))) + 0) + ((list-empty? benches) (println "There are no benchmarks.") 0) + ((menu? o) + (def file (match benches ((Cons b _) (shown-file (record-ref b file))) (Nil module))) + (bench-menu file benches o (fun () (read-line/opt (parameter-ref current-input-port)))) + 0) + ((some? (record-ref o cases)) + (def wanted (match (record-ref o cases) ((Some is) is) (None []))) + (match (cases-numbered benches wanted) + ((Ok chosen) (run-and-report chosen o)) + ((Err why) (println #"bench: --cases: ${why}") 2))) + (else + (def chosen (filtered benches (record-ref o filters))) + (if (= (case-count chosen) 0) + (let () (println "No benchmark's name contains any of the names given.") 0) + (run-and-report chosen o))))) + +;; The cases numbered `wanted`, in `(list)`'s numbering, as selections: a +;; benchmark's chosen cases together, in the order the file has them. +(: (-> (List Bench) (Vec int) (Result string (List Selection)))) +(defun (cases-numbered benches wanted) + (def total (fold (fun (b n) (+ n (list-length (cases-of b)))) 0 benches)) + (match (find (fun (i) (or (< i 0) (>= i total))) (vec->list wanted)) + ((Some i) (Err #"there is no case ${i}: there are ${total}")) + (None + (Ok (loop (:for b benches) + (:with first 0 (+ first (list-length (cases-of b)))) + (:let picked + (loop (:for c (cases-of b)) + (:with i first (+ i 1)) + (:acc (listing c) #:when (any? (fun (w) (= w i)) wanted)))) + (:acc (listing (Selection (bench b) (cases picked))) + #:when (not (list-empty? picked)))))))) + +;; A path as a person reads it: its last part. +(: (-> string string)) +(defun (shown-file path) + (match (string-search-backward path "/") + ((Some at) (substring path (string-cursor-next path at) (string-cursor-end path))) + (None path))) + +;; `(run-benches)` — the file's `main`: runs the benchmarks this module +;; registered. +(def/macro (run-benches form inject compare) + (syntax-match form + ((_) + #'(begin + (: ,(inject 'main) (-> (Vec string) int)) + (defun (,(inject 'main) args) (run-registered-benches (current-module) args)))) + (bad (syntax-error bad "run-benches takes nothing: (run-benches) at the end of a benchmark file.")))) + +;; --------------------------------------------------------------------------- +;; time-it +;; --------------------------------------------------------------------------- + +;; `(time-it body ...)`: the body as an anonymous benchmark, measured at once +;; with `--quick`'s targets, and one line. A body whose first run takes more +;; than a second is not run again: that run is the measurement. +(: (-> string (-> long void) void)) +(defun (time-it-run label batch) + (def run (CaseRun (batch batch) (call None) (alloc ""))) + (def first (time-batch run 1L)) + (if (>= first 1000000000L) + (println (str label ": " (time-text (cast System.Double first)) + " (one cold run: a body over a second is run once)")) + (let () + (def o (match (parse-bench-args (vec-empty)) ((Ok o) o) ((Err e) (panic! e)))) + (def m (measure-calibrated run (bench-settings (record-set o (scale 0.5))))) + (def xs (record-ref m samples)) + (def med (median xs)) + (println (str label ": " (time-text med) + " ± " (fixed (if (> med 0.0) (* 100.0 (/ (mad xs) med)) 0.0) 1) "%" + " (min " (time-text (vec-fold min (vec-ref xs 0) xs)) "), " + (bytes-text (/ (cast System.Double (record-ref m bytes)) (cast System.Double (record-ref m calls)))) + "/call, " (int->string (vec-length xs)) " samples"))))) + +;; The label of a `time-it`: the body as it was written, cut short. +(: (-> (List Syntax) Syntax)) +(defun (time-it-label forms) + (def text (string-join (list->vec (list-map syntax->string forms)) " ")) + (SStr (if (> (string-count text) 50) (str (list->string (list-take (string->list text) 47)) "...") text))) + +(: (-> (List %a) int (List %a))) +(defun (list-take xs n) + (match xs + ((Cons x more) #:when (> n 0) (cons x (list-take more (- n 1)))) + (_ (list)))) + +;; A batch: the body `k` times, its value kept. +(: (-> (List Syntax) Syntax)) +(defun (time-it-batch forms) + #'(fun (k) + (let lp ((i 0L)) + (when (< i k) + (time-it-keep (begin ,@forms)) + (lp (+ i 1L)))))) + +;; `(time-it body ...)`: the body measured as an anonymous benchmark, on one +;; line. `#:times n` and `#:warm-up-rounds w` ask for exactly that many runs +;; instead, untimed then timed, as `time-it` always did. +;; +;; A function, so that `(std stopwatch)`'s `time-it` is this one too: a macro +;; is only seen by the modules that import its own, and programs import +;; `time-it` from `(std stopwatch)`. +(def/macro (time-it form inject compare) + (time-it-expansion form)) + +(: (-> Syntax Syntax)) +(defun (time-it-expansion form) + (syntax-match form + ((_ #:times n #:warm-up-rounds w first body ...) + #'(time-it-fixed ,(time-it-label (cons first body)) ,n ,w ,(time-it-batch (cons first body)))) + ((_ #:warm-up-rounds w #:times n first body ...) + #'(time-it-fixed ,(time-it-label (cons first body)) ,n ,w ,(time-it-batch (cons first body)))) + ((_ #:times n first body ...) + #'(time-it-fixed ,(time-it-label (cons first body)) ,n 0 ,(time-it-batch (cons first body)))) + ((_ #:warm-up-rounds w first body ...) + #'(time-it-fixed ,(time-it-label (cons first body)) 1 ,w ,(time-it-batch (cons first body)))) + ((_ first body ...) + #'(time-it-run ,(time-it-label (cons first body)) ,(time-it-batch (cons first body)))) + (bad (syntax-error bad "time-it takes a body: (time-it [#:times n] [#:warm-up-rounds w] expr ...)")))) + +;; `(time-it #:times n #:warm-up-rounds w body ...)`: exactly `w` untimed runs +;; and `n` timed ones, as `time-it` always did, reported on the new line. +(: (-> string int int (-> long void) void)) +(defun (time-it-fixed label times warm batch) + (def run (CaseRun (batch batch) (call None) (alloc ""))) + (loop (:for _ (up-from 0 #:to warm)) (:do ((record-ref run batch) 1L))) + (def taken (loop (:for _ (up-from 0 #:to times)) (:acc out (vecing (take-sample run 1L))) => out)) + (def xs (vec-map (fun (x) (cast System.Double (record-ref x ns))) taken)) + (def bytes (vec-fold (fun (x acc) (+ acc (record-ref x bytes))) 0L taken)) + (println (str label ": " (time-text (median xs)) + (if (> times 1) (str " (min " (time-text (vec-fold min (vec-ref xs 0) xs)) ")") "") + ", " (bytes-text (/ (cast System.Double bytes) (cast System.Double times))) "/call, " + (int->string times) (if (= times 1) " run" " runs")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/bench/stats.bjo — the statistics of a benchmark run. +;; +;; Summaries of one set of timings, and the comparison of two. Pure functions +;; over vecs of doubles, with nothing of a runner in them, so that they can be +;; tested against values computed elsewhere and used without `(std bench)`. +;; +;; Every test here makes no assumption about the shape of the timings, which +;; are skewed and have long tails: medians rather than means, rank tests +;; rather than t-tests, and the bootstrap for intervals. See +;; `Docs/Benchmark-design.org`, "The statistics of a comparison". + +(export + ;; One set of samples + median quantile mean std-dev mad tukey-outliers + ;; Intervals + bootstrap-ratio bootstrap-ratio/paired + ;; Tests + wilcoxon-p mann-whitney-p t-test-p t-critical holm normal-sf) + +;; Every export is documented there; the module doc is too. +(:doc #:file "stats.bjodoc") + +(import (std prelude)) +(import (std maths)) +(import (std random)) + +(import/extern + (sort-doubles! (: Bjolang.Runtime.BjoBench.SortDoubles (-> (Array double) void))) + (arg-sort (: Bjolang.Runtime.BjoBench.ArgSort (-> (Array double) (Array int)))) + (clr-vec->array (: BjolangRuntime.vecsubgtarray (-> (Vec double) (Array double))))) + +;; --------------------------------------------------------------------------- +;; One set of samples +;; --------------------------------------------------------------------------- + +;; A sorted copy, as an array: what every order statistic below reads. +(: (-> (Vec double) (Array double))) +(defun (sorted xs) + (def a (clr-vec->array xs)) + (sort-doubles! a) + a) + +;; The `q`-quantile of a sorted array, interpolated linearly between the two +;; samples around it: R's type 7, which is NumPy's default. +(: (-> (Array double) double double)) +(defun (sorted-quantile a q) + (def n (array-length a)) + (cond + ((= n 0) 0.0) + ((= n 1) (array-ref a 0)) + (else + (def h (* q (cast System.Double (- n 1)))) + (def lo (exact-floor h)) + (def hi (if (>= (+ lo 1) n) lo (+ lo 1))) + (def frac (- h (cast System.Double lo))) + (+ (array-ref a lo) (* frac (- (array-ref a hi) (array-ref a lo))))))) + +(: (-> (Vec double) double double)) +(defun (quantile xs q) (sorted-quantile (sorted xs) q)) + +(: (-> (Vec double) double)) +(defun (median xs) (sorted-quantile (sorted xs) 0.5)) + +(: (-> (Vec double) double)) +(defun (mean xs) + (if (vec-empty? xs) + 0.0 + (/ (vec-fold + 0.0 xs) (cast System.Double (vec-length xs))))) + +;; The sample standard deviation, over n - 1. +(: (-> (Vec double) double)) +(defun (std-dev xs) + (def n (vec-length xs)) + (if (< n 2) + 0.0 + (let ((m (mean xs))) + (sqrt (/ (vec-fold (fun (x acc) (+ acc (* (- x m) (- x m)))) 0.0 xs) + (cast System.Double (- n 1))))))) + +;; The median absolute deviation from the median, unscaled. +(: (-> (Vec double) double)) +(defun (mad xs) + (def m (median xs)) + (median (vec-map (fun (x) (abs (- x m))) xs))) + +;; How many samples lie more than 1.5 inter-quartile ranges outside the +;; quartiles: Tukey's fences. +(: (-> (Vec double) int)) +(defun (tukey-outliers xs) + (def a (sorted xs)) + (def q1 (sorted-quantile a 0.25)) + (def q3 (sorted-quantile a 0.75)) + (def fence (* 1.5 (- q3 q1))) + (loop (:for x xs) + (:acc n (counting x) #:when (or (< x (- q1 fence)) (> x (+ q3 fence)))) + => n)) + +;; --------------------------------------------------------------------------- +;; The bootstrap +;; --------------------------------------------------------------------------- + +;; The median of `n` draws, with replacement, from `from`, using `scratch`. +(: (-> (Array double) (Array double) RandomSource double)) +(defun (resampled-median from scratch rng) + (def n (array-length from)) + (loop (:for i (up-from 0 #:to n)) + (:do (array-set! scratch i (array-ref from (random-int 0 (- n 1) #:rng rng))))) + (sort-doubles! scratch) + (sorted-quantile scratch 0.5)) + +;; The 2.5th and 97.5th percentiles of `ratios`. +(: (-> (Array double) (Tuple double double))) +(defun (interval ratios) + (sort-doubles! ratios) + (Tuple (sorted-quantile ratios 0.025) (sorted-quantile ratios 0.975))) + +;; A 95% interval for median(b) / median(a), from `resamples` resamplings of +;; each, independently, with replacement. Seeded, so a run is repeatable. +(: (-> (Vec double) (Vec double) (#:resamples int) (#:seed ulong) + (Tuple double double))) +(defun (bootstrap-ratio a b #:resamples 10000 #:seed 1UL) + (def rng (make-random/seed64 seed)) + (def aa (clr-vec->array a)) + (def ba (clr-vec->array b)) + (def sa (make-array (array-length aa))) + (def sb (make-array (array-length ba))) + (def ratios (make-array resamples)) + (loop (:for r (up-from 0 #:to resamples)) + (:do (array-set! ratios r (/ (resampled-median ba sb rng) (resampled-median aa sa rng))))) + (interval ratios)) + +;; The same for paired samples, `a[i]` and `b[i]` taken together, as an A/B +;; run's rounds are: each resampling draws whole pairs, so that what a round +;; shared stays shared. +(: (-> (Vec double) (Vec double) (#:resamples int) (#:seed ulong) + (Tuple double double))) +(defun (bootstrap-ratio/paired a b #:resamples 10000 #:seed 1UL) + (def rng (make-random/seed64 seed)) + (def aa (clr-vec->array a)) + (def ba (clr-vec->array b)) + (def n (min (array-length aa) (array-length ba))) + (def sa (make-array n)) + (def sb (make-array n)) + (def ratios (make-array resamples)) + (loop (:for r (up-from 0 #:to resamples)) + (:do (loop (:for i (up-from 0 #:to n)) + (:let j (random-int 0 (- n 1) #:rng rng)) + (:do (array-set! sa i (array-ref aa j)) + (array-set! sb i (array-ref ba j)))) + (sort-doubles! sa) + (sort-doubles! sb) + (array-set! ratios r (/ (sorted-quantile sb 0.5) (sorted-quantile sa 0.5))))) + (interval ratios)) + +;; --------------------------------------------------------------------------- +;; The normal distribution +;; --------------------------------------------------------------------------- + +;; erfc(x), from Numerical Recipes' Chebyshev fit: a relative error below +;; 1.2e-7 everywhere, tails included, which is what a small p-value needs. +(: (-> double double)) +(defun (erfc x) + (def z (abs x)) + (def t (/ 1.0 (+ 1.0 (* 0.5 z)))) + (def poly + (+ -1.26551223 + (* t (+ 1.00002368 + (* t (+ 0.37409196 + (* t (+ 0.09678418 + (* t (+ -0.18628806 + (* t (+ 0.27886807 + (* t (+ -1.13520398 + (* t (+ 1.48851587 + (* t (+ -0.82215223 + (* t 0.17087277))))))))))))))))))) + (def r (* t (exp (+ (- 0.0 (* z z)) poly)))) + (if (>= x 0.0) r (- 2.0 r))) + +;; P(Z > z) for a standard normal Z. +(: (-> double double)) +(defun (normal-sf z) (* 0.5 (erfc (/ z (sqrt 2.0))))) + +;; --------------------------------------------------------------------------- +;; Student's t +;; --------------------------------------------------------------------------- +;; +;; For a handful of values that are each a median of many samples, such as the +;; ratios of `bjo bench --against --processes P`: few enough that a rank test +;; cannot reach a small p (2/2^P at best), and close enough to normal that the +;; t-test is fair. + +;; log Γ(x) for x > 0, Lanczos' approximation (g = 7, nine terms): about 15 +;; correct digits. +(: (-> double double)) +(defun (log-gamma x) + (def cs [0.99999999999980993 676.5203681218851 -1259.1392167224028 771.32342877765313 + -176.61502916214059 12.507343278686905 -0.13857109526572012 + 9.9843695780195716e-6 1.5056327351493116e-7]) + (def y (- x 1.0)) + (def a (loop (:for i (up-from 1 #:to 9)) + (:acc s (folding (vec-ref cs 0) (+ s (/ (vec-ref cs i) (+ y (cast System.Double i)))))) + => s)) + (def t (+ y 7.5)) + (+ (* 0.5 (loge (* 2.0 3.141592653589793))) + (- (* (+ y 0.5) (loge t)) t) + (loge a))) + +;; The continued fraction of the incomplete beta function, by Lentz's method, +;; as Numerical Recipes' `betacf`. +(: (-> double double double double)) +(defun (beta-fraction a b x) + (def tiny 1e-300) + (defun (floor-tiny (: v double)) : double (if (< (abs v) tiny) tiny v)) + (def qab (+ a b)) + (def qap (+ a 1.0)) + (def qam (- a 1.0)) + (let go ((m 1) (c 1.0) (d (/ 1.0 (floor-tiny (- 1.0 (/ (* qab x) qap))))) (h (/ 1.0 (floor-tiny (- 1.0 (/ (* qab x) qap)))))) + (if (> m 300) + h + (let () + (def fm (cast System.Double m)) + (def m2 (* 2.0 fm)) + (def aa (/ (* fm (- b fm) x) (* (+ qam m2) (+ a m2)))) + (def d1 (/ 1.0 (floor-tiny (+ 1.0 (* aa d))))) + (def c1 (floor-tiny (+ 1.0 (/ aa c)))) + (def h1 (* h (* d1 c1))) + (def ab (/ (- 0.0 (* (+ a fm) (+ qab fm) x)) (* (+ a m2) (+ qap m2)))) + (def d2 (/ 1.0 (floor-tiny (+ 1.0 (* ab d1))))) + (def c2 (floor-tiny (+ 1.0 (/ ab c1)))) + (def del (* d2 c2)) + (def h2 (* h1 del)) + (if (< (abs (- del 1.0)) 1e-15) h2 (go (+ m 1) c2 d2 h2)))))) + +;; The regularized incomplete beta function I_x(a, b). +(: (-> double double double double)) +(defun (incomplete-beta a b x) + (cond + ((<= x 0.0) 0.0) + ((>= x 1.0) 1.0) + (else + (def front (exp (+ (- (log-gamma (+ a b)) (log-gamma a) (log-gamma b)) + (* a (loge x)) (* b (loge (- 1.0 x)))))) + (if (< x (/ (+ a 1.0) (+ a b 2.0))) + (/ (* front (beta-fraction a b x)) a) + (- 1.0 (/ (* front (beta-fraction b a (- 1.0 x))) b)))))) + +;; P(|T| > t) for Student's T with `df` degrees of freedom. +(: (-> double int double)) +(defun (t-two-sided t df) + (def v (cast System.Double df)) + (incomplete-beta (* 0.5 v) 0.5 (/ v (+ v (* t t))))) + +;; The two-sided p-value of a one-sample t-test of `xs` against a mean of 0. +;; SciPy's `ttest_1samp(xs, 0)` gives the same. +(: (-> (Vec double) double)) +(defun (t-test-p xs) + (def n (vec-length xs)) + (if (< n 2) + 1.0 + (let () + (def m (mean xs)) + (def sd (std-dev xs)) + (cond + ((> sd 0.0) (t-two-sided (/ (abs m) (/ sd (sqrt (cast System.Double n)))) (- n 1))) + ((= m 0.0) 1.0) + (else 0.0))))) + +;; The t with P(|T| > t) = alpha, for `df` degrees of freedom: what a mean is +;; widened by, times its standard error, for a 1 - alpha interval. SciPy's +;; `t.ppf(1 - alpha/2, df)`. +(: (-> double int double)) +(defun (t-critical alpha df) + ;; P(|T| > t) falls as t grows; bisect on it. + (let go ((lo 0.0) (hi 1e6) (i 0)) + (def mid (* 0.5 (+ lo hi))) + (cond + ((>= i 200) mid) + ((> (t-two-sided mid df) alpha) (go mid hi (+ i 1))) + (else (go lo mid (+ i 1)))))) + +;; --------------------------------------------------------------------------- +;; Ranks +;; --------------------------------------------------------------------------- + +;; The ranks of `xs`, 1-based, a tie getting the mean of the ranks it spans; +;; and the sum of t³ - t over the groups of t tied values, which the tests' +;; variances are corrected by. +(: (-> (Array double) (Tuple (Array double) double))) +(defun (ranks xs) + (def n (array-length xs)) + ;; The indices in the order of their values. + (def order (arg-sort xs)) + (def out (make-array n #:fill 0.0)) + (def/mutable ties 0.0) + (let go ((i 0)) + (when (< i n) + (def v (array-ref xs (array-ref order i))) + (def j (let run ((j i)) + (if (and (< (+ j 1) n) (= (array-ref xs (array-ref order (+ j 1))) v)) (run (+ j 1)) j))) + (def rank (/ (cast System.Double (+ i j 2)) 2.0)) + (loop (:for k (up-from i #:to (+ j 1))) + (:do (array-set! out (array-ref order k) rank))) + (def t (cast System.Double (+ (- j i) 1))) + (set! ties (+ ties (- (* t t t) t))) + (go (+ j 1)))) + (Tuple out ties)) + +;; --------------------------------------------------------------------------- +;; Wilcoxon signed-rank +;; --------------------------------------------------------------------------- + +;; The two-sided p-value of the Wilcoxon signed-rank test that the +;; differences `ds` are centred on zero. A zero difference is dropped, as +;; Wilcoxon's own method does. Exact, from the distribution of the signed-rank +;; sum, when at most 50 differences are left and none are tied; otherwise the +;; normal approximation, its variance corrected for ties, with no continuity +;; correction. SciPy's `wilcoxon` with its defaults gives the same. +(: (-> (Vec double) double)) +(defun (wilcoxon-p ds) + (def nonzero (vec-filter (fun (d) (not (= d 0.0))) ds)) + (def n (vec-length nonzero)) + (if (= n 0) + 1.0 + (let () + (def (Tuple rs ties) (ranks (clr-vec->array (vec-map abs nonzero)))) + (def w-plus + (loop (:for i (up-from 0 #:to n)) + (:acc s (summing (array-ref rs i)) #:when (> (vec-ref nonzero i) 0.0)) + => s)) + (def total (/ (cast System.Double (* n (+ n 1))) 2.0)) + (def w (min w-plus (- total w-plus))) + (if (and (<= n 50) (= ties 0.0)) + (wilcoxon-exact-p n w) + (let () + (def nd (cast System.Double n)) + (def mu (/ (* nd (+ nd 1.0)) 4.0)) + (def var (- (/ (* nd (+ nd 1.0) (+ (* 2.0 nd) 1.0)) 24.0) (/ ties 48.0))) + (if (<= var 0.0) + 1.0 + (min 1.0 (* 2.0 (normal-sf (/ (abs (- w mu)) (sqrt var))))))))))) + +;; 2·P(W ≤ w) for the signed-rank sum W of n untied differences: the number +;; of subsets of 1..n with each sum, by counting. +(: (-> int double double)) +(defun (wilcoxon-exact-p n w) + (def max-sum (/ (* n (+ n 1)) 2)) + (def counts (make-array (+ max-sum 1) #:fill 0.0)) + (array-set! counts 0 1.0) + (loop (:for k (up-from 1 #:to (+ n 1))) + (:do (loop (:for s (down-from max-sum #:to k)) + (:do (array-set! counts s (+ (array-ref counts s) (array-ref counts (- s k)))))))) + (def total (pow 2.0 (cast System.Double n))) + (def limit (exact-floor w)) + (def below + (loop (:for s (up-from 0 #:to (+ limit 1))) + (:acc c (summing (array-ref counts s))) + => c)) + (min 1.0 (* 2.0 (/ below total)))) + +;; --------------------------------------------------------------------------- +;; Mann–Whitney U +;; --------------------------------------------------------------------------- + +;; The two-sided p-value of the Mann–Whitney U test that `xs` and `ys` come +;; from the same distribution: the normal approximation, with the continuity +;; correction and the variance corrected for ties. SciPy's `mannwhitneyu` +;; with `method="asymptotic"` gives the same. Benchmark runs have tens of +;; samples or more, where the approximation is good. +(: (-> (Vec double) (Vec double) double)) +(defun (mann-whitney-p xs ys) + (def n1 (vec-length xs)) + (def n2 (vec-length ys)) + (if (or (= n1 0) (= n2 0)) + 1.0 + (let () + (def both (clr-vec->array (vec-append xs ys))) + (def (Tuple rs ties) (ranks both)) + (def r1 (loop (:for i (up-from 0 #:to n1)) (:acc s (summing (array-ref rs i))) => s)) + (def a (cast System.Double n1)) + (def b (cast System.Double n2)) + (def n (+ a b)) + (def u1 (- r1 (/ (* a (+ a 1.0)) 2.0))) + (def mu (/ (* a b) 2.0)) + (def var (* (/ (* a b) 12.0) (- (+ n 1.0) (/ ties (* n (- n 1.0)))))) + (if (<= var 0.0) + 1.0 + (let ((z (/ (max 0.0 (- (abs (- u1 mu)) 0.5)) (sqrt var)))) + (min 1.0 (* 2.0 (normal-sf z)))))))) + +;; --------------------------------------------------------------------------- +;; Many comparisons +;; --------------------------------------------------------------------------- + +;; Holm's step-down correction of `ps`, in their own order: the smallest is +;; multiplied by m, the next by m - 1, and so on, each at least the one +;; before it, none above 1. +(: (-> (Vec double) (Vec double))) +(defun (holm ps) + (def m (vec-length ps)) + (def order (arg-sort (clr-vec->array ps))) + (def out (make-array m #:fill 1.0)) + (let go ((k 0) (running 0.0)) + (when (< k m) + (def i (array-ref order k)) + (def adjusted (min 1.0 (max running (* (cast System.Double (- m k)) (vec-ref ps i))))) + (array-set! out i adjusted) + (go (+ k 1) adjusted))) + (array->vec out)) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/bench/store.bjo — a benchmark run kept in a file, and a run compared +;; with one. +;; +;; A saved run is bjodat, one tagged form, a case to a line: +;; +;; (bench-run +;; (version 1) +;; (saved "2026-10-16T12:00:00Z") +;; (machine "3f9c...") +;; (header (build "Release") (dotnet ".NET 10.0.4") ...) +;; (case (name "sort") (labels ("n" "1000")) (k 512) (once #f) +;; (bytes 0.0) (samples [41213.0 41180.5 ...])) +;; ...) +;; +;; The samples rather than a summary: a comparison needs the distribution. +;; It is `(std bench)` that measures and prints; this module knows only runs +;; as data, so that the two do not import each other. + +(export SavedRun SavedCase Comparison Verdict CompareResult PairedCase + saved-run->string parse-saved-run write-saved-run! read-saved-run + case-key compare-runs compare-paired) + +;; Every export is documented there; the module doc is too. +(:doc #:file "store.bjodoc") + +(import (std prelude)) +(import (std maths)) +(import (std bench stats)) +(import (text bjodat-core)) + +;; --------------------------------------------------------------------------- +;; Runs as data +;; --------------------------------------------------------------------------- + +;; One benchmark at one combination of its parameters. `samples` are +;; nanoseconds per call, in the order they were taken; `bytes` is allocated +;; bytes per call. +(type (: SavedCase (Record (: name string) + (: labels (List (Tuple string string))) + (: k long) + (: once bool) + (: bytes double) + (: samples (Vec double))))) + +;; A run: when, where, and what it measured. `header` holds the facts of +;; the machine and the build, by name: `build`, `dotnet`, `arch`, `cores`, +;; `gc-server`, `cpu`, `os`, `governor`. +(type (: SavedRun (Record (: saved string) + (: machine string) + (: header (List (Tuple string string))) + (: cases (List SavedCase))))) + +(: (-> SavedCase string)) +(defun (case-name c) (record-ref c name)) + +(: (-> SavedCase (List (Tuple string string)))) +(defun (case-labels c) (record-ref c labels)) + +(: (-> SavedCase (Vec double))) +(defun (case-samples c) (record-ref c samples)) + +;; What a case is known by across runs: its name and its parameters, as the +;; report shows them. +(: (-> SavedCase string)) +(defun (case-key c) + (string-join (list->vec (cons (case-name c) + (list-map (fun (l) (match l ((Tuple p v) (str p "=" v)))) + (case-labels c)))) + " ")) + +;; --------------------------------------------------------------------------- +;; Writing +;; --------------------------------------------------------------------------- + +(def file-version 1L) + +(: (-> string Bjodat)) +(defun (sym s) (BjoSym (string->symbol s))) + +(: (-> string Bjodat Bjodat)) +(defun (clause name value) (BjoList (list (sym name) value))) + +(: (-> (Tuple string string) Bjodat)) +(defun (pair->bjodat p) (match p ((Tuple a b) (BjoList (list (BjoStr a) (BjoStr b)))))) + +(: (-> SavedCase Bjodat)) +(defun (case->bjodat c) + (BjoList (list (sym "case") + (clause "name" (BjoStr (case-name c))) + (BjoList (cons (sym "labels") (list-map pair->bjodat (case-labels c)))) + (clause "k" (BjoInt (record-ref c k))) + (clause "once" (BjoBool (record-ref c once))) + (clause "bytes" (BjoFloat (record-ref c bytes))) + (clause "samples" (BjoVec (vec-map BjoFloat (case-samples c))))))) + +(: (-> (List (Tuple string string)) Bjodat)) +(defun (header->bjodat facts) + (BjoList (cons (sym "header") + (list-map (fun (f) (match f ((Tuple n v) (clause n (BjoStr v))))) facts)))) + +;; The text of a saved run: a case to a line, so that a diff of two files +;; says which cases changed. +(: (-> SavedRun string)) +(defun (saved-run->string r) + (def lines + (list-append + (list (bjodat->string (clause "version" (BjoInt file-version))) + (bjodat->string (clause "saved" (BjoStr (record-ref r saved)))) + (bjodat->string (clause "machine" (BjoStr (record-ref r machine)))) + (bjodat->string (header->bjodat (record-ref r header)))) + (list-map (fun (c) (bjodat->string (case->bjodat c))) (record-ref r cases)))) + (str "(bench-run\n" + (string-join (list->vec (list-map (fun (l) (str " " l)) lines)) "\n") + ")\n")) + +(: (-> string SavedRun void)) +(defun (write-saved-run! path r) (file-write-text path (saved-run->string r))) + +;; --------------------------------------------------------------------------- +;; Reading +;; --------------------------------------------------------------------------- + +;; The value of clause `name` among `(name value)` clauses. +(: (-> (List Bjodat) string (Option Bjodat))) +(defun (clause-ref clauses name) + (match clauses + ((Cons (BjoList (Cons (BjoSym s) (Cons v Nil))) more) + (if (= (symbol->string s) name) (Some v) (clause-ref more name))) + ((Cons _ more) (clause-ref more name)) + (Nil None))) + +;; The clauses of the form `(name ...)` among them, each without its name. +(: (-> (List Bjodat) string (List (List Bjodat)))) +(defun (forms-named clauses name) + (loop (:for c clauses) + (:acc (listing (match c ((BjoList (Cons _ rest)) rest) (_ (list)))) + #:when (match c + ((BjoList (Cons (BjoSym s) _)) (= (symbol->string s) name)) + (_ #f))))) + +(: (-> Bjodat (Option double))) +(defun (number d) + (match d + ((BjoFloat x) (Some x)) + ((BjoInt n) (Some (cast System.Double n))) + (_ None))) + +(: (-> Bjodat (Option (Tuple string string)))) +(defun (string-pair d) + (match d + ((BjoList (Cons (BjoStr a) (Cons (BjoStr b) Nil))) (Some (Tuple a b))) + (_ None))) + +(: (-> (List (Option %a)) (Option (List %a)))) +(defun (all-some xs) + (match xs + (Nil (Some (list))) + ((Cons (Some x) more) (option-map (fun (rest) (cons x rest)) (all-some more))) + ((Cons None _) None))) + +(: (-> (List Bjodat) (Result string SavedCase))) +(defun (bjodat->case clauses) + (def labels + (match (find (fun (c) (match c ((BjoList (Cons (BjoSym s) _)) (= (symbol->string s) "labels")) (_ #f))) + clauses) + ((Some (BjoList (Cons _ pairs))) (all-some (list-map string-pair pairs))) + (_ None))) + (match (Tuple (clause-ref clauses "name") labels (clause-ref clauses "k") + (clause-ref clauses "once") (clause-ref clauses "bytes") (clause-ref clauses "samples")) + ((Tuple (Some (BjoStr name)) (Some labels) (Some (BjoInt k)) (Some (BjoBool once)) + (Some bytes) (Some (BjoVec samples))) + (match (Tuple (number bytes) (all-some (list-map number (vec->list samples)))) + ((Tuple (Some b) (Some xs)) + (Ok (SavedCase (name name) (labels labels) (k k) (once once) (bytes b) (samples (list->vec xs))))) + (_ (Err #"the case ${name} has a sample or a size that is not a number")))) + (_ (Err "a case lacks one of name, labels, k, once, bytes and samples, or has one of the wrong kind")))) + +(: (-> (List (Result string %a)) (Result string (List %a)))) +(defun (results rs) + (match rs + (Nil (Ok (list))) + ((Cons (Ok x) more) (match (results more) ((Ok rest) (Ok (cons x rest))) ((Err e) (Err e)))) + ((Cons (Err e) _) (Err e)))) + +(def not-a-run "it is not a saved benchmark run: that is one (bench-run ...) form") + +(: (-> string (Result string SavedRun))) +(defun (parse-saved-run text) + (match (parse-bjodat text) + ((Err e) (Err e)) + ((Ok (BjoList (Cons (BjoSym head) clauses))) + (if (not (= (symbol->string head) "bench-run")) + (Err not-a-run) + (match (Tuple (clause-ref clauses "version") (clause-ref clauses "saved") (clause-ref clauses "machine")) + ((Tuple (Some (BjoInt v)) (Some (BjoStr saved)) (Some (BjoStr machine))) + (if (> v file-version) + (Err #"it is version ${v} of the format, and this program reads version ${file-version}") + (saved-run-of clauses saved machine))) + (_ (Err "it lacks its version, the time it was saved, or its machine"))))) + (_ (Err not-a-run)))) + + +(: (-> (List Bjodat) string string (Result string SavedRun))) +(defun (saved-run-of clauses saved machine) + (def header + (match (forms-named clauses "header") + ((Cons facts _) + (loop (:for f facts) + (:acc (listing (match f + ((BjoList (Cons (BjoSym n) (Cons (BjoStr v) Nil))) (Tuple (symbol->string n) v)) + (_ (Tuple "" "")))) + #:when (match f ((BjoList (Cons (BjoSym _) (Cons (BjoStr _) Nil))) #t) (_ #f))))) + (Nil (list)))) + (match (results (list-map bjodat->case (forms-named clauses "case"))) + ((Ok cases) (Ok (SavedRun (saved saved) (machine machine) (header header) (cases cases)))) + ((Err e) (Err e)))) + +;; The run saved in `path`, or what is wrong with it. +(: (-> string (Result string SavedRun))) +(defun (read-saved-run path) + (if (not (file-exists? path)) + (Err #"${path} does not exist") + (match (try (file-read-text path) #:catch (System.Exception)) + ((Ok text) + (result-map-err (fun (e) #"${path}: ${e}") (parse-saved-run text))) + (_ (Err #"${path} could not be read"))))) + +;; --------------------------------------------------------------------------- +;; Comparing +;; --------------------------------------------------------------------------- + +;; What a comparison concludes: a change that is significant and larger +;; than the threshold, one that is significant but smaller, or none shown. +(type (: Verdict (Union Faster Slower SmallChange NoChange))) + +;; One case compared. `change` is the ratio of the medians less one, `now` +;; against `was`, and `lo` and `hi` its 95% interval the same way; `p` is +;; corrected for the run's number of comparisons, `p-raw` is not. +(type (: Comparison (Record (: name string) + (: labels (List (Tuple string string))) + (: was double) + (: now double) + (: change double) + (: lo double) + (: hi double) + (: p-raw double) + (: p double) + (: verdict Verdict) + ;; With several process pairs, the standard + ;; deviation of their log ratios: how far apart + ;; processes were. -1 when it was not measured. + (: spread double)))) + +;; A run compared with a saved one. `only-now` and `only-saved` are the +;; cases one side has and the other has not, by `case-key`. `warnings` are +;; the facts of the two runs that differ; `other-machine` is set when the +;; machine keys do, and then no verdict is to be believed. +(type (: CompareResult (Record (: comparisons (List Comparison)) + (: only-now (List string)) + (: only-saved (List string)) + (: warnings (List string)) + (: other-machine bool)))) + +(: (-> (List (Tuple string string)) string string)) +(defun (fact facts name) + (match (find (fun (f) (match f ((Tuple n _) (= n name)))) facts) + ((Some (Tuple _ v)) v) + (None ""))) + +;; The facts that a machine key leaves out and that move times: said when +;; they differ. +(def watched-facts (list (Tuple "build" "the build") + (Tuple "dotnet" "the .NET version") + (Tuple "gc-server" "server GC") + (Tuple "governor" "the CPU frequency governor"))) + +(: (-> SavedRun SavedRun (List string))) +(defun (fact-warnings saved now) + (loop (:for w watched-facts) + (:let (Tuple name what) w) + (:let a (fact (record-ref saved header) name)) + (:let b (fact (record-ref now header) name)) + (:acc (listing #"${what} differs: ${a} in the saved run, ${b} now") + #:when (not (= a b))))) + +(: (-> double double double double Verdict)) +(defun (verdict p change alpha threshold) + (cond + ((>= p alpha) NoChange) + ((<= (abs change) threshold) SmallChange) + ((> change 0.0) Slower) + (else Faster))) + +(: (-> (List SavedCase) string (Option SavedCase))) +(defun (find-case cases key) (find (fun (c) (= (case-key c) key)) cases)) + +;; `now` compared with `saved`, case by case: the Mann–Whitney U test on the +;; two sets of samples, since a saved run has no pairs, and a bootstrap +;; interval of the ratio of the medians; the p-values corrected by Holm's +;; method for the number of cases compared. +(: (-> SavedRun SavedRun double double CompareResult)) +(defun (compare-runs saved now alpha threshold) + (def saved-cases (record-ref saved cases)) + (def now-cases (record-ref now cases)) + (def pairs + (loop (:for c now-cases) + (:acc (listing (Tuple (match (find-case saved-cases (case-key c)) ((Some s) s) (None c)) c)) + #:when (some? (find-case saved-cases (case-key c)))))) + (def raw + (list->vec (list-map (fun (p) (match p ((Tuple a b) (mann-whitney-p (case-samples a) (case-samples b))))) + pairs))) + (def corrected (holm raw)) + (def comparisons + (loop (:for p pairs) + (:with i 0 (+ i 1)) + (:let (Tuple a b) p) + (:let was (median (case-samples a))) + (:let now-median (median (case-samples b))) + (:let change (if (> was 0.0) (- (/ now-median was) 1.0) 0.0)) + (:let (Tuple lo hi) (bootstrap-ratio (case-samples a) (case-samples b))) + (:acc (listing (Comparison (name (case-name b)) (labels (case-labels b)) + (was was) (now now-median) (change change) + (lo (- lo 1.0)) (hi (- hi 1.0)) + (p-raw (vec-ref raw i)) (p (vec-ref corrected i)) + (verdict (verdict (vec-ref corrected i) change alpha threshold)) + (spread -1.0)))))) + (CompareResult + (comparisons comparisons) + (only-now (loop (:for c now-cases) + (:acc (listing (case-key c)) #:when (none? (find-case saved-cases (case-key c)))))) + (only-saved (loop (:for c saved-cases) + (:acc (listing (case-key c)) #:when (none? (find-case now-cases (case-key c)))))) + (warnings (fact-warnings saved now)) + (other-machine (not (= (record-ref saved machine) (record-ref now machine)))))) + +;; --------------------------------------------------------------------------- +;; Comparing in rounds: A/B +;; --------------------------------------------------------------------------- + +;; One case of an A/B run: for each process pair, A's and B's per-call times, +;; a sample of each a round, in the order of the rounds. +(type (: PairedCase (Record (: name string) + (: labels (List (Tuple string string))) + (: a (Vec (Vec double))) + (: b (Vec (Vec double)))))) + +(: (-> double double double)) +(defun (log-ratio a b) (if (and (> a 0.0) (> b 0.0)) (loge (/ b a)) 0.0)) + +;; The comparison of one case before the correction: was, now, change, the +;; interval, the raw p, and the spread between processes. +(type (: Paired (Record (: was double) (: now double) (: change double) + (: lo double) (: hi double) (: p double) (: spread double)))) + +(: (-> PairedCase Paired)) +(defun (paired-one c) + (def pairs (vec-length (record-ref c a))) + (if (= pairs 1) + ;; One process a side: the rounds are the pairs. The Wilcoxon test on + ;; each round's difference of log times, and a bootstrap that draws + ;; whole rounds. + (let () + (def a (vec-ref (record-ref c a) 0)) + (def b (vec-ref (record-ref c b) 0)) + (def n (min (vec-length a) (vec-length b))) + (def was (median a)) + (def now (median b)) + (def (Tuple lo hi) (bootstrap-ratio/paired a b)) + (Paired (was was) (now now) (change (if (> was 0.0) (- (/ now was) 1.0) 0.0)) + (lo (- lo 1.0)) (hi (- hi 1.0)) + (p (wilcoxon-p {vecing (log-ratio (vec-ref a i) (vec-ref b i)) (:for i (range 0 n))})) + (spread -1.0))) + ;; Several process pairs: one ratio of medians each, and the t-test + ;; and t-interval on their logs, which a difference between processes + ;; is in. + (let () + (def logs {vecing (log-ratio (median (vec-ref (record-ref c a) j)) (median (vec-ref (record-ref c b) j))) + (:for j (range 0 pairs))}) + (def m (mean logs)) + (def sd (std-dev logs)) + (def half (* (t-critical 0.05 (- pairs 1)) (/ sd (sqrt (cast System.Double pairs))))) + (Paired (was (median (vec-map median (record-ref c a)))) + (now (median (vec-map median (record-ref c b)))) + (change (- (exp m) 1.0)) + (lo (- (exp (- m half)) 1.0)) (hi (- (exp (+ m half)) 1.0)) + (p (t-test-p logs)) + (spread sd))))) + +;; An A/B run's cases compared, A as `was` and B as `now`, the p-values +;; corrected by Holm's method. `only-a` and `only-b` are the cases one side +;; has and the other has not. +(: (-> (List PairedCase) (List string) (List string) double double CompareResult)) +(defun (compare-paired cases only-a only-b alpha threshold) + (def results (list->vec (list-map paired-one cases))) + (def corrected (holm (vec-map (fun (r) (record-ref r p)) results))) + (CompareResult + (comparisons + (loop (:for c cases) + (:with i 0 (+ i 1)) + (:let r (vec-ref results i)) + (:acc (listing (Comparison (name (record-ref c name)) (labels (record-ref c labels)) + (was (record-ref r was)) (now (record-ref r now)) + (change (record-ref r change)) + (lo (record-ref r lo)) (hi (record-ref r hi)) + (p-raw (record-ref r p)) (p (vec-ref corrected i)) + (verdict (verdict (vec-ref corrected i) (record-ref r change) alpha threshold)) + (spread (record-ref r spread))))))) + (only-now only-b) + (only-saved only-a) + (warnings (list)) + (other-machine #f))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; Ordering for common .NET types. +;; +;; The time types are ordered but their names are not exported, because +;; (std datetime) exports a DateTime of its own and a module may import both. +;; Code that names the .NET type writes System.DateTime or declares an alias; +;; the implementations here apply to the type whatever it is called. +(export Guid Version) + +(import (std prelude)) + +;; Provides Ord trait implementations for common .NET types. +(import/class + (DateTime (: System.DateTime)) + (DateTimeOffset (: System.DateTimeOffset)) + (TimeSpan (: System.TimeSpan)) + (DateOnly (: System.DateOnly)) + (TimeOnly (: System.TimeOnly)) + (Guid (: System.Guid)) + (Version (: System.Version))) + +(:doc #:file "clr-ord.bjodoc") + +;; Implements Ord using CompareTo. +(impl (Ord DateTime) + (defun (compare a b) (.CompareTo a b))) + +(impl (Ord DateTimeOffset) + (defun (compare a b) (.CompareTo a b))) + +(impl (Ord TimeSpan) + (defun (compare a b) (.CompareTo a b))) + +(impl (Ord DateOnly) + (defun (compare a b) (.CompareTo a b))) + +(impl (Ord TimeOnly) + (defun (compare a b) (.CompareTo a b))) + +(impl (Ord Guid) + (defun (compare a b) (.CompareTo a b))) + +;; Implements Ord for Version using CompareTo. +(impl (Ord Version) + (defun (compare a b) (.CompareTo a b))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; Values that combine in order: the result of a run over the elements of `a` +;; followed by those of `b`. +;; +;; - `combine` is associative, not commutative. +;; - `combine` changes neither argument. The result may share structure with +;; them. +;; - Implemented only where a type has one meaning. A type that combines +;; some other way is a type of its own. +;; - `Array` has none: it is mutable. Write the parts into one array instead. + +(import (std prelude)) +(import (std hashset)) +(import (std orderedset)) + +(:doc #:file "combine.bjodoc") + +(import/extern + (iarray-adopt (: Bjolang.Runtime.IArrayModule.Adopt (-> (Array %a) (IArray %a))))) + +(export Combinable combine combine-all) + +;; - `combine a b`: `a`'s elements, then `b`'s. +;; - `combine-all xs`: the values of `xs` combined in order. `xs` is not +;; empty. Equal to `combine` from the left; an implementation overrides it +;; to copy once. +(def/trait (Combinable %a) + (: combine (-> %a %a %a)) + (: combine-all (-> (Vec %a) %a)) + (defun (combine-all xs) + (loop (:for i (range 1 (vec-length xs))) + (:acc out (folding (vec-ref xs 0) (combine out (vec-ref xs i))))))) + +(impl (Combinable Unit) + (defun (combine a b) unit) + (defun (combine-all xs) unit)) + +;; Concatenation shares the nodes of both vectors. +(impl (Combinable (Vec %a)) + (defun (combine a b) (vec-append a b))) + +;; From the right: each list but the last is copied once. +(impl (Combinable (List %a)) + (defun (combine a b) (list-append a b)) + (defun (combine-all xs) + (def n (vec-length xs)) + (loop (:for i (down-from (- n 2) #:to 0)) + (:acc out (folding (vec-ref xs (- n 1)) (list-append (vec-ref xs i) out)))))) + +(impl (Combinable string) + (defun (combine a b) (string-append a b)) + (defun (combine-all xs) (string-join xs ""))) + +(impl (Combinable (IArray %a)) + (defun (combine a b) (combine-all [a b])) + (defun (combine-all xs) + (def out (make-array (loop (:for x xs) (:acc n (summing (iarray-length x)))))) + (loop (:for x xs) + (:with at 0 (+ at (iarray-length x))) + (:do (loop (:for i (range 0 (iarray-length x))) + (:do (array-set! out (+ at i) (iarray-ref x i)))))) + (iarray-adopt out))) + +(impl (Combinable (HashSet %a)) + (defun (combine a b) (hashset-union a b))) + +(impl (Combinable (OrderedSet %a)) + (defun (combine a b) (orderedset-union a b))) + +;; Tuples combine element by element. + +(impl (Combinable (Tuple %a %b)) + (where (Combinable %a) (Combinable %b)) + (defun (combine x y) + (def (a1 b1) x) + (def (a2 b2) y) + (Tuple (combine a1 a2) (combine b1 b2)))) + +(impl (Combinable (Tuple %a %b %c)) + (where (Combinable %a) (Combinable %b) (Combinable %c)) + (defun (combine x y) + (def (a1 b1 c1) x) + (def (a2 b2 c2) y) + (Tuple (combine a1 a2) (combine b1 b2) (combine c1 c2)))) + +(impl (Combinable (Tuple %a %b %c %d)) + (where (Combinable %a) (Combinable %b) (Combinable %c) (Combinable %d)) + (defun (combine x y) + (def (a1 b1 c1 d1) x) + (def (a2 b2 c2 d2) y) + (Tuple (combine a1 a2) (combine b1 b2) (combine c1 c2) (combine d1 d2)))) + +(impl (Combinable (Tuple %a %b %c %d %e)) + (where (Combinable %a) (Combinable %b) (Combinable %c) (Combinable %d) (Combinable %e)) + (defun (combine x y) + (def (a1 b1 c1 d1 e1) x) + (def (a2 b2 c2 d2 e2) y) + (Tuple (combine a1 a2) (combine b1 b2) (combine c1 c2) (combine d1 d2) (combine e1 e2)))) + +(impl (Combinable (Tuple %a %b %c %d %e %f)) + (where (Combinable %a) (Combinable %b) (Combinable %c) (Combinable %d) (Combinable %e) (Combinable %f)) + (defun (combine x y) + (def (a1 b1 c1 d1 e1 f1) x) + (def (a2 b2 c2 d2 e2 f2) y) + (Tuple (combine a1 a2) (combine b1 b2) (combine c1 c2) (combine d1 d2) (combine e1 e2) (combine f1 f2)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/core-macros.bjo +;; +;; Macros below the prelude. Built without it, so the prelude and the files it +;; includes can use them. The prelude re-exports each one. + +(import (std syntax-match)) + +(:doc #:file "core-macros.bjodoc") + +(export) + +;; `(cond (test body ...) ... (:def pattern expr) ... (else body ...))` +;; +;; Cond requires else. +;; +;; `(:match pattern expr body ...)`, with an optional `#:when guard` after +;; `expr`, is a test that binds: it answers when `expr` matches and the guard +;; holds, and falls through to the next clause otherwise. What it binds is in +;; scope in its guard and body only. +;; +;; Two clauses do not answer anything. `(:def pattern expr)` binds, and +;; `(:do body ...)` runs something; both are reached only when every test +;; above them has failed, and what a `:def` binds is in scope in every clause +;; below it. They exist because the alternative is a new `(else ...)` and a +;; new `cond` inside it for every value a later test needs — the same +;; decisions, indented once per binding. +;; +;; A `:def` clause *is* a `def`: the sequel it swallows is the rest of the +;; cond, so a refutable pattern takes the same failure part it takes anywhere +;; else: `(:def (Ok v) (parse s) :propagate)` makes the whole cond the `Err`, +;; and `(:def (Some n) (lookup k) :default 0)` binds `n` and never answers. +(def/macro (cond form inject compare) + (syntax-match form + ((_ clause ...) (cond-arms clause form)) + (bad (syntax-error bad "cond takes clauses: (cond (test body ...) ... (else body ...))")))) + +(: (-> (List Syntax) Syntax Syntax)) +(defun (cond-arms clauses whole) + (match clauses + (Nil + (syntax-error whole "cond needs a final (else ...) clause: every arm answers something, so there has to be a last answer.")) + ((Cons clause rest) + (syntax-match clause + ;; Binds. The rest of the cond is the body the `def` swallows, which is + ;; what puts the name in scope below and nowhere else. + ((:def pattern expr fail ...) + #'(let () (def ,pattern ,expr ,@fail) ,(cond-arms rest whole))) + ((:def junk ...) + (syntax-error clause "(:def ...) binds: (:def name expr), or (:def pattern expr failure-part), such as :default value or :propagate.")) + ;; Answers when `expr` matches, and falls through to the next clause + ;; when it does not. Unlike `:def` it swallows nothing: what it binds + ;; is in scope in its guard and body only. The rest of the cond is the + ;; fallback arm, so it is written once. + ((:match pattern expr #:when guard body ...) + #'(match ,expr + (,pattern #:when ,guard ,(one-form body clause)) + (,(SSym (string->symbol "_")) ,(cond-arms rest whole)))) + ((:match pattern expr body ...) + #'(match ,expr + (,pattern ,(one-form body clause)) + (,(SSym (string->symbol "_")) ,(cond-arms rest whole)))) + ((:match junk ...) + (syntax-error clause "(:match ...) is (:match pattern expr body ...), or (:match pattern expr #:when guard body ...).")) + ;; Runs. + ((:do first body ...) + #'(let () ,first ,@body ,(cond-arms rest whole))) + ((:do) + (syntax-error clause "(:do ...) needs something to run.")) + ;; Else clause. It always answers, so a clause after it could never be + ;; reached. + (('else body ...) + (match rest + (Nil (one-form body clause)) + ((Cons after _) + (syntax-error after "This clause comes after (else ...), so it is never reached. Move it above the else, or remove it.")))) + ((test body ...) #'(if ,test ,(one-form body clause) ,(cond-arms rest whole))) + (bad (syntax-error bad "A cond clause is (test body ...), (:match pattern expr body ...) to answer on a match, (:def pattern expr) to bind, (:do body ...) to run something, or (else body ...) for the last one.")))))) + +;; Several forms as one. +(: (-> (List Syntax) Syntax Syntax)) +(defun (one-form body whole) + (match body + (Nil (syntax-error whole "This clause has nothing to do.")) + ((Cons only Nil) only) + (_ #'(let () ,@body)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/crypto/hash.bjo — digests, HMAC and password hashing. +;; +;; The work is done by .NET's System.Security.Cryptography, called directly. +;; A string is hashed as its UTF-8 bytes. +;; +;; Passwords are hashed with PBKDF2, because it is the password hash that .NET +;; has. The result is a PHC string that holds the algorithm, the iteration +;; count and the salt, so a hash made with old parameters still verifies. +;; PBKDF2 at 600000 iterations takes a core for a few hundred milliseconds. In +;; a bjoroutine, the password functions run on a thread of their own, so the +;; fiber pool is not blocked. +;; +;; `derive-key` is PBKDF2 with a salt you give: the same bytes each time. + +(export + ;; Transparent: Md5, Sha1, Sha256, Sha384, Sha512, Sha3-256, Sha3-384, Sha3-512. + HashAlgorithm + hash-algorithm-supported? digest-length + ;; One-shot + hash-bytes hash-string hash-file + hmac-bytes hmac-string + ;; Incremental + Hasher make-hasher make-hmac-hasher hasher-add! hasher-add-string! hasher-finish! + ;; Comparing and printing + constant-time-equal? bytes->hex hex->bytes + ;; Passwords + password-hash password-verify password-needs-rehash? + default-password-iterations + ;; Keys + derive-key) + +(import (std prelude)) +(import (only (std random) random-bytes!)) + +(:doc #:file "hash.bjodoc") + +;; MD5 and SHA-1 are broken for collision resistance. They are here for +;; protocols and checksums that need them. +(type (: HashAlgorithm (Union Md5 Sha1 Sha256 Sha384 Sha512 Sha3-256 Sha3-384 Sha3-512))) + +(import/class + (Hasher (: System.Security.Cryptography.IncrementalHash)) + (HashAlgorithmName (: System.Security.Cryptography.HashAlgorithmName (-> string HashAlgorithmName))) + (Stream (: System.IO.Stream)) + ;; Path, mode, access, share, buffer size, and whether reads are async. + (FileStream (: System.IO.FileStream (-> string FileMode FileAccess FileShare int bool FileStream))) + (FileMode (: System.IO.FileMode)) + (FileAccess (: System.IO.FileAccess)) + (FileShare (: System.IO.FileShare)) + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +(import/extern + (clr-sha3-256? (: System.Security.Cryptography.SHA3_256.IsSupported #:get)) + (clr-sha3-384? (: System.Security.Cryptography.SHA3_384.IsSupported #:get)) + (clr-sha3-512? (: System.Security.Cryptography.SHA3_512.IsSupported #:get)) + (clr-hash (: System.Security.Cryptography.CryptographicOperations.HashData + (-> HashAlgorithmName (Array byte) (Array byte)))) + (clr-hash-stream (: System.Security.Cryptography.CryptographicOperations.HashData + (-> HashAlgorithmName Stream (Array byte)) #:blocking)) + (clr-hash-stream-await (: System.Security.Cryptography.CryptographicOperations.HashDataAsync + (-> HashAlgorithmName Stream (Array byte)) #:async)) + (clr-hmac (: System.Security.Cryptography.CryptographicOperations.HmacData + (-> HashAlgorithmName (Array byte) (Array byte) (Array byte)))) + (clr-make-hasher (: System.Security.Cryptography.IncrementalHash.CreateHash + (-> HashAlgorithmName Hasher))) + (clr-make-hmac (: System.Security.Cryptography.IncrementalHash.CreateHMAC + (-> HashAlgorithmName (Array byte) Hasher))) + (clr-hasher-add (: System.Security.Cryptography.IncrementalHash.AppendData (-> Hasher (Array byte) void))) + ;; The digest; the hasher is reset, and an HMAC hasher keeps its key. + (clr-hasher-finish (: System.Security.Cryptography.IncrementalHash.GetHashAndReset (-> Hasher (Array byte)))) + (clr-fixed-equal (: System.Security.Cryptography.CryptographicOperations.FixedTimeEquals + (-> (RoSpan byte) (RoSpan byte) bool))) + (clr-pbkdf2 (: System.Security.Cryptography.Rfc2898DeriveBytes.Pbkdf2 + (-> (Array byte) (Array byte) int HashAlgorithmName int (Array byte)))) + (clr-to-hex (: System.Convert.ToHexStringLower (-> (Array byte) string))) + ;; Raises `FormatException` for an odd length or a character that is not a hex digit. + (clr-from-hex (: System.Convert.FromHexString (-> string (Array byte)))) + (clr-to-base64 (: System.Convert.ToBase64String (-> (Array byte) string))) + ;; Raises `FormatException` for text that is not base64. + (clr-from-base64 (: System.Convert.FromBase64String (-> string (Array byte))))) + +;; .net algorithm names. +(: (-> HashAlgorithm string)) +(defun (algorithm-name a) + (match a + (Md5 "MD5") + (Sha1 "SHA1") + (Sha256 "SHA256") + (Sha384 "SHA384") + (Sha512 "SHA512") + (Sha3-256 "SHA3-256") + (Sha3-384 "SHA3-384") + (Sha3-512 "SHA3-512"))) + +(: (-> HashAlgorithm HashAlgorithmName)) +(defun (clr-name a) (HashAlgorithmName. (algorithm-name a))) + +;; SHA-3 needs OpenSSL 1.1.1 or later, or Windows 11. The others are always there. +(: (-> HashAlgorithm bool)) +(defun (hash-algorithm-supported? a) + (match a + (Sha3-256 clr-sha3-256?) + (Sha3-384 clr-sha3-384?) + (Sha3-512 clr-sha3-512?) + (_ #t))) + +;; In bytes. +(: (-> HashAlgorithm int)) +(defun (digest-length a) + (match a + (Md5 16) + (Sha1 20) + (Sha256 32) + (Sha384 48) + (Sha512 64) + (Sha3-256 32) + (Sha3-384 48) + (Sha3-512 64))) + +;; --------------------------------------------------------------------------- +;; One-shot +;; --------------------------------------------------------------------------- + +(: (-> (Array byte) (#:algorithm HashAlgorithm) (Array byte))) +(defun (hash-bytes data #:algorithm Sha256) + (clr-hash (clr-name algorithm) data)) + +(: (-> string (#:algorithm HashAlgorithm) (Array byte))) +(defun (hash-string text #:algorithm Sha256) + (clr-hash (clr-name algorithm) (string->bytes text))) + +;; The file opened for reading, other readers allowed, as `File.OpenRead` +;; opens it. With `async`, reads do not park a thread. +(: (-> string bool FileStream)) +(defun (open-for-hash path async) + (FileStream. path FileMode.Open FileAccess.Read FileShare.Read 4096 async)) + +;; The file is read a buffer at a time, so a large file is never all in memory. +;; Raises as opening the file does, for example `FileNotFoundException`. +(: (-> string (#:algorithm HashAlgorithm) (Array byte))) +(defbjouble (hash-file path #:algorithm Sha256) + (#:sync (let ((file (open-for-hash path #f))) + (try (clr-hash-stream (clr-name algorithm) (cast Stream file)) + #:finally (.Dispose file)))) + (#:bjo (let ((file (open-for-hash path #t))) + (try (clr-hash-stream-await (clr-name algorithm) (cast Stream file)) + #:finally (.Dispose file))))) + +(: (-> (Array byte) (Array byte) (#:algorithm HashAlgorithm) (Array byte))) +(defun (hmac-bytes key data #:algorithm Sha256) + (clr-hmac (clr-name algorithm) key data)) + +(: (-> (Array byte) string (#:algorithm HashAlgorithm) (Array byte))) +(defun (hmac-string key text #:algorithm Sha256) + (clr-hmac (clr-name algorithm) key (string->bytes text))) + +;; --------------------------------------------------------------------------- +;; Incremental +;; --------------------------------------------------------------------------- +;; +;; A hasher is mutable. Do not share one between fibers without a lock. + +(: (-> (#:algorithm HashAlgorithm) Hasher)) +(defun (make-hasher #:algorithm Sha256) + (clr-make-hasher (clr-name algorithm))) + +(: (-> (Array byte) (#:algorithm HashAlgorithm) Hasher)) +(defun (make-hmac-hasher key #:algorithm Sha256) + (clr-make-hmac (clr-name algorithm) key)) + +(: (-> Hasher (Array byte) void)) +(defun (hasher-add! h data) (clr-hasher-add h data)) + +(: (-> Hasher string void)) +(defun (hasher-add-string! h text) (clr-hasher-add h (string->bytes text))) + +;; The digest of all that was added. The hasher is then empty, and can be used +;; again. An HMAC hasher keeps its key. +(: (-> Hasher (Array byte))) +(defun (hasher-finish! h) (clr-hasher-finish h)) + +;; --------------------------------------------------------------------------- +;; Comparing and printing +;; --------------------------------------------------------------------------- + +;; Use this to compare a secret, such as an HMAC tag or a token. The time does +;; not depend on where the arrays differ. Arrays of different lengths are not +;; equal, and the time can show the lengths. +(: (-> (Array byte) (Array byte) bool)) +(defun (constant-time-equal? a b) (clr-fixed-equal (array->ro-span a) (array->ro-span b))) + +;; Lower case. +(: (-> (Array byte) string)) +(defun (bytes->hex bytes) (clr-to-hex bytes)) + +;; Either case. `None` for an odd length or a character that is not a hex digit. +(: (-> string (Option (Array byte)))) +(defun (hex->bytes text) + (result->option (try (clr-from-hex text) #:catch (System.FormatException)))) + +;; --------------------------------------------------------------------------- +;; Passwords +;; --------------------------------------------------------------------------- + +;; The OWASP recommendation for PBKDF2-HMAC-SHA256 in 2023. +(: int) +(def default-password-iterations 600000) + +;; 16 bytes is the size that NIST SP 800-132 recommends as a minimum. +(: int) +(def salt-length 16) + +;; A stored password is a PHC string: `$pbkdf2-sha256$i=600000$$`. +;; The salt and the hash are standard base64 without padding, as the PHC +;; format specifies. + +(: (-> string %a)) +(defun (refuse message) (raise (cast Exception (ArgumentException. message)))) + +;; The PHC id of a password algorithm. Raises `ArgumentException` for an +;; algorithm that is not SHA-256, SHA-384 or SHA-512. +(: (-> HashAlgorithm string)) +(defun (phc-id a) + (match a + (Sha256 "pbkdf2-sha256") + (Sha384 "pbkdf2-sha384") + (Sha512 "pbkdf2-sha512") + (_ (refuse (str "Passwords are hashed with SHA256, SHA384 or SHA512, not " + (algorithm-name a) "."))))) + +(: (-> string (Option HashAlgorithm))) +(defun (algorithm-of-phc-id id) + (cond + ((= id "pbkdf2-sha256") (Some Sha256)) + ((= id "pbkdf2-sha384") (Some Sha384)) + ((= id "pbkdf2-sha512") (Some Sha512)) + (else None))) + +;; The length of a password hash: the digest length. Raises +;; `ArgumentException` for an algorithm that is not a password algorithm. +(: (-> HashAlgorithm int)) +(defun (password-hash-length a) + (match a + ((or Sha256 Sha384 Sha512) (digest-length a)) + (_ (refuse (str "Not a password algorithm: " (algorithm-name a) "."))))) + +;; Base64 without the `=` padding at the end. +(: (-> (Array byte) string)) +(defun (to-base64 bytes) + (def padding (match (% (array-length bytes) 3) (0 0) (1 2) (_ 1))) + (string-drop-right (clr-to-base64 bytes) padding)) + +;; Base64 with or without its padding. `None` for text that is not base64. +(: (-> string (Option (Array byte)))) +(defun (from-base64 text) + (def padded + (match (% (string-length text) 4) + (0 (Some text)) + (2 (Some (str text "=="))) + (3 (Some (str text "="))) + (_ None))) + (match padded + ((Some t) + (result->option (try (clr-from-base64 t) #:catch (System.FormatException)))) + (None None))) + +;; One or more ASCII digits, as an int of at least 1. `None` for anything +;; else: a sign, a space, or a number too big for an int. +(: (-> string (Option int))) +(defun (positive-int text) + (def bytes (string->bytes text)) + (def n (array-length bytes)) + (let go ((i 0) (v 0L)) + (cond + ((= i n) (if (and (> n 0) (>= v 1L)) (Some (cast int v)) None)) + (else + (def b (cast int (array-ref bytes i))) + (def next (+ (* v 10L) (cast long (- b 48)))) + (if (and (>= b 48) (<= b 57) (<= next 2147483647L)) + (go (+ i 1) next) + None))))) + +(: (-> string (Array byte) HashAlgorithm int int (Array byte))) +(defun (pbkdf2 password salt algorithm iterations length) + (clr-pbkdf2 (string->bytes password) salt iterations (clr-name algorithm) length)) + +;; What a PHC string holds. +(type (: Phc (Record (: phc-algorithm HashAlgorithm) + (: phc-iterations int) + (: phc-salt (Array byte)) + (: phc-hash (Array byte))))) + +;; Raises `ArgumentException` when `encoded` is not a PHC string of PBKDF2 +;; with SHA-256, SHA-384 or SHA-512, at least 1 iteration, and a salt and a +;; hash that are not empty. +(: (-> string Phc)) +(defun (parse-phc encoded) + (def parts (string-split encoded "$")) + (def fail "Not a password hash: expected $pbkdf2-$i=$$.") + ;; The leading `$` makes the first part empty. + (if (not (and (= (vec-length parts) 5) (= (vec-ref parts 0) ""))) + (refuse fail) + (let ((algorithm (algorithm-of-phc-id (vec-ref parts 1))) + (count (if (string-starts-with? (vec-ref parts 2) "i=") + (positive-int (string-drop (vec-ref parts 2) 2)) + None)) + (salt (from-base64 (vec-ref parts 3))) + (hash (from-base64 (vec-ref parts 4)))) + (match (Tuple algorithm count salt hash) + ((Tuple (Some a) (Some n) (Some s) (Some h)) + (if (or (= (array-length s) 0) (= (array-length h) 0)) + (refuse fail) + (Phc (phc-algorithm a) (phc-iterations n) (phc-salt s) (phc-hash h)))) + (_ (refuse fail)))))) + +;; Raises `ArgumentException` for an algorithm that is not a password +;; algorithm, fewer than 1 iteration, or a salt shorter than 8 bytes. +(: (-> string (Array byte) HashAlgorithm int string)) +(defun (encode-password-hash password salt algorithm iterations) + (def id (phc-id algorithm)) + (when (< iterations 1) (refuse "The iteration count must be at least 1.")) + (when (< (array-length salt) 8) (refuse "The salt must be at least 8 bytes.")) + (def digest (pbkdf2 password salt algorithm iterations (password-hash-length algorithm))) + (str "$" id "$i=" (int->string iterations) "$" (to-base64 salt) "$" (to-base64 digest))) + +(: (-> string string bool)) +(defun (password-verify-now password stored) + (def phc (parse-phc stored)) + (def expected (record-ref phc phc-hash)) + (def candidate (pbkdf2 password (record-ref phc phc-salt) (record-ref phc phc-algorithm) + (record-ref phc phc-iterations) (array-length expected))) + (constant-time-equal? candidate expected)) + +;; Raises `ArgumentException` for an algorithm that is not a password +;; algorithm, fewer than 1 iteration, or a length less than 1. +(: (-> string (Array byte) HashAlgorithm int int (Array byte))) +(defun (derive-key-now password salt algorithm iterations length) + (ignore (password-hash-length algorithm)) + (when (< iterations 1) (refuse "The iteration count must be at least 1.")) + (when (< length 1) (refuse "The key length must be at least 1.")) + (pbkdf2 password salt algorithm iterations length)) + +(: (-> string HashAlgorithm int string)) +(defun (password-hash-now password algorithm iterations) + (def salt (make-array salt-length)) + (random-bytes! salt) + (encode-password-hash password salt algorithm iterations)) + +;; The value of a thread's result, or its exception raised again. +(: (-> (Result Exception %a) %a)) +(defun (unwrap-thread r) + (match r + ((Ok v) v) + ((Err e) (raise e)))) + +(: (-> string (#:algorithm HashAlgorithm) (#:iterations int) string)) +(defbjouble (password-hash password #:algorithm Sha256 #:iterations default-password-iterations) + (#:sync (password-hash-now password algorithm iterations)) + (#:bjo (unwrap-thread (sync (spawn/thread #(password-hash-now password algorithm iterations)))))) + +;; Raises `ArgumentException` when `stored` is not a hash from `password-hash`. +;; This is an error in the stored data, not a wrong password. +(: (-> string string bool)) +(defbjouble (password-verify password stored) + (#:sync (password-verify-now password stored)) + (#:bjo (unwrap-thread (sync (spawn/thread #(password-verify-now password stored)))))) + +;; True when `stored` was made with another algorithm or with fewer iterations. +;; Check this after a correct `password-verify`, and store a new hash of the +;; password if it is true. +(: (-> string (#:algorithm HashAlgorithm) (#:iterations int) bool)) +(defun (password-needs-rehash? stored #:algorithm Sha256 #:iterations default-password-iterations) + (def phc (parse-phc stored)) + (or (not (= (algorithm-name (record-ref phc phc-algorithm)) (algorithm-name algorithm))) + (< (record-ref phc phc-iterations) iterations) + (not (= (array-length (record-ref phc phc-hash)) (password-hash-length algorithm))))) + +;; --------------------------------------------------------------------------- +;; Keys +;; --------------------------------------------------------------------------- + +;; PBKDF2 bytes of `password` with the salt given. Same password and salt, +;; same bytes. Runs on a thread of its own in a bjoroutine, as +;; `password-hash` does. +(: (-> string (Array byte) (#:algorithm HashAlgorithm) (#:iterations int) (#:length int) (Array byte))) +(defbjouble (derive-key password salt #:algorithm Sha256 #:iterations default-password-iterations + #:length 32) + (#:sync (derive-key-now password salt algorithm iterations length)) + (#:bjo (unwrap-thread + (sync (spawn/thread #(derive-key-now password salt algorithm iterations length)))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/datetime.bjo — dates, times, instants, zones and durations. +;; +;; Every value lies between 0001-01-01T00:00 and 9999-12-31T23:59:59.9999999, +;; at a precision of 100 ns. A result outside that range raises an +;; OverflowException from a function that returns the value itself, and is None +;; from a constructor, a parser or an `/opt` function. Nothing is clamped +;; silently, except month arithmetic landing past the end of a month +;; (2026-01-31 + 1 month = 2026-02-28) and time-of-day arithmetic, which wraps +;; at midnight. +;; +;; Text is formatted and parsed here, digit by digit, rather than by .NET, so +;; no culture is involved. +;; +;; Its reference is its docs, in datetime.bjodoc. +;; +;; Open: +;; - ->weekday answers an int, so (= (->weekday d) 0) compiles and is never +;; true. A Weekday union, Monday to Sunday, would make it a type error, at +;; the cost of arithmetic on weekdays. +;; - date->text writes (week) beside (year) without a word, so +;; '((year 4) "-W" (week 2)) writes 2027-01-01 as 2027-W53, a week 2027 +;; does not have, where text->date refuses that format. Beside (month) and +;; (day) the week is right, so the writer cannot refuse it by itself. +;; - A format is checked each time it reads, as a list cannot carry the +;; check, which is about a third of a reader's time; a reader costs about +;; ten times an iso8601-> one. Reading has no optional parts either, such +;; as seconds that may be missing; the ISO 8601 readers have those. +;; - utc-datetime->instant takes any DateTime, so a local time goes in and is +;; read as UTC without a word. + +(import (std prelude)) + +(import/class + (ClrDate (: System.DateOnly (-> int int int ClrDate))) + (ClrTime (: System.TimeOnly (-> long ClrTime))) + (ClrDateTime (: System.DateTime (-> long DateTimeKind ClrDateTime))) + (DateTimeKind (: System.DateTimeKind)) + (ClrTimeSpan (: System.TimeSpan (-> long ClrTimeSpan))) + (TimeZoneInfo (: System.TimeZoneInfo)) + (OverflowException (: System.OverflowException (-> string OverflowException))) + (ArgumentException (: System.ArgumentException (-> string ArgumentException))) + (ClrLock (: System.Threading.Lock (-> ClrLock))) + ;; A .NET StringBuilder of its own, because the prelude's cannot be given a + ;; capacity, and it takes a UTF-16 unit, which a digit is. + (TextBuilder (: System.Text.StringBuilder (-> int TextBuilder)))) + +;; A body an importer may copy (a trait method here, or a constrained +;; function) names .NET members through these aliases rather than through +;; helpers of this module: an alias travels under a name nobody can write, +;; while a helper would be exported under its own. +(import/extern + (clr-date-from-day-number (: System.DateOnly.FromDayNumber (-> int ClrDate))) + (clr-date-from-datetime (: System.DateOnly.FromDateTime (-> ClrDateTime ClrDate))) + (clr-time-from-datetime (: System.TimeOnly.FromDateTime (-> ClrDateTime ClrTime))) + (clr-time-add (: System.TimeOnly.Add (-> ClrTime ClrTimeSpan ClrTime))) + (clr-time-hour (: System.TimeOnly.Hour (-> ClrTime int) #:get)) + (clr-time-minute (: System.TimeOnly.Minute (-> ClrTime int) #:get)) + (clr-time-second (: System.TimeOnly.Second (-> ClrTime int) #:get)) + (clr-time-millisecond (: System.TimeOnly.Millisecond (-> ClrTime int) #:get)) + (clr-time-microsecond (: System.TimeOnly.Microsecond (-> ClrTime int) #:get)) + (clr-time-nanosecond (: System.TimeOnly.Nanosecond (-> ClrTime int) #:get)) + (clr-date-day-number (: System.DateOnly.DayNumber (-> ClrDate int) #:get)) + (clr-date-year (: System.DateOnly.Year (-> ClrDate int) #:get)) + (clr-date-month (: System.DateOnly.Month (-> ClrDate int) #:get)) + (clr-date-day (: System.DateOnly.Day (-> ClrDate int) #:get)) + (clr-date-day-of-year (: System.DateOnly.DayOfYear (-> ClrDate int) #:get)) + (clr-date-add-months (: System.DateOnly.AddMonths (-> ClrDate int ClrDate))) + (clr-date-add-years (: System.DateOnly.AddYears (-> ClrDate int ClrDate))) + (clr-days-in-month (: System.DateTime.DaysInMonth (-> int int int))) + (clr-time-ticks (: System.TimeOnly.Ticks (-> ClrTime long) #:get)) + (clr-datetime-ticks (: System.DateTime.Ticks (-> ClrDateTime long) #:get)) + (clr-timespan-ticks (: System.TimeSpan.Ticks (-> ClrTimeSpan long) #:get)) + (clr-tz-try-find + (: System.TimeZoneInfo.TryFindSystemTimeZoneById + (-> string (out TimeZoneInfo) (Option TimeZoneInfo)))) + (clr-tz-utc (: System.TimeZoneInfo.Utc #:get)) + (clr-tz-id (: System.TimeZoneInfo.Id (-> TimeZoneInfo string) #:get)) + (clr-tz-has-iana-id (: System.TimeZoneInfo.HasIanaId (-> TimeZoneInfo bool) #:get)) + (clr-tz-utc-offset (: System.TimeZoneInfo.GetUtcOffset (-> TimeZoneInfo ClrDateTime ClrTimeSpan))) + (clr-at-utc (: BjolangRuntime.atsubtime (-> ClrDateTime (Event Unit)))) + (clr-lock-enter (: System.Threading.Lock.Enter (-> ClrLock void))) + (clr-lock-exit (: System.Threading.Lock.Exit (-> ClrLock void))) + (clr-add-unit (: System.Text.StringBuilder.Append (-> TextBuilder System.Char TextBuilder))) + (clr-add-text (: System.Text.StringBuilder.Append (-> TextBuilder string TextBuilder))) + (clr-builder-text (: System.Text.StringBuilder.ToString (-> TextBuilder string)))) + +;; Every export is documented there; the module doc is too. +(:doc #:file "datetime.bjodoc") + +;; The prelude's, which timers take too. Offered here as well, so that an +;; import of the time types from this module names it as before. +(re-export Duration) + +(export + ;; Types + Date Time DateTime Instant TimeZone Moment + DatePeriod Period Gap Overlap Resolver + ;; Traits + HasDate ->date HasTime ->time HasInstant ->instant + DateArith +date-period DateDiff ->date-in-zone-of + TimeArith +duration + ;; Construction + date time datetime date+time->datetime ->datetime + duration date-period period + ;; Zones and moments + timezone utc timezone-id timezone-offset-at + resolver datetime->moment datetime->moment/opt instant->moment + +date-period/resolver -date-period/resolver + ->utc-offset ->timezone adjust-timezone + ;; Instants + instant->utc-datetime utc-datetime->instant + instant->unix-seconds unix-seconds->instant + instant->unix-milliseconds unix-milliseconds->instant ->unix + instant->unix-microseconds unix-microseconds->instant + date->unix-days unix-days->date + ;; The clock + system-clock-in clock-in fixed-clock manual-clock clock-advance! + clock-timezone now now/moment today + ;; Timers + at-time + ;; Accessors + ->year ->month ->day ->weekday ->yday + ->week ->week-year weeks-in-year week-date + ->hours ->minutes ->seconds ->milliseconds ->nanoseconds + ;; Arithmetic + +years +months +weeks +days -years -months -weeks -days -date-period + +hours +minutes +seconds +milliseconds + -hours -minutes -seconds -milliseconds -duration + +period -period + duration->hours duration->minutes duration->seconds duration->milliseconds + duration-negate + ;; Differences + years-between months-between weeks-between days-between + duration-between hours-between minutes-between seconds-between milliseconds-between + ;; Text + iso8601->date iso8601->time iso8601->datetime iso8601->instant iso8601->moment + iso8601->duration iso8601->date-period iso8601->period + instant->http-date http-date->instant + ;; Text of your own + ;; DatePart is a grammar. Its export also exports DateField, DateNumber, + ;; DateBareNumber, RomanNumber, TimePart, TimeField, TimeBareNumber, + ;; DateTimePart, DateTimeField, MomentPart and InstantPart. + DatePart DateNames + date->text time->text datetime->text moment->text instant->text + text->date text->time text->datetime text->instant text->moment text->moment/resolver + english-names swedish-names current-date-names) + + +;; =========================================================================== +;; Types +;; =========================================================================== + +;; The value types are opaque so that every value comes from a function that +;; checked it: a Moment's offset is always one its zone has at that time. +(type (: Date #:opaque (Struct (: v ClrDate)))) +(type (: Time #:opaque (Struct (: v ClrTime)))) +;; Kind is always Unspecified. +(type (: DateTime #:opaque (Struct (: v ClrDateTime)))) +;; UTC ticks, not a UTC DateTime, because DateTime equality ignores Kind. +(type (: Instant #:opaque (Struct (: ticks long)))) +(type (: TimeZone #:opaque (Struct (: tz TimeZoneInfo)))) +(type (: Moment #:opaque (Struct (: local DateTime) (: offset Duration) (: zone TimeZone)))) + +;; Any mix of amounts is a valid period, so these are transparent. Structs, +;; so that the one +days builds is not an allocation. +(type (: DatePeriod (Struct (: years int) (: months int) (: weeks int) (: days int)))) +(type (: Period (Struct (: calendar DatePeriod) (: exact Duration)))) + +;; What datetime->moment does with a local time that a zone skips (a gap) or +;; has twice (an overlap). +(type (: Gap (Union GapPush GapNextValid))) +(type (: Overlap (Union OverlapEarlier OverlapLater))) +(type (: Resolver (Record (: gap Gap) (: overlap Overlap)))) + + +;; =========================================================================== +;; Ranges +;; =========================================================================== + +(: long) +(def ticks-per-day 864000000000L) +(: long) +(def ticks-per-hour 36000000000L) +(: long) +(def ticks-per-minute 600000000L) +(: long) +(def ticks-per-second 10000000L) +(: long) +(def ticks-per-millisecond 10000L) + +;; 9999-12-31T23:59:59.9999999 in ticks, and 9999-12-31 as a day number. +(: long) +(def max-ticks 3155378975999999999L) +(: long) +(def max-day-number 3652058L) + +;; 1970-01-01T00:00Z in ticks. +(: long) +(def unix-epoch-ticks 621355968000000000L) + +;; Every Duration stays within ±long-max ticks, about ±29 227 years. Leaving +;; out long's minimum gives every duration a negation, so each one prints and +;; reads back. +(: long) +(def long-max 9223372036854775807L) + +(: (-> int long)) +(defun (long-of n) (cast System.Int64 n)) + +(: (-> long int)) +(defun (int-of n) (cast System.Int32 n)) + +(: (-> string %a)) +(defun (range-error what) + (raise (cast Exception (OverflowException. (str "(std datetime): " what))))) + +(: (-> string %a)) +(defun (calendar-error what) + (range-error (str what " is outside 0001-01-01T00:00 to 9999-12-31T23:59:59.9999999"))) + +(: (-> %a)) +(defun (duration-error) + (range-error "a duration is outside ±29 227 years (±9 223 372 036 854 775 807 ticks)")) + +(: (-> long bool)) +(defun (in-calendar? t) (and (>= t 0L) (<= t max-ticks))) + +;; n × unit for a positive unit, or None past the range of a Duration. +(: (-> long long (Option long))) +(defun (checked-mul n unit) + (def limit (/ long-max unit)) + (if (or (> n limit) (< n (- limit))) None (Some (* n unit)))) + +;; a + b, or None past the range of a Duration. +(: (-> long long (Option long))) +(defun (checked-add a b) + (if (if (> b 0L) (> a (- long-max b)) (< a (- (- long-max) b))) + None + (Some (+ a b)))) + +(: (-> long long long)) +(defun (units->ticks n unit) + (match (checked-mul n unit) + ((Some t) t) + (None (duration-error)))) + +(: (-> long long long)) +(defun (plus-ticks a b) + (match (checked-add a b) + ((Some t) t) + (None (duration-error)))) + +;; A tick count moved by a duration, for DateTime and Instant. The sum can +;; wrap only above long's maximum, and then it lands below zero, so the one +;; range check catches it. +(: (-> long long string long)) +(defun (shift-ticks t d what) + (def moved (+ t d)) + (if (in-calendar? moved) moved (calendar-error what))) + + +;; =========================================================================== +;; Construction +;; =========================================================================== + +(: (-> long DateTime)) +(defun (ticks->datetime t) (DateTime (v (ClrDateTime. t DateTimeKind.Unspecified)))) + +(: (-> DateTime long)) +(defun (datetime-ticks dt) (clr-datetime-ticks (struct-ref dt v))) + +(: (-> Date long)) +(defun (day-number d) (long-of (clr-date-day-number (struct-ref d v)))) + +(: (-> long Date)) +(defun (day-number->date n) (Date (v (clr-date-from-day-number (int-of n))))) + +(: (-> int int int bool)) +(defun (valid-date? y m d) + (and (>= y 1) (<= y 9999) (>= m 1) (<= m 12) (>= d 1) (<= d (clr-days-in-month y m)))) + +(: (-> int int int int bool)) +(defun (valid-time? h m s ns) + (and (>= h 0) (< h 24) (>= m 0) (< m 60) (>= s 0) (< s 60) + (>= ns 0) (< ns 1000000000))) + +;; Nanoseconds are cut to the 100 ns a tick holds. +(: (-> int int int int long)) +(defun (time-of-day-ticks h m s ns) + (+ (+ (* (long-of h) ticks-per-hour) (* (long-of m) ticks-per-minute)) + (+ (* (long-of s) ticks-per-second) (long-of (/ ns 100))))) + +(: (-> int int int (Option Date))) +(defun (date y m d) + (if (valid-date? y m d) (Some (Date (v (ClrDate. y m d)))) None)) + +(: (-> int int (#:seconds int) (#:nanoseconds int) (Option Time))) +(defun (time h m #:seconds 0 #:nanoseconds 0) + (if (valid-time? h m seconds nanoseconds) + (Some (Time (v (ClrTime. (time-of-day-ticks h m seconds nanoseconds))))) + None)) + +(: (-> int int int (#:hours int) (#:minutes int) (#:seconds int) + (#:nanoseconds int) (Option DateTime))) +(defun (datetime y mo d #:hours 0 #:minutes 0 #:seconds 0 #:nanoseconds 0) + (if (and (valid-date? y mo d) (valid-time? hours minutes seconds nanoseconds)) + (Some (ticks->datetime + (+ (* (long-of (clr-date-day-number (ClrDate. y mo d))) ticks-per-day) + (time-of-day-ticks hours minutes seconds nanoseconds)))) + None)) + +(: (-> Date Time DateTime)) +(defun (date+time->datetime d t) + (ticks->datetime (+ (* (day-number d) ticks-per-day) (clr-time-ticks (struct-ref t v))))) + +(: (-> (#:days long) (#:hours long) (#:minutes long) (#:seconds long) + (#:milliseconds long) (#:nanoseconds long) Duration)) +(defun (duration #:days 0L #:hours 0L #:minutes 0L #:seconds 0L #:milliseconds 0L + #:nanoseconds 0L) + (ticks->duration + (plus-ticks (plus-ticks (plus-ticks (units->ticks days ticks-per-day) + (units->ticks hours ticks-per-hour)) + (plus-ticks (units->ticks minutes ticks-per-minute) + (units->ticks seconds ticks-per-second))) + (plus-ticks (units->ticks milliseconds ticks-per-millisecond) + (/ nanoseconds 100L))))) + +(: (-> (#:years int) (#:months int) (#:weeks int) (#:days int) DatePeriod)) +(defun (date-period #:years 0 #:months 0 #:weeks 0 #:days 0) + (DatePeriod (years years) (months months) (weeks weeks) (days days))) + +(: (-> (#:years int) (#:months int) (#:weeks int) (#:days int) + (#:hours long) (#:minutes long) (#:seconds long) + (#:milliseconds long) (#:nanoseconds long) Period)) +(defun (period #:years 0 #:months 0 #:weeks 0 #:days 0 + #:hours 0L #:minutes 0L #:seconds 0L #:milliseconds 0L #:nanoseconds 0L) + (Period (calendar (DatePeriod (years years) (months months) (weeks weeks) (days days))) + (exact (duration #:hours hours #:minutes minutes #:seconds seconds + #:milliseconds milliseconds #:nanoseconds nanoseconds)))) + + +;; =========================================================================== +;; Traits +;; =========================================================================== + +(def/trait (HasDate %a) + (: ->date (-> %a Date))) + +(def/trait (HasTime %a) + (: ->time (-> %a Time))) + +(def/trait (HasInstant %a) + (: ->instant (-> %a Instant))) + +(def/trait (DateArith %a) + (: +date-period (-> %a DatePeriod %a))) + +;; The date of b where a is, which calendar differences compare with a's own: +;; a moment b is first moved into a's zone, so two moments of one instant are +;; the same day wherever they are. +(def/trait (DateDiff %a) + (: ->date-in-zone-of (-> %a %a Date))) + +(def/trait (TimeArith %a) + (: +duration (-> %a Duration %a))) + + +;; =========================================================================== +;; Date +;; =========================================================================== + +;; Each step is checked before .NET takes it, because DateOnly would throw its +;; own error, and because weeks × 7 overflows an int. +(: (-> ClrDate int ClrDate)) +(defun (plus-years c n) + (def y (+ (long-of (clr-date-year c)) (long-of n))) + (cond ((= n 0) c) + ((and (>= y 1L) (<= y 9999L)) (clr-date-add-years c n)) + (else (calendar-error "a date plus years")))) + +;; Months are counted from year 0, so year 1 January is month 12. +(: (-> ClrDate int ClrDate)) +(defun (plus-months c n) + (def m (+ (+ (* (long-of (clr-date-year c)) 12L) (long-of (- (clr-date-month c) 1))) + (long-of n))) + (cond ((= n 0) c) + ((and (>= m 12L) (<= m 119999L)) (clr-date-add-months c n)) + (else (calendar-error "a date plus months")))) + +(: (-> ClrDate long ClrDate)) +(defun (plus-days c n) + (def dn (+ (long-of (clr-date-day-number c)) n)) + (cond ((= n 0L) c) + ((and (>= dn 0L) (<= dn max-day-number)) (clr-date-from-day-number (int-of dn))) + (else (calendar-error "a date plus days")))) + +;; Largest unit first: years, then months, then weeks and days together. +(: (-> ClrDate DatePeriod ClrDate)) +(defun (clr-date-plus-period c p) + (plus-days (plus-months (plus-years c (record-ref p years)) (record-ref p months)) + (+ (* (long-of (record-ref p weeks)) 7L) (long-of (record-ref p days))))) + +;; This and the other `type+unit` functions are the trait methods at one type. +;; They are exported because the impls' bodies, which importers copy, call +;; them. +(: (-> Date DatePeriod Date)) +(defun (date+date-period d p) (Date (v (clr-date-plus-period (struct-ref d v) p)))) + +(impl (HasDate Date) + (defun (->date d) d)) + +(impl (DateArith Date) + (defun (+date-period d p) (date+date-period d p))) + +(impl (DateDiff Date) + (defun (->date-in-zone-of b a) b)) + +(impl (Ord Date) + (defun (compare a b) + (compare (clr-date-day-number (struct-ref a v)) (clr-date-day-number (struct-ref b v))))) + +(impl (Written Date) + (defun (written d) (date->string (struct-ref d v))) + (defun (write-into d b) (stringbuilder-add-string! b (written d)))) + + +;; =========================================================================== +;; Time +;; =========================================================================== + +(impl (HasTime Time) + (defun (->time t) t)) + +;; Wraps at midnight, as TimeOnly.Add does. +(impl (TimeArith Time) + (defun (+duration t d) (Time (v (clr-time-add (struct-ref t v) (ClrTimeSpan. (duration-ticks d))))))) + +(impl (Ord Time) + (defun (compare a b) + (compare (clr-time-ticks (struct-ref a v)) (clr-time-ticks (struct-ref b v))))) + +(impl (Written Time) + (defun (written t) (time-of-day->string (clr-time-ticks (struct-ref t v)))) + (defun (write-into t b) (stringbuilder-add-string! b (written t)))) + + +;; =========================================================================== +;; DateTime +;; =========================================================================== + +(impl (HasDate DateTime) + (defun (->date dt) (Date (v (clr-date-from-datetime (struct-ref dt v)))))) + +(impl (HasTime DateTime) + (defun (->time dt) (Time (v (clr-time-from-datetime (struct-ref dt v)))))) + +(: (-> DateTime DatePeriod DateTime)) +(defun (datetime+date-period dt p) + (def t (datetime-ticks dt)) + (def day (clr-date-plus-period (clr-date-from-day-number (int-of (/ t ticks-per-day))) p)) + (ticks->datetime (+ (* (long-of (clr-date-day-number day)) ticks-per-day) + (% t ticks-per-day)))) + +(: (-> DateTime Duration DateTime)) +(defun (datetime+duration dt d) + (ticks->datetime (shift-ticks (datetime-ticks dt) (duration-ticks d) + "a datetime plus a duration"))) + +(impl (DateArith DateTime) + (defun (+date-period dt p) (datetime+date-period dt p))) + +(impl (DateDiff DateTime) + (defun (->date-in-zone-of b a) (->date b))) + +(impl (TimeArith DateTime) + (defun (+duration dt d) (datetime+duration dt d))) + +(impl (Ord DateTime) + (defun (compare a b) (compare (datetime-ticks a) (datetime-ticks b)))) + +(impl (Written DateTime) + (defun (written dt) (ticks->iso-datetime (datetime-ticks dt))) + (defun (write-into dt b) (stringbuilder-add-string! b (written dt)))) + + +;; =========================================================================== +;; Instant +;; =========================================================================== + +(: (-> Instant long)) +(defun (instant-ticks i) (struct-ref i ticks)) + +(: (-> long string Instant)) +(defun (ticks->instant t what) + (if (in-calendar? t) (Instant (ticks t)) (calendar-error what))) + +(: (-> Instant Duration Instant)) +(defun (instant+duration i d) + (Instant (ticks (shift-ticks (instant-ticks i) (duration-ticks d) + "an instant plus a duration")))) + +(impl (HasInstant Instant) + (defun (->instant i) i)) + +(impl (TimeArith Instant) + (defun (+duration i d) (instant+duration i d))) + +(impl (Ord Instant) + (defun (compare a b) (compare (instant-ticks a) (instant-ticks b)))) + +(impl (Written Instant) + (defun (written i) (ticks->iso-instant (instant-ticks i))) + (defun (write-into i b) (stringbuilder-add-string! b (written i)))) + +(: (-> Instant DateTime)) +(defun (instant->utc-datetime i) (ticks->datetime (instant-ticks i))) + +(: (-> DateTime Instant)) +(defun (utc-datetime->instant dt) (Instant (ticks (datetime-ticks dt)))) + +;; Rounds toward the past, so the second before 1970 is -1 rather than 0. +(: (-> long long long)) +(defun (floor-div a b) + (def q (/ a b)) + (if (< (% a b) 0L) (- q 1L) q)) + +(: (-> Instant long)) +(defun (instant->unix-seconds i) + (floor-div (- (instant-ticks i) unix-epoch-ticks) ticks-per-second)) + +(: (-> Instant long)) +(defun (instant->unix-milliseconds i) + (floor-div (- (instant-ticks i) unix-epoch-ticks) ticks-per-millisecond)) + +;; The bounds are 0001-01-01T00:00Z and the last whole unit of 9999, so the +;; multiplication cannot overflow. +(: (-> long (Option Instant))) +(defun (unix-seconds->instant s) + (if (and (>= s -62135596800L) (<= s 253402300799L)) + (Some (Instant (ticks (+ (* s ticks-per-second) unix-epoch-ticks)))) + None)) + +(: (-> long (Option Instant))) +(defun (unix-milliseconds->instant ms) + (if (and (>= ms -62135596800000L) (<= ms 253402300799999L)) + (Some (Instant (ticks (+ (* ms ticks-per-millisecond) unix-epoch-ticks)))) + None)) + +(: (-> Instant long)) +(defun (instant->unix-microseconds i) + (floor-div (- (instant-ticks i) unix-epoch-ticks) 10L)) + +(: (-> long (Option Instant))) +(defun (unix-microseconds->instant us) + (if (and (>= us -62135596800000000L) (<= us 253402300799999999L)) + (Some (Instant (ticks (+ (* us 10L) unix-epoch-ticks)))) + None)) + +;; Days since 1970-01-01, which is day number 719 162. A column of dates in +;; Apache Arrow, Polars and R holds these. +(: long) +(def unix-epoch-day-number 719162L) + +(: (-> Date long)) +(defun (date->unix-days d) (- (day-number d) unix-epoch-day-number)) + +(: (-> long (Option Date))) +(defun (unix-days->date n) + (def dn (+ n unix-epoch-day-number)) + (if (and (>= dn 0L) (<= dn max-day-number)) (Some (day-number->date dn)) None)) + +;; Unix seconds of an instant or a moment, rounded as instant->unix-seconds +;; rounds, which it calls because an importer copies this body. +(: (-> %a long) (where (HasInstant %a))) +(defun (->unix x) (instant->unix-seconds (->instant x))) + + +;; =========================================================================== +;; Duration +;; =========================================================================== + +;; Checked here rather than by TimeSpan.Add, which lets a sum reach -2^63 +;; ticks: the one TimeSpan with no negation, which could not be printed or +;; read back. +(: (-> Duration Duration Duration)) +(defun (duration+duration a b) (ticks->duration (plus-ticks (duration-ticks a) (duration-ticks b)))) + +;; (+hours (duration #:minutes 30) 1) is PT1H30M. +(impl (TimeArith Duration) + (defun (+duration a b) (duration+duration a b))) + +;; Whole units, toward zero. +(: (-> Duration long)) +(defun (duration->hours d) (/ (duration-ticks d) ticks-per-hour)) + +(: (-> Duration long)) +(defun (duration->minutes d) (/ (duration-ticks d) ticks-per-minute)) + +(: (-> Duration long)) +(defun (duration->seconds d) (/ (duration-ticks d) ticks-per-second)) + +(: (-> Duration long)) +(defun (duration->milliseconds d) (/ (duration-ticks d) ticks-per-millisecond)) + +;; Never raises, as no duration reaches -2^63 ticks. +(: (-> Duration Duration)) +(defun (duration-negate d) (ticks->duration (- (duration-ticks d)))) + + +;; =========================================================================== +;; Periods +;; =========================================================================== + +(: (-> DatePeriod DatePeriod)) +(defun (date-period-negate p) + (DatePeriod (years (- (record-ref p years))) (months (- (record-ref p months))) + (weeks (- (record-ref p weeks))) (days (- (record-ref p days))))) + +(impl (Written DatePeriod) + (defun (written p) (date-period->string p)) + (defun (write-into p b) (stringbuilder-add-string! b (written p)))) + +(impl (Written Period) + (defun (written p) (period->string p)) + (defun (write-into p b) (stringbuilder-add-string! b (written p)))) + + +;; =========================================================================== +;; Accessors and arithmetic +;; =========================================================================== + +(: (-> %a DateTime) (where (HasDate %a) (HasTime %a))) +(defun (->datetime x) (date+time->datetime (->date x) (->time x))) + +(: (-> %a int) (where (HasDate %a))) +(defun (->year x) (clr-date-year (struct-ref (->date x) v))) + +(: (-> %a int) (where (HasDate %a))) +(defun (->month x) (clr-date-month (struct-ref (->date x) v))) + +(: (-> %a int) (where (HasDate %a))) +(defun (->day x) (clr-date-day (struct-ref (->date x) v))) + +;; ISO 8601: 1 is Monday, 7 is Sunday. Day number 0, 0001-01-01, was a Monday. +(: (-> %a int) (where (HasDate %a))) +(defun (->weekday x) (+ (% (clr-date-day-number (struct-ref (->date x) v)) 7) 1)) + +(: (-> %a int) (where (HasDate %a))) +(defun (->yday x) (clr-date-day-of-year (struct-ref (->date x) v))) + +;; ISO 8601 weeks run Monday to Sunday, and a week belongs to the year its +;; Thursday is in, so week 1 holds the year's first Thursday. n - n mod 7 is +;; the Monday, as day number 0 was one. Every date in range has its Thursday +;; in range, since 0001-01-01 was a Monday and 9999-12-31 a Friday. +(: (-> %a int) (where (HasDate %a))) +(defun (->week x) + (def n (clr-date-day-number (struct-ref (->date x) v))) + (+ (/ (- (clr-date-day-of-year (clr-date-from-day-number (+ (- n (% n 7)) 3))) 1) 7) 1)) + +;; The year ->week counts in. It is not always ->year: 2027-01-01 is in week +;; 53 of 2026. +(: (-> %a int) (where (HasDate %a))) +(defun (->week-year x) + (def n (clr-date-day-number (struct-ref (->date x) v))) + (clr-date-year (clr-date-from-day-number (+ (- n (% n 7)) 3)))) + +;; 52 or 53. December 28 is always in its year's last week. +(: (-> int int)) +(defun (weeks-in-year y) + (if (and (>= y 1) (<= y 9999)) + (->week (Date (v (ClrDate. y 12 28)))) + (range-error (str "year " (displayed y) " is outside 1 to 9999")))) + +;; Day d (1 is Monday) of week w of week-year y. Week 1 starts on the Monday +;; on or before January 4. +(: (-> int int int (Option Date))) +(defun (week-date y w d) + (if (and (>= y 1) (<= y 9999) (>= w 1) (<= w (weeks-in-year y)) (>= d 1) (<= d 7)) + (let* ((jan4 (long-of (clr-date-day-number (ClrDate. y 1 4)))) + (n (+ (- jan4 (% jan4 7L)) (long-of (+ (* (- w 1) 7) (- d 1)))))) + ;; The last week of 9999 ends in 10000. + (if (<= n max-day-number) (Some (day-number->date n)) None)) + None)) + +(: (-> %a int) (where (HasTime %a))) +(defun (->hours x) (clr-time-hour (struct-ref (->time x) v))) + +(: (-> %a int) (where (HasTime %a))) +(defun (->minutes x) (clr-time-minute (struct-ref (->time x) v))) + +(: (-> %a int) (where (HasTime %a))) +(defun (->seconds x) (clr-time-second (struct-ref (->time x) v))) + +(: (-> %a int) (where (HasTime %a))) +(defun (->milliseconds x) (clr-time-millisecond (struct-ref (->time x) v))) + +;; The whole fraction of the second, so always a multiple of 100. +(: (-> %a int) (where (HasTime %a))) +(defun (->nanoseconds x) + (def t (struct-ref (->time x) v)) + (+ (* (clr-time-millisecond t) 1000000) + (+ (* (clr-time-microsecond t) 1000) (clr-time-nanosecond t)))) + +(: (-> %a int %a) (where (DateArith %a))) +(defun (+years x n) (+date-period x (DatePeriod (years n) (months 0) (weeks 0) (days 0)))) + +(: (-> %a int %a) (where (DateArith %a))) +(defun (+months x n) (+date-period x (DatePeriod (years 0) (months n) (weeks 0) (days 0)))) + +(: (-> %a int %a) (where (DateArith %a))) +(defun (+weeks x n) (+date-period x (DatePeriod (years 0) (months 0) (weeks n) (days 0)))) + +(: (-> %a int %a) (where (DateArith %a))) +(defun (+days x n) (+date-period x (DatePeriod (years 0) (months 0) (weeks 0) (days n)))) + +(: (-> %a int %a) (where (DateArith %a))) +(defun (-years x n) (+years x (- n))) + +(: (-> %a int %a) (where (DateArith %a))) +(defun (-months x n) (+months x (- n))) + +(: (-> %a int %a) (where (DateArith %a))) +(defun (-weeks x n) (+weeks x (- n))) + +(: (-> %a int %a) (where (DateArith %a))) +(defun (-days x n) (+days x (- n))) + +(: (-> %a DatePeriod %a) (where (DateArith %a))) +(defun (-date-period x p) + (+date-period x (DatePeriod (years (- (record-ref p years))) (months (- (record-ref p months))) + (weeks (- (record-ref p weeks))) (days (- (record-ref p days)))))) + +(: (-> %a long %a) (where (TimeArith %a))) +(defun (+hours x n) (+duration x (duration #:hours n))) + +(: (-> %a long %a) (where (TimeArith %a))) +(defun (+minutes x n) (+duration x (duration #:minutes n))) + +(: (-> %a long %a) (where (TimeArith %a))) +(defun (+seconds x n) (+duration x (duration #:seconds n))) + +(: (-> %a long %a) (where (TimeArith %a))) +(defun (+milliseconds x n) (+duration x (duration #:milliseconds n))) + +;; Negated after the range check, so long's minimum is refused rather than +;; wrapped back to itself. +(: (-> %a long %a) (where (TimeArith %a))) +(defun (-hours x n) (+duration x (duration-negate (duration #:hours n)))) + +(: (-> %a long %a) (where (TimeArith %a))) +(defun (-minutes x n) (+duration x (duration-negate (duration #:minutes n)))) + +(: (-> %a long %a) (where (TimeArith %a))) +(defun (-seconds x n) (+duration x (duration-negate (duration #:seconds n)))) + +(: (-> %a long %a) (where (TimeArith %a))) +(defun (-milliseconds x n) (+duration x (duration-negate (duration #:milliseconds n)))) + +(: (-> %a Duration %a) (where (TimeArith %a))) +(defun (-duration x d) (+duration x (duration-negate d))) + +(: (-> %a Period %a) (where (DateArith %a) (TimeArith %a))) +(defun (+period x p) + (+duration (+date-period x (record-ref p calendar)) (record-ref p exact))) + +(: (-> %a Period %a) (where (DateArith %a) (TimeArith %a))) +(defun (-period x p) + (-duration (-date-period x (record-ref p calendar)) (record-ref p exact))) + + +;; =========================================================================== +;; Differences +;; =========================================================================== + +;; Calendar differences read the dates only: 23:00 to 01:00 the next day is +;; one day. Both arguments have one type, and b's date is read where a is. +(: (-> %a %a int) (where (HasDate %a) (DateDiff %a))) +(defun (days-between a b) + (- (clr-date-day-number (struct-ref (->date-in-zone-of b a) v)) + (clr-date-day-number (struct-ref (->date a) v)))) + +(: (-> %a %a int) (where (HasDate %a) (DateDiff %a))) +(defun (weeks-between a b) (/ (days-between a b) 7)) + +;; A month counts once the day of the month is reached, as in java.time, so +;; swapping the arguments only flips the sign: 2026-01-31 to 2026-02-28 is 0. +(: (-> %a %a int) (where (HasDate %a) (DateDiff %a))) +(defun (months-between a b) + (def x (struct-ref (->date a) v)) + (def y (struct-ref (->date-in-zone-of b a) v)) + (def months (- (+ (* (clr-date-year y) 12) (clr-date-month y)) + (+ (* (clr-date-year x) 12) (clr-date-month x)))) + (def days (- (clr-date-day y) (clr-date-day x))) + (cond ((and (> months 0) (< days 0)) (- months 1)) + ((and (< months 0) (> days 0)) (+ months 1)) + (else months))) + +(: (-> %a %a int) (where (HasDate %a) (DateDiff %a))) +(defun (years-between a b) (/ (months-between a b) 12)) + +;; The time that passed from a to b, between their instants, so a moment and +;; an instant mix, and a DateTime, which has no instant, does not compile. Two +;; instants are less than 10 000 years apart, so the difference fits. +(: (-> %a %b Duration) (where (HasInstant %a) (HasInstant %b))) +(defun (duration-between a b) + (ticks->duration (- (struct-ref (->instant b) ticks) (struct-ref (->instant a) ticks)))) + +(: (-> %a %b long) (where (HasInstant %a) (HasInstant %b))) +(defun (hours-between a b) (duration->hours (duration-between a b))) + +(: (-> %a %b long) (where (HasInstant %a) (HasInstant %b))) +(defun (minutes-between a b) (duration->minutes (duration-between a b))) + +(: (-> %a %b long) (where (HasInstant %a) (HasInstant %b))) +(defun (seconds-between a b) (duration->seconds (duration-between a b))) + +(: (-> %a %b long) (where (HasInstant %a) (HasInstant %b))) +(defun (milliseconds-between a b) (duration->milliseconds (duration-between a b))) + + +;; =========================================================================== +;; Zones +;; =========================================================================== + +(: TimeZone) +(def utc (TimeZone (tz clr-tz-utc))) + +;; IANA ids only: .NET would also take a Windows id such as "W. Europe +;; Standard Time", which HasIanaId refuses. +(: (-> string (Option TimeZone))) +(defun (timezone id) + (match (clr-tz-try-find id) + ((Some z) (if (clr-tz-has-iana-id z) (Some (TimeZone (tz z))) None)) + (None None))) + +(: (-> TimeZone string)) +(defun (timezone-id z) (clr-tz-id (struct-ref z tz))) + +(impl (Eq TimeZone) + (defun (= a b) (= (timezone-id a) (timezone-id b))) + (defun (eq-hash z) (eq-hash (timezone-id z)))) + +(impl (Written TimeZone) + (defun (written z) (timezone-id z)) + (defun (write-into z b) (stringbuilder-add-string! b (written z)))) + +;; The offset in z at the UTC tick count u, in ticks. u is clamped into the +;; calendar, because local-fit looks a day past a local time on the first or +;; last day; no zone changes offset within a day of either end, so the answer +;; is the same. +(: (-> TimeZoneInfo long long)) +(defun (offset-at z u) + (def clamped (if (< u 0L) 0L (if (> u max-ticks) max-ticks u))) + (clr-timespan-ticks (clr-tz-utc-offset z (ClrDateTime. clamped DateTimeKind.Utc)))) + +(: (-> TimeZone Instant Duration)) +(defun (timezone-offset-at z i) (ticks->duration (offset-at (struct-ref z tz) (instant-ticks i)))) + +;; Where a local time lands in a zone: at one offset, or two in an overlap +;; (the larger first, since its instant is the earlier), or in a gap between +;; the offset before and the offset after. +(type (: Fit (Union (: Fits long long) (: Skipped long long)))) + +(: (-> long (List long) bool)) +(defun (member-long? x xs) + (and (not (list-empty? xs)) + (or (= x (list-head xs)) (member-long? x (list-tail xs))))) + +;; An offset o fits the local tick count l when the zone has offset o at the +;; instant l - o. The candidates are the offsets a day either side of l, which +;; is every offset in reach while a zone changes offset at most once in 48 h: +;; the tz database's closest two changes are 95 h apart. When a lookup answers +;; with a third offset it becomes a candidate too, which catches most layouts +;; with two changes in 48 h, but not all. +(: (-> TimeZoneInfo long Fit)) +(defun (local-fit z l) + (def before (offset-at z (- l ticks-per-day))) + (def after (offset-at z (+ l ticks-per-day))) + (def at-before (offset-at z (- l before))) + (if (and (= before after) (= at-before before)) + (Fits before before) + (let* ((at-after (offset-at z (- l after))) + (candidates (list-foldl (fun (o seen) (if (member-long? o seen) seen (Cons o seen))) + '() (list before after at-before at-after))) + (fits (list-filter (fun (o) (= (offset-at z (- l o)) o)) candidates))) + (if (list-empty? fits) + (Skipped before at-before) + (Fits (list-foldl max (list-head fits) fits) (list-foldl min (list-head fits) fits)))))) + +;; The first instant at offset `after`, by bisection between an instant still +;; before the change and one after it. +(: (-> TimeZoneInfo long long long long)) +(defun (change-ticks z lo hi after) + (if (<= (- hi lo) 1L) + hi + (let ((mid (+ lo (/ (- hi lo) 2L)))) + (if (= (offset-at z mid) after) + (change-ticks z lo mid after) + (change-ticks z mid hi after))))) + + +;; =========================================================================== +;; Moments +;; =========================================================================== + +(: (-> Moment long)) +(defun (moment-local-ticks m) (datetime-ticks (struct-ref m local))) + +(: (-> Moment long)) +(defun (moment-offset-ticks m) (duration-ticks (struct-ref m offset))) + +(: (-> Moment long)) +(defun (moment-instant-ticks m) (- (moment-local-ticks m) (moment-offset-ticks m))) + +;; A moment from a local time and an offset that fits it. Near either end of +;; the calendar the instant can fall outside it: 0001-01-01T00:00 in Tokyo is +;; the day before. +(: (-> long long TimeZone Moment)) +(defun (fitted-moment l o z) + (if (in-calendar? (- l o)) + (Moment (local (ticks->datetime l)) (offset (ticks->duration o)) (zone z)) + (calendar-error "the instant of a local time"))) + +(: (-> Instant TimeZone Moment)) +(defun (instant->moment i z) + (def u (instant-ticks i)) + (def o (offset-at (struct-ref z tz) u)) + (if (in-calendar? (+ u o)) + (Moment (local (ticks->datetime (+ u o))) (offset (ticks->duration o)) (zone z)) + (calendar-error "the local time of an instant"))) + +(: (-> Gap Overlap Resolver)) +(defun (resolver g o) (Resolver (gap g) (overlap o))) + +;; GapPush moves the local time forward by the length of the gap: the instant +;; it names at the offset before the gap, read at the offset after it. +(: (-> long TimeZone Resolver Moment)) +(defun (resolve-local l z r) + (match (local-fit (struct-ref z tz) l) + ((Fits earlier later) + (fitted-moment l (match (record-ref r overlap) + (OverlapEarlier earlier) + (OverlapLater later)) + z)) + ((Skipped before after) + (match (record-ref r gap) + (GapPush (instant->moment (ticks->instant (- l before) "the instant of a local time") z)) + (GapNextValid + (instant->moment (Instant (ticks (change-ticks (struct-ref z tz) (- l after) (- l before) after))) + z)))))) + +(: (-> DateTime TimeZone Resolver Moment)) +(defun (datetime->moment dt z r) (resolve-local (datetime-ticks dt) z r)) + +(: (-> DateTime TimeZone (Option Moment))) +(defun (datetime->moment/opt dt z) + (def l (datetime-ticks dt)) + (match (local-fit (struct-ref z tz) l) + ((Fits earlier later) + (if (and (= earlier later) (in-calendar? (- l earlier))) + (Some (Moment (local dt) (offset (ticks->duration earlier)) (zone z))) + None)) + (_ None))) + +(: (-> Moment DatePeriod Resolver Moment)) +(defun (+date-period/resolver m p r) + (datetime->moment (datetime+date-period (struct-ref m local) p) (struct-ref m zone) r)) + +(: (-> Moment DatePeriod Resolver Moment)) +(defun (-date-period/resolver m p r) (+date-period/resolver m (date-period-negate p) r)) + +;; Date arithmetic on a moment keeps its offset through an overlap if that +;; offset still fits, takes the earlier one if not, and pushes forward through +;; a gap. +(: (-> Moment DatePeriod Moment)) +(defun (moment+date-period m p) + (def l (datetime-ticks (datetime+date-period (struct-ref m local) p))) + (def z (struct-ref m zone)) + (def kept (moment-offset-ticks m)) + (match (local-fit (struct-ref z tz) l) + ((Fits earlier later) + (fitted-moment l (if (or (= kept earlier) (= kept later)) kept earlier) z)) + ((Skipped before after) + (instant->moment (ticks->instant (- l before) "the instant of a local time") z)))) + +(: (-> Moment Duration)) +(defun (->utc-offset m) (struct-ref m offset)) + +(: (-> Moment TimeZone)) +(defun (->timezone m) (struct-ref m zone)) + +(: (-> Moment TimeZone Moment)) +(defun (adjust-timezone m z) + (instant->moment (Instant (ticks (moment-instant-ticks m))) z)) + +;; Exact: through the instant, and back into the same zone. +(: (-> Moment Duration Moment)) +(defun (moment+duration m d) + (instant->moment + (Instant (ticks (shift-ticks (moment-instant-ticks m) (duration-ticks d) + "a moment plus a duration"))) + (struct-ref m zone))) + +(impl (HasDate Moment) + (defun (->date m) (->date (struct-ref m local)))) + +(impl (HasTime Moment) + (defun (->time m) (->time (struct-ref m local)))) + +(impl (HasInstant Moment) + (defun (->instant m) + (Instant (ticks (- (clr-datetime-ticks (struct-ref (struct-ref m local) v)) + (duration-ticks (struct-ref m offset))))))) + +(impl (DateArith Moment) + (defun (+date-period m p) (moment+date-period m p))) + +;; Near either end of the calendar, the move can raise as instant->moment does. +(impl (DateDiff Moment) + (defun (->date-in-zone-of b a) + (if (= (->timezone b) (->timezone a)) + (->date b) + (->date (adjust-timezone b (->timezone a)))))) + +(impl (TimeArith Moment) + (defun (+duration m d) (moment+duration m d))) + +;; Every field, so 12:00+02:00[Europe/Stockholm] and 10:00+00:00[UTC] are two +;; values although they are one instant. +(impl (Eq Moment) + (defun (= a b) + (and (= (moment-local-ticks a) (moment-local-ticks b)) + (= (moment-offset-ticks a) (moment-offset-ticks b)) + (= (struct-ref a zone) (struct-ref b zone)))) + (defun (eq-hash m) + (hash-combine (eq-hash (moment-local-ticks m)) + (hash-combine (eq-hash (moment-offset-ticks m)) (eq-hash (struct-ref m zone)))))) + +;; By instant, then local time, then zone id, so that compare is 0 exactly +;; when = holds. +(impl (Ord Moment) + (defun (compare a b) + (def by-instant (compare (moment-instant-ticks a) (moment-instant-ticks b))) + (def by-local (compare (moment-local-ticks a) (moment-local-ticks b))) + (cond ((not (= by-instant 0)) by-instant) + ((not (= by-local 0)) by-local) + (else (compare (timezone-id (struct-ref a zone)) (timezone-id (struct-ref b zone))))))) + +(impl (Written Moment) + (defun (written m) (moment->string m)) + (defun (write-into m b) (stringbuilder-add-string! b (written m)))) + +(impl (Written Gap) + (defun (written g) (match g (GapPush "GapPush") (GapNextValid "GapNextValid"))) + (defun (write-into g b) (stringbuilder-add-string! b (written g)))) + +(impl (Written Overlap) + (defun (written o) (match o (OverlapEarlier "OverlapEarlier") (OverlapLater "OverlapLater"))) + (defun (write-into o b) (stringbuilder-add-string! b (written o)))) + +(impl (Written Resolver) + (defun (written r) + (str "(resolver " (written (record-ref r gap)) " " (written (record-ref r overlap)) ")")) + (defun (write-into r b) (stringbuilder-add-string! b (written r)))) + + +;; =========================================================================== +;; The clock +;; =========================================================================== + +;; The functions that read the time take #:clock, defaulting to the value of +;; the prelude's current-clock at the call, so a program that binds the +;; parameter changes no call site. They are the only functions here that are +;; not pure. + +(: (-> TimeZone (-> TimeZoneInfo))) +(defun (zone-reader z) + (def info (struct-ref z tz)) + (fun () info)) + +;; c, with today in z. +(: (-> Clock TimeZone Clock)) +(defun (clock-in c z) (record-set c (read-zone (zone-reader z)))) + +;; The system's time in zone z, so that a program on a machine set to UTC can +;; bind current-clock once and have (today) be z's date everywhere. +(: (-> TimeZone Clock)) +(defun (system-clock-in z) (clock-in system-clock z)) + +;; Always the same instant. Its monotonic reading stands still too. +(: (-> Instant TimeZone Clock)) +(defun (fixed-clock i z) + (def t (instant-ticks i)) + (make-clock #:monotonic-ms (fun () 0L) + #:utc-ticks (fun () t) + #:zone (zone-reader z))) + +(: (-> ClrLock (-> %a) %a)) +(defun (locked lock body) + (clr-lock-enter lock) + (try (body) #:finally (clr-lock-exit lock))) + +;; Stands still until clock-advance! moves it. Read and written under its lock, +;; so fibers on other threads may share one: an advance is whole, and visible +;; to the next read. +;; +;; The monotonic reading starts at 0 and moves only on a positive advance, so a +;; step back of the wall clock never makes an elapsed time negative. +(: (-> Instant TimeZone Clock)) +(defun (manual-clock i z) + (def wall (make-box i)) + (def mono (make-box 0L)) + (def lock (ClrLock.)) + (make-clock + #:monotonic-ms (fun () (/ (locked lock (fun () (box-ref mono))) ticks-per-millisecond)) + #:monotonic-ns (Some (fun () (* (locked lock (fun () (box-ref mono))) 100L))) + #:utc-ticks (fun () (instant-ticks (locked lock (fun () (box-ref wall))))) + #:zone (zone-reader z) + #:advance (Some (fun (t) + (clr-lock-enter lock) + (try (let () + (box-set! wall (instant+duration (box-ref wall) (ticks->duration t))) + (when (> t 0L) (box-set! mono (+ (box-ref mono) t)))) + #:finally (clr-lock-exit lock)))))) + +;; A negative duration steps the clock back, as a corrected wall clock does. +(: (-> Clock Duration void)) +(defun (clock-advance! c d) + (match (record-ref c advance-ticks) + ((Some advance) (advance (duration-ticks d))) + (None (raise (cast Exception + (ArgumentException. + "(std datetime): clock-advance! moves a clock made with #:advance, as a manual clock is, and this one has none.")))))) + +(: (-> (#:clock Clock) TimeZone)) +(defun (clock-timezone #:clock (parameter-ref current-clock)) + (TimeZone (tz ((record-ref clock read-zone))))) + +(: (-> (#:clock Clock) Instant)) +(defun (now #:clock (parameter-ref current-clock)) + (ticks->instant ((record-ref clock read-utc-ticks)) "a clock's reading")) + +(: (-> (#:clock Clock) (#:tz TimeZone) Moment)) +(defun (now/moment #:clock (parameter-ref current-clock) #:tz (clock-timezone #:clock clock)) + (instant->moment (now #:clock clock) tz)) + +(: (-> (#:clock Clock) (#:tz TimeZone) Date)) +(defun (today #:clock (parameter-ref current-clock) #:tz (clock-timezone #:clock clock)) + (->date (now/moment #:clock clock #:tz tz))) + + +;; =========================================================================== +;; Timers +;; =========================================================================== + +;; Ready at an instant. The time left is worked out at each sync against the +;; system's clock, which is what a timer waits by, and not against +;; `current-clock`: a clock that is fixed or moved by hand does not move a +;; timer. +(: (-> Instant (Event Unit))) +(defun (at-time i) (clr-at-utc (ClrDateTime. (struct-ref i ticks) DateTimeKind.Utc))) + + +;; =========================================================================== +;; Writing ISO 8601 +;; =========================================================================== +;; +;; A writer appends to one StringBuilder sized for its text, a digit at a time, +;; so writing a value allocates the builder and the string, not a string per +;; field. + +(: (-> int TextBuilder)) +(defun (new-text capacity) (TextBuilder. capacity)) + +(: (-> TextBuilder string)) +(defun (text-of sb) (clr-builder-text sb)) + +(: (-> TextBuilder string void)) +(defun (add-text! sb s) (ignore (clr-add-text sb s))) + +(: (-> TextBuilder long void)) +(defun (add-digit! sb d) (ignore (clr-add-unit sb (cast System.Char (+ 48 (int-of d)))))) + +;; The number of decimal digits in n >= 0, 0 having one. +(: (-> long int)) +(defun (digit-count n) (if (< n 10L) 1 (+ 1 (digit-count (/ n 10L))))) + +;; The largest power of ten not above n >= 0, or 1. +(: (-> long long)) +(defun (top-power n) (if (< n 10L) 1L (* 10L (top-power (/ n 10L))))) + +;; n >= 0 in decimal, with zeros in front up to width digits. +(: (-> TextBuilder long int void)) +(defun (add-padded! sb n width) + (let zeros ((k (digit-count n))) + (when (< k width) + (add-digit! sb 0L) + (zeros (+ k 1)))) + (let digits ((p (top-power n))) + (when (> p 0L) + (add-digit! sb (% (/ n p) 10L)) + (digits (/ p 10L))))) + +(: (-> TextBuilder ClrDate void)) +(defun (add-date! sb c) + (add-padded! sb (long-of (clr-date-year c)) 4) + (add-text! sb "-") + (add-padded! sb (long-of (clr-date-month c)) 2) + (add-text! sb "-") + (add-padded! sb (long-of (clr-date-day c)) 2)) + +;; ".5" for half a second: up to seven digits, without trailing zeros, and +;; nothing for a whole second. +(: (-> TextBuilder long void)) +(defun (add-fraction! sb f) + (unless (= f 0L) + (let loop ((f f) (width 7)) + (if (= (% f 10L) 0L) + (loop (/ f 10L) (- width 1)) + (begin + (add-text! sb ".") + (add-padded! sb f width)))))) + +;; hh:mm:ss of a tick count within a day. +(: (-> TextBuilder long void)) +(defun (add-clock! sb t) + (add-padded! sb (/ t ticks-per-hour) 2) + (add-text! sb ":") + (add-padded! sb (% (/ t ticks-per-minute) 60L) 2) + (add-text! sb ":") + (add-padded! sb (% (/ t ticks-per-second) 60L) 2)) + +(: (-> TextBuilder long void)) +(defun (add-time-of-day! sb t) + (add-clock! sb t) + (add-fraction! sb (% t ticks-per-second))) + +(: (-> TextBuilder long void)) +(defun (add-datetime! sb t) + (add-date! sb (clr-date-from-day-number (int-of (/ t ticks-per-day)))) + (add-text! sb "T") + (add-time-of-day! sb (% t ticks-per-day))) + +;; ±hh:mm, and :ss when there are seconds. +(: (-> TextBuilder long void)) +(defun (add-offset! sb o) + (def a (abs o)) + (def s (% (/ a ticks-per-second) 60L)) + (add-text! sb (if (< o 0L) "-" "+")) + (add-padded! sb (/ a ticks-per-hour) 2) + (add-text! sb ":") + (add-padded! sb (% (/ a ticks-per-minute) 60L) 2) + (unless (= s 0L) + (add-text! sb ":") + (add-padded! sb s 2))) + +(: (-> ClrDate string)) +(defun (date->string c) + (def sb (new-text 10)) + (add-date! sb c) + (text-of sb)) + +(: (-> long string)) +(defun (time-of-day->string t) + (def sb (new-text 16)) + (add-time-of-day! sb t) + (text-of sb)) + +(: (-> long string)) +(defun (ticks->iso-datetime t) + (def sb (new-text 27)) + (add-datetime! sb t) + (text-of sb)) + +(: (-> long string)) +(defun (ticks->iso-instant t) + (def sb (new-text 28)) + (add-datetime! sb t) + (add-text! sb "Z") + (text-of sb)) + +;; 38 characters hold the longest local time and offset, and the brackets. +(: (-> Moment string)) +(defun (moment->string m) + (def id (timezone-id (struct-ref m zone))) + (def sb (new-text (+ 38 (string-length id)))) + (add-datetime! sb (moment-local-ticks m)) + (add-offset! sb (moment-offset-ticks m)) + (add-text! sb "[") + (add-text! sb id) + (add-text! sb "]") + (text-of sb)) + +;; n and its unit, with a minus when n is negative, and nothing for 0. +(: (-> TextBuilder long string void)) +(defun (add-amount! sb n unit) + (unless (= n 0L) + (when (< n 0L) (add-text! sb "-")) + (add-padded! sb (abs n) 1) + (add-text! sb unit))) + +;; The time components of a non-negative tick count, each prefixed by sign. +(: (-> TextBuilder long string void)) +(defun (add-time-parts! sb a sign) + (def h (/ a ticks-per-hour)) + (def m (% (/ a ticks-per-minute) 60L)) + (def s (% (/ a ticks-per-second) 60L)) + (def f (% a ticks-per-second)) + (unless (= h 0L) + (add-text! sb sign) + (add-padded! sb h 1) + (add-text! sb "H")) + (unless (= m 0L) + (add-text! sb sign) + (add-padded! sb m 1) + (add-text! sb "M")) + (unless (and (= s 0L) (= f 0L)) + (add-text! sb sign) + (add-padded! sb s 1) + (add-fraction! sb f) + (add-text! sb "S"))) + +;; Each component carries its own sign, since a period may mix them: P1M-3D. +(: (-> TextBuilder DatePeriod void)) +(defun (add-date-parts! sb p) + (add-amount! sb (long-of (record-ref p years)) "Y") + (add-amount! sb (long-of (record-ref p months)) "M") + (add-amount! sb (long-of (record-ref p weeks)) "W") + (add-amount! sb (long-of (record-ref p days)) "D")) + +(: (-> DatePeriod bool)) +(defun (date-period-zero? p) + (and (= (record-ref p years) 0) (= (record-ref p months) 0) + (= (record-ref p weeks) 0) (= (record-ref p days) 0))) + +(: (-> DatePeriod string)) +(defun (date-period->string p) + (if (date-period-zero? p) + "P0D" + (let ((sb (new-text 48))) + (add-text! sb "P") + (add-date-parts! sb p) + (text-of sb)))) + +;; P1DT3H. A negative exact part signs each of its components: P1DT-3H. +(: (-> Period string)) +(defun (period->string p) + (def dates (record-ref p calendar)) + (def t (duration-ticks (record-ref p exact))) + (if (and (date-period-zero? dates) (= t 0L)) + "P0D" + (let ((sb (new-text 80))) + (add-text! sb "P") + (add-date-parts! sb dates) + (unless (= t 0L) + (add-text! sb "T") + (add-time-parts! sb (abs t) (if (< t 0L) "-" ""))) + (text-of sb)))) + + +;; =========================================================================== +;; Reading ISO 8601 +;; =========================================================================== +;; +;; Each reader takes the string and a cursor into it, and answers what it read +;; with the cursor after it, or None. Option and Tuple are structs, so reading +;; allocates nothing but a moment's zone id. Only the extended format is read +;; (2026-10-25T02:30:00, not 20261025T023000), with an upper-case T and Z. + +(: (-> char int)) +(defun (ascii-digit c) + (def n (char->int c)) + (if (and (>= n 48) (<= n 57)) (- n 48) -1)) + +(: (-> string StringCursor int)) +(defun (next-digit s c) + (if (string-cursor-end? s c) -1 (ascii-digit (string-cursor-ref s c)))) + +(: (-> string StringCursor char bool)) +(defun (starts-with-char? s c ch) + (and (not (string-cursor-end? s c)) (char=? (string-cursor-ref s c) ch))) + +(: (-> string StringCursor char (Option StringCursor))) +(defun (skip-char s c ch) + (if (starts-with-char? s c ch) (Some (string-cursor-next s c)) None)) + +;; Exactly n digits. +(: (-> string StringCursor int (Option (Tuple int StringCursor)))) +(defun (read-fixed s c n) + (let loop ((c c) (left n) (acc 0)) + (def d (next-digit s c)) + (cond ((= left 0) (Some (Tuple acc c))) + ((< d 0) None) + (else (loop (string-cursor-next s c) (- left 1) (+ (* acc 10) d)))))) + +;; One or more digits, or None past long's range. +(: (-> string StringCursor (Option (Tuple long StringCursor)))) +(defun (read-number s c) + (let loop ((c c) (count 0) (acc 0L)) + (def d (long-of (next-digit s c))) + (cond ((< d 0L) (if (= count 0) None (Some (Tuple acc c)))) + ((> acc (/ (- long-max d) 10L)) None) + (else (loop (string-cursor-next s c) (+ count 1) (+ (* acc 10L) d)))))) + +;; The digits after a decimal point or comma, as ticks. Digits past the +;; seventh are read and dropped, as a tick holds no less than 100 ns. +(: (-> string StringCursor (Option (Tuple long StringCursor)))) +(defun (read-fraction s c) + (let loop ((c c) (count 0) (acc 0L)) + (def d (long-of (next-digit s c))) + (cond ((>= d 0L) + (loop (string-cursor-next s c) (+ count 1) (if (< count 7) (+ (* acc 10L) d) acc))) + ((= count 0) None) + (else (Some (Tuple (scale-fraction acc count) c)))))) + +(: (-> long int long)) +(defun (scale-fraction f digits) + (if (>= digits 7) f (scale-fraction (* f 10L) (+ digits 1)))) + +;; An optional fraction: nothing, or a separator and at least one digit. +(: (-> string StringCursor (Option (Tuple long StringCursor)))) +(defun (read-optional-fraction s c) + (if (or (starts-with-char? s c #\.) (starts-with-char? s c #\,)) + (read-fraction s (string-cursor-next s c)) + (Some (Tuple 0L c)))) + +;; YYYY-MM-DD, as a day number. +(: (-> string StringCursor (Option (Tuple long StringCursor)))) +(defun (read-date s c) + (def* ((Some (Tuple y c1)) (read-fixed s c 4) :propagate) + ((Some c2) (skip-char s c1 #\-) :propagate) + ((Some (Tuple m c3)) (read-fixed s c2 2) :propagate) + ((Some c4) (skip-char s c3 #\-) :propagate) + ((Some (Tuple d c5)) (read-fixed s c4 2) :propagate)) + (if (valid-date? y m d) + (Some (Tuple (long-of (clr-date-day-number (ClrDate. y m d))) c5)) + None)) + +;; hh:mm, hh:mm:ss or hh:mm:ss.fff, as ticks since midnight. No leap second. +(: (-> string StringCursor (Option (Tuple long StringCursor)))) +(defun (read-time s c) + (def* ((Some (Tuple h c1)) (read-fixed s c 2) :propagate) + ((Some c2) (skip-char s c1 #\:) :propagate) + ((Some (Tuple m c3)) (read-fixed s c2 2) :propagate)) + (cond ((not (and (< h 24) (< m 60))) None) + ((not (starts-with-char? s c3 #\:)) (Some (Tuple (time-of-day-ticks h m 0 0) c3))) + (:def (Some (Tuple sec c4)) (read-fixed s (string-cursor-next s c3) 2) :propagate) + (:def (Some (Tuple f c5)) (read-optional-fraction s c4) :propagate) + ((>= sec 60) None) + (else (Some (Tuple (+ (time-of-day-ticks h m sec 0) f) c5))))) + +;; A date, T, and a time, as local ticks. +(: (-> string StringCursor (Option (Tuple long StringCursor)))) +(defun (read-datetime s c) + (def* ((Some (Tuple day c1)) (read-date s c) :propagate) + ((Some c2) (skip-char s c1 #\T) :propagate) + ((Some (Tuple t c3)) (read-time s c2) :propagate)) + (Some (Tuple (+ (* day ticks-per-day) t) c3))) + +;; Z, or ±hh:mm with an optional :ss, as ticks. +(: (-> string StringCursor (Option (Tuple long StringCursor)))) +(defun (read-offset s c) + (cond ((starts-with-char? s c #\Z) (Some (Tuple 0L (string-cursor-next s c)))) + ((starts-with-char? s c #\+) (read-hhmm s (string-cursor-next s c) 1L)) + ((starts-with-char? s c #\-) (read-hhmm s (string-cursor-next s c) -1L)) + (else None))) + +(: (-> string StringCursor long (Option (Tuple long StringCursor)))) +(defun (read-hhmm s c sign) + (def* ((Some (Tuple h c1)) (read-fixed s c 2) :propagate) + ((Some c2) (skip-char s c1 #\:) :propagate) + ((Some (Tuple m c3)) (read-fixed s c2 2) :propagate)) + (cond ((not (and (< h 24) (< m 60))) None) + ((not (starts-with-char? s c3 #\:)) + (Some (Tuple (* sign (time-of-day-ticks h m 0 0)) c3))) + (:def (Some (Tuple sec c4)) (read-fixed s (string-cursor-next s c3) 2) :propagate) + ((>= sec 60) None) + (else (Some (Tuple (* sign (time-of-day-ticks h m sec 0)) c4))))) + +;; The text up to the character ch, and the cursor after ch. +(: (-> string StringCursor char (Option (Tuple string StringCursor)))) +(defun (read-until s c ch) + (let loop ((e c)) + (cond ((string-cursor-end? s e) None) + ((char=? (string-cursor-ref s e) ch) + (Some (Tuple (substring s c e) (string-cursor-next s e)))) + (else (loop (string-cursor-next s e)))))) + +;; A reader's value when it read to the end of s. +(: (-> string (Option (Tuple %a StringCursor)) (Option %a))) +(defun (whole s r) + (match r + ((Some (Tuple v rest)) (if (string-cursor-end? s rest) (Some v) None)) + (None None))) + +(: (-> string (Option Date))) +(defun (iso8601->date s) + (def (Some day) (whole s (read-date s (string-cursor-start s))) :propagate) + (Some (day-number->date day))) + +(: (-> string (Option Time))) +(defun (iso8601->time s) + (def (Some t) (whole s (read-time s (string-cursor-start s))) :propagate) + (Some (Time (v (ClrTime. t))))) + +(: (-> string (Option DateTime))) +(defun (iso8601->datetime s) + (def (Some t) (whole s (read-datetime s (string-cursor-start s))) :propagate) + (Some (ticks->datetime t))) + +;; Any offset, converted to UTC. +(: (-> string (Option Instant))) +(defun (iso8601->instant s) + (def* ((Some (Tuple l c)) (read-datetime s (string-cursor-start s)) :propagate) + ((Some o) (whole s (read-offset s c)) :propagate)) + (if (in-calendar? (- l o)) (Some (Instant (ticks (- l o)))) None)) + +;; The offset has to be one the zone has at that local time, so an overlap +;; reads either way and a gap reads neither. +(: (-> string (Option Moment))) +(defun (iso8601->moment s) + (def* ((Some (Tuple l c1)) (read-datetime s (string-cursor-start s)) :propagate) + ((Some (Tuple o c2)) (read-offset s c1) :propagate) + ((Some c3) (skip-char s c2 #\[) :propagate) + ((Some id) (whole s (read-until s c3 #\])) :propagate) + ((Some z) (timezone id) :propagate)) + (if (and (in-calendar? (- l o)) (= (offset-at (struct-ref z tz) (- l o)) o)) + (Some (Moment (local (ticks->datetime l)) (offset (ticks->duration o)) (zone z))) + None)) + +;; An optional sign: -1 for a minus, 1 otherwise. +(: (-> string StringCursor (Tuple long StringCursor))) +(defun (read-sign s c) + (cond ((starts-with-char? s c #\-) (Tuple -1L (string-cursor-next s c))) + ((starts-with-char? s c #\+) (Tuple 1L (string-cursor-next s c))) + (else (Tuple 1L c)))) + +;; A unit's place in P1Y2M3W4D, or -1. +(: (-> string StringCursor int)) +(defun (date-unit s c) + (cond ((starts-with-char? s c #\Y) 0) + ((starts-with-char? s c #\M) 1) + ((starts-with-char? s c #\W) 2) + ((starts-with-char? s c #\D) 3) + (else -1))) + +;; A unit's place in T1H2M3S, or -1. +(: (-> string StringCursor int)) +(defun (time-unit s c) + (cond ((starts-with-char? s c #\H) 0) + ((starts-with-char? s c #\M) 1) + ((starts-with-char? s c #\S) 2) + (else -1))) + +;; The date components of a period, each optional and in the order Y M W D, +;; each with its own sign, which multiplies with the period's. Stops at T or +;; at the end. next is the place of the first unit still allowed. +(: (-> string StringCursor long (Option (Tuple (Tuple DatePeriod int) StringCursor)))) +(defun (read-date-parts s c sign) + (let loop ((c c) (next 0) (p (date-period)) (count 0)) + (if (or (string-cursor-end? s c) (starts-with-char? s c #\T)) + (Some (Tuple (Tuple p count) c)) + (let ((signed (read-sign s c))) + (def (Tuple own c1) signed) + (def (Some (Tuple n c2)) (read-number s c1) :propagate) + (def value (* (* sign own) n)) + (def unit (date-unit s c2)) + (if (or (< unit next) (> value 2147483647L) (< value -2147483648L)) + None + (loop (string-cursor-next s c2) (+ unit 1) + (set-date-part p unit (int-of value)) (+ count 1))))))) + +(: (-> DatePeriod int int DatePeriod)) +(defun (set-date-part p unit n) + (cond ((= unit 0) (record-set p (years n))) + ((= unit 1) (record-set p (months n))) + ((= unit 2) (record-set p (weeks n))) + (else (record-set p (days n))))) + +;; The time components after T, in the order H M S, as ticks. Only seconds +;; take a fraction. +(: (-> string StringCursor long (Option (Tuple (Tuple long int) StringCursor)))) +(defun (read-time-parts s c sign) + (let loop ((c c) (next 0) (total 0L) (count 0)) + (if (string-cursor-end? s c) + (Some (Tuple (Tuple total count) c)) + (let ((signed (read-sign s c))) + (def (Tuple own c1) signed) + (def* ((Some (Tuple n c2)) (read-number s c1) :propagate) + ((Some (Tuple f c3)) (read-optional-fraction s c2) :propagate)) + (def unit (time-unit s c3)) + (def per (cond ((= unit 0) ticks-per-hour) + ((= unit 1) ticks-per-minute) + (else ticks-per-second))) + (cond ((< unit next) None) + ((and (> f 0L) (not (= unit 2))) None) + (:def (Some whole-ticks) (checked-mul n per) :propagate) + (:def (Some magnitude) (checked-add whole-ticks f) :propagate) + (:def (Some sum) (checked-add total (* (* sign own) magnitude)) :propagate) + (else (loop (string-cursor-next s c3) (+ unit 1) sum (+ count 1)))))))) + +;; [sign] P [date parts] [T time parts]: the date period, the ticks, and +;; whether each part was there. A struct, so reading one allocates nothing. +(type (: PeriodText (Struct (: dates DatePeriod) (: date-count int) + (: ticks long) (: time-count int) (: has-t bool)))) + +(: (-> string (Option PeriodText))) +(defun (read-period-text s) + (def (Tuple sign c0) (read-sign s (string-cursor-start s))) + (def* ((Some c1) (skip-char s c0 #\P) :propagate) + ((Some (Tuple (Tuple dates date-count) c2)) (read-date-parts s c1 sign) :propagate)) + (if (string-cursor-end? s c2) + (Some (PeriodText (dates dates) (date-count date-count) (ticks 0L) (time-count 0) + (has-t #f))) + (let ((after-t (string-cursor-next s c2))) + (def (Some (Tuple (Tuple t time-count) c3)) (read-time-parts s after-t sign) :propagate) + (Some (PeriodText (dates dates) (date-count date-count) (ticks t) + (time-count time-count) (has-t #t)))))) + +;; PT26H30M. Days are a calendar amount, so P1D is not a duration. +(: (-> string (Option Duration))) +(defun (iso8601->duration s) + (def (Some p) (read-period-text s) :propagate) + (if (and (= (record-ref p date-count) 0) (> (record-ref p time-count) 0)) + (Some (ticks->duration (record-ref p ticks))) + None)) + +(: (-> string (Option DatePeriod))) +(defun (iso8601->date-period s) + (def (Some p) (read-period-text s) :propagate) + (if (and (> (record-ref p date-count) 0) (not (record-ref p has-t))) + (Some (record-ref p dates)) + None)) + +;; P1DT3H. A T needs a time part after it. +(: (-> string (Option Period))) +(defun (iso8601->period s) + (def (Some p) (read-period-text s) :propagate) + (if (and (> (+ (record-ref p date-count) (record-ref p time-count)) 0) + (or (not (record-ref p has-t)) (> (record-ref p time-count) 0))) + (Some (Period (calendar (record-ref p dates)) (exact (ticks->duration (record-ref p ticks))))) + None)) + + +;; =========================================================================== +;; HTTP dates +;; =========================================================================== +;; +;; IMF-fixdate (RFC 9110): Sun, 25 Oct 2026 01:30:00 GMT. The two obsolete +;; forms, RFC 850 and asctime, are not read. + +;; Numbered as ->weekday numbers them. +(: (-> int string)) +(defun (day-name n) + (match n (1 "Mon") (2 "Tue") (3 "Wed") (4 "Thu") (5 "Fri") (6 "Sat") (_ "Sun"))) + +(: (-> int string)) +(defun (month-name n) + (match n + (1 "Jan") (2 "Feb") (3 "Mar") (4 "Apr") (5 "May") (6 "Jun") + (7 "Jul") (8 "Aug") (9 "Sep") (10 "Oct") (11 "Nov") (_ "Dec"))) + +;; The fraction of a second is dropped, as the format has none. +(: (-> Instant string)) +(defun (instant->http-date i) + (def t (instant-ticks i)) + (def d (day-number->date (/ t ticks-per-day))) + (def c (struct-ref d v)) + (def sb (new-text 29)) + (add-text! sb (day-name (->weekday d))) + (add-text! sb ", ") + (add-padded! sb (long-of (clr-date-day c)) 2) + (add-text! sb " ") + (add-text! sb (month-name (clr-date-month c))) + (add-text! sb " ") + (add-padded! sb (long-of (clr-date-year c)) 4) + (add-text! sb " ") + (add-clock! sb (% t ticks-per-day)) + (add-text! sb " GMT") + (text-of sb)) + +;; The cursor n characters on, or None when s ends first. +(: (-> string StringCursor int (Option StringCursor))) +(defun (skip-chars s c n) + (cond ((= n 0) (Some c)) + ((string-cursor-end? s c) None) + (else (skip-chars s (string-cursor-next s c) (- n 1))))) + +;; The cursor after the text want, when s has it at c. +(: (-> string StringCursor string (Option StringCursor))) +(defun (skip-string s c want) + (let loop ((c c) (w (string-cursor-start want))) + (cond ((string-cursor-end? want w) (Some c)) + ((starts-with-char? s c (string-cursor-ref want w)) + (loop (string-cursor-next s c) (string-cursor-next want w))) + (else None)))) + +;; A month's name, as its number. +(: (-> string StringCursor (Option (Tuple int StringCursor)))) +(defun (read-month s c) + (let loop ((m 1)) + (if (> m 12) + None + (match (skip-string s c (month-name m)) + ((Some after) (Some (Tuple m after))) + (None (loop (+ m 1))))))) + +;; The day name must be the date's own, so it is checked once the date is read. +(: (-> string (Option Instant))) +(defun (http-date->instant s) + (def c0 (string-cursor-start s)) + (def* ((Some c1) (skip-chars s c0 3) :propagate) + ((Some c2) (skip-string s c1 ", ") :propagate) + ((Some (Tuple d c3)) (read-fixed s c2 2) :propagate) + ((Some c4) (skip-char s c3 #\space) :propagate) + ((Some (Tuple m c5)) (read-month s c4) :propagate) + ((Some c6) (skip-char s c5 #\space) :propagate) + ((Some (Tuple y c7)) (read-fixed s c6 4) :propagate) + ((Some c8) (skip-char s c7 #\space) :propagate) + ((Some (Tuple h c9)) (read-fixed s c8 2) :propagate) + ((Some c10) (skip-char s c9 #\:) :propagate) + ((Some (Tuple mi c11)) (read-fixed s c10 2) :propagate) + ((Some c12) (skip-char s c11 #\:) :propagate) + ((Some (Tuple sec c13)) (read-fixed s c12 2) :propagate) + ((Some c14) (skip-string s c13 " GMT") :propagate)) + (if (and (string-cursor-end? s c14) (valid-date? y m d) (valid-time? h mi sec 0)) + (let ((day (long-of (clr-date-day-number (ClrDate. y m d))))) + (if (some? (skip-string s c0 (day-name (->weekday (day-number->date day))))) + (Some (Instant (ticks (+ (* day ticks-per-day) (time-of-day-ticks h mi sec 0))))) + None)) + None)) + + +;; =========================================================================== +;; Text of your own +;; =========================================================================== +;; +;; A format is a quoted list. The grammar `DatePart` reads it into the parts +;; below. A keyword names a field, and a string or a character is written as +;; it is. +;; +;; (date->text d '((year 4) "-" (month 2) "-" (day 2))) ;; 2026-03-05 +;; +;; A number after the name of a field is a width, filled with zeros. A bare +;; name, such as `year`, has no width. There is one production for each kind +;; of value, so a format that asks a date for its hours is a compile error +;; where it is written. +;; +;; The text and the characters are in the parts only: `DatePart`, +;; `TimePart`, `DateTimePart`, `MomentPart` and `InstantPart`. The fields are +;; in `DateField`, `TimeField` and `DateTimeField`, which have no text. Thus +;; the grammar knows the part that a string goes in. +(def/grammar DatePart + (DatePart (:or (: DateOf DateField) (: DateText string) (: DateChar char))) + + ;; A date's numbers, its Roman numerals and its names, from a DateNames. + (DateField (:or (: DateNum DateNumber) + (: DateBare DateBareNumber) + (: DateRoman (roman RomanNumber)) + (: MonthName 'month-name) + (: MonthAbbr 'month-abbr) + (: WeekdayName 'weekday-name) + (: WeekdayAbbr 'weekday-abbr))) + + ;; The numbers of a date, with a width: (year 4). A width list that is + ;; empty, (year), is the same as a bare year. + (DateNumber (:or (: NumYear (year int ...)) + (: NumMonth (month int ...)) + (: NumDay (day int ...)) + (: NumWeekday (weekday int ...)) + (: NumYday (yday int ...)) + (: NumWeek (week int ...)) + (: NumWeekYear (week-year int ...)))) + + ;; The numbers of a date without a width: year. + (DateBareNumber (:or (: BareYear 'year) + (: BareMonth 'month) + (: BareDay 'day) + (: BareWeekday 'weekday) + (: BareYday 'yday) + (: BareWeek 'week) + (: BareWeekYear 'week-year))) + + ;; What (roman ...) writes. No width, as a numeral has no zeros to fill + ;; with. + (RomanNumber (:or (: RomanYear 'year) (: RomanMonth 'month) (: RomanDay 'day))) + + (TimePart (:or (: TimeOf TimeField) (: TimeText string) (: TimeChar char))) + + ;; (fraction n) is n digits of the second's fraction, cut, not rounded, + ;; with zeros past the seventh. A bare fraction is ISO 8601's, point + ;; included: .5, and nothing for a whole second. + (TimeField (:or (: TimeHours (hours int ...)) + (: TimeHours12 (hours12 int ...)) + (: TimeMinutes (minutes int ...)) + (: TimeSeconds (seconds int ...)) + (: TimeFraction (fraction int ...)) + (: TimeAmPm 'am-pm) + (: TimeBare TimeBareNumber))) + + ;; The numbers of a time without a width: hours. + (TimeBareNumber (:or (: BareHours 'hours) + (: BareHours12 'hours12) + (: BareMinutes 'minutes) + (: BareSeconds 'seconds) + (: BareFraction 'fraction))) + + (DateTimePart (:or (: DateTimeOf DateTimeField) (: DateTimeText string) (: DateTimeChar char))) + + ;; The fields of a date and of a time. + (DateTimeField (:or (: DateTimeDate DateField) (: DateTimeTime TimeField))) + + ;; A date and a time, the offset as ISO 8601 writes it (+02:00), the zone's + ;; id, and the Unix time in seconds or milliseconds, rounded toward the + ;; past. + (MomentPart (:or (: MomentDateTime DateTimeField) + (: MomentOffset 'offset) + (: MomentZone 'zone) + (: MomentUnix 'unix) + (: MomentUnixMs 'unix-ms) + (: MomentText string) + (: MomentChar char))) + + ;; A date and a time in UTC, whose offset is written Z, and the Unix time. + (InstantPart (:or (: InstantDateTime DateTimeField) + (: InstantOffset 'offset) + (: InstantUnix 'unix) + (: InstantUnixMs 'unix-ms) + (: InstantText string) + (: InstantChar char)))) + +;; A bare number of a date is the number without a width. +(: (-> DateBareNumber DateNumber)) +(defun (bare-date-number b) + (match b + (BareYear (NumYear Nil)) + (BareMonth (NumMonth Nil)) + (BareDay (NumDay Nil)) + (BareWeekday (NumWeekday Nil)) + (BareYday (NumYday Nil)) + (BareWeek (NumWeek Nil)) + (BareWeekYear (NumWeekYear Nil)))) + +;; A bare number of a time is the number without a width. +(: (-> TimeBareNumber TimeField)) +(defun (bare-time-field b) + (match b + (BareHours (TimeHours Nil)) + (BareHours12 (TimeHours12 Nil)) + (BareMinutes (TimeMinutes Nil)) + (BareSeconds (TimeSeconds Nil)) + (BareFraction (TimeFraction Nil)))) + +;; Twelve months from January and seven weekdays from Monday, and what am-pm +;; writes. A table of other lengths raises when a name is taken from it. +(type (: DateNames (Struct (: months (Vec string)) (: months-short (Vec string)) + (: weekdays (Vec string)) (: weekdays-short (Vec string)) + (: am string) (: pm string)))) + +(: DateNames) +(def english-names + (DateNames (months ["January" "February" "March" "April" "May" "June" "July" + "August" "September" "October" "November" "December"]) + (months-short ["Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct" + "Nov" "Dec"]) + (weekdays ["Monday" "Tuesday" "Wednesday" "Thursday" "Friday" "Saturday" + "Sunday"]) + (weekdays-short ["Mon" "Tue" "Wed" "Thu" "Fri" "Sat" "Sun"]) + (am "AM") + (pm "PM"))) + +(: DateNames) +(def swedish-names + (DateNames (months ["januari" "februari" "mars" "april" "maj" "juni" "juli" + "augusti" "september" "oktober" "november" "december"]) + (months-short ["jan" "feb" "mar" "apr" "maj" "jun" "jul" "aug" "sep" "okt" + "nov" "dec"]) + (weekdays ["måndag" "tisdag" "onsdag" "torsdag" "fredag" "lördag" "söndag"]) + (weekdays-short ["mån" "tis" "ons" "tor" "fre" "lör" "sön"]) + (am "fm") + (pm "em"))) + +;; After english-names, which is its first value. +(: (Param DateNames)) +(def current-date-names (make-parameter english-names)) + +(: (-> string %a)) +(defun (argument-error what) + (raise (cast Exception (ArgumentException. (str "(std datetime): " what))))) + +;; The width a part was written with, 0 for none. +(: (-> (List int) string int)) +(defun (part-width ws tag) + (cond ((list-empty? ws) 0) + ((list-empty? (list-tail ws)) (list-head ws)) + (else (argument-error (str "(" tag " ...) takes one width, and was given " + (displayed (list-length ws)) "."))))) + +;; A table of names, which must have count of them. +(: (-> (Vec string) int string (Vec string))) +(defun (full-table names count what) + (if (= (vec-length names) count) + names + (argument-error (str "a DateNames has " (displayed count) " " what ", and this one has " + (displayed (vec-length names)) ".")))) + +;; The i:th name, counting from 1, of a table that must have count of them. +(: (-> (Vec string) int int string string)) +(defun (name-at names count i what) (vec-ref (full-table names count what) (- i 1))) + +;; MMXXVI. Numerals stop at MMMCMXCIX, and a larger year goes on adding Ms, +;; 4000 being MMMM, since nothing else is standard. +(: (List (Tuple int string))) +(def roman-steps + (list (Tuple 1000 "M") (Tuple 900 "CM") (Tuple 500 "D") (Tuple 400 "CD") + (Tuple 100 "C") (Tuple 90 "XC") (Tuple 50 "L") (Tuple 40 "XL") + (Tuple 10 "X") (Tuple 9 "IX") (Tuple 5 "V") (Tuple 4 "IV") (Tuple 1 "I"))) + +(: (-> TextBuilder int void)) +(defun (add-roman! sb n) + (let loop ((n n) (steps roman-steps)) + (unless (list-empty? steps) + (match (list-head steps) + ((Tuple value numeral) + (if (>= n value) + (begin + (add-text! sb numeral) + (loop (- n value) steps)) + (loop n (list-tail steps)))))))) + +;; The first n digits of a second's fraction f, in ticks: cut, not rounded, +;; and zeros past the seventh. +(: (-> TextBuilder long int void)) +(defun (add-fraction-digits! sb f n) + (let loop ((i 0) (scale 1000000L)) + (when (< i n) + (add-digit! sb (if (> scale 0L) (% (/ f scale) 10L) 0L)) + (loop (+ i 1) (/ scale 10L))))) + +(: (-> TextBuilder Date DateNumber void)) +(defun (add-date-number! sb d n) + (match n + ((NumYear w) (add-padded! sb (long-of (->year d)) (part-width w "year"))) + ((NumMonth w) (add-padded! sb (long-of (->month d)) (part-width w "month"))) + ((NumDay w) (add-padded! sb (long-of (->day d)) (part-width w "day"))) + ((NumWeekday w) (add-padded! sb (long-of (->weekday d)) (part-width w "weekday"))) + ((NumYday w) (add-padded! sb (long-of (->yday d)) (part-width w "yday"))) + ((NumWeek w) (add-padded! sb (long-of (->week d)) (part-width w "week"))) + ((NumWeekYear w) (add-padded! sb (long-of (->week-year d)) (part-width w "week-year"))))) + +(: (-> TextBuilder Date DateNames DateField void)) +(defun (add-date-field! sb d names p) + (match p + ((DateNum n) (add-date-number! sb d n)) + ((DateBare b) (add-date-number! sb d (bare-date-number b))) + ((DateRoman RomanYear) (add-roman! sb (->year d))) + ((DateRoman RomanMonth) (add-roman! sb (->month d))) + ((DateRoman RomanDay) (add-roman! sb (->day d))) + (MonthName (add-text! sb (name-at (record-ref names months) 12 (->month d) "months"))) + (MonthAbbr (add-text! sb (name-at (record-ref names months-short) 12 (->month d) "months"))) + (WeekdayName (add-text! sb (name-at (record-ref names weekdays) 7 (->weekday d) "weekdays"))) + (WeekdayAbbr + (add-text! sb (name-at (record-ref names weekdays-short) 7 (->weekday d) "weekdays"))))) + +(: (-> TextBuilder Date DateNames DatePart void)) +(defun (add-date-part! sb d names p) + (match p + ((DateOf f) (add-date-field! sb d names f)) + ((DateText s) (add-text! sb s)) + ((DateChar c) (add-text! sb (char->string c))))) + +;; t is the ticks since midnight. +(: (-> TextBuilder long DateNames TimeField void)) +(defun (add-time-field! sb t names p) + (def h (/ t ticks-per-hour)) + (match p + ((TimeHours w) (add-padded! sb h (part-width w "hours"))) + ((TimeHours12 w) (add-padded! sb (if (= (% h 12L) 0L) 12L (% h 12L)) (part-width w "hours12"))) + ((TimeMinutes w) (add-padded! sb (% (/ t ticks-per-minute) 60L) (part-width w "minutes"))) + ((TimeSeconds w) (add-padded! sb (% (/ t ticks-per-second) 60L) (part-width w "seconds"))) + ((TimeFraction w) + (if (list-empty? w) + (add-fraction! sb (% t ticks-per-second)) + (add-fraction-digits! sb (% t ticks-per-second) (part-width w "fraction")))) + (TimeAmPm (add-text! sb (if (< h 12L) (record-ref names am) (record-ref names pm)))) + ((TimeBare b) (add-time-field! sb t names (bare-time-field b))))) + +;; t is the ticks since midnight. +(: (-> TextBuilder long DateNames TimePart void)) +(defun (add-time-part! sb t names p) + (match p + ((TimeOf f) (add-time-field! sb t names f)) + ((TimeText s) (add-text! sb s)) + ((TimeChar c) (add-text! sb (char->string c))))) + +;; t is local ticks. +(: (-> TextBuilder long DateNames DateTimeField void)) +(defun (add-datetime-field! sb t names p) + (match p + ((DateTimeDate df) (add-date-field! sb (day-number->date (/ t ticks-per-day)) names df)) + ((DateTimeTime tf) (add-time-field! sb (% t ticks-per-day) names tf)))) + +;; t is local ticks. +(: (-> TextBuilder long DateNames DateTimePart void)) +(defun (add-datetime-part! sb t names p) + (match p + ((DateTimeOf f) (add-datetime-field! sb t names f)) + ((DateTimeText s) (add-text! sb s)) + ((DateTimeChar c) (add-text! sb (char->string c))))) + +;; n in decimal, with a minus when it is negative. +(: (-> TextBuilder long void)) +(defun (add-signed! sb n) + (when (< n 0L) (add-text! sb "-")) + (add-padded! sb (abs n) 1)) + +(: (-> TextBuilder Moment DateNames MomentPart void)) +(defun (add-moment-part! sb m names p) + (match p + ((MomentDateTime dtf) (add-datetime-field! sb (moment-local-ticks m) names dtf)) + (MomentOffset (add-offset! sb (moment-offset-ticks m))) + (MomentZone (add-text! sb (timezone-id (struct-ref m zone)))) + (MomentUnix + (add-signed! sb (instant->unix-seconds (Instant (ticks (moment-instant-ticks m)))))) + (MomentUnixMs + (add-signed! sb (instant->unix-milliseconds (Instant (ticks (moment-instant-ticks m)))))) + ((MomentText s) (add-text! sb s)) + ((MomentChar c) (add-text! sb (char->string c))))) + +(: (-> TextBuilder Instant DateNames InstantPart void)) +(defun (add-instant-part! sb i names p) + (match p + ((InstantDateTime dtf) (add-datetime-field! sb (instant-ticks i) names dtf)) + (InstantOffset (add-text! sb "Z")) + (InstantUnix (add-signed! sb (instant->unix-seconds i))) + (InstantUnixMs (add-signed! sb (instant->unix-milliseconds i))) + ((InstantText s) (add-text! sb s)) + ((InstantChar c) (add-text! sb (char->string c))))) + +;; One function per type rather than one over the traits, so a moment's date +;; is written with (date->text (->date m) ...), which says whose date it is. +(: (-> Date (List DatePart) (#:names DateNames) string)) +(defun (date->text d parts #:names (parameter-ref current-date-names)) + (def sb (new-text 32)) + (let loop ((ps parts)) + (unless (list-empty? ps) + (add-date-part! sb d names (list-head ps)) + (loop (list-tail ps)))) + (text-of sb)) + +(: (-> Time (List TimePart) (#:names DateNames) string)) +(defun (time->text t parts #:names (parameter-ref current-date-names)) + (def sb (new-text 32)) + (let loop ((ps parts)) + (unless (list-empty? ps) + (add-time-part! sb (clr-time-ticks (struct-ref t v)) names (list-head ps)) + (loop (list-tail ps)))) + (text-of sb)) + +(: (-> DateTime (List DateTimePart) (#:names DateNames) string)) +(defun (datetime->text dt parts #:names (parameter-ref current-date-names)) + (def sb (new-text 48)) + (let loop ((ps parts)) + (unless (list-empty? ps) + (add-datetime-part! sb (datetime-ticks dt) names (list-head ps)) + (loop (list-tail ps)))) + (text-of sb)) + +(: (-> Moment (List MomentPart) (#:names DateNames) string)) +(defun (moment->text m parts #:names (parameter-ref current-date-names)) + (def sb (new-text 64)) + (let loop ((ps parts)) + (unless (list-empty? ps) + (add-moment-part! sb m names (list-head ps)) + (loop (list-tail ps)))) + (text-of sb)) + +;; The date and time are UTC's, so the format that writes a moment's local +;; time writes the instant as `written` does. +(: (-> Instant (List InstantPart) (#:names DateNames) string)) +(defun (instant->text i parts #:names (parameter-ref current-date-names)) + (def sb (new-text 48)) + (let loop ((ps parts)) + (unless (list-empty? ps) + (add-instant-part! sb i names (list-head ps)) + (loop (list-tail ps)))) + (text-of sb)) + + +;; =========================================================================== +;; Reading text of your own +;; =========================================================================== +;; +;; A reader sees each part of a format as a Field, whichever union it came +;; from, so one reader serves every format. It checks the format first, so a +;; format that cannot be read raises whatever the text, and then walks the +;; text: the parts in order, and the whole text used. The fields read so far +;; are kept in a Seen, a struct, and put together at the end, so reading +;; allocates nothing but a zone id. + +;; What a part reads. A match on it tests its arms in order, so text, +;; characters and the common fields come first where that matters. +(type (: FieldKind (Union FText FChar + FYear FMonth FDay FWeekday FYday FWeek FWeekYear + FRomanYear FRomanMonth FRomanDay + FMonthName FMonthAbbr FWeekdayName FWeekdayAbbr + FHours FHours12 FMinutes FSeconds FFraction FAmPm + FOffset FZone FUnix FUnixMs))) + +;; A part as a reader sees it: its width, 0 for none; the most digits it +;; reads, 0 when it is not a number; and its text or character when it is +;; one. A bare fraction, which can be nothing at all, has the width -1, so +;; that it is told apart from (fraction 0). The functions that make a Field +;; know its kind, so they say its most digits, and no reader asks again. +(type (: Field (Struct (: kind FieldKind) (: width int) (: most int) (: text string) (: ch char)))) + +(: (-> FieldKind int Field)) +(defun (field k w) (Field (kind k) (width w) (most 0) (text "") (ch #\nul))) + +(: (-> FieldKind int int Field)) +(defun (number-field k most w) (Field (kind k) (width w) (most most) (text "") (ch #\nul))) + +(: (-> string Field)) +(defun (text-field s) (Field (kind FText) (width 0) (most 0) (text s) (ch #\nul))) + +(: (-> char Field)) +(defun (char-field c) (Field (kind FChar) (width 0) (most 0) (text "") (ch c))) + +(: (-> DateNumber Field)) +(defun (date-number-field n) + (match n + ((NumYear w) (number-field FYear 4 (part-width w "year"))) + ((NumMonth w) (number-field FMonth 2 (part-width w "month"))) + ((NumDay w) (number-field FDay 2 (part-width w "day"))) + ((NumWeekday w) (number-field FWeekday 1 (part-width w "weekday"))) + ((NumYday w) (number-field FYday 3 (part-width w "yday"))) + ((NumWeek w) (number-field FWeek 2 (part-width w "week"))) + ((NumWeekYear w) (number-field FWeekYear 4 (part-width w "week-year"))))) + +(: (-> DateField Field)) +(defun (date-field-field p) + (match p + ((DateNum n) (date-number-field n)) + ((DateBare b) (date-number-field (bare-date-number b))) + ((DateRoman RomanYear) (field FRomanYear 0)) + ((DateRoman RomanMonth) (field FRomanMonth 0)) + ((DateRoman RomanDay) (field FRomanDay 0)) + (MonthName (field FMonthName 0)) + (MonthAbbr (field FMonthAbbr 0)) + (WeekdayName (field FWeekdayName 0)) + (WeekdayAbbr (field FWeekdayAbbr 0)))) + +(: (-> DatePart Field)) +(defun (date-part-field p) + (match p + ((DateText s) (text-field s)) + ((DateChar c) (char-field c)) + ((DateOf f) (date-field-field f)))) + +(: (-> TimeField Field)) +(defun (time-field-field p) + (match p + ((TimeHours w) (number-field FHours 2 (part-width w "hours"))) + ((TimeHours12 w) (number-field FHours12 2 (part-width w "hours12"))) + ((TimeMinutes w) (number-field FMinutes 2 (part-width w "minutes"))) + ((TimeSeconds w) (number-field FSeconds 2 (part-width w "seconds"))) + ((TimeFraction w) + (field FFraction (if (list-empty? w) -1 (max (part-width w "fraction") 0)))) + (TimeAmPm (field FAmPm 0)) + ((TimeBare b) (time-field-field (bare-time-field b))))) + +(: (-> TimePart Field)) +(defun (time-part-field p) + (match p + ((TimeText s) (text-field s)) + ((TimeChar c) (char-field c)) + ((TimeOf f) (time-field-field f)))) + +(: (-> DateTimeField Field)) +(defun (datetime-field-field p) + (match p + ((DateTimeDate df) (date-field-field df)) + ((DateTimeTime tf) (time-field-field tf)))) + +(: (-> DateTimePart Field)) +(defun (datetime-part-field p) + (match p + ((DateTimeText s) (text-field s)) + ((DateTimeChar c) (char-field c)) + ((DateTimeOf f) (datetime-field-field f)))) + +(: (-> MomentPart Field)) +(defun (moment-part-field p) + (match p + ((MomentText s) (text-field s)) + ((MomentChar c) (char-field c)) + ((MomentDateTime dtf) (datetime-field-field dtf)) + (MomentOffset (field FOffset 0)) + (MomentZone (field FZone 0)) + (MomentUnix (field FUnix 0)) + (MomentUnixMs (field FUnixMs 0)))) + +(: (-> InstantPart Field)) +(defun (instant-part-field p) + (match p + ((InstantText s) (text-field s)) + ((InstantChar c) (char-field c)) + ((InstantDateTime dtf) (datetime-field-field dtf)) + (InstantOffset (field FOffset 0)) + (InstantUnix (field FUnix 0)) + (InstantUnixMs (field FUnixMs 0)))) + +;; Each format union's parts as Fields, text first, as separators are the +;; most common part. A trait rather than a function argument, so that each +;; reader is compiled for its own union. +(def/trait (FormatPart %p) + (: part-field (-> %p Field))) + +(impl (FormatPart DatePart) + (defun (part-field p) (date-part-field p))) + +(impl (FormatPart TimePart) + (defun (part-field p) (time-part-field p))) + +(impl (FormatPart DateTimePart) + (defun (part-field p) (datetime-part-field p))) + +(impl (FormatPart MomentPart) + (defun (part-field p) (moment-part-field p))) + +(impl (FormatPart InstantPart) + (defun (part-field p) (instant-part-field p))) + +(: (-> string int string)) +(defun (with-width name w) (if (> w 0) (str "(" name " " (displayed w) ")") name)) + +;; A part as a format writes it, for a message. +(: (-> Field string)) +(defun (field-name f) + (def w (struct-ref f width)) + (match (struct-ref f kind) + (FText (str "the text \"" (struct-ref f text) "\"")) + (FChar (str "the character " (char->string (struct-ref f ch)))) + (FYear (with-width "year" w)) + (FMonth (with-width "month" w)) + (FDay (with-width "day" w)) + (FWeekday (with-width "weekday" w)) + (FYday (with-width "yday" w)) + (FWeek (with-width "week" w)) + (FWeekYear (with-width "week-year" w)) + (FRomanYear "(roman year)") + (FRomanMonth "(roman month)") + (FRomanDay "(roman day)") + (FMonthName "month-name") + (FMonthAbbr "month-abbr") + (FWeekdayName "weekday-name") + (FWeekdayAbbr "weekday-abbr") + (FHours (with-width "hours" w)) + (FHours12 (with-width "hours12" w)) + (FMinutes (with-width "minutes" w)) + (FSeconds (with-width "seconds" w)) + (FFraction (if (< w 0) "fraction" (str "(fraction " (displayed w) ")"))) + (FAmPm "am-pm") + (FOffset "offset") + (FZone "zone") + (FUnix "unix") + (FUnixMs "unix-ms"))) + +;; The kinds of field a format has, one bit per kind that the checks below +;; tell apart. +(: int) +(def b-year 1) +;; month, day and yday, by number, numeral or name: the fields within a year. +(: int) +(def b-in-year 2) +(: int) +(def b-week-year 4) +(: int) +(def b-week 8) +(: int) +(def b-weekday 16) +(: int) +(def b-hours 32) +(: int) +(def b-hours12 64) +(: int) +(def b-am-pm 128) +;; minutes, seconds and the fraction: the fields within an hour. +(: int) +(def b-in-hour 256) +(: int) +(def b-offset 512) +(: int) +(def b-zone 1024) +(: int) +(def b-unix 2048) +(: int) +(def b-unix-ms 4096) + +(: (-> FieldKind int)) +(defun (field-bits k) + (match k + (FText 0) + (FChar 0) + (FYear b-year) + (FRomanYear b-year) + (FMonth b-in-year) + (FDay b-in-year) + (FYday b-in-year) + (FRomanMonth b-in-year) + (FRomanDay b-in-year) + (FMonthName b-in-year) + (FMonthAbbr b-in-year) + (FWeekYear b-week-year) + (FWeek b-week) + (FWeekday b-weekday) + (FWeekdayName b-weekday) + (FWeekdayAbbr b-weekday) + (FHours b-hours) + (FHours12 b-hours12) + (FAmPm b-am-pm) + (FMinutes b-in-hour) + (FSeconds b-in-hour) + (FFraction b-in-hour) + (FOffset b-offset) + (FZone b-zone) + (FUnix b-unix) + (FUnixMs b-unix-ms))) + +(: (-> int int bool)) +(defun (has? bits b) (> (bitwise-and bits b) 0)) + +;; The characters a part can go on taking once it has begun, as bits: a +;; part of one fixed length takes none. +(: int) +(def t-digits 1) +(: int) +(def t-roman 2) +;; A decimal point or comma, which a bare fraction may begin with. +(: int) +(def t-point 4) +;; The characters of an IANA id. +(: int) +(def t-zone 8) +;; The colon before an offset's seconds. +(: int) +(def t-colon 16) + +(: (-> char int)) +(defun (roman-value c) + (cond ((char=? c #\I) 1) + ((char=? c #\V) 5) + ((char=? c #\X) 10) + ((char=? c #\L) 50) + ((char=? c #\C) 100) + ((char=? c #\D) 500) + ((char=? c #\M) 1000) + (else 0))) + +;; ASCII letters and digits, and / _ - +, as in Etc/GMT+5 and +;; America/Port-au-Prince. +(: (-> char bool)) +(defun (zone-char? c) + (def n (char->int c)) + (or (and (>= n 65) (<= n 90)) (and (>= n 97) (<= n 122)) (and (>= n 48) (<= n 57)) + (= n 47) (= n 95) (= n 45) (= n 43))) + +(: (-> int char bool)) +(defun (takes? taken c) + (or (and (has? taken t-digits) (>= (ascii-digit c) 0)) + (and (has? taken t-roman) (> (roman-value c) 0)) + (and (has? taken t-point) (or (char=? c #\.) (char=? c #\,))) + (and (has? taken t-zone) (zone-char? c)) + (and (has? taken t-colon) (char=? c #\:)))) + +;; A number reads from its width, or 1, up to its most digits or its width, +;; so it has one length only when its width is at least its most. A bare +;; fraction is nothing, or a point and as many digits as there are, and an +;; offset may have seconds after a colon. +(: (-> Field int)) +(defun (field-takes f) + (def k (struct-ref f kind)) + (def w (struct-ref f width)) + (def most (struct-ref f most)) + (if (> most 0) + (if (< (max w 1) most) t-digits 0) + (match k + (FText 0) + (FChar 0) + (FRomanYear t-roman) + (FRomanMonth t-roman) + (FRomanDay t-roman) + (FFraction (if (< w 0) (bitwise-ior t-digits t-point) 0)) + (FOffset t-colon) + (FZone t-zone) + (FUnix t-digits) + (FUnixMs t-digits) + (_ 0)))) + +(: (-> Field bool)) +(defun (can-be-empty? f) + (match (struct-ref f kind) + (FText (string-empty? (struct-ref f text))) + (FFraction (<= (struct-ref f width) 0)) + (_ #f))) + +(: (-> string char)) +(defun (first-char s) (string-cursor-ref s (string-cursor-start s))) + +(: (-> string int bool)) +(defun (name-starts? name taken) + (and (not (string-empty? name)) (takes? taken (first-char name)))) + +(: (-> (Vec string) int bool)) +(defun (any-name-starts? table taken) + (let loop ((i 0)) + (and (< i (vec-length table)) + (or (name-starts? (vec-ref table i) taken) (loop (+ i 1)))))) + +;; The table a name reads from, which must be whole. +(: (-> DateNames FieldKind (Vec string))) +(defun (name-table names k) + (match k + (FMonthName (full-table (record-ref names months) 12 "months")) + (FMonthAbbr (full-table (record-ref names months-short) 12 "months")) + (FWeekdayName (full-table (record-ref names weekdays) 7 "weekdays")) + (_ (full-table (record-ref names weekdays-short) 7 "weekdays")))) + +(: (-> FieldKind bool)) +(defun (name-kind? k) + (match k + (FText #f) + (FChar #f) + (FMonthName #t) + (FMonthAbbr #t) + (FWeekdayName #t) + (FWeekdayAbbr #t) + (_ #f))) + +;; Whether a part can begin with a character in taken. Digits, Roman +;; letters and the letters an id begins with are each one class, so one of +;; them stands for all: M is a Roman letter and begins MST. +(: (-> Field DateNames int bool)) +(defun (starts-within? f names taken) + (def k (struct-ref f kind)) + (def w (struct-ref f width)) + (cond ((> (struct-ref f most) 0) (takes? taken #\0)) + ((name-kind? k) (any-name-starts? (name-table names k) taken)) + (else + (match k + (FText (name-starts? (struct-ref f text) taken)) + (FChar (takes? taken (struct-ref f ch))) + (FRomanYear (takes? taken #\M)) + (FRomanMonth (takes? taken #\M)) + (FRomanDay (takes? taken #\M)) + (FFraction (cond ((< w 0) (or (takes? taken #\.) (takes? taken #\,))) + ((> w 0) (takes? taken #\0)) + (else #f))) + (FAmPm (or (name-starts? (record-ref names am) taken) + (name-starts? (record-ref names pm) taken))) + (FOffset (or (takes? taken #\Z) (takes? taken #\+) (takes? taken #\-))) + (FZone (takes? taken #\M)) + (FUnix (or (takes? taken #\-) (takes? taken #\0))) + (FUnixMs (or (takes? taken #\-) (takes? taken #\0))) + (_ #f))))) + +(: (-> Field Field %a)) +(defun (two-readings-error before after) + (def most (struct-ref before most)) + (argument-error + (str (field-name before) " can be of more than one length, and " (field-name after) + " after it can begin with a character " (field-name before) + " could also take, so a text could be read two ways. " + (if (> most 0) + (str "A width of " (displayed most) ", or text between the two, reads one way.") + "Text between the two reads one way.")))) + +;; The kinds of field a format has, as bits, once it is known that no text +;; can be read with it two ways: no part that can be of more than one length +;; is followed by a part that can begin with a character it could also take. +;; A part that can be empty passes what could be taken on to the next. +(: (-> (List %p) DateNames int) (where (FormatPart %p))) +(defun (format-bits parts names) + (let loop ((ps parts) (bits 0) (taken 0) (before (text-field ""))) + (if (list-empty? ps) + bits + (let ((f (part-field (list-head ps)))) + (def k (struct-ref f kind)) + (when (name-kind? k) (ignore (name-table names k))) + (when (and (> taken 0) (starts-within? f names taken)) + (two-readings-error before f)) + (def own (field-takes f)) + (loop (list-tail ps) + (bitwise-ior bits (field-bits k)) + (if (can-be-empty? f) (bitwise-ior taken own) own) + (if (> own 0) f before)))))) + +;; An offset or a Unix time not read, which no reading can be. +(: long) +(def no-ticks -9223372036854775808L) + +;; The fields read so far, each -1 until it is read, or no-ticks for those +;; in ticks. pm is 0 for the morning's word and 1 for the afternoon's, the +;; fraction is in ticks, read to fraction-digits digits, and unix is the +;; instant's UTC ticks, from either kind of Unix time. The zone is the span of +;; text its id was read from, empty until then, and is looked up once at the +;; end. So a Seen holds no reference, and .NET need not clear every copy of +;; it a function might make each time the function is called. +(type (: Seen (Struct (: year int) (: month int) (: day int) (: weekday int) (: yday int) + (: week int) (: week-year int) + (: hours int) (: hours12 int) (: pm int) (: minutes int) (: seconds int) + (: fraction long) (: fraction-digits int) + (: offset long) (: unix long) + (: zone-start StringCursor) (: zone-end StringCursor)))) + +(: Seen) +(def unseen + (Seen (year -1) (month -1) (day -1) (weekday -1) (yday -1) (week -1) (week-year -1) + (hours -1) (hours12 -1) (pm -1) (minutes -1) (seconds -1) + (fraction -1L) (fraction-digits 0) + (offset no-ticks) (unix no-ticks) + (zone-start (string-cursor-start "")) (zone-end (string-cursor-start "")))) + +;; From least to most ASCII digits, as many as there are. The value stops +;; growing at eighteen digits, which no field reaches. +(: (-> string StringCursor int int (Option (Tuple long StringCursor)))) +(defun (read-digits s c least most) + (let loop ((c c) (count 0) (acc 0L)) + (def d (next-digit s c)) + (if (or (= count most) (< d 0)) + (if (>= count least) (Some (Tuple acc c)) None) + (loop (string-cursor-next s c) (+ count 1) + (if (>= acc 100000000000000000L) acc (+ (* acc 10L) (long-of d))))))) + +;; Exactly n digits of a second's fraction, as ticks. Digits past the +;; seventh are read and dropped, as a tick holds no less than 100 ns. +(: (-> string StringCursor int (Option (Tuple long StringCursor)))) +(defun (read-fraction-digits s c n) + (let loop ((c c) (i 0) (acc 0L)) + (def d (next-digit s c)) + (cond ((= i n) (Some (Tuple (scale-fraction acc (min n 7)) c))) + ((< d 0) None) + (else (loop (string-cursor-next s c) (+ i 1) + (if (< i 7) (+ (* acc 10L) (long-of d)) acc)))))) + +;; The cursor after n's numeral as add-roman! writes it, when s has it at c. +(: (-> string StringCursor int (Option StringCursor))) +(defun (skip-roman s c n) + (let loop ((c c) (n n) (steps roman-steps)) + (if (list-empty? steps) + (Some c) + (match (list-head steps) + ((Tuple value numeral) + (cond ((< n value) (loop c n (list-tail steps))) + (:def (Some after) (skip-string s c numeral) :propagate) + (else (loop after (- n value) steps)))))))) + +;; A run of Roman letters, read only as add-roman! writes its value: MCM, +;; not MDCCCC, and IV, not IIII. The value stops growing past 100 000, and +;; is None there, as no field reaches it. +(: (-> string StringCursor (Option (Tuple long StringCursor)))) +(defun (read-roman s c) + (let loop ((e c) (total 0) (prev 0)) + (def v (if (string-cursor-end? s e) 0 (roman-value (string-cursor-ref s e)))) + (cond ((> v 0) + (loop (string-cursor-next s e) + (cond ((> total 100000) total) + ((> v prev) (+ total (- v (* 2 prev)))) + (else (+ total v))) + v)) + ((or (< total 1) (> total 100000)) None) + (:def (Some after) (skip-roman s c total) :propagate) + ((string-cursor=? after e) (Some (Tuple (long-of total) e))) + (else None)))) + +;; The number, from 1, of the longest name in the table that s has at c. +(: (-> string StringCursor (Vec string) (Option (Tuple long StringCursor)))) +(defun (read-name s c table) + (let loop ((i 0) (best 0) (best-end c)) + (if (= i (vec-length table)) + (if (= best 0) None (Some (Tuple (long-of best) best-end))) + (match (skip-string s c (vec-ref table i)) + ((Some e) (if (or (= best 0) (string-cursor>? e best-end)) + (loop (+ i 1) (+ i 1) e) + (loop (+ i 1) best best-end))) + (None (loop (+ i 1) best best-end)))))) + +;; 0 for the morning's word and 1 for the afternoon's, the longer when both +;; match. +(: (-> string StringCursor DateNames (Option (Tuple long StringCursor)))) +(defun (read-am-pm s c names) + (match (Tuple (skip-string s c (record-ref names am)) (skip-string s c (record-ref names pm))) + ((Tuple (Some a) (Some p)) (if (string-cursor>? p a) (Some (Tuple 1L p)) (Some (Tuple 0L a)))) + ((Tuple (Some a) None) (Some (Tuple 0L a))) + ((Tuple None (Some p)) (Some (Tuple 1L p))) + (_ None))) + +;; A Unix time, an optional minus and up to 12 digits of seconds or 15 of +;; milliseconds, as UTC ticks: None outside the years 1 to 9999, as +;; unix-seconds->instant answers. +(: (-> string StringCursor bool (Option (Tuple long StringCursor)))) +(defun (read-unix s c ms) + (def minus (starts-with-char? s c #\-)) + (def (Some (Tuple n c1)) + (read-digits s (if minus (string-cursor-next s c) c) 1 (if ms 15 12)) :propagate) + (def signed (if minus (- n) n)) + (def (Some i) + (if ms (unix-milliseconds->instant signed) (unix-seconds->instant signed)) :propagate) + (Some (Tuple (instant-ticks i) c1))) + +;; The end of the longest run, at least one long, of the characters an IANA +;; id is made of. +(: (-> string StringCursor (Option StringCursor))) +(defun (skip-zone s c) + (let loop ((e c)) + (cond ((and (not (string-cursor-end? s e)) (zone-char? (string-cursor-ref s e))) + (loop (string-cursor-next s e))) + ((string-cursor=? e c) None) + (else (Some e))))) + +;; Whether s has the same text from a to a-end as from b to b-end. +(: (-> string StringCursor StringCursor StringCursor StringCursor bool)) +(defun (same-span? s a a-end b b-end) + (cond ((string-cursor=? a a-end) (string-cursor=? b b-end)) + ((string-cursor=? b b-end) #f) + ((char=? (string-cursor-ref s a) (string-cursor-ref s b)) + (same-span? s (string-cursor-next s a) a-end (string-cursor-next s b) b-end)) + (else #f))) + +(: (-> (Option StringCursor) (Option (Tuple long StringCursor)))) +(defun (zero-after r) + (option-map (fun (c) (Tuple 0L c)) r)) + +;; What one part reads at c, as a number, and the cursor after it. Text and +;; characters read as 0, and a zone is read by skip-zone. +(: (-> string StringCursor DateNames Field (Option (Tuple long StringCursor)))) +(defun (read-value s c names f) + (def k (struct-ref f kind)) + (def w (struct-ref f width)) + (def most (struct-ref f most)) + (cond ((> most 0) (read-digits s c (max w 1) (max w most))) + ((name-kind? k) (read-name s c (name-table names k))) + (else + (match k + (FText (zero-after (skip-string s c (struct-ref f text)))) + (FChar (zero-after (skip-char s c (struct-ref f ch)))) + (FFraction (if (< w 0) (read-optional-fraction s c) (read-fraction-digits s c w))) + (FAmPm (read-am-pm s c names)) + (FOffset (read-offset s c)) + (FUnix (read-unix s c #f)) + (FUnixMs (read-unix s c #t)) + (FZone None) + (_ (read-roman s c)))))) + +;; Whether a field read as old, or not read when -1, agrees with v. +(: (-> int int bool)) +(defun (agrees? old v) (or (< old 0) (= old v))) + +(: (-> long long bool)) +(defun (ticks-agree? old t) (or (= old no-ticks) (= old t))) + +;; The ticks in the last of 7 - digits places, which a fraction read to +;; digits digits does not say. +(: (-> int long)) +(defun (fraction-unit digits) (if (>= digits 7) 1L (* 10L (fraction-unit (+ digits 1))))) + +;; A fraction read twice has to agree to the fewer digits of the two, and +;; the one with more is kept: .5 and .5000 agree, and .5 and .4 do not. +(: (-> Seen long int (Option Seen))) +(defun (put-fraction seen f digits) + (def old (struct-ref seen fraction)) + (def old-digits (struct-ref seen fraction-digits)) + (def unit (fraction-unit (min digits old-digits))) + (cond ((or (< old 0L) (> digits old-digits)) + (if (or (< old 0L) (= (/ old unit) (/ f unit))) + (Some (record-set seen (fraction f) (fraction-digits digits))) + None)) + ((= (/ old unit) (/ f unit)) (Some seen)) + (else None))) + +;; seen with the field f read as v, or None when it was read before as +;; something else. A value too large for any field stays too large. The +;; check comes first, so a Seen, which is large, is copied only to change it. +(: (-> Seen Field long (Option Seen))) +(defun (put-value seen f v) + (def n (if (or (> v 1000000000L) (< v 0L)) 1000000000 (int-of v))) + (def w (struct-ref f width)) + (match (struct-ref f kind) + (FYear (if (agrees? (struct-ref seen year) n) (Some (record-set seen (year n))) None)) + (FMonth (if (agrees? (struct-ref seen month) n) (Some (record-set seen (month n))) None)) + (FDay (if (agrees? (struct-ref seen day) n) (Some (record-set seen (day n))) None)) + (FHours (if (agrees? (struct-ref seen hours) n) (Some (record-set seen (hours n))) None)) + (FMinutes (if (agrees? (struct-ref seen minutes) n) (Some (record-set seen (minutes n))) None)) + (FSeconds (if (agrees? (struct-ref seen seconds) n) (Some (record-set seen (seconds n))) None)) + ;; A bare fraction says every digit there is: .5 is .5000000. + (FFraction (put-fraction seen v (if (< w 0) 7 (min w 7)))) + (FOffset (if (ticks-agree? (struct-ref seen offset) v) (Some (record-set seen (offset v))) None)) + (FUnix (if (ticks-agree? (struct-ref seen unix) v) (Some (record-set seen (unix v))) None)) + (FUnixMs (if (ticks-agree? (struct-ref seen unix) v) (Some (record-set seen (unix v))) None)) + (FRomanYear (if (agrees? (struct-ref seen year) n) (Some (record-set seen (year n))) None)) + (FRomanMonth (if (agrees? (struct-ref seen month) n) (Some (record-set seen (month n))) None)) + (FRomanDay (if (agrees? (struct-ref seen day) n) (Some (record-set seen (day n))) None)) + (FMonthName (if (agrees? (struct-ref seen month) n) (Some (record-set seen (month n))) None)) + (FMonthAbbr (if (agrees? (struct-ref seen month) n) (Some (record-set seen (month n))) None)) + (FWeekday (if (agrees? (struct-ref seen weekday) n) (Some (record-set seen (weekday n))) None)) + (FWeekdayName + (if (agrees? (struct-ref seen weekday) n) (Some (record-set seen (weekday n))) None)) + (FWeekdayAbbr + (if (agrees? (struct-ref seen weekday) n) (Some (record-set seen (weekday n))) None)) + (FYday (if (agrees? (struct-ref seen yday) n) (Some (record-set seen (yday n))) None)) + (FWeek (if (agrees? (struct-ref seen week) n) (Some (record-set seen (week n))) None)) + (FWeekYear + (if (agrees? (struct-ref seen week-year) n) (Some (record-set seen (week-year n))) None)) + (FHours12 (if (agrees? (struct-ref seen hours12) n) (Some (record-set seen (hours12 n))) None)) + (FAmPm (if (agrees? (struct-ref seen pm) n) (Some (record-set seen (pm n))) None)) + ;; Text is read by read-format, and a zone by put-zone. + (_ (Some seen)))) + +;; seen with the zone read from c to e, when no other was read before. +(: (-> string Seen StringCursor StringCursor (Option Seen))) +(defun (put-zone s seen c e) + (def start (struct-ref seen zone-start)) + (def end (struct-ref seen zone-end)) + (cond ((string-cursor=? start end) (Some (record-set seen (zone-start c) (zone-end e)))) + ((same-span? s start end c e) (Some seen)) + (else None))) + +;; The fields of a text in a format's parts, when the text has every part +;; and nothing after. Text is only skipped, and a field's value is read +;; before it is put, so a Seen is copied no more than it has to be. +(: (-> string (List %p) DateNames (Option Seen)) (where (FormatPart %p))) +(defun (read-format s parts names) + (let loop ((ps parts) (c (string-cursor-start s)) (seen unseen)) + (if (list-empty? ps) + (if (string-cursor-end? s c) (Some seen) None) + (let ((f (part-field (list-head ps)))) + (match (struct-ref f kind) + (FText + (match (skip-string s c (struct-ref f text)) + ((Some c1) (loop (list-tail ps) c1 seen)) + (None None))) + (FChar + (match (skip-char s c (struct-ref f ch)) + ((Some c1) (loop (list-tail ps) c1 seen)) + (None None))) + (FZone + (def* ((Some c1) (skip-zone s c) :propagate) + ((Some seen1) (put-zone s seen c c1) :propagate)) + (loop (list-tail ps) c1 seen1)) + (_ + (def* ((Some (Tuple v c1)) (read-value s c names f) :propagate) + ((Some seen1) (put-value seen f v) :propagate)) + (loop (list-tail ps) c1 seen1))))))) + +;; A date needs its largest unit. Month, day and yday count within year, +;; and week within week-year; either may be read beside a date the other +;; kind of year makes, and is checked against it. +(: (-> int void)) +(defun (check-date-bits bits) + (def year? (has? bits b-year)) + (def week-year? (has? bits b-week-year)) + (def in-year? (has? bits b-in-year)) + (def week? (has? bits b-week)) + (unless (or year? week-year?) + (argument-error "a format read as a date needs year, (roman year) or week-year.")) + (when (and in-year? (not year?) (not week?)) + (argument-error + "month, day and yday count within year, and this format has week-year instead: write year, or read a week date with week-year and week.")) + (when (and week? (not week-year?) (not in-year?)) + (argument-error + "week counts within week-year, and this format has year instead: write week-year, or read a calendar date with year, month and day."))) + +;; A time needs its hours: hours, or hours12 with am-pm, since 1:05 on a +;; twelve-hour clock could be either. A datetime's format may leave the time +;; out, which is midnight, and then optional is true. +(: (-> int bool void)) +(defun (check-time-bits bits optional) + (def hours? (or (has? bits b-hours) (has? bits b-hours12))) + (when (and (has? bits b-hours12) (not (has? bits b-am-pm))) + (argument-error "hours12 needs am-pm beside it, as 1:05 could be either.")) + (unless (or hours? optional) + (argument-error "a format read as a time needs hours, or hours12 with am-pm.")) + (when (and (not hours?) (has? bits (bitwise-ior b-in-hour b-am-pm))) + (argument-error + "minutes, seconds, fraction and am-pm count within hours, and this format has no hours or hours12."))) + +;; An instant is its Unix time alone, or a date and a time. A Unix time +;; beside another field of the instant could say another instant, so a +;; format with one has no other field but a moment's zone. +(: (-> int void)) +(defun (check-instant-bits bits) + (def unix (bitwise-ior b-unix b-unix-ms)) + (def others (- bits (bitwise-and bits (bitwise-ior unix b-zone)))) + (when (and (has? bits b-unix) (has? bits b-unix-ms)) + (argument-error "unix and unix-ms each say the whole instant, so a format reads one of them.")) + (when (and (has? bits unix) (> others 0)) + (argument-error + "unix and unix-ms say the whole instant, and a field beside them could say another, so a format that reads one has no other field but text and a moment's zone.")) + (unless (has? bits unix) + (check-date-bits bits) + (check-time-bits bits #t))) + +;; A moment needs its zone, which is never guessed. +(: (-> int void)) +(defun (check-moment-bits bits) + (unless (has? bits b-zone) + (argument-error + "a format read as a moment needs zone. A text with an offset and no zone is read with text->instant, and put in a zone with instant->moment.")) + (check-instant-bits bits)) + +(: (-> int int)) +(defun (first-value n) (if (< n 0) 1 n)) + +(: (-> int int int (Option long))) +(defun (calendar-day y m d) + (if (valid-date? y m d) (Some (long-of (clr-date-day-number (ClrDate. y m d)))) None)) + +(: (-> int int (Option long))) +(defun (ordinal-day y yd) + (if (and (>= y 1) (<= y 9999) (>= yd 1) (<= yd (clr-date-day-of-year (ClrDate. y 12 31)))) + (Some (+ (long-of (clr-date-day-number (ClrDate. y 1 1))) (long-of (- yd 1)))) + None)) + +;; The day the date fields make: year, month and day; year and yday; or +;; week-year, week and weekday. A field left out is its first value. +(: (-> Seen (Option long))) +(defun (seen-day-number seen) + (def y (struct-ref seen year)) + (def m (struct-ref seen month)) + (def d (struct-ref seen day)) + (def yd (struct-ref seen yday)) + (def wy (struct-ref seen week-year)) + (cond ((and (>= y 0) (or (>= m 0) (>= d 0))) (calendar-day y (first-value m) (first-value d))) + ((and (>= y 0) (>= yd 0)) (ordinal-day y yd)) + ((>= wy 0) + (option-map day-number (week-date wy (first-value (struct-ref seen week)) + (first-value (struct-ref seen weekday))))) + ((>= y 0) (calendar-day y 1 1)) + (else None))) + +;; Whether every date field read is the date's own. A field not read is -1, +;; and its accessor is not called. +(: (-> Seen Date bool)) +(defun (date-agrees? seen d) + (and (or (< (struct-ref seen year) 0) (= (struct-ref seen year) (->year d))) + (or (< (struct-ref seen month) 0) (= (struct-ref seen month) (->month d))) + (or (< (struct-ref seen day) 0) (= (struct-ref seen day) (->day d))) + (or (< (struct-ref seen weekday) 0) (= (struct-ref seen weekday) (->weekday d))) + (or (< (struct-ref seen yday) 0) (= (struct-ref seen yday) (->yday d))) + (or (< (struct-ref seen week) 0) (= (struct-ref seen week) (->week d))) + (or (< (struct-ref seen week-year) 0) (= (struct-ref seen week-year) (->week-year d))))) + +(: (-> Seen (Option Date))) +(defun (seen->date seen) + (def (Some n) (seen-day-number seen) :propagate) + (def d (day-number->date n)) + (if (date-agrees? seen d) (Some d) None)) + +;; The time of day the fields make, in ticks since midnight: hours, or +;; hours12 and am-pm, where 12 AM is 00 and 12 PM is 12; minutes, seconds +;; and the fraction left out are 0. am-pm beside hours has to agree with it. +(: (-> Seen (Option long))) +(defun (seen-time-ticks seen) + (def h24 (struct-ref seen hours)) + (def h12 (struct-ref seen hours12)) + (def pm (struct-ref seen pm)) + (def h (cond ((< h12 0) (max h24 0)) + ((or (< h12 1) (> h12 12)) -1) + (else (+ (% h12 12) (* (max pm 0) 12))))) + (def m (max (struct-ref seen minutes) 0)) + (def sec (max (struct-ref seen seconds) 0)) + (cond ((not (and (agrees? h24 h) (valid-time? h m sec 0))) None) + ((and (>= pm 0) (not (= pm (/ h 12)))) None) + (else (Some (+ (time-of-day-ticks h m sec 0) (max (struct-ref seen fraction) 0L)))))) + +;; Local ticks: a date and a time, midnight when the format has none. +(: (-> Seen (Option long))) +(defun (seen-local-ticks seen) + (def* ((Some d) (seen->date seen) :propagate) + ((Some t) (seen-time-ticks seen) :propagate)) + (Some (+ (* (day-number d) ticks-per-day) t))) + +;; The instant's UTC ticks: its Unix time, or the local time at the offset +;; read, or in UTC when the format has none, as instant->text writes it. +(: (-> Seen (Option long))) +(defun (seen-instant-ticks seen) + (def u (struct-ref seen unix)) + (def o (struct-ref seen offset)) + (cond ((not (= u no-ticks)) (Some u)) + (:def (Some l) (seen-local-ticks seen) :propagate) + ((= o no-ticks) (Some l)) + ((in-calendar? (- l o)) (Some (- l o))) + (else None))) + +;; resolve-local, None rather than raising where a local time and its +;; instant cannot both be in the calendar. +(: (-> long TimeZone Resolver (Option Moment))) +(defun (resolve-local/opt l z r) + (result->option (try (resolve-local l z r) #:catch (System.OverflowException)))) + +;; A moment in the zone read from s: the Unix time's instant there; or the +;; local time at the offset read, when the zone has that offset then; or the +;; local time alone, which in a gap or an overlap is None unless a resolver +;; says. An id the machine does not know is None. +(: (-> string Seen (Option Resolver) (Option Moment))) +(defun (seen->moment s seen r) + (def (Some z) + (timezone (substring s (struct-ref seen zone-start) (struct-ref seen zone-end))) + :propagate) + (def zi (struct-ref z tz)) + (def u (struct-ref seen unix)) + (def o (struct-ref seen offset)) + (cond ((not (= u no-ticks)) + (let ((uo (offset-at zi u))) + (if (in-calendar? (+ u uo)) + (Some (Moment (local (ticks->datetime (+ u uo))) (offset (ticks->duration uo)) + (zone z))) + None))) + (:def (Some l) (seen-local-ticks seen) :propagate) + ((not (= o no-ticks)) + (if (and (in-calendar? (- l o)) (= (offset-at zi (- l o)) o)) + (Some (Moment (local (ticks->datetime l)) (offset (ticks->duration o)) (zone z))) + None)) + (else + (match r + ((Some resolve) (resolve-local/opt l z resolve)) + (None (datetime->moment/opt (ticks->datetime l) z)))))) + +;; The reader of date->text's formats. +(: (-> string (List DatePart) (#:names DateNames) (Option Date))) +(defun (text->date s parts #:names (parameter-ref current-date-names)) + (check-date-bits (format-bits parts names)) + (def (Some seen) (read-format s parts names) :propagate) + (seen->date seen)) + +(: (-> string (List TimePart) (#:names DateNames) (Option Time))) +(defun (text->time s parts #:names (parameter-ref current-date-names)) + (check-time-bits (format-bits parts names) #f) + (def* ((Some seen) (read-format s parts names) :propagate) + ((Some t) (seen-time-ticks seen) :propagate)) + (Some (Time (v (ClrTime. t))))) + +(: (-> string (List DateTimePart) (#:names DateNames) (Option DateTime))) +(defun (text->datetime s parts #:names (parameter-ref current-date-names)) + (def bits (format-bits parts names)) + (check-date-bits bits) + (check-time-bits bits #t) + (def* ((Some seen) (read-format s parts names) :propagate) + ((Some l) (seen-local-ticks seen) :propagate)) + (Some (ticks->datetime l))) + +;; The fields are UTC's unless the format has an offset, as instant->text +;; writes them. +(: (-> string (List InstantPart) (#:names DateNames) (Option Instant))) +(defun (text->instant s parts #:names (parameter-ref current-date-names)) + (check-instant-bits (format-bits parts names)) + (def* ((Some seen) (read-format s parts names) :propagate) + ((Some u) (seen-instant-ticks seen) :propagate)) + (Some (Instant (ticks u)))) + +;; With an offset, the offset has to be one the zone has at that local time, +;; as iso8601->moment demands. Without one, a local time in a gap or an +;; overlap is None. +(: (-> string (List MomentPart) (#:names DateNames) (Option Moment))) +(defun (text->moment s parts #:names (parameter-ref current-date-names)) + (check-moment-bits (format-bits parts names)) + (def (Some seen) (read-format s parts names) :propagate) + (seen->moment s seen None)) + +;; A local time without an offset found as the resolver says, as +;; datetime->moment finds it. +(: (-> string (List MomentPart) Resolver (#:names DateNames) (Option Moment))) +(defun (text->moment/resolver s parts r #:names (parameter-ref current-date-names)) + (check-moment-bits (format-bits parts names)) + (def (Some seen) (read-format s parts names) :propagate) + (seen->moment s seen (Some r))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/effect.bjo — tail-resumptive effect handlers. +;; +;; An effect operation is a parameter holding a function. `with-handler` is +;; `parameterize`, and performing an operation reads the parameter and calls the +;; function on the performer's own stack. No continuation is captured, so a +;; handler can return once (resume) or throw (abort) and nothing else. +;; +;; What that rules out, and rules out for good on this backend: generators, +;; backtracking, and resuming an operation twice. Nothing here can be extended +;; to give them — there is no continuation to hold on to. +;; +;; This module imports neither `prelude` nor anything that imports it, and that +;; is what lets `prelude` declare effects of its own: a macro cannot be used in +;; the module that defines it, so `defeffect` has to live below the module with +;; the most to declare. The parameter machinery a template reaches for — +;; `make-parameter`, `make-cold-parameter`, `parameter-ref`, `Param` and +;; `parameterize` — is built in rather than imported, so an expansion resolves +;; wherever it lands. + +(import (std eq)) +(import (std syntax-match)) + +(import/extern + (str-concat (: System.String.Concat))) + +(import/class + (Exception (: System.Exception)) + (Unhandled (: Bjolang.Runtime.Unhandled (-> string Unhandled)))) + +(:doc #:file "effect.bjodoc") + +(export unhandled-effect) + +(: (-> string string string)) +(defun (cat a b) (str-concat a b)) + +;; What an operation declared with no `#:default` does when it is performed. +;; Written here rather than in the template so that the message is in one place. +(: (-> string %a)) +(defun (unhandled-effect operation) + (raise (cast Exception (Unhandled. operation)))) + +;; --------------------------------------------------------------------------- +;; Reading a declaration +;; --------------------------------------------------------------------------- + +(: (-> Syntax string)) +(defun (ident-name s) + (match s + ((SSym name) (symbol->string name)) + (_ ""))) + +;; The head of an arrow: "->" or "-bjo->". +(: (-> Syntax string)) +(defun (arrow-head arrow) + (match arrow + ((SList (Cons head _)) (ident-name head)) + (_ ""))) + +(: (-> Syntax bool)) +(defun (suspending-arrow? arrow) + (= (arrow-head arrow) "-bjo->")) + +(: (-> Syntax bool)) +(defun (arrow? arrow) + (if (= (arrow-head arrow) "->") #t (suspending-arrow? arrow))) + +;; Everything but the head and the return type. +(: (-> Syntax int)) +(defun (arrow-arity arrow) + (match arrow + ((SList items) (- (list-length items) 2)) + (_ 0))) + +;; The parameter that holds an operation's handler. +;; +;; `op/log` is the cell and `log` is the wrapper that reads it. Both are +;; ordinary module-level names: a module that wants another module to be able to +;; handle its effect exports both. +;; +;; Built through `inject`, so the name is *not* renamed. `defeffect` and +;; `with-handler` are separate invocations with separate rename tables, and a +;; renamed cell would give each of them a spelling the other could not write. +(: (-> (-> Symbol Syntax) Syntax Syntax)) +(defun (op-cell inject name) + (inject (string->symbol (cat "op/" (ident-name name))))) + +;; The wrapper's parameters. Like every binder a macro introduces they are +;; renamed, to `effect-arg0__164` and the like, which is harmless here: the only +;; thing in scope inside the wrapper is the wrapper. A doc of the operation names +;; them itself, as it would a builtin's. +(: (-> int int (List Syntax))) +(defun (wrapper-args i n) + (if (>= i n) + Nil + (Cons (SSym (string->symbol (cat "effect-arg" (.ToString i)))) + (wrapper-args (+ i 1) n)))) + +;; --------------------------------------------------------------------------- +;; The clauses of a `defeffect` +;; --------------------------------------------------------------------------- + +(: (-> (List Syntax) (List Syntax))) +(defun (rest-after clauses) + (match clauses + (Nil Nil) + ((Cons _ rest) rest))) + +;; The items of the form following a keyword clause. +(: (-> (List Syntax) (List Syntax))) +(defun (clause-items clauses) + (match clauses + (Nil Nil) + ((Cons c _) + (match c + ((SList items) items) + (_ (list (syntax-error c "a #:default or #:cold clause takes a parenthesized list."))))))) + +;; The operation declarations, which is everything that is not a keyword clause +;; or the form one introduces. +(: (-> (List Syntax) (List Syntax))) +(defun (operation-clauses clauses) + (match clauses + (Nil Nil) + ((Cons c rest) + (syntax-match c + (#:default (operation-clauses (rest-after rest))) + (#:cold (operation-clauses (rest-after rest))) + (#:adopt (operation-clauses (rest-after rest))) + (_ (Cons c (operation-clauses rest))))))) + +(: (-> (List Syntax) (List Syntax))) +(defun (default-clauses clauses) + (match clauses + (Nil Nil) + ((Cons c rest) + (syntax-match c + (#:default (clause-items rest)) + (_ (default-clauses rest)))))) + +(: (-> (List Syntax) (List Syntax))) +(defun (cold-clauses clauses) + (match clauses + (Nil Nil) + ((Cons c rest) + (syntax-match c + (#:cold (clause-items rest)) + (_ (cold-clauses rest)))))) + +(: (-> (List Syntax) (List Syntax))) +(defun (adopt-clauses clauses) + (match clauses + (Nil Nil) + ((Cons c rest) + (syntax-match c + (#:adopt (clause-items rest)) + (_ (adopt-clauses rest)))))) + +(: (-> Syntax (List Syntax) bool)) +(defun (named-in? name names) + (match names + (Nil #f) + ((Cons n rest) (if (syntax-ident=? n name) #t (named-in? name rest))))) + +(: (-> Syntax (List Syntax) (Option Syntax))) +(defun (default-for name pairs) + (match pairs + (Nil None) + ((Cons pair rest) + (syntax-match pair + ((op handler) (if (syntax-ident=? op name) (Some handler) (default-for name rest))) + (bad (Some (syntax-error bad "a #:default clause pairs an operation with a handler: (log write-it)"))))))) + +;; --------------------------------------------------------------------------- +;; What one operation expands to +;; --------------------------------------------------------------------------- + +;; The default is a lambda of the operation's own colour: colours are not +;; subtypes here, so an ordinary `fun` does not stand in for a `(-bjo-> ...)`. +(: (-> Syntax Syntax (List Syntax) (List Syntax) Syntax)) +(defun (handler-default name arrow args defaults) + (match (default-for name defaults) + ((Some handler) handler) + (None + (if (suspending-arrow? arrow) + #'(bjoroutine (,@args) (unhandled-effect ,(SStr (ident-name name)))) + #'(fun (,@args) (unhandled-effect ,(SStr (ident-name name)))))))) + +;; Three definitions per operation, and nothing per effect: the cell, the +;; wrapper's signature, and the wrapper. +;; +;; The wrapper is a named function rather than a bare parameter read so that the +;; declared arrow is checked at every call, and so that the colour machinery +;; sees a function: a `(-bjo-> ...)` operation performed inside a `defun` is then +;; the ordinary "cannot suspend here" error. +;; +;; The handler goes into a local before it is applied. A suspending call is only +;; awaited when its callee is a name, so `((parameter-ref cell) x)` would emit a +;; `Fiber` where the value was wanted. +;; +;; `#:adopt` intercepts the value returned by a handler and passes it through a +;; named function before returning it to the caller. It is for operations that +;; return *resources*: the resource can be registered on the current scope, so +;; that one a handler made is cleaned up as a real one would be. +(: (-> Syntax Syntax (List Syntax) (List Syntax) Syntax)) +(defun (wrapper-body name cell args adopts) + (match (default-for name adopts) + (None #'(let ((handler (parameter-ref ,cell))) (handler ,@args))) + ((Some adopt) #'(let ((handler (parameter-ref ,cell))) (,adopt (handler ,@args)))))) + +(: (-> (-> Symbol Syntax) Syntax (List Syntax) (List Syntax) (List Syntax) (List Syntax))) +(defun (operation-decls inject clause defaults colds adopts) + (syntax-match clause + ((name arrow) + (if (not (arrow? arrow)) + (list (syntax-error arrow "an operation's type is an arrow: (-> ...) for one performable anywhere, (-bjo-> ...) for one only a bjoroutine may perform.")) + (let ((cell (op-cell inject name)) + (args (wrapper-args 0 (arrow-arity arrow)))) + (list #'(: ,cell (Param ,arrow)) + (if (named-in? name colds) + #'(def ,cell (make-cold-parameter ,(handler-default name arrow args defaults))) + #'(def ,cell (make-parameter ,(handler-default name arrow args defaults)))) + #'(: ,name ,arrow) + (if (suspending-arrow? arrow) + #'(defbjo (,name ,@args) ,(wrapper-body name cell args adopts)) + #'(defun (,name ,@args) ,(wrapper-body name cell args adopts))))))) + (bad (list (syntax-error bad "an operation is written (name arrow), as in (log (-> string void))."))))) + +(: (-> (-> Symbol Syntax) (List Syntax) (List Syntax) (List Syntax) (List Syntax) (List Syntax))) +(defun (effect-decls inject ops defaults colds adopts) + (match ops + (Nil Nil) + ((Cons op rest) + (syntax-splice (operation-decls inject op defaults colds adopts) + (effect-decls inject rest defaults colds adopts))))) + +;; --------------------------------------------------------------------------- +;; The macros +;; --------------------------------------------------------------------------- + +;; `(defeffect Name (op arrow) ... #:default ((op handler) ...) #:cold (op ...))` +;; +;; (defeffect Log +;; (log (-> string void)) +;; (audit (-bjo-> string void)) +;; #:default ((log (fun (s) (display s)))) +;; #:cold (audit)) +;; +;; The effect name is documentation: nothing is emitted for it, and there is no +;; form that installs a whole effect at once. Handlers are installed one +;; operation at a time with `with-handler`. +;; +;; An operation with no `#:default` raises `Bjolang.Runtime.Unhandled` when it is +;; performed with nothing installed. Give a default when there is one sensible +;; meaning; leave it out when running with no handler should fail loudly. +;; +;; An operation's colour is its declared arrow, and a handler has to match it. +;; A `(-> ...)` operation takes `fun` handlers and can be performed anywhere, +;; including inside a `defun`. A `(-bjo-> ...)` operation takes `bjoroutine` +;; handlers and can only be performed from a bjoroutine; an ordinary `fun` is +;; *not* lifted into one, because the two arrows are different calling +;; conventions rather than a subtype pair. +;; +;; Prefer a sync operation. Anything a `defun` has to be able to perform must be +;; one, and an operation that has to return something suspendable can return a +;; value whose own methods are colour-aware — a port, say — rather than +;; suspending itself. +;; +;; Every operation spends one of the dynamic environment's 31 cheap parameter +;; ids. `#:cold` gives that claim up, for an operation its author knows is not +;; performed in a loop: the ids are a budget, and a program with many rarely +;; performed effects should not push its ports out of the cheap slots. +;; +;; `#:adopt ((op fn))` intercepts the value returned by a handler and passes it +;; through `fn` before returning it to the caller. An operation that returns a +;; *resource* needs it when a handler may build one that nothing owns - a +;; string port, say - since otherwise that resource is never released and a +;; leak test over the handler means nothing. An operation that returns the raw +;; material instead, and leaves building and registering the resource to the +;; caller, does not: the prelude's `FS` openers answer a `ClrStream` for that +;; reason. +(def/macro (defeffect form inject compare) + (syntax-match form + ((_ name clause ...) + (if (list-empty? clause) + (syntax-error form "defeffect needs at least one operation: (defeffect Log (log (-> string void)))") + #'(begin + ,@(effect-decls inject + (operation-clauses clause) + (default-clauses clause) + (cold-clauses clause) + (adopt-clauses clause))))) + (bad (syntax-error bad "defeffect takes an effect name and one or more operations: (defeffect Log (log (-> string void)) #:default ((log write-it)))")))) + +;; `(with-handler ((op handler) ...) body ...)` — install handlers for the +;; extent of the body. +;; +;; This is `parameterize` on the operations' cells: the bindings are +;; simultaneous, and the environment is restored however the body ends. +;; +;; A handler runs on the *performer's* stack. So a handler that throws unwinds +;; the performer rather than the installer, and inside a spawned fiber that is a +;; child failure like any other. +;; +;; # A handler frame is not a scope +;; +;; A scope waits for its children; this does not. So +;; +;; (with-cancel (cancel) +;; (with-handler ((log (collect-into buf))) +;; (spawn (work))) +;; (report buf)) +;; +;; returns from `with-handler` as soon as the spawn is queued, and `report` +;; reads a buffer the child is still filling. The child does not lose the +;; handler — a spawn takes the environment as it stood at the spawn, so the +;; handler travels with it and outlives the form that installed it — which is +;; why the read is early rather than empty. +;; +;; **A handler that collects must enclose the scope that owns the performers.** +;; Put the `with-handler` outside the `with-cancel`, and read after both have +;; returned. `parameterize` behaves the same way; a collecting handler is only +;; where it shows. +;; +;; # Forwarding +;; +;; A handler runs with itself installed, so one that performs its own operation +;; calls itself: +;; +;; (with-handler ((log (fun (s) (log (str "[app] " s))))) ...) ;; loops +;; +;; The handler installed before it is `(handler-of log)`. The bindings of a +;; `with-handler` are evaluated before any of them is installed, so it can be +;; read in the binding itself: +;; +;; (with-handler ((log (let ((outer (handler-of log))) +;; (fun (s) (outer (str "[app] " s)))))) +;; ...) +;; +;; # The cell is found by spelling +;; +;; `with-handler` derives the cell from the operation's written name: `log` +;; binds `op/log`. That is a convention, not a binding, so an operation imported +;; under a prefix or a rename has no cell of that spelling and `with-handler` +;; cannot find it. The fallback is the expansion itself: +;; +;; (parameterize ((p:op/log collect)) ...) +;; +;; `op/log` is an ordinary exported binding and follows every rename. +;; +;; The same convention is why a macro that writes a `with-handler` has to +;; `inject` the operation name: a template renames `log`, and a renamed `log` +;; derives a cell nothing declares. +;; +;; (with-handler ((,(inject (string->symbol "log")) collect)) ...) +;; +;; A name taken from the macro's own input is already unrenamed. +;; `std/simpletest`'s `with-fake-fs` is the worked example. +(def/macro (with-handler form inject compare) + (syntax-match form + ((_ ((op handler) ...) first body ...) + (if (list-empty? op) + (syntax-error form "with-handler installs at least one handler: (with-handler ((log collect)) body ...)") + #'(parameterize ,(SList (handler-bindings inject op handler)) ,first ,@body))) + (bad (syntax-error bad "with-handler takes a list of (operation handler) pairs and a body: (with-handler ((log collect)) body ...)")))) + +(: (-> (-> Symbol Syntax) (List Syntax) (List Syntax) (List Syntax))) +(defun (handler-bindings inject ops handlers) + (match ops + (Nil Nil) + ((Cons op orest) + (match handlers + (Nil Nil) + ((Cons h hrest) (Cons #'(,(op-cell inject op) ,h) (handler-bindings inject orest hrest))))))) + +;; `(handler-of op)` - the handler installed for `op` here. +;; +;; `(parameter-ref op/x)`, with the cell derived the way `with-handler` derives +;; it. Read it where a `with-handler` is written to forward to the handler that +;; one replaces. +(def/macro (handler-of form inject compare) + (syntax-match form + ((_ op) #'(parameter-ref ,(op-cell inject op))) + (bad (syntax-error bad "handler-of takes an operation name: (handler-of log)")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/eq.bjo — equality and hashing. +(export = eq-hash + Ord compare less? greater? at-most? at-least? + least greatest list-sort vec-sort vec-sort-by array-sort! array-sort-by! + list-sort-unstable vec-sort-unstable array-sort-unstable! + list-min list-max + not) + +(:doc #:file "eq.bjodoc") + +;; Here, and not in the prelude, so that the modules the prelude imports +;; have it too. The prelude re-exports it. +(: (-> bool bool)) +(defun (not b) #:inline (if b #f #t)) + +;; --------------------------------------------------------------------------- +;; Eq — the one equality +;; --------------------------------------------------------------------------- +;; Trait for equality and hashing. +(def/trait (Eq %a) + (: = (-> %a %a bool)) + (: eq-hash (-> %a int))) + + +;; --------------------------------------------------------------------------- +;; Ord — the one ordering +;; --------------------------------------------------------------------------- +;; Trait for ordering. +(def/trait (Ord %a) + (where (Eq %a)) + (: compare (-> %a %a int))) + +;; Ord predicates. +(: (-> %a %a bool) (where (Ord %a))) +(defun (less? a b) (< (compare a b) 0)) + +(: (-> %a %a bool) (where (Ord %a))) +(defun (greater? a b) (> (compare a b) 0)) + +(: (-> %a %a bool) (where (Ord %a))) +(defun (at-most? a b) (<= (compare a b) 0)) + +(: (-> %a %a bool) (where (Ord %a))) +(defun (at-least? a b) (>= (compare a b) 0)) + +;; least and greatest (ties to left). +(: (-> %a %a %a) (where (Ord %a))) +(defun (least a b) (if (at-most? a b) a b)) + +(: (-> %a %a %a) (where (Ord %a))) +(defun (greatest a b) (if (at-least? a b) a b)) + +;; --- Sorting --------------------------------------------------------------- +;; Two kinds, in the order of the elements' Ord: +;; +;; - `array-sort!`, `list-sort`, `vec-sort` (and `mutablevec-sort!`): stable, +;; equal elements keep their order. The merge sort in `array-sort!`. It is +;; exported, so the compiler copies it into each caller at the element type, +;; and `compare` is a direct call. +;; - The `-unstable` ones: .NET's introsort (`Bjolang.Runtime.SortModule`), +;; with `(ord-comparer)`. Equal elements can change places. Faster, and for +;; a type whose equal values are the same value, such as an int, the result +;; is the same. +;; +;; The list and vec sorts sort a copy, so when `compare` raises their input is +;; as it was. + +(import/class + ((Comparer %a) (: System.Collections.Generic.IComparer))) + +(import/extern + (clr-sort-unstable! (: Bjolang.Runtime.SortModule.SortUnstable (-> (Array %a) (Comparer %a) void))) + (clr-vec->array (: BjolangRuntime.vecsubgtarray (-> (Vec %a) (Array %a)))) + (clr-list->array (: BjolangRuntime.listsubgtarray (-> (List %a) (Array %a))))) + +;; Sorts `arr` in place, in the order of the elements' Ord. Stable. +;; +;; - Insertion sort over runs of 16, then bottom-up merges between `arr` and +;; one buffer. A merge takes from the left run on a tie, which keeps the sort +;; stable. +;; - When `compare` raises, the contents of `arr` are unspecified: an element +;; can be lost or be there twice. +(: (-> (Array %a) void) (where (Ord %a))) +(defun (array-sort! arr) + (def n (array-length arr)) + (def run 16) + + (defun (smaller (: a int) b) (if (< a b) a b)) + + ;; The elements in [lo, hi), in order. + (defun (insert-run lo hi) + (let next ((i (+ lo 1))) + (when (< i hi) + (let ((x (array-ref arr i))) + (let shift ((j (- i 1))) + (if (if (>= j lo) (> (compare (array-ref arr j) x) 0) #f) + (let () + (array-set! arr (+ j 1) (array-ref arr j)) + (shift (- j 1))) + (array-set! arr (+ j 1) x)))) + (next (+ i 1))))) + + ;; src[lo, mid) and src[mid, hi) into dst[lo, hi). + (defun (merge-runs (: src (Array %a)) dst lo mid hi) + (let go ((l lo) (r mid) (k lo)) + (when (< k hi) + (if (if (< l mid) + (if (< r hi) (<= (compare (array-ref src l) (array-ref src r)) 0) #t) + #f) + (let () + (array-set! dst k (array-ref src l)) + (go (+ l 1) r (+ k 1))) + (let () + (array-set! dst k (array-ref src r)) + (go l (+ r 1) (+ k 1))))))) + + (defun (merge-pass src dst width) + (let go ((lo 0)) + (if (< lo n) + (let () + (merge-runs src dst lo (smaller (+ lo width) n) (smaller (+ lo (* 2 width)) n)) + (go (+ lo (* 2 width)))) + dst))) + + (let runs ((lo 0)) + (when (< lo n) + (insert-run lo (smaller (+ lo run) n)) + (runs (+ lo run)))) + (when (> n run) + (def sorted + (let go ((src arr) (dst (make-array n)) (width run)) + (if (< width n) + (go (merge-pass src dst width) src (* width 2)) + src))) + (when (not (eq? sorted arr)) + (let copy ((i 0)) + (when (< i n) + (array-set! arr i (array-ref sorted i)) + (copy (+ i 1))))))) + +;; Sorts `arr` in place, in the order of the elements' Ord. Not stable. +;; +;; - When `compare` raises, it raises the same exception, and the contents of +;; `arr` are unspecified. +(: (-> (Array %a) void) (where (Ord %a))) +(defun (array-sort-unstable! arr) (clr-sort-unstable! arr (ord-comparer))) + +(: (-> (List %a) (List %a)) (where (Ord %a))) +(defun (list-sort xs) + (def n (list-length xs)) + + (defun (fill arr) + (let go ((i 0) (rest xs)) + (if (< i n) + (let () + (array-set! arr i (list-head rest)) + (go (+ i 1) (list-tail rest))) + arr))) + + (defun (sorted) + (def (: arr (Array %a)) (fill (make-array n))) + (array-sort! arr) + (let go ((i (- n 1)) (acc Nil)) + (if (< i 0) + acc + (go (- i 1) (Cons (array-ref arr i) acc))))) + + (if (< n 2) xs (sorted))) + +;; The same merge sort over an array, with the comparison passed in: negative +;; when the first comes first, zero when they tie, positive otherwise. Stable, +;; since ties take the left run. Answers the array that holds the result, which +;; is arr or one of the same length. +(: (-> (-> %a %a int) (Array %a) (Array %a))) +;; +;; The local functions take the array and the comparison as arguments, and +;; capture nothing. A captured variable is a field that .NET reads again +;; after each call of `cmp`, and checks the index against again. +(defun (sort-array-by cmp arr) + (def n (array-length arr)) + (def run 16) + + (defun (smaller (: a int) b) (if (< a b) a b)) + + ;; The elements in [lo, hi), in order, as `array-sort!` sorts its runs. + (defun (insert-run (: c (-> %a %a int)) (: a (Array %a)) lo hi) + (let next ((i (+ lo 1))) + (when (< i hi) + (let ((x (array-ref a i))) + (let shift ((j (- i 1))) + (if (if (>= j lo) (> (c (array-ref a j) x) 0) #f) + (let () + (array-set! a (+ j 1) (array-ref a j)) + (shift (- j 1))) + (array-set! a (+ j 1) x)))) + (next (+ i 1))))) + + (defun (merge-runs (: c (-> %a %a int)) (: src (Array %a)) dst lo mid hi) + (let go ((l lo) (r mid) (k lo)) + (if (< k hi) + (if (if (< l mid) + (if (< r hi) (<= (c (array-ref src l) (array-ref src r)) 0) #t) + #f) + (let () + (array-set! dst k (array-ref src l)) + (go (+ l 1) r (+ k 1))) + (let () + (array-set! dst k (array-ref src r)) + (go l (+ r 1) (+ k 1)))) + k))) + + (defun (merge-pass (: c (-> %a %a int)) src dst (: m int) width) + (let go ((lo 0)) + (if (< lo m) + (go (merge-runs c src dst lo (smaller (+ lo width) m) (smaller (+ lo (* 2 width)) m))) + dst))) + + (let runs ((lo 0)) + (when (< lo n) + (insert-run cmp arr lo (smaller (+ lo run) n)) + (runs (+ lo run)))) + (if (> n run) + (let go ((src arr) (dst (make-array n)) (width run)) + (if (< width n) + (go (merge-pass cmp src dst n width) src (* width 2)) + src)) + arr)) + +;; Sorts `arr` in place, in the order of a comparison, as `sort-array-by` +;; orders it. Stable. When `cmp` raises, the contents of `arr` are +;; unspecified. +(: (-> (-> %a %a int) (Array %a) void)) +(defun (array-sort-by! cmp arr) + (def sorted (sort-array-by cmp arr)) + (when (not (eq? sorted arr)) + (let copy ((i 0)) + (when (< i (array-length arr)) + (array-set! arr i (array-ref sorted i)) + (copy (+ i 1)))))) + +;; A vec in order of a comparison, as `sort-array-by` orders an array. +(: (-> (-> %a %a int) (Vec %a) (Vec %a))) +(defun (vec-sort-by cmp v) + (def n (vec-length v)) + (if (< n 2) + v + (let () + (def arr (make-array n)) + (let fill ((i 0)) + (when (< i n) + (array-set! arr i (vec-ref v i)) + (fill (+ i 1)))) + (array->vec (sort-array-by cmp arr))))) + +;; A vec in the order of its elements' Ord. Stable. +(: (-> (Vec %a) (Vec %a)) (where (Ord %a))) +(defun (vec-sort v) + (if (< (vec-length v) 2) + v + (let ((arr (clr-vec->array v))) + (array-sort! arr) + (array->vec arr)))) + +;; A vec in the order of its elements' Ord. Not stable. +(: (-> (Vec %a) (Vec %a)) (where (Ord %a))) +(defun (vec-sort-unstable v) + (if (< (vec-length v) 2) + v + (let ((arr (clr-vec->array v))) + (array-sort-unstable! arr) + (array->vec arr)))) + +;; A list in the order of its elements' Ord. Not stable. +(: (-> (List %a) (List %a)) (where (Ord %a))) +(defun (list-sort-unstable xs) + (if (if (list-empty? xs) #t (list-empty? (list-tail xs))) + xs + (let ((arr (clr-list->array xs))) + (array-sort-unstable! arr) + (let go ((i (- (array-length arr) 1)) (acc Nil)) + (if (< i 0) + acc + (go (- i 1) (Cons (array-ref arr i) acc))))))) + +;; Private left fold for list min/max. +(: (-> (-> %a %b %b) %b (List %a) %b)) +(defun (fold-left f init xs) + (let go ((rest xs) (acc init)) + (if (list-empty? rest) + acc + (go (list-tail rest) (f (list-head rest) acc))))) + +(: (-> (List %a) (Option %a)) (where (Ord %a))) +(defun (list-min xs) + (if (list-empty? xs) + None + (Some (fold-left least (list-head xs) (list-tail xs))))) + +(: (-> (List %a) (Option %a)) (where (Ord %a))) +(defun (list-max xs) + (if (list-empty? xs) + None + (Some (fold-left greatest (list-head xs) (list-tail xs))))) + + +;; --- The blanket ----------------------------------------------------------- +;; Default to .NET equality. +(impl (Eq %a) + (defun (= a b) (clr-equals a b)) + (defun (eq-hash a) (clr-hash a))) + + +;; --- The primitives -------------------------------------------------------- +;; Eq for primitives. + +(impl (Eq int) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +(impl (Eq long) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +(impl (Eq short) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +(impl (Eq byte) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +(impl (Eq uint) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +(impl (Eq ulong) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +(impl (Eq ushort) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +;; Eq for double uses Equals (NaN == NaN). +(impl (Eq double) + (defun (= a b) (clr-equals a b)) + (defun (eq-hash a) (clr-hash a))) + +(impl (Eq bool) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +;; Eq for string. +(impl (Eq string) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +(impl (Eq char) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +;; Eq for Keyword and Symbol. +(impl (Eq Keyword) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +(impl (Eq Symbol) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +(impl (Eq StringCursor) + (defun (= a b) (clr-eq a b)) + (defun (eq-hash a) (clr-hash a))) + +;; Eq for void. +(impl (Eq void) + (defun (= a b) #t) + (defun (eq-hash a) 0)) + + +;; --- Ord for the primitives ------------------------------------------------ +;; Delegates to IComparable. + +(impl (Ord int) (defun (compare a b) (.CompareTo a b))) +(impl (Ord long) (defun (compare a b) (.CompareTo a b))) +(impl (Ord short) (defun (compare a b) (.CompareTo a b))) +(impl (Ord byte) (defun (compare a b) (.CompareTo a b))) +(impl (Ord uint) (defun (compare a b) (.CompareTo a b))) +(impl (Ord ulong) (defun (compare a b) (.CompareTo a b))) +(impl (Ord ushort) (defun (compare a b) (.CompareTo a b))) + +;; Ord for double uses CompareTo. +(impl (Ord double) (defun (compare a b) (.CompareTo a b))) + +(impl (Ord bool) (defun (compare a b) (.CompareTo a b))) + +;; Ord for string compares code points, never the ambient culture: the same +;; answer everywhere, and the order an OrderedMap of strings keeps. +(impl (Ord string) (defun (compare a b) (.CompareTo a b))) + +(impl (Ord char) (defun (compare a b) (.CompareTo a b))) + +;; Ord for the four numbers past the eight. Their Eq is the blanket's, which is +;; their own Equals; their order is their own CompareTo, as for the primitives. +(impl (Ord decimal) (defun (compare a b) (.CompareTo a b))) +(impl (Ord bigint) (defun (compare a b) (.CompareTo a b))) +(impl (Ord rational) (defun (compare a b) (.CompareTo a b))) +(impl (Ord int128) (defun (compare a b) (.CompareTo a b))) +(impl (Ord uint128) (defun (compare a b) (.CompareTo a b))) + +;; And for sbyte, which has no name of its own: a program imports it with +;; import/class. The alias is not exported. +(import/class (ClrSByte (: System.SByte))) +(impl (Ord ClrSByte) (defun (compare a b) (.CompareTo a b))) + + +;; --- The containers -------------------------------------------------------- +;; Container equality using element Eq. Uses builtins only. + +;; A loop rather than recursion through `and`, so a long list does not +;; overflow the stack. Two lists that share a tail stop comparing there. +(impl (Eq (List %a)) + (where (Eq %a)) + (defun (= xs ys) + (let go ((a xs) (b ys)) + (if (eq? a b) + #t + (if (list-empty? a) + (list-empty? b) + (if (list-empty? b) + #f + (if (= (list-head a) (list-head b)) + (go (list-tail a) (list-tail b)) + #f)))))) + (defun (eq-hash xs) + (fold-left (fun (x h) (hash-combine h (eq-hash x))) 17 xs))) + +;; Two cursors walk the leaves in step: no tree lookup per element, and no +;; recursion per element through `and`. +(impl (Eq (Vec %a)) + (where (Eq %a)) + (defun (= xs ys) + (if (eq? xs ys) + #t + (if (= (vec-length xs) (vec-length ys)) + (let go ((cx (vec-cursor xs)) (cy (vec-cursor ys))) + (if (vec-cursor-done? cx) + #t + (if (vec-cursor-done? cy) + #f + (if (= (vec-cursor-current cx) (vec-cursor-current cy)) + (go (vec-cursor-next cx) (vec-cursor-next cy)) + #f)))) + #f))) + ;; Manual indexing. + (defun (eq-hash xs) + (defun (hash-from i h) + (if (= i (vec-length xs)) + h + (hash-from (+ i 1) (hash-combine h (eq-hash (vec-ref xs i)))))) + (hash-from 0 17))) + +(impl (Eq (Option %a)) + (where (Eq %a)) + (defun (= a b) + (match a + ((Some x) + (match b + ((Some y) (= x y)) + (None #f))) + (None (none? b)))) + (defun (eq-hash o) + (match o + ((Some x) (hash-combine 1 (eq-hash x))) + (None 0)))) + +;; Eq for Tuples. +(impl (Eq (Tuple %a %b)) + (where (Eq %a) (Eq %b)) + (defun (= x y) + (def (a1 b1) x) + (def (a2 b2) y) + (and (= a1 a2) (= b1 b2))) + (defun (eq-hash x) + (def (a b) x) + (hash-combine (eq-hash a) (eq-hash b)))) + +(impl (Eq (Tuple %a %b %c)) + (where (Eq %a) (Eq %b) (Eq %c)) + (defun (= x y) + (def (a1 b1 c1) x) + (def (a2 b2 c2) y) + (and (= a1 a2) (and (= b1 b2) (= c1 c2)))) + (defun (eq-hash x) + (def (a b c) x) + (hash-combine (hash-combine (eq-hash a) (eq-hash b)) (eq-hash c)))) + +(impl (Eq (Tuple %a %b %c %d)) + (where (Eq %a) (Eq %b) (Eq %c) (Eq %d)) + (defun (= x y) + (def (a1 b1 c1 d1) x) + (def (a2 b2 c2 d2) y) + (and (and (= a1 a2) (= b1 b2)) (and (= c1 c2) (= d1 d2)))) + (defun (eq-hash x) + (def (a b c d) x) + (hash-combine (hash-combine (eq-hash a) (eq-hash b)) + (hash-combine (eq-hash c) (eq-hash d))))) + +;; Ord for Tuples: lexicographic. The first items decide, and a later item is +;; compared only when every item before it is equal. +(impl (Ord (Tuple %a %b)) + (where (Ord %a) (Ord %b)) + (defun (compare x y) + (def (a1 b1) x) + (def (a2 b2) y) + (def c (compare a1 a2)) + (if (= c 0) (compare b1 b2) c))) + +(impl (Ord (Tuple %a %b %c)) + (where (Ord %a) (Ord %b) (Ord %c)) + (defun (compare x y) + (def (a1 b1 c1) x) + (def (a2 b2 c2) y) + (def ca (compare a1 a2)) + (if (= ca 0) + (let ((cb (compare b1 b2))) + (if (= cb 0) (compare c1 c2) cb)) + ca))) + +(impl (Ord (Tuple %a %b %c %d)) + (where (Ord %a) (Ord %b) (Ord %c) (Ord %d)) + (defun (compare x y) + (def (a1 b1 c1 d1) x) + (def (a2 b2 c2 d2) y) + (def ca (compare a1 a2)) + (if (= ca 0) + (let ((cb (compare b1 b2))) + (if (= cb 0) + (let ((cc (compare c1 c2))) + (if (= cc 0) (compare d1 d2) cc)) + cb)) + ca))) + +(impl (Eq (Result %e %a)) + (where (Eq %e) (Eq %a)) + (defun (= a b) + (match a + ((Ok x) + (match b + ((Ok y) (= x y)) + ((Err _) #f))) + ((Err x) + (match b + ((Ok _) #f) + ((Err y) (= x y)))))) + (defun (eq-hash r) + (match r + ((Ok x) (hash-combine 1 (eq-hash x))) + ((Err e) (hash-combine 2 (eq-hash e)))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/fmt.bjo — formatting. Layer 1: Interpolation. Layer 2: Layout. +(import (std prelude)) +(import (std syntax-match)) +(export + ;; Exported types. + Disp Sink Block Align TableLayout TableStyle TableRules BorderChars ColumnWidth Overflow + + ;; Layer 1 — interpolation + println* + ;; Running blocks + render show render->string run-block run-blocks + ;; Settings + with-fmt with-fmt* with-pad + current-pad current-width current-ellipsis current-string-width + current-precision current-comma current-comma-sep current-decimal-sep + current-signed terminal-width + ;; The block algebra + nothing each captured width-of + ;; Vertical and horizontal space + nl fl space-to tab-to + ;; Fitting to a width + padded/left padded/right padded/both + trimmed/left trimmed/right trimmed/both + fitted/left fitted/right fitted/both + ;; Sequences + joined joined/prefix joined/suffix joined/last + ;; Numbers + numeric numeric/comma bytes percent bar sparkline + ;; Paragraphs and tables + wrapped justified columnar tabular tabular/rows + table-layout table-widths table-row table-top table-rule table-bottom + ;; A table as a literal + Table TableCell table-block show-table) + +;; Every export is documented there; the module doc is too. +(:doc #:file "fmt.bjodoc") + + +;; =========================================================================== +;; Layer 1 — interpolation +;; =========================================================================== +;; +;; What an interpolation literal can hold: one atom of each of these types. +(def/grammar Disp + (:or (: DStr string) (: DInt int) (: DDouble double) (: DChar char) (: DBool bool))) + +(: (-> Disp string)) +(defun (disp->string d) + (match d + ((DStr s) s) + ((DInt n) (int->string n)) + ((DDouble x) (double->string x)) + ((DChar c) (char->string c)) + ((DBool b) (if b "#t" "#f")))) + +;; Formats to string. +(: (-> (Vec Disp) string)) +(defun (str* ds) + (let ((b (stringbuilder-empty))) + (ignore + (fold (fun (d acc) (stringbuilder-add-string! acc (disp->string d)) acc) + b + ds)) + (stringbuilder->string b))) + +(: (-> (Vec Disp) void)) +(defun (println* ds) (displayln (str* ds))) + + +;; =========================================================================== +;; Layer 2 — layout +;; =========================================================================== + +;; Output sink. +(type (: Sink (-> string void))) + +;; Layout blocks. A vec of blocks is a block: its blocks in order. BRun runs +;; combinators. +(def/grammar Block + (:or (: BStr string) (: BInt int) (: BChar char) (: BMany (Vec Block)) + (: BRun ,(-> Sink int int)))) + +;; Computes column position. +(: (-> int string int)) +(defun (column-after col s) + (string-fold (fun (ch c) (if (char=? ch #\newline) 0 (+ c 1))) col s)) + +(: (-> Sink int Block int)) +(defun (run-block out col b) + (match b + ((BStr s) (let () (out s) (column-after col s))) + ((BInt n) (run-block out col (BStr (int->string n)))) + ((BChar c) (run-block out col (BStr (char->string c)))) + ((BMany bs) (run-blocks out col bs)) + ((BRun f) (f out col)))) + +(: (-> Sink int (Vec Block) int)) +(defun (run-blocks out col bs) + (fold (fun (b c) (run-block out c b)) col bs)) + +;; `#:sync`, and it cannot be otherwise: the sink is stored in a `Block`'s +;; `BRun` case, and a stored function's colour is its type — so no `-?->` can +;; go there and no suspending copy of this could ever await. Rendering from a +;; fiber is `render->string` plus a write you make yourself. +(: render #:sync (-> OutputPort Block void)) +(defun (render port b) + (ignore (run-block #(write-string port &1) 0 b))) + +(: (-> Block string)) +(defun (render->string b) + (let ((sb (stringbuilder-empty))) + (ignore (run-block #(stringbuilder-add-string! sb &1) 0 b)) + (stringbuilder->string sb))) + +;; Renders to current output port. `#:sync` for `render`'s reason, and stated +;; here too rather than left to follow from it: this is the exported one, and +;; what it promises a caller should be readable at it. +;; +(: show #:sync (-> Block void)) +(defun (show b) (render (parameter-ref current-output-port) b)) + + +;; --- Settings -------------------------------------------------------------- +(: (-> (Param %a) %a Block Block)) +(defun (with-fmt p v b) + (BRun (fun (out col) + (parameterize ((p v)) + (run-block out col b))))) + +;; What padding is made of. +(: (Param char)) +(def current-pad (make-parameter #\space)) + +(: (-> char Block Block)) +(defun (with-pad c b) (with-fmt current-pad c b)) + +;; `(with-fmt* ((current-width 30) (current-pad #\.)) block ...)`: with-fmt +;; for each setting, the first outermost, around the blocks in order. +(def/macro (with-fmt* form inject compare) + (syntax-match form + ((_ () body ...) + #'[,@body]) + ((_ ((p v) more ...) body ...) + #'(with-fmt ,p ,v (with-fmt* (,@more) ,@body))) + (bad (syntax-error bad "with-fmt* takes settings and blocks: (with-fmt* ((current-width 30)) block ...)")))) + +;; The right margin: what `wrapped`, `justified` and `columnar` fill up to. +(: (Param int)) +(def current-width (make-parameter 78)) + +;; Put in place of whatever a `trimmed` cut off. Empty means cut silently. +(: (Param string)) +(def current-ellipsis (make-cold-parameter "")) + +;; Calculates string width in columns. +(: (Param (-> string int))) +(def current-string-width (make-cold-parameter #(string-count &1))) + +;; --- Number settings ------------------------------------------------------- + +;; Digits after the decimal separator. Negative means as many as the number +;; needs, which is what `double->string` would have given. +(: (Param int)) +(def current-precision (make-cold-parameter -1)) + +;; Digits per group in the integer part. Zero means no grouping. +(: (Param int)) +(def current-comma (make-cold-parameter 0)) + +(: (Param char)) +(def current-comma-sep (make-cold-parameter #\,)) + +(: (Param char)) +(def current-decimal-sep (make-cold-parameter #\.)) + +;; A leading `+` on a number that has no sign of its own. +(: (Param bool)) +(def current-signed (make-cold-parameter #f)) + + +;; --- Measuring ------------------------------------------------------------- + +;; Returns width of string. +(: (-> string int)) +(defun (width-of s) + (let ((measure (parameter-ref current-string-width))) + (measure s))) + +;; Generates padding of specified width. +(: (-> int string)) +(defun (padding w) + (def* (c (parameter-ref current-pad)) + (cw (width-of (char->string c))) + (b (stringbuilder-empty))) + (when (> cw 0) + (let loop ((room w)) + (when (>= room cw) + (stringbuilder-add! b c) + (loop (- room cw))))) + (stringbuilder->string b)) + +;; Gets prefix/suffix fitting w columns. +(: (-> string int string)) +(defun (prefix-of-width s w) + (let loop ((c (string-cursor-start s)) (room w)) + (if (string-cursor-end? s c) + s + (let ((cw (width-of (char->string (string-cursor-ref s c))))) + (if (<= cw room) + (loop (string-cursor-next s c) (- room cw)) + (substring s (string-cursor-start s) c)))))) + +(: (-> string int string)) +(defun (suffix-of-width s w) + (let loop ((c (string-cursor-end s)) (room w)) + (cond + ((string-cursor=? c (string-cursor-start s)) + s) + (:def p (string-cursor-prev s c)) + (:def cw (width-of (char->string (string-cursor-ref s p)))) + ((<= cw room) (loop p (- room cw))) + (else (substring s c (string-cursor-end s)))))) + ;; (if (string-cursor=? c (string-cursor-start s)) + ;; s + ;; (let* ((p (string-cursor-prev s c)) + ;; (cw (width-of (char->string (string-cursor-ref s p))))) + ;; (if (<= cw room) + ;; (loop p (- room cw)) + ;; (substring s c (string-cursor-end s))))))) + +(: (-> int string string)) +(defun (pad/left w s) (string-append (padding (- w (width-of s))) s)) + +(: (-> int string string)) +(defun (pad/right w s) (string-append s (padding (- w (width-of s))))) + +;; Centred, with the odd column on the right — SRFI 166's rule. +(: (-> int string string)) +(defun (pad/both w s) + (let* ((missing (- w (width-of s))) + (left (/ missing 2))) + (string-append (padding left) (string-append s (padding (- missing left)))))) + +;; Trims string to w columns. +(: (-> int string string)) +(defun (trim/left w s) + (if (<= (width-of s) w) + s + (let ((e (prefix-of-width (parameter-ref current-ellipsis) w))) + (string-append e (suffix-of-width s (- w (width-of e))))))) + +(: (-> int string string)) +(defun (trim/right w s) + (if (<= (width-of s) w) + s + (let ((e (prefix-of-width (parameter-ref current-ellipsis) w))) + (string-append (prefix-of-width s (- w (width-of e))) e)))) + +;; The middle survives, and both ends say so. +(: (-> int string string)) +(defun (trim/both w s) + (if (<= (width-of s) w) + s + (let* ((e (prefix-of-width (parameter-ref current-ellipsis) (/ w 2))) + (room (- w (* 2 (width-of e)))) + (dropped (- (width-of s) (max room 0))) + (kept (suffix-of-width s (- (width-of s) (/ dropped 2))))) + (string-append e (string-append (prefix-of-width kept (max room 0)) e))))) + + +;; --- The block algebra ----------------------------------------------------- + +;; Nothing, written. What a conditional arm answers when it has nothing to say. +(: Block) +(def nothing (BRun (fun (out col) col))) + +;; Blocks in order, as one. What a vec of blocks is anyway; the name is for +;; where a function is wanted, as in (vec-map each rows). +(: (-> (Vec Block) Block)) +(defun (each bs) (BMany bs)) + +;; Renders the content and applies f. +(: (-> (-> string Block) Block Block)) +(defun (captured f b) + (BRun (fun (out col) (run-block out col (f (render->string b)))))) + + +;; --- Vertical and horizontal space ----------------------------------------- + +(: Block) +(def nl (BStr "\n")) + +;; Outputs newline if not at start of line. +(: Block) +(def fl + (BRun (fun (out col) + (if (= col 0) col (run-block out col nl))))) + +;; Pads to specified column. +(: (-> int Block)) +(defun (space-to goal) + (BRun (fun (out col) + (if (>= col goal) + col + (run-block out col (BStr (padding (- goal col)))))))) + +;; Moves to next tab stop. +(: (-> (#:width int) Block)) +(defun (tab-to #:width 8) + (BRun (fun (out col) + (run-block out col (BStr (padding (- (* width (+ 1 (/ col width))) + col))))))) + + +;; --- Fitting to a width ---------------------------------------------------- + +(: (-> int Block Block)) +(defun (padded/left w b) (captured (fun (s) (BStr (pad/left w s))) b)) + +(: (-> int Block Block)) +(defun (padded/right w b) (captured (fun (s) (BStr (pad/right w s))) b)) + +(: (-> int Block Block)) +(defun (padded/both w b) (captured (fun (s) (BStr (pad/both w s))) b)) + +(: (-> int Block Block)) +(defun (trimmed/left w b) (captured (fun (s) (BStr (trim/left w s))) b)) + +(: (-> int Block Block)) +(defun (trimmed/right w b) (captured (fun (s) (BStr (trim/right w s))) b)) + +(: (-> int Block Block)) +(defun (trimmed/both w b) (captured (fun (s) (BStr (trim/both w s))) b)) + +(: (-> int Block Block)) +(defun (fitted/left w b) + (captured (fun (s) (BStr (pad/left w (trim/left w s)))) b)) + +(: (-> int Block Block)) +(defun (fitted/right w b) + (captured (fun (s) (BStr (pad/right w (trim/right w s)))) b)) + +(: (-> int Block Block)) +(defun (fitted/both w b) + (captured (fun (s) (BStr (pad/both w (trim/both w s)))) b)) + + +;; --- Sequences ------------------------------------------------------------- + +;; Every element, with `sep` between them. +(: (-> (-> %a Block) (List %a) Block Block)) +(defun (joined f xs sep) (joined/last f f xs sep)) + +;; `sep` before each element, including the first. +(: (-> (-> %a Block) (List %a) Block Block)) +(defun (joined/prefix f xs sep) + (BRun (fun (out col) + (list-foldl (fun (x c) (run-block out (run-block out c sep) (f x))) + col + xs)))) + +;; Appends separator after elements. +(: (-> (-> %a Block) (List %a) Block Block)) +(defun (joined/suffix f xs sep) + (BRun (fun (out col) + (list-foldl (fun (x c) (run-block out (run-block out c (f x)) sep)) + col + xs)))) + +;; Joins elements with custom last element formatter. +(: (-> (-> %a Block) (-> %a Block) (List %a) Block Block)) +(defun (joined/last f last-f xs sep) + (BRun (fun (out col) + (let loop ((ys xs) (c col) (first? #t)) + (match ys + (Nil c) + ((Cons x Nil) + (run-block out (if first? c (run-block out c sep)) (last-f x))) + ((Cons x rest) + (loop rest + (run-block out (if first? c (run-block out c sep)) (f x)) + #f))))))) + + +;; --- Numbers --------------------------------------------------------------- +(import/extern + (clr-invariant (: System.Globalization.CultureInfo.InvariantCulture #:get))) + +;; `n` digits at a time from the right, `sep` between the groups. +(: (-> string int char string)) +(defun (grouped digits n sep) + (if (<= n 0) + digits + (let ((b (stringbuilder-empty))) + ;; Backwards, because the groups are counted from the right; the string + ;; comes out reversed and is turned around once at the end. + (let loop ((c (string-cursor-end digits)) (i 0)) + (unless (string-cursor=? c (string-cursor-start digits)) + (let ((p (string-cursor-prev digits c))) + (when (and (> i 0) (= 0 (% i n))) + (stringbuilder-add! b sep)) + (stringbuilder-add! b (string-cursor-ref digits p)) + (loop p (+ i 1))))) + (string-reverse (stringbuilder->string b))))) + +;; Formats number blocks. +;; +;; It is `double` rather than anything wider: an `int` crosses with +;; `(cast double n)`, and a number being formatted is on its way to a string. +(: (-> double (#:precision int) (#:comma int) (#:comma-sep char) + (#:decimal-sep char) (#:signed bool) Block)) +(defun (numeric x #:precision -1 #:comma -1 #:comma-sep #\nul + #:decimal-sep #\nul #:signed #f) + (BRun (fun (out col) + (run-block + out + col + (BStr (formatted x + (if (< precision 0) (parameter-ref current-precision) precision) + (if (< comma 0) (parameter-ref current-comma) comma) + (if (char=? comma-sep #\nul) + (parameter-ref current-comma-sep) + comma-sep) + (if (char=? decimal-sep #\nul) + (parameter-ref current-decimal-sep) + decimal-sep) + (or signed (parameter-ref current-signed)))))))) + +(: (-> double int int char char bool string)) +(defun (formatted x precision comma comma-sep decimal-sep signed) + (let* ((raw (if (< precision 0) + (.ToString x clr-invariant) + (.ToString x (string-append "F" (int->string precision)) clr-invariant))) + (negative? (string-starts-with? raw "-")) + (body (if negative? (string-drop-first raw) raw)) + (parts (split-at-point body)) + (whole (first-of parts)) + (fraction (second-of parts))) + (string-append + (if negative? "-" (if signed "+" "")) + (string-append + (grouped whole comma comma-sep) + (if (string-empty? fraction) + "" + (string-append (char->string decimal-sep) fraction)))))) + +;; The common case, and the reason `comma` is spelled that way: thousands. +(: (-> double Block)) +(defun (numeric/comma x) (numeric x #:comma 3)) + +;; The invariant culture's decimal point is `.`, so this is a split on one +;; character and not a search for whatever the machine calls a separator. +(: (-> string (Tuple string string))) +(defun (split-at-point s) + (match (string-find (fun (c) (char=? c #\.)) s) + ((Some c) (Tuple (substring s (string-cursor-start s) c) + (substring s (string-cursor-next s c) (string-cursor-end s)))) + (None (Tuple s "")))) + +(: (-> (Tuple string string) string)) +(defun (first-of t) (let (((a _) t)) a)) + +(: (-> (Tuple string string) string)) +(defun (second-of t) (let (((_ b) t)) b)) + +(: (-> string string)) +(defun (string-drop-first s) + (if (string-empty? s) + s + (substring s + (string-cursor-next s (string-cursor-start s)) + (string-cursor-end s)))) + + +;; --- Sizes, shares, bars ---------------------------------------------------- + +;; A number of bytes in the largest unit it is at least one of: `512 B`, +;; `1.2 KiB`, `3.4 GiB`. With `#:si`, the units are powers of 1000: `1.2 kB`. +(: (-> long (#:si bool) (#:precision int) Block)) +(defun (bytes n #:si #f #:precision 1) + (BRun (fun (out col) + (def step (if si 1000.0 1024.0)) + (def units (if si ["B" "kB" "MB" "GB" "TB" "PB" "EB"] ["B" "KiB" "MiB" "GiB" "TiB" "PiB" "EiB"])) + (def negative? (< n 0L)) + (def size (cast double (if negative? (- 0L n) n))) + (def text + (let lp ((x size) (u 0)) + (cond + ((= u 0) (if (< x step) (str (long->string (cast long x)) " B") (lp (/ x step) 1))) + ((and (>= x step) (< u (- (vec-length units) 1))) (lp (/ x step) (+ u 1))) + (else (str (formatted x (max precision 0) 0 #\, (parameter-ref current-decimal-sep) #f) + " " (vec-ref units u)))))) + (run-block out col (BStr (if negative? (str "-" text) text)))))) + +;; A share, written as a percentage: 0.125 is `12.5%`. +(: (-> double (#:precision int) Block)) +(defun (percent x #:precision 1) + (BRun (fun (out col) + (run-block out col + (BStr (str (formatted (* x 100.0) (max precision 0) 0 #\, + (parameter-ref current-decimal-sep) #f) + "%")))))) + +;; The eighths of a block, from one eighth to seven. +(def eighths ["▏" "▎" "▍" "▌" "▋" "▊" "▉"]) + +;; A bar `width` columns wide, filled to `fraction` of it to an eighth of a +;; column, and `empty` in the rest. A fraction outside 0 to 1 is the nearest +;; end. +(: (-> double (#:width int) (#:empty string) Block)) +(defun (bar fraction #:width 10 #:empty " ") + (BRun (fun (out col) + (def* (f (max (min fraction 1.0) 0.0)) + (total (cast int (+ (* f (cast double (* width 8))) 0.5))) + (full (/ total 8)) + (part (% total 8)) + (b (stringbuilder-empty))) + (loop (:for _ (range 0 full)) (:do (stringbuilder-add-string! b "█"))) + (when (> part 0) (stringbuilder-add-string! b (vec-ref eighths (- part 1)))) + (def used (+ full (if (> part 0) 1 0))) + (loop (:for _ (range 0 (- width used))) (:do (stringbuilder-add-string! b empty))) + (run-block out col (BStr (stringbuilder->string b)))))) + +(def levels ["▁" "▂" "▃" "▄" "▅" "▆" "▇" "█"]) + +;; The values as a line of bars of eight heights, the smallest the lowest and +;; the largest the highest. Values that are all the same are all in the +;; middle. +(: (-> (Vec double) Block)) +(defun (sparkline xs) + (BRun (fun (out col) + (def* (first (if (> (vec-length xs) 0) (vec-ref xs 0) 0.0)) + (lo (vec-fold min first xs)) + (hi (vec-fold max first xs)) + (b (stringbuilder-empty))) + (loop (:for x xs) + (:do (stringbuilder-add-string! + b + (if (<= hi lo) + (vec-ref levels 3) + (vec-ref levels (min 7 (cast int (* 8.0 (/ (- x lo) (- hi lo)))))))))) + (run-block out col (BStr (stringbuilder->string b)))))) + + +;; --- Paragraphs ------------------------------------------------------------ + +;; The runs of non-whitespace in `s`, in order. +(: (-> string (Vec string))) +(defun (words-of s) + (def out (vecbuilder-empty)) + (let lp ((c (string-cursor-start s)) (start (string-cursor-start s)) (in? #f)) + (if (string-cursor-end? s c) + ;; todo: add a (cond ...) here + (when in? (vecbuilder-add! out (substring s start c))) + (if (char-whitespace? (string-cursor-ref s c)) + (let () + (when in? (vecbuilder-add! out (substring s start c))) + (lp (string-cursor-next s c) (string-cursor-next s c) #f)) + (lp (string-cursor-next s c) (if in? start c) #t)))) + (vecbuilder->vec out)) + +;; Packs words into lines. +(: (-> (Vec string) int int (List (List string)))) +(defun (packed ws first width) + (let lp ((i 0) (line Nil) (used 0) (budget first) (lines Nil)) + (if (>= i (vec-length ws)) + (list-reverse (if (list-empty? line) lines (Cons (list-reverse line) lines))) + (let* ((w (vec-ref ws i)) + (needed (if (= used 0) (width-of w) (+ used 1 (width-of w))))) + (if (or (= used 0) (<= needed budget)) + (lp (+ i 1) (Cons w line) needed budget lines) + (lp i Nil 0 width (Cons (list-reverse line) lines))))))) + +;; The words with a single pad character between them. +(: (-> (List string) string)) +(defun (spaced ws) + (def b (stringbuilder-empty)) + (def gap (padding 1)) + (loop (:with first? #t #f) + (:for w ws) + (:do + (unless first? (stringbuilder-add-string! b gap)) + (stringbuilder-add-string! b w))) + (stringbuilder->string b)) + +;; Stretches spaced words to budget. +(: (-> (List string) int string)) +(defun (stretched ws budget) + (let ((gaps (- (list-length ws) 1))) + (if (<= gaps 0) + (spaced ws) + (let* ((text (list-foldl (fun (w acc) (+ acc (width-of w))) 0 ws)) + (room (max (- budget text) gaps)) + (base (/ room gaps)) + (extra (% room gaps)) + (b (stringbuilder-empty))) + ;; TODO: rewrite as a {folding ...} + (ignore (list-foldl (fun (w i) + (when (> i 0) + (stringbuilder-add-string! + b + (padding (+ base (if (<= i extra) 1 0))))) + (stringbuilder-add-string! b w) + (+ i 1)) + 0 + ws)) + (stringbuilder->string b))))) + +(: (-> (Vec string) bool Block)) +(defun (wrap ws justify?) + (BRun (fun (out col) + (let* ((width (parameter-ref current-width)) + (lines (packed ws (- width col) width))) + (let loop ((ls lines) (c col) (first? #t)) + (match ls + (Nil c) + ((Cons line rest) + ;; Todo: rewrite using (def ...) + (let* ((c1 (if first? c (run-block out c nl))) + (budget (if first? (- width col) width)) + (text (if (and justify? (not (list-empty? rest))) + (stretched line budget) + (spaced line)))) + (loop rest (run-block out c1 (BStr text)) #f))))))))) + +;; Filled to the right margin, ragged on the right. +(: (-> Block Block)) +(defun (wrapped b) (captured (fun (s) (wrap (words-of s) #f)) b)) + +;; Filled to the right margin and flush against both, the last line excepted. +(: (-> Block Block)) +(defun (justified b) (captured (fun (s) (wrap (words-of s) #t)) b)) + + +;; --- Tables ---------------------------------------------------------------- +;; +;; They differ only in where the widths come from: `columnar` divides +;; `current-width` evenly and cuts whatever does not fit, `tabular` gives each +;; column the width its own content asks for and cuts nothing. + +;; The lines of `s`, without the empty one a trailing newline leaves behind. +(: (-> string (Vec string))) +(defun (lines-of s) + (def* (parts (string-split s "\n")) + (n (vec-length parts))) + (if (and (> n 1) (string-empty? (vec-ref parts (- n 1)))) + (vec-slice parts 0 (- n 1)) + parts)) + +;; Where a cell sits in its column. `(on #\.)` puts a character of every cell +;; in one column, as the decimal point is in a column of numbers. +(def/grammar Align + (:or (: AlignLeft 'left) (: AlignRight 'right) (: AlignCentre 'centre) (: AlignOn (on char)))) + +;; The alignment of column `i`: left unless said otherwise. +(: (-> (Vec Align) int Align)) +(defun (align-at aligns i) + (if (< i (vec-length aligns)) (vec-ref aligns i) AlignLeft)) + +;; A cell fitted to its column. A cut keeps the end an alignment leans on, +;; as `fitted/left` and the rest do. The last column is not padded on the +;; right: there is nothing after it to line up. +(: (-> Align int bool bool string string)) +(defun (fit-cell a w cut? last? text) + (def s (if cut? + (match a + (AlignLeft (trim/right w text)) + (AlignRight (trim/left w text)) + (AlignCentre (trim/both w text)) + ((AlignOn _) (trim/left w text))) + text)) + (match a + (AlignLeft (if last? s (pad/right w s))) + (AlignRight (pad/left w s)) + (AlignCentre (if last? + (string-append (padding (/ (- w (width-of s)) 2)) s) + (pad/both w s))) + ;; Without the widths on either side of the character, which these + ;; tables do not measure, the cell is aligned to the right. + ((AlignOn _) (pad/left w s)))) + +(: (-> (Vec (Vec string)) (Vec int) (Vec Align) string bool Block)) +(defun (laid-out cells widths aligns sep cut?) + (BRun (fun (out col) + (def rows (vec-fold (fun (ls acc) (max acc (vec-length ls))) 0 cells)) + (let lp ((r 0) (c (run-block out col fl))) + (if (>= r rows) + c + (lp (+ r 1) + (run-block out + (run-block out c (BStr (row-of cells widths aligns sep cut? r))) + nl))))))) + +;; Line `r` of a row. Empty cells at the end of it are left out, separators +;; and all, so that no line ends in blanks. +(: (-> (Vec (Vec string)) (Vec int) (Vec Align) string bool int string)) +(defun (row-of cells widths aligns sep cut? r) + (def* (b (stringbuilder-empty)) + (texts (vec-map (fun (ls) (if (< r (vec-length ls)) (vec-ref ls r) "")) cells)) + (n (let lp ((n (vec-length texts))) + (if (and (> n 0) (string-empty? (vec-ref texts (- n 1)))) (lp (- n 1)) n)))) + (let lp ((i 0)) + (when (< i n) + (when (> i 0) (stringbuilder-add-string! b sep)) + (stringbuilder-add-string! + b + (fit-cell (align-at aligns i) (vec-ref widths i) cut? (= i (- n 1)) (vec-ref texts i))) + (lp (+ i 1)))) + (stringbuilder->string b)) + +;; Each cell's lines. +(: (-> (Vec (Vec Block)) (Vec (Vec string)))) +(defun (rendered-cells cells) + (vec-map (fun (c) (lines-of (render->string c))) cells)) + +;; The lines of a row's cells, a block each. +(: (-> (Vec Block) (Vec (Vec string)))) +(defun (rendered-row row) + (vec-map (fun (c) (lines-of (render->string [c]))) row)) + +(: (-> (Vec string) int)) +(defun (widest ls) (vec-fold (fun (l acc) (max acc (width-of l))) 0 ls)) + +;; Renders columns. +(: (-> (Vec (Vec Block)) (#:sep string) (#:align (Vec Align)) Block)) +(defun (columnar cols #:sep " " #:align []) + (BRun (fun (out col) + (let* ((cells (rendered-cells cols)) + (n (vec-length cells)) + (room (- (parameter-ref current-width) + (* (width-of sep) (max (- n 1) 0)))) + (each-width (if (> n 0) (max (/ room n) 0) 0))) + (run-block out + col + (laid-out cells (vec-map (fun (ls) each-width) cells) align sep #t)))))) + +;; --- Borders ----------------------------------------------------------------- +;; +;; A bordered table is lines of three kinds: the cells of a row, a horizontal +;; line made of a fill repeated to each column's width, and the edges and +;; joins around both. `BorderChars` is every piece of them, so a style is a +;; value and a custom one is written as one. + +(type (: BorderChars (Record (: top-left string) (: top string) (: top-join string) (: top-right string) + (: left string) (: column string) (: right string) + (: rule-left string) (: rule string) (: rule-join string) (: rule-right string) + (: bottom-left string) (: bottom string) (: bottom-join string) + (: bottom-right string) + (: marks bool)))) + +(def/grammar TableStyle + (:or (: StyleNone 'none) (: StyleSimple 'simple) (: StyleAscii 'ascii) (: StyleLight 'light) + (: StyleHeavy 'heavy) (: StyleDouble 'double) (: StyleRounded 'rounded) + (: StyleMarkdown 'markdown) (: StyleCustom ,BorderChars))) + +;; Which horizontal rules a table has inside it: under the header, between +;; every two rows, or none. +(def/grammar TableRules + (:or (: RulesHeader 'header) (: RulesEvery 'every) (: RulesNone 'none))) + +;; How wide a column is: as its content asks, a number of columns, at least +;; or at most a number, or a share of what the other columns leave of +;; `current-width`. +(def/grammar ColumnWidth + (:or (: WidthAuto 'auto) (: WidthFixed int) (: WidthAtLeast (at-least int)) + (: WidthAtMost (at-most int)) (: WidthShare (share int)))) + +;; What a cell wider than its column does: pushes the rest of its row out, +;; is cut, or is wrapped onto more lines. A table that cuts or wraps is made +;; to fit `current-width`. +(def/grammar Overflow + (:or (: OverflowPush 'push) (: OverflowCut 'cut) (: OverflowWrap 'wrap))) + +;; `n` spaces, whatever the pad character is: a border's spacing is part of +;; the border. +(: (-> int string)) +(defun (spaces n) + (def b (stringbuilder-empty)) + (loop (:for _ (range 0 n)) (:do (stringbuilder-add! b #\space))) + (stringbuilder->string b)) + +;; A box drawn with the line `h`, the line `v`, and the nine places they meet, +;; as on a keypad: the top row, the middle row and the bottom row, each left, +;; middle and right. A cell has a space on each side of it. +(: (-> string string string string string string string string string string string BorderChars)) +(defun (box-chars h v tl tj tr ml mj mr bl bj br) + (BorderChars (top-left (str tl h)) (top h) (top-join (str h tj h)) (top-right (str h tr)) + (left (str v " ")) (column (str " " v " ")) (right (str " " v)) + (rule-left (str ml h)) (rule h) (rule-join (str h mj h)) (rule-right (str h mr)) + (bottom-left (str bl h)) (bottom h) (bottom-join (str h bj h)) (bottom-right (str h br)) + (marks #f))) + +;; No edges: `sep` between the columns, and a rule of `fill` under the header +;; when `fill` is not empty. +(: (-> string string BorderChars)) +(defun (open-chars sep fill) + (BorderChars (top-left "") (top "") (top-join "") (top-right "") + (left "") (column sep) (right "") + (rule-left "") (rule fill) (rule-join (spaces (width-of sep))) (rule-right "") + (bottom-left "") (bottom "") (bottom-join "") (bottom-right "") + (marks #f))) + +;; The pieces a style is drawn with. `sep` is what `StyleNone` puts between +;; the columns. +(: (-> TableStyle string BorderChars)) +(defun (border-chars style sep) + (match style + (StyleNone (open-chars sep "")) + (StyleSimple (open-chars " " "─")) + (StyleAscii (box-chars "-" "|" "+" "+" "+" "+" "+" "+" "+" "+" "+")) + (StyleLight (box-chars "─" "│" "┌" "┬" "┐" "├" "┼" "┤" "└" "┴" "┘")) + (StyleHeavy (box-chars "━" "┃" "┏" "┳" "┓" "┣" "╋" "┫" "┗" "┻" "┛")) + (StyleDouble (box-chars "═" "║" "╔" "╦" "╗" "╠" "╬" "╣" "╚" "╩" "╝")) + (StyleRounded (box-chars "─" "│" "╭" "┬" "╮" "├" "┼" "┤" "╰" "┴" "╯")) + (StyleMarkdown + (BorderChars (top-left "") (top "") (top-join "") (top-right "") + (left "| ") (column " | ") (right " |") + (rule-left "| ") (rule "-") (rule-join " | ") (rule-right " |") + (bottom-left "") (bottom "") (bottom-join "") (bottom-right "") + (marks #t))) + ((StyleCustom chars) chars))) + +;; Whether a table is laid out as the tables without borders always were: +;; no borders, no header, no column aligned on a character, which only the +;; bordered layout measures, and no width rules or overflow. +(: (-> TableStyle (Vec Block) (Vec Align) (Vec ColumnWidth) Overflow bool)) +(defun (plain-table? style titles aligns widths overflow) + (and (match style (StyleNone #t) (_ #f)) + (= (vec-length titles) 0) + (not (any? (fun (a) (match a ((AlignOn _) #t) (_ #f))) aligns)) + (= (vec-length widths) 0) + (match overflow (OverflowPush #t) (_ #f)))) + +;; A column's width, and for one aligned on a character, the widest part of a +;; line before the character and the widest from it on. +(type (: ColumnFit (Record (: width int) (: before int) (: after int)))) + +;; A line cut where `c` first is: the part before it, and the rest from it +;; on. A line without one is all before. +(: (-> char string (Tuple string string))) +(defun (split-on c s) + (match (string-find (fun (x) (char=? x c)) s) + ((Some at) (Tuple (substring s (string-cursor-start s) at) (substring s at (string-cursor-end s)))) + (None (Tuple s "")))) + +(: (-> (Vec (Vec string)) int (Vec string))) +(defun (cell-at cells i) + (if (< i (vec-length cells)) (vec-ref cells i) [])) + +;; Every line of column `i`, in all the rows. +(: (-> (Vec (Vec (Vec string))) int (Vec string))) +(defun (column-lines rows i) + (def b (vecbuilder-empty)) + (loop (:for row rows) + (:do (loop (:for l (cell-at row i)) (:do (vecbuilder-add! b l))))) + (vecbuilder->vec b)) + +(: (-> Align (Vec string) (Vec string) bool ColumnFit)) +(defun (column-fit a title lines marks) + (def* (parts (match a + ((AlignOn c) + (vec-fold (fun (l acc) + (def (Tuple b af) (split-on c l)) + (Tuple (max (fst acc) (width-of b)) (max (snd acc) (width-of af)))) + (Tuple 0 0) lines)) + (_ (Tuple 0 0)))) + (content (match a + ((AlignOn _) (+ (fst parts) (snd parts))) + (_ (widest lines))))) + (ColumnFit (width (max content (max (widest title) (if marks 3 0)))) + (before (fst parts)) + (after (snd parts)))) + +(: (-> (Vec Align) (Vec (Vec string)) (Vec (Vec (Vec string))) bool (Vec ColumnFit))) +(defun (column-fits aligns titles rows marks) + (def n (vec-fold (fun (r acc) (max acc (vec-length r))) (vec-length titles) rows)) + {vecing (column-fit (align-at aligns i) (cell-at titles i) (column-lines rows i) marks) + (:for i (range 0 n))}) + +;; A title is aligned as its column is, and one over a column aligned on a +;; character to the right, where the numbers under it end. +(: (-> Align Align)) +(defun (title-align a) + (match a + ((AlignOn _) AlignRight) + (_ a))) + +;; One line of a cell, as wide as its column. `open-end?` is a last cell with +;; nothing after it, which is not padded on the right. +(: (-> Align ColumnFit bool string string)) +(defun (fit-line a fit open-end? text) + (def w (record-ref fit width)) + (match a + ((AlignOn c) + (def* (parts (split-on c text)) + (before (fst parts)) + (span (+ (record-ref fit before) (record-ref fit after))) + (core (str (padding (- (record-ref fit before) (width-of before))) before (snd parts)))) + (str (padding (- w span)) (if open-end? core (pad/right span core)))) + (_ (fit-cell a w #f open-end? text)))) + +;; One line of a row: the edges, the separators and each cell's line `k`. +;; Without edges, empty cells at the end are left out and the last one is not +;; padded, as in a table without borders. +(: (-> BorderChars (Vec Align) (Vec ColumnFit) (Vec string) string)) +(defun (row-line chars aligns fits texts) + (def* (open? (and (string-empty? (record-ref chars left)) (string-empty? (record-ref chars right)))) + (n (if open? + (let lp ((n (vec-length texts))) + (if (and (> n 0) (string-empty? (vec-ref texts (- n 1)))) (lp (- n 1)) n)) + (vec-length fits))) + (b (stringbuilder-empty))) + (stringbuilder-add-string! b (record-ref chars left)) + (loop (:for i (range 0 n)) + (:do (when (> i 0) (stringbuilder-add-string! b (record-ref chars column))) + (stringbuilder-add-string! + b + (fit-line (align-at aligns i) (vec-ref fits i) (and open? (= i (- n 1))) + (if (< i (vec-length texts)) (vec-ref texts i) ""))))) + (stringbuilder-add-string! b (record-ref chars right)) + (stringbuilder->string b)) + +;; The lines of a row, as many as its tallest cell has. +(: (-> BorderChars (Vec Align) (Vec ColumnFit) (Vec (Vec string)) (Vec string))) +(defun (row-lines chars aligns fits cells) + (def height (vec-fold (fun (c acc) (max acc (vec-length c))) 1 cells)) + {vecing (row-line chars aligns fits + (vec-map (fun (c) (if (< k (vec-length c)) (vec-ref c k) "")) cells)) + (:for k (range 0 height))}) + +;; `fill` repeated to `w` columns. +(: (-> string int string)) +(defun (repeated fill w) + (def* (fw (width-of fill)) + (b (stringbuilder-empty))) + (when (> fw 0) + (loop (:for _ (range 0 (/ w fw))) (:do (stringbuilder-add-string! b fill)))) + (stringbuilder->string b)) + +;; A column's part of the rule under a Markdown header, which says how the +;; column is aligned. +(: (-> Align int string)) +(defun (marked a w) + (match a + (AlignLeft (str ":" (repeated "-" (- w 1)))) + (AlignCentre (str ":" (repeated "-" (- w 2)) ":")) + (_ (str (repeated "-" (- w 1)) ":")))) + +;; A horizontal line: `fill` across each column, `join` where the columns +;; meet, and the two ends. +(: (-> string string string string bool (Vec Align) (Vec ColumnFit) string)) +(defun (hline lft fill join rgt marks aligns fits) + (def b (stringbuilder-empty)) + (stringbuilder-add-string! b lft) + (loop (:for i (range 0 (vec-length fits))) + (:do (when (> i 0) (stringbuilder-add-string! b join)) + (def w (record-ref (vec-ref fits i) width)) + (stringbuilder-add-string! b (if marks (marked (align-at aligns i) w) (repeated fill w))))) + (stringbuilder-add-string! b rgt) + (stringbuilder->string b)) + +(: (-> BorderChars (Vec Align) (Vec ColumnFit) string)) +(defun (top-line chars aligns fits) + (hline (record-ref chars top-left) (record-ref chars top) (record-ref chars top-join) + (record-ref chars top-right) #f aligns fits)) + +(: (-> BorderChars bool (Vec Align) (Vec ColumnFit) string)) +(defun (rule-line chars marks aligns fits) + (hline (record-ref chars rule-left) (record-ref chars rule) (record-ref chars rule-join) + (record-ref chars rule-right) marks aligns fits)) + +(: (-> BorderChars (Vec Align) (Vec ColumnFit) string)) +(defun (bottom-line chars aligns fits) + (hline (record-ref chars bottom-left) (record-ref chars bottom) (record-ref chars bottom-join) + (record-ref chars bottom-right) #f aligns fits)) + +;; The lines, each ended by a newline, starting on a fresh line. +(: (-> (Vec string) Block)) +(defun (lines-block lines) + (BRun (fun (out col) + (vec-fold (fun (l c) (run-block out (run-block out c (BStr l)) nl)) (run-block out col fl) lines)))) + +;; --- Column widths and overflow ---------------------------------------------- + +(: (-> (Vec ColumnWidth) int ColumnWidth)) +(defun (width-rule-at rules i) + (if (< i (vec-length rules)) (vec-ref rules i) WidthAuto)) + +(: (-> ColumnWidth bool)) +(defun (shared? rule) + (match rule ((WidthShare _) #t) (_ #f))) + +;; The narrowest a column may be made to fit the table into its width. +(: (-> ColumnWidth int)) +(defun (narrowest rule) + (match rule + ((WidthFixed n) (max n 1)) + ((WidthAtLeast n) (max n 1)) + (_ 1))) + +;; The width of each column: its content's, `natural`, as its rule says, with +;; the shares dividing what the others leave of `room`. When `fit?`, the +;; widest column that may shrink loses a column until the table fits `room`. +(: (-> (Vec int) (Vec ColumnWidth) int bool (Vec int))) +(defun (settled-widths natural rules room fit?) + (def* (n (vec-length natural)) + (base {vecing (match (width-rule-at rules i) + ((WidthFixed k) (max k 1)) + ((WidthAtLeast k) (max (vec-ref natural i) k)) + ((WidthAtMost k) (max (min (vec-ref natural i) k) 1)) + ((WidthShare _) 0) + (WidthAuto (vec-ref natural i))) + (:for i (range 0 n))}) + (shares (loop (:for i (range 0 n)) + (:acc (summing (match (width-rule-at rules i) ((WidthShare s) (max s 1)) (_ 0))))))) + (defun (shrinkable? ws i) + (and (not (shared? (width-rule-at rules i))) + (match (width-rule-at rules i) ((WidthFixed _) #f) (_ #t)) + (> (vec-ref ws i) (narrowest (width-rule-at rules i))))) + (def* (fitted (if (not fit?) + base + (let lp ((ws base)) + (def used (vec-fold + 0 ws)) + (def wanted (+ used (if (> shares 0) n 0))) + (def widest-i + (loop (:for i (range 0 n)) + (:when (shrinkable? ws i)) + (:acc best (folding -1 (if (or (< best 0) (> (vec-ref ws i) (vec-ref ws best))) i best))))) + (if (or (<= wanted room) (< widest-i 0)) + ws + (lp (vec-set ws widest-i (- (vec-ref ws widest-i) 1)))))))) + (if (= shares 0) + fitted + (let () + (def left (max (- room (vec-fold + 0 fitted)) 0)) + {vecing (match (width-rule-at rules i) + ((WidthShare s) (max (/ (* left (max s 1)) shares) 1)) + (_ (vec-ref fitted i))) + (:for i (range 0 n))}))) + +;; `word` in pieces no wider than `w`. A character wider than `w` is a piece +;; of its own. +(: (-> int string (List string))) +(defun (chopped w word) + (let lp ((c (string-cursor-start word)) (start (string-cursor-start word)) (used 0) (acc Nil)) + (cond + ((string-cursor-end? word c) + (list-reverse (if (string-cursor=? start c) acc (Cons (substring word start c) acc)))) + (:def cw (width-of (char->string (string-cursor-ref word c)))) + ((and (> used 0) (> (+ used cw) w)) (lp c c 0 (Cons (substring word start c) acc))) + (else (lp (string-cursor-next word c) start (+ used cw) acc))))) + +;; A line broken at its spaces into lines no wider than `w`. +(: (-> int string (Vec string))) +(defun (wrapped-line w line) + (if (<= (width-of line) w) + [line] + (let () + (def pieces (vecbuilder-empty)) + (loop (:for word (words-of line)) + (:do (loop (:for p (chopped w word)) (:do (vecbuilder-add! pieces p))))) + (list->vec (list-map spaced (packed (vecbuilder->vec pieces) w w)))))) + +;; A cell's lines made to fit a column of `w`, as `overflow` says. +(: (-> Overflow Align int (Vec string) (Vec string))) +(defun (overflowed overflow a w lines) + (match (Tuple overflow a) + ((Tuple _ (AlignOn _)) lines) + ((Tuple OverflowPush _) lines) + ((Tuple OverflowCut AlignLeft) (vec-map (fun (l) (trim/right w l)) lines)) + ((Tuple OverflowCut AlignCentre) (vec-map (fun (l) (trim/both w l)) lines)) + ((Tuple OverflowCut _) (vec-map (fun (l) (trim/left w l)) lines)) + ((Tuple OverflowWrap _) + (vec-fold (fun (l acc) (vec-append acc (wrapped-line w l))) [] lines)))) + +;; A row's cells made to fit their columns. +(: (-> Overflow (Vec Align) (Vec int) (Vec (Vec string)) (Vec (Vec string)))) +(defun (overflowed-row overflow aligns widths cells) + {vecing (overflowed overflow (align-at aligns i) + (if (< i (vec-length widths)) (vec-ref widths i) 1) + (vec-ref cells i)) + (:for i (range 0 (vec-length cells)))}) + +;; How many columns the edges and separators of a row of `n` cells take. +(: (-> BorderChars int int)) +(defun (border-width chars n) + (+ (width-of (record-ref chars left)) + (+ (width-of (record-ref chars right)) + (* (max (- n 1) 0) (width-of (record-ref chars column)))))) + +;; A table with borders, its cells rendered already: a header of `titles` +;; when there are any, and `rows`. A horizontal line is drawn where its fill +;; is not empty. The columns are as wide as `widths` says, and a cell wider +;; than its column does what `overflow` says. +(: (-> BorderChars (Vec Align) (Vec (Vec string)) (Vec (Vec (Vec string))) TableRules + (Vec ColumnWidth) Overflow Block)) +(defun (bordered chars aligns titles rows rules widths overflow) + (def* (marks (record-ref chars marks)) + (natural (column-fits aligns titles rows marks)) + (fit? (match overflow (OverflowPush #f) (_ #t))) + (settled (vec-map (fun (w) (if marks (max w 3) w)) + (settled-widths (vec-map (fun (f) (record-ref f width)) natural) widths + (- (parameter-ref current-width) + (border-width chars (vec-length natural))) + fit?))) + (fits {vecing (ColumnFit (width (vec-ref settled i)) + (before (record-ref (vec-ref natural i) before)) + (after (record-ref (vec-ref natural i) after))) + (:for i (range 0 (vec-length natural)))}) + (title-aligns {vecing (title-align (align-at aligns i)) (:for i (range 0 (vec-length fits)))}) + (titles (overflowed-row overflow title-aligns settled titles)) + (rows (vec-map (fun (row) (overflowed-row overflow aligns settled row)) rows)) + (ruled? (not (string-empty? (record-ref chars rule)))) + (under-header? (match rules (RulesNone #f) (_ #t))) + (between-rows? (match rules (RulesEvery #t) (_ #f))) + (lines (vecbuilder-empty))) + (defun (add-all ls) (loop (:for l ls) (:do (vecbuilder-add! lines l)))) + (unless (string-empty? (record-ref chars top)) + (vecbuilder-add! lines (top-line chars aligns fits))) + (when (> (vec-length titles) 0) + (add-all (row-lines chars title-aligns fits titles)) + (when (and ruled? under-header?) + (vecbuilder-add! lines (rule-line chars marks aligns fits)))) + (loop (:for (Tuple r row) (enumerate rows)) + (:do (when (and (> r 0) ruled? (not marks) between-rows?) + (vecbuilder-add! lines (rule-line chars #f aligns fits))) + (add-all (row-lines chars aligns fits row)))) + (unless (string-empty? (record-ref chars bottom)) + (vecbuilder-add! lines (bottom-line chars aligns fits))) + (lines-block (vecbuilder->vec lines))) + +;; Columns of lines as rows of one-line cells: the `r`th line of each column +;; is row `r`. +(: (-> (Vec (Vec string)) (Vec (Vec (Vec string))))) +(defun (columns->rows cols) + (def height (vec-fold (fun (c acc) (max acc (vec-length c))) 0 cols)) + {vecing (vec-map (fun (c) (if (< r (vec-length c)) [(vec-ref c r)] [])) cols) + (:for r (range 0 height))}) + +(: (-> (Vec (Vec Block)) (#:sep string) (#:align (Vec Align)) (#:style TableStyle) + (#:titles (Vec Block)) (#:rules TableRules) (#:widths (Vec ColumnWidth)) (#:overflow Overflow) + Block)) +(defun (tabular cols #:sep " " #:align [] #:style StyleNone #:titles [] #:rules RulesHeader + #:widths [] #:overflow OverflowPush) + (BRun (fun (out col) + (def cells (rendered-cells cols)) + (if (plain-table? style titles align widths overflow) + (run-block out col (laid-out cells (vec-map widest cells) align sep #f)) + (run-block out col (bordered (border-chars style sep) align (rendered-row titles) + (columns->rows cells) rules widths overflow)))))) + +;; --- Tables a row at a time -------------------------------------------------- +;; +;; A table whose rows are known one by one: the widths are settled first, +;; and each row is then laid out on its own, so it can be written the moment +;; it is known. + +(type (: TableLayout (Record (: widths (Vec int)) + (: aligns (Vec Align)) + (: sep string) + (: style TableStyle)))) + +(: (-> (Vec int) (#:sep string) (#:align (Vec Align)) (#:style TableStyle) TableLayout)) +(defun (table-layout widths #:sep " " #:align [] #:style StyleNone) + (TableLayout (widths widths) (aligns align) (sep sep) (style style))) + +;; A layout's columns as fits. The widths on either side of a character are +;; not known a row at a time, so a column aligned on one is aligned right. +(: (-> TableLayout (Vec Align))) +(defun (layout-aligns layout) + (def aligns (record-ref layout aligns)) + {vecing (title-align (align-at aligns i)) (:for i (range 0 (vec-length (record-ref layout widths))))}) + +(: (-> TableLayout (Vec ColumnFit))) +(defun (layout-fits layout) + (vec-map (fun (w) (ColumnFit (width w) (before 0) (after 0))) (record-ref layout widths))) + +(: (-> TableLayout BorderChars)) +(defun (layout-chars layout) + (border-chars (record-ref layout style) (record-ref layout sep))) + +;; The line above a table laid out a row at a time, or nothing for a style +;; without one. +(: (-> TableLayout Block)) +(defun (table-top layout) + (def chars (layout-chars layout)) + (if (string-empty? (record-ref chars top)) + nothing + (lines-block [(top-line chars (layout-aligns layout) (layout-fits layout))]))) + +;; A rule across the table: under a header row, or between two rows. +(: (-> TableLayout Block)) +(defun (table-rule layout) + (def chars (layout-chars layout)) + (if (string-empty? (record-ref chars rule)) + nothing + (lines-block [(rule-line chars (record-ref chars marks) (layout-aligns layout) (layout-fits layout))]))) + +;; The line under a table laid out a row at a time. +(: (-> TableLayout Block)) +(defun (table-bottom layout) + (def chars (layout-chars layout)) + (if (string-empty? (record-ref chars bottom)) + nothing + (lines-block [(bottom-line chars (layout-aligns layout) (layout-fits layout))]))) + +;; The width each column of these rows asks for: its widest line. +(: (-> (Vec (Vec Block)) (Vec int))) +(defun (table-widths rows) + (vec-fold (fun (row acc) + (def ws (vec-map widest (rendered-row row))) + (def n (max (vec-length acc) (vec-length ws))) + {vecing (max (if (< i (vec-length acc)) (vec-ref acc i) 0) + (if (< i (vec-length ws)) (vec-ref ws i) 0)) + (:for i (range 0 n))}) + [] rows)) + +;; One row, laid out to `layout`. A cell wider than its column is not cut: +;; it pushes the rest of its row out. Cells past the layout's columns get +;; no width and are left as they are. +;; +;; With a style that has borders, a row is drawn between them, every cell +;; padded to its column, and a cell past the layout's columns is left out. +(: (-> TableLayout (Vec Block) Block)) +(defun (table-row layout row) + (BRun (fun (out col) + (def cells (rendered-row row)) + (if (plain-table? (record-ref layout style) [] [] [] OverflowPush) + (let () + (def ws (record-ref layout widths)) + (def widths {vecing (if (< i (vec-length ws)) (vec-ref ws i) 0) + (:for i (range 0 (vec-length cells)))}) + (run-block out col (laid-out cells widths (record-ref layout aligns) (record-ref layout sep) #f))) + (run-block out col (lines-block (row-lines (layout-chars layout) (layout-aligns layout) + (layout-fits layout) cells))))))) + +;; `tabular`, given rows rather than columns. +(: (-> (Vec (Vec Block)) (#:sep string) (#:align (Vec Align)) (#:style TableStyle) + (#:titles (Vec Block)) (#:rules TableRules) (#:widths (Vec ColumnWidth)) (#:overflow Overflow) + Block)) +(defun (tabular/rows rows #:sep " " #:align [] #:style StyleNone #:titles [] #:rules RulesHeader + #:widths [] #:overflow OverflowPush) + ;; Measured when it runs, where the settings it is run under are bound. + (BRun (fun (out col) + (if (plain-table? style titles align widths overflow) + (let () + (def layout (table-layout (table-widths rows) #:sep sep #:align align)) + (run-blocks out col (vec-map (fun (row) (table-row layout row)) rows))) + (run-block out col (bordered (border-chars style sep) align (rendered-row titles) + (vec-map rendered-row rows) rules widths overflow)))))) + +;; --- Width in a terminal --------------------------------------------------- +;; +;; The code points whose width in a terminal is not one column, as triples of +;; the first, the last and the width: 0 for combining marks, format and +;; control characters and the skin tone modifiers, which join what is before +;; them, and 2 for East Asian Wide and Fullwidth characters and the regional +;; indicators. Generated from Python's `unicodedata` (Unicode 16.0) and checked +;; in, as `(std rx)` checks in its tables. +(: (Array int)) +(def width-ranges + #[ + 0 31 0 127 159 0 173 173 0 768 879 0 1155 1161 0 1425 1469 0 1471 1471 0 1473 1474 0 + 1476 1477 0 1479 1479 0 1536 1541 0 1552 1562 0 1564 1564 0 1611 1631 0 1648 1648 0 + 1750 1757 0 1759 1764 0 1767 1768 0 1770 1773 0 1807 1807 0 1809 1809 0 1840 1866 0 + 1958 1968 0 2027 2035 0 2045 2045 0 2070 2073 0 2075 2083 0 2085 2087 0 2089 2093 0 + 2137 2139 0 2192 2193 0 2199 2207 0 2250 2306 0 2362 2362 0 2364 2364 0 2369 2376 0 + 2381 2381 0 2385 2391 0 2402 2403 0 2433 2433 0 2492 2492 0 2497 2500 0 2509 2509 0 + 2530 2531 0 2558 2558 0 2561 2562 0 2620 2620 0 2625 2626 0 2631 2632 0 2635 2637 0 + 2641 2641 0 2672 2673 0 2677 2677 0 2689 2690 0 2748 2748 0 2753 2757 0 2759 2760 0 + 2765 2765 0 2786 2787 0 2810 2815 0 2817 2817 0 2876 2876 0 2879 2879 0 2881 2884 0 + 2893 2893 0 2901 2902 0 2914 2915 0 2946 2946 0 3008 3008 0 3021 3021 0 3072 3072 0 + 3076 3076 0 3132 3132 0 3134 3136 0 3142 3144 0 3146 3149 0 3157 3158 0 3170 3171 0 + 3201 3201 0 3260 3260 0 3263 3263 0 3270 3270 0 3276 3277 0 3298 3299 0 3328 3329 0 + 3387 3388 0 3393 3396 0 3405 3405 0 3426 3427 0 3457 3457 0 3530 3530 0 3538 3540 0 + 3542 3542 0 3633 3633 0 3636 3642 0 3655 3662 0 3761 3761 0 3764 3772 0 3784 3790 0 + 3864 3865 0 3893 3893 0 3895 3895 0 3897 3897 0 3953 3966 0 3968 3972 0 3974 3975 0 + 3981 3991 0 3993 4028 0 4038 4038 0 4141 4144 0 4146 4151 0 4153 4154 0 4157 4158 0 + 4184 4185 0 4190 4192 0 4209 4212 0 4226 4226 0 4229 4230 0 4237 4237 0 4253 4253 0 + 4352 4447 2 4448 4607 0 4957 4959 0 5906 5908 0 5938 5939 0 5970 5971 0 6002 6003 0 + 6068 6069 0 6071 6077 0 6086 6086 0 6089 6099 0 6109 6109 0 6155 6159 0 6277 6278 0 + 6313 6313 0 6432 6434 0 6439 6440 0 6450 6450 0 6457 6459 0 6679 6680 0 6683 6683 0 + 6742 6742 0 6744 6750 0 6752 6752 0 6754 6754 0 6757 6764 0 6771 6780 0 6783 6783 0 + 6832 6862 0 6912 6915 0 6964 6964 0 6966 6970 0 6972 6972 0 6978 6978 0 7019 7027 0 + 7040 7041 0 7074 7077 0 7080 7081 0 7083 7085 0 7142 7142 0 7144 7145 0 7149 7149 0 + 7151 7153 0 7212 7219 0 7222 7223 0 7376 7378 0 7380 7392 0 7394 7400 0 7405 7405 0 + 7412 7412 0 7416 7417 0 7616 7679 0 8203 8207 0 8234 8238 0 8288 8292 0 8294 8303 0 + 8400 8432 0 8986 8987 2 9001 9002 2 9193 9196 2 9200 9200 2 9203 9203 2 9725 9726 2 + 9748 9749 2 9776 9783 2 9800 9811 2 9855 9855 2 9866 9871 2 9875 9875 2 9889 9889 2 + 9898 9899 2 9917 9918 2 9924 9925 2 9934 9934 2 9940 9940 2 9962 9962 2 9970 9971 2 + 9973 9973 2 9978 9978 2 9981 9981 2 9989 9989 2 9994 9995 2 10024 10024 2 10060 10060 2 + 10062 10062 2 10067 10069 2 10071 10071 2 10133 10135 2 10160 10160 2 10175 10175 2 + 11035 11036 2 11088 11088 2 11093 11093 2 11503 11505 0 11647 11647 0 11744 11775 0 + 11904 11929 2 11931 12019 2 12032 12245 2 12272 12329 2 12330 12333 0 12334 12350 2 + 12353 12438 2 12441 12442 0 12443 12543 2 12549 12591 2 12593 12686 2 12688 12773 2 + 12783 12830 2 12832 12871 2 12880 42124 2 42128 42182 2 42607 42610 0 42612 42621 0 + 42654 42655 0 42736 42737 0 43010 43010 0 43014 43014 0 43019 43019 0 43045 43046 0 + 43052 43052 0 43204 43205 0 43232 43249 0 43263 43263 0 43302 43309 0 43335 43345 0 + 43360 43388 2 43392 43394 0 43443 43443 0 43446 43449 0 43452 43453 0 43493 43493 0 + 43561 43566 0 43569 43570 0 43573 43574 0 43587 43587 0 43596 43596 0 43644 43644 0 + 43696 43696 0 43698 43700 0 43703 43704 0 43710 43711 0 43713 43713 0 43756 43757 0 + 43766 43766 0 44005 44005 0 44008 44008 0 44013 44013 0 44032 55203 2 63744 64255 2 + 64286 64286 0 65024 65039 0 65040 65049 2 65056 65071 0 65072 65106 2 65108 65126 2 + 65128 65131 2 65279 65279 0 65281 65376 2 65504 65510 2 65529 65531 0 66045 66045 0 + 66272 66272 0 66422 66426 0 68097 68099 0 68101 68102 0 68108 68111 0 68152 68154 0 + 68159 68159 0 68325 68326 0 68900 68903 0 68969 68973 0 69291 69292 0 69372 69375 0 + 69446 69456 0 69506 69509 0 69633 69633 0 69688 69702 0 69744 69744 0 69747 69748 0 + 69759 69761 0 69811 69814 0 69817 69818 0 69821 69821 0 69826 69826 0 69837 69837 0 + 69888 69890 0 69927 69931 0 69933 69940 0 70003 70003 0 70016 70017 0 70070 70078 0 + 70089 70092 0 70095 70095 0 70191 70193 0 70196 70196 0 70198 70199 0 70206 70206 0 + 70209 70209 0 70367 70367 0 70371 70378 0 70400 70401 0 70459 70460 0 70464 70464 0 + 70502 70508 0 70512 70516 0 70587 70592 0 70606 70606 0 70608 70608 0 70610 70610 0 + 70625 70626 0 70712 70719 0 70722 70724 0 70726 70726 0 70750 70750 0 70835 70840 0 + 70842 70842 0 70847 70848 0 70850 70851 0 71090 71093 0 71100 71101 0 71103 71104 0 + 71132 71133 0 71219 71226 0 71229 71229 0 71231 71232 0 71339 71339 0 71341 71341 0 + 71344 71349 0 71351 71351 0 71453 71453 0 71455 71455 0 71458 71461 0 71463 71467 0 + 71727 71735 0 71737 71738 0 71995 71996 0 71998 71998 0 72003 72003 0 72148 72151 0 + 72154 72155 0 72160 72160 0 72193 72202 0 72243 72248 0 72251 72254 0 72263 72263 0 + 72273 72278 0 72281 72283 0 72330 72342 0 72344 72345 0 72752 72758 0 72760 72765 0 + 72767 72767 0 72850 72871 0 72874 72880 0 72882 72883 0 72885 72886 0 73009 73014 0 + 73018 73018 0 73020 73021 0 73023 73029 0 73031 73031 0 73104 73105 0 73109 73109 0 + 73111 73111 0 73459 73460 0 73472 73473 0 73526 73530 0 73536 73536 0 73538 73538 0 + 73562 73562 0 78896 78912 0 78919 78933 0 90398 90409 0 90413 90415 0 92912 92916 0 + 92976 92982 0 94031 94031 0 94095 94098 0 94176 94179 2 94180 94180 0 94192 94193 2 + 94208 100343 2 100352 101589 2 101631 101640 2 110576 110579 2 110581 110587 2 + 110589 110590 2 110592 110882 2 110898 110898 2 110928 110930 2 110933 110933 2 + 110948 110951 2 110960 111355 2 113821 113822 0 113824 113827 0 118528 118573 0 + 118576 118598 0 119143 119145 0 119155 119170 0 119173 119179 0 119210 119213 0 + 119362 119364 0 119552 119638 2 119648 119670 2 121344 121398 0 121403 121452 0 + 121461 121461 0 121476 121476 0 121499 121503 0 121505 121519 0 122880 122886 0 + 122888 122904 0 122907 122913 0 122915 122916 0 122918 122922 0 123023 123023 0 + 123184 123190 0 123566 123566 0 123628 123631 0 124140 124143 0 124398 124399 0 + 125136 125142 0 125252 125258 0 126980 126980 2 127183 127183 2 127374 127374 2 + 127377 127386 2 127462 127490 2 127504 127547 2 127552 127560 2 127568 127569 2 + 127584 127589 2 127744 127776 2 127789 127797 2 127799 127868 2 127870 127891 2 + 127904 127946 2 127951 127955 2 127968 127984 2 127988 127988 2 127992 127994 2 + 127995 127999 0 128000 128062 2 128064 128064 2 128066 128252 2 128255 128317 2 + 128331 128334 2 128336 128359 2 128378 128378 2 128405 128406 2 128420 128420 2 + 128507 128591 2 128640 128709 2 128716 128716 2 128720 128722 2 128725 128727 2 + 128732 128735 2 128747 128748 2 128756 128764 2 128992 129003 2 129008 129008 2 + 129292 129338 2 129340 129349 2 129351 129535 2 129648 129660 2 129664 129673 2 + 129679 129734 2 129742 129756 2 129759 129769 2 129776 129784 2 131072 196605 2 + 196608 262141 2 917505 917505 0 917536 917631 0 917760 917999 0]) + +;; The width of one code point, by a binary search of `width-ranges`. +(: (-> int int)) +(defun (code-point-width cp) + (if (and (>= cp 32) (< cp 127)) + 1 + (searched-width cp))) + +(: (-> int int)) +(defun (searched-width cp) + (let lp ((lo 0) (hi (- (/ (array-length width-ranges) 3) 1))) + (if (> lo hi) + 1 + (let () + (def mid (/ (+ lo hi) 2)) + (cond + ((< cp (ref width-ranges (* 3 mid))) (lp lo (- mid 1))) + ((> cp (ref width-ranges (+ (* 3 mid) 1))) (lp (+ mid 1) hi)) + (else (ref width-ranges (+ (* 3 mid) 2)))))))) + +;; The cursor after the escape sequence that starts at `c`, an ESC: a CSI +;; ends at a character from @ to ~, an OSC at BEL or at ESC and a backslash, +;; and any other is ESC and one character. +(: (-> string StringCursor StringCursor)) +(defun (after-escape s c) + (def next (string-cursor-next s c)) + (defun (char-at k) (if (string-cursor-end? s k) 0 (char->int (string-cursor-ref s k)))) + (cond + ((string-cursor-end? s next) next) + ((= (char-at next) 91) + (let lp ((k (string-cursor-next s next))) + (cond + ((string-cursor-end? s k) k) + ((and (>= (char-at k) 64) (<= (char-at k) 126)) (string-cursor-next s k)) + (else (lp (string-cursor-next s k)))))) + ((= (char-at next) 93) + (let lp ((k (string-cursor-next s next))) + (cond + ((string-cursor-end? s k) k) + ((= (char-at k) 7) (string-cursor-next s k)) + ((and (= (char-at k) 27) + (let ((k2 (string-cursor-next s k))) + (and (not (string-cursor-end? s k2)) (= (char-at k2) 92)))) + (string-cursor-next s (string-cursor-next s k))) + (else (lp (string-cursor-next s k)))))) + (else (string-cursor-next s next)))) + +;; How many columns `s` takes in a terminal. +(: (-> string int)) +(defun (terminal-width s) + (let lp ((c (string-cursor-start s)) (total 0) (last 0) (joined? #f) (open-flag? #f)) + (if (string-cursor-end? s c) + total + (let () + (def cp (char->int (string-cursor-ref s c))) + (def next (string-cursor-next s c)) + (cond + ((= cp 27) (lp (after-escape s c) total last joined? open-flag?)) + ((= cp 8205) (lp next total last #t #f)) + ((= cp 65039) + (if (= last 1) (lp next (+ total 1) 2 #f #f) (lp next total last #f #f))) + (joined? (lp next total last #f #f)) + ((and (>= cp 127462) (<= cp 127487)) + (if open-flag? (lp next total last #f #f) (lp next (+ total 2) 2 #f #t))) + (else + (def w (code-point-width cp)) + (lp next (+ total w) (if (> w 0) w last) #f #f))))))) + +;; =========================================================================== +;; A table as a literal +;; =========================================================================== +;; +;; `(show-table '(table (style rounded) header (column "Item") ... (rows ,rows)))`. +;; The grammar checks the literal where it is written; `table-block` lays out +;; the value with `tabular/rows`. + +(def/grammar Table + (Table (table (: options TableOptions) (: columns TableColumn ...+) (: rows (rows TableRow ...)))) + (TableOptions (:each (: style (:? (style TableStyle))) + (: header (:? 'header)) + (: rules (:? (rules TableRules))) + (: width (:? (width int))) + (: overflow (:? (overflow Overflow))))) + (TableColumn (column (: title string) (: options ColumnOptions))) + (ColumnOptions (:each (: align (:? (align Align))) + (: width (:? (width ColumnWidth))) + (: kind (:? ColumnKind)))) + (ColumnKind (:or (: KindNumber (number NumberFormat)) + (: KindBytes 'bytes) + (: KindPercent (percent (:? (precision int)))) + (: KindBar (bar BarFormat)))) + (NumberFormat (:each (: precision (:? (precision int))) + (: comma (:? (comma int))) + (: signed (:? 'signed)))) + (BarFormat (:each (: limit (:? (max double))) + (: size (:? (width int))))) + (TableRow (:or (: RowOf (row TableCell ...)) + (: RowSpliced ,(Vec TableCell)) + (: RowsSpliced ,(Vec (Vec TableCell))))) + (TableCell (:or (: CellText string) (: CellInt int) (: CellNum double) (: CellBlocks ,(Vec Block))))) + +;; A cell shown as it is. +(: (-> TableCell Block)) +(defun (plain-cell c) + (match c + ((CellText s) (BStr s)) + ((CellInt n) (BInt n)) + ((CellNum x) (numeric x)) + ((CellBlocks bs) (each bs)))) + +(: (-> TableCell (Option double))) +(defun (cell-number c) + (match c + ((CellInt n) (Some (cast double n))) + ((CellNum x) (Some x)) + (_ None))) + +;; A cell shown as its column's kind says. A cell that is not a number is +;; shown as it is in any column. +(: (-> (Option ColumnKind) TableCell Block)) +(defun (kind-cell kind c) + (match (Tuple kind (cell-number c)) + ((Tuple (Some (KindNumber f)) (Some x)) + (numeric x #:precision (option-ref-or (record-ref f precision) -1) + #:comma (option-ref-or (record-ref f comma) -1) + #:signed (record-ref f signed))) + ((Tuple (Some KindBytes) (Some x)) (bytes (cast long x))) + ((Tuple (Some (KindPercent p)) (Some x)) (percent x #:precision (option-ref-or p 1))) + ((Tuple (Some (KindBar b)) (Some x)) + (bar (/ x (option-ref-or (record-ref b limit) 1.0)) #:width (option-ref-or (record-ref b size) 10))) + (_ (plain-cell c)))) + +;; A column's alignment: the one it says, or to the right for a column of +;; numbers, sizes or shares, and to the left for anything else. +(: (-> TableColumn Align)) +(defun (column-align col) + (def o (record-ref col options)) + (match (record-ref o align) + ((Some a) a) + (None (match (record-ref o kind) + ((Some (KindBar _)) AlignLeft) + ((Some _) AlignRight) + (None AlignLeft))))) + +;; The table's rows, a block a cell, each shown as its column's kind says. +(: (-> (Vec TableColumn) (Vec (Vec TableCell)) (Vec (Vec Block)))) +(defun (table-cells cols rows) + (vec-map (fun (row) + {vecing (kind-cell (if (< i (vec-length cols)) + (record-ref (record-ref (vec-ref cols i) options) kind) + None) + (vec-ref row i)) + (:for i (range 0 (vec-length row)))}) + rows)) + +;; A table literal as a block, laid out as `tabular/rows` lays out a table. +(: (-> Table Block)) +(defun (table-block t) + (def* (o (record-ref t options)) + (cols (list->vec (record-ref t columns))) + (rows (list-foldl (fun (r acc) + (match r + ((RowOf cells) (vec-add acc (list->vec cells))) + ((RowSpliced cells) (vec-add acc cells)) + ((RowsSpliced more) (vec-append acc more)))) + [] + (record-ref t rows))) + (laid (tabular/rows (table-cells cols rows) + #:titles (if (record-ref o header) + (vec-map (fun (c) (BStr (record-ref c title))) cols) + []) + #:style (option-ref-or (record-ref o style) StyleNone) + #:rules (option-ref-or (record-ref o rules) RulesHeader) + #:align (vec-map column-align cols) + #:widths (vec-map (fun (c) (option-ref-or (record-ref (record-ref c options) width) + WidthAuto)) + cols) + #:overflow (option-ref-or (record-ref o overflow) OverflowPush)))) + (match (record-ref o width) + ((Some w) (with-fmt current-width w [laid])) + (None laid))) + +;; Shows a table literal on the current output port. `#:sync` for `show`'s +;; reason. +(: show-table #:sync (-> Table void)) +(defun (show-table t) (show [(table-block t)])) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/hashset.bjo — the unordered hash set. +;; +;; A CHAMP trie: a node holds up to 32 slots, picked by 5 bits of the +;; element's hash at its depth. A slot holds an element or a child. +;; +;; - The nodes are C# classes in BjolangRuntime/HashTries, one class for +;; each size, so that a node is one object. `NodeOps` does the operations +;; on one node. This module does everything that spans nodes. +;; - A leaf holds only elements. A branch holds elements and children. A +;; collision node holds elements whose full hashes are equal. +;; - A child is never a leaf of one element: removal puts such an element in +;; the parent's slot. The empty set is the empty leaf. +;; - A transient owns the nodes it copied, and edits those in place. + +(export + ;; The types. + HashSet HashSetBuilder TransientHashSet HashSetCursor HashSetting + + ;; Sets + hashset-empty hashset-add hashset-add-all hashset-remove hashset-remove-all + hashset-contains? hashset-length hashset-empty? hashset-clear hashset-union + hashset-map hashset-filter hashset-fold hashset-for-each hashset-iter hashset-any? hashset-find + hashset-split + ;; Conversions + list->hashset hashset->list vec->hashset hashset->vec seq->hashset hashset->seq + hashset->str + ;; Builders — the same shape as ListBuilder + hashsetbuilder-empty hashset->hashsetbuilder hashsetbuilder-add! hashsetbuilder-add-all! + hashsetbuilder-length hashsetbuilder->hashset + ;; Transients — the set's interface, in place + transienthashset-empty hashset->transienthashset transienthashset->hashset + transienthashset-add! transienthashset-add-all! transienthashset-remove! + transienthashset-remove-all! transienthashset-contains? transienthashset-length + transienthashset-empty?) + +(import (std prelude)) + +(:doc #:file "hashset.bjodoc") + +(import/class + ((Node %a) (: Bjolang.Runtime.HashTries.Node)) + ;; What a set compares its elements with. See `eq-comparer`. + ((EqualityComparer %a) (: System.Collections.Generic.IEqualityComparer))) + +(import/extern + (owner-next (: Bjolang.Runtime.HashTries.NodeOps.OwnerNext (-> long))) + (empty-node (: Bjolang.Runtime.HashTries.NodeOps.Empty (-> (Node %a)))) + (node-kind (: Bjolang.Runtime.HashTries.NodeOps.Kind (-> (Node %a) int))) + (node-size (: Bjolang.Runtime.HashTries.NodeOps.Size (-> (Node %a) int))) + (child-count (: Bjolang.Runtime.HashTries.NodeOps.ChildCount (-> (Node %a) int))) + (element (: Bjolang.Runtime.HashTries.NodeOps.Element (-> (Node %a) int %a))) + (child (: Bjolang.Runtime.HashTries.NodeOps.Child (-> (Node %a) int (Node %a)))) + (element-map (: Bjolang.Runtime.HashTries.NodeOps.ElementMap (-> (Node %a) int))) + (child-map (: Bjolang.Runtime.HashTries.NodeOps.ChildMap (-> (Node %a) int))) + (collision-hash (: Bjolang.Runtime.HashTries.NodeOps.CollisionHash (-> (Node %a) int))) + (same-node? (: Bjolang.Runtime.HashTries.NodeOps.Same (-> (Node %a) (Node %a) bool))) + (fragment (: Bjolang.Runtime.HashTries.NodeOps.Fragment (-> int int int))) + (hash-of (: Bjolang.Runtime.HashTries.NodeOps.Hash (-> (EqualityComparer %a) %a int))) + (same-element? (: Bjolang.Runtime.HashTries.NodeOps.Equal (-> (EqualityComparer %a) %a %a bool))) + (same-comparer? (: Bjolang.Runtime.HashTries.NodeOps.SameComparer (-> (EqualityComparer %a) (EqualityComparer %a) bool))) + (probe (: Bjolang.Runtime.HashTries.NodeOps.Probe (-> (Node %a) %a int int (EqualityComparer %a) int))) + (element-at (: Bjolang.Runtime.HashTries.NodeOps.ElementAt (-> (Node %a) int int %a))) + (with-element (: Bjolang.Runtime.HashTries.NodeOps.WithElement (-> (Node %a) %a int int long (Node %a)))) + (with-subtree (: Bjolang.Runtime.HashTries.NodeOps.WithSubtree (-> (Node %a) int int (Node %a) long (Node %a)))) + (with-child (: Bjolang.Runtime.HashTries.NodeOps.WithChild (-> (Node %a) int (Node %a) long (Node %a)))) + (leaf-of-one (: Bjolang.Runtime.HashTries.NodeOps.Single (-> %a int int long (Node %a)))) + (leaf-of-two (: Bjolang.Runtime.HashTries.NodeOps.Pair (-> %a int %a int int long (Node %a)))) + (branch-around (: Bjolang.Runtime.HashTries.NodeOps.Wrap (-> (Node %a) int int long (Node %a)))) + (branch-of-two (: Bjolang.Runtime.HashTries.NodeOps.Fork (-> %a int (Node %a) int int long (Node %a)))) + (collision-of-two (: Bjolang.Runtime.HashTries.NodeOps.Collision (-> %a %a int long (Node %a)))) + (without-element (: Bjolang.Runtime.HashTries.NodeOps.WithoutElement + (-> (Node %a) %a int int (EqualityComparer %a) long (Node %a)))) + (without-child (: Bjolang.Runtime.HashTries.NodeOps.WithoutChild (-> (Node %a) int int int long (Node %a)))) + (with-child-inlined (: Bjolang.Runtime.HashTries.NodeOps.WithChildInlined + (-> (Node %a) int %a int int long (Node %a)))) + (assemble (: Bjolang.Runtime.HashTries.NodeOps.Assemble + (-> int (Array %a) int (Array (Node %a)) int long (Node %a)))) + (collision-of (: Bjolang.Runtime.HashTries.NodeOps.CollisionOf (-> (Array %a) int int long (Node %a))))) + +;; The kinds `node-kind` returns. +(def leaf-kind 0) +(def collision-kind 2) + +;; The owner of the nodes no transient owns. They are never edited in place. +(: no-owner long) +(def no-owner 0) + +(: (-> (Node %a) bool)) +(defun (leaf? n) (= (node-kind n) leaf-kind)) + +(: (-> (Node %a) bool)) +(defun (collision? n) (= (node-kind n) collision-kind)) + +;; The elements under `n`. Nodes hold no counts, so this visits every node. +(: (-> (Node %a) int)) +(defun (count-under n) + (loop (:for i (range 0 (child-count n))) + (:acc c (folding (node-size n) (+ c (count-under (child n i))))) + => c)) + +;; --------------------------------------------------------------------------- +;; The set +;; --------------------------------------------------------------------------- + +;; - `count` is -1 for a part made by `hashset-split` until it is first asked +;; for, so that the part counts itself in its own fiber. It is written at +;; most once, with the same value by every thread that writes it. +(type (: (HashSet %a) #:opaque + (Record (: root (Node %a)) + (: count int #:mutable) + (: comparer (EqualityComparer %a))))) + +(: (-> (EqualityComparer %a) (HashSet %a))) +(defun (empty-with cmp) (HashSet (root (empty-node)) (count 0) (comparer cmp))) + +;; What makes a set out of nothing asks for the element's Eq, and the set +;; compares elements with it — the same answer `=` gives, also for two cases of +;; a union that `=` equates. One made from a set keeps that set's. +(: (-> (HashSet %a)) (where (Eq %a))) +(defun (hashset-empty) (empty-with (eq-comparer))) + +;; An empty set that compares elements as `s` does. +(: (-> (HashSet %a) (HashSet %a))) +(defun (hashset-clear s) (empty-with (.comparer s))) + +(: (-> (HashSet %a) int)) +(defun (hashset-length s) + (def n (.count s)) + (if (>= n 0) + n + (begin + (def counted (count-under (.root s))) + (record-set! s (count counted)) + counted))) + +(: (-> (HashSet %a) bool)) +(defun (hashset-empty? s) (= (hashset-length s) 0)) + +;; --------------------------------------------------------------------------- +;; Lookup +;; --------------------------------------------------------------------------- + +(: (-> (Node %a) %a int int (EqualityComparer %a) bool)) +(defun (contains-in? n x h shift cmp) + (def c (probe n x h shift cmp)) + (if (>= c 0) + (contains-in? (child n c) x h (+ shift 5) cmp) + (= c -1))) + +(: (-> (HashSet %a) %a bool)) +(defun (hashset-contains? s x) + (def cmp (.comparer s)) + (contains-in? (.root s) x (hash-of cmp x) 0 cmp)) + +;; --------------------------------------------------------------------------- +;; Add and remove +;; --------------------------------------------------------------------------- + +;; A node at `shift` of the two different elements `a` and `b`: a leaf if +;; their fragments differ there, a collision node if their hashes are equal, +;; else a branch around the node one level down. +(: (-> %a int %a int int long (Node %a))) +(defun (join a ha b hb shift owner) + (cond + ((= ha hb) (collision-of-two a b ha owner)) + ((= (fragment ha shift) (fragment hb shift)) + (branch-around (join a ha b hb (+ shift 5) owner) ha shift owner)) + (else (leaf-of-two a ha b hb shift owner)))) + +;; The collision node `col`, at `shift`, and `x`, whose hash differs from its +;; elements' hash, under one node at `shift`. +(: (-> (Node %a) %a int int long (Node %a))) +(defun (split-collision col x hx shift owner) + (def hc (collision-hash col)) + (if (= (fragment hc shift) (fragment hx shift)) + (branch-around (split-collision col x hx (+ shift 5) owner) hc shift owner) + (branch-of-two x hx col hc shift owner))) + +;; Adds `x` to the subtree `n` at `shift`. Returns the new subtree, and +;; whether `x` is new. An equal element already there is kept, and `n` +;; itself returned. +(: (-> (Node %a) %a int int (EqualityComparer %a) long (Tuple (Node %a) bool))) +(defun (insert n x h shift cmp owner) + (def c (probe n x h shift cmp)) + (cond + ((>= c 0) + (def ch (child n c)) + (def (Tuple r added?) (insert ch x h (+ shift 5) cmp owner)) + (Tuple (if (same-node? r ch) n (with-child n c r owner)) added?)) + ((= c -1) (Tuple n #f)) + ((= c -2) (Tuple (with-element n x h shift owner) #t)) + ((collision? n) (Tuple (split-collision n x h shift owner) #t)) + (else + (def y (element-at n h shift)) + (Tuple (with-subtree n h shift (join y (hash-of cmp y) x h (+ shift 5) owner) owner) #t)))) + +;; Removes the element equal to `x` from the subtree `n` at `shift`. Returns +;; the new subtree, and whether there was one. A child left empty is +;; dropped, and a child left with one element is put in its parent's slot. +(: (-> (Node %a) %a int int (EqualityComparer %a) long (Tuple (Node %a) bool))) +(defun (delete n x h shift cmp owner) + (def c (probe n x h shift cmp)) + (cond + ((= c -1) (Tuple (without-element n x h shift cmp owner) #t)) + ((< c 0) (Tuple n #f)) + (else + (def ch (child n c)) + (def (Tuple r removed?) (delete ch x h (+ shift 5) cmp owner)) + (cond + ((not removed?) (Tuple n #f)) + ((and (leaf? r) (= (node-size r) 0)) (Tuple (without-child n c h shift owner) #t)) + ((and (leaf? r) (= (node-size r) 1)) (Tuple (with-child-inlined n c (element r 0) h shift owner) #t)) + ((same-node? r ch) (Tuple n #t)) + (else (Tuple (with-child n c r owner) #t)))))) + +(: (-> (HashSet %a) %a (HashSet %a))) +(defun (hashset-add s x) + (def cmp (.comparer s)) + (def (Tuple r added?) (insert (.root s) x (hash-of cmp x) 0 cmp no-owner)) + (if added? + (HashSet (root r) (count (+ (hashset-length s) 1)) (comparer cmp)) + s)) + +(: (-> (HashSet %a) %a (HashSet %a))) +(defun (hashset-remove s x) + (def cmp (.comparer s)) + (def (Tuple r removed?) (delete (.root s) x (hash-of cmp x) 0 cmp no-owner)) + (if removed? + (HashSet (root r) (count (- (hashset-length s) 1)) (comparer cmp)) + s)) + +(: (-> (HashSet %a) (Seq %a) (HashSet %a))) +(defun (hashset-add-all s xs) + (def t (hashset->transienthashset s)) + (transienthashset-add-all! t xs) + (transienthashset->hashset t)) + +(: (-> (HashSet %a) (Seq %a) (HashSet %a))) +(defun (hashset-remove-all s xs) + (def t (hashset->transienthashset s)) + (transienthashset-remove-all! t xs) + (transienthashset->hashset t)) + +;; --------------------------------------------------------------------------- +;; Transients +;; --------------------------------------------------------------------------- + +;; A set that edits in place the nodes it copied. +;; `transienthashset->hashset` returns a set of its contents and gives it a +;; new owner, so that the set does not change after. +(type (: (TransientHashSet %a) #:opaque + (Record (: root (Node %a) #:mutable) + (: count int #:mutable) + (: owner long #:mutable) + (: comparer (EqualityComparer %a))))) + +(: (-> (TransientHashSet %a)) (where (Eq %a))) +(defun (transienthashset-empty) (hashset->transienthashset (hashset-empty))) + +(: (-> (HashSet %a) (TransientHashSet %a))) +(defun (hashset->transienthashset s) + (TransientHashSet (root (.root s)) (count (hashset-length s)) (owner (owner-next)) (comparer (.comparer s)))) + +(: (-> (TransientHashSet %a) (HashSet %a))) +(defun (transienthashset->hashset t) + (record-set! t (owner (owner-next))) + (HashSet (root (.root t)) (count (.count t)) (comparer (.comparer t)))) + +(: (-> (TransientHashSet %a) %a void)) +(defun (transienthashset-add! t x) + (def cmp (.comparer t)) + (def (Tuple r added?) (insert (.root t) x (hash-of cmp x) 0 cmp (.owner t))) + (record-set! t (root r)) + (when added? (record-set! t (count (+ (.count t) 1))))) + +(: (-> (TransientHashSet %a) %a void)) +(defun (transienthashset-remove! t x) + (def cmp (.comparer t)) + (def (Tuple r removed?) (delete (.root t) x (hash-of cmp x) 0 cmp (.owner t))) + (when removed? (record-set! t (root r) (count (- (.count t) 1))))) + +(: (-> (TransientHashSet %a) (Seq %a) void)) +(defun (transienthashset-add-all! t xs) + (loop (:for x xs) (:do (transienthashset-add! t x)))) + +(: (-> (TransientHashSet %a) (Seq %a) void)) +(defun (transienthashset-remove-all! t xs) + (loop (:for x xs) (:do (transienthashset-remove! t x)))) + +(: (-> (TransientHashSet %a) %a bool)) +(defun (transienthashset-contains? t x) + (def cmp (.comparer t)) + (contains-in? (.root t) x (hash-of cmp x) 0 cmp)) + +(: (-> (TransientHashSet %a) int)) +(defun (transienthashset-length t) (.count t)) + +(: (-> (TransientHashSet %a) bool)) +(defun (transienthashset-empty? t) (= (.count t) 0)) + +;; --------------------------------------------------------------------------- +;; Cursors +;; --------------------------------------------------------------------------- + +;; A position in an iteration of a set, a struct, so that an iteration +;; allocates nothing. The elements come in the trie's order: a node's own +;; elements, then those under its children, in order. +;; +;; - The cursor is on element `i` of `node`, of `end` elements. +;; - `path` is the child index at each level above `node`, 5 bits a level +;; above the low 4 bits, which hold the depth of `node`. `last-run` marks a +;; node with no elements after it: a root that is a leaf. +;; - At the end of a node, `advance` follows `path` from the root again. +(type (: (HashSetCursor %a) #:opaque + (Struct (: node (Node %a)) + (: i int) + (: end int) + (: root (Node %a)) + (: path long)))) + +(def depth-bits 4) +(def index-bits 5) +(: last-run long) +(def last-run -1) + +(: (-> long int int)) +(defun (index-at path level) + (cast System.Int32 (bitwise-and (shift-right-logical path (+ depth-bits (* index-bits level))) 31))) + +;; `path` with the index `index` at `level`, and the depth `depth`. +(: (-> long int int int long)) +(defun (path-to path level index depth) + (def shift (+ depth-bits (* index-bits level))) + (def (: mask long) (bitwise-ior (shift-left 31 shift) 15)) + (bitwise-ior (bitwise-and path (bitwise-not mask)) + (shift-left (cast System.Int64 index) shift) + (cast System.Int64 depth))) + +(: (-> (Node %a) (HashSetCursor %a))) +(defun (cursor-of root) + (if (leaf? root) + (HashSetCursor (node root) (i 0) (end (node-size root)) (root root) (path last-run)) + (enter root root 0))) + +;; The cursor on the first element of `n`, at `path`, or after it if it has +;; none. +(: (-> (Node %a) (Node %a) long (HashSetCursor %a))) +(defun (enter n root path) + (def size (node-size n)) + (if (> size 0) + (HashSetCursor (node n) (i 0) (end size) (root root) (path path)) + (advance root path))) + +;; The cursor on the first element after the node at `path`: under its first +;; child, else under the next sibling of it or of its nearest ancestor that +;; has one. A done cursor if there is none. +(: (-> (Node %a) long (HashSetCursor %a))) +(defun (advance root path) + (def depth (cast System.Int32 (bitwise-and path 15))) + ;; The deepest level whose child index can move right, and its node. + (def/mutable level -1) + (def/mutable parent root) + (def/mutable n root) + (loop (:for d (range 0 depth)) + (:do (def i (index-at path d)) + (when (< (+ i 1) (child-count n)) + (set! level d) + (set! parent n)) + (set! n (child n i)))) + (cond + ((> (child-count n) 0) (enter (child n 0) root (path-to path depth 0 (+ depth 1)))) + ((>= level 0) + (def next (+ (index-at path level) 1)) + (enter (child parent next) root (path-to path level next (+ level 1)))) + (else (HashSetCursor (node root) (i 0) (end 0) (root root) (path last-run))))) + +(: (-> (HashSetCursor %a) bool)) +(defun (cursor-done? c) (>= (.i c) (.end c))) + +(: (-> (HashSetCursor %a) %a)) +(defun (cursor-element c) (element (.node c) (.i c))) + +(: (-> (HashSetCursor %a) (HashSetCursor %a))) +(defun (cursor-step c) + (def i (+ (.i c) 1)) + (if (or (< i (.end c)) (= (.path c) last-run)) + (record-set c (i i)) + (advance (.root c) (.path c)))) + +(impl (Iterable (HashSet %a)) + (type %elem %a) + (type %cursor (HashSetCursor %a)) + (defun (iterable-start s) (cursor-of (.root s))) + (defun (iterable-done? s c) (cursor-done? c)) + (defun (iterable-current s c) (cursor-element c)) + (defun (iterable-next s c) (cursor-step c)) + (defun (count s) (hashset-length s))) + +;; --------------------------------------------------------------------------- +;; Iteration and rebuilding +;; --------------------------------------------------------------------------- + +;; Folds `f` over the elements in the trie's order. Recursive over the trie, +;; which reads each node once. +(: (-> (-?-> %a %s %s) %s (HashSet %a) %s)) +(defun (hashset-fold f init s) (fold-node f init (.root s))) + +(: (-> (-?-> %a %s %s) %s (Node %a) %s)) +(defun (fold-node f acc n) + (def own (loop (:for i (range 0 (node-size n))) + (:acc s (folding acc (f (element n i) s))) + => s)) + (loop (:for i (range 0 (child-count n))) + (:acc s (folding own (fold-node f s (child n i)))) + => s)) + +(: (-> (-?-> %a void) (HashSet %a) void)) +(defun (hashset-for-each f s) + (loop (:for x s) (:do (f x)))) + +;; Calls `f` until it returns #f. Returns whether it never did. +(: (-> (-?-> %a bool) (HashSet %a) bool)) +(defun (hashset-iter f s) + (let go ((c (cursor-of (.root s)))) + (cond + ((cursor-done? c) #t) + ((f (cursor-element c)) (go (cursor-step c))) + (else #f)))) + +(: (-> (-?-> %a bool) (HashSet %a) bool)) +(defun (hashset-any? pred s) + (let go ((c (cursor-of (.root s)))) + (cond + ((cursor-done? c) #f) + ((pred (cursor-element c)) #t) + (else (go (cursor-step c)))))) + +;; The first element in the trie's order that `pred` accepts. +(: (-> (-?-> %a bool) (HashSet %a) (Option %a))) +(defun (hashset-find pred s) + (let go ((c (cursor-of (.root s)))) + (cond + ((cursor-done? c) None) + (:def x (cursor-element c)) + ((pred x) (Some x)) + (else (go (cursor-step c)))))) + +;; `f` of each element, compared by `%b`'s Eq. +(: (-> (-?-> %a %b) (HashSet %a) (HashSet %b)) (where (Eq %b))) +(defun (hashset-map f s) + (def b (builder-with (eq-comparer) (max 16 (hashset-length s)))) + (loop (:for x s) (:do (hashsetbuilder-add! b (f x)))) + (hashsetbuilder->hashset b)) + +;; The elements `keep?` accepts. Returns `s` when it accepts every element. +(: (-> (-?-> %a bool) (HashSet %a) (HashSet %a))) +(defun (hashset-filter keep? s) + (def b (builder-with (.comparer s) (max 16 (hashset-length s)))) + (loop (:for x s) (:do (when (keep? x) (hashsetbuilder-add! b x)))) + (if (= (.count b) (hashset-length s)) s (hashsetbuilder->hashset b))) + +;; --------------------------------------------------------------------------- +;; Union +;; --------------------------------------------------------------------------- + +(: (-> int int bool)) +(defun (has? map bit) (not (= (bitwise-and map bit) 0))) + +;; The levels of leaves and branches: shifts 0 to 30. Below them are only +;; collision nodes. +(def levels 7) + +;; The union of the subtrees `a` and `b`, both at `shift`. Of two equal +;; elements the one in `a` is kept. Returns the union, and how many elements +;; of `b` are in `a`. Returns `a` itself when `b` adds nothing to it. +;; - `xs` and `kids` hold 32 slots for each level, where a node at that level +;; gathers its elements and children. +(: (-> (EqualityComparer %a) (Node %a) (Node %a) int (Array %a) (Array (Node %a)) (Tuple (Node %a) int))) +(defun (union-of cmp a b shift xs kids) + (cond + ((same-node? a b) (Tuple a (count-under a))) + ((collision? a) (union-collision cmp a b shift)) + ((collision? b) (union-into-collision cmp a b shift)) + (else (union-nodes cmp a b shift xs kids)))) + +;; The union of the collision node `a` and the node `b`. +(: (-> (EqualityComparer %a) (Node %a) (Node %a) int (Tuple (Node %a) int))) +(defun (union-collision cmp a b shift) + (def h (collision-hash a)) + (cond + ((and (collision? b) (= (collision-hash b) h)) + (loop (:for i (range 0 (node-size b))) + (:acc r (folding (Tuple a 0) + (begin + (def (Tuple n o) r) + (def y (element b i)) + (if (contains-in? n y h shift cmp) + (Tuple n (+ o 1)) + (Tuple (with-element n y h shift no-owner) o))))) + => r)) + ((and (leaf? b) (= (node-size b) 1)) + (def y (element b 0)) + (def hy (hash-of cmp y)) + (cond + ((not (= hy h)) (Tuple (split-collision a y hy shift no-owner) 0)) + ((contains-in? a y h shift cmp) (Tuple a 1)) + (else (Tuple (with-element a y h shift no-owner) 0)))) + (else + ;; The elements of `a` into `b`, each replacing an equal one in `b`. + (loop (:for i (range 0 (node-size a))) + (:acc r (folding (Tuple b 0) + (begin + (def (Tuple n o) r) + (def x (element a i)) + (def (Tuple without removed?) (delete n x h shift cmp no-owner)) + (def (Tuple added-to _) (insert without x h shift cmp no-owner)) + (Tuple added-to (if removed? (+ o 1) o))))) + => r)))) + +;; The union of the leaf or branch `a` and the collision node `b`. +(: (-> (EqualityComparer %a) (Node %a) (Node %a) int (Tuple (Node %a) int))) +(defun (union-into-collision cmp a b shift) + (def h (collision-hash b)) + (if (and (leaf? a) (= (node-size a) 1)) + (begin + (def x (element a 0)) + (def hx (hash-of cmp x)) + (cond + ((not (= hx h)) (Tuple (split-collision b x hx shift no-owner) 0)) + ((contains-in? b x h shift cmp) + (def n (node-size b)) + (def (: xs (Array %a)) (make-array n)) + (loop (:for i (range 0 n)) + (:do (def y (element b i)) + (array-set! xs i (if (same-element? cmp x y) x y)))) + (Tuple (collision-of xs n h no-owner) 1)) + (else (Tuple (with-element b x h shift no-owner) 0)))) + ;; Insert keeps an equal element of `a`. + (loop (:for i (range 0 (node-size b))) + (:acc r (folding (Tuple a 0) + (begin + (def (Tuple n o) r) + (def (Tuple added-to added?) (insert n (element b i) h shift cmp no-owner)) + (Tuple added-to (if added? o (+ o 1)))))) + => r))) + +;; The union of two leaves or branches, slot by slot. +(: (-> (EqualityComparer %a) (Node %a) (Node %a) int (Array %a) (Array (Node %a)) (Tuple (Node %a) int))) +(defun (union-nodes cmp a b shift xs kids) + (def ea (element-map a)) + (def ca (child-map a)) + (def eb (element-map b)) + (def cb (child-map b)) + (def down (+ shift 5)) + (def base (* 32 (/ shift 5))) + (def/mutable ia 0) + (def/mutable ja 0) + (def/mutable ib 0) + (def/mutable jb 0) + ;; The next free slots of this level in `xs` and `kids`. + (def/mutable ne base) + (def/mutable nk base) + (def/mutable em 0) + (def/mutable cm 0) + (def/mutable changed? #f) + (def/mutable overlap 0) + ;; The occupied slots, lowest first. + (def/mutable m (bitwise-ior (bitwise-ior ea ca) (bitwise-ior eb cb))) + (loop (:while (not (= m 0))) + (:do (def bit (bitwise-and m (- 0 m))) + (set! m (bitwise-xor m bit)) + (def ea? (has? ea bit)) + (def ca? (has? ca bit)) + (def eb? (has? eb bit)) + (def cb? (has? cb bit)) + (cond + ((and ea? eb?) + (def x (element a ia)) + (def y (element b ib)) + (if (same-element? cmp x y) + (begin (set! overlap (+ overlap 1)) + (array-set! xs ne x) (set! ne (+ ne 1)) (set! em (bitwise-ior em bit))) + (begin + (set! changed? #t) + (array-set! kids nk (join x (hash-of cmp x) y (hash-of cmp y) down no-owner)) + (set! nk (+ nk 1)) + (set! cm (bitwise-ior cm bit))))) + ((or ca? cb?) + ;; A child on either side, and a child or an element on the other. + (def left (cond (ca? (child a ja)) + (ea? (begin (def x (element a ia)) (leaf-of-one x (hash-of cmp x) down no-owner))) + (else (child b jb)))) + (def right (cond (cb? (child b jb)) + (eb? (begin (def y (element b ib)) (leaf-of-one y (hash-of cmp y) down no-owner))) + (else left))) + (def (Tuple n o) + (if (and (or ca? ea?) (or cb? eb?)) + (union-of cmp left right down xs kids) + (Tuple left 0))) + (set! overlap (+ overlap o)) + (unless (and ca? (same-node? n (child a ja))) (set! changed? #t)) + (array-set! kids nk n) + (set! nk (+ nk 1)) + (set! cm (bitwise-ior cm bit))) + (ea? (array-set! xs ne (element a ia)) (set! ne (+ ne 1)) (set! em (bitwise-ior em bit))) + (eb? (set! changed? #t) + (array-set! xs ne (element b ib)) (set! ne (+ ne 1)) (set! em (bitwise-ior em bit))) + (else unit)) + (when ea? (set! ia (+ ia 1))) + (when ca? (set! ja (+ ja 1))) + (when eb? (set! ib (+ ib 1))) + (when cb? (set! jb (+ jb 1))))) + (if changed? + (Tuple (assemble em xs cm kids base no-owner) overlap) + (Tuple a overlap))) + + + +;; The elements of `s` and of `other`. Of two equal elements the one in `s` +;; is kept, and the union compares elements as `s` does. Two sets that +;; compare alike share the nodes that only one of them has. +(: (-> (HashSet %a) (HashSet %a) (HashSet %a))) +(defun (hashset-union s other) + (def cmp (.comparer s)) + (cond + ((hashset-empty? other) s) + ((not (same-comparer? cmp (.comparer other))) + (def t (hashset->transienthashset s)) + (loop (:for x other) (:do (transienthashset-add! t x))) + (transienthashset->hashset t)) + ((hashset-empty? s) other) + (else + (def (Tuple r overlap) (union-of cmp (.root s) (.root other) 0 (make-array (* 32 levels)) (make-array (* 32 levels)))) + (cond + ((same-node? r (.root s)) s) + ((same-node? r (.root other)) other) + (else (HashSet (root r) + (count (- (+ (hashset-length s) (hashset-length other)) overlap)) + (comparer cmp))))))) + +;; --------------------------------------------------------------------------- +;; Builders +;; --------------------------------------------------------------------------- + +;; Elements added with their hashes, made into a trie at the end by sorting +;; them on 5 bits of hash a level. +(type (: (HashSetBuilder %a) #:opaque + (Record (: hashes (Array int) #:mutable) + (: elements (Array %a) #:mutable) + (: count int #:mutable) + (: comparer (EqualityComparer %a))))) + +(: (-> (EqualityComparer %a) int (HashSetBuilder %a))) +(defun (builder-with cmp capacity) + (HashSetBuilder (hashes (make-array capacity)) (elements (make-array capacity)) (count 0) (comparer cmp))) + +(: (-> (HashSetBuilder %a)) (where (Eq %a))) +(defun (hashsetbuilder-empty) (builder-with (eq-comparer) 16)) + +;; A builder that holds the elements of `s`. +(: (-> (HashSet %a) (HashSetBuilder %a))) +(defun (hashset->hashsetbuilder s) + (def b (builder-with (.comparer s) (max 16 (hashset-length s)))) + (loop (:for x s) (:do (hashsetbuilder-add! b x))) + b) + +(: (-> (HashSetBuilder %a) %a void)) +(defun (hashsetbuilder-add! b x) + (def n (.count b)) + (when (= n (array-length (.elements b))) + (record-set! b (hashes (array-resize (.hashes b) (* 2 n))) (elements (array-resize (.elements b) (* 2 n))))) + (array-set! (.hashes b) n (hash-of (.comparer b) x)) + (array-set! (.elements b) n x) + (record-set! b (count (+ n 1)))) + +(: (-> (HashSetBuilder %a) (Seq %a) void)) +(defun (hashsetbuilder-add-all! b xs) + (loop (:for x xs) (:do (hashsetbuilder-add! b x)))) + +;; The number of elements added, an element added twice counted twice. +(: (-> (HashSetBuilder %a) int)) +(defun (hashsetbuilder-length b) (.count b)) + +(: (-> (Array int) int int bool)) +(defun (one-hash? hs from n) + (def h (array-ref hs from)) + (loop (:for i (range (+ from 1) (+ from n))) + (:acc ok (folding #t (and ok (= (array-ref hs i) h)))) + => ok)) + +;; The distinct elements of `xs` from `from`, `n` of them, into the start of +;; `out`. Of equal elements the last one is kept. Returns how many. +(: (-> (EqualityComparer %a) (Array %a) int int (Array %a) int)) +(defun (distinct-into cmp xs from n out) + (def/mutable k 0) + (loop (:for i (range from (+ from n))) + (:do (def x (array-ref xs i)) + (def at (let find ((j 0)) + (cond ((= j k) -1) + ((same-element? cmp (array-ref out j) x) j) + (else (find (+ j 1)))))) + (if (< at 0) + (begin (array-set! out k x) (set! k (+ k 1))) + (array-set! out at x)))) + k) + +;; Groups this small are sorted in place by insertion rather than counted +;; into 32 buckets. +(def small-group 12) + +;; Where `build-node` sorts and gathers: 32 slots for each level. +(type (: (BuildScratch %a) #:opaque + (Record (: counts (Array int)) + (: starts (Array int)) + (: ends (Array int)) + (: own (Array %a)) + (: kids (Array (Node %a)))))) + +;; The node at `shift` of the `n` entries of `hs` and `xs` from `from`, which +;; have at least two hashes. Sorts them into `to-hs` and `to-xs` on the +;; fragment at `shift`, and builds each group's child from there, with the +;; arrays swapped. Returns the node and how many elements it holds. +(: (-> (EqualityComparer %a) (Array int) (Array %a) (Array int) (Array %a) int int int (BuildScratch %a) + (Tuple (Node %a) int))) +(defun (build-node cmp hs xs to-hs to-xs from n shift sc) + (if (<= n small-group) + (build-small cmp hs xs to-hs to-xs from n shift sc) + (build-sorted cmp hs xs to-hs to-xs from n shift sc))) + +;; `build-node` for a small group: sorts it in place on the fragment at +;; `shift`, then takes its runs of one fragment in order. +(: (-> (EqualityComparer %a) (Array int) (Array %a) (Array int) (Array %a) int int int (BuildScratch %a) + (Tuple (Node %a) int))) +(defun (build-small cmp hs xs to-hs to-xs from n shift sc) + (def end (+ from n)) + (loop (:for i (range (+ from 1) end)) + (:do (def h (array-ref hs i)) + (def x (array-ref xs i)) + (def f (fragment h shift)) + (def/mutable j i) + (loop (:while (and (> j from) (> (fragment (array-ref hs (- j 1)) shift) f))) + (:do (array-set! hs j (array-ref hs (- j 1))) + (array-set! xs j (array-ref xs (- j 1))) + (set! j (- j 1)))) + (array-set! hs j h) + (array-set! xs j x))) + (def base (* 32 (/ shift 5))) + (def own (.own sc)) + (def kids (.kids sc)) + (def/mutable ne base) + (def/mutable nk base) + (def/mutable em 0) + (def/mutable cm 0) + (def/mutable total 0) + (def/mutable i from) + (loop (:while (< i end)) + (:do (def f (fragment (array-ref hs i) shift)) + (def/mutable j (+ i 1)) + (loop (:while (and (< j end) (= (fragment (array-ref hs j) shift) f))) (:do (set! j (+ j 1)))) + (def c (- j i)) + (def bit (shift-left 1 f)) + (cond + ((= c 1) + (set! total (+ total 1)) + (array-set! own ne (array-ref xs i)) + (set! ne (+ ne 1)) + (set! em (bitwise-ior em bit))) + ((one-hash? hs i c) + (def (: group (Array %a)) (make-array c)) + (def k (distinct-into cmp xs i c group)) + (set! total (+ total k)) + (if (= k 1) + (begin (array-set! own ne (array-ref group 0)) (set! ne (+ ne 1)) (set! em (bitwise-ior em bit))) + (begin (array-set! kids nk (collision-of group k (array-ref hs i) no-owner)) + (set! nk (+ nk 1)) + (set! cm (bitwise-ior cm bit))))) + (else + (def (Tuple kid k) (build-node cmp hs xs to-hs to-xs i c (+ shift 5) sc)) + (set! total (+ total k)) + (array-set! kids nk kid) + (set! nk (+ nk 1)) + (set! cm (bitwise-ior cm bit)))) + (set! i j))) + (Tuple (assemble em own cm kids base no-owner) total)) + +;; `build-node` for a big group: counts it into 32 buckets on the fragment +;; at `shift` and moves it into `to-hs` and `to-xs` by bucket. +(: (-> (EqualityComparer %a) (Array int) (Array %a) (Array int) (Array %a) int int int (BuildScratch %a) + (Tuple (Node %a) int))) +(defun (build-sorted cmp hs xs to-hs to-xs from n shift sc) + (def base (* 32 (/ shift 5))) + (def counts (.counts sc)) + (def starts (.starts sc)) + (def ends (.ends sc)) + (def own (.own sc)) + (def kids (.kids sc)) + (loop (:for f (range base (+ base 32))) (:do (array-set! counts f 0))) + (loop (:for i (range from (+ from n))) + (:do (def f (+ base (fragment (array-ref hs i) shift))) + (array-set! counts f (+ (array-ref counts f) 1)))) + (def/mutable o from) + (loop (:for f (range base (+ base 32))) + (:do (array-set! starts f o) + (array-set! ends f o) + (set! o (+ o (array-ref counts f))))) + (loop (:for i (range from (+ from n))) + (:do (def f (+ base (fragment (array-ref hs i) shift))) + (def p (array-ref ends f)) + (array-set! to-hs p (array-ref hs i)) + (array-set! to-xs p (array-ref xs i)) + (array-set! ends f (+ p 1)))) + (def/mutable ne base) + (def/mutable nk base) + (def/mutable em 0) + (def/mutable cm 0) + (def/mutable total 0) + (loop (:for f (range 0 32)) + (:do (def c (array-ref counts (+ base f))) + (def s (array-ref starts (+ base f))) + (def bit (shift-left 1 f)) + (cond + ((= c 0) unit) + ((= c 1) + (set! total (+ total 1)) + (array-set! own ne (array-ref to-xs s)) + (set! ne (+ ne 1)) + (set! em (bitwise-ior em bit))) + ((one-hash? to-hs s c) + (def (: group (Array %a)) (make-array c)) + (def k (distinct-into cmp to-xs s c group)) + (set! total (+ total k)) + (if (= k 1) + (begin (array-set! own ne (array-ref group 0)) (set! ne (+ ne 1)) (set! em (bitwise-ior em bit))) + (begin (array-set! kids nk (collision-of group k (array-ref to-hs s) no-owner)) + (set! nk (+ nk 1)) + (set! cm (bitwise-ior cm bit))))) + (else + (def (Tuple kid k) (build-node cmp to-hs to-xs hs xs s c (+ shift 5) sc)) + (set! total (+ total k)) + (array-set! kids nk kid) + (set! nk (+ nk 1)) + (set! cm (bitwise-ior cm bit)))))) + (Tuple (assemble em own cm kids base no-owner) total)) + +(: (-> (HashSetBuilder %a) (HashSet %a))) +(defun (hashsetbuilder->hashset b) + (def cmp (.comparer b)) + (def n (.count b)) + (def hs (.hashes b)) + (def xs (.elements b)) + (cond + ((= n 0) (empty-with cmp)) + ((one-hash? hs 0 n) + (def (: group (Array %a)) (make-array n)) + (def k (distinct-into cmp xs 0 n group)) + (def h (array-ref hs 0)) + (HashSet (root (if (= k 1) (leaf-of-one (array-ref group 0) h 0 no-owner) (collision-of group k h no-owner))) + (count k) + (comparer cmp))) + (else + ;; The build reorders the builder's entries, which leaves the builder + ;; holding the same elements. + (def sc (BuildScratch (counts (make-array (* 32 levels))) (starts (make-array (* 32 levels))) + (ends (make-array (* 32 levels))) (own (make-array (* 32 levels))) + (kids (make-array (* 32 levels))))) + (def (Tuple root k) (build-node cmp hs xs (make-array n) (make-array n) 0 n 0 sc)) + (HashSet (root root) (count k) (comparer cmp))))) + +;; --------------------------------------------------------------------------- +;; Splitting +;; --------------------------------------------------------------------------- + +;; A slot of some node, for `hashset-split`: an element, or a child. +;; - `path` holds the fragments of the nodes above, 5 bits a level, the +;; root's lowest. `depth` is the level of the node; 0 is the root. +;; - `node` is the child, or the node that holds the element. +;; - `index` is the element's index in `node`, or -1 for a child. +(type (: (SplitUnit %a) #:opaque + (Struct (: path long) + (: node (Node %a)) + (: depth int) + (: bit int) + (: index int)))) + +(: (-> (SplitUnit %a) bool)) +(defun (unit-element? u) (>= (.index u) 0)) + +;; Whether the unit is a leaf or a branch, which can be replaced by its slots. +(: (-> (SplitUnit %a) bool)) +(defun (expands? u) (and (not (unit-element? u)) (not (collision? (.node u))))) + +;; The slots of `n`, at `depth` under `path`, onto `out`: its elements, then +;; its children, as iteration has them. +(: (-> (Node %a) int long (ListBuilder (SplitUnit %a)) void)) +(defun (add-units n depth path out) + (def/mutable m (element-map n)) + (def/mutable i 0) + (loop (:while (not (= m 0))) + (:do (listbuilder-add! out (SplitUnit (path path) (node n) (depth depth) (bit (trailing-zero-count m)) (index i))) + (set! i (+ i 1)) + (set! m (bitwise-and m (- m 1))))) + (set! m (child-map n)) + (set! i 0) + (loop (:while (not (= m 0))) + (:do (listbuilder-add! out (SplitUnit (path path) (node (child n i)) (depth depth) (bit (trailing-zero-count m)) (index -1))) + (set! i (+ i 1)) + (set! m (bitwise-and m (- m 1)))))) + +(: (-> int int)) +(defun (trailing-zero-count m) + (let go ((m m) (k 0)) (if (= (bitwise-and m 1) 1) k (go (shift-right-logical m 1) (+ k 1))))) + +;; The node at `depth` for the units from `from` below `to`, which all lie +;; at or below it on one path. A node of one element and no child, below the +;; root, is returned as a leaf of it with #t, for the parent's slot, as +;; removal would put it. +(: (-> (Array (SplitUnit %a)) int int int (Tuple (Node %a) bool))) +(defun (build-part units from to depth) + (def (: own (Array %a)) (make-array 32)) + (def (: kids (Array (Node %a))) (make-array 32)) + ;; The slots by fragment first, then packed in fragment order. + (def (: by-bit-x (Array %a)) (make-array 32)) + (def (: by-bit-n (Array (Node %a))) (make-array 32)) + (def/mutable em 0) + (def/mutable cm 0) + (let go ((i from)) + (when (< i to) + (def u (array-ref units i)) + (if (= (.depth u) depth) + (begin + (if (unit-element? u) + (begin (array-set! by-bit-x (.bit u) (element (.node u) (.index u))) + (set! em (bitwise-ior em (shift-left 1 (.bit u))))) + (begin (array-set! by-bit-n (.bit u) (.node u)) + (set! cm (bitwise-ior cm (shift-left 1 (.bit u)))))) + (go (+ i 1))) + (begin + (def f (fragment-of (.path u) depth)) + (def j (let scan ((j (+ i 1))) + (if (and (< j to) + (> (.depth (array-ref units j)) depth) + (= (fragment-of (.path (array-ref units j)) depth) f)) + (scan (+ j 1)) + j))) + (def (Tuple piece inline?) (build-part units i j (+ depth 1))) + (if inline? + (begin (array-set! by-bit-x f (element piece 0)) + (set! em (bitwise-ior em (shift-left 1 f)))) + (begin (array-set! by-bit-n f piece) + (set! cm (bitwise-ior cm (shift-left 1 f))))) + (go j))))) + (def/mutable ne 0) + (def/mutable nk 0) + (loop (:for f (range 0 32)) + (:do (def bit (shift-left 1 f)) + (when (has? em bit) (array-set! own ne (array-ref by-bit-x f)) (set! ne (+ ne 1))) + (when (has? cm bit) (array-set! kids nk (array-ref by-bit-n f)) (set! nk (+ nk 1))))) + (def n (assemble em own cm kids 0 no-owner)) + (Tuple n (and (> depth 0) (= cm 0) (= ne 1)))) + +(: (-> (ListBuilder %a) (Array %a))) +(defun (array-of b) (loop (:for x (listbuilder->list b)) (:acc (arraying x)))) + +(: (-> long int int)) +(defun (fragment-of path depth) + (cast System.Int32 (bitwise-and (shift-right-logical path (* 5 depth)) 31))) + +;; About `k` sets that together hold each element of `s` once, for work on +;; them at the same time. They share the nodes of `s`; only the nodes on the +;; way down to a cut are new. +;; +;; - The units are the slots of the root. While a child holds more than +;; count / k elements on average, every leaf and branch child is replaced +;; by its own slots. A collision node stays one unit, of its known size. +;; - The parts are runs of units, cut where the running weight passes +;; i * count / k. Hashes that are not spread make parts of different +;; sizes, not wrong ones. +;; - A part's count is not known until it is first asked for. +(: (-> (HashSet %a) int (Array (HashSet %a)))) +(defun (hashset-split s k) + (def size (hashset-length s)) + (def root (.root s)) + (def cmp (.comparer s)) + (if (or (<= k 1) (<= size 1) (collision? root)) + (vec->array [s]) + (begin + (def target (/ (cast double size) (cast double k))) + (def (: units-of-root (ListBuilder (SplitUnit %a))) (listbuilder-empty)) + (add-units root 0 0 units-of-root) + (: child-weight (-> (Array (SplitUnit %a)) (Tuple double int))) + (defun (child-weight us) + (def/mutable rest size) + (def/mutable children 0) + (loop (:for u us) + (:do (cond ((unit-element? u) (set! rest (- rest 1))) + ((collision? (.node u)) (set! rest (- rest (node-size (.node u))))) + (else (set! children (+ children 1)))))) + (Tuple (if (= children 0) 0.0 (max 1.0 (/ (cast double rest) (cast double children)))) children)) + (def (Tuple units weight) + (let expand ((us (array-of units-of-root))) + (def (Tuple w children) (child-weight us)) + (if (or (<= w target) (= children 0)) + (Tuple us w) + (begin + (def out (listbuilder-empty)) + (loop (:for u us) + (:do (if (expands? u) + (add-units (.node u) (+ (.depth u) 1) + (bitwise-ior (.path u) (shift-left (cast System.Int64 (.bit u)) (* 5 (.depth u)))) + out) + (listbuilder-add! out u)))) + (expand (array-of out)))))) + (def weights (loop (:for u units) + (:acc (arraying (cond ((unit-element? u) 1.0) + ((collision? (.node u)) (cast double (node-size (.node u)))) + (else weight)))))) + (def total (loop (:for w weights) (:acc (summing w)))) + (def n (array-length units)) + (def parts (min k n)) + (def out (listbuilder-empty)) + (def/mutable made 0) + (def/mutable start 0) + (def/mutable done 0.0) + (loop (:for j (range 0 n)) + (:do (set! done (+ done (array-ref weights j))) + (when (or (= j (- n 1)) + (and (< made (- parts 1)) + (>= done (/ (* total (cast double (+ made 1))) (cast double parts))))) + (def (Tuple part _) (build-part units start (+ j 1) 0)) + (listbuilder-add! out (HashSet (root part) (count -1) (comparer cmp))) + (set! made (+ made 1)) + (set! start (+ j 1))))) + (array-of out)))) + +;; --------------------------------------------------------------------------- +;; Conversions +;; --------------------------------------------------------------------------- + +(: (-> (Seq %a) (HashSet %a)) (where (Eq %a))) +(defun (seq->hashset xs) + (def b (hashsetbuilder-empty)) + (hashsetbuilder-add-all! b xs) + (hashsetbuilder->hashset b)) + +(: (-> (HashSet %a) (Seq %a))) +(defun (hashset->seq s) (->seq s)) + +(: (-> (List %a) (HashSet %a)) (where (Eq %a))) +(defun (list->hashset xs) (loop (:for x xs) (:acc (hash-setting x)))) + +(: (-> (HashSet %a) (List %a))) +(defun (hashset->list s) (->list s)) + +(: (-> (Vec %a) (HashSet %a)) (where (Eq %a))) +(defun (vec->hashset v) (loop (:for x v) (:acc (hash-setting x)))) + +(: (-> (HashSet %a) (Vec %a))) +(defun (hashset->vec s) (->vec s)) + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- + +(impl (Refable (HashSet %a)) + (type %index %a) + (type %out bool) + (defun (ref set key) + (hashset-contains? set key))) + +(impl (Refable (TransientHashSet %a)) + (type %index %a) + (type %out bool) + (defun (ref set key) + (transienthashset-contains? set key))) + +(impl (Collection (HashSet %a)) + (type %content %a) + (defun (length s) (hashset-length s)) + (defun (empty? s) (hashset-empty? s))) + +(type (: (HashSetting %a) (Union hash-setting))) + +(impl (Collector (HashSetting %a)) + (where (Eq %a)) + (type %elem %a) + (type %acc (HashSetBuilder %a)) + (type %out (HashSet %a)) + (defun (collector-init c) (hashsetbuilder-empty)) + (defun (collector-step c acc x) (hashsetbuilder-add! acc x) acc) + (defun (collector-finish c acc) (hashsetbuilder->hashset acc))) + +;; The mutable sinks, one element at a time. `Addable!` is the prelude's +;; trait; this module owns the types, which is what licenses the impls. +(impl (Addable! (HashSetBuilder %a)) + (type %elem %a) + (defun (add! sb elem) + (hashsetbuilder-add! sb elem))) + +(impl (Addable! (TransientHashSet %a)) + (type %elem %a) + (defun (add! ts elem) + (transienthashset-add! ts elem))) + +;; And the persistent set itself. +(impl (Addable (HashSet %a)) + (type %elem %a) + (defun (add s elem) + (hashset-add s elem))) + +;; Equality for sets. +(impl (Eq (HashSet %a)) + (where (Eq %a)) + (defun (= a b) + (and (= (hashset-length a) (hashset-length b)) + (hashset-iter (fun (x) (hashset-contains? b x)) a))) + (defun (eq-hash s) + (hashset-fold (fun (x h) (bitwise-xor h (eq-hash x))) 0 s))) + +;; `#{a b c}`, in the order of the trie. +(impl (Written (HashSet %a)) + (where (Written %a)) + (defun (write-into s b) (elements-print-into "#{" s "}" b (fun (x b) (write-into x b))))) + +(impl (Displayed (HashSet %a)) + (where (Displayed %a)) + (defun (display-into s b) (elements-print-into "#{" s "}" b (fun (x b) (display-into x b))))) + +(: (-> (HashSet %a) string) (where (Displayed %a))) +(defun (hashset->str s) (displayed s)) + +(:deprecated hashset->str "use `displayed`") +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/http.bjo — HTTP requests. +;; +;; A wrapper over System.Net.Http +;; +;; Two things are deliberately not what .NET does. A status code is a value, +;; never an exception — `EnsureSuccessStatusCode` is not imported and 404 is +;; not a failure. And `HttpClient.Timeout` is switched off, because a deadline +;; belongs to the CML system bjolang uses. + +(export + ;; The types + Method Body Custom Pair Request Response Client + + ;; Writing a request + request http-get http-post + + ;; Sending one + fetch fetching + + ;; Reading the answer + response-status response-ok? response-reason + response-header response-headers + response-text response-port response-bytes-port response-close + + ;; Scoped + with-response + + ;; The client + make-client current-http-client) + +(import (std prelude)) +(import (std syntax-match)) +(import (only (text encodings) utf-8 encoding-named)) +(import (only (std url) encode-query-part encode-form)) + +;; Every export is documented there; the module doc is too. +(:doc #:file "http.bjodoc") + +;; =========================================================================== +;; Interop +;; =========================================================================== + +(import/class + (SocketsHandler (: System.Net.Http.SocketsHttpHandler (-> SocketsHandler))) + (MessageHandler (: System.Net.Http.HttpMessageHandler)) + (Client (: System.Net.Http.HttpClient (-> MessageHandler Client))) + (Verb (: System.Net.Http.HttpMethod (-> string Verb))) + (Message (: System.Net.Http.HttpRequestMessage (-> Verb Uri Message))) + (Reply (: System.Net.Http.HttpResponseMessage)) + (Content (: System.Net.Http.HttpContent)) + (StringContent (: System.Net.Http.StringContent (-> string Encoding StringContent))) + (Headers (: System.Net.Http.Headers.HttpHeaders)) + (RequestHeaders (: System.Net.Http.Headers.HttpRequestHeaders)) + (ReplyHeaders (: System.Net.Http.Headers.HttpResponseHeaders)) + (ContentHeaders (: System.Net.Http.Headers.HttpContentHeaders)) + (Completion (: System.Net.Http.HttpCompletionOption)) + (StatusCode (: System.Net.HttpStatusCode)) + (Decompression (: System.Net.DecompressionMethods)) + (Span (: System.TimeSpan)) + (ClrStream (: System.IO.Stream)) + (Uri (: System.Uri (-> string Uri) #:exceptions (System.UriFormatException))) + (Failure (: System.Net.Http.HttpRequestException (-> string Failure))) + (Object (: System.Object)) + ;; How .NET hands over a header collection: one name, several values. + ((Pairing %k %v) (: System.Collections.Generic.KeyValuePair))) + +(import/extern + ;; Enum values and other statics. + (headers-read (: System.Net.Http.HttpCompletionOption.ResponseHeadersRead #:get)) + (decompress-all (: System.Net.DecompressionMethods.All #:get)) + (no-timeout (: System.Threading.Timeout.InfiniteTimeSpan #:get)) + ;; Bytes in a declared encoding as an input port: read as they are when + ;; they are UTF-8, transcoded to UTF-8 otherwise. + (text-port-over (: Bjolang.Runtime.InputPorts.Over (-> ClrStream Encoding InputPort))) + ;; Named `span-seconds` rather than `seconds`: `std/prelude` exports a + ;; `seconds` that answers milliseconds, for a deadline, and two bindings of + ;; one name that differ only in what they return is the kind of collision + ;; nothing reports usefully. + (span-seconds (: System.TimeSpan.FromSeconds (-> double Span))) + + ;; The client and the handler under it. + (handler-lifetime! (: System.Net.Http.SocketsHttpHandler.PooledConnectionLifetime + (-> SocketsHandler Span void) #:set)) + (handler-decompress! (: System.Net.Http.SocketsHttpHandler.AutomaticDecompression + (-> SocketsHandler Decompression void) #:set)) + (handler-redirects! (: System.Net.Http.SocketsHttpHandler.AllowAutoRedirect + (-> SocketsHandler bool void) #:set)) + (client-timeout! (: System.Net.Http.HttpClient.Timeout (-> Client Span void) #:set)) + + ;; The call. Declared with two arguments after the receiver, so the overload + ;; taking a third — the CancellationToken — is the one selected, and the + ;; ambient token is filled in at the call site. + (send (: System.Net.Http.HttpClient.SendAsync (-> Client Message Completion Reply) #:async)) + + ;; Building a request. + (url-scheme (: System.Uri.Scheme (-> Uri string) #:get)) + (message-headers (: System.Net.Http.HttpRequestMessage.Headers (-> Message RequestHeaders) #:get)) + (message-content! (: System.Net.Http.HttpRequestMessage.Content (-> Message Content void) #:set)) + (content-headers (: System.Net.Http.HttpContent.Headers (-> Content ContentHeaders) #:get)) + (header-add! (: System.Net.Http.Headers.HttpHeaders.TryAddWithoutValidation + (-> Headers string string bool))) + (header-remove! (: System.Net.Http.Headers.HttpHeaders.Remove (-> Headers string bool))) + + ;; Reading a reply. + (reply-status (: System.Net.Http.HttpResponseMessage.StatusCode (-> Reply StatusCode) #:get)) + (reply-reason (: System.Net.Http.HttpResponseMessage.ReasonPhrase (-> Reply string) #:get)) + (reply-headers (: System.Net.Http.HttpResponseMessage.Headers (-> Reply ReplyHeaders) #:get)) + (reply-content (: System.Net.Http.HttpResponseMessage.Content (-> Reply Content) #:get)) + + ;; The body, both ways. A `defbjouble` pair: the synchronous one parks a + ;; thread, which is right for a plain `defun`, and the other suspends. + (clr-stream (: System.Net.Http.HttpContent.ReadAsStream (-> Content ClrStream))) + (clr-stream-await (: System.Net.Http.HttpContent.ReadAsStreamAsync (-> Content ClrStream) #:async))) + +;; Nothing in the language builds an `Exception`, and every rejection below is +;; this module's own rather than .NET's. +(: (-> string Exception)) +(defun (http-error message) + (cast Exception (Failure. message))) + +;; .NET says null where it has nothing: a reason phrase over HTTP/2, a header +;; that is not there. A .NET string becomes a Bjolang one at the call, and a +;; null becomes "" there. +(: (-> string string)) +(defun (text-or-empty s) s) + + +;; =========================================================================== +;; The client +;; =========================================================================== + +;; One handler, one client, and a connection that does not live forever: a +;; pooled connection outlives DNS, so a client that never recycles keeps +;; talking to an address that stopped being right. +(: (-> (#:connection-seconds double) (#:redirects bool) Client)) +(defun (make-client #:connection-seconds 120.0 #:redirects #t) + (def handler (SocketsHandler.)) + (handler-lifetime! handler (span-seconds connection-seconds)) + (handler-decompress! handler decompress-all) + (handler-redirects! handler redirects) + (def client (Client. (cast MessageHandler handler))) + ;; Off, on purpose. It applies to every call made through the client, + ;; cannot be varied per request, and arrives as a cancellation that + ;; cannot be told from a real one. Race a `(timeout d)` instead. + (client-timeout! client no-timeout) + client) + +;; Parameterize another one in for a proxy, a test double or a second set of +;; connection settings. +(: (Param Client)) +(def current-http-client (make-parameter (make-client))) + + +;; =========================================================================== +;; What you write +;; =========================================================================== + +;; A name and a value: a header, a query parameter or a form field. Written +;; `("Accept" . "application/json")`. +(type (: Pair (Alias (Tuple string string)))) + +(type (: Method (Enum GET POST PUT PATCH DELETE HEAD))) + +;; A body .NET has no name for: your own content type, your own text. +(type (: Custom (Record (: content-type string) (: text string)))) + +(type (: Body (Union Empty + (: Text string) + ;; Already serialized: there is no JSON writer to call yet, + ;; and inventing one here would be the wrong module. Named + ;; `JsonText` rather than `Json` because `(text json)` + (: JsonText string) + (: Form (Vec Pair)) + (: Other Custom)))) + +(type (: Request (Record (: method Method) + (: url string) + (: query (Vec Pair)) + (: headers (Vec Pair)) + (: body Body)))) + +(: (-> Method string (#:query (Vec Pair)) (#:headers (Vec Pair)) (#:body Body) Request)) +(defun (request method url #:query (vec-empty) #:headers (vec-empty) #:body Empty) + (Request (method method) (url url) (query query) (headers headers) (body body))) + +(: (-> string (#:query (Vec Pair)) (#:headers (Vec Pair)) Request)) +(defun (http-get url #:query (vec-empty) #:headers (vec-empty)) + (request GET url #:query query #:headers headers)) + +(: (-> string Body (#:query (Vec Pair)) (#:headers (Vec Pair)) Request)) +(defun (http-post url body #:query (vec-empty) #:headers (vec-empty)) + (request POST url #:query query #:headers headers #:body body)) + +(: (-> Method string)) +(defun (method-name m) + (match m + (GET "GET") + (POST "POST") + (PUT "PUT") + (PATCH "PATCH") + (DELETE "DELETE") + (HEAD "HEAD"))) + + +;; =========================================================================== +;; Turning one into an HttpRequestMessage +;; =========================================================================== + +;; Pairs as `encode` writes each name and value, joined with `&`. A query is +;; encoded by `encode-query-part` and a form by `encode-form`, from (std url): +;; both encode `&`, `=` and `+`, so a value cannot begin another pair. +(: (-> (-> string string) (Vec Pair) string)) +(defun (encode-pairs encode pairs) + (string-join (vec-map (fun (pair) + (match pair + ((Tuple name value) (str (encode name) "=" (encode value))))) + pairs) + "&")) + +(: (-> Request string)) +(defun (full-url req) + (def base (record-ref req url)) + (def query (record-ref req query)) + (cond + ((empty? query) base) + (else + ;; The query goes before a fragment. After the `#` it would be part of + ;; the fragment, which is not sent. + (def parts (string-split base "#")) + (def head (vec-ref parts 0)) + (def fragment (if (> (vec-length parts) 1) (str "#" (string-join (vec-slice parts 1 (- (vec-length parts) 1)) "#")) "")) + ;; A url may already carry a query, in which case ours is more of it. + (str head (if (string-contains? head "?") "&" "?") + (encode-pairs encode-query-part query) + fragment)))) + +;; Attaches a body. Helper function for the header functions below +(: (-> Message Body (Option Content))) +(defun (attach-body! message body) + (match body + (Empty None) + ((Text text) (Some (attach! message text "text/plain; charset=utf-8"))) + ((JsonText text) (Some (attach! message text "application/json; charset=utf-8"))) + ((Form fields) (Some (attach! message (encode-pairs encode-form fields) + "application/x-www-form-urlencoded"))) + ((Other custom) (Some (attach! message + (record-ref custom text) + (record-ref custom content-type)))))) + +(: (-> Message string string Content)) +(defun (attach! message text content-type) + (def content (cast Content (StringContent. text utf-8))) + (message-content! message content) + ;; StringContent has already put `text/plain` there. + (ignore (header-remove! (cast Headers (content-headers content)) "Content-Type")) + (ignore (header-add! (cast Headers (content-headers content)) "Content-Type" content-type)) + content) + +;; Which end of the request a header belongs on is .NET's business, not the +;; caller's: `Accept` goes on the request and `Content-Type` on the content. +;; `TryAddWithoutValidation` returning false is how it says "the other one". +(: (-> Message (Option Content) Pair (Result Exception Unit))) +(defun (add-header! message content pair) + (match pair + ((Tuple name value) + (if (header-add! (cast Headers (message-headers message)) name value) + (Ok unit) + (match content + (None (Err (http-error + (str "'" name "' is a header about a body, and this request has none")))) + ((Some body) + ;; Remove first, or the content type set above is sent twice. + (ignore (header-remove! (cast Headers (content-headers body)) name)) + (if (header-add! (cast Headers (content-headers body)) name value) + (Ok unit) + (Err (http-error (str "not a usable header name: " name)))))))))) + +(: (-> Message (Option Content) (Vec Pair) (Result Exception Unit))) +(defun (add-headers! message content headers) + (loop (:for pair headers) + (:acc outcome (folding (Ok unit) + (match outcome + ((Err e) (Err e)) + ((Ok done) (add-header! message content pair))))))) + +(: (-> Uri bool)) +(defun (web-url? url) + (def scheme (url-scheme url)) + (or (= scheme "http") (= scheme "https"))) + +(: (-> Request (Result Exception Message))) +(defun (message-of req) + (match (Uri. (full-url req)) + ((Err e) (Err (http-error (str "not a url: " (record-ref req url))))) + ((Ok url) + ;; `new Uri` is happy with `file:///etc/passwd`, which is not something an + ;; HTTP library should be sending anywhere. + (if (not (web-url? url)) + (Err (http-error (str "not an http or https url: " (record-ref req url)))) + (let* ((message (Message. (Verb. (method-name (record-ref req method))) url)) + (content (attach-body! message (record-ref req body)))) + (match (add-headers! message content (record-ref req headers)) + ((Err e) (Err e)) + ((Ok added) (Ok message)))))))) + + +;; =========================================================================== +;; The answer +;; =========================================================================== + +;; Opaque: the CLR reply inside owns a live connection, and handing it out +;; would make closing it somebody else's problem by accident. +(type + (: Response #:opaque + (Record (: status int) + (: reason string) + (: headers (Map string string)) + (: raw Reply)))) + +(: (-> (Seq System.String) string)) +(defun (join-values values) + (string-join (loop (:for value values) (:acc all (vecing (clr-string->string value)))) ", ")) + +;; Makes both headers and into lower-cased, folded with ", ". Header names are +;; case-insensitive on the wire +;; +;; Set-Cookie is the exception the RFC makes and this does not: folded cookies +;; cannot be taken apart again. When something here reads cookies it needs a +;; call of its own. +(: (-> (Seq (Pairing System.String (Seq System.String))) (Map string string) (Map string string))) +(defun (collect-headers headers into) + (loop (:for header headers) + (:acc all (folding into + (map-set all + (string-downcase (.-Key header)) + (join-values (.-Value header))))))) + +(: (-> Reply Response)) +(defun (response-of raw) + (Response (status (cast int (reply-status raw))) + (reason (text-or-empty (reply-reason raw))) + (headers (collect-headers + (cast (Seq (Pairing System.String (Seq System.String))) (content-headers (reply-content raw))) + (collect-headers + (cast (Seq (Pairing System.String (Seq System.String))) (reply-headers raw)) + (map-empty)))) + (raw raw))) + +(: (-> Response int)) +(defun (response-status r) (record-ref r status)) + +;; 2xx. Nothing here throws on a 404: which codes are failures is the caller's +;; business, and plenty of them are not. +(: (-> Response bool)) +(defun (response-ok? r) + (def code (record-ref r status)) + (and (>= code 200) (< code 300))) + +;; Empty over HTTP/2, which has no reason phrase. Do not parse it. +(: (-> Response string)) +(defun (response-reason r) (record-ref r reason)) + +(: (-> Response (Map string string))) +(defun (response-headers r) (record-ref r headers)) + +;; The name is lower-cased for you, so "Content-Type" and "content-type" +;; find the same header. +(: (-> Response string (Option string))) +(defun (response-header r name) + (map-try-ref (record-ref r headers) (string-downcase name))) + + +;; =========================================================================== +;; The body +;; =========================================================================== + +(: (-> Content ClrStream)) +(defbjouble (body-stream content) + (#:sync (clr-stream content)) + (#:bjo (clr-stream-await content))) + +;; What the server said the body was encoded as: any name or alias +;; `encoding-named` knows, and UTF-8 when it named none. A charset nothing +;; here knows is an error rather than a guess, since text decoded by the wrong +;; encoding is wrong without a word; `response-bytes-port` reads such a body. +(: (-> Response Encoding)) +(defun (charset-of r) + (match (response-header r "content-type") + (None utf-8) + ((Some content-type) + (match (charset-name content-type) + (None utf-8) + ((Some name) + (match (encoding-named name) + ((Some found) found) + (None + (raise (http-error + (str "the response declares charset=" name + ", which is not an encoding (text encodings) knows. " + "response-bytes-port reads its bytes as they are")))))))))) + +(: (-> string (Option string))) +(defun (charset-name content-type) + (loop (:for part (string-split content-type ";")) + (:acc found (folding None + (let ((trimmed (string-trim part))) + (if (string-starts-with? (string-downcase trimmed) "charset=") + (Some (strip-quotes (string-trim (substring-after trimmed "=")))) + found)))))) + +(: (-> string string string)) +(defun (substring-after s sep) + (def parts (string-split s sep)) + (if (< (length parts) 2) + "" + (vec-ref parts 1))) + +;; charset="utf-8" is as legal as charset=utf-8, and an encoding name has +;; no quotes of its own to protect. +(: (-> string string)) +(defun (strip-quotes s) + (string-replace s "\"" "")) + +;; The body as a port, so that `read-line`, `port->seq`, `port->stream` and +;; `read-json` all work on it. Reading suspends inside a bjoroutine and blocks +;; outside one, exactly as reading a file does. +;; +;; Decoded by the charset the server declared, and only by it: a byte order +;; mark is not treated specially. A UTF-8 body is the port's bytes as they +;; arrived; another charset is transcoded to UTF-8 on the way in. A charset +;; that no encoding has raises. +(: (-> Response InputPort)) +(defun (response-port r) + (def body (reply-content (record-ref r raw))) + (text-port-over (body-stream body) (charset-of r))) + +;; The body's bytes as they arrived, whatever the charset says, as Racket's +;; HTTP client hands them over: for a body that is not text, or text to decode +;; yourself with `reencode-input-port` or `bytes->string`. +(: (-> Response InputPort)) +(defun (response-bytes-port r) + (def body (reply-content (record-ref r raw))) + (text-port-over (body-stream body) utf-8)) + +(: (-> Response string)) +(defun (response-text r) + (read-all (response-port r))) + +;; A response holds a connection until it is closed. `ignore` does this, so a +;; response dropped in statement position releases it rather than leaking it. +(: (-> Response void)) +(defun (response-close r) + (.Dispose (record-ref r raw))) + +(impl (Discard Response) + (defun (ignore r) (response-close r))) + + +;; =========================================================================== +;; Sending one +;; =========================================================================== + +;; The request as a cml event: nothing is sent until the branch is reached, and a +;; branch that loses cancels the request it started. +;; +;; `guard` is not decoration. An `HttpRequestMessage` may be sent once, and an +;; event may be synced any number of times — every retry, every losing branch +;; that is tried again. Building the message inside the generator is what makes +;; the second sync work. +(: (-> Request (Event (Result Exception Response)))) +(defun (fetching req) + (guard + (fun () + (match (message-of req) + ((Err e) (always (Err e))) + ((Ok message) + (wrap (task->event (send (parameter-ref current-http-client) message headers-read)) + (fun (outcome) + (match outcome + ((Ok raw) (Ok (response-of raw))) + ((Err e) (Err e)))))))))) + +;; Send it and wait for the answer, in whichever colour the caller is. +(: (-> Request (Result Exception Response))) +(defbjouble (fetch req) + (#:sync (sync/blocking (fetching req))) + (#:bjo (sync (fetching req)))) + + +;; =========================================================================== +;; Scoped +;; =========================================================================== + +;; `(with-response (r (fetch (http-get url))) body ...)` — closes the response on +;; the way out, however the body leaves. +(def/macro (with-response form inject compare) + (syntax-match form + ((_ (name spec) first body ...) + #'(let () + (def answer ,spec) + (try + (let ((,name answer)) + ,first + ,@body) + #:finally (match answer + ((Ok got) (response-close got)) + ((Err failed) unit))))) + (bad (syntax-error bad "with-response names a response and takes a body: (with-response (r (fetch (http-get url))) body ...)")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/inbox.bjo — the door .NET code knocks on. +;; +;; A thread that is not a fiber — Kestrel's, a timer's, a `FileSystemWatcher`'s +;; — hands an item to an inbox and returns. A fiber receives it, with its own +;; dynamic environment: its handlers, its parameters, its cancellation scope. +;; +;; THE RULE FOR A LAMBDA .NET CALLS. It may call the functions in this module +;; and nothing else. Everything here that a foreign thread uses is an ordinary +;; `defun`, not a bjoroutine, and reads neither parameters nor effect handlers — +;; because a thread that is not a fiber has none. A `(log ...)` written in such +;; a lambda does not fail: it quietly goes to the default handler, even inside a +;; `with-handler`. Post the work, and do the logging in the fiber. +;; +;; See `BjolangRuntime/Cml/Inbox.cs` for how it is done, and the Readme's Inbox +;; section for what to write. + +(export + ;; Making one, and the policy it needs. `WhenFull` is transparent: its two + ;; cases are written wherever a `make-inbox` is. + WhenFull + make-inbox + + ;; Putting something in. `inbox-post!` and `inbox-send` are safe from any + ;; thread; `inbox-send-evt` is the fiber's form. + inbox-post! inbox-send inbox-send/token inbox-send-evt + + ;; Taking it out, and the end of the stream. + inbox-recv inbox-closed + + ;; Ending it. + inbox-close! inbox-fail! inbox-abort! + + ;; What was lost, and subscribing to an `IObservable`. + inbox-dropped-count inbox-observer make-observer IObserver + + ;; Request and reply. + inbox-call inbox-call/token + inbox-try-call inbox-try-call/token + call-request call-reply! call-fail! call-abandoned + + ;; The token an `inbox-call/token` is given: a caller's own, such as + ;; Kestrel's `RequestAborted`. + CancellationToken) + +(import (std prelude)) + +(:doc #:file "inbox.bjodoc") + +(import/class + (CancellationToken (: System.Threading.CancellationToken)) + ((IObserver %a) (: System.IObserver))) + +;; `Inbox`, `Call`, `Task` and `Event` are spelled by the compiler: they are +;; runtime types the language has a name for, like `Chan` and `Promise`. See +;; `DotNetInterop.genericTypeCorrespondence`. +(import/extern + (clr-make-inbox (: Bjoml.InboxModule.Make (-> int bool (Inbox %a)))) + (clr-post (: Bjoml.InboxModule.Post (-> (Inbox %a) %a bool))) + (clr-send (: Bjoml.InboxModule.Send (-> (Inbox %a) %a (Task Unit)))) + (clr-send/token (: Bjoml.InboxModule.SendWithToken (-> (Inbox %a) %a CancellationToken (Task Unit)))) + (clr-send-evt (: Bjoml.InboxModule.SendEvent (-> (Inbox %a) %a (Event Unit)))) + (clr-recv (: Bjoml.InboxModule.Receive (-> (Inbox %a) (Event %a)))) + (clr-closed (: Bjoml.InboxModule.Closed (-> (Inbox %a) (Event (Option Exception))))) + (clr-close! (: Bjoml.InboxModule.Close (-> (Inbox %a) void))) + (clr-fail! (: Bjoml.InboxModule.Fail (-> (Inbox %a) Exception void))) + (clr-abort! (: Bjoml.InboxModule.Abort (-> (Inbox %a) void))) + (clr-dropped-count (: Bjoml.InboxModule.DroppedCount (-> (Inbox %a) int))) + (clr-make-observer (: Bjoml.InboxModule.MakeObserver + (-> (-> %a void) (-> Exception void) (-> void) (IObserver %a)))) + + (clr-call (: Bjoml.InboxModule.CallInbox (-> (Inbox (Call %q %r)) %q (Task %r)))) + (clr-call/token (: Bjoml.InboxModule.CallInboxWithToken + (-> (Inbox (Call %q %r)) %q CancellationToken (Task %r)))) + (clr-try-call (: Bjoml.InboxModule.TryCallInbox + (-> (Inbox (Call %q %r)) %q (Option (Task %r))))) + (clr-try-call/token (: Bjoml.InboxModule.TryCallInboxWithToken + (-> (Inbox (Call %q %r)) %q CancellationToken (Option (Task %r))))) + (clr-call-request (: Bjoml.InboxModule.CallRequest (-> (Call %q %r) %q))) + (clr-call-reply! (: Bjoml.InboxModule.CallReply (-> (Call %q %r) %r bool))) + (clr-call-fail! (: Bjoml.InboxModule.CallFail (-> (Call %q %r) Exception bool))) + (clr-call-abandoned (: Bjoml.InboxModule.CallAbandoned (-> (Call %q %r) (Event Unit))))) + +;; --------------------------------------------------------------------------- +;; Making one +;; --------------------------------------------------------------------------- + +;; What a full inbox does to an item `inbox-post!` or `inbox-call` brings. +;; +;; It says nothing about `inbox-send` or `inbox-send-evt`: those wait for room +;; and never drop anything. +(type (: WhenFull (Union DropNewest DropOldest))) + +;; An inbox holding at most `capacity` queued items, with `capacity > 0`. +;; +;; There is no unbounded inbox and no default capacity, because an event source +;; faster than its consumer must not grow memory without limit — a +;; `FileSystemWatcher` during a large `git checkout` fires thousands of events. +;; The number is yours to pick: for an HTTP server it is how many requests may +;; wait before the answer is 503. +(: (-> int WhenFull (Inbox %a))) +(defun (make-inbox capacity policy) + (clr-make-inbox capacity (match policy + (DropNewest #f) + (DropOldest #t)))) + +;; --------------------------------------------------------------------------- +;; Putting something in +;; --------------------------------------------------------------------------- + +;; Hands `x` over without ever waiting. Safe from any thread, including one with +;; no fiber context at all. +;; +;; `#t` when the item was queued or handed straight to a waiting receiver. `#f` +;; when the inbox is closed, or full under `DropNewest`. +(: (-> (Inbox %a) %a bool)) +(defun (inbox-post! ib x) (clr-post ib x)) + +;; The same handover for a caller that can wait, because it returns a `Task`: +;; the task completes once `x` is in. On a full inbox it stays pending until a +;; receiver makes room, which is what makes this real backpressure — each +;; waiting producer holds exactly one item. +;; +;; On a closed inbox, or an `inbox-abort!` while waiting, the task is cancelled. +(: (-> (Inbox %a) %a (Task Unit))) +(defun (inbox-send ib x) (clr-send ib x)) + +;; The same, with the caller's own token. If it fires while the send is waiting +;; for room, the item never enters the queue and the task is cancelled. +(: (-> (Inbox %a) %a CancellationToken (Task Unit))) +(defun (inbox-send/token ib x token) (clr-send/token ib x token)) + +;; The fiber's form of a send, as a CML event: it commits when there is room, +;; and in a `choose` that another branch wins, `x` is *not* queued. +;; +;; That is why a fiber gets an event rather than the task above. A task cannot +;; be withdrawn, so `(task->event (inbox-send ib x))` in a losing branch would +;; queue `x` anyway, later, with nobody expecting it. +;; +;; On a closed inbox it never commits. A fiber that may meet one should choose +;; it together with `inbox-closed`. +(: (-> (Inbox %a) %a (Event Unit))) +(defun (inbox-send-evt ib x) (clr-send-evt ib x)) + +;; --------------------------------------------------------------------------- +;; Taking it out +;; --------------------------------------------------------------------------- + +;; A real CML event: in a `choose` that another branch wins, nothing is +;; consumed and the item stays for the next receiver, exactly as a channel +;; receive does. +;; +;; On a closed, empty inbox it never fires. `inbox-closed` is how the end is +;; noticed. +(: (-> (Inbox %a) (Event %a))) +(defun (inbox-recv ib) (clr-recv ib)) + +;; Fires once the inbox is closed *and* empty, and from then on commits +;; immediately on every `sync`. `None` after `inbox-close!` or `inbox-abort!`, +;; and `(Some e)` after `(inbox-fail! ib e)`. +;; +;; The usual receive loop is +;; +;; (sync (choose (wrap (inbox-recv ib) Some) +;; (wrap (inbox-closed ib) (fun (_) None)))) +;; +;; It is a separate event rather than an `Option` from `inbox-recv` so that code +;; which never closes its inbox — an HTTP worker stopped by cancelling its scope +;; — keeps the simple form. +(: (-> (Inbox %a) (Event (Option Exception)))) +(defun (inbox-closed ib) (clr-closed ib)) + +;; --------------------------------------------------------------------------- +;; Ending it +;; --------------------------------------------------------------------------- + +;; A graceful end. Later posts answer `#f` and later sends and calls get a +;; cancelled task, but what is already queued stays and receivers can still take +;; it; a sender already waiting for room still gets in. `inbox-closed` fires +;; once all of that has been received. +(: (-> (Inbox %a) void)) +(defun (inbox-close! ib) (clr-close! ib)) + +;; The same as `inbox-close!`, but `inbox-closed` yields `(Some e)`. +;; +;; The error is kept beside the queue rather than in it, so no full policy can +;; drop the one item that says what went wrong. +(: (-> (Inbox %a) Exception void)) +(defun (inbox-fail! ib e) (clr-fail! ib e)) + +;; A hard stop: closed, and everything queued dropped. Queued calls and waiting +;; `inbox-send` tasks are cancelled, a waiting `inbox-send-evt` never commits, +;; and `inbox-closed` then fires with `None` — or with the earlier error, if +;; `inbox-fail!` came first. +(: (-> (Inbox %a) void)) +(defun (inbox-abort! ib) (clr-abort! ib)) + +;; --------------------------------------------------------------------------- +;; What was lost +;; --------------------------------------------------------------------------- + +;; How many items the two dropping policies have thrown away, so that a program +;; can report its losses: "dropped 37 progress updates". `inbox-abort!` is not +;; counted — that is the program stopping, not the inbox overflowing. +(: (-> (Inbox %a) int)) +(defun (inbox-dropped-count ib) (clr-dropped-count ib)) + +;; --------------------------------------------------------------------------- +;; Subscribing to an IObservable +;; --------------------------------------------------------------------------- + +;; An `IObserver` made of three functions. Subscribing to an `IObservable` +;; needs an object implementing the interface, which Bjolang cannot write, and +;; base .NET has no `Subscribe(Action)`. +;; +;; All three run on the observable's thread, so they follow the rule every +;; foreign callback does: post, close or fail, and nothing else. +(: (-> (-> %a void) (-> Exception void) (-> void) (IObserver %a))) +(defun (make-observer on-next on-error on-completed) + (clr-make-observer on-next on-error on-completed)) + +;; The inbox as an observer, for `(.Subscribe observable (inbox-observer ib))`. +;; `OnNext` posts, so the inbox's policy applies; `OnError` fails it; and +;; `OnCompleted` closes it. +(: (-> (Inbox %a) (IObserver %a))) +(defun (inbox-observer ib) + (make-observer (fun (x) (ignore (inbox-post! ib x))) + (fun (e) (inbox-fail! ib e)) + (fun () (inbox-close! ib)))) + +;; --------------------------------------------------------------------------- +;; Request and reply +;; --------------------------------------------------------------------------- +;; +;; A request/reply inbox is an inbox whose items are `Call`s, so capacity, the +;; full policy, closing and receiving work exactly as they do for any other. +;; +;; A fiber that takes a `Call` must answer it — typically in `#:finally`, so +;; that a raise answers it too. Otherwise the .NET caller waits forever: a +;; browser request that never finishes. + +;; Posts a call and hands back the task its caller waits on. Safe from any +;; thread. When the inbox is full the policy decides, and the task faults with a +;; `Bjoml.InboxFullException`; when it is closed, the task is cancelled. +(: (-> (Inbox (Call %q %r)) %q (Task %r))) +(defun (inbox-call ib request) (clr-call ib request)) + +;; The same, with the caller's own `CancellationToken` — Kestrel's +;; `RequestAborted`. What the token does depends on where the call is: +;; +;; * still queued — it is removed and the task is cancelled, so no fiber ever +;; sees a call nobody is waiting for; +;; * already taken — the task is *not* completed. Only `call-reply!` and +;; `call-fail!` complete it, and `call-abandoned` fires instead, for the +;; receiver to do as it sees fit. This is what keeps an HTTP handler from +;; writing into a context that has already been recycled into the next +;; request. +(: (-> (Inbox (Call %q %r)) %q CancellationToken (Task %r))) +(defun (inbox-call/token ib request token) (clr-call/token ib request token)) + +;; The same two, with the refusal as an answer rather than as a fault on the +;; task: `None` when the inbox was full under `DropNewest` or closed, and +;; nothing was posted. +;; +;; This is `inbox-post!`'s `#f` for a call, and it is here because a faulted +;; task is the wrong shape for the caller who most needs to know. A .NET +;; callback that hands its task straight back — an HTTP middleware is the case +;; — never awaits it, so it cannot see the fault, and the exception goes to +;; whatever the framework does with an unhandled one. A web server that answers +;; 500 where it meant 503 is telling the client to give up rather than to come +;; back: +;; +;; (match (inbox-try-call/token requests ctx (request-aborted ctx)) +;; ((Some answering) (cast Task answering)) +;; (None (set-status! (response-of ctx) 503) completed-task)) +;; +;; Under `DropOldest` the answer is `(Some ...)`: this call went in, and it is +;; an *older* one whose task faults. `inbox-post!` means the same thing by `#t`. +(: (-> (Inbox (Call %q %r)) %q (Option (Task %r)))) +(defun (inbox-try-call ib request) (clr-try-call ib request)) + +(: (-> (Inbox (Call %q %r)) %q CancellationToken (Option (Task %r)))) +(defun (inbox-try-call/token ib request token) (clr-try-call/token ib request token)) + +;; What the caller asked. +(: (-> (Call %q %r) %q)) +(defun (call-request c) (clr-call-request c)) + +;; Answers the call. The first completion wins and returns `#t`; a later one +;; returns `#f` and does nothing, so a timeout and a handler can race to answer +;; safely. +(: (-> (Call %q %r) %r bool)) +(defun (call-reply! c value) (clr-call-reply! c value)) + +;; The same, with an exception for the caller to receive. +(: (-> (Call %q %r) Exception bool)) +(defun (call-fail! c e) (clr-call-fail! c e)) + +;; Fires once the caller's token has fired *after* this call was taken. It never +;; fires for a call made without a token, and never for one cancelled while it +;; was still queued — nobody had it to be told. +(: (-> (Call %q %r) (Event Unit))) +(defun (call-abandoned c) (clr-call-abandoned c)) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/interop.bjo — what .NET wants and Bjolang cannot write. +;; +;; A Bjolang function is a Func. A .NET API that wants an Action, an +;; EventHandler or a delegate type of its own gets one from `as-delegate`. +;; A parameter that wants null gets it from `clr-default`. The work is in the +;; runtime (`BjoInterop.cs`). +(import (std prelude)) + +(:doc #:file "interop.bjodoc") + +(export Delegate ClrType + as-delegate as-delegate/type + add-event-handler! remove-event-handler! + clr-default) + +(import/class + (Delegate (: System.Delegate)) + (ClrType (: System.Type))) + +(import/extern + (clr-default (: Bjolang.Runtime.BjoInterop.Default (-> %a))) + (as-delegate (: Bjolang.Runtime.BjoInterop.As (-> %f %d))) + (as-delegate/type (: Bjolang.Runtime.BjoInterop.Convert (-> ClrType %f Delegate))) + (add-event-handler! (: Bjolang.Runtime.BjoInterop.AddEventHandler + (-> %t string %f Delegate))) + (remove-event-handler! (: Bjolang.Runtime.BjoInterop.RemoveEventHandler + (-> %t string Delegate void)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/maths.bjo — Math functions. + +;; Import Eq trait for `=`. +(import (std eq)) + +(:doc #:file "maths.bjodoc") + +(export + ;; Num + abs sign min max clamp zero? even? odd? + sqr positive? negative? + ;; Integer division, and the integer functions + quotient remainder modulo quotient/remainder + gcd lcm expt integer-sqrt + ;; Bits + Bits popcount trailing-zeros leading-zeros + ;; Doubles: what they are, and pi + nan? infinite? finite? pi + ;; Exponential and logarithmic + cbrt exp pow loge log-base log2 log10 + ;; Rounding, and rounding to an int + floor ceiling truncate round + exact-floor exact-ceiling exact-truncate exact-round + ;; Trigonometric + sin cos tan asin acos atan atan2 + ;; Hyperbolic + sinh cosh tanh asinh acosh atanh) + +;; --------------------------------------------------------------------------- +;; Num +;; --------------------------------------------------------------------------- + +;; `Num` interface. A number can be compared with `<`: `INumber` extends +;; `IComparisonOperators`, which is `Ordered`. +(def/trait (Num %a) + (where (Ordered %a)) + (#:clr-constraint (System.Numerics.INumber %a)) + (: abs (-> %a %a) #:clr-member Abs) + (: sign (-> %a int) #:clr-member Sign) + (: min (-> %a %a %a) #:clr-member Min) + (: max (-> %a %a %a) #:clr-member Max) + (: clamp (-> %a %a %a %a) #:clr-member Clamp) + ;; Parity tests. + (: zero? (-> %a bool) #:clr-member IsZero) + (: even? (-> %a bool) #:clr-member IsEvenInteger) + (: odd? (-> %a bool) #:clr-member IsOddInteger)) + + +;; Todo: `min`/`max` over a sequence. + +;; At any unit: the square of a length is an area. +(: (-> (num %a %u) (num %a %u^2)) (where (Num %a))) +(defun (sqr x) (* x x)) + +;; Not `INumber.IsPositive`, which holds for zero. At any unit: zero is a +;; number of every unit. +(: (-> (num %a %u) bool) (where (Num %a))) +(defun (positive? x) (> x 0)) + +(: (-> (num %a %u) bool) (where (Num %a))) +(defun (negative? x) (< x 0)) + +;; --------------------------------------------------------------------------- +;; Integer division +;; --------------------------------------------------------------------------- +;; +;; Racket's names. `quotient` and `remainder` are `/` and `%` at an integer +;; type, named, so that they can be passed and so that a reader sees which +;; convention is meant; `modulo` is the one the operators do not give. R7RS's +;; `truncate/` and `floor/` families are the same operations under other names. + +(import/class + (Exception (: System.Exception)) + (ArgumentOutOfRangeException (: System.ArgumentOutOfRangeException))) + +(: (-> string string %a)) +(defun (out-of-range parameter why) + (raise (cast Exception (ArgumentOutOfRangeException. parameter why)))) + +(: (-> %a %a %a) (where (Integral %a))) +(defun (quotient a b) (/ a b)) + +(: (-> %a %a %a) (where (Integral %a))) +(defun (remainder a b) (% a b)) + +;; The remainder moved into the divisor's sign: a nonzero remainder of the +;; other sign is one divisor away from it. +(: (-> %a %a %a) (where (Integral %a))) +(defun (modulo a b) + (let ((r (% a b))) + (if (and (not (zero? r)) (not (= (< r 0) (< b 0)))) + (+ r b) + r))) + +(: (-> %a %a (Tuple %a %a)) (where (Integral %a))) +(defun (quotient/remainder a b) + (Tuple (/ a b) (% a b))) + +;; --------------------------------------------------------------------------- +;; The integer functions +;; --------------------------------------------------------------------------- + +;; Euclid's algorithm, on the magnitudes. +(: (-> %a %a %a) (where (Integral %a))) +(defun (gcd a b) + (let go ((x (abs a)) (y (abs b))) + (if (zero? y) x (go y (% x y))))) + +;; Divided before it is multiplied, so that the product overflows only when the +;; answer does. +(: (-> %a %a %a) (where (Integral %a))) +(defun (lcm a b) + (if (or (zero? a) (zero? b)) + 0 + (abs (* (/ a (gcd a b)) b)))) + +;; By squaring: one multiplication per bit of the exponent, and one more for +;; each bit that is set. +(: (-> %a int %a) (where (Integral %a))) +(defun (expt base n) + (when (< n 0) + (out-of-range "n" "expt: the exponent is negative, and an integer power has to be one.")) + (let go ((b base) (k n) (acc 1)) + (if (= k 0) + acc + (go (* b b) (shift-right k 1) (if (odd? k) (* acc b) acc))))) + +;; Newton's method from above. The start, n/2 + 1, is at least the root for +;; every n of 2 or more, and nothing on the way adds past n, so it cannot +;; overflow even at the type's largest value. +(: (-> %a %a) (where (Integral %a))) +(defun (integer-sqrt n) + (when (< n 0) + (out-of-range "n" "integer-sqrt: the number is negative.")) + (if (< n 2) + n + (let go ((x (+ (/ n 2) 1))) + (let ((y (/ (+ x (/ n x)) 2))) + (if (>= y x) x (go y)))))) + +;; --------------------------------------------------------------------------- +;; Bits +;; --------------------------------------------------------------------------- + +;; The bits of an integer, answered by .NET's generic math for every integer +;; type: `(popcount 13)` is 3 and `(popcount 13L)` is 3L. +(def/trait (Bits %a) + (#:clr-constraint (System.Numerics.IBinaryInteger %a)) + (: popcount (-> %a %a) #:clr-member PopCount) + (: trailing-zeros (-> %a %a) #:clr-member TrailingZeroCount) + (: leading-zeros (-> %a %a) #:clr-member LeadingZeroCount)) + +;; --------------------------------------------------------------------------- +;; The double functions +;; --------------------------------------------------------------------------- + +;; What a double is. +(import/extern + (nan? (: System.Double.IsNaN)) + (infinite? (: System.Double.IsInfinity)) + (finite? (: System.Double.IsFinite)) + (pi (: System.Math.PI #:get))) + +;; Exponential and logarithmic. +;; +;; One name per base, and none of them overloaded. `System.Math.Log` is two +;; methods rather than one — the natural logarithm and the log to a given base — +;; and binding both to `log` made the meaning depend on how many arguments the +;; call happened to pass. `loge` completes `log2` and `log10`, and `log-base` +;; says what its second argument is. +(import/extern + (clr-cbrt (: System.Math.Cbrt (-> double double))) + (exp (: System.Math.Exp)) + (pow (: System.Math.Pow)) + (loge (: System.Math.Log (-> double double))) + (log-base (: System.Math.Log (-> double double double))) + (log2 (: System.Math.Log2)) + (log10 (: System.Math.Log10))) + +;; The cube root of a number in m^3 is in m. +(: cbrt (-> (double %u^3) (double %u))) +(defun (cbrt x) (cast (double %u) (clr-cbrt (cast double x)))) + +;; Rounding. +(import/extern + (floor (: System.Math.Floor)) + (ceiling (: System.Math.Ceiling)) + (truncate (: System.Math.Truncate)) + (round (: System.Math.Round))) + +;; Rounding to an int. `Convert.ToInt32` refuses NaN, the infinities and +;; anything out of an int's range with an OverflowException, where a cast +;; would answer a number. The value it is given is already whole, so its own +;; rounding never applies. +(import/extern + (double->int (: System.Convert.ToInt32 (-> double int)))) + +(: (-> double int)) +(defun (exact-floor x) (double->int (floor x))) + +(: (-> double int)) +(defun (exact-ceiling x) (double->int (ceiling x))) + +(: (-> double int)) +(defun (exact-truncate x) (double->int (truncate x))) + +(: (-> double int)) +(defun (exact-round x) (double->int (round x))) + +;; Trigonometric. +(import/extern + (sin (: System.Math.Sin)) + (cos (: System.Math.Cos)) + (tan (: System.Math.Tan)) + (asin (: System.Math.Asin)) + (acos (: System.Math.Acos)) + (atan (: System.Math.Atan)) + (atan2 (: System.Math.Atan2))) + +;; Hyperbolic. +(import/extern + (sinh (: System.Math.Sinh)) + (cosh (: System.Math.Cosh)) + (tanh (: System.Math.Tanh)) + (asinh (: System.Math.Asinh)) + (acosh (: System.Math.Acosh)) + (atanh (: System.Math.Atanh))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/monad.bjo +;; +;; Monad trait, stock instances, and (do ...) macro. +(import (std prelude)) +(import (std syntax-match)) + +(:doc #:file "monad.bjodoc") + +(export pure bind) + +;; Monad trait definition. +(def/trait (Monad (%m %a)) + (: pure (-> %a (%m %a))) + (: bind (-> (%m %a) (-?-> %a (%m %b)) (%m %b)))) + +;; `k` is handed to `list-map` rather than called inside a lambda. A lambda's +;; text is fixed and its colour is not — `(fun ...)` is wrong in the suspending +;; copy and `(bjoroutine ...)` in the ordinary one — so a callback written here +;; could only ever be one of them. +(impl (Monad (List %a)) + (defun (pure x) (Cons x Nil)) + (defun (bind m k) + (list-foldr (fun (xs ys) (list-append xs ys)) Nil (list-map k m)))) + +(impl (Monad (Option %a)) + (defun (pure x) (Some x)) + (defun (bind m k) + (match m + ((Some v) (k v)) + (None None)))) + +;; Short-circuits on Err. +(impl (Monad (Result %e %a)) + (defun (pure x) (Ok x)) + (defun (bind m k) + (match m + ((Ok v) (k v)) + ((Err e) (Err e))))) + +;; --------------------------------------------------------------------------- +;; The block +;; --------------------------------------------------------------------------- +;; Macro for (do ...) blocks. +(def/macro (do form inject compare) + (syntax-match form + ((_ step ...) (do-block step form)) + (bad (syntax-error bad "do takes a block of forms: (do (:bind x m) ... (:return e))")))) + +;; Builds the block chain. +(: (-> (List Syntax) Syntax Syntax)) +(defun (do-block steps whole) + (match steps + (Nil (syntax-error whole "A (do ...) block is empty. It has to answer something.")) + ((Cons step Nil) (do-last step)) + ((Cons step rest) (do-step step (do-block rest whole))))) + +(: (-> Syntax bool)) +(defun (identifier? s) + (match s + ((SSym _) #t) + (_ #f))) + +;; Checks if form is keyword-headed. +(: (-> Syntax bool)) +(defun (clause? s) + (match s + ((SList (Cons head _)) + (match head + ((SKey _) #t) + (_ #f))) + (_ #f))) + +(: (-> Syntax Syntax)) +(defun (unknown-clause step) + (syntax-error step "This is not a (do ...) form. They are :bind, :match, :let, :then and :return.")) + +;; The block's last form, which is its value. +(: (-> Syntax Syntax)) +(defun (do-last step) + (syntax-match step + ((:return expr) #'(pure ,expr)) + ((:return bad ...) (syntax-error step "(:return expr) takes one expression.")) + ((:bind bad ...) + (syntax-error step "A (do ...) block cannot end with (:bind ...). Its last form is the block's value.")) + ((:match bad ...) + (syntax-error step "A (do ...) block cannot end with (:match ...). Its last form is the block's value.")) + ((:let bad ...) + (syntax-error step "A (do ...) block cannot end with (:let ...). Its last form is the block's value.")) + ((:then bad ...) + (syntax-error step "A (do ...) block cannot end with (:then ...). Its last form is the block's value.")) + ;; Reports keyword errors. + (last (if (clause? last) (unknown-clause last) last)))) + +;; A step, and everything after it. +(: (-> Syntax Syntax Syntax)) +(defun (do-step step rest) + (syntax-match step + ;; :bind takes an identifier. + ((:bind name expr) + (if (identifier? name) + #'(bind ,expr (fun (,name) ,rest)) + (syntax-error step "(:bind name expr) takes a plain identifier, not a pattern."))) + ((:bind bad ...) (syntax-error step "(:bind name expr) takes an identifier and one expression.")) + + ;; :match takes a pattern. + ((:match pattern expr) + #'(bind ,expr (fun (bound) (match bound (,pattern ,rest))))) + ((:match bad ...) (syntax-error step "(:match pattern expr) takes one pattern and one expression.")) + + ((:let name expr) + (if (identifier? name) + #'(let ((,name ,expr)) ,rest) + (syntax-error step "(:let name expr) takes a plain identifier, not a pattern."))) + ((:let bad ...) (syntax-error step "(:let name expr) takes an identifier and one expression.")) + + ;; :then for variable-less bind. + ((:then expr) #'(bind ,expr (fun (_) ,rest))) + ((:then bad ...) (syntax-error step "(:then expr) takes one expression.")) + + ((:return bad ...) + (syntax-error step "(:return ...) is the last form of its (do ...) block, and this one is not.")) + + ;; Plain form statement. + (other (if (clause? other) (unknown-clause other) #'(let ((_ ,other)) ,rest))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; A double-ended queue that is written in place. +;; +;; Two lists: `front` holds the front of the deque in order, `back` holds the +;; rear reversed, so the deque's contents are `front ++ reverse back`. Pushing +;; at either end is a cons. Popping takes the head of the list on that end, and +;; when that list is empty the other one is split to refill it. +;; +;; **This is the mutable one.** `(deque-push-back! d x)` writes `d`; it answers +;; nothing, and everyone holding `d` sees the change. The persistent shape of +;; the same structure is what `List` and `Vec` already are, and a program that +;; wants a queue it can keep old versions of should use one of those with an +;; explicit fold. What this buys is the thing they cannot: a worklist that a +;; recursive walk pushes onto and a loop pops from, without either of them +;; having to thread a new deque back out. +;; +;; **Split in half, not moved wholesale.** Refilling an empty side by moving +;; *all* of the other one is the textbook queue trick, and it is quadratic on a +;; deque: pop one end, pop the other, and every pop moves the whole structure +;; back again. Moving half leaves both ends stocked, so the next n operations at +;; either end are free. Every operation is O(1) amortized. +;; +;; **Not a key.** `Deque` has mutable fields, so `eq-hash` refuses and a deque +;; cannot go in a `Map` or a `HashSet`. See the `Eq` implementation at the bottom. +(import (std prelude)) + +(:doc #:file "deque.bjodoc") + +(export + Deque + deque deque-length deque-empty? deque-clear! + deque-push-front! deque-push-back! + deque-pop-first! deque-pop-last! deque-try-pop-first! deque-try-pop-last! + deque-peek-front! deque-peek-back! + deque-contains? + list->deque deque->list deque-fold deque-for-each + deque->str) + +;; `#:opaque`, so the two lists and the count stay here: which half an element +;; is in is a fact about the representation and not about the deque, and a +;; caller that could read `front` would be reading an implementation detail that +;; every push and pop rearranges. The name still crosses — a signature may say +;; `(Deque int)`, and the trait implementations below still dispatch — so what +;; is lost is only the ability to take one apart. +;; +;; That also settles the mutable fields. A mutable field may be written only by +;; the module that declared it; opacity makes reading one the same rule, and the +;; operations below are the whole interface. +;; +;; `size` is carried rather than counted so that `deque-length` is O(1). It is +;; the one field that is not derivable from the others in constant time. +(type + (: (Deque %a) #:opaque + (Record (: front (List %a) #:mutable) + (: back (List %a) #:mutable) + (: size int #:mutable)))) + +(: (-> (Deque %a))) +(defun (deque) + (Deque (front Nil) (back Nil) (size 0))) + +(: (-> (Deque %a) int)) +(defun (deque-length d) (record-ref d size)) + +(: (-> (Deque %a) bool)) +(defun (deque-empty? d) (= (record-ref d size) 0)) + +(: (-> (Deque %a) void)) +(defun (deque-clear! d) + (record-set! d (front Nil) (back Nil) (size 0))) + +;; --------------------------------------------------------------------------- +;; Splitting +;; --------------------------------------------------------------------------- +;; +;; The prelude has no `list-take` or `list-drop`, deliberately — they invite the +;; O(n) indexing a cons list cannot support — so the two walks a split needs are +;; here. Both run once per rebalance, which is once per n operations. + +;; Collected reversed and turned once, so a long prefix does not recurse once +;; per element. +(: (-> int (List %a) (List %a))) +(defun (take-n k xs) + (let go ((k k) (rest xs) (acc Nil)) + (if (= k 0) + (list-reverse acc) + (match rest + (Nil (list-reverse acc)) + ((Cons x more) (go (- k 1) more (Cons x acc))))))) + +(: (-> int (List %a) (List %a))) +(defun (drop-n k xs) + (if (= k 0) + xs + (match xs + (Nil Nil) + ((Cons _ rest) (drop-n (- k 1) rest))))) + +;; `front` is empty and everything is on `back`, reversed. Keep the rear half +;; where it is and hand the front half over, in order. +;; +;; `k` elements stay on `back`, and since `back` is the deque reversed its first +;; `k` are the deque's *last* `k` — so the ones to move are the rest, reversed +;; back into order. With `k = n / 2` the front is left non-empty for any n ≥ 1: +;; a single element has `k = 0` and moves across whole. +;; +;; Both fields are written by one `record-set!`, so the deque is never briefly +;; holding an element twice. +(: (-> (Deque %a) void)) +(defun (refill-front! d) + (when (and (list-empty? (record-ref d front)) + (not (list-empty? (record-ref d back)))) + (def b (record-ref d back)) + (def k (/ (record-ref d size) 2)) + (record-set! d + (front (list-reverse (drop-n k b))) + (back (take-n k b))))) + +;; The mirror image: everything is on `front`, in order, and the rear half moves +;; across reversed. +(: (-> (Deque %a) void)) +(defun (refill-back! d) + (when (and (list-empty? (record-ref d back)) + (not (list-empty? (record-ref d front)))) + (def f (record-ref d front)) + (def k (/ (record-ref d size) 2)) + (record-set! d + (front (take-n k f)) + (back (list-reverse (drop-n k f)))))) + +;; --------------------------------------------------------------------------- +;; The ends +;; --------------------------------------------------------------------------- + +(: (-> (Deque %a) %a void)) +(defun (deque-push-front! d x) + (record-set! d + (front (Cons x (record-ref d front))) + (size (+ (record-ref d size) 1)))) + +(: (-> (Deque %a) %a void)) +(defun (deque-push-back! d x) + (record-set! d + (back (Cons x (record-ref d back))) + (size (+ (record-ref d size) 1)))) + +(: (-> (Deque %a) (Option %a))) +(defun (deque-try-pop-first! d) + (refill-front! d) + (match (record-ref d front) + (Nil None) + ((Cons x rest) + (record-set! d + (front rest) + (size (- (record-ref d size) 1))) + (Some x)))) + +(: (-> (Deque %a) (Option %a))) +(defun (deque-try-pop-last! d) + (refill-back! d) + (match (record-ref d back) + (Nil None) + ((Cons x rest) + (record-set! d + (back rest) + (size (- (record-ref d size) 1))) + (Some x)))) + +(import/class + (EmptyDeque (: System.InvalidOperationException (-> string EmptyDeque)))) + +;; The front element, removed. +;; Raises InvalidOperationException when the deque is empty. +(: (-> (Deque %a) %a)) +(defun (deque-pop-first! d) + (match (deque-try-pop-first! d) + ((Some x) x) + (None (raise (cast Exception (EmptyDeque. "deque-pop-first!: the deque is empty.")))))) + +;; The back element, removed. +;; Raises InvalidOperationException when the deque is empty. +(: (-> (Deque %a) %a)) +(defun (deque-pop-last! d) + (match (deque-try-pop-last! d) + ((Some x) x) + (None (raise (cast Exception (EmptyDeque. "deque-pop-last!: the deque is empty.")))))) + +;; A look that does not consume — but it still rebalances, hence the `!`. +;; +;; Refilling changes how the deque is *stored* and not what it holds, so this is +;; a read as far as the caller is concerned. Reading the front of an empty +;; `front` without refilling would mean walking to the end of `back`, which is +;; the O(n) this structure exists to avoid. +(: (-> (Deque %a) (Option %a))) +(defun (deque-peek-front! d) + (refill-front! d) + (match (record-ref d front) + (Nil None) + ((Cons x _) (Some x)))) + +(: (-> (Deque %a) (Option %a))) +(defun (deque-peek-back! d) + (refill-back! d) + (match (record-ref d back) + (Nil None) + ((Cons x _) (Some x)))) + +;; --------------------------------------------------------------------------- +;; Conversions and walks +;; --------------------------------------------------------------------------- +;; +;; None of these rebalance: they read both halves as they stand, so a walk is +;; not a write and a deque may be folded from anywhere. + +;; `front` reversed, then consed element by element onto `back` in order: the +;; front lands first, and nothing recurses per element. +(: (-> (Deque %a) (List %a))) +(defun (deque->list d) + (list-foldl (fun (x acc) (Cons x acc)) + (list-reverse (record-ref d back)) + (list-reverse (record-ref d front)))) + +(: (-> (List %a) (Deque %a))) +(defun (list->deque xs) + (Deque (front xs) (back Nil) (size (list-length xs)))) + +;; Front to back. The rear half is reversed once rather than per element. +(: (-> (-?-> %a %b %b) %b (Deque %a) %b)) +(defun (deque-fold f init d) + (list-foldl f (list-foldl f init (record-ref d front)) + (list-reverse (record-ref d back)))) + +(: (-> (-?-> %a void) (Deque %a) void)) +(defun (deque-for-each f d) + (list-for-each f (record-ref d front)) + (list-for-each f (list-reverse (record-ref d back)))) + +;; Whether the deque holds `x`, compared with `=`. The order of the two lists +;; does not matter here, so neither is reversed. +(: (-> (Deque %a) %a bool) (where (Eq %a))) +(defun (deque-contains? d x) + (or (list-contains? (record-ref d front) x) (list-contains? (record-ref d back) x))) + +;; --------------------------------------------------------------------------- +;; Equality, and the hash there is not +;; --------------------------------------------------------------------------- +;; +;; Two deques are equal when they hold the same elements in the same order. +;; *Where* those elements sit is a fact about the representation: pushing three +;; items on the back and popping one leaves a deque that is `=` to one built by +;; three pushes on the front, and it would be absurd for it not to be. +;; +;; This implementation is not decoration, then — without it a deque would fall +;; to the blanket, which is .NET's equality, which for this record is a +;; field-wise comparison of `front`, `back` and `size`. Two deques holding the +;; same elements but split differently would have compared unequal, silently. +;; +;; `eq-hash` refuses. There is no honest number for a value whose fields move: +;; a `Map` keyed on a deque would lose the entry at the next push. The +;; implementation is compiled into `Deque` as its `Equals` and +;; `GetHashCode`, so .NET's equality is this one too, and a `Map` or a `HashSet` +;; that asks for the hash gets the refusal. +(impl (Eq (Deque %a)) + (where (Eq %a)) + (defun (= a b) + (and (= (deque-length a) (deque-length b)) + (= (deque->list a) (deque->list b)))) + (defun (eq-hash d) (unhashable "Deque"))) + +;; `#`, front first. +(impl (Written (Deque %a)) + (where (Written %a)) + (defun (write-into d b) + (elements-print-into "#<" (deque->list d) ">" b (fun (x b) (write-into x b))))) + +(impl (Displayed (Deque %a)) + (where (Displayed %a)) + (defun (display-into d b) + (elements-print-into "#<" (deque->list d) ">" b (fun (x b) (display-into x b))))) + +(: (-> (Deque %a) string) (where (Displayed %a))) +(defun (deque->str d) (displayed d)) + +(:deprecated deque->str "use `displayed`") +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/mutable/heap.bjo — the priority queue. +;; +;; A `MutableHeap` *is* a .NET `PriorityQueue`: an array-backed quaternary +;; min-heap. Pushing and popping are O(log n), reading the least is O(1). +;; +;; **An element and a priority, kept apart.** `(mutableheap-add! h task 3)` +;; files `task` under priority 3, and the two come back together. Ordering the +;; element itself would mean every element type needing an order, and a +;; scheduler's task has none — the priority is the thing being ordered, and this +;; is the one structure in the library where they are different values. +;; +;; **Least first.** `mutableheap-pop!` answers the smallest priority. A largest +;; -first heap is the same structure under a negated comparison, which +;; `mutableheap-by` takes. +;; +;; **Ordering is .NET's unless you say otherwise.** `mutableheap` uses +;; `Comparer

.Default`, which is what `Map` and `HashSet` do for equality. +;; `mutableheap-by` takes a comparison, and is how a priority is ordered by the +;; `Ord` trait or by anything else. +;; +;; **No `Eq`, and deliberately.** Two heaps holding the same pairs are not +;; distinguishable from the outside without draining both, and draining is a +;; write. `=` therefore falls to the blanket, which for a `PriorityQueue` +;; is reference identity — which is honest, unlike a structural answer nothing +;; could compute without destroying its arguments. +(import (std prelude)) +(import (only (std interop) as-delegate)) + +(:doc #:file "heap.bjodoc") + +(export + MutableHeap + mutableheap mutableheap-by + mutableheap-length mutableheap-empty? mutableheap-clear! + mutableheap-add! mutableheap-add-all! mutableheap-add-pop! + mutableheap-peek mutableheap-pop! mutableheap-try-pop! + mutableheap-copy + list->mutableheap mutableheap->unordered-list + seq->mutableheap mutableheap->unordered-seq + mutableheap->str) + +;; The type is .NET's, so a `MutableHeap` handed to any other `import/extern` +;; taking a `PriorityQueue` is accepted as it stands. +(import/class + ((MutableHeap %a %p) (: System.Collections.Generic.PriorityQueue)) + ((UnorderedItems %a %p) (: System.Collections.Generic.PriorityQueue`2+UnorderedItemsCollection)) + ((Comparison %a) (: System.Comparison)) + ((Comparer %a) (: System.Collections.Generic.Comparer)) + ((IComparer %a) (: System.Collections.Generic.IComparer))) + +;; --------------------------------------------------------------------------- +;; The heap +;; --------------------------------------------------------------------------- + +;; `PriorityQueue`'s own members. +(import/extern + (mutableheap-length (: System.Collections.Generic.PriorityQueue.Count (-> (MutableHeap %a %p) int) #:get)) + (mutableheap-add! (: System.Collections.Generic.PriorityQueue.Enqueue (-> (MutableHeap %a %p) %a %p void))) + (mutableheap-add-all! (: System.Collections.Generic.PriorityQueue.EnqueueRange (-> (MutableHeap %a %p) (Seq (Tuple %a %p)) void))) + ;; Pushes and pops as one rebalance, answering what came off — which may be + ;; the element just pushed. This is how a heap is held to a bound. + (mutableheap-add-pop! (: System.Collections.Generic.PriorityQueue.EnqueueDequeue (-> (MutableHeap %a %p) %a %p %a))) + (mutableheap-clear! (: System.Collections.Generic.PriorityQueue.Clear (-> (MutableHeap %a %p) void))) + ;; The least element and its priority, left where they are. + (mutableheap-peek (: System.Collections.Generic.PriorityQueue.TryPeek + (-> (MutableHeap %a %p) (out %a) (out %p) (Option (Tuple %a %p))))) + ;; The least element and its priority, removed. `None` leaves the heap as it + ;; was, so a drain can pop until it stops. + (mutableheap-try-pop! (: System.Collections.Generic.PriorityQueue.TryDequeue + (-> (MutableHeap %a %p) (out %a) (out %p) (Option (Tuple %a %p))))) + (clr-items (: System.Collections.Generic.PriorityQueue.UnorderedItems (-> (MutableHeap %a %p) (UnorderedItems %a %p)) #:get)) + (clr-comparer (: System.Collections.Generic.PriorityQueue.Comparer (-> (MutableHeap %a %p) (IComparer %p)) #:get)) + (clr-comparer-of (: System.Collections.Generic.Comparer.Create (-> (Comparison %p) (Comparer %p))))) + +;; Ordered by `Comparer

.Default`, least priority first. +(: (-> (MutableHeap %a %p))) +(defun (mutableheap) (MutableHeap.)) + +(import/class + (EmptyHeap (: System.InvalidOperationException (-> string EmptyHeap)))) + +;; The least element and its priority, removed. +;; Raises InvalidOperationException when the heap is empty. +(: (-> (MutableHeap %a %p) (Tuple %a %p))) +(defun (mutableheap-pop! h) + (match (mutableheap-try-pop! h) + ((Some entry) entry) + (None (raise (cast Exception (EmptyHeap. "mutableheap-pop!: the heap is empty.")))))) + +;; Ordered by `cmp`, which is how a priority is ordered by the `Ord` trait +;; rather than by .NET. +(: (-> (-> %p %p int) (MutableHeap %a %p))) +(defun (mutableheap-by cmp) + (MutableHeap. (cast (IComparer %p) (clr-comparer-of (: (as-delegate cmp) (Comparison %p)))))) + +(: (-> (MutableHeap %a %p) bool)) +(defun (mutableheap-empty? h) #:inline (= (mutableheap-length h) 0)) + +;; In *no* order: heap order is only observable by popping, and a walk that +;; looked ordered would be a lie the structure cannot tell. +(: (-> (MutableHeap %a %p) (Seq (Tuple %a %p)))) +(defun (mutableheap->unordered-seq h) (cast (Seq (Tuple %a %p)) (clr-items h))) + +(: (-> (Seq (Tuple %a %p)) (MutableHeap %a %p))) +(defun (seq->mutableheap xs) + (def h (mutableheap)) + (mutableheap-add-all! h xs) + h) + +;; The copy keeps the comparer, so a heap built by `mutableheap-by` copies to +;; one ordered the same way. +(: (-> (MutableHeap %a %p) (MutableHeap %a %p))) +(defun (mutableheap-copy h) (MutableHeap. (mutableheap->unordered-seq h) (clr-comparer h))) + +;; --------------------------------------------------------------------------- +;; Conversions +;; --------------------------------------------------------------------------- + +(: (-> (List (Tuple %a %p)) (MutableHeap %a %p))) +(defun (list->mutableheap xs) (seq->mutableheap (list->seq xs))) + +(: (-> (MutableHeap %a %p) (List (Tuple %a %p)))) +(defun (mutableheap->unordered-list h) (->list (mutableheap->unordered-seq h))) + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- +;; +;; `Collection`, `Written` and `Displayed` only. There is no `Iterable`: the walk a heap has is +;; unordered, and a caller reaching for `for-each` would get an order it had +;; every reason to expect was the heap's. `mutableheap->unordered-seq` is that +;; walk under a name that says so, and is `Iterable` by being a `Seq`. +;; +;; No `Addable!` either, since adding to a heap takes a priority as well as an +;; element and the trait's `add!` has one argument. + +(impl (Collection (MutableHeap %a %p)) + (type %content (Tuple %a %p)) + (defun (length h) (mutableheap-length h)) + (defun (empty? h) (mutableheap-empty? h))) + +;; `#mheap(element priority ...)`, unordered, for the reason above. Each +;; element and priority is added by the function given for it. +(: (-> (MutableHeap %a %p) StringBuilder (-> %a StringBuilder void) (-> %p StringBuilder void) void)) +(defun (mutableheap-print-into h b each-elem each-priority) + (stringbuilder-add-string! b "#mheap(") + (loop (:with first? #t #f) + (:for entry (mutableheap->unordered-seq h)) + (:do (def (x p) entry) + (unless first? (stringbuilder-add-string! b " ")) + (each-elem x b) + (stringbuilder-add-string! b " ") + (each-priority p b))) + (stringbuilder-add-string! b ")")) + +(impl (Written (MutableHeap %a %p)) + (where (Written %a) (Written %p)) + (defun (write-into h b) + (mutableheap-print-into h b (fun (x b) (write-into x b)) (fun (p b) (write-into p b))))) + +(impl (Displayed (MutableHeap %a %p)) + (where (Displayed %a) (Displayed %p)) + (defun (display-into h b) + (mutableheap-print-into h b (fun (x b) (display-into x b)) (fun (p b) (display-into p b))))) + +(: (-> (MutableHeap %a %p) string) (where (Displayed %a) (Displayed %p))) +(defun (mutableheap->str h) (displayed h)) + +(:deprecated mutableheap->str "use `displayed`") +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/mutable/map.bjo — the hash table, written in place. +;; +;; A `MutableMap` *is* a .NET `Dictionary`: open addressing over buckets, +;; O(1) expected for every operation below. `mutablemap` makes a subclass that +;; also keeps an entry for the key `null`, which a `Dictionary` refuses. The +;; one case of a union that carries nothing is often `null` at run time, and +;; this lets it be a key. +;; +;; **This is the mutable one.** `(mutablemap-set! m k v)` writes `m`; it answers +;; nothing, and everyone holding `m` sees the change. `Map` is the persistent +;; structure of the same shape. +;; +;; **What `TransientMap` is not.** A transient is a `Map` being built: it is +;; made from a persistent map, filled, and frozen back into one. This is a table +;; that is never frozen — entries go in and come out for as long as the program +;; holds it, which is what a cache, an index or a counter wants. +;; +;; **Keys are filed under .NET's equality**, `EqualityComparer.Default`, not +;; under the `Eq` implementation in `std/eq`. That is the same equality `Map` and +;; `HashSet` file their keys under, so the three agree with each other — but a +;; hand-written `Eq` that departs from .NET's is not consulted here. +(import (std prelude)) + +(:doc #:file "map.bjodoc") + +(export + MutableMap MutableMapEnumerator + mutablemap mutablemap-with-capacity + mutablemap-length mutablemap-empty? + mutablemap-ref mutablemap-try-ref mutablemap-ref-or + mutablemap-set! mutablemap-add-new! mutablemap-try-add-new! + mutablemap-remove! mutablemap-clear! + mutablemap-alter! mutablemap-update! + mutablemap-contains? mutablemap-has-key? mutablemap-keys mutablemap-values mutablemap-copy + mutablemap-enumerator mutablemap-enumerator-next! mutablemap-enumerator-current + mutablemap-for-each + list->mutablemap mutablemap->list vec->mutablemap mutablemap->vec + seq->mutablemap mutablemap->seq map->mutablemap mutablemap->map + mutablemap->str) + +;; The type is .NET's, so a `MutableMap` handed to any other `import/extern` +;; taking a `Dictionary` is accepted as it stands. Such an extern does not +;; see the entry for a `null` key. +(import/class + ((MutableMap %k %v) (: System.Collections.Generic.Dictionary)) + ((MutableMapEnumerator %k %v) (: Bjolang.Runtime.MutableMapEnumerator))) + +;; --------------------------------------------------------------------------- +;; The map +;; --------------------------------------------------------------------------- + +(import/extern + (mutablemap (: Bjolang.Runtime.MutableMapModule.Empty (-> (MutableMap %k %v)))) + (mutablemap-with-capacity (: Bjolang.Runtime.MutableMapModule.WithCapacity (-> int (MutableMap %k %v)))) + (mutablemap-length (: Bjolang.Runtime.MutableMapModule.Count (-> (MutableMap %k %v) int))) + (mutablemap-empty? (: Bjolang.Runtime.MutableMapModule.IsEmpty (-> (MutableMap %k %v) bool))) + ;; Throws when the key is absent, which is what `Refable` asks. The two + ;; answers that are not an exception are below it. + (mutablemap-ref (: Bjolang.Runtime.MutableMapModule.Get (-> (MutableMap %k %v) %k %v))) + (mutablemap-ref-or (: Bjolang.Runtime.MutableMapModule.GetOr (-> (MutableMap %k %v) %k %v %v))) + ;; Replaces whatever was under the key. + (mutablemap-set! (: Bjolang.Runtime.MutableMapModule.Set (-> (MutableMap %k %v) %k %v void))) + ;; Adds a key that must be new, and raises when it is not. To add or replace, + ;; there is `mutablemap-set!`, which is also what `Addable!`'s `add!` does. + (mutablemap-add-new! (: Bjolang.Runtime.MutableMapModule.AddNew (-> (MutableMap %k %v) %k %v void))) + ;; The value already under the key, or None when the entry was added. What + ;; `mutablemap-try-add-new!` is made of. + (add-or-existing! (: Bjolang.Runtime.MutableMapModule.AddOrExisting (-> (MutableMap %k %v) %k %v (out %v) (Option %v)))) + (mutablemap-remove! (: Bjolang.Runtime.MutableMapModule.Remove (-> (MutableMap %k %v) %k bool))) + (mutablemap-clear! (: Bjolang.Runtime.MutableMapModule.Clear (-> (MutableMap %k %v) void))) + (mutablemap-contains? (: Bjolang.Runtime.MutableMapModule.ContainsKey (-> (MutableMap %k %v) %k bool))) + ;; Views, not copies: each walks the map as it stands when it is walked, so + ;; writing the map during one is an error .NET raises. + (mutablemap-keys (: Bjolang.Runtime.MutableMapModule.Keys (-> (MutableMap %k %v) (Seq %k)))) + (mutablemap-values (: Bjolang.Runtime.MutableMapModule.Values (-> (MutableMap %k %v) (Seq %v)))) + (mutablemap-copy (: Bjolang.Runtime.MutableMapModule.Copy (-> (MutableMap %k %v) (MutableMap %k %v)))) + (mutablemap->seq (: Bjolang.Runtime.MutableMapModule.AsEnumerable (-> (MutableMap %k %v) (Seq (Tuple %k %v))))) + ;; Later pairs win, so this and repeated `mutablemap-set!` agree. + (seq->mutablemap (: Bjolang.Runtime.MutableMapModule.FromEnumerable (-> (Seq (Tuple %k %v)) (MutableMap %k %v)))) + + (mutablemap-try-ref (: Bjolang.Runtime.MutableMapModule.TryGet (-> (MutableMap %k %v) %k (out %v) (Option %v)))) + + ;; A walk with .NET's protocol: `next!` is `MoveNext`, `current` is `Current`. + (mutablemap-enumerator (: Bjolang.Runtime.MutableMapModule.Enumerator (-> (MutableMap %k %v) (MutableMapEnumerator %k %v)))) + (mutablemap-enumerator-next! (: Bjolang.Runtime.MutableMapModule.MoveNext (-> (MutableMapEnumerator %k %v) bool))) + (mutablemap-enumerator-current (: Bjolang.Runtime.MutableMapModule.Current (-> (MutableMapEnumerator %k %v) (Tuple %k %v))))) + +;; `mutablemap-add-new!` with the failure as a value: `(Err v)` carries the +;; value the key already has, which the lookup found anyway. +(: (-> (MutableMap %k %v) %k %v (Result %v Unit))) +(defun (mutablemap-try-add-new! m k v) + (match (add-or-existing! m k v) + ((Some old) (Err old)) + (None (Ok unit)))) + +;; The entry of `k` decided by `f`, written in place: +;; - `f` is given `(Some v)` for the value of `k`, or `None` if `k` is absent. +;; - `(Some v)` from `f` sets `k` to `v`, and `None` removes `k`. +;; - A lookup, then a set or a remove. +(: (-> (MutableMap %k %v) %k (-?-> (Option %v) (Option %v)) void)) +(defun (mutablemap-alter! m k f) + (def old (mutablemap-try-ref m k)) + (match (f old) + ((Some v) (mutablemap-set! m k v)) + (None (when (some? old) (ignore (mutablemap-remove! m k)))))) + +;; `k` set to `f` of its value, in place. A missing key raises +;; `KeyNotFoundException`, unless `#:default (Some d)` is given: then `f` is +;; applied to `d`. +(: (-> (MutableMap %k %v) %k (-?-> %v %v) (#:default (Option %v)) void)) +(defun (mutablemap-update! m k f #:default None) + (def old (match (mutablemap-try-ref m k) + ((Some v) v) + (None (match default + ((Some d) d) + ;; Raises KeyNotFoundException. + (None (mutablemap-ref m k)))))) + (mutablemap-set! m k (f old))) + +;; The key and the value as two arguments, which is what a caller of a map +;; almost always wants and what the `Iterable` walk over pairs cannot give. +(: (-> (-?-> %k %v void) (MutableMap %k %v) void)) +(defun (mutablemap-for-each f m) + (def e (mutablemap-enumerator m)) + (let go () + (when (mutablemap-enumerator-next! e) + (def (k v) (mutablemap-enumerator-current e)) + (f k v) + (go)))) + +;; --------------------------------------------------------------------------- +;; Conversions +;; --------------------------------------------------------------------------- + +(: (-> (List (Tuple %k %v)) (MutableMap %k %v))) +(defun (list->mutablemap xs) (seq->mutablemap (list->seq xs))) + +(: (-> (MutableMap %k %v) (List (Tuple %k %v)))) +(defun (mutablemap->list m) (->list (mutablemap->seq m))) + +(: (-> (Vec (Tuple %k %v)) (MutableMap %k %v))) +(defun (vec->mutablemap v) (seq->mutablemap (vec->seq v))) + +(: (-> (MutableMap %k %v) (Vec (Tuple %k %v)))) +(defun (mutablemap->vec m) (->vec (mutablemap->seq m))) + +(: (-> (Map %k %v) (MutableMap %k %v))) +(defun (map->mutablemap m) (seq->mutablemap (map->seq m))) + +(: (-> (MutableMap %k %v) (Map %k %v))) +(defun (mutablemap->map m) (->map (mutablemap->seq m))) + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- + +(: (-> (MutableMap %k %v) %k bool)) +(defun (mutablemap-has-key? m k) (mutablemap-contains? m k)) + +(impl (Refable (MutableMap %k %v)) + (type %index %k) + (type %out %v) + (defun (ref m key) + (mutablemap-ref m key)) + (defun (try-ref m key) + (mutablemap-try-ref m key))) + +;; Written in place, so `(set! m k v)` writes `m`, and adds `k` if it was not +;; there. +(impl (Settable! (MutableMap %k %v)) + (type %index %k) + (type %in %v) + (defun (ref-set! m k v) + (mutablemap-set! m k v))) + +(impl (Iterable (MutableMap %k %v)) + (type %elem (Tuple %k %v)) + ;; The enumerator is the cursor, and it steps in `done?`, which the + ;; protocol calls once per step before `current`. See the `Seq` impl. + (type %cursor (MutableMapEnumerator %k %v)) + (defun (iterable-start m) (mutablemap-enumerator m)) + (defun (iterable-done? m e) (not (mutablemap-enumerator-next! e))) + (defun (iterable-current m e) (mutablemap-enumerator-current e)) + (defun (iterable-next m e) e)) + +(impl (Collection (MutableMap %k %v)) + (type %content (Tuple %k %v)) + (defun (length m) (mutablemap-length m)) + (defun (empty? m) (mutablemap-empty? m))) + +;; Set semantics, as `Addable`'s do everywhere: a key already there is replaced. +(impl (Addable! (MutableMap %k %v)) + (type %elem (Tuple %k %v)) + (defun (add! m pair) + (def (k v) pair) + (mutablemap-set! m k v))) + +;; Two maps are equal when they hold the same keys with equal values. +;; +;; Without this they would fall to the blanket, which for a `Dictionary` is +;; reference identity. The walk is one-sided because the lengths are compared +;; first: every key of `a` being in `b` with an equal value, at equal counts, +;; leaves `b` no room for a key of its own. +;; +;; The *keys* are matched by .NET's equality, since finding one is a lookup; +;; only the values go through `Eq`. `eq-hash` refuses, as `Deque`'s does. +(impl (Eq (MutableMap %k %v)) + (where (Eq %v)) + (defun (= a b) + (and (= (mutablemap-length a) (mutablemap-length b)) + (let ((e (mutablemap-enumerator a))) + (let go () + (cond + ((not (mutablemap-enumerator-next! e)) #t) + (else + (def (k v) (mutablemap-enumerator-current e)) + (match (mutablemap-try-ref b k) + (None #f) + ((Some w) (if (= v w) (go) #f))))))))) + (defun (eq-hash m) (unhashable "MutableMap"))) + +;; `#mmap(k v k v)`: each key and its value, flat, with a space between all. +;; Each key and value is added by the function given for it. +(: (-> (MutableMap %k %v) StringBuilder (-> %k StringBuilder void) (-> %v StringBuilder void) void)) +(defun (mutablemap-print-into m b each-key each-value) + (stringbuilder-add-string! b "#mmap(") + (loop (:with first? #t #f) + (:for entry m) + (:do (def (k v) entry) + (unless first? (stringbuilder-add-string! b " ")) + (each-key k b) + (stringbuilder-add-string! b " ") + (each-value v b))) + (stringbuilder-add-string! b ")")) + +(impl (Written (MutableMap %k %v)) + (where (Written %k) (Written %v)) + (defun (write-into m b) + (mutablemap-print-into m b (fun (k b) (write-into k b)) (fun (v b) (write-into v b))))) + +(impl (Displayed (MutableMap %k %v)) + (where (Displayed %k) (Displayed %v)) + (defun (display-into m b) + (mutablemap-print-into m b (fun (k b) (display-into k b)) (fun (v b) (display-into v b))))) + +(: (-> (MutableMap %k %v) string) (where (Displayed %k) (Displayed %v))) +(defun (mutablemap->str m) (displayed m)) + +(:deprecated mutablemap->str "use `displayed`") + +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/mutable/set.bjo — the hash set, written in place. +;; +;; A `MutableSet` *is* a .NET `HashSet`. O(1) expected membership, and set +;; algebra that rewrites the receiver rather than building a new set. +;; +;; **This is the mutable one.** `(mutableset-add! s x)` writes `s`; everyone +;; holding `s` sees the change. `HashSet` is the persistent structure of the same +;; shape, and `TransientHashSet` is a `HashSet` being built — made from one, filled, and +;; frozen back. This is the one that is never frozen. +;; +;; **The set algebra writes the left-hand side.** `(mutableset-union! a b)` is +;; `a` becoming the union, not a third set: rebuilding is what the persistent +;; `hashset-union` is for, and doing it in place is the reason to reach for this. +;; +;; **Members are filed under .NET's equality**, `EqualityComparer.Default`, +;; not under the `Eq` implementation in `std/eq` — the same equality `Map` and +;; `HashSet` file their keys under. +(import (std prelude)) +;; For `hashset->seq` and `seq->hashset`, which the two conversions at the bottom are +;; written with. The persistent set is a module of its own. +(import (std hashset)) +(import (only (std interop) as-delegate)) + +(:doc #:file "set.bjodoc") + +(export + MutableSet MutableSetEnumerator + mutableset + mutableset-length mutableset-empty? + mutableset-add! mutableset-add-new! mutableset-try-add-new! + mutableset-add-all! mutableset-remove! mutableset-remove-all! + mutableset-remove-where! mutableset-clear! mutableset-contains? + mutableset-union! mutableset-intersect! mutableset-difference! + mutableset-symmetric-difference! + mutableset-subset? mutableset-superset? mutableset-overlaps? + mutableset-copy + mutableset-enumerator mutableset-enumerator-next! mutableset-enumerator-current + mutableset-for-each + list->mutableset mutableset->list vec->mutableset mutableset->vec + seq->mutableset mutableset->seq hashset->mutableset mutableset->hashset + mutableset->str) + +;; The type is .NET's, so a `MutableSet` handed to any other `import/extern` +;; taking a `HashSet` is accepted as it stands. +(import/class + ((MutableSet %a) (: System.Collections.Generic.HashSet)) + ((MutableSetEnumerator %a) (: Bjolang.Runtime.MutableSetEnumerator)) + ((Predicate %a) (: System.Predicate)) + (ClrObject (: System.Object)) + (ArgumentException (: System.ArgumentException (-> string string ArgumentException)))) + +;; --------------------------------------------------------------------------- +;; The set +;; --------------------------------------------------------------------------- + +;; `HashSet`'s own members. The ones that take another collection take a +;; `Seq`, so a set is given to them as one. +(import/extern + (mutableset-length (: System.Collections.Generic.HashSet.Count (-> (MutableSet %a) int) #:get)) + (clr-add (: System.Collections.Generic.HashSet.Add (-> (MutableSet %a) %a bool))) + (clr-try-get (: System.Collections.Generic.HashSet.TryGetValue (-> (MutableSet %a) %a (out %a) (Option %a)))) + (mutableset-add-all! (: System.Collections.Generic.HashSet.UnionWith (-> (MutableSet %a) (Seq %a) void))) + (mutableset-remove! (: System.Collections.Generic.HashSet.Remove (-> (MutableSet %a) %a bool))) + (mutableset-remove-all! (: System.Collections.Generic.HashSet.ExceptWith (-> (MutableSet %a) (Seq %a) void))) + (clr-remove-where (: System.Collections.Generic.HashSet.RemoveWhere (-> (MutableSet %a) (Predicate %a) int))) + (mutableset-clear! (: System.Collections.Generic.HashSet.Clear (-> (MutableSet %a) void))) + (mutableset-contains? (: System.Collections.Generic.HashSet.Contains (-> (MutableSet %a) %a bool))) + (clr-intersect (: System.Collections.Generic.HashSet.IntersectWith (-> (MutableSet %a) (Seq %a) void))) + (clr-symmetric-except (: System.Collections.Generic.HashSet.SymmetricExceptWith (-> (MutableSet %a) (Seq %a) void))) + (clr-subset? (: System.Collections.Generic.HashSet.IsSubsetOf (-> (MutableSet %a) (Seq %a) bool))) + (clr-superset? (: System.Collections.Generic.HashSet.IsSupersetOf (-> (MutableSet %a) (Seq %a) bool))) + (clr-overlaps? (: System.Collections.Generic.HashSet.Overlaps (-> (MutableSet %a) (Seq %a) bool))) + (clr-text (: System.Object.ToString (-> ClrObject string))) + + ;; A walk with .NET's protocol: `next!` is `MoveNext`, `current` is `Current`. + (mutableset-enumerator (: Bjolang.Runtime.MutableSetModule.Enumerator (-> (MutableSet %a) (MutableSetEnumerator %a)))) + (mutableset-enumerator-next! (: Bjolang.Runtime.MutableSetModule.MoveNext (-> (MutableSetEnumerator %a) bool))) + (mutableset-enumerator-current (: Bjolang.Runtime.MutableSetModule.Current (-> (MutableSetEnumerator %a) %a)))) + +(: (-> (MutableSet %a))) +(defun (mutableset) (MutableSet.)) + +(: (-> (Seq %a) (MutableSet %a))) +(defun (seq->mutableset xs) (MutableSet. xs)) + +;; The set itself, seen as a sequence: nothing is copied. +(: (-> (MutableSet %a) (Seq %a))) +(defun (mutableset->seq s) #:inline (cast (Seq %a) s)) + +;; A set with the same members. .NET copies another `HashSet` with the same +;; equality without hashing its members again. +(: (-> (MutableSet %a) (MutableSet %a))) +(defun (mutableset-copy s) (MutableSet. (mutableset->seq s))) + +(: (-> (MutableSet %a) bool)) +(defun (mutableset-empty? s) #:inline (= (mutableset-length s) 0)) + +;; Makes x a member. Adding one that is already a member is not a failure: +;; afterwards it is a member, which is all `add!` says. +(: (-> (MutableSet %a) %a void)) +(defun (mutableset-add! s x) #:inline (ignore (clr-add s x))) + +;; The equal element already there, or None when x was added. What +;; `mutableset-try-add-new!` is made of. +(: (-> (MutableSet %a) %a (Option %a))) +(defun (add-or-existing! s x) + (match (clr-try-get s x) + ((Some old) (Some old)) + (None (ignore (clr-add s x)) None))) + +;; Adds an element that must be new, and raises when it is not. +(: (-> (MutableSet %a) %a void)) +(defun (mutableset-add-new! s x) + (when (not (clr-add s x)) + (raise (cast Exception + (ArgumentException. (str "The element is already in the set: " + (clr-text (cast ClrObject x))) + "value"))))) + +;; Removes every member that `pred` accepts, and returns how many it removed. +(: (-> (MutableSet %a) (-> %a bool) int)) +(defun (mutableset-remove-where! s pred) + (clr-remove-where s (: (as-delegate pred) (Predicate %a)))) + +;; --- Set algebra, written into the first argument --- + +(: (-> (MutableSet %a) (MutableSet %a) void)) +(defun (mutableset-union! a b) (mutableset-add-all! a (mutableset->seq b))) + +(: (-> (MutableSet %a) (MutableSet %a) void)) +(defun (mutableset-intersect! a b) (clr-intersect a (mutableset->seq b))) + +(: (-> (MutableSet %a) (MutableSet %a) void)) +(defun (mutableset-difference! a b) (mutableset-remove-all! a (mutableset->seq b))) + +(: (-> (MutableSet %a) (MutableSet %a) void)) +(defun (mutableset-symmetric-difference! a b) (clr-symmetric-except a (mutableset->seq b))) + +;; --- Comparisons, which write nothing --- + +(: (-> (MutableSet %a) (MutableSet %a) bool)) +(defun (mutableset-subset? a b) (clr-subset? a (mutableset->seq b))) + +(: (-> (MutableSet %a) (MutableSet %a) bool)) +(defun (mutableset-superset? a b) (clr-superset? a (mutableset->seq b))) + +(: (-> (MutableSet %a) (MutableSet %a) bool)) +(defun (mutableset-overlaps? a b) (clr-overlaps? a (mutableset->seq b))) + +;; `mutableset-add-new!` with the failure as a value: `(Err y)` carries the +;; equal element already in the set, which is what interning a value needs. +(: (-> (MutableSet %a) %a (Result %a Unit))) +(defun (mutableset-try-add-new! s x) + (match (add-or-existing! s x) + ((Some old) (Err old)) + (None (Ok unit)))) + +;; A walk whose callback may suspend. The `Iterable` default would do as well; +;; this one names the set first, matching `hashset-for-each`. +(: (-> (-?-> %a void) (MutableSet %a) void)) +(defun (mutableset-for-each f s) + (def e (mutableset-enumerator s)) + (let go () + (when (mutableset-enumerator-next! e) + (f (mutableset-enumerator-current e)) + (go)))) + +;; --------------------------------------------------------------------------- +;; Conversions +;; --------------------------------------------------------------------------- + +(: (-> (List %a) (MutableSet %a))) +(defun (list->mutableset xs) (seq->mutableset (list->seq xs))) + +(: (-> (MutableSet %a) (List %a))) +(defun (mutableset->list s) (->list (mutableset->seq s))) + +(: (-> (Vec %a) (MutableSet %a))) +(defun (vec->mutableset v) (seq->mutableset (vec->seq v))) + +(: (-> (MutableSet %a) (Vec %a))) +(defun (mutableset->vec s) (->vec (mutableset->seq s))) + +(: (-> (HashSet %a) (MutableSet %a))) +(defun (hashset->mutableset s) (seq->mutableset (hashset->seq s))) + +(: (-> (MutableSet %a) (HashSet %a))) +(defun (mutableset->hashset s) (seq->hashset (mutableset->seq s))) + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- + +(impl (Refable (MutableSet %a)) + (type %index %a) + (type %out bool) + (defun (ref s elem) + (mutableset-contains? s elem))) + +(impl (Iterable (MutableSet %a)) + (type %elem %a) + ;; The enumerator is the cursor, and it steps in `done?`, which the + ;; protocol calls once per step before `current`. See the `Seq` impl. + (type %cursor (MutableSetEnumerator %a)) + (defun (iterable-start s) (mutableset-enumerator s)) + (defun (iterable-done? s e) (not (mutableset-enumerator-next! e))) + (defun (iterable-current s e) (mutableset-enumerator-current e)) + (defun (iterable-next s e) e)) + +(impl (Collection (MutableSet %a)) + (type %content %a) + (defun (length s) (mutableset-length s)) + (defun (empty? s) (mutableset-empty? s))) + +(impl (Addable! (MutableSet %a)) + (type %elem %a) + (defun (add! s elem) + (mutableset-add! s elem))) + +;; Two sets are equal when they hold the same elements. Same size, and every +;; element of `a` in `b` — which leaves `b` no room for one of its own. +;; +;; Membership is .NET's equality, since finding an element is a lookup. Without +;; this implementation the whole comparison would be .NET's too, and for a +;; `HashSet` that is reference identity. `eq-hash` refuses, as `Deque`'s does. +(impl (Eq (MutableSet %a)) + (defun (= a b) + (and (= (mutableset-length a) (mutableset-length b)) + (let ((e (mutableset-enumerator a))) + (let go () + (cond + ((not (mutableset-enumerator-next! e)) #t) + ((mutableset-contains? b (mutableset-enumerator-current e)) (go)) + (else #f)))))) + (defun (eq-hash s) (unhashable "MutableSet"))) + +;; `#mset(a b c)`, in whatever order the table holds them. +(impl (Written (MutableSet %a)) + (where (Written %a)) + (defun (write-into s b) (elements-print-into "#mset(" s ")" b (fun (x b) (write-into x b))))) + +(impl (Displayed (MutableSet %a)) + (where (Displayed %a)) + (defun (display-into s b) (elements-print-into "#mset(" s ")" b (fun (x b) (display-into x b))))) + +(: (-> (MutableSet %a) string) (where (Displayed %a))) +(defun (mutableset->str s) (displayed s)) + +(:deprecated mutableset->str "use `displayed`") +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/mutable/vec.bjo — the growable array, written in place. +;; +;; A `MutableVec` *is* a .NET `List`: a contiguous array that doubles when it +;; fills. Indexing and appending are O(1), inserting and removing at an index +;; are O(n) in what follows the index. +;; +;; **This is the mutable one.** `(mutablevec-add! v x)` writes `v`; it answers +;; nothing, and everyone holding `v` sees the change. `Vec` is the persistent +;; structure of the same shape, and a program that wants to keep old versions +;; should use that. +;; +;; **What the existing builders are not.** `VecBuilder` and `ListBuilder` are +;; append-only: an element goes in and nothing comes out until the collection is +;; finished. This is the one that can be written at an index, sorted, and had +;; elements removed from the middle — a scratch buffer rather than a funnel. +;; +;; **Not a key.** The `Eq` implementation at the bottom is structural, and +;; `eq-hash` refuses: there is no honest number for a value whose elements move. +(import (std prelude)) +(import (only (std interop) as-delegate)) + +(:doc #:file "vec.bjodoc") + +(export + MutableVec + mutablevec mutablevec-with-capacity + mutablevec-length mutablevec-empty? + mutablevec-ref mutablevec-try-ref mutablevec-set! + mutablevec-add! mutablevec-add-all! mutablevec-insert! + mutablevec-remove-at! mutablevec-remove! mutablevec-remove-where! + mutablevec-pop! mutablevec-try-pop! mutablevec-clear! + mutablevec-contains? mutablevec-index-of + mutablevec-reverse! mutablevec-sort! mutablevec-sort-unstable! mutablevec-sort-by! + mutablevec-copy + mutablevec-grow! mutablevec-span + list->mutablevec mutablevec->list vec->mutablevec mutablevec->vec + mutablevec->iarray + seq->mutablevec mutablevec->seq mutablevec->array + mutablevec->str) + +;; The type is .NET's, not an alias for a record of our own: a `MutableVec` +;; handed to any other `import/extern` that takes a `List` is accepted as it +;; stands, with nothing to unwrap. +(import/class + ((MutableVec %a) (: System.Collections.Generic.List)) + ((Predicate %a) (: System.Predicate))) + +;; --------------------------------------------------------------------------- +;; The vector +;; --------------------------------------------------------------------------- + +;; `List`'s own members. `get_Item` and `set_Item` are its indexer. +(import/extern + (mutablevec-length (: System.Collections.Generic.List.Count (-> (MutableVec %a) int) #:get)) + ;; Throws when the index is out of range, which is what `Refable` asks of an + ;; indexed collection. `mutablevec-try-ref` is the checked one. + (mutablevec-ref (: System.Collections.Generic.List.get_Item (-> (MutableVec %a) int %a))) + (mutablevec-set! (: System.Collections.Generic.List.set_Item (-> (MutableVec %a) int %a void))) + (mutablevec-add! (: System.Collections.Generic.List.Add (-> (MutableVec %a) %a void))) + (mutablevec-add-all! (: System.Collections.Generic.List.AddRange (-> (MutableVec %a) (Seq %a) void))) + (mutablevec-insert! (: System.Collections.Generic.List.Insert (-> (MutableVec %a) int %a void))) + (mutablevec-remove-at! (: System.Collections.Generic.List.RemoveAt (-> (MutableVec %a) int void))) + ;; Removes the first element equal to `x`, and answers whether there was one. + (mutablevec-remove! (: System.Collections.Generic.List.Remove (-> (MutableVec %a) %a bool))) + (mutablevec-clear! (: System.Collections.Generic.List.Clear (-> (MutableVec %a) void))) + (mutablevec-contains? (: System.Collections.Generic.List.Contains (-> (MutableVec %a) %a bool))) + ;; -1 when there is none, which is what .NET answers and what an `Option` + ;; would cost an allocation to say. + (mutablevec-index-of (: System.Collections.Generic.List.IndexOf (-> (MutableVec %a) %a int))) + (mutablevec-reverse! (: System.Collections.Generic.List.Reverse (-> (MutableVec %a) void))) + (mutablevec->array (: System.Collections.Generic.List.ToArray (-> (MutableVec %a) (Array %a)))) + (clr-remove-all (: System.Collections.Generic.List.RemoveAll (-> (MutableVec %a) (Predicate %a) int))) + ;; The way to build an `IArray` one element at a time: add to a vector, then + ;; copy it once. `iarray-add` copies the whole array each time. + (mutablevec->iarray (: Bjolang.Runtime.IArrayModule.FromList (-> (MutableVec %a) (IArray %a)))) + ;; The elements where the vector keeps them. Valid until the vector grows. + (clr-as-span (: System.Runtime.InteropServices.CollectionsMarshal.AsSpan (-> (MutableVec %a) (Span %a)))) + (clr-set-count (: System.Runtime.InteropServices.CollectionsMarshal.SetCount (-> (MutableVec %a) int void)))) + +(: (-> (MutableVec %a))) +(defun (mutablevec) (MutableVec.)) + +;; Room for `capacity` elements before the first regrow. The length is still 0. +(: (-> int (MutableVec %a))) +(defun (mutablevec-with-capacity capacity) (MutableVec. capacity)) + +(: (-> (Seq %a) (MutableVec %a))) +(defun (seq->mutablevec xs) (MutableVec. xs)) + +;; The vector itself, seen as a sequence: nothing is copied. +(: (-> (MutableVec %a) (Seq %a))) +(defun (mutablevec->seq v) #:inline (cast (Seq %a) v)) + +(: (-> (MutableVec %a) (MutableVec %a))) +(defun (mutablevec-copy v) (MutableVec. (mutablevec->seq v))) + +(: (-> (MutableVec %a) bool)) +(defun (mutablevec-empty? v) #:inline (= (mutablevec-length v) 0)) + +(: (-> (MutableVec %a) int (Option %a))) +(defun (mutablevec-try-ref v i) #:inline + (if (and (>= i 0) (< i (mutablevec-length v))) (Some (mutablevec-ref v i)) None)) + +;; The last element, removed. `None` leaves the vector as it was, so a +;; stack-shaped loop can pop until it stops. +(: (-> (MutableVec %a) (Option %a))) +(defun (mutablevec-try-pop! v) + (def last (- (mutablevec-length v) 1)) + (if (< last 0) + None + (let ((x (mutablevec-ref v last))) + (mutablevec-remove-at! v last) + (Some x)))) + +(import/class + (EmptyVec (: System.InvalidOperationException (-> string EmptyVec)))) + +;; The last element, removed. +;; Raises InvalidOperationException when the vector is empty. +(: (-> (MutableVec %a) %a)) +(defun (mutablevec-pop! v) + (match (mutablevec-try-pop! v) + ((Some x) x) + (None (raise (cast Exception (EmptyVec. "mutablevec-pop!: the vector is empty.")))))) + +;; Removes every element that `pred` accepts, and returns how many it removed. +(: (-> (MutableVec %a) (-> %a bool) int)) +(defun (mutablevec-remove-where! v pred) + (clr-remove-all v (: (as-delegate pred) (Predicate %a)))) + +(import/class + (ArgumentOutOfRangeException (: System.ArgumentOutOfRangeException + (-> string string ArgumentOutOfRangeException)))) + +;; The elements of `sorted`, which has the vector's length, into the vector. +(: (-> (MutableVec %a) (Array %a) void)) +(defun (write-back! xs sorted) + (def items (clr-as-span xs)) + (loop (:for i (range 0 (array-length sorted))) + (:do (span-set! items i (array-ref sorted i))))) + +;; Sorted by the element's own `Ord`. `mutablevec-sort-by!` takes the comparison +;; directly, for an order that is not the type's. Both are stable, as +;; `list-sort` is: equal elements keep their order. A copy is sorted with +;; `array-sort!` and written back once the sort is done, so when a comparison +;; raises, the vector is as it was. +(: (-> (MutableVec %a) void) (where (Ord %a))) +(defun (mutablevec-sort! xs) + (def sorted (mutablevec->array xs)) + (array-sort! sorted) + (write-back! xs sorted)) + +;; As `mutablevec-sort!`, with `array-sort-unstable!`: equal elements can +;; change places. When a comparison raises, the vector is as it was. +(: (-> (MutableVec %a) void) (where (Ord %a))) +(defun (mutablevec-sort-unstable! xs) + (def sorted (mutablevec->array xs)) + (array-sort-unstable! sorted) + (write-back! xs sorted)) + +;; As `mutablevec-sort!`, in the order of `cmp`, with `array-sort-by!`. +;; Stable. When `cmp` raises, the vector is as it was. +(: (-> (MutableVec %a) (-> %a %a int) void)) +(defun (mutablevec-sort-by! v cmp) + (def sorted (mutablevec->array v)) + (array-sort-by! cmp sorted) + (write-back! v sorted)) + +;; --------------------------------------------------------------------------- +;; Blocks +;; --------------------------------------------------------------------------- +;; +;; A loop that fills a vector a block at a time grows the vector, and then +;; writes the new elements through a span. + +;; Adds `n` elements at the end. Their values are unspecified: the storage +;; can keep values from before. Write each one before you read it. +;; Raises ArgumentOutOfRangeException when `n` is less than 0. +(: (-> (MutableVec %a) int void)) +(defun (mutablevec-grow! xs n) + (when (< n 0) + (raise (cast Exception + (ArgumentOutOfRangeException. + "n" (str "mutablevec-grow!: the count is " (int->string n) ", and it must be 0 or more."))))) + (clr-set-count xs (+ (mutablevec-length xs) n))) + +;; The `n` elements from index `start`, as a span over the storage of the +;; vector. A write through the span writes the vector. The span is valid +;; until the vector changes its length. +;; Raises ArgumentOutOfRangeException when the range is not in the vector. +(: (-> (MutableVec %a) int int (Span %a))) +(defun (mutablevec-span xs start n) + (span-slice (clr-as-span xs) #:from start #:to (+ start n))) + +;; --------------------------------------------------------------------------- +;; Conversions +;; --------------------------------------------------------------------------- + +(: (-> (List %a) (MutableVec %a))) +(defun (list->mutablevec xs) (seq->mutablevec (list->seq xs))) + +(: (-> (MutableVec %a) (List %a))) +(defun (mutablevec->list xs) (->list (mutablevec->seq xs))) + +(: (-> (Vec %a) (MutableVec %a))) +(defun (vec->mutablevec v) (seq->mutablevec (vec->seq v))) + +(: (-> (MutableVec %a) (Vec %a))) +(defun (mutablevec->vec xs) (->vec (mutablevec->seq xs))) + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- + +(impl (Refable (MutableVec %a)) + (type %index int) + (type %out %a) + (defun (ref xs index) + (mutablevec-ref xs index)) + (defun (try-ref xs index) + (mutablevec-try-ref xs index))) + +;; Written in place, so `(set! v i x)` writes `v`. An index out of range throws. +(impl (Settable! (MutableVec %a)) + (type %index int) + (type %in %a) + (defun (ref-set! v i x) + (mutablevec-set! v i x))) + +;; An `int` cursor, as `Array` has: an index and an array read is what this +;; collection makes cheap, so there is nothing a cursor object would save. +;; +;; A walk reads the length at every step rather than once, so a vector written +;; *during* its own walk is walked as it now stands. Adding an element inside a +;; `for-each` over the same vector therefore does not end the loop early — nor +;; does it end at all, which is the usual hazard of writing a collection you +;; are reading. Shrinking it past the cursor ends the walk. +(impl (Iterable (MutableVec %a)) + (type %elem %a) + (type %cursor int) + (defun (iterable-start xs) 0) + (defun (iterable-done? xs index) (>= index (mutablevec-length xs))) + (defun (iterable-current xs index) (mutablevec-ref xs index)) + (defun (iterable-next xs index) (+ index 1))) + +;; The elements are one .NET array, so a loop over them can be a vector loop. +;; Such a loop does not add to the vector, so the span stays valid. +(impl (Contiguous (MutableVec %a)) + (type %value %a) + (defun (contiguous? xs) #t) + (defun (contiguous-values xs) (span->ro-span (clr-as-span xs))) + (defun (contiguous-valid xs) (array->ro-span (make-array 0)))) + +(impl (Collection (MutableVec %a)) + (type %content %a) + (defun (length xs) (mutablevec-length xs)) + (defun (empty? xs) (mutablevec-empty? xs))) + +(impl (Addable! (MutableVec %a)) + (type %elem %a) + (defun (add! xs elem) + (mutablevec-add! xs elem))) + +;; Two vectors are equal when they hold the same elements in the same order. +;; +;; Without this they would fall to the blanket, which is .NET's equality, which +;; for a `List` is reference identity — two vectors built the same way from +;; the same elements would have compared unequal. +;; +;; `eq-hash` refuses, exactly as `Deque`'s does: there is no honest number for a +;; value whose elements move, and a `Map` keyed on one would ask .NET rather +;; than asking this. +(impl (Eq (MutableVec %a)) + (where (Eq %a)) + (defun (= a b) + (def len (mutablevec-length a)) + (if (= len (mutablevec-length b)) + (let lp ((i 0)) + (cond + ((= i len) #t) + ((= (mutablevec-ref a i) (mutablevec-ref b i)) (lp (+ i 1))) + (else #f))) + #f)) + (defun (eq-hash xs) (unhashable "MutableVec"))) + +;; `#mvec(a b c)`, in order. +(impl (Written (MutableVec %a)) + (where (Written %a)) + (defun (write-into xs b) (elements-print-into "#mvec(" xs ")" b (fun (x b) (write-into x b))))) + +(impl (Displayed (MutableVec %a)) + (where (Displayed %a)) + (defun (display-into xs b) (elements-print-into "#mvec(" xs ")" b (fun (x b) (display-into x b))))) + +(: (-> (MutableVec %a) string) (where (Displayed %a))) +(defun (mutablevec->str xs) (displayed xs)) + +(:deprecated mutablevec->str "use `displayed`") +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/net.bjo — TCP, as two byte ports. +;; +;; Connecting gives you a `(Tuple InputPort OutputPort)` and nothing +;; else — no socket object, which is why this module is `(std net)` and not +;; `(std socket)`. That is the whole design: a protocol written against +;; `(std ports)` runs over TCP without knowing it is TCP, and `byte-pipe` can +;; stand in for a network in a test. +;; +;; Everything you do with a connection comes from `(std ports)` and the prelude: +;; `read-some`, `read-bytes-evt`, `write-bytes!`, `flush-port`, `shutdown!`, +;; `limited`, and `read-line` on the same input port as `read-bytes`. Import +;; both. +;; +;; (match (tcp-connect "127.0.0.1" 8080) +;; ((Ok (Tuple in out)) +;; (write-bytes! out request) +;; (flush-port out) +;; (shutdown! out) ; a FIN: "I have finished asking" +;; (read-some in)) ; ...and the answer still arrives +;; ((Err e) ...)) +;; +;; THE INVARIANT THE ACCEPT SIDE TURNS ON, which is the byte port's one layer +;; out: +;; +;; A connection the operating system has handed over belongs to the listener, +;; not to the event that asked for it. An accept never takes a connection from +;; the kernel — it takes one from the listener's slot, and only when it wins. +;; +;; So `accept-evt` is a real CML event: a branch of a `choose` that loses accepts +;; nothing, and the connection is still there for the next acceptor. +;; +;; `BjolangRuntime/BjoNet.cs` is where that is done. + +(export + ;; Connecting. + tcp-connect + + ;; A connection somebody else made. + stream->ports + + ;; Listening. + Listener + tcp-listen accept-evt accept + listener-closed listener-port close-listener! + + ;; Who is at the other end. + peer-address) + +(import (std prelude)) +(import (std ports)) + +(:doc #:file "net.bjodoc") + +(import/class + (Listener (: Bjolang.Runtime.BjoTcpListener)) + ;; Opaque, and deliberately not exported: a connection *is* its two ports, and + ;; a third value to pass around would be the thing this module exists not to + ;; have. It appears below only between the runtime and the tuple. + (Connection (: Bjolang.Runtime.BjoConnection (-> ClrStream string Connection))) + (Exception (: System.Exception))) + +(import/extern + ;; Connecting. `#:async`, so the ambient cancellation token is filled in at + ;; the call site and `with-deadline` works on a connect with no keyword of its + ;; own. `OperationCanceledException` is deliberately *not* in the list: a + ;; deadline is a raise, not a `Result`. + (clr-connect (: Bjolang.Runtime.Net.Connect (-> string int Connection) #:async + #:exceptions (System.Net.Sockets.SocketException + System.ArgumentOutOfRangeException))) + + ;; Listening. + (clr-listen (: Bjolang.Runtime.Net.Listen (-> string int int Listener) + #:exceptions (System.Net.Sockets.SocketException + System.ArgumentOutOfRangeException))) + (clr-accept-evt (: Bjolang.Runtime.Net.AcceptEvent + (-> Listener (Event (Result Exception Connection))))) + (clr-closed-evt (: Bjolang.Runtime.Net.ClosedEvent (-> Listener (Event Unit)))) + (clr-listener-port (: Bjolang.Runtime.Net.ListenerPort (-> Listener int))) + (clr-close-listener (: Bjolang.Runtime.Net.CloseListener (-> Listener void))) + + ;; A connection's two halves, and who is behind them. + (clr-conn-in (: Bjolang.Runtime.Net.ConnectionInput (-> Connection InputPort))) + (clr-conn-out (: Bjolang.Runtime.Net.ConnectionOutput (-> Connection OutputPort))) + (clr-peer (: Bjolang.Runtime.Net.PeerAddress (-> InputPort (Option string)))) + + ;; Ownership. Called from here, on the fiber, and never from inside the + ;; awaited connect: after an await the continuation is on a pool thread, where + ;; the ambient scope is not this fiber's and there would be nothing to + ;; register on. + (clr-own-connection (: BjolangRuntime.OwnConnection (-> Connection Connection))) + (clr-own-listener (: BjolangRuntime.OwnListener (-> Listener Listener)))) + +;; --------------------------------------------------------------------------- +;; A connection is its two ports +;; --------------------------------------------------------------------------- + +;; The socket under the pair goes when **both** halves have been closed, or when +;; the scope that owns the connection ends, whichever comes first. That is what +;; makes the obvious spelling mean the obvious thing: +;; +;; (with-open ((i in) (o out)) +;; ...) ; both closed here, so the handle goes here +;; +;; Closing one half does not close the other — "I have finished listening, now +;; let me answer" has to work, and it is the same fact `shutdown!` exists for. +(: (-> Connection (Tuple InputPort OutputPort))) +(defun (connection->ports c) + (Tuple (clr-conn-in c) (clr-conn-out c))) + +;; --------------------------------------------------------------------------- +;; Connecting +;; --------------------------------------------------------------------------- + +;; Connects to `host` on `port`, and answers the two ports over it. +;; +;; `host` may be an address or a name. A name is resolved and its addresses are +;; tried in order, the first that answers winning; there is no happy-eyeballs +;; racing, and an address literal asks nothing of the resolver at all. +;; +;; The connection is owned by the ambient scope, so a program that forgets to +;; close it still closes it when the scope returns. There is no `#:deadline`: +;; `sync` watches the ambient token, so `with-deadline` already bounds this. +;; +;; Nagle is switched off on the socket. A byte port has a write buffer of its +;; own and `flush-port` is the program saying "send this now", so leaving Nagle on +;; would hold a flushed small write back waiting for an ACK — exactly wrong for +;; a request/response protocol. +(: (-bjo-> string int (Result Exception (Tuple InputPort OutputPort)))) +(defbjo (tcp-connect host port) + ;; Raises here if there is no scope, before a socket exists to leak. + (ignore (current-scope)) + (match (clr-connect host port) + ((Ok c) (Ok (connection->ports (clr-own-connection c)))) + ((Err e) (Err e)))) + +;; --------------------------------------------------------------------------- +;; A connection somebody else made +;; --------------------------------------------------------------------------- + +;; The two ports over a duplex `ClrStream` that something other than this module +;; connected: the stream an HTTP server hands over after `101 Switching +;; Protocols`, or a `NetworkStream`. +;; +;; `stream->input-port` and `stream->output-port` each own the stream +;; they wrap, so the first half closed would take the stream from the other. +;; This shares it the way `tcp-connect` does: the stream is disposed when both +;; halves have been closed, or when the ambient scope ends. +;; +;; `peer` is what `peer-address` answers for the input port; `""` means `None`. +(: (-> ClrStream (#:peer string) (Tuple InputPort OutputPort))) +(defun (stream->ports s #:peer "") + (ignore (current-scope)) + (connection->ports (clr-own-connection (Connection. s peer)))) + +;; --------------------------------------------------------------------------- +;; Listening +;; --------------------------------------------------------------------------- + +;; Binds and listens. `port` may be 0, in which case the operating system picks +;; one and `listener-port` says which. +;; +;; The listener is owned by the ambient scope, and what the scope does at the end +;; is the *hard* stop — see `close-listener!` for the other one. +;; +;; A host name here is resolved synchronously, because binding happens once at +;; startup. An address literal — `"127.0.0.1"`, `"0.0.0.0"`, `"::1"` — asks +;; nothing of the resolver. +(: (-> string int (#:backlog int) (Result Exception Listener))) +(defun (tcp-listen host port #:backlog 128) + (ignore (current-scope)) + (match (clr-listen host port backlog) + ((Ok l) (Ok (clr-own-listener l))) + ((Err e) (Err e)))) + +;; The port actually bound, which is the interesting question when 0 was asked +;; for. Still answerable after the listener is closed. +(: (-> Listener int)) +(defun (listener-port l) (clr-listener-port l)) + +;; A real CML event: in a `choose` that another branch wins, nothing is accepted +;; and the connection stays for the next acceptor, exactly as a channel receive +;; does. It is also *ephemeral* — each `sync` asks the listener a new question, +;; so a server loop accepts successive connections rather than winning with a +;; remembered one forever. +;; +;; WHY THE VALUE IS A `Result`. A listening socket can fail, and the CML resume +;; path carries a value rather than a raise — so a failure that lands while a +;; fiber is parked in a `choose` has to arrive as one. `promise-join`, `blocking` +;; and the byte ports' reads all answer a `Result` for the same reason. +;; +;; On a closed listener it never commits. `listener-closed` is how a loop +;; notices, exactly as `inbox-closed` is for an inbox: +;; +;; (let go () +;; (match (sync (choose (wrap (accept-evt l) Some) +;; (wrap (listener-closed l) (fun (u) None)))) +;; ((Some (Ok conn)) (spawn (handle conn)) (go)) +;; ((Some (Err e)) (log (str "accept failed: " (.-Message e))) (go)) +;; (None 'stopped))) +(: (-> Listener (Event (Result Exception (Tuple InputPort OutputPort))))) +(defun (accept-evt l) + (wrap (clr-accept-evt l) + #(match & + ((Ok c) (Ok (connection->ports c))) + ((Err e) (Err e))))) + +;; Fires once the listener has stopped accepting, and from then on commits +;; immediately on every `sync`. +(: (-> Listener (Event Unit))) +(defun (listener-closed l) (clr-closed-evt l)) + +(: (-> (Result Exception (Tuple InputPort OutputPort)) + (Tuple InputPort OutputPort))) +(defun (accept-or-raise r) + (match r + ((Ok conn) conn) + ((Err e) (raise e)))) + +;; One connection, for when there is nothing to choose between. Defined by +;; syncing the event above: a convenience, not a second implementation. The +;; failure is raised here, on the caller's own stack. +;; +;; On a closed listener this waits forever, because the event never commits. +;; A program that may meet one uses `accept-evt` under a `choose` with +;; `listener-closed`. +(: (-> Listener (Tuple InputPort OutputPort))) +(defbjouble (accept l) + (#:sync (accept-or-raise (sync/blocking (accept-evt l)))) + (#:bjo (accept-or-raise (sync (accept-evt l))))) + +;; Stops accepting, and says so. +;; +;; GRACEFUL, and that is the whole difference from letting the scope end: +;; connections already accepted are **not** touched, because they belong to +;; whoever is handling them and killing a request half-answered is not what +;; "stop listening" means. That is `inbox-close!`'s meaning, one layer out. +;; +;; So a server drains by closing the listener, letting its handlers finish, and +;; leaving the scope — which closes the listening socket and anything still open. +(: (-> Listener void)) +(defun (close-listener! l) (clr-close-listener l)) + +;; --------------------------------------------------------------------------- +;; Who is at the other end +;; --------------------------------------------------------------------------- + +;; The peer, as the address and port it connected from, for a log line. +;; +;; It hangs off the input port because the tuple is fixed at two and there is +;; nowhere else to put it. `None` for a port that is not a connection's — a +;; file, a pipe, a `limited` view — and for a `stream->ports` given no peer. +(: (-> InputPort (Option string))) +(defun (peer-address p) + (match (clr-peer p) + ((Some "") None) + (found found))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/orderedmap.bjo — the sorted map: a persistent B+ tree, keyed in the keys' +;; order. Lookups are O(log n). +;; +;; - The tree is a B+ tree over C# nodes in the runtime, through +;; `Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps`. The nodes stay in C# +;; because an internal node holds its keys and children in inline arrays. +;; - Everything that spans several nodes is here: descent, splits, merges, +;; rotations, iteration, building and joining trees. +;; - A leaf holds up to 64 entries, an internal node up to 31 keys and 32 +;; children. An edit that leaves a node other than the root with fewer +;; than 8 keys takes a key from a sibling or merges with it. A tree that is +;; built, split or joined may have smaller nodes at its right edge. +;; - An edit copies the nodes on its path. A transient's owner lets it edit +;; the nodes it copied in place. The owner `owner-none` makes `editable` +;; copy every node it is given. +;; - A leaf's value type is not in its C# base type. Every leaf is annotated +;; with `(Leaf %k %v)` where a `%v` is in scope, and a function with no +;; `%v` in its signature otherwise takes `vt`, a `(Option %v)` that is +;; always `None` and only names the type. + +(import (std prelude)) + +(:doc #:file "orderedmap.bjodoc") + +(export + OrderedMap OrderedMapCursor OrderedMapRange OrderedMapBuilder TransientOrderedMap OrderedMapping + ;; Maps + orderedmap-empty orderedmap-length orderedmap-empty? orderedmap-clear + orderedmap-ref orderedmap-ref-or orderedmap-try-ref orderedmap-contains? orderedmap-has-key? + orderedmap-set orderedmap-set-all orderedmap-remove orderedmap-remove-all + orderedmap-alter orderedmap-update + orderedmap-keys orderedmap-values + orderedmap-merge orderedmap-merge-with + ;; Order + orderedmap-min orderedmap-max orderedmap-successor orderedmap-predecessor + orderedmap-range orderedmap-from orderedmap-until + ;; Iteration and rebuilding + orderedmap-map orderedmap-map-values orderedmap-filter orderedmap-fold + orderedmap-for-each orderedmap-iter orderedmap-any? orderedmap-find + orderedmap-cursor orderedmap-cursor-done? orderedmap-cursor-current orderedmap-cursor-next + ;; Parts, for work on them at the same time + orderedmap-split orderedmap-concat + ;; Conversions + list->orderedmap orderedmap->list vec->orderedmap orderedmap->vec seq->orderedmap orderedmap->seq + map->orderedmap orderedmap->map orderedmap->str + ;; Builders + orderedmapbuilder-empty orderedmap->orderedmapbuilder orderedmapbuilder-add! orderedmapbuilder-add-all! + orderedmapbuilder-length orderedmapbuilder->orderedmap + ;; Transients + transientorderedmap-empty orderedmap->transientorderedmap transientorderedmap->orderedmap + transientorderedmap-set! transientorderedmap-set-all! transientorderedmap-remove! + transientorderedmap-alter! transientorderedmap-update! + transientorderedmap-remove-all! transientorderedmap-contains? transientorderedmap-has-key? transientorderedmap-ref + transientorderedmap-ref-or transientorderedmap-try-ref transientorderedmap-length + transientorderedmap-empty? transientorderedmap-filter!) + +(import/class + (Exception (: System.Exception)) + (KeyNotFoundException (: System.Collections.Generic.KeyNotFoundException)) + (ArgumentException (: System.ArgumentException)) + (InvalidOperationException (: System.InvalidOperationException)) + ((Node %k) (: Bjolang.Runtime.OrderedTrees.MapNodes.Node)) + ((Leaf %k %v) (: Bjolang.Runtime.OrderedTrees.MapNodes.LeafNode)) + (Owner (: Bjolang.Runtime.OrderedTrees.MapNodes.OwnerId)) + ((Comparer %a) (: System.Collections.Generic.IComparer))) + +(import/extern + (owner-none (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.OwnerNone (-> Owner))) + (owner-next (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.OwnerNext (-> Owner))) + (editable (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Editable (-> (Node %k) Owner (Node %k)))) + (new-leaf (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.NewLeaf (-> Owner (Leaf %k %v)))) + (new-internal (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.NewInternal (-> Owner (Node %k)))) + (as-leaf (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.AsLeaf (-> (Node %k) (Leaf %k %v)))) + (leaf-node (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafAsNode (-> (Leaf %k %v) (Node %k)))) + (leaf? (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.IsLeaf (-> (Node %k) bool))) + (node-count (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Count (-> (Node %k) int))) + (leaf-key (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafKey (-> (Leaf %k %v) int %k))) + (leaf-value (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafValue (-> (Leaf %k %v) int %v))) + (leaf-keys (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafKeys (-> (Leaf %k %v) (Array %k)))) + (leaf-values (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafValues (-> (Leaf %k %v) (Array %v)))) + (node-key (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Key (-> (Node %k) int %k))) + (key-of (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.KeyOf (-> (Node %k) int %k))) + (child (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Child (-> (Node %k) int (Node %k)))) + (leaf-find (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafFind (-> (Leaf %k %v) %k (Comparer %k) int))) + (route (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Route (-> (Node %k) %k (Comparer %k) int))) + (compare-keys (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.Compare (-> (Comparer %k) %k %k int))) + (set-count! (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.SetCount (-> (Node %k) int void))) + (leaf-set! (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafSet (-> (Leaf %k %v) int %k %v void))) + (leaf-set-value! (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafSetValue (-> (Leaf %k %v) int %v void))) + (set-key! (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.SetKey (-> (Node %k) int %k void))) + (set-child! (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.SetChild (-> (Node %k) int (Node %k) void))) + (leaf-move! (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.LeafMove (-> (Leaf %k %v) int (Leaf %k %v) int int void))) + (key-move! (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.KeyMove (-> (Node %k) int (Node %k) int int void))) + (child-move! (: Bjolang.Runtime.OrderedTrees.MapNodes.NodeOps.ChildMove (-> (Node %k) int (Node %k) int int void)))) + +;; Raise the .NET exceptions the C# map raised, so that a caller's handler +;; still catches them. +(: (-> %a)) +(defun (key-not-found) + (raise (cast Exception (KeyNotFoundException. "The key is not in the map.")))) + +(: (-> string %a)) +(defun (bad-argument why) (raise (cast Exception (ArgumentException. why)))) + +(def leaf-capacity 64) +(def internal-capacity 32) +(def merge-threshold 8) + +;; --------------------------------------------------------------------------- +;; The map +;; --------------------------------------------------------------------------- + +;; `%v` is the value type of the leaves under `root`. +(type (: (OrderedMap %k %v) #:opaque + (Record (: root (Node %k)) + (: count int) + (: comparer (Comparer %k))))) + +(: (-> (Comparer %k) (OrderedMap %k %v))) +(defun (empty-with cmp) + (def (: l (Leaf %k %v)) (new-leaf (owner-none))) + (OrderedMap (root (leaf-node l)) (count 0) (comparer cmp))) + +(: (-> (OrderedMap %k %v)) (where (Ord %k))) +(defun (orderedmap-empty) (empty-with (ord-comparer))) + +;; An empty map with the order of `m`. +(: (-> (OrderedMap %k %v) (OrderedMap %k %v))) +(defun (orderedmap-clear m) (empty-with (.comparer m))) + +(: (-> (OrderedMap %k %v) int)) +(defun (orderedmap-length m) (.count m)) + +(: (-> (OrderedMap %k %v) bool)) +(defun (orderedmap-empty? m) (= (.count m) 0)) + +;; --------------------------------------------------------------------------- +;; Lookups +;; --------------------------------------------------------------------------- + +;; The leaf under `node` that `key` belongs in. +(: (-> (Comparer %k) (Node %k) %k (Leaf %k %v))) +(defun (leaf-for cmp node key) + (if (leaf? node) + (as-leaf node) + (leaf-for cmp (child node (route node key cmp)) key))) + +(: (-> (Comparer %k) (Node %k) %k (Option %v))) +(defun (try-ref-in cmp root key) + (def (: l (Leaf %k %v)) (leaf-for cmp root key)) + (def i (leaf-find l key cmp)) + (if (>= i 0) (Some (leaf-value l i)) None)) + +(: (-> (Comparer %k) (Node %k) %k %v %v)) +(defun (ref-or-in cmp root key fallback) + (def (: l (Leaf %k %v)) (leaf-for cmp root key)) + (def i (leaf-find l key cmp)) + (if (>= i 0) (leaf-value l i) fallback)) + +(: (-> (Comparer %k) (Node %k) %k %v)) +(defun (ref-in cmp root key) + (def (: l (Leaf %k %v)) (leaf-for cmp root key)) + (def i (leaf-find l key cmp)) + (if (>= i 0) (leaf-value l i) (key-not-found))) + +(: (-> (Comparer %k) (Node %k) %k (Option %v) bool)) +(defun (contains-in? cmp root key vt) + (def (: l (Leaf %k %v)) (leaf-for cmp root key)) + (>= (leaf-find l key cmp) 0)) + +(: (-> (OrderedMap %k %v) %k (Option %v))) +(defun (orderedmap-try-ref m k) (try-ref-in (.comparer m) (.root m) k)) + +;; Raises when `key` is not in `m`. +(: (-> (OrderedMap %k %v) %k %v)) +(defun (orderedmap-ref m k) (ref-in (.comparer m) (.root m) k)) + +(: (-> (OrderedMap %k %v) %k %v %v)) +(defun (orderedmap-ref-or m k default) (ref-or-in (.comparer m) (.root m) k default)) + +(: (-> (OrderedMap %k %v) %k bool)) +(defun (orderedmap-contains? m k) + (def (: vt (Option %v)) None) + (contains-in? (.comparer m) (.root m) k vt)) + +(: (-> (OrderedMap %k %v) %k bool)) +(defun (orderedmap-has-key? m key) (orderedmap-contains? m key)) + +(: (-> (Node %k) (Node %k))) +(defun (leftmost node) (if (leaf? node) node (leftmost (child node 0)))) + +(: (-> (Node %k) (Node %k))) +(defun (rightmost node) (if (leaf? node) node (rightmost (child node (node-count node))))) + +(: (-> (Node %k) %k)) +(defun (first-key node) (key-of (leftmost node) 0)) + +(: (-> (Node %k) (Option (Tuple %k %v)))) +(defun (min-in node) + (def (: l (Leaf %k %v)) (as-leaf (leftmost node))) + (if (= (leaf-count l) 0) None (Some (Tuple (leaf-key l 0) (leaf-value l 0))))) + +(: (-> (Node %k) (Option (Tuple %k %v)))) +(defun (max-in node) + (def (: l (Leaf %k %v)) (as-leaf (rightmost node))) + (def n (leaf-count l)) + (if (= n 0) None (Some (Tuple (leaf-key l (- n 1)) (leaf-value l (- n 1)))))) + +(: (-> (OrderedMap %k %v) (Option (Tuple %k %v)))) +(defun (orderedmap-min m) (min-in (.root m))) + +(: (-> (OrderedMap %k %v) (Option (Tuple %k %v)))) +(defun (orderedmap-max m) (max-in (.root m))) + +;; The entry with the least key greater than `key`. +(: (-> (OrderedMap %k %v) %k (Option (Tuple %k %v)))) +(defun (orderedmap-successor m k) (successor-in (.comparer m) (.root m) k)) + +(: (-> (Comparer %k) (Node %k) %k (Option (Tuple %k %v)))) +(defun (successor-in cmp node key) + (if (leaf? node) + (begin + (def (: l (Leaf %k %v)) (as-leaf node)) + (def i (leaf-find l key cmp)) + (def at (if (>= i 0) (+ i 1) (- -1 i))) + (if (< at (leaf-count l)) (Some (Tuple (leaf-key l at) (leaf-value l at))) None)) + (begin + (def r (route node key cmp)) + (def found (successor-in cmp (child node r) key)) + (if (and (none? found) (< r (node-count node))) + (min-in (child node (+ r 1))) + found)))) + +;; The entry with the greatest key less than `key`. +(: (-> (OrderedMap %k %v) %k (Option (Tuple %k %v)))) +(defun (orderedmap-predecessor m k) (predecessor-in (.comparer m) (.root m) k)) + +(: (-> (Comparer %k) (Node %k) %k (Option (Tuple %k %v)))) +(defun (predecessor-in cmp node key) + (if (leaf? node) + (begin + (def (: l (Leaf %k %v)) (as-leaf node)) + (def i (leaf-find l key cmp)) + (def at (if (>= i 0) i (- -1 i))) + (if (> at 0) (Some (Tuple (leaf-key l (- at 1)) (leaf-value l (- at 1)))) None)) + (begin + (def r (route node key cmp)) + (def found (predecessor-in cmp (child node r) key)) + (if (and (none? found) (> r 0)) + (max-in (child node (- r 1))) + found)))) + +;; --------------------------------------------------------------------------- +;; Slots in one node +;; --------------------------------------------------------------------------- + +(: (-> (Leaf %k %v) int)) +(defun (leaf-count l) (node-count (leaf-node l))) + +(: (-> (Leaf %k %v) int %k %v void)) +(defun (leaf-insert! l i key value) + (def n (leaf-count l)) + (when (< i n) (leaf-move! l i l (+ i 1) (- n i))) + (leaf-set! l i key value) + (set-count! (leaf-node l) (+ n 1))) + +(: (-> (Leaf %k %v) int void)) +(defun (leaf-remove! l i) + (def n (leaf-count l)) + (when (> (- n i 1) 0) (leaf-move! l (+ i 1) l i (- n i 1))) + (set-count! (leaf-node l) (- n 1))) + +;; Puts `sep` at key slot `i` and `right` at child slot `i + 1`. +(: (-> (Node %k) int %k (Node %k) void)) +(defun (internal-insert! node i sep right) + (def n (node-count node)) + (when (< i n) + (key-move! node i node (+ i 1) (- n i)) + (child-move! node (+ i 1) node (+ i 2) (- n i))) + (set-key! node i sep) + (set-child! node (+ i 1) right) + (set-count! node (+ n 1))) + +;; --------------------------------------------------------------------------- +;; Set +;; --------------------------------------------------------------------------- + +;; What `insert!` did: +;; - `(Tuple c leaf _)` with `c` negative: the key is at slot `-1 - c` of +;; `leaf`, and its value is not written. The path to `leaf` is editable. +;; - `(Tuple 1 _ _)`: the key is added. +;; - `(Tuple 2 right sep)`: the key is added and the node split. `right` is +;; the new right half, and `sep` its first key. +(: (-> (Comparer %k) (Node %k) %k %v Owner (Tuple int (Node %k) %k))) +(defun (insert! cmp node key value owner) + (if (leaf? node) + (insert-leaf! cmp (as-leaf node) key value owner) + (begin + (def i (route node key cmp)) + (def c (editable (child node i) owner)) + (set-child! node i c) + (def r (insert! cmp c key value owner)) + (def (Tuple code right sep) r) + (cond + ((< code 2) r) + ((< (node-count node) (- internal-capacity 1)) + (internal-insert! node i sep right) + (Tuple 1 node key)) + (else (split-internal! node i sep right owner)))))) + +(: (-> (Comparer %k) (Leaf %k %v) %k %v Owner (Tuple int (Node %k) %k))) +(defun (insert-leaf! cmp l key value owner) + (def i (leaf-find l key cmp)) + (cond + ((>= i 0) (Tuple (- -1 i) (leaf-node l) key)) + ((< (leaf-count l) leaf-capacity) + (leaf-insert! l (- -1 i) key value) + (Tuple 1 (leaf-node l) key)) + (else (split-leaf! l (- -1 i) key value owner)))) + +;; Splits a full leaf and inserts at slot `i`. An insert at either end +;; leaves the full half as it is, so that ascending or descending inserts +;; fill their leaves. +(: (-> (Leaf %k %v) int %k %v Owner (Tuple int (Node %k) %k))) +(defun (split-leaf! left i key value owner) + (def (: right (Leaf %k %v)) (new-leaf owner)) + (def total (leaf-count left)) + (def at (cond ((= i total) total) ((= i 0) 0) (else (shift-right-logical total 1)))) + (def moving (- total at)) + (when (> moving 0) (leaf-move! left at right 0 moving)) + (set-count! (leaf-node left) at) + (set-count! (leaf-node right) moving) + (if (or (< i at) (and (= at 0) (= i 0))) + (leaf-insert! left i key value) + (leaf-insert! right (- i at) key value)) + (Tuple 2 (leaf-node right) (leaf-key right 0))) + +;; Splits a full internal node, and puts `sep` and `new-child` at slot `i`. +(: (-> (Node %k) int %k (Node %k) Owner (Tuple int (Node %k) %k))) +(defun (split-internal! left i sep new-child owner) + (def right (new-internal owner)) + (def n (node-count left)) + (def at (shift-right-logical n 1)) + (def up (node-key left at)) + (def moving (- n at 1)) + (key-move! left (+ at 1) right 0 moving) + (child-move! left (+ at 1) right 0 (+ moving 1)) + (set-count! left at) + (set-count! right moving) + (if (<= i at) + (internal-insert! left i sep new-child) + (internal-insert! right (- i (+ at 1)) sep new-child)) + (Tuple 2 right up)) + +(: (-> (Node %k) %k (Node %k) Owner (Node %k))) +(defun (new-root left sep right owner) + (def r (new-internal owner)) + (set-key! r 0 sep) + (set-child! r 0 left) + (set-child! r 1 right) + (set-count! r 1) + r) + +;; Sets `key` in the tree under `root`. Returns the new root, and whether +;; the key is new. +(: (-> (Comparer %k) (Node %k) %k %v Owner (Tuple (Node %k) bool))) +(defun (set-in cmp root key value owner) + (def r (editable root owner)) + (def (Tuple code right sep) (insert! cmp r key value owner)) + (cond + ((< code 0) + (def (: l (Leaf %k %v)) (as-leaf right)) + (leaf-set-value! l (- -1 code) value) + (Tuple r #f)) + ((= code 2) (Tuple (new-root r sep right owner) #t)) + (else (Tuple r #t)))) + +(: (-> (OrderedMap %k %v) %k %v (OrderedMap %k %v))) +(defun (orderedmap-set m k v) + (def (Tuple root added?) (set-in (.comparer m) (.root m) k v (owner-none))) + (OrderedMap (root root) + (count (if added? (+ (.count m) 1) (.count m))) + (comparer (.comparer m)))) + +;; --------------------------------------------------------------------------- +;; Remove +;; --------------------------------------------------------------------------- + +;; Removes `key` under `node`, which `owner` may edit. Returns 0 if the key +;; is not there, 1 if it is removed, and 2 if it is removed and `node` is +;; left with fewer than `merge-threshold` keys. +(: (-> (Comparer %k) (Node %k) %k Owner (Option %v) int)) +(defun (remove! cmp node key owner vt) + (if (leaf? node) + (begin + (def (: l (Leaf %k %v)) (as-leaf node)) + (def i (leaf-find l key cmp)) + (cond + ((< i 0) 0) + (else + (leaf-remove! l i) + (if (< (leaf-count l) merge-threshold) 2 1)))) + (begin + (def i (route node key cmp)) + (def c (editable (child node i) owner)) + (set-child! node i c) + (def r (remove! cmp c key owner vt)) + (cond + ((< r 2) r) + ((underflow! node i owner vt) 2) + (else 1))))) + +(: (-> (Node %k) bool)) +(defun (can-lend? node) (> (node-count node) (+ merge-threshold 1))) + +;; Child `i` of `parent` has too few keys. Takes one from a sibling that has +;; enough, or else merges with a sibling. Returns whether `parent` has too few +;; keys after. +(: (-> (Node %k) int Owner (Option %v) bool)) +(defun (underflow! parent i owner vt) + (cond + ((< i (node-count parent)) + (def right (editable (child parent (+ i 1)) owner)) + (set-child! parent (+ i 1) right) + (balance! parent i (child parent i) right #t vt)) + ((> i 0) + (def left (editable (child parent (- i 1)) owner)) + (set-child! parent (- i 1) left) + (balance! parent (- i 1) left (child parent i) #f vt)) + (else #t))) + +;; `left` and `right` are the children on each side of key `s` of `parent`. +;; The one `from-right?` names lends a key, or the two merge. +(: (-> (Node %k) int (Node %k) (Node %k) bool (Option %v) bool)) +(defun (balance! parent s left right from-right? vt) + (cond + ((and from-right? (can-lend? right)) (rotate-left! parent s left right vt) #f) + ((and (not from-right?) (can-lend? left)) (rotate-right! parent s left right vt) #f) + (else + (merge! parent s left right vt) + (< (node-count parent) merge-threshold)))) + +;; Moves the first entry of `right` to the end of `left`. +(: (-> (Node %k) int (Node %k) (Node %k) (Option %v) void)) +(defun (rotate-left! parent s left right vt) + (if (leaf? left) + (begin + (def (: l (Leaf %k %v)) (as-leaf left)) + (def (: r (Leaf %k %v)) (as-leaf right)) + (leaf-insert! l (leaf-count l) (leaf-key r 0) (leaf-value r 0)) + (leaf-remove! r 0) + (set-key! parent s (leaf-key r 0))) + (begin + (internal-insert! left (node-count left) (node-key parent s) (child right 0)) + (set-key! parent s (node-key right 0)) + (def rn (node-count right)) + (child-move! right 1 right 0 rn) + (when (> rn 1) (key-move! right 1 right 0 (- rn 1))) + (set-count! right (- rn 1))))) + +;; Moves the last entry of `left` to the front of `right`. +(: (-> (Node %k) int (Node %k) (Node %k) (Option %v) void)) +(defun (rotate-right! parent s left right vt) + (if (leaf? left) + (begin + (def (: l (Leaf %k %v)) (as-leaf left)) + (def (: r (Leaf %k %v)) (as-leaf right)) + (def last (- (leaf-count l) 1)) + (leaf-insert! r 0 (leaf-key l last) (leaf-value l last)) + (leaf-remove! l last) + (set-key! parent s (leaf-key r 0))) + (begin + (def last (- (node-count left) 1)) + (def rn (node-count right)) + (key-move! right 0 right 1 rn) + (set-key! right 0 (node-key parent s)) + (child-move! right 0 right 1 (+ rn 1)) + (set-child! right 0 (child left (+ last 1))) + (set-count! right (+ rn 1)) + (set-key! parent s (node-key left last)) + (set-count! left last)))) + +;; Moves all of `right` to the end of `left`, and removes key `s` of +;; `parent` and the child after it. +(: (-> (Node %k) int (Node %k) (Node %k) (Option %v) void)) +(defun (merge! parent s left right vt) + (def ln (node-count left)) + (def rn (node-count right)) + (if (leaf? left) + (begin + (def (: l (Leaf %k %v)) (as-leaf left)) + (def (: r (Leaf %k %v)) (as-leaf right)) + (leaf-move! r 0 l ln rn) + (set-count! left (+ ln rn))) + (begin + (set-key! left ln (node-key parent s)) + (key-move! right 0 left (+ ln 1) rn) + (child-move! right 0 left (+ ln 1) (+ rn 1)) + (set-count! left (+ ln 1 rn)))) + (def pn (node-count parent)) + (def moving (- pn s 1)) + (when (> moving 0) + (key-move! parent (+ s 1) parent s moving) + (child-move! parent (+ s 2) parent (+ s 1) moving)) + (set-count! parent (- pn 1))) + +;; Removes `key` from the tree under `root`. Returns the new root, and +;; whether the key was there. When it was not, the root is `root`. +(: (-> (Comparer %k) (Node %k) %k Owner (Option %v) (Tuple (Node %k) bool))) +(defun (remove-in cmp root key owner vt) + (def r (editable root owner)) + (def code (remove! cmp r key owner vt)) + (cond + ((= code 0) (Tuple root #f)) + ((and (= code 2) (not (leaf? r)) (= (node-count r) 0)) (Tuple (child r 0) #t)) + (else (Tuple r #t)))) + +(: (-> (OrderedMap %k %v) %k (OrderedMap %k %v))) +(defun (orderedmap-remove m k) + (def (: vt (Option %v)) None) + (def (Tuple root removed?) (remove-in (.comparer m) (.root m) k (owner-none) vt)) + (if removed? + (OrderedMap (root root) (count (- (.count m) 1)) (comparer (.comparer m))) + m)) + +;; --------------------------------------------------------------------------- +;; Alter and update +;; --------------------------------------------------------------------------- +;; +;; A read descent, then a set or a remove. The read allocates nothing, and the +;; write copies the path as `orderedmap-set` and `orderedmap-remove` do. + +;; The value an update applies its function to: the key's value, else the +;; default. Raises `KeyNotFoundException` when there is neither. +(: (-> (Option %v) (Option %v) %v)) +(defun (update-start old default) + (match old + ((Some v) v) + (None (match default + ((Some d) d) + (None (key-not-found)))))) + +;; `m` with the entry of `k` decided by `f`: +;; - `f` is given `(Some v)` for the value of `k`, or `None` if `k` is absent. +;; - `(Some v)` from `f` sets `k` to `v`, and `None` removes `k`. +;; - `None` for an absent key returns `m` itself. +(: (-> (OrderedMap %k %v) %k (-?-> (Option %v) (Option %v)) (OrderedMap %k %v))) +(defun (orderedmap-alter m k f) + (def old (orderedmap-try-ref m k)) + (match (f old) + ((Some v) (orderedmap-set m k v)) + (None (if (some? old) (orderedmap-remove m k) m)))) + +;; `m` with `k` set to `f` of its value. A missing key raises +;; `KeyNotFoundException`, unless `#:default (Some d)` is given: then `f` is +;; applied to `d`. +(: (-> (OrderedMap %k %v) %k (-?-> %v %v) (#:default (Option %v)) (OrderedMap %k %v))) +(defun (orderedmap-update m k f #:default None) + (orderedmap-set m k (f (update-start (orderedmap-try-ref m k) default)))) + +;; --------------------------------------------------------------------------- +;; The cursor +;; --------------------------------------------------------------------------- + +;; A position in an iteration in key order. A value: `orderedmap-cursor-next` returns +;; the next one. +;; +;; - It holds the key and value arrays of one leaf, so a step inside a leaf +;; is an increment and a compare. +;; - `path` is the child index at each level above the leaf, 6 bits a level +;; above the low 4 bits, which hold the leaf's depth. `last-leaf` marks a +;; leaf with no leaf after it. +;; - At the end of a leaf, `cursor-after` follows `path` from the root again. +(type (: (OrderedMapCursor %k %v) #:opaque + (Struct (: keys (Array %k)) + (: values (Array %v)) + (: i int) + (: end int) + (: root (Node %k)) + (: path long)))) + +(def depth-bits 4) +(def index-bits 6) +(def max-depth 10) +(: last-leaf long) +(def last-leaf -1) + +(: (-> long int int)) +(defun (index-at path level) + (cast System.Int32 (bitwise-and (shift-right-logical path (+ depth-bits (* index-bits level))) 63))) + +(: (-> long int int long)) +(defun (with-index path level index) + (def shift (+ depth-bits (* index-bits level))) + (def (: mask long) (shift-left 63 shift)) + (bitwise-ior (bitwise-and path (bitwise-not mask)) + (shift-left (cast System.Int64 index) shift))) + +(: (-> long int long)) +(defun (with-depth path depth) + (def (: depth-mask long) 15) + (bitwise-ior (bitwise-and path (bitwise-not depth-mask)) (cast System.Int64 depth))) + +(: (-> (Leaf %k %v) int (Node %k) long (OrderedMapCursor %k %v))) +(defun (cursor-at l i root path) + (OrderedMapCursor (keys (leaf-keys l)) (values (leaf-values l)) (i i) (end (leaf-count l)) + (root root) (path path))) + +(: (-> (Node %k) (OrderedMapCursor %k %v))) +(defun (cursor-of root) + (if (leaf? root) + (begin + (def (: l (Leaf %k %v)) (as-leaf root)) + (cursor-at l 0 root last-leaf)) + (descend root root 0 0))) + +(: (-> (OrderedMap %k %v) (OrderedMapCursor %k %v))) +(defun (orderedmap-cursor m) (cursor-of (.root m))) + +;; The cursor on the first entry of the leftmost leaf under `node`, which is +;; at `level`, or on the entry after it if that leaf is empty. +(: (-> (Node %k) (Node %k) int long (OrderedMapCursor %k %v))) +(defun (descend node root level path) + (cond + ((leaf? node) + (def (: l (Leaf %k %v)) (as-leaf node)) + (def at (with-depth path level)) + (if (> (leaf-count l) 0) + (cursor-at l 0 root at) + (advance l root at))) + ((= level max-depth) + (raise (cast Exception (InvalidOperationException. "The map is deeper than a cursor can walk.")))) + (else (descend (child node 0) root (+ level 1) (with-index path level 0))))) + +;; The cursor on the first entry after the leaf `l` at `path`, or a done +;; cursor on `l`. +(: (-> (Leaf %k %v) (Node %k) long (OrderedMapCursor %k %v))) +(defun (advance l root path) + (def depth (cast System.Int32 (bitwise-and path 15))) + ;; The deepest level whose child index can move right, and its node. + (def/mutable level -1) + (def/mutable parent root) + (def/mutable node root) + (loop (:for d (range 0 depth)) + (:do (def i (index-at path d)) + (when (< i (node-count node)) + (set! level d) + (set! parent node)) + (set! node (child node i)))) + (if (< level 0) + (cursor-at l (leaf-count l) root last-leaf) + (begin + (def next (+ (index-at path level) 1)) + (descend (child parent next) root (+ level 1) (with-index path level next))))) + +;; The cursor on the first entry whose key is not less than `key`, or with +;; `after?` greater than `key`. +(: (-> (Comparer %k) (Node %k) (Node %k) %k bool int long (OrderedMapCursor %k %v))) +(defun (cursor-from cmp node root key after? level path) + (if (leaf? node) + (begin + (def (: l (Leaf %k %v)) (as-leaf node)) + (def i (leaf-find l key cmp)) + (def at (cond ((< i 0) (- -1 i)) (after? (+ i 1)) (else i))) + (def p (if (= level 0) last-leaf (with-depth path level))) + (if (or (< at (leaf-count l)) (= level 0)) + (cursor-at l at root p) + (advance l root p))) + (begin + (def r (route node key cmp)) + (cursor-from cmp (child node r) root key after? (+ level 1) (with-index path level r))))) + +(: (-> (OrderedMapCursor %k %v) bool)) +(defun (orderedmap-cursor-done? c) (>= (.i c) (.end c))) + +(: (-> (OrderedMapCursor %k %v) (Tuple %k %v))) +(defun (orderedmap-cursor-current c) + (Tuple (array-ref (.keys c) (.i c)) (array-ref (.values c) (.i c)))) + +(: (-> (OrderedMapCursor %k %v) %k)) +(defun (cursor-key c) (array-ref (.keys c) (.i c))) + +(: (-> (OrderedMapCursor %k %v) (OrderedMapCursor %k %v))) +(defun (orderedmap-cursor-next c) + (def i (+ (.i c) 1)) + (if (or (< i (.end c)) (= (.path c) last-leaf)) + (record-set c (i i)) + (cursor-after c))) + +;; Out of line, so that a step inside a leaf stays small. +(: (-> (OrderedMapCursor %k %v) (OrderedMapCursor %k %v))) +(defun (cursor-after c) + (def root (.root c)) + (def depth (cast System.Int32 (bitwise-and (.path c) 15))) + (def/mutable node root) + (loop (:for d (range 0 depth)) (:do (set! node (child node (index-at (.path c) d))))) + (def (: l (Leaf %k %v)) (as-leaf node)) + (advance l root (.path c))) + +(impl (Iterable (OrderedMap %k %v)) + (type %elem (Tuple %k %v)) + (type %cursor (OrderedMapCursor %k %v)) + (defun (iterable-start m) (orderedmap-cursor m)) + (defun (iterable-done? m c) (orderedmap-cursor-done? c)) + (defun (iterable-current m c) (orderedmap-cursor-current c)) + (defun (iterable-next m c) (orderedmap-cursor-next c)) + (defun (count m) (orderedmap-length m))) + +;; --------------------------------------------------------------------------- +;; Ranges +;; --------------------------------------------------------------------------- + +;; The entries from `from` to `to`, both included. `orderedmap-range`, +;; `orderedmap-from` and `orderedmap-until` return one, not a `Seq`: a loop over it +;; inlines the cursor. `(->seq r)` returns a `Seq` of it. +;; +;; - `start` is the cursor on the first entry. +;; - `end-i` and `end-path` are the position of the first entry after the +;; range. A cursor's position is its slot and path, and two cursors on one +;; entry have the same position, so a step compares two numbers and no key. +;; - With no upper bound, `end-i` is -1, which no cursor has. +;; - When `from` is greater than `to`, `start` is the end. +(type (: (OrderedMapRange %k %v) #:opaque + (Struct (: start (OrderedMapCursor %k %v)) (: end-i int) (: end-path long)))) + +(impl (Iterable (OrderedMapRange %k %v)) + (type %elem (Tuple %k %v)) + (type %cursor (OrderedMapCursor %k %v)) + (defun (iterable-start r) (.start r)) + (defun (iterable-done? r c) + (or (orderedmap-cursor-done? c) (and (= (.i c) (.end-i r)) (= (.path c) (.end-path r))))) + (defun (iterable-current r c) (orderedmap-cursor-current c)) + (defun (iterable-next r c) (orderedmap-cursor-next c))) + +(: (-> (OrderedMap %k %v) %k bool (OrderedMapCursor %k %v))) +(defun (cursor-at-key m key after?) (cursor-from (.comparer m) (.root m) (.root m) key after? 0 0)) + +(: (-> (OrderedMapCursor %k %v) (OrderedMapCursor %k %v) (OrderedMapRange %k %v))) +(defun (range-of start end) + (OrderedMapRange (start start) (end-i (.i end)) (end-path (.path end)))) + +(: (-> (OrderedMap %k %v) %k %k (OrderedMapRange %k %v))) +(defun (orderedmap-range m lo hi) + (def end (cursor-at-key m hi #t)) + (range-of (if (> (compare-keys (.comparer m) lo hi) 0) end (cursor-at-key m lo #f)) end)) + +(: (-> (OrderedMap %k %v) %k (OrderedMapRange %k %v))) +(defun (orderedmap-from m lo) + (OrderedMapRange (start (cursor-at-key m lo #f)) (end-i -1) (end-path 0))) + +(: (-> (OrderedMap %k %v) %k (OrderedMapRange %k %v))) +(defun (orderedmap-until m hi) (range-of (orderedmap-cursor m) (cursor-at-key m hi #t))) + +(: (-> (OrderedMap %k %v) (Seq %k))) +(defun (orderedmap-keys m) (seql (:for (Tuple k _) m) (:yield k))) + +(: (-> (OrderedMap %k %v) (Seq %v))) +(defun (orderedmap-values m) (seql (:for (Tuple _ v) m) (:yield v))) + +;; --------------------------------------------------------------------------- +;; Building trees +;; --------------------------------------------------------------------------- + +;; One tree over nodes of one depth, in key order: internal nodes of up to 32 +;; children each, as even as they can be, level by level up to one root. +(: (-> (Array (Node %k)) (Node %k))) +(defun (over nodes) + (def n (array-length nodes)) + (if (= n 1) + (array-ref nodes 0) + (begin + (def parents (/ (+ n (- internal-capacity 1)) internal-capacity)) + (def up (make-array parents)) + (def/mutable offset 0) + (loop (:for p (range 0 parents)) + (:do (def size (+ (/ n parents) (if (< p (% n parents)) 1 0))) + (def parent (new-internal (owner-none))) + (loop (:for c (range 0 size)) + (:do (def node (array-ref nodes (+ offset c))) + (set-child! parent c node) + (when (> c 0) (set-key! parent (- c 1) (first-key node))))) + (set-count! parent (- size 1)) + (array-set! up p parent) + (set! offset (+ offset size)))) + (over up)))) + +;; A map of the first `n` keys and values, in ascending key order, in full +;; leaves. +(: (-> (Array %k) (Array %v) int (Comparer %k) (OrderedMap %k %v))) +(defun (from-sorted ks vs n cmp) + (if (= n 0) + (empty-with cmp) + (begin + (def count (/ (+ n (- leaf-capacity 1)) leaf-capacity)) + (def leaves (make-array count)) + (loop (:for i (range 0 count)) + (:do (def (: l (Leaf %k %v)) (new-leaf (owner-none))) + (def at (* i leaf-capacity)) + (def size (min leaf-capacity (- n at))) + (array-copy! ks at (leaf-keys l) 0 size) + (array-copy! vs at (leaf-values l) 0 size) + (set-count! (leaf-node l) size) + (array-set! leaves i (leaf-node l)))) + (OrderedMap (root (over leaves)) (count n) (comparer cmp))))) + +;; --------------------------------------------------------------------------- +;; Builders +;; --------------------------------------------------------------------------- + +;; Entries added in any order, made into a map at the end. The last value of +;; a key wins. Entries added in ascending key order need no sort. +(type (: (OrderedMapBuilder %k %v) #:opaque + (Record (: keys (Array %k) #:mutable) + (: values (Array %v) #:mutable) + (: count int #:mutable) + (: ascending? bool #:mutable) + (: comparer (Comparer %k))))) + +(: (-> (Comparer %k) int (OrderedMapBuilder %k %v))) +(defun (builder-with cmp capacity) + (OrderedMapBuilder (keys (make-array capacity)) (values (make-array capacity)) (count 0) + (ascending? #t) (comparer cmp))) + +(: (-> (OrderedMapBuilder %k %v)) (where (Ord %k))) +(defun (orderedmapbuilder-empty) (builder-with (ord-comparer) 16)) + +;; A builder that holds the entries of `m`. +(: (-> (OrderedMap %k %v) (OrderedMapBuilder %k %v))) +(defun (orderedmap->orderedmapbuilder m) + (def b (builder-with (.comparer m) (max 16 (.count m)))) + (loop (:for (Tuple k v) m) (:do (orderedmapbuilder-add! b k v))) + b) + +(: (-> (OrderedMapBuilder %k %v) %k %v void)) +(defun (orderedmapbuilder-add! b k v) + (def n (.count b)) + (when (= n (array-length (.keys b))) + (record-set! b (keys (array-resize (.keys b) (* 2 n))) (values (array-resize (.values b) (* 2 n))))) + (when (and (.ascending? b) (> n 0) + (>= (compare-keys (.comparer b) (array-ref (.keys b) (- n 1)) k) 0)) + (record-set! b (ascending? #f))) + (array-set! (.keys b) n k) + (array-set! (.values b) n v) + (record-set! b (count (+ n 1)))) + +(: (-> (OrderedMapBuilder %k %v) (Seq (Tuple %k %v)) void)) +(defun (orderedmapbuilder-add-all! b kvs) + (loop (:for (Tuple k v) kvs) (:do (orderedmapbuilder-add! b k v)))) + +;; The number of entries added, a key added twice counted twice. +(: (-> (OrderedMapBuilder %k %v) int)) +(defun (orderedmapbuilder-length b) (.count b)) + +(: (-> (OrderedMapBuilder %k %v) (OrderedMap %k %v))) +(defun (orderedmapbuilder->orderedmap b) + (def cmp (.comparer b)) + (def n (.count b)) + (def ks (.keys b)) + (def vs (.values b)) + (if (.ascending? b) + (from-sorted ks vs n cmp) + (begin + ;; The slots in key order, equal keys in the order they were added. + (def order (loop (:for i (range 0 n)) (:acc (arraying i)))) + (array-sort-by! (fun (x y) (compare-keys cmp (array-ref ks x) (array-ref ks y))) order) + (def uk (make-array n)) + (def uv (make-array n)) + (def/mutable u 0) + (loop (:for j order) + (:do (when (or (= u 0) (not (= (compare-keys cmp (array-ref ks j) (array-ref uk (- u 1))) 0))) + (set! u (+ u 1))) + (array-set! uk (- u 1) (array-ref ks j)) + (array-set! uv (- u 1) (array-ref vs j)))) + (from-sorted uk uv u cmp)))) + +;; --------------------------------------------------------------------------- +;; Transients +;; --------------------------------------------------------------------------- + +;; A map that edits in place the nodes it copied. `transientorderedmap->orderedmap` +;; returns a map of its contents and gives it a new owner, so that the map +;; does not change after. +(type (: (TransientOrderedMap %k %v) #:opaque + (Record (: root (Node %k) #:mutable) + (: count int #:mutable) + (: owner Owner #:mutable) + (: comparer (Comparer %k))))) + +(: (-> (TransientOrderedMap %k %v)) (where (Ord %k))) +(defun (transientorderedmap-empty) (orderedmap->transientorderedmap (orderedmap-empty))) + +(: (-> (OrderedMap %k %v) (TransientOrderedMap %k %v))) +(defun (orderedmap->transientorderedmap m) + (TransientOrderedMap (root (.root m)) (count (.count m)) (owner (owner-next)) (comparer (.comparer m)))) + +(: (-> (TransientOrderedMap %k %v) (OrderedMap %k %v))) +(defun (transientorderedmap->orderedmap t) + (record-set! t (owner (owner-next))) + (OrderedMap (root (.root t)) (count (.count t)) (comparer (.comparer t)))) + +(: (-> (TransientOrderedMap %k %v) %k %v void)) +(defun (transientorderedmap-set! t k v) + (def (Tuple root added?) (set-in (.comparer t) (.root t) k v (.owner t))) + (record-set! t (root root)) + (when added? (record-set! t (count (+ (.count t) 1))))) + +(: (-> (TransientOrderedMap %k %v) %k void)) +(defun (transientorderedmap-remove! t k) + (def (: vt (Option %v)) None) + (def (Tuple root removed?) (remove-in (.comparer t) (.root t) k (.owner t) vt)) + (when removed? (record-set! t (root root) (count (- (.count t) 1))))) + +;; `orderedmap-alter` in place. +(: (-> (TransientOrderedMap %k %v) %k (-?-> (Option %v) (Option %v)) void)) +(defun (transientorderedmap-alter! t k f) + (def old (transientorderedmap-try-ref t k)) + (match (f old) + ((Some v) (transientorderedmap-set! t k v)) + (None (when (some? old) (transientorderedmap-remove! t k))))) + +;; `orderedmap-update` in place. +(: (-> (TransientOrderedMap %k %v) %k (-?-> %v %v) (#:default (Option %v)) void)) +(defun (transientorderedmap-update! t k f #:default None) + (transientorderedmap-set! t k (f (update-start (transientorderedmap-try-ref t k) default)))) + +(: (-> (TransientOrderedMap %k %v) (Seq (Tuple %k %v)) void)) +(defun (transientorderedmap-set-all! t kvs) + (loop (:for (Tuple k v) kvs) (:do (transientorderedmap-set! t k v)))) + +(: (-> (TransientOrderedMap %k %v) (Seq %k) void)) +(defun (transientorderedmap-remove-all! t ks) + (loop (:for k ks) (:do (transientorderedmap-remove! t k)))) + +(: (-> (TransientOrderedMap %k %v) %k bool)) +(defun (transientorderedmap-contains? t k) + (def (: vt (Option %v)) None) + (contains-in? (.comparer t) (.root t) k vt)) + +(: (-> (TransientOrderedMap %k %v) %k bool)) +(defun (transientorderedmap-has-key? t k) (transientorderedmap-contains? t k)) + +(: (-> (TransientOrderedMap %k %v) %k %v)) +(defun (transientorderedmap-ref t k) (ref-in (.comparer t) (.root t) k)) + +(: (-> (TransientOrderedMap %k %v) %k %v %v)) +(defun (transientorderedmap-ref-or t k default) (ref-or-in (.comparer t) (.root t) k default)) + +(: (-> (TransientOrderedMap %k %v) %k (Option %v))) +(defun (transientorderedmap-try-ref t k) (try-ref-in (.comparer t) (.root t) k)) + +(: (-> (TransientOrderedMap %k %v) int)) +(defun (transientorderedmap-length t) (.count t)) + +(: (-> (TransientOrderedMap %k %v) bool)) +(defun (transientorderedmap-empty? t) (= (.count t) 0)) + +;; Removes the entries `keep?` rejects. Calls it in key order, before it +;; removes any. +(: (-> (-?-> %k %v bool) (TransientOrderedMap %k %v) void)) +(defun (transientorderedmap-filter! keep? t) + (def/mutable dropped '()) + (loop (:for (Tuple k v) (OrderedMap (root (.root t)) (count (.count t)) (comparer (.comparer t)))) + (:do (unless (keep? k v) (set! dropped (cons k dropped))))) + (loop (:for k dropped) (:do (transientorderedmap-remove! t k)))) + +;; --------------------------------------------------------------------------- +;; Iteration +;; --------------------------------------------------------------------------- + +;; Folds `f` over the entries in key order. Recursive over the tree, which +;; is about twice as fast as the cursor: it reads each leaf's arrays once. +(: (-> (-?-> %k %v %s %s) %s (OrderedMap %k %v) %s)) +(defun (orderedmap-fold f init m) (fold-node f init (.root m))) + +(: (-> (-?-> %k %v %s %s) %s (Node %k) %s)) +(defun (fold-node f acc node) + (if (leaf? node) + (begin + (def (: l (Leaf %k %v)) (as-leaf node)) + (def ks (leaf-keys l)) + (def vs (leaf-values l)) + (loop (:for i (range 0 (leaf-count l))) + (:acc s (folding acc (f (array-ref ks i) (array-ref vs i) s))) + => s)) + (loop (:for i (range 0 (+ (node-count node) 1))) + (:acc s (folding acc (fold-node f s (child node i)))) + => s))) + +(: (-> (-?-> %k %v void) (OrderedMap %k %v) void)) +(defun (orderedmap-for-each f m) + (loop (:for (Tuple k v) m) (:do (f k v)))) + +;; Calls `f` in key order until it returns #f. Returns whether it never did. +(: (-> (-?-> %k %v bool) (OrderedMap %k %v) bool)) +(defun (orderedmap-iter f m) + (let go ((c (orderedmap-cursor m))) + (cond + ((orderedmap-cursor-done? c) #t) + (:def (Tuple k v) (orderedmap-cursor-current c)) + ((f k v) (go (orderedmap-cursor-next c))) + (else #f)))) + +(: (-> (-?-> %k %v bool) (OrderedMap %k %v) bool)) +(defun (orderedmap-any? pred m) + (let go ((c (orderedmap-cursor m))) + (cond + ((orderedmap-cursor-done? c) #f) + (:def (Tuple k v) (orderedmap-cursor-current c)) + ((pred k v) #t) + (else (go (orderedmap-cursor-next c)))))) + +;; The first entry in key order that `pred` accepts. +(: (-> (-?-> %k %v bool) (OrderedMap %k %v) (Option (Tuple %k %v)))) +(defun (orderedmap-find pred m) + (let go ((c (orderedmap-cursor m))) + (cond + ((orderedmap-cursor-done? c) None) + (:def (Tuple k v) (orderedmap-cursor-current c)) + ((pred k v) (Some (Tuple k v))) + (else (go (orderedmap-cursor-next c)))))) + +;; A map of the same keys, with `f` of each entry. A tree of the same shape: +;; the keys are copied and `f` is called in key order. +(: (-> (-?-> %k %v %w) (OrderedMap %k %v) (OrderedMap %k %w))) +(defun (orderedmap-map f m) + (def (: vt (Option %v)) None) + (def (: wt (Option %w)) None) + (OrderedMap (root (map-node f (.root m) vt wt)) (count (.count m)) (comparer (.comparer m)))) + +(: (-> (-?-> %k %v %w) (Node %k) (Option %v) (Option %w) (Node %k))) +(defun (map-node f node vt wt) + (if (leaf? node) + (begin + (def (: l (Leaf %k %v)) (as-leaf node)) + (def (: out (Leaf %k %w)) (new-leaf (owner-none))) + (def n (leaf-count l)) + (def ks (leaf-keys l)) + (def vs (leaf-values l)) + (def ws (leaf-values out)) + (array-copy! ks 0 (leaf-keys out) 0 n) + (loop (:for i (range 0 n)) (:do (array-set! ws i (f (array-ref ks i) (array-ref vs i))))) + (set-count! (leaf-node out) n) + (leaf-node out)) + (begin + (def out (new-internal (owner-none))) + (def n (node-count node)) + (key-move! node 0 out 0 n) + (loop (:for i (range 0 (+ n 1))) (:do (set-child! out i (map-node f (child node i) vt wt)))) + (set-count! out n) + out))) + +(: (-> (-?-> %v %w) (OrderedMap %k %v) (OrderedMap %k %w))) +(defun (orderedmap-map-values f m) + (def (: vt (Option %v)) None) + (def (: wt (Option %w)) None) + (OrderedMap (root (map-values-node f (.root m) vt wt)) (count (.count m)) (comparer (.comparer m)))) + +(: (-> (-?-> %v %w) (Node %k) (Option %v) (Option %w) (Node %k))) +(defun (map-values-node f node vt wt) + (if (leaf? node) + (begin + (def (: l (Leaf %k %v)) (as-leaf node)) + (def (: out (Leaf %k %w)) (new-leaf (owner-none))) + (def n (leaf-count l)) + (def vs (leaf-values l)) + (def ws (leaf-values out)) + (array-copy! (leaf-keys l) 0 (leaf-keys out) 0 n) + (loop (:for i (range 0 n)) (:do (array-set! ws i (f (array-ref vs i))))) + (set-count! (leaf-node out) n) + (leaf-node out)) + (begin + (def out (new-internal (owner-none))) + (def n (node-count node)) + (key-move! node 0 out 0 n) + (loop (:for i (range 0 (+ n 1))) (:do (set-child! out i (map-values-node f (child node i) vt wt)))) + (set-count! out n) + out))) + +;; The entries `keep?` accepts, in new leaves filled one after the other. +;; Returns `m` when it accepts every entry. +(: (-> (-?-> %k %v bool) (OrderedMap %k %v) (OrderedMap %k %v))) +(defun (orderedmap-filter keep? m) + (def n (.count m)) + (def (: ks (Array %k)) (make-array n)) + (def (: vs (Array %v)) (make-array n)) + (def/mutable kept 0) + (loop (:for (Tuple k v) m) + (:do (when (keep? k v) + (array-set! ks kept k) + (array-set! vs kept v) + (set! kept (+ kept 1))))) + (if (= kept n) m (from-sorted ks vs kept (.comparer m)))) + +;; --------------------------------------------------------------------------- +;; Merges +;; --------------------------------------------------------------------------- + +;; The entries of all the maps. The last map's value of a key wins. +(: (-> (OrderedMap %k %v) (OrderedMap %k %v) #:rest (OrderedMap %k %v) (OrderedMap %k %v))) +(defun (orderedmap-merge m other #:rest more) + (merge-all (fun (k a b) b) (inputs-of m other more))) + +;; The entries of all the maps. The values of a key are combined by `f`, +;; first to last. +(: (-> (-?-> %k %v %v %v) (OrderedMap %k %v) (OrderedMap %k %v) #:rest (OrderedMap %k %v) (OrderedMap %k %v))) +(defun (orderedmap-merge-with f m other #:rest more) + (merge-all f (inputs-of m other more))) + +(: (-> (OrderedMap %k %v) (OrderedMap %k %v) (Array (OrderedMap %k %v)) (Array (OrderedMap %k %v)))) +(defun (inputs-of m other more) + (def all (cons m (cons other (loop (:for x more) (:acc (listing x)))))) + (def kept (loop (:for x all) (:when (> (.count x) 0)) (:acc (arraying x)))) + (if (= (array-length kept) 0) (vec->array [m]) kept)) + +;; At most this many inputs are iterated at once, and only when the inputs +;; other than the biggest hold this percent of the keys or more. Else the +;; others are set into a transient of the biggest. +(def merge-max-inputs 32) +(def merge-percent 30) + +(: (-> (-?-> %k %v %v %v) (Array (OrderedMap %k %v)) (OrderedMap %k %v))) +(defun (merge-all f inputs) + (def k (array-length inputs)) + (cond + ((= k 1) (array-ref inputs 0)) + (else + (def total (loop (:for x inputs) (:acc (summing (.count x))))) + (def biggest (loop (:for x inputs) (:acc b (folding 0 (max b (.count x)))) => b)) + (if (and (<= k merge-max-inputs) (>= (* 100 (- total biggest)) (* merge-percent total))) + (merge-walk f inputs total) + (merge-into f inputs))))) + +;; The inputs set one after the other into a transient of the first, the +;; values of a key combined in input order. +(: (-> (-?-> %k %v %v %v) (Array (OrderedMap %k %v)) (OrderedMap %k %v))) +(defun (merge-into f inputs) + (def t (orderedmap->transientorderedmap (array-ref inputs 0))) + (loop (:for i (range 1 (array-length inputs))) + (:do (loop (:for (Tuple k v) (array-ref inputs i)) + (:do (combine! f t k v))))) + (transientorderedmap->orderedmap t)) + +;; Sets `k` in `t` to `v`, or to `(f k old v)` if it is there, by one +;; descent. +(: (-> (-?-> %k %v %v %v) (TransientOrderedMap %k %v) %k %v void)) +(defun (combine! f t k v) + (def owner (.owner t)) + (def r (editable (.root t) owner)) + (def (Tuple code right sep) (insert! (.comparer t) r k v owner)) + (cond + ((< code 0) + (record-set! t (root r)) + (def (: l (Leaf %k %v)) (as-leaf right)) + (def i (- -1 code)) + (leaf-set-value! l i (f k (leaf-value l i) v))) + (else + (record-set! t (root (if (= code 2) (new-root r sep right owner) r)) + (count (+ (.count t) 1)))))) + +;; The inputs iterated in key order at once into a builder, which needs no +;; sort. The least key is found by a scan of the cursors. +(: (-> (-?-> %k %v %v %v) (Array (OrderedMap %k %v)) int (OrderedMap %k %v))) +(defun (merge-walk f inputs total) + (def k (array-length inputs)) + (def cmp (.comparer (array-ref inputs 0))) + (def cursors (loop (:for x inputs) (:acc (arraying (orderedmap-cursor x))))) + (def b (builder-with cmp total)) + (def/mutable live k) + (loop (:while (> live 0)) + (:do + ;; The least key of the cursors that are not done. + (def/mutable least -1) + (loop (:for i (range 0 k)) + (:do (def c (array-ref cursors i)) + (when (and (not (orderedmap-cursor-done? c)) + (or (< least 0) + (< (compare-keys cmp (cursor-key c) (cursor-key (array-ref cursors least))) 0))) + (set! least i)))) + (def key (cursor-key (array-ref cursors least))) + ;; Its values, in input order, and the cursors stepped past it. + (def/mutable value (tuple-second (orderedmap-cursor-current (array-ref cursors least)))) + (array-set! cursors least (orderedmap-cursor-next (array-ref cursors least))) + (when (orderedmap-cursor-done? (array-ref cursors least)) (set! live (- live 1))) + (loop (:for i (range (+ least 1) k)) + (:do (def c (array-ref cursors i)) + (when (and (not (orderedmap-cursor-done? c)) (= (compare-keys cmp (cursor-key c) key) 0)) + (set! value (f key value (tuple-second (orderedmap-cursor-current c)))) + (array-set! cursors i (orderedmap-cursor-next c)) + (when (orderedmap-cursor-done? (array-ref cursors i)) (set! live (- live 1)))))) + (orderedmapbuilder-add! b key value))) + (orderedmapbuilder->orderedmap b)) + +(: (-> (Tuple %a %b) %b)) +(defun (tuple-second t) (def (Tuple _ b) t) b) + +;; --------------------------------------------------------------------------- +;; Parts +;; --------------------------------------------------------------------------- + +(: (-> (Node %k) int)) +(defun (count-keys node) + (if (leaf? node) + (node-count node) + (loop (:for i (range 0 (+ (node-count node) 1))) (:acc (summing (count-keys (child node i))))))) + +;; The leaves under an internal node, or the keys of a leaf. +(: (-> (Node %k) int)) +(defun (weight node) + (cond + ((leaf? node) (node-count node)) + ((leaf? (child node 0)) (+ (node-count node) 1)) + (else (loop (:for i (range 0 (+ (node-count node) 1))) (:acc (summing (weight (child node i)))))))) + +;; About `k` maps in key order that together hold the entries of `m` once. +;; They share its nodes: a part is a run of the nodes of the highest level +;; with at least `k` nodes, under new internal nodes. The runs are cut where +;; the running weight passes i * total / k, so the parts are close in size. +(: (-> (OrderedMap %k %v) int (Array (OrderedMap %k %v)))) +(defun (orderedmap-split m k) + (def root (.root m)) + (if (or (<= k 1) (leaf? root)) + (vec->array [m]) + (begin + (def level (level-of (vec->array [root]) k)) + (def n (array-length level)) + (if (= n 1) + (vec->array [m]) + (begin + (def weights (loop (:for x level) (:acc (arraying (weight x))))) + (def total (loop (:for w weights) (:acc (summing w)))) + (def parts (min k n)) + (def/mutable out '()) + (def/mutable made 0) + (def/mutable start 0) + (def/mutable done 0) + (loop (:for j (range 0 n)) + (:do (set! done (+ done (array-ref weights j))) + (when (or (= j (- n 1)) + (and (< made (- parts 1)) + (>= (* (cast System.Int64 done) (cast System.Int64 parts)) + (* (cast System.Int64 total) (cast System.Int64 (+ made 1)))))) + (def node (over (loop (:for x (range start (+ j 1))) (:acc (arraying (array-ref level x)))))) + (set! out (cons (OrderedMap (root node) (count (count-keys node)) (comparer (.comparer m))) out)) + (set! made (+ made 1)) + (set! start (+ j 1))))) + (loop (:for x (list-reverse out)) (:acc (arraying x)))))))) + +;; The highest level, from `level` down, with at least `k` nodes, or the +;; leaves. +(: (-> (Array (Node %k)) int (Array (Node %k)))) +(defun (level-of level k) + (if (or (>= (array-length level) k) (leaf? (array-ref level 0))) + level + (begin + (def/mutable below '()) + (loop (:for x level) + (:do (loop (:for i (range 0 (+ (node-count x) 1))) + (:do (set! below (cons (child x i) below)))))) + (level-of (list->nodes (list-reverse below)) k)))) + +;; One map of the maps' entries, in order. Each map's keys must be less than +;; every key of the next. The leaves are kept: an empty one is dropped, and +;; a small one is copied together with its neighbour when the two fit in one +;; leaf. +(: (-> (Array (OrderedMap %k %v)) (OrderedMap %k %v))) +(defun (orderedmap-concat maps) + (when (= (array-length maps) 0) (bad-argument "orderedmap-concat needs at least one map.")) + (def cmp (.comparer (array-ref maps 0))) + (def (: vt (Option %v)) None) + (def/mutable leaves '()) + (def/mutable count 0) + (def/mutable (: last (Option %k)) None) + (loop (:for m maps) + (:when (> (.count m) 0)) + (:do (def lo (first-key (.root m))) + (when (and (some? last) (>= (compare-keys cmp (option-ref-or last lo) lo) 0)) + (bad-argument "The maps are not in key order: a key of one is not less than every key of the next.")) + (def hi (key-of (rightmost (.root m)) (- (node-count (rightmost (.root m))) 1))) + (set! last (Some hi)) + (set! leaves (leaves-of (.root m) leaves)) + (set! count (+ count (.count m))))) + (def joined (join-small (list-reverse leaves) vt)) + (if (list-empty? joined) + (empty-with cmp) + (OrderedMap (root (over (list->nodes joined))) (count count) (comparer cmp)))) + +(: (-> (List (Node %k)) (Array (Node %k)))) +(defun (list->nodes xs) (loop (:for x xs) (:acc (arraying x)))) + +;; The non-empty leaves under `node`, in reverse key order, onto `acc`. +(: (-> (Node %k) (List (Node %k)) (List (Node %k)))) +(defun (leaves-of node acc) + (cond + ((leaf? node) (if (> (node-count node) 0) (cons node acc) acc)) + (else (loop (:for i (range 0 (+ (node-count node) 1))) + (:acc a (folding acc (leaves-of (child node i) a))) + => a)))) + +;; `leaves` with each small leaf copied together with the one before it when +;; the two fit in one leaf. +(: (-> (List (Node %k)) (Option %v) (List (Node %k)))) +(defun (join-small leaves vt) + (list-reverse + (loop (:for x leaves) + (:acc out (folding '() + (if (and (not (list-empty? out)) + (or (< (node-count (list-head out)) merge-threshold) + (< (node-count x) merge-threshold)) + (<= (+ (node-count (list-head out)) (node-count x)) leaf-capacity)) + (cons (both (list-head out) x vt) (list-tail out)) + (cons x out)))) + => out))) + +;; A new leaf with the entries of `a` and then those of `b`. +(: (-> (Node %k) (Node %k) (Option %v) (Node %k))) +(defun (both a b vt) + (def (: la (Leaf %k %v)) (as-leaf a)) + (def (: lb (Leaf %k %v)) (as-leaf b)) + (def (: l (Leaf %k %v)) (new-leaf (owner-none))) + (def na (leaf-count la)) + (def nb (leaf-count lb)) + (leaf-move! la 0 l 0 na) + (leaf-move! lb 0 l na nb) + (set-count! (leaf-node l) (+ na nb)) + (leaf-node l)) + +;; --------------------------------------------------------------------------- +;; Conversions +;; --------------------------------------------------------------------------- + +;; Sets each entry, in order, by a transient. +(: (-> (OrderedMap %k %v) (Seq (Tuple %k %v)) (OrderedMap %k %v))) +(defun (orderedmap-set-all m kvs) + (def t (orderedmap->transientorderedmap m)) + (transientorderedmap-set-all! t kvs) + (transientorderedmap->orderedmap t)) + +(: (-> (OrderedMap %k %v) (Seq %k) (OrderedMap %k %v))) +(defun (orderedmap-remove-all m ks) + (def t (orderedmap->transientorderedmap m)) + (transientorderedmap-remove-all! t ks) + (transientorderedmap->orderedmap t)) + +(: (-> (List (Tuple %k %v)) (OrderedMap %k %v)) (where (Ord %k))) +(defun (list->orderedmap xs) + (def b (orderedmapbuilder-empty)) + (loop (:for (Tuple k v) xs) (:do (orderedmapbuilder-add! b k v))) + (orderedmapbuilder->orderedmap b)) + +(: (-> (Vec (Tuple %k %v)) (OrderedMap %k %v)) (where (Ord %k))) +(defun (vec->orderedmap v) + (def b (orderedmapbuilder-empty)) + (loop (:for (Tuple key value) v) (:do (orderedmapbuilder-add! b key value))) + (orderedmapbuilder->orderedmap b)) + +(: (-> (Seq (Tuple %k %v)) (OrderedMap %k %v)) (where (Ord %k))) +(defun (seq->orderedmap xs) + (def b (orderedmapbuilder-empty)) + (loop (:for (Tuple k v) xs) (:do (orderedmapbuilder-add! b k v))) + (orderedmapbuilder->orderedmap b)) + +(: (-> (Map %k %v) (OrderedMap %k %v)) (where (Ord %k))) +(defun (map->orderedmap m) + (def b (orderedmapbuilder-empty)) + (map-for-each (fun (k v) (orderedmapbuilder-add! b k v)) m) + (orderedmapbuilder->orderedmap b)) + +(: (-> (OrderedMap %k %v) (List (Tuple %k %v)))) +(defun (orderedmap->list m) (->list m)) + +(: (-> (OrderedMap %k %v) (Vec (Tuple %k %v)))) +(defun (orderedmap->vec m) (->vec m)) + +(: (-> (OrderedMap %k %v) (Seq (Tuple %k %v)))) +(defun (orderedmap->seq m) (->seq m)) + +(: (-> (OrderedMap %k %v) (Map %k %v))) +(defun (orderedmap->map m) + (def mb (mapbuilder-empty)) + (loop (:for (Tuple k v) m) (:do (mapbuilder-add! mb k v))) + (mapbuilder->map mb)) + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- + +(impl (Refable (OrderedMap %k %v)) + (type %index %k) + (type %out %v) + (defun (ref m key) (orderedmap-ref m key)) + (defun (try-ref m key) (orderedmap-try-ref m key))) + +(impl (Settable (OrderedMap %k %v)) + (type %index %k) + (type %in %v) + (defun (ref-set m key v) (orderedmap-set m key v))) + +(impl (Refable (TransientOrderedMap %k %v)) + (type %index %k) + (type %out %v) + (defun (ref t key) (transientorderedmap-ref t key)) + (defun (try-ref t key) (transientorderedmap-try-ref t key))) + +(impl (Collection (OrderedMap %k %v)) + (type %content (Tuple %k %v)) + (defun (length m) (orderedmap-length m)) + (defun (empty? m) (orderedmap-empty? m))) + +(type (: (OrderedMapping %k %v) (Union ordered-mapping))) + +(impl (Collector (OrderedMapping %k %v)) + (where (Ord %k)) + (type %elem (Tuple %k %v)) + (type %acc (OrderedMapBuilder %k %v)) + (type %out (OrderedMap %k %v)) + (defun (collector-init c) (orderedmapbuilder-empty)) + (defun (collector-step c acc kv) + (def (Tuple k v) kv) + (orderedmapbuilder-add! acc k v) + acc) + (defun (collector-finish c acc) (orderedmapbuilder->orderedmap acc))) + +(impl (Addable! (OrderedMapBuilder %k %v)) + (type %elem (Tuple %k %v)) + (defun (add! b kv) (def (Tuple k v) kv) (orderedmapbuilder-add! b k v))) + +(impl (Addable! (TransientOrderedMap %k %v)) + (type %elem (Tuple %k %v)) + (defun (add! t kv) (def (Tuple k v) kv) (transientorderedmap-set! t k v))) + +(impl (Addable (OrderedMap %k %v)) + (type %elem (Tuple %k %v)) + (defun (add m kv) (def (Tuple k v) kv) (orderedmap-set m k v))) + +;; In the type's module, so that it is the map's .NET `Equals` and +;; `GetHashCode` too. +(impl (Eq (OrderedMap %k %v)) + (where (Eq %k) (Eq %v)) + (defun (= a b) + (and (= (orderedmap-length a) (orderedmap-length b)) + (= (orderedmap->list a) (orderedmap->list b)))) + (defun (eq-hash m) + (orderedmap-fold (fun (k v h) (hash-combine h (hash-combine (eq-hash k) (eq-hash v)))) 17 m))) + +;; `#map((k v) ...)` in ascending key order, as a `Map` prints. +(impl (Written (OrderedMap %k %v)) + (where (Written %k) (Written %v)) + (defun (write-into m b) + (entries-print-into "#map(" m ")" b (fun (k b) (write-into k b)) (fun (v b) (write-into v b))))) + +(impl (Displayed (OrderedMap %k %v)) + (where (Displayed %k) (Displayed %v)) + (defun (display-into m b) + (entries-print-into "#map(" m ")" b (fun (k b) (display-into k b)) (fun (v b) (display-into v b))))) + +(: (-> (OrderedMap %k %v) string) (where (Displayed %k) (Displayed %v))) +(defun (orderedmap->str m) (displayed m)) + +(:deprecated orderedmap->str "use `displayed`") +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/orderedset.bjo — the sorted set: a persistent B+ tree of elements, in +;; their order. Lookups are O(log n). +;; +;; - The tree of `(std orderedmap)` without values, over C# nodes in the +;; runtime, through `Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps`. The +;; nodes stay in C# because an internal node holds its keys and children +;; in inline arrays. +;; - Everything that spans several nodes is here: descent, splits, merges, +;; rotations, iteration, building and joining trees, and set algebra. +;; - A leaf holds up to 64 elements, an internal node up to 31 keys and 32 +;; children. An edit that leaves a node other than the root with fewer +;; than 8 keys takes a key from a sibling or merges with it. A tree that is +;; built, split or joined may have smaller nodes at its right edge. +;; - An edit copies the nodes on its path. A transient's owner lets it edit +;; the nodes it copied in place. The owner `owner-none` makes `editable` +;; copy every node it is given. + +(import (std prelude)) + +(:doc #:file "orderedset.bjodoc") + +(export + ;; The types. + OrderedSet OrderedSetBuilder TransientOrderedSet OrderedSetCursor OrderedSetRange + OrderedSetting + + ;; Sets + orderedset-empty orderedset-add orderedset-add-all orderedset-remove + orderedset-remove-all orderedset-contains? orderedset-length orderedset-empty? + orderedset-clear + ;; Set algebra + orderedset-union orderedset-intersection orderedset-difference + orderedset-symmetric-difference + ;; Order — what the tree is for + orderedset-min orderedset-max orderedset-successor orderedset-predecessor + orderedset-range orderedset-from orderedset-until + ;; Walking and rebuilding + orderedset-map orderedset-filter orderedset-fold orderedset-for-each + orderedset-iter orderedset-any? orderedset-find + ;; Parts, for work on them at the same time + orderedset-split orderedset-concat + ;; Conversions + list->orderedset orderedset->list vec->orderedset orderedset->vec + seq->orderedset orderedset->seq + orderedset->str + ;; Builders — the same shape as ListBuilder + orderedsetbuilder-empty orderedset->orderedsetbuilder orderedsetbuilder-add! + orderedsetbuilder-add-all! orderedsetbuilder-length orderedsetbuilder->orderedset + ;; Transients — the set's interface, in place + transientorderedset-empty orderedset->transientorderedset + transientorderedset->orderedset transientorderedset-add! + transientorderedset-add-all! transientorderedset-remove! + transientorderedset-remove-all! transientorderedset-contains? + transientorderedset-length transientorderedset-empty? + transientorderedset-min transientorderedset-max + transientorderedset-filter!) + +(import/class + (Exception (: System.Exception)) + (ArgumentException (: System.ArgumentException)) + (InvalidOperationException (: System.InvalidOperationException)) + ((Node %k) (: Bjolang.Runtime.OrderedTrees.SetNodes.Node)) + ((Leaf %k) (: Bjolang.Runtime.OrderedTrees.SetNodes.LeafNode)) + (Owner (: Bjolang.Runtime.OrderedTrees.SetNodes.OwnerId)) + ;; What a set orders its elements by. See `ord-comparer`. + ((Comparer %a) (: System.Collections.Generic.IComparer))) + +(import/extern + (owner-none (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.OwnerNone (-> Owner))) + (owner-next (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.OwnerNext (-> Owner))) + (editable (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Editable (-> (Node %k) Owner (Node %k)))) + (new-leaf (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.NewLeaf (-> Owner (Leaf %k)))) + (new-internal (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.NewInternal (-> Owner (Node %k)))) + (as-leaf (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.AsLeaf (-> (Node %k) (Leaf %k)))) + (leaf-node (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafAsNode (-> (Leaf %k) (Node %k)))) + (leaf? (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.IsLeaf (-> (Node %k) bool))) + (node-count (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Count (-> (Node %k) int))) + (leaf-key (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafKey (-> (Leaf %k) int %k))) + (leaf-keys (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafKeys (-> (Leaf %k) (Array %k)))) + (node-key (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Key (-> (Node %k) int %k))) + (key-of (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.KeyOf (-> (Node %k) int %k))) + (child (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Child (-> (Node %k) int (Node %k)))) + (leaf-find (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafFind (-> (Leaf %k) %k (Comparer %k) int))) + (route (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Route (-> (Node %k) %k (Comparer %k) int))) + (compare-keys (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.Compare (-> (Comparer %k) %k %k int))) + (set-count! (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.SetCount (-> (Node %k) int void))) + (leaf-set! (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafSet (-> (Leaf %k) int %k void))) + (set-key! (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.SetKey (-> (Node %k) int %k void))) + (set-child! (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.SetChild (-> (Node %k) int (Node %k) void))) + (leaf-move! (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.LeafMove (-> (Leaf %k) int (Leaf %k) int int void))) + (key-move! (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.KeyMove (-> (Node %k) int (Node %k) int int void))) + (child-move! (: Bjolang.Runtime.OrderedTrees.SetNodes.NodeOps.ChildMove (-> (Node %k) int (Node %k) int int void))) + (log2 (: System.Math.Log2 (-> double double)))) + +;; Raises the .NET exception the C# set raised, so that a caller's handler +;; still catches it. +(: (-> string %a)) +(defun (bad-argument why) (raise (cast Exception (ArgumentException. why)))) + +(def leaf-capacity 64) +(def internal-capacity 32) +(def merge-threshold 8) + +;; --------------------------------------------------------------------------- +;; The set +;; --------------------------------------------------------------------------- + +(type (: (OrderedSet %a) #:opaque + (Record (: root (Node %a)) + (: count int) + (: comparer (Comparer %a))))) + +(: (-> (Comparer %a) (OrderedSet %a))) +(defun (empty-with cmp) + (def (: l (Leaf %a)) (new-leaf (owner-none))) + (OrderedSet (root (leaf-node l)) (count 0) (comparer cmp))) + +(: (-> (OrderedSet %a)) (where (Ord %a))) +(defun (orderedset-empty) (empty-with (ord-comparer))) + +;; An empty set with the order of `s`. +(: (-> (OrderedSet %a) (OrderedSet %a))) +(defun (orderedset-clear s) (empty-with (.comparer s))) + +(: (-> (OrderedSet %a) int)) +(defun (orderedset-length s) (.count s)) + +(: (-> (OrderedSet %a) bool)) +(defun (orderedset-empty? s) (= (.count s) 0)) + +;; --------------------------------------------------------------------------- +;; Lookups +;; --------------------------------------------------------------------------- + +;; The leaf under `node` that `key` belongs in. +(: (-> (Comparer %k) (Node %k) %k (Leaf %k))) +(defun (leaf-for cmp node key) + (if (leaf? node) + (as-leaf node) + (leaf-for cmp (child node (route node key cmp)) key))) + +(: (-> (Comparer %k) (Node %k) %k bool)) +(defun (contains-in? cmp root key) + (>= (leaf-find (leaf-for cmp root key) key cmp) 0)) + +(: (-> (OrderedSet %a) %a bool)) +(defun (orderedset-contains? s x) (contains-in? (.comparer s) (.root s) x)) + +(: (-> (Node %k) (Node %k))) +(defun (leftmost node) (if (leaf? node) node (leftmost (child node 0)))) + +(: (-> (Node %k) (Node %k))) +(defun (rightmost node) (if (leaf? node) node (rightmost (child node (node-count node))))) + +(: (-> (Node %k) %k)) +(defun (first-key node) (key-of (leftmost node) 0)) + +(: (-> (Node %k) (Option %k))) +(defun (min-in node) + (def l (as-leaf (leftmost node))) + (if (= (leaf-count l) 0) None (Some (leaf-key l 0)))) + +(: (-> (Node %k) (Option %k))) +(defun (max-in node) + (def l (as-leaf (rightmost node))) + (def n (leaf-count l)) + (if (= n 0) None (Some (leaf-key l (- n 1))))) + +(: (-> (OrderedSet %a) (Option %a))) +(defun (orderedset-min s) (min-in (.root s))) + +(: (-> (OrderedSet %a) (Option %a))) +(defun (orderedset-max s) (max-in (.root s))) + +;; The least element greater than `x`. +(: (-> (OrderedSet %a) %a (Option %a))) +(defun (orderedset-successor s x) (successor-in (.comparer s) (.root s) x)) + +(: (-> (Comparer %k) (Node %k) %k (Option %k))) +(defun (successor-in cmp node key) + (if (leaf? node) + (begin + (def l (as-leaf node)) + (def i (leaf-find l key cmp)) + (def at (if (>= i 0) (+ i 1) (- -1 i))) + (if (< at (leaf-count l)) (Some (leaf-key l at)) None)) + (begin + (def r (route node key cmp)) + (def found (successor-in cmp (child node r) key)) + (if (and (none? found) (< r (node-count node))) + (min-in (child node (+ r 1))) + found)))) + +;; The greatest element less than `x`. +(: (-> (OrderedSet %a) %a (Option %a))) +(defun (orderedset-predecessor s x) (predecessor-in (.comparer s) (.root s) x)) + +(: (-> (Comparer %k) (Node %k) %k (Option %k))) +(defun (predecessor-in cmp node key) + (if (leaf? node) + (begin + (def l (as-leaf node)) + (def i (leaf-find l key cmp)) + (def at (if (>= i 0) i (- -1 i))) + (if (> at 0) (Some (leaf-key l (- at 1))) None)) + (begin + (def r (route node key cmp)) + (def found (predecessor-in cmp (child node r) key)) + (if (and (none? found) (> r 0)) + (max-in (child node (- r 1))) + found)))) + +;; --------------------------------------------------------------------------- +;; Slots in one node +;; --------------------------------------------------------------------------- + +(: (-> (Leaf %k) int)) +(defun (leaf-count l) (node-count (leaf-node l))) + +(: (-> (Leaf %k) int %k void)) +(defun (leaf-insert! l i key) + (def n (leaf-count l)) + (when (< i n) (leaf-move! l i l (+ i 1) (- n i))) + (leaf-set! l i key) + (set-count! (leaf-node l) (+ n 1))) + +(: (-> (Leaf %k) int void)) +(defun (leaf-remove! l i) + (def n (leaf-count l)) + (when (> (- n i 1) 0) (leaf-move! l (+ i 1) l i (- n i 1))) + (set-count! (leaf-node l) (- n 1))) + +;; Puts `sep` at key slot `i` and `right` at child slot `i + 1`. +(: (-> (Node %k) int %k (Node %k) void)) +(defun (internal-insert! node i sep right) + (def n (node-count node)) + (when (< i n) + (key-move! node i node (+ i 1) (- n i)) + (child-move! node (+ i 1) node (+ i 2) (- n i))) + (set-key! node i sep) + (set-child! node (+ i 1) right) + (set-count! node (+ n 1))) + +;; --------------------------------------------------------------------------- +;; Add +;; --------------------------------------------------------------------------- + +;; What `insert!` did: +;; - `(Tuple 0 _ _)`: the key was there. +;; - `(Tuple 1 _ _)`: the key is added. +;; - `(Tuple 2 right sep)`: the key is added and the node split. `right` is +;; the new right half, and `sep` its first key. +(: (-> (Comparer %k) (Node %k) %k Owner (Tuple int (Node %k) %k))) +(defun (insert! cmp node key owner) + (if (leaf? node) + (insert-leaf! cmp (as-leaf node) key owner) + (begin + (def i (route node key cmp)) + (def c (editable (child node i) owner)) + (set-child! node i c) + (def r (insert! cmp c key owner)) + (def (Tuple code right sep) r) + (cond + ((< code 2) r) + ((< (node-count node) (- internal-capacity 1)) + (internal-insert! node i sep right) + (Tuple 1 node key)) + (else (split-internal! node i sep right owner)))))) + +(: (-> (Comparer %k) (Leaf %k) %k Owner (Tuple int (Node %k) %k))) +(defun (insert-leaf! cmp l key owner) + (def i (leaf-find l key cmp)) + (cond + ((>= i 0) (Tuple 0 (leaf-node l) key)) + ((< (leaf-count l) leaf-capacity) + (leaf-insert! l (- -1 i) key) + (Tuple 1 (leaf-node l) key)) + (else (split-leaf! l (- -1 i) key owner)))) + +;; Splits a full leaf and inserts at slot `i`. An insert at either end +;; leaves the full half as it is, so that ascending or descending inserts +;; fill their leaves. +(: (-> (Leaf %k) int %k Owner (Tuple int (Node %k) %k))) +(defun (split-leaf! left i key owner) + (def (: right (Leaf %k)) (new-leaf owner)) + (def total (leaf-count left)) + (def at (cond ((= i total) total) ((= i 0) 0) (else (shift-right-logical total 1)))) + (def moving (- total at)) + (when (> moving 0) (leaf-move! left at right 0 moving)) + (set-count! (leaf-node left) at) + (set-count! (leaf-node right) moving) + (if (or (< i at) (and (= at 0) (= i 0))) + (leaf-insert! left i key) + (leaf-insert! right (- i at) key)) + (Tuple 2 (leaf-node right) (leaf-key right 0))) + +;; Splits a full internal node, and puts `sep` and `new-child` at slot `i`. +(: (-> (Node %k) int %k (Node %k) Owner (Tuple int (Node %k) %k))) +(defun (split-internal! left i sep new-child owner) + (def right (new-internal owner)) + (def n (node-count left)) + (def at (shift-right-logical n 1)) + (def up (node-key left at)) + (def moving (- n at 1)) + (key-move! left (+ at 1) right 0 moving) + (child-move! left (+ at 1) right 0 (+ moving 1)) + (set-count! left at) + (set-count! right moving) + (if (<= i at) + (internal-insert! left i sep new-child) + (internal-insert! right (- i (+ at 1)) sep new-child)) + (Tuple 2 right up)) + +(: (-> (Node %k) %k (Node %k) Owner (Node %k))) +(defun (new-root left sep right owner) + (def r (new-internal owner)) + (set-key! r 0 sep) + (set-child! r 0 left) + (set-child! r 1 right) + (set-count! r 1) + r) + +;; Adds `key` to the tree under `root`. Returns the new root, and whether +;; the key is new. +(: (-> (Comparer %k) (Node %k) %k Owner (Tuple (Node %k) bool))) +(defun (add-in cmp root key owner) + (def r (editable root owner)) + (def (Tuple code right sep) (insert! cmp r key owner)) + (cond + ((= code 0) (Tuple r #f)) + ((= code 2) (Tuple (new-root r sep right owner) #t)) + (else (Tuple r #t)))) + +(: (-> (OrderedSet %a) %a (OrderedSet %a))) +(defun (orderedset-add s x) + (def (Tuple root added?) (add-in (.comparer s) (.root s) x (owner-none))) + (OrderedSet (root root) + (count (if added? (+ (.count s) 1) (.count s))) + (comparer (.comparer s)))) + +;; --------------------------------------------------------------------------- +;; Remove +;; --------------------------------------------------------------------------- + +;; Removes `key` under `node`, which `owner` may edit. Returns 0 if the key +;; is not there, 1 if it is removed, and 2 if it is removed and `node` is +;; left with fewer than `merge-threshold` keys. +(: (-> (Comparer %k) (Node %k) %k Owner int)) +(defun (remove! cmp node key owner) + (if (leaf? node) + (begin + (def l (as-leaf node)) + (def i (leaf-find l key cmp)) + (cond + ((< i 0) 0) + (else + (leaf-remove! l i) + (if (< (leaf-count l) merge-threshold) 2 1)))) + (begin + (def i (route node key cmp)) + (def c (editable (child node i) owner)) + (set-child! node i c) + (def r (remove! cmp c key owner)) + (cond + ((< r 2) r) + ((underflow! node i owner) 2) + (else 1))))) + +(: (-> (Node %k) bool)) +(defun (can-lend? node) (> (node-count node) (+ merge-threshold 1))) + +;; Child `i` of `parent` has too few keys. Takes one from a sibling that has +;; enough, or else merges with a sibling. Returns whether `parent` has too few +;; keys after. +(: (-> (Node %k) int Owner bool)) +(defun (underflow! parent i owner) + (cond + ((< i (node-count parent)) + (def right (editable (child parent (+ i 1)) owner)) + (set-child! parent (+ i 1) right) + (balance! parent i (child parent i) right #t)) + ((> i 0) + (def left (editable (child parent (- i 1)) owner)) + (set-child! parent (- i 1) left) + (balance! parent (- i 1) left (child parent i) #f)) + (else #t))) + +;; `left` and `right` are the children on each side of key `s` of `parent`. +;; The one `from-right?` names lends a key, or the two merge. +(: (-> (Node %k) int (Node %k) (Node %k) bool bool)) +(defun (balance! parent s left right from-right?) + (cond + ((and from-right? (can-lend? right)) (rotate-left! parent s left right) #f) + ((and (not from-right?) (can-lend? left)) (rotate-right! parent s left right) #f) + (else + (merge! parent s left right) + (< (node-count parent) merge-threshold)))) + +;; Moves the first key of `right` to the end of `left`. +(: (-> (Node %k) int (Node %k) (Node %k) void)) +(defun (rotate-left! parent s left right) + (if (leaf? left) + (begin + (def l (as-leaf left)) + (def r (as-leaf right)) + (leaf-insert! l (leaf-count l) (leaf-key r 0)) + (leaf-remove! r 0) + (set-key! parent s (leaf-key r 0))) + (begin + (internal-insert! left (node-count left) (node-key parent s) (child right 0)) + (set-key! parent s (node-key right 0)) + (def rn (node-count right)) + (child-move! right 1 right 0 rn) + (when (> rn 1) (key-move! right 1 right 0 (- rn 1))) + (set-count! right (- rn 1))))) + +;; Moves the last key of `left` to the front of `right`. +(: (-> (Node %k) int (Node %k) (Node %k) void)) +(defun (rotate-right! parent s left right) + (if (leaf? left) + (begin + (def l (as-leaf left)) + (def r (as-leaf right)) + (def last (- (leaf-count l) 1)) + (leaf-insert! r 0 (leaf-key l last)) + (leaf-remove! l last) + (set-key! parent s (leaf-key r 0))) + (begin + (def last (- (node-count left) 1)) + (def rn (node-count right)) + (key-move! right 0 right 1 rn) + (set-key! right 0 (node-key parent s)) + (child-move! right 0 right 1 (+ rn 1)) + (set-child! right 0 (child left (+ last 1))) + (set-count! right (+ rn 1)) + (set-key! parent s (node-key left last)) + (set-count! left last)))) + +;; Moves all of `right` to the end of `left`, and removes key `s` of +;; `parent` and the child after it. +(: (-> (Node %k) int (Node %k) (Node %k) void)) +(defun (merge! parent s left right) + (def ln (node-count left)) + (def rn (node-count right)) + (if (leaf? left) + (begin + (leaf-move! (as-leaf right) 0 (as-leaf left) ln rn) + (set-count! left (+ ln rn))) + (begin + (set-key! left ln (node-key parent s)) + (key-move! right 0 left (+ ln 1) rn) + (child-move! right 0 left (+ ln 1) (+ rn 1)) + (set-count! left (+ ln 1 rn)))) + (def pn (node-count parent)) + (def moving (- pn s 1)) + (when (> moving 0) + (key-move! parent (+ s 1) parent s moving) + (child-move! parent (+ s 2) parent (+ s 1) moving)) + (set-count! parent (- pn 1))) + +;; Removes `key` from the tree under `root`. Returns the new root, and +;; whether the key was there. When it was not, the root is `root`. +(: (-> (Comparer %k) (Node %k) %k Owner (Tuple (Node %k) bool))) +(defun (remove-in cmp root key owner) + (def r (editable root owner)) + (def code (remove! cmp r key owner)) + (cond + ((= code 0) (Tuple root #f)) + ((and (= code 2) (not (leaf? r)) (= (node-count r) 0)) (Tuple (child r 0) #t)) + (else (Tuple r #t)))) + +(: (-> (OrderedSet %a) %a (OrderedSet %a))) +(defun (orderedset-remove s x) + (def (Tuple root removed?) (remove-in (.comparer s) (.root s) x (owner-none))) + (if removed? + (OrderedSet (root root) (count (- (.count s) 1)) (comparer (.comparer s))) + s)) + +;; --------------------------------------------------------------------------- +;; The cursor +;; --------------------------------------------------------------------------- + +;; A position in an iteration in order. A value: the next step returns the +;; next one. +;; +;; - It holds the key array of one leaf, so a step inside a leaf is an +;; increment and a compare. +;; - `path` is the child index at each level above the leaf, 6 bits a level +;; above the low 4 bits, which hold the leaf's depth. `last-leaf` marks a +;; leaf with no leaf after it. +;; - At the end of a leaf, `cursor-after` follows `path` from the root again. +(type (: (OrderedSetCursor %a) #:opaque + (Struct (: keys (Array %a)) + (: i int) + (: end int) + (: root (Node %a)) + (: path long)))) + +(def depth-bits 4) +(def index-bits 6) +(def max-depth 10) +(: last-leaf long) +(def last-leaf -1) + +(: (-> long int int)) +(defun (index-at path level) + (cast System.Int32 (bitwise-and (shift-right-logical path (+ depth-bits (* index-bits level))) 63))) + +(: (-> long int int long)) +(defun (with-index path level index) + (def shift (+ depth-bits (* index-bits level))) + (def (: mask long) (shift-left 63 shift)) + (bitwise-ior (bitwise-and path (bitwise-not mask)) + (shift-left (cast System.Int64 index) shift))) + +(: (-> long int long)) +(defun (with-depth path depth) + (def (: depth-mask long) 15) + (bitwise-ior (bitwise-and path (bitwise-not depth-mask)) (cast System.Int64 depth))) + +(: (-> (Leaf %k) int (Node %k) long (OrderedSetCursor %k))) +(defun (cursor-at l i root path) + (OrderedSetCursor (keys (leaf-keys l)) (i i) (end (leaf-count l)) (root root) (path path))) + +(: (-> (Node %k) (OrderedSetCursor %k))) +(defun (cursor-of root) + (if (leaf? root) + (cursor-at (as-leaf root) 0 root last-leaf) + (descend root root 0 0))) + +;; The cursor on the first key of the leftmost leaf under `node`, which is +;; at `level`, or on the key after it if that leaf is empty. +(: (-> (Node %k) (Node %k) int long (OrderedSetCursor %k))) +(defun (descend node root level path) + (cond + ((leaf? node) + (def l (as-leaf node)) + (def at (with-depth path level)) + (if (> (leaf-count l) 0) + (cursor-at l 0 root at) + (advance l root at))) + ((= level max-depth) + (raise (cast Exception (InvalidOperationException. "The set is deeper than a cursor can walk.")))) + (else (descend (child node 0) root (+ level 1) (with-index path level 0))))) + +;; The cursor on the first key after the leaf `l` at `path`, or a done +;; cursor on `l`. +(: (-> (Leaf %k) (Node %k) long (OrderedSetCursor %k))) +(defun (advance l root path) + (def depth (cast System.Int32 (bitwise-and path 15))) + ;; The deepest level whose child index can move right, and its node. + (def/mutable level -1) + (def/mutable parent root) + (def/mutable node root) + (loop (:for d (range 0 depth)) + (:do (def i (index-at path d)) + (when (< i (node-count node)) + (set! level d) + (set! parent node)) + (set! node (child node i)))) + (if (< level 0) + (cursor-at l (leaf-count l) root last-leaf) + (begin + (def next (+ (index-at path level) 1)) + (descend (child parent next) root (+ level 1) (with-index path level next))))) + +;; The cursor on the first key not less than `key`, or with `after?` +;; greater than `key`. +(: (-> (Comparer %k) (Node %k) (Node %k) %k bool int long (OrderedSetCursor %k))) +(defun (cursor-from cmp node root key after? level path) + (if (leaf? node) + (begin + (def l (as-leaf node)) + (def i (leaf-find l key cmp)) + (def at (cond ((< i 0) (- -1 i)) (after? (+ i 1)) (else i))) + (def p (if (= level 0) last-leaf (with-depth path level))) + (if (or (< at (leaf-count l)) (= level 0)) + (cursor-at l at root p) + (advance l root p))) + (begin + (def r (route node key cmp)) + (cursor-from cmp (child node r) root key after? (+ level 1) (with-index path level r))))) + +(: (-> (OrderedSetCursor %k) bool)) +(defun (cursor-done? c) (>= (.i c) (.end c))) + +(: (-> (OrderedSetCursor %k) %k)) +(defun (cursor-key c) (array-ref (.keys c) (.i c))) + +(: (-> (OrderedSetCursor %k) (OrderedSetCursor %k))) +(defun (cursor-step c) + (def i (+ (.i c) 1)) + (if (or (< i (.end c)) (= (.path c) last-leaf)) + (record-set c (i i)) + (cursor-after c))) + +;; Out of line, so that a step inside a leaf stays small. +(: (-> (OrderedSetCursor %k) (OrderedSetCursor %k))) +(defun (cursor-after c) + (def root (.root c)) + (def depth (cast System.Int32 (bitwise-and (.path c) 15))) + (def/mutable node root) + (loop (:for d (range 0 depth)) (:do (set! node (child node (index-at (.path c) d))))) + (advance (as-leaf node) root (.path c))) + +(impl (Iterable (OrderedSet %a)) + (type %elem %a) + (type %cursor (OrderedSetCursor %a)) + (defun (iterable-start s) (cursor-of (.root s))) + (defun (iterable-done? s c) (cursor-done? c)) + (defun (iterable-current s c) (cursor-key c)) + (defun (iterable-next s c) (cursor-step c)) + (defun (count s) (orderedset-length s))) + +;; --------------------------------------------------------------------------- +;; Ranges +;; --------------------------------------------------------------------------- + +;; The elements from one bound to another, both included. `orderedset-range`, +;; `orderedset-from` and `orderedset-until` return one, not a `Seq`: a loop +;; over it inlines the cursor. `(->seq r)` returns a `Seq` of it. +;; +;; - `start` is the cursor on the first element. +;; - `end-i` and `end-path` are the position of the first element after the +;; range. A cursor's position is its slot and path, and two cursors on one +;; element have the same position, so a step compares two numbers and no +;; element. +;; - With no upper bound, `end-i` is -1, which no cursor has. +;; - When the lower bound is greater than the upper, `start` is the end. +(type (: (OrderedSetRange %a) #:opaque + (Struct (: start (OrderedSetCursor %a)) (: end-i int) (: end-path long)))) + +(impl (Iterable (OrderedSetRange %a)) + (type %elem %a) + (type %cursor (OrderedSetCursor %a)) + (defun (iterable-start r) (.start r)) + (defun (iterable-done? r c) + (or (cursor-done? c) (and (= (.i c) (.end-i r)) (= (.path c) (.end-path r))))) + (defun (iterable-current r c) (cursor-key c)) + (defun (iterable-next r c) (cursor-step c))) + +(: (-> (OrderedSet %a) %a bool (OrderedSetCursor %a))) +(defun (cursor-at-key s key after?) (cursor-from (.comparer s) (.root s) (.root s) key after? 0 0)) + +(: (-> (OrderedSetCursor %a) (OrderedSetCursor %a) (OrderedSetRange %a))) +(defun (range-of start end) + (OrderedSetRange (start start) (end-i (.i end)) (end-path (.path end)))) + +(: (-> (OrderedSet %a) %a %a (OrderedSetRange %a))) +(defun (orderedset-range s lo hi) + (def end (cursor-at-key s hi #t)) + (range-of (if (> (compare-keys (.comparer s) lo hi) 0) end (cursor-at-key s lo #f)) end)) + +(: (-> (OrderedSet %a) %a (OrderedSetRange %a))) +(defun (orderedset-from s lo) + (OrderedSetRange (start (cursor-at-key s lo #f)) (end-i -1) (end-path 0))) + +(: (-> (OrderedSet %a) %a (OrderedSetRange %a))) +(defun (orderedset-until s hi) (range-of (cursor-of (.root s)) (cursor-at-key s hi #t))) + +;; --------------------------------------------------------------------------- +;; Building trees +;; --------------------------------------------------------------------------- + +;; One tree over nodes of one depth, in order: internal nodes of up to 32 +;; children each, as even as they can be, level by level up to one root. +(: (-> (Array (Node %k)) (Node %k))) +(defun (over nodes) + (def n (array-length nodes)) + (if (= n 1) + (array-ref nodes 0) + (begin + (def parents (/ (+ n (- internal-capacity 1)) internal-capacity)) + (def up (make-array parents)) + (def/mutable offset 0) + (loop (:for p (range 0 parents)) + (:do (def size (+ (/ n parents) (if (< p (% n parents)) 1 0))) + (def parent (new-internal (owner-none))) + (loop (:for c (range 0 size)) + (:do (def node (array-ref nodes (+ offset c))) + (set-child! parent c node) + (when (> c 0) (set-key! parent (- c 1) (first-key node))))) + (set-count! parent (- size 1)) + (array-set! up p parent) + (set! offset (+ offset size)))) + (over up)))) + +;; A set of the first `n` elements of `ks`, which are in ascending order and +;; distinct, in full leaves. +(: (-> (Array %a) int (Comparer %a) (OrderedSet %a))) +(defun (from-sorted ks n cmp) + (if (= n 0) + (empty-with cmp) + (begin + (def count (/ (+ n (- leaf-capacity 1)) leaf-capacity)) + (def leaves (make-array count)) + (loop (:for i (range 0 count)) + (:do (def (: l (Leaf %a)) (new-leaf (owner-none))) + (def at (* i leaf-capacity)) + (def size (min leaf-capacity (- n at))) + (array-copy! ks at (leaf-keys l) 0 size) + (set-count! (leaf-node l) size) + (array-set! leaves i (leaf-node l)))) + (OrderedSet (root (over leaves)) (count n) (comparer cmp))))) + +;; The elements of `s`, in order, in a new array. +(: (-> (OrderedSet %a) (Array %a))) +(defun (elements-of s) + (def (: out (Array %a)) (make-array (.count s))) + (def/mutable i 0) + (loop (:for x s) (:do (array-set! out i x) (set! i (+ i 1)))) + out) + +;; --------------------------------------------------------------------------- +;; Builders +;; --------------------------------------------------------------------------- + +;; Elements added in any order, made into a set at the end. Elements added +;; in ascending order need no sort. +(type (: (OrderedSetBuilder %a) #:opaque + (Record (: keys (Array %a) #:mutable) + (: count int #:mutable) + (: ascending? bool #:mutable) + (: comparer (Comparer %a))))) + +(: (-> (Comparer %a) int (OrderedSetBuilder %a))) +(defun (builder-with cmp capacity) + (OrderedSetBuilder (keys (make-array capacity)) (count 0) (ascending? #t) (comparer cmp))) + +(: (-> (OrderedSetBuilder %a)) (where (Ord %a))) +(defun (orderedsetbuilder-empty) (builder-with (ord-comparer) 16)) + +;; A builder that holds the elements of `s`. +(: (-> (OrderedSet %a) (OrderedSetBuilder %a))) +(defun (orderedset->orderedsetbuilder s) + (def b (builder-with (.comparer s) (max 16 (.count s)))) + (loop (:for x s) (:do (orderedsetbuilder-add! b x))) + b) + +;; "Ascending" means strictly: an element equal to the one before makes the +;; build sort, which drops the repeat. +(: (-> (OrderedSetBuilder %a) %a void)) +(defun (orderedsetbuilder-add! b x) + (def n (.count b)) + (when (= n (array-length (.keys b))) + (record-set! b (keys (array-resize (.keys b) (* 2 n))))) + (when (and (.ascending? b) (> n 0) + (>= (compare-keys (.comparer b) (array-ref (.keys b) (- n 1)) x) 0)) + (record-set! b (ascending? #f))) + (array-set! (.keys b) n x) + (record-set! b (count (+ n 1)))) + +(: (-> (OrderedSetBuilder %a) (Seq %a) void)) +(defun (orderedsetbuilder-add-all! b xs) + (loop (:for x xs) (:do (orderedsetbuilder-add! b x)))) + +;; The number of elements added, an element added twice counted twice. +(: (-> (OrderedSetBuilder %a) int)) +(defun (orderedsetbuilder-length b) (.count b)) + +(: (-> (OrderedSetBuilder %a) (OrderedSet %a))) +(defun (orderedsetbuilder->orderedset b) + (def cmp (.comparer b)) + (def n (.count b)) + (if (.ascending? b) + (from-sorted (.keys b) n cmp) + (begin + (def ks (array-resize (.keys b) n)) + (array-sort-by! (fun (x y) (compare-keys cmp x y)) ks) + ;; Drops repeats in place. + (def/mutable u 0) + (loop (:for i (range 0 n)) + (:do (when (or (= u 0) (not (= (compare-keys cmp (array-ref ks i) (array-ref ks (- u 1))) 0))) + (array-set! ks u (array-ref ks i)) + (set! u (+ u 1))))) + (from-sorted ks u cmp)))) + +;; --------------------------------------------------------------------------- +;; Transients +;; --------------------------------------------------------------------------- + +;; A set that edits in place the nodes it copied. +;; `transientorderedset->orderedset` returns a set of its contents and gives +;; it a new owner, so that the set does not change after. +(type (: (TransientOrderedSet %a) #:opaque + (Record (: root (Node %a) #:mutable) + (: count int #:mutable) + (: owner Owner #:mutable) + (: comparer (Comparer %a))))) + +(: (-> (TransientOrderedSet %a)) (where (Ord %a))) +(defun (transientorderedset-empty) (orderedset->transientorderedset (orderedset-empty))) + +(: (-> (OrderedSet %a) (TransientOrderedSet %a))) +(defun (orderedset->transientorderedset s) + (TransientOrderedSet (root (.root s)) (count (.count s)) (owner (owner-next)) (comparer (.comparer s)))) + +(: (-> (TransientOrderedSet %a) (OrderedSet %a))) +(defun (transientorderedset->orderedset t) + (record-set! t (owner (owner-next))) + (OrderedSet (root (.root t)) (count (.count t)) (comparer (.comparer t)))) + +(: (-> (TransientOrderedSet %a) %a void)) +(defun (transientorderedset-add! t x) + (def (Tuple root added?) (add-in (.comparer t) (.root t) x (.owner t))) + (record-set! t (root root)) + (when added? (record-set! t (count (+ (.count t) 1))))) + +(: (-> (TransientOrderedSet %a) %a void)) +(defun (transientorderedset-remove! t x) + (def (Tuple root removed?) (remove-in (.comparer t) (.root t) x (.owner t))) + (when removed? (record-set! t (root root) (count (- (.count t) 1))))) + +(: (-> (TransientOrderedSet %a) (Seq %a) void)) +(defun (transientorderedset-add-all! t xs) + (loop (:for x xs) (:do (transientorderedset-add! t x)))) + +(: (-> (TransientOrderedSet %a) (Seq %a) void)) +(defun (transientorderedset-remove-all! t xs) + (loop (:for x xs) (:do (transientorderedset-remove! t x)))) + +(: (-> (TransientOrderedSet %a) %a bool)) +(defun (transientorderedset-contains? t x) (contains-in? (.comparer t) (.root t) x)) + +(: (-> (TransientOrderedSet %a) int)) +(defun (transientorderedset-length t) (.count t)) + +(: (-> (TransientOrderedSet %a) bool)) +(defun (transientorderedset-empty? t) (= (.count t) 0)) + +(: (-> (TransientOrderedSet %a) (Option %a))) +(defun (transientorderedset-min t) (min-in (.root t))) + +(: (-> (TransientOrderedSet %a) (Option %a))) +(defun (transientorderedset-max t) (max-in (.root t))) + +;; Removes the elements `keep?` rejects. Calls it in order, before it +;; removes any. +(: (-> (-?-> %a bool) (TransientOrderedSet %a) void)) +(defun (transientorderedset-filter! keep? t) + (def/mutable dropped '()) + (loop (:for x (OrderedSet (root (.root t)) (count (.count t)) (comparer (.comparer t)))) + (:do (unless (keep? x) (set! dropped (cons x dropped))))) + (loop (:for x dropped) (:do (transientorderedset-remove! t x)))) + +(: (-> (OrderedSet %a) (Seq %a) (OrderedSet %a))) +(defun (orderedset-add-all s xs) + (def t (orderedset->transientorderedset s)) + (transientorderedset-add-all! t xs) + (transientorderedset->orderedset t)) + +(: (-> (OrderedSet %a) (Seq %a) (OrderedSet %a))) +(defun (orderedset-remove-all s xs) + (def t (orderedset->transientorderedset s)) + (transientorderedset-remove-all! t xs) + (transientorderedset->orderedset t)) + +;; --------------------------------------------------------------------------- +;; Iteration and rebuilding +;; --------------------------------------------------------------------------- + +;; Folds `f` over the elements in order. Recursive over the tree, which is +;; about twice as fast as the cursor: it reads each leaf's array once. +(: (-> (-?-> %a %s %s) %s (OrderedSet %a) %s)) +(defun (orderedset-fold f init s) (fold-node f init (.root s))) + +(: (-> (-?-> %a %s %s) %s (Node %a) %s)) +(defun (fold-node f acc node) + (if (leaf? node) + (begin + (def l (as-leaf node)) + (def ks (leaf-keys l)) + (loop (:for i (range 0 (leaf-count l))) + (:acc s (folding acc (f (array-ref ks i) s))) + => s)) + (loop (:for i (range 0 (+ (node-count node) 1))) + (:acc s (folding acc (fold-node f s (child node i)))) + => s))) + +(: (-> (-?-> %a void) (OrderedSet %a) void)) +(defun (orderedset-for-each f s) + (loop (:for x s) (:do (f x)))) + +;; Calls `f` in order until it returns #f. Returns whether it never did. +(: (-> (-?-> %a bool) (OrderedSet %a) bool)) +(defun (orderedset-iter f s) + (let go ((c (cursor-of (.root s)))) + (cond + ((cursor-done? c) #t) + ((f (cursor-key c)) (go (cursor-step c))) + (else #f)))) + +(: (-> (-?-> %a bool) (OrderedSet %a) bool)) +(defun (orderedset-any? pred s) + (let go ((c (cursor-of (.root s)))) + (cond + ((cursor-done? c) #f) + ((pred (cursor-key c)) #t) + (else (go (cursor-step c)))))) + +;; The first element in order that `pred` accepts. +(: (-> (-?-> %a bool) (OrderedSet %a) (Option %a))) +(defun (orderedset-find pred s) + (let go ((c (cursor-of (.root s)))) + (cond + ((cursor-done? c) None) + (:def x (cursor-key c)) + ((pred x) (Some x)) + (else (go (cursor-step c)))))) + +;; `f` of each element, ordered by `%b`'s order. `f` is called in order. +(: (-> (-?-> %a %b) (OrderedSet %a) (OrderedSet %b)) (where (Ord %b))) +(defun (orderedset-map f s) + (def b (builder-with (ord-comparer) (max 16 (.count s)))) + (loop (:for x s) (:do (orderedsetbuilder-add! b (f x)))) + (orderedsetbuilder->orderedset b)) + +;; The elements `keep?` accepts, in new leaves filled one after the other. +;; Returns `s` when it accepts every element. +(: (-> (-?-> %a bool) (OrderedSet %a) (OrderedSet %a))) +(defun (orderedset-filter keep? s) + (def n (.count s)) + (def (: ks (Array %a)) (make-array n)) + (def/mutable kept 0) + (loop (:for x s) + (:do (when (keep? x) + (array-set! ks kept x) + (set! kept (+ kept 1))))) + (if (= kept n) s (from-sorted ks kept (.comparer s)))) + +;; --------------------------------------------------------------------------- +;; Set algebra +;; --------------------------------------------------------------------------- + +;; The sets that have elements, or `s` alone when none has. +(: (-> (OrderedSet %a) (OrderedSet %a) (Array (OrderedSet %a)) (Array (OrderedSet %a)))) +(defun (inputs-of s other more) + (def all (cons s (cons other (loop (:for x more) (:acc (listing x)))))) + (def kept (loop (:for x all) (:when (> (.count x) 0)) (:acc (arraying x)))) + (if (= (array-length kept) 0) (vec->array [s]) kept)) + +;; At most this many sets are iterated at once, and only when the sets other +;; than the biggest hold this percent of the elements or more. Else the +;; others are added to a transient of the biggest. +(def union-max-inputs 32) +(def union-percent 30) + +;; The elements of any of the sets. +(: (-> (OrderedSet %a) (OrderedSet %a) #:rest (OrderedSet %a) (OrderedSet %a))) +(defun (orderedset-union s other #:rest more) + (def inputs (inputs-of s other more)) + (def k (array-length inputs)) + (cond + ((= k 1) (array-ref inputs 0)) + (else + (def total (loop (:for x inputs) (:acc (summing (.count x))))) + (def/mutable biggest 0) + (loop (:for i (range 1 k)) + (:do (when (> (.count (array-ref inputs i)) (.count (array-ref inputs biggest))) + (set! biggest i)))) + (if (and (<= k union-max-inputs) + (>= (* 100 (- total (.count (array-ref inputs biggest)))) (* union-percent total))) + (union-walk inputs total) + (union-into inputs biggest))))) + +;; The sets other than `biggest` added to a transient of it. +(: (-> (Array (OrderedSet %a)) int (OrderedSet %a))) +(defun (union-into inputs biggest) + (def t (orderedset->transientorderedset (array-ref inputs biggest))) + (loop (:for i (range 0 (array-length inputs))) + (:do (unless (= i biggest) + (loop (:for x (array-ref inputs i)) (:do (transientorderedset-add! t x)))))) + (transientorderedset->orderedset t)) + +;; The sets iterated in order at once into an array. The least element is +;; found by a scan of the cursors. +(: (-> (Array (OrderedSet %a)) int (OrderedSet %a))) +(defun (union-walk inputs total) + (def k (array-length inputs)) + (def cmp (.comparer (array-ref inputs 0))) + (def cursors (loop (:for x inputs) (:acc (arraying (cursor-of (.root x)))))) + (def (: out (Array %a)) (make-array total)) + (def/mutable n 0) + (def/mutable live k) + (loop (:while (> live 0)) + (:do + (def/mutable least -1) + (loop (:for i (range 0 k)) + (:do (def c (array-ref cursors i)) + (when (and (not (cursor-done? c)) + (or (< least 0) + (< (compare-keys cmp (cursor-key c) (cursor-key (array-ref cursors least))) 0))) + (set! least i)))) + (def x (cursor-key (array-ref cursors least))) + (array-set! out n x) + (set! n (+ n 1)) + ;; Every cursor on `x` steps past it. + (loop (:for i (range least k)) + (:do (def c (array-ref cursors i)) + (when (and (not (cursor-done? c)) (= (compare-keys cmp (cursor-key c) x) 0)) + (def next (cursor-step c)) + (array-set! cursors i next) + (when (cursor-done? next) (set! live (- live 1)))))))) + (from-sorted out n cmp)) + +;; Keeps the first `n` elements of `kept`, in order, that `against` has +;; (`keep?` #t) or lacks (#f), at the front of `kept`. Returns how many. +;; +;; - A probe of `against` costs a descent, about 20 steps of an iteration of +;; it. So few elements are probed, and many are compared with an iteration +;; of `against` in order. +(: (-> (Array %a) int (OrderedSet %a) bool int)) +(defun (keep-in! kept n against keep?) + (def cmp (.comparer against)) + (def/mutable w 0) + (if (< (* (cast long n) 20) (cast long (.count against))) + (loop (:for i (range 0 n)) + (:do (def x (array-ref kept i)) + (when (= (contains-in? cmp (.root against) x) keep?) + (array-set! kept w x) + (set! w (+ w 1))))) + (begin + (def/mutable c (cursor-of (.root against))) + (loop (:for i (range 0 n)) + (:do (def x (array-ref kept i)) + (loop (:while (and (not (cursor-done? c)) (< (compare-keys cmp (cursor-key c) x) 0))) + (:do (set! c (cursor-step c)))) + (def there? (and (not (cursor-done? c)) (= (compare-keys cmp (cursor-key c) x) 0))) + (when (= there? keep?) + (array-set! kept w x) + (set! w (+ w 1))))))) + w) + +;; The elements in every set. Starts from the smallest set, since the result +;; can only shrink, and keeps what each other set has. +(: (-> (OrderedSet %a) (OrderedSet %a) #:rest (OrderedSet %a) (OrderedSet %a))) +(defun (orderedset-intersection s other #:rest more) + (def all (loop (:for x (cons s (cons other (loop (:for x more) (:acc (listing x)))))) (:acc (arraying x)))) + (def/mutable least 0) + (loop (:for i (range 1 (array-length all))) + (:do (when (< (.count (array-ref all i)) (.count (array-ref all least))) + (set! least i)))) + (def smallest (array-ref all least)) + (def size (.count smallest)) + (def kept (elements-of smallest)) + (def/mutable n size) + (loop (:for i (range 0 (array-length all))) + (:while (> n 0)) + (:do (unless (= i least) (set! n (keep-in! kept n (array-ref all i) #t))))) + (if (= n size) smallest (from-sorted kept n (.comparer smallest)))) + +;; The elements of `s` that no other set has. +;; +;; - Removes the others' elements from a transient of `s` when they are +;; few: a removal costs a descent and copies of nodes, about three probes. +;; - Else keeps each element of `s` that each other set lacks. +(: (-> (OrderedSet %a) (OrderedSet %a) #:rest (OrderedSet %a) (OrderedSet %a))) +(defun (orderedset-difference s other #:rest more) + (def others + (loop (:for x (cons other (loop (:for x more) (:acc (listing x))))) + (:when (> (.count x) 0)) + (:acc (arraying x)))) + (def size (.count s)) + (cond + ((or (= size 0) (= (array-length others) 0)) s) + (else + (def depth (log2 (cast double (+ size 2)))) + (def removing + (loop (:for o others) (:acc (summing (* 3.0 (cast double (.count o)) depth))))) + (def filtering + (loop (:for o others) + (:acc (summing (min (* (cast double size) (log2 (cast double (+ (.count o) 2)))) + (cast double (+ size (.count o)))))))) + (if (< removing filtering) + (begin + (def t (orderedset->transientorderedset s)) + (loop (:for o others) (:do (loop (:for x o) (:do (transientorderedset-remove! t x))))) + (transientorderedset->orderedset t)) + (begin + (def kept (elements-of s)) + (def/mutable n size) + (loop (:for o others) + (:while (> n 0)) + (:do (set! n (keep-in! kept n o #f)))) + (if (= n size) s (from-sorted kept n (.comparer s)))))))) + +;; The elements of one of the two sets and not of both, by an iteration of +;; both in order. +(: (-> (OrderedSet %a) (OrderedSet %a) (OrderedSet %a))) +(defun (orderedset-symmetric-difference s other) + (def cmp (.comparer s)) + (def (: out (Array %a)) (make-array (+ (.count s) (.count other)))) + (def/mutable n 0) + (: put! (-> %a void)) + (defun (put! x) (array-set! out n x) (set! n (+ n 1))) + (let go ((a (cursor-of (.root s))) (c (cursor-of (.root other)))) + (cond + ((and (cursor-done? a) (cursor-done? c)) unit) + ((cursor-done? a) (put! (cursor-key c)) (go a (cursor-step c))) + ((cursor-done? c) (put! (cursor-key a)) (go (cursor-step a) c)) + (else + (def order (compare-keys cmp (cursor-key a) (cursor-key c))) + (cond + ((< order 0) (put! (cursor-key a)) (go (cursor-step a) c)) + ((> order 0) (put! (cursor-key c)) (go a (cursor-step c))) + (else (go (cursor-step a) (cursor-step c))))))) + (from-sorted out n cmp)) + +;; --------------------------------------------------------------------------- +;; Parts +;; --------------------------------------------------------------------------- + +(: (-> (Node %k) int)) +(defun (count-keys node) + (if (leaf? node) + (node-count node) + (loop (:for i (range 0 (+ (node-count node) 1))) (:acc (summing (count-keys (child node i))))))) + +;; The leaves under an internal node, or the keys of a leaf. +(: (-> (Node %k) int)) +(defun (weight node) + (cond + ((leaf? node) (node-count node)) + ((leaf? (child node 0)) (+ (node-count node) 1)) + (else (loop (:for i (range 0 (+ (node-count node) 1))) (:acc (summing (weight (child node i)))))))) + +;; About `k` sets in order that together hold the elements of `s` once. +;; They share its nodes: a part is a run of the nodes of the highest level +;; with at least `k` nodes, under new internal nodes. The runs are cut where +;; the running weight passes i * total / k, so the parts are close in size. +(: (-> (OrderedSet %a) int (Array (OrderedSet %a)))) +(defun (orderedset-split s k) + (def root (.root s)) + (if (or (<= k 1) (leaf? root)) + (vec->array [s]) + (begin + (def level (level-of (vec->array [root]) k)) + (def n (array-length level)) + (if (= n 1) + (vec->array [s]) + (begin + (def weights (loop (:for x level) (:acc (arraying (weight x))))) + (def total (loop (:for w weights) (:acc (summing w)))) + (def parts (min k n)) + (def/mutable out '()) + (def/mutable made 0) + (def/mutable start 0) + (def/mutable done 0) + (loop (:for j (range 0 n)) + (:do (set! done (+ done (array-ref weights j))) + (when (or (= j (- n 1)) + (and (< made (- parts 1)) + (>= (* (cast System.Int64 done) (cast System.Int64 parts)) + (* (cast System.Int64 total) (cast System.Int64 (+ made 1)))))) + (def node (over (loop (:for x (range start (+ j 1))) (:acc (arraying (array-ref level x)))))) + (set! out (cons (OrderedSet (root node) (count (count-keys node)) (comparer (.comparer s))) out)) + (set! made (+ made 1)) + (set! start (+ j 1))))) + (loop (:for x (list-reverse out)) (:acc (arraying x)))))))) + +;; The highest level, from `level` down, with at least `k` nodes, or the +;; leaves. +(: (-> (Array (Node %k)) int (Array (Node %k)))) +(defun (level-of level k) + (if (or (>= (array-length level) k) (leaf? (array-ref level 0))) + level + (begin + (def/mutable below '()) + (loop (:for x level) + (:do (loop (:for i (range 0 (+ (node-count x) 1))) + (:do (set! below (cons (child x i) below)))))) + (level-of (list->nodes (list-reverse below)) k)))) + +;; One set of the sets' elements, in order. Each set's elements must be less +;; than every element of the next. The leaves are kept: an empty one is +;; dropped, and a small one is copied together with its neighbour when the +;; two fit in one leaf. +(: (-> (Array (OrderedSet %a)) (OrderedSet %a))) +(defun (orderedset-concat sets) + (when (= (array-length sets) 0) (bad-argument "orderedset-concat needs at least one set.")) + (def cmp (.comparer (array-ref sets 0))) + (def/mutable leaves '()) + (def/mutable count 0) + (def/mutable (: last (Option %a)) None) + (loop (:for s sets) + (:when (> (.count s) 0)) + (:do (def lo (first-key (.root s))) + (when (and (some? last) (>= (compare-keys cmp (option-ref-or last lo) lo) 0)) + (bad-argument "The sets are not in order: an element of one is not less than every element of the next.")) + (def hi (key-of (rightmost (.root s)) (- (node-count (rightmost (.root s))) 1))) + (set! last (Some hi)) + (set! leaves (leaves-of (.root s) leaves)) + (set! count (+ count (.count s))))) + (def joined (join-small (list-reverse leaves))) + (if (list-empty? joined) + (empty-with cmp) + (OrderedSet (root (over (list->nodes joined))) (count count) (comparer cmp)))) + +(: (-> (List (Node %k)) (Array (Node %k)))) +(defun (list->nodes xs) (loop (:for x xs) (:acc (arraying x)))) + +;; The non-empty leaves under `node`, in reverse order, onto `acc`. +(: (-> (Node %k) (List (Node %k)) (List (Node %k)))) +(defun (leaves-of node acc) + (cond + ((leaf? node) (if (> (node-count node) 0) (cons node acc) acc)) + (else (loop (:for i (range 0 (+ (node-count node) 1))) + (:acc a (folding acc (leaves-of (child node i) a))) + => a)))) + +;; `leaves` with each small leaf copied together with the one before it when +;; the two fit in one leaf. +(: (-> (List (Node %k)) (List (Node %k)))) +(defun (join-small leaves) + (list-reverse + (loop (:for x leaves) + (:acc out (folding '() + (if (and (not (list-empty? out)) + (or (< (node-count (list-head out)) merge-threshold) + (< (node-count x) merge-threshold)) + (<= (+ (node-count (list-head out)) (node-count x)) leaf-capacity)) + (cons (both (list-head out) x) (list-tail out)) + (cons x out)))) + => out))) + +;; A new leaf with the keys of `a` and then those of `b`. +(: (-> (Node %k) (Node %k) (Node %k))) +(defun (both a b) + (def la (as-leaf a)) + (def lb (as-leaf b)) + (def (: l (Leaf %k)) (new-leaf (owner-none))) + (def na (leaf-count la)) + (def nb (leaf-count lb)) + (leaf-move! la 0 l 0 na) + (leaf-move! lb 0 l na nb) + (set-count! (leaf-node l) (+ na nb)) + (leaf-node l)) + +;; --------------------------------------------------------------------------- +;; Conversions +;; --------------------------------------------------------------------------- + +(: (-> (List %a) (OrderedSet %a)) (where (Ord %a))) +(defun (list->orderedset xs) + (def b (orderedsetbuilder-empty)) + (loop (:for x xs) (:do (orderedsetbuilder-add! b x))) + (orderedsetbuilder->orderedset b)) + +(: (-> (OrderedSet %a) (List %a))) +(defun (orderedset->list s) (->list s)) + +(: (-> (Vec %a) (OrderedSet %a)) (where (Ord %a))) +(defun (vec->orderedset v) + (def b (orderedsetbuilder-empty)) + (loop (:for x v) (:do (orderedsetbuilder-add! b x))) + (orderedsetbuilder->orderedset b)) + +(: (-> (OrderedSet %a) (Vec %a))) +(defun (orderedset->vec s) (->vec s)) + +(: (-> (Seq %a) (OrderedSet %a)) (where (Ord %a))) +(defun (seq->orderedset xs) + (def b (orderedsetbuilder-empty)) + (orderedsetbuilder-add-all! b xs) + (orderedsetbuilder->orderedset b)) + +(: (-> (OrderedSet %a) (Seq %a))) +(defun (orderedset->seq s) (->seq s)) + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- + +(impl (Refable (OrderedSet %a)) + (type %index %a) + (type %out bool) + (defun (ref set key) + (orderedset-contains? set key))) + +(impl (Refable (TransientOrderedSet %a)) + (type %index %a) + (type %out bool) + (defun (ref set key) + (transientorderedset-contains? set key))) + +(impl (Collection (OrderedSet %a)) + (type %content %a) + (defun (length s) (orderedset-length s)) + (defun (empty? s) (orderedset-empty? s))) + +(type (: (OrderedSetting %a) (Union ordered-setting))) + +(impl (Collector (OrderedSetting %a)) + (where (Ord %a)) + (type %elem %a) + (type %acc (OrderedSetBuilder %a)) + (type %out (OrderedSet %a)) + (defun (collector-init c) (orderedsetbuilder-empty)) + (defun (collector-step c acc x) (orderedsetbuilder-add! acc x) acc) + (defun (collector-finish c acc) (orderedsetbuilder->orderedset acc))) + +;; `Addable!`, adds an element at a time. +(impl (Addable! (OrderedSetBuilder %a)) + (type %elem %a) + (defun (add! sb elem) + (orderedsetbuilder-add! sb elem))) + +(impl (Addable! (TransientOrderedSet %a)) + (type %elem %a) + (defun (add! ts elem) + (transientorderedset-add! ts elem))) + +;; `Addable` adds an element immutably. +(impl (Addable (OrderedSet %a)) + (type %elem %a) + (defun (add s elem) + (orderedset-add s elem))) + +;; In the type's module, so that it is the set's .NET `Equals` and +;; `GetHashCode` too. +(impl (Eq (OrderedSet %a)) + (where (Eq %a)) + (defun (= a b) + (and (= (orderedset-length a) (orderedset-length b)) + (= (orderedset->list a) (orderedset->list b)))) + (defun (eq-hash s) + (orderedset-fold (fun (x h) (hash-combine h (eq-hash x))) 17 s))) + +;; `#{a b c}`, in ascending order. +(impl (Written (OrderedSet %a)) + (where (Written %a)) + (defun (write-into s b) (elements-print-into "#{" s "}" b (fun (x b) (write-into x b))))) + +(impl (Displayed (OrderedSet %a)) + (where (Displayed %a)) + (defun (display-into s b) (elements-print-into "#{" s "}" b (fun (x b) (display-into x b))))) + +(: (-> (OrderedSet %a) string) (where (Displayed %a))) +(defun (orderedset->str s) (displayed s)) + +(:deprecated orderedset->str "use `displayed`") +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; The operating system the program runs on, the machine and the process, and +;; the user's standard folders. +;; +;; - Which system: .NET's `OperatingSystem.IsXYZ`. Each check is a call of the +;; .NET method itself, at the call. .NET's core library is built for one +;; OS, so the method returns a constant, and the JIT removes the branch that +;; cannot run. A program is built once and chooses at run time. +;; - The machine and the process: `RuntimeInformation`, `Environment` and the +;; GC's memory limit. +;; - The folders: `Environment.GetFolderPath`, which follows the XDG +;; variables on Linux, and the cache folder, which it does not have. +;; - The signals: .NET's `PosixSignalRegistration`, through +;; `Bjolang.Runtime.SignalGroup`, and an inbox for each kind of signal. + +(import (std prelude)) +(import (std syntax-match)) +(import (std inbox)) + +(:doc #:file "os.bjodoc") + +(export + ;; Which system. + windows? linux? macos? freebsd? android? ios? mac-catalyst? tvos? watchos? browser? wasi? + os-platform? + windows-version-at-least? macos-version-at-least? freebsd-version-at-least? + android-version-at-least? ios-version-at-least? mac-catalyst-version-at-least? + tvos-version-at-least? watchos-version-at-least? os-platform-version-at-least? + ;; The machine and the process. + os-description os-version os-architecture process-architecture runtime-identifier + dotnet-description processor-count available-memory machine-name user-name + process-id process-path privileged? + ;; The user's folders. + home-directory config-directory data-directory cache-directory + documents-directory desktop-directory temp-directory make-temp-directory + ;; Signals. + Signal Signals signal-subscribe signal-recv signal-unsubscribe! signal-name + on-signal with-signal-cancel + ;; Named by the template of `with-signal-cancel`, so they have to be + ;; reachable where it expands. + signal-cancel-arm! signal-cancel-result) + +(import/extern + (windows? (: System.OperatingSystem.IsWindows (-> bool))) + (linux? (: System.OperatingSystem.IsLinux (-> bool))) + (macos? (: System.OperatingSystem.IsMacOS (-> bool))) + (freebsd? (: System.OperatingSystem.IsFreeBSD (-> bool))) + (android? (: System.OperatingSystem.IsAndroid (-> bool))) + (ios? (: System.OperatingSystem.IsIOS (-> bool))) + (mac-catalyst? (: System.OperatingSystem.IsMacCatalyst (-> bool))) + (tvos? (: System.OperatingSystem.IsTvOS (-> bool))) + (watchos? (: System.OperatingSystem.IsWatchOS (-> bool))) + (browser? (: System.OperatingSystem.IsBrowser (-> bool))) + (wasi? (: System.OperatingSystem.IsWasi (-> bool))) + (os-platform? (: System.OperatingSystem.IsOSPlatform (-> string bool)))) + +;; The version checks take .NET's trailing optional parameters as keywords. +;; Each wrapper is copied into its calls, so the call is .NET's. +(import/extern + (clr-windows-at-least (: System.OperatingSystem.IsWindowsVersionAtLeast (-> int int int int bool))) + (clr-macos-at-least (: System.OperatingSystem.IsMacOSVersionAtLeast (-> int int int bool))) + (clr-freebsd-at-least (: System.OperatingSystem.IsFreeBSDVersionAtLeast (-> int int int int bool))) + (clr-android-at-least (: System.OperatingSystem.IsAndroidVersionAtLeast (-> int int int int bool))) + (clr-ios-at-least (: System.OperatingSystem.IsIOSVersionAtLeast (-> int int int bool))) + (clr-mac-catalyst-at-least (: System.OperatingSystem.IsMacCatalystVersionAtLeast (-> int int int bool))) + (clr-tvos-at-least (: System.OperatingSystem.IsTvOSVersionAtLeast (-> int int int bool))) + (clr-watchos-at-least (: System.OperatingSystem.IsWatchOSVersionAtLeast (-> int int int bool))) + (clr-platform-at-least (: System.OperatingSystem.IsOSPlatformVersionAtLeast + (-> string int int int int bool)))) + +(: (-> int (#:minor int) (#:build int) (#:revision int) bool)) +(defun (windows-version-at-least? major #:minor 0 #:build 0 #:revision 0) #:inline + (clr-windows-at-least major minor build revision)) + +(: (-> int (#:minor int) (#:build int) bool)) +(defun (macos-version-at-least? major #:minor 0 #:build 0) #:inline + (clr-macos-at-least major minor build)) + +(: (-> int (#:minor int) (#:build int) (#:revision int) bool)) +(defun (freebsd-version-at-least? major #:minor 0 #:build 0 #:revision 0) #:inline + (clr-freebsd-at-least major minor build revision)) + +(: (-> int (#:minor int) (#:build int) (#:revision int) bool)) +(defun (android-version-at-least? major #:minor 0 #:build 0 #:revision 0) #:inline + (clr-android-at-least major minor build revision)) + +(: (-> int (#:minor int) (#:build int) bool)) +(defun (ios-version-at-least? major #:minor 0 #:build 0) #:inline + (clr-ios-at-least major minor build)) + +(: (-> int (#:minor int) (#:build int) bool)) +(defun (mac-catalyst-version-at-least? major #:minor 0 #:build 0) #:inline + (clr-mac-catalyst-at-least major minor build)) + +(: (-> int (#:minor int) (#:build int) bool)) +(defun (tvos-version-at-least? major #:minor 0 #:build 0) #:inline + (clr-tvos-at-least major minor build)) + +(: (-> int (#:minor int) (#:build int) bool)) +(defun (watchos-version-at-least? major #:minor 0 #:build 0) #:inline + (clr-watchos-at-least major minor build)) + +(: (-> string int (#:minor int) (#:build int) (#:revision int) bool)) +(defun (os-platform-version-at-least? platform major #:minor 0 #:build 0 #:revision 0) #:inline + (clr-platform-at-least platform major minor build revision)) + +;; --------------------------------------------------------------------------- +;; The machine and the process +;; --------------------------------------------------------------------------- + +(import/class + (Architecture (: System.Runtime.InteropServices.Architecture)) + (ClrVersion (: System.Version)) + (ClrOperatingSystem (: System.OperatingSystem)) + (GCMemoryInfo (: System.GCMemoryInfo))) + +;; A static property is imported as a value. Each is given as a function, as +;; the checks above are, and copied into its calls, so the call is the read of +;; the property. +(import/extern + (clr-os-description (: System.Runtime.InteropServices.RuntimeInformation.OSDescription (-> string) #:get)) + (clr-runtime-identifier + (: System.Runtime.InteropServices.RuntimeInformation.RuntimeIdentifier (-> string) #:get)) + (clr-dotnet-description + (: System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription (-> string) #:get)) + (clr-os-architecture (: System.Runtime.InteropServices.RuntimeInformation.OSArchitecture (-> Architecture) #:get)) + (clr-process-architecture + (: System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture (-> Architecture) #:get)) + (clr-os (: System.Environment.OSVersion (-> ClrOperatingSystem) #:get)) + (clr-os-version (: System.OperatingSystem.Version (-> ClrOperatingSystem ClrVersion) #:get)) + (clr-major (: System.Version.Major (-> ClrVersion int) #:get)) + (clr-minor (: System.Version.Minor (-> ClrVersion int) #:get)) + (clr-build (: System.Version.Build (-> ClrVersion int) #:get)) + (clr-revision (: System.Version.Revision (-> ClrVersion int) #:get)) + (clr-processor-count (: System.Environment.ProcessorCount (-> int) #:get)) + (clr-machine-name (: System.Environment.MachineName (-> string) #:get)) + (clr-user-name (: System.Environment.UserName (-> string) #:get)) + (clr-process-id (: System.Environment.ProcessId (-> int) #:get)) + (clr-process-path (: System.Environment.ProcessPath (-> string) #:get)) + (clr-privileged? (: System.Environment.IsPrivilegedProcess (-> bool) #:get)) + (clr-memory-info (: System.GC.GetGCMemoryInfo (-> GCMemoryInfo))) + (clr-total-available (: System.GCMemoryInfo.TotalAvailableMemoryBytes (-> GCMemoryInfo long) #:get))) + +(: (-> string)) +(defun (os-description) #:inline clr-os-description) + +(: (-> string)) +(defun (runtime-identifier) #:inline clr-runtime-identifier) + +(: (-> string)) +(defun (dotnet-description) #:inline clr-dotnet-description) + +(: (-> int)) +(defun (processor-count) #:inline clr-processor-count) + +(: (-> string)) +(defun (machine-name) #:inline clr-machine-name) + +(: (-> string)) +(defun (user-name) #:inline clr-user-name) + +(: (-> int)) +(defun (process-id) #:inline clr-process-id) + +(: (-> bool)) +(defun (privileged?) #:inline clr-privileged?) + +;; .NET's name of an architecture, in lower case, as a runtime identifier +;; writes it. +(: (-> Architecture string)) +(defun (architecture-name a) (string-downcase (.ToString a))) + +(: (-> string)) +(defun (os-architecture) (architecture-name clr-os-architecture)) + +(: (-> string)) +(defun (process-architecture) (architecture-name clr-process-architecture)) + +;; The parts of the version that .NET has: a part it does not know is -1, and +;; is left out. +(: (-> (Vec int))) +(defun (os-version) + (def v (clr-os-version clr-os)) + (vec-filter (fun (n) (>= n 0)) [(clr-major v) (clr-minor v) (clr-build v) (clr-revision v)])) + +;; The memory the GC may use: the machine's, or the limit of the container or +;; of the process. +(: (-> long)) +(defun (available-memory) (clr-total-available (clr-memory-info))) + +;; .NET has null when it cannot tell, which arrives as "". +(: (-> (Option string))) +(defun (process-path) + (def p clr-process-path) + (if (string-empty? p) None (Some p))) + +;; --------------------------------------------------------------------------- +;; The user's folders +;; --------------------------------------------------------------------------- + +(import/class + (SpecialFolder (: System.Environment+SpecialFolder)) + (SpecialFolderOption (: System.Environment+SpecialFolderOption)) + (ClrDirectoryInfo (: System.IO.DirectoryInfo)) + (InvalidOperationException (: System.InvalidOperationException (-> string InvalidOperationException)))) + +(import/extern + (clr-folder-path (: System.Environment.GetFolderPath (-> SpecialFolder SpecialFolderOption string))) + (clr-temp-path (: System.IO.Path.GetTempPath (-> string))) + (clr-trim-separator (: System.IO.Path.TrimEndingDirectorySeparator (-> string string))) + (clr-rooted? (: System.IO.Path.IsPathRooted (-> string bool))) + (clr-create-temp (: System.IO.Directory.CreateTempSubdirectory (-> string ClrDirectoryInfo))) + (clr-full-name (: System.IO.DirectoryInfo.FullName (-> ClrDirectoryInfo string) #:get))) + +;; The path of a special folder, whether or not it exists yet. Raises when +;; the system has none, as when HOME is not set and the user has no home in +;; the password database. +(: (-> SpecialFolder string string)) +(defun (folder-path folder what) + (def p (clr-folder-path folder SpecialFolderOption.DoNotVerify)) + (if (string-empty? p) + (raise (cast Exception (InvalidOperationException. (str "This system has no " what ".")))) + p)) + +(: (-> string)) +(defun (home-directory) (folder-path SpecialFolder.UserProfile "home directory")) + +(: (-> string)) +(defun (config-directory) (folder-path SpecialFolder.ApplicationData "folder for configuration")) + +(: (-> string)) +(defun (data-directory) (folder-path SpecialFolder.LocalApplicationData "folder for data")) + +(: (-> string)) +(defun (documents-directory) (folder-path SpecialFolder.MyDocuments "folder for documents")) + +(: (-> string)) +(defun (desktop-directory) (folder-path SpecialFolder.Desktop "desktop folder")) + +;; .NET has no special folder for a cache. On Windows it is the local data +;; folder, on macOS `~/Library/Caches`, and elsewhere `XDG_CACHE_HOME`, or +;; `~/.cache` when that is not set or not an absolute path, as the XDG Base +;; Directory specification says. +(: (-> string)) +(defun (cache-directory) + (cond + ((windows?) (data-directory)) + ((macos?) (path-combine (home-directory) "Library" "Caches")) + (else + (match (get-environment-variable "XDG_CACHE_HOME") + ((Some p) (if (clr-rooted? p) p (path-combine (home-directory) ".cache"))) + (None (path-combine (home-directory) ".cache")))))) + +;; The folder for temporary files: `TMPDIR` on Unix, `TMP` or `TEMP` on +;; Windows, else the system's. Without the separator .NET ends it with. +(: (-> string)) +(defun (temp-directory) (clr-trim-separator (clr-temp-path))) + +;; A new, empty directory in the temp folder, with a name of its own after +;; `prefix`. On Unix only its owner may read or write it. +(: (-> (#:prefix string) string)) +(defun (make-temp-directory #:prefix "") + (clr-full-name (clr-create-temp prefix))) + +;; --------------------------------------------------------------------------- +;; Signals +;; --------------------------------------------------------------------------- +;; +;; - While a subscription lives, its signals do not do their default action, +;; as ending or stopping the process. The ambient scope owns the +;; subscription, so the default action is back when the scope ends. +;; - .NET calls the handler on a thread that is not a fiber. The handler posts +;; the signal to an inbox with room for one item. Each kind of signal has +;; its own inbox, so a signal of a kind that already waits is merged with +;; it, as in OCaml and Racket. A fiber receives from the inboxes. +;; - `with-signal-cancel` cancels its scope at the first signal and gives the +;; signals back to the system, so a second signal does the default action. + +(type (: Signal (Union Hangup Interrupt Quit Terminate Child Continue WindowChanged + TerminalInput TerminalOutput TerminalStop User1 User2 + (: OtherSignal int)))) + +(import/class + (SignalGroup (: Bjolang.Runtime.SignalGroup (-> SignalGroup))) + (PlatformNotSupportedException + (: System.PlatformNotSupportedException (-> string PlatformNotSupportedException))) + (ArgumentException (: System.ArgumentException (-> string ArgumentException))) + (HandlerFailed (: System.InvalidOperationException (-> string Exception HandlerFailed)))) + +(import/extern + (clr-group-add! (: Bjolang.Runtime.SignalGroup.Add (-> SignalGroup int bool (-> void) void))) + (clr-cancellation? (: Bjolang.Runtime.SignalGroup.IsCancellation (-> Exception bool)))) + +(type (: Signals #:opaque (Struct (: group SignalGroup) (: inboxes (Vec (Inbox Signal)))))) + +(: (-> string PlatformNotSupportedException)) +(defun (unsupported message) (PlatformNotSupportedException. message)) + +(: (-> Signal string)) +(defun (signal-name s) + (match s + (Hangup "SIGHUP") + (Interrupt "SIGINT") + (Quit "SIGQUIT") + (Terminate "SIGTERM") + (Child "SIGCHLD") + (Continue "SIGCONT") + (WindowChanged "SIGWINCH") + (TerminalInput "SIGTTIN") + (TerminalOutput "SIGTTOU") + (TerminalStop "SIGTSTP") + (User1 "SIGUSR1") + (User2 "SIGUSR2") + ((OtherSignal n) (str "signal " (int->string n))))) + +;; The four that Windows has: the console's close button, Ctrl-C, Ctrl-Break, +;; and log off or shut down. +(: (-> Signal bool)) +(defun (windows-signal? s) + (match s + (Hangup #t) + (Interrupt #t) + (Quit #t) + (Terminate #t) + (_ #f))) + +;; Whether the default action of the signal ends or stops the process, which +;; a subscription prevents. The default action of the other three is to do +;; nothing, and .NET's own handling of them stays. +(: (-> Signal bool)) +(defun (cancels-default? s) + (match s + (Child #f) + (Continue #f) + (WindowChanged #f) + (_ #t))) + +;; SIGUSR1 and SIGUSR2 have one number on Linux and another on the BSDs and +;; Apple's systems. +(: (-> Signal int int int)) +(defun (user-signal-code s linux bsd) + (cond + ((or (linux?) (android?)) linux) + ((or (macos?) (freebsd?) (ios?) (tvos?) (watchos?) (mac-catalyst?)) bsd) + (else + (raise (cast Exception (unsupported (str "The number of " (signal-name s) " on this system" + " is not known. Use (OtherSignal n)."))))))) + +;; The number of the signal for .NET's `PosixSignal`: negative for .NET's own +;; names, and the system's number for the others. +;; +;; - Raises when this system cannot have the signal. +;; - Raises for `(OtherSignal n)` with `n` below 1: .NET reads a negative +;; number as one of its own names. +(: (-> Signal int)) +(defun (signal-code s) + (cond + ((or (browser?) (wasi?)) + (raise (cast Exception (unsupported "This system has no signals.")))) + ((and (windows?) (not (windows-signal? s))) + (raise (cast Exception (unsupported (str "Windows has no " (signal-name s) "."))))) + (else + (match s + (Hangup -1) + (Interrupt -2) + (Quit -3) + (Terminate -4) + (Child -5) + (Continue -6) + (WindowChanged -7) + (TerminalInput -8) + (TerminalOutput -9) + (TerminalStop -10) + (User1 (user-signal-code s 10 30)) + (User2 (user-signal-code s 12 31)) + ((OtherSignal n) + (if (> n 0) + n + (raise (cast Exception (ArgumentException. + (str "OtherSignal takes a signal number above 0, not " + (int->string n) ".")))))))))) + +;; Each signal once, in the order given. +(: (-> (Vec Signal) (Vec Signal))) +(defun (distinct-signals signals) + (let go ((rest (vec->list signals)) (kept [])) + (match rest + (Nil kept) + ((Cons s more) (go more (if (vec-contains? kept s) kept (vec-add kept s))))))) + +;; Registers the handler in `handlers` for each signal in `kinds`, whose +;; numbers are `codes`. Raises when the system cannot handle one, as SIGKILL. +;; The caller disposes the group then. +(: (-> SignalGroup (Vec Signal) (Vec int) (Vec (-> void)) void)) +(defun (add-handlers! group kinds codes handlers) + (loop (:for s kinds) + (:for code codes) + (:for handler handlers) + (:do (match (try (clr-group-add! group code (cancels-default? s) handler) + #:catch (System.Exception)) + ((Ok _) unit) + ((Err e) + (raise (cast Exception (unsupported (str "This system cannot handle " (signal-name s) + ": " (.-Message e)))))))))) + +;; Subscribes to `signals`. +;; +;; - While the subscription lives, the signals do not do their default +;; action. The ambient scope owns it, and ends it when the scope ends. +;; - Raises before it registers anything when a signal is not on this +;; system, and when there is no ambient scope. +(: (-> (Vec Signal) Signals)) +(defun (signal-subscribe signals) + (def kinds (distinct-signals signals)) + (def codes (->vec (map signal-code kinds))) + (def inboxes (->vec (map (fun (_) (make-inbox 1 DropNewest)) kinds))) + (def subscription (Signals (group (SignalGroup.)) (inboxes inboxes))) + (def owned (own! (fun () (signal-unsubscribe! subscription)))) + (def handlers + (loop (:for s kinds) + (:for ib inboxes) + (:acc hs (vecing (fun () (ignore (inbox-post! ib s))))))) + (match (try (add-handlers! (struct-ref subscription group) kinds codes handlers) + #:catch (System.Exception)) + ((Ok _) subscription) + ((Err e) (release! owned) (raise e)))) + +;; The event of the next signal. In a `choose` that another branch wins, it +;; takes nothing. When signals of more than one kind wait, which one comes +;; first is not given. After the subscription ends, it never fires. +(: (-> Signals (Event Signal))) +(defun (signal-recv s) + (match (vec->list (struct-ref s inboxes)) + (Nil (never)) + ((Cons ib more) + (let go ((rest more) (e (inbox-recv ib))) + (match rest + (Nil e) + ((Cons next after) (go after (choose e (inbox-recv next))))))))) + +;; Ends the subscription now: the signals do their default action again, and +;; signals that wait are dropped. A second call does nothing. +(: (-> Signals void)) +(defun (signal-unsubscribe! s) + (.Dispose (struct-ref s group)) + (for-each inbox-abort! (struct-ref s inboxes))) + +;; Calls `handler` with each of `signals` that comes, for as long as the +;; ambient scope lives. +;; +;; - A daemon of the scope calls it, so it has the dynamic environment of +;; this call, and the scope does not wait for it. The subscription and the +;; daemon end together, with the scope. +;; - One call at a time. A signal that comes during a call waits, merged +;; with the others of its kind. +;; - The handler is a bjoroutine, and an ordinary function is one too. Not +;; `-?->`: with a bjoroutine, that would make this a bjoroutine, though it +;; only gives the handler to the daemon, and `main` could not call it. +(: (-> (Vec Signal) (-bjo-> Signal void) void)) +(defun (on-signal signals handler) + (def subscription (signal-subscribe signals)) + (spawn/daemon (handle-signals subscription handler))) + +;; The daemon of `on-signal`. When the handler raises, it ends the +;; subscription, so the signals do their default action again, and raises +;; for the report of the daemon. Nothing else would receive the signals, and +;; they would be lost without a word. A cancellation is raised as it is: the +;; scope ends. +(: (-bjo-> Signals (-bjo-> Signal void) void)) +(defbjo (handle-signals subscription handler) + (def s (sync (signal-recv subscription))) + (match (try (handler s) #:catch (System.Exception)) + ((Ok _) (handle-signals subscription handler)) + ((Err e) + (signal-unsubscribe! subscription) + (if (clr-cancellation? e) + (raise e) + (raise (cast Exception (HandlerFailed. (str "The handler of " (signal-name s) " raised, so " + "its signals do their default action again.") + e))))))) + +;; `(with-signal-cancel signals body ...)` - a scope that the first of +;; `signals` cancels. +;; +;; - Returns `(Ok v)` when the body returns `v`, and `(Err s)` when the signal +;; `s` stopped the body. The scope's reason is `(Requested "SIGINT")` and +;; the like. +;; - At the first signal it gives the signals back to the system, so a second +;; signal does the default action, as ending the process at once. +;; - A failure of the body is raised again, also after a signal. +(def/macro (with-signal-cancel form inject compare) + (syntax-match form + ((_ signals first body ...) + #'(let () + (def kinds ,signals) + (def scope (scope-open! 0)) + (signal-cancel-result + kinds + (try (scope-form scope (cancel) + (signal-cancel-arm! kinds cancel) + ,first ,@body) + #:catch (System.Exception)) + (scope-token scope)))) + (bad (syntax-error bad "with-signal-cancel takes a Vec of signals and a body: (with-signal-cancel [Interrupt Terminate] body ...)")))) + +;; Subscribes the scope that `cancel` cancels to `signals`. The first signal +;; ends the subscription and then cancels the scope, so the signals do their +;; default action again before anything sees the cancellation. +(: (-> (Vec Signal) (-> CancelReason void) void)) +(defun (signal-cancel-arm! signals cancel) + (def kinds (distinct-signals signals)) + (def codes (->vec (map signal-code kinds))) + (def group (own-disposable (SignalGroup.))) + (add-handlers! group kinds codes + (->vec (map (fun (s) (fun () + (.Dispose group) + (cancel (Requested (signal-name s))))) + kinds)))) + +;; The value of `with-signal-cancel`, from the outcome of its scope and the +;; scope's token. +;; +;; - The body returned: its value. +;; - One of `signals` cancelled the scope, and the body left because of it: +;; that signal. +;; - Else: the body's exception, raised again. +(: (-> (Vec Signal) (Result Exception %a) CancelToken (Result Signal %a))) +(defun (signal-cancel-result signals outcome token) + (match outcome + ((Ok v) (Ok v)) + ((Err e) + (match (cancel-reason token) + ((Some (Requested who)) + (match (find (fun (s) (= (signal-name s) who)) signals) + ((Some s) (if (clr-cancellation? e) (Err s) (raise e))) + (None (raise e)))) + (_ (raise e)))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; Work on the parts of a collection at the same time. +;; +;; A `Chunkable` collection splits into parts that together hold each element +;; once, in the collection's order when it has one. Both run a function on +;; each part on threads made for work that uses a core: +;; +;; - `in-parts` returns the results in the order of the parts. +;; - `in-parallel` returns them combined in that order, by `Combinable`. +;; +;; The functions over a whole collection join the parts for the caller: +;; +;; - `par-map` and `par-filter` return a collection of the type they are +;; given. Each type has its own way to join its parts. +;; - `par-for-each` runs a function on each element. +;; +;; An array splits into views, not copies. A part of an array gives a span of +;; its own elements, so the parts of one array can be written in place at the +;; same time. A span cannot go to another thread, so each part makes its own. +;; A vector, a map or a set splits into collections of its own type, which +;; share its nodes. + +(import (std prelude)) +(import (std hashset)) +(import (std orderedmap)) +(import (std orderedset)) +(import (std atomic)) +(import (std inbox)) +(import (std combine)) + +(:doc #:file "parallel.bjodoc") + +(import/extern + (processor-count (: System.Environment.ProcessorCount #:get)) + (parse-int (: System.Int32.TryParse (-> string (out int) (Option int)))) + (make-jobs-inbox (: Bjoml.InboxModule.MakeNewestFirst (-> int bool (Inbox %a)))) + (spin-before-parking (: BjolangRuntime.SpinBeforeParking (-> int void))) + (iarray-adopt (: Bjolang.Runtime.IArrayModule.Adopt (-> (Array %a) (IArray %a))))) + +(import/class + ;; The accumulator of `arraying`, which the prelude does not export. + ((ArrayBuilder %a) (: Bjolang.Runtime.ArrayBuilder))) + +(import/extern + (arraybuilder-empty (: Bjolang.Runtime.ArrayBuilderModule.Empty (-> (ArrayBuilder %a)))) + (arraybuilder-add (: Bjolang.Runtime.ArrayBuilderModule.Add (-> (ArrayBuilder %a) %a (ArrayBuilder %a)))) + (arraybuilder-count (: Bjolang.Runtime.ArrayBuilderModule.Count (-> (ArrayBuilder %a) int))) + (arraybuilder-span (: Bjolang.Runtime.ArrayBuilderModule.Span (-> (ArrayBuilder %a) int int (Span %a))))) + +(re-export combine combine-all) + +(export Chunkable chunk-size chunks + in-parts in-parallel computing-each compute-thread-count + ArrayPart part-array part-lo part-hi part-length part->span part->ro-span + IArrayPart iarray-part-lo iarray-part-hi iarray-part-length + par-for-each + ParallelMap par-map ParallelFilter par-filter) + +;; --------------------------------------------------------------------------- +;; The trait +;; --------------------------------------------------------------------------- + +;; A collection that splits into parts. +;; +;; - `chunk-size`: the number of elements, or a good estimate. +;; - `chunks c k`: about k parts, in the collection's order when it has one, +;; that hold each element once. Fewer when there are fewer elements than k, +;; and at least one, which may be empty. +(def/trait (Chunkable %c) + (type %chunk) + (: chunk-size (-> %c int)) + (: chunks (-> %c int (Vec %chunk)))) + +;; The number of parts to make of n elements when k are asked for: at least +;; one, and no more than there are elements. +(: (-> int int int)) +(defun (part-count n k) + (max 1 (min k n))) + +;; Where part i of k starts, as an offset into n elements. Parts differ in size +;; by one at most. In a long, because i times n can pass `int-max`. +(: (-> int int int int)) +(defun (part-start n k i) + (cast int (/ (* (cast long n) (cast long i)) (cast long k)))) + +;; --------------------------------------------------------------------------- +;; The compute threads +;; --------------------------------------------------------------------------- + +;; Threads for work that uses a core, made at the first call and kept. +;; +;; - Each is a `spawn/thread` that takes jobs from `jobs` for ever. A +;; detached fiber syncs its event, as `spawn/thread` starts its thread when +;; synced. The fiber is suspended for ever, and holds nothing. +;; - They are not the threads of the run queues, so a job holds no pump, and +;; fibers go on running while the jobs compute. +;; - `jobs` is an inbox, which several threads can wait on, and which a post +;; does not wait for. It wakes the thread that waited last, which is the +;; one most likely to be spinning still, as a thread spins a little before +;; it parks: then a post makes no system call to wake it. +;; - A thread that has no job spins for 50 us before it parks. A call that +;; comes within that time finds it awake: it starts at once, and the post +;; makes no system call. Measured on 24 empty parts after 10 us of other +;; work: 14 to 6 us for each call. The cost is a core for 50 us after the +;; last job of each thread. +;; - A job is a closure over the dynamic environment of the code that made +;; it, and runs in that environment. +;; - A job posted to a full inbox is dropped. That is safe because a call +;; always has a thread that takes its parts: in ordinary code the calling +;; thread, and in a bjoroutine the first runner, which waits for room +;; when the inbox is full. + +;; How many compute threads there are: BJOLANG_COMPUTE_THREADS, or one less +;; than the logical cores, and at least one. The core left over is for the +;; thread that calls, which takes parts too in ordinary code, and for the +;; fibers. Read once, when the module is loaded. +;; +;; Measured with bjostat's bench/latency.bjo, 24 logical cores: with 24 +;; compute threads, a fiber that sleeps 1 ms woke up to 7.2-8.6 ms late in +;; every run; with 23, up to 1.9-2.7 ms, at the same number of loops. +(def default-thread-count (max 1 (- processor-count 1))) + +(def thread-count + (match (get-environment-variable "BJOLANG_COMPUTE_THREADS") + ((Some s) (match (parse-int s) + ((Some n) (if (> n 0) n default-thread-count)) + (None default-thread-count))) + (None default-thread-count))) + +(: (-> int)) +(defun (compute-thread-count) thread-count) + +(def jobs (: (make-jobs-inbox 4096 #f) (Inbox (-> Unit)))) +(def started (make-atomic 0)) + +;; A compute thread: runs jobs for ever. A job raises nothing: the parts it +;; runs keep their exceptions. +(def spin-us 50) + +(: (-> Unit)) +(defun (compute-loop) + (spin-before-parking spin-us) + (let go () + ((sync/blocking (inbox-recv jobs))) + (go))) + +(: (-bjo-> void)) +(defbjo (keep-compute-thread) + (ignore (sync (spawn/thread compute-loop)))) + +(: (-> void)) +(defun (start-compute-threads!) + (when (= (atomic-compare-and-swap! started 0 1) 0) + (loop (:for _ (range 0 thread-count)) + (:do (spawn/detached (keep-compute-thread)))))) + +;; --------------------------------------------------------------------------- +;; Running the parts +;; --------------------------------------------------------------------------- + +;; Runs `(work k)` for each k from 0 to n - 1 on the compute threads, and +;; returns the values in the order of k. +;; +;; - A runner is a job that takes parts by number until none is left. The +;; caller posts runners one after the other while a part is left that no +;; runner has taken, up to one for each compute thread, or for each part +;; when there are fewer. A post that wakes a thread that sleeps is a system +;; call of some microseconds, and the thread starts some microseconds +;; later: so when the parts are small, the first runners take them all and +;; the caller stops posting. A runner that a thread takes after the parts +;; are all taken ends at once. +;; - In a bjoroutine the fiber is suspended until the parts have run, and +;; holds nothing. +;; - In ordinary code the thread posts one runner fewer, then takes the parts +;; that are left itself, and then waits only for the parts that other +;; threads have taken. So a part can call `computing-each` itself: a call +;; does not wait for runners that no thread is free to take, and there is +;; no deadlock when calls nest. +;; - A part runs in the dynamic environment of the call. +;; - When a part raises, the parts not started do not run, and the exception +;; is raised here when the parts that started have ended. A part that runs +;; cannot be stopped. In a bjoroutine, the parts not started when the scope +;; is cancelled do not run, and `Cancelled` is raised. +(: computing-each (-> int (-> int %a) (Vec %a))) +(defun (computing-each n work) + (if (<= n 0) + [] + (let () + (start-compute-threads!) + (def results (make-array n)) + (def next (make-atomic -1)) + (def ended (make-atomic 0)) + (def failed (make-atomic 0)) + (def failure (: (make-array 1) (Array Exception))) + (def skipped (make-atomic 0)) + (def done (: (make-inbox 1 DropNewest) (Inbox Unit))) + (def token (parameter-ref current-cancel)) + (def env (current-dynamic-environment)) + ;; Takes parts until none is left. + (def run-parts + (fun () + (let go () + (def k (atomic-increment! next)) + (when (< k n) + (cond + ((not (= (atomic-ref failed) 0)) unit) + ((cancelled? token) (ignore (atomic-increment! skipped))) + (else (match (try (array-set! results k (work k)) #:catch (System.Exception)) + ((Ok _) unit) + ((Err e) (when (= (atomic-compare-and-swap! failed 0 1) 0) + (array-set! failure 0 e)))))) + (when (= (atomic-increment! ended) n) + (ignore (inbox-post! done unit))) + (go))) + unit)) + (def runner (fun () (call-with-dynamic-environment env run-parts))) + ;; At most one runner for each part and one for each compute thread, + ;; and in ordinary code one less, as the calling thread takes parts + ;; too. + (def runners (by-colour (#:sync (min (- n 1) thread-count)) + (#:bjo (min n thread-count)))) + (def posted (make-atomic 0)) + ;; Posts runners while a part is left that no runner has taken. + (def post-runners + (fun () + (let go () + (when (and (< (+ (atomic-ref next) 1) n) + (<= (atomic-increment! posted) runners)) + (ignore (inbox-post! jobs runner)) + (go))))) + (by-colour + (#:sync (begin + (post-runners) + (run-parts) + (sync/blocking (inbox-recv done)))) + (#:bjo (begin + (ignore (atomic-increment! posted)) + (unless (inbox-post! jobs runner) + (sync (inbox-send-evt jobs runner))) + (post-runners) + (sync (inbox-recv done)) + (when (> (atomic-ref skipped) 0) (sync (: (never) (Event Unit))))))) + (when (= (atomic-ref failed) 1) (raise (array-ref failure 0))) + (array->vec results)))) + +;; How many parts to ask of a collection of `n` elements. +;; - `size` > 0: enough parts that none need be longer than `size`. +;; - else: `parts`. +(: (-> int int int int)) +(defun (parts-asked n parts size) + (if (> size 0) + (max 1 (+ (/ n size) (if (= 0 (% n size)) 0 1))) + parts)) + +;; Runs `f` on each part of `c` on the compute threads, and returns the +;; results in the order of the parts. The parts run by `computing-each`. +;; +;; - `#:parts`: how many parts to ask for. Default: four for each compute +;; thread. The collection may make fewer. +;; - `#:size`: instead, ceil(n / size) parts of n elements, so that the parts +;; do not depend on the machine. +(: in-parts (-> %c (-> %p %r) (#:parts int) (#:size int) (Vec %r)) (where (Chunkable %c #:chunk %p))) +(defun (in-parts c f #:parts (* 4 thread-count) #:size 0) + (def ps (chunks c (parts-asked (chunk-size c) parts size))) + (computing-each (vec-length ps) (fun (i) (f (vec-ref ps i))))) + +;; `in-parts`, with the results combined in the order of the parts. +(: in-parallel (-> %c (-> %p %r) (#:parts int) (#:size int) %r) + (where (Chunkable %c #:chunk %p) (Combinable %r))) +(defun (in-parallel c f #:parts (* 4 thread-count) #:size 0) + (combine-all (in-parts c f #:parts parts #:size size))) + +;; --------------------------------------------------------------------------- +;; Ranges and vectors +;; --------------------------------------------------------------------------- + +(impl (Chunkable Range) + (type %chunk Range) + (defun (chunk-size r) (max 0 (- (record-ref r hi) (record-ref r lo)))) + (defun (chunks r k) + (def lo (record-ref r lo)) + (def n (max 0 (- (record-ref r hi) lo))) + (def m (part-count n k)) + (loop (:for i (range 0 m)) + (:acc parts (vecing (range (+ lo (part-start n m i)) + (+ lo (part-start n m (+ i 1))))))))) + +;; A part of a vector is a slice of it, which shares its nodes. +(impl (Chunkable (Vec %a)) + (type %chunk (Vec %a)) + (defun (chunk-size v) (vec-length v)) + (defun (chunks v k) + (def n (vec-length v)) + (def m (part-count n k)) + (loop (:for i (range 0 m)) + (:acc parts (vecing (vec-slice v + (part-start n m i) + (- (part-start n m (+ i 1)) (part-start n m i)))))))) + +;; --------------------------------------------------------------------------- +;; Arrays: parts are views +;; --------------------------------------------------------------------------- + +;; The elements of `array` from `lo` up to `hi`. A view: it holds the array, +;; not a copy of the elements. +(type (: (ArrayPart %a) (Struct (: array (Array %a)) (: lo int) (: hi int)))) + +(: (-> (ArrayPart %a) (Array %a))) +(defun (part-array p) (record-ref p array)) + +;; The index in the array of the part's first element. +(: (-> (ArrayPart %a) int)) +(defun (part-lo p) (record-ref p lo)) + +;; The index in the array after the part's last element. +(: (-> (ArrayPart %a) int)) +(defun (part-hi p) (record-ref p hi)) + +(: (-> (ArrayPart %a) int)) +(defun (part-length p) (- (record-ref p hi) (record-ref p lo))) + +;; A span of the part's elements, which writes into the array. Index 0 is the +;; part's first element. Made where it is used: a span cannot go to another +;; thread or into a closure. +(: (-> (ArrayPart %a) (Span %a))) +(defun (part->span p) + (array->span (record-ref p array) #:from (record-ref p lo) #:to (record-ref p hi))) + +(: (-> (ArrayPart %a) (RoSpan %a))) +(defun (part->ro-span p) + (array->ro-span (record-ref p array) #:from (record-ref p lo) #:to (record-ref p hi))) + +;; The part's elements, in order. The cursor is the index into the array. +(impl (Iterable (ArrayPart %a)) + (type %elem %a) + (type %cursor int) + (defun (iterable-start p) (record-ref p lo)) + (defun (iterable-done? p index) (>= index (record-ref p hi))) + (defun (iterable-current p index) (array-ref (record-ref p array) index)) + (defun (iterable-next p index) (+ index 1)) + (defun (count p) (- (record-ref p hi) (record-ref p lo)))) + +(impl (Chunkable (Array %a)) + (type %chunk (ArrayPart %a)) + (defun (chunk-size a) (array-length a)) + (defun (chunks a k) + (def n (array-length a)) + (def m (part-count n k)) + (loop (:for i (range 0 m)) + (:acc parts (vecing (ArrayPart (array a) + (lo (part-start n m i)) + (hi (part-start n m (+ i 1))))))))) + +;; --------------------------------------------------------------------------- +;; Immutable arrays: parts are views too +;; --------------------------------------------------------------------------- + +(type (: (IArrayPart %a) (Struct (: iarray (IArray %a)) (: lo int) (: hi int)))) + +(: (-> (IArrayPart %a) int)) +(defun (iarray-part-lo p) (record-ref p lo)) + +(: (-> (IArrayPart %a) int)) +(defun (iarray-part-hi p) (record-ref p hi)) + +(: (-> (IArrayPart %a) int)) +(defun (iarray-part-length p) (- (record-ref p hi) (record-ref p lo))) + +(impl (Iterable (IArrayPart %a)) + (type %elem %a) + (type %cursor int) + (defun (iterable-start p) (record-ref p lo)) + (defun (iterable-done? p index) (>= index (record-ref p hi))) + (defun (iterable-current p index) (iarray-ref (record-ref p iarray) index)) + (defun (iterable-next p index) (+ index 1)) + (defun (count p) (- (record-ref p hi) (record-ref p lo)))) + +(impl (Chunkable (IArray %a)) + (type %chunk (IArrayPart %a)) + (defun (chunk-size a) (iarray-length a)) + (defun (chunks a k) + (def n (iarray-length a)) + (def m (part-count n k)) + (loop (:for i (range 0 m)) + (:acc parts (vecing (IArrayPart (iarray a) + (lo (part-start n m i)) + (hi (part-start n m (+ i 1))))))))) + +;; --------------------------------------------------------------------------- +;; Maps and sets: parts are maps and sets +;; --------------------------------------------------------------------------- + +;; The split is in the runtime. It cuts along the tree's own nodes, so the +;; parts share them, and their sizes are close but not equal. A tree too small +;; to cut, one leaf, is one part. + +;; The parts of a hash map or a hash set are in no order that means +;; anything, as the map's own order does not. +(impl (Chunkable (Map %k %v)) + (type %chunk (Map %k %v)) + (defun (chunk-size m) (map-length m)) + (defun (chunks m k) (array->vec (map-split m k)))) + +(impl (Chunkable (HashSet %a)) + (type %chunk (HashSet %a)) + (defun (chunk-size s) (hashset-length s)) + (defun (chunks s k) (array->vec (hashset-split s k)))) + +;; The parts of an ordered map or set are in key order: every key of a part +;; is less than every key of the next. +(impl (Chunkable (OrderedMap %k %v)) + (type %chunk (OrderedMap %k %v)) + (defun (chunk-size m) (orderedmap-length m)) + (defun (chunks m k) (array->vec (orderedmap-split m k)))) + +(impl (Chunkable (OrderedSet %a)) + (type %chunk (OrderedSet %a)) + (defun (chunk-size s) (orderedset-length s)) + (defun (chunks s k) (array->vec (orderedset-split s k)))) + +;; --------------------------------------------------------------------------- +;; Over the elements of a whole collection +;; --------------------------------------------------------------------------- + +;; Runs `f` on each element of `c`, on the compute threads. The elements of +;; a part are taken in order, and the parts at the same time. +(: par-for-each (-> (-> %e void) %c void) + (where (Chunkable %c #:chunk %p) + (Iterable %p #:elem %e))) +(defun (par-for-each f c) + (ignore (in-parallel c + (fun (part) + (let go ((cur (iterable-start part))) + (if (iterable-done? part cur) + unit + (begin + (f (iterable-current part cur)) + (go (iterable-next part cur))))))))) + +;; The parts joined from the first to the last: `(join (join p0 p1) p2)`. +;; There is at least one part. +(: (-> (Vec %a) (-> %a %a %a) %a)) +(defun (joined parts join) + (loop (:for i (range 1 (vec-length parts))) + (:acc out (folding (vec-ref parts 0) (join out (vec-ref parts i)))))) + +;; --------------------------------------------------------------------------- +;; A collection of the type given +;; --------------------------------------------------------------------------- + +;; A collection whose elements a function can change on the compute threads. +;; +;; - `par-map f c`: a collection of the same type, with `(f x)` in place of +;; each element x, in the same order. `f` runs on the compute threads. +;; - Of a map, the values are the elements: the keys stay as they are. +;; - A set is not a `ParallelMap`, as a set of the values of `f` needs their +;; `Eq` or `Ord`. Use `in-parallel` with a loop into `hash-setting` or +;; `ordered-setting`. +(def/trait (ParallelMap (%c %a)) + (: par-map (-> (-> %a %b) (%c %a) (%c %b)))) + +;; A collection whose elements a test can select on the compute threads. +;; +;; - `par-filter keep? c`: a collection of the same type, with the elements +;; for which `(keep? x)` is true, in the same order. `keep?` runs on the +;; compute threads. +;; - Of a map, `keep?` is given the value. To test the key too, use +;; `in-parts` with `map-filter`, and `map-merge` the parts. +(def/trait (ParallelFilter (%c %a)) + (: par-filter (-> (-> %a bool) (%c %a) (%c %a)))) + +;; --- Vectors: the parts are joined by concatenation, which shares their +;; nodes. + +(impl (ParallelMap (Vec %a)) + (defun (par-map f v) + (in-parallel v (fun (p) (vec-map f p))))) + +(impl (ParallelFilter (Vec %a)) + (defun (par-filter keep? v) + (in-parallel v (fun (p) (vec-filter keep? p))))) + +;; --- Arrays: `par-map` makes the result first, and each part writes its +;; own range of it. `par-filter` collects each part's elements, and then +;; copies them into an array of their number. + +;; Writes `(f x)` into `out` for each element x of part `p` of `a`, at the +;; same index. +(: (-> (-> %a %b) (ArrayPart %a) (Array %b) Unit)) +(defun (map-part-into f p out) + (def a (record-ref p array)) + (loop (:for i (range (record-ref p lo) (record-ref p hi))) + (:do (array-set! out i (f (array-ref a i))))) + unit) + +(: (-> (-> %a %b) (IArrayPart %a) (Array %b) Unit)) +(defun (map-iarray-part-into f p out) + (def a (record-ref p iarray)) + (loop (:for i (range (record-ref p lo) (record-ref p hi))) + (:do (array-set! out i (f (iarray-ref a i))))) + unit) + +;; The elements of the builders, one builder after the other, in an array of +;; their number. +(: (-> (Vec (ArrayBuilder %a)) (Array %a))) +(defun (builders->array parts) + (def out (make-array (loop (:for b parts) (:acc n (summing (arraybuilder-count b)))))) + (loop (:for b parts) + (:with at 0 (+ at (arraybuilder-count b))) + (:do (let ((from (arraybuilder-span b 0 (arraybuilder-count b)))) + (loop (:for i (range 0 (span-length from))) + (:do (array-set! out (+ at i) (span-ref from i))))))) + out) + +(: (-> (-> %a bool) (ArrayPart %a) (ArrayBuilder %a))) +(defun (filter-part keep? p) + (loop (:for x p) + (:when (keep? x)) + (:acc kept (folding (arraybuilder-empty) (arraybuilder-add kept x))))) + +(: (-> (-> %a bool) (IArrayPart %a) (ArrayBuilder %a))) +(defun (filter-iarray-part keep? p) + (loop (:for x p) + (:when (keep? x)) + (:acc kept (folding (arraybuilder-empty) (arraybuilder-add kept x))))) + +(impl (ParallelMap (Array %a)) + (defun (par-map f a) + (def out (make-array (array-length a))) + (ignore (in-parts a (fun (p) (map-part-into f p out)))) + out)) + +(impl (ParallelFilter (Array %a)) + (defun (par-filter keep? a) + (builders->array (in-parts a (fun (p) (filter-part keep? p)))))) + +;; The array that the parts write is not seen by any other code, so it +;; becomes the immutable array with no copy. +(impl (ParallelMap (IArray %a)) + (defun (par-map f a) + (def out (make-array (iarray-length a))) + (ignore (in-parts a (fun (p) (map-iarray-part-into f p out)))) + (iarray-adopt out))) + +(impl (ParallelFilter (IArray %a)) + (defun (par-filter keep? a) + (iarray-adopt (builders->array (in-parts a (fun (p) (filter-iarray-part keep? p))))))) + +;; --- Hash maps and sets: the parts have no key in common, so a merge +;; shares their nodes. + +(impl (ParallelMap (Map %k %a)) + (defun (par-map f m) + (joined (in-parts m (fun (p) (map-map-values f p))) (fun (a b) (map-merge a b))))) + +(impl (ParallelFilter (Map %k %a)) + (defun (par-filter keep? m) + (joined (in-parts m (fun (p) (map-filter (fun (k v) (keep? v)) p))) + (fun (a b) (map-merge a b))))) + +(impl (ParallelFilter (HashSet %a)) + (defun (par-filter keep? s) + (in-parallel s (fun (p) (hashset-filter keep? p))))) + +;; --- Ordered maps and sets: the parts are in key order, so a concat joins +;; them by their leaves, under new internal nodes. It compares one pair of +;; keys at each join, and copies only small leaves, together with a +;; neighbour. + +(impl (ParallelMap (OrderedMap %k %a)) + (defun (par-map f m) + (orderedmap-concat (vec->array (in-parts m (fun (p) (orderedmap-map-values f p))))))) + +(impl (ParallelFilter (OrderedMap %k %a)) + (defun (par-filter keep? m) + (orderedmap-concat (vec->array (in-parts m (fun (p) (orderedmap-filter (fun (k v) (keep? v)) p))))))) + +(impl (ParallelFilter (OrderedSet %a)) + (defun (par-filter keep? s) + (orderedset-concat (vec->array (in-parts s (fun (p) (orderedset-filter keep? p))))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/ports.bjo — draining and streaming a port, its bytes, and byte output. +;; +;; An `InputPort` is bytes, as a Racket port is: `read-line` and `read-char` +;; decode UTF-8 from it, and `read-some`, `read-bytes` and `peek-bytes` take +;; the bytes themselves, on the same port and in any order. The byte reads are +;; CML events, so a read can be a branch of a `choose`. +;; +;; The invariant the port turns on, and the reason a read is safe inside a +;; `choose`: +;; +;; Every byte that leaves the operating system lands in the port's buffer. A +;; read never takes bytes from the underlying stream — it takes them from the +;; buffer, and only when it wins. +;; +;; `BjolangRuntime/BjoInputPort.cs` is where that is done. +;; +;; An `OutputPort` is bytes too: `write-string` and `write-char` write UTF-8 +;; to it, and `write-bytes!` writes bytes as they are, on the same port and in +;; any order. + +(export + ;; Draining + port->list port->vec port->lines + ;; Streaming + port->seq file->seq + port->stream + + ;; --- Bytes --------------------------------------------------------------- + + ;; The types. `Encoding` is the prelude's, its values `(text encodings)`'s. + ClrStream + + ;; Reading, as events. These are what go in a `choose`. + read-some-evt read-bytes-evt eof-evt + ;; Reading, for when there is nothing to choose between. + read-some read-bytes + ;; Looking without taking. + peek-bytes + + ;; Writing. + write-bytes! + + ;; The half-close. + shutdown! + + ;; Composition. + limited byte-pipe + + ;; Sources. + stream->input-port stream->output-port + + ;; Encodings. + reencode-input-port reencode-output-port) + +(import (std prelude)) +(import (std stream)) + +(:doc #:file "ports.bjodoc") + +;; Exception import. +(import/class + (Exception (: System.Exception))) + +;; --------------------------------------------------------------------------- +;; Draining +;; --------------------------------------------------------------------------- +;; +;; Drains items. + + +;; Drains a port. +(: (-> (-?-> InputPort %a) InputPort (List %a))) +(defun (port->list reader p) + (loop (:with i 1 1) + (:finish (port-eof? p)) + (:acc lst (listing (reader p))))) + + +(: (-> (-?-> InputPort %a) InputPort (Vec %a))) +(defun (port->vec reader p) + (loop (:with i 1 1) + (:finish (port-eof? p)) + (:acc vec (vecing (reader p))))) + +;; The lines left in `port`, ended as `#:mode` says (`Any` by default), as +;; Racket's `port->lines` takes a `#:line-mode`. +(: (-> InputPort (#:mode LineMode) (Vec string))) +(defun (port->lines port #:mode Any) + (port->vec (fun (p) (read-line p #:mode mode)) port)) + +;; --------------------------------------------------------------------------- +;; Streaming +;; --------------------------------------------------------------------------- + +;; Streams items from port as a Seq. +(: port->seq #:sync (-> (-> InputPort %a) InputPort (Seq %a))) +(defun (port->seq reader p) + (seql (:with i 0 0) + (:finish (port-eof? p)) + (:yield (reader p)))) + +;; --------------------------------------------------------------------------- +;; Streaming that suspends +;; --------------------------------------------------------------------------- + +;; The items `reader` reads from `p` until EOF, as a `Stream`. +;; +;; - Each item is read when the loop asks for it, so a loop left early leaves +;; the port just after the last item it took. +;; - What `reader` raises is raised in the loop. +;; - The port is not closed. +(: (-> (-bjo-> InputPort %a) InputPort (Stream %a))) +(defun (port->stream reader p) + (make-stream (bjoroutine (out) + (let go () + (unless (port-eof? p) + (emit! out (reader p)) + (go)))))) + +;; Streams file contents as Seq. +(: file->seq #:sync (-> (-> InputPort %a) string (Result Exception (Seq %a)))) +(defun (file->seq reader path) + (match (open-input-file path) + ((Ok probe) + ;; First we just check if the file exists + (close-input-port probe) + (Ok (file-read/seq reader path))) + ((Err e) (Err e)))) + +;; =========================================================================== +;; Bytes +;; =========================================================================== +;; +;; An input port owns its buffer, and a read is a cursor move inside that +;; buffer and nothing else — which is what makes a read safe inside a +;; `choose`, and why a refill that lands after its branch has already lost is +;; harmless rather than data loss. +;; +;; The port is a class in `BjolangRuntime/BjoInputPort.cs`, the way `Inbox` is +;; a class with `(std inbox)` over it, and its type, `InputPort`, is the +;; prelude's. What is here is the typed surface of its byte reads. +;; +;; This module still imports `(std prelude)` and nothing else, so it keeps its +;; place in `build_std.sh` above `(std inbox)`: the event vocabulary is all +;; prelude builtins, and the runtime class is reached by `import/extern`, which +;; is not a module dependency. + +(import/class + ;; The pipe object, which exists only so that `byte-pipe` can take its two + ;; halves off it. Not exported: a pipe *is* its two ports. + (BytePipe (: Bjolang.Runtime.BjoBytePipe)) + (ClrStream (: System.IO.Stream))) + +(import/extern + ;; Sources. + (clr-from-stream (: Bjolang.Runtime.BytePorts.FromStream (-> ClrStream InputPort))) + (clr-to-stream (: Bjolang.Runtime.OutputPorts.FromStream (-> ClrStream OutputPort))) + (clr-make-pipe (: Bjolang.Runtime.BytePorts.MakePipe (-> BytePipe))) + (clr-pipe-in (: Bjolang.Runtime.BytePorts.PipeInput (-> BytePipe InputPort))) + (clr-pipe-out (: Bjolang.Runtime.BytePorts.PipeOutput (-> BytePipe OutputPort))) + (clr-limited (: Bjolang.Runtime.BytePorts.Limited (-> InputPort int InputPort))) + + ;; The events. + (clr-read-some-evt (: Bjolang.Runtime.BytePorts.ReadSomeEvent + (-> InputPort (Event (Result Exception (Option (Array byte))))))) + (clr-read-bytes-evt (: Bjolang.Runtime.BytePorts.ReadBytesEvent + (-> InputPort int (Event (Result Exception (Option (Array byte))))))) + (clr-eof-evt (: Bjolang.Runtime.BytePorts.EofEvent (-> InputPort (Event Unit)))) + + ;; The questions that are not events, in sync/async pairs. + (clr-peek (: Bjolang.Runtime.BytePorts.PeekBytes + (-> InputPort int int (Option (Array byte))) #:blocking)) + (clr-peek-await (: Bjolang.Runtime.BytePorts.PeekBytesAsync + (-> InputPort int int (Option (Array byte))) #:async)) + + ;; Writing. + (clr-write-bytes (: Bjolang.Runtime.BjoOutputPort.WriteBytes + (-> OutputPort (Array byte) void) #:blocking)) + (clr-write-bytes-await (: Bjolang.Runtime.BjoOutputPort.WriteBytesAsync + (-> OutputPort (Array byte) void) #:async)) + (clr-shutdown (: Bjolang.Runtime.BjoOutputPort.Shutdown (-> OutputPort void) #:blocking)) + (clr-shutdown-await (: Bjolang.Runtime.BjoOutputPort.ShutdownAsync (-> OutputPort void) #:async)) + + ;; Encodings. + (clr-reencode (: Bjolang.Runtime.InputPorts.Reencode (-> InputPort Encoding InputPort))) + (clr-reencode-out (: Bjolang.Runtime.OutputPorts.Reencode (-> OutputPort Encoding OutputPort)))) + +;; --------------------------------------------------------------------------- +;; Reading, as events +;; --------------------------------------------------------------------------- +;; +;; These are what go in a `choose`, and they are real CML events: a branch that +;; loses consumes nothing, and the bytes stay in the buffer for the next reader. +;; They are also *ephemeral* — each `sync` asks the port a new question, so a +;; loop over one reads successive bytes rather than winning with a remembered +;; answer forever. That is exactly what `(task->event ...)` cannot do, which is +;; why a read is not one. +;; +;; WHY THE VALUE IS A `Result`. A stream can fail, and the CML resume path +;; carries a value rather than a raise — so a failure that lands while a fiber +;; is parked in a `choose` has to arrive as one. `promise-join`, `blocking` and +;; `task->event` all answer a `Result` for the same reason. `read-some` and +;; `read-bytes` below unwrap it and raise on the fiber's own stack, which is +;; what an ordinary read wants. + +;; Whatever is available, never an empty array, and `None` once there is no more. +(: (-> InputPort (Event (Result Exception (Option (Array byte)))))) +(defun (read-some-evt p) (clr-read-some-evt p)) + +;; Exactly `n` bytes. At end of input the short remainder comes back as +;; `(Some ...)` and the call after it as `None`, so the two are separate facts +;; rather than one ambiguous answer. +;; +;; `n` must be at least 1. +(: (-> InputPort int (Event (Result Exception (Option (Array byte)))))) +(defun (read-bytes-evt p n) (clr-read-bytes-evt p n)) + +;; Fires once the port is finished — cleanly, or because it failed. It says only +;; that, which is what makes it the thing to wait on for "the other end has gone +;; away"; a read beside it is what says which of the two happened. +(: (-> InputPort (Event Unit))) +(defun (eof-evt p) (clr-eof-evt p)) + +;; --------------------------------------------------------------------------- +;; Reading, for when there is nothing to choose between +;; --------------------------------------------------------------------------- +;; +;; Defined by syncing the events above: conveniences, not a second +;; implementation. The `defbjouble` is what picks the right kind of wait — +;; `sync/blocking` parks the calling thread for an ordinary `defun`, `sync` +;; suspends the fiber — and the raise happens after either of them returns, on +;; the caller's own stack. + +(: (-> (Result Exception (Option (Array byte))) (Option (Array byte)))) +(defun (byte-read-or-raise r) + (match r + ((Ok bytes) bytes) + ((Err e) (raise e)))) + +;; Whatever is available, waiting for at least one byte. `None` at end of input. +(: (-> InputPort (Option (Array byte)))) +(defbjouble (read-some p) + (#:sync (byte-read-or-raise (sync/blocking (clr-read-some-evt p)))) + (#:bjo (byte-read-or-raise (sync (clr-read-some-evt p))))) + +;; Exactly `n` bytes, or the short remainder at end of input, or `None`. +(: (-> InputPort int (Option (Array byte)))) +(defbjouble (read-bytes p n) + (#:sync (byte-read-or-raise (sync/blocking (clr-read-bytes-evt p n)))) + (#:bjo (byte-read-or-raise (sync (clr-read-bytes-evt p n))))) + +;; --------------------------------------------------------------------------- +;; Looking without taking +;; --------------------------------------------------------------------------- + +;; `skip` bytes past the cursor, then `count` of them, leaving the cursor where +;; it was. Short at end of input, and `None` when there is nothing at all beyond +;; the skip. +;; +;; This is the mechanism a read is, with the cursor move left out — which is why +;; "peek, decide, then read" loses nothing: the sniffed bytes are still in the +;; buffer, for `read-bytes` and `read-line` alike. +(: (-> InputPort int int (Option (Array byte)))) +(defbjouble (peek-bytes p skip count) + (#:sync (clr-peek p skip count)) + (#:bjo (clr-peek-await p skip count))) + +;; --------------------------------------------------------------------------- +;; Writing +;; --------------------------------------------------------------------------- +;; +;; Buffered, so most writes are a copy. A write that fills the buffer drains it, +;; and that is a syscall — which is why these have suspending twins at all. + +(: (-> OutputPort (Array byte) void)) +(defbjouble (write-bytes! p bytes) + (#:sync (clr-write-bytes p bytes)) + (#:bjo (clr-write-bytes-await p bytes))) + + +;; `flush-port` and `close-output-port` are the prelude's: there is one kind of +;; output port. + +;; --------------------------------------------------------------------------- +;; The half-close +;; --------------------------------------------------------------------------- + +;; Ends the write side and leaves the read side open. +;; +;; A protocol needs this and `close` cannot say it: "I have finished speaking, +;; now answer me" is a half-close, not a closed connection. Everything buffered +;; goes out first, and a write after it is refused rather than dropped. A stream +;; with no half-close of its own gets the flush and nothing more, which is the +;; most that can honestly be done for a file. +(: (-> OutputPort void)) +(defbjouble (shutdown! p) + (#:sync (clr-shutdown p)) + (#:bjo (clr-shutdown-await p))) + +;; --------------------------------------------------------------------------- +;; Composition +;; --------------------------------------------------------------------------- + +;; At most `n` bytes of `p`, and then end of input. Once the bound is reached, +;; `p` itself is positioned exactly where the limit ended — the limited port +;; reads *through* `p` rather than around it, so nothing it buffered is lost or +;; double-counted. +;; +;; The result owns nothing: closing it does not close `p`. +(: (-> InputPort int InputPort)) +(defun (limited p n) (clr-limited p n)) + +;; A port pair with nothing underneath: what is written to the output can be +;; read from the input. +;; +;; Unbounded and in memory, which is what a string port is one layer up — the GC +;; prices it, nothing registers it on a scope, and a write never waits. Closing +;; the output half, or `shutdown!` on it, is what the input half sees as end of +;; input. +(: (-> (Tuple InputPort OutputPort))) +(defun (byte-pipe) + (def pipe (clr-make-pipe)) + (Tuple (clr-pipe-in pipe) (clr-pipe-out pipe))) + +;; --------------------------------------------------------------------------- +;; Sources +;; --------------------------------------------------------------------------- +;; +;; A file is opened with the prelude's `open-input-file` and `open-output-file`: +;; there is one kind of input port and one kind of output port, and a file port +;; takes bytes as readily as text. + +;; A port over a `ClrStream` somebody else handed over — a `NetworkStream`, a +;; request body, a `MemoryStream`. This pair is what `(std net)` and a web +;; framework reach for, and it is the only way into this module from outside it. +;; +;; Owned like a file, because a `ClrStream` is a handle in every case that matters. +(: (-> ClrStream InputPort)) +(defun (stream->input-port s) + (ignore (current-scope)) + (clr-from-stream s)) + +(: (-> ClrStream OutputPort)) +(defun (stream->output-port s) + (ignore (current-scope)) + (clr-to-stream s)) + +;; --------------------------------------------------------------------------- +;; Encodings +;; --------------------------------------------------------------------------- + +;; A port whose bytes are `p`'s read as `encoding` and re-encoded as UTF-8, as +;; Racket's `reencode-input-port` makes one: the way to read text that is not +;; UTF-8, since every port's text reads decode UTF-8. +;; +;; Reads *through* `p`, so whatever `p` had buffered — from a `peek-bytes`, or +;; a refill that overshot — is not skipped: sniff the first bytes, then +;; re-encode the rest. Closing the new port does not close `p`. +;; +;; `encoding` is any value from `(text encodings)`. Bytes that do not decode +;; read as U+FFFD, as they do on every port; `bytes->string` is the strict +;; one. A byte order mark is not treated specially. +(: (-> InputPort Encoding InputPort)) +(defun (reencode-input-port p encoding) + (clr-reencode p encoding)) + +;; Output's counterpart, as Racket's `reencode-output-port` makes one: what is +;; written to the new port is UTF-8 re-encoded as `encoding` on its way into +;; `p`. A character the encoding lacks is written as its replacement. Closing +;; the new port flushes it into `p` and leaves `p` open. +(: (-> OutputPort Encoding OutputPort)) +(defun (reencode-output-port p encoding) + (clr-reencode-out p encoding)) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/prelude.bjo +;; Standard library entry point +;; Macros defined here CANNOT be used in this file. Those of (std core-macros), +;; such as cond, can. +(export + ;; Types. + Map MapBuilder TransientMap MapCursor TransientVec + Range RangeBy + ArraySlice ReverseArraySlice VecSlice ReverseVecSlice + VecBackCursor StringWalk ListTails PortReads UpFrom DownFrom + Enumerated Zipped Cycled Pairwise Repeated Chained + Listing ListingReverse Vecing VecingSmall Arraying Mapping Stringing Summing Counting Folding + Maximizing Minimizing MaximizingPairs MinimizingPairs Tallying GroupingPairs + VecingInto ListingInto ListingReverseInto MappingInto + InputPort OutputPort Exception Encoding + + ;; Iterable + iterable-start iterable-done? iterable-current iterable-next iterable-end iterable-cleanup + fold map filter take drop take-while drop-while + seq-enumerate flat-map any? all? find for-each ->seq + ->list ->vec count first ->map + ;; Adapters + enumerate zip in-cycle pairwise in-repeat chain + ;; Cursor. + cursor-start cursor-end cursor-end? cursor-ref cursor-next cursor-prev + ;; Collector + collector-init collector-step collector-finish + ;; Vector forms + contiguous? contiguous-values contiguous-valid + block-count block-grow block-span block-none! + ;; `#:into` on a collector + collector-into + ;; Collection + length empty? + ;; Addable! and Addable + add! add-all! + add add-all + ;; Discard. + ignore + ;; Range + range range-by + ;; Iterating part of a collection, or backwards, or without one. + ;; Written in iterators.protobjo, which this file includes. + in-array in-reverse-array in-vec in-reverse-vec + over-list in-port up-from down-from + int-max int-min + ;; The cursors those walk with, named because an inlined impl body reaches them + vec-back-cursor vec-back-cursor-done? vec-back-cursor-current vec-back-cursor-next + string-walk string-walk-done? string-walk-current string-walk-next + + print println ->str list->str + ;; Written and Displayed + written write-into displayed display-into + elements-print-into entries-print-into + list-append + + ;; List operations. + list-map list-filter list-foldl list-foldr list-for-each list-unfold + list-contains? + + ;; Vec operations. + vec-take vec-drop vec-reverse vec->array vec-try-pop vec-try-pop-first + vec-contains? array-contains? iarray-contains? transientvec-contains? + vec-add vec-add-first vec-insert vec-remove-at vec-slice vec-split + vec-map vec-filter vec-fold vec-reduce + vec-for-each vec-iter + + ;; Seq operations. + seq-fold seq-for-each + + ;; Maps. + map-empty map-ref map-ref-or map-try-ref + map-set map-add-new map-try-add-new map-set-all map-remove map-remove-all + map-alter map-update + map-contains? map-has-key? map-length map-empty? map-clear + map-keys map-values map-merge map-merge-with + ;; Walking and rebuilding + map-map map-map-values map-filter map-fold + map-for-each map-iter map-any? map-find map-split + ;; The cursor `Iterable` walks with + map-cursor map-cursor-done? map-cursor-current map-cursor-next + ;; Conversions. + list->map list->vec map->list vec->list vec->map map->vec seq->map map->seq + map->str map-entry->str + ;; Builders. + mapbuilder-empty map->mapbuilder mapbuilder-add! mapbuilder-add-all! + mapbuilder-length mapbuilder->map + ;; The vec's transient. + transientvec-empty vec->transientvec transientvec->vec + transientvec-length transientvec-empty? transientvec-ref transientvec-set! + transientvec-add! transientvec-add-all! transientvec-pop! transientvec-try-pop! + transientvec-add-first! transientvec-pop-first! transientvec-try-pop-first! + ;; The immutable array. + IArray + iarray-empty iarray-length iarray-empty? iarray-ref iarray-try-ref + iarray-set iarray-add iarray-map + subarray->iarray array->iarray iarray->array seq->iarray iarray->seq + vec->iarray iarray->vec list->iarray iarray->list + ;; Arrays. + array-copy! + ;; Spans. + Span RoSpan + array->span array->ro-span span->ro-span + span-length span-ref span-set! span-slice span->array span-> + ro-span-length ro-span-ref ro-span-slice ro-span->array ro-span-> + ;; Transients. + transientmap-empty map->transientmap transientmap->map + transientmap-set! transientmap-set-all! transientmap-remove! + transientmap-alter! transientmap-update! + transientmap-remove-all! transientmap-contains? transientmap-has-key? transientmap-ref + transientmap-ref-or transientmap-try-ref transientmap-length + transientmap-empty? transientmap-filter! + + ;; Results + ok? err? result-ref result-ref-or result-map result-map-err result->option + option-map option-filter option-or option->result + ;; Pairs + fst snd + + ;; Box + Box + make-box box-ref box-set! + + ;; Refable, Settable, Settable!, Derefable, Derefable! + ref try-ref + ref-set + ref-set! + deref try-deref + deref-set! + + ;; Strings + str string-append string-length + string-contains? string-starts-with? string-ends-with? string-replace + string-upcase string-downcase + string-trim string-trim-left string-trim-right + string-trim/slice string-trim-left/slice string-trim-right/slice string-copy + string-pad-left string-pad-right + string-split string-split/slice string-join + string-split-once string-split-once/slice + ;; Taking and dropping characters. + string-take string-drop string-take-right string-drop-right + string-take/slice string-drop/slice string-take-right/slice string-drop-right/slice + string-cursor-forward string-cursor-back + ;; Characters. + char=? char? char<=? char>=? + char-ci=? char-ci? char-ci<=? char-ci>=? + ;; String cursors. + string-cursor=? string-cursor? string-cursor<=? string-cursor>=? + string-fold string-fold-right string-for-each string-find + string->list list->string string->vec string-reverse + ;; Conversions + int->string long->string double->string byte->string number->string + string->int string->long string->double + decimal->string string->decimal bigint->string string->bigint + int128->string string->int128 uint128->string string->uint128 + bytes->string string->bytes bytes-utf-8? + Utf8Chunk bytes->utf8-chunk utf8-chunk-length utf8-chunk-valid-end utf8-chunk-substring + ;; Math. + sqrt + ;; Files + file-read-text file-write-text file-append-text file-exists? file-delete file-move + file-info file-copy file-read-bytes file-write-bytes file-append-bytes + file-read-lines file-write-lines + file-read/seq file-read-lines/seq + ;; Paths + path-absolute path-combine path-filename path-file-extension + ;; Directories + directory-exists? directory-create directory-delete directory-delete-tree + directory-files directory-subdirectories directory-walk + ;; The environment + get-environment-variable set-environment-variable! + current-directory set-current-directory! + ;; Open ports. + open-input-file open-input-string open-output-file open-output-string + ;; Ports — closing + close-input-port close-output-port flush-port + ;; Ports — the openers past the FS effect + open-input-file/raw open-output-file/raw WriteMode + call-with-input-file call-with-output-file + ;; The FS effect: its operations, their cells, and the defaults + FsKind FsInfo MoveMode + open-read-stream op/open-read-stream open-read-stream/raw + open-write-stream op/open-write-stream open-write-stream/raw + fs-info op/fs-info fs-info/raw + fs-list op/fs-list fs-list/raw + fs-create-directory op/fs-create-directory fs-create-directory/raw + fs-delete-file op/fs-delete-file fs-delete-file/raw + fs-delete-directory op/fs-delete-directory fs-delete-directory/raw + fs-move op/fs-move fs-move/raw + ;; Ports — input + port-eof? read-line read-line/opt LineMode read-char read-char/opt + peek-char peek-char/opt read-all port-fold + ;; Ports — output + write-string write-char writeln get-output-string get-output-bytes + + ;; Durations, which every timer takes. + Duration ticks->duration duration-ticks milliseconds seconds minutes hours + timeout duration->timer-ms + + ;; Effects the standard library performs. The `op/` name of each is the + ;; parameter `with-handler` binds, and is exported for that reason. + warn op/warn + log op/log + getenv op/getenv + + ;; The clock + Clock make-clock system-clock current-clock monotonic-ms monotonic-ns + + ;; The dynamic environment, taken here and run in on another thread. + DynamicEnvironment current-dynamic-environment call-with-dynamic-environment + + promise-join/sync panic! + + ;; What a grammar's checked literal calls. See (def/grammar ...). + grammar-checked) + + +(import/extern (clr-panic (: BjolangRuntime.Panic (-> string int %a)))) + + +(: (-> string (#:exit-code int) %a)) +(defun (panic! msg #:exit-code 1) + (clr-panic msg exit-code)) + +;; The value of a grammar's check on a literal. The compiler ran the same +;; check on the same literal and refused an Err, so the Err case is not +;; reached unless the check gives different results for one value. +(: (-> (Result string %a) %a)) +(defun (grammar-checked r) + (match r + ((Ok v) v) + ((Err e) (panic! (str "a check of a grammar gave Err for a literal it gave Ok for when compiled: " e))))) + + +;; The `written` fallback, invariant. `System.Convert.ToString` reads the ambient +;; culture, which is how a `short` or a `float` — types with no conversion of +;; their own below — would print differently on a differently configured +;; machine. Generic, because a union case that is `null` at run time is named +;; by the type the value has, and the value alone does not say it. +(import/extern + (clr-show (: Bjolang.Runtime.BjoNum.Show (-> %a string)))) + +;; The docs: the prelude's own definitions and the builtins beside them, by +;; area. The builtins are documented here because every program gets them with +;; the prelude, and their docs are published in its assembly. +(:doc #:file "prelude.bjodoc") +(:doc #:file "prelude-core.bjodoc") +(:doc #:file "prelude-seqs.bjodoc") +(:doc #:file "prelude-lists.bjodoc") +(:doc #:file "prelude-vecs.bjodoc") +(:doc #:file "prelude-maps.bjodoc") +(:doc #:file "prelude-strings.bjodoc") +(:doc #:file "prelude-chars.bjodoc") +(:doc #:file "prelude-files.bjodoc") +(:doc #:file "prelude-ports.bjodoc") +(:doc #:file "prelude-concurrency.bjodoc") +(:doc #:file "prelude-syntax.bjodoc") + +;; Before the includes, which use them: cond and the rest of the macros the +;; prelude needs itself. +(import (std core-macros)) +(re-export cond) + +;; First, and before the includes. = is a trait and needs to be imported +(import (std eq)) +;; re-export ord and eq traits, and not +(re-export = eq-hash not + compare less? greater? at-most? at-least? + least greatest list-sort vec-sort vec-sort-by array-sort! array-sort-by! + list-sort-unstable vec-sort-unstable array-sort-unstable! + list-min list-max) + + +;; List append. +(: (-> (List %a) (List %a) #:rest (List %a) (List %a))) +(defun (list-append xs ys #:rest more) + ;; Each list but the last is copied, once, and the last is shared: the + ;; lists are joined from the right. + (def n (array-length more)) + (if (= n 0) + (list-foldr Cons ys xs) + (loop (:for i (down-from (- n 2) #:to 0)) + (:acc acc (folding (array-ref more (- n 1)) + (list-foldr Cons acc (array-ref more i)))) + => (list-foldr Cons (list-foldr Cons acc ys) xs)))) + +;; --------------------------------------------------------------------------- +;; Written and Displayed +;; --------------------------------------------------------------------------- +;; `written` is a value as the reader reads it, as Scheme's `write` writes +;; it: a string in quotes, a char as `#\a`, a long as `21L`. `displayed` is +;; the value for a person to read, as `display` writes it: a string is its +;; text. Every type is written; a type is displayed as it is written unless +;; it says otherwise. +;; +;; Each trait has two methods. `write-into` adds the value to a builder, and +;; is what a container calls on its elements, so that a nested value is +;; written into one buffer. `written` answers a string, through `write-into` +;; and a builder the thread keeps. A type whose text it has at hand, such as +;; a number, gives `written` too, and no builder is used. +(import/extern + (clr-builder-rent (: Bjolang.Runtime.BjoShow.Rent)) + (clr-builder-finish (: Bjolang.Runtime.BjoShow.Finish)) + (clr-write-string-into (: Bjolang.Runtime.BjoShow.WriteStringInto)) + (clr-written-char (: Bjolang.Runtime.BjoShow.WrittenChar)) + (clr-written-double (: Bjolang.Runtime.BjoShow.WrittenDouble)) + (clr-written-number (: Bjolang.Runtime.BjoShow.WrittenNumber (-> %a string string)))) + +(def/trait (Written %a) + (: write-into (-> %a StringBuilder void)) + (: written (-> %a string)) + (defun (written x) + (def b (clr-builder-rent)) + (write-into x b) + (clr-builder-finish b))) + +(def/trait (Displayed %a) + (where (Written %a)) + (: display-into (-> %a StringBuilder void)) + (: displayed (-> %a string)) + (defun (displayed x) + (def b (clr-builder-rent)) + (display-into x b) + (clr-builder-finish b))) + +;; A .NET type is written as it writes itself, in the invariant culture. +(impl (Written %a) + (defun (written x) (clr-show x)) + (defun (write-into x b) (stringbuilder-add-string! b (clr-show x)))) + +;; Every type is displayed as it is written, unless it says otherwise. +(impl (Displayed %a) + (defun (displayed x) (written x)) + (defun (display-into x b) (write-into x b))) + +;; A function, as Scheme writes one. No blanket reaches a function type, so it +;; is written and displayed by these, inside a container as well. +(impl (Written Procedure) + (defun (written f) "#") + (defun (write-into f b) (stringbuilder-add-string! b "#"))) + +(impl (Displayed Procedure) + (defun (displayed f) "#") + (defun (display-into f b) (stringbuilder-add-string! b "#"))) + +;; Each number with the suffix that gives its type back, and displayed +;; without it. +(impl (Written int) + (defun (written n) (int->string n)) + (defun (write-into n b) (stringbuilder-add-string! b (int->string n)))) + +(impl (Written long) + (defun (written n) (clr-written-number n "L")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed long) + (defun (displayed n) (long->string n)) + (defun (display-into n b) (stringbuilder-add-long! b n))) + +(impl (Written byte) + (defun (written n) (clr-written-number n "uy")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed byte) + (defun (displayed n) (byte->string n)) + (defun (display-into n b) (stringbuilder-add-string! b (byte->string n)))) + +(impl (Written short) + (defun (written n) (clr-written-number n "s")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed short) + (defun (displayed n) (clr-show n)) + (defun (display-into n b) (stringbuilder-add-string! b (clr-show n)))) + +(impl (Written ushort) + (defun (written n) (clr-written-number n "us")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed ushort) + (defun (displayed n) (clr-show n)) + (defun (display-into n b) (stringbuilder-add-string! b (clr-show n)))) + +(impl (Written uint) + (defun (written n) (clr-written-number n "u")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed uint) + (defun (displayed n) (clr-show n)) + (defun (display-into n b) (stringbuilder-add-string! b (clr-show n)))) + +(impl (Written ulong) + (defun (written n) (clr-written-number n "UL")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed ulong) + (defun (displayed n) (clr-show n)) + (defun (display-into n b) (stringbuilder-add-string! b (clr-show n)))) + +(impl (Written decimal) + (defun (written n) (clr-written-number n "M")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed decimal) + (defun (displayed n) (clr-show n)) + (defun (display-into n b) (stringbuilder-add-string! b (clr-show n)))) + +(impl (Written bigint) + (defun (written n) (clr-written-number n "I")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed bigint) + (defun (displayed n) (clr-show n)) + (defun (display-into n b) (stringbuilder-add-string! b (clr-show n)))) + +;; A float, an int128 and a uint128 have no suffix, so each is written with +;; its type, as a value with its type written is read: `(: 2.5 float)`. +(import/extern + (clr-written-single (: Bjolang.Runtime.BjoShow.WrittenSingle))) + +(impl (Written float) + (defun (written n) (str "(: " (clr-written-single n) " float)")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed float) + (defun (displayed n) (clr-show n)) + (defun (display-into n b) (stringbuilder-add-string! b (clr-show n)))) + +(import/extern + (clr-written-rational (: Bjolang.Runtime.BjoRational.Written (-> rational string)))) + +(impl (Written rational) + (defun (written n) (clr-written-rational n)) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed rational) + (defun (displayed n) (clr-show n)) + (defun (display-into n b) (stringbuilder-add-string! b (clr-show n)))) + +(impl (Written int128) + (defun (written n) (str "(: " (clr-written-number n "") " int128)")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed int128) + (defun (displayed n) (clr-show n)) + (defun (display-into n b) (stringbuilder-add-string! b (clr-show n)))) + +(impl (Written uint128) + (defun (written n) (str "(: " (clr-written-number n "") " uint128)")) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(impl (Displayed uint128) + (defun (displayed n) (clr-show n)) + (defun (display-into n b) (stringbuilder-add-string! b (clr-show n)))) + +;; A double is written with a point, so that it is not read back as an int. +(impl (Written double) + (defun (written n) (clr-written-double n)) + (defun (write-into n b) (stringbuilder-add-string! b (clr-written-double n)))) + +(impl (Displayed double) + (defun (displayed n) (double->string n)) + (defun (display-into n b) (stringbuilder-add-double! b n))) + +;; As the language writes a boolean, not as .NET does. +(impl (Written bool) + (defun (written b) (if b "#t" "#f")) + (defun (write-into x b) (stringbuilder-add-string! b (if x "#t" "#f")))) + +;; A string in quotes, and displayed as its text. +(impl (Written string) + (defun (write-into s b) (clr-write-string-into b s))) + +(impl (Displayed string) + (defun (displayed s) s) + (defun (display-into s b) (stringbuilder-add-string! b s))) + +(impl (Written Char) + (defun (written c) (clr-written-char c)) + (defun (write-into c b) (stringbuilder-add-string! b (clr-written-char c)))) + +(impl (Displayed Char) + (defun (displayed c) (char->string c)) + (defun (display-into c b) (stringbuilder-add! b c))) + +(impl (Written Keyword) + (defun (written k) (str "#:" (keyword->string k))) + (defun (write-into k b) + (stringbuilder-add-string! b "#:") + (stringbuilder-add-string! b (keyword->string k)))) + +(impl (Displayed Keyword) + (defun (displayed k) (keyword->string k)) + (defun (display-into k b) (stringbuilder-add-string! b (keyword->string k)))) + +(impl (Written Symbol) + (defun (written s) (symbol->string s)) + (defun (write-into s b) (stringbuilder-add-string! b (symbol->string s)))) + +;; Container formats are further down. + +;; --------------------------------------------------------------------------- +;; ->str +;; --------------------------------------------------------------------------- +;; What `displayed` was called before there was `written`. An implementation +;; of its own is still called by `->str`, but not by `displayed`, `println` +;; or an interpolation, which is why it is deprecated. +(def/trait (->str %a) + (where (Displayed %a)) + (: ->str (-> %a string))) + +(impl (->str %a) + (defun (->str x) (displayed x))) + +(:deprecated ->str "use `displayed`, and implement `Displayed` or `Written`") + + +;; --------------------------------------------------------------------------- +;; Discard +;; --------------------------------------------------------------------------- +;; Discard value. +(def/trait (Discard %a) + (: ignore (-> %a void))) + +(impl (Discard %a) + (defun (ignore x) unit)) + +;; Detach promise. +(impl (Discard (Promise %a)) + (defun (ignore p) (detach p))) + +;; Writes a value as `displayed` gives it. +(: (-> %a void)) +(defun (print x) + (display (displayed x))) + +(: (-> %a void)) +(defun (println x) + (displayln (displayed x))) + + +;; --------------------------------------------------------------------------- +;; TransientVec — a vec changed in place +;; --------------------------------------------------------------------------- +;; The builder a `VecBuilder` is, under its own name, with the ends a builder +;; has no use for. +(import/class + ((TransientVec %a) (: Bjolang.Runtime.Rrb.RrbTransient))) + +(import/extern + (transientvec-empty (: Bjolang.Runtime.Rrb.TransientVecModule.Empty (-> (TransientVec %a)))) + (vec->transientvec (: Bjolang.Runtime.Rrb.TransientVecModule.FromList (-> (Vec %a) (TransientVec %a)))) + (transientvec->vec (: Bjolang.Runtime.Rrb.TransientVecModule.ToList (-> (TransientVec %a) (Vec %a)))) + (transientvec-length (: Bjolang.Runtime.Rrb.TransientVecModule.Count (-> (TransientVec %a) int))) + (transientvec-empty? (: Bjolang.Runtime.Rrb.TransientVecModule.IsEmpty (-> (TransientVec %a) bool))) + (transientvec-ref (: Bjolang.Runtime.Rrb.TransientVecModule.Ref (-> (TransientVec %a) int %a))) + (transientvec-set! (: Bjolang.Runtime.Rrb.TransientVecModule.Set (-> (TransientVec %a) int %a void))) + (transientvec-add! (: Bjolang.Runtime.Rrb.TransientVecModule.Add (-> (TransientVec %a) %a void))) + (transientvec-add-all! (: Bjolang.Runtime.Rrb.TransientVecModule.AddRange (-> (TransientVec %a) (Seq %a) void))) + (transientvec-add-first! (: Bjolang.Runtime.Rrb.TransientVecModule.Prepend (-> (TransientVec %a) %a void))) + (transientvec-pop! (: Bjolang.Runtime.Rrb.TransientVecModule.Pop (-> (TransientVec %a) %a))) + (transientvec-pop-first! (: Bjolang.Runtime.Rrb.TransientVecModule.PopFirst (-> (TransientVec %a) %a)))) + +;; The last element, removed, or `None` when the transient is empty. `None` +;; leaves the transient as it was. +(: (-> (TransientVec %a) (Option %a))) +(defun (transientvec-try-pop! t) + (if (transientvec-empty? t) None (Some (transientvec-pop! t)))) + +;; The first element, removed, or `None` when the transient is empty. `None` +;; leaves the transient as it was. +(: (-> (TransientVec %a) (Option %a))) +(defun (transientvec-try-pop-first! t) + (if (transientvec-empty? t) None (Some (transientvec-pop-first! t)))) + +;; --------------------------------------------------------------------------- +;; IArray — the immutable array +;; --------------------------------------------------------------------------- +;; A struct around .NET's `ImmutableArray`: one array that is not written +;; after it is made. Indexing is an array read, and the value is one reference +;; wide. Every change copies the whole array, so it is for values that are +;; built once and then read. `Vec` is the one to change. The impls are with +;; those of `Array`. Why it is a wrapper is in `BjoIArray.cs`. +(import/class + ((IArray %a) (: Bjolang.Runtime.IArray))) + +(import/extern + (iarray-empty (: Bjolang.Runtime.IArrayModule.Empty (-> (IArray %a)))) + (iarray-length (: Bjolang.Runtime.IArrayModule.Count (-> (IArray %a) int))) + (iarray-empty? (: Bjolang.Runtime.IArrayModule.IsEmpty (-> (IArray %a) bool))) + ;; Throws when the index is out of range. `iarray-try-ref` is the checked one. + (iarray-ref (: Bjolang.Runtime.IArrayModule.Get (-> (IArray %a) int %a))) + (iarray-try-ref (: Bjolang.Runtime.IArrayModule.TryGet (-> (IArray %a) int (out %a) (Option %a)))) + ;; A copy with one element replaced, or one added at the end. + (iarray-set (: Bjolang.Runtime.IArrayModule.SetItem (-> (IArray %a) int %a (IArray %a)))) + (iarray-add (: Bjolang.Runtime.IArrayModule.Add (-> (IArray %a) %a (IArray %a)))) + (iarray-map (: Bjolang.Runtime.IArrayModule.Map (-> (-> %a %b) (IArray %a) (IArray %b)))) + ;; The array, start and count, as `subarray->vec` takes them. The elements + ;; are copied, so the array can be written again. + (subarray->iarray (: Bjolang.Runtime.IArrayModule.FromSubarray (-> (Array %a) int int (IArray %a)))) + (array->iarray (: Bjolang.Runtime.IArrayModule.FromArray (-> (Array %a) (IArray %a)))) + ;; A new array, which the caller can write. + (iarray->array (: Bjolang.Runtime.IArrayModule.ToArray (-> (IArray %a) (Array %a)))) + (seq->iarray (: Bjolang.Runtime.IArrayModule.FromEnumerable (-> (Seq %a) (IArray %a)))) + (iarray->seq (: Bjolang.Runtime.IArrayModule.AsEnumerable (-> (IArray %a) (Seq %a))))) + +(: (-> (Vec %a) (IArray %a))) +(defun (vec->iarray v) (seq->iarray (vec->seq v))) + +(: (-> (List %a) (IArray %a))) +(defun (list->iarray xs) (seq->iarray (list->seq xs))) + +;; --------------------------------------------------------------------------- +;; Arrays +;; --------------------------------------------------------------------------- +;; The other array operations (`make-array`, `array-ref`, `array-resize` ...) +;; are compiler builtins. +(import/extern + (array-copy! (: Bjolang.Runtime.BjoArray.Copy (-> (Array %a) int (Array %a) int int void)))) + +;; --------------------------------------------------------------------------- +;; Spans +;; --------------------------------------------------------------------------- +;; A view of part of an array, which is valid only while the call that made +;; it runs. `SpanCheck` refuses one where it could outlive that call. A +;; `Span` and a `RoSpan` have separate functions, because neither converts to +;; the other implicitly; `span->ro-span` converts explicitly. +(import/class + ((Span %a) (: System.Span)) + ((RoSpan %a) (: System.ReadOnlySpan))) + +(import/extern + (clr-array->span (: Bjolang.Runtime.Spans.FromArray (-> (Array %a) int int (Span %a)))) + (clr-array->ro-span (: Bjolang.Runtime.Spans.ReadOnlyFromArray (-> (Array %a) int int (RoSpan %a)))) + (clr-span-slice (: Bjolang.Runtime.Spans.Slice (-> (Span %a) int int (Span %a)))) + (clr-ro-span-slice (: Bjolang.Runtime.Spans.Slice (-> (RoSpan %a) int int (RoSpan %a)))) + (span-length (: Bjolang.Runtime.Spans.Length (-> (Span %a) int))) + (span-ref (: Bjolang.Runtime.Spans.Ref (-> (Span %a) int %a))) + (span-set! (: Bjolang.Runtime.Spans.Set (-> (Span %a) int %a void))) + (span->array (: Bjolang.Runtime.Spans.ToArray (-> (Span %a) (Array %a)))) + (span->ro-span (: Bjolang.Runtime.Spans.ReadOnly (-> (Span %a) (RoSpan %a)))) + (ro-span-length (: Bjolang.Runtime.Spans.Length (-> (RoSpan %a) int))) + (ro-span-ref (: Bjolang.Runtime.Spans.Ref (-> (RoSpan %a) int %a))) + (ro-span->array (: Bjolang.Runtime.Spans.ToArray (-> (RoSpan %a) (Array %a)))) + (clr-span-> (: Bjolang.Runtime.Spans.As (-> (Span %a) (Span %b)))) + (clr-ro-span-> (: Bjolang.Runtime.Spans.As (-> (RoSpan %a) (RoSpan %b))))) + +;; The same memory as numbers of another type, as a (Span byte) as a +;; (Span long). The use gives the type. Nothing is copied: a write through +;; one span is a write through the other, and a read-only span gives a +;; read-only one. Only plain numbers: the integers, float and double, which +;; have no references in them, and whose every bit pattern is a value. A +;; decimal or a bigint, and a length in bytes that is not a whole number of +;; the new elements, is a panic. +(: (-> (Span %a) (Span %b)) (where (Num %a) (Num %b))) +(defun (span-> s) (clr-span-> s)) + +(: (-> (RoSpan %a) (RoSpan %b)) (where (Num %a) (Num %b))) +(defun (ro-span-> s) (clr-ro-span-> s)) + +(: (-> (Array %a) (#:from int) (#:to int) (Span %a))) +(defun (array->span a #:from 0 #:to (array-length a)) + (clr-array->span a from to)) + +(: (-> (Array %a) (#:from int) (#:to int) (RoSpan %a))) +(defun (array->ro-span a #:from 0 #:to (array-length a)) + (clr-array->ro-span a from to)) + +(: (-> (Span %a) (#:from int) (#:to int) (Span %a))) +(defun (span-slice s #:from 0 #:to (span-length s)) + (clr-span-slice s from to)) + +(: (-> (RoSpan %a) (#:from int) (#:to int) (RoSpan %a))) +(defun (ro-span-slice s #:from 0 #:to (ro-span-length s)) + (clr-ro-span-slice s from to)) + +;; --------------------------------------------------------------------------- +;; Map — the unordered map +;; --------------------------------------------------------------------------- +;; +;; A hash trie written in Bjolang, over the C# nodes of +;; `Bjolang.Runtime.HashTries`. Included rather than imported: the prelude +;; defines `Map`, and a module of its own would have to import the prelude +;; that is still being compiled. +(include "hashmap.protobjo") + +(: (-> (Map %k %v) %k bool)) +(defun (map-has-key? m key) (map-contains? m key)) + +(: (-> (TransientMap %k %v) %k bool)) +(defun (transientmap-has-key? m key) (transientmap-contains? m key)) + +;; `map-add-new` with the failure as a value: `(Err v)` carries the value the +;; key already has. +(: (-> (Map %k %v) %k %v (Result %v (Map %k %v)))) +(defun (map-try-add-new m k v) + (match (map-try-ref m k) + ((Some old) (Err old)) + (None (Ok (map-set m k v))))) + +;; Two maps are equal when they hold the same entries. + +(impl (Eq (Map %k %v)) + (where (Eq %k) (Eq %v)) + (defun (= a b) + (and (= (map-length a) (map-length b)) + (map-iter (fun (k v) + (match (map-try-ref b k) + ((Some w) (= v w)) + (None #f))) + a))) + ;; Hash map. + (defun (eq-hash m) + (map-fold (fun (k v h) (bitwise-xor h (hash-combine (eq-hash k) (eq-hash v)))) 0 m))) + +;; Two immutable arrays are equal when they hold equal elements in the same +;; order. The value cannot change, so it has a hash. +(impl (Eq (IArray %a)) + (where (Eq %a)) + (defun (= a b) + (def len (iarray-length a)) + (if (= len (iarray-length b)) + (let go ((i 0)) + (if (= i len) + #t + (if (= (iarray-ref a i) (iarray-ref b i)) + (go (+ i 1)) + #f))) + #f)) + (defun (eq-hash xs) + (def len (iarray-length xs)) + (let go ((i 0) (h 17)) + (if (= i len) + h + (go (+ i 1) (hash-combine h (eq-hash (iarray-ref xs i)))))))) + + + + +;; --------------------------------------------------------------------------- +;; Collector +;; --------------------------------------------------------------------------- + +;; Collector interface. +(def/trait (Collector %c) + (type %elem) + (type %acc) + (type %out) + (: collector-init (-> %c %acc)) + (: collector-step (-> %c %acc %elem %acc)) + (: collector-finish (-> %c %acc %out))) + +;; --------------------------------------------------------------------------- +;; Vector forms of drivers and collectors +;; --------------------------------------------------------------------------- +;; +;; A loop whose drivers are Contiguous and whose collectors are +;; BlockCollectors can be compiled to vector instructions. Each implementation +;; is a promise of its author: the compiler does not prove it. See +;; Docs/Vectorisation-design.org. + +;; An Iterable whose values are in memory one after another. The Iterable's +;; cursor is an int that starts at 0 and steps by 1, and the walk is done when +;; the cursor is the length of the values. The element at cursor i is +;; values[i], or, when the element is an (Option %value), (Some values[i]) if +;; bit i of the valid bitmap is set and None if it is not. +(def/trait (Contiguous %s) + (type %value) + ;; Whether the values are in one span now. If not, the loop walks them one + ;; by one. + (: contiguous? (-> %s bool)) + (: contiguous-values (-> %s (RoSpan %value))) + ;; The valid bitmap, bit i of byte i/8 for element i, the least + ;; significant bit first. Empty when no element is None. + (: contiguous-valid (-> %s (RoSpan byte)))) + +;; A Collector whose accumulator can take a block of elements at a time. The +;; collector's element is %value or (Option %value). +(def/trait (BlockCollector %c) + (type %value) + (type %acc) + ;; The number of elements taken in so far. + (: block-count (-> %c %acc int)) + ;; The accumulator with room for n more elements at the end. They are as if + ;; taken in: not None, with values that are written next. + (: block-grow (-> %c %acc int %acc)) + ;; The elements from start, n of them, to write. + (: block-span (-> %c %acc int int (Span %value))) + ;; Makes element i None. + (: block-none! (-> %c %acc int void))) + +;; Collectors. + +(type (: (Listing %a) (Union listing))) + +;; Listing. +(impl (Collector (Listing %a)) + (type %elem %a) + (type %acc (ListBuilder %a)) + (type %out (List %a)) + (defun (collector-init c) (listbuilder-empty)) + (defun (collector-step c acc x) (listbuilder-add! acc x) acc) + (defun (collector-finish c acc) (listbuilder->list acc))) + +;; Listing in reverse: each value consed onto the list so far, so the list is +;; the values last first, and there is no builder to finish. +(type (: (ListingReverse %a) (Union listing/reverse))) + +(impl (Collector (ListingReverse %a)) + (type %elem %a) + (type %acc (List %a)) + (type %out (List %a)) + (defun (collector-init c) Nil) + (defun (collector-step c acc x) (Cons x acc)) + (defun (collector-finish c acc) acc)) + +;; Vecing. +(type (: (Vecing %a) (Union vecing))) + +(impl (Collector (Vecing %a)) + (type %elem %a) + (type %acc (VecBuilder %a)) + (type %out (Vec %a)) + (defun (collector-init c) (vecbuilder-empty)) + (defun (collector-step c acc x) (vecbuilder-add! acc x) acc) + (defun (collector-finish c acc) (vecbuilder->vec acc))) + +;; Arraying: the elements in an array. Its accumulator takes a block of +;; elements at a time, so a loop of numbers into it can be a vector loop. +(type (: (Arraying %a) (Union arraying))) + +(import/class + ((ArrayBuilder %a) (: Bjolang.Runtime.ArrayBuilder))) + +(import/extern + (arraybuilder-empty (: Bjolang.Runtime.ArrayBuilderModule.Empty (-> (ArrayBuilder %a)))) + (arraybuilder-add (: Bjolang.Runtime.ArrayBuilderModule.Add (-> (ArrayBuilder %a) %a (ArrayBuilder %a)))) + (arraybuilder-count (: Bjolang.Runtime.ArrayBuilderModule.Count (-> (ArrayBuilder %a) int))) + (arraybuilder-grow (: Bjolang.Runtime.ArrayBuilderModule.Grow (-> (ArrayBuilder %a) int (ArrayBuilder %a)))) + (arraybuilder-span (: Bjolang.Runtime.ArrayBuilderModule.Span (-> (ArrayBuilder %a) int int (Span %a)))) + (arraybuilder->array (: Bjolang.Runtime.ArrayBuilderModule.ToArray (-> (ArrayBuilder %a) (Array %a)))) + (arraybuilder-no-none! (: Bjolang.Runtime.ArrayBuilderModule.NoNone (-> (ArrayBuilder %a) int void)))) + +(impl (Collector (Arraying %a)) + (type %elem %a) + (type %acc (ArrayBuilder %a)) + (type %out (Array %a)) + (defun (collector-init c) (arraybuilder-empty)) + (defun (collector-step c acc x) (arraybuilder-add acc x)) + (defun (collector-finish c acc) (arraybuilder->array acc))) + +(impl (BlockCollector (Arraying %a)) + (type %value %a) + (type %acc (ArrayBuilder %a)) + (defun (block-count c acc) (arraybuilder-count acc)) + (defun (block-grow c acc n) (arraybuilder-grow acc n)) + (defun (block-span c acc start n) (arraybuilder-span acc start n)) + (defun (block-none! c acc i) (arraybuilder-no-none! acc i))) + +;; Vecing into a builder for a vec that is likely small: its tail starts at 4 +;; items and doubles, where `vecing` takes a leaf of 32 at the first item. +(type (: (VecingSmall %a) (Union vecing/small))) + +(impl (Collector (VecingSmall %a)) + (type %elem %a) + (type %acc (VecBuilder %a)) + (type %out (Vec %a)) + (defun (collector-init c) (vecbuilder-empty/small)) + (defun (collector-step c acc x) (vecbuilder-add! acc x) acc) + (defun (collector-finish c acc) (vecbuilder->vec acc))) + +;; Mapping. +(type (: (Mapping %k %v) (Union mapping))) + +(impl (Collector (Mapping %k %v)) + (where (Eq %k)) + (type %elem (Tuple %k %v)) + (type %acc (MapBuilder %k %v)) + (type %out (Map %k %v)) + (defun (collector-init c) (mapbuilder-empty)) + (defun (collector-step c acc kv) + (def (k v) kv) + (mapbuilder-add! acc k v) + acc) + (defun (collector-finish c acc) (mapbuilder->map acc))) + +;; --- Starting from a collection: `#:into` --- +;; +;; `(vecing x #:into v)` adds to the back of `v` rather than to an empty vec, +;; and so do `listing` to a list and `mapping` to a map; `listing/reverse` +;; conses onto the front of its list, which costs nothing at all. The loop +;; makes the collector as `(collector-into vecing v)`, and the builder is made +;; from `v` once, before the first element: every step is the step it was. +;; +;; A collector takes `#:into` by implementing this, which says what it can +;; start from and what collector that makes. One that does not is a type error +;; naming it. +(def/trait (CollectorInto %c) + (type %seed) + (type %into) + (: collector-into (-> %c %seed %into))) + +(type (: (VecingInto %a) (Struct (: seed (Vec %a))))) + +(impl (Collector (VecingInto %a)) + (type %elem %a) + (type %acc (VecBuilder %a)) + (type %out (Vec %a)) + (defun (collector-init c) (vec->vecbuilder (record-ref c seed))) + (defun (collector-step c acc x) (vecbuilder-add! acc x) acc) + (defun (collector-finish c acc) (vecbuilder->vec acc))) + +(impl (CollectorInto (Vecing %a)) + (type %seed (Vec %a)) + (type %into (VecingInto %a)) + (defun (collector-into c seed) (VecingInto (seed seed)))) + +;; The vec it starts from says how big it is, so `/small' has nothing to add. +(impl (CollectorInto (VecingSmall %a)) + (type %seed (Vec %a)) + (type %into (VecingInto %a)) + (defun (collector-into c seed) (VecingInto (seed seed)))) + +;; A list grows only at the front, so adding to its back copies it, once. +(type (: (ListingInto %a) (Struct (: seed (List %a))))) + +(impl (Collector (ListingInto %a)) + (type %elem %a) + (type %acc (ListBuilder %a)) + (type %out (List %a)) + (defun (collector-init c) (list->listbuilder (record-ref c seed))) + (defun (collector-step c acc x) (listbuilder-add! acc x) acc) + (defun (collector-finish c acc) (listbuilder->list acc))) + +(impl (CollectorInto (Listing %a)) + (type %seed (List %a)) + (type %into (ListingInto %a)) + (defun (collector-into c seed) (ListingInto (seed seed)))) + +;; Consing onto the front shares the list it starts from: nothing is copied. +(type (: (ListingReverseInto %a) (Struct (: seed (List %a))))) + +(impl (Collector (ListingReverseInto %a)) + (type %elem %a) + (type %acc (List %a)) + (type %out (List %a)) + (defun (collector-init c) (record-ref c seed)) + (defun (collector-step c acc x) (Cons x acc)) + (defun (collector-finish c acc) acc)) + +(impl (CollectorInto (ListingReverse %a)) + (type %seed (List %a)) + (type %into (ListingReverseInto %a)) + (defun (collector-into c seed) (ListingReverseInto (seed seed)))) + +;; The map's own key comparer goes with it, so no `Eq' is asked for. +(type (: (MappingInto %k %v) (Struct (: seed (Map %k %v))))) + +(impl (Collector (MappingInto %k %v)) + (type %elem (Tuple %k %v)) + (type %acc (MapBuilder %k %v)) + (type %out (Map %k %v)) + (defun (collector-init c) (map->mapbuilder (record-ref c seed))) + (defun (collector-step c acc kv) + (def (k v) kv) + (mapbuilder-add! acc k v) + acc) + (defun (collector-finish c acc) (mapbuilder->map acc))) + +(impl (CollectorInto (Mapping %k %v)) + (type %seed (Map %k %v)) + (type %into (MappingInto %k %v)) + (defun (collector-into c seed) (MappingInto (seed seed)))) + +;; Stringing. +(type (: Stringing (Union stringing))) + +(impl (Collector Stringing) + (type %elem char) + (type %acc StringBuilder) + (type %out string) + (defun (collector-init c) (stringbuilder-empty)) + (defun (collector-step c acc x) (stringbuilder-add! acc x) acc) + (defun (collector-finish c acc) (stringbuilder->string acc))) + +;; Summing and counting work at every numeric type. A sum has the type of what +;; it adds. A count has the type its use asks for, and is an int when nothing +;; asks, as a literal is. +(type (: (Summing %a) (Union summing))) + +(impl (Collector (Summing %a)) + (where (Num %a)) + (type %elem %a) + (type %acc %a) + (type %out %a) + (defun (collector-init c) 0) + (defun (collector-step c acc x) (+ acc x)) + (defun (collector-finish c acc) acc)) + +(type (: (Counting %a %n) (Union counting))) + +(impl (Collector (Counting %a %n)) + (where (Num %n)) + (type %elem %a) + (type %acc %n) + (type %out %n) + (defun (collector-init c) 0) + (defun (collector-step c acc x) (+ acc 1)) + (defun (collector-finish c acc) acc)) + +;; Folding. +(type (: (Folding %a) (Union (: folding %a)))) + +(impl (Collector (Folding %a)) + (type %elem %a) + (type %acc %a) + (type %out %a) + (defun (collector-init c) (match c ((folding s) s))) + (defun (collector-step c acc x) x) + (defun (collector-finish c acc) acc)) + + + + +;; --------------------------------------------------------------------------- +;; Iterable +;; --------------------------------------------------------------------------- +;; Iterable protocol. What the compiler does, is that it fuses +;; subsequent seql, so doing (-> an-iterable map map map filter map take) should fuse +;; to only ONE IEnumerable, meaning only one of the shitty awful state machines. +(def/trait (Iterable %s) + (type %elem) + (type %cursor) + (: iterable-start (-> %s %cursor)) + ;; A walk calls this once per step, before `current`. So a cursor may be a + ;; .NET enumerator that steps here, as the `Seq` impl's does; its + ;; `iterable-next` then answers the same enumerator. + (: iterable-done? (-> %s %cursor bool)) + (: iterable-current (-> %s %cursor %elem)) + (: iterable-next (-> %s %cursor %cursor)) + ;; A loop calls this on each cursor it stops before `done?` answered true: + ;; at a `:finish`, an `:abandon`, or when another driver of the level ended + ;; first. It may also come after `done?` answered true, or a second time, + ;; and must then do nothing: a `zip` ends the longer side when the shorter + ;; one ends. A cursor that holds a resource for the walk releases it here. + (: iterable-end (-> %s %cursor void)) + ;; A loop calls this, in place of `iterable-end`, on each cursor that is + ;; live when an exception or a non-local exit (such as `ret` of a + ;; `with-return`) leaves it. A cursor that holds a resource releases it here + ;; as well. With the default, which does nothing, the loop has no `try` + ;; around it and keeps no record of its cursors. + (: iterable-cleanup (-> %s %cursor void)) + (: ->seq (-> %s (Seq %elem))) + (: fold (-> (-> %elem %acc %acc) %acc %s %acc)) + (: map (-> (-> %elem %b) %s (Seq %b))) + (: filter (-> (-> %elem bool) %s (Seq %elem))) + (: take (-> int %s (Seq %elem))) + (: drop (-> int %s (Seq %elem))) + (: take-while (-> (-> %elem bool) %s (Seq %elem))) + (: drop-while (-> (-> %elem bool) %s (Seq %elem))) + (: seq-enumerate (-> %s (Seq (Tuple int %elem)))) + (: flat-map (-> (-> %elem (Seq %b)) %s (Seq %b))) + ;; The methods below return a value rather than a sequence, so their defaults + ;; are ordinary loops instead of `seql`s. A `seql` becomes a C# iterator, + ;; which cannot await, so only these can take a callback of either colour. + (: any? (-> (-?-> %elem bool) %s bool)) + (: all? (-> (-?-> %elem bool) %s bool)) + (: find (-> (-?-> %elem bool) %s (Option %elem))) + (: for-each (-> (-?-> %elem void) %s void)) + (: ->list (-> %s (List %elem))) + (: ->vec (-> %s (Vec %elem))) + (: count (-> %s int)) + (: first (-> %s (Option %elem))) + + ;; Most cursors hold nothing that needs release. + (defun (iterable-end s c) (when #f ())) + (defun (iterable-cleanup s c) (when #f ())) + (defun (->seq s) + (seql (:for e s) + (:yield e))) + (defun (fold f acc coll) + (loop (:for elem coll) + (:acc res (folding acc (f elem res))))) + (defun (map f s) + (seql (:for e s) + (:yield (f e)))) + (defun (filter f s) + (seql (:for e s) + (:when (f e)) + (:yield e))) + ;; The count is tested before the walk pulls, so the element after the n-th + ;; is never asked for: over a port it stays for the next reader. + (defun (take n s) + (seql (:with i 0 (+ i 1) (>= i n)) + (:for elem s) + (:yield elem))) + ;; this is of course not efficient, but hey. + (defun (drop n s) + (seql (:with i 1 (+ i 1)) + (:for elem s) + (:when (> i n)) + (:yield elem))) + (defun (take-while f s) + (seql (:for elem s) + (:finish (not (f elem))) + (:yield elem))) + + ;; The flag is made inside the `seq`, whose body runs once per walk, so that + ;; walking the result twice drops the same elements both times. + (defun (drop-while f s) + (seq + (def/mutable dropping #t) + (loop (:for elem s) + (:do (when (and dropping (not (f elem))) + (set! dropping #f))) + (:when (not dropping)) + (:do (yield elem))))) + + ;; A `seql`, so that a loop over it fuses with a chain of others, as + ;; `(seq-enumerate (map f xs))` does. `enumerate` makes an adapter instead. + (defun (seq-enumerate s) + (seql (:with i 0 (+ i 1)) + (:for e s) + (:yield (Tuple i e)))) + + (defun (flat-map f s) + (seql (:for e s) + (:subloop) + (:for x (f e)) + (:yield x))) + + ;; `folding` keeps the last stepped value, so the accumulator holds the + ;; current element's answer and `:finish` stops on it. + (defun (any? f s) + (loop (:for e s) + (:acc found (folding #f (f e))) + (:finish found))) + + (defun (all? f s) + (loop (:for e s) + (:acc ok (folding #t (f e))) + (:finish (not ok)))) + + (defun (find f s) + (loop (:for e s) + (:acc got (folding None (if (f e) (Some e) None))) + (:finish (match got ((Some _) #t) (None #f))))) + + (defun (for-each f s) + (loop (:for e s) + (:do (ignore (f e))))) + + (defun (->list s) + (loop (:for e s) + (:acc xs (listing e)))) + + (defun (->vec s) + (loop (:for e s) + (:acc xs (vecing e)))) + + ;; Walks the whole of `s`. A collection that knows its length answers it. + (defun (count s) + (loop (:for e s) + (:acc n (counting e)))) + + ;; Stops on the first element, so the walk pulls only that one. + (defun (first s) + (loop (:for e s) + (:acc got (folding None (Some e))) + (:finish #t))) + ) + +;; Declared before the other implementations because the inherited +;; `flat-map` default walks the `(Seq %b)` its callback returns, and an +;; implementation is only in scope for the ones checked after it. +(impl (Iterable (Seq %a)) + (type %elem %a) + ;; The walk's cursor is the enumerator itself. The protocol calls `done?` + ;; once per step, before `current`, so `done?` is where it steps, and + ;; `next` has nothing left to do. + (type %cursor (SeqEnumerator %a)) + (defun (iterable-start s) (seq-enumerator s)) + (defun (iterable-done? s e) (not (seq-enumerator-next! e))) + (defun (iterable-current s e) (seq-enumerator-current e)) + (defun (iterable-next s e) e) + ;; Disposes the enumerator, which closes a file the sequence opened for + ;; this walk. + (defun (iterable-end s e) (seq-enumerator-end! e)) + ;; The same on an exception, so that a file is closed then too. + (defun (iterable-cleanup s e) (seq-enumerator-end! e)) + + (defun (->seq s) s)) + +(impl (Iterable (List %a)) + (type %elem %a) + (type %cursor (List %a)) + (defun (iterable-start s) s) + (defun (iterable-done? s c) (list-empty? c)) + (defun (iterable-current s c) (list-head c)) + (defun (iterable-next s c) (list-tail c)) + (defun (->list s) s)) + +;; A struct cursor that walks the vec's leaf arrays: an array read per +;; element, and nothing allocated for the walk. +(impl (Iterable (Vec %a)) + (type %elem %a) + (type %cursor (VecCursor %a)) + (defun (iterable-start s) (vec-cursor s)) + (defun (iterable-done? s c) (vec-cursor-done? c)) + (defun (iterable-current s c) (vec-cursor-current c)) + (defun (iterable-next s c) (vec-cursor-next c)) + (defun (->vec s) s) + (defun (count s) (vec-length s))) + +;; `>=` rather than `=`: the JIT drops the bounds check on `array-ref` only +;; when the loop test is `index < length`. +(impl (Iterable (Array %a)) + (type %elem %a) + (type %cursor int) + (defun (iterable-start s) 0) + (defun (iterable-done? s index) (>= index (array-length s))) + (defun (iterable-current s index) (array-ref s index)) + (defun (iterable-next s index) (+ index 1)) + (defun (count s) (array-length s))) + +;; An `int` cursor, as `Array` has. +(impl (Iterable (IArray %a)) + (type %elem %a) + (type %cursor int) + (defun (iterable-start s) 0) + (defun (iterable-done? s index) (>= index (iarray-length s))) + (defun (iterable-current s index) (iarray-ref s index)) + (defun (iterable-next s index) (+ index 1)) + (defun (count s) (iarray-length s))) + +(: (-> (IArray %a) (Vec %a))) +(defun (iarray->vec xs) (->vec xs)) + +(: (-> (IArray %a) (List %a))) +(defun (iarray->list xs) (->list xs)) + +;; Range of ints, in steps of one. A step of one reaches `hi` before it can +;; pass `int-max`, so the walk tests `(>= c hi)`, the test the JIT knows from a +;; loop over an array's indices. +(type (: Range (Struct (: lo int) (: hi int)))) + +;; Inlined, so that a loop over a range has its two ints and no struct (see +;; `Simplify`, "A record that is only read"). +(: (-> int int Range)) +(defun (range lo hi) #:inline + (Range (lo lo) (hi hi))) + +(impl (Iterable Range) + (type %elem int) + (type %cursor int) + (defun (iterable-start s) (record-ref s lo)) + (defun (iterable-done? s c) (>= c (record-ref s hi))) + (defun (iterable-current s c) c) + (defun (iterable-next s c) (+ c 1))) + +;; Range of ints, in steps of `by`. A step past the last value can pass +;; `int-max`, so the walk counts in a long, where it cannot wrap round below +;; `hi`: a value below `hi` and a step of at most `int-max` stay far below +;; `long-max`. +(type (: RangeBy (Struct (: lo int) (: hi int) (: by int)))) + +;; A range counts up, so a step of zero or less would never reach `hi`. The +;; message is made in `range-by-panic`, so that `range-by` is small enough for +;; the JIT to inline at the start of a loop. +(: (-> int int int RangeBy)) +(defun (range-by lo hi by) + (when (<= by 0) + (range-by-panic by)) + (RangeBy (lo lo) (hi hi) (by by))) + +(: (-> int void)) +(defun (range-by-panic by) + (panic! #"range-by: the step is ${by}, and a range counts up from lo to hi. down-from counts down.")) + +;; How many values a non-empty range holds, in longs, where `hi - lo` cannot +;; overflow. +(: (-> int int int long)) +(defun (range-by-count lo hi by) + (+ (/ (- (- (cast long hi) (cast long lo)) 1L) (cast long by)) 1L)) + +(impl (Iterable RangeBy) + (type %elem int) + (type %cursor long) + (defun (iterable-start s) (cast long (record-ref s lo))) + (defun (iterable-done? s c) (>= c (cast long (record-ref s hi)))) + (defun (iterable-current s c) (cast int c)) + (defun (iterable-next s c) (+ c (cast long (record-ref s by))))) + +;; Map elements. +(impl (Iterable (Map %k %v)) + (type %elem (Tuple %k %v)) + (type %cursor (MapCursor %k %v)) + (defun (iterable-start s) (map-cursor s)) + (defun (iterable-done? s c) (map-cursor-done? c)) + (defun (iterable-current s c) (map-cursor-current c)) + (defun (iterable-next s c) (map-cursor-next c)) + (defun (count s) (map-length s))) + +;; String characters. Not a `StringCursor`, which asks the string at every +;; step whether it is a slice: a walk takes its bytes once and decodes each +;; character as it steps onto it. +(import/class + (StringWalk (: Bjolang.Runtime.StringWalk))) + +(import/extern + (string-walk (: Bjolang.Runtime.StringWalks.Start (-> string StringWalk))) + (string-walk-done? (: Bjolang.Runtime.StringWalks.Done (-> StringWalk bool))) + (string-walk-current (: Bjolang.Runtime.StringWalks.Current (-> StringWalk char))) + (string-walk-next (: Bjolang.Runtime.StringWalks.Next (-> StringWalk StringWalk)))) + +(impl (Iterable string) + (type %elem char) + (type %cursor StringWalk) + (defun (iterable-start s) (string-walk s)) + (defun (iterable-done? s c) (string-walk-done? c)) + (defun (iterable-current s c) (string-walk-current c)) + (defun (iterable-next s c) (string-walk-next c))) + +;; An iterable of pairs as a map. Not a method of `Iterable`: its where clause +;; says the element is a pair, and a member may not constrain the +;; implementor. Copied into the call, so that it fuses with a chain under it. +(: (-> %s (Map %k %v)) (where (Eq %k) (Iterable %s #:elem (Tuple %k %v)))) +(defun (->map s) #:inline + (loop (:for e s) + (:acc m (mapping e)))) + +;; --------------------------------------------------------------------------- +;; Adapters +;; --------------------------------------------------------------------------- +;; +;; An adapter is a struct around iterables, and an iterable itself: its +;; element and cursor types are made from theirs, and its methods call +;; theirs. A loop over one at a known type inlines it with what it wraps, so +;; it allocates nothing; through a dictionary its class holds the dictionaries +;; of what it wraps. `iterable-end` and `iterable-cleanup` are passed on, so +;; an adapter over a sequence that holds a file closes it as the sequence +;; would, and one over iterables that hold nothing keeps the loop free of a +;; guard. The cursor is taken apart with a `def` rather than `fst` and `snd` +;; where it is passed on: `Simplify` can see that the binding does nothing, +;; and so that a cleanup which does nothing leaves no guard. + +;; The elements of `inner`, each with its index from 0. +(type (: (Enumerated %s) (Struct (: inner %s)))) + +(: (-> %s (Enumerated %s)) (where (Iterable %s))) +(defun (enumerate s) (Enumerated (inner s))) + +(impl (Iterable (Enumerated %s)) (where (Iterable %s)) + (type %elem (Tuple int (assoc Iterable elem %s))) + (type %cursor (Tuple int (assoc Iterable cursor %s))) + (defun (iterable-start e) (Tuple 0 (iterable-start (record-ref e inner)))) + (defun (iterable-done? e c) (iterable-done? (record-ref e inner) (snd c))) + (defun (iterable-current e c) (Tuple (fst c) (iterable-current (record-ref e inner) (snd c)))) + (defun (iterable-next e c) (Tuple (+ (fst c) 1) (iterable-next (record-ref e inner) (snd c)))) + (defun (iterable-end e c) + (def (Tuple _ k) c) + (iterable-end (record-ref e inner) k)) + (defun (iterable-cleanup e c) + (def (Tuple _ k) c) + (iterable-cleanup (record-ref e inner) k))) + +;; The elements of `left` and `right` in pairs, until either ends. +(type (: (Zipped %l %r) (Struct (: left %l) (: right %r)))) + +(: (-> %l %r (Zipped %l %r)) (where (Iterable %l) (Iterable %r))) +(defun (zip l r) (Zipped (left l) (right r))) + +;; `done?` asks the left side first. When one side ends, the other is ended +;; there and then, so that a sequence holding a file is closed although its +;; walk did not finish; the loop ends no cursor whose `done?` answered true. +;; When the right side is the shorter, the left has read one element more, +;; which nothing uses. +(impl (Iterable (Zipped %l %r)) (where (Iterable %l) (Iterable %r)) + (type %elem (Tuple (assoc Iterable elem %l) (assoc Iterable elem %r))) + (type %cursor (Tuple (assoc Iterable cursor %l) (assoc Iterable cursor %r))) + (defun (iterable-start z) + (Tuple (iterable-start (record-ref z left)) (iterable-start (record-ref z right)))) + ;; Written with `or` and `and` rather than `if`s: where both ends do nothing, + ;; `Simplify` drops them, and the test is one expression, as two lockstep + ;; `:for`s would make it. + (defun (iterable-done? z c) + (def (Tuple kl kr) c) + (or (and (iterable-done? (record-ref z left) kl) + (begin (iterable-end (record-ref z right) kr) #t)) + (and (iterable-done? (record-ref z right) kr) + (begin (iterable-end (record-ref z left) kl) #t)))) + (defun (iterable-current z c) + (Tuple (iterable-current (record-ref z left) (fst c)) + (iterable-current (record-ref z right) (snd c)))) + (defun (iterable-next z c) + (Tuple (iterable-next (record-ref z left) (fst c)) + (iterable-next (record-ref z right) (snd c)))) + (defun (iterable-end z c) + (def (Tuple kl kr) c) + (iterable-end (record-ref z left) kl) + (iterable-end (record-ref z right) kr)) + (defun (iterable-cleanup z c) + (def (Tuple kl kr) c) + (iterable-cleanup (record-ref z left) kl) + (iterable-cleanup (record-ref z right) kr))) + +;; Each element of `inner` with the one after it: (Tuple a b), one fewer +;; than `inner` has, in one walk of it. +(type (: (Pairwise %s) (Struct (: inner %s)))) + +(: (-> %s (Pairwise %s)) (where (Iterable %s))) +(defun (pairwise s) (Pairwise (inner s))) + +;; The cursor is the element before and the inner cursor at the one after. +;; `start` takes the first element, so it asks the inner `done?` of the first +;; cursor itself; `None` before says the inner iterable was empty, and no +;; `done?` is asked again of that cursor. `current` and `next` both read the +;; element at the inner cursor. +(impl (Iterable (Pairwise %s)) (where (Iterable %s)) + (type %elem (Tuple (assoc Iterable elem %s) (assoc Iterable elem %s))) + (type %cursor (Tuple (Option (assoc Iterable elem %s)) (assoc Iterable cursor %s))) + ;; One tuple, not one per branch, and `done?` with `or` rather than a + ;; `match`: over an array, the loop is then faster than one that indexes + ;; the array twice. + (defun (iterable-start w) + (def k (iterable-start (record-ref w inner))) + (def empty (iterable-done? (record-ref w inner) k)) + (Tuple (if empty None (Some (iterable-current (record-ref w inner) k))) + (if empty k (iterable-next (record-ref w inner) k)))) + (defun (iterable-done? w c) + (def (Tuple before k) c) + (or (none? before) (iterable-done? (record-ref w inner) k))) + (defun (iterable-current w c) + (def (Tuple before k) c) + (Tuple (option-ref before) (iterable-current (record-ref w inner) k))) + (defun (iterable-next w c) + (def (Tuple _ k) c) + (Tuple (Some (iterable-current (record-ref w inner) k)) + (iterable-next (record-ref w inner) k))) + (defun (iterable-end w c) + (def (Tuple _ k) c) + (iterable-end (record-ref w inner) k)) + (defun (iterable-cleanup w c) + (def (Tuple _ k) c) + (iterable-cleanup (record-ref w inner) k))) + +;; The elements of `first` and then those of `second`: two iterables with one +;; element type, of any two kinds. +(type (: (Chained %a %b) (Struct (: first %a) (: second %b)))) + +(: (-> %a %b (Chained %a %b)) (where (Iterable %a #:elem %e) (Iterable %b #:elem %e))) +(defun (chain a b) (Chained (first a) (second b))) + +;; The cursor is `(Tuple on-second? cur-a cur-b)`. +;; +;; Because `iterable-done?` cannot mutate the cursor, transitions from `first` to +;; `second` are determined ahead of time: +;; - `iterable-start` and `iterable-next` eagerly check if `first` has finished, +;; setting `on-second?` to `#t` when moving to `second`. +;; - `iterable-done?` only checks completion of the active side (via `cur-b` once +;; `on-second?` is true). +;; - `iterable-cleanup` safely cleans up both iterables when the walk terminates. +(impl (Iterable (Chained %a %b)) + (where (Iterable %a) (Iterable %b #:elem (assoc Iterable elem %a))) + (type %elem (assoc Iterable elem %a)) + (type %cursor (Tuple bool (assoc Iterable cursor %a) (assoc Iterable cursor %b))) + (defun (iterable-start h) + (def ka (iterable-start (record-ref h first))) + (Tuple (iterable-done? (record-ref h first) ka) ka (iterable-start (record-ref h second)))) + (defun (iterable-done? h c) + (def (Tuple on-second _ kb) c) + (and on-second (iterable-done? (record-ref h second) kb))) + (defun (iterable-current h c) + (def (Tuple on-second ka kb) c) + (if on-second + (iterable-current (record-ref h second) kb) + (iterable-current (record-ref h first) ka))) + (defun (iterable-next h c) + (def (Tuple on-second ka kb) c) + (if on-second + (Tuple #t ka (iterable-next (record-ref h second) kb)) + (let ((next-a (iterable-next (record-ref h first) ka))) + (Tuple (iterable-done? (record-ref h first) next-a) next-a kb)))) + ;; Both sides, whichever the walk is on: an end of a cursor whose `done?` + ;; answered true does nothing, as the protocol says, and with no test + ;; here, ends that do nothing leave the loop no guard. + (defun (iterable-end h c) + (def (Tuple _ ka kb) c) + (iterable-end (record-ref h first) ka) + (iterable-end (record-ref h second) kb)) + (defun (iterable-cleanup h c) + (def (Tuple _ ka kb) c) + (iterable-cleanup (record-ref h first) ka) + (iterable-cleanup (record-ref h second) kb))) + +;; `item`, `times` times, or without end when `times` is `repeat-endless`, +;; which is what `in-repeat` gives when no count is written. It is +;; `int-min`, which no count written as a count is. +(type (: (Repeated %a) (Struct (: item %a) (: times int)))) + +(: int) +(def repeat-endless -2147483648) + +(: (-> %a (#:times int) (Repeated %a))) +(defun (in-repeat x #:times repeat-endless) (Repeated (item x) (times times))) + +(impl (Iterable (Repeated %a)) + (type %elem %a) + (type %cursor int) + (defun (iterable-start r) 0) + (defun (iterable-done? r i) + (and (not (= (record-ref r times) repeat-endless)) (>= i (record-ref r times)))) + (defun (iterable-current r i) (record-ref r item)) + (defun (iterable-next r i) (+ i 1))) + +;; The elements of `inner`, over and over: when a walk of it ends, the next +;; one starts. It ends when a walk yields nothing, so the cycle of an empty +;; iterable is empty, and so is the next pass over a port that is used up. +(type (: (Cycled %s) (Struct (: inner %s)))) + +(: (-> %s (Cycled %s)) (where (Iterable %s))) +(defun (in-cycle s) (Cycled (inner s))) + +;; `done?` of the inner iterable is asked once for each of its cursors, as +;; the protocol wants: a cursor over a sequence pulls when asked. So `next` +;; asks it of the cursor it makes, and the flag in the cursor says that it +;; answered false; when it answered true, `next` starts the next walk, and +;; `done?` asks of that cursor in turn. `next` therefore pulls the element +;; after the current one before a loop asks for it, as `zip` does on its +;; left side. +(impl (Iterable (Cycled %s)) (where (Iterable %s)) + (type %elem (assoc Iterable elem %s)) + (type %cursor (Tuple bool (assoc Iterable cursor %s))) + (defun (iterable-start y) (Tuple #f (iterable-start (record-ref y inner)))) + (defun (iterable-done? y c) + (def (Tuple asked k) c) + (and (not asked) (iterable-done? (record-ref y inner) k))) + (defun (iterable-current y c) (iterable-current (record-ref y inner) (snd c))) + (defun (iterable-next y c) + (def (Tuple _ k) c) + (def n (iterable-next (record-ref y inner) k)) + (if (iterable-done? (record-ref y inner) n) + (Tuple #f (iterable-start (record-ref y inner))) + (Tuple #t n))) + (defun (iterable-end y c) + (def (Tuple _ k) c) + (iterable-end (record-ref y inner) k)) + (defun (iterable-cleanup y c) + (def (Tuple _ k) c) + (iterable-cleanup (record-ref y inner) k))) + + +;; --------------------------------------------------------------------------- +;; Cursor +;; --------------------------------------------------------------------------- + +;; Cursor. +(def/trait (Cursor %s) + (type %elem) + (type %cursor) + (: cursor-start (-> %s %cursor)) + (: cursor-end (-> %s %cursor)) + (: cursor-end? (-> %s %cursor bool)) + (: cursor-ref (-> %s %cursor %elem)) + (: cursor-next (-> %s %cursor %cursor)) + (: cursor-prev (-> %s %cursor %cursor))) + +(impl (Cursor string) + (type %elem char) + (type %cursor StringCursor) + (defun (cursor-start s) (string-cursor-start s)) + (defun (cursor-end s) (string-cursor-end s)) + (defun (cursor-end? s c) (string-cursor-end? s c)) + (defun (cursor-ref s c) (string-cursor-ref s c)) + (defun (cursor-next s c) (string-cursor-next s c)) + (defun (cursor-prev s c) (string-cursor-prev s c))) + + +;; --------------------------------------------------------------------------- +;; Lists, where the callback may suspend +;; --------------------------------------------------------------------------- +;; The walks below are written #:inline where their body is past the size the +;; compiler copies by itself: copied, the loop is in the caller with the +;; callback's body in it, and no delegate is made or called. +;; +;; List map. +(: (-> (-?-> %a %b) (List %a) (List %b))) +(defun (list-map f xs) + (loop (:for elem xs) + (:acc lst (listing (f elem))))) + +(: (-> (-?-> %a bool) (List %a) (List %a))) +(defun (list-filter keep? xs) #:inline + (loop (:for elem xs) + (:when (keep? elem)) + (:acc acc (listing elem)))) + +;; List foldl. +(: (-> (-?-> %a %b %b) %b (List %a) %b)) +(defun (list-foldl f init xs) #:inline + (loop (:for elem xs) + (:acc acc (folding init (f elem acc))))) + +;; List foldr. +(: (-> (-?-> %a %b %b) %b (List %a) %b)) +(defun (list-foldr f init xs) #:inline + (loop (:for elem (list-reverse xs)) + (:acc acc (folding init (f elem acc))))) + +(: (-> (-?-> %a Unit) (List %a) Unit)) +(defun (list-for-each f xs) + (loop (:for elem xs) + (:do (f elem)))) + +;; The list seq-unfold would make, without the lazy seq. The update is +;; evaluated after the iteration, so it reads the state the :finish-let bound. +(: (-> (-?-> %s (Option (Tuple %a %s))) %s (List %a))) +(defun (list-unfold f state) + (loop (:with step (f state) (f next)) + (:finish-let (Some (Tuple x next)) step) + (:acc out (listing x)))) + +;; --------------------------------------------------------------------------- +;; Vecs, where the callback may suspend +;; --------------------------------------------------------------------------- +;; Vec functions. + +(import/extern + (clr-vec-map (: BjolangRuntime.RrbMap (-> (-> %a %b) (Vec %a) (Vec %b)))) + ;; A vec of the same shape as the first, holding the array's items in + ;; order: what `clr-vec-map` makes, for items made by a function that may + ;; suspend, which no extern can call. + (clr-vec-map-shape (: Bjolang.Runtime.Rrb.RrbFun.MapShape (-> (Vec %a) (Array %b) (Vec %b)))) + (clr-vec-filter (: BjolangRuntime.RrbFilter (-> (-> %a bool) (Vec %a) (Vec %a)))) + (clr-vec-fold (: BjolangRuntime.RrbFold (-> (-> %a %b %b) %b (Vec %a) %b))) + (clr-vec-reduce (: BjolangRuntime.RrbReduce (-> (-> %a %a %a) (Vec %a) %a))) + (clr-vec-for-each (: BjolangRuntime.RrbForEach (-> (-> %a Unit) (Vec %a) int int Unit))) + (clr-vec-iter (: BjolangRuntime.RrbIter (-> (-> %a bool) (Vec %a) bool))) + ;; The checks the ordinary copies make, for the suspending copies to make + ;; too, so that both raise the same thing. + (clr-vec-check-reduce (: BjolangRuntime.RrbCheckReduce (-> (Vec %a) Unit))) + (clr-vec-check-range (: BjolangRuntime.RrbCheckRange (-> (Vec %a) int int Unit))) + ;; Append one element at the end — the persistent counterpart of + ;; `vecbuilder-add!`, and what `Addable`'s impl for Vec delegates to. + (vec-add (: Bjolang.Runtime.Rrb.RrbFun.Add (-> (Vec %a) %a (Vec %a)))) + ;; Add one element at the front — the persistent counterpart of + ;; `transientvec-add-first!`, as `vec-pop-first` is of `transientvec-pop-first!`. + (vec-add-first (: Bjolang.Runtime.Rrb.RrbFun.Prepend (-> (Vec %a) %a (Vec %a))))) + +;; Vec map. +(: (-> (-?-> %a %b) (Vec %a) (Vec %b))) +;; +;; Not inlined: `clr-vec-map` copies the vec's arrays at their exact size, and +;; for the short vecs most programs map that is faster than a loop into a +;; builder, even with the call through the delegate. +;; +;; The suspending copy keeps the vec's shape too: the items go into an array +;; in order, and `clr-vec-map-shape` lays them out as `v` is laid out. So the +;; halves of a split, mapped in bjoroutines, merge back into a dense vec. +(defbjouble (vec-map f v) + #:no-inline + (#:sync (clr-vec-map f v)) + (#:bjo + (def out (make-array (vec-length v))) + (loop (:for elem v) + (:with i 0 (+ i 1)) + (:do (array-set! out i (f elem)))) + (clr-vec-map-shape v out))) + +(: (-> (-?-> %a bool) (Vec %a) (Vec %a))) +(defbjouble (vec-filter keep? v) + #:inline + (#:sync (clr-vec-filter keep? v)) + (#:bjo + (loop (:for elem v) + (:when (keep? elem)) + (:acc acc (vecing/small elem))))) + +(: (-> (-?-> %a %b %b) %b (Vec %a) %b)) +(defbjouble (vec-fold f init v) + #:inline + (#:sync (clr-vec-fold f init v)) + (#:bjo + (loop (:for elem v) + (:acc acc (folding init (f elem acc)))))) + +;; Vec reduce. +(: (-> (-?-> %a %a %a) (Vec %a) %a)) +(defbjouble (vec-reduce f v) + (#:sync (clr-vec-reduce f v)) + (#:bjo + (ignore (clr-vec-check-reduce v)) + (let go ((c (vec-cursor-slice v 1 (- (vec-length v) 1))) (acc (vec-ref v 0))) + (if (vec-cursor-done? c) + acc + (go (vec-cursor-next c) (f (vec-cursor-current c) acc)))))) + +;; The elements from `#:from` up to `#:to`, front to back. Raises +;; ArgumentOutOfRangeException, before `f` is first called, unless +;; 0 <= from <= to <= length. +(: (-> (-?-> %a Unit) (Vec %a) (#:from int) (#:to int) Unit)) +(defbjouble (vec-for-each f v #:from 0 #:to (vec-length v)) + #:inline + (#:sync (clr-vec-for-each f v from to)) + (#:bjo + (ignore (clr-vec-check-range v from to)) + (let go ((c (vec-cursor-slice v from (- to from)))) + (if (vec-cursor-done? c) + unit + (let () + (ignore (f (vec-cursor-current c))) + (go (vec-cursor-next c))))))) + +;; Vec iter. +(: (-> (-?-> %a bool) (Vec %a) bool)) +(defbjouble (vec-iter f v) + #:inline + (#:sync (clr-vec-iter f v)) + (#:bjo + (loop (:for elem v) + (:acc acc (folding #t (f elem))) + (:finish (not acc))))) + +;; --------------------------------------------------------------------------- +;; Seqs, where the callback may suspend +;; --------------------------------------------------------------------------- +;; +;; The eager two. `seq-unfold` is lazy and stays a builtin: its callback runs +;; when the walker pulls, which is somewhere else. + +(: (-> (-?-> %a %b %b) %b (Seq %a) %b)) +(defun (seq-fold f init s) + (let ((e (seq-enumerator s))) + (let go ((acc init)) + (if (seq-enumerator-next! e) + (go (f (seq-enumerator-current e) acc)) + acc)))) + +(: (-> (-?-> %a Unit) (Seq %a) Unit)) +(defun (seq-for-each f s) + (let ((e (seq-enumerator s))) + (let go () + (if (seq-enumerator-next! e) + (let () + (ignore (f (seq-enumerator-current e))) + (go)) + unit)))) + + +;; --------------------------------------------------------------------------- +;; Conversions +;; --------------------------------------------------------------------------- +;; Conversions. + +(: (-> (List (Tuple %k %v)) (Map %k %v))) +(defun (list->map xs) + (loop (:for kvp xs) + (:acc map (mapping kvp)))) + +(: (-> (List %a) (Vec %a))) +(defun (list->vec lst) + {vecing e (:for e lst)}) + +(: (-> (Map %k %v) (List (Tuple %k %v)))) +(defun (map->list m) + {listing kvp (:for kvp m)}) + +(: (-> (Vec (Tuple %k %v)) (Map %k %v))) +(defun (vec->map v) + (loop (:for kvp v) + (:acc map (mapping kvp)))) + +;; Faster than a loop over the indices: `vec-for-each` walks the vec itself, +;; and the builder adds in place. +(: (-> (Vec %a) (List %a))) +(defun (vec->list v) + (def b (listbuilder-empty)) + (vec-for-each (fun (elem) (listbuilder-add! b elem)) v ) + (listbuilder->list b)) + +(: (-> (Map %k %v) (Vec (Tuple %k %v)))) +(defun (map->vec m) + {vecing kvp (:for kvp m)}) + +;; vec-insert, vec-remove-at, vec-slice and vec-split, over the runtime's RRB +;; nodes. +(include "vec.protobjo") + + +;; -------------------------------------------------------------------------- +;; Results and options +;; -------------------------------------------------------------------------- + +;; `some?` and `none?`, for the other two-case builtin. + +(: (-> (Result %e %a) bool)) +(defun (ok? r) + (match r + ((Ok v) #t) + ((Err e) #f))) + +(: (-> (Result %e %a) bool)) +(defun (err? r) + (match r + ((Ok v) #f) + ((Err e) #t))) + +;; The `Ok` value through `f`, which may suspend; an `Err` returns itself. +(: (-> (-?-> %a %b) (Result %e %a) (Result %e %b))) +(defun (result-map f r) + (match r + ((Ok v) (Ok (f v))) + ((Err e) (Err e)))) + +;; The `Err` value through `f`, which may suspend; an `Ok` returns itself. +(: (-> (-?-> %e %f) (Result %e %a) (Result %f %a))) +(defun (result-map-err f r) + (match r + ((Ok v) (Ok v)) + ((Err e) (Err (f e))))) + +(import/class + (ResultRefError (: System.InvalidOperationException (-> string ResultRefError)))) + +;; The `Ok` value. An `Err` raises InvalidOperationException, as +;; `option-ref` does for `None`. +(: (-> (Result %e %a) %a)) +(defun (result-ref r) + (match r + ((Ok v) v) + ((Err _) (raise (cast Exception (ResultRefError. "result-ref on Err")))))) + +;; The `Ok` value, or `fallback` for an `Err`. `fallback` is evaluated +;; first, as for any call. +(: (-> (Result %e %a) %a %a)) +(defun (result-ref-or r fallback) + (match r + ((Ok v) v) + ((Err _) fallback))) + +;; `(Some v)` for `(Ok v)`, and `None` for an `Err`, whose value is dropped. +(: (-> (Result %e %a) (Option %a))) +(defun (result->option r) + (match r + ((Ok v) (Some v)) + ((Err _) None))) + +;; The `Some` value through `f`, which may suspend; `None` returns itself. +(: (-> (-?-> %a %b) (Option %a) (Option %b))) +(defun (option-map f opt) + (match opt + ((Some v) (Some (f v))) + (None None))) + +;; `opt` if `pred` holds for its value, else `None`. `pred` may suspend. +(: (-> (-?-> %a bool) (Option %a) (Option %a))) +(defun (option-filter pred opt) + (match opt + ((Some v) (if (pred v) opt None)) + (None None))) + +;; `opt` if it is a `Some`, else `other`. Both are evaluated, as for any +;; call. +(: (-> (Option %a) (Option %a) (Option %a))) +(defun (option-or opt other) + (match opt + ((Some _) opt) + (None other))) + +;; `(Ok v)` for `(Some v)`, and `(Err e)` for `None`. +(: (-> (Option %a) %e (Result %e %a))) +(defun (option->result opt e) + (match opt + ((Some v) (Ok v)) + (None (Err e)))) + +;; --- Pairs --- + +(: (-> (Tuple %a %b) %a)) +(defun (fst p) + (match p ((Tuple a _) a))) + +(: (-> (Tuple %a %b) %b)) +(defun (snd p) + (match p ((Tuple _ b) b))) + + +;; -------------------------------------------------------------------------- +;; Boxes - a 1-width array +;; -------------------------------------------------------------------------- + +(type (: (Box %a) #:opaque (Struct (: field (Array %a))))) +(impl (Written (Box %a)) + (where (Written %a)) + (defun (write-into box b) + (stringbuilder-add-string! b "(Box ") + (write-into (box-ref box) b) + (stringbuilder-add-string! b ")"))) + +(impl (Displayed (Box %a)) + (where (Displayed %a)) + (defun (display-into box b) + (stringbuilder-add-string! b "(Box ") + (display-into (box-ref box) b) + (stringbuilder-add-string! b ")"))) +(: (-> %a (Box %a))) +(defun (make-box elem) + (def arr (make-array 1)) + (array-set! arr 0 elem) + (Box (field arr))) + +(: (-> (Box %a) %a)) +(defun (box-ref b) + (-> (struct-ref b field) + (array-ref & 0))) + +(: (-> (Box %a) %a Unit)) +(defun (box-set! box val) + (def arr (struct-ref box field)) + (array-set! arr 0 val)) + + + +;; -------------------------------------------------------------------------- +;; Refable and Derefable trait - a "generic" ref function +;; -------------------------------------------------------------------------- +(def/trait (Refable %c) + (type %index) + (type %out) + (: ref (-> %c %index %out)) + (: try-ref (-> %c %index (Option %out))) + + ;; The default: `ref`'s answer as a `Some`, so an impl whose `ref` raises out + ;; of bounds needs no `try-ref` of its own. + (defun (try-ref coll i) + (Some (ref coll i)))) + +(impl (Refable (Array %i)) + (type %index int) + (type %out %i) + (defun (ref arr i) + (array-ref arr i))) + +(impl (Refable (IArray %i)) + (type %index int) + (type %out %i) + (defun (ref xs i) + (iarray-ref xs i)) + (defun (try-ref xs i) + (iarray-try-ref xs i))) + +(impl (Refable (Vec %i)) + (type %index int) + (type %out %i) + (defun (ref vec i) + (vec-ref vec i)) + (defun (try-ref vec i) + (if (and (>= i 0) (< i (vec-length vec))) + (Some (vec-ref vec i)) + None))) + +(impl (Refable (Map %k %v)) + (type %index %k) + (type %out %v) + (defun (ref map k) + (map-ref map k)) + (defun (try-ref map k) + (map-try-ref map k))) + +(impl (Refable string) + (type %index StringCursor) + (type %out char) + (defun (ref str cur) + (string-cursor-ref str cur)) + (defun (try-ref str cur) + (if (string-cursor-end? str cur) + None + (Some (string-cursor-ref str cur))))) + +;; The collections written in place, read by the same `ref`, so that `get` +;; and `try-get` read them and a `set!` path can go through them. +(impl (Refable (VecBuilder %i)) + (type %index int) + (type %out %i) + (defun (ref vb i) + (vecbuilder-ref vb i)) + (defun (try-ref vb i) + (if (and (>= i 0) (< i (vecbuilder-length vb))) + (Some (vecbuilder-ref vb i)) + None))) + +(impl (Refable (TransientVec %i)) + (type %index int) + (type %out %i) + (defun (ref t i) + (transientvec-ref t i)) + (defun (try-ref t i) + (if (and (>= i 0) (< i (transientvec-length t))) + (Some (transientvec-ref t i)) + None))) + +(impl (Refable (TransientMap %k %v)) + (type %index %k) + (type %out %v) + (defun (ref t k) + (transientmap-ref t k)) + (defun (try-ref t k) + (transientmap-try-ref t k))) + +;; Settable — the persistent write that goes with `ref` +;; +;; `(ref-set coll i x)` answers a copy of `coll` with `x` at `i`, and leaves +;; `coll` alone. It is a trait of its own rather than a method of `Refable` +;; because many things that can be read by index cannot sensibly be copied +;; with one slot changed: a string read by cursor, a set read by membership, +;; or a mutable collection, whose write is `!` and in place. +;; +;; `set` and `update` write through it. A Map gains a key it did not have; a +;; Vec index out of range throws. +(def/trait (Settable %c) + (type %index) + (type %in) + (: ref-set (-> %c %index %in %c))) + +(impl (Settable (Vec %i)) + (type %index int) + (type %in %i) + (defun (ref-set vec i x) + (vec-set vec i x))) + +(impl (Settable (Map %k %v)) + (type %index %k) + (type %in %v) + (defun (ref-set map k v) + (map-set map k v))) + +;; Settable! — the write in place that goes with `ref` +;; +;; `(ref-set! coll i x)` writes `x` at `i` in `coll` itself, as `Addable!` +;; adds to a collection in place where `Addable` answers a copy. `set!` with a +;; path writes its last key through it. A persistent collection has no impl, +;; because it has no place to write: `ref-set` answers its copy. +;; +;; A Map, transient or mutable, gains a key it did not have; an Array or a Vec +;; index out of range throws. +(def/trait (Settable! %c) + (type %index) + (type %in) + (: ref-set! (-> %c %index %in void))) + +(impl (Settable! (Array %i)) + (type %index int) + (type %in %i) + (defun (ref-set! arr i x) + (array-set! arr i x))) + +(impl (Settable! (VecBuilder %i)) + (type %index int) + (type %in %i) + (defun (ref-set! vb i x) + (vecbuilder-set! vb i x))) + +(impl (Settable! (TransientVec %i)) + (type %index int) + (type %in %i) + (defun (ref-set! t i x) + (transientvec-set! t i x))) + +(impl (Settable! (TransientMap %k %v)) + (type %index %k) + (type %in %v) + (defun (ref-set! t k v) + (transientmap-set! t k v))) + +(def/trait (Derefable %a) + (type %out) + (: deref (-> %a %out)) + (: try-deref (-> %a (Option %out))) + (defun (try-deref p) (Some (deref p)))) + +(impl (Derefable (Param %b)) + (type %out %b) + (defun (deref p) (parameter-ref p))) + +(impl (Derefable (Box %b)) + (type %out %b) + (defun (deref b) (box-ref b))) + +;; Derefable! — the write in place that goes with `deref` +;; +;; `(deref-set! cell x)` puts `x` in `cell`, as `Settable!` goes with +;; `Settable`. `(set! name value)` writes through it when `name` is not a +;; `def/mutable`: the binding cannot change, so what it holds does. A Param has +;; no impl, since a Param is rebound for an extent with `parameterize` and +;; not written. +(def/trait (Derefable! %a) + (type %in) + (: deref-set! (-> %a %in void))) + +(impl (Derefable! (Box %b)) + (type %in %b) + (defun (deref-set! b x) (box-set! b x))) + + +;; --------------------------------------------------------------------------- +;; Maths +;; --------------------------------------------------------------------------- + +;; Math. + +(import (std maths)) + +;; Remember: exporting one trait function exposes ALL functions. +(re-export abs sign min max clamp zero?) + +;; Square root. The root of a number in m^2 is in m. +(import/extern + (clr-sqrt (: System.Math.Sqrt (-> double double)))) + +(: sqrt (-> (double %u^2) (double %u))) +(defun (sqrt x) (cast (double %u) (clr-sqrt (cast double x)))) + +;; --------------------------------------------------------------------------- +;; Strings, conversions, files and paths +;; --------------------------------------------------------------------------- +;; Wrappers. + +;; --- Strings --- + +;; Invariant throughout. The ambient culture reads and writes "1.5" as "1,5" +;; where the decimal separator is a comma, and writes -100 as "−100" with +;; U+2212 where sv-SE does, so a number written by one machine is unreadable on +;; the next. +(import/extern + (clr-concat (: BjoString.Utf8String.Concat)) + (clr-concat-all (: BjolangRuntime.StringConcat)) + (clr-parse-int (: BjoString.Utf8Number.ParseInt (-> string int))) + (clr-parse-long (: Bjolang.Runtime.BjoNum.ParseLong)) + (clr-parse-double (: BjoString.Utf8Number.ParseDouble (-> string double))) + (clr-double-string (: BjoString.Utf8Number.Format (-> double string))) + (clr-int-string (: BjoString.Utf8Number.Format (-> int string))) + (clr-long-string (: BjoString.Utf8Number.Format (-> long string))) + (clr-byte-string (: BjoString.Utf8Number.Format (-> byte string))) + (clr-parse-decimal (: Bjolang.Runtime.BjoNum.ParseDecimal)) + (clr-parse-bigint (: Bjolang.Runtime.BjoNum.ParseInteger (-> string bigint))) + (clr-parse-int128 (: Bjolang.Runtime.BjoNum.ParseInteger (-> string int128))) + (clr-parse-uint128 (: Bjolang.Runtime.BjoNum.ParseInteger (-> string uint128))) + (clr-format-number (: Bjolang.Runtime.BjoNum.FormatNumber (-> %a string)))) + +(: (-> string string string)) +(defun (string-append a b) (clr-concat a b)) + +;; Variadic concatenation. +(: (-> #:rest string string)) +(defun (str #:rest parts) (clr-concat-all parts)) + +;; String length in storage units: bytes of UTF-8. +(: (-> string int)) +(defun (string-length s) (.-ByteLength s)) + +;; --- Searching and rewriting ----------------------------------------------- +;; Ordinal: bytes of UTF-8 compared as they are. + +(import/extern + (clr-contains (: BjolangRuntime.StringContains)) + (clr-starts-with (: BjolangRuntime.StringStartsWith)) + (clr-ends-with (: BjolangRuntime.StringEndsWith)) + (clr-replace (: BjolangRuntime.StringReplace))) + +(: (-> string string bool)) +(defun (string-contains? s needle) (clr-contains s needle)) + +(: (-> string string bool)) +(defun (string-starts-with? s prefix) (clr-starts-with s prefix)) + +(: (-> string string bool)) +(defun (string-ends-with? s suffix) (clr-ends-with s suffix)) + +;; String replace. +(: (-> string string string string)) +(defun (string-replace s old new) (clr-replace s old new)) + +;; --- Case ------------------------------------------------------------------ +;; Invariant case. +(: (-> string string)) +(defun (string-upcase s) (.ToUpperInvariant s)) + +(: (-> string string)) +(defun (string-downcase s) (.ToLowerInvariant s)) + +;; --- Trimming and padding -------------------------------------------------- +;; Trim whitespace. +(: (-> string string)) +(defun (string-trim s) (.Trim s)) + +(: (-> string string)) +(defun (string-trim-left s) (.TrimStart s)) + +(: (-> string string)) +(defun (string-trim-right s) (.TrimEnd s)) + +;; The /slice forms share the string's bytes rather than copying them, so +;; what they answer keeps the whole string alive. `string-copy` gives such a +;; string bytes of its own, and is the string itself when it has them. +(: (-> string string)) +(defun (string-trim/slice s) (.TrimSlice s)) + +(: (-> string string)) +(defun (string-trim-left/slice s) (.TrimStartSlice s)) + +(: (-> string string)) +(defun (string-trim-right/slice s) (.TrimEndSlice s)) + +(: (-> string string)) +(defun (string-copy s) (.Copy s)) + +;; --- Taking and dropping characters ---------------------------------------- +;; A count of characters from one end of a string. The cursors walk the count, +;; so the cost is in the count and not in the length of the string. A count +;; past the end takes or drops the whole string; a negative count is refused. +;; The /slice forms share the string's bytes, as the trims' do. + +(: (-> string int void)) +(defun (check-count who n) + (when (< n 0) + (count-panic who n))) + +(: (-> string int void)) +(defun (count-panic who n) + (panic! #"${who}: the count is ${n}, and a count of characters or elements is 0 or more.")) + +;; The cursor n characters after c, or the end when there are fewer. +(: (-> string StringCursor int StringCursor)) +(defun (forward-at-most s c n) + (let go ((c c) (k n)) + (if (or (= k 0) (string-cursor-end? s c)) + c + (go (string-cursor-next s c) (- k 1))))) + +;; The cursor n characters before c, or the start when there are fewer. +(: (-> string StringCursor int StringCursor)) +(defun (back-at-most s c n) + (def start (string-cursor-start s)) + (let go ((c c) (k n)) + (if (or (= k 0) (string-cursor=? c start)) + c + (go (string-cursor-prev s c) (- k 1))))) + +(: (-> string StringCursor int StringCursor)) +(defun (string-cursor-forward s c n) + (check-count "string-cursor-forward" n) + (let go ((c c) (k n)) + (if (= k 0) c (go (string-cursor-next s c) (- k 1))))) + +(: (-> string StringCursor int StringCursor)) +(defun (string-cursor-back s c n) + (check-count "string-cursor-back" n) + (let go ((c c) (k n)) + (if (= k 0) c (go (string-cursor-prev s c) (- k 1))))) + +(: (-> string int string)) +(defun (string-take s n) + (check-count "string-take" n) + (substring s (string-cursor-start s) (forward-at-most s (string-cursor-start s) n))) + +(: (-> string int string)) +(defun (string-drop s n) + (check-count "string-drop" n) + (substring s (forward-at-most s (string-cursor-start s) n) (string-cursor-end s))) + +(: (-> string int string)) +(defun (string-take-right s n) + (check-count "string-take-right" n) + (substring s (back-at-most s (string-cursor-end s) n) (string-cursor-end s))) + +(: (-> string int string)) +(defun (string-drop-right s n) + (check-count "string-drop-right" n) + (substring s (string-cursor-start s) (back-at-most s (string-cursor-end s) n))) + +(: (-> string int string)) +(defun (string-take/slice s n) + (check-count "string-take/slice" n) + (substring/slice s (string-cursor-start s) (forward-at-most s (string-cursor-start s) n))) + +(: (-> string int string)) +(defun (string-drop/slice s n) + (check-count "string-drop/slice" n) + (substring/slice s (forward-at-most s (string-cursor-start s) n) (string-cursor-end s))) + +(: (-> string int string)) +(defun (string-take-right/slice s n) + (check-count "string-take-right/slice" n) + (substring/slice s (back-at-most s (string-cursor-end s) n) (string-cursor-end s))) + +(: (-> string int string)) +(defun (string-drop-right/slice s n) + (check-count "string-drop-right/slice" n) + (substring/slice s (string-cursor-start s) (back-at-most s (string-cursor-end s) n))) + +;; The text before the first sep and the text after it, or None when sep is +;; not in s. +(: (-> string string (Option (Tuple string string)))) +(defun (string-split-once s sep) + (option-map (fun (c) + (Tuple (substring s (string-cursor-start s) c) + (substring s (string-cursor-forward s c (string-count sep)) (string-cursor-end s)))) + (string-search s sep))) + +(: (-> string string (Option (Tuple string string)))) +(defun (string-split-once/slice s sep) + (option-map (fun (c) + (Tuple (substring/slice s (string-cursor-start s) c) + (substring/slice s (string-cursor-forward s c (string-count sep)) (string-cursor-end s)))) + (string-search s sep))) + +;; Pad width. +(: (-> string int (#:with char) string)) +(defun (string-pad-left s width #:with #\space) + (let ((b (stringbuilder-empty))) + (string-pad-fill! b with (- width (string-count s))) + (stringbuilder-add-string! b s) + (stringbuilder->string b))) + +(: (-> string int (#:with char) string)) +(defun (string-pad-right s width #:with #\space) + (let ((b (stringbuilder-empty))) + (stringbuilder-add-string! b s) + (string-pad-fill! b with (- width (string-count s))) + (stringbuilder->string b))) + +;; Pad fill. +(: (-> StringBuilder char int void)) +(defun (string-pad-fill! b c n) + (let fill ((i 0)) + (when (< i n) + (stringbuilder-add! b c) + (fill (+ i 1))))) + +;; --- Splitting and joining ------------------------------------------------- + +;; String split, empty fields kept. The runtime gives the array the split +;; makes to the vec without copying it, which only it can do safely. +(import/extern + (clr-split (: BjolangRuntime.SplitToVec (-> string string (Vec string)))) + (clr-split/slices (: BjolangRuntime.SplitSlicesToVec (-> string string (Vec string))))) + +(: (-> string string (Vec string))) +(defun (string-split s sep) (clr-split s sep)) + +;; The fields as slices of `s`, sharing its bytes. +(: (-> string string (Vec string))) +(defun (string-split/slice s sep) (clr-split/slices s sep)) + +(: (-> (Vec string) string string)) +(defun (string-join parts sep) + (def b (stringbuilder-empty)) + (loop (:with first? #t #f) + (:for part parts) + (:do (unless first? (stringbuilder-add-string! b sep)) + (stringbuilder-add-string! b part))) + (stringbuilder->string b)) + +;; --- Conversions --- +;; ToString wrappers. Each is invariant, so each round-trips through the +;; matching reader on any machine. See the externs above. + +(: (-> int string)) +(defun (int->string n) (clr-int-string n)) + +(: (-> long string)) +(defun (long->string n) (clr-long-string n)) + +(: (-> double string)) +(defun (double->string n) (clr-double-string n)) + +(: (-> byte string)) +(defun (byte->string n) (clr-byte-string n)) + +;; Number to string. +(: (-> int string)) +(defun (number->string n) (int->string n)) + +(: (-> string int)) +(defun (string->int s) (clr-parse-int s)) + +(: (-> string long)) +(defun (string->long s) (clr-parse-long s)) + +(: (-> string double)) +(defun (string->double s) (clr-parse-double s)) + +;; The four numbers past the eight. A decimal is written and read with a point +;; and no exponent when it has none: (decimal->string 1.50M) is "1.50", as +;; .NET keeps a decimal's scale. +(: (-> decimal string)) +(defun (decimal->string n) (clr-format-number n)) + +(: (-> string decimal)) +(defun (string->decimal s) (clr-parse-decimal s)) + +(: (-> bigint string)) +(defun (bigint->string n) (clr-format-number n)) + +(: (-> string bigint)) +(defun (string->bigint s) (clr-parse-bigint s)) + +(: (-> int128 string)) +(defun (int128->string n) (clr-format-number n)) + +(: (-> string int128)) +(defun (string->int128 s) (clr-parse-int128 s)) + +(: (-> uint128 string)) +(defun (uint128->string n) (clr-format-number n)) + +(: (-> string uint128)) +(defun (string->uint128 s) (clr-parse-uint128 s)) + +;; --- Bytes and strings --- +;; +;; An encoding is a .NET `System.Text.Encoding`. `(text encodings)` has one +;; value for each encoding .NET knows, and finds them by name; UTF-8 is the +;; default here and needs no import. +;; +;; Both directions are strict, as Racket's `bytes->string/utf-8` is: what +;; does not decode, or has no bytes in the encoding, is an error naming its +;; offset. `#:replace (Some c)` puts `c` there instead. A port is the +;; lenient one: it reads what does not decode as U+FFFD. + +(type (: Encoding System.Text.Encoding)) + +(import/extern + (clr-utf-8 (: Bjolang.Runtime.Encodings.Cp65001 #:get)) + (clr-decode (: Bjolang.Runtime.Encodings.Decode (-> (Array byte) int int Encoding string))) + (clr-decode/replace (: Bjolang.Runtime.Encodings.Decode (-> (Array byte) int int Encoding char string))) + (clr-encode (: Bjolang.Runtime.Encodings.Encode (-> string Encoding (Array byte)))) + (clr-encode/replace (: Bjolang.Runtime.Encodings.Encode (-> string Encoding char (Array byte)))) + ;; Whether the bytes from `start` to `end` are UTF-8, which is what + ;; `bytes->string` would accept. Vectorized: a reader that copies runs of + ;; bytes checks a run with it in one call. + (bytes-utf-8? (: Bjolang.Runtime.Encodings.IsUtf8 (-> (Array byte) int int bool)))) + +;; The text of the bytes from `#:start` to `#:end`, in `#:encoding`. A string +;; is UTF-8, so from UTF-8 this checks the bytes and copies them. A byte order +;; mark is a character like any other. +(: (-> (Array byte) (#:encoding Encoding) (#:start int) (#:end int) (#:replace (Option char)) string)) +(defun (bytes->string bytes #:encoding clr-utf-8 #:start 0 #:end (array-length bytes) #:replace None) + (match replace + (None (clr-decode bytes start end encoding)) + ((Some c) (clr-decode/replace bytes start end encoding c)))) + +;; The string's bytes in `#:encoding`, without a byte order mark. To UTF-8 +;; this is a copy of the string's own bytes. +(: (-> string (#:encoding Encoding) (#:replace (Option char)) (Array byte))) +(defun (string->bytes s #:encoding clr-utf-8 #:replace None) + (match replace + (None (clr-encode s encoding)) + ((Some c) (clr-encode/replace s encoding c)))) + +;; A copy of bytes, checked to be UTF-8 once, for a reader that makes many +;; strings from one buffer: each string is taken by byte offset and is not +;; checked again. The chunk owns its copy, and takes a string only between +;; character boundaries in its valid part, so a string made from it is valid +;; whatever the offsets are. `BjolangRuntime/Utf8Chunk.cs` says why. +(type (: Utf8Chunk Bjolang.Runtime.Utf8Chunk)) + +(import/extern + (clr-utf8-chunk (: Bjolang.Runtime.Utf8Chunk.FromBytes (-> (Array byte) int int Utf8Chunk))) + (utf8-chunk-length (: Bjolang.Runtime.Utf8Chunk.Length (-> Utf8Chunk int) #:get)) + (utf8-chunk-valid-end (: Bjolang.Runtime.Utf8Chunk.ValidEnd (-> Utf8Chunk int) #:get)) + (utf8-chunk-substring (: Bjolang.Runtime.Utf8Chunk.Substring (-> Utf8Chunk int int string)))) + +(: (-> (Array byte) (#:start int) (#:end int) Utf8Chunk)) +(defun (bytes->utf8-chunk bytes #:start 0 #:end (array-length bytes)) + (clr-utf8-chunk bytes start end)) + +;; --------------------------------------------------------------------------- +;; Characters +;; --------------------------------------------------------------------------- +;; Character ordering. +(: (-> char char bool)) +(defun (char=? a b) (= a b)) + +(: (-> char char bool)) +(defun (char char char bool)) +(defun (char>? a b) (> a b)) + +(: (-> char char bool)) +(defun (char<=? a b) (<= a b)) + +(: (-> char char bool)) +(defun (char>=? a b) (>= a b)) + +;; Case-insensitive comparison. +(: (-> char char bool)) +(defun (char-ci=? a b) (= (char-foldcase a) (char-foldcase b))) + +(: (-> char char bool)) +(defun (char-ci char char bool)) +(defun (char-ci>? a b) (> (char-foldcase a) (char-foldcase b))) + +(: (-> char char bool)) +(defun (char-ci<=? a b) (<= (char-foldcase a) (char-foldcase b))) + +(: (-> char char bool)) +(defun (char-ci>=? a b) (>= (char-foldcase a) (char-foldcase b))) + +;; --------------------------------------------------------------------------- +;; String cursors +;; --------------------------------------------------------------------------- +;; String cursors. +(: (-> StringCursor StringCursor bool)) +(defun (string-cursor=? a b) (= a b)) + +(: (-> StringCursor StringCursor bool)) +(defun (string-cursor StringCursor StringCursor bool)) +(defun (string-cursor>? a b) (> a b)) + +(: (-> StringCursor StringCursor bool)) +(defun (string-cursor<=? a b) (<= a b)) + +(: (-> StringCursor StringCursor bool)) +(defun (string-cursor>=? a b) (>= a b)) + +;; --- Traversal ------------------------------------------------------------- + +;; String fold. +;; The forward walks below ask for the character and the step in one call, so +;; that the unit at the cursor is read and bounds-checked once per character +;; rather than twice. +(: (-> (-> char %a %a) %a string %a)) +(defun (string-fold f init s) + (let loop ((c (string-cursor-start s)) (acc init)) + (if (string-cursor-end? s c) + acc + (match (string-cursor-ref+next s c) + ((Tuple ch next) (loop next (f ch acc))))))) + +;; String fold right. +(: (-> (-> char %a %a) %a string %a)) +(defun (string-fold-right f init s) + (let loop ((c (string-cursor-end s)) (acc init)) + (if (string-cursor=? c (string-cursor-start s)) + acc + (let ((p (string-cursor-prev s c))) + (loop p (f (string-cursor-ref s p) acc)))))) + +(: (-> (-> char void) string void)) +(defun (string-for-each f s) + ;; (loop (:for ch s) + ;; (:do (f ch))) + (let loop ((c (string-cursor-start s))) + (unless (string-cursor-end? s c) + (def (Tuple ch next) (string-cursor-ref+next s c)) + (f ch) + (loop next)))) + +;; The cursor on the first character of `s` that `pred` holds for. +(: (-> (-> char bool) string (Option StringCursor))) +(defun (string-find pred s) + (let loop ((c (string-cursor-start s))) + (if (string-cursor-end? s c) + None + (match (string-cursor-ref+next s c) + ((Tuple ch next) + (if (pred ch) + (Some c) + (loop next))))))) + +;; --- Conversions ----------------------------------------------------------- + +(: (-> string (List char))) +(defun (string->list s) + (string-fold-right cons Nil s)) + +(: (-> (List char) string)) +(defun (list->string cs) + (def b (stringbuilder-empty)) + (list-for-each #(stringbuilder-add! b &1) cs) + (stringbuilder->string b)) + +(: (-> string (Vec char))) +(defun (string->vec s) + (def b (vecbuilder-empty)) + (string-for-each #(vecbuilder-add! b &1) s) + (vecbuilder->vec b)) + +;; Reverse string. +(: (-> string string)) +(defun (string-reverse s) (.Reverse s)) + +;; --- File operations. +;; +;; Everything in this section and the directory one below reaches the disk +;; through the operations of the `FS` effect, declared with the ports further +;; down, so a fake filesystem answers for all of it. + +;; How an output file is opened. +;; +;; - `Truncate`: create the file, or empty it if it exists. +;; - `Append`: create the file, or write after what it holds. +;; - `CreateNew`: create the file, and fail if it exists. The check and the +;; create are one system call, so two writers cannot both succeed. +(type (: WriteMode (Union Truncate Append CreateNew))) + +;; What a path names, when it names anything. +(type (: FsKind (Union FsFile FsDirectory))) + +;; What a move does when something is already at the target. +;; +;; - `MoveNew`: fails, and changes nothing. +;; - `MoveReplace`: a file at the target is replaced, in the one rename that +;; moves the new one there, so nothing ever sees the target missing. A +;; directory is never replaced, whichever of the two it is. +(type (: MoveMode (Union MoveNew MoveReplace))) + +;; What `(fs-info path)` answers about a path that names something. One answer +;; to one question, so that a handler cannot say a path is a file and then +;; forget to say how big it is. +;; +;; - `size`: in bytes. 0 for a directory. +;; - `modified-ms`: the last write, in Unix milliseconds, UTC. Milliseconds +;; because `(std datetime)`'s `unix-milliseconds->instant` takes them, and +;; the prelude cannot name an `Instant` itself. +;; - `link?`: the path itself is a symbolic link. The other fields are what +;; .NET reports for it; `directory-walk` does not enter one, so a link back up +;; the tree cannot loop it. +(type (: FsInfo (Record (: kind FsKind) + (: size long) + (: modified-ms long) + (: link? bool)))) + +;; The operations answer a `Result`; the functions in these two sections raise +;; the `Err`, as the .NET methods they replaced did. +(: (-> (Result System.Exception %a) %a)) +(defun (raise-err r) + (match r + ((Ok v) v) + ((Err e) (raise e)))) + +;; Read all text. +(: (-> string string)) +(defun (file-read-text path) + (match (open-input-file path) + ((Ok opened) (with-open ((p opened)) (read-all p))) + ((Err e) (raise e)))) + +;; The flush is explicit so that the suspending copy awaits it. Closing would +;; flush too, but synchronously, on whatever thread runs the close. +(: (-> string string WriteMode void)) +(defun (write-whole-file path contents mode) + (match (open-output-file path #:mode mode) + ((Ok opened) (with-open ((p opened)) + (write-string p contents) + (flush-port p))) + ((Err e) (raise e)))) + +;; Write text. +(: (-> string string void)) +(defun (file-write-text path contents) (write-whole-file path contents Truncate)) + +(: (-> string string void)) +(defun (file-append-text path contents) (write-whole-file path contents Append)) + +(: (-> string (Option FsInfo))) +(defun (file-info path) (fs-info path)) + +;; What a path names, for the functions below that only ask that. +(: (-> string (Option FsKind))) +(defun (kind-of path) + (option-map (fun (info) (record-ref info kind)) (fs-info path))) + +(: (-> string bool)) +(defun (file-exists? path) + (match (kind-of path) + ((Some FsFile) #t) + (_ #f))) + +;; A missing file is not an error, as with `File.Delete`. +(: (-> string void)) +(defun (file-delete path) (raise-err (fs-delete-file path))) + +;; Moves or renames a file or a directory. Fails if `to` exists, unless +;; `#:mode MoveReplace` says to replace a file there. +(: (-> string string (#:mode MoveMode) void)) +(defun (file-move from to #:mode MoveNew) (raise-err (fs-move from to mode))) + +;; --- Paths --- +;; Path strings. + +(import/extern + (clr-full-path (: System.IO.Path.GetFullPath)) + (clr-path-combine (: System.IO.Path.Combine)) + (clr-file-name (: System.IO.Path.GetFileName)) + (clr-file-extension (: System.IO.Path.GetExtension))) + +(: (-> string string)) +(defun (path-absolute path) (clr-full-path path)) + +;; Path combine. +(: (-> #:rest string string)) +(defun (path-combine #:rest parts) (clr-path-combine parts)) + +;; Path filename. +(: (-> string (Option string))) +(defun (path-filename path) + (def name (clr-file-name path)) + (if (string-empty? name) None (Some name))) + +;; File extension. +(: (-> string (Option string))) +(defun (path-file-extension path) + (def ext (clr-file-extension path)) + (if (string-empty? ext) None (Some ext))) + +;; --- Directories --- + +(: (-> string bool)) +(defun (directory-exists? path) + (match (kind-of path) + ((Some FsDirectory) #t) + (_ #f))) + +;; Create directory. +(: (-> string void)) +(defun (directory-create path) (raise-err (fs-create-directory path))) + +;; Delete directory. +(: (-> string void)) +(defun (directory-delete path) (raise-err (fs-delete-directory path))) + +;; Built from the operations rather than handed to `Directory.Delete` with its +;; recursive flag, so that a handler sees every deletion. +;; +;; A symbolic link to a directory is removed as a link and never entered, and +;; that includes `path` itself. Entering one would delete what it points at: a +;; link to the home directory inside a build directory would empty the home +;; directory, and a link back up the tree would never end. +(: (-> string void)) +(defun (directory-delete-tree path) + (if (directory-link? path) + (directory-delete path) + (let () + (loop (:for entry (raise-err (fs-list path))) + (:do (match (fs-info entry) + ((Some info) + (match (record-ref info kind) + (FsDirectory (directory-delete-tree entry)) + (FsFile (file-delete entry)))) + ;; A link to nothing, which `file-delete` removes. + (None (file-delete entry))))) + (directory-delete path)))) + +(: (-> string bool)) +(defun (directory-link? path) + (match (fs-info path) + ((Some info) (and (record-ref info link?) + (match (record-ref info kind) (FsDirectory #t) (FsFile #f)))) + (None #f))) + +(: (-> string FsKind (Vec string))) +(defun (entries-of-kind path kind) + (loop (:for entry (raise-err (fs-list path))) + (:when (match (Tuple (kind-of entry) kind) + ((Tuple (Some FsFile) FsFile) #t) + ((Tuple (Some FsDirectory) FsDirectory) #t) + (_ #f))) + (:acc found (vecing entry)))) + +;; Directory files. +(: (-> string (Vec string))) +(defun (directory-files path) (entries-of-kind path FsFile)) + +(: (-> string (Vec string))) +(defun (directory-subdirectories path) (entries-of-kind path FsDirectory)) + +;; --- The environment --- + +(import/extern + (clr-get-env (: System.Environment.GetEnvironmentVariable)) + (clr-set-env (: System.Environment.SetEnvironmentVariable)) + (clr-empty? (: System.String.IsNullOrEmpty)) + (clr-cwd (: System.Environment.CurrentDirectory #:get)) + (clr-set-cwd (: System.Environment.CurrentDirectory #:set))) + +(: (-> string (Option string))) +(defun (get-environment-variable name) + (def value (clr-get-env name)) + (if (clr-empty? value) + None + (Some value))) + +;; Set environment variable. +(: (-> string string void)) +(defun (set-environment-variable! name value) (clr-set-env name value)) + +(: (-> string)) +(defun (current-directory) clr-cwd) + +(: (-> string void)) +(defun (set-current-directory! path) (clr-set-cwd path)) + +;; --------------------------------------------------------------------------- +;; Ports +;; --------------------------------------------------------------------------- + +(import/class + ;; What the `FS` effect's openers answer, and what a file port is over. + (ClrStream (: System.IO.Stream)) + ;; `IOException` covers `CreateNew` finding the file already there, as well + ;; as the not-found exceptions below it. + (FileStream (: System.IO.FileStream (-> string FileMode FileAccess FileShare FileStream) + #:exceptions (System.IO.IOException + System.UnauthorizedAccessException))) + (FileMode (: System.IO.FileMode)) + (FileAccess (: System.IO.FileAccess)) + (FileShare (: System.IO.FileShare)) + + (Exception (: System.Exception))) + +;; Port types. +;; +;; One input port type and one output port type, as in Racket: a port is +;; bytes. Text reads decode UTF-8 from an input port, and text writes encode +;; UTF-8 into an output port, so byte and text operations mix on one port. +;; They are also a `System.IO.TextReader` and a `System.IO.TextWriter`, so a +;; .NET API that takes one takes a port. +(type + (: InputPort Bjolang.Runtime.BjoInputPort) + (: OutputPort Bjolang.Runtime.BjoOutputPort)) + +;; What ends a line: Racket's `read-line` modes. `Any` is the default +;; everywhere a line is read. +(type (: LineMode (Union Any AnyOne Linefeed Return ReturnLinefeed))) + +;; --- Opening --- +;; +;; A file port is owned by the ambient scope: when the scope returns, the file +;; is closed, whether the program said so or not. A string port is not - it is +;; memory, and the GC prices memory. +;; +;; The scope is read before the file is opened, so a program with no scope to +;; put the port in fails without having opened anything. + +(import/extern + ;; A port over a stream, registered on the ambient scope. + (owned-port-over (: Bjolang.Runtime.BytePorts.FromStream (-> ClrStream InputPort))) + (string-port (: Bjolang.Runtime.InputPorts.FromString (-> string InputPort))) + (owned-port-out (: Bjolang.Runtime.OutputPorts.FromStream (-> ClrStream OutputPort))) + (string-port-out (: Bjolang.Runtime.OutputPorts.OpenString (-> OutputPort))) + (output-bytes (: Bjolang.Runtime.OutputPorts.GetBytes (-> OutputPort (Array byte)))) + (close-in (: BjolangRuntime.CloseInput (-> InputPort void))) + (close-out (: BjolangRuntime.CloseOutput (-> OutputPort void))) + (close-out-await (: BjolangRuntime.CloseOutputAsync (-> OutputPort void) #:async))) + +;; An input port over a stream an opener answered, registered on the ambient +;; scope. Whatever the stream is - a file, or a `MemoryStream` a fake +;; filesystem made - the port is owned the same way. +(: (-> (Result Exception ClrStream) (Result Exception InputPort))) +(defun (stream->input-port opened) + (match opened + ((Ok s) (Ok (owned-port-over s))) + ((Err e) (Err e)))) + +;; The same for an output port: a file port is buffered a block at a time, and +;; closing it, or its scope ending, writes what is left. +(: (-> (Result Exception ClrStream) (Result Exception OutputPort))) +(defun (stream->output-port opened) + (match opened + ((Ok s) (Ok (owned-port-out s))) + ((Err e) (Err e)))) + +;; Open input string: a port over the string's own UTF-8 bytes, shared rather +;; than copied. Memory only, so nothing registers it. +(: (-> string InputPort)) +(defun (open-input-string contents) + (string-port contents)) + +;; Open output string: a port that keeps what is written to it, read back with +;; `get-output-string` or `get-output-bytes`. Memory only, so nothing +;; registers it. +(: (-> OutputPort)) +(defun (open-output-string) + (string-port-out)) + +;; --- Closing --- +;; +;; Closing goes through the owner handle rather than calling `.Dispose`. A +;; direct dispose would close the stream and leave the registration on the +;; scope's list, so a long-lived scope would collect one spent entry per file it +;; had already finished with. +;; +;; On a port nothing owns this disposes, which is what a string port wants. On +;; one of the standard ports it flushes and stops: nothing in the language +;; should be able to take stdout away from the rest of the program. +(: (-> InputPort void)) +(defun (close-input-port p) (close-in p)) + +;; What is still buffered goes out first. On a fiber that write suspends, so the +;; close works over a stream that refuses blocking I/O, as Kestrel's do; the +;; port is closed even when the write fails, and the failure raised after. +;; `with-open` and a scope ending close with a blocking write, which after a +;; `flush-port` has nothing left to do. +(: (-> OutputPort void)) +(defbjouble (close-output-port p) + (#:sync (close-out p)) + (#:bjo (close-out-await p))) + +;; --- The filesystem is an effect --- +;; +;; All filesystem operations route through the `FS` algebraic effect, allowing +;; callers to intercept disk operations (e.g., in-memory mocks for tests, sandbox +;; write restrictions, or build dependency tracking). When unhandled, operations +;; fall back to their `/raw` defaults using standard .NET calls. +;; +;; Design: +;; - Openers return raw `ClrStream`s rather than ports. Higher-level functions +;; (like `open-input-file` or `(std ports)` byte openers) construct ports and +;; register them in the ambient scope, allowing mock handlers to work uniformly +;; across text and byte I/O. +;; - Effect operations are synchronous; coroutine suspension occurs only when reading +;; or writing through the underlying streams/ports. +;; - Effect operations return `(Result Exception %a)` values, whereas higher-level +;; prelude wrappers unwrap them and raise exceptions on error. + +(import/class + (DirectoryInfo (: System.IO.DirectoryInfo (-> string DirectoryInfo))) + (FileInfo (: System.IO.FileInfo (-> string FileInfo)))) + +(import/extern + (clr-file-exists (: System.IO.File.Exists (-> string bool))) + (clr-dir-exists (: System.IO.Directory.Exists (-> string bool))) + (clr-dir-entries (: System.IO.Directory.GetFileSystemEntries (-> string (Array string)) + #:exceptions (System.IO.IOException System.UnauthorizedAccessException))) + (clr-dir-create (: System.IO.Directory.CreateDirectory (-> string DirectoryInfo) + #:exceptions (System.IO.IOException System.UnauthorizedAccessException))) + (clr-file-delete (: System.IO.File.Delete (-> string void) + #:exceptions (System.IO.IOException System.UnauthorizedAccessException))) + (clr-dir-delete (: System.IO.Directory.Delete (-> string void) + #:exceptions (System.IO.IOException System.UnauthorizedAccessException))) + (clr-file-move (: System.IO.File.Move (-> string string bool void) + #:exceptions (System.IO.IOException System.UnauthorizedAccessException))) + (clr-dir-move (: System.IO.Directory.Move (-> string string void) + #:exceptions (System.IO.IOException System.UnauthorizedAccessException)))) + +;; Read-only, and other readers allowed alongside, as `StreamReader`'s own +;; path constructor opens. +(: (-> string (Result Exception ClrStream))) +(defun (open-read-stream/raw path) + (match (FileStream. path FileMode.Open FileAccess.Read FileShare.Read) + ((Ok s) (Ok (cast ClrStream s))) + ((Err e) (Err e)))) + +(: (-> WriteMode FileMode)) +(defun (write-mode->file-mode mode) + (match mode + (Truncate FileMode.Create) + (Append FileMode.Append) + (CreateNew FileMode.CreateNew))) + +;; Write-only, readers allowed alongside, as `StreamWriter`'s path constructor +;; opens. +(: (-> string WriteMode (Result Exception ClrStream))) +(defun (open-write-stream/raw path mode) + (match (FileStream. path (write-mode->file-mode mode) FileAccess.Write FileShare.Read) + ((Ok s) (Ok (cast ClrStream s))) + ((Err e) (Err e)))) + +;; .NET ticks at 1970-01-01T00:00Z, and per millisecond. +(: long) +(def unix-epoch-ticks 621355968000000000L) + +;; Rounds toward the past, so a write just before 1970 is -1 and not 0. +(: (-> long long)) +(defun (ticks->unix-ms ticks) + (def d (- ticks unix-epoch-ticks)) + (def q (/ d 10000L)) + (if (< (% d 10000L) 0L) (- q 1L) q)) + +;; `File.Exists` and `Directory.Exists` first, because they never throw: a +;; path that is no path at all, "" say, is `None` rather than an +;; `ArgumentException` out of the `FileInfo` constructor. The info object then +;; reads the path once, on its first property, so size and time agree with +;; each other; a file deleted in between is `None`. +;; Where a chain of symbolic links ends: `path` itself when it is no link. A +;; link's target is relative to the directory the link is in. At most 40 +;; steps, as Linux allows, so that a loop of links ends. Read step by step +;; because `ResolveLinkTarget` answers null for a link to nothing, and there is +;; no null to test here. +(: (-> string string)) +(defun (link-end path) + (let loop ((p path) (steps 0)) + (def target (.-LinkTarget (FileInfo. p))) + (if (or (clr-empty? target) (= steps 40)) + p + (loop (match (path-directory p) + ((Some dir) (path-combine dir target)) + (None target)) + (+ steps 1))))) + +;; A link's size and time are those of the file it leads to, which is what +;; reading it reads; `link?` is what says it is one. A link to nothing keeps +;; its own. +(: (-> string (Option FsInfo))) +(defun (file-info/raw path) + (def own (FileInfo. path)) + (def link (not (clr-empty? (.-LinkTarget own)))) + (def end (if link (FileInfo. (link-end path)) own)) + (def f (if (.-Exists end) end own)) + (if (.-Exists own) + (Some (FsInfo (kind FsFile) + (size (.-Length f)) + (modified-ms (ticks->unix-ms (.-Ticks (.-LastWriteTimeUtc f)))) + (link? link))) + None)) + +(: (-> string (Option FsInfo))) +(defun (directory-info/raw path) + (def own (DirectoryInfo. path)) + (def link (not (clr-empty? (.-LinkTarget own)))) + (def end (if link (DirectoryInfo. (link-end path)) own)) + (def d (if (.-Exists end) end own)) + (if (.-Exists own) + (Some (FsInfo (kind FsDirectory) + (size 0L) + (modified-ms (ticks->unix-ms (.-Ticks (.-LastWriteTimeUtc d)))) + (link? link))) + None)) + +(: (-> string (Option FsInfo))) +(defun (fs-info/raw path) + (if (clr-file-exists path) + (file-info/raw path) + (if (clr-dir-exists path) (directory-info/raw path) None))) + +;; Whole paths, files and directories both, in the order the OS gives them. +(: (-> string (Result Exception (Vec string)))) +(defun (fs-list/raw path) + (match (clr-dir-entries path) + ((Ok names) (Ok (loop (:for name names) (:acc found (vecing name))))) + ((Err e) (Err e)))) + +;; Creates every missing directory along the path. One that is already there is +;; not an error. +(: (-> string (Result Exception Unit))) +(defun (fs-create-directory/raw path) + (match (clr-dir-create path) + ((Ok _) (Ok unit)) + ((Err e) (Err e)))) + +;; What a guarded `void` call answers, as the `Unit` an operation declares. +(: (-> (Result Exception %a) (Result Exception Unit))) +(defun (unit-result r) + (match r + ((Ok _) (Ok unit)) + ((Err e) (Err e)))) + +;; A missing file is not an error. +(: (-> string (Result Exception Unit))) +(defun (fs-delete-file/raw path) (unit-result (clr-file-delete path))) + +;; An empty directory. A missing or non-empty one is an `IOException`. +(: (-> string (Result Exception Unit))) +(defun (fs-delete-directory/raw path) (unit-result (clr-dir-delete path))) + +;; A file or a directory. Fails if `to` exists, unless `MoveReplace` asks to +;; replace a file there. `Directory.Move` never replaces, which is the rule. +(: (-> string string MoveMode (Result Exception Unit))) +(defun (fs-move/raw from to mode) + (unit-result (if (clr-dir-exists from) + (clr-dir-move from to) + (clr-file-move from to (match mode (MoveNew #f) (MoveReplace #t)))))) + +(defeffect FS + (open-read-stream (-> string (Result Exception ClrStream))) + (open-write-stream (-> string WriteMode (Result Exception ClrStream))) + (fs-info (-> string (Option FsInfo))) + (fs-list (-> string (Result Exception (Vec string)))) + (fs-create-directory (-> string (Result Exception Unit))) + (fs-delete-file (-> string (Result Exception Unit))) + (fs-delete-directory (-> string (Result Exception Unit))) + (fs-move (-> string string MoveMode (Result Exception Unit))) + #:default ((open-read-stream open-read-stream/raw) + (open-write-stream open-write-stream/raw) + (fs-info fs-info/raw) + (fs-list fs-list/raw) + (fs-create-directory fs-create-directory/raw) + (fs-delete-file fs-delete-file/raw) + (fs-delete-directory fs-delete-directory/raw) + (fs-move fs-move/raw)) + #:cold (open-read-stream open-write-stream fs-info fs-list + fs-create-directory fs-delete-file fs-delete-directory fs-move)) + +;; --- Opening a file port --- +;; +;; The scope is read before the perform, so a program with nowhere to put the +;; port fails without having opened anything. + +(: (-> string (Result Exception InputPort))) +(defun (open-input-file path) + (ignore (current-scope)) + (stream->input-port (open-read-stream path))) + +;; `(open-output-file path)` truncates; `#:mode Append` or `#:mode CreateNew` +;; says otherwise. +(: (-> string (#:mode WriteMode) (Result Exception OutputPort))) +(defun (open-output-file path #:mode Truncate) + (ignore (current-scope)) + (stream->output-port (open-write-stream path mode))) + +;; The same, past whatever handler is installed. The `/raw` says so. +(: (-> string (Result Exception InputPort))) +(defun (open-input-file/raw path) + (ignore (current-scope)) + (stream->input-port (open-read-stream/raw path))) + +(: (-> string WriteMode (Result Exception OutputPort))) +(defun (open-output-file/raw path mode) + (ignore (current-scope)) + (stream->output-port (open-write-stream/raw path mode))) + +;; Opens a file, passes the open port to `proc`, and guarantees port closure +;; on normal return or exception via `with-open`. +;; +;; Unlike `with-scope`, `with-open` avoids allocating a new fiber scope, allowing +;; synchronous functions to call it without requiring a coroutine context. +;; +;; `proc` is suspension-polymorphic (`-?->`): both synchronous and async/suspending +;; variants are compiled, ensuring suspending bodies receive identical cleanup. +(: (-> string (-?-> InputPort %a) (Result Exception %a))) +(defun (call-with-input-file path proc) + (match (open-input-file path) + ((Ok p) (with-open ((it p)) (Ok (proc it)))) + ((Err e) (Err e)))) + +(: (-> string (-?-> OutputPort %a) (#:mode WriteMode) (Result Exception %a))) +(defun (call-with-output-file path proc #:mode Truncate) + (match (open-output-file path #:mode mode) + ((Ok p) (with-open ((it p)) (Ok (proc it)))) + ((Err e) (Err e)))) + +;; --- Bytes, copies and lines --- +;; +;; Because the prelude has no byte port type, byte I/O and file copying operate +;; directly on `ClrStream`s returned by the `FS` effect openers (`open-read-stream` +;; and `open-write-stream`). +;; +;; Like the text port functions, these helpers provide both sync and suspending +;; (async) variants backed by the streams' async methods. When used in `with-open`, +;; a stream is explicitly flushed before exiting so pending writes are awaited +;; asynchronously rather than blocking during synchronous disposal. + +(import/class + (MemoryStream (: System.IO.MemoryStream (-> MemoryStream))) + (IOException (: System.IO.IOException (-> string IOException)))) + +(import/extern + (clr-copy-to (: System.IO.Stream.CopyTo (-> ClrStream ClrStream void) #:blocking)) + (clr-copy-to-await (: System.IO.Stream.CopyToAsync (-> ClrStream ClrStream void) #:async)) + (clr-stream-write (: System.IO.Stream.Write (-> ClrStream (Array byte) int int void) #:blocking)) + (clr-stream-write-await (: System.IO.Stream.WriteAsync (-> ClrStream (Array byte) int int void) #:async)) + (clr-stream-flush (: System.IO.Stream.Flush (-> ClrStream void) #:blocking)) + (clr-stream-flush-await (: System.IO.Stream.FlushAsync (-> ClrStream void) #:async))) + +(: (-> ClrStream ClrStream void)) +(defbjouble (stream-copy! from to) + (#:sync (clr-copy-to from to)) + (#:bjo (clr-copy-to-await from to))) + +(: (-> ClrStream (Array byte) void)) +(defbjouble (stream-write! s bytes) + (#:sync (clr-stream-write s bytes 0 (array-length bytes))) + (#:bjo (clr-stream-write-await s bytes 0 (array-length bytes)))) + +(: (-> ClrStream void)) +(defbjouble (stream-flush! s) + (#:sync (clr-stream-flush s)) + (#:bjo (clr-stream-flush-await s))) + +(import/extern + (clr-stream-read (: System.IO.Stream.Read (-> ClrStream (Array byte) int int int) #:blocking)) + (clr-stream-read-await (: System.IO.Stream.ReadAsync (-> ClrStream (Array byte) int int int) #:async)) + (clr-stream-length (: System.IO.Stream.Length (-> ClrStream long) #:get)) + (clr-new-guid (: System.Guid.NewGuid (-> System.Guid))) + (clr-guid-text (: System.Guid.ToString (-> System.Guid string string)))) + +;; Up to `n` bytes into the start of `buffer`; the number read, 0 at the end. +(: (-> ClrStream (Array byte) int int)) +(defbjouble (stream-read-some! s buffer n) + (#:sync (clr-stream-read s buffer 0 n)) + (#:bjo (clr-stream-read-await s buffer 0 n))) + +;; The first `n` bytes of `buffer`. +(: (-> ClrStream (Array byte) int void)) +(defbjouble (stream-write-some! s buffer n) + (#:sync (clr-stream-write s buffer 0 n)) + (#:bjo (clr-stream-write-await s buffer 0 n))) + +;; At most `count` bytes: what the source held when it was opened. +;; +;; - `Stream.CopyTo` copies to the end of the source. When the target is the +;; source under another name, opened to append, the end moves away as fast +;; as the copy writes, and the file grows until the disk is full. Copying +;; the length that the source had when it was opened ends. +;; - Stops early when the source ends before `count` bytes. +(: (-> ClrStream ClrStream long void)) +(defun (stream-copy-count! from to count) + (def buffer (make-array (cast int (min 81920L (max count 1L))))) + (let go ((left count)) + (when (> left 0L) + (def n (stream-read-some! from buffer (cast int (min (cast long (array-length buffer)) left)))) + (when (> n 0) + (stream-write-some! to buffer n) + (go (- left (cast long n))))))) + +;; Copies a file from `from` to `to`. +;; +;; `#:mode` controls destination behavior: +;; - `CreateNew` (default): Fails if `to` exists, preventing accidental overwrite. +;; - `Truncate`: Overwrites `to` via an atomic replace. +;; - `Append`: Appends to `to`. +;; +;; Safety & Aliasing: +;; - Direct self-copies are rejected. +;; - The source is opened first; a missing source leaves `to` untouched. +;; - If `to` is an alias or symlink to `from`, in-place truncation could empty +;; the source before copying. To prevent this: +;; * `Truncate` copies to a temporary sibling file and moves it over `to` +;; (replacing any symlink rather than writing through it). +;; * `Append` copies only the source's initial byte length, preventing an +;; infinite loop when appending a file to itself. +(: (-> string string (#:mode WriteMode) void)) +(defun (file-copy from to #:mode CreateNew) + (when (= (path-absolute from) (path-absolute to)) + (raise (cast Exception (IOException. (string-append "cannot copy a file onto itself: " from))))) + (with-open ((in (raise-err (open-read-stream from)))) + (def count (clr-stream-length in)) + (match mode + (Truncate + (def fresh (str to ".copy-" (clr-guid-text (clr-new-guid) "N"))) + (match (try (begin + (with-open ((out (raise-err (open-write-stream fresh CreateNew)))) + (stream-copy-count! in out count) + (stream-flush! out)) + (raise-err (fs-move fresh to MoveReplace))) + #:catch (System.Exception)) + ((Ok _) unit) + ((Err e) + (ignore (fs-delete-file fresh)) + (raise e)))) + (_ + (with-open ((out (raise-err (open-write-stream to mode)))) + (stream-copy-count! in out count) + (stream-flush! out)))))) + +(: (-> string (Array byte))) +(defun (file-read-bytes path) + (with-open ((in (raise-err (open-read-stream path)))) + (def buffer (MemoryStream.)) + (stream-copy! in (cast ClrStream buffer)) + (.ToArray buffer))) + +(: (-> string (Array byte) WriteMode void)) +(defun (write-whole-bytes path bytes mode) + (with-open ((out (raise-err (open-write-stream path mode)))) + (stream-write! out bytes) + (stream-flush! out))) + +(: (-> string (Array byte) void)) +(defun (file-write-bytes path bytes) (write-whole-bytes path bytes Truncate)) + +(: (-> string (Array byte) void)) +(defun (file-append-bytes path bytes) (write-whole-bytes path bytes Append)) + +;; The lines, without their line ends. What ends a line is `#:mode`'s to say; +;; by default "\n", "\r" and "\r\n" all do. A last line needs no end, and a +;; final line end makes no empty line after it. +(: (-> string (#:mode LineMode) (Vec string))) +(defun (file-read-lines path #:mode Any) + (match (open-input-file path) + ((Ok opened) (with-open ((p opened)) + (def lines (vecbuilder-empty)) + (let next () + (unless (port-eof? p) + (vecbuilder-add! lines (read-line p #:mode mode)) + (next))) + (vecbuilder->vec lines))) + ((Err e) (raise e)))) + +;; Each line followed by "\n", the last one too, on every OS: the same lines +;; make the same bytes wherever they are written, and `file-read-lines` gives +;; them back as long as none holds a line end of its own. +(: (-> string (Vec string) void)) +(defun (file-write-lines path lines) + (match (open-output-file path) + ((Ok opened) (with-open ((p opened)) + (loop (:for line lines) + (:do (write-string p line) + (write-string p "\n"))) + (flush-port p))) + ((Err e) (raise e)))) + +;; Returns a lazy `Seq` of items produced by repeatedly calling `(reader port)`. +;; +;; The file is opened inside the `seq`, meaning each iteration re-opens the file +;; from the start. The file is closed when iteration reaches EOF, stops early, +;; or encounters an exception. If an iterator is abandoned without cleanup, the +;; port remains open until the ambient scope ends. +;; +;; Because opening happens inside the sequence, open errors (such as a missing +;; file) raise on the first pull rather than when `file-read/seq` is called. +(: file-read/seq #:sync (-> (-> InputPort %a) string (Seq %a))) +(defun (file-read/seq reader path) + (seq + (def file (raise-err (open-input-file path))) + (with-open ((p file)) + (let loop () + (unless (port-eof? p) + (yield (reader p)) + (loop)))))) + +(: file-read-lines/seq #:sync (-> string (#:mode LineMode) (Seq string))) +(defun (file-read-lines/seq path #:mode Any) + (file-read/seq (fun (p) (read-line p #:mode mode)) path)) + +(: (-> string (List string))) +(defun (sorted-entries dir) (list-sort (vec->list (raise-err (fs-list dir))))) + +;; Lazily yields all files under `dir` in depth-first order. +;; +;; Behavior & Guarantees: +;; - Deterministic: Directory entries are sorted ordinally, guaranteeing identical +;; traversal order across all operating systems and virtual filesystems. +;; - Filtering (`#:into?`): Called on each directory before entering; return `#f` +;; to skip a subtree (e.g. `.git`). +;; - Cycle Prevention: Directory symlinks are never followed. +;; - Lazy Traversal: Subdirectories are listed on demand using the ambient `FS` +;; handler, so stopping iteration early avoids unneeded filesystem I/O. +(: directory-walk #:sync (-> string (#:into? (-> string bool)) (Seq string))) +(defun (directory-walk dir #:into? (fun (d) #t)) + (seq + (let loop ((todo (sorted-entries dir))) + (unless (list-empty? todo) + (def entry (list-head todo)) + (def rest (list-tail todo)) + (match (fs-info entry) + ((Some info) + (match (record-ref info kind) + (FsFile (yield entry) + (loop rest)) + (FsDirectory + (if (and (not (record-ref info link?)) (into? entry)) + (loop (list-append (sorted-entries entry) rest)) + (loop rest))))) + ;; Gone since it was listed. + (None (loop rest))))))) + +;; Flush. +(import/extern + (clr-flush (: Bjolang.Runtime.BjoOutputPort.Flush (-> OutputPort void) #:blocking)) + (clr-flush-await (: Bjolang.Runtime.BjoOutputPort.FlushValueAsync (-> OutputPort void) #:async))) + +(: (-> OutputPort void)) +(defbjouble (flush-port p) + (#:sync (clr-flush p)) + (#:bjo (clr-flush-await p))) + +;; --- Input --- +;; These are the leaves of our IO. The bottom calls, so to say. +(import/extern + (clr-eof (: Bjolang.Runtime.BjoInputPort.Eof (-> InputPort bool) #:blocking)) + (clr-eof-await (: Bjolang.Runtime.BjoInputPort.EofAsync (-> InputPort bool) #:async)) + (clr-line (: Bjolang.Runtime.InputPorts.ReadLineOrThrow (-> InputPort int string) #:blocking)) + (clr-line-await (: Bjolang.Runtime.InputPorts.ReadLineOrThrowAsync (-> InputPort int string) #:async)) + (clr-char (: Bjolang.Runtime.InputPorts.ReadCharOrThrow (-> InputPort char) #:blocking)) + (clr-char-await (: Bjolang.Runtime.InputPorts.ReadCharOrThrowAsync (-> InputPort char) #:async)) + (clr-rest (: Bjolang.Runtime.BjoInputPort.ReadToEndUtf8 (-> InputPort string) #:blocking)) + (clr-rest-await (: Bjolang.Runtime.BjoInputPort.ReadToEndUtf8Async (-> InputPort string) #:async)) + (clr-line-opt (: Bjolang.Runtime.InputPorts.ReadLineOpt (-> InputPort int (Option string)) #:blocking)) + (clr-line-opt-await (: Bjolang.Runtime.InputPorts.ReadLineOptAsync (-> InputPort int (Option string)) #:async)) + (clr-char-opt (: Bjolang.Runtime.InputPorts.ReadCharOpt (-> InputPort (Option char)) #:blocking)) + (clr-char-opt-await (: Bjolang.Runtime.InputPorts.ReadCharOptAsync (-> InputPort (Option char)) #:async)) + (clr-peek-char (: Bjolang.Runtime.InputPorts.PeekCharOrThrow (-> InputPort char) #:blocking)) + (clr-peek-char-await (: Bjolang.Runtime.InputPorts.PeekCharOrThrowAsync (-> InputPort char) #:async)) + (clr-peek-char-opt (: Bjolang.Runtime.InputPorts.PeekCharOpt (-> InputPort (Option char)) #:blocking)) + (clr-peek-char-opt-await (: Bjolang.Runtime.InputPorts.PeekCharOptAsync (-> InputPort (Option char)) #:async))) + +;; Port eof. +(: (-> InputPort bool)) +(defbjouble (port-eof? p) + (#:sync (clr-eof p)) + (#:bjo (clr-eof-await p))) + +;; `LineMode` as `Bjolang.Runtime.LineMode` numbers it. +(: (-> LineMode int)) +(defun (line-mode-code mode) + (match mode + (Any 0) + (AnyOne 1) + (Linefeed 2) + (Return 3) + (ReturnLinefeed 4))) + +;; Where a reader of one argument is expected, `read-line` is the function +;; that calls it with the port, and the mode is `Any`: `(port->list read-line +;; p)`. For another mode, hand it over as `(fun (p) (read-line p #:mode m))`. +(: (-> InputPort (#:mode LineMode) string)) +(defbjouble (read-line p #:mode Any) + (#:sync (clr-line p (line-mode-code mode))) + (#:bjo (clr-line-await p (line-mode-code mode)))) + +;; One read, not `port-eof?` and then `read-line`: between those two another +;; fiber reading the same port can take the last line. +(: (-> InputPort (#:mode LineMode) (Option string))) +(defbjouble (read-line/opt p #:mode Any) + (#:sync (clr-line-opt p (line-mode-code mode))) + (#:bjo (clr-line-opt-await p (line-mode-code mode)))) + +(: (-> InputPort char)) +(defbjouble (read-char p) + (#:sync (clr-char p)) + (#:bjo (clr-char-await p))) + +;; One read, for the reason `read-line/opt` is. +(: (-> InputPort (Option char))) +(defbjouble (read-char/opt p) + (#:sync (clr-char-opt p)) + (#:bjo (clr-char-opt-await p))) + +;; The character `read-char` would answer, left in the port, as Racket's +;; `peek-char`. Decoded from the port's buffer: it waits for the bytes a +;; character needs and takes none of them. +(: (-> InputPort char)) +(defbjouble (peek-char p) + (#:sync (clr-peek-char p)) + (#:bjo (clr-peek-char-await p))) + +(: (-> InputPort (Option char))) +(defbjouble (peek-char/opt p) + (#:sync (clr-peek-char-opt p)) + (#:bjo (clr-peek-char-opt-await p))) + +;; Drains a port +(: (-> InputPort string)) +(defbjouble (read-all p) + (#:sync (clr-rest p)) + (#:bjo (clr-rest-await p))) + +;; Port fold. +(: (-> (-?-> InputPort %a %a) %a InputPort %a)) +(defun (port-fold f init port) + (let loop ((acc init)) + (if (port-eof? port) + acc + (loop (f port acc))))) + +;; --- Output --- + +;; Port write operations provide both synchronous (`#:sync`) and suspending (`#:async`) +;; implementations via `defbjouble`. +;; +;; Writes are buffered in memory and non-blocking until the buffer fills. When a buffer +;; drains (triggering a syscall), the async variant suspends the coroutine rather than +;; blocking the host thread. +;; +;; In contrast, pure string formatters (`written` and `displayed`) output to a +;; thread-local `StringBuilder`, remaining strictly synchronous and side-effect free. +(import/extern + (clr-write (: Bjolang.Runtime.BjoOutputPort.WriteText (-> OutputPort string void) #:blocking)) + (clr-write-await (: Bjolang.Runtime.BjoOutputPort.WriteTextAsync (-> OutputPort string void) #:async)) + (clr-writec (: Bjolang.Runtime.BjoOutputPort.WriteScalar (-> OutputPort char void) #:blocking)) + (clr-writec-await (: Bjolang.Runtime.BjoOutputPort.WriteScalarAsync (-> OutputPort char void) #:async)) + (clr-writeln (: Bjolang.Runtime.BjoOutputPort.WriteTextLine (-> OutputPort string void) #:blocking)) + (clr-writeln-await (: Bjolang.Runtime.BjoOutputPort.WriteTextLineAsync (-> OutputPort string void) #:async))) + +(: (-> OutputPort string void)) +(defbjouble (write-string p s) + (#:sync (clr-write p s)) + (#:bjo (clr-write-await p s))) + +;; Write char. +(: (-> OutputPort char void)) +(defbjouble (write-char p c) + (#:sync (clr-writec p c)) + (#:bjo (clr-writec-await p c))) + +(: (-> OutputPort string void)) +(defbjouble (writeln p s) + (#:sync (clr-writeln p s)) + (#:bjo (clr-writeln-await p s))) + +;; What a string port has been given, as text. Bytes written to it with +;; `write-bytes!` that are not UTF-8 read as U+FFFD, as Racket's do. +(: (-> OutputPort string)) +(defun (get-output-string p) (writer->string p)) + +;; What a string port has been given, as bytes. +(: (-> OutputPort (Array byte))) +(defun (get-output-bytes p) (output-bytes p)) + + +;; --------------------------------------------------------------------------- +;; Effects +;; --------------------------------------------------------------------------- +;; +;; An operation is a parameter holding a function, and performing an effect +;; calls that function on the performer's own stack. +;; (std effect) has the whole of the mechanism; what is here is the handful +;; of operations the standard library performs itself. +;; +;; What is an effect and what is a parameter? +;; If the thing is a value the caller reads, it is a parameter. If it is a +;; behaviour the caller decides - with a default worth naming, a possible +;; abort, and a signature worth checking - it is an effect. So errors stay +;; try/raise, state stays parameterize and def/mutable, the three +;; standard ports stay parameters, and random stays the parameter +;; (std random) already has: it is swappable, it has no abort case, and an +;; effect would put a wrapper function over a read that is already one. +;; The clock is a parameter for the same reason. +;; +;; Installing a handler needs (import (std effect)) for with-handler. A +;; macro is not a binding, so it cannot be re-exported from here. + +(import (std effect)) + +;; `(warn "...")` - something is wrong, and carrying on is the default. +;; +;; Warn continues by default, but this is overridable. +;; +;; (log "...") - a line of diagnostics, and nobody's decision. +;; +;; Both are synchronous. A diagnostic that suspends is a promise answered here, +;; and acted on wherever it is awaited. +(defeffect Diagnostics + (warn (-> string void)) + (log (-> string void)) + #:default ((warn (fun (s) (writeln (parameter-ref current-error-port) s))) + (log (fun (s) (writeln (parameter-ref current-error-port) s))))) + +;; (getenv "HOME") - the process environment, as an effect so that a test can +;; answer for it without writing to the real one. +;; +;; Cold: read at startup, not in a loop. +(defeffect Env + (getenv (-> string (Option string))) + #:default ((getenv get-environment-variable)) + #:cold (getenv)) + + +;; --------------------------------------------------------------------------- +;; The clock +;; --------------------------------------------------------------------------- +;; +;; A `Clock` is a record of functions, so a program can make its own. +;; `monotonic-ms` and `monotonic-ns` read it here, and (std datetime)'s `now` +;; and `today` read the same one, so binding `current-clock` moves both. +;; +;; - `read-monotonic-ms`: milliseconds from an unspecified start. Never +;; decreases; only differences mean anything. +;; - `read-monotonic-ns`: the same clock in nanoseconds, at the resolution +;; the clock has. A clock made with only `#:monotonic-ms` reads it as its +;; milliseconds times a million, so a fake clock gives the same exact +;; elapsed time in both units. +;; - `read-utc-ticks`: the wall clock, in .NET ticks since 0001-01-01 UTC. May +;; step back. +;; - `read-zone`: the zone (std datetime)'s `today` is in. +;; - `advance-ticks`: what (std datetime)'s `clock-advance!` calls, with a +;; number of ticks. `None` for a clock that cannot be moved. +(type (: Clock (Record (: read-monotonic-ms (-> long)) + (: read-monotonic-ns (-> long)) + (: read-utc-ticks (-> long)) + (: read-zone (-> System.TimeZoneInfo)) + (: advance-ticks (Option (-> long void)))))) + +(import/extern + (clock-ms (: BjolangRuntime.ClockMs (-> long))) + (clock-ns (: BjolangRuntime.ClockNs (-> long))) + (clr-utc-now (: System.DateTime.UtcNow #:get)) + (clr-local-zone (: System.TimeZoneInfo.Local #:get))) + +(: (-> long)) +(defun (system-utc-ticks) (.-Ticks clr-utc-now)) + +(: (-> System.TimeZoneInfo)) +(defun (system-zone) clr-local-zone) + +;; Each part not given is the system clock's, but for the nanoseconds: when +;; `#:monotonic-ns` is not given, they are read from the milliseconds, so that +;; a clock given only `#:monotonic-ms` agrees with itself. +(: (-> (#:monotonic-ms (-> long)) + (#:monotonic-ns (Option (-> long))) + (#:utc-ticks (-> long)) + (#:zone (-> System.TimeZoneInfo)) + (#:advance (Option (-> long void))) + Clock)) +(defun (make-clock #:monotonic-ms clock-ms #:monotonic-ns None #:utc-ticks system-utc-ticks + #:zone system-zone #:advance None) + (Clock (read-monotonic-ms monotonic-ms) + (read-monotonic-ns (match monotonic-ns + ((Some ns) ns) + (None (fun () (* (monotonic-ms) 1000000L))))) + (read-utc-ticks utc-ticks) + (read-zone zone) + (advance-ticks advance))) + +;; The machine's clock, in the machine's zone. +(: Clock) +(def system-clock (make-clock #:monotonic-ns (Some clock-ns))) + +;; Cold: nothing reads the time in a loop. +(: (Param Clock)) +(def current-clock (make-cold-parameter system-clock)) + +;; Not called `now`: that is (std datetime)'s wall-clock `Instant`. +(: (-> (#:clock Clock) long)) +(defun (monotonic-ms #:clock (parameter-ref current-clock)) + ((record-ref clock read-monotonic-ms))) + +;; Nanoseconds on the same clock: for timing what takes less than a +;; millisecond. +(: (-> (#:clock Clock) long)) +(defun (monotonic-ns #:clock (parameter-ref current-clock)) + ((record-ref clock read-monotonic-ns))) + + +;; --------------------------------------------------------------------------- +;; Iterating something other than the whole of a collection +;; --------------------------------------------------------------------------- +;; +;; `in-array`, `in-vec` and their reverses, `over-list`, `in-port`, `up-from` +;; and `down-from`. Included rather than imported: they are `Iterable` +;; implementations, and an implementation belongs with the trait — a module of +;; its own would have to import the prelude it is part of. What is above stays +;; here: the walk of a whole collection is what the collection itself is for. +;; +;; Here rather than beside those implementations because `in-port` names +;; `InputPort`, which the section above introduces. +(include "iterators.protobjo") + + +;; --------------------------------------------------------------------------- +;; Collection +;; --------------------------------------------------------------------------- +;; +;; Collection trait. +(def/trait (Collection %a) + (type %content) + (: length (-> %a int)) + (: empty? (-> %a bool))) + +(impl (Collection (List %a)) + (type %content %a) + (defun (length lst) + (list-length lst)) + (defun (empty? lst) + (list-empty? lst))) + +(impl (Collection (Array %a)) + (type %content %a) + (defun (length arr) + (array-length arr)) + (defun (empty? arr) + (= 0 (array-length arr)))) + +(impl (Collection (IArray %a)) + (type %content %a) + (defun (length xs) + (iarray-length xs)) + (defun (empty? xs) + (iarray-empty? xs))) + +(impl (Collection (Vec %a)) + (type %content %a) + (defun (length vec) + (vec-length vec)) + (defun (empty? vec) + (vec-empty? vec))) + +(impl (Collection (VecBuilder %a)) + (type %content %a) + (defun (length vb) + (vecbuilder-length vb)) + (defun (empty? vb) + (= (vecbuilder-length vb) 0))) + +(impl (Collection (TransientVec %a)) + (type %content %a) + (defun (length t) + (transientvec-length t)) + (defun (empty? t) + (transientvec-empty? t))) + +(impl (Collection (ListBuilder %a)) + (type %content %a) + (defun (length lb) + (listbuilder-length lb)) + (defun (empty? lb) + (= (listbuilder-length lb) 0))) + +;; Map collection. +(impl (Collection (Map %k %v)) + (type %content (Tuple %k %v)) + (defun (length m) + (map-length m)) + (defun (empty? m) + (map-empty? m))) + +(impl (Collection (TransientMap %k %v)) + (type %content (%k . %v)) + (defun (length tm) + (transientmap-length tm)) + (defun (empty? tm) + (transientmap-empty? tm))) +(impl (Collection (MapBuilder %k %v)) + (type %content (%k . %v)) + (defun (length mb) + (mapbuilder-length mb)) + (defun (empty? mb) + (= (mapbuilder-length mb) 0))) + +;; Range collection. +(impl (Collection Range) + (type %content int) + (defun (length r) + (def lo (record-ref r lo)) + (def hi (record-ref r hi)) + (if (<= hi lo) + 0 + (range-length-of (- (cast long hi) (cast long lo))))) + (defun (empty? r) + (>= (record-ref r lo) (record-ref r hi)))) + +(impl (Collection RangeBy) + (type %content int) + (defun (length r) + (def lo (record-ref r lo)) + (def hi (record-ref r hi)) + (if (<= hi lo) + 0 + (range-length-of (range-by-count lo hi (record-ref r by))))) + (defun (empty? r) + (>= (record-ref r lo) (record-ref r hi)))) + +;; A range from below zero to above it can hold more than `int-max` values, +;; which a length cannot give. +(: (-> long int)) +(defun (range-length-of n) + (when (> n (cast long int-max)) + (panic! #"length: the range holds ${n} values, more than an int can count.")) + (cast int n)) + +;; String collection. +(impl (Collection string) + (type %content char) + (defun (length s) + (string-count s)) + (defun (empty? s) + (string-empty? s))) + +;; Seq collection. +(impl (Collection (Seq %a)) + (type %content %a) + (defun (length s) + (count s)) + (defun (empty? s) + (seq-empty? s))) + + +;;---------------------------------------------------------------------------- +;; Addable! trait +;; --------------------------------------------------------------------------- +;; +;; `add!` never fails. Afterwards the sink holds the element: a sequence has +;; it appended, a set has it as a member (which it may already have been), +;; and a map has the key mapped to the value, replacing an older one. An +;; operation that refuses what is already there is named `add-new` and is +;; not part of the trait. +(def/trait (Addable! %coll) + (type %elem) + (: add! (-> %coll %elem void)) + ;; Everything an `Iterable` holds, added in the order it yields. The + ;; `(where ...)` is member-level: each call site brings its own source type, + ;; with `#:elem` pinned to this trait's `%elem` — the element the source + ;; yields *is* the element the sink takes, which no function-level clause + ;; can say. The cursor is nobody's business, so it is pinned to a fresh + ;; variable. + (: add-all! (-> %coll %s void) + (where (Iterable %s #:elem %elem #:cursor %k))) + (defun (add-all! coll xs) + (let go ((cur (iterable-start xs))) + (unless (iterable-done? xs cur) + (add! coll (iterable-current xs cur)) + (go (iterable-next xs cur)))))) + +(impl (Addable! StringBuilder) + (type %elem Char) + (defun (add! sb s) + (stringbuilder-add! sb s))) + +(impl (Addable! (VecBuilder %a)) + (type %elem %a) + (defun (add! vb elem) + (vecbuilder-add! vb elem))) + +(impl (Addable! (TransientVec %a)) + (type %elem %a) + (defun (add! t elem) + (transientvec-add! t elem))) + +(impl (Addable! (TransientMap %k %v)) + (type %elem (%k . %v)) + (defun (add! tm tup) + (let (((k v) tup)) + (transientmap-set! tm k v)))) + +(impl (Addable! (MapBuilder %k %v)) + (type %elem (%k . %v)) + (defun (add! mb tup) + (let (((k v) tup)) + (mapbuilder-add! mb k v)))) + +(impl (Addable! (ListBuilder %a)) + (type %elem %a) + (defun (add! lb elem) + (listbuilder-add! lb elem))) + +;; The set and ordered-collection sinks live with their types, in (std hashset), +;; (std orderedset) and (std orderedmap): `Addable!` is this module's trait, +;; so a module that owns the type may implement it. + +;;---------------------------------------------------------------------------- +;; Addable — the persistent counterpart +;; --------------------------------------------------------------------------- +;; +;; The same shape without the `!`: `add` answers a new collection and touches +;; nothing. It never fails, as `add!` never does: a map-shaped collection +;; replaces a key already there, and a set already holding the element +;; answers an equal set. A caller who wants a duplicate key refused writes +;; `map-add-new`, which raises, or `map-try-add-new`, which answers a Result. +(def/trait (Addable %coll) + (type %elem) + (: add (-> %coll %elem %coll)) + (: add-all (-> %coll %s %coll) + (where (Iterable %s #:elem %elem #:cursor %k))) + ;; A mutable cell rather than a named-let accumulator, deliberately: the + ;; cell is initialised from `coll`, so its type is the implementor's from + ;; the first line, and `(add acc e)` resolves before the loop is even + ;; checked. A let-bound accumulator is a fresh variable at the moment the + ;; recursive call is inferred, which leaves `add`'s element projection + ;; nothing to resolve against. + (defun (add-all coll xs) + (def/mutable acc coll) + (let go ((cur (iterable-start xs))) + (unless (iterable-done? xs cur) + (set! acc (add acc (iterable-current xs cur))) + (go (iterable-next xs cur)))) + acc)) + +(impl (Addable (Vec %a)) + (type %elem %a) + (defun (add v elem) + (vec-add v elem))) + +(impl (Addable (Map %k %v)) + (type %elem (%k . %v)) + (defun (add m tup) + (let (((k v) tup)) + (map-set m k v)))) + +;; As with the sinks, the set and ordered-collection impls live with their +;; types, in (std hashset), (std orderedset) and (std orderedmap). + + + +;; --------------------------------------------------------------------------- +;; Control +;; --------------------------------------------------------------------------- +;; +;; Macros. +(import (std syntax-match)) + +;; Several forms as one. +(: (-> (List Syntax) Syntax Syntax)) +(defun (one-form body whole) + (match body + (Nil (syntax-error whole "This clause has nothing to do.")) + ((Cons only Nil) only) + (_ #'(let () ,@body)))) + +;; `(if-let ((name expr)) then else)` +;; +;; `name` bound to the `Some` value of `expr`, or `else` on a `None`. +(def/macro (if-let form inject compare) + (syntax-match form + ((_ (name value) then else) + #'(match ,value ((Some ,name) ,then) (None ,else))) + (bad (syntax-error bad "if-let takes one binding, a then and an else: (if-let (name expr) then else)")))) + +;; `(when-let (name expr) body ...)` +;; The body run with `name` bound to a `Some` value, and nothing on `None`. +(def/macro (when-let form inject compare) + (syntax-match form + ((_ (name value) first body ...) + ;; Evaluate value once. + #'(let ((found ,value)) + (when (some? found) + (let ((,name (option-ref found))) + ,first + ,@body)))) + (bad (syntax-error bad "when-let takes one binding and a body: (when-let (name expr) body ...)")))) + +;; `(some-> value step ...)` — `->`, stopping at the first `None`. +;; +;; Threading option. +(def/macro (some-> form inject compare) + (syntax-match form + ((_ value step ...) (some-steps value step)) + (bad (syntax-error bad "some-> takes a value and steps: (some-> value (f &) (g &))")))) + +(: (-> Syntax (List Syntax) Syntax)) +(defun (some-steps value steps) + (match steps + ;; No steps. + (Nil #'(Some ,value)) + ;; Last step. + ((Cons step Nil) #'(-> ,value ,step)) + ((Cons step rest) + ;; Step binding. + #'(match (-> ,value ,step) + (None None) + ((Some it) ,(some-steps #'it rest)))))) + +;; `(try-> value step ...)` — `->`, stopping at the first `Err`. +;; +;; Threading result. Same as some-> but for Result. +(def/macro (try-> form inject compare) + (syntax-match form + ((_ value step ...) (try-steps value step)) + (bad (syntax-error bad "try-> takes a value and steps: (try-> value (f &) (g &))")))) + +(: (-> Syntax (List Syntax) Syntax)) +(defun (try-steps value steps) + (match steps + ;; No steps. + (Nil #'(Ok ,value)) + ;; Last step. + ((Cons step Nil) #'(-> ,value ,step)) + ((Cons step rest) + ;; The error is rebuilt rather than passed along, because the two Results + ;; differ in their Ok type. + #'(match (-> ,value ,step) + ((Err e) (Err e)) + ((Ok it) ,(try-steps #'it rest)))))) + +;; --------------------------------------------------------------------------- +;; Collector macros and the collectors they expand to +;; --------------------------------------------------------------------------- + +;; Expands a collector macro's `(:acc [name] (head arg ...) [#:when cond])`. +;; +;; - `build` gets the name (a list of zero or one), the head's arguments and +;; the condition, and returns the clauses. +;; - `build` returns None for arguments it does not take. That, and a clause +;; of any other shape, is a syntax error with `usage`. +(: (-> Syntax string (-> (List Syntax) (List Syntax) (Option Syntax) (Option Syntax)) Syntax)) +(defun (expand-acc-clause form usage build) + (def built + (syntax-match form + ((_ (_ arg ...)) (build (list) arg None)) + ((_ (_ arg ...) #:when c) (build (list) arg (Some c))) + ((_ name (_ arg ...)) (build (list name) arg None)) + ((_ name (_ arg ...) #:when c) (build (list name) arg (Some c))) + (_ None))) + (match built + ((Some clauses) clauses) + (None (syntax-error form usage)))) + +;; `(:acc ,@name ,collector [#:when ,cond])`. +(: (-> (List Syntax) Syntax (Option Syntax) Syntax)) +(defun (acc-form name collector when) + (match when + ((Some c) #'(:acc ,@name ,collector #:when ,c)) + (None #'(:acc ,@name ,collector)))) + +;; `test`, or `(if ,cond ,test ,otherwise)` with a condition. +(: (-> (Option Syntax) Syntax Syntax Syntax)) +(defun (acc-gated when test otherwise) + (match when + ((Some c) #'(if ,c ,test ,otherwise)) + (None test))) + +;; `(:acc found (someing expr))`: the first `(Some v)` that `expr`, an +;; Option, evaluates to, which ends the loop, or `None` when the loop ends +;; first. +;; +;; - A `folding` of `expr`, and a `:finish` after it. Every round before the +;; last gave `None`, so the fold needs no condition. +;; - `expr` is evaluated once a round. With `#:when cond`, a round `cond` does +;; not hold for does not evaluate it. +(def/acc-clause (someing form inject compare) + (expand-acc-clause form "someing takes one Option: (:acc found (someing (map-try-ref m k))), or {someing (map-try-ref m k) (:for k ks)}" + (fun (name args when) + (match args + ((Cons expr Nil) + (Some #'(begin (:let got ,(acc-gated when expr #'None)) + (:acc ,@name (folding None got)) + (:finish (some? got))))) + (_ None))))) + +;; `(:acc found (finding x))`: `(Some x)` for the first round that reaches it, +;; which ends the loop, or `None` when the loop ends first. +;; +;; - `(someing (Some x))`. +;; - With `#:when cond`, `cond` is evaluated once, and gates both: a round it +;; does not hold for neither finds nor finishes. +(def/acc-clause (finding form inject compare) + (expand-acc-clause form "finding takes one expression: (:acc found (finding x)), or {finding x (:for x xs)}" + (fun (name args when) + (match args + ((Cons x Nil) (Some (acc-form name #'(someing (Some ,x)) when))) + (_ None))))) + +;; `(:acc all? (anding test))`: whether `test` holds in every round. The +;; first round it does not hold in ends the loop. #t for no rounds. +;; +;; - With `#:when cond`, a round `cond` does not hold for is not tested. +(def/acc-clause (anding form inject compare) + (expand-acc-clause form "anding takes one test: (:acc all? (anding (even? x))), or {anding (even? x) (:for x xs)}" + (fun (name args when) + (match args + ((Cons test Nil) + (Some #'(begin (:let ok? ,(acc-gated when test #'#t)) + (:acc ,@name (folding #t ok?)) + (:finish (not ok?))))) + (_ None))))) + +;; `(:acc any? (oring test))`: whether `test` holds in some round. The first +;; round it holds in ends the loop. #f for no rounds. +;; +;; - With `#:when cond`, a round `cond` does not hold for is not tested. +(def/acc-clause (oring form inject compare) + (expand-acc-clause form "oring takes one test: (:acc any? (oring (even? x))), or {oring (even? x) (:for x xs)}" + (fun (name args when) + (match args + ((Cons test Nil) + (Some #'(begin (:let ok? ,(acc-gated when test #'#f)) + (:acc ,@name (folding #f ok?)) + (:finish ok?)))) + (_ None))))) + +;; `(:acc firsts (taking n x))`: a list of the first `n` values, in order. The +;; round that takes the `n`th ends the loop. +;; +;; - `n` is evaluated each round, before the value. +;; - With `#:when cond`, only the rounds `cond` holds for are taken. +;; - An `n` of 0 or less takes nothing, but still ends the loop only after +;; its first round. +(def/acc-clause (taking form inject compare) + (expand-acc-clause form "taking takes a count and an expression: (:acc firsts (taking 3 x)), or {taking 3 x (:for x xs)}" + (fun (name args when) + (def acc (match name ((Cons n Nil) n) (_ #'taken))) + (match args + ((Cons n (Cons x Nil)) + (Some #'(begin (:let limit ,n) + ,(acc-form (list acc) #'(listing ,x) + (Some (acc-gated when #'(< (length ,acc) limit) #'#f))) + (:finish (>= (length ,acc) limit))))) + (_ None))))) + +;; `(:acc best (maximizing-by key x))`: `(Some x)` of the greatest `key`, or +;; `None` for no rounds. Of equal keys the first is kept. +;; +;; - `key` is evaluated once a round, and is compared by Ord. +;; - `maximizing/pairs` of `(Tuple key x)`. +(def/acc-clause (maximizing-by form inject compare) + (expand-acc-clause form "maximizing-by takes a key and an expression: (:acc best (maximizing-by (score x) x))" + (fun (name args when) + (match args + ((Cons key (Cons x Nil)) (Some (acc-form name #'(maximizing/pairs (Tuple ,key ,x)) when))) + (_ None))))) + +;; `(:acc best (minimizing-by key x))`: `(Some x)` of the least `key`, or +;; `None` for no rounds. Of equal keys the first is kept. +;; +;; - `key` is evaluated once a round, and is compared by Ord. +;; - `minimizing/pairs` of `(Tuple key x)`. +(def/acc-clause (minimizing-by form inject compare) + (expand-acc-clause form "minimizing-by takes a key and an expression: (:acc best (minimizing-by (cost x) x))" + (fun (name args when) + (match args + ((Cons key (Cons x Nil)) (Some (acc-form name #'(minimizing/pairs (Tuple ,key ,x)) when))) + (_ None))))) + +;; `(:acc groups (grouping key x))`: a map from each `key` to the list of its +;; `x`s, in the order they came. +;; +;; - `key` is evaluated once a round, and is compared by Eq. +;; - `grouping/pairs` of `(Tuple key x)`. +(def/acc-clause (grouping form inject compare) + (expand-acc-clause form "grouping takes a key and an expression: (:acc groups (grouping (string-length w) w))" + (fun (name args when) + (match args + ((Cons key (Cons x Nil)) (Some (acc-form name #'(grouping/pairs (Tuple ,key ,x)) when))) + (_ None))))) + +;; The greatest or least value by Ord, or None. Of equal ones the first is kept. +(type (: (Maximizing %a) (Union maximizing))) + +(impl (Collector (Maximizing %a)) + (where (Ord %a)) + (type %elem %a) + (type %acc (Option %a)) + (type %out (Option %a)) + (defun (collector-init c) None) + (defun (collector-step c acc x) + (match acc + ((Some best) (if (greater? x best) (Some x) acc)) + (None (Some x)))) + (defun (collector-finish c acc) acc)) + +(type (: (Minimizing %a) (Union minimizing))) + +(impl (Collector (Minimizing %a)) + (where (Ord %a)) + (type %elem %a) + (type %acc (Option %a)) + (type %out (Option %a)) + (defun (collector-init c) None) + (defun (collector-step c acc x) + (match acc + ((Some best) (if (less? x best) (Some x) acc)) + (None (Some x)))) + (defun (collector-finish c acc) acc)) + +;; Of `(Tuple key value)`s: the value of the greatest or least key by Ord, or +;; None. Of equal keys the first is kept. +(type (: (MaximizingPairs %k %v) (Union maximizing/pairs))) + +(impl (Collector (MaximizingPairs %k %v)) + (where (Ord %k)) + (type %elem (Tuple %k %v)) + (type %acc (Option (Tuple %k %v))) + (type %out (Option %v)) + (defun (collector-init c) None) + (defun (collector-step c acc kv) + (match acc + ((Some best) (if (greater? (fst kv) (fst best)) (Some kv) acc)) + (None (Some kv)))) + (defun (collector-finish c acc) + (match acc + ((Some (Tuple _ v)) (Some v)) + (None None)))) + +(type (: (MinimizingPairs %k %v) (Union minimizing/pairs))) + +(impl (Collector (MinimizingPairs %k %v)) + (where (Ord %k)) + (type %elem (Tuple %k %v)) + (type %acc (Option (Tuple %k %v))) + (type %out (Option %v)) + (defun (collector-init c) None) + (defun (collector-step c acc kv) + (match acc + ((Some best) (if (less? (fst kv) (fst best)) (Some kv) acc)) + (None (Some kv)))) + (defun (collector-finish c acc) + (match acc + ((Some (Tuple _ v)) (Some v)) + (None None)))) + +;; A map from each value to how many times it came. +(type (: (Tallying %a) (Union tallying))) + +(impl (Collector (Tallying %a)) + (where (Eq %a)) + (type %elem %a) + (type %acc (TransientMap %a int)) + (type %out (Map %a int)) + (defun (collector-init c) (transientmap-empty)) + (defun (collector-step c acc x) + (transientmap-set! acc x (+ (transientmap-ref-or acc x 0) 1)) + acc) + (defun (collector-finish c acc) (transientmap->map acc))) + +;; Of `(Tuple key value)`s: a map from each key to the list of its values, in +;; the order they came. +(type (: (GroupingPairs %k %v) (Union grouping/pairs))) + +(impl (Collector (GroupingPairs %k %v)) + (where (Eq %k)) + (type %elem (Tuple %k %v)) + (type %acc (TransientMap %k (ListBuilder %v))) + (type %out (Map %k (List %v))) + (defun (collector-init c) (transientmap-empty)) + (defun (collector-step c acc kv) + (def (k v) kv) + (match (transientmap-try-ref acc k) + ((Some b) (listbuilder-add! b v)) + (None + (def b (listbuilder-empty)) + (listbuilder-add! b v) + (transientmap-set! acc k b))) + acc) + (defun (collector-finish c acc) + (map-map-values listbuilder->list (transientmap->map acc)))) + +;; --------------------------------------------------------------------------- +;; Paths: get, try-get, set, update and set! +;; --------------------------------------------------------------------------- +;; +;; Path navigation for nested data structures: +;; - `(get target step ...)`: Reads a nested value along a path. +;; - `(set target step ... value)`: Returns a copy of `target` with the nested value +;; replaced (leaving the original untouched). +;; - `(update target step ... f)`: Returns a copy of `target` with the nested value +;; updated via `(f current-value)`. +;; +;; Steps are resolved purely by syntactic form before type checking: +;; - Field step (`.name`): Accesses record/struct fields or .NET members via `(.name x)` +;; and writes via `record-set`. +;; - Key step (any other expression): Accesses keyed collections (`Refable`) via `ref` +;; and writes (`Settable`) via `ref-set`. +;; +;; Semantics: +;; - `try-get`: Returns `(Some value)` or `None` if any key in the path is missing +;; (evaluating keys lazily from left to right and stopping at the first `None`). +;; - `set` and `update`: Evaluate target, path keys (left-to-right), and value/fn +;; exactly once. Missing intermediate keys raise an error; `set` on the final key +;; adds it if absent on maps. +;; - `set!`: In-place mutation along the path via `ref-set!` (`Settable!`) or `record-set!`. + +;; `(get target step ...)`, and `(get target)`, which is `(deref target)`. +(def/macro (get form inject compare) + (syntax-match form + ((_ target) + #'(deref ,target)) + ((_ target first rest ...) + (path-reads target (Cons first rest))) + (bad (syntax-error bad "get needs a target: (get target step ...), where a step is .field or a key, or (get target) to read a Box or a Param.")))) + +;; `(try-get target step ...)` +(def/macro (try-get form inject compare) + (syntax-match form + ((_ target first rest ...) + (path-tries target (Cons first rest))) + (bad (syntax-error bad "try-get reads along a path and needs at least one step: (try-get target step ...), where a step is .field or a key.")))) + +;; `(set target step ... value)` +(def/macro (set form inject compare) + (syntax-match form + ((_ target first second rest ...) + (path-write inject target (Cons first (Cons second rest)) #f)) + (bad (syntax-error bad "set needs a target, at least one step and a value: (set target step ... value).")))) + +;; `(update target step ... f)` +(def/macro (update form inject compare) + (syntax-match form + ((_ target first second rest ...) + (path-write inject target (Cons first (Cons second rest)) #t)) + (bad (syntax-error bad "update needs a target, at least one step and a function: (update target step ... f).")))) + +;; True for a field step: an identifier written with a leading dot. `...` and a +;; lone `.` are punctuation, not identifiers, so they never get here. +(: (-> Syntax bool)) +(defun (path-dotted? step) + (match step + ((SSym s) + (let ((name (symbol->string s))) + (and (> (string-length name) 1) (string-starts-with? name ".")))) + (_ #f))) + +;; A `.-Name` step, which reads a .NET property. +(: (-> Syntax bool)) +(defun (path-property? step) + (match step + ((SSym s) (string-starts-with? (symbol->string s) ".-")) + (_ #f))) + +;; Rejects a step that cannot be read: punctuation such as `.` or `...`. +(: (-> Syntax Syntax)) +(defun (path-check step) + (match step + ((SPunct p) + (syntax-error step (str "'" p "' is not a path step. A step is .field or a key expression."))) + (_ step))) + +;; One step's read of `acc`. +(: (-> Syntax Syntax Syntax)) +(defun (path-read acc step) + (syntax-at + (if (path-dotted? (path-check step)) + #'(,step ,acc) + #'(ref ,acc ,step)) + step)) + +(: (-> Syntax (List Syntax) Syntax)) +(defun (path-reads acc steps) + (match steps + (Nil acc) + ((Cons step rest) (path-reads (path-read acc step) rest)))) + +;; Every key read is a `try-ref`, and the first `None` is the answer. The last +;; key's Option is already the answer, so it is not taken apart and rebuilt. +(: (-> Syntax (List Syntax) Syntax)) +(defun (path-tries acc steps) + (match steps + (Nil #'(Some ,acc)) + ((Cons step rest) + ;; cond is not available here. + (if (path-dotted? (path-check step)) + (path-tries (path-read acc step) rest) + (let ((probe (syntax-at #'(try-ref ,acc ,step) step))) + (match rest + (Nil probe) + (_ #'(match ,probe + (None None) + ((Some it) ,(path-tries #'it rest)))))))))) + +;; A name for the expansion's own temporaries. Built from a string so that each +;; depth gets a name of its own: hygiene gives every distinct name one fresh +;; spelling per expansion, so a single template name would shadow itself. +(: (-> string int Syntax)) +(defun (path-temp prefix i) + (SSym (string->symbol (str prefix (displayed i))))) + +;; The field a `.name` step writes. A hygiene mark — `.name__37`, when the +;; step was written in another macro's template — is stripped, since a field +;; name is never renamed. Injected, so that this macro's hygiene leaves it +;; alone too. +(: (-> (-> Symbol Syntax) Syntax Syntax)) +(defun (path-field inject step) + (def name + (match step + ((SSym s) + (def dotted (symbol->string s)) + (substring dotted (string-cursor-next dotted (string-cursor-start dotted)) (string-cursor-end dotted))) + (_ ""))) + (def start (string-cursor-start name)) + (inject + (string->symbol + (match (string-search-backward name "__") + ((Some mark) + (if (and (string-cursor>? mark start) + (path-digits? name (string-cursor-next name (string-cursor-next name mark)))) + (substring name start mark) + name)) + (None name))))) + +;; Is everything in `s` from `c` on an ASCII digit, and is there anything? +(: (-> string StringCursor bool)) +(defun (path-digits? s c) + (and (not (string-cursor-end? s c)) + (let go ((c c)) + (if (string-cursor-end? s c) + #t + (let ((ch (string-cursor-ref s c))) + (and (char>=? ch #\0) (char<=? ch #\9) (go (string-cursor-next s c)))))))) + +;; `set` and `update`. `items` is the steps followed by the value or function. +;; +;; Everything that is evaluated is bound first, in order — the target, each +;; key, then the value — and the copy is then built from the bound names, so +;; that nothing is evaluated twice or out of order. +(: (-> (-> Symbol Syntax) Syntax (List Syntax) bool Syntax)) +(defun (path-write inject target items update?) + (def reversed (list-reverse items)) + (def value (list-head reversed)) + (def steps (list-reverse (list-tail reversed))) + (def named (path-key-names steps 1 #t)) + (def obj (path-temp "obj" 0)) + (def new (path-temp "new" 0)) + (def (: leaf (-> Syntax Syntax)) + (if update? + (fun (old) #'(,new ,old)) + (fun (old) new))) + (def body (path-body inject obj 0 named steps leaf)) + #'(let ((,obj ,target)) + ,(list-foldr (fun (binding inner) #'(let (,binding) ,inner)) + #'(let ((,new ,value)) ,body) + (path-key-bindings named steps)))) + +;; The steps, each key replaced by the name it is bound to. A field step is +;; kept. When `copying?`, a .NET property is refused: `set` copies with +;; `record-set`, which has no way to copy a .NET object. `set!` copies nothing +;; and only reads the steps before its last, so it reads a property there. +(: (-> (List Syntax) int bool (List Syntax))) +(defun (path-key-names steps i copying?) + (match steps + (Nil Nil) + ((Cons step rest) + (if (path-dotted? (path-check step)) + (if (and copying? (path-property? step)) + (Cons (syntax-error step + (str (syntax->string step) " is a .NET property. set and update copy along the path with record-set, which only copies a Bjolang record or struct; set the property itself, or build the new value with get.")) + Nil) + (Cons step (path-key-names rest (+ i 1) copying?))) + (Cons (path-temp "key" i) (path-key-names rest (+ i 1) copying?)))))) + +;; `(key-n expr)` for each key step, in order. +(: (-> (List Syntax) (List Syntax) (List Syntax))) +(defun (path-key-bindings named steps) + (match named + (Nil Nil) + ((Cons name names) + (def step (list-head steps)) + (def rest (path-key-bindings names (list-tail steps))) + (if (path-dotted? step) + rest + (Cons #'(,name ,step) rest))))) + +;; The copy of `obj` with the rest of the path written. `named` and `steps` run +;; side by side: `named` is what the code uses, and `steps` is where each step +;; was written. +(: (-> (-> Symbol Syntax) Syntax int (List Syntax) (List Syntax) (-> Syntax Syntax) Syntax)) +(defun (path-body inject obj depth named steps leaf) + (match named + (Nil (syntax-error obj "internal error: a path ran out of steps.")) + ((Cons key names) + (def step (list-head steps)) + (def rest (list-tail steps)) + (def inner (path-temp "obj" (+ depth 1))) + (def field? (path-dotted? step)) + (def old (if field? #'(,step ,obj) #'(ref ,obj ,key))) + (def value + (match rest + (Nil (leaf old)) + (_ #'(let ((,inner ,old)) + ,(path-body inject inner (+ depth 1) names rest leaf))))) + (syntax-at + (if field? + #'(record-set ,obj (,(path-field inject step) ,value)) + #'(ref-set ,obj ,key ,value)) + step)))) + +;; `(set! target step ... value)`: the parser hands `set!` with three or more +;; operands here, and keeps `(set! name value)`, which writes a variable. +;; +;; Nothing is copied. The steps before the last are read, as `get` reads them, +;; and the last is written in place: a key with `ref-set!` (`Settable!`), a +;; field with `record-set!`, so the field has to be `#:mutable`. The target, +;; each key from left to right and then the value are each evaluated once, +;; before anything is read, as in `set`. +(def/hash-extend (set! form inject compare) + (syntax-match form + ((_ target first second rest ...) + (path-write! inject target (Cons first (Cons second rest)))) + (bad (syntax-error bad "set! writes a variable, (set! name value), or along a path in place, (set! target step ... value), where a step is .field or a key.")))) + +(: (-> (-> Symbol Syntax) Syntax (List Syntax) Syntax)) +(defun (path-write! inject target items) + (def reversed (list-reverse items)) + (def value (list-head reversed)) + (def steps (list-reverse (list-tail reversed))) + (def named (path-key-names steps 1 #f)) + (def obj (path-temp "obj" 0)) + (def new (path-temp "new" 0)) + #'(let ((,obj ,target)) + ,(list-foldr (fun (binding inner) #'(let (,binding) ,inner)) + #'(let ((,new ,value)) ,(path-store! inject obj named steps new)) + (path-key-bindings named steps)))) + +;; The reads of the steps before the last, then the write of the last. The +;; container a field is written in is bound to a name first, because +;; `record-set!` takes the name of a record and not an expression. +(: (-> (-> Symbol Syntax) Syntax (List Syntax) (List Syntax) Syntax Syntax)) +(defun (path-store! inject acc named steps new) + (match named + (Nil (syntax-error acc "internal error: a path ran out of steps.")) + ((Cons key names) + (def step (list-head steps)) + (def rest (list-tail steps)) + (def field? (path-dotted? step)) + (match rest + (Nil + ;; cond is not available here. + (syntax-at + (if (not field?) + #'(ref-set! ,acc ,key ,new) + (if (path-property? step) + (syntax-error step + (str (syntax->string step) " is a .NET property, and set! writes the last step with record-set!, which writes a field of a Bjolang record. Call the property's setter, (.set_Name obj value), instead.")) + ;; The write carries the step's position, which is also + ;; what makes it the writing module's: `record-set!` may + ;; write only the fields of a record its own module declared. + (let ((dest (path-temp "dest" 0))) + #'(let ((,dest ,acc)) + ,(syntax-at #'(record-set! ,dest (,(path-field inject step) ,new)) step))))) + step)) + (_ + (def read (syntax-at (if field? #'(,step ,acc) #'(ref ,acc ,key)) step)) + (path-store! inject read names rest new)))))) + +;; --------------------------------------------------------------------------- +;; Hash macros +;; --------------------------------------------------------------------------- +;; +;; `#name(args...)` calls a hash macro, its own namespace beside the functions: +;; `#map` and `map` are two things. `(def/hash-extend (name form inject compare) +;; body)` writes one; like any macro it cannot be used in the module that +;; defines it, so nothing below uses these four. +;; +;; The parser hands an operator with three or more operands to the first three: +;; `(+ a b c)` is `#fl(+ a b c)` and `(< a b c)` is `#ch(< a b c)`. + +;; `#fl(op a b c ...)` — `(op (op a b) c)`, folded left. +(def/hash-extend (fl form inject compare) + (syntax-match form + ((_ op a b rest ...) (fold-left-forms op #'(,op ,a ,b) rest)) + (bad (syntax-error bad "#fl takes an operator and at least two operands: #fl(+ a b c).")))) + +(: (-> Syntax Syntax (List Syntax) Syntax)) +(defun (fold-left-forms op acc rest) + (match rest + (Nil acc) + ((Cons x more) (fold-left-forms op #'(,op ,acc ,x) more)))) + +;; `#fr(op a b c ...)` — `(op a (op b c))`, folded right. +(def/hash-extend (fr form inject compare) + (syntax-match form + ((_ op a b rest ...) (fold-right-forms op a (Cons b rest))) + (bad (syntax-error bad "#fr takes an operator and at least two operands: #fr(Cons 1 2 Nil).")))) + +(: (-> Syntax Syntax (List Syntax) Syntax)) +(defun (fold-right-forms op x rest) + (match rest + (Nil x) + ((Cons y more) #'(,op ,x ,(fold-right-forms op y more))))) + +;; `#ch(op a b c ...)` — `(op a b)`, and `(op b c)`, and so on, each tested only +;; if the one before it held. A middle operand that is not an atom is bound +;; once, right before its first use. The operands are evaluated left to right: +;; with three or more, a first operand that is not an atom is bound before the +;; second is evaluated, and a name is read before the operand after it is. +;; +;; Three template names for the temporaries. `t` and `u` hold operands and are +;; taken in turn: hygiene gives a template name one fresh spelling per +;; expansion, so a single name would shadow itself and `(op t t)` would compare +;; a value to itself. `s` holds a name's value, and is never needed again once +;; the operand after the name is bound. +(def/hash-extend (ch form inject compare) + (syntax-match form + ((_ op a b) #'(,op ,a ,b)) + ((_ op a b rest ...) + (if (syntax-atom? a) + (chain-forms op a (syntax-constant? a) (Cons b rest) #t) + #'(let ((u ,a)) ,(chain-forms op #'u #t (Cons b rest) #t)))) + (bad (syntax-error bad "#ch takes an operator and at least two operands: #ch(< a b c).")))) + +;; `prev` is safe to read twice: the first operand, an atom, or a temporary. +;; `fixed` says whether reading it later gives what reading it now would: a +;; constant and a temporary do, and a name does not if the operand after it +;; sets it. So a name is bound before an operand that is not an atom is +;; evaluated. The last comparison needs no binding: a call evaluates its +;; arguments left to right. +;; `use-t` says which temporary is free; the other may be `prev`. +(: (-> Syntax Syntax bool (List Syntax) bool Syntax)) +(defun (chain-forms op prev fixed rest use-t) + (match rest + (Nil prev) + ((Cons x Nil) #'(,op ,prev ,x)) + ((Cons x more) + (if (syntax-atom? x) + #'(if (,op ,prev ,x) ,(chain-forms op x (syntax-constant? x) more use-t) #f) + (if (not fixed) + #'(let ((s ,prev)) ,(chain-forms op #'s #t rest use-t)) + (if use-t + #'(let ((t ,x)) (if (,op ,prev t) ,(chain-forms op #'t #t more #f) #f)) + #'(let ((u ,x)) (if (,op ,prev u) ,(chain-forms op #'u #t more #t) #f)))))))) + +;; Cheap to evaluate twice: anything but a list. +(: (-> Syntax bool)) +(defun (syntax-atom? s) + (match s + ((SList _) #f) + (_ #t))) + +;; The same each time it is evaluated: an atom that is not a name. +(: (-> Syntax bool)) +(defun (syntax-constant? s) + (match s + ((SList _) #f) + ((SSym _) #f) + (_ #t))) + +;; `#map((k v) ...)` — a `Map`. An entry is `(k v)`, `[k v]` or `(k . v)`. +(def/hash-extend (map form inject compare) + (syntax-match form + ((_) #'(map-empty)) + ((_ entry ...) #'(list->map ,(map-entries entry))))) + +(: (-> (List Syntax) Syntax)) +(defun (map-entries entries) + (match entries + (Nil #'Nil) + ((Cons entry rest) + (syntax-match entry + (('vec-literal k v) #'(Cons (Tuple ,k ,v) ,(map-entries rest))) + (('Tuple k v) #'(Cons (Tuple ,k ,v) ,(map-entries rest))) + ((k v) #'(Cons (Tuple ,k ,v) ,(map-entries rest))) + (bad (syntax-error bad "A map entry is (key value), [key value] or (key . value).")))))) + +;; --------------------------------------------------------------------------- +;; Cancellation scopes +;; --------------------------------------------------------------------------- +;; +;; A scope owns the fibers started inside it. `(spawn ...)` and `(bjo ...)` put +;; a fiber into the current scope, and the scope does not return until every one +;; of those fibers has finished. So this +;; +;; (with-cancel (cancel) +;; (spawn (process a)) +;; (spawn (process b))) +;; +;; waits for both, and the work is over when the form returns. +;; +;; Stopping early is what the cancel thunk is for: +;; +;; (with-cancel (cancel) +;; (spawn (worker jobs answers)) +;; (def answer (sync (chan-recv answers))) +;; (cancel (Requested "got it")) +;; answer) +;; +;; The cost of waiting first rather than cancelling first: a scope whose children +;; never end, and which never calls `cancel`, hangs here. That is the intended +;; trade. A hang says where it is; a scope that returned while its children ran +;; on says nothing at all. +;; +;; Waiting is the whole of what the four forms do. In a bjoroutine the wait +;; suspends; in ordinary code it parks the thread, as the end of a `defun` +;; `main` does. `scope-form` writes both, with `by-colour`. + +;; The shared expansion of `with-cancel`, `with-deadline` and `with-shield`. +;; +;; `opener` is the call that makes the scope, and it is the only difference +;; between the three: a linked scope, a linked scope with a timer, or an +;; unlinked one. Everything after that — install, run, close, restore — is the +;; same, which is the point of writing it once. +;; +;; The two nested `try`s are one mechanism each. The inner one turns a failure in +;; the body into a value, so that the close can be told about it and can cancel +;; the other children before waiting for them. The outer one restores the +;; dynamic binding however the form is left; a fired token left installed would +;; make every later scope look cancelled. +;; +;; `(raise failure)` puts the body's exception back with the stack it arrived +;; with. It is not reached when the close raised first, which is what happens +;; when children failed too — the close reports them together. +(def/macro (scope-form form inject compare) + (syntax-match form + ((_ opener (cancel) first body ...) + #'(let () + (def scope ,opener) + (def saved (scope-push! scope)) + (def ,cancel (scope-canceller scope)) + (try + (match (try (let () ,first ,@body) #:catch (System.Exception)) + ((Ok answer) + (by-colour (#:sync (scope-close!/blocking scope None)) + (#:bjo (scope-close! scope None))) + answer) + ((Err failure) + (by-colour (#:sync (scope-close!/blocking scope (Some failure))) + (#:bjo (scope-close! scope (Some failure)))) + (raise failure))) + #:finally (dyn-restore! saved)))) + (bad (syntax-error bad "scope-form takes an opener, a cancel thunk name and a body. It is what with-scope, with-cancel, with-deadline and with-shield expand to; write one of those.")))) + +;; `(with-scope body ...)` - a scope with no cancel thunk. +;; +;; The plain form, and the one to put around resources. Everything the other +;; three do about fibers it does too: it waits for what was spawned in it, and +;; it releases what was owned in it, newest first. +;; +;; The cancel thunk is bound to a name the template made up, so nothing in the +;; body can reach it. A scope that wants to stop early says so with +;; `with-cancel`; a scope that only wants ownership says so with this. +(def/macro (with-scope form inject compare) + (syntax-match form + ((_ first body ...) + #'(scope-form (scope-open! 0) (no-cancel) ,first ,@body)) + (bad (syntax-error bad "with-scope takes a body: (with-scope body ...). For a cancel thunk write with-cancel.")))) + +;; `(with-cancel (cancel) body ...)` — a scope that owns what it starts. +(def/macro (with-cancel form inject compare) + (syntax-match form + ((_ (cancel) first body ...) + #'(scope-form (scope-open! 0) (,cancel) ,first ,@body)) + (bad (syntax-error bad "with-cancel names its cancel thunk and takes a body: (with-cancel (cancel) body ...)")))) + +;; `(with-deadline duration body ...)` — the same scope, whose token also fires +;; on its own once `duration` has passed, with `Deadline`. +;; +;; The timer belongs to the scope rather than to a fiber racing it. A watcher +;; fiber cannot work here: the scope waits for its children before it fires its +;; token, and the watcher would be waiting for that token — so each would be +;; waiting for the other. +(def/macro (with-deadline form inject compare) + (syntax-match form + ((_ ms first body ...) + #'(scope-form (deadline-open! (duration->timer-ms ,ms)) (fire) ,first ,@body)) + (bad (syntax-error bad "with-deadline takes a Duration and a body: (with-deadline (seconds 5) body ...)")))) + +;; --------------------------------------------------------------------------- +;; The dynamic environment, for another thread +;; --------------------------------------------------------------------------- + +;; Opaque: taken by `current-dynamic-environment`, used by +;; `call-with-dynamic-environment`, and nothing else. +(import/class + (DynamicEnvironment (: Bjolang.Runtime.DynamicEnvironment))) + +(import/extern + (current-dynamic-environment (: Bjolang.Runtime.DynamicEnvironments.Current (-> DynamicEnvironment))) + (call-with-dynamic-environment (: Bjolang.Runtime.DynamicEnvironments.Call (-> DynamicEnvironment (-> %a) %a)))) + +;; `(with-shield body ...)` and `(with-shield #:within ms body ...)` — a scope +;; whose token is *not* linked to its parent's. +;; +;; This is how cleanup runs after a cancellation. Every `sync` consults the +;; ambient token, so once a scope has been cancelled every `sync` in it raises — +;; including the ones in the code that is trying to tidy up. Inside a shield the +;; ambient token is a fresh one that has not fired, so a `sync` works again: +;; +;; (with-shield #:within (seconds 5) +;; (spawn (flush-buffer a)) +;; (spawn (flush-buffer b))) +;; +;; It is the same object as any other scope with one flag different, and that is +;; deliberate: `(current-cancel)` inside a shield also reads as "not cancelled", +;; so a hand-written `until-cancelled` agrees with what `sync` is doing rather +;; than contradicting it. +;; +;; The deadline is not optional and defaults to thirty seconds. A shield with no +;; deadline is unreachable from outside by construction, which would make the +;; program impossible to kill. +;; +;; A shield is a scope, so it waits for its own children like any other. +(def/macro (with-shield form inject compare) + (syntax-match form + ((_ #:within ms first body ...) + #'(scope-form (shield-open! (duration->timer-ms ,ms)) (fire) ,first ,@body)) + ((_ first body ...) + #'(with-shield #:within (seconds 30) ,first ,@body)) + (bad (syntax-error bad "with-shield takes a body and an optional deadline: (with-shield #:within (seconds 5) body ...)")))) + +;; --------------------------------------------------------------------------- +;; Scope Resource Management +;; --------------------------------------------------------------------------- +;; +;; Scopes manage unmanaged resources (OS handles, file descriptors, network connections) +;; whose lifetimes must be strictly bound rather than left to garbage collection. +;; +;; Cleanup Lifecycle: +;; - Registered cleanup thunks run when the scope exits, after all spawned fibers +;; have completed. +;; - Cleanup runs in LIFO order (newest first), ensuring wrappers (e.g., buffered ports) +;; close before their underlying resources (e.g., streams). +;; - Cleanup thunks are synchronous and non-suspending. If a cleanup callback throws, +;; the exception is recorded and remaining cleanup thunks continue to run. +;; +;; Scope Boundaries & Fibers: +;; - Fibers do NOT introduce their own scopes. A resource opened inside `(spawn ...)` +;; or `(bjo ...)` belongs to the ambient enclosing scope and outlives the spawned fiber. +;; - For workloads that create many short-lived resources within a long-lived operation, +;; use nested child scopes (`(with-scope ...)`) per work unit to reclaim resources promptly. +;; - A closing scope refuses new fibers (`spawn`), and a closed scope refuses new resource +;; registrations (`own!`). + +;; `(in-scope s body ...)` - run the body with `s` as the ambient scope. +;; +;; The way to own something in a scope other than the innermost one, and the +;; way back in from a .NET callback, which runs on a foreign thread with no +;; ambient scope at all: +;; +;; (def home (current-scope)) +;; (on-event (fun (e) (in-scope home (handle e)))) +;; +;; This installs a scope; it does not open or close one. Whoever opened `s` is +;; still the one that waits for it. +;; +;; A form and not a keyword argument on `spawn` and `own!`: Bjolang has no +;; rest-keywords, so a `#:scope` could not be forwarded through a wrapper, and +;; the two would drift apart. +(def/macro (in-scope form inject compare) + (syntax-match form + ((_ scope first body ...) + #'(let () + (def saved (scope-push! ,scope)) + (try (let () ,first ,@body) + #:finally (dyn-restore! saved)))) + (bad (syntax-error bad "in-scope takes a scope and a body: (in-scope s body ...)")))) + +;; `(own-disposable x)` - register `.Dispose` on the ambient scope, and hand +;; `x` straight back. +;; +;; A form rather than a function so that `.Dispose` is resolved at the call +;; site, the way `with-open` resolves it. A function would need the whole +;; `System.IDisposable` constraint apparatus to say the same thing. +;; +;; `with-open` is the other spelling: that one for a resource whose lifetime is +;; one block, this one for a resource whose lifetime is the scope's. +(def/macro (own-disposable form inject compare) + (syntax-match form + ((_ x) + ;; `unit` after the call, not the call itself: `.Dispose` is a void .NET + ;; method, so it has no value for the thunk to return. + #'(let ((it ,x)) + (ignore (own! (fun () (.Dispose it) unit))) + it)) + (bad (syntax-error bad "own-disposable takes one expression: (own-disposable (StreamReader. path))")))) + +;; --------------------------------------------------------------------------- +;; Durations +;; --------------------------------------------------------------------------- +;; +;; A length of time, which is what every timer takes. A number of milliseconds +;; is a number whose unit is in no type: `5000` is easy to misread, and a count +;; that overflows becomes a different deadline without a word. A Duration has +;; its unit, and making one past its range raises. +;; +;; In .NET ticks of 100 ns, within ±long-max ticks, about ±29 227 years. Leaving +;; out long's minimum gives every duration a negation, so each one prints and +;; reads back. (std datetime) adds the arithmetic, the calendar and the text. + +(import/class + (DurationSpan (: System.TimeSpan (-> long DurationSpan))) + (DurationOverflow (: System.OverflowException (-> string DurationOverflow)))) + +(import/extern + (clr-duration-span-ticks (: System.TimeSpan.Ticks (-> DurationSpan long) #:get))) + +(type (: Duration #:opaque (Struct (: v DurationSpan)))) + +(: (-> %a)) +(defun (duration-overflow) + (raise (cast Exception (DurationOverflow. "a duration is outside ±29 227 years (±9 223 372 036 854 775 807 ticks)")))) + +;; The duration of `t` ticks. Raises at long's minimum, the one tick count +;; with no negation. +(: (-> long Duration)) +(defun (ticks->duration t) + (when (= t -9223372036854775808L) (duration-overflow)) + (Duration (v (DurationSpan. t)))) + +(: (-> Duration long)) +(defun (duration-ticks d) (clr-duration-span-ticks (struct-ref d v))) + +;; n units of `unit` ticks, or a raise past the range of a Duration. +(: (-> long long Duration)) +(defun (units->duration n unit) + (def limit (/ 9223372036854775807L unit)) + (when (or (> n limit) (< n (- limit))) (duration-overflow)) + (Duration (v (DurationSpan. (* n unit))))) + +(: (-> long Duration)) +(defun (milliseconds n) (units->duration n 10000L)) + +(: (-> long Duration)) +(defun (seconds n) (units->duration n 10000000L)) + +(: (-> long Duration)) +(defun (minutes n) (units->duration n 600000000L)) + +(: (-> long Duration)) +(defun (hours n) (units->duration n 36000000000L)) + +(impl (Ord Duration) + (defun (compare a b) (compare (duration-ticks a) (duration-ticks b)))) + +;; ISO 8601: PT26H30M, PT0.5S, with one leading sign: -PT1H30M. +(impl (Written Duration) + (defun (written d) (duration->iso (duration-ticks d))) + (defun (write-into d b) (stringbuilder-add-string! b (written d)))) + +(: (-> long string)) +(defun (duration->iso t) + (if (= t 0L) + "PT0S" + (let () + (def a (if (< t 0L) (- t) t)) + (def h (/ a 36000000000L)) + (def m (% (/ a 600000000L) 60L)) + (def s (% (/ a 10000000L) 60L)) + (def f (% a 10000000L)) + (string-append + (if (< t 0L) "-PT" "PT") + (string-append + (if (= h 0L) "" #"${h}H") + (string-append + (if (= m 0L) "" #"${m}M") + (if (and (= s 0L) (= f 0L)) "" #"${s}${(tick-fraction f)}S"))))))) + +;; ".5" for half a second: up to seven digits, without trailing zeros, and +;; nothing for a whole second. +(: (-> long string)) +(defun (tick-fraction f) + (if (= f 0L) + "" + (let trim ((f f) (width 7)) + (if (= (% f 10L) 0L) + (trim (/ f 10L) (- width 1)) + (let pad ((digits (displayed f))) + (if (< (string-length digits) width) + (pad (string-append "0" digits)) + (string-append "." digits))))))) + +;; Milliseconds for a timer, which counts them in a long. A part of a +;; millisecond counts as a whole one, so that a positive duration does not +;; become zero, which a timer takes as a time already passed. +(: (-> Duration long)) +(defun (duration->timer-ms d) + (def t (duration-ticks d)) + (if (<= t 0L) 0L (+ (/ (- t 1L) 10000L) 1L))) + +(import/extern + (clr-timeout-ms (: BjolangRuntime.timeout (-> long (Event Unit))))) + +;; An event that is ready `d` after each sync of it. Zero or less is ready at +;; once. +(: (-> Duration (Event Unit))) +(defun (timeout d) (clr-timeout-ms (duration->timer-ms d))) + + +;; --------------------------------------------------------------------------- +;; Select +;; --------------------------------------------------------------------------- + +;; `(select branch ...)` - sync on the events of the branches, then run the +;; body of the branch whose event committed. +;; +;; - Branches: +;; - `(event pattern body ...)`: `event` is any `(Event %a)`. When it +;; commits, its value is matched against `pattern` and `body` runs. +;; - `(#:timeout duration body ...)`: a branch on `(timeout duration)`. +;; - `(#:else body ...)`: runs when no other branch can commit at once. The +;; sync does not wait. It must be the last branch. +;; - Types: each event has its own value type. The bodies have one type, which +;; is the type of the form. +;; - Order: the events are offered in the order written. When several can +;; commit at once, the first one wins, as with `choose`. +;; - Bodies run after the sync, in the code that the form is in. In a +;; bjoroutine a body can suspend. A branch that loses runs no body, and a +;; `with-nack` in its event fires. +;; - Colour: in a bjoroutine the form suspends, as `sync`. In a `defun` it +;; parks the thread, as `sync/blocking`. +;; - Errors: a pattern that can fail to match makes the form a non-exhaustive +;; `match`, which is a compile error. +;; +;; Expansion: `wrap` tags each value with nested `Ok` and `Err`, one `choose` +;; takes the tagged events, and a `match` on the tag runs the body. The +;; compiler emits a `choose` of two to four `wrap`s at a `sync` as one +;; `Cml.WrapChooseOnce` event, so a warm sync of up to four branches +;; allocates nothing. +(def/macro (select form inject compare) + (syntax-match form + ((_) (syntax-error form "select takes branches: (select (event pattern body ...) ... (#:timeout duration body ...) (#:else body ...))")) + ((_ (#:else body ...)) (syntax-error form "select needs a branch with an event, not only (#:else ...). Without one, write the body.")) + ((_ branch ...) (select-form (select-arms branch))) + (bad (syntax-error bad "select takes branches: (select (event pattern body ...) ... (#:timeout duration body ...) (#:else body ...))")))) + +;; One branch of a `select`: its event, its pattern and its body as one form. +(type (: SelectArm (Union (: Arm Syntax Syntax Syntax)))) + +;; The branches of a `select`, in the order written. +(: (-> (List Syntax) (List SelectArm))) +(defun (select-arms branches) + (match branches + (Nil Nil) + ((Cons branch rest) + (syntax-match branch + ((#:else body ...) + (match rest + (Nil (list (Arm #'(always unit) (SSym (string->symbol "_")) (one-form body branch)))) + ((Cons after _) + (list (select-arm-error after "(#:else ...) must be the last branch of a select: it runs only when no branch above it can commit at once."))))) + ((#:timeout duration body ...) + (Cons (Arm #'(timeout ,duration) (SSym (string->symbol "_")) (one-form body branch)) + (select-arms rest))) + ((#:timeout) + (list (select-arm-error branch "(#:timeout ...) is (#:timeout duration body ...)."))) + ((event pattern body ...) + (match event + ((SKey _) + (list (select-arm-error event "A select branch starts with an event, #:timeout or #:else."))) + (_ (Cons (Arm event pattern (one-form body branch)) (select-arms rest))))) + (bad + (list (select-arm-error bad "A select branch is (event pattern body ...), (#:timeout duration body ...) or (#:else body ...)."))))))) + +;; Raises the syntax error. Typed as an arm, for the lists above. +(: (-> Syntax string SelectArm)) +(defun (select-arm-error at message) + (def stx (syntax-error at message)) + (Arm stx stx stx)) + +;; The `match` on the sync of all the arms. +(: (-> (List SelectArm) Syntax)) +(defun (select-form arms) + (def last (- (list-length arms) 1)) + (def event + (match arms + ((Cons (Arm only _ _) Nil) only) + (_ #'(choose ,@(select-wraps arms 0 last))))) + #'(match (by-colour (#:sync (sync/blocking ,event)) (#:bjo (sync ,event))) + ,@(select-clauses arms 0 last))) + +;; Each arm's event, with its value tagged by `select-tag`. +(: (-> (List SelectArm) int int (List Syntax))) +(defun (select-wraps arms i last) + (match arms + (Nil Nil) + ((Cons (Arm event _ _) rest) + (Cons #'(wrap ,event (fun (v) ,(select-tag i last #'v))) + (select-wraps rest (+ i 1) last))))) + +;; Each arm's `match` clause: the arm's tag, and a `match` of the value on the +;; arm's pattern. The inner `match` is what a pattern that can fail is +;; reported on, so the message names the event's values, not the tags. +(: (-> (List SelectArm) int int (List Syntax))) +(defun (select-clauses arms i last) + (match arms + (Nil Nil) + ((Cons (Arm _ pattern body) rest) + (Cons #'(,(select-tag i last #'v) (match v (,pattern ,body))) + (select-clauses rest (+ i 1) last))))) + +;; The tag of arm `i` of `last + 1` arms around `x`, as an expression or a +;; pattern. Arm 0 is `(Ok x)`, arm 1 is `(Err (Ok x))`, and the last arm is +;; `x` inside `last` `Err`s. One arm has no tag. +(: (-> int int Syntax Syntax)) +(defun (select-tag i last x) + (if (= last 0) + x + (if (= i 0) + #'(Ok ,x) + #'(Err ,(select-tag (- i 1) (- last 1) x))))) + + +;; --------------------------------------------------------------------------- +;; Container formats +;; --------------------------------------------------------------------------- +;; A container is written with its elements written, and displayed with them +;; displayed, as Scheme's `write` and `display` do: `(written (list "a"))` is +;; `("a")` and `(displayed (list "a"))` is `(a)`. One function lays out each +;; kind of container, and is given the method that adds an element. + +;; The elements of `xs` between `open` and `close`, with a space between two. +(: (-> string %s string StringBuilder (-> %a StringBuilder void) void) + (where (Iterable %s #:elem %a))) +(defun (elements-print-into open xs close b each) + (stringbuilder-add-string! b open) + (loop (:with first? #t #f) + (:for x xs) + (:do (unless first? (stringbuilder-add-string! b " ")) + (each x b))) + (stringbuilder-add-string! b close)) + +;; `(1 2 3)`. No quote, so a nested list is `((1) (2 3))` and the empty list +;; is `()`. +(impl (Written (List %a)) + (where (Written %a)) + (defun (write-into l b) (elements-print-into "(" l ")" b (fun (x b) (write-into x b))))) + +(impl (Displayed (List %a)) + (where (Displayed %a)) + (defun (display-into l b) (elements-print-into "(" l ")" b (fun (x b) (display-into x b))))) + +;; `[1 2 3]`, as the literal writes it. +(impl (Written (Vec %a)) + (where (Written %a)) + (defun (write-into v b) (elements-print-into "[" v "]" b (fun (x b) (write-into x b))))) + +(impl (Displayed (Vec %a)) + (where (Displayed %a)) + (defun (display-into v b) (elements-print-into "[" v "]" b (fun (x b) (display-into x b))))) + +;; `#[1 2 3]`, as the literal writes it. +(impl (Written (Array %a)) + (where (Written %a)) + (defun (write-into a b) (elements-print-into "#[" a "]" b (fun (x b) (write-into x b))))) + +(impl (Displayed (Array %a)) + (where (Displayed %a)) + (defun (display-into a b) (elements-print-into "#[" a "]" b (fun (x b) (display-into x b))))) + +;; `#iarray[1 2 3]`. There is no literal; the name says which collection it is. +(impl (Written (IArray %a)) + (where (Written %a)) + (defun (write-into a b) (elements-print-into "#iarray[" a "]" b (fun (x b) (write-into x b))))) + +(impl (Displayed (IArray %a)) + (where (Displayed %a)) + (defun (display-into a b) (elements-print-into "#iarray[" a "]" b (fun (x b) (display-into x b))))) + +;; A sequence is not walked: walking it could do anything, or never end. +(impl (Written (Seq %a)) + (defun (written s) "(Seq ...)") + (defun (write-into s b) (stringbuilder-add-string! b "(Seq ...)"))) + +;; The entries of `xs` between `open` and `close`, each in parentheses, a key +;; and its value with a space between. +(: (-> string %s string StringBuilder (-> %k StringBuilder void) (-> %v StringBuilder void) void) + (where (Iterable %s #:elem (Tuple %k %v)))) +(defun (entries-print-into open xs close b each-key each-value) + (stringbuilder-add-string! b open) + (loop (:with first? #t #f) + (:for entry xs) + (:do (def (k v) entry) + (unless first? (stringbuilder-add-string! b " ")) + (stringbuilder-add-string! b "(") + (each-key k b) + (stringbuilder-add-string! b " ") + (each-value v b) + (stringbuilder-add-string! b ")"))) + (stringbuilder-add-string! b close)) + +;; `#map((k v) ...)`, as the `#map` literal writes it. +(impl (Written (Map %k %v)) + (where (Written %k) (Written %v)) + (defun (write-into m b) + (entries-print-into "#map(" m ")" b (fun (k b) (write-into k b)) (fun (v b) (write-into v b))))) + +(impl (Displayed (Map %k %v)) + (where (Displayed %k) (Displayed %v)) + (defun (display-into m b) + (entries-print-into "#map(" m ")" b (fun (k b) (display-into k b)) (fun (v b) (display-into v b))))) + +;; A case of a union the runtime declares, in constructor syntax: `(Some 1)` +;; and `None`, `(Ok 1)` and `(Err "no")`. +(: (-> string %a StringBuilder (-> %a StringBuilder void) void)) +(defun (case-print-into name x b each) + (stringbuilder-add-string! b "(") + (stringbuilder-add-string! b name) + (stringbuilder-add-string! b " ") + (each x b) + (stringbuilder-add-string! b ")")) + +(impl (Written (Option %a)) + (where (Written %a)) + (defun (write-into o b) + (match o + ((Some x) (case-print-into "Some" x b (fun (x b) (write-into x b)))) + (None (stringbuilder-add-string! b "None"))))) + +(impl (Displayed (Option %a)) + (where (Displayed %a)) + (defun (display-into o b) + (match o + ((Some x) (case-print-into "Some" x b (fun (x b) (display-into x b)))) + (None (stringbuilder-add-string! b "None"))))) + +(impl (Written (Result %e %a)) + (where (Written %e) (Written %a)) + (defun (write-into r b) + (match r + ((Ok x) (case-print-into "Ok" x b (fun (x b) (write-into x b)))) + ((Err e) (case-print-into "Err" e b (fun (e b) (write-into e b))))))) + +(impl (Displayed (Result %e %a)) + (where (Displayed %e) (Displayed %a)) + (defun (display-into r b) + (match r + ((Ok x) (case-print-into "Ok" x b (fun (x b) (display-into x b)))) + ((Err e) (case-print-into "Err" e b (fun (e b) (display-into e b))))))) + +;; `(Tuple 1 "a")`, as a tuple is built. +(impl (Written (Tuple %a %b)) + (where (Written %a) (Written %b)) + (defun (write-into t b) + (def (x y) t) + (stringbuilder-add-string! b "(Tuple ") + (write-into x b) + (stringbuilder-add-string! b " ") + (write-into y b) + (stringbuilder-add-string! b ")"))) + +(impl (Displayed (Tuple %a %b)) + (where (Displayed %a) (Displayed %b)) + (defun (display-into t b) + (def (x y) t) + (stringbuilder-add-string! b "(Tuple ") + (display-into x b) + (stringbuilder-add-string! b " ") + (display-into y b) + (stringbuilder-add-string! b ")"))) + +(impl (Written (Tuple %a %b %c)) + (where (Written %a) (Written %b) (Written %c)) + (defun (write-into t b) + (def (Tuple x y z) t) + (stringbuilder-add-string! b "(Tuple ") + (write-into x b) + (stringbuilder-add-string! b " ") + (write-into y b) + (stringbuilder-add-string! b " ") + (write-into z b) + (stringbuilder-add-string! b ")"))) + +(impl (Displayed (Tuple %a %b %c)) + (where (Displayed %a) (Displayed %b) (Displayed %c)) + (defun (display-into t b) + (def (Tuple x y z) t) + (stringbuilder-add-string! b "(Tuple ") + (display-into x b) + (stringbuilder-add-string! b " ") + (display-into y b) + (stringbuilder-add-string! b " ") + (display-into z b) + (stringbuilder-add-string! b ")"))) + +;; Four elements, as far as `Eq` and `Ord` go. +(impl (Written (Tuple %a %b %c %d)) + (where (Written %a) (Written %b) (Written %c) (Written %d)) + (defun (write-into t b) + (def (Tuple w x y z) t) + (stringbuilder-add-string! b "(Tuple ") + (write-into w b) + (stringbuilder-add-string! b " ") + (write-into x b) + (stringbuilder-add-string! b " ") + (write-into y b) + (stringbuilder-add-string! b " ") + (write-into z b) + (stringbuilder-add-string! b ")"))) + +(impl (Displayed (Tuple %a %b %c %d)) + (where (Displayed %a) (Displayed %b) (Displayed %c) (Displayed %d)) + (defun (display-into t b) + (def (Tuple w x y z) t) + (stringbuilder-add-string! b "(Tuple ") + (display-into w b) + (stringbuilder-add-string! b " ") + (display-into x b) + (stringbuilder-add-string! b " ") + (display-into y b) + (stringbuilder-add-string! b " ") + (display-into z b) + (stringbuilder-add-string! b ")"))) + +;; What `displayed` was called for these before it took every type. +(: (-> (List %a) string) (where (Displayed %a))) +(defun (list->str l) (displayed l)) + +(: (-> (Map %k %v) string) (where (Displayed %k) (Displayed %v))) +(defun (map->str m) (displayed m)) + +(: (-> (Tuple %k %v) string) (where (Displayed %k) (Displayed %v))) +(defun (map-entry->str entry) + (def (k v) entry) + (str (displayed k) " " (displayed v))) + +(:deprecated list->str "use `displayed`") +(:deprecated map->str "use `displayed`") +(:deprecated map-entry->str "use `displayed` on the key and the value") + +;; Sequential binding +;; +;; Sequential binding macros. + +;; `(let* ((a 1) (b (+ a 1))) ...)` — each binding is in scope for the next. +(def/macro (let* form compare inject) + (syntax-match form + ((_ () body ...) + #'(let () ,@body)) + ((_ (bind) body ...) + #'(let (,bind) ,@body)) + ((_ (bind binds ...) body ...) + #'(let (,bind) + (let* (,@binds) ,@body))))) + +;; parameterize* behavior. +(def/macro (parameterize* form compare inject) + (syntax-match form + ((_ () body ...) + #'(let () ,@body)) + ((_ (bind) body ...) + #'(parameterize (,bind) ,@body)) + ((_ (bind binds ...) body ...) + #'(parameterize (,bind) + (parameterize* (,@binds) ,@body))))) + +;; --------------------------------------------------------------------------- +;; type/derive +;; --------------------------------------------------------------------------- + +(: (-> string Syntax)) +(defun (derive-sym name) (SSym (string->symbol name))) + +;; Refuse invalid syntax. +(: (-> Syntax string (List Syntax))) +(defun (derive-fail form message) + (ignore (syntax-error form message)) + Nil) + +;; Head type variables. +(: (-> Syntax (List Syntax))) +(defun (derive-head-vars head) + (match head + ((SList items) + (match items + (Nil Nil) + ((Cons _ vars) vars))) + (_ Nil))) + +;; Type name. +(: (-> Syntax Syntax)) +(defun (derive-head-name head) + (match head + ((SList items) + (match items + ((Cons name _) name) + (Nil head))) + (_ head))) + +;; Hash combine fold. +(: (-> (List Syntax) Syntax Syntax)) +(defun (derive-hash-fold parts acc) + (match parts + (Nil acc) + ((Cons p rest) (derive-hash-fold rest #'(hash-combine ,acc ,p))))) + +;; --- Records and structs --- + +;; Field names. +(: (-> (List Syntax) (List Syntax))) +(defun (derive-field-names fields) + (list-map (fun (f) + (syntax-match f + ((_ name _ marker ...) name) + (bad (syntax-error bad "a record field is written (: name type), optionally followed by #:mutable.")))) + fields)) + +;; Any mutable fields. +(: (-> (List Syntax) bool)) +(defun (derive-any-mutable? fields) + (def (Cons f rest) fields + :leave-with #f) + (if (syntax-match f + ((_ _ _ marker markers ...) #t) + (_ #f)) + #t + (derive-any-mutable? rest))) + +;; Eq record methods. +(: (-> Syntax (List Syntax) bool (List Syntax))) +(defun (derive-eq-record head names mutable?) + (def equals + #'(defun (= a b) + (and ,@(list-map (fun (n) #'(= (record-ref a ,n) (record-ref b ,n))) names)))) + (def hashed + (if mutable? + #'(defun (eq-hash v) (unhashable ,(SStr (syntax->string (derive-head-name head))))) + #'(defun (eq-hash v) + ,(derive-hash-fold (list-map (fun (n) #'(eq-hash (record-ref v ,n))) names) #'17)))) + (Cons equals (Cons hashed Nil))) + +;; --- Unions --- + +;; A case is a bare name or `(: Name type ... markers ...)`. +(: (-> Syntax Syntax)) +(defun (derive-case-name c) + (def* ((SList items) c :leave-with c) + ((Cons _ (Cons name _)) items :leave-with c)) + name) + +;; The payload types, which end at the first marker: `#:tag` takes a name after +;; it, which is not a type. +(: (-> Syntax (List Syntax))) +(defun (derive-payload c) + (def* ((SList items) c :leave-with Nil) + ((Cons _ (Cons _ rest)) items :leave-with Nil)) + (derive-types-before-markers rest)) + +(: (-> (List Syntax) (List Syntax))) +(defun (derive-types-before-markers items) + (match items + ((Cons (SKey _) _) Nil) + ((Cons t rest) (Cons t (derive-types-before-markers rest))) + (Nil Nil))) + +;; Binders. +(: (-> string int int (List Syntax))) +(defun (derive-binders side i n) + (if (= i n) + Nil + (Cons (derive-sym (string-append side (int->string i))) + (derive-binders side (+ i 1) n)))) + +(: (-> (List Syntax) (List Syntax) (List Syntax))) +(defun (derive-eq-pairs ls rs) + (def* ((Cons l lrest) ls :leave-with Nil) + ((Cons r rrest) rs :leave-with Nil)) + (Cons #'(= ,l ,r) (derive-eq-pairs lrest rrest))) + +;; Union arm eq. +(: (-> Syntax Syntax)) +(defun (derive-eq-union-arm c) + (def name (derive-case-name c)) + (def n (list-length (derive-payload c))) + (def ls (derive-binders "__derived-l" 0 n)) + (def rs (derive-binders "__derived-r" 0 n)) + #'((,name ,@ls) + (match b + ((,name ,@rs) (and ,@(derive-eq-pairs ls rs))) + (,(derive-sym "_") #f)))) + +;; Hash union arms. +(: (-> (List Syntax) int (List Syntax))) +(defun (derive-hash-union-arms cases index) + (def (Cons c rest) cases :leave-with Nil) + (def name (derive-case-name c)) + (def ls (derive-binders "__derived-l" 0 (list-length (derive-payload c)))) + (def tag (SInt (int->string index))) + (Cons #'((,name ,@ls) + ,(derive-hash-fold (list-map (fun (l) #'(eq-hash ,l)) ls) tag)) + (derive-hash-union-arms rest (+ index 1)))) + +(: (-> (List Syntax) (List Syntax))) +(defun (derive-eq-union cases) + (def equals + #'(defun (= a b) (match a ,@(list-map derive-eq-union-arm cases)))) + (def hashed + #'(defun (eq-hash v) (match v ,@(derive-hash-union-arms cases 0)))) + (Cons equals (Cons hashed Nil))) + +;; --- Ordering --- + +;; Ord chain. +(: (-> (List Syntax) int Syntax)) +(defun (derive-ord-chain parts i) + (def* ((Cons p rest) parts :leave-with #'0) + (c (derive-sym (string-append "__derived-c" (int->string i))))) + (if (list-empty? rest) + p + #'(let ((,c ,p)) + (if (= ,c 0) ,(derive-ord-chain rest (+ i 1)) ,c)))) + +;; Ord record. +(: (-> (List Syntax) (List Syntax))) +(defun (derive-ord-record names) + (Cons #'(defun (compare a b) + ,(derive-ord-chain + (list-map (fun (n) #'(compare (record-ref a ,n) (record-ref b ,n))) names) + 0)) + Nil)) + +;; Wildcards. +(: (-> int (List Syntax))) +(defun (derive-wildcards n) + (if (= n 0) + Nil + (Cons (derive-sym "_") (derive-wildcards (- n 1))))) + +;; Index arms. +(: (-> (List Syntax) int (List Syntax))) +(defun (derive-ord-index-arms cases index) + (def (Cons c rest) cases :leave-with Nil) + (Cons #'((,(derive-case-name c) ,@(derive-wildcards (list-length (derive-payload c)))) + ,(SInt (int->string index))) + (derive-ord-index-arms rest (+ index 1)))) + +(: (-> (List Syntax) (List Syntax) (List Syntax))) +(defun (derive-ord-pairs ls rs) + (def* ((Cons l lrest) ls :leave-with Nil) + ((Cons r rrest) rs :leave-with Nil)) + (Cons #'(compare ,l ,r) (derive-ord-pairs lrest rrest))) + +;; Ord union arm. +(: (-> Syntax Syntax)) +(defun (derive-ord-union-arm c) + (def name (derive-case-name c)) + (def n (list-length (derive-payload c))) + (def ls (derive-binders "__derived-l" 0 n)) + (def rs (derive-binders "__derived-r" 0 n)) + #'((,name ,@ls) + (match b + ((,name ,@rs) ,(derive-ord-chain (derive-ord-pairs ls rs) 0)) + (,(derive-sym "_") 0)))) + +;; Ord union. +(: (-> (List Syntax) (List Syntax))) +(defun (derive-ord-union cases) + (def ia (derive-sym "__derived-ia")) + (def ib (derive-sym "__derived-ib")) + (def index-arms (derive-ord-index-arms cases 0)) + (Cons #'(defun (compare a b) + (let ((,ia (match a ,@index-arms)) + (,ib (match b ,@index-arms))) + (if (= ,ia ,ib) + (match a ,@(list-map derive-ord-union-arm cases)) + (compare ,ia ,ib)))) + Nil)) + +;; --- Putting one implementation together --- + +;; Shape error. +(: (-> Syntax (List Syntax))) +(defun (derive-shape-error bad) + (derive-fail bad "a type that derives has to be a Record, a Struct or a Union. An alias is a second spelling of a type rather than a type of its own — derive for the type it names.")) + +;; Eq methods. +(: (-> Syntax Syntax (List Syntax))) +(defun (derive-eq-methods head shape) + (syntax-match shape + (('Record fields ...) (derive-eq-record head (derive-field-names fields) (derive-any-mutable? fields))) + (('Struct fields ...) (derive-eq-record head (derive-field-names fields) (derive-any-mutable? fields))) + (('Union cases ...) (derive-eq-union cases)) + (('Enum cases ...) (derive-eq-union cases)) + (('Sum cases ...) (derive-eq-union cases)) + (bad (derive-shape-error bad)))) + +(: (-> Syntax (List Syntax))) +(defun (derive-ord-methods shape) + (syntax-match shape + (('Record fields ...) (derive-ord-record (derive-field-names fields))) + (('Struct fields ...) (derive-ord-record (derive-field-names fields))) + (('Union cases ...) (derive-ord-union cases)) + (('Enum cases ...) (derive-ord-union cases)) + (('Sum cases ...) (derive-ord-union cases)) + (bad (derive-shape-error bad)))) + +(: (-> Syntax Syntax Syntax (List Syntax))) +(defun (derive-methods trait head shape) + (if (syntax-ident=? trait (derive-sym "Eq")) + (derive-eq-methods head shape) + (if (syntax-ident=? trait (derive-sym "Ord")) + (derive-ord-methods shape) + (derive-fail trait "the traits that can be derived are Eq and Ord.")))) + +;; Derive conditional implementation. A newtype's is the type it wraps, +;; whatever the trait. +(: (-> Syntax Syntax Syntax)) +(defun (derive-impl trait typedef) + (syntax-match typedef + ((_ head ('Newtype underlying)) + #'(impl (,trait ,head) #:via ,underlying)) + ((_ head shape) + #'(impl (,trait ,head) + (where ,@(list-map (fun (v) #'(,trait ,v)) (derive-head-vars head))) + ,@(derive-methods trait head shape))) + (bad (syntax-error bad "a type declaration is written (: Name (Record ...)).")))) + +(: (-> (List Syntax) (List Syntax) (List Syntax))) +(defun (derive-impls traits typedefs) + (match traits + (Nil Nil) + ((Cons t rest) + (list-append (list-map (fun (td) (derive-impl t td)) typedefs) + (derive-impls rest typedefs))))) + +(def/macro (type/derive form inject compare) + (syntax-match form + ((_ (trait ...) typedef ...) + #'(begin + (type ,@typedef) + ,@(derive-impls trait typedef))) + (bad (syntax-error bad "type/derive takes a list of traits and one or more type declarations, as in (type/derive (Eq) (: Point (Record (: x int)))).")))) + +;; ------------------------------------------------------------- +;; Extra thingies for cml +;; + +(: (-> (Promise %a) %a)) +(defun (promise-join/sync p) + (match (sync/blocking (promise-join p)) + ((Err e) (raise e)) + ((Ok v) v))) + + +;; Pattern matching thingies +;; +;; `Has` matches a `Refable`: the key is read with `try-ref`, and the pattern is +;; matched against the value when the key is there. +;; (match my-map +;; ((Has :my-key pat) (println #"The value of :my-key is ${pat}")) +;; (_ (panic! "it didn't have it"))) +(def/pattern (Has form inject compare) + (syntax-match form + ((_ key p) #'(:view (try-ref & ,key) (Some ,p))) + (bad (syntax-error bad "(Has key pattern) matches a Refable that holds key.")))) + +(def/pattern (Has* form inject compare) + (syntax-match form + ((_) (syntax-error form "(Has* (key pattern) ...) needs at least one key.")) + ((_ (key pat) ...) + #'(and ,@{listing #'(Has ,k ,p) (:for k key) (:for p pat)})) + (bad (syntax-error bad "(Has* (key pattern) ...) pairs each key with a pattern for its value.")))) + +;; --------------------------------------------------------------------------- +;; Taking, dropping and reversing a vec +;; --------------------------------------------------------------------------- +;; +;; A count past the end takes or drops the whole vec, as for a string; a +;; negative count is refused. + +(: (-> (Vec %a) int (Vec %a))) +(defun (vec-take v n) + (check-count "vec-take" n) + (def len (vec-length v)) + (vec-slice v 0 (if (< n len) n len))) + +(: (-> (Vec %a) int (Vec %a))) +(defun (vec-drop v n) + (check-count "vec-drop" n) + (def len (vec-length v)) + (def k (if (< n len) n len)) + (vec-slice v k (- len k))) + +(: (-> (Vec %a) (Vec %a))) +(defun (vec-reverse v) + {vecing x (:for x (in-reverse-vec v))}) + +;; The vec without its last element, or `None` when it is empty. +;; `vec-pop`, which raises on an empty vec, is a compiler builtin. +(: (-> (Vec %a) (Option (Vec %a)))) +(defun (vec-try-pop v) + (if (vec-empty? v) None (Some (vec-pop v)))) + +;; The vec without its first element, or `None` when it is empty. +(: (-> (Vec %a) (Option (Vec %a)))) +(defun (vec-try-pop-first v) + (if (vec-empty? v) None (Some (vec-pop-first v)))) + +;; --------------------------------------------------------------------------- +;; Membership +;; --------------------------------------------------------------------------- +;; +;; Whether a collection holds an element. Each compares with `=` and stops at +;; the first match. Written as loops: `oring` is defined in this file, and a +;; collector macro cannot be used where it is defined. + +(: (-> (List %a) %a bool) (where (Eq %a))) +(defun (list-contains? xs x) + (match xs + (Nil #f) + ((Cons y rest) (if (= y x) #t (list-contains? rest x))))) + +(: (-> (Vec %a) %a bool) (where (Eq %a))) +(defun (vec-contains? v x) + (let go ((c (vec-cursor v))) + (if (vec-cursor-done? c) + #f + (if (= (vec-cursor-current c) x) #t (go (vec-cursor-next c)))))) + +(: (-> (Array %a) %a bool) (where (Eq %a))) +(defun (array-contains? a x) + (def len (array-length a)) + (let go ((i 0)) + (if (= i len) + #f + (if (= (array-ref a i) x) #t (go (+ i 1)))))) + +(: (-> (IArray %a) %a bool) (where (Eq %a))) +(defun (iarray-contains? a x) + (def len (iarray-length a)) + (let go ((i 0)) + (if (= i len) + #f + (if (= (iarray-ref a i) x) #t (go (+ i 1)))))) + +(: (-> (TransientVec %a) %a bool) (where (Eq %a))) +(defun (transientvec-contains? t x) + (def len (transientvec-length t)) + (let go ((i 0)) + (if (= i len) + #f + (if (= (transientvec-ref t i) x) #t (go (+ i 1)))))) + +;; The elements in a new array, which the vec does not share. The runtime's, +;; which copies the leaves in one pass. +(import/extern + (vec->array (: BjolangRuntime.vecsubgtarray (-> (Vec %a) (Array %a))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/priority-queue.bjo — the persistent priority queue. +;; +;; A pairing heap. A cell holds an element, its priority, and a list of cells +;; that come out after it. +;; +;; Equal priorities come out in the order they were added: each cell has a +;; number from its queue's counter, and the number breaks a tie. +;; +;; Taking the least is O(log n) amortised, which holds when each queue is used +;; once. n adds leave a root with n children. Taking its least walks them +;; all; a queue kept and used again pays that each time. `priorityqueue-balance` +;; rebuilds a queue in O(n) as a tournament, so that the root has about log2(n) +;; children. A queue made from many entries at once is built that way. +;; +;; A cell's children are a chain of `child` and `sibling` fields, so a link +;; changes two fields. A transient is the same tree with an owner: a cell made +;; by the transient carries its owner and is changed in place, and any other +;; cell a link reaches is copied once. A persistent operation has an owner of +;; its own while it runs, so it copies a cell at most once too. Freezing hands the tree to +;; a persistent queue and gives the transient a new owner, so nothing the +;; queue holds is changed again. It does not balance: that would make it O(n). + +(import (std prelude)) + +(:doc #:file "priority-queue.bjodoc") + +(export + PriorityQueue TransientPriorityQueue PriorityQueueBuilder + ;; Named by functions an importer inlines. Opaque: nothing outside this + ;; module can use them. + PriorityQueueTree PriorityQueueCell PriorityQueueOwner + priorityqueue-empty priorityqueue-by + priorityqueue-length priorityqueue-empty? priorityqueue-peek + priorityqueue-add priorityqueue-add-all priorityqueue-merge + priorityqueue-pop priorityqueue-try-pop priorityqueue-remove-min + priorityqueue-balance priorityqueue-balanced? + list->priorityqueue priorityqueue->list priorityqueue->unordered-list + priorityqueue->str + transientpriorityqueue-empty transientpriorityqueue-by + priorityqueue->transientpriorityqueue transientpriorityqueue->priorityqueue + transientpriorityqueue-length transientpriorityqueue-empty? transientpriorityqueue-peek + transientpriorityqueue-add! transientpriorityqueue-add-all! + transientpriorityqueue-pop! transientpriorityqueue-try-pop! + priorityqueuebuilder-empty priorityqueuebuilder-by + priorityqueuebuilder-add! priorityqueuebuilder-length priorityqueuebuilder->priorityqueue) + +;; --------------------------------------------------------------------------- +;; The tree +;; --------------------------------------------------------------------------- + +;; What cells are owned by, compared with `eq?`. It has to have a field, but "transient" is never used +(type (: PriorityQueueOwner #:opaque (Record (: transient bool)))) + +(: (-> PriorityQueueOwner)) +(defun (new-owner) (PriorityQueueOwner (transient #t))) + +;; A cell's children are a chain: `child` is the first, and each child's +;; `sibling` the next. Linking two cells then changes two fields and +;; allocates nothing, when both are owned. `Tip` is the empty tree, and is +;; `null`; a cell is reached through its `Node`, which an operation keeps +;; when it changes the cell in place. +(type-rec + (: (PriorityQueueTree %a %p) #:opaque + (Union Tip (: Node (PriorityQueueCell %a %p)))) + (: (PriorityQueueCell %a %p) #:opaque + (Record (: item %a) + (: priority %p) + (: seq long) + (: child (PriorityQueueTree %a %p) #:mutable) + (: sibling (PriorityQueueTree %a %p) #:mutable) + (: owner PriorityQueueOwner)))) + +;; Whether `x` comes out before `y`: by priority, then by the order they were +;; added. +(: (-> (-> %p %p int) (PriorityQueueCell %a %p) (PriorityQueueCell %a %p) bool)) +(defun (before? order x y) + (def c (order (record-ref x priority) (record-ref y priority))) + (or (< c 0) (and (= c 0) (< (record-ref x seq) (record-ref y seq))))) + +(: (-> PriorityQueueOwner %a %p long (PriorityQueueTree %a %p))) +(defun (leaf owner x p seq) + (Node (PriorityQueueCell (item x) (priority p) (seq seq) (child Tip) (sibling Tip) (owner owner)))) + +;; `t`, whose cell is `c`, with a new first child and next sibling. In place +;; when `owner` owns the cell; otherwise a copy that `owner` owns. +(: (-> PriorityQueueOwner (PriorityQueueTree %a %p) (PriorityQueueCell %a %p) + (PriorityQueueTree %a %p) (PriorityQueueTree %a %p) + (PriorityQueueTree %a %p))) +(defun (relinked owner t c first next) + (if (eq? (record-ref c owner) owner) + (begin + (record-set! c (child first) (sibling next)) + t) + (Node (PriorityQueueCell (item (record-ref c item)) (priority (record-ref c priority)) + (seq (record-ref c seq)) (child first) (sibling next) + (owner owner))))) + +;; Two cells as one heap whose next sibling is `next`: the one that comes out +;; later becomes the first child of the other. +(: (-> (-> %p %p int) PriorityQueueOwner + (PriorityQueueTree %a %p) (PriorityQueueCell %a %p) + (PriorityQueueTree %a %p) (PriorityQueueCell %a %p) + (PriorityQueueTree %a %p) + (PriorityQueueTree %a %p))) +(defun (link order owner a ca b cb next) + (if (before? order cb ca) + (adopt owner b cb a ca next) + (adopt owner a ca b cb next))) + +(: (-> PriorityQueueOwner + (PriorityQueueTree %a %p) (PriorityQueueCell %a %p) + (PriorityQueueTree %a %p) (PriorityQueueCell %a %p) + (PriorityQueueTree %a %p) + (PriorityQueueTree %a %p))) +(defun (adopt owner w cw l cl next) + (relinked owner w cw + (relinked owner l cl (record-ref cl child) (record-ref cw child)) + next)) + +(: (-> (PriorityQueueTree %a %p) (PriorityQueueCell %a %p))) +(defun (cell-of t) + (match t + ((Node c) c) + (Tip (panic! "priority-queue: a cell of the empty tree")))) + +;; `t` with no next sibling. +(: (-> PriorityQueueOwner (PriorityQueueTree %a %p) (PriorityQueueTree %a %p))) +(defun (alone owner t) + (def (Node c) t :propagate) + (match (record-ref c sibling) + (Tip t) + (_ (relinked owner t c (record-ref c child) Tip)))) + +;; Two roots, whose siblings are Tip, as one. +(: (-> (-> %p %p int) PriorityQueueOwner (PriorityQueueTree %a %p) (PriorityQueueTree %a %p) + (PriorityQueueTree %a %p))) +(defun (merge order owner a b) + (def (Node ca) a :leave-with b) + (def (Node cb) b :leave-with a) + (link order owner a ca b cb Tip)) + +;; A chain of children as one heap, in two passes. The first links +;; neighbours and chains the results through their siblings, last first; the +;; second links that chain from its head, which is the last pair to the +;; first. A link reads the next sibling before it changes it. +(: (-> (-> %p %p int) PriorityQueueOwner (PriorityQueueTree %a %p) + (PriorityQueueTree %a %p))) +(defun (link-pairs order owner chain) + (let pair ((t chain) (done Tip)) + (def (Node ca) t :leave-with (combine order owner done)) + (def b (record-ref ca sibling)) + (match b + (Tip (combine order owner (relinked owner t ca (record-ref ca child) done))) + ((Node cb) + (def rest (record-ref cb sibling)) + (pair rest (link order owner t ca b cb done)))))) + +(: (-> (-> %p %p int) PriorityQueueOwner (PriorityQueueTree %a %p) + (PriorityQueueTree %a %p))) +(defun (combine order owner done) + (def (Node first) done :propagate) + (let go ((acc done) (cacc first) (rest (record-ref first sibling))) + (def (Node cr) rest :leave-with (alone owner acc)) + (def more (record-ref cr sibling)) + (def linked (link order owner rest cr acc cacc Tip)) + (go linked (cell-of linked) more))) + +(: (-> (-> %p %p int) PriorityQueueOwner (PriorityQueueTree %a %p) + (PriorityQueueTree %a %p))) +(defun (without-root order owner t) + (def (Node c) t :leave-with Tip) + (link-pairs order owner (record-ref c child))) + +(: (-> (PriorityQueueTree %a %p) (Option (Tuple %a %p)))) +(defun (root-entry t) + (def (Node c) t :leave-with None) + (Some (Tuple (record-ref c item) (record-ref c priority)))) + +;; One round of a tournament: neighbours linked, so the list is half as long. +(: (-> (-> %p %p int) PriorityQueueOwner (List (PriorityQueueTree %a %p)) + (List (PriorityQueueTree %a %p)))) +(defun (link-round order owner ts) + (let go ((rest ts) (out Nil)) + (match rest + (Nil out) + ((Cons t Nil) (Cons t out)) + ((Cons a (Cons b more)) + (go more (Cons (link order owner a (cell-of a) b (cell-of b) Tip) out)))))) + +;; Rounds until one tree is left. Round k makes roots with k children, so the +;; winner has about log2(n) children and the tree is about log2(n) deep. n-1 +;; links in all. +(: (-> (-> %p %p int) PriorityQueueOwner (List (PriorityQueueTree %a %p)) + (PriorityQueueTree %a %p))) +(defun (tournament order owner ts) + (match ts + (Nil Tip) + ((Cons t Nil) t) + (_ (tournament order owner (link-round order owner ts))))) + +;; The entries as a balanced tree, numbered from `first` in the order given. +(: (-> (-> %p %p int) PriorityQueueOwner (List (Tuple %a %p)) long + (PriorityQueueTree %a %p))) +(defun (build order owner entries first) + (def leaves + (loop (:for rest entries) + (:with seq first (+ seq 1L)) + (:let (x . p) rest) + (:acc out (listing/reverse (leaf owner x p seq))))) + (tournament order owner leaves)) + +;; Every cell, in no useful order. A worklist rather than recursion: a chain +;; of children can be as long as the queue. +(: (-> (PriorityQueueTree %a %p) (List (PriorityQueueCell %a %p)))) +(defun (tree-cells t) + (let walk ((todo (Cons t Nil)) (out Nil)) + (match todo + (Nil out) + ((Cons Tip rest) (walk rest out)) + ((Cons (Node c) rest) + (walk (Cons (record-ref c child) (Cons (record-ref c sibling) rest)) (Cons c out)))))) + +;; --------------------------------------------------------------------------- +;; The persistent queue +;; --------------------------------------------------------------------------- + +;; `next` is the number the next entry gets. `balanced` is #t when the tree +;; came out of a tournament and nothing has been done to it since. +(type (: (PriorityQueue %a %p) #:opaque + (Record (: tree (PriorityQueueTree %a %p)) + (: order (-> %p %p int)) + (: size int) + (: next long) + (: balanced bool)))) + +(: (-> (-> %p %p int) (PriorityQueue %a %p))) +(defun (priorityqueue-by order) + (PriorityQueue (tree Tip) (order order) (size 0) (next 0L) (balanced #t))) + +(: (-> (PriorityQueue %a %p)) (where (Ord %p))) +(defun (priorityqueue-empty) + (priorityqueue-by (fun (a b) (compare a b)))) + +(: (-> (PriorityQueue %a %p) int)) +(defun (priorityqueue-length q) (record-ref q size)) + +(: (-> (PriorityQueue %a %p) bool)) +(defun (priorityqueue-empty? q) (= (record-ref q size) 0)) + +(: (-> (PriorityQueue %a %p) (Option (Tuple %a %p)))) +(defun (priorityqueue-peek q) (root-entry (record-ref q tree))) + +(: (-> (PriorityQueue %a %p) bool)) +(defun (priorityqueue-balanced? q) (record-ref q balanced)) + +(: (-> (PriorityQueue %a %p) (PriorityQueueTree %a %p) int long bool (PriorityQueue %a %p))) +(defun (changed q t size next balanced) + (PriorityQueue (tree t) (order (record-ref q order)) (size size) (next next) (balanced balanced))) + +(: (-> (PriorityQueue %a %p) %a %p (PriorityQueue %a %p))) +(defun (priorityqueue-add q x p) + (def next (record-ref q next)) + (changed q + (let ((owner (new-owner))) + (merge (record-ref q order) owner (record-ref q tree) (leaf owner x p next))) + (+ (record-ref q size) 1) (+ next 1L) #f)) + +;; The entries are built into a balanced tree of their own, with an owner of +;; their own so that the build links in place, and that tree is linked in. +(: (-> (PriorityQueue %a %p) (List (Tuple %a %p)) (PriorityQueue %a %p))) +(defun (priorityqueue-add-all q entries) + (def order (record-ref q order)) + (def next (record-ref q next)) + (def n (list-length entries)) + (changed q + (let ((owner (new-owner))) + (merge order owner (record-ref q tree) (build order owner entries next))) + (+ (record-ref q size) n) (+ next (cast long n)) + (or (= n 0) (and (priorityqueue-empty? q) (record-ref q balanced))))) + +(: (-> (PriorityQueue %a %p) (PriorityQueue %a %p) (PriorityQueue %a %p))) +(defun (priorityqueue-merge a b) + (cond + ((priorityqueue-empty? b) a) + ((priorityqueue-empty? a) (changed a (record-ref b tree) (record-ref b size) + (record-ref b next) (record-ref b balanced))) + (else + (def na (record-ref a next)) + (def nb (record-ref b next)) + (changed a + (merge (record-ref a order) (new-owner) (record-ref a tree) (record-ref b tree)) + (+ (record-ref a size) (record-ref b size)) (if (> na nb) na nb) #f)))) + +(: (-> (PriorityQueue %a %p) (PriorityQueue %a %p))) +(defun (priorityqueue-remove-min q) + (match (record-ref q tree) + (Tip q) + (t (changed q (without-root (record-ref q order) (new-owner) t) + (- (record-ref q size) 1) (record-ref q next) #f)))) + +(: (-> (PriorityQueue %a %p) (Option (Tuple %a %p (PriorityQueue %a %p))))) +(defun (priorityqueue-try-pop q) + (match (record-ref q tree) + (Tip None) + ((Node c) + (Some (Tuple (record-ref c item) (record-ref c priority) (priorityqueue-remove-min q)))))) + +(import/class + (EmptyQueue (: System.InvalidOperationException (-> string EmptyQueue)))) + +;; The entry that comes out first, and the queue without it. +;; Raises InvalidOperationException when the queue is empty. +(: (-> (PriorityQueue %a %p) (Tuple %a %p (PriorityQueue %a %p)))) +(defun (priorityqueue-pop q) + (match (priorityqueue-try-pop q) + ((Some entry) entry) + (None (raise (cast Exception (EmptyQueue. "priorityqueue-pop: the queue is empty.")))))) + +;; O(n), and a field read on a queue that is balanced already. The cells are +;; made again, keeping their numbers: the old ones may be in other queues. +(: (-> (PriorityQueue %a %p) (PriorityQueue %a %p))) +(defun (priorityqueue-balance q) + (if (record-ref q balanced) + q + (let ((owner (new-owner))) + (changed q + (tournament (record-ref q order) owner + (list-map (fun (c) (leaf owner (record-ref c item) (record-ref c priority) + (record-ref c seq))) + (tree-cells (record-ref q tree)))) + (record-ref q size) (record-ref q next) #t)))) + +(: (-> (List (Tuple %a %p)) (PriorityQueue %a %p)) (where (Ord %p))) +(defun (list->priorityqueue entries) + (priorityqueue-add-all (priorityqueue-empty) entries)) + +;; In the order they come out. Drained through a transient, which copies a +;; cell when it first wins a link and then adds to the copy, rather than +;; copying a cell for every link. +(: (-> (PriorityQueue %a %p) (List (Tuple %a %p)))) +(defun (priorityqueue->list q) + (def t (priorityqueue->transientpriorityqueue q)) + (let go ((out Nil)) + (match (transientpriorityqueue-try-pop! t) + (None (list-reverse out)) + ((Some entry) (go (Cons entry out)))))) + +(: (-> (PriorityQueue %a %p) (List (Tuple %a %p)))) +(defun (priorityqueue->unordered-list q) + (list-map (fun (c) (Tuple (record-ref c item) (record-ref c priority))) + (tree-cells (record-ref q tree)))) + +;; --------------------------------------------------------------------------- +;; The transient +;; --------------------------------------------------------------------------- + +(type (: (TransientPriorityQueue %a %p) #:opaque + (Record (: tree (PriorityQueueTree %a %p) #:mutable) + (: order (-> %p %p int)) + (: size int #:mutable) + (: next long #:mutable) + (: owner PriorityQueueOwner #:mutable)))) + +(: (-> (PriorityQueue %a %p) (TransientPriorityQueue %a %p))) +(defun (priorityqueue->transientpriorityqueue q) + (TransientPriorityQueue (tree (record-ref q tree)) (order (record-ref q order)) + (size (record-ref q size)) (next (record-ref q next)) + (owner (new-owner)))) + +(: (-> (-> %p %p int) (TransientPriorityQueue %a %p))) +(defun (transientpriorityqueue-by order) + (priorityqueue->transientpriorityqueue (priorityqueue-by order))) + +(: (-> (TransientPriorityQueue %a %p)) (where (Ord %p))) +(defun (transientpriorityqueue-empty) + (priorityqueue->transientpriorityqueue (priorityqueue-empty))) + +;; O(1). The transient goes on with a new owner, so its next change copies +;; what it reaches rather than changing the queue handed out. +(: (-> (TransientPriorityQueue %a %p) (PriorityQueue %a %p))) +(defun (transientpriorityqueue->priorityqueue t) + (record-set! t (owner (new-owner))) + (PriorityQueue (tree (record-ref t tree)) (order (record-ref t order)) + (size (record-ref t size)) (next (record-ref t next)) (balanced #f))) + +(: (-> (TransientPriorityQueue %a %p) int)) +(defun (transientpriorityqueue-length t) (record-ref t size)) + +(: (-> (TransientPriorityQueue %a %p) bool)) +(defun (transientpriorityqueue-empty? t) (= (record-ref t size) 0)) + +(: (-> (TransientPriorityQueue %a %p) (Option (Tuple %a %p)))) +(defun (transientpriorityqueue-peek t) (root-entry (record-ref t tree))) + +(: (-> (TransientPriorityQueue %a %p) %a %p void)) +(defun (transientpriorityqueue-add! t x p) + (def owner (record-ref t owner)) + (def next (record-ref t next)) + (record-set! t + (tree (merge (record-ref t order) owner (record-ref t tree) (leaf owner x p next))) + (size (+ (record-ref t size) 1)) + (next (+ next 1L)))) + +(: (-> (TransientPriorityQueue %a %p) (List (Tuple %a %p)) void)) +(defun (transientpriorityqueue-add-all! t entries) + (def order (record-ref t order)) + (def owner (record-ref t owner)) + (def next (record-ref t next)) + (def n (list-length entries)) + (record-set! t + (tree (merge order owner (record-ref t tree) (build order owner entries next))) + (size (+ (record-ref t size) n)) + (next (+ next (cast long n))))) + +;; `None` leaves the transient as it was. +(: (-> (TransientPriorityQueue %a %p) (Option (Tuple %a %p)))) +(defun (transientpriorityqueue-try-pop! t) + (match (record-ref t tree) + (Tip None) + ((Node c) + (record-set! t + (tree (link-pairs (record-ref t order) (record-ref t owner) (record-ref c child))) + (size (- (record-ref t size) 1))) + (Some (Tuple (record-ref c item) (record-ref c priority)))))) + +;; The entry that comes out first, removed in place. +;; Raises InvalidOperationException when the transient is empty. +(: (-> (TransientPriorityQueue %a %p) (Tuple %a %p))) +(defun (transientpriorityqueue-pop! t) + (match (transientpriorityqueue-try-pop! t) + ((Some entry) entry) + (None (raise (cast Exception (EmptyQueue. "transientpriorityqueue-pop!: the queue is empty.")))))) + +;; --------------------------------------------------------------------------- +;; The builder +;; --------------------------------------------------------------------------- + +;; The entries, last first, and how many. +(type (: (PriorityQueueBuilder %a %p) #:opaque + (Record (: entries (List (Tuple %a %p)) #:mutable) + (: count int #:mutable) + (: order (-> %p %p int))))) + +(: (-> (-> %p %p int) (PriorityQueueBuilder %a %p))) +(defun (priorityqueuebuilder-by order) + (PriorityQueueBuilder (entries Nil) (count 0) (order order))) + +(: (-> (PriorityQueueBuilder %a %p)) (where (Ord %p))) +(defun (priorityqueuebuilder-empty) + (priorityqueuebuilder-by (fun (a b) (compare a b)))) + +(: (-> (PriorityQueueBuilder %a %p) %a %p void)) +(defun (priorityqueuebuilder-add! b x p) + (record-set! b + (entries (Cons (Tuple x p) (record-ref b entries))) + (count (+ (record-ref b count) 1)))) + +(: (-> (PriorityQueueBuilder %a %p) int)) +(defun (priorityqueuebuilder-length b) (record-ref b count)) + +;; O(n), and balanced. The builder keeps its entries, and may be added to and +;; built again. +(: (-> (PriorityQueueBuilder %a %p) (PriorityQueue %a %p))) +(defun (priorityqueuebuilder->priorityqueue b) + (priorityqueue-add-all (priorityqueue-by (record-ref b order)) + (list-reverse (record-ref b entries)))) + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- + +(impl (Collection (PriorityQueue %a %p)) + (type %content (Tuple %a %p)) + (defun (length q) (priorityqueue-length q)) + (defun (empty? q) (priorityqueue-empty? q))) + +;; In the order the entries come out. The cursor is the rest of the queue, so +;; a walk leaves the queue itself alone. Each cursor is used once, so the +;; amortised bound holds. +(impl (Iterable (PriorityQueue %a %p)) + (type %elem (Tuple %a %p)) + (type %cursor (PriorityQueue %a %p)) + (defun (iterable-start q) q) + (defun (iterable-done? q c) (priorityqueue-empty? c)) + (defun (iterable-current q c) + (match (priorityqueue-peek c) + ((Some entry) entry) + (None (panic! "priority-queue: iterable-current past the end")))) + (defun (iterable-next q c) (priorityqueue-remove-min c))) + +;; Equal when the same entries come out in the same order. The shape of the +;; tree, and the numbers that break ties, are not compared. +(impl (Eq (PriorityQueue %a %p)) + (where (Eq %a) (Eq %p)) + (defun (= a b) + (and (= (priorityqueue-length a) (priorityqueue-length b)) + (= (priorityqueue->list a) (priorityqueue->list b)))) + (defun (eq-hash q) + (list-foldl (fun (entry acc) (hash-combine acc (eq-hash entry))) 17 (priorityqueue->list q)))) + +;; `#priorityqueue(element priority ...)`, in the order they come out. Each +;; element and priority is added by the function given for it. +(: (-> (PriorityQueue %a %p) StringBuilder (-> %a StringBuilder void) (-> %p StringBuilder void) void)) +(defun (priorityqueue-print-into q b each-elem each-priority) + (stringbuilder-add-string! b "#priorityqueue(") + (loop (:with first? #t #f) + (:for entry q) + (:do (def (x p) entry) + (unless first? (stringbuilder-add-string! b " ")) + (each-elem x b) + (stringbuilder-add-string! b " ") + (each-priority p b))) + (stringbuilder-add-string! b ")")) + +(impl (Written (PriorityQueue %a %p)) + (where (Written %a) (Written %p)) + (defun (write-into q b) + (priorityqueue-print-into q b (fun (x b) (write-into x b)) (fun (p b) (write-into p b))))) + +(impl (Displayed (PriorityQueue %a %p)) + (where (Displayed %a) (Displayed %p)) + (defun (display-into q b) + (priorityqueue-print-into q b (fun (x b) (display-into x b)) (fun (p b) (display-into p b))))) + +(: (-> (PriorityQueue %a %p) string) (where (Displayed %a) (Displayed %p))) +(defun (priorityqueue->str q) (displayed q)) + +(:deprecated priorityqueue->str "use `displayed`") +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/random.bjo — random numbers. +;; +;; The default source is ChaCha20 seeded with 256 bits from the OS CSPRNG, one +;; generator per thread, drawn from without a lock. Any user-supplied rng uses +;; a lock. Every function here takes `#:rng`, defaulting to +;; `(parameter-ref current-random)`, so a program that wants a different source +;; binds the parameter and changes no call site. +;; +;; By default, a thread get's it's own generator, so that reading from the +;; RandomSource is unconstrained and not risky. +;; +;; **`current-random` holds no generator.** Its value is the *ambient* source, +;; which resolves to the calling thread's generator at each draw. That is what +;; makes the default safe on a bjoroutine: a fiber may resume on a different +;; thread than it suspended on, so a source read out of the parameter before a +;; suspension would otherwise go on drawing from the old thread's generator +;; while another fiber ran on that thread and drew from it too. +;; +;; A source from `make-random` and its neighbours does name a generator, and two +;; fibers may hold it at once, so every operation on one takes its lock. The +;; lock is per *operation*: a shuffle is a contiguous slice of the stream rather +;; than one interleaved with another fiber's. +;; +;; **Locked is not reproducible.** Sharing a seeded source between fibers is +;; safe, but which fiber gets which value of the stream is the scheduler's +;; business. A run that has to replay wants `random-split` per fiber. +;; +;; **Seeded means reproducible, not unguessable.** `make-random` takes 256 bits +;; from the OS. `make-random/seed64` and `make-random/seed256` take what they +;; are given, and are for replaying a run. If you seed make-random with 256 random +;; bits yourself, it is safe. + +(export + RandomSource + current-random + ;; Making one + make-random make-random/seed64 make-random/seed256 random-split + ;; Drawing + random-int random-double random-u32 random-u64 + random-bool random-chance? + random-bytes! shuffle-array! shuffle-vec random-ref + random-string random-string/chars + ;; Binding one + with-random-seed) + +(import (std prelude)) +(import (std syntax-match)) + +;; Every export is documented there; the module doc is too. +(:doc #:file "random.bjodoc") + +(import/class + (RandomSource (: Bjolang.Runtime.RandomSource)) + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +(import/extern + (ambient-random (: Bjolang.Runtime.RandomModule.Ambient (-> RandomSource))) + (make-random (: Bjolang.Runtime.RandomModule.Fresh (-> RandomSource))) + (make-random/seed64 (: Bjolang.Runtime.RandomModule.FromSeed64 (-> ulong RandomSource))) + (make-random/seed256 + (: Bjolang.Runtime.RandomModule.FromSeed256 (-> ulong ulong ulong ulong RandomSource))) + (random-split (: Bjolang.Runtime.RandomModule.Split (-> RandomSource RandomSource))) + + (clr-random-range (: Bjolang.Runtime.RandomModule.Range (-> RandomSource int int int))) + (clr-random-double (: Bjolang.Runtime.RandomModule.Double (-> RandomSource double))) + (clr-random-u32 (: Bjolang.Runtime.RandomModule.U32 (-> RandomSource uint))) + (clr-random-u64 (: Bjolang.Runtime.RandomModule.U64 (-> RandomSource ulong))) + (clr-random-bool (: Bjolang.Runtime.RandomModule.Bool (-> RandomSource bool))) + (clr-random-chance (: Bjolang.Runtime.RandomModule.Chance (-> RandomSource double bool))) + (clr-random-fill (: Bjolang.Runtime.RandomModule.Fill (-> RandomSource (Array byte) void))) + (clr-random-shuffle (: Bjolang.Runtime.RandomModule.Shuffle (-> RandomSource (Array %a) void))) + (clr-random-string (: Bjolang.Runtime.RandomModule.Str (-> RandomSource int string)))) + +;; `make-parameter`. Takes a hot parameter. +;; The default ambient-random is null, meaning every read reads the threadlocal. +;; Parameterizing current-random to (ambient-random) is thus always safe. +(: (Param RandomSource)) +(def current-random (make-parameter (ambient-random))) + +;; --------------------------------------------------------------------------- +;; Drawing +;; --------------------------------------------------------------------------- + +;; Inclusive at both ends. Lemire's method, so no modulo bias. +(: (-> int int (#:rng RandomSource) int)) +(defun (random-int lo hi #:rng (parameter-ref current-random)) + (clr-random-range rng lo hi)) + +;; [0, 1), with 53 bits of randomness. +(: (-> (#:rng RandomSource) double)) +(defun (random-double #:rng (parameter-ref current-random)) + (clr-random-double rng)) + +(: (-> (#:rng RandomSource) uint)) +(defun (random-u32 #:rng (parameter-ref current-random)) + (clr-random-u32 rng)) + +(: (-> (#:rng RandomSource) ulong)) +(defun (random-u64 #:rng (parameter-ref current-random)) + (clr-random-u64 rng)) + +;; One bit per call, taken 64 at a time. +(: (-> (#:rng RandomSource) bool)) +(defun (random-bool #:rng (parameter-ref current-random)) + (clr-random-bool rng)) + +;; True with probability `p`. Outside [0, 1] it is the nearer of the two, and +;; costs no draw. +(: (-> double (#:rng RandomSource) bool)) +(defun (random-chance? p #:rng (parameter-ref current-random)) + (clr-random-chance rng p)) + +;; Fills the array in place. Whole blocks go straight into it, so this is the +;; way to ask for many bytes at once. +(: (-> (Array byte) (#:rng RandomSource) void)) +(defun (random-bytes! bytes #:rng (parameter-ref current-random)) + (clr-random-fill rng bytes)) + +;; Fisher-Yates, in place. Answers nothing: the array it was given is the +;; shuffled one. +(: (-> (Array %a) (#:rng RandomSource) void)) +(defun (shuffle-array! a #:rng (parameter-ref current-random)) + (clr-random-shuffle rng a)) + +;; A `Vec` is persistent, so this answers a new one and leaves `v` alone. +;; +;; Walks the vec, and places it's elements into an array of the +;; same size randomly. +(: (-> (Vec %a) (#:rng RandomSource) (Vec %a))) +(defun (shuffle-vec v #:rng (parameter-ref current-random)) + (cond + ((< (vec-length v) 2) v) + (else + (def a (make-array (vec-length v))) + (loop (:with i 0 (+ i 1)) + (:for x v) + (:do (array-set! a i x))) + (clr-random-shuffle rng a) + (array->vec a)))) + +;; A string over `[A-Za-z0-9-_]`. The shim writes the UTF-8 bytes directly. +(: (-> int (#:rng RandomSource) string)) +(defun (random-string length #:rng (parameter-ref current-random)) + (clr-random-string rng length)) + +;; --------------------------------------------------------------------------- +;; Over a collection +;; --------------------------------------------------------------------------- +;; +;; Written here rather than in the shim: each is an index and a draw, and +;; Bjolang's `char` is a `BjoChar`, which does not cross as a `char[]`. + +;; An element of `v`. Randomly picked. +;; This could be a trait function, but currently we cannot constrain +;; Refable to integer keys. +(: (-> (Vec %a) (#:rng RandomSource) %a)) +(defun (random-ref v #:rng (parameter-ref current-random)) + (when (= (vec-length v) 0) + (raise (cast Exception (ArgumentException. "random-ref was given an empty vec.")))) + (vec-ref v (clr-random-range rng 0 (- (vec-length v) 1)))) + +;; A string over an alphabet of the caller's own. +(: (-> int (Vec char) (#:rng RandomSource) string)) +(defun (random-string/chars length alphabet #:rng (parameter-ref current-random)) + (when (= (vec-length alphabet) 0) + (raise (cast Exception (ArgumentException. "random-string/chars was given an empty alphabet.")))) + ;; The loop below counts up to `length`, so a negative one never ends. + (when (< length 0) + (raise (cast Exception (ArgumentException. "random-string/chars was given a negative length.")))) + (def sb (stringbuilder-empty)) + (def top (- (vec-length alphabet) 1)) + (let go ((i 0)) + (cond + ((= i length) (stringbuilder->string sb)) + (else + (stringbuilder-add! sb (vec-ref alphabet (clr-random-range rng 0 top))) + (go (+ i 1)))))) + +;; --------------------------------------------------------------------------- +;; Binding a source +;; --------------------------------------------------------------------------- + +;; `(with-random-seed 42UL body ...)` — the extent draws from a generator that +;; replays. +;; +;; One generator for the whole extent. Spawning a fiber here is apeshit bad +;; since fibers should NOT share state. It is at least safe — the source is +;; locked — but the order the fibers draw in is the scheduler's, so the run +;; stops replaying, which is the only reason to have seeded it. A fiber wants +;; `(random-split ...)` or a seed of its own. +(def/macro (with-random-seed form inject compare) + (syntax-match form + ((_ seed first body ...) + #'(parameterize ((current-random (make-random/seed64 ,seed))) + ,first + ,@body)) + (bad (syntax-error bad "with-random-seed takes a seed and a body: (with-random-seed 42UL body ...)")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/rational.bjo — exact fractions over bigint: the built-in `rational`, +;; with functions by name. + +(import (std prelude) + (std maths)) + +(:doc #:file "rational.bjodoc") + +(export + Rational + ;; Construction and parts + rational int->rational bigint->rational + rational-num rational-den + ;; Arithmetic + rational+ rational- rational* rational/ + rational-negate rational-recip rational-abs rational-expt + ;; Predicates + rational-zero? rational-integer? rational-sign + ;; Rounding to a bigint + rational-floor rational-ceiling rational-truncate rational-round + ;; Conversions + rational->double rational->decimal + rational->string string->rational) + +(import/class + (Exception (: System.Exception)) + (DivideByZeroException (: System.DivideByZeroException (-> string DivideByZeroException))) + (OverflowException (: System.OverflowException (-> string OverflowException))) + (BjoRational (: Bjolang.Runtime.BjoRational (-> bigint bigint BjoRational)))) + +(import/extern + (clr-bit-length (: System.Numerics.BigInteger.GetBitLength (-> bigint long))) + (clr-scale-b (: System.Math.ScaleB (-> double int double))) + (clr-numerator (: Bjolang.Runtime.BjoRational.NumeratorOf (-> rational bigint))) + (clr-denominator (: Bjolang.Runtime.BjoRational.DenominatorOf (-> rational bigint)))) + +;; The built-in: den > 0, gcd(num, den) = 1, zero is 0/1. Eq, Ord, Written and +;; Displayed are the prelude's. +(type (: Rational rational)) + +(: (-> string %a)) +(defun (divide-by-zero what) + (raise (cast Exception (DivideByZeroException. (str "(std rational): " what))))) + +;; --------------------------------------------------------------------------- +;; Construction +;; --------------------------------------------------------------------------- + +;; Normalizes sign and common factors. +(: (-> bigint bigint Rational)) +(defun (rational n d) + (when (zero? d) + (divide-by-zero "the denominator is zero.")) + (BjoRational. n d)) + +(: (-> bigint Rational)) +(defun (bigint->rational n) (BjoRational. n 1I)) + +(: (-> int Rational)) +(defun (int->rational n) (bigint->rational (cast System.Numerics.BigInteger n))) + +(: (-> Rational bigint)) +(defun (rational-num x) (clr-numerator x)) + +(: (-> Rational bigint)) +(defun (rational-den x) (clr-denominator x)) + +;; --------------------------------------------------------------------------- +;; Arithmetic +;; --------------------------------------------------------------------------- + +(: (-> Rational Rational Rational)) +(defun (rational+ a b) (+ a b)) + +(: (-> Rational Rational Rational)) +(defun (rational- a b) (- a b)) + +(: (-> Rational Rational Rational)) +(defun (rational* a b) (* a b)) + +;; Raises DivideByZeroException when b is zero. +(: (-> Rational Rational Rational)) +(defun (rational/ a b) (rational* a (rational-recip b))) + +(: (-> Rational Rational)) +(defun (rational-negate x) (- x)) + +;; Raises DivideByZeroException for zero. +(: (-> Rational Rational)) +(defun (rational-recip x) + (when (rational-zero? x) + (divide-by-zero "the reciprocal of zero.")) + (rational (rational-den x) (rational-num x))) + +(: (-> Rational Rational)) +(defun (rational-abs x) (abs x)) + +;; Any int exponent. A negative one takes the reciprocal; zero to a negative +;; power raises DivideByZeroException. +(: (-> Rational int Rational)) +(defun (rational-expt x n) + (if (< n 0) + (rational-recip (rational-expt x (- 0 n))) + (rational (expt (rational-num x) n) (expt (rational-den x) n)))) + +;; --------------------------------------------------------------------------- +;; Predicates +;; --------------------------------------------------------------------------- + +(: (-> Rational bool)) +(defun (rational-zero? x) (zero? x)) + +(: (-> Rational bool)) +(defun (rational-integer? x) (= (rational-den x) 1I)) + +;; -1, 0 or 1. +(: (-> Rational int)) +(defun (rational-sign x) (sign x)) + +;; --------------------------------------------------------------------------- +;; Rounding to a bigint +;; --------------------------------------------------------------------------- + +(: (-> Rational bigint)) +(defun (rational-floor x) + (def n (rational-num x)) + (def d (rational-den x)) + (def q (/ n d)) + (if (< (% n d) 0) (- q 1I) q)) + +(: (-> Rational bigint)) +(defun (rational-ceiling x) + (- 0I (rational-floor (rational-negate x)))) + +(: (-> Rational bigint)) +(defun (rational-truncate x) + (/ (rational-num x) (rational-den x))) + +;; Ties to even. +(: (-> Rational bigint)) +(defun (rational-round x) + (def f (rational-floor x)) + (def twice-frac (* 2I (- (rational-num x) (* f (rational-den x))))) + (def c (compare twice-frac (rational-den x))) + (if (or (> c 0) (and (= c 0) (odd? f))) (+ f 1I) f)) + +;; --------------------------------------------------------------------------- +;; Conversions +;; --------------------------------------------------------------------------- + +(: (-> bigint int)) +(defun (bit-length n) (cast System.Int32 (clr-bit-length n))) + +;; q/d rounded to an integer, ties to even, given q = floor(x) and r its remainder. +(: (-> bigint bigint bigint bigint)) +(defun (round-quotient q r d) + (def c (compare (* 2I r) d)) + (if (or (> c 0) (and (= c 0) (odd? q))) (+ q 1I) q)) + +;; Nearest double, ties to even, subnormals included. Rounds once. +;; Too large for a double: infinity. +(: (-> Rational double)) +(defun (rational->double x) + (def n (abs (rational-num x))) + (def d (rational-den x)) + (def mag + (if (zero? n) + 0.0 + (let* ((e (- (bit-length n) (bit-length d))) ; n/d in [2^(e-1), 2^(e+1)) + (k (max (- e 54) -1074)) ; exponent of q's last bit + (num (if (< k 0) (shift-left n (- 0 k)) n)) + (den (if (< k 0) d (shift-left d k))) + (q (/ num den)) + (r (% num den)) + (extra (max 0 (- (bit-length q) 53)))) + (if (= extra 0) + (clr-scale-b (cast System.Double (round-quotient q r den)) k) + (let* ((unit (shift-left 1I extra)) + (low (% q unit)) + (high (shift-right q extra)) + (half (shift-right unit 1)) + (c (compare low half)) + (up (or (> c 0) + (and (= c 0) (not (zero? r))) + (and (= c 0) (zero? r) (odd? high))))) + (clr-scale-b (cast System.Double (if up (+ high 1I) high)) (+ k extra))))))) + (if (< (rational-num x) 0) (- 0.0 mag) mag)) + +(def decimal-limit (shift-left 1I 96)) + +;; The digits of q with a point before the last s of them. +(: (-> bigint int string)) +(defun (point-digits q s) + (def digits (string-pad-left (bigint->string q) (+ s 1) #:with #\0)) + (if (= s 0) + digits + (str (string-drop-right digits s) "." (string-take-right digits s)))) + +;; Nearest decimal, ties to even, at the largest scale that fits, with +;; trailing zeros removed. Rounds once. +;; Raises OverflowException when the integer part does not fit. +(: (-> Rational decimal)) +(defun (rational->decimal x) + (def n (abs (rational-num x))) + (def d (rational-den x)) + (def (q s) + (let down ((s 28)) + (def scaled (* n (expt 10I s))) + (def q (round-quotient (/ scaled d) (% scaled d) d)) + (cond + ((< q decimal-limit) (Tuple q s)) + ((= s 0) (raise (cast Exception (OverflowException. + (str "(std rational): " (rational->string x) + " is too large for a decimal."))))) + (else (down (- s 1)))))) + (def (q* s*) + (let strip ((q q) (s s)) + (if (and (> s 0) (zero? (% q 10I))) + (strip (/ q 10I) (- s 1)) + (Tuple q s)))) + (string->decimal (str (if (< (rational-num x) 0) "-" "") (point-digits q* s*)))) + +;; "3", "-1/3". +(: (-> Rational string)) +(defun (rational->string x) + (if (rational-integer? x) + (bigint->string (rational-num x)) + (str (bigint->string (rational-num x)) "/" (bigint->string (rational-den x))))) + +;; --------------------------------------------------------------------------- +;; Parsing +;; --------------------------------------------------------------------------- + +(: (-> char bool)) +(defun (digit? c) (and (char>=? c #\0) (char<=? c #\9))) + +(: (-> char bigint)) +(defun (digit-value c) (cast System.Numerics.BigInteger (- (char->int c) 48))) + +;; Digits from the front of cs: (Tuple value count rest). +(: (-> (List char) (Tuple bigint int (List char)))) +(defun (read-digits cs) + (let go ((cs cs) (v 0I) (k 0)) + (match cs + ((Cons c rest) #:when (digit? c) (go rest (+ (* v 10I) (digit-value c)) (+ k 1))) + (_ (Tuple v k cs))))) + +;; An optional sign: (Tuple negative? rest). +(: (-> (List char) (Tuple bool (List char)))) +(defun (read-sign cs) + (match cs + ((Cons c rest) #:when (char=? c #\-) (Tuple #t rest)) + ((Cons c rest) #:when (char=? c #\+) (Tuple #f rest)) + (_ (Tuple #f cs)))) + +;; sign? digits ("." digits)? (("e" | "E") sign? digits)?, at least one +;; mantissa digit, and nothing after. +(: (-> (List char) (Option Rational))) +(defun (read-decimal cs) + (def (neg? cs1) (read-sign cs)) + (def (int-part int-count cs2) (read-digits cs1)) + (def (frac-part frac-count cs3) + (match cs2 + ((Cons c rest) #:when (char=? c #\.) (read-digits rest)) + (_ (Tuple 0I 0 cs2)))) + (def exponent + (match cs3 + (Nil (Some 0)) + ((Cons c rest) #:when (or (char=? c #\e) (char=? c #\E)) + (def (eneg? cs4) (read-sign rest)) + (def (ev ecount cs5) (read-digits cs4)) + (if (and (> ecount 0) (list-empty? cs5) (< ev 100000I)) + (Some (* (if eneg? -1 1) (cast System.Int32 ev))) + None)) + (_ None))) + (match exponent + ((Some ex) #:when (> (+ int-count frac-count) 0) + (def mantissa (rational (+ (* int-part (expt 10I frac-count)) frac-part) + (expt 10I frac-count))) + (def scaled (rational* mantissa (rational-expt (bigint->rational 10I) ex))) + (Some (if neg? (rational-negate scaled) scaled))) + (_ None))) + +;; "3", "-0.3048", "1e-3", "2.5E+10", "1|60", "1/3". The two sides of | or / +;; are decimals; no space anywhere. None for anything else, and for a zero +;; denominator. +(: (-> string (Option Rational))) +(defun (string->rational s) + (def (a b) + (match (string-split-once s "|") + ((Some (Tuple x y)) (Tuple x (Some y))) + (None (match (string-split-once s "/") + ((Some (Tuple x y)) (Tuple x (Some y))) + (None (Tuple s None)))))) + (match (Tuple (read-decimal (string->list a)) b) + ((Tuple (Some x) None) (Some x)) + ((Tuple (Some x) (Some text)) + (match (read-decimal (string->list text)) + ((Some y) #:when (not (rational-zero? y)) (Some (rational/ x y))) + (_ None))) + (_ None))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/run.bjo — run external programs. +(import (std prelude)) +(import (std syntax-match)) +(import (std stream)) + +;; Every export is documented there; the module doc is too. +(:doc #:file "run.bjodoc") + +(export + ;; The types. `Form` is a grammar. Its export also exports `Word`, and a + ;; literal in another module is read by the grammar there. + Word Form Cmd Filter Proc + + ;; Writing a form + cmd + + ;; Running one, blocking + run run/string run/strings run/status run/output + + ;; A running form + proc-input proc-output wait wait/all kill + + ;; ...and from inside a bjoroutine + Reader exited running run/channel by-lines run/stream + + ;; Scoped + with-run) + + +;; =========================================================================== +;; Interop +;; =========================================================================== + +(import/class + (Process (: System.Diagnostics.Process)) + (ProcessStartInfo (: System.Diagnostics.ProcessStartInfo (-> ProcessStartInfo))) + ;; Stream types for Process. + (ClrStream (: System.IO.Stream)) + (StreamReader (: System.IO.StreamReader)) + ;; An input port over a stream, registered on no scope: a child's pipes are + ;; `run`'s to close, not the ambient scope's. + (PipePort (: Bjolang.Runtime.BjoInputPort (-> ClrStream PipePort))) + (StreamWriter (: System.IO.StreamWriter)) + ;; An output port over a stream, registered on no scope, for the same + ;; reason: a child's stdin and a redirect's file are `run`'s to close. + (PortOver (: Bjolang.Runtime.BjoOutputPort (-> ClrStream bool PortOver))) + (BufferMode (: Bjolang.Runtime.BufferMode)) + ;; An output port joined to an input port. See `BjolangRuntime/BjoProcess.cs`. + (BjoPipe (: BjoPipe)) + (InvalidOperationException + (: System.InvalidOperationException (-> string InvalidOperationException)))) + +(import/extern + (settings-program! (: System.Diagnostics.ProcessStartInfo.FileName + (-> ProcessStartInfo string void) #:set)) + (settings-redirect-input! (: System.Diagnostics.ProcessStartInfo.RedirectStandardInput + (-> ProcessStartInfo bool void) #:set)) + (settings-redirect-output! (: System.Diagnostics.ProcessStartInfo.RedirectStandardOutput + (-> ProcessStartInfo bool void) #:set)) + (settings-redirect-errors! (: System.Diagnostics.ProcessStartInfo.RedirectStandardError + (-> ProcessStartInfo bool void) #:set)) + (settings-use-shell! (: System.Diagnostics.ProcessStartInfo.UseShellExecute + (-> ProcessStartInfo bool void) #:set)) + ;; Add process arguments. + (settings-argument (: BjoProc.AddArgument + (-> ProcessStartInfo string ProcessStartInfo))) + + ;; Starts process. + (process-start (: System.Diagnostics.Process.Start + (-> ProcessStartInfo Process) + #:exceptions (System.ComponentModel.Win32Exception + System.InvalidOperationException))) + ;; Awaits process exit. + (process-await (: System.Diagnostics.Process.WaitForExitAsync (-> Process void) #:async)) + (process-code (: System.Diagnostics.Process.ExitCode (-> Process int) #:get)) + (process-stdin (: System.Diagnostics.Process.StandardInput (-> Process StreamWriter) #:get)) + ;; The pipe under a child's input writer, written by nothing yet. + (writer-bytes (: System.IO.StreamWriter.BaseStream (-> StreamWriter ClrStream) #:get)) + (port-mode! (: Bjolang.Runtime.BjoOutputPort.Mode (-> OutputPort BufferMode void) #:set)) + ;; The synchronous close. What `run` closes on a caller's behalf is a child's + ;; stdin or a pipe's writer, both unbuffered, so there is nothing left to + ;; write and no reason for the close to suspend. + (close-unbuffered (: BjolangRuntime.CloseOutput (-> OutputPort void))) + (unbuffered (: Bjolang.Runtime.BufferMode.None BufferMode #:get)) + (process-stdout (: System.Diagnostics.Process.StandardOutput (-> Process StreamReader) #:get)) + (process-stderr (: System.Diagnostics.Process.StandardError (-> Process StreamReader) #:get)) + ;; The pipe under a child's output reader, read by nothing yet. + (reader-bytes (: System.IO.StreamReader.BaseStream (-> StreamReader ClrStream) #:get)) + ;; Killing something that has already exited is the ordinary case when a + ;; later stage fails to start, so the raise is turned into a value here. + (process-kill (: System.Diagnostics.Process.Kill (-> Process void) + #:exceptions (System.InvalidOperationException + System.NotSupportedException + System.ComponentModel.Win32Exception))) + + (make-pipe (: BjoPipe.Create (-> BjoPipe))) + (pipe-input (: BjoPipe.Reader (-> BjoPipe InputPort) #:get)) + (pipe-output (: BjoPipe.Writer (-> BjoPipe OutputPort) #:get)) + + ;; Not `double->string`, which is `.ToString()` and so follows the machine's + ;; locale: on a Swedish one `(sleep 0.5)` would ask for `0,5`. + (number->argument (: BjoProc.Argument (-> double string))) + + ;; Async pumps. + (copy-stream (: BjoProc.PumpAsync (-> InputPort OutputPort void) #:async)) + (copy-into (: BjoProc.PumpIntoAsync (-> InputPort OutputPort void) #:async)) + (read-lines (: BjoProc.ReadLinesAsync (-> InputPort (Array string)) #:async)) + (read-rest (: Bjolang.Runtime.BjoInputPort.ReadToEndUtf8Async (-> InputPort string) #:async))) + +;; A child's output as an input port over the pipe's own bytes, as Racket's +;; subprocess ports are: its text reads decode UTF-8, whatever encoding .NET +;; would have guessed the child writes. +(: (-> StreamReader InputPort)) +(defun (output-port reader) (cast InputPort (PipePort. (reader-bytes reader)))) + +;; Nothing in the language builds an `Exception`, and every rejection below is +;; this module's own rather than .NET's, so they need a carrier. +(: (-> string Exception)) +(defun (run-error message) + (cast Exception (InvalidOperationException. message))) + + +;; =========================================================================== +;; What you write +;; =========================================================================== + +;; A command written out, for when a program name is computed rather than +;; spelled: `,(cmd program "-l")`. An unquoted value has no literal shape and +;; is selected by its type, and `Cmd` is the only case carrying a record. +(type (: Cmd (Record (: program string) (: arguments (Vec string))))) + +;; A filter stage. +(type (: Filter (-bjo-> InputPort OutputPort Unit))) + +;; A form is one thing to run. You write it as a quoted literal, and this +;; grammar reads it. The keywords `pipe`, `from-file` and the four after them +;; name a case. The compiler checks the keyword, the number of items and each +;; item where you write the form. A list without one of these keywords is a +;; command, so `'(sort)` and `'(echo "x")` are commands. A keyword has +;; priority over a program name. To run a program with the name `pipe`, write +;; `"pipe"`. +(def/grammar Form + (Form (:or (: FPipe (pipe Form ...)) + (: FFrom (from-file string Form)) + (: FInto (into-file string Form)) + (: FAppend (append-to-file string Form)) + (: FErrInto (errors-into-file string Form)) + (: FErrAppend (errors-append-to-file string Form)) + (: FSym symbol) ;; a bare program name + (: FStr string) ;; ...written as a string + (: FArgv (Vec string)) ;; ...or a computed argv + (: FCmdValue ,Cmd) ;; a bare ,(cmd ...) + (: FFilter ,Filter) ;; a bare ,filter + (: FCmd (Word ...+)))) ;; (echo "x") + ;; The words of a command. + (Word (:or (: WSym symbol) (: WStr string) (: WInt int) (: WDouble double) + (: WArgv (Vec string)) (: WCmd ,Cmd) (: WFilter ,Filter) + (: WForm (Word ...))))) + +(: (-> string #:rest string Cmd)) +(defun (cmd program #:rest arguments) + (Cmd (program program) + (arguments (loop (:for a arguments) (:acc out (vecing a)))))) + + +;; =========================================================================== +;; What it parses to +;; =========================================================================== + +;; Parsed AST node. +(type (: Redirection (Union RFrom RInto RAppend RErrInto RErrAppend))) + +(type-rec + (: Node (Union (: NCmd Cmd) + (: NFilter Filter) + (: NPipe (List Node)) + (: NRedirect Redirect))) + (: Redirect (Record (: how Redirection) (: path string) (: inner Node)))) + +(: (-> Word string)) +(defun (word->text w) + (match w + ((WSym s) (symbol->string s)) + ((WStr s) (str "\"" s "\"")) + ((WInt n) (int->string n)) + ((WDouble x) (number->argument x)) + ((WArgv v) "a spliced vector") + ((WCmd c) "an unquoted command") + ((WFilter f) "an unquoted filter") + ((WForm ws) "a nested form"))) + +(: (-> (Vec string) (Result Exception Node))) +(defun (command-of argv) + (if (vec-empty? argv) + (Err (run-error "a command with no program name")) + (Ok (NCmd (Cmd (program (vec-ref argv 0)) + (arguments (vec-slice argv 1 (- (vec-length argv) 1)))))))) + +;; One form, translated to a node. The grammar read each keyword where the +;; form was written. What is left is the shape of the pipeline and the words +;; of a command, which the grammar gives as a flat list. +(: (-> Form (Result Exception Node))) +(defun (form->node form) + (match form + ((FSym s) (command-of [(symbol->string s)])) + ((FStr s) (command-of [s])) + ((FArgv v) (command-of v)) + ((FCmdValue c) (Ok (NCmd c))) + ((FFilter f) (Ok (NFilter f))) + ((FCmd words) (parse-words words)) + ((FPipe forms) (parse-stages forms)) + ((FFrom path inner) (form->redirect RFrom path inner)) + ((FInto path inner) (form->redirect RInto path inner)) + ((FAppend path inner) (form->redirect RAppend path inner)) + ((FErrInto path inner) (form->redirect RErrInto path inner)) + ((FErrAppend path inner) (form->redirect RErrAppend path inner)))) + +(: (-> (List Form) (Result Exception Node))) +(defun (parse-stages forms) + (match forms + (Nil (Err (run-error "(pipe ...) needs at least one thing to run"))) + ((Cons only Nil) (form->node only)) + (_ (match (form->each forms) + ((Err e) (Err e)) + ((Ok nodes) (Ok (NPipe nodes))))))) + +(: (-> (List Form) (Result Exception (List Node)))) +(defun (form->each forms) + (match forms + (Nil (Ok Nil)) + ((Cons form rest) + (def (Ok node) (form->node form) :propagate) + (def (Ok more) (form->each rest) :propagate) + (Ok (Cons node more))))) + +;; A redirection is its file name and the one form it wraps. +(: (-> Redirection string Form (Result Exception Node))) +(defun (form->redirect how path inner) + (match (form->node inner) + ((Err e) (Err e)) + ((Ok node) (Ok (NRedirect (Redirect (how how) (path path) (inner node))))))) + +;; The words of one command, flattened. The first is the program and the rest +;; are its arguments, each one argument whatever it contains — nothing here +;; re-splits on whitespace. +(: (-> string)) +(defun (tag-names) + "pipe, from-file, into-file, append-to-file, errors-into-file, errors-append-to-file") + +;; Adds a hint about the keywords to an error. +(: (-> (List Word) Exception Exception)) +(defun (with-tag-hint ws original) + (match ws + ((Cons (WSym s) rest) + (run-error (str (.-Message original) + " If `" (symbol->string s) "` was meant as a keyword, the keywords are: " + (tag-names) "."))) + (_ original))) + +(: (-> (List Word) (Result Exception Node))) +(defun (parse-command ws) + (match (words->argv ws (vec-empty)) + ((Err e) (Err (with-tag-hint ws e))) + ((Ok argv) (command-of argv)))) + +(: (-> (List Word) (Vec string) (Result Exception (Vec string)))) +(defun (words->argv ws acc) + (match ws + (Nil (Ok acc)) + ((Cons w rest) + (match w + ((WSym s) (words->argv rest (vec-add acc (symbol->string s)))) + ((WStr s) (words->argv rest (vec-add acc s))) + ((WInt n) (words->argv rest (vec-add acc (int->string n)))) + ((WDouble x) (words->argv rest (vec-add acc (number->argument x)))) + ((WArgv v) (words->argv rest (vec-append acc v))) + (_ (Err (run-error + (str "a command's words have to be plain — this one contains " + (word->text w) + ", which belongs in a (pipe ...) rather than among arguments.")))))))) + + + +;; A form written as a list is a command, or one thing that is a whole stage on +;; its own: `'(,my-filter)` and `'(,(cmd ...))`. The grammar read the keywords +;; before this runs, so a list's head is never one here — `'(sort)` and +;; `'(echo "x")` are commands, and `'(pipe ...)` never reaches this at all. +(: (-> (List Word) (Result Exception Node))) +(defun (parse-words ws) + (match ws + (Nil (Err (run-error "an empty form: there is nothing here to run"))) + ((Cons head rest) + (match head + ((WFilter f) (stage-alone (NFilter f) rest "a filter")) + ((WCmd c) (stage-alone (NCmd c) rest "an unquoted ,(cmd ...)")) + ((WForm ws) + (Err (run-error + "this form begins with another form. To run several things in turn, name it: (pipe (cat \"a\") (wc -l))"))) + ((WSym s) (parse-command ws)) + ((WStr s) (parse-command ws)) + ((WArgv v) (parse-command ws)) + (_ (Err (run-error + (str "a program name has to be a word, and this form begins with " + (word->text head))))))))) + +;; A form that is one thing already carries everything it needs, so a word +;; after it is a pipeline nobody named. +(: (-> Node (List Word) string (Result Exception Node))) +(defun (stage-alone node rest what) + (match rest + (Nil (Ok node)) + (_ (Err (run-error + (str what + " is a whole stage on its own, so nothing may follow it here. To run several things in turn, name it: (pipe ...)")))))) + + +;; =========================================================================== +;; Running it +;; =========================================================================== + +;; Started process state. +(type + (: Proc #:opaque + (Record (: input (Option OutputPort)) + (: output (Option InputPort)) + (: children (Vec Process)) + (: exits (Vec (Promise Unit))) + (: waits (Vec (Promise Unit)))))) + +;; One node, wired but not yet connected to its neighbours. A command and a +;; filter differ only in where `sink` and `source` come from, which is what +;; makes the pipeline loop have no cases in it. +(type + (: Wired #:opaque + (Record (: sink (Option OutputPort)) + (: source (Option InputPort)) + (: children (Vec Process)) + (: exits (Vec (Promise Unit))) + (: waits (Vec (Promise Unit)))))) + +;; Waits for child exit once. +(: (-> Process Unit)) +(defbjo (awaiting-exit child) + (process-await child) + unit) + +(: (-> Cmd (Option OutputPort) (Result Exception Wired))) +(defun (start-command c errors) + (def settings (ProcessStartInfo.)) + (settings-program! settings (record-ref c program)) + (loop (:for arg (record-ref c arguments)) + (:do (ignore (settings-argument settings arg)))) + ;; `UseShellExecute` off is what makes redirection legal at all, and also + ;; what makes the program name resolve against PATH rather than against a + ;; desktop association. + (settings-use-shell! settings #f) + (settings-redirect-input! settings #t) + (settings-redirect-output! settings #t) + (when (some? errors) + (settings-redirect-errors! settings #t)) + (match (process-start settings) + ((Err e) (Err e)) + ((Ok child) + (Ok (Wired (sink (Some (child-input child))) + (source (Some (output-port (process-stdout child)))) + (children [child]) + (exits [(bjo (awaiting-exit child))]) + ;; Handle stderr copying. + (waits (match errors + ((Some w) [(bjo (pump-into (output-port (process-stderr child)) w))]) + (None (vec-empty))))))))) + +;; Two pipes, because a filter has no ends of its own: one for what is written +;; to it and one for what it writes. +(: (-> Filter Wired)) +(defun (start-filter f) + (def in (make-pipe)) + (def out (make-pipe)) + (def done (bjo (f (pipe-input in) (pipe-output out)))) + (Wired (sink (Some (pipe-output in))) + (source (Some (pipe-input out))) + (children (vec-empty)) + (exits (vec-empty)) + (waits [(bjo (close-after done (pipe-output out)))]))) + +;; Closes and raises errors. +(: (-> (Promise Unit) OutputPort Unit)) +(defbjo (close-after done out) + (match (sync (promise-join done)) + ((Ok u) (close-output-port out) unit) + ((Err e) (close-output-port out) (raise e)))) + +;; `PumpAsync` closes its target on exhaustion, which is the whole reason it is +;; a junction: an unclosed stdin leaves the process downstream waiting forever. +(: (-> InputPort OutputPort Unit)) +(defbjo (pump from to) + (copy-stream from to) + unit) + +;; The shared-destination pump. Closing is `close-when-all-done`'s job. +(: (-> InputPort OutputPort Unit)) +(defbjo (pump-into from to) + (copy-into from to) + unit) + +(: (-> (Vec (Promise Unit)) OutputPort Unit)) +(defbjo (close-when-all-done ps out) + ;; The failure is already carried by one of `ps`, which is in this form's + ;; `waits` too, so it is not re-raised here — only the file is closed. + (ignore (join-all ps)) + (close-output-port out) + unit) + +(: (-> (Vec Process) void)) +(defun (kill-all children) + (loop (:for child children) + (:do (ignore (process-kill child))))) + +;; Undoes a form that was wired and will not run, because a stage or a file +;; after it failed. Its processes are killed and its two ends closed: a filter +;; then reads to the end of its input and finishes, where it would otherwise +;; wait for input that never comes, and the scope that started it would wait +;; for it. +(: (-> Wired void)) +(defun (abandon w) + (kill-all (record-ref w children)) + (when-let (to (record-ref w sink)) (close-output-port to)) + (when-let (from (record-ref w source)) (close-input-port from))) + +;; Connect wired stages front to back, pumping each one's output into the next +;; one's input. +(: (-> (List Node) (Option OutputPort) (Result Exception Wired))) +(defun (join-stages nodes errors) + (match nodes + (Nil (Err (run-error "a pipeline with no stages"))) + ((Cons first rest) + (match (wire first errors) + ((Err e) (Err e)) + ((Ok head) (join-onto head rest errors)))))) + +(: (-> Wired (List Node) (Option OutputPort) (Result Exception Wired))) +(defun (join-onto joined nodes errors) + (match nodes + (Nil (Ok joined)) + ((Cons node more) + (match (wire node errors) + ;; Whatever is already running would otherwise sit on a stdin nobody + ;; will ever close. + ((Err e) (abandon joined) (Err e)) + ((Ok w) + (match (junction (record-ref joined source) (record-ref w sink)) + ((Err e) + (abandon joined) + (abandon w) + (Err e)) + ((Ok link) + (join-onto + (record-set joined + (source (record-ref w source)) + (children (vec-append (record-ref joined children) (record-ref w children))) + (exits (vec-append (record-ref joined exits) (record-ref w exits))) + (waits (vec-add (vec-append (record-ref joined waits) (record-ref w waits)) link))) + more + errors)))))))) + +;; Creates stream junction. +(: (-> (Option InputPort) (Option OutputPort) (Result Exception (Promise Unit)))) +(defun (junction upstream downstream) + (match upstream + (None (Err (run-error + "the stage before this one has its output redirected to a file, so there is nothing to pipe onwards"))) + ((Some from) + (match downstream + (None (Err (run-error + "this stage has its input redirected from a file, so nothing can be piped into it"))) + ((Some to) (Ok (bjo (pump from to)))))))) + +;; Through the prelude's `FS` effect, as every file is opened, so a handler +;; that answers for the disk answers for a redirect too. +(: (-> string bool (Result Exception OutputPort))) +(defun (open-for-writing path append) + (match (open-write-stream path (if append Append Truncate)) + ((Err e) (Err e)) + ((Ok stream) (Ok (cast OutputPort (PortOver. stream #t)))))) + +;; A child's standard input as a port: unbuffered, as .NET's writer over it +;; was, because what reads the other end is another process and there is no +;; later moment at which we would know to flush. Closing it closes the pipe, +;; which is how the child sees the end of its input. +(: (-> Process OutputPort)) +(defun (child-input child) + (def port (cast OutputPort (PortOver. (writer-bytes (process-stdin child)) #t))) + (port-mode! port unbuffered) + port) + +(: (-> Node (Option OutputPort) (Result Exception Wired))) +(defun (wire node errors) + (match node + ((NCmd c) (start-command c errors)) + ((NFilter f) (Ok (start-filter f))) + ((NPipe nodes) (join-stages nodes errors)) + ((NRedirect r) (wire-redirect r errors)))) + +(: (-> Redirect (Option OutputPort) (Result Exception Wired))) +(defun (wire-redirect r errors) + (def path (record-ref r path)) + (def inner (record-ref r inner)) + (match (record-ref r how) + (RFrom + (def (Ok file) (open-input-file path) :propagate) + (redirect-input inner errors file)) + (RInto (redirect-output inner errors path #f)) + (RAppend (redirect-output inner errors path #t)) + (RErrInto + (def (Ok file) (open-for-writing path #f) :propagate) + (redirect-errors inner file)) + (RErrAppend + (def (Ok file) (open-for-writing path #t) :propagate) + (redirect-errors inner file)))) + +;; The file replaces the caller as the thing writing in, so `sink` goes away. +;; A form that does not start leaves the file closed. +(: (-> Node (Option OutputPort) InputPort (Result Exception Wired))) +(defun (redirect-input inner errors file) + (match (wire inner errors) + ((Err e) (close-input-port file) (Err e)) + ((Ok w) + (match (record-ref w sink) + (None (abandon w) + (close-input-port file) + (Err (run-error "this form already takes its input from somewhere, so it cannot also take it from a file"))) + ((Some to) + (Ok (record-set w + (sink None) + (waits (vec-add (record-ref w waits) (bjo (pump file to))))))))))) + +;; ...and the file replaces the caller as the thing reading out, so `source` +;; goes away. The file is opened once the form has started and has an output +;; to give it, since opening it to write truncates it: a form that does not +;; start leaves the file as it was. +(: (-> Node (Option OutputPort) string bool (Result Exception Wired))) +(defun (redirect-output inner errors path append) + (def (Ok w) (wire inner errors) :propagate) + (match (record-ref w source) + (None (abandon w) + (Err (run-error "this form already sends its output somewhere, so it cannot also send it to a file"))) + ((Some from) + (match (open-for-writing path append) + ((Err e) (abandon w) (Err e)) + ((Ok file) + (Ok (record-set w + (source None) + (waits (vec-add (record-ref w waits) (bjo (pump from file))))))))))) + +;; Standard error is not a stream this module wires between stages — every +;; command under here writes into the one file — so neither end changes. +;; +;; Unlike `redirect-output`, the file is opened before the form starts: it is +;; the error sink each command is started with. A form that does not start +;; leaves it closed, but already truncated. +(: (-> Node OutputPort (Result Exception Wired))) +(defun (redirect-errors inner file) + ;; Every command under here writes to the one file from a fiber of its own; + ;; a port takes each write whole, so they can share it as it is. + (def shared file) + + (match (wire inner (Some shared)) + ((Err e) (close-output-port shared) (Err e)) + ((Ok w) + (Ok (record-set w + (waits (vec-add (record-ref w waits) + (bjo (close-when-all-done (record-ref w waits) shared))))))))) + +;; Parses and runs form. +(: (-> Form (Result Exception Proc))) +(defun (run form) + (def (Ok node) (form->node form) :propagate) + (def (Ok w) (wire node None) :propagate) + (Ok (Proc (input (record-ref w sink)) + (output (record-ref w source)) + (children (record-ref w children)) + (exits (record-ref w exits)) + (waits (record-ref w waits))))) + +;; Where to write the form's input, unless a `from-file` took it. Closing it is +;; what ends the first stage. +(: (-> Proc (Option OutputPort))) +(defun (proc-input p) (record-ref p input)) + +;; Where to read the form's output, unless an `into-file` took it. It has to be +;; drained, or the last stage blocks writing and `wait` never returns. +(: (-> Proc (Option InputPort))) +(defun (proc-output p) (record-ref p output)) + +;; --------------------------------------------------------------------------- +;; Waiting, written once and seen from two sides +;; --------------------------------------------------------------------------- +;; +;; Checks if process finished. + +(: (-> (Vec (Promise Unit)) (Result Exception Unit))) +(defbjo (join-all ps) + (let go ((i 0)) + (if (= i (vec-length ps)) + (Ok unit) + (match (sync (promise-join (vec-ref ps i))) + ((Err e) (Err e)) + ((Ok u) (go (+ i 1))))))) + +;; Awaits process completion. +(: (-> Proc (Result Exception int))) +(defbjo (waiting p) + (def (Ok _) (join-all (record-ref p exits)) :propagate) + (def (Ok _) (join-all (record-ref p waits)) :propagate) + (def children (record-ref p children)) + (Ok (if (vec-empty? children) + 0 + (process-code (vec-ref children (- (vec-length children) 1)))))) + +(: (-> Proc (Result Exception (Vec int)))) +(defbjo (waiting/all p) + (def (Ok _) (join-all (record-ref p exits)) :propagate) + (def (Ok _) (join-all (record-ref p waits)) :propagate) + (Ok (vec-map (fun (c) (process-code c)) (record-ref p children)))) + +;; Gets blocking result. +(: blocking-result #:sync (-> (Promise (Result Exception %a)) (Result Exception %a))) +(defun (blocking-result p) + (match (sync/blocking (promise-join p)) + ((Err e) (Err e)) + ((Ok answer) answer))) + +;; Waits for last command exit. +(: (-> Proc (Result Exception int))) +(defun (wait p) (blocking-result (bjo (waiting p)))) + +;; Every command's exit code, in the order they were started. Filters +;; contribute nothing — their failures arrive as the `Err` instead. +(: (-> Proc (Result Exception (Vec int)))) +(defun (wait/all p) (blocking-result (bjo (waiting/all p)))) + +;; Returns process exit event. +(: (-> Proc (Event (Result Exception int)))) +(defun (exited p) + (wrap (promise-join (bjo (waiting p))) + (fun (r) (match r ((Err e) (Err e)) ((Ok answer) answer))))) + +;; Stop everything. The pumps end on their own once the streams they are +;; copying do. +(: (-> Proc void)) +(defun (kill p) (kill-all (record-ref p children))) + +;; A background job added to a form after it has started, so that `wait` and +;; `exited` cover it too. `run/channel`'s reader is the one caller. +(: (-> Proc (Promise Unit) Proc)) +(defun (with-wait p extra) + (record-set p (waits (vec-add (record-ref p waits) extra)))) + + +;; =========================================================================== +;; The whole thing at once +;; =========================================================================== +;; +;; --- Process runners --- + +(: (-> Proc void)) +(defun (close-input p) + (match (proc-input p) + ((Some w) (close-unbuffered w)) + (None unit))) + +(: (-> Form (Result Exception string))) +(defun (run/string form) + (match (run form) + ((Err e) (Err e)) + ((Ok p) + (close-input p) + (blocking-result (bjo (collecting p)))))) + +;; Read the output, then wait. The order is what matters: a last stage blocked +;; on a full pipe never exits, so waiting first would wait forever. +(: (-> Proc (Result Exception string))) +(defbjo (collecting p) + (let ((text (match (proc-output p) + ((Some out) (read-rest out)) + (None "")))) + (match (waiting p) + ((Err e) (Err e)) + ((Ok code) (Ok text))))) + +;; The output and the exit code, together. +;; +;; `run/string` answers the output and drops the code, which is enough for a +;; command that is expected to work. It is not enough for one whose failure is +;; an ordinary answer — `git show v1.0.0:manifest.bjodat` on a tag that has no +;; manifest prints nothing and exits non-zero, and that is not the same as a +;; tag whose manifest is empty. Both halves are here so that the caller does +;; not have to run the thing twice to get them. +(: (-> Form (Result Exception (Tuple int string)))) +(defun (run/output form) + (match (run form) + ((Err e) (Err e)) + ((Ok p) + (close-input p) + (blocking-result (bjo (collecting/code p)))))) + +(: (-> Proc (Result Exception (Tuple int string)))) +(defbjo (collecting/code p) + (let ((text (match (proc-output p) + ((Some out) (read-rest out)) + (None "")))) + (match (waiting p) + ((Err e) (Err e)) + ((Ok code) (Ok (Tuple code text)))))) + +(: (-> Form (Result Exception (Vec string)))) +(defun (run/strings form) + (match (run form) + ((Err e) (Err e)) + ((Ok p) + (close-input p) + (blocking-result (bjo (collecting/lines p)))))) + +(: (-> Proc (Result Exception (Vec string)))) +(defbjo (collecting/lines p) + (let ((lines (match (proc-output p) + ((Some out) (loop (:for l (read-lines out)) (:acc v (vecing l)))) + (None (vec-empty))))) + (match (waiting p) + ((Err e) (Err e)) + ((Ok code) (Ok lines))))) + +;; Runs for status code. +(: (-> Form (Result Exception int))) +(defun (run/status form) + (match (run form) + ((Err e) (Err e)) + ((Ok p) + (close-input p) + (blocking-result (bjo (draining p)))))) + +(: (-> Proc (Result Exception int))) +(defbjo (draining p) + (match (proc-output p) + ((Some out) (ignore (read-rest out))) + (None unit)) + (waiting p)) + + +;; =========================================================================== +;; A form as a channel +;; =========================================================================== + +;; Reader streams items to channel. +(type (: (Reader %a) (-bjo-> InputPort (Chan %a) Unit))) + +;; Runs and streams to channel. +(: (-> Form (Reader %a) (Result Exception (Tuple (Chan %a) Proc)))) +(defun (run/channel form read) + (match (run form) + ((Err e) (Err e)) + ((Ok p) + (let ((items (make-chan))) + (match (proc-output p) + ;; Handles redirect output. + (None (kill p) + (Err (run-error + "this form sends its output to a file, so there is nothing left to read onto a channel"))) + ((Some out) + (Ok (Tuple items + (with-wait p (bjo (read out items))))))))))) + +;; The stock `Reader`: one item per line, `\r\n` and a missing last newline +;; handled by `read-line`'s default mode rather than by guessing. Each line is +;; sent as soon as it is read: a program whose output does not end, `tail -f`, +;; still delivers its lines, and the output is never held whole. +(: (Reader string)) +(defbjo (by-lines out items) + (let next () + (match (read-line/opt out) + ((Some line) + (sync (chan-send items line)) + (next)) + (None unit)))) + + +;; =========================================================================== +;; A form as a stream +;; =========================================================================== + +;; The items `reader` reads from a form's output, as a `Stream`. Each loop +;; over it runs the form afresh. +;; +;; - The form's input is closed at once. Each item is read when the loop asks +;; for it. +;; - The exit code is dropped, as `run/strings` drops it. +;; - Lifecycle: the form is killed however the loop ends, early included. +;; When the loop returns, its processes have exited. See `reaping`. +;; - Raises in the loop: when the form cannot start, its output goes to a +;; file, a filter or a redirection fails, or `reader` raises. +(: (-> Form (-bjo-> InputPort %a) (Stream %a))) +(defun (run/stream form reader) + (make-stream (bjoroutine (out) (stream-form form reader out)))) + +(: (-> Form (-bjo-> InputPort %a) (Emitter %a) void)) +(defbjo (stream-form form reader out) + (match (run form) + ((Err e) (raise e)) + ((Ok p) + (close-input p) + (try + (match (proc-output p) + (None (raise (run-error "this form sends its output to a file, so there is nothing left to stream"))) + ((Some port) + (let go () + (unless (port-eof? port) + (emit! out (reader port)) + (go))) + (match (waiting p) + ((Err e) (raise e)) + ((Ok _) unit)))) + #:finally (reaping p))))) + +;; The longest time `reaping` waits for a killed form's processes to exit. +(def reap-grace (seconds 5)) + +;; Awaits the exit of each of `children`. +(: (-> (Vec Process) Unit)) +(defbjo (awaiting-exits children) + (loop (:for child children) (:do (process-await child))) + unit) + +;; Kills the form, and waits until its processes have exited. +;; +;; - After this returns, the processes are dead and reaped, unless they took +;; longer than `reap-grace` to exit. +;; - The wait runs in a shield, so it also works in a fiber that has been +;; cancelled. +;; - Does not raise when the grace passes. +;; - It waits for the processes only, not for the form's pumps and filters. +;; A pump can wait for a pipe that a grandchild process still holds open. +(: (-> Proc void)) +(defbjo (reaping p) + (kill p) + ;; The shield's deadline is longer than `reap-grace`, so the timeout always + ;; commits first and the shield never raises `Cancelled`. The inner scope + ;; cancels the awaits that are still waiting when the grace passes. + (with-shield #:within (seconds 6) + (with-cancel (stop) + (def exited (bjo (awaiting-exits (record-ref p children)))) + (ignore (sync (choose (wrap (promise-join exited) (fun (_) unit)) (timeout reap-grace)))) + (ignore (stop (Requested "the grace for the form to exit has passed")))))) + + +;; =========================================================================== +;; A form as one event +;; =========================================================================== + +;; Stops on event. +(: (-> (Event Unit) Proc (Promise (Result Exception string)) Unit)) +(defbjo (reap-on-loss nack p done) + (when (sync (choose (wrap nack (fun (u) #t)) + (wrap (promise-join done) (fun (r) #f)))) + (kill p)) + unit) + +;; Wraps process in event. +(: (-> Form (Event (Result Exception string)))) +(defun (running form) + (with-nack + (fun (nack) + (match (run form) + ((Err e) (always (Err e))) + ((Ok p) + (close-input p) + (let ((done (bjo (collecting p)))) + (ignore (bjo (reap-on-loss nack p done))) + (wrap (promise-join done) + (fun (r) (match r ((Err e) (Err e)) ((Ok answer) answer)))))))))) + + +;; =========================================================================== +;; Scoped +;; =========================================================================== + +;; Runs form scoped to body. +(def/macro (with-run form inject compare) + (syntax-match form + ((_ (name spec) first body ...) + ;; `name` came from the input, so hygiene leaves it alone and the body sees + ;; it. `started` is the template's and is renamed apart. + #'(let () + (def started (run ,spec)) + (try + (let ((,name started)) + ,first + ,@body) + #:finally (match started + ((Ok p) (kill p)) + ((Err e) unit))))) + (bad (syntax-error bad "with-run names a form and takes a body: (with-run (p '(ls \"-l\")) body ...)")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/rx.bjo — regular expressions in s-expression notation. +;; +;; #rx(seq :bos (+ :digit) (lit "-") ($ (** 4 4 :digit)) :eos) +;; +;; `#rx(...)` is read, checked and compiled to a pattern at compile time; a bad +;; pattern is a compile error pointing at the sub-form that is wrong. What the +;; expansion evaluates is `(rx-compile "" "")`. +;; +;; Anything that takes no argument is a keyword — `:bos`, +;; `:any`, `:digit`. Anything that takes one is a symbol heading a form — +;; `(seq ...)`, `(+ ...)`, `(in ...)`. A string or a character literal matches +;; itself: a literal sequence in pattern position, the set of its characters +;; inside `(in ...)`. +;; +;; The engine is BjoRx (`BjolangRuntime/BjoRx`): linear time, leftmost-first, +;; over the string's UTF-8 as it is. It has no backreferences, lookarounds, +;; atomic groups or conditionals. This to not tie (std rx) +;; to a backtracking engine for good, and nothing here can emit one. +;; +;; The pattern is BjoRx's tree as text (`BjoRx.Syntax.HirText`): literals, +;; classes as scalar ranges, sequence, alternation, repetition, groups and the +;; two anchors, nothing else. Classes are written out from the tables at the +;; bottom of this file, so `:digit` means the same thing on every runtime and +;; the meaning is readable here. The alphabet is the Unicode scalar, as it is +;; for `char` and the string cursors. + +(export + ;; The types. + Rx RxMatch + + ;; Matching. + rx-match rx-match? rx-search rx-search? rx-matches + rx-replace rx-replace-with rx-split + + ;; Reading a match. + rx-text rx-input rx-start rx-end + rx-group rx-group-start rx-group-end rx-group/n rx-group-count + + ;; What `#rx(...)` expands into, and what a test reads back. + rx-compile rx-pattern) + +(import (std prelude)) +(import (std syntax-match)) + +;; Every export is documented there; the module doc is too. +(:doc #:file "rx.bjodoc") + +(import/class + (Rx (: Bjolang.Runtime.BjoRegex)) + (RxMatch (: Bjolang.Runtime.BjoMatch))) + +(import/extern + (clr-rx-compile (: Bjolang.Runtime.BjoRegexModule.Compile (-> string string Rx))) + (clr-rx-pattern (: Bjolang.Runtime.BjoRegexModule.Pattern (-> Rx string))) + (clr-rx-search (: Bjolang.Runtime.BjoRegexModule.Search (-> Rx string (Option RxMatch)))) + (clr-rx-search? (: Bjolang.Runtime.BjoRegexModule.IsSearchMatch (-> Rx string bool))) + (clr-rx-whole (: Bjolang.Runtime.BjoRegexModule.MatchWhole (-> Rx string (Option RxMatch)))) + (clr-rx-whole? (: Bjolang.Runtime.BjoRegexModule.IsWholeMatch (-> Rx string bool))) + (clr-rx-matches (: Bjolang.Runtime.BjoRegexModule.Matches (-> Rx string (Array RxMatch)))) + (clr-rx-replace (: Bjolang.Runtime.BjoRegexModule.Replace (-> Rx string string string))) + (clr-rx-replace-with + (: Bjolang.Runtime.BjoRegexModule.ReplaceWith (-> Rx string (-> RxMatch string) string))) + (clr-rx-split (: Bjolang.Runtime.BjoRegexModule.Split (-> Rx string (Array string)))) + (clr-rx-text (: Bjolang.Runtime.BjoRegexModule.Text (-> RxMatch string))) + (clr-rx-input (: Bjolang.Runtime.BjoRegexModule.Input (-> RxMatch string))) + (clr-rx-start (: Bjolang.Runtime.BjoRegexModule.Start (-> RxMatch StringCursor))) + (clr-rx-end (: Bjolang.Runtime.BjoRegexModule.End (-> RxMatch StringCursor))) + (clr-rx-count (: Bjolang.Runtime.BjoRegexModule.Count (-> RxMatch int))) + (clr-rx-group-text (: Bjolang.Runtime.BjoRegexModule.GroupText (-> RxMatch int (Option string)))) + (clr-rx-group-start (: Bjolang.Runtime.BjoRegexModule.GroupStart (-> RxMatch int (Option StringCursor)))) + (clr-rx-group-end (: Bjolang.Runtime.BjoRegexModule.GroupEnd (-> RxMatch int (Option StringCursor)))) + (clr-rx-index-of (: Bjolang.Runtime.BjoRegexModule.MatchIndexOfName (-> RxMatch string int)))) + +;; --------------------------------------------------------------------------- +;; The runtime surface +;; --------------------------------------------------------------------------- +;; +;; Wrappers rather than the `import/extern` aliases directly, because a macro +;; template naming one of these resolves it to this module by the second +;; renaming rule, and there has to be a def of this module to resolve to. +;; TODO i guess. +;; +;; +;; Compiles a pattern. `names` is the Bjolang name of each capture group in +;; order, comma-separated — a group `($ ...)` left unnamed contributes an empty +;; one. Commas cannot occur in a keyword, so it wont' be a part of a name. +(: (-> string string Rx)) +(defun (rx-compile pattern names) (clr-rx-compile pattern names)) + +;; The pattern text this was compiled to. For testing purposes. +(: (-> Rx string)) +(defun (rx-pattern rx) (clr-rx-pattern rx)) + +;; The leftmost match anywhere in the string. +(: (-> Rx string (Option RxMatch))) +(defun (rx-search rx s) (clr-rx-search rx s)) + +(: (-> Rx string bool)) +(defun (rx-search? rx s) (clr-rx-search? rx s)) + +;; The match against the whole string, anchored at both ends. +(: (-> Rx string (Option RxMatch))) +(defun (rx-match rx s) (clr-rx-whole rx s)) + +(: (-> Rx string bool)) +(defun (rx-match? rx s) (clr-rx-whole? rx s)) + +;; Every non-overlapping match, left to right. +(: (-> Rx string (Vec RxMatch))) +(defun (rx-matches rx s) (array->vec (clr-rx-matches rx s))) + +;; Every match replaced by `replacement`, which is literal text: there is no +;; `$1` substitution syntax. Use `rx-replace-with` to build the +;; replacement from the match. +(: (-> Rx string string string)) +(defun (rx-replace rx s replacement) (clr-rx-replace rx s replacement)) + +(: (-> Rx string (-> RxMatch string) string)) +(defun (rx-replace-with rx s f) (clr-rx-replace-with rx s f)) + +;; The pieces between the matches. An empty match returns [] +(: (-> Rx string (Vec string))) +(defun (rx-split rx s) (array->vec (clr-rx-split rx s))) + +(: (-> RxMatch string)) +(defun (rx-text m) (clr-rx-text m)) + +(: (-> RxMatch string)) +(defun (rx-input m) (clr-rx-input m)) + +(: (-> RxMatch StringCursor)) +(defun (rx-start m) (clr-rx-start m)) + +(: (-> RxMatch StringCursor)) +(defun (rx-end m) (clr-rx-end m)) + +(: (-> RxMatch int)) +(defun (rx-group-count m) (clr-rx-count m)) + +;; The text of the group `(=> :name ...)` named, or `None` if that group took +;; no part in the match. An unknown name is `None` as well. +(: (-> RxMatch Keyword (Option string))) +(defun (rx-group m name) (clr-rx-group-text m (clr-rx-index-of m (keyword->string name)))) + +(: (-> RxMatch Keyword (Option StringCursor))) +(defun (rx-group-start m name) (clr-rx-group-start m (clr-rx-index-of m (keyword->string name)))) + +(: (-> RxMatch Keyword (Option StringCursor))) +(defun (rx-group-end m name) (clr-rx-group-end m (clr-rx-index-of m (keyword->string name)))) + +;; The text of the nth group, counted from zero in the order the groups open. +(: (-> RxMatch int (Option string))) +(defun (rx-group/n m n) (clr-rx-group-text m n)) + +;; --------------------------------------------------------------------------- +;; Character sets +;; --------------------------------------------------------------------------- +;; +;; A set is a sorted, disjoint list of inclusive scalar ranges. It never +;; contains a surrogate code point: `BjoChar` rejects those, so no set the +;; language can write holds one. + +(type + (: Cset (Record (: ranges (List (Tuple int int)))))) + +;; U+0000–U+D7FF and U+E000–U+10FFFF: every scalar. +(def universe (list (Tuple 0 55295) (Tuple 57344 1114111))) + +(: (-> (Tuple int int) int)) +(defun (range-lo r) (match r ((Tuple lo _) lo))) + +(: (-> (Tuple int int) int)) +(defun (range-hi r) (match r ((Tuple _ hi) hi))) + +(: (-> int int int)) +(defun (imin a b) (if (< a b) a b)) + +(: (-> int int int)) +(defun (imax a b) (if (> a b) a b)) + +(: (-> (Tuple int int) (List (Tuple int int)) (List (Tuple int int)))) +(defun (ranges-insert r sorted) + (cond + ((list-empty? sorted) (list r)) + ((<= (range-lo r) (range-lo (list-head sorted))) + (Cons r sorted)) + (else (Cons (list-head sorted) (ranges-insert r (list-tail sorted)))))) + +;; Insertion sort: a set written by hand has a handful of ranges, and a table +;; read out of the hex below is already ascending, so this is the best case +;; either way. +(: (-> (List (Tuple int int)) (List (Tuple int int)))) +(defun (ranges-sort rs) + (match rs + (Nil Nil) + ((Cons r more) (ranges-insert r (ranges-sort more))))) + +;; Adjacent as well as overlapping runs are merged, so that `[a-b][c-d]` with +;; `c = b + 1` comes out as one range and two sets built differently compare +;; the same. +(: (-> (List (Tuple int int)) (List (Tuple int int)))) +(defun (ranges-merge rs) + (match rs + (Nil Nil) + ((Cons r Nil) (list r)) + ((Cons a (Cons b more)) + (if (<= (range-lo b) (+ (range-hi a) 1)) + (ranges-merge (Cons (Tuple (range-lo a) (imax (range-hi a) (range-hi b))) more)) + (Cons a (ranges-merge (Cons b more))))))) + +(: (-> (List (Tuple int int)) (List (Tuple int int)))) +(defun (ranges-norm rs) (ranges-merge (ranges-sort rs))) + +;; The parts of `lo`–`hi` that `rs` does not cover. +(: (-> int int (List (Tuple int int)) (List (Tuple int int)))) +(defun (ranges-gaps lo hi rs) + (match rs + (Nil (if (<= lo hi) (list (Tuple lo hi)) Nil)) + ((Cons r more) + (def before (if (< lo (range-lo r)) (list (Tuple lo (imin hi (- (range-lo r) 1)))) Nil)) + (list-append before (ranges-gaps (imax lo (+ (range-hi r) 1)) hi more))))) + +(: (-> (List (Tuple int int)) (List (Tuple int int)) (List (Tuple int int)))) +(defun (ranges-inter a b) + (if (or (list-empty? a) (list-empty? b)) + Nil + (let ((x (list-head a)) (y (list-head b))) + (def lo (imax (range-lo x) (range-lo y))) + (def hi (imin (range-hi x) (range-hi y))) + (def rest (if (< (range-hi x) (range-hi y)) + (ranges-inter (list-tail a) b) + (ranges-inter a (list-tail b)))) + (if (<= lo hi) (Cons (Tuple lo hi) rest) rest)))) + +;; Normalizing and clipping to the universe here is what lets every other +;; operation assume both. What does that even mean, Claude? +(: (-> (List (Tuple int int)) Cset)) +(defun (cs-make rs) + (Cset (ranges (ranges-inter (ranges-norm rs) universe)))) + +(: (-> Cset (List (Tuple int int)))) +(defun (cs-ranges c) (record-ref c ranges)) + +(: (-> Cset Cset Cset)) +(defun (cs-union a b) + (cs-make (list-append (cs-ranges a) (cs-ranges b)))) + +(: (-> Cset Cset Cset)) +(defun (cs-inter a b) + (cs-make (ranges-inter (cs-ranges a) (cs-ranges b)))) + +(: (-> Cset Cset)) +(defun (cs-complement a) + (cs-make (ranges-gaps 0 1114111 (cs-ranges a)))) + +(: (-> Cset Cset Cset)) +(defun (cs-diff a b) (cs-inter a (cs-complement b))) + +(: (-> Cset bool)) +(defun (cs-empty? c) (list-empty? (cs-ranges c))) + +(: (-> int Cset)) +(defun (cs-char n) (cs-make (list (Tuple n n)))) + +;; --------------------------------------------------------------------------- +;; Reading the tables +;; --------------------------------------------------------------------------- +;; +;; A table is one hex string, four digits per endpoint and eight per inclusive +;; range, rather than a list of literal tuples. Six hundred ranges written out +;; as forms is six hundred forms for the compiler to read on every build of +;; this module, and the answer is the same every time. + +(: (-> char int)) +(defun (hex-val c) + (def n (char->int c)) + (cond ((and (>= n 48) (<= n 57)) (- n 48)) + ((and (>= n 65) (<= n 70)) (+ 10 (- n 65))) + (else 0))) + +(: (-> char char char char int)) +(defun (hex4-val a b c d) + (+ (* 4096 (hex-val a)) + (+ (* 256 (hex-val b)) + (+ (* 16 (hex-val c)) (hex-val d))))) + +(: (-> (List char) (List (Tuple int int)))) +(defun (hex-chunks cs) + (match cs + ((Cons a (Cons b (Cons c (Cons d (Cons e (Cons f (Cons g (Cons h rest)))))))) + (Cons (Tuple (hex4-val a b c d) (hex4-val e f g h)) (hex-chunks rest))) + (_ Nil))) + +(: (-> string (List (Tuple int int)))) +(defun (hex-ranges text) (hex-chunks (string->list text))) + +(: (-> string Cset)) +(defun (table-set text) (cs-make (hex-ranges text))) + +;; --------------------------------------------------------------------------- +;; The tree +;; --------------------------------------------------------------------------- + +(type + (: Rxn (Union + RxEmpty + (: RxLit string) + (: RxSet Cset) + (: RxSeq (List Rxn)) + (: RxAlt (List Rxn)) + ;; body, least, most (-1 for unbounded), greedy + (: RxRep Rxn int int bool) + ;; the name the user gave, "" for `($ ...)` + (: RxGroup string Rxn) + RxBos + RxEos))) + +;; --------------------------------------------------------------------------- +;; Emitting +;; --------------------------------------------------------------------------- +;; +;; The text of BjoRx's tree, one parenthesized form per node: `(l 61 62)` a +;; literal, `(c 30-39)` a class, `(s ...)` a sequence, `(a ...)` an +;; alternation, `(r least most g|l body)` a repetition, `(g n body)` a group, +;; `(A)` and `(z)` the anchors, `(e)` nothing. Scalars are hexadecimal. + +(: (-> int string)) +(defun (hex-digit-string n) + (def d (% n 16)) + (char->string (if (< d 10) (int->char (+ 48 d)) (int->char (+ 55 d))))) + +(: (-> int string)) +(defun (hex n) + (if (< n 16) + (hex-digit-string n) + (string-append (hex (/ n 16)) (hex-digit-string n)))) + +(: (-> (List string) string)) +(defun (concat-strings parts) (list-foldl (fun (part acc) (string-append acc part)) "" parts)) + +(: (-> (List string) string string)) +(defun (join-strings parts sep) + (match parts + (Nil "") + ((Cons p Nil) p) + ((Cons p more) (str p sep (join-strings more sep))))) + +(: (-> (Tuple int int) string)) +(defun (emit-range r) + (if (= (range-lo r) (range-hi r)) + (str " " (hex (range-lo r))) + (str " " (hex (range-lo r)) "-" (hex (range-hi r))))) + +(: (-> Cset string)) +(defun (emit-set c) (str "(c" (concat-strings (list-map emit-range (cs-ranges c))) ")")) + +(: (-> string string)) +(defun (emit-lit text) + (str "(l" (concat-strings (list-map (fun (c) (str " " (hex (char->int c)))) (string->list text))) ")")) + +(: (-> string (List Rxn) (Box (List string)) string)) +(defun (emit-list head parts names) + (match parts + (Nil "(e)") + ((Cons p Nil) (emit p names)) + (_ (str "(" head " " (join-strings (list-map (fun (p) (emit p names)) parts) " ") ")")))) + +;; `names` accumulates the Bjolang name of each group, most recent first. A +;; group's number is one more than how many were open before it, so the count +;; is read before the body is walked and nested groups number after their +;; parent. +(: (-> Rxn (Box (List string)) string)) +(defun (emit node names) + (match node + (RxEmpty "(e)") + ((RxLit text) (emit-lit text)) + ((RxSet c) (emit-set c)) + ((RxSeq parts) (emit-list "s" parts names)) + ((RxAlt parts) (emit-list "a" parts names)) + ((RxRep body least most greedy) + (str "(r " (int->string least) " " (if (< most 0) "-" (int->string most)) + (if greedy " g " " l ") (emit body names) ")")) + ((RxGroup name body) + (def n (+ 1 (list-length (box-ref names)))) + (box-set! names (Cons name (box-ref names))) + (str "(g " (int->string n) " " (emit body names) ")")) + (RxBos "(A)") + (RxEos "(z)"))) + +;; --------------------------------------------------------------------------- +;; Reading a set +;; --------------------------------------------------------------------------- + +(: (-> Syntax (Option string))) +(defun (head-name form) + (match form + ((SList items) + (if (list-empty? items) + None + (match (list-head items) + ((SSym s) (Some (symbol->string s))) + (_ None)))) + (_ None))) + +(: (-> Syntax (List Syntax))) +(defun (form-args form) + (match form + ((SList items) (if (list-empty? items) Nil (list-tail items))) + (_ Nil))) + +;; A character usable as a set member or a range endpoint. +(: (-> Syntax char int)) +(defun (set-scalar form c) + (ignore form) + (char->int c)) + +;; `syntax-error` types as a `Syntax` and never returns, so a branch that owes +;; an `int` has to discard it. +(: (-> Syntax int)) +(defun (ignore-int s) (ignore s) 0) + +(: (-> Syntax Cset)) +(defun (ignore-cset s) (ignore s) (cs-make Nil)) + +(: (-> Syntax (List Syntax) (List int))) +(defun (chars-of whole items) + (match items + (Nil Nil) + ((Cons item more) + (list-append + (match item + ((SChar c) (list (set-scalar item c))) + ((SStr text) (list-map (fun (c) (set-scalar item c)) (string->list text))) + (_ (list (ignore-int + (syntax-error item + "A range endpoint is a character or a string of them: (/ #\\a #\\z) or (/ \"az\")."))))) + (chars-of whole more))))) + +;; `(/ #\a #\z #\0 #\9)` and `(/ "az09")` are the same set: the endpoints are +;; taken in pairs however they were written. +(: (-> Syntax (List int) Cset)) +(defun (pairs->cset whole ns) + (match ns + (Nil (cs-make Nil)) + ((Cons lo (Cons hi more)) + (if (> lo hi) + (ignore-cset + (syntax-error whole "A range runs upwards, and this one does not.")) + (cs-union (cs-make (list (Tuple lo hi))) (pairs->cset whole more)))) + (_ (ignore-cset + (syntax-error whole "A range takes its endpoints in pairs, and this one has an odd number."))))) + +(: (-> (List Syntax) Cset)) +(defun (union-of items) + (list-foldl (fun (item acc) (cs-union acc (parse-set item))) (cs-make Nil) items)) + +(: (-> Syntax Cset)) +(defun (parse-set form) + (match form + ((SChar c) (cs-char (set-scalar form c))) + ((SStr text) (list-foldl (fun (c acc) (cs-union acc (cs-char (set-scalar form c)))) + (cs-make Nil) + (string->list text))) + ((SKey k) + (match (named-class (keyword->string k)) + ((Some c) c) + (None (ignore-cset + (syntax-error form (str "There is no character set called " + (keyword->string k) ".")))))) + ((SList _) (parse-set-form form)) + (_ (ignore-cset + (syntax-error form + "A character set is a character, a string, a named set like :digit, or a form like (in ...), (/ ...), (~ ...), (- ...) or (& ...)."))))) + +(: (-> Syntax Cset)) +(defun (parse-set-form form) + (def args (form-args form)) + (match (head-name form) + (None (ignore-cset (syntax-error form "A character set form is headed by a name."))) + ((Some name) + (cond + ((= name "in") (union-of args)) + ((= name "or") (union-of args)) + ((= name "/") (pairs->cset form (chars-of form args))) + ((= name "~") (cs-complement (union-of args))) + ((= name "-") (set-difference form args)) + ((= name "&") (set-intersection form args)) + (else (ignore-cset + (syntax-error form + (str "'" name "' is not a character set operation. Use (in ...), (/ ...), (~ ...), (- ...) or (& ...).")))))))) + +(: (-> Syntax (List Syntax) Cset)) +(defun (set-difference form args) + (if (list-empty? args) + (ignore-cset (syntax-error form "(- ...) takes a set to subtract from.")) + (cs-diff (parse-set (list-head args)) (union-of (list-tail args))))) + +(: (-> Syntax (List Syntax) Cset)) +(defun (set-intersection form args) + (if (list-empty? args) + (ignore-cset (syntax-error form "(& ...) takes at least one set.")) + (list-foldl (fun (item acc) (cs-inter acc (parse-set item))) + (parse-set (list-head args)) + (list-tail args)))) + +;; A set reaching pattern position has to be able to match something, and an +;; empty one is a mistake that would otherwise silently never match. +(: (-> Syntax Cset Rxn)) +(defun (set-node form c) + (if (cs-empty? c) + (ignore-node (syntax-error form "This character set is empty, so it can never match.")) + (RxSet c))) + +(: (-> Syntax Rxn)) +(defun (ignore-node s) (ignore s) RxEmpty) + +;; --------------------------------------------------------------------------- +;; Reading a pattern +;; --------------------------------------------------------------------------- + +(: (-> Syntax Rxn)) +(defun (parse form) + (match form + ((SStr text) (RxLit text)) + ((SChar c) (RxLit (char->string c))) + ((SKey k) (parse-atom form (keyword->string k))) + ((SList _) (parse-form form)) + ((SSym s) + (ignore-node + (syntax-error form + (str "'" (symbol->string s) "' is a symbol, and an rx that takes no argument is a keyword. Did you mean :" + (symbol->string s) "?")))) + ((SPunct _) + (ignore-node + (syntax-error form + "An rx is built at compile time, so it cannot splice a value in. Dynamically built patterns are not supported."))) + (_ (ignore-node (syntax-error form "This is not an rx."))))) + +(: (-> Syntax string Rxn)) +(defun (parse-atom form name) + (cond + ((= name "bos") RxBos) + ((= name "eos") RxEos) + (else + (match (named-class name) + ((Some c) (set-node form c)) + (None + (ignore-node + (syntax-error form + (str "There is no rx called :" name + ". The ones taking no argument are :bos :eos :any :nonl and the named sets.")))))))) + +(: (-> (List Syntax) (List Rxn))) +(defun (parse-all items) (list-map parse items)) + +(: (-> Syntax Rxn)) +(defun (parse-form form) + (def args (form-args form)) + (match (head-name form) + (None + (ignore-node + (syntax-error form + "An rx form is headed by a name: (seq ...), (or ...), (+ ...), (in ...) and so on."))) + ((Some name) + (cond + ((= name "seq") (RxSeq (parse-all args))) + ((= name "or") (RxAlt (parse-all args))) + ((= name "lit") (RxLit (literal-text form args))) + ((= name "*") (RxRep (one-arg form args) 0 -1 #t)) + ((= name "+") (RxRep (one-arg form args) 1 -1 #t)) + ((= name "?") (RxRep (one-arg form args) 0 1 #t)) + ((= name "*?") (RxRep (one-arg form args) 0 -1 #f)) + ((= name "+?") (RxRep (one-arg form args) 1 -1 #f)) + ((= name "??") (RxRep (one-arg form args) 0 1 #f)) + ((= name "**") (bounded form args)) + ((= name "=") (exactly form args)) + ((= name ">=") (at-least form args)) + ((= name "$") (RxGroup "" (RxSeq (parse-all args)))) + ((= name "=>") (named-group form args)) + ((= name "in") (set-node form (union-of args))) + ((= name "/") (set-node form (pairs->cset form (chars-of form args)))) + ((= name "~") (set-node form (cs-complement (union-of args)))) + ((= name "-") (set-node form (set-difference form args))) + ((= name "&") (set-node form (set-intersection form args))) + (else + (ignore-node + (syntax-error form + (str "'" name "' is not an rx form. If you meant a set or an anchor, those are keywords: :" + name ".")))))))) + +(: (-> Syntax (List Syntax) Rxn)) +(defun (one-arg form args) + (match args + ((Cons a Nil) (parse a)) + (_ (RxSeq (parse-all args))))) + +(: (-> Syntax (List Syntax) string)) +(defun (literal-text form args) + (match args + ((Cons a Nil) + (match a + ((SStr text) text) + ((SChar c) (char->string c)) + (_ (ignore-string (syntax-error form "(lit ...) takes one string."))))) + (_ (ignore-string (syntax-error form "(lit ...) takes one string."))))) + +(: (-> Syntax string)) +(defun (ignore-string s) (ignore s) "") + +(: (-> Syntax Syntax int)) +(defun (count-of whole s) + (match s + ((SInt text) (string->int text)) + (_ (ignore-int (syntax-error whole "A repetition count is a whole number."))))) + +(: (-> Syntax (List Syntax) Rxn)) +(defun (bounded form args) + (match args + ((Cons lo (Cons hi rest)) + (RxRep (one-arg form rest) (count-of form lo) (count-of form hi) #t)) + (_ (ignore-node (syntax-error form "(** least most rx) takes two counts and a pattern."))))) + +(: (-> Syntax (List Syntax) Rxn)) +(defun (exactly form args) + (match args + ((Cons n rest) (let ((k (count-of form n))) (RxRep (one-arg form rest) k k #t))) + (_ (ignore-node (syntax-error form "(= n rx) takes a count and a pattern."))))) + +(: (-> Syntax (List Syntax) Rxn)) +(defun (at-least form args) + (match args + ((Cons n rest) (RxRep (one-arg form rest) (count-of form n) -1 #t)) + (_ (ignore-node (syntax-error form "(>= n rx) takes a count and a pattern."))))) + +;; The name is a keyword for the same reason every other constant of this +;; sublanguage is one: it is a name used as data, not a binding, and a keyword +;; can never be mistaken for one. +(: (-> Syntax (List Syntax) Rxn)) +(defun (named-group form args) + (match args + ((Cons n rest) + (match n + ((SKey k) (RxGroup (keyword->string k) (RxSeq (parse-all rest)))) + (_ (ignore-node + (syntax-error form "(=> :name rx) names its group with a keyword."))))) + (_ (ignore-node (syntax-error form "(=> :name rx) takes a name and a pattern."))))) + +;; --------------------------------------------------------------------------- +;; The macros +;; --------------------------------------------------------------------------- + +(: (-> (List string) string)) +(defun (names-text names) (join-strings names ",")) + +;; `#rx(seq a b)` reads as `(#rx seq a b)`, so the arguments are the form's own +;; contents. One argument is the form itself, which is what makes `#rx(:digit)` +;; and `#rx((+ :digit))` work as well as `#rx(+ :digit)`. +;; +;; The compile is hoisted: both arguments are literals, so the regex is built +;; the first time the expression is reached and every later evaluation reuses +;; it. Without that, every evaluation looks the pattern up in `BjoRegex`'s +;; cache, which hashes and compares the whole pattern string, and an emitted +;; character class can be thousands of characters long. A pattern in a match +;; clause is evaluated once per match attempt, so it pays that every time. +(: (-> Syntax Syntax)) +(defun (rx-expansion form) + (def node (parse form)) + (def names (make-box Nil)) + (def pattern (emit node names)) + #'(please-hoist-i-promise-i-am-not-naughty + (rx-compile ,(SStr pattern) ,(SStr (names-text (list-reverse (box-ref names))))))) + +(def/hash-extend (rx form inject compare) + (match (form-args form) + (Nil (syntax-error form "#rx takes a pattern: #rx(seq :bos (+ :digit) :eos).")) + ((Cons only Nil) (rx-expansion only)) + (items (rx-expansion (SList items))))) + +;; `(Rx re p)` matches `re` against the whole string, `(RxIn re p)` searches for +;; it. Both hand the inner pattern an `(Option RxMatch)`, so a clause reads +;; `(Rx phone (Some m))`. +;; +;; A view covers nothing for exhaustiveness, so a `match` using either still +;; needs a clause that matches anything. +(def/pattern (Rx form inject compare) + (syntax-match form + ((_ re p) #'(:view (rx-match ,re &) ,p)) + (bad (syntax-error bad "(Rx regex pattern) matches regex against the whole string.")))) + +(def/pattern (RxIn form inject compare) + (syntax-match form + ((_ re p) #'(:view (rx-search ,re &) ,p)) + (bad (syntax-error bad "(RxIn regex pattern) searches the string for regex.")))) + +;; --------------------------------------------------------------------------- +;; The tables +;; --------------------------------------------------------------------------- +;; +;; Generated from .NET's `CharUnicodeInfo`, so that `:alpha` and +;; `char-alphabetic?` agree. This is to avoid walking the BMP on each build. + +(def tbl-alpha + (str + "0041005A0061007A00AA00AA00B500B500BA00BA00C000D600D800F600F802C102C602D1" + "02E002E402EC02EC02EE02EE0370037403760377037A037D037F037F038603860388038A" + "038C038C038E03A103A303F503F70481048A052F05310556055905590560058805D005EA" + "05EF05F20620064A066E066F067106D306D506D506E506E606EE06EF06FA06FC06FF06FF" + "071007100712072F074D07A507B107B107CA07EA07F407F507FA07FA08000815081A081A" + "0824082408280828084008580860086A087008870889088E08A008C909040939093D093D" + "0950095009580961097109800985098C098F0990099309A809AA09B009B209B209B609B9" + "09BD09BD09CE09CE09DC09DD09DF09E109F009F109FC09FC0A050A0A0A0F0A100A130A28" + "0A2A0A300A320A330A350A360A380A390A590A5C0A5E0A5E0A720A740A850A8D0A8F0A91" + "0A930AA80AAA0AB00AB20AB30AB50AB90ABD0ABD0AD00AD00AE00AE10AF90AF90B050B0C" + "0B0F0B100B130B280B2A0B300B320B330B350B390B3D0B3D0B5C0B5D0B5F0B610B710B71" + "0B830B830B850B8A0B8E0B900B920B950B990B9A0B9C0B9C0B9E0B9F0BA30BA40BA80BAA" + "0BAE0BB90BD00BD00C050C0C0C0E0C100C120C280C2A0C390C3D0C3D0C580C5A0C5D0C5D" + "0C600C610C800C800C850C8C0C8E0C900C920CA80CAA0CB30CB50CB90CBD0CBD0CDD0CDE" + "0CE00CE10CF10CF20D040D0C0D0E0D100D120D3A0D3D0D3D0D4E0D4E0D540D560D5F0D61" + "0D7A0D7F0D850D960D9A0DB10DB30DBB0DBD0DBD0DC00DC60E010E300E320E330E400E46" + "0E810E820E840E840E860E8A0E8C0EA30EA50EA50EA70EB00EB20EB30EBD0EBD0EC00EC4" + "0EC60EC60EDC0EDF0F000F000F400F470F490F6C0F880F8C1000102A103F103F10501055" + "105A105D1061106110651066106E107010751081108E108E10A010C510C710C710CD10CD" + "10D010FA10FC1248124A124D1250125612581258125A125D12601288128A128D129012B0" + "12B212B512B812BE12C012C012C212C512C812D612D81310131213151318135A1380138F" + "13A013F513F813FD1401166C166F167F1681169A16A016EA16F116F817001711171F1731" + "174017511760176C176E1770178017B317D717D717DC17DC1820187818801884188718A8" + "18AA18AA18B018F51900191E1950196D19701974198019AB19B019C91A001A161A201A54" + "1AA71AA71B051B331B451B4C1B831BA01BAE1BAF1BBA1BE51C001C231C4D1C4F1C5A1C7D" + "1C801C8A1C901CBA1CBD1CBF1CE91CEC1CEE1CF31CF51CF61CFA1CFA1D001DBF1E001F15" + "1F181F1D1F201F451F481F4D1F501F571F591F591F5B1F5B1F5D1F5D1F5F1F7D1F801FB4" + "1FB61FBC1FBE1FBE1FC21FC41FC61FCC1FD01FD31FD61FDB1FE01FEC1FF21FF41FF61FFC" + "20712071207F207F2090209C2102210221072107210A2113211521152119211D21242124" + "2126212621282128212A212D212F2139213C213F21452149214E214E218321842C002CE4" + "2CEB2CEE2CF22CF32D002D252D272D272D2D2D2D2D302D672D6F2D6F2D802D962DA02DA6" + "2DA82DAE2DB02DB62DB82DBE2DC02DC62DC82DCE2DD02DD62DD82DDE2E2F2E2F30053006" + "30313035303B303C30413096309D309F30A130FA30FC30FF3105312F3131318E31A031BF" + "31F031FF34004DBF4E00A48CA4D0A4FDA500A60CA610A61FA62AA62BA640A66EA67FA69D" + "A6A0A6E5A717A71FA722A788A78BA7CDA7D0A7D1A7D3A7D3A7D5A7DCA7F2A801A803A805" + "A807A80AA80CA822A840A873A882A8B3A8F2A8F7A8FBA8FBA8FDA8FEA90AA925A930A946" + "A960A97CA984A9B2A9CFA9CFA9E0A9E4A9E6A9EFA9FAA9FEAA00AA28AA40AA42AA44AA4B" + "AA60AA76AA7AAA7AAA7EAAAFAAB1AAB1AAB5AAB6AAB9AABDAAC0AAC0AAC2AAC2AADBAADD" + "AAE0AAEAAAF2AAF4AB01AB06AB09AB0EAB11AB16AB20AB26AB28AB2EAB30AB5AAB5CAB69" + "AB70ABE2AC00D7A3D7B0D7C6D7CBD7FBF900FA6DFA70FAD9FB00FB06FB13FB17FB1DFB1D" + "FB1FFB28FB2AFB36FB38FB3CFB3EFB3EFB40FB41FB43FB44FB46FBB1FBD3FD3DFD50FD8F" + "FD92FDC7FDF0FDFBFE70FE74FE76FEFCFF21FF3AFF41FF5AFF66FFBEFFC2FFC7FFCAFFCF" + "FFD2FFD7FFDAFFDC" + )) + +(def tbl-digit + (str + "003000390660066906F006F907C007C90966096F09E609EF0A660A6F0AE60AEF0B660B6F" + "0BE60BEF0C660C6F0CE60CEF0D660D6F0DE60DEF0E500E590ED00ED90F200F2910401049" + "1090109917E017E9181018191946194F19D019D91A801A891A901A991B501B591BB01BB9" + "1C401C491C501C59A620A629A8D0A8D9A900A909A9D0A9D9A9F0A9F9AA50AA59ABF0ABF9" + "FF10FF19" + )) + +(def tbl-alnum + (str + "003000390041005A0061007A00AA00AA00B500B500BA00BA00C000D600D800F600F802C1" + "02C602D102E002E402EC02EC02EE02EE0370037403760377037A037D037F037F03860386" + "0388038A038C038C038E03A103A303F503F70481048A052F053105560559055905600588" + "05D005EA05EF05F20620064A06600669066E066F067106D306D506D506E506E606EE06FC" + "06FF06FF071007100712072F074D07A507B107B107C007EA07F407F507FA07FA08000815" + "081A081A0824082408280828084008580860086A087008870889088E08A008C909040939" + "093D093D09500950095809610966096F097109800985098C098F0990099309A809AA09B0" + "09B209B209B609B909BD09BD09CE09CE09DC09DD09DF09E109E609F109FC09FC0A050A0A" + "0A0F0A100A130A280A2A0A300A320A330A350A360A380A390A590A5C0A5E0A5E0A660A6F" + "0A720A740A850A8D0A8F0A910A930AA80AAA0AB00AB20AB30AB50AB90ABD0ABD0AD00AD0" + "0AE00AE10AE60AEF0AF90AF90B050B0C0B0F0B100B130B280B2A0B300B320B330B350B39" + "0B3D0B3D0B5C0B5D0B5F0B610B660B6F0B710B710B830B830B850B8A0B8E0B900B920B95" + "0B990B9A0B9C0B9C0B9E0B9F0BA30BA40BA80BAA0BAE0BB90BD00BD00BE60BEF0C050C0C" + "0C0E0C100C120C280C2A0C390C3D0C3D0C580C5A0C5D0C5D0C600C610C660C6F0C800C80" + "0C850C8C0C8E0C900C920CA80CAA0CB30CB50CB90CBD0CBD0CDD0CDE0CE00CE10CE60CEF" + "0CF10CF20D040D0C0D0E0D100D120D3A0D3D0D3D0D4E0D4E0D540D560D5F0D610D660D6F" + "0D7A0D7F0D850D960D9A0DB10DB30DBB0DBD0DBD0DC00DC60DE60DEF0E010E300E320E33" + "0E400E460E500E590E810E820E840E840E860E8A0E8C0EA30EA50EA50EA70EB00EB20EB3" + "0EBD0EBD0EC00EC40EC60EC60ED00ED90EDC0EDF0F000F000F200F290F400F470F490F6C" + "0F880F8C1000102A103F104910501055105A105D1061106110651066106E107010751081" + "108E108E1090109910A010C510C710C710CD10CD10D010FA10FC1248124A124D12501256" + "12581258125A125D12601288128A128D129012B012B212B512B812BE12C012C012C212C5" + "12C812D612D81310131213151318135A1380138F13A013F513F813FD1401166C166F167F" + "1681169A16A016EA16F116F817001711171F1731174017511760176C176E1770178017B3" + "17D717D717DC17DC17E017E9181018191820187818801884188718A818AA18AA18B018F5" + "1900191E1946196D19701974198019AB19B019C919D019D91A001A161A201A541A801A89" + "1A901A991AA71AA71B051B331B451B4C1B501B591B831BA01BAE1BE51C001C231C401C49" + "1C4D1C7D1C801C8A1C901CBA1CBD1CBF1CE91CEC1CEE1CF31CF51CF61CFA1CFA1D001DBF" + "1E001F151F181F1D1F201F451F481F4D1F501F571F591F591F5B1F5B1F5D1F5D1F5F1F7D" + "1F801FB41FB61FBC1FBE1FBE1FC21FC41FC61FCC1FD01FD31FD61FDB1FE01FEC1FF21FF4" + "1FF61FFC20712071207F207F2090209C2102210221072107210A2113211521152119211D" + "212421242126212621282128212A212D212F2139213C213F21452149214E214E21832184" + "2C002CE42CEB2CEE2CF22CF32D002D252D272D272D2D2D2D2D302D672D6F2D6F2D802D96" + "2DA02DA62DA82DAE2DB02DB62DB82DBE2DC02DC62DC82DCE2DD02DD62DD82DDE2E2F2E2F" + "3005300630313035303B303C30413096309D309F30A130FA30FC30FF3105312F3131318E" + "31A031BF31F031FF34004DBF4E00A48CA4D0A4FDA500A60CA610A62BA640A66EA67FA69D" + "A6A0A6E5A717A71FA722A788A78BA7CDA7D0A7D1A7D3A7D3A7D5A7DCA7F2A801A803A805" + "A807A80AA80CA822A840A873A882A8B3A8D0A8D9A8F2A8F7A8FBA8FBA8FDA8FEA900A925" + "A930A946A960A97CA984A9B2A9CFA9D9A9E0A9E4A9E6A9FEAA00AA28AA40AA42AA44AA4B" + "AA50AA59AA60AA76AA7AAA7AAA7EAAAFAAB1AAB1AAB5AAB6AAB9AABDAAC0AAC0AAC2AAC2" + "AADBAADDAAE0AAEAAAF2AAF4AB01AB06AB09AB0EAB11AB16AB20AB26AB28AB2EAB30AB5A" + "AB5CAB69AB70ABE2ABF0ABF9AC00D7A3D7B0D7C6D7CBD7FBF900FA6DFA70FAD9FB00FB06" + "FB13FB17FB1DFB1DFB1FFB28FB2AFB36FB38FB3CFB3EFB3EFB40FB41FB43FB44FB46FBB1" + "FBD3FD3DFD50FD8FFD92FDC7FDF0FDFBFE70FE74FE76FEFCFF10FF19FF21FF3AFF41FF5A" + "FF66FFBEFFC2FFC7FFCAFFCFFFD2FFD7FFDAFFDC" + )) + +(def tbl-upper + (str + "0041005A00C000D600D800DE0100010001020102010401040106010601080108010A010A" + "010C010C010E010E0110011001120112011401140116011601180118011A011A011C011C" + "011E011E0120012001220122012401240126012601280128012A012A012C012C012E012E" + "0130013001320132013401340136013601390139013B013B013D013D013F013F01410141" + "014301430145014501470147014A014A014C014C014E014E015001500152015201540154" + "0156015601580158015A015A015C015C015E015E01600160016201620164016401660166" + "01680168016A016A016C016C016E016E0170017001720172017401740176017601780179" + "017B017B017D017D0181018201840184018601870189018B018E01910193019401960198" + "019C019D019F01A001A201A201A401A401A601A701A901A901AC01AC01AE01AF01B101B3" + "01B501B501B701B801BC01BC01C401C401C701C701CA01CA01CD01CD01CF01CF01D101D1" + "01D301D301D501D501D701D701D901D901DB01DB01DE01DE01E001E001E201E201E401E4" + "01E601E601E801E801EA01EA01EC01EC01EE01EE01F101F101F401F401F601F801FA01FA" + "01FC01FC01FE01FE0200020002020202020402040206020602080208020A020A020C020C" + "020E020E0210021002120212021402140216021602180218021A021A021C021C021E021E" + "0220022002220222022402240226022602280228022A022A022C022C022E022E02300230" + "02320232023A023B023D023E024102410243024602480248024A024A024C024C024E024E" + "037003700372037203760376037F037F038603860388038A038C038C038E038F039103A1" + "03A303AB03CF03CF03D203D403D803D803DA03DA03DC03DC03DE03DE03E003E003E203E2" + "03E403E403E603E603E803E803EA03EA03EC03EC03EE03EE03F403F403F703F703F903FA" + "03FD042F0460046004620462046404640466046604680468046A046A046C046C046E046E" + "0470047004720472047404740476047604780478047A047A047C047C047E047E04800480" + "048A048A048C048C048E048E0490049004920492049404940496049604980498049A049A" + "049C049C049E049E04A004A004A204A204A404A404A604A604A804A804AA04AA04AC04AC" + "04AE04AE04B004B004B204B204B404B404B604B604B804B804BA04BA04BC04BC04BE04BE" + "04C004C104C304C304C504C504C704C704C904C904CB04CB04CD04CD04D004D004D204D2" + "04D404D404D604D604D804D804DA04DA04DC04DC04DE04DE04E004E004E204E204E404E4" + "04E604E604E804E804EA04EA04EC04EC04EE04EE04F004F004F204F204F404F404F604F6" + "04F804F804FA04FA04FC04FC04FE04FE0500050005020502050405040506050605080508" + "050A050A050C050C050E050E0510051005120512051405140516051605180518051A051A" + "051C051C051E051E0520052005220522052405240526052605280528052A052A052C052C" + "052E052E0531055610A010C510C710C710CD10CD13A013F51C891C891C901CBA1CBD1CBF" + "1E001E001E021E021E041E041E061E061E081E081E0A1E0A1E0C1E0C1E0E1E0E1E101E10" + "1E121E121E141E141E161E161E181E181E1A1E1A1E1C1E1C1E1E1E1E1E201E201E221E22" + "1E241E241E261E261E281E281E2A1E2A1E2C1E2C1E2E1E2E1E301E301E321E321E341E34" + "1E361E361E381E381E3A1E3A1E3C1E3C1E3E1E3E1E401E401E421E421E441E441E461E46" + "1E481E481E4A1E4A1E4C1E4C1E4E1E4E1E501E501E521E521E541E541E561E561E581E58" + "1E5A1E5A1E5C1E5C1E5E1E5E1E601E601E621E621E641E641E661E661E681E681E6A1E6A" + "1E6C1E6C1E6E1E6E1E701E701E721E721E741E741E761E761E781E781E7A1E7A1E7C1E7C" + "1E7E1E7E1E801E801E821E821E841E841E861E861E881E881E8A1E8A1E8C1E8C1E8E1E8E" + "1E901E901E921E921E941E941E9E1E9E1EA01EA01EA21EA21EA41EA41EA61EA61EA81EA8" + "1EAA1EAA1EAC1EAC1EAE1EAE1EB01EB01EB21EB21EB41EB41EB61EB61EB81EB81EBA1EBA" + "1EBC1EBC1EBE1EBE1EC01EC01EC21EC21EC41EC41EC61EC61EC81EC81ECA1ECA1ECC1ECC" + "1ECE1ECE1ED01ED01ED21ED21ED41ED41ED61ED61ED81ED81EDA1EDA1EDC1EDC1EDE1EDE" + "1EE01EE01EE21EE21EE41EE41EE61EE61EE81EE81EEA1EEA1EEC1EEC1EEE1EEE1EF01EF0" + "1EF21EF21EF41EF41EF61EF61EF81EF81EFA1EFA1EFC1EFC1EFE1EFE1F081F0F1F181F1D" + "1F281F2F1F381F3F1F481F4D1F591F591F5B1F5B1F5D1F5D1F5F1F5F1F681F6F1FB81FBB" + "1FC81FCB1FD81FDB1FE81FEC1FF81FFB2102210221072107210B210D2110211221152115" + "2119211D212421242126212621282128212A212D21302133213E213F2145214521832183" + "2C002C2F2C602C602C622C642C672C672C692C692C6B2C6B2C6D2C702C722C722C752C75" + "2C7E2C802C822C822C842C842C862C862C882C882C8A2C8A2C8C2C8C2C8E2C8E2C902C90" + "2C922C922C942C942C962C962C982C982C9A2C9A2C9C2C9C2C9E2C9E2CA02CA02CA22CA2" + "2CA42CA42CA62CA62CA82CA82CAA2CAA2CAC2CAC2CAE2CAE2CB02CB02CB22CB22CB42CB4" + "2CB62CB62CB82CB82CBA2CBA2CBC2CBC2CBE2CBE2CC02CC02CC22CC22CC42CC42CC62CC6" + "2CC82CC82CCA2CCA2CCC2CCC2CCE2CCE2CD02CD02CD22CD22CD42CD42CD62CD62CD82CD8" + "2CDA2CDA2CDC2CDC2CDE2CDE2CE02CE02CE22CE22CEB2CEB2CED2CED2CF22CF2A640A640" + "A642A642A644A644A646A646A648A648A64AA64AA64CA64CA64EA64EA650A650A652A652" + "A654A654A656A656A658A658A65AA65AA65CA65CA65EA65EA660A660A662A662A664A664" + "A666A666A668A668A66AA66AA66CA66CA680A680A682A682A684A684A686A686A688A688" + "A68AA68AA68CA68CA68EA68EA690A690A692A692A694A694A696A696A698A698A69AA69A" + "A722A722A724A724A726A726A728A728A72AA72AA72CA72CA72EA72EA732A732A734A734" + "A736A736A738A738A73AA73AA73CA73CA73EA73EA740A740A742A742A744A744A746A746" + "A748A748A74AA74AA74CA74CA74EA74EA750A750A752A752A754A754A756A756A758A758" + "A75AA75AA75CA75CA75EA75EA760A760A762A762A764A764A766A766A768A768A76AA76A" + "A76CA76CA76EA76EA779A779A77BA77BA77DA77EA780A780A782A782A784A784A786A786" + "A78BA78BA78DA78DA790A790A792A792A796A796A798A798A79AA79AA79CA79CA79EA79E" + "A7A0A7A0A7A2A7A2A7A4A7A4A7A6A7A6A7A8A7A8A7AAA7AEA7B0A7B4A7B6A7B6A7B8A7B8" + "A7BAA7BAA7BCA7BCA7BEA7BEA7C0A7C0A7C2A7C2A7C4A7C7A7C9A7C9A7CBA7CCA7D0A7D0" + "A7D6A7D6A7D8A7D8A7DAA7DAA7DCA7DCA7F5A7F5FF21FF3A" + )) + +(def tbl-lower + (str + "0061007A00B500B500DF00F600F800FF0101010101030103010501050107010701090109" + "010B010B010D010D010F010F0111011101130113011501150117011701190119011B011B" + "011D011D011F011F0121012101230123012501250127012701290129012B012B012D012D" + "012F012F01310131013301330135013501370138013A013A013C013C013E013E01400140" + "01420142014401440146014601480149014B014B014D014D014F014F0151015101530153" + "015501550157015701590159015B015B015D015D015F015F016101610163016301650165" + "0167016701690169016B016B016D016D016F016F01710171017301730175017501770177" + "017A017A017C017C017E0180018301830185018501880188018C018D0192019201950195" + "0199019B019E019E01A101A101A301A301A501A501A801A801AA01AB01AD01AD01B001B0" + "01B401B401B601B601B901BA01BD01BF01C601C601C901C901CC01CC01CE01CE01D001D0" + "01D201D201D401D401D601D601D801D801DA01DA01DC01DD01DF01DF01E101E101E301E3" + "01E501E501E701E701E901E901EB01EB01ED01ED01EF01F001F301F301F501F501F901F9" + "01FB01FB01FD01FD01FF01FF0201020102030203020502050207020702090209020B020B" + "020D020D020F020F0211021102130213021502150217021702190219021B021B021D021D" + "021F021F0221022102230223022502250227022702290229022B022B022D022D022F022F" + "0231023102330239023C023C023F0240024202420247024702490249024B024B024D024D" + "024F0293029502AF037103710373037303770377037B037D0390039003AC03CE03D003D1" + "03D503D703D903D903DB03DB03DD03DD03DF03DF03E103E103E303E303E503E503E703E7" + "03E903E903EB03EB03ED03ED03EF03F303F503F503F803F803FB03FC0430045F04610461" + "04630463046504650467046704690469046B046B046D046D046F046F0471047104730473" + "047504750477047704790479047B047B047D047D047F047F04810481048B048B048D048D" + "048F048F0491049104930493049504950497049704990499049B049B049D049D049F049F" + "04A104A104A304A304A504A504A704A704A904A904AB04AB04AD04AD04AF04AF04B104B1" + "04B304B304B504B504B704B704B904B904BB04BB04BD04BD04BF04BF04C204C204C404C4" + "04C604C604C804C804CA04CA04CC04CC04CE04CF04D104D104D304D304D504D504D704D7" + "04D904D904DB04DB04DD04DD04DF04DF04E104E104E304E304E504E504E704E704E904E9" + "04EB04EB04ED04ED04EF04EF04F104F104F304F304F504F504F704F704F904F904FB04FB" + "04FD04FD04FF04FF0501050105030503050505050507050705090509050B050B050D050D" + "050F050F0511051105130513051505150517051705190519051B051B051D051D051F051F" + "0521052105230523052505250527052705290529052B052B052D052D052F052F05600588" + "10D010FA10FD10FF13F813FD1C801C881C8A1C8A1D001D2B1D6B1D771D791D9A1E011E01" + "1E031E031E051E051E071E071E091E091E0B1E0B1E0D1E0D1E0F1E0F1E111E111E131E13" + "1E151E151E171E171E191E191E1B1E1B1E1D1E1D1E1F1E1F1E211E211E231E231E251E25" + "1E271E271E291E291E2B1E2B1E2D1E2D1E2F1E2F1E311E311E331E331E351E351E371E37" + "1E391E391E3B1E3B1E3D1E3D1E3F1E3F1E411E411E431E431E451E451E471E471E491E49" + "1E4B1E4B1E4D1E4D1E4F1E4F1E511E511E531E531E551E551E571E571E591E591E5B1E5B" + "1E5D1E5D1E5F1E5F1E611E611E631E631E651E651E671E671E691E691E6B1E6B1E6D1E6D" + "1E6F1E6F1E711E711E731E731E751E751E771E771E791E791E7B1E7B1E7D1E7D1E7F1E7F" + "1E811E811E831E831E851E851E871E871E891E891E8B1E8B1E8D1E8D1E8F1E8F1E911E91" + "1E931E931E951E9D1E9F1E9F1EA11EA11EA31EA31EA51EA51EA71EA71EA91EA91EAB1EAB" + "1EAD1EAD1EAF1EAF1EB11EB11EB31EB31EB51EB51EB71EB71EB91EB91EBB1EBB1EBD1EBD" + "1EBF1EBF1EC11EC11EC31EC31EC51EC51EC71EC71EC91EC91ECB1ECB1ECD1ECD1ECF1ECF" + "1ED11ED11ED31ED31ED51ED51ED71ED71ED91ED91EDB1EDB1EDD1EDD1EDF1EDF1EE11EE1" + "1EE31EE31EE51EE51EE71EE71EE91EE91EEB1EEB1EED1EED1EEF1EEF1EF11EF11EF31EF3" + "1EF51EF51EF71EF71EF91EF91EFB1EFB1EFD1EFD1EFF1F071F101F151F201F271F301F37" + "1F401F451F501F571F601F671F701F7D1F801F871F901F971FA01FA71FB01FB41FB61FB7" + "1FBE1FBE1FC21FC41FC61FC71FD01FD31FD61FD71FE01FE71FF21FF41FF61FF7210A210A" + "210E210F21132113212F212F2134213421392139213C213D21462149214E214E21842184" + "2C302C5F2C612C612C652C662C682C682C6A2C6A2C6C2C6C2C712C712C732C742C762C7B" + "2C812C812C832C832C852C852C872C872C892C892C8B2C8B2C8D2C8D2C8F2C8F2C912C91" + "2C932C932C952C952C972C972C992C992C9B2C9B2C9D2C9D2C9F2C9F2CA12CA12CA32CA3" + "2CA52CA52CA72CA72CA92CA92CAB2CAB2CAD2CAD2CAF2CAF2CB12CB12CB32CB32CB52CB5" + "2CB72CB72CB92CB92CBB2CBB2CBD2CBD2CBF2CBF2CC12CC12CC32CC32CC52CC52CC72CC7" + "2CC92CC92CCB2CCB2CCD2CCD2CCF2CCF2CD12CD12CD32CD32CD52CD52CD72CD72CD92CD9" + "2CDB2CDB2CDD2CDD2CDF2CDF2CE12CE12CE32CE42CEC2CEC2CEE2CEE2CF32CF32D002D25" + "2D272D272D2D2D2DA641A641A643A643A645A645A647A647A649A649A64BA64BA64DA64D" + "A64FA64FA651A651A653A653A655A655A657A657A659A659A65BA65BA65DA65DA65FA65F" + "A661A661A663A663A665A665A667A667A669A669A66BA66BA66DA66DA681A681A683A683" + "A685A685A687A687A689A689A68BA68BA68DA68DA68FA68FA691A691A693A693A695A695" + "A697A697A699A699A69BA69BA723A723A725A725A727A727A729A729A72BA72BA72DA72D" + "A72FA731A733A733A735A735A737A737A739A739A73BA73BA73DA73DA73FA73FA741A741" + "A743A743A745A745A747A747A749A749A74BA74BA74DA74DA74FA74FA751A751A753A753" + "A755A755A757A757A759A759A75BA75BA75DA75DA75FA75FA761A761A763A763A765A765" + "A767A767A769A769A76BA76BA76DA76DA76FA76FA771A778A77AA77AA77CA77CA77FA77F" + "A781A781A783A783A785A785A787A787A78CA78CA78EA78EA791A791A793A795A797A797" + "A799A799A79BA79BA79DA79DA79FA79FA7A1A7A1A7A3A7A3A7A5A7A5A7A7A7A7A7A9A7A9" + "A7AFA7AFA7B5A7B5A7B7A7B7A7B9A7B9A7BBA7BBA7BDA7BDA7BFA7BFA7C1A7C1A7C3A7C3" + "A7C8A7C8A7CAA7CAA7CDA7CDA7D1A7D1A7D3A7D3A7D5A7D5A7D7A7D7A7D9A7D9A7DBA7DB" + "A7F6A7F6A7FAA7FAAB30AB5AAB60AB68AB70ABBFFB00FB06FB13FB17FF41FF5A" + )) + +(def tbl-space + (str + "0009000D002000200085008500A000A0168016802000200A20282029202F202F205F205F" + "30003000" + )) + +(def tbl-word + (str + "003000390041005A005F005F0061007A00AA00AA00B500B500BA00BA00C000D600D800F6" + "00F802C102C602D102E002E402EC02EC02EE02EE0300037403760377037A037D037F037F" + "038603860388038A038C038C038E03A103A303F503F7048104830487048A052F05310556" + "0559055905600588059105BD05BF05BF05C105C205C405C505C705C705D005EA05EF05F2" + "0610061A06200669066E06D306D506DC06DF06E806EA06FC06FF06FF0710074A074D07B1" + "07C007F507FA07FA07FD07FD0800082D0840085B0860086A087008870889088E089708E1" + "08E309020904093A093C093D09410948094D094D095009630966096F097109810985098C" + "098F0990099309A809AA09B009B209B209B609B909BC09BD09C109C409CD09CE09DC09DD" + "09DF09E309E609F109FC09FC09FE09FE0A010A020A050A0A0A0F0A100A130A280A2A0A30" + "0A320A330A350A360A380A390A3C0A3C0A410A420A470A480A4B0A4D0A510A510A590A5C" + "0A5E0A5E0A660A750A810A820A850A8D0A8F0A910A930AA80AAA0AB00AB20AB30AB50AB9" + "0ABC0ABD0AC10AC50AC70AC80ACD0ACD0AD00AD00AE00AE30AE60AEF0AF90AFF0B010B01" + "0B050B0C0B0F0B100B130B280B2A0B300B320B330B350B390B3C0B3D0B3F0B3F0B410B44" + "0B4D0B4D0B550B560B5C0B5D0B5F0B630B660B6F0B710B710B820B830B850B8A0B8E0B90" + "0B920B950B990B9A0B9C0B9C0B9E0B9F0BA30BA40BA80BAA0BAE0BB90BC00BC00BCD0BCD" + "0BD00BD00BE60BEF0C000C000C040C0C0C0E0C100C120C280C2A0C390C3C0C400C460C48" + "0C4A0C4D0C550C560C580C5A0C5D0C5D0C600C630C660C6F0C800C810C850C8C0C8E0C90" + "0C920CA80CAA0CB30CB50CB90CBC0CBD0CBF0CBF0CC60CC60CCC0CCD0CDD0CDE0CE00CE3" + "0CE60CEF0CF10CF20D000D010D040D0C0D0E0D100D120D3D0D410D440D4D0D4E0D540D56" + "0D5F0D630D660D6F0D7A0D7F0D810D810D850D960D9A0DB10DB30DBB0DBD0DBD0DC00DC6" + "0DCA0DCA0DD20DD40DD60DD60DE60DEF0E010E3A0E400E4E0E500E590E810E820E840E84" + "0E860E8A0E8C0EA30EA50EA50EA70EBD0EC00EC40EC60EC60EC80ECE0ED00ED90EDC0EDF" + "0F000F000F180F190F200F290F350F350F370F370F390F390F400F470F490F6C0F710F7E" + "0F800F840F860F970F990FBC0FC60FC61000102A102D1030103210371039103A103D1049" + "105010551058106110651066106E108210851086108D108E10901099109D109D10A010C5" + "10C710C710CD10CD10D010FA10FC1248124A124D1250125612581258125A125D12601288" + "128A128D129012B012B212B512B812BE12C012C012C212C512C812D612D8131013121315" + "1318135A135D135F1380138F13A013F513F813FD1401166C166F167F1681169A16A016EA" + "16F116F817001714171F1733174017531760176C176E177017721773178017B517B717BD" + "17C617C617C917D317D717D717DC17DD17E017E9180B180D180F181918201878188018AA" + "18B018F51900191E1920192219271928193219321939193B1946196D19701974198019AB" + "19B019C919D019D91A001A181A1B1A1B1A201A541A561A561A581A5E1A601A601A621A62" + "1A651A6C1A731A7C1A7F1A891A901A991AA71AA71AB01ABD1ABF1ACE1B001B031B051B34" + "1B361B3A1B3C1B3C1B421B421B451B4C1B501B591B6B1B731B801B811B831BA01BA21BA5" + "1BA81BA91BAB1BE61BE81BE91BED1BED1BEF1BF11C001C231C2C1C331C361C371C401C49" + "1C4D1C7D1C801C8A1C901CBA1CBD1CBF1CD01CD21CD41CE01CE21CF61CF81CFA1D001F15" + "1F181F1D1F201F451F481F4D1F501F571F591F591F5B1F5B1F5D1F5D1F5F1F7D1F801FB4" + "1FB61FBC1FBE1FBE1FC21FC41FC61FCC1FD01FD31FD61FDB1FE01FEC1FF21FF41FF61FFC" + "203F20402054205420712071207F207F2090209C20D020DC20E120E120E520F021022102" + "21072107210A2113211521152119211D212421242126212621282128212A212D212F2139" + "213C213F21452149214E214E218321842C002CE42CEB2CF32D002D252D272D272D2D2D2D" + "2D302D672D6F2D6F2D7F2D962DA02DA62DA82DAE2DB02DB62DB82DBE2DC02DC62DC82DCE" + "2DD02DD62DD82DDE2DE02DFF2E2F2E2F30053006302A302D30313035303B303C30413096" + "3099309A309D309F30A130FA30FC30FF3105312F3131318E31A031BF31F031FF34004DBF" + "4E00A48CA4D0A4FDA500A60CA610A62BA640A66FA674A67DA67FA6E5A6F0A6F1A717A71F" + "A722A788A78BA7CDA7D0A7D1A7D3A7D3A7D5A7DCA7F2A822A825A826A82CA82CA840A873" + "A882A8B3A8C4A8C5A8D0A8D9A8E0A8F7A8FBA8FBA8FDA92DA930A951A960A97CA980A982" + "A984A9B3A9B6A9B9A9BCA9BDA9CFA9D9A9E0A9FEAA00AA2EAA31AA32AA35AA36AA40AA4C" + "AA50AA59AA60AA76AA7AAA7AAA7CAA7CAA7EAAC2AADBAADDAAE0AAEAAAECAAEDAAF2AAF4" + "AAF6AAF6AB01AB06AB09AB0EAB11AB16AB20AB26AB28AB2EAB30AB5AAB5CAB69AB70ABE2" + "ABE5ABE5ABE8ABE8ABEDABEDABF0ABF9AC00D7A3D7B0D7C6D7CBD7FBF900FA6DFA70FAD9" + "FB00FB06FB13FB17FB1DFB28FB2AFB36FB38FB3CFB3EFB3EFB40FB41FB43FB44FB46FBB1" + "FBD3FD3DFD50FD8FFD92FDC7FDF0FDFBFE00FE0FFE20FE2FFE33FE34FE4DFE4FFE70FE74" + "FE76FEFCFF10FF19FF21FF3AFF3FFF3FFF41FF5AFF66FFBEFFC2FFC7FFCAFFCFFFD2FFD7" + "FFDAFFDC" + )) + +(def tbl-control + (str + "0000001F007F009F" + )) + +;; --------------------------------------------------------------------------- +;; The named sets +;; --------------------------------------------------------------------------- + +(def cs-alpha (table-set tbl-alpha)) +(def cs-digit (table-set tbl-digit)) +(def cs-alnum (table-set tbl-alnum)) +(def cs-upper (table-set tbl-upper)) +(def cs-lower (table-set tbl-lower)) +(def cs-space (table-set tbl-space)) +(def cs-word (table-set tbl-word)) +(def cs-control (table-set tbl-control)) + +(def cs-ascii (cs-make (list (Tuple 0 127)))) +(def cs-hex-digit (cs-make (list (Tuple 48 57) (Tuple 65 70) (Tuple 97 102)))) + +;; `:digit` is every Unicode decimal digit, which is what `numeric` means in +;; SRE and what `char-numeric?` answers. A program that wants the ten ASCII +;; ones usually means exactly that, so the narrowing has a name of its own +;; rather than being spelled `(& :digit :ascii)` at every use. +(def cs-ascii-digit (cs-inter cs-digit cs-ascii)) +(def cs-ascii-alpha (cs-inter cs-alpha cs-ascii)) +(def cs-ascii-alnum (cs-inter cs-alnum cs-ascii)) + +;; One scalar, whatever it is. `:nonl` is the same set without U+000A, which is +;; the only character either of them treats as a line break. +(def cs-any (cs-make universe)) +(def cs-nonl (cs-make (ranges-gaps 0 1114111 (list (Tuple 10 10))))) + +(: (-> string (Option Cset))) +(defun (named-class name) + (cond + ((= name "any") (Some cs-any)) + ((= name "nonl") (Some cs-nonl)) + ((= name "alpha") (Some cs-alpha)) + ((= name "alphabetic") (Some cs-alpha)) + ((= name "digit") (Some cs-digit)) + ((= name "numeric") (Some cs-digit)) + ((= name "alnum") (Some cs-alnum)) + ((= name "alphanumeric") (Some cs-alnum)) + ((= name "upper") (Some cs-upper)) + ((= name "upper-case") (Some cs-upper)) + ((= name "lower") (Some cs-lower)) + ((= name "lower-case") (Some cs-lower)) + ((= name "space") (Some cs-space)) + ((= name "whitespace") (Some cs-space)) + ((= name "word") (Some cs-word)) + ((= name "control") (Some cs-control)) + ((= name "ascii") (Some cs-ascii)) + ((= name "hex-digit") (Some cs-hex-digit)) + ((= name "ascii-digit") (Some cs-ascii-digit)) + ((= name "ascii-alpha") (Some cs-ascii-alpha)) + ((= name "ascii-alnum") (Some cs-ascii-alnum)) + (else None))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/scan.bjo — reading a line by a pattern, after OCaml's Scanf. +;; +;; `(scan line '(long "-" long))` reads two longs with a dash between them and +;; answers them as a tuple. The pattern is a quoted list, read when the macro +;; expands: the expansion is straight-line code over the line's bytes, one +;; step per item, with no pattern left to interpret at run time. The steps +;; are written #:inline, so each one is copied into the code that scans. A number step answers a value and the next position as a value +;; tuple, so a pattern of numbers allocates nothing. A text step answers only +;; the next position, and the value is a slice made after it, so text that +;; `(:skip ...)` drops allocates nothing either. +;; +;; The code is built in continuation-passing style: each item is given what to +;; do with its value and the position after it, and what to do when it does +;; not match. Where the rest of the pattern would be copied into two places — +;; after a repetition, an option or a choice — it is a local function instead, +;; a join point, so the expansion grows with the pattern and not faster. +(import (std prelude) (std syntax-match) (std interop)) + +(:doc #:file "scan.bjodoc") + +(export scan-lit scan-byte scan-lit8 scan-ws scan-int scan-long scan-double scan-char scan-word + scan-rest scan-chars scan-until scan-until-byte scan-slice scan-end? scan-fail scan-message) + +;; --------------------------------------------------------------------------- +;; The steps +;; --------------------------------------------------------------------------- + +;; What the expansion calls. They are exported, because a template reaches a +;; helper of its own module only through the module's exports. They are +;; #:inline, so an importer copies each step into the code that scans, and +;; the constant width that the macro gives is known there. +;; +;; A position is a byte offset into the line's UTF-8 bytes, from 0 to the +;; byte length. Each step takes the position it starts at and answers the +;; position after what it read, or -1 when the text there does not match. A +;; step that reads a number or a character answers it with the position, as +;; a tuple, so nothing is allocated. A step that reads text answers only the +;; position, and `scan-slice` makes the text when the pattern keeps it. +;; +;; A step that takes a width reads no more than that many characters, and +;; reads to the end of the line when the width is negative. +;; +;; A step never raises on text that does not match. The macro decides what a +;; failure does: `scan` calls `scan-fail`, and `try-scan` makes an `Err` with +;; `scan-message`. + +(import/class + (FormatException (: System.FormatException (-> string FormatException)))) + +;; The line's bytes are read-only, and a part of the line is made by +;; `SliceBytes`, which refuses an offset in the middle of a character. +(import/extern + (line-bytes (: BjoString.Utf8String.AsSpan (-> string (RoSpan byte)))) + (line-length (: BjoString.Utf8String.ByteLength (-> string int) #:get)) + (line-part (: BjoString.Utf8String.SliceBytes (-> string int int string))) + (bytes-part (: Bjolang.Runtime.Spans.Slice (-> (RoSpan byte) int int (RoSpan byte)))) + (bytes-start-with? (: System.MemoryExtensions.StartsWith (-> (RoSpan byte) (RoSpan byte) bool))) + (bytes-find (: System.MemoryExtensions.IndexOf (-> (RoSpan byte) byte int))) + (bytes-long (: System.Buffers.Binary.BinaryPrimitives.ReadInt64LittleEndian (-> (RoSpan byte) long))) + (bytes-double + (: System.Buffers.Text.Utf8Parser.TryParse + (-> (RoSpan byte) (out double) (out int) System.Char (Tuple bool double int))))) + +;; Whether a byte is whitespace, as a space in a pattern matches it: the +;; ASCII blanks. One unsigned compare tests a range, as one branch. +(: (-> int bool)) +(defun (blank? b) #:inline (or (= b 32) (<= (cast uint (- b 9)) 4u))) + +(: (-> (RoSpan byte) int int)) +(defun (byte-at b i) #:inline (cast int (ro-span-ref b i))) + +;; The end of `width` characters from `pos`, or the end of the bytes. +(: (-> (RoSpan byte) int int int)) +(defun (count-end b pos width) + (def n (ro-span-length b)) + (let go ((i pos) (k 0)) + (if (and (< k width) (< i n)) + (let ((c (byte-at b i))) + (go (+ i (cond ((< c 128) 1) ((>= c 240) 4) ((>= c 224) 3) (else 2))) (+ k 1))) + i))) + +;; The bytes of the line that a step with a width can read: all of them when +;; the width is negative. A loop that stops at their length tests each index +;; once, because .NET then knows it is in the span. +(: (-> string int int (RoSpan byte))) +(defun (within s pos width) #:inline + (def b (line-bytes s)) + (if (< width 0) b (bytes-part b 0 (count-end b pos width)))) + +;; The text of `literal`, exactly. +(: (-> string int string int)) +(defun (scan-lit s pos literal) #:inline + (def b (line-bytes s)) + (def want (line-bytes literal)) + (if (bytes-start-with? (bytes-part b pos (ro-span-length b)) want) + (+ pos (ro-span-length want)) + -1)) + +;; One ASCII byte, exactly. What a literal of one character compiles to. +(: (-> string int int int)) +(defun (scan-byte s pos code) #:inline + (def b (line-bytes s)) + (if (and (< pos (ro-span-length b)) (= (byte-at b pos) code)) (+ pos 1) -1)) + +;; A literal of 2 to 8 bytes, exactly. `bytes` holds the literal's bytes with +;; the first in the lowest byte, so that one 64-bit read and compare matches +;; it. +(: (-> string int long int int)) +(defun (scan-lit8 s pos bytes length) #:inline + (def b (line-bytes s)) + (def left (- (ro-span-length b) pos)) + (cond + ((< left length) -1) + ((< left 8) (lit-tail b pos bytes length)) + ((= (bitwise-and (bytes-long (bytes-part b pos (+ pos 8))) + (if (= length 8) -1L (- (shift-left 1L (* 8 length)) 1L))) + bytes) + (+ pos length)) + (else -1))) + +;; `scan-lit8` with fewer than 8 bytes left in the line. Out of line, so that +;; each literal of a pattern does not copy the loop. +(: (-> (RoSpan byte) int long int int)) +(defun (lit-tail b pos bytes length) + (let go ((i 0) (acc 0L)) + (cond + ((< i length) + (go (+ i 1) (bitwise-ior acc (shift-left (cast long (ro-span-ref b (+ pos i))) (* 8 i))))) + ((= acc bytes) (+ pos length)) + (else -1)))) + +;; Any amount of whitespace, none included. Never fails. +(: (-> string int int)) +(defun (scan-ws s pos) #:inline + (def b (line-bytes s)) + (def n (ro-span-length b)) + (let go ((i pos)) + (cond + ((>= i n) i) + ((blank? (byte-at b i)) (go (+ i 1))) + (else i)))) + +;; The digits from `first` to `stop`, more than 18 of them, as a number of +;; at most `hi`, or `hi + 1` when the sign is a minus. Out of line, because +;; most of such digits are leading zeros or too many. The digits are added up +;; below zero, where the least long fits, and each one is checked. +(: (-> (RoSpan byte) int int bool long (Tuple long int))) +(defun (long-digits b first stop negative? hi) + (def lo (if negative? (- (- 0L hi) 1L) (- 0L hi))) + (let go ((i first) (acc 0L)) + (cond + ((= i stop) (Tuple (if negative? acc (- 0L acc)) i)) + (else + (def d (cast long (- (byte-at b i) 48))) + (if (< acc (/ (+ lo d) 10L)) + (Tuple 0L -1) + (go (+ i 1) (- (* acc 10L) d))))))) + +;; An optional sign and one or more decimal digits from `pos`, as a number of +;; at most `hi`, or `hi + 1` when the sign is a minus, and the position after +;; them. The position is -1 when there is no digit or the number is too big. +;; Up to 18 digits cannot overflow a long, so the check comes after the loop. +(: (-> (RoSpan byte) int long (Tuple long int))) +(defun (read-digits b pos hi) #:inline + (def* (end (ro-span-length b)) + (c (if (< pos end) (byte-at b pos) 0)) + (negative? (= c 45)) + (first (if (or negative? (= c 43)) (+ pos 1) pos))) + (def (acc . stop) + (let go ((i first) (acc 0L)) + (if (< i end) + (let ((d (- (byte-at b i) 48))) + (if (<= (cast uint d) 9u) (go (+ i 1) (+ (* acc 10L) (cast long d))) (Tuple acc i))) + (Tuple acc i)))) + (cond + ((= stop first) (Tuple 0L -1)) + ((> (- stop first) 18) (long-digits b first stop negative? hi)) + ((> (if negative? (- acc 1L) acc) hi) (Tuple 0L -1)) + (negative? (Tuple (- 0L acc) stop)) + (else (Tuple acc stop)))) + +;; An int: an optional sign and decimal digits. A number too big for an int +;; does not match. +(: (-> string int int (Tuple int int))) +(defun (scan-int s pos width) #:inline + (def (v . q) (read-digits (within s pos width) pos 2147483647L)) + (Tuple (cast int v) q)) + +;; A long, as `scan-int` reads an int. +(: (-> string int int (Tuple long int))) +(defun (scan-long s pos width) #:inline + (read-digits (within s pos width) pos 9223372036854775807L)) + +;; A double, as `string->double` reads one: `1.5`, `-2`, `3e8`. +(: (-> string int int (Tuple double int))) +(defun (scan-double s pos width) #:inline + (def b (within s pos width)) + (let (((ok? v used) (bytes-double (bytes-part b pos (ro-span-length b)) (clr-default)))) + (if ok? (Tuple v (+ pos used)) (Tuple 0.0 -1)))) + +;; The low 6 bits of a continuation byte. +(: (-> (RoSpan byte) int int)) +(defun (continued b i) #:inline (bitwise-and (byte-at b i) 63)) + +;; The character that starts with the byte `c` at `pos`, which is not ASCII, +;; and the position after it. The line is valid UTF-8. +(: (-> (RoSpan byte) int int (Tuple char int))) +(defun (decode-char b pos c) + (cond + ((>= c 240) + (Tuple (int->char (bitwise-ior (bitwise-ior (shift-left (bitwise-and c 7) 18) + (shift-left (continued b (+ pos 1)) 12)) + (bitwise-ior (shift-left (continued b (+ pos 2)) 6) + (continued b (+ pos 3))))) + (+ pos 4))) + ((>= c 224) + (Tuple (int->char (bitwise-ior (bitwise-ior (shift-left (bitwise-and c 15) 12) + (shift-left (continued b (+ pos 1)) 6)) + (continued b (+ pos 2)))) + (+ pos 3))) + (else + (Tuple (int->char (bitwise-ior (shift-left (bitwise-and c 31) 6) (continued b (+ pos 1)))) + (+ pos 2))))) + +;; One character, whatever it is. +(: (-> string int (Tuple char int))) +(defun (scan-char s pos) #:inline + (def b (line-bytes s)) + (if (>= pos (ro-span-length b)) + (Tuple (int->char 0) -1) + (let ((c (byte-at b pos))) + (if (< c 128) (Tuple (int->char c) (+ pos 1)) (decode-char b pos c))))) + +;; One or more characters up to whitespace or the end. +(: (-> string int int int)) +(defun (scan-word s pos width) #:inline + (def b (within s pos width)) + (def end (ro-span-length b)) + (def stop + (let go ((i pos)) + (cond + ((>= i end) i) + ((blank? (byte-at b i)) i) + (else (go (+ i 1)))))) + (if (= stop pos) -1 stop)) + +;; The rest of the line, which may be empty. Never fails. +(: (-> string int int int)) +(defun (scan-rest s pos width) #:inline + (if (< width 0) (line-length s) (count-end (line-bytes s) pos width))) + +;; Whether an ASCII set holds a byte. The set is a 128-bit mask in two +;; halves: bit c of `low` is character c, and bit c - 64 of `high` is +;; character c from 64. A byte from 128 up is in no set. +(: (-> int long long bool)) +(defun (in-set? c low high) #:inline + (if (< c 64) + (not (= (bitwise-and (shift-right-logical low c) 1L) 0L)) + (and (< c 128) (not (= (bitwise-and (shift-right-logical high (- c 64)) 1L) 0L))))) + +;; One or more characters of an ASCII set. +(: (-> string int long long int int)) +(defun (scan-chars s pos low high width) #:inline + (def b (within s pos width)) + (def end (ro-span-length b)) + (def stop + (let go ((i pos)) + (cond + ((>= i end) i) + ((in-set? (byte-at b i) low high) (go (+ i 1))) + (else i)))) + (if (= stop pos) -1 stop)) + +;; Any characters up to one of an ASCII set or the end, none included. Never +;; fails. A character that is not ASCII is in no set, so the end is never +;; inside one. +(: (-> string int long long int int)) +(defun (scan-until s pos low high width) #:inline + (def b (within s pos width)) + (def end (ro-span-length b)) + (let go ((i pos)) + (cond + ((>= i end) i) + ((in-set? (byte-at b i) low high) i) + (else (go (+ i 1)))))) + +;; Any characters up to one ASCII byte or the end, none included. Never +;; fails. A vectorized search, which is what `scan-until` with one character +;; compiles to. +(: (-> string int int int int)) +(defun (scan-until-byte s pos stop width) #:inline + (def b (within s pos width)) + (def end (ro-span-length b)) + (def found (bytes-find (bytes-part b pos end) (cast byte stop))) + (if (< found 0) end (+ pos found))) + +;; The text from `from` to `to`, which shares the line's bytes. What a text +;; item's value is. +(: (-> string int int string)) +(defun (scan-slice s from to) #:inline (line-part s from (- to from))) + +;; Whether `pos` is the end of the line. +(: (-> string int bool)) +(defun (scan-end? s pos) #:inline (= pos (line-length s))) + +;; Why the line does not match: what was expected, at which column, and the +;; line. The column counts characters from 1. +(: (-> string int string string)) +(defun (scan-message s pos expected) + (def b (line-bytes s)) + (def column + (let go ((i 0) (k 1)) + (if (< i pos) + (go (+ i 1) (if (= (bitwise-and (byte-at b i) 192) 128) k (+ k 1))) + k))) + (str "scan: expected " expected " at column " (int->string column) " of \"" s "\"")) + +;; Raises a FormatException that says why the line does not match. +(: (-> string int string %a)) +(defun (scan-fail s pos expected) + (raise (cast Exception (FormatException. (scan-message s pos expected))))) + +;; --------------------------------------------------------------------------- +;; The pattern +;; --------------------------------------------------------------------------- + +;; One item of a pattern. An `(Option int)` is the item's width: the most +;; characters it reads. `ISet` is the characters of `(chars ...)`, or of +;; `(until ...)` when its flag is set, and the text an error says it +;; expected. +(type (: Item (Union (: ILit string) + IWs + (: IRead string (Option int)) + (: ISet string bool string (Option int)) + (: IGroup (List Item)) + (: IRep Item (Option Item)) + (: IOpt Item) + (: IOr (List Item)) + (: ISkip Item)))) + +;; Whether the item reads a value, which the result then holds. +(: (-> Item bool)) +(defun (valued? item) + (match item + ((ILit _) #f) + (IWs #f) + ((IRead _ _) #t) + ((ISet _ _ _ _) #t) + ((IGroup items) (any? valued? items)) + ((IRep _ _) #t) + ((IOpt inner) (valued? inner)) + ((IOr alts) (valued? (list-head alts))) + ((ISkip _) #f))) + +;; What an error says the item was expected to be. +(: (-> Item string)) +(defun (describe item) + (match item + ((ILit t) (str "\"" t "\"")) + (IWs "whitespace") + ((IRead kind _) + (case kind + (("int") "an int") + (("long") "a long") + (("double") "a double") + (("char") "a character") + (("word") "a word") + (else "the rest of the line"))) + ((ISet _ _ what _) what) + ((IGroup items) (if (list-empty? items) "nothing" (describe (list-head items)))) + ((IRep inner _) (describe inner)) + ((IOpt inner) (describe inner)) + ((IOr alts) (str "one of " (string-join (list->vec (list-map describe alts)) ", "))) + ((ISkip inner) (describe inner)))) + +(: (-> char bool)) +(defun (space? c) (or (char=? c #\space) (char=? c #\tab))) + +;; A string of a pattern: the text between its spaces, each exactly, and a +;; run of spaces as any amount of whitespace. +(: (-> string (List Item))) +(defun (string-items text) + (defun (flush lit acc) + (if (list-empty? lit) acc (Cons (ILit (list->string (list-reverse lit))) acc))) + (defun (skip cs) + (match cs + ((Cons c rest) #:when (space? c) (skip rest)) + (_ cs))) + (let go ((cs (string->list text)) (lit Nil) (acc Nil)) + (match cs + (Nil (list-reverse (flush lit acc))) + ((Cons c rest) + (cond ((space? c) + (go (skip rest) Nil (Cons IWs (flush lit acc)))) + (else (go rest (Cons c lit) acc))))))) + +;; The bit of an ASCII character in a set's two halves. +(: (-> int (Tuple long long))) +(defun (char-bit code) + (if (< code 64) + (Tuple (shift-left 1L code) 0L) + (Tuple 0L (shift-left 1L (- code 64))))) + +;; The characters a `(chars ...)` keyword names. +(: (-> string (Option string))) +(defun (char-class name) + (case name + (("digit") (Some "0123456789")) + (("lower") (Some "abcdefghijklmnopqrstuvwxyz")) + (("upper") (Some "ABCDEFGHIJKLMNOPQRSTUVWXYZ")) + (("alpha") (Some "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")) + (("alnum") (Some "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")) + (("hex") (Some "0123456789abcdefABCDEF")) + (else None))) + +;; A set of ASCII characters as the two halves `in-set?` tests a byte +;; against. +(: (-> string (Tuple long long))) +(defun (mask text) + (string-fold + (fun (c acc) + (def (l . h) (char-bit (char->int c))) + (Tuple (bitwise-ior (fst acc) l) (bitwise-ior (snd acc) h))) + (Tuple 0L 0L) text)) + +;; The characters of a string of a set, with `a-z` as all the characters from +;; `a` to `z`. A `-` at the start or the end is itself. +(: (-> string string Syntax string)) +(defun (with-ranges text form at) + (defun (span from to) + (let go ((c to) (acc Nil)) + (if (< c from) (list->string acc) (go (- c 1) (Cons (int->char c) acc))))) + (let go ((cs (string->list text)) (acc "")) + (match cs + ((Cons a (Cons dash (Cons b rest))) #:when (char=? dash #\-) + (def* (from (char->int a)) + (to (char->int b))) + (when (> from to) + (ignore (ret-error at (str form " has the range " (char->string a) "-" (char->string b) ", which goes backwards.")))) + (go rest (str acc (span from to)))) + ((Cons c rest) (go rest (str acc (char->string c)))) + (Nil acc)))) + +;; `(chars ".#")`, `(chars "a-z" :digit)` and the same for `(until ...)`: +;; the characters of the strings and the classes. `form` names the item in +;; errors. +(: (-> string bool (List Syntax) Syntax Item)) +(defun (set-item form until? parts whole) + (def text + (fold + (fun (part acc) + (match part + ((SStr s) (str acc (with-ranges s form part))) + ((SKey k) + (match (char-class (displayed k)) + ((Some cs) (str acc cs)) + (None (ret-error part (str form " knows the classes :digit, :lower, :upper, :alpha, :alnum and :hex, not :" (displayed k) "."))))) + (_ (ret-error part (str form " takes strings of characters and class keywords, such as (chars \".#\"), (chars \"a-z\" \"_\") or (until \",\")."))))) + "" parts)) + (when (string-empty? text) + (ignore (ret-error whole (str form " needs at least one character.")))) + (string-for-each + (fun (c) + (when (>= (char->int c) 128) + (ignore (ret-error whole (str form " takes ASCII characters only, and " (char->string c) " is not one."))))) + text) + (def written + (string-join + (list->vec (list-map (fun (part) + (match part + ((SStr s) (str "\"" s "\"")) + ((SKey k) (str ":" (displayed k))) + (_ ""))) + parts)) + " ")) + (ISet text until? (if until? (str "text up to one of " written) (str "one of " written)) None)) + +;; A syntax error, as a value of any type: the transformer raises it, and the +;; expander reports it at the form it names. +(: (-> Syntax string %a)) +(defun (ret-error at message) + (ignore (syntax-error at message)) + (panic! "unreachable: syntax-error does not return")) + +(: (-> Syntax string bool)) +(defun (keyword? s name) + (match s + ((SKey k) (= (displayed k) name)) + (_ #f))) + +(def items-are + "The items are strings, int, long, double, char, word, rest, (chars ...), (until ...), a group (...), (:? ...), (:or ...) and (:skip ...).") + +;; One item, without what follows it. +(: (-> Syntax Item)) +(defun (parse-item s) + (match s + ((SStr text) + (match (string-items text) + ((Cons only Nil) only) + (items (IGroup items)))) + ((SSym name) + (def kind (symbol->string name)) + (if (any? #(= & kind) (list "int" "long" "double" "char" "word" "rest")) + (IRead kind None) + (ret-error s (str "'" kind "' is not an item of a scan pattern. " items-are)))) + ((SList (Cons head rest)) + (cond ((keyword? head "?") + (IOpt (group-of (parse-items rest s)))) + ((keyword? head "or") (or-item rest s)) + ((keyword? head "skip") + (ISkip (group-of (parse-items rest s)))) + ((named? head "chars") (set-item "(chars ...)" #f rest s)) + ((named? head "until") (set-item "(until ...)" #t rest s)) + (else (IGroup (parse-items (Cons head rest) s))))) + ((SList Nil) (IGroup Nil)) + (_ (ret-error s (str "This is not an item of a scan pattern. " items-are))))) + +(: (-> Syntax string bool)) +(defun (named? s name) + (match s + ((SSym n) (= (symbol->string n) name)) + (_ #f))) + +;; `item #:width n`: the item reads no more than n characters. +(: (-> Item int Syntax Item)) +(defun (with-width item n at) + (def message "#:width goes after int, long, double, word, rest, (chars ...) or (until ...).") + (match item + ((IRead _ (Some _)) (ret-error at "This item has a #:width already.")) + ((ISet _ _ _ (Some _)) (ret-error at "This item has a #:width already.")) + ((IRead kind None) + (if (= kind "char") (ret-error at message) (IRead kind (Some n)))) + ((ISet text until? what None) (ISet text until? what (Some n))) + (_ (ret-error at message)))) + +;; `(:or a b ...)`: each choice is one item, and they all read a value or +;; none does. +(: (-> (List Syntax) Syntax Item)) +(defun (or-item choices whole) + (def alts (list-map (fun (a) (group-of (parse-items (list a) a))) choices)) + (when (list-empty? alts) + (ignore (ret-error whole "(:or ...) needs at least one choice."))) + (when (not (all? #(= (valued? &) (valued? (list-head alts))) alts)) + (ignore (ret-error whole "(:or ...) needs every choice to read a value, or none of them."))) + (IOr alts)) + +;; A list of items as one item. +(: (-> (List Item) Item)) +(defun (group-of items) + (match items + ((Cons only Nil) only) + (_ (IGroup items)))) + +;; The items of a list, with `item ...` and `item ... #:sep sep` read as +;; repetitions of the item before them, and `item #:width n` as the item +;; with a width. +(: (-> (List Syntax) Syntax (List Item))) +(defun (parse-items xs whole) + (let go ((xs xs) (acc Nil)) + (match xs + (Nil (list-reverse acc)) + ((Cons k (Cons (SInt digits) more)) #:when (and (keyword? k "width") (> (string->int digits) 0)) + (match acc + (Nil (ret-error k "#:width gives a width to the item before it, and there is none.")) + ((Cons prev before) (go more (Cons (with-width prev (string->int digits) k) before))))) + ((Cons k _) #:when (keyword? k "width") + (ret-error k "#:width needs a number of characters after it, as in int #:width 4.")) + ((Cons (SPunct "...") rest) + (match acc + (Nil (ret-error whole "A '...' repeats the item before it, and there is none.")) + ((Cons prev before) + (when (not (valued? prev)) + (ignore (ret-error whole "A '...' repeats an item that reads a value, and a string reads none. Put the string in a group with something that does, as in (\"(\" int \")\") ..."))) + (match rest + ((Cons k (Cons sep more)) #:when (keyword? k "sep") + (go more (Cons (IRep prev (Some (parse-item sep))) before))) + ((Cons k Nil) #:when (keyword? k "sep") + (ret-error whole "#:sep needs the separator after it, as in int ... #:sep \",\".")) + (_ (go rest (Cons (IRep prev None) before))))))) + ((Cons x rest) (go rest (Cons (parse-item x) acc)))))) + +;; --------------------------------------------------------------------------- +;; The expansion +;; --------------------------------------------------------------------------- + +;; The line's name in the expansion, and a counter for the expansion's own +;; temporaries. Each temporary is built from a string, so that each one gets a +;; name of its own: hygiene gives every distinct name one fresh spelling per +;; expansion, and one template name used at every depth would shadow itself. +;; `keep` is false inside `(:skip ...)`: there an item gives no value, so a +;; text item makes no slice and a repetition no vec. +(type (: Ctx (Record (: line Syntax) (: counter (Array int)) (: keep bool)))) + +;; Whether the item gives a value here. +(: (-> Ctx Item bool)) +(defun (holds? ctx item) (and (.keep ctx) (valued? item))) + +(: (-> Ctx string Syntax)) +(defun (fresh ctx prefix) + (def n (ref (.counter ctx) 0)) + (array-set! (.counter ctx) 0 (+ n 1)) + (SSym (string->symbol (str "scan-" prefix (displayed n))))) + +;; What an item does with its value and the position after it, and what it +;; does when it does not match. +(type (: OnValue (-> (Option Syntax) Syntax Syntax))) +(type (: OnMiss (-> Syntax string Syntax))) + +;; A width as the steps take it: -1 for none. +(: (-> (Option int) Syntax)) +(defun (width-literal width) + (match width + ((Some n) (SInt (int->string n))) + (None (SInt "-1")))) + +(: (-> int Syntax)) +(defun (int-literal n) (SInt (int->string n))) + +(: (-> long Syntax)) +(defun (long-literal n) (SInt (str (long->string n) "L"))) + +;; Up to 8 byte values as one long, the first in the lowest byte, as +;; `scan-lit8` reads them. +(: (-> (List int) long)) +(defun (packed codes) + (list-foldr (fun (code acc) (bitwise-ior (shift-left acc 8) (cast System.Int64 code))) 0L codes)) + +;; Code for `item` at position `p`. +(: (-> Ctx Item Syntax OnValue OnMiss Syntax)) +(defun (gen ctx item p ok fail) + (def s (.line ctx)) + (match item + ((ILit text) + (def q (fresh ctx "q")) + (def codes (list-map char->int (string->list text))) + (def n (list-length codes)) + (def step + (cond ((not (all? #(< & 128) codes)) #'(scan-lit ,s ,p ,(SStr text))) + ((= n 1) #'(scan-byte ,s ,p ,(int-literal (list-head codes)))) + ((<= n 8) #'(scan-lit8 ,s ,p ,(long-literal (packed codes)) ,(int-literal n))) + (else #'(scan-lit ,s ,p ,(SStr text))))) + #'(let ((,q ,step)) + (if (< ,q 0) ,(fail p (describe item)) ,(ok None q)))) + (IWs + (def q (fresh ctx "q")) + #'(let ((,q (scan-ws ,s ,p))) ,(ok None q))) + ((IRead kind width) + (def w (width-literal width)) + (case kind + (("int") (gen-number ctx item #'(scan-int ,s ,p ,w) p ok fail)) + (("long") (gen-number ctx item #'(scan-long ,s ,p ,w) p ok fail)) + (("double") (gen-number ctx item #'(scan-double ,s ,p ,w) p ok fail)) + (("char") (gen-number ctx item #'(scan-char ,s ,p) p ok fail)) + (("word") (gen-text ctx item #'(scan-word ,s ,p ,w) p ok fail)) + (else (gen-text ctx item #'(scan-rest ,s ,p ,w) p ok fail)))) + ((ISet text until? _ width) + (def w (width-literal width)) + (def (low . high) (mask text)) + (def step + (cond ((not until?) #'(scan-chars ,s ,p ,(long-literal low) ,(long-literal high) ,w)) + ((= (string-length text) 1) + #'(scan-until-byte ,s ,p ,(int-literal (char->int (list-head (string->list text)))) ,w)) + (else #'(scan-until ,s ,p ,(long-literal low) ,(long-literal high) ,w)))) + (gen-text ctx item step p ok fail)) + ((IGroup items) (gen-seq ctx items p Nil ok fail)) + ((IOpt inner) (gen-opt ctx inner p ok)) + ((IOr alts) (gen-or ctx alts item p ok fail)) + ((IRep inner sep) (gen-rep ctx inner sep p ok)) + ((ISkip inner) + (gen (Ctx (line s) (counter (.counter ctx)) (keep #f)) inner p + (fun (_ q) (ok None q)) fail)))) + +;; A step that answers a value and the position after it. +(: (-> Ctx Item Syntax Syntax OnValue OnMiss Syntax)) +(defun (gen-number ctx item step p ok fail) + (def* (v (fresh ctx "v")) + (q (fresh ctx "q"))) + #'(match ,step + ((,v . ,q) (if (< ,q 0) ,(fail p (describe item)) ,(ok (if (.keep ctx) (Some v) None) q))))) + +;; A step that answers the position after its text. The value is the slice +;; from `p` to there, made only where the value is used. +(: (-> Ctx Item Syntax Syntax OnValue OnMiss Syntax)) +(defun (gen-text ctx item step p ok fail) + (def* (s (.line ctx)) + (q (fresh ctx "q"))) + #'(let ((,q ,step)) + (if (< ,q 0) + ,(fail p (describe item)) + ,(ok (if (.keep ctx) (Some #'(scan-slice ,s ,p ,q)) None) q)))) + +;; Code for items in order. A group's value is its one value, or a tuple of +;; its values, or none. +(: (-> Ctx (List Item) Syntax (List Syntax) OnValue OnMiss Syntax)) +(defun (gen-seq ctx items p vals ok fail) + (match items + (Nil + (match (list-reverse vals) + (Nil (ok None p)) + ((Cons only Nil) (ok (Some only) p)) + (all (ok (Some #'(Tuple ,@all)) p)))) + ((Cons item rest) + (gen ctx item p + (fun (v q) + (gen-seq ctx rest q (match v ((Some x) (Cons x vals)) (None vals)) ok fail)) + fail)))) + +;; A join point: a local function `k` that runs `ok` on its arguments, so +;; that several paths can continue with one copy of it. `valued?` says +;; whether it takes a value before the position. +(: (-> Ctx bool OnValue (-> (-> (Option Syntax) Syntax Syntax) Syntax) Syntax)) +(defun (join ctx valued? ok body) + (def* (k (fresh ctx "k")) + (kv (fresh ctx "kv")) + (kp (fresh ctx "kp"))) + (if valued? + #'(let () + (defun (,k ,kv ,kp) ,(ok (Some kv) kp)) + ,(body (fun (v q) #'(,k ,(option-ref v) ,q)))) + #'(let () + (defun (,k ,kp) ,(ok None kp)) + ,(body (fun (v q) #'(,k ,q)))))) + +;; `(:? item)`: the item's value in a `Some`, or `None` and no text read. +(: (-> Ctx Item Syntax OnValue Syntax)) +(defun (gen-opt ctx inner p ok) + (def valued (holds? ctx inner)) + (join ctx valued ok + (fun (k) + (gen ctx inner p + (fun (v q) (k (option-map (fun (x) #'(Some ,x)) v) q)) + (fun (_ _) (k (if valued (Some #'None) None) p)))))) + +;; `(:or a b ...)`: the first choice that matches, each tried at `p`. +(: (-> Ctx (List Item) Item Syntax OnValue OnMiss Syntax)) +(defun (gen-or ctx alts whole p ok fail) + (join ctx (valued? whole) ok + (fun (k) + (let next ((alts alts)) + (match alts + (Nil (fail p (describe whole))) + ((Cons a rest) (gen ctx a p k (fun (_ _) (next rest))))))))) + +;; `item ...` and `item ... #:sep sep`: the items as a vec, as many as match, +;; none included. A separator not followed by an item is not read. The loop +;; is a named let, so it runs as a loop however many items there are. Where +;; the value is not kept, the loop only moves past the items. +(: (-> Ctx Item (Option Item) Syntax OnValue Syntax)) +(defun (gen-rep ctx inner sep p ok) + ;; The loop reads an item at `at`, and stops at `back` when the item does + ;; not match: `back` is the position before the separator, so that a + ;; separator not followed by an item is not read. The item and the + ;; separator are each written once, so nested repetitions grow with the + ;; pattern. + (def* (keep (.keep ctx)) + (acc (fresh ctx "acc")) + (back (fresh ctx "back")) + (at (fresh ctx "at")) + (more (fresh ctx "more"))) + (join ctx keep ok + (fun (k) + (defun (stop pos) (k (if keep (Some #'(vecbuilder->vec ,acc)) None) pos)) + ;; After the separator at `q`, or none: round again, or stop at `q`. + (defun (next q) + (match sep + (None #'(,more ,q ,q)) + ((Some sep-item) + (gen ctx sep-item q (fun (_ sq) #'(,more ,q ,sq)) (fun (_ _) (stop q)))))) + ;; After an item: add its value and go on from `q`. A round that + ;; reads no text, as `rest` at the end does, ends the repetition + ;; without its value, because it would match forever. + (defun (add v q) + #'(if (= ,q ,back) + ,(stop back) + ,(if keep + #'(let () + (vecbuilder-add! ,acc ,(option-ref v)) + ,(next q)) + (next q)))) + (def loop + #'(let ,more ((,back ,p) (,at ,p)) + ,(gen ctx inner at add (fun (_ _) (stop back))))) + (if keep + #'(let ((,acc (vecbuilder-empty/small))) ,loop) + loop)))) + +;; The whole expansion: the items from position 0, then the end of the line. +(: (-> Syntax Syntax Syntax bool Syntax)) +(defun (expand form line pattern raising?) + (def quoted + (match pattern + ((SList (Cons (SSym q) items)) + (if (= (symbol->string q) "quoted-list") (Some items) None)) + (_ None))) + (def items + (match quoted + ((Some items) (parse-items items pattern)) + (None (ret-error pattern "The pattern of a scan is a quoted list, such as '(int \",\" int).")))) + (def ctx (Ctx (line (SSym (string->symbol "scan-line"))) (counter (make-array 1 #:fill 0)) (keep #t))) + (def s (.line ctx)) + (defun (fail p expected) + (if raising? + #'(scan-fail ,s ,p ,(SStr expected)) + #'(Err (scan-message ,s ,p ,(SStr expected))))) + (defun (finish v p) + (def value (match v ((Some x) x) (None #'unit))) + #'(if (scan-end? ,s ,p) + ,(if raising? value #'(Ok ,value)) + ,(fail p "the end of the line"))) + #'(let ((,s ,line)) + ,(gen-seq ctx items #'0 Nil finish fail))) + +;; `(scan line '(pattern ...))`: the values the pattern reads, or raises a +;; FormatException that says where the line does not match. +(def/macro (scan form inject compare) + (syntax-match form + ((_ line pattern) (expand form line pattern #t)) + (bad (syntax-error bad "scan takes a line and a quoted pattern, as in (scan line '(int \",\" int)).")))) + +;; `(try-scan line '(pattern ...))`: the same values in an `Ok`, or an `Err` +;; with the message `scan` would raise. +(def/macro (try-scan form inject compare) + (syntax-match form + ((_ line pattern) (expand form line pattern #f)) + (bad (syntax-error bad "try-scan takes a line and a quoted pattern, as in (try-scan line '(int \",\" int)).")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + + +;; std/simd.bjo — vectors of numbers, a fixed number of lanes wide. +;; +;; A lane type is a .NET `Vector128` or `Vector256`: 128 or 256 bits of +;; numbers of one type. `Float64x4` is four doubles and `Int8x32` is 32 signed +;; bytes. An operation on one is one CPU instruction where the CPU has it. +;; +;; The operations are the methods of a few traits, which both widths +;; implement: `Simd` for what every element type has, `SimdFloat` and +;; `SimdInt` for what only floats or only integers have, `SimdBitwise` for +;; vectors and masks alike, and `SimdMask` for masks. So `simd-add` adds two +;; `Float64x2`s or two `Int32x8`s. What makes a vector — `simd256-load`, +;; `simd128-splat` — says its width, as there is no argument to take it from, +;; and so does a conversion, whose result is a vector of another type. +;; +;; A comparison gives a mask, `(Mask128 %a)` or `(Mask256 %a)`, which is a +;; type of its own: it can be combined, counted and selected with, but not +;; added up as numbers. +;; +;; The design follows OxCaml's SIMD library where that fits a language with +;; traits and unsigned types: typed masks, operations only where they mean +;; something, and conversions named for what they do with what does not fit. +;; +;; A 128-bit vector is fast on every x86-64 and ARM64 CPU. A 256-bit one needs +;; AVX2: elsewhere .NET emulates it, which is much slower than scalar code. +;; `simd256-accelerated?` says which. A Native AOT build has AVX2 only with +;; `(aot-cpu ...)`. +(import (std prelude) + (std syntax-match) + (std mutable vec)) + +(:doc #:file "simd.bjodoc") + +(export + Simd128 Simd256 Mask128 Mask256 + Float64x2 Float32x4 Int64x2 Int32x4 Int16x8 Int8x16 + UInt64x2 UInt32x4 UInt16x8 UInt8x16 + Float64x4 Float32x8 Int64x4 Int32x8 Int16x16 Int8x32 + UInt64x4 UInt32x8 UInt16x16 UInt8x32 + ;; The element types + Lane IntLane FloatLane + ;; What every vector has + Simd + simd-add simd-sub simd-mul simd-scale simd-neg simd-abs simd-min simd-max + simd-eq simd-ne simd-lt simd-le simd-gt simd-ge simd-select + simd-sum simd-dot simd-ref simd-with simd-store! simd-lanes + ;; Bits, of vectors and masks + SimdBitwise + simd-and simd-or simd-xor simd-not simd-and-not + ;; Masks + SimdMask + simd-any? simd-all? simd-count-true simd-first-true simd-mask-bits simd-mask->lanes + ;; Floats only + SimdFloat + simd-div simd-sqrt simd-floor simd-ceiling simd-round simd-truncate simd-nan? simd-mul-add + ;; Integers only + SimdInt + simd-shift-left simd-shift-right simd-shift-right-logical + simd-add-saturating simd-sub-saturating simd-shuffle + ;; Byte tables + ByteLane SimdLookup simd-lookup + ;; The same bits as another element type + SimdBits + simd-bits->int8 simd-bits->uint8 simd-bits->int16 simd-bits->uint16 + simd-bits->int32 simd-bits->uint32 simd-bits->int64 simd-bits->uint64 + simd-bits->float32 simd-bits->float64 + ;; Conversions + SimdWiden SimdNarrow SimdIntToFloat SimdFloatToInt + simd128-widen-lower simd128-widen-upper simd256-widen-lower simd256-widen-upper + simd128-narrow-wrapping simd128-narrow-saturating + simd256-narrow-wrapping simd256-narrow-saturating + simd128-int->float simd256-int->float + simd128-float->int-truncating simd256-float->int-truncating + simd128-float->uint-truncating simd256-float->uint-truncating + ;; Where vectors are loaded from + SimdSource simd-source-length + ;; Making one + simd128-splat simd128-zero simd128-of simd128-load simd128-lanes simd128-whole + simd256-splat simd256-zero simd256-of simd256-load simd256-lanes simd256-whole + simd128-accelerated? simd256-accelerated?) + +(import/class + ((Simd128 %a) (: System.Runtime.Intrinsics.Vector128)) + ((Simd256 %a) (: System.Runtime.Intrinsics.Vector256))) + +;; A comparison's answer: a vector of the same type, with every bit of a lane +;; set where the comparison holds and none where it does not, kept apart from +;; the vectors so that it is not mistaken for numbers. Only this module makes +;; one. The JIT keeps it in a register as it keeps the vector. +(type (: (Mask128 %a) #:opaque (Struct (: lanes (Simd128 %a))))) +(type (: (Mask256 %a) #:opaque (Struct (: lanes (Simd256 %a))))) + +;; --------------------------------------------------------------------------- +;; The lane types +;; --------------------------------------------------------------------------- +;; +;; Named as OxCaml names them: the element's type and bits, and the lanes. + +(type (: Float64x2 (Simd128 double))) +(type (: Float32x4 (Simd128 System.Single))) +(type (: Int64x2 (Simd128 long))) +(type (: Int32x4 (Simd128 int))) +(type (: Int16x8 (Simd128 short))) +(type (: Int8x16 (Simd128 System.SByte))) +(type (: UInt64x2 (Simd128 ulong))) +(type (: UInt32x4 (Simd128 uint))) +(type (: UInt16x8 (Simd128 ushort))) +(type (: UInt8x16 (Simd128 byte))) + +(type (: Float64x4 (Simd256 double))) +(type (: Float32x8 (Simd256 System.Single))) +(type (: Int64x4 (Simd256 long))) +(type (: Int32x8 (Simd256 int))) +(type (: Int16x16 (Simd256 short))) +(type (: Int8x32 (Simd256 System.SByte))) +(type (: UInt64x4 (Simd256 ulong))) +(type (: UInt32x8 (Simd256 uint))) +(type (: UInt16x16 (Simd256 ushort))) +(type (: UInt8x32 (Simd256 byte))) + +;; --------------------------------------------------------------------------- +;; The element types +;; --------------------------------------------------------------------------- +;; +;; .NET's vectors hold the primitive numbers below and nothing else: a vector +;; of another type is refused when it is made, at run time. `Lane` says which +;; types they are, so a vector of a char or a string is refused when it is +;; compiled. `IntLane` and `FloatLane` divide them, for the operations that +;; mean something for one kind only. None has methods, and nothing outside +;; this module implements them. + +(def/trait (Lane %a)) +(impl (Lane double)) +(impl (Lane System.Single)) +(impl (Lane long)) +(impl (Lane int)) +(impl (Lane short)) +(impl (Lane System.SByte)) +(impl (Lane ulong)) +(impl (Lane uint)) +(impl (Lane ushort)) +(impl (Lane byte)) + +(def/trait (IntLane %a)) +(impl (IntLane long)) +(impl (IntLane int)) +(impl (IntLane short)) +(impl (IntLane System.SByte)) +(impl (IntLane ulong)) +(impl (IntLane uint)) +(impl (IntLane ushort)) +(impl (IntLane byte)) + +(def/trait (FloatLane %a)) +(impl (FloatLane double)) +(impl (FloatLane System.Single)) + +;; The element type `simd-lookup` takes its indices in. +(def/trait (ByteLane %a)) +(impl (ByteLane byte)) + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- + +;; What every vector has. A comparison gives a `%mask`, which `simd-select` +;; takes: a lane from its second argument where the mask is set, and from its +;; third where it is not. +(def/trait (Simd %v) + (type %elem) + (type %mask) + (: simd-add (-> %v %v %v)) + (: simd-sub (-> %v %v %v)) + (: simd-mul (-> %v %v %v)) + (: simd-scale (-> %v %elem %v)) + (: simd-neg (-> %v %v)) + (: simd-abs (-> %v %v)) + (: simd-min (-> %v %v %v)) + (: simd-max (-> %v %v %v)) + (: simd-eq (-> %v %v %mask)) + (: simd-ne (-> %v %v %mask)) + (: simd-lt (-> %v %v %mask)) + (: simd-le (-> %v %v %mask)) + (: simd-gt (-> %v %v %mask)) + (: simd-ge (-> %v %v %mask)) + (: simd-select (-> %mask %v %v %v)) + (: simd-sum (-> %v %elem)) + (: simd-dot (-> %v %v %elem)) + (: simd-ref (-> %v int %elem)) + (: simd-with (-> %v int %elem %v)) + (: simd-store! (-> %v (Array %elem) int void)) + (: simd-lanes (-> %v int))) + +;; The bitwise operations, which vectors and masks both have. Two masks +;; combine into a mask. +(def/trait (SimdBitwise %v) + (: simd-and (-> %v %v %v)) + (: simd-or (-> %v %v %v)) + (: simd-xor (-> %v %v %v)) + (: simd-not (-> %v %v)) + (: simd-and-not (-> %v %v %v))) + +;; Reading a mask. A lane is set when every bit of it is, and the top bit says +;; it, so each reader is one move-mask instruction and an integer operation. +(def/trait (SimdMask %m) + (type %vec) + (: simd-any? (-> %m bool)) + (: simd-all? (-> %m bool)) + (: simd-count-true (-> %m int)) + (: simd-first-true (-> %m int)) + (: simd-mask-bits (-> %m int)) + (: simd-mask->lanes (-> %m %vec))) + +;; What only float lanes have. Integer lanes divide one lane at a time, with +;; no instruction for it, and raise on a zero; that is not offered as a +;; vector operation. +(def/trait (SimdFloat %v) + (type %mask) + (: simd-div (-> %v %v %v)) + (: simd-sqrt (-> %v %v)) + (: simd-floor (-> %v %v)) + (: simd-ceiling (-> %v %v)) + (: simd-round (-> %v %v)) + (: simd-truncate (-> %v %v)) + (: simd-nan? (-> %v %mask)) + (: simd-mul-add (-> %v %v %v %v))) + +;; What only integer lanes have. A shift of a float lane would shift the bits +;; of its encoding. A shuffle's indices are of the vector's own type, which +;; for a float vector would be another type; a float vector is shuffled as +;; the same bits read as integers, with `simd-bits->int64` and back. +(def/trait (SimdInt %v) + (: simd-shift-left (-> %v int %v)) + (: simd-shift-right (-> %v int %v)) + (: simd-shift-right-logical (-> %v int %v)) + (: simd-add-saturating (-> %v %v %v)) + (: simd-sub-saturating (-> %v %v %v)) + (: simd-shuffle (-> %v %v %v))) + +;; A lookup in a table of 16 bytes, one for each byte of the indices. +(def/trait (SimdLookup %v) + (: simd-lookup (-> (Simd128 byte) %v %v))) + +;; The same bits, read as lanes of another type. The lanes are as many as the +;; bits allow: an `Int64x2` is a `UInt8x16`. +(def/trait (SimdBits %v) + (type %int8) (type %uint8) (type %int16) (type %uint16) (type %int32) + (type %uint32) (type %int64) (type %uint64) (type %float32) (type %float64) + (: simd-bits->int8 (-> %v %int8)) + (: simd-bits->uint8 (-> %v %uint8)) + (: simd-bits->int16 (-> %v %int16)) + (: simd-bits->uint16 (-> %v %uint16)) + (: simd-bits->int32 (-> %v %int32)) + (: simd-bits->uint32 (-> %v %uint32)) + (: simd-bits->int64 (-> %v %int64)) + (: simd-bits->uint64 (-> %v %uint64)) + (: simd-bits->float32 (-> %v %float32)) + (: simd-bits->float64 (-> %v %float64))) + +;; --------------------------------------------------------------------------- +;; 128 bits +;; --------------------------------------------------------------------------- + +(import/extern + (v128-add (: System.Runtime.Intrinsics.Vector128.Add (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-sub (: System.Runtime.Intrinsics.Vector128.Subtract (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-mul (: System.Runtime.Intrinsics.Vector128.Multiply (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-div (: System.Runtime.Intrinsics.Vector128.Divide (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-scale (: System.Runtime.Intrinsics.Vector128.Multiply (-> (Simd128 %a) %a (Simd128 %a)))) + (v128-neg (: System.Runtime.Intrinsics.Vector128.Negate (-> (Simd128 %a) (Simd128 %a)))) + (v128-abs (: System.Runtime.Intrinsics.Vector128.Abs (-> (Simd128 %a) (Simd128 %a)))) + (v128-min (: System.Runtime.Intrinsics.Vector128.Min (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-max (: System.Runtime.Intrinsics.Vector128.Max (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-sqrt (: System.Runtime.Intrinsics.Vector128.Sqrt (-> (Simd128 %a) (Simd128 %a)))) + (v128-and (: System.Runtime.Intrinsics.Vector128.BitwiseAnd (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-or (: System.Runtime.Intrinsics.Vector128.BitwiseOr (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-xor (: System.Runtime.Intrinsics.Vector128.Xor (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-not (: System.Runtime.Intrinsics.Vector128.OnesComplement (-> (Simd128 %a) (Simd128 %a)))) + (v128-and-not (: System.Runtime.Intrinsics.Vector128.AndNot (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-eq (: System.Runtime.Intrinsics.Vector128.Equals (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-lt (: System.Runtime.Intrinsics.Vector128.LessThan (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-le (: System.Runtime.Intrinsics.Vector128.LessThanOrEqual (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-gt (: System.Runtime.Intrinsics.Vector128.GreaterThan (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-ge (: System.Runtime.Intrinsics.Vector128.GreaterThanOrEqual (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-nan? (: System.Runtime.Intrinsics.Vector128.IsNaN (-> (Simd128 %a) (Simd128 %a)))) + (v128-select (: System.Runtime.Intrinsics.Vector128.ConditionalSelect (-> (Simd128 %a) (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-sum (: System.Runtime.Intrinsics.Vector128.Sum (-> (Simd128 %a) %a))) + (v128-dot (: System.Runtime.Intrinsics.Vector128.Dot (-> (Simd128 %a) (Simd128 %a) %a))) + (v128-ref (: System.Runtime.Intrinsics.Vector128.GetElement (-> (Simd128 %a) int %a))) + (v128-with (: System.Runtime.Intrinsics.Vector128.WithElement (-> (Simd128 %a) int %a (Simd128 %a)))) + (v128-store! (: System.Runtime.Intrinsics.Vector128.CopyTo (-> (Simd128 %a) (Array %a) int void))) + (v128-add-saturating (: System.Runtime.Intrinsics.Vector128.AddSaturate (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-sub-saturating (: System.Runtime.Intrinsics.Vector128.SubtractSaturate (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-shuffle (: Bjolang.Runtime.BjoSimd.Shuffle128 (-> (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-lookup (: Bjolang.Runtime.BjoSimd.Lookup128 (-> (Simd128 byte) (Simd128 %a) (Simd128 %a)))) + (v128-as (: System.Runtime.Intrinsics.Vector128.As (-> (Simd128 %a) (Simd128 %b)))) + (v128-floor (: Bjolang.Runtime.BjoSimd.Floor128 (-> (Simd128 %a) (Simd128 %a)))) + (v128-ceiling (: Bjolang.Runtime.BjoSimd.Ceiling128 (-> (Simd128 %a) (Simd128 %a)))) + (v128-round (: Bjolang.Runtime.BjoSimd.Round128 (-> (Simd128 %a) (Simd128 %a)))) + (v128-truncate (: Bjolang.Runtime.BjoSimd.Truncate128 (-> (Simd128 %a) (Simd128 %a)))) + (v128-mul-add (: Bjolang.Runtime.BjoSimd.MulAdd128 (-> (Simd128 %a) (Simd128 %a) (Simd128 %a) (Simd128 %a)))) + (v128-lanes (: Bjolang.Runtime.BjoSimd.Lanes128 (-> (Simd128 %a) int))) + (v128-any? (: Bjolang.Runtime.BjoSimd.Any128 (-> (Simd128 %a) bool))) + (v128-all? (: Bjolang.Runtime.BjoSimd.All128 (-> (Simd128 %a) bool))) + (v128-count-true (: Bjolang.Runtime.BjoSimd.CountTrue128 (-> (Simd128 %a) int))) + (v128-first-true (: Bjolang.Runtime.BjoSimd.FirstTrue128 (-> (Simd128 %a) int))) + (v128-splat (: System.Runtime.Intrinsics.Vector128.Create (-> %a (Simd128 %a)))) + (v128-zero (: Bjolang.Runtime.BjoSimd.Zero128 (-> (Simd128 %a)))) + (v128-mask-bits (: Bjolang.Runtime.BjoSimd.MaskBits128 (-> (Simd128 %a) int))) + (v128-shift-left (: Bjolang.Runtime.BjoSimd.ShiftLeft128 (-> (Simd128 %a) int (Simd128 %a)))) + (v128-shift-right (: Bjolang.Runtime.BjoSimd.ShiftRight128 (-> (Simd128 %a) int (Simd128 %a)))) + (v128-shift-right-logical (: Bjolang.Runtime.BjoSimd.ShiftRightLogical128 (-> (Simd128 %a) int (Simd128 %a)))) + (v128-load-array (: System.Runtime.Intrinsics.Vector128.Create (-> (Array %a) int (Simd128 %a)))) + (v128-load-list (: Bjolang.Runtime.BjoSimd.LoadList128 (-> (MutableVec %a) int (Simd128 %a)))) + (v128-array-lanes (: Bjolang.Runtime.BjoSimd.ArrayLanes128 (-> (Array %a) int))) + (v128-list-lanes (: Bjolang.Runtime.BjoSimd.ListLanes128 (-> (MutableVec %a) int))) + (simd128-accelerated? (: System.Runtime.Intrinsics.Vector128.IsHardwareAccelerated #:get))) + +(: (-> %a (Simd128 %a)) (where (Lane %a))) +(defun (simd128-splat x) (v128-splat x)) + +(: (-> (Simd128 %a)) (where (Lane %a))) +(defun (simd128-zero) (v128-zero)) + +;; The vector a mask holds, and the mask of a vector of a comparison's lanes. +(: (-> (Simd128 %a) (Mask128 %a))) +(defun (m128 v) (Mask128 (lanes v))) + +(: (-> (Mask128 %a) (Simd128 %a))) +(defun (lanes128 m) (record-ref m lanes)) + +(impl (Simd (Simd128 %a)) (where (Lane %a)) + (type %elem %a) + (type %mask (Mask128 %a)) + (defun (simd-add a b) (v128-add a b)) + (defun (simd-sub a b) (v128-sub a b)) + (defun (simd-mul a b) (v128-mul a b)) + (defun (simd-scale a b) (v128-scale a b)) + (defun (simd-neg a) (v128-neg a)) + (defun (simd-abs a) (v128-abs a)) + (defun (simd-min a b) (v128-min a b)) + (defun (simd-max a b) (v128-max a b)) + (defun (simd-eq a b) (m128 (v128-eq a b))) + (defun (simd-ne a b) (m128 (v128-not (v128-eq a b)))) + (defun (simd-lt a b) (m128 (v128-lt a b))) + (defun (simd-le a b) (m128 (v128-le a b))) + (defun (simd-gt a b) (m128 (v128-gt a b))) + (defun (simd-ge a b) (m128 (v128-ge a b))) + (defun (simd-select m a b) (v128-select (lanes128 m) a b)) + (defun (simd-sum a) (v128-sum a)) + (defun (simd-dot a b) (v128-dot a b)) + (defun (simd-ref a i) (v128-ref a i)) + (defun (simd-with a i x) (v128-with a i x)) + (defun (simd-store! a b c) (v128-store! a b c)) + (defun (simd-lanes a) (v128-lanes a))) + +(impl (SimdBitwise (Simd128 %a)) (where (Lane %a)) + (defun (simd-and a b) (v128-and a b)) + (defun (simd-or a b) (v128-or a b)) + (defun (simd-xor a b) (v128-xor a b)) + (defun (simd-not a) (v128-not a)) + (defun (simd-and-not a b) (v128-and-not a b))) + +(impl (SimdBitwise (Mask128 %a)) (where (Lane %a)) + (defun (simd-and a b) (m128 (v128-and (lanes128 a) (lanes128 b)))) + (defun (simd-or a b) (m128 (v128-or (lanes128 a) (lanes128 b)))) + (defun (simd-xor a b) (m128 (v128-xor (lanes128 a) (lanes128 b)))) + (defun (simd-not a) (m128 (v128-not (lanes128 a)))) + (defun (simd-and-not a b) (m128 (v128-and-not (lanes128 a) (lanes128 b))))) + +(impl (SimdMask (Mask128 %a)) (where (Lane %a)) + (type %vec (Simd128 %a)) + (defun (simd-any? m) (v128-any? (lanes128 m))) + (defun (simd-all? m) (v128-all? (lanes128 m))) + (defun (simd-count-true m) (v128-count-true (lanes128 m))) + (defun (simd-first-true m) (v128-first-true (lanes128 m))) + (defun (simd-mask-bits m) (v128-mask-bits (lanes128 m))) + (defun (simd-mask->lanes m) (lanes128 m))) + +(impl (SimdFloat (Simd128 %a)) (where (FloatLane %a)) + (type %mask (Mask128 %a)) + (defun (simd-div a b) (v128-div a b)) + (defun (simd-sqrt a) (v128-sqrt a)) + (defun (simd-floor a) (v128-floor a)) + (defun (simd-ceiling a) (v128-ceiling a)) + (defun (simd-round a) (v128-round a)) + (defun (simd-truncate a) (v128-truncate a)) + (defun (simd-nan? a) (m128 (v128-nan? a))) + (defun (simd-mul-add a b c) (v128-mul-add a b c))) + +(impl (SimdInt (Simd128 %a)) (where (IntLane %a)) + (defun (simd-shift-left a n) (v128-shift-left a n)) + (defun (simd-shift-right a n) (v128-shift-right a n)) + (defun (simd-shift-right-logical a n) (v128-shift-right-logical a n)) + (defun (simd-add-saturating a b) (v128-add-saturating a b)) + (defun (simd-sub-saturating a b) (v128-sub-saturating a b)) + (defun (simd-shuffle v idx) (v128-shuffle v idx))) + +(impl (SimdLookup (Simd128 %a)) (where (ByteLane %a)) + (defun (simd-lookup table idx) (v128-lookup table idx))) + +(impl (SimdBits (Simd128 %a)) (where (Lane %a)) + (type %int8 (Simd128 System.SByte)) + (type %uint8 (Simd128 byte)) + (type %int16 (Simd128 short)) + (type %uint16 (Simd128 ushort)) + (type %int32 (Simd128 int)) + (type %uint32 (Simd128 uint)) + (type %int64 (Simd128 long)) + (type %uint64 (Simd128 ulong)) + (type %float32 (Simd128 System.Single)) + (type %float64 (Simd128 double)) + (defun (simd-bits->int8 v) (v128-as v)) + (defun (simd-bits->uint8 v) (v128-as v)) + (defun (simd-bits->int16 v) (v128-as v)) + (defun (simd-bits->uint16 v) (v128-as v)) + (defun (simd-bits->int32 v) (v128-as v)) + (defun (simd-bits->uint32 v) (v128-as v)) + (defun (simd-bits->int64 v) (v128-as v)) + (defun (simd-bits->uint64 v) (v128-as v)) + (defun (simd-bits->float32 v) (v128-as v)) + (defun (simd-bits->float64 v) (v128-as v))) + +;; --------------------------------------------------------------------------- +;; 256 bits +;; --------------------------------------------------------------------------- + +(import/extern + (v256-add (: System.Runtime.Intrinsics.Vector256.Add (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-sub (: System.Runtime.Intrinsics.Vector256.Subtract (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-mul (: System.Runtime.Intrinsics.Vector256.Multiply (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-div (: System.Runtime.Intrinsics.Vector256.Divide (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-scale (: System.Runtime.Intrinsics.Vector256.Multiply (-> (Simd256 %a) %a (Simd256 %a)))) + (v256-neg (: System.Runtime.Intrinsics.Vector256.Negate (-> (Simd256 %a) (Simd256 %a)))) + (v256-abs (: System.Runtime.Intrinsics.Vector256.Abs (-> (Simd256 %a) (Simd256 %a)))) + (v256-min (: System.Runtime.Intrinsics.Vector256.Min (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-max (: System.Runtime.Intrinsics.Vector256.Max (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-sqrt (: System.Runtime.Intrinsics.Vector256.Sqrt (-> (Simd256 %a) (Simd256 %a)))) + (v256-and (: System.Runtime.Intrinsics.Vector256.BitwiseAnd (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-or (: System.Runtime.Intrinsics.Vector256.BitwiseOr (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-xor (: System.Runtime.Intrinsics.Vector256.Xor (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-not (: System.Runtime.Intrinsics.Vector256.OnesComplement (-> (Simd256 %a) (Simd256 %a)))) + (v256-and-not (: System.Runtime.Intrinsics.Vector256.AndNot (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-eq (: System.Runtime.Intrinsics.Vector256.Equals (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-lt (: System.Runtime.Intrinsics.Vector256.LessThan (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-le (: System.Runtime.Intrinsics.Vector256.LessThanOrEqual (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-gt (: System.Runtime.Intrinsics.Vector256.GreaterThan (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-ge (: System.Runtime.Intrinsics.Vector256.GreaterThanOrEqual (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-nan? (: System.Runtime.Intrinsics.Vector256.IsNaN (-> (Simd256 %a) (Simd256 %a)))) + (v256-select (: System.Runtime.Intrinsics.Vector256.ConditionalSelect (-> (Simd256 %a) (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-sum (: System.Runtime.Intrinsics.Vector256.Sum (-> (Simd256 %a) %a))) + (v256-dot (: System.Runtime.Intrinsics.Vector256.Dot (-> (Simd256 %a) (Simd256 %a) %a))) + (v256-ref (: System.Runtime.Intrinsics.Vector256.GetElement (-> (Simd256 %a) int %a))) + (v256-with (: System.Runtime.Intrinsics.Vector256.WithElement (-> (Simd256 %a) int %a (Simd256 %a)))) + (v256-store! (: System.Runtime.Intrinsics.Vector256.CopyTo (-> (Simd256 %a) (Array %a) int void))) + (v256-add-saturating (: System.Runtime.Intrinsics.Vector256.AddSaturate (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-sub-saturating (: System.Runtime.Intrinsics.Vector256.SubtractSaturate (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-shuffle (: Bjolang.Runtime.BjoSimd.Shuffle256 (-> (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-lookup (: Bjolang.Runtime.BjoSimd.Lookup256 (-> (Simd128 byte) (Simd256 %a) (Simd256 %a)))) + (v256-as (: System.Runtime.Intrinsics.Vector256.As (-> (Simd256 %a) (Simd256 %b)))) + (v256-floor (: Bjolang.Runtime.BjoSimd.Floor256 (-> (Simd256 %a) (Simd256 %a)))) + (v256-ceiling (: Bjolang.Runtime.BjoSimd.Ceiling256 (-> (Simd256 %a) (Simd256 %a)))) + (v256-round (: Bjolang.Runtime.BjoSimd.Round256 (-> (Simd256 %a) (Simd256 %a)))) + (v256-truncate (: Bjolang.Runtime.BjoSimd.Truncate256 (-> (Simd256 %a) (Simd256 %a)))) + (v256-mul-add (: Bjolang.Runtime.BjoSimd.MulAdd256 (-> (Simd256 %a) (Simd256 %a) (Simd256 %a) (Simd256 %a)))) + (v256-lanes (: Bjolang.Runtime.BjoSimd.Lanes256 (-> (Simd256 %a) int))) + (v256-any? (: Bjolang.Runtime.BjoSimd.Any256 (-> (Simd256 %a) bool))) + (v256-all? (: Bjolang.Runtime.BjoSimd.All256 (-> (Simd256 %a) bool))) + (v256-count-true (: Bjolang.Runtime.BjoSimd.CountTrue256 (-> (Simd256 %a) int))) + (v256-first-true (: Bjolang.Runtime.BjoSimd.FirstTrue256 (-> (Simd256 %a) int))) + (v256-splat (: System.Runtime.Intrinsics.Vector256.Create (-> %a (Simd256 %a)))) + (v256-zero (: Bjolang.Runtime.BjoSimd.Zero256 (-> (Simd256 %a)))) + (v256-mask-bits (: Bjolang.Runtime.BjoSimd.MaskBits256 (-> (Simd256 %a) int))) + (v256-shift-left (: Bjolang.Runtime.BjoSimd.ShiftLeft256 (-> (Simd256 %a) int (Simd256 %a)))) + (v256-shift-right (: Bjolang.Runtime.BjoSimd.ShiftRight256 (-> (Simd256 %a) int (Simd256 %a)))) + (v256-shift-right-logical (: Bjolang.Runtime.BjoSimd.ShiftRightLogical256 (-> (Simd256 %a) int (Simd256 %a)))) + (v256-load-array (: System.Runtime.Intrinsics.Vector256.Create (-> (Array %a) int (Simd256 %a)))) + (v256-load-list (: Bjolang.Runtime.BjoSimd.LoadList256 (-> (MutableVec %a) int (Simd256 %a)))) + (v256-array-lanes (: Bjolang.Runtime.BjoSimd.ArrayLanes256 (-> (Array %a) int))) + (v256-list-lanes (: Bjolang.Runtime.BjoSimd.ListLanes256 (-> (MutableVec %a) int))) + (simd256-accelerated? (: System.Runtime.Intrinsics.Vector256.IsHardwareAccelerated #:get))) + +(: (-> %a (Simd256 %a)) (where (Lane %a))) +(defun (simd256-splat x) (v256-splat x)) + +(: (-> (Simd256 %a)) (where (Lane %a))) +(defun (simd256-zero) (v256-zero)) + +;; The vector a mask holds, and the mask of a vector of a comparison's lanes. +(: (-> (Simd256 %a) (Mask256 %a))) +(defun (m256 v) (Mask256 (lanes v))) + +(: (-> (Mask256 %a) (Simd256 %a))) +(defun (lanes256 m) (record-ref m lanes)) + +(impl (Simd (Simd256 %a)) (where (Lane %a)) + (type %elem %a) + (type %mask (Mask256 %a)) + (defun (simd-add a b) (v256-add a b)) + (defun (simd-sub a b) (v256-sub a b)) + (defun (simd-mul a b) (v256-mul a b)) + (defun (simd-scale a b) (v256-scale a b)) + (defun (simd-neg a) (v256-neg a)) + (defun (simd-abs a) (v256-abs a)) + (defun (simd-min a b) (v256-min a b)) + (defun (simd-max a b) (v256-max a b)) + (defun (simd-eq a b) (m256 (v256-eq a b))) + (defun (simd-ne a b) (m256 (v256-not (v256-eq a b)))) + (defun (simd-lt a b) (m256 (v256-lt a b))) + (defun (simd-le a b) (m256 (v256-le a b))) + (defun (simd-gt a b) (m256 (v256-gt a b))) + (defun (simd-ge a b) (m256 (v256-ge a b))) + (defun (simd-select m a b) (v256-select (lanes256 m) a b)) + (defun (simd-sum a) (v256-sum a)) + (defun (simd-dot a b) (v256-dot a b)) + (defun (simd-ref a i) (v256-ref a i)) + (defun (simd-with a i x) (v256-with a i x)) + (defun (simd-store! a b c) (v256-store! a b c)) + (defun (simd-lanes a) (v256-lanes a))) + +(impl (SimdBitwise (Simd256 %a)) (where (Lane %a)) + (defun (simd-and a b) (v256-and a b)) + (defun (simd-or a b) (v256-or a b)) + (defun (simd-xor a b) (v256-xor a b)) + (defun (simd-not a) (v256-not a)) + (defun (simd-and-not a b) (v256-and-not a b))) + +(impl (SimdBitwise (Mask256 %a)) (where (Lane %a)) + (defun (simd-and a b) (m256 (v256-and (lanes256 a) (lanes256 b)))) + (defun (simd-or a b) (m256 (v256-or (lanes256 a) (lanes256 b)))) + (defun (simd-xor a b) (m256 (v256-xor (lanes256 a) (lanes256 b)))) + (defun (simd-not a) (m256 (v256-not (lanes256 a)))) + (defun (simd-and-not a b) (m256 (v256-and-not (lanes256 a) (lanes256 b))))) + +(impl (SimdMask (Mask256 %a)) (where (Lane %a)) + (type %vec (Simd256 %a)) + (defun (simd-any? m) (v256-any? (lanes256 m))) + (defun (simd-all? m) (v256-all? (lanes256 m))) + (defun (simd-count-true m) (v256-count-true (lanes256 m))) + (defun (simd-first-true m) (v256-first-true (lanes256 m))) + (defun (simd-mask-bits m) (v256-mask-bits (lanes256 m))) + (defun (simd-mask->lanes m) (lanes256 m))) + +(impl (SimdFloat (Simd256 %a)) (where (FloatLane %a)) + (type %mask (Mask256 %a)) + (defun (simd-div a b) (v256-div a b)) + (defun (simd-sqrt a) (v256-sqrt a)) + (defun (simd-floor a) (v256-floor a)) + (defun (simd-ceiling a) (v256-ceiling a)) + (defun (simd-round a) (v256-round a)) + (defun (simd-truncate a) (v256-truncate a)) + (defun (simd-nan? a) (m256 (v256-nan? a))) + (defun (simd-mul-add a b c) (v256-mul-add a b c))) + +(impl (SimdInt (Simd256 %a)) (where (IntLane %a)) + (defun (simd-shift-left a n) (v256-shift-left a n)) + (defun (simd-shift-right a n) (v256-shift-right a n)) + (defun (simd-shift-right-logical a n) (v256-shift-right-logical a n)) + (defun (simd-add-saturating a b) (v256-add-saturating a b)) + (defun (simd-sub-saturating a b) (v256-sub-saturating a b)) + (defun (simd-shuffle v idx) (v256-shuffle v idx))) + +(impl (SimdLookup (Simd256 %a)) (where (ByteLane %a)) + (defun (simd-lookup table idx) (v256-lookup table idx))) + +(impl (SimdBits (Simd256 %a)) (where (Lane %a)) + (type %int8 (Simd256 System.SByte)) + (type %uint8 (Simd256 byte)) + (type %int16 (Simd256 short)) + (type %uint16 (Simd256 ushort)) + (type %int32 (Simd256 int)) + (type %uint32 (Simd256 uint)) + (type %int64 (Simd256 long)) + (type %uint64 (Simd256 ulong)) + (type %float32 (Simd256 System.Single)) + (type %float64 (Simd256 double)) + (defun (simd-bits->int8 v) (v256-as v)) + (defun (simd-bits->uint8 v) (v256-as v)) + (defun (simd-bits->int16 v) (v256-as v)) + (defun (simd-bits->uint16 v) (v256-as v)) + (defun (simd-bits->int32 v) (v256-as v)) + (defun (simd-bits->uint32 v) (v256-as v)) + (defun (simd-bits->int64 v) (v256-as v)) + (defun (simd-bits->uint64 v) (v256-as v)) + (defun (simd-bits->float32 v) (v256-as v)) + (defun (simd-bits->float64 v) (v256-as v))) + +;; --------------------------------------------------------------------------- +;; Conversions +;; --------------------------------------------------------------------------- +;; +;; A conversion gives a vector of another element type, which depends on the +;; one it is given. So these traits are over the element type, with an impl +;; for each, and a method for each width: the name says the width, as the +;; name of what makes a vector does. +;; +;; Each name says what happens to a value that does not fit. Widening keeps +;; every value: a signed lane is sign-extended and an unsigned one is +;; zero-extended, by its type. Narrowing takes two vectors, the lower half +;; first, and either wraps, keeping the low bits, or saturates, giving the +;; nearest value that fits. + +;; The lower or the upper half of the lanes, each widened to twice the bits. +(def/trait (SimdWiden %a) + (type %wide) + (: simd128-widen-lower (-> (Simd128 %a) (Simd128 %wide))) + (: simd128-widen-upper (-> (Simd128 %a) (Simd128 %wide))) + (: simd256-widen-lower (-> (Simd256 %a) (Simd256 %wide))) + (: simd256-widen-upper (-> (Simd256 %a) (Simd256 %wide)))) + +;; Two vectors into one of half the bits a lane. +(def/trait (SimdNarrow %a) + (type %narrow) + (: simd128-narrow-wrapping (-> (Simd128 %a) (Simd128 %a) (Simd128 %narrow))) + (: simd128-narrow-saturating (-> (Simd128 %a) (Simd128 %a) (Simd128 %narrow))) + (: simd256-narrow-wrapping (-> (Simd256 %a) (Simd256 %a) (Simd256 %narrow))) + (: simd256-narrow-saturating (-> (Simd256 %a) (Simd256 %a) (Simd256 %narrow)))) + +;; Integer lanes to float lanes of the same bits, rounded to the nearest float +;; where the integer has more digits than the float holds. +(def/trait (SimdIntToFloat %a) + (type %float) + (: simd128-int->float (-> (Simd128 %a) (Simd128 %float))) + (: simd256-int->float (-> (Simd256 %a) (Simd256 %float)))) + +;; Float lanes to integer lanes of the same bits, toward zero. A value past the +;; integer type's range gives its least or greatest value, and NaN gives 0. +(def/trait (SimdFloatToInt %a) + (type %int) + (type %uint) + (: simd128-float->int-truncating (-> (Simd128 %a) (Simd128 %int))) + (: simd256-float->int-truncating (-> (Simd256 %a) (Simd256 %int))) + (: simd128-float->uint-truncating (-> (Simd128 %a) (Simd128 %uint))) + (: simd256-float->uint-truncating (-> (Simd256 %a) (Simd256 %uint)))) + +(import/extern + (widen128-lower-sb (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 System.SByte) (Simd128 short)))) + (widen128-upper-sb (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 System.SByte) (Simd128 short)))) + (widen256-lower-sb (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 System.SByte) (Simd256 short)))) + (widen256-upper-sb (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 System.SByte) (Simd256 short))))) +(impl (SimdWiden System.SByte) + (type %wide short) + (defun (simd128-widen-lower v) (widen128-lower-sb v)) + (defun (simd128-widen-upper v) (widen128-upper-sb v)) + (defun (simd256-widen-lower v) (widen256-lower-sb v)) + (defun (simd256-widen-upper v) (widen256-upper-sb v))) + +(import/extern + (widen128-lower-b (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 byte) (Simd128 ushort)))) + (widen128-upper-b (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 byte) (Simd128 ushort)))) + (widen256-lower-b (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 byte) (Simd256 ushort)))) + (widen256-upper-b (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 byte) (Simd256 ushort))))) +(impl (SimdWiden byte) + (type %wide ushort) + (defun (simd128-widen-lower v) (widen128-lower-b v)) + (defun (simd128-widen-upper v) (widen128-upper-b v)) + (defun (simd256-widen-lower v) (widen256-lower-b v)) + (defun (simd256-widen-upper v) (widen256-upper-b v))) + +(import/extern + (widen128-lower-s (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 short) (Simd128 int)))) + (widen128-upper-s (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 short) (Simd128 int)))) + (widen256-lower-s (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 short) (Simd256 int)))) + (widen256-upper-s (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 short) (Simd256 int))))) +(impl (SimdWiden short) + (type %wide int) + (defun (simd128-widen-lower v) (widen128-lower-s v)) + (defun (simd128-widen-upper v) (widen128-upper-s v)) + (defun (simd256-widen-lower v) (widen256-lower-s v)) + (defun (simd256-widen-upper v) (widen256-upper-s v))) + +(import/extern + (widen128-lower-us (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 ushort) (Simd128 uint)))) + (widen128-upper-us (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 ushort) (Simd128 uint)))) + (widen256-lower-us (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 ushort) (Simd256 uint)))) + (widen256-upper-us (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 ushort) (Simd256 uint))))) +(impl (SimdWiden ushort) + (type %wide uint) + (defun (simd128-widen-lower v) (widen128-lower-us v)) + (defun (simd128-widen-upper v) (widen128-upper-us v)) + (defun (simd256-widen-lower v) (widen256-lower-us v)) + (defun (simd256-widen-upper v) (widen256-upper-us v))) + +(import/extern + (widen128-lower-i (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 int) (Simd128 long)))) + (widen128-upper-i (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 int) (Simd128 long)))) + (widen256-lower-i (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 int) (Simd256 long)))) + (widen256-upper-i (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 int) (Simd256 long))))) +(impl (SimdWiden int) + (type %wide long) + (defun (simd128-widen-lower v) (widen128-lower-i v)) + (defun (simd128-widen-upper v) (widen128-upper-i v)) + (defun (simd256-widen-lower v) (widen256-lower-i v)) + (defun (simd256-widen-upper v) (widen256-upper-i v))) + +(import/extern + (widen128-lower-ui (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 uint) (Simd128 ulong)))) + (widen128-upper-ui (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 uint) (Simd128 ulong)))) + (widen256-lower-ui (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 uint) (Simd256 ulong)))) + (widen256-upper-ui (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 uint) (Simd256 ulong))))) +(impl (SimdWiden uint) + (type %wide ulong) + (defun (simd128-widen-lower v) (widen128-lower-ui v)) + (defun (simd128-widen-upper v) (widen128-upper-ui v)) + (defun (simd256-widen-lower v) (widen256-lower-ui v)) + (defun (simd256-widen-upper v) (widen256-upper-ui v))) + +(import/extern + (widen128-lower-f (: System.Runtime.Intrinsics.Vector128.WidenLower (-> (Simd128 System.Single) (Simd128 double)))) + (widen128-upper-f (: System.Runtime.Intrinsics.Vector128.WidenUpper (-> (Simd128 System.Single) (Simd128 double)))) + (widen256-lower-f (: System.Runtime.Intrinsics.Vector256.WidenLower (-> (Simd256 System.Single) (Simd256 double)))) + (widen256-upper-f (: System.Runtime.Intrinsics.Vector256.WidenUpper (-> (Simd256 System.Single) (Simd256 double))))) +(impl (SimdWiden System.Single) + (type %wide double) + (defun (simd128-widen-lower v) (widen128-lower-f v)) + (defun (simd128-widen-upper v) (widen128-upper-f v)) + (defun (simd256-widen-lower v) (widen256-lower-f v)) + (defun (simd256-widen-upper v) (widen256-upper-f v))) + +(import/extern + (narrow128-wrapping-s (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 short) (Simd128 short) (Simd128 System.SByte)))) + (narrow128-saturating-s (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 short) (Simd128 short) (Simd128 System.SByte)))) + (narrow256-wrapping-s (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 short) (Simd256 short) (Simd256 System.SByte)))) + (narrow256-saturating-s (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 short) (Simd256 short) (Simd256 System.SByte))))) +(impl (SimdNarrow short) + (type %narrow System.SByte) + (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-s a b)) + (defun (simd128-narrow-saturating a b) (narrow128-saturating-s a b)) + (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-s a b)) + (defun (simd256-narrow-saturating a b) (narrow256-saturating-s a b))) + +(import/extern + (narrow128-wrapping-us (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 ushort) (Simd128 ushort) (Simd128 byte)))) + (narrow128-saturating-us (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 ushort) (Simd128 ushort) (Simd128 byte)))) + (narrow256-wrapping-us (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 ushort) (Simd256 ushort) (Simd256 byte)))) + (narrow256-saturating-us (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 ushort) (Simd256 ushort) (Simd256 byte))))) +(impl (SimdNarrow ushort) + (type %narrow byte) + (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-us a b)) + (defun (simd128-narrow-saturating a b) (narrow128-saturating-us a b)) + (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-us a b)) + (defun (simd256-narrow-saturating a b) (narrow256-saturating-us a b))) + +(import/extern + (narrow128-wrapping-i (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 int) (Simd128 int) (Simd128 short)))) + (narrow128-saturating-i (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 int) (Simd128 int) (Simd128 short)))) + (narrow256-wrapping-i (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 int) (Simd256 int) (Simd256 short)))) + (narrow256-saturating-i (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 int) (Simd256 int) (Simd256 short))))) +(impl (SimdNarrow int) + (type %narrow short) + (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-i a b)) + (defun (simd128-narrow-saturating a b) (narrow128-saturating-i a b)) + (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-i a b)) + (defun (simd256-narrow-saturating a b) (narrow256-saturating-i a b))) + +(import/extern + (narrow128-wrapping-ui (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 uint) (Simd128 uint) (Simd128 ushort)))) + (narrow128-saturating-ui (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 uint) (Simd128 uint) (Simd128 ushort)))) + (narrow256-wrapping-ui (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 uint) (Simd256 uint) (Simd256 ushort)))) + (narrow256-saturating-ui (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 uint) (Simd256 uint) (Simd256 ushort))))) +(impl (SimdNarrow uint) + (type %narrow ushort) + (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-ui a b)) + (defun (simd128-narrow-saturating a b) (narrow128-saturating-ui a b)) + (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-ui a b)) + (defun (simd256-narrow-saturating a b) (narrow256-saturating-ui a b))) + +(import/extern + (narrow128-wrapping-l (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 long) (Simd128 long) (Simd128 int)))) + (narrow128-saturating-l (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 long) (Simd128 long) (Simd128 int)))) + (narrow256-wrapping-l (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 long) (Simd256 long) (Simd256 int)))) + (narrow256-saturating-l (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 long) (Simd256 long) (Simd256 int))))) +(impl (SimdNarrow long) + (type %narrow int) + (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-l a b)) + (defun (simd128-narrow-saturating a b) (narrow128-saturating-l a b)) + (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-l a b)) + (defun (simd256-narrow-saturating a b) (narrow256-saturating-l a b))) + +(import/extern + (narrow128-wrapping-ul (: System.Runtime.Intrinsics.Vector128.Narrow (-> (Simd128 ulong) (Simd128 ulong) (Simd128 uint)))) + (narrow128-saturating-ul (: System.Runtime.Intrinsics.Vector128.NarrowWithSaturation (-> (Simd128 ulong) (Simd128 ulong) (Simd128 uint)))) + (narrow256-wrapping-ul (: System.Runtime.Intrinsics.Vector256.Narrow (-> (Simd256 ulong) (Simd256 ulong) (Simd256 uint)))) + (narrow256-saturating-ul (: System.Runtime.Intrinsics.Vector256.NarrowWithSaturation (-> (Simd256 ulong) (Simd256 ulong) (Simd256 uint))))) +(impl (SimdNarrow ulong) + (type %narrow uint) + (defun (simd128-narrow-wrapping a b) (narrow128-wrapping-ul a b)) + (defun (simd128-narrow-saturating a b) (narrow128-saturating-ul a b)) + (defun (simd256-narrow-wrapping a b) (narrow256-wrapping-ul a b)) + (defun (simd256-narrow-saturating a b) (narrow256-saturating-ul a b))) + +(import/extern + (int->float128-i (: System.Runtime.Intrinsics.Vector128.ConvertToSingle (-> (Simd128 int) (Simd128 System.Single)))) + (int->float256-i (: System.Runtime.Intrinsics.Vector256.ConvertToSingle (-> (Simd256 int) (Simd256 System.Single))))) +(impl (SimdIntToFloat int) + (type %float System.Single) + (defun (simd128-int->float v) (int->float128-i v)) + (defun (simd256-int->float v) (int->float256-i v))) + +(import/extern + (int->float128-ui (: System.Runtime.Intrinsics.Vector128.ConvertToSingle (-> (Simd128 uint) (Simd128 System.Single)))) + (int->float256-ui (: System.Runtime.Intrinsics.Vector256.ConvertToSingle (-> (Simd256 uint) (Simd256 System.Single))))) +(impl (SimdIntToFloat uint) + (type %float System.Single) + (defun (simd128-int->float v) (int->float128-ui v)) + (defun (simd256-int->float v) (int->float256-ui v))) + +(import/extern + (int->float128-l (: System.Runtime.Intrinsics.Vector128.ConvertToDouble (-> (Simd128 long) (Simd128 double)))) + (int->float256-l (: System.Runtime.Intrinsics.Vector256.ConvertToDouble (-> (Simd256 long) (Simd256 double))))) +(impl (SimdIntToFloat long) + (type %float double) + (defun (simd128-int->float v) (int->float128-l v)) + (defun (simd256-int->float v) (int->float256-l v))) + +(import/extern + (int->float128-ul (: System.Runtime.Intrinsics.Vector128.ConvertToDouble (-> (Simd128 ulong) (Simd128 double)))) + (int->float256-ul (: System.Runtime.Intrinsics.Vector256.ConvertToDouble (-> (Simd256 ulong) (Simd256 double))))) +(impl (SimdIntToFloat ulong) + (type %float double) + (defun (simd128-int->float v) (int->float128-ul v)) + (defun (simd256-int->float v) (int->float256-ul v))) + +(import/extern + (float->int128-f (: System.Runtime.Intrinsics.Vector128.ConvertToInt32 (-> (Simd128 System.Single) (Simd128 int)))) + (float->uint128-f (: System.Runtime.Intrinsics.Vector128.ConvertToUInt32 (-> (Simd128 System.Single) (Simd128 uint)))) + (float->int256-f (: System.Runtime.Intrinsics.Vector256.ConvertToInt32 (-> (Simd256 System.Single) (Simd256 int)))) + (float->uint256-f (: System.Runtime.Intrinsics.Vector256.ConvertToUInt32 (-> (Simd256 System.Single) (Simd256 uint))))) +(impl (SimdFloatToInt System.Single) + (type %int int) + (type %uint uint) + (defun (simd128-float->int-truncating v) (float->int128-f v)) + (defun (simd128-float->uint-truncating v) (float->uint128-f v)) + (defun (simd256-float->int-truncating v) (float->int256-f v)) + (defun (simd256-float->uint-truncating v) (float->uint256-f v))) + +(import/extern + (float->int128-d (: System.Runtime.Intrinsics.Vector128.ConvertToInt64 (-> (Simd128 double) (Simd128 long)))) + (float->uint128-d (: System.Runtime.Intrinsics.Vector128.ConvertToUInt64 (-> (Simd128 double) (Simd128 ulong)))) + (float->int256-d (: System.Runtime.Intrinsics.Vector256.ConvertToInt64 (-> (Simd256 double) (Simd256 long)))) + (float->uint256-d (: System.Runtime.Intrinsics.Vector256.ConvertToUInt64 (-> (Simd256 double) (Simd256 ulong))))) +(impl (SimdFloatToInt double) + (type %int long) + (type %uint ulong) + (defun (simd128-float->int-truncating v) (float->int128-d v)) + (defun (simd128-float->uint-truncating v) (float->uint128-d v)) + (defun (simd256-float->int-truncating v) (float->int256-d v)) + (defun (simd256-float->uint-truncating v) (float->uint256-d v))) + +;; --------------------------------------------------------------------------- +;; A vector of given lanes +;; --------------------------------------------------------------------------- +;; +;; `(simd128-of 1 2 3 4)`: a vector with these lanes, from lane 0. Each value +;; is written into a vector of zeros with `simd-with`. A lane not given is 0, +;; and a value past the last lane raises `ArgumentOutOfRangeException`. With +;; constants, the JIT folds the writes into one constant vector. + +(: (-> Syntax int (List Syntax) Syntax)) +(defun (lanes-of acc i xs) + (match xs + (Nil acc) + ((Cons x rest) + (lanes-of #'(simd-with ,acc ,(SInt (int->string i)) ,x) (+ i 1) rest)))) + +(def/macro (simd128-of form inject compare) + (syntax-match form + ((_ first rest ...) (lanes-of #'(simd128-zero) 0 (Cons first rest))) + (bad (syntax-error bad "simd128-of takes the lanes, one or more: (simd128-of 1 2 3 4).")))) + +(def/macro (simd256-of form inject compare) + (syntax-match form + ((_ first rest ...) (lanes-of #'(simd256-zero) 0 (Cons first rest))) + (bad (syntax-error bad "simd256-of takes the lanes, one or more: (simd256-of 1 2 3 4).")))) + +;; --------------------------------------------------------------------------- +;; Where vectors are loaded from +;; --------------------------------------------------------------------------- +;; +;; A source is a run of numbers in memory, one after the other: an array or a +;; `MutableVec`. Every load raises `ArgumentOutOfRangeException` when fewer +;; than a vector's lanes are left. A string is not a source: its positions +;; are cursors, and a lane of its bytes would be a byte index. + +(def/trait (SimdSource %s) + (type %elem) + (: simd128-load (-> %s int (Simd128 %elem))) + (: simd256-load (-> %s int (Simd256 %elem))) + (: simd128-lanes (-> %s int)) + (: simd256-lanes (-> %s int)) + (: simd-source-length (-> %s int))) + +(impl (SimdSource (Array %a)) (where (Lane %a)) + (type %elem %a) + (defun (simd128-load a i) (v128-load-array a i)) + (defun (simd256-load a i) (v256-load-array a i)) + (defun (simd128-lanes a) (v128-array-lanes a)) + (defun (simd256-lanes a) (v256-array-lanes a)) + (defun (simd-source-length a) (array-length a))) + +(impl (SimdSource (MutableVec %a)) (where (Lane %a)) + (type %elem %a) + (defun (simd128-load v i) (v128-load-list v i)) + (defun (simd256-load v i) (v256-load-list v i)) + (defun (simd128-lanes v) (v128-list-lanes v)) + (defun (simd256-lanes v) (v256-list-lanes v)) + (defun (simd-source-length v) (mutablevec-length v))) + +;; Where the elements after the last whole vector start. +(: (-> %s int) (where (SimdSource %s))) +(defun (simd128-whole s) + (def w (simd128-lanes s)) + (* w (/ (simd-source-length s) w))) + +(: (-> %s int) (where (SimdSource %s))) +(defun (simd256-whole s) + (def w (simd256-lanes s)) + (* w (/ (simd-source-length s) w))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/simpletest.bjo — basic test assertion module. + +(export expect expect-true fail + default-test-deadline overran? + FakeFs fake-fs fake-fs-written fake-fs-contents fake-fs-set-modified! + ;; The handlers, one per `FS` operation. Exported because the + ;; `with-fake-fs` templates name them. + fake-fs-reader fake-fs-writer fake-fs-info fake-fs-list + fake-fs-create-directory fake-fs-delete-file fake-fs-delete-directory + fake-fs-move) + +(import (std prelude)) +(import (std effect)) +(import (std syntax-match)) + +(:doc #:file "simpletest.bjodoc") + +;; A failure makes the program exit 1 even when `main` answers 0, so that a +;; test run is a command a shell or CI can check. +(import/extern + (set-exit-code! (: System.Environment.ExitCode (-> int void) #:set))) + +(: (-> string void)) +(defun (failed! message) + (println message) + (set-exit-code! 1)) + +;; Asserts equality and prints result. +(: (-> string %a %a void) (where (Eq %a) (Written %a))) +(defun (expect label actual wanted) + (if (= actual wanted) + (println #"ok: ${label}") + ;; Written, so that "1" and 1 cannot read the same. + (failed! #"FAILURE: ${label} gave ${(written actual)} but wanted ${(written wanted)}"))) + +;; Asserts boolean and prints result. +(: (-> string bool void)) +(defun (expect-true label ok) + (if ok + (println #"ok: ${label}") + (failed! #"FAILURE: ${label}"))) + +;; Marks a failing branch. +(: (-> string void)) +(defun (fail label) + (failed! #"FAILURE: ${label}")) + +;; --------------------------------------------------------------------------- +;; A test body, under a scope +;; --------------------------------------------------------------------------- +;; +;; `(with-test "name" body ...)` runs the body under a deadline and asserts that +;; it did not leave a resource behind: +;; +;; (with-test "reads the fixture" +;; (with-fake-fs (("a.txt" "hello")) +;; (expect "read it" (call-with-input-file "a.txt" read-all) (Ok "hello")))) +;; +;; Four things come from the scope rather than from the harness. A test gets a +;; timeout, so a leaked fiber is a failure that names the test instead of a +;; suite that hangs. A fiber the body spawned is waited for, so an assertion +;; inside one cannot be missed. Resources the body opened are released. And +;; several failures in one body are reported together rather than only the +;; first. +;; +;; We check if the test left any resources open *before* the scope finishes. +;; If we checked after, the scope would have already silently cleaned them up! +;; We want tests to explicitly fail if they forget to close their files. +;; +;; You can disable this check using `#:open-ok` (useful if you are specifically +;; testing that cleanup happens on scope-close). +;; +;; `#:within d`, a Duration, overrides the deadline. +;; +;; We catch `System.Exception` here to handle `Cancelled` exceptions caused by timeouts. +;; +;; Normally, catching and hiding a cancellation is a terrible idea in structured +;; concurrency. But since this is the top-level test runner, we *want* to catch +;; it. Doing so lets us report a clean failure with the test's name, instead +;; of crashing and dumping an unhelpful stack trace. + +(: (-> Duration)) +(defun (default-test-deadline) (seconds 5)) + +(: (-> Syntax Syntax)) +(defun (leak-check name) + #'(let ((left (scope-owned-count (current-scope)))) + (unless (= left 0) + (fail (string-append ,name (string-append " left " (string-append (int->string left) " resource(s) open"))))))) + +;; Whether the deadline was what ended the scope. +;; +;; A fiber the body leaked is *cancelled* by the deadline rather than failing, +;; and a cancelled child is not a failure - so the scope closes cleanly and the +;; test would otherwise pass, slowly. The token carries the reason, so the +;; scope is kept in a box and the reason read after it has closed. +(: (-> (Box (Option Scope)) bool)) +(defun (overran? holder) + (match (box-ref holder) + (None #f) + ((Some s) + (match (cancel-reason (scope-token s)) + ((Some Deadline) #t) + (_ #f))))) + +(: (-> Syntax Syntax bool (List Syntax) Syntax)) +(defun (test-expansion name ms check body) + (let ((core (if check + #'(let () ,@body ,(leak-check name) unit) + #'(let () ,@body unit)))) + #'(let ((holder (make-box None))) + (match (try (with-deadline ,ms (let () (box-set! holder (Some (current-scope))) ,core)) + #:catch (System.Exception)) + ((Ok _) + (when (overran? holder) + (fail (string-append ,name " ran past its deadline: something it started never finished")))) + ((Err e) (fail (string-append ,name (string-append " did not finish: " (.-Message e))))))))) + +(def/macro (with-test form inject compare) + (syntax-match form + ((_ name #:within ms #:open-ok first body ...) + (test-expansion name ms #f (cons first body))) + ((_ name #:within ms first body ...) + (test-expansion name ms #t (cons first body))) + ((_ name #:open-ok first body ...) + (test-expansion name #'(default-test-deadline) #f (cons first body))) + ((_ name first body ...) + (test-expansion name #'(default-test-deadline) #t (cons first body))) + (bad (syntax-error bad "with-test takes a name and a body: (with-test \"name\" body ...), optionally #:within d, a Duration, and #:open-ok.")))) + +;; --------------------------------------------------------------------------- +;; A filesystem that is not one +;; --------------------------------------------------------------------------- +;; +;; `with-fake-fs` installs a handler for every operation of the prelude's `FS` +;; effect, so everything that reaches the disk through it - the text and byte +;; openers, `file-read-text` and its neighbours, `file-exists?`, `file-delete`, +;; `file-move` and the directory functions - answers out of a map instead: +;; +;; (with-fake-fs (("notes.txt" "a b c") +;; ("empty.txt" "")) +;; (expect "read it back" (call-with-input-file "notes.txt" read-all) (Ok "a b c"))) +;; +;; A seeded path is a file, and every directory above one exists. A path +;; written inside it is readable afterwards. `with-fake-fs-of` installs an +;; existing `FakeFs`, so that `(fake-fs-written fs "out.txt")` can read back +;; what was written - which works after the port has been closed, because +;; `MemoryStream.ToArray` does. +;; +;; Paths are compared as text after one normalization: backslashes become +;; slashes, and a leading "./" and a trailing "/" are dropped. "" and "." are +;; the root. Nothing resolves "..", and a relative path is not made absolute, so +;; "a.txt" and "/tmp/a.txt" are different files. +;; +;; A failure answers the exception type the .NET call would raise, so a test +;; that matches on the error matches either way. +;; +;; The fake has a clock of its own, so that a test about modified times does +;; not depend on how fast it runs. It starts at 2000-01-01T00:00:00Z, which is +;; every seed's time, and each write opened and each directory created moves it +;; one second on and stamps the path: something written later is always newer. +;; A move keeps the times it moves, as a rename does on a disk. +;; `(fake-fs-set-modified! fs path ms)` sets one by hand. +;; +;; The openers answer a `MemoryStream`, and the ports over it are built and +;; owned by the same code that builds them over a file - so a fake port is +;; registered on the scope the open happened in, and a leak test over a fake +;; filesystem means what it means over a real one. +;; +;; `with-fake-fs` is a handler frame, not a scope: it does not wait for a fiber +;; spawned inside it. The fiber keeps the fake - a spawn takes the environment +;; as it stood at the spawn - but the body's assertions, and `with-test`'s leak +;; check, run before the fiber is done. So a fiber that reads or writes the fake +;; goes inside a `with-scope` of its own, inside `with-fake-fs`. No scope is +;; opened here, because one would hide from `with-test` what the body left open. + +(import/class + (ClrStream (: System.IO.Stream)) + (MemoryStream (: System.IO.MemoryStream (-> MemoryStream))) + (Encoding (: System.Text.Encoding)) + (FileNotFound (: System.IO.FileNotFoundException (-> string FileNotFound))) + (DirectoryNotFound (: System.IO.DirectoryNotFoundException (-> string DirectoryNotFound))) + (IOException (: System.IO.IOException (-> string IOException))) + (AccessDenied (: System.UnauthorizedAccessException (-> string AccessDenied)))) + +(import/extern + (clr-utf8 (: System.Text.Encoding.UTF8 #:get)) + (stream-position! (: System.IO.Stream.Position (-> ClrStream long void) #:set))) + +;; A seeded file holds its text. A written one holds the stream it was written +;; through, which is read when the file is: the writer may still be open, and a +;; read sees what has reached the stream so far, as a read of a real file sees +;; what has been flushed. +(type (: FakeEntry (Union (: FakeText string) (: FakeWritten MemoryStream) FakeDir))) + +;; Mutable because a write has to be visible to a later read. `modified` holds +;; the time of every path something stamped; any other path has the clock's +;; start. +(type + (: FakeFs #:opaque + (Record (: entries (Map string FakeEntry) #:mutable) + (: modified (Map string long) #:mutable) + (: clock long #:mutable)))) + +;; 2000-01-01T00:00:00Z, in Unix milliseconds. +(: long) +(def fake-epoch-ms 946684800000L) + +;; What follows `prefix` in `s`, which starts with it. +(: (-> string string string)) +(defun (after-prefix s prefix) + (let go ((c (string-cursor-start s)) (n (string-count prefix))) + (if (= n 0) + (substring s c (string-cursor-end s)) + (go (string-cursor-next s c) (- n 1))))) + +(: (-> string string)) +(defun (fake-path path) + (let loop ((p (string-replace path "\\" "/"))) + (cond + ((= p ".") "") + ((string-starts-with? p "./") (loop (after-prefix p "./"))) + ((and (string-ends-with? p "/") (> (string-length p) 1)) + (loop (substring p (string-cursor-start p) (string-cursor-prev p (string-cursor-end p))))) + (else p)))) + +(: (-> string string)) +(defun (fake-parent path) + (match (string-search-backward path "/") + ((Some at) (substring path (string-cursor-start path) at)) + (None ""))) + +;; "" for the root, so that a child of the root is its own name. +(: (-> string string)) +(defun (fake-prefix path) + (if (string-empty? path) "" (str path "/"))) + +;; Every directory above `path` gets an entry of its own, so that it outlives +;; its contents: emptying a directory does not delete it. +(: (-> (Map string FakeEntry) string (Map string FakeEntry))) +(defun (fake-with-ancestors m path) + (let loop ((m m) (dir (fake-parent path))) + (if (or (string-empty? dir) (map-has-key? m dir)) + m + (loop (map-set m dir FakeDir) (fake-parent dir))))) + +(: (-> (List (Tuple string string)) FakeFs)) +(defun (fake-fs seeds) + (FakeFs (entries (list-foldl (fun (seed m) + (def (raw text) seed) + (def path (fake-path raw)) + (map-set (fake-with-ancestors m path) path (FakeText text))) + (map-empty) + seeds)) + (modified (map-empty)) + (clock fake-epoch-ms))) + +(: (-> FakeFs (Map string FakeEntry))) +(defun (fake-entries fs) (record-ref fs entries)) + +(: (-> FakeFs (Map string FakeEntry) void)) +(defun (fake-update! fs m) (record-set! fs (entries m))) + +;; Moves the clock one second on, and gives `path` the new time. +(: (-> FakeFs string void)) +(defun (fake-stamp! fs path) + (def now (+ (record-ref fs clock) 1000L)) + (record-set! fs (clock now) (modified (map-set (record-ref fs modified) path now)))) + +(: (-> FakeFs string void)) +(defun (fake-forget-time! fs path) + (record-set! fs (modified (map-remove (record-ref fs modified) path)))) + +(: (-> FakeFs string long void)) +(defun (fake-fs-set-modified! fs path ms) + (record-set! fs (modified (map-set (record-ref fs modified) (fake-path path) ms)))) + +(: (-> (Array byte) string)) +(defun (fake-bytes->text bytes) (.GetString (cast Encoding clr-utf8) bytes)) + +(: (-> string (Array byte))) +(defun (fake-text->bytes text) (.GetBytes (cast Encoding clr-utf8) text)) + +;; What a file entry holds. `None` for a directory. +(: (-> FakeEntry (Option string))) +(defun (fake-entry-text entry) + (match entry + ((FakeText text) (Some text)) + ((FakeWritten s) (Some (fake-bytes->text (.ToArray s)))) + (FakeDir None))) + +;; What a file entry holds, as bytes. Reading and appending go through this +;; rather than through the text, so that bytes that are not UTF-8 come back as +;; they were written. +(: (-> FakeEntry (Option (Array byte)))) +(defun (fake-entry-bytes entry) + (match entry + ((FakeText text) (Some (fake-text->bytes text))) + ((FakeWritten s) (Some (.ToArray s))) + (FakeDir None))) + +(: (-> FakeFs string (Option (Array byte)))) +(defun (fake-bytes-at fs path) + (match (map-try-ref (fake-entries fs) path) + ((Some entry) (fake-entry-bytes entry)) + (None None))) + +;; In bytes, as a file on a disk would be. +(: (-> FakeEntry long)) +(defun (fake-entry-size entry) + (match (fake-entry-bytes entry) + ((Some bytes) (cast System.Int64 (array-length bytes))) + (None 0L))) + +;; A path that is no entry of its own is still a directory when something is +;; below it. +(: (-> FakeFs string (Option FsKind))) +(defun (fake-kind fs path) + (if (string-empty? path) + (Some FsDirectory) + (match (map-try-ref (fake-entries fs) path) + ((Some FakeDir) (Some FsDirectory)) + ((Some _) (Some FsFile)) + (None (if (map-any? (fun (k v) (string-starts-with? k (fake-prefix path))) + (fake-entries fs)) + (Some FsDirectory) + None))))) + +(: (-> FakeFs string (Option FsInfo))) +(defun (fake-info fs path) + (option-map (fun (k) + (FsInfo (kind k) + (size (match (map-try-ref (fake-entries fs) path) + ((Some entry) (fake-entry-size entry)) + (None 0L))) + (modified-ms (map-ref-or (record-ref fs modified) path fake-epoch-ms)) + (link? #f))) + (fake-kind fs path))) + +(: (-> FakeFs string bool)) +(defun (fake-dir? fs path) + (match (fake-kind fs path) + ((Some FsDirectory) #t) + (_ #f))) + +;; What a file holds, whether it was seeded or written. +(: (-> FakeFs string (Option string))) +(defun (fake-fs-contents fs path) + (match (map-try-ref (fake-entries fs) (fake-path path)) + ((Some entry) (fake-entry-text entry)) + (None None))) + +(: (-> (Array byte) MemoryStream)) +(defun (fake-memory-stream bytes) + (def s (MemoryStream.)) + (.Write s bytes 0 (array-length bytes)) + (stream-position! (cast ClrStream s) 0L) + s) + +(: (-> Exception (Result Exception %a))) +(defun (fake-error e) (Err e)) + +(: (-> string (Result Exception %a))) +(defun (not-found path) + (fake-error (cast Exception (FileNotFound. #"the fake filesystem has no file ${path}")))) + +(: (-> string (Result Exception %a))) +(defun (no-directory path) + (fake-error (cast Exception (DirectoryNotFound. #"the fake filesystem has no directory ${path}")))) + +(: (-> string (Result Exception %a))) +(defun (io-error message) + (fake-error (cast Exception (IOException. message)))) + +(: (-> string (Result Exception %a))) +(defun (access-denied path) + (fake-error (cast Exception (AccessDenied. #"the fake filesystem cannot do that to directory ${path}")))) + +;; --- The handlers -------------------------------------------------------------- + +(: (-> FakeFs (-> string (Result Exception ClrStream)))) +(defun (fake-fs-reader fs) + (fun (raw) + (def path (fake-path raw)) + (match (fake-kind fs path) + ((Some FsDirectory) (access-denied path)) + ((Some FsFile) (match (fake-bytes-at fs path) + ((Some bytes) (Ok (cast ClrStream (fake-memory-stream bytes)))) + (None (not-found path)))) + (None (if (fake-dir? fs (fake-parent path)) (not-found path) (no-directory path)))))) + +;; `Append` starts the new stream with what the path held. +(: (-> FakeFs (-> string WriteMode (Result Exception ClrStream)))) +(defun (fake-fs-writer fs) + (fun (raw mode) + (def path (fake-path raw)) + (def kind (fake-kind fs path)) + (cond + ((not (fake-dir? fs (fake-parent path))) (no-directory path)) + (else + (match (Tuple mode kind) + ((Tuple _ (Some FsDirectory)) (access-denied path)) + ((Tuple CreateNew (Some FsFile)) (io-error #"the fake filesystem already has ${path}")) + (_ + (def s (match (Tuple mode (fake-bytes-at fs path)) + ((Tuple Append (Some bytes)) (let ((m (fake-memory-stream bytes))) + (stream-position! (cast ClrStream m) (.-Length m)) + m)) + (_ (MemoryStream.)))) + (fake-update! fs (map-set (fake-entries fs) path (FakeWritten s))) + (fake-stamp! fs path) + (Ok (cast ClrStream s)))))))) + +(: (-> FakeFs (-> string (Option FsInfo)))) +(defun (fake-fs-info fs) + (fun (raw) (fake-info fs (fake-path raw)))) + +;; Whole paths, as `Directory.GetFileSystemEntries` gives them, sorted. +(: (-> FakeFs (-> string (Result Exception (Vec string))))) +(defun (fake-fs-list fs) + (fun (raw) + (def path (fake-path raw)) + (def prefix (fake-prefix path)) + (if (not (fake-dir? fs path)) + (no-directory path) + (let ((children (map-fold (fun (k v found) + (if (and (string-starts-with? k prefix) (not (= k path))) + (let* ((rest (after-prefix k prefix)) + (child (match (string-search rest "/") + ((Some at) (substring rest (string-cursor-start rest) at)) + (None rest)))) + (map-set found (str prefix child) #t)) + found)) + (map-empty) + (fake-entries fs)))) + (Ok (list->vec (list-sort (loop (:for child (map-keys children)) + (:acc (listing child)))))))))) + +;; Every missing directory along the path, as `Directory.CreateDirectory`. +(: (-> FakeFs (-> string (Result Exception Unit)))) +(defun (fake-fs-create-directory fs) + (fun (raw) + (def path (fake-path raw)) + (match (fake-kind fs path) + ((Some FsFile) (io-error #"the fake filesystem has a file at ${path}")) + ((Some FsDirectory) (Ok unit)) + (None (fake-update! fs (map-set (fake-with-ancestors (fake-entries fs) path) path FakeDir)) + (fake-stamp! fs path) + (Ok unit))))) + +;; A missing file is not an error, as with `File.Delete`. +(: (-> FakeFs (-> string (Result Exception Unit)))) +(defun (fake-fs-delete-file fs) + (fun (raw) + (def path (fake-path raw)) + (match (fake-kind fs path) + ((Some FsDirectory) (access-denied path)) + ((Some FsFile) (fake-update! fs (map-remove (fake-entries fs) path)) + (fake-forget-time! fs path) + (Ok unit)) + (None (Ok unit))))) + +(: (-> FakeFs (-> string (Result Exception Unit)))) +(defun (fake-fs-delete-directory fs) + (fun (raw) + (def path (fake-path raw)) + (def prefix (fake-prefix path)) + (match (fake-kind fs path) + ((Some FsFile) (io-error #"the fake filesystem has a file, not a directory, at ${path}")) + (None (no-directory path)) + ((Some FsDirectory) + (cond + ((string-empty? path) (io-error "the fake filesystem's root cannot be deleted")) + ((map-any? (fun (k v) (string-starts-with? k prefix)) (fake-entries fs)) + (io-error #"the fake directory ${path} is not empty")) + (else (fake-update! fs (map-remove (fake-entries fs) path)) + (fake-forget-time! fs path) + (Ok unit))))))) + +;; `m` with `from`, and everything below it, renamed to `to`. Generic, because +;; the entries and their times move by the same renaming. +(: (-> (Map string %v) string string (Map string %v))) +(defun (fake-renamed m from to) + (def from-prefix (fake-prefix from)) + (map-fold (fun (k v acc) + (cond + ((= k from) (map-set acc to v)) + ((string-starts-with? k from-prefix) + (map-set acc (str to "/" (after-prefix k from-prefix)) v)) + (else (map-set acc k v)))) + (map-empty) + m)) + +;; A file, or a directory and everything below it. Fails if `to` exists, +;; unless `MoveReplace` asks to replace a file there with a file. The source is +;; checked first, as `File.Move` does. +(: (-> FakeFs (-> string string MoveMode (Result Exception Unit)))) +(defun (fake-fs-move fs) + (fun (raw-from raw-to mode) + (def from (fake-path raw-from)) + (def to (fake-path raw-to)) + (def from-kind (fake-kind fs from)) + (def to-kind (fake-kind fs to)) + (def replacing (match (Tuple mode from-kind to-kind) + ((Tuple MoveReplace (Some FsFile) (Some FsFile)) #t) + (_ #f))) + (cond + ((none? from-kind) (not-found from)) + ((and (some? to-kind) (not replacing)) (io-error #"the fake filesystem already has ${to}")) + ((not (fake-dir? fs (fake-parent to))) (no-directory to)) + (else + (when replacing + (fake-update! fs (map-remove (fake-entries fs) to)) + (fake-forget-time! fs to)) + (fake-update! fs (fake-renamed (fake-entries fs) from to)) + (record-set! fs (modified (fake-renamed (record-ref fs modified) from to))) + (Ok unit))))) + +;; What was written to `path`, or `None` if nothing was. A seeded file nobody +;; wrote is `None`. +(: (-> FakeFs string (Option string))) +(defun (fake-fs-written fs path) + (match (map-try-ref (fake-entries fs) (fake-path path)) + ((Some (FakeWritten s)) (Some (fake-bytes->text (.ToArray s)))) + (_ None))) + +;; --- The forms ----------------------------------------------------------------- +;; +;; The operation names go through `inject`, so they arrive at `with-handler` +;; unrenamed. `with-handler` derives the cell it binds from the name it is +;; given, and a renamed `open-read-stream` would derive a cell nothing +;; declares. Any macro that writes a `with-handler` has to do this. + +(: (-> (-> Symbol Syntax) string Syntax)) +(defun (fake-op inject name) (inject (string->symbol name))) + +;; `(with-fake-fs-of fs body ...)`: install `fs` for the body. +(def/macro (with-fake-fs-of form inject compare) + (syntax-match form + ((_ fs first body ...) + #'(let ((the-fs ,fs)) + (with-handler ((,(fake-op inject "open-read-stream") (fake-fs-reader the-fs)) + (,(fake-op inject "open-write-stream") (fake-fs-writer the-fs)) + (,(fake-op inject "fs-info") (fake-fs-info the-fs)) + (,(fake-op inject "fs-list") (fake-fs-list the-fs)) + (,(fake-op inject "fs-create-directory") (fake-fs-create-directory the-fs)) + (,(fake-op inject "fs-delete-file") (fake-fs-delete-file the-fs)) + (,(fake-op inject "fs-delete-directory") (fake-fs-delete-directory the-fs)) + (,(fake-op inject "fs-move") (fake-fs-move the-fs))) + ,first ,@body))) + (bad (syntax-error bad "with-fake-fs-of takes a FakeFs and a body: (with-fake-fs-of fs body ...)")))) + +;; `(with-fake-fs ((path contents) ...) body ...)`: a fresh one, seeded. +(def/macro (with-fake-fs form inject compare) + (syntax-match form + ((_ ((path text) ...) first body ...) + #'(with-fake-fs-of (fake-fs (list ,@{listing #'(Tuple ,p ,t) (:for p path) (:for t text)})) + ,first ,@body)) + (bad (syntax-error bad "with-fake-fs takes a list of (path contents) pairs and a body: (with-fake-fs ((\"a.txt\" \"hello\")) body ...)")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/stm.bjo — software transactional memory. +;; +;; A `(Loc T)` is a `Bjolang.Runtime.Loc` and a `Tx` is a +;; `Bjolang.Runtime.Tx`: the log of one run of a transaction. The design is the +;; one of kcas (https://github.com/ocaml-multicore/kcas), whose licence is in +;; BjolangRuntime/BjoStm.cs. A transaction that retries is a CML event that is +;; not available yet; see `AtomicallyEvent` in the same file. +(import (std prelude)) + +(:doc #:file "stm.bjodoc") + +(export + Loc Tx + make-loc loc-ref + tx-ref tx-set! tx-update! tx-swap! tx-retry tx-or-else + atomically-evt atomically try-atomically) + +(import/class + ((Loc %a) (: Bjolang.Runtime.Loc)) + (Tx (: Bjolang.Runtime.Tx))) + +(import/extern + (make-loc (: Bjolang.Runtime.StmModule.MakeLoc (-> %a (Loc %a)))) + (loc-ref (: Bjolang.Runtime.StmModule.Ref (-> (Loc %a) %a))) + (tx-ref (: Bjolang.Runtime.StmModule.Get (-> Tx (Loc %a) %a))) + (tx-set! (: Bjolang.Runtime.StmModule.Set (-> Tx (Loc %a) %a void))) + (tx-update! (: Bjolang.Runtime.StmModule.Update (-> Tx (Loc %a) (-> %a %a) %a))) + (tx-swap! (: Bjolang.Runtime.StmModule.Swap (-> Tx (Loc %a) %a %a))) + (tx-retry (: Bjolang.Runtime.StmModule.Retry (-> Tx %a))) + (tx-or-else (: Bjolang.Runtime.StmModule.OrElse (-> Tx (-> Tx %a) (-> Tx %a) %a))) + (atomically-evt (: Bjolang.Runtime.StmModule.AtomicallyEvt (-> (-> Tx %a) (Event %a)))) + (try-atomically (: Bjolang.Runtime.StmModule.TryAtomically (-> (-> Tx %a) (Option %a))))) + +;; Run and commit the transaction, and wait while it retries. Defined by +;; syncing `atomically-evt`, in the colour of the caller. +(: (-> (-> Tx %a) %a)) +(defbjouble (atomically f) + (#:sync (sync/blocking (atomically-evt f))) + (#:bjo (sync (atomically-evt f)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/stm/counter.bjo — a counter that transactions add to without +;; conflicting with each other. +;; +;; The value is the sum of several locations, the stripes. An add writes only +;; the stripe of the CPU that the transaction runs on, so transactions on +;; different CPUs that add to the counter do not conflict. A read sums all +;; stripes, so it conflicts with every add. This is the Accumulator of +;; kcas_data (https://github.com/ocaml-multicore/kcas), which selects the +;; stripe by OCaml domain. +;; +;; A transaction can run again on another CPU, and then adds to another +;; stripe. That is correct: only the sum has a meaning. +(import (std prelude)) +(import (std stm)) + +(:doc #:file "counter.bjodoc") + +(export + TCounter + make-tcounter + tcounter-add! tcounter-sub! tcounter-ref tcounter-set!) + +(import/extern + (processor-id (: System.Threading.Thread.GetCurrentProcessorId (-> int))) + (processor-count (: System.Environment.ProcessorCount int #:get))) + +(type (: (TCounter %a) #:opaque + (Record (: stripes (Vec (Loc %a)))))) + +;; One stripe for each CPU, rounded up to a power of two, so that the stripe of +;; a CPU is a mask of its id. +(: (-> int)) +(defun (stripe-count) + (let lp ((n 1)) + (if (>= n processor-count) n (lp (* n 2))))) + +(: (-> %a (TCounter %a)) (where (Num %a))) +(defun (make-tcounter initial) + (def zero (- initial initial)) + (TCounter (stripes (loop (:for i (range 0 (stripe-count))) + (:acc v (vecing (make-loc (if (= i 0) initial zero)))))))) + +(: (-> (TCounter %a) (Loc %a))) +(defun (own-stripe c) + (def stripes (record-ref c stripes)) + (vec-ref stripes (bitwise-and (processor-id) (- (vec-length stripes) 1)))) + +(: (-> Tx (TCounter %a) %a void) (where (Num %a))) +(defun (tcounter-add! tx c n) + (ignore (tx-update! tx (own-stripe c) #(+ & n)))) + +(: (-> Tx (TCounter %a) %a void) (where (Num %a))) +(defun (tcounter-sub! tx c n) + (ignore (tx-update! tx (own-stripe c) #(- & n)))) + +(: (-> Tx (TCounter %a) %a) (where (Num %a))) +(defun (tcounter-ref tx c) + (def stripes (record-ref c stripes)) + (loop (:for i (range 1 (vec-length stripes))) + (:acc sum (folding (tx-ref tx (vec-ref stripes 0)) + (+ sum (tx-ref tx (vec-ref stripes i))))) + => sum)) + +(: (-> Tx (TCounter %a) %a void) (where (Num %a))) +(defun (tcounter-set! tx c value) + (def stripes (record-ref c stripes)) + (def zero (- value value)) + (loop (:for i (range 0 (vec-length stripes))) + (:do (tx-set! tx (vec-ref stripes i) (if (= i 0) value zero))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/stm/map.bjo — a hash map for transactions. +;; +;; A hash array mapped trie in which each branch is a location of its own, as +;; in the StmMap of Haskell's stm-containers. A branch has a 32-bit bitmap and +;; one slot for each bit that is set. Level n uses bits 5n to 5n+4 of the hash +;; to select a slot: the key's `eq-hash`, mixed (see `mix`). A slot holds one +;; entry, a bucket of entries with the same hash, or the location of the next +;; branch. +;; +;; An operation reads the branches on the path to its key, and writes only the +;; branch where the slot changes. So two transactions that change different +;; keys conflict only if they change the same branch. A new key goes into the +;; root branch while its slot there is empty, so a small map is mostly the +;; root, and transactions that add keys to it conflict often. In a large map +;; the root slots point to branches, and the writes are spread over them. +;; +;; A branch that becomes empty is removed from its parent. A branch with one +;; entry left is not merged into its parent: that would write the parent. +;; +;; The size is a `TCounter`, whose stripes are selected by CPU. One location +;; would be written by every transaction that adds or removes a key, so all of +;; them would conflict. `tmap-length` reads all stripes. +(import (std prelude)) +(import (std stm)) +(import (std stm counter)) + +(:doc #:file "map.bjodoc") + +(export + TMap + ;; Named by the constrained functions, whose bodies an importer + ;; specialises. Opaque: nothing outside this module can use them. + TMapSlot TMapBranch TMapChange + make-tmap + tmap-ref tmap-try-ref tmap-ref-or tmap-contains? tmap-has-key? + tmap-set! tmap-remove! tmap-update! + tmap-length tmap-empty? tmap-fold tmap->list) + +(import/extern + (popcount (: System.Int32.PopCount (-> int int)))) + +(import/class + (KeyNotFound (: System.Collections.Generic.KeyNotFoundException + (-> string KeyNotFound)))) + +(type-rec + (: (TMapSlot %k %v) #:opaque + (Union (: Leaf int %k %v) + (: Bucket int (List (Tuple %k %v))) + (: Child (Loc (TMapBranch %k %v))))) + (: (TMapBranch %k %v) #:opaque + (Struct (: bitmap int) (: slots (Vec (TMapSlot %k %v)))))) + +;; What a change does to one slot. +(type (: (TMapChange %k %v) #:opaque + (Union Same + (: Put (TMapSlot %k %v) int) + Drop))) + +(type (: (TMap %k %v) #:opaque + (Record (: root (Loc (TMapBranch %k %v))) + (: count (TCounter int))))) + +;; --------------------------------------------------------------------------- +;; Branches +;; --------------------------------------------------------------------------- + +(: (-> (TMapBranch %k %v))) +(defun (empty-branch) (TMapBranch (bitmap 0) (slots (vec-empty)))) + +;; The hash the map uses: `eq-hash` with every bit spread over all bits, by +;; the finaliser of MurmurHash3 (Austin Appleby, public domain). The `eq-hash` +;; of an int is the int, so keys that are near each other differ only in their +;; low bits. Without the mix, fibers that add such keys at the same time would +;; write the same branches. The mix is a bijection, so +;; keys with equal hashes still share a bucket. +(: (-> int int)) +(defun (mix h0) + (def h1 (* (bitwise-xor h0 (shift-right-logical h0 16)) -2048144789)) + (def h2 (* (bitwise-xor h1 (shift-right-logical h1 13)) -1028477387)) + (bitwise-xor h2 (shift-right-logical h2 16))) + +;; The slot index of hash `h` at `level`: 0 to 31. +(: (-> int int int)) +(defun (index-at h level) + (bitwise-and (shift-right-logical h (* 5 level)) 31)) + +;; Where the slot for `bit` is in the vec: the number of set bits below it. +(: (-> int int int)) +(defun (position bitmap bit) + (popcount (bitwise-and bitmap (- bit 1)))) + +(: (-> (TMapSlot %k %v) int)) +(defun (slot-hash slot) + (match slot + ((Leaf h _ _) h) + ((Bucket h _) h) + ((Child _) 0))) + +(: (-> (TMapSlot %k %v) (TMapSlot %k %v) (Vec (TMapSlot %k %v)))) +(defun (two-slots a b) + (vec-add (vec-add (vec-empty) a) b)) + +;; A branch at `level` that holds `a` and `b`, whose hashes differ. While the +;; two hashes select the same slot, each level gets a branch of one slot. +(: (-> int (TMapSlot %k %v) (TMapSlot %k %v) (TMapBranch %k %v))) +(defun (split level a b) + (def ia (index-at (slot-hash a) level)) + (def ib (index-at (slot-hash b) level)) + (if (= ia ib) + (TMapBranch (bitmap (shift-left 1 ia)) + (slots (vec-add (vec-empty) (Child (make-loc (split (+ level 1) a b)))))) + (TMapBranch (bitmap (bitwise-ior (shift-left 1 ia) (shift-left 1 ib))) + (slots (if (< ia ib) (two-slots a b) (two-slots b a)))))) + +;; --------------------------------------------------------------------------- +;; Buckets +;; --------------------------------------------------------------------------- + +(: (-> %k (List (Tuple %k %v)) (Option %v)) (where (Eq %k))) +(defun (bucket-find k entries) + (match entries + (Nil None) + ((Cons (Tuple k2 v) rest) (if (= k k2) (Some v) (bucket-find k rest))))) + +(: (-> %k (List (Tuple %k %v)) (List (Tuple %k %v))) (where (Eq %k))) +(defun (bucket-without k entries) + (match entries + (Nil Nil) + ((Cons (Tuple k2 v) rest) + (if (= k k2) rest (Cons (Tuple k2 v) (bucket-without k rest)))))) + +;; --------------------------------------------------------------------------- +;; Lookup and change +;; --------------------------------------------------------------------------- + +(: (-> Tx (Loc (TMapBranch %k %v)) int %k int (Option %v)) (where (Eq %k))) +(defun (find-in tx loc h k level) + (def b (tx-ref tx loc)) + (def bitmap (struct-ref b bitmap)) + (def bit (shift-left 1 (index-at h level))) + (if (= 0 (bitwise-and bitmap bit)) + None + (match (vec-ref (struct-ref b slots) (position bitmap bit)) + ((Leaf h2 k2 v) (if (and (= h h2) (= k k2)) (Some v) None)) + ((Bucket h2 entries) (if (= h h2) (bucket-find k entries) None)) + ((Child sub) (find-in tx sub h k (+ level 1)))))) + +;; What `f` does to a slot that is not a branch, at `level`. +(: (-> (TMapSlot %k %v) int %k (-> (Option %v) (Option %v)) int (TMapChange %k %v)) + (where (Eq %k))) +(defun (change-slot slot h k f level) + (match slot + ((Leaf h2 k2 v2) + (if (and (= h h2) (= k k2)) + (match (f (Some v2)) + (None Drop) + ((Some v) (Put (Leaf h k v) 0))) + (match (f None) + (None Same) + ((Some v) + (if (= h h2) + (Put (Bucket h (list (Tuple k v) (Tuple k2 v2))) 1) + (Put (Child (make-loc (split (+ level 1) slot (Leaf h k v)))) 1)))))) + ((Bucket h2 entries) + (if (not (= h h2)) + (match (f None) + (None Same) + ((Some v) (Put (Child (make-loc (split (+ level 1) slot (Leaf h k v)))) 1))) + (let ((before (bucket-find k entries)) + (rest (bucket-without k entries))) + (match (f before) + (None + (match before + (None Same) + ((Some _) + (match rest + ((Cons (Tuple k1 v1) Nil) (Put (Leaf h k1 v1) -1)) + (_ (Put (Bucket h rest) -1)))))) + ((Some v) + (Put (Bucket h (Cons (Tuple k v) rest)) + (match before (None 1) ((Some _) 0)))))))) + ((Child _) Same))) + +;; Applies `f` to the entry for `k` in the branch at `loc`. Returns the change +;; of the size: -1, 0 or 1. +(: (-> Tx (Loc (TMapBranch %k %v)) int %k (-> (Option %v) (Option %v)) int int) + (where (Eq %k))) +(defun (change-in tx loc h k f level) + (def b (tx-ref tx loc)) + (def bitmap (struct-ref b bitmap)) + (def slots (struct-ref b slots)) + (def bit (shift-left 1 (index-at h level))) + (def pos (position bitmap bit)) + (if (= 0 (bitwise-and bitmap bit)) + (match (f None) + (None 0) + ((Some v) + (tx-set! tx loc (TMapBranch (bitmap (bitwise-ior bitmap bit)) (slots (vec-insert slots pos (Leaf h k v))))) + 1)) + (match (vec-ref slots pos) + ((Child sub) + (def d (change-in tx sub h k f (+ level 1))) + ;; A branch that is now empty is taken out of its parent. + (when (and (= d -1) (= 0 (struct-ref (tx-ref tx sub) bitmap))) + (tx-set! tx loc (TMapBranch (bitmap (bitwise-xor bitmap bit)) (slots (vec-remove-at slots pos))))) + d) + (slot + (match (change-slot slot h k f level) + (Same 0) + ((Put s d) + (tx-set! tx loc (TMapBranch (bitmap bitmap) (slots (vec-set slots pos s)))) + d) + (Drop + (tx-set! tx loc (TMapBranch (bitmap (bitwise-xor bitmap bit)) (slots (vec-remove-at slots pos)))) + -1)))))) + +(: (-> Tx (Loc (TMapBranch %k %v)) (-> %k %v %s %s) %s %s)) +(defun (fold-in tx loc f acc) + (loop (:for slot (in-vec (struct-ref (tx-ref tx loc) slots))) + (:acc out (folding acc + (match slot + ((Leaf _ k v) (f k v out)) + ((Bucket _ entries) + (list-foldl (fun (e a) (match e ((Tuple k v) (f k v a)))) out entries)) + ((Child sub) (fold-in tx sub f out))))) + => out)) + +;; --------------------------------------------------------------------------- +;; The map +;; --------------------------------------------------------------------------- + +(: (-> (TMap %k %v))) +(defun (make-tmap) + (TMap (root (make-loc (empty-branch))) + (count (make-tcounter 0)))) + +(: (-> Tx (TMap %k %v) %k (Option %v)) (where (Eq %k))) +(defun (tmap-try-ref tx m k) + (find-in tx (record-ref m root) (mix (eq-hash k)) k 0)) + +;; Not constrained, so that it is not specialised in the importer, where the +;; class `KeyNotFound` is not known. +(: (-> %v)) +(defun (key-not-found) + (raise (cast Exception (KeyNotFound. "tmap-ref: the key is not in the map")))) + +(: (-> Tx (TMap %k %v) %k %v) (where (Eq %k))) +(defun (tmap-ref tx m k) + (match (tmap-try-ref tx m k) + ((Some v) v) + (None (key-not-found)))) + +(: (-> Tx (TMap %k %v) %k %v %v) (where (Eq %k))) +(defun (tmap-ref-or tx m k default) + (match (tmap-try-ref tx m k) + ((Some v) v) + (None default))) + +(: (-> Tx (TMap %k %v) %k bool) (where (Eq %k))) +(defun (tmap-contains? tx m k) + (match (tmap-try-ref tx m k) + ((Some _) #t) + (None #f))) + +(: (-> Tx (TMap %k %v) %k bool) (where (Eq %k))) +(defun (tmap-has-key? tx m k) (tmap-contains? tx m k)) + +(: (-> Tx (TMap %k %v) %k (-> (Option %v) (Option %v)) void) (where (Eq %k))) +(defun (tmap-update! tx m k f) + (def h (mix (eq-hash k))) + (def d (change-in tx (record-ref m root) h k f 0)) + (when (not (= d 0)) + (tcounter-add! tx (record-ref m count) d))) + +(: (-> Tx (TMap %k %v) %k %v void) (where (Eq %k))) +(defun (tmap-set! tx m k v) + (tmap-update! tx m k (fun (_) (Some v)))) + +(: (-> Tx (TMap %k %v) %k void) (where (Eq %k))) +(defun (tmap-remove! tx m k) + (tmap-update! tx m k (fun (_) None))) + +(: (-> Tx (TMap %k %v) int)) +(defun (tmap-length tx m) + (tcounter-ref tx (record-ref m count))) + +;; Reads only the root: a branch that becomes empty is removed, so the root is +;; empty when the map is. +(: (-> Tx (TMap %k %v) bool)) +(defun (tmap-empty? tx m) + (= 0 (struct-ref (tx-ref tx (record-ref m root)) bitmap))) + +(: (-> Tx (-> %k %v %s %s) %s (TMap %k %v) %s)) +(defun (tmap-fold tx f init m) + (fold-in tx (record-ref m root) f init)) + +(: (-> Tx (TMap %k %v) (List (Tuple %k %v)))) +(defun (tmap->list tx m) + (tmap-fold tx (fun (k v acc) (Cons (Tuple k v) acc)) Nil m)) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/stm/queue.bjo — a FIFO queue for transactions. +;; +;; Two locations: `front` holds the front of the queue in order, and `back` +;; holds the rear reversed. A push writes only `back` and a pop writes only +;; `front`, so a producer and a consumer conflict only when the front is +;; empty. Then a pop moves all of `back` to `front`, reversed. Each item is +;; moved once, so every operation is O(1) amortized. +;; +;; Each side keeps its count, so the length is O(1). +;; +;; A bounded queue counts its free places in two locations, as the TBQueue of +;; Haskell's stm package does. `write` is the free places that pushes use, +;; and `freed` is the places that pops have freed since. A push takes a place +;; from `write`, and reads `freed` only when `write` is 0. So pushes and pops +;; do not conflict on the capacity until the producers run out of places. +(import (std prelude)) +(import (std stm)) + +(:doc #:file "queue.bjodoc") + +(export + TQueue + make-queue make-bounded-queue + ;; In a transaction. + queue-push! queue-pop! queue-try-pop! queue-peek queue-try-peek + queue-pop-all! queue-length queue-empty? + ;; Outside one, as the channel operations are named. + queue-send queue-recv queue-put queue-get) + +(import/class + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +(type (: (Side %a) (Struct (: count int) (: items (List %a))))) + +(type (: Room (Record (: write (Loc int)) (: freed (Loc int))))) + +(type (: (TQueue %a) #:opaque + (Record (: front (Loc (Side %a))) + (: back (Loc (Side %a))) + (: room (Option Room))))) + +;; --------------------------------------------------------------------------- +;; Making one +;; --------------------------------------------------------------------------- + +(: (-> (Side %a))) +(defun (empty-side) (Side (count 0) (items (list)))) + +(: (-> (TQueue %a))) +(defun (make-queue) + (TQueue (front (make-loc (empty-side))) + (back (make-loc (empty-side))) + (room None))) + +(: (-> int (TQueue %a))) +(defun (make-bounded-queue capacity) + (when (< capacity 1) + (raise (cast Exception (ArgumentException. "make-bounded-queue: the capacity must be at least 1")))) + (TQueue (front (make-loc (empty-side))) + (back (make-loc (empty-side))) + (room (Some (Room (write (make-loc capacity)) (freed (make-loc 0))))))) + +;; --------------------------------------------------------------------------- +;; Capacity +;; --------------------------------------------------------------------------- + +;; Takes one free place, or retries when there is none. +(: (-> Tx (Option Room) void)) +(defun (take-place! tx room) + (match room + (None unit) + ((Some r) + (def w (tx-ref tx (record-ref r write))) + (if (> w 0) + (tx-set! tx (record-ref r write) (- w 1)) + (let ((f (tx-ref tx (record-ref r freed)))) + (if (= f 0) + (tx-retry tx) + (begin + (tx-set! tx (record-ref r freed) 0) + (tx-set! tx (record-ref r write) (- f 1))))))))) + +(: (-> Tx (Option Room) int void)) +(defun (free-places! tx room n) + (match room + (None unit) + ((Some r) (ignore (tx-update! tx (record-ref r freed) #(+ & n)))))) + +;; --------------------------------------------------------------------------- +;; In a transaction +;; --------------------------------------------------------------------------- + +;; The front side, with the back moved to it when it is empty. Moves nothing +;; when both are empty, so that a pop of an empty queue writes nothing. +(: (-> Tx (TQueue %a) (Side %a))) +(defun (stocked-front tx q) + (def f (tx-ref tx (record-ref q front))) + (if (> (struct-ref f count) 0) + f + (let ((b (tx-ref tx (record-ref q back)))) + (if (= (struct-ref b count) 0) + f + (let ((moved (Side (count (struct-ref b count)) + (items (list-reverse (struct-ref b items)))))) + (tx-set! tx (record-ref q back) (empty-side)) + (tx-set! tx (record-ref q front) moved) + moved))))) + +(: (-> Tx (TQueue %a) %a void)) +(defun (queue-push! tx q x) + (take-place! tx (record-ref q room)) + (def b (tx-ref tx (record-ref q back))) + (tx-set! tx (record-ref q back) + (Side (count (+ (struct-ref b count) 1)) + (items (Cons x (struct-ref b items)))))) + +(: (-> Tx (TQueue %a) (Option %a))) +(defun (queue-try-pop! tx q) + (def f (stocked-front tx q)) + (match (struct-ref f items) + ((Cons x rest) + (tx-set! tx (record-ref q front) (Side (count (- (struct-ref f count) 1)) (items rest))) + (free-places! tx (record-ref q room) 1) + (Some x)) + (Nil None))) + +(: (-> Tx (TQueue %a) %a)) +(defun (queue-pop! tx q) + (match (queue-try-pop! tx q) + ((Some x) x) + (None (tx-retry tx)))) + +(: (-> Tx (TQueue %a) (Option %a))) +(defun (queue-try-peek tx q) + (match (struct-ref (stocked-front tx q) items) + ((Cons x _) (Some x)) + (Nil None))) + +(: (-> Tx (TQueue %a) %a)) +(defun (queue-peek tx q) + (match (queue-try-peek tx q) + ((Some x) x) + (None (tx-retry tx)))) + +(: (-> Tx (TQueue %a) (List %a))) +(defun (queue-pop-all! tx q) + (def f (tx-ref tx (record-ref q front))) + (def b (tx-ref tx (record-ref q back))) + (def n (+ (struct-ref f count) (struct-ref b count))) + (if (= n 0) + (list) + (begin + (tx-set! tx (record-ref q front) (empty-side)) + (tx-set! tx (record-ref q back) (empty-side)) + (free-places! tx (record-ref q room) n) + (list-append (struct-ref f items) (list-reverse (struct-ref b items)))))) + +(: (-> Tx (TQueue %a) int)) +(defun (queue-length tx q) + (+ (struct-ref (tx-ref tx (record-ref q front)) count) + (struct-ref (tx-ref tx (record-ref q back)) count))) + +(: (-> Tx (TQueue %a) bool)) +(defun (queue-empty? tx q) (= (queue-length tx q) 0)) + +;; --------------------------------------------------------------------------- +;; Outside a transaction +;; --------------------------------------------------------------------------- + +(: (-> (TQueue %a) %a (Event Unit))) +(defun (queue-send q x) + (atomically-evt (fun (tx) (queue-push! tx q x)))) + +(: (-> (TQueue %a) (Event %a))) +(defun (queue-recv q) + (atomically-evt (fun (tx) (queue-pop! tx q)))) + +(: (-> (TQueue %a) %a void)) +(defbjouble (queue-put q x) + (#:sync (sync/blocking (queue-send q x))) + (#:bjo (sync (queue-send q x)))) + +(: (-> (TQueue %a) %a)) +(defbjouble (queue-get q) + (#:sync (sync/blocking (queue-recv q))) + (#:bjo (sync (queue-recv q)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +(import (std prelude)) +(import (std syntax-match)) +(import (std bench)) + +(import/class + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +(import/extern (get-memory-usage (: System.GC.GetAllocatedBytesForCurrentThread))) + +(:doc #:file "stopwatch.bjodoc") + +(export Stopwatch stopwatch stopwatch-start! stopwatch-stop! stopwatch-ns stopwatch-us stopwatch-ms + ns->string get-memory-usage) + +;; A stopwatch reads the prelude's `monotonic-ns` from `current-clock` rather +;; than holding a `System.Diagnostics.Stopwatch`, so that a test can bind a +;; clock and get an exact elapsed time instead of a real one. +;; +;; `started` is the reading in nanoseconds the watch was last started at, or -1 +;; when it is not running; `elapsed` is the nanoseconds earlier runs added up +;; to. Both mutable, because a stopwatch is a thing that is started and stopped +;; rather than a value that is replaced. The other units are read from the +;; nanoseconds, so that a run under a millisecond is not lost. +(type + (: Stopwatch (Record (: started long #:mutable) + (: elapsed long #:mutable)))) + +(: (-> Stopwatch)) +(defun (stopwatch) + (Stopwatch (started -1L) (elapsed 0L))) + +(: (-> Stopwatch bool)) +(defun (running? sw) + (>= (record-ref sw started) 0L)) + +;; Starting a running watch does nothing, as restarting a .NET one does not. +(: (-> Stopwatch void)) +(defun (stopwatch-start! sw) + (unless (running? sw) + (record-set! sw (started (monotonic-ns))))) + +(: (-> Stopwatch void)) +(defun (stopwatch-stop! sw) + (when (running? sw) + (record-set! sw + (elapsed (+ (record-ref sw elapsed) (- (monotonic-ns) (record-ref sw started)))) + (started -1L)))) + +;; The reading while running includes the segment in progress, so a watch can be +;; read without stopping it. +(: (-> Stopwatch long)) +(defun (stopwatch-ns sw) + (if (running? sw) + (+ (record-ref sw elapsed) (- (monotonic-ns) (record-ref sw started))) + (record-ref sw elapsed))) + +(: (-> Stopwatch long)) +(defun (stopwatch-us sw) + (/ (stopwatch-ns sw) 1000L)) + +(: (-> Stopwatch long)) +(defun (stopwatch-ms sw) + (/ (stopwatch-ns sw) 1000000L)) + +;; `(ref sw 'ms)` — the elapsed time under the name of a unit. +(impl (Refable Stopwatch) + (type %index Symbol) + (type %out long) + (defun (ref sw sym) + (case sym + ('ns (stopwatch-ns sw)) + ('us (stopwatch-us sw)) + ('ms (stopwatch-ms sw)) + ('s (/ (stopwatch-ns sw) 1000000000L)) + (else + (raise + (cast Exception + (ArgumentException. + (str "A stopwatch is read with 'ns, 'us, 'ms or 's, and this was '" + (displayed sym) ".")))))))) + +;; A duration in the largest unit that keeps it at 1 or more, with one decimal +;; under 100: "850 ns", "12.5 us", "3.2 ms", "1.5 s". +(: (-> long string)) +(defun (ns->string ns) + (def magnitude (abs ns)) + (cond + ((< magnitude 1000L) (str (displayed ns) " ns")) + ((< magnitude 1000000L) (scaled ns 1000L " us")) + ((< magnitude 1000000000L) (scaled ns 1000000L " ms")) + (else (scaled ns 1000000000L " s")))) + +;; ns in units of `unit` nanoseconds: one decimal under 100, none above. +(: (-> long long string string)) +(defun (scaled ns unit suffix) + (def sign (if (< ns 0L) "-" "")) + (def tenths (/ (* (abs ns) 10L) unit)) + (if (< tenths 1000L) + (str sign (displayed (/ tenths 10L)) "." (displayed (% tenths 10L)) suffix) + (str sign (displayed (/ tenths 10L)) suffix))) + + +;; `time-it` is `(std bench)`'s now, measured as a benchmark is: this one is +;; the same expansion, so that a program that imports this module for it goes +;; on finding it here. A macro is seen only where its own module is imported, +;; and what its expansion names has to be reachable from there, so those are +;; re-exported. +(def/macro (time-it form inject compare) + (time-it-expansion form)) + +(re-export time-it-run time-it-fixed time-it-keep) + +;; (std stream): loops over values that work which suspends produces. +;; +;; - A `Stream` is a recipe for a producer: a bjoroutine handed an `Emitter`. +;; Holding one runs nothing. Each loop over it starts the producer in a +;; fiber of its own, receives what it emits, and stops it when the loop +;; ends, however the loop ends. +;; - `in-chan` reads a channel that something else fills, until an end event. +;; The loop does not own the producer, and leaving early does not stop it. +;; +;; Both are drivers: `(:for x s)`. A loop in a bjoroutine suspends at each +;; receive. Anywhere else the receive parks the thread. +(import (std prelude)) +(import (std syntax-match)) + +(:doc #:file "stream.bjodoc") + +(export Stream Emitter StreamRun ChanReads Item + make-stream emit! in-chan) + +;; What the producer's wrapper sends: a value, the end, or what the +;; producer raised. +(type (: (Item %a) (Union (: Next %a) Done (: Failed Exception)))) + +;; Where a producer sends its values, and receives the loop's requests for +;; the next one. +(type (: (Emitter %a) #:opaque (Struct (: chan (Chan (Item %a))) (: demand (Chan Unit))))) + +;; The producer's body. Each loop runs it afresh. +(type (: (Stream %a) #:opaque (Struct (: body (-bjo-> (Emitter %a) void))))) + +;; A loop's cursor over a stream: the running producer's channels, the cancel +;; thunk that stops it, its promise, and the item the loop is at. +(type (: (StreamRun %a) #:opaque + (Record (: chan (Chan (Item %a))) + (: demand (Chan Unit)) + (: stop (-> CancelReason void)) + (: producer (Promise Unit)) + (: item (Item %a) #:mutable)))) + +;; A channel read until `end` commits. +(type (: (ChanReads %a %e) #:opaque (Struct (: chan (Chan %a)) (: end (Event %e))))) + +;; A stream whose producer is `body`. +(: (-> (-bjo-> (Emitter %a) void) (Stream %a))) +(defun (make-stream body) (Stream (body body))) + +;; Sends `v` to the loop, then waits until the loop asks for the next value. +;; +;; - The producer computes nothing the loop has not asked for. +;; - Raises `Cancelled` when the loop has ended, so the producer unwinds and +;; its own releases run. +(: (-> (Emitter %a) %a void)) +(defbjo (emit! out v) #:inline + (sync (chan-send (.chan out) (Next v))) + (sync (chan-recv (.demand out)))) + +;; The two halves of a receive: a bjoroutine's loop suspends, any other code +;; parks the thread. +(: (-> (Chan %a) %a)) +(defbjouble (chan-take c) + (#:sync (sync/blocking (chan-recv c))) + (#:bjo (sync (chan-recv c)))) + +;; Asks the producer for its next item, and receives it. +(: (-> (Chan Unit) (Chan %a) %a)) +(defbjouble (chan-ask demand c) + (#:sync (sync/blocking (chan-send demand unit)) (sync/blocking (chan-recv c))) + (#:bjo (sync (chan-send demand unit)) (sync (chan-recv c)))) + +(: (-> (Chan %a) (Event %e) (Option %a))) +(defbjouble (chan-take-until c end) + (#:sync (sync/blocking (choose (wrap (chan-recv c) (fun (x) (Some x))) (wrap end (fun (_) None))))) + (#:bjo (sync (choose (wrap (chan-recv c) (fun (x) (Some x))) (wrap end (fun (_) None)))))) + +;; Runs the producer's body, then sends the end: `Done`, or `Failed` with what +;; the body raised. A body that was cancelled sends nothing: the loop is gone. +;; +;; - The loop asked for the first item by starting the producer, and for each +;; later one through `demand`, so the end is always asked for. +(: (-> (-bjo-> (Emitter %a) void) (Chan (Item %a)) (Chan Unit) CancelToken void)) +(defbjo (run-producer body out demand token) + (match (try (body (Emitter (chan out) (demand demand))) #:catch (System.Exception)) + ((Ok _) (sync (chan-send out Done))) + ((Err e) (if (cancelled? token) (raise e) (sync (chan-send out (Failed e))))))) + +;; Stores an item in the cursor. A `Failed` is raised here, in the loop. +(: (-> (StreamRun %a) (Item %a) (StreamRun %a))) +(defun (stream-set! run item) + (match item + ((Failed e) (raise e)) + (_ (record-set! run (item item)) run))) + +;; The longest time the end of a loop waits for its producer to unwind. +(def producer-grace (seconds 5)) + +;; Waits until `p` lands or `d` has passed, whichever is first. +(: (-> (Promise %a) Duration void)) +(defbjouble (await-within p d) + (#:sync (ignore (sync/blocking (choose (wrap (promise-join p) (fun (_) unit)) (timeout d))))) + (#:bjo (ignore (sync (choose (wrap (promise-join p) (fun (_) unit)) (timeout d)))))) + +;; Stops the producer, and waits until it has unwound. +;; +;; - After this returns, the producer's releases have run, unless it took +;; longer than `producer-grace` to unwind. +;; - The wait runs in a shield, so it also works when the loop is left +;; because its scope was cancelled. +;; - Does not raise when the grace passes. The producer then goes on +;; unwinding, and the enclosing scope waits for it. +(: (-> (StreamRun %a) CancelReason void)) +(defun (stop-producer run reason) + ((.stop run) reason) + ;; The shield's deadline is longer than `producer-grace`, so the timeout + ;; always commits first and the shield never raises `Cancelled`. + (with-shield #:within (seconds 6) + (await-within (.producer run) producer-grace))) + +;; A stream as a driver. +;; +;; - `iterable-start` starts the producer under a cancel token of its own, +;; linked to the ambient one, and receives the first item. +;; - `iterable-next` asks for the next item, and receives it. +;; - `iterable-end` and `iterable-cleanup` stop it and wait until it has +;; unwound: at a `:finish`, an `:abandon`, a driver beside it that ran out, +;; an exception or a `ret`. See `stop-producer`. +(impl (Iterable (Stream %a)) + (type %elem %a) + (type %cursor (StreamRun %a)) + (defun (iterable-start s) + (def (Tuple stop token) (make-cancel)) + (link-cancel (parameter-ref current-cancel) token) + (def c (make-chan)) + (def demand (make-chan)) + (def producer (parameterize ((current-cancel token)) (bjo (run-producer (.body s) c demand token)))) + (stream-set! (StreamRun (chan c) (demand demand) (stop stop) (producer producer) (item Done)) (chan-take c))) + (defun (iterable-done? s run) (match (.item run) (Done #t) (_ #f))) + (defun (iterable-current s run) + (match (.item run) + ((Next x) x) + (_ (panic! "iterable-current of a stream that has ended")))) + (defun (iterable-next s run) (stream-set! run (chan-ask (.demand run) (.chan run)))) + (defun (iterable-end s run) (stop-producer run (Requested "the loop over the stream ended"))) + (defun (iterable-cleanup s run) (stop-producer run (Requested "the loop over the stream was left")))) + +;; The values received on `c` until `end` commits. +(: (-> (Chan %a) (Event %e) (ChanReads %a %e))) +(defun (in-chan c end) (ChanReads (chan c) (end end))) + +(impl (Iterable (ChanReads %a %e)) + (type %elem %a) + (type %cursor (Option %a)) + (defun (iterable-start s) (chan-take-until (.chan s) (.end s))) + (defun (iterable-done? s c) (none? c)) + (defun (iterable-current s c) (option-ref c)) + (defun (iterable-next s c) (chan-take-until (.chan s) (.end s)))) + +;; `{streaming expr clause ...}`: a stream of `expr` for each round of a loop +;; of the clauses. The loop runs in the producer, so its clauses may suspend. +;; +;; - A loose `:when` arrives as a last `(:when ...)` clause. +(def/macro (streaming form inject compare) + (syntax-match form + ((_ expr clause ...) + #'(make-stream (bjoroutine (out) (loop ,@clause (:do (emit! out ,expr)))))) + (bad (syntax-error bad "streaming takes an expression and loop clauses: {streaming (* x x) (:for x xs)}")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/syntax-match.bjo +;; +;; Pattern matching for macro definitions. + +;; Imports eq. +(import (std eq)) + +(import/extern + (str-concat (: System.String.Concat))) + +(:doc #:file "syntax-match.bjodoc") + +(export all-of) + +(: (-> string string string)) +(defun (cat a b) (str-concat a b)) + +;; --------------------------------------------------------------------------- +;; What the expansion calls +;; --------------------------------------------------------------------------- + +;; all-of accepts when f accepts all items, else None. +(: (-> (List %a) (-> %a (Option %b)) (Option (List %b)))) +(defun (all-of items f) + (match items + (Nil (Some Nil)) + ((Cons item rest) + (def (Some value) (f item) :propagate) + (def (Some values) (all-of rest f) :propagate) + (Some (Cons value values))))) + +;; --------------------------------------------------------------------------- +;; Reading a pattern +;; --------------------------------------------------------------------------- + +;; Checks for ellipsis. +(: (-> Syntax bool)) +(defun (ellipsis? item) + (match item + ((SPunct text) (= text "...")) + (_ #f))) + +(: (-> Syntax bool)) +(defun (wildcard? p) + (match p + ((SSym name) (= (symbol->string name) "_")) + (_ #f))) + +;; Checks if pattern cannot fail. +(: (-> Syntax bool)) +(defun (irrefutable? p) + (match p + ((SSym _) #t) + (_ #f))) + +(: (-> Syntax Syntax bool)) +(defun (same-name? a b) + (match a + ((SSym x) + (match b + ((SSym y) (= (symbol->string x) (symbol->string y))) + (_ #f))) + (_ #f))) + +(: (-> (List Syntax) (List Syntax))) +(defun (init-of items) + (match items + (Nil Nil) + ((Cons _ Nil) Nil) + ((Cons item rest) (Cons item (init-of rest))))) + +(: (-> (List Syntax) (Option Syntax))) +(defun (last-of items) + (match items + (Nil None) + ((Cons item Nil) (Some item)) + ((Cons _ rest) (last-of rest)))) + +;; Private right fold. +(: (-> (-> %a %b %b) %b (List %a) %b)) +(defun (fold-right f init xs) + (let go ((rest (list-reverse xs)) (acc init)) + (if (list-empty? rest) + acc + (go (list-tail rest) (f (list-head rest) acc))))) + +(: (-> (List Syntax) int)) +(defun (count-ellipses items) + (let go ((rest items) (n 0)) + (if (list-empty? rest) + n + (go (list-tail rest) (if (ellipsis? (list-head rest)) (+ n 1) n))))) + +;; Extracts repeated pattern before trailing ellipsis. +(: (-> (List Syntax) Syntax (Option Syntax))) +(defun (repeated-of items whole) + (def count (count-ellipses items)) + (if (= count 0) + None + (if (> count 1) + (Some (syntax-error whole "A form repeats only its last pattern, so it may hold only one `...`.")) + (match (last-of items) + (None None) + ((Some final) + (if (not (ellipsis? final)) + (Some (syntax-error whole "`...` repeats the pattern before it, so it may only come last.")) + (match (last-of (init-of items)) + (None (Some (syntax-error whole "`...` repeats the pattern before it, and this form has none."))) + ((Some repeated) (Some repeated))))))))) + +;; Items matched one for one. +(: (-> (List Syntax) Syntax (List Syntax))) +(defun (fixed-of items whole) + (match (repeated-of items whole) + (None items) + ((Some _) (init-of (init-of items))))) + +;; Every variable a pattern binds, in the order written. +(: (-> Syntax (List Syntax))) +(defun (pattern-vars p) + (match p + ((SSym _) (if (wildcard? p) Nil (list p))) + ((SList items) + (fold-right (fun (item found) (syntax-splice (pattern-vars item) found)) Nil items)) + (_ Nil))) + +(: (-> Syntax (List Syntax) bool)) +(defun (bound-already? var seen) + (match seen + (Nil #f) + ((Cons other rest) (if (same-name? var other) #t (bound-already? var rest))))) + +(: (-> Syntax bool)) +(defun (uppercase-name? var) + (match var + ((SSym name) + (def text (symbol->string name)) + (if (string-empty? text) + #f + (char-upper-case? (string-cursor-ref text (string-cursor-start text))))) + (_ #f))) + +;; Checked bound variables. +(: (-> Syntax (List Syntax))) +(defun (checked-vars p) + (fold-right + (fun (var seen) + (if (uppercase-name? var) + (Cons (syntax-error var "A pattern variable is lowercase; write 'name to match that identifier itself.") seen) + (if (bound-already? var seen) + (Cons (syntax-error var "This pattern binds the same name twice.") seen) + (Cons var seen)))) + Nil + (pattern-vars p))) + +;; --------------------------------------------------------------------------- +;; Names the expansion binds +;; --------------------------------------------------------------------------- + +;; Positional name. +(: (-> string string Syntax)) +(defun (at-path prefix path) + (SSym (string->symbol (cat prefix path)))) + +(: (-> string int string)) +(defun (sub-path path index) + (cat path (cat "-" (.ToString index)))) + +;; The path of the pattern a `...` repeats: the form's last item. +(: (-> (List Syntax) Syntax string string)) +(defun (repeated-path items whole path) + (sub-path path (list-length (fixed-of items whole)))) + +;; --------------------------------------------------------------------------- +;; What a pattern compiles to +;; --------------------------------------------------------------------------- + +(: (-> Syntax string Syntax)) +(defun (structural p path) + (match p + ((SSym _) p) + ;; Bound not tested. + ((SDatum _) (at-path "sm-lit" path)) + ((SInt text) #'(SInt ,(SStr text))) + ((SStr _) #'(SStr ,p)) + ((SChar _) #'(SChar ,p)) + ((SKey _) #'(SKey ,p)) + ((SList items) #'(SList ,(cons-chain (fixed-of items p) path 0 (tail-of items p path)))) + (_ (syntax-error p "This is not a pattern.")))) + +;; Matches known length form. +(: (-> (List Syntax) string int Syntax Syntax)) +(defun (cons-chain items path index tail) + (match items + (Nil tail) + ((Cons item rest) + #'(Cons ,(structural item (sub-path path index)) ,(cons-chain rest path (+ index 1) tail))))) + +;; Ends cons chain. +(: (-> (List Syntax) Syntax string Syntax)) +(defun (tail-of items whole path) + (match (repeated-of items whole) + (None #'Nil) + ((Some repeated) + (if (irrefutable? repeated) repeated (at-path "sm-rest" path))))) + +(: (-> Syntax string (List Syntax))) +(defun (checks p path) + (match p + ((SDatum _) (list #'(syntax-ident=? ,(at-path "sm-lit" path) (SDatum ,p)))) + ((SList items) + (syntax-splice (fixed-checks (fixed-of items p) path 0) (repeat-check items p path))) + (_ Nil))) + +(: (-> (List Syntax) string int (List Syntax))) +(defun (fixed-checks items path index) + (match items + (Nil Nil) + ((Cons item rest) + (syntax-splice (checks item (sub-path path index)) (fixed-checks rest path (+ index 1)))))) + +;; Checks repeated items match pattern. +(: (-> (List Syntax) Syntax string (List Syntax))) +(defun (repeat-check items whole path) + (match (repeated-of items whole) + (None Nil) + ((Some repeated) + (if (irrefutable? repeated) + Nil + (list #'(some? ,(fold-over items whole path repeated #'#t))))))) + +;; Folds ellipsis over compound. +(: (-> (List Syntax) Syntax string Syntax Syntax Syntax)) +(defun (fold-over items whole path repeated value) + (def item (at-path "sm-item" path)) + (def inner (repeated-path items whole path)) + #'(all-of ,(at-path "sm-rest" path) + (fun (,item) + (match ,item + (,(structural repeated inner) ,(answer (checks repeated inner) #'(Some ,value))) + (_ None))))) + +;; Item answer. +(: (-> (List Syntax) Syntax Syntax)) +(defun (answer tests some) + (if (list-empty? tests) some #'(if ,(conjunction tests) ,some None))) + +(: (-> (List Syntax) Syntax)) +(defun (conjunction tests) + (match tests + (Nil #'#t) + ((Cons only Nil) only) + (_ #'(and ,@tests)))) + +;; Variable value source. +(: (-> Syntax Syntax string (Option Syntax))) +(defun (value-of p var path) + (match p + ((SSym _) (if (same-name? p var) (Some p) None)) + ((SList items) + (match (fixed-value (fixed-of items p) var path 0) + ((Some found) (Some found)) + (None (repeated-value items p var path)))) + (_ None))) + +(: (-> (List Syntax) Syntax string int (Option Syntax))) +(defun (fixed-value items var path index) + (match items + (Nil None) + ((Cons item rest) + (match (value-of item var (sub-path path index)) + ((Some found) (Some found)) + (None (fixed-value rest var path (+ index 1))))))) + +(: (-> (List Syntax) Syntax Syntax string (Option Syntax))) +(defun (repeated-value items whole var path) + (def (Some repeated) (repeated-of items whole) :propagate) + (if (irrefutable? repeated) + ;; The tail binder, bound by the pattern like any other variable. + (if (same-name? repeated var) (Some repeated) None) + (match (value-of repeated var (repeated-path items whole path)) + (None None) + ((Some inner) + ;; option-ref takes answer. + (Some #'(option-ref ,(fold-over items whole path repeated inner))))))) + +;; --------------------------------------------------------------------------- +;; A clause +;; --------------------------------------------------------------------------- + +;; Clause bindings. +(: (-> Syntax (List Syntax))) +(defun (binds p) + (fold-right + (fun (var bindings) + (match (value-of p var "") + (None bindings) + ((Some value) + (if (same-name? value var) bindings (Cons #'(,var ,value) bindings))))) + Nil + (checked-vars p))) + +(: (-> (List Syntax) Syntax)) +(defun (single-body bodies) + (match bodies + ((Cons body Nil) body) + (_ #'(let () ,@bodies)))) + +(: (-> Syntax Syntax)) +(defun (compile-clause clause) + (match clause + ((SList (Cons p bodies)) + (if (list-empty? bodies) + (syntax-error clause "A syntax-match clause is (pattern body ...), and this one has no body.") + (let ((tests (checks p "")) + (bindings (binds p)) + (body (single-body bodies))) + (def whole (if (list-empty? bindings) body #'(let ,(SList bindings) ,body))) + (if (list-empty? tests) + #'(,(structural p "") ,whole) + #'(,(structural p "") #:when ,(conjunction tests) ,whole))))) + (_ (syntax-error clause "A syntax-match clause is (pattern body ...).")))) + +;; True if last clause covers everything. +(: (-> (List Syntax) bool)) +(defun (covered? clauses) + (match (last-of clauses) + (None #f) + ((Some clause) + (match clause + ((SList (Cons p _)) (irrefutable? p)) + (_ #f))))) + +(: (-> (List Syntax) (List Syntax))) +(defun (arms clauses) + (syntax-splice + ;; Built backwards from the reversed input, so it comes out in order. + (let go ((rest (list-reverse clauses)) (acc Nil)) + (if (list-empty? rest) + acc + (go (list-tail rest) (cons (compile-clause (list-head rest)) acc)))) + (if (covered? clauses) + Nil + ;; Reports unmatched form. + (list #'(sm-unmatched (syntax-error sm-unmatched "No clause of this syntax-match matched.")))))) + +;; --------------------------------------------------------------------------- +;; The macro +;; --------------------------------------------------------------------------- + +;; syntax-match macro. +(def/macro (syntax-match form inject compare) + (match form + ((SList (Cons _ (Cons subject clauses))) + (if (list-empty? clauses) + (syntax-error form "syntax-match needs at least one clause.") + #'(match ,subject ,@(arms clauses)))) + (_ (syntax-error form "syntax-match takes a subject and at least one clause: (syntax-match form (pattern body ...) ...)")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/test.bjo — tests that are silent unless they fail. +;; +;; (import (std prelude)) +;; (import (std test)) +;; +;; (deftest "parse adds" +;; (check (= (parse "1+2") (Ok 3)))) +;; +;; (deftest "a bad number is an error" +;; (check-match (parse "x") (Err _))) +;; +;; (run-tests) +;; +;; `deftest` registers a test where it is written, and `(run-tests)` is the +;; file's `main`: it runs every test the module registered, in source order. A +;; test that passes prints nothing. A test that fails prints the test, where it +;; is, and every check in it that failed, with the values the check saw. The +;; run ends with one line, and exits 1 if anything failed. +;; +;; Arguments to the program choose tests: a test runs when its name contains +;; one of them. With none, every test runs. Arguments that start with `--` are +;; flags; see `parse-test-args`. +;; +;; Each test runs in a scope of its own, as `with-test` in `(std simpletest)` +;; does: under a deadline (`#:within d`, a Duration, 5 seconds by default), waiting for +;; the fibers it started, and failing if it left a resource open (unless +;; `#:open-ok`). A test that raises fails with the exception's message, and the +;; tests after it still run. +;; +;; Property tests are in `(std test prop)`, generators in `(std test gen)`, +;; and output snapshots in `(std test expect)`. They hook into a run through +;; `running-test`, `current-test-options`, `test-seed` and `at-run-end!`. + +(export deftest run-tests + check check-match check-raises check-fail check-approx check-output + Test TestResult TestStatus registered-tests module-tests run-test run-registered-tests + test-name test-module test-file test-line test-result-test test-result-failures + test-result-status test-failed! + ;; What the extensions of a run read + TestOptions parse-test-args current-test-options running-test test-seed + test-time-left next-test-index! take-test-output! + mix-seed string-hash at-run-end! failures-into + ;; Named by the templates of the macros above, so they have to be + ;; reachable where those expand. + register-test! check-failed! deftest-cases-row check-raises-message approx-close?) + +(import (std prelude)) +(import (std syntax-match)) +(import (std random)) + +(:doc #:file "test.bjodoc") + +;; A failure makes the program exit 1 even when `main` answers 0. +(import/extern + (set-exit-code! (: System.Environment.ExitCode (-> int void) #:set))) + +;; --------------------------------------------------------------------------- +;; Tests and their registry +;; --------------------------------------------------------------------------- + +;; A registered test. `body` is the test's forms, run when the test runs. +;; `skip` and `todo` are the reasons given with `#:skip` and `#:todo`, and +;; empty when the test has none. `capture` is #t when the body uses +;; `expect-output`: its output then goes to a string port that +;; `take-test-output!` reads. +(type (: Test (Record (: name string) + (: module string) + (: file string) + (: line int) + (: within (Option Duration)) + (: open-ok bool) + (: skip string) + (: todo string) + (: slow bool) + (: capture bool) + (: body (-> void))))) + +(: (-> Test string)) +(defun (test-name t) (record-ref t name)) + +(: (-> Test string)) +(defun (test-module t) (record-ref t module)) + +(: (-> Test string)) +(defun (test-file t) (record-ref t file)) + +(: (-> Test int)) +(defun (test-line t) (record-ref t line)) + +;; Newest first. A box rather than a `def/mutable`, so that `register-test!` +;; can be inlined into the module that calls it. +(: (Box (List Test))) +(def registry (make-box (list))) + +;; Adds a test. It answers a number only so that `deftest` can bind it at the +;; top level, which is what makes it run before `main`. +(: (-> string string string int (Option Duration) bool string string bool bool (-> void) int)) +(defun (register-test! name module file line within open-ok skip todo slow capture body) + (box-set! registry + (cons (Test (name name) (module module) (file file) (line line) + (within within) (open-ok open-ok) (skip skip) (todo todo) + (slow slow) (capture capture) (body body)) + (box-ref registry))) + 0) + +;; Every test registered so far, in the order they were written. +(: (-> (List Test))) +(defun (registered-tests) (list-reverse (box-ref registry))) + +;; The tests one module registered, by the key `(current-module)` answers +;; there, in the order they were written. Every module that imports this one +;; registers into the same list, and each runs only its own. +(: (-> string (List Test))) +(defun (module-tests module) + (list-filter (fun (t) (= (record-ref t module) module)) (registered-tests))) + +;; --------------------------------------------------------------------------- +;; The options of a run +;; --------------------------------------------------------------------------- + +;; What the arguments of a test program asked for. `seed` is always set: from +;; `--seed`, from BJO_TEST_SEED, or from the OS. `runs` and `shrink-limit` are +;; 0 when not given, which means each `for-all`'s own default. +(type (: TestOptions (Record (: filters (Vec string)) + (: seed ulong) + (: runs int) + (: shrink-limit int) + (: stats bool) + (: quick bool) + (: promote bool) + (: store bool) + (: junit (Option string))))) + +;; A number in decimal, or in hex after 0x. +(: (-> string (Option ulong))) +(defun (parse-ulong s) + (def chars (string->list s)) + (def hex? (match chars ((Cons #\0 (Cons (or #\x #\X) _)) #t) (_ #f))) + (def digits (if hex? (list-tail (list-tail chars)) chars)) + (def base (if hex? 16UL 10UL)) + (if (list-empty? digits) + None + (let go ((left digits) (acc 0UL)) + (match left + (Nil (Some acc)) + ((Cons ch more) + (def d (char->int ch)) + (def v (cond ((and (>= d 48) (<= d 57)) (- d 48)) + ((and hex? (>= d 97) (<= d 102)) (- d 87)) + ((and hex? (>= d 65) (<= d 70)) (- d 55)) + (else -1))) + (if (< v 0) + None + (go more (+ (* acc base) (cast System.UInt64 v))))))))) + +(: (-> string string (Result string int))) +(defun (parse-count flag s) + (match (parse-ulong s) + ((Some n) #:when (< n 1000000000UL) (Ok (cast System.Int32 n))) + (_ (Err #"${flag} takes a number, and this is not one: ${s}")))) + +;; The seed of a run that gives none: BJO_TEST_SEED, else 64 bits from the OS. +(: (-> (Result string ulong))) +(defun (fresh-seed) + (match (get-environment-variable "BJO_TEST_SEED") + ((Some s) + (option->result (parse-ulong s) #"BJO_TEST_SEED is not a number: ${s}")) + (None (Ok (random-u64 #:rng (make-random)))))) + +;; The arguments of a test program: flags, and name filters. Flags start with +;; `--`: `--seed n`, `--runs n`, `--shrink-limit n`, `--junit file`, and +;; `--stats`, `--quick`, `--promote`, `--no-store`. +(: (-> (Vec string) (Result string TestOptions))) +(defun (parse-test-args args) + (def (Ok seed0) (fresh-seed) :propagate) + (let go ((left (vec->list args)) (filters (list)) (seed seed0) (runs 0) (shrink 0) + (stats #f) (quick #f) (promote #f) (store #t) (junit None)) + (defun (value-of flag more k) + (match more + ((Cons v after) (k v after)) + (Nil (Err #"${flag} needs a value.")))) + (match left + (Nil + (Ok (TestOptions (filters (list->vec (list-reverse filters))) (seed seed) (runs runs) + (shrink-limit shrink) (stats stats) (quick quick) (promote promote) + (store store) (junit junit)))) + ((Cons a more) + (case a + ("--seed" + (value-of a more + (fun (v after) + (match (parse-ulong v) + ((Some n) (go after filters n runs shrink stats quick promote store junit)) + (None (Err #"--seed takes a number, in decimal or 0x hex: ${v}")))))) + ("--runs" + (value-of a more + (fun (v after) + (match (parse-count a v) + ((Ok n) (go after filters seed n shrink stats quick promote store junit)) + ((Err e) (Err e)))))) + ("--shrink-limit" + (value-of a more + (fun (v after) + (match (parse-count a v) + ((Ok n) (go after filters seed runs n stats quick promote store junit)) + ((Err e) (Err e)))))) + ("--junit" + (value-of a more + (fun (v after) (go after filters seed runs shrink stats quick promote store (Some v))))) + ("--stats" (go more filters seed runs shrink #t quick promote store junit)) + ("--quick" (go more filters seed runs shrink stats #t promote store junit)) + ("--promote" (go more filters seed runs shrink stats quick #t store junit)) + ("--no-store" (go more filters seed runs shrink stats quick promote #f junit)) + (else + (if (string-starts-with? a "--") + (Err #"unknown flag ${a}. The flags are --seed, --runs, --shrink-limit, --junit, --stats, --quick, --promote and --no-store.") + (go more (cons a filters) seed runs shrink stats quick promote store junit)))))))) + +;; The options of a run with no arguments, for tests run with `run-test` +;; outside `(run-tests)`. +(: (-> TestOptions)) +(defun (default-options) + (match (parse-test-args (vec-empty)) + ((Ok o) o) + ((Err why) (panic! why)))) + +(: (Box TestOptions)) +(def options-box (make-box (default-options))) + +;; The options of the run in progress. +(: (-> TestOptions)) +(defun (current-test-options) (box-ref options-box)) + +;; --------------------------------------------------------------------------- +;; Seeds +;; --------------------------------------------------------------------------- + +;; FNV-1a over the string's characters. Not `GetHashCode`, which .NET +;; randomizes per process: a test's seed has to be the same in every run. +(: (-> string ulong)) +(defun (string-hash s) + (string-fold (fun (ch h) + (* (bitwise-xor h (cast System.UInt64 (char->int ch))) 1099511628211UL)) + 14695981039346656037UL + s)) + +;; SplitMix64's finalizer over a xor b: two seeds that differ in one bit give +;; unrelated results. +(: (-> ulong ulong ulong)) +(defun (mix-seed a b) + (def z0 (bitwise-xor a b)) + (def z1 (* (bitwise-xor z0 (shift-right z0 30)) 13787848793156543929UL)) + (def z2 (* (bitwise-xor z1 (shift-right z1 27)) 10723151780598845931UL)) + (bitwise-xor z2 (shift-right z2 31))) + +;; The seed of one test: the run's seed and the test's name. So choosing +;; tests, or adding one, does not change what the others draw. +(: (-> Test ulong)) +(defun (test-seed t) + (mix-seed (record-ref (current-test-options) seed) (string-hash (record-ref t name)))) + +;; --------------------------------------------------------------------------- +;; Failures +;; --------------------------------------------------------------------------- + +;; The failures of the test that is running, newest first, or `None` when no +;; test is running. A check outside a test reports at once. +(: (Box (Option (Box (List string))))) +(def current-failures (make-box None)) + +;; The test that is running, or `None`. +(: (Box (Option Test))) +(def running-box (make-box None)) + +(: (-> (Option Test))) +(defun (running-test) (box-ref running-box)) + +;; When the running test started, and its deadline in milliseconds. +(: (Box (Tuple long long))) +(def running-clock (make-box (Tuple 0L 0L))) + +;; The milliseconds left before the running test's deadline, or a day when no +;; test runs. Shrinking stops early when this gets low. +(: (-> long)) +(defun (test-time-left) + (match (box-ref running-box) + (None 86400000L) + ((Some _) + (match (box-ref running-clock) + ((Tuple started within) + (- (+ started within) (monotonic-ms))))))) + +;; The port a capturing test writes to, and how far `take-test-output!` has +;; read it. +(: (Box (Option (Tuple OutputPort StringCursor)))) +(def output-box (make-box None)) + +;; What the running test wrote since it started, or since the last call. None +;; when the test does not capture its output. +(: (-> (Option string))) +(defun (take-test-output!) + (match (box-ref output-box) + (None None) + ((Some (Tuple port from)) + (def text (get-output-string port)) + (box-set! output-box (Some (Tuple port (string-cursor-end text)))) + (Some (substring text from (string-cursor-end text)))))) + +;; How many times `next-test-index!` was called with each key in the running +;; test. A `for-all` takes its position among those with the same bindings +;; from it, to find its stored cases. +(: (Box (Map string int))) +(def index-box (make-box (map-empty))) + +(: (-> string int)) +(defun (next-test-index! key) + (def counts (box-ref index-box)) + (def i (map-ref-or counts key 0)) + (box-set! index-box (map-set counts key (+ i 1))) + i) + +;; Runs `body` with its failures going to `failures` instead of to the running +;; test, and puts the running test's back afterwards, also when `body` raises. +;; `for-all` runs each case this way, to decide which failures to keep. +(: (-> (Box (List string)) (-> %a) %a)) +(defun (failures-into failures body) + (def outer (box-ref current-failures)) + (box-set! current-failures (Some failures)) + (match (try (body) #:catch (System.Exception)) + ((Ok v) (box-set! current-failures outer) v) + ((Err e) (box-set! current-failures outer) (raise e)))) + +;; A source file as a report names it: by its file name, which is enough to +;; find it and short enough to read. +(: (-> string string)) +(defun (shown-file path) + (match (path-filename path) + ((Some name) name) + (None path))) + +;; Records a failure of the running test, or prints it and fails the program +;; when there is none. +(: (-> string void)) +(defun (test-failed! message) + (match (box-ref current-failures) + ((Some failures) (box-set! failures (cons message (box-ref failures)))) + (None + (println message) + (set-exit-code! 1)))) + +;; The report of one failed check: where it is, what it said, and each value +;; it saw, as source text and value. +(: (-> string int string (List (Tuple string string)) void)) +(defun (check-failed! file line source seen) + (def width + (loop (:for (Tuple src _) seen) + (:acc w (folding 0 (let ((n (string-length src))) (if (> n w) n w)))))) + (def lines + (loop (:for (Tuple src value) seen) + (:acc out (listing #" ${(string-pad-right src width)} => ${value}")))) + (test-failed! + (string-join (list->vec (cons #" ${(shown-file file)}:${line}: ${source}" lines)) "\n"))) + +;; --------------------------------------------------------------------------- +;; Checks +;; --------------------------------------------------------------------------- + +;; The source position of a form, as literals a template can splice. +(: (-> Syntax Syntax)) +(defun (file-of form) (SStr (syntax-file form))) + +(: (-> Syntax Syntax)) +(defun (line-of form) (SInt (int->string (syntax-line form)))) + +(: (-> Syntax Syntax)) +(defun (source-of form) (SStr (syntax->string form))) + +;; A comparison, with both operands evaluated once, in order, and shown when +;; it does not hold. +(: (-> Syntax Syntax Syntax Syntax Syntax)) +(defun (check-comparison form op a b) + #'(let ((left ,a) + (right ,b)) + (unless (,op left right) + (check-failed! ,(file-of form) ,(line-of form) ,(source-of form) + (list (Tuple ,(source-of a) (displayed left)) + (Tuple ,(source-of b) (displayed right))))))) + +;; `(check expr)` passes when `expr` is #t. When it is a comparison — `=`, `<`, +;; `>`, `<=` or `>=` — a failure shows both operands' values; otherwise it says +;; that the expression was #f. A failed check does not stop the test. +(def/macro (check form inject compare) + (syntax-match form + ((_ ('= a b)) (check-comparison form #'= a b)) + ((_ ('< a b)) (check-comparison form #'< a b)) + ((_ ('> a b)) (check-comparison form #'> a b)) + ((_ ('<= a b)) (check-comparison form #'<= a b)) + ((_ ('>= a b)) (check-comparison form #'>= a b)) + ((_ expr) + #'(unless ,expr + (check-failed! ,(file-of form) ,(line-of form) ,(source-of form) + (list (Tuple ,(source-of expr) "#f"))))) + (bad (syntax-error bad "check takes one expression: (check (= (f x) 3)).")))) + +;; `(check-match expr pattern)` passes when the value matches the pattern. A +;; failure shows the value. +(def/macro (check-match form inject compare) + (syntax-match form + ((_ expr pattern) + #'(let ((value ,expr)) + (match value + (,pattern unit) + (_ (check-failed! ,(file-of form) ,(line-of form) ,(source-of form) + (list (Tuple ,(source-of expr) (displayed value)))))))) + (bad (syntax-error bad "check-match takes an expression and a pattern: (check-match (parse \"x\") (Err _)).")))) + +;; The part of `check-raises` that `#:message` adds: the exception was of the +;; right type, and now its message has to contain `part`. +(: (-> string int string string string void)) +(defun (check-raises-message file line source message part) + (unless (string-contains? message part) + (check-failed! file line source + (list (Tuple "message" message) (Tuple "should contain" part))))) + +;; `(check-raises Type expr)` passes when evaluating `expr` raises a `Type`, +;; a .NET exception type. A failure shows what it answered instead. With +;; `#:message "part"`, the exception's message has to contain `part` too. +(def/macro (check-raises form inject compare) + (syntax-match form + ((_ type expr #:message part) + #'(match (try ,expr #:catch (,type)) + ((Ok value) + (check-failed! ,(file-of form) ,(line-of form) ,(source-of form) + (list (Tuple ,(source-of expr) (str "answered " (displayed value) " and raised nothing"))))) + ((Err e) (check-raises-message ,(file-of form) ,(line-of form) ,(source-of form) (.-Message e) ,part)))) + ((_ type expr) + #'(match (try ,expr #:catch (,type)) + ((Ok value) + (check-failed! ,(file-of form) ,(line-of form) ,(source-of form) + (list (Tuple ,(source-of expr) (str "answered " (displayed value) " and raised nothing"))))) + ((Err _) unit))) + (bad (syntax-error bad "check-raises takes an exception type and an expression: (check-raises System.FormatException (parse-int \"x\")), optionally #:message \"part\" after them.")))) + +;; `(check-fail "why")` fails where it stands: for a branch a test must not +;; reach. +(def/macro (check-fail form inject compare) + (syntax-match form + ((_ why) + #'(check-failed! ,(file-of form) ,(line-of form) ,(source-of form) + (list (Tuple "reason" ,why)))) + (bad (syntax-error bad "check-fail takes a reason: (check-fail \"this branch is unreachable\").")))) + +;; Whether two doubles are within `eps` of each other: absolutely near 0, and +;; relative to the larger of the two away from it. Equal infinities are close; +;; NaN is close to nothing. +(: (-> double double double bool)) +(defun (approx-close? a b eps) + (or (= a b) + (let* ((diff (abs (- a b))) + (scale (max 1.0 (max (abs a) (abs b))))) + (<= diff (* eps scale))))) + +;; `(check-approx a b)` passes when two doubles are within 1e-9 of each other, +;; relative to their size when that is above 1. `#:eps e` changes the bound. +(: (-> Syntax Syntax Syntax Syntax Syntax)) +(defun (check-approx-expansion form a b eps) + #'(let ((left ,a) (right ,b)) + (unless (approx-close? left right ,eps) + (check-failed! ,(file-of form) ,(line-of form) ,(source-of form) + (list (Tuple ,(source-of a) (displayed left)) + (Tuple ,(source-of b) (displayed right))))))) + +(def/macro (check-approx form inject compare) + (syntax-match form + ((_ a b #:eps eps) (check-approx-expansion form a b eps)) + ((_ a b) (check-approx-expansion form a b #'1e-9)) + (bad (syntax-error bad "check-approx takes two doubles: (check-approx (sqrt 2.0) 1.4142135623730951), optionally #:eps 1e-6 after them.")))) + +;; `(check-output expr "text")` passes when what `expr` writes to +;; `current-output-port` is the text. +(def/macro (check-output form inject compare) + (syntax-match form + ((_ expr text) + #'(let* ((port (open-output-string)) + (ignored (parameterize ((current-output-port port)) ,expr unit)) + (written (get-output-string port)) + (wanted ,text)) + (unless (= written wanted) + (check-failed! ,(file-of form) ,(line-of form) ,(source-of form) + (list (Tuple "it wrote" (displayed written)) + (Tuple "wanted" (displayed wanted))))))) + (bad (syntax-error bad "check-output takes an expression and a string: (check-output (display \"hi\") \"hi\").")))) + +;; --------------------------------------------------------------------------- +;; deftest +;; --------------------------------------------------------------------------- + +;; The deadline of a test that gives none. +(: Duration) +(def default-test-deadline (seconds 5)) + +;; What the options of one `deftest` came to. `cases` is the `#:cases` table: +;; the column names, then the rows. +(type (: DeftestOptions (Record (: within Syntax) + (: open-ok bool) + (: skip Syntax) + (: todo Syntax) + (: slow bool) + (: cases (Option (Tuple (List Syntax) (List Syntax)))) + (: body (List Syntax))))) + +(: (-> (List Syntax) DeftestOptions)) +(defun (no-options body) + (DeftestOptions (within #'None) (open-ok #f) (skip #'"") (todo #'"") (slow #f) + (cases None) (body body))) + +;; Reads the options after a test's name, in any order, up to the body. +(: (-> Syntax (List Syntax) DeftestOptions)) +(defun (deftest-options form items) (options-from form items (no-options (list)))) + +(: (-> Syntax (List Syntax) DeftestOptions DeftestOptions)) +(defun (options-from form items o) + (let ((go (fun (more o2) (options-from form more o2)))) + (syntax-match (SList items) + ((#:within d more ...) (go more (record-set o (within #'(Some ,d))))) + ((#:open-ok more ...) (go more (record-set o (open-ok #t)))) + ((#:skip why more ...) (go more (record-set o (skip why)))) + ((#:todo why more ...) (go more (record-set o (todo why)))) + ((#:slow more ...) (go more (record-set o (slow #t)))) + ((#:cases ((column ...) row ...) more ...) (go more (with-cases form column row o))) + ((first more ...) (record-set o (body (cons first more)))) + (_ (ignore (syntax-error form "deftest takes a name and a body: (deftest \"name\" body ...), with options such as #:within (seconds 2), #:open-ok, #:skip \"why\", #:todo \"why\", #:slow and #:cases after the name.")) + o)))) + +;; `#:cases ((a b) row ...)`: the first list names the columns, and each row +;; after it gives them values. +(: (-> Syntax (List Syntax) (List Syntax) DeftestOptions DeftestOptions)) +(defun (with-cases form columns rows o) + (def width (list-length columns)) + (when (list-empty? rows) + (ignore (syntax-error form "#:cases has no rows. It is the column names, then rows of as many values: #:cases ((input expected) (\"1\" 1) (\"2\" 2))."))) + (loop (:for row rows) + (:do (match row + ((SList items) + (unless (= (list-length items) width) + (ignore (syntax-error row #"This row of #:cases has ${(list-length items)} values, and there are ${width} columns.")))) + (_ (ignore (syntax-error row "A row of #:cases is a list of values, one for each column.")))))) + (record-set o (cases (Some (Tuple columns rows))))) + +;; The name of one row of a `#:cases` test, and its values, for its report. +(: (-> int (List (Tuple string string)) string)) +(defun (deftest-cases-row i seen) + (string-join + (list->vec (cons #" case ${i}:" + (list-map (fun (p) (match p ((Tuple n v) #" ${n} = ${v}"))) seen))) + "\n")) + +;; One row's body: the columns bound to the row's values. A column that the +;; body uses as the pattern of a `check-match` is spliced in instead, so a row +;; can hold a pattern such as `(Err _)`. +(: (-> Syntax int (List Syntax) Syntax (List Syntax) Syntax)) +(defun (case-body form i columns row body) + (def values (syntax-match row ((v ...) v) (_ (list)))) + (def pairs (zip-syntax columns values)) + (def pattern-columns (list-filter (fun (p) (used-as-pattern? (fst p) body)) pairs)) + (def value-columns (list-filter (fun (p) (not (used-as-pattern? (fst p) body))) pairs)) + (def spliced (list-map (fun (b) (splice-patterns pattern-columns b)) body)) + #'(let ((case-failures (make-box (list)))) + (failures-into case-failures + (fun () + (let* ,(SList (list-map (fun (p) #'(,(fst p) ,(snd p))) value-columns)) + ,@spliced + unit))) + (unless (list-empty? (box-ref case-failures)) + (test-failed! + (string-join + (list->vec + (cons (deftest-cases-row ,(SInt (int->string i)) + (list ,@(list-map (fun (p) #'(Tuple ,(SStr (syntax->string (fst p))) + ,(SStr (syntax->string (snd p))))) + pairs))) + (list-reverse (box-ref case-failures)))) + "\n"))))) + + +(: (-> (List Syntax) (List Syntax) (List (Tuple Syntax Syntax)))) +(defun (zip-syntax xs ys) + (match (Tuple xs ys) + ((Tuple (Cons x xm) (Cons y ym)) (cons (Tuple x y) (zip-syntax xm ym))) + (_ (list)))) + +;; Whether `name` stands in the pattern position of a `check-match` anywhere +;; in the forms. +(: (-> Syntax (List Syntax) bool)) +(defun (used-as-pattern? name forms) + (any? (fun (f) + (or (match (check-match-pattern f) + ((Some p) (syntax-ident=? p name)) + (None #f)) + (match f + ((SList items) (used-as-pattern? name items)) + (_ #f)))) + forms)) + +;; The pattern of a `(check-match expr pattern)` whose pattern is a name. +(: (-> Syntax (Option Syntax))) +(defun (check-match-pattern f) + (match f + ((SList (Cons (SSym head) (Cons _ (Cons (SSym p) Nil)))) + (if (= (symbol->string head) "check-match") (Some (SSym p)) None)) + (_ None))) + +;; The forms with each pattern column in a `check-match` replaced by its +;; row's value. +(: (-> (List (Tuple Syntax Syntax)) Syntax Syntax)) +(defun (splice-patterns columns form) + (match (check-match-pattern form) + ((Some p) + (match form + ((SList (Cons head (Cons e _))) + (match (list-filter (fun (c) (syntax-ident=? (fst c) p)) columns) + ((Cons c _) (syntax-at (SList (list head (splice-patterns columns e) (snd c))) form)) + (Nil form))) + (_ form))) + (None + (match form + ((SList items) (syntax-at (SList (list-map (fun (x) (splice-patterns columns x)) items)) form)) + (_ form))))) + +(: (-> Syntax Syntax DeftestOptions Syntax)) +(defun (deftest-expansion form name o) + (def body + (match (record-ref o cases) + (None (record-ref o body)) + ((Some (Tuple columns rows)) + (loop (:with i 1 (+ i 1)) + (:for row rows) + (:acc out (listing (case-body form i columns row (record-ref o body)))))))) + #'(def registered + (register-test! ,name (current-module) ,(file-of form) ,(line-of form) + ,(record-ref o within) ,(if (record-ref o open-ok) #'#t #'#f) + ,(record-ref o skip) ,(record-ref o todo) ,(if (record-ref o slow) #'#t #'#f) + ,(if (mentions? "expect-output" body) #'#t #'#f) + (fun () ,@body unit)))) + +;; Whether a name appears anywhere in the forms. +(: (-> string (List Syntax) bool)) +(defun (mentions? name forms) + (any? (fun (f) + (match f + ((SSym s) (= (symbol->string s) name)) + ((SList items) (mentions? name items)) + (_ #f))) + forms)) + +;; `(deftest "name" body ...)`, with options after the name in any order: +;; `#:within d` (a Duration), `#:open-ok`, `#:skip "why"`, `#:todo "why"`, `#:slow` and +;; `#:cases ((column ...) row ...)`. Registers the test; `(run-tests)` runs it. +(def/macro (deftest form inject compare) + (syntax-match form + ((_ name first more ...) + (deftest-expansion form name (deftest-options form (cons first more)))) + (bad (syntax-error bad "deftest takes a name and a body: (deftest \"name\" body ...).")))) + +;; --------------------------------------------------------------------------- +;; Running +;; --------------------------------------------------------------------------- + +;; How a test came out. `Todo` is a `#:todo` test that failed, as it was +;; expected to, and `Fixed` one that passed. +(type (: TestStatus (Union Passed Failed Skipped Todo Fixed))) + +;; What running one test came to: its failures, in the order they happened. +(type (: TestResult (Record (: test Test) + (: status TestStatus) + (: failures (List string))))) + +(: (-> TestResult Test)) +(defun (test-result-test r) (record-ref r test)) + +(: (-> TestResult TestStatus)) +(defun (test-result-status r) (record-ref r status)) + +(: (-> TestResult (List string))) +(defun (test-result-failures r) (record-ref r failures)) + +;; Whether the deadline was what ended the scope. A fiber the test leaked is +;; cancelled by the deadline rather than failing, so the scope closes cleanly +;; and the test would otherwise pass, slowly. +(: (-> (Box (Option Scope)) bool)) +(defun (overran? holder) + (match (box-ref holder) + (None #f) + ((Some s) + (match (cancel-reason (scope-token s)) + ((Some Deadline) #t) + (_ #f))))) + +;; Runs one test in a scope of its own and answers what it came to. Prints +;; nothing. A `#:skip` test does not run. +(: (-> Test TestResult)) +(defun (run-test t) + (if (not (string-empty? (record-ref t skip))) + (TestResult (test t) (status Skipped) (failures (list))) + (run-test-body t))) + +(: (-> Test TestResult)) +(defun (run-test-body t) + (def failures (make-box (list))) + (def outer (box-ref current-failures)) + (def outer-test (box-ref running-box)) + (box-set! current-failures (Some failures)) + (box-set! running-box (Some t)) + (def holder (make-box None)) + (def body (record-ref t body)) + (def within (match (record-ref t within) ((Some d) d) (None default-test-deadline))) + (def outer-clock (box-ref running-clock)) + (def outer-index (box-ref index-box)) + (box-set! running-clock (Tuple (monotonic-ms) (duration->timer-ms within))) + (box-set! index-box (map-empty)) + (def outer-output (box-ref output-box)) + (def port (open-output-string)) + (box-set! output-box (if (record-ref t capture) (Some (Tuple port (string-cursor-start ""))) None)) + (match (try (with-deadline within + (box-set! holder (Some (current-scope))) + (if (record-ref t capture) + (parameterize ((current-output-port port)) (body)) + (body)) + (unless (record-ref t open-ok) + (let ((left (scope-owned-count (current-scope)))) + (unless (= left 0) + (test-failed! #" it left ${left} resource(s) open"))))) + #:catch (System.Exception)) + ((Ok _) + (when (overran? holder) + (test-failed! #" it ran past its deadline of ${within}: something it started never finished"))) + ((Err e) (test-failed! #" it raised: ${(.-Message e)}"))) + (box-set! current-failures outer) + (box-set! running-box outer-test) + (box-set! running-clock outer-clock) + (box-set! index-box outer-index) + ;; What no `expect-output` took is printed, so that a test's prints are + ;; not lost. + (when (record-ref t capture) + (match (take-test-output!) + ((Some text) (unless (string-empty? text) (display text))) + (None unit))) + (box-set! output-box outer-output) + (def failed (list-reverse (box-ref failures))) + (def todo? (not (string-empty? (record-ref t todo)))) + (TestResult (test t) + (status (cond ((and todo? (list-empty? failed)) Fixed) + (todo? Todo) + ((list-empty? failed) Passed) + (else Failed))) + (failures failed))) + +;; Whether a test is chosen by the filters: by a name that contains one of +;; them, or by there being none. `--quick` leaves out `#:slow` tests. +(: (-> TestOptions Test bool)) +(defun (chosen? o t) + (def filters (record-ref o filters)) + (and (not (and (record-ref o quick) (record-ref t slow))) + (or (vec-empty? filters) + (any? (fun (f) (string-contains? (record-ref t name) f)) filters)))) + +;; What extensions run when a run ends, oldest first: `(std test prop)` writes +;; its failure file, `(std test expect)` its promotions. +(: (Box (List (Tuple string (-> void))))) +(def run-end-hooks (make-box (list))) + +(: (-> (Tuple string (-> void)) string)) +(defun (hook-key h) (match h ((Tuple k _) k))) + +;; Adds `hook` to what runs when `run-registered-tests` ends, once for each +;; `key`: a second hook with the same key is ignored. +(: (-> string (-> void) void)) +(defun (at-run-end! key hook) + (unless (any? (fun (h) (= (hook-key h) key)) (box-ref run-end-hooks)) + (box-set! run-end-hooks (cons (Tuple key hook) (box-ref run-end-hooks))))) + +;; Text for an XML attribute or element. +(: (-> string string)) +(defun (xml-escape s) + (string-replace (string-replace (string-replace (string-replace s "&" "&") "<" "<") + ">" ">") + "\"" """)) + +;; The results as JUnit XML, the format CI servers read. +(: (-> string (List TestResult) long string)) +(defun (junit-xml module results ms) + (def failed (list-length (list-filter (fun (r) (match (record-ref r status) (Failed #t) (_ #f))) results))) + (def skipped (list-length (list-filter (fun (r) (match (record-ref r status) ((or Skipped Todo) #t) (_ #f))) results))) + (def cases + (list-map + (fun (r) + (def t (record-ref r test)) + (def head #" \n " + (xml-escape (string-join (list->vec (record-ref r failures)) "\n")) + "\n ")) + (Skipped (str head ">\n \n ")) + (Todo (str head ">\n \n ")) + (_ (str head "/>")))) + results)) + (str "\n" + "\n" + " string (list-length results)) + "\" failures=\"" (int->string failed) "\" skipped=\"" (int->string skipped) "\" time=\"" + (double->string (/ (cast System.Double ms) 1000.0)) "\">\n" + (string-join (list->vec cases) "\n") + (if (list-empty? cases) "" "\n") + " \n\n")) + +(: (-> TestStatus TestResult bool)) +(defun (status-is? s r) + (match (Tuple s (record-ref r status)) + ((Tuple Passed Passed) #t) + ((Tuple Failed Failed) #t) + ((Tuple Skipped Skipped) #t) + ((Tuple Todo Todo) #t) + ((Tuple Fixed Fixed) #t) + (_ #f))) + +;; The last line of a run. +(: (-> (List TestResult) long string)) +(defun (summary-line results ms) + (def total (list-length results)) + (def failed (list-length (list-filter (fun (r) (status-is? Failed r)) results))) + (def skipped (list-length (list-filter (fun (r) (status-is? Skipped r)) results))) + (def todo (list-length (list-filter (fun (r) (or (status-is? Todo r) (status-is? Fixed r))) results))) + (def extra + (str (if (> skipped 0) #", ${skipped} skipped" "") + (if (> todo 0) #", ${todo} todo" ""))) + (if (= failed 0) + #"${(- total (+ skipped todo))} test(s) passed${extra} in ${ms} ms." + #"${failed} of ${total} test(s) failed${extra} in ${ms} ms.")) + +;; Runs the tests `module` registered that the arguments choose, prints each +;; failure as it happens and one line at the end, and answers the exit code: 0 +;; when every test passed, 1 when one failed, 2 when the arguments are wrong. +(: (-> string (Vec string) int)) +(defun (run-registered-tests module args) + (match (parse-test-args args) + ((Err why) (println #"test: ${why}") 2) + ((Ok o) (run-with-options module o)))) + +(: (-> string TestOptions int)) +(defun (run-with-options module o) + (box-set! options-box o) + (def started (monotonic-ms)) + (def tests (list-filter (fun (t) (chosen? o t)) (module-tests module))) + (def results + (loop (:for t tests) + (:let result (run-test t)) + (:do (match (record-ref result status) + (Failed + (println #"FAIL ${(record-ref t name)} (${(shown-file (record-ref t file))}:${(record-ref t line)})") + (loop (:for f (record-ref result failures)) (:do (println f)))) + (Fixed + (println #"FIXED ${(record-ref t name)} (${(shown-file (record-ref t file))}:${(record-ref t line)}): it passes, so its #:todo can go")) + (_ unit))) + (:acc out (listing result)))) + (loop (:for h (list-reverse (box-ref run-end-hooks))) + (:do (match h ((Tuple _ hook) (hook))))) + (def ms (- (monotonic-ms) started)) + (match (record-ref o junit) + ((Some path) (file-write-text path (junit-xml module results ms))) + (None unit)) + (def failed (list-length (list-filter (fun (r) (status-is? Failed r)) results))) + (if (list-empty? tests) + (println (if (vec-empty? (record-ref o filters)) + "There are no tests." + "No test's name contains any of the names given.")) + (println (summary-line results ms))) + (if (= failed 0) 0 1)) + +;; `(run-tests)` — the file's `main`: runs every test this module registered. +;; A module that imports another with tests does not run those. +(def/macro (run-tests form inject compare) + (syntax-match form + ((_) + ;; `main` is injected: a template's own names are renamed, and the + ;; program's entry point has to be the name the compiler looks for. + #'(begin + (: ,(inject 'main) (-> (Vec string) int)) + (defun (,(inject 'main) args) (run-registered-tests (current-module) args)))) + (bad (syntax-error bad "run-tests takes nothing: (run-tests) at the end of a test file.")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/test/expect.bjo — output compared with a text in the test. +;; +;; (deftest "printing a card" +;; (print-value (Card (rank 12) (suit Hearts))) +;; (expect-output """ +;; (Card (rank 12) (suit Hearts)) +;; """)) +;; +;; A test whose body names `expect-output` writes its output to a string port +;; (see `take-test-output!` in `(std test)`). Each `expect-output` compares +;; what was written since the test started, or since the previous one, with +;; its text. With `--promote`, a text that differs is not a failure: the run +;; writes the output into the test file in its place, when it ends. + +(export expect-output + ;; Named by the template of the macro above, so it has to be + ;; reachable where that expands. + expect-output-check!) + +(import (std prelude)) +(import (std syntax-match)) +(import (std test)) + +(:doc #:file "expect.bjodoc") + +;; --------------------------------------------------------------------------- +;; Comparing +;; --------------------------------------------------------------------------- + +;; Text as it is compared: its lines, without a carriage return or other +;; whitespace at their ends, and without empty lines at its start and end. A +;; raw string drops the line breaks next to its quotes, so this keeps a +;; printed final newline from being a difference. +(: (-> string (Vec string))) +(defun (compared-lines text) + (def lines (vec->list (vec-map string-trim-right (string-split text "\n")))) + (defun (drop-empty xs) + (match xs + ((Cons "" more) (drop-empty more)) + (_ xs))) + (list->vec (list-reverse (drop-empty (list-reverse (drop-empty lines)))))) + +;; A line diff of two texts: a line only in `wanted` starts with -, one only +;; in `got` with +, and one in both with a space. The longest common +;; subsequence decides which lines are in both. +(: (-> (Vec string) (Vec string) (List string))) +(defun (diff-lines wanted got) + (def n (vec-length wanted)) + (def m (vec-length got)) + (if (> (* n m) 1000000) + ;; Too large to diff in reasonable time: both, in full. + (list-append (list-map (fun (l) (str "-" l)) (vec->list wanted)) + (list-map (fun (l) (str "+" l)) (vec->list got))) + (let ((table (make-array (* (+ n 1) (+ m 1))))) + (defun (at i j) (array-ref table (+ (* i (+ m 1)) j))) + ;; table[i][j] is the length of the common subsequence of the lines + ;; from i on and from j on. + (loop (:for back-i (range 0 (+ n 1))) + (:let i (- n back-i)) + (:do (loop (:for back-j (range 0 (+ m 1))) + (:let j (- m back-j)) + (:do (array-set! table (+ (* i (+ m 1)) j) + (cond + ((or (= i n) (= j m)) 0) + ((= (vec-ref wanted i) (vec-ref got j)) (+ 1 (at (+ i 1) (+ j 1)))) + (else (max (at (+ i 1) j) (at i (+ j 1)))))))))) + (let go ((i 0) (j 0) (out (list))) + (cond + ((and (= i n) (= j m)) (list-reverse out)) + ((= i n) (go i (+ j 1) (cons (str "+" (vec-ref got j)) out))) + ((= j m) (go (+ i 1) j (cons (str "-" (vec-ref wanted i)) out))) + ((= (vec-ref wanted i) (vec-ref got j)) (go (+ i 1) (+ j 1) (cons (str " " (vec-ref wanted i)) out))) + ((>= (at (+ i 1) j) (at i (+ j 1))) (go (+ i 1) j (cons (str "-" (vec-ref wanted i)) out))) + (else (go i (+ j 1) (cons (str "+" (vec-ref got j)) out)))))))) + +;; One `expect-output` whose text differed: where it is, the text it had when +;; the test ran, and what the test wrote instead. The old text is how the +;; promoter finds the form, and how it knows the file has not changed since. +(type (: Promotion (Record (: line int) (: wanted string) (: got string)))) + +;; The output a run promotes, by file, newest first. +(: (Box (List (Tuple string Promotion)))) +(def promotions (make-box (list))) + +(: (-> string string)) +(defun (shown-name path) + (match (path-filename path) + ((Some name) name) + (None path))) + +;; What `expect-output` expands to. +(: (-> string int string void)) +(defun (expect-output-check! file line wanted) + (def where #"${(shown-name file)}:${line}") + (match (take-test-output!) + (None + (test-failed! #" ${where}: expect-output has no output to read. It reads the output of a test whose deftest body names it.")) + ((Some got) + (def w (compared-lines wanted)) + (def g (compared-lines got)) + (unless (= w g) + (if (record-ref (current-test-options) promote) + (begin + (box-set! promotions (cons (Tuple file (Promotion (line line) (wanted wanted) (got got))) + (box-ref promotions))) + (at-run-end! "std test expect: promote" promote-all!)) + (test-failed! + (string-join + (list->vec (cons #" ${where}: the output is not the expected text (- expected, + written; --promote updates the text):" + (list-map (fun (l) (str " " l)) (diff-lines w g)))) + "\n"))))))) + +;; `(expect-output text)` — the output since the test started, or since the +;; previous `expect-output`, is the text. +(def/macro (expect-output form inject compare) + (syntax-match form + ((_ text) + #'(expect-output-check! ,(SStr (syntax-file form)) ,(SInt (int->string (syntax-line form))) ,text)) + (bad (syntax-error bad "expect-output takes a string: (expect-output \"\"\" ... \"\"\").")))) + +;; --------------------------------------------------------------------------- +;; Promoting +;; --------------------------------------------------------------------------- + +;; The index in `cs` of the first occurrence of `needle` at or after `from`. +(: (-> (Vec char) (Vec char) int (Option int))) +(defun (vec-search cs needle from) + (def n (vec-length cs)) + (def k (vec-length needle)) + (let go ((i from)) + (cond + ((> (+ i k) n) None) + ((loop (:for j (range 0 k)) (:acc ok (folding #t (and ok (char=? (vec-ref cs (+ i j)) (vec-ref needle j)))))) + (Some i)) + (else (go (+ i 1)))))) + +;; The index where line `line` (from 1) starts. +(: (-> (Vec char) int int)) +(defun (line-start cs line) + (let go ((i 0) (l 1)) + (cond + ((= l line) i) + ((>= i (vec-length cs)) i) + ((char=? (vec-ref cs i) #\newline) (go (+ i 1) (+ l 1))) + (else (go (+ i 1) l))))) + +;; How many quotes stand in a row from i. +(: (-> (Vec char) int int)) +(defun (quote-run cs i) + (let go ((j i)) + (if (and (< j (vec-length cs)) (char=? (vec-ref cs j) #\")) (go (+ j 1)) (- j i)))) + +;; Where the string literal that starts at i ends: the index after its +;; closing quotes. A run of three or more quotes is a raw string, closed by a +;; run as long. +(: (-> (Vec char) int (Option int))) +(defun (literal-end cs i) + (def q (quote-run cs i)) + (def n (vec-length cs)) + (cond + ((= q 0) None) + ((= q 2) (Some (+ i 2))) + ((= q 1) + (let go ((j (+ i 1))) + (cond + ((>= j n) None) + ((char=? (vec-ref cs j) #\\) (go (+ j 2))) + ((char=? (vec-ref cs j) #\") (Some (+ j 1))) + (else (go (+ j 1)))))) + (else + (let go ((j (+ i q))) + (cond + ((>= j n) None) + ((>= (quote-run cs j) q) (Some (+ j q))) + (else (go (+ j 1)))))))) + +;; The longest run of quotes in a text. +(: (-> string int)) +(defun (longest-quote-run text) + (def cs (string->vec text)) + (let go ((i 0) (best 0)) + (if (>= i (vec-length cs)) + best + (let ((q (quote-run cs i))) + (if (> q 0) (go (+ i q) (max q best)) (go (+ i 1) best)))))) + +(: (-> string int string)) +(defun (repeat-string s n) + (string-join (list->vec (loop (:for _ (range 0 n)) (:acc out (listing s)))) "")) + +;; The text as a literal for the test file. One line without a quote, a +;; backslash or a tab is a plain string. Anything else is a raw string whose +;; lines start at `margin` columns, with the closing quotes on a line of their +;; own at the margin; its quotes are one more than the longest run in the +;; text, and at least three. +(: (-> string int string)) +(defun (raw-literal text margin) + (def lines (compared-lines text)) + (cond + ((vec-empty? lines) "\"\"") + ((and (= (vec-length lines) 1) + (not (string-contains? (vec-ref lines 0) "\"")) + (not (string-contains? (vec-ref lines 0) "\\")) + (not (string-contains? (vec-ref lines 0) "\t"))) + (str "\"" (vec-ref lines 0) "\"")) + (else + (let* ((q (repeat-string "\"" (max 3 (+ 1 (longest-quote-run text))))) + (pad (repeat-string " " margin))) + (str q "\n" + (string-join (vec-map (fun (l) (if (string-empty? l) "" (str pad l))) lines) "\n") + "\n" pad q))))) + +;; What the string literal from i to end holds: a raw string without its +;; margin, or an escaped string with its escapes undone. +(: (-> (Vec char) int int string)) +(defun (literal-text cs i end) + (def q (quote-run cs i)) + (defun (chars from to) (list->string (vec->list (vec-slice cs from (- to from))))) + (cond + ((= q 2) "") + ((= q 1) (unescape (chars (+ i 1) (- end 1)))) + (else + (def inside (chars (+ i q) (- end q))) + (if (string-starts-with? inside "\n") + ;; Over several lines: the opening quotes end their line, and the + ;; closing ones stand at the margin on a line of their own. + (let* ((lines (vec->list (string-split inside "\n"))) + (body (list-tail lines)) + (margin (string-length (list-head (list-reverse body)))) + (text (list-reverse (list-tail (list-reverse body))))) + (string-join (list->vec (list-map (fun (l) (drop-chars l margin)) text)) "\n")) + inside)))) + +;; The string without its first n characters, or empty when it is shorter. +(: (-> string int string)) +(defun (drop-chars s n) + (let go ((left (string->list s)) (k n)) + (match left + ((Cons _ more) #:when (> k 0) (go more (- k 1))) + (_ (list->string left))))) + +;; An escaped string's text: \n, \t, \r, \0, \\ and \" undone. Other escapes +;; are kept as written, which is enough to compare texts. +(: (-> string string)) +(defun (unescape s) + (let go ((left (string->list s)) (out (list))) + (match left + (Nil (list->string (list-reverse out))) + ((Cons #\\ (Cons c more)) + (go more (cons (case c + (#\n #\newline) + (#\t #\tab) + (#\r #\return) + (#\0 (int->char 0)) + (else c)) + out))) + ((Cons c more) (go more (cons c out)))))) + +;; The places on line `line` (from 1) where an `(expect-output` starts. +(: (-> (Vec char) int (List int))) +(defun (forms-on-line cs line) + (def head (string->vec "(expect-output")) + (def start (line-start cs line)) + (def stop (line-start cs (+ line 1))) + (let go ((from start) (out (list))) + (match (vec-search cs head from) + ((Some at) (if (< at stop) (go (+ at 1) (cons at out)) (list-reverse out))) + (None (list-reverse out))))) + +;; Where the `expect-output` on `line` that still holds `wanted` is: the +;; start of the form, and the start and end of its literal. None when no form +;; there holds it, because the file changed since the test was compiled. +(: (-> (Vec char) int string (Option (Tuple int (Tuple int int))))) +(defun (find-form cs line wanted) + (def head-length (string-length "(expect-output")) + (loop (:for at (forms-on-line cs line)) + (:acc found + (folding None + (match found + ((Some _) found) + (None + (match (literal-after cs (+ at head-length)) + (None None) + ((Some (Tuple i end)) + (if (= (compared-lines (literal-text cs i end)) (compared-lines wanted)) + (Some (Tuple at (Tuple i end))) + None))))))))) + +;; The text with the literal from i to end replaced by `got`, at the margin of +;; a form that starts at `at`. +(: (-> (Vec char) int int int int string string)) +(defun (replace-literal cs line at i end got) + (str (list->string (vec->list (vec-slice cs 0 i))) + (raw-literal got (+ (- at (line-start cs line)) 2)) + (list->string (vec->list (vec-slice cs end (- (vec-length cs) end)))))) + +;; Where the literal after i starts and ends, past whitespace. +(: (-> (Vec char) int (Option (Tuple int int)))) +(defun (literal-after cs i) + (cond + ((>= i (vec-length cs)) None) + ((char-whitespace? (vec-ref cs i)) (literal-after cs (+ i 1))) + (else + (option-map (fun (end) (Tuple i end)) (literal-end cs i))))) + +;; Writes every promotion into its file, from the last line to the first so +;; that a replacement does not move the lines of those still to come. Forms on +;; one line go from the right, for the same reason: a raw string over several +;; lines moves what follows it on its line. +(: (-> void)) +(defun (promote-all!) + (def by-file + (list-foldl (fun (p m) + (match p + ((Tuple file entry) (map-set m file (cons entry (map-ref-or m file (list))))))) + (map-empty) + (box-ref promotions))) + (map-for-each + (fun (file entries) + (match (promote-file (file-read-text file) file (list-sort-desc entries)) + ((Tuple new-text n) + (when (> n 0) + (file-write-text file new-text) + (println #"promote: wrote ${n} text(s) into ${(shown-name file)}"))))) + by-file) + (box-set! promotions (list))) + +;; The text with the promotions written into it, and how many were. The +;; promotions come by line, the last first. +(: (-> string string (List Promotion) (Tuple string int))) +(defun (promote-file text file sorted) + (match sorted + (Nil (Tuple text 0)) + ((Cons p _) + (def line (record-ref p line)) + (def here (list-filter (fun (q) (= (record-ref q line) line)) sorted)) + (def rest (list-filter (fun (q) (not (= (record-ref q line) line))) sorted)) + (def cs (string->vec text)) + ;; Each form found on the line, from the right. + (def found + (loop (:for q here) + (:acc out (listing (Tuple q (find-form cs line (record-ref q wanted))))))) + (loop (:for f found) + (:do (match f + ((Tuple q None) + (println #"promote: the expect-output at ${(shown-name file)}:${line} no longer holds the text the test ran with, so it was left alone")) + (_ unit)))) + (def placed (sort-by-start-desc + (loop (:for f found) + (:when-let (Tuple q (Some place)) f) + (:acc out (listing (Tuple q place)))))) + (def new-text + (list-foldl (fun (f t) + (match f + ((Tuple q (Tuple at (Tuple i end))) + (replace-literal (string->vec t) line at i end (record-ref q got))))) + text + placed)) + (match (promote-file new-text file rest) + ((Tuple final n) (Tuple final (+ n (list-length placed)))))))) + +(: (-> (List (Tuple Promotion (Tuple int (Tuple int int)))) + (List (Tuple Promotion (Tuple int (Tuple int int)))))) +(defun (sort-by-start-desc xs) + (list-foldl insert-by-start (list) xs)) + +(: (-> (Tuple Promotion (Tuple int (Tuple int int))) + (List (Tuple Promotion (Tuple int (Tuple int int)))) + (List (Tuple Promotion (Tuple int (Tuple int int)))))) +(defun (insert-by-start e sorted) + (match sorted + (Nil (list e)) + ((Cons f more) + (if (>= (start-of e) (start-of f)) (cons e sorted) (cons f (insert-by-start e more)))))) + +(: (-> (Tuple Promotion (Tuple int (Tuple int int))) int)) +(defun (start-of e) (match e ((Tuple _ (Tuple at _)) at))) + +;; The promotions by line, the last first. A form promoted twice, at the same +;; line with the same old text, keeps the first. +(: (-> (List Promotion) (List Promotion))) +(defun (list-sort-desc xs) + (list-foldl insert-promotion (list) (list-reverse xs))) + +(: (-> Promotion (List Promotion) (List Promotion))) +(defun (insert-promotion e sorted) + (match sorted + (Nil (list e)) + ((Cons f more) + (cond + ((and (= (record-ref e line) (record-ref f line)) (= (record-ref e wanted) (record-ref f wanted))) sorted) + ((>= (record-ref e line) (record-ref f line)) (cons e sorted)) + (else (cons f (insert-promotion e more))))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/test/gen.bjo — generators of test values. +;; +;; A generator does not read a RandomSource. It asks a `Choices` for numbers +;; with `draw`, and the `Choices` either takes them from a source and records +;; them, or replays a recorded vector. `(std test prop)` shrinks a failing +;; case by making the recorded vector smaller and replaying it, so no +;; generator needs a shrinker of its own. +;; +;; That works only when a smaller number gives a simpler value. Every +;; generator here keeps to that rule: 0 is the simplest choice at every draw. +;; A replayed vector that runs out answers 0 for every further draw, so a +;; shorter vector gives a simpler value too. + +(export + ;; The recorder + Choices recording-choices replaying-choices + draw choices-size choices-draws choices-discarded? discard! + ;; Generators, and running one + Gen make-gen gen-run gen-sample gen-samples + ;; Combinators + gen-pure gen-map gen-bind gen-let gen-such-that gen-one-of gen-frequency + gen-element gen-sized gen-resize gen-recursive + ;; Numbers + gen-bool gen-int gen-int-range gen-long gen-long-range gen-ulong gen-double + gen-bigint + ;; Text + gen-char gen-ascii-char gen-string gen-ascii-string gen-string-of + ;; Shapes + gen-unit gen-option gen-result gen-tuple2 gen-tuple3 gen-tuple4 + ;; Sequences + gen-list gen-list-of-length gen-vec gen-array gen-shuffle + ;; Maps and sets + gen-map-of gen-hashset gen-orderedmap gen-orderedset + ;; Defaults by type + Arbitrary arbitrary) + +(import (std prelude)) +(import (std random)) +(import (std syntax-match)) +(import (std hashset)) +(import (std orderedset)) +(import (std orderedmap)) + +(import/class + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +(:doc #:file "gen.bjodoc") + +;; --------------------------------------------------------------------------- +;; The recorder +;; --------------------------------------------------------------------------- + +;; `source` is the source a recording draws from, `None` when replaying. +;; `replay` is the vector a replay reads, and `pos` how far it has read. +;; `recorded` holds every answer given, in both modes: after a replay it is +;; the part of the vector that was used, and the zeros answered after the +;; vector ran out. That is what a shrinker keeps. +(type + (: Choices #:opaque + (Record (: source (Option RandomSource)) + (: replay (Vec ulong)) + (: pos int #:mutable) + (: recorded (VecBuilder ulong)) + (: size int #:mutable) + (: discarded bool #:mutable)))) + +;; A generator that draws more than this many numbers is stopped, and its run +;; is discarded. A generator that never stops drawing would otherwise run +;; until the test's deadline. +(: int) +(def max-draws 10000) + +(: (-> RandomSource int Choices)) +(defun (recording-choices rng size) + (Choices (source (Some rng)) (replay (vec-empty)) (pos 0) + (recorded (vecbuilder-empty)) (size size) (discarded #f))) + +(: (-> (Vec ulong) int Choices)) +(defun (replaying-choices draws size) + (Choices (source None) (replay draws) (pos 0) + (recorded (vecbuilder-empty)) (size size) (discarded #f))) + +(: (-> Choices int)) +(defun (choices-size c) (record-ref c size)) + +(: (-> Choices (Vec ulong))) +(defun (choices-draws c) (vecbuilder->vec (record-ref c recorded))) + +(: (-> Choices bool)) +(defun (choices-discarded? c) (record-ref c discarded)) + +;; Marks the run as discarded and leaves it. The exception is only the way +;; out: whoever runs the generator looks at `choices-discarded?`, not at the +;; exception. +(: (-> Choices string %a)) +(defun (discard! c why) + (record-set! c (discarded #t)) + (raise (cast Exception (ArgumentException. (str "discarded: " why))))) + +;; A number in [0, bound). A bound of 0 means every ulong. +(: (-> Choices ulong ulong)) +(defun (draw c bound) (draw/forced c bound None)) + +;; `draw`, except that a recording records `forced` instead of drawing, when +;; it is given. A replay reads the vector as `draw` does. A generator uses it +;; to record a value it chose itself in the draws another way to that value +;; would make, so that a replay, and so shrinking, sees no difference. +(: (-> Choices ulong (Option ulong) ulong)) +(defun (draw/forced c bound forced) + (def recorded (record-ref c recorded)) + (when (>= (vecbuilder-length recorded) max-draws) + (discard! c "the generator drew too many numbers")) + (def raw + (match (record-ref c source) + ((Some rng) + (match forced + ((Some v) v) + (None (random-u64 #:rng rng)))) + (None + (def replay (record-ref c replay)) + (def pos (record-ref c pos)) + (record-set! c (pos (+ pos 1))) + (if (< pos (vec-length replay)) (vec-ref replay pos) 0UL)))) + (def value (if (= bound 0UL) raw (% raw bound))) + (vecbuilder-add! recorded value) + value) + +;; A number in [0, n) from the source of a recording, not recorded. Only a +;; choice that a forced draw records may be made this way. A replay answers 0. +(: (-> Choices int int)) +(defun (unrecorded-below c n) + (match (record-ref c source) + ((Some rng) (cast System.Int32 (% (random-u64 #:rng rng) (cast System.UInt64 n)))) + (None 0))) + +;; A number in [0, n), for an n above 0. +(: (-> Choices int int)) +(defun (draw-below c n) + (cast System.Int32 (draw c (cast System.UInt64 n)))) + +;; --------------------------------------------------------------------------- +;; Generators +;; --------------------------------------------------------------------------- + +(type (: (Gen %a) #:opaque (Record (: run (-> Choices %a))))) + +(: (-> (-> Choices %a) (Gen %a))) +(defun (make-gen f) (Gen (run f))) + +(: (-> (Gen %a) Choices %a)) +(defun (gen-run g c) ((record-ref g run) c)) + +;; One value, drawn from a fresh source unless `rng` is given. +(: (-> (Gen %a) (#:size int) (#:rng RandomSource) %a)) +(defun (gen-sample g #:size 30 #:rng (make-random)) + (gen-run g (recording-choices rng size))) + +;; `n` values, at sizes that grow from 1 to `size`, as a property's runs do. +(: (-> (Gen %a) int (#:size int) (#:rng RandomSource) (List %a))) +(defun (gen-samples g n #:size 100 #:rng (make-random)) + (loop (:for i (range 0 n)) + (:acc out (listing (gen-run g (recording-choices (random-split rng) + (size-of-run i n size))))))) + +;; The size of run i of n: 1 at the first run, `top` at the last. +(: (-> int int int int)) +(defun (size-of-run i n top) + (if (<= n 1) top (+ 1 (/ (* i (- top 1)) (- n 1))))) + +;; --------------------------------------------------------------------------- +;; Combinators +;; --------------------------------------------------------------------------- + +(: (-> %a (Gen %a))) +(defun (gen-pure x) (make-gen (fun (c) x))) + +(: (-> (-> %a %b) (Gen %a) (Gen %b))) +(defun (gen-map f g) (make-gen (fun (c) (f (gen-run g c))))) + +(: (-> (Gen %a) (-> %a (Gen %b)) (Gen %b))) +(defun (gen-bind g f) (make-gen (fun (c) (gen-run (f (gen-run g c)) c)))) + +;; `(gen-let ((name gen) ...) body ...)` — a generator that draws each +;; binding in order, as `let*` binds, and answers the body. A later generator +;; may use an earlier value. `Gen` is not a `Monad`: `bind`'s callback may be a +;; bjoroutine, and a generator runs inside an ordinary function. +(def/macro (gen-let form inject compare) + (syntax-match form + ((_ ((name g) ...) first body ...) + #'(make-gen + (fun (choices) + (let* ,(SList (list-map (fun (pair) + (syntax-match pair + ((n x) #'(,n (gen-run ,x choices))) + (bad (syntax-error bad "a binding is (name generator).")))) + (match form + ((SList (Cons _ (Cons (SList bindings) _))) bindings) + (_ (list))))) + ,first + ,@body)))) + (bad (syntax-error bad "gen-let takes bindings and a body: (gen-let ((x gen-int) (y (gen-int-range 0 x))) (Tuple x y)).")))) + +;; Retries until `pred` holds. After 100 tries the run is discarded, since a +;; generator that rarely satisfies `pred` tests little. +(: (-> (-> %a bool) (Gen %a) (Gen %a))) +(defun (gen-such-that pred g) + (make-gen + (fun (c) + (let go ((tries 0)) + (if (= tries 100) + (discard! c "gen-such-that found no value in 100 tries") + (let ((x (gen-run g c))) + (if (pred x) x (go (+ tries 1))))))))) + +;; One of the generators. The first is the simplest. +(: (-> (List (Gen %a)) (Gen %a))) +(defun (gen-one-of gens) + (when (list-empty? gens) + (raise (cast Exception (ArgumentException. "gen-one-of was given no generators.")))) + (def v (list->vec gens)) + (make-gen (fun (c) (gen-run (vec-ref v (draw-below c (vec-length v))) c)))) + +;; One of the generators, by weight. The first is the simplest. +(: (-> (List (Tuple int (Gen %a))) (Gen %a))) +(defun (gen-frequency choices) + (def total (loop (:for (Tuple w _) choices) (:acc n (summing w)))) + (when (any? (fun (p) (match p ((Tuple w _) (< w 0)))) choices) + (raise (cast Exception (ArgumentException. "gen-frequency was given a weight below 0.")))) + (when (<= total 0) + (raise (cast Exception (ArgumentException. "gen-frequency needs a weight above 0.")))) + (make-gen + (fun (c) + (let go ((left choices) (n (draw-below c total))) + (match left + ((Cons (Tuple w g) more) (if (< n w) (gen-run g c) (go more (- n w)))) + ;; Not reached: the draw is below the total of the weights, and + ;; none of them is below 0. + (Nil (raise (cast Exception (ArgumentException. "gen-frequency ran out of weights."))))))))) + +;; One of the values. The first is the simplest. +(: (-> (List %a) (Gen %a))) +(defun (gen-element xs) + (when (list-empty? xs) + (raise (cast Exception (ArgumentException. "gen-element was given no values.")))) + (def v (list->vec xs)) + (make-gen (fun (c) (vec-ref v (draw-below c (vec-length v)))))) + +(: (-> (-> int (Gen %a)) (Gen %a))) +(defun (gen-sized f) (make-gen (fun (c) (gen-run (f (choices-size c)) c)))) + +;; `g` with the size `n`. The size goes back to what it was afterwards. +(: (-> int (Gen %a) (Gen %a))) +(defun (gen-resize n g) + (make-gen + (fun (c) + (def outer (choices-size c)) + (record-set! c (size n)) + (def x (gen-run g c)) + (record-set! c (size outer)) + x))) + +;; A generator of a recursive type. At size 0 it is `base`. Above, a draw +;; chooses `base` or `(step self)`, where `self` is this generator at half the +;; size. 0 chooses `base`, so a case shrinks toward it. +(: (-> (Gen %a) (-> (Gen %a) (Gen %a)) (Gen %a))) +(defun (gen-recursive base step) + (make-gen + (fun (c) + (def n (choices-size c)) + (if (or (<= n 0) (= (draw c 4UL) 0UL)) + (gen-run base c) + (gen-run (step (gen-resize (/ n 2) (gen-recursive base step))) c))))) + +;; --------------------------------------------------------------------------- +;; Numbers +;; --------------------------------------------------------------------------- + +(: (Gen bool)) +(def gen-bool (make-gen (fun (c) (= (draw c 2UL) 1UL)))) + +;; A long in [lo, hi]. 0 gives the value nearest 0: lo when the range is all +;; above 0, hi when it is all below, and 0 itself otherwise. A range across 0 +;; draws the sign first, positive at 0, then the distance from 0. +(: (-> Choices long long long)) +(defun (draw-long-range c lo hi) (draw-long-range/forced c lo hi None)) + +;; The draw that records `target`, as `f` turns it into a draw. +(: (-> (Option long) (-> long long) (Option ulong))) +(defun (forced-by target f) + (option-map (fun (t) (cast System.UInt64 (f t))) target)) + +;; `draw-long-range`, recording `target` when given: the draws it records are +;; those that give `target` when replayed. +(: (-> Choices long long (Option long) long)) +(defun (draw-long-range/forced c lo hi target) + (defun (forced f) (forced-by target f)) + (cond + ((> lo hi) + (raise (cast Exception (ArgumentException. #"An empty range: ${lo} to ${hi}.")))) + ((>= lo 0L) + ;; hi - lo + 1 as a ulong, which is 0, meaning every ulong, only when + ;; the range is all of them. + (+ lo (cast System.Int64 (draw/forced c (+ (cast System.UInt64 (- hi lo)) 1UL) + (forced (fun (t) (- t lo))))))) + ((<= hi 0L) + (- hi (cast System.Int64 (draw/forced c (+ (cast System.UInt64 (- hi lo)) 1UL) + (forced (fun (t) (- hi t))))))) + ((= (draw/forced c 2UL (forced (fun (t) (if (>= t 0L) 0L 1L)))) 0UL) + (cast System.Int64 (draw/forced c (+ (cast System.UInt64 hi) 1UL) (forced (fun (t) t))))) + (else + ;; -(lo + 1) does not overflow, even at long-min. + (- -1L (cast System.Int64 (draw/forced c (+ (cast System.UInt64 (- -1L lo)) 1UL) + (forced (fun (t) (- -1L t))))))))) + +(: (-> long long (Gen long))) +(defun (gen-long-range lo hi) + (when (> lo hi) + (raise (cast Exception (ArgumentException. #"gen-long-range: ${lo} is above ${hi}.")))) + (make-gen (fun (c) (draw-long-range c lo hi)))) + +(: (-> int int (Gen int))) +(defun (gen-int-range lo hi) + (when (> lo hi) + (raise (cast Exception (ArgumentException. #"gen-int-range: ${lo} is above ${hi}.")))) + (make-gen (fun (c) (cast System.Int32 (draw-long-range c (cast System.Int64 lo) (cast System.Int64 hi)))))) + +;; The distance from 0 that a value of size n reaches: n * n, so a size of 100 +;; reaches 10000. A size at or below 0 reaches 0. +(: (-> int long)) +(defun (reach n) + (if (<= n 0) 0L (* (cast System.Int64 n) (cast System.Int64 n)))) + +;; An edge value: one of `edges`, drawn with `pad` more draws of 0 after it. +;; An edge has to cost as many draws as the longest other way to a value, or +;; shrinking, which prefers fewer draws, would turn a value such as 1000 into +;; an edge such as int-max. +(: (-> Choices (Vec %a) int %a)) +(defun (draw-edge c edges pad) + (def x (vec-ref edges (draw-below c (vec-length edges)))) + (loop (:for _ (range 0 pad)) (:do (ignore (draw c 1UL)))) + x) + +;; A sized whole number, with the edges of its type now and then. The first +;; draw chooses: 0 to 7 a value within the size's reach, 8 any value of the +;; type, 9 an edge. An edge is recorded as kind 8 would record it, so the two +;; replay alike: shrinking turns the 9 into an 8 and keeps the value, and can +;; then make the value smaller. +(: (-> Choices long long (List long) long)) +(defun (draw-sized-long c lo hi edges) + (def kind (draw c 10UL)) + (cond + ((< kind 8UL) + (def r (reach (choices-size c))) + (draw-long-range c (if (< (- 0L r) lo) lo (- 0L r)) (if (> r hi) hi r))) + ((= kind 8UL) (draw-long-range c lo hi)) + (else + (def v (list->vec edges)) + (draw-long-range/forced c lo hi (Some (vec-ref v (unrecorded-below c (vec-length v)))))))) + +(: (Gen long)) +(def gen-long + (make-gen + (fun (c) + (draw-sized-long c -9223372036854775808L 9223372036854775807L + (list 0L 1L -1L 9223372036854775807L -9223372036854775808L + 2147483647L -2147483648L 4294967296L))))) + +(: (Gen int)) +(def gen-int + (make-gen + (fun (c) + (cast System.Int32 + (draw-sized-long c -2147483648L 2147483647L + (list 0L 1L -1L 2147483647L -2147483648L 255L 256L 65535L)))))) + +;; Any ulong. 0 to 7 within the size's reach, 8 any, 9 an edge. +(: (Gen ulong)) +(def gen-ulong + (make-gen + (fun (c) + (def kind (draw c 10UL)) + (cond + ((< kind 8UL) (draw c (+ (cast System.UInt64 (reach (choices-size c))) 1UL))) + ((= kind 8UL) (draw c 0UL)) + (else + ;; Recorded as kind 8 records a value: see draw-sized-long. + (def v (list->vec (list 0UL 1UL 18446744073709551615UL 9223372036854775808UL 4294967296UL))) + (draw/forced c 0UL (Some (vec-ref v (unrecorded-below c (vec-length v)))))))))) + +;; A bigint: a long most of the time, and a product of longs beyond the range +;; of a long otherwise. +(: (Gen bigint)) +(def gen-bigint + (make-gen + (fun (c) + (def low (cast System.Numerics.BigInteger (gen-run gen-long c))) + (if (= (draw c 4UL) 3UL) + (* low (+ (cast System.Numerics.BigInteger (gen-run gen-long c)) 9223372036854775808I)) + low)))) + +;; A double. Most draws give a whole part within the size's reach and a +;; fraction; 0 gives 0.0. An edge now and then: 0.0, ±1, the largest and +;; smallest values, the smallest above 0. `#:special` adds NaN and the +;; infinities to the edges. +(: (-> (#:special bool) (Gen double))) +(defun (gen-double #:special #f) + (def edges + (list->vec + (list-append (list 0.0 1.0 -1.0 1.7976931348623157e308 -1.7976931348623157e308 5e-324 -0.0) + (if special (list (/ 0.0 0.0) (/ 1.0 0.0) (/ -1.0 0.0)) (list))))) + (make-gen + (fun (c) + (def kind (draw c 10UL)) + (if (= kind 9UL) + ;; Padded to the four draws of the other way: sign, distance and + ;; fraction after the kind. + (draw-edge c edges 2) + (let* ((r (reach (choices-size c))) + (whole (draw-long-range c (- 0L r) r)) + ;; 53 bits, so every fraction is a double exactly. + (frac (/ (cast System.Double (draw c 9007199254740992UL)) 9007199254740992.0))) + (if (< whole 0L) + (- (cast System.Double whole) frac) + (+ (cast System.Double whole) frac))))))) + +;; --------------------------------------------------------------------------- +;; Text +;; --------------------------------------------------------------------------- + +;; Printable ASCII, from #\a: the 95 characters from space to ~, turned so +;; that 0 is a letter, the character a case most often shrinks to. +(: (-> Choices char)) +(defun (draw-ascii c) + (def d (cast System.Int32 (draw c 95UL))) + (int->char (+ 32 (% (+ d 65) 95)))) + +(: (Gen char)) +(def gen-ascii-char (make-gen draw-ascii)) + +;; Characters that find bugs in text code: the empty character, a combining +;; accent, a character outside the BMP (two UTF-16 units), a right-to-left +;; letter, the replacement character, a non-breaking space, a final sigma, +;; and a capital I with a dot, whose lower case is two characters. +(: (Vec char)) +(def odd-chars + (list->vec (list (int->char 0) (int->char 0x301) (int->char 0x1F600) (int->char 0x5D0) + (int->char 0xFFFD) (int->char 0xA0) (int->char 0x3C2) (int->char 0x130)))) + +;; Any Unicode scalar value. 0 to 7 printable ASCII, 8 one of the odd +;; characters above, 9 any scalar value. +(: (Gen char)) +(def gen-char + (make-gen + (fun (c) + (def kind (draw c 10UL)) + (cond + ((< kind 8UL) (draw-ascii c)) + ((= kind 8UL) (vec-ref odd-chars (draw-below c (vec-length odd-chars)))) + (else + ;; Every scalar value: 0 to 0x10FFFF without the surrogates. + (def n (draw-below c 1112064)) + (int->char (if (< n 0xD800) n (+ n 2048)))))))) + +(: (-> (Gen char) (Gen string))) +(defun (gen-string-of g) (gen-map list->string (gen-list g))) + +(: (Gen string)) +(def gen-string (gen-string-of gen-char)) + +(: (Gen string)) +(def gen-ascii-string (gen-string-of gen-ascii-char)) + +;; --------------------------------------------------------------------------- +;; Shapes +;; --------------------------------------------------------------------------- + +(: (Gen void)) +(def gen-unit (make-gen (fun (c) unit))) + +;; None at 0, and a quarter of the time. +(: (-> (Gen %a) (Gen (Option %a)))) +(defun (gen-option g) + (make-gen (fun (c) (if (= (draw c 4UL) 0UL) None (Some (gen-run g c)))))) + +;; Ok at 0, and half of the time. +(: (-> (Gen %e) (Gen %a) (Gen (Result %e %a)))) +(defun (gen-result ge ga) + (make-gen (fun (c) (if (= (draw c 2UL) 0UL) (Ok (gen-run ga c)) (Err (gen-run ge c)))))) + +(: (-> (Gen %a) (Gen %b) (Gen (Tuple %a %b)))) +(defun (gen-tuple2 ga gb) + (make-gen (fun (c) (let* ((a (gen-run ga c)) (b (gen-run gb c))) (Tuple a b))))) + +(: (-> (Gen %a) (Gen %b) (Gen %c) (Gen (Tuple %a %b %c)))) +(defun (gen-tuple3 ga gb gc) + (make-gen (fun (c) (let* ((a (gen-run ga c)) (b (gen-run gb c)) (x (gen-run gc c))) + (Tuple a b x))))) + +(: (-> (Gen %a) (Gen %b) (Gen %c) (Gen %d) (Gen (Tuple %a %b %c %d)))) +(defun (gen-tuple4 ga gb gc gd) + (make-gen (fun (c) (let* ((a (gen-run ga c)) (b (gen-run gb c)) + (x (gen-run gc c)) (d (gen-run gd c))) + (Tuple a b x d))))) + +;; --------------------------------------------------------------------------- +;; Sequences +;; --------------------------------------------------------------------------- + +;; The longest list a generator makes, whatever the size. +(: int) +(def max-length 1000) + +;; A list whose length grows with the size: 1 + size / 5 on average. Before +;; each element a draw says whether there is one more, and 0 says no. So +;; deleting draws makes the list shorter, and setting one to 0 ends it there. +(: (-> (Gen %a) (Gen (List %a)))) +(defun (gen-list g) + (make-gen + (fun (c) + (def mean (cast System.UInt64 (+ 1 (/ (choices-size c) 5)))) + (def out (listbuilder-empty)) + (let go ((n 0)) + (if (and (< n max-length) (> (draw c (+ mean 1UL)) 0UL)) + (begin + (listbuilder-add! out (gen-run g c)) + (go (+ n 1))) + (listbuilder->list out)))))) + +(: (-> int (Gen %a) (Gen (List %a)))) +(defun (gen-list-of-length n g) + (make-gen + (fun (c) + (loop (:for _ (range 0 n)) + (:acc out (listing (gen-run g c))))))) + +(: (-> (Gen %a) (Gen (Vec %a)))) +(defun (gen-vec g) (gen-map list->vec (gen-list g))) + +(: (-> (Gen %a) (Gen (Array %a)))) +(defun (gen-array g) + (gen-map (fun (xs) + (def v (list->vec xs)) + (def a (make-array (vec-length v))) + (loop (:with i 0 (+ i 1)) + (:for x v) + (:do (array-set! a i x))) + a) + (gen-list g))) + +;; A permutation of `xs`. Fisher-Yates from the front: element i swaps with +;; one at or after it, and a draw of 0 leaves it where it is. So the order +;; shrinks toward the order `xs` has. +(: (-> (List %a) (Gen (List %a)))) +(defun (gen-shuffle xs) + (make-gen + (fun (c) + (def v (list->vec xs)) + (def n (vec-length v)) + (def a (make-array n)) + (loop (:with i 0 (+ i 1)) + (:for x v) + (:do (array-set! a i x))) + (loop (:for i (range 0 n)) + (:do (let* ((j (+ i (draw-below c (- n i)))) + (x (array-ref a i))) + (array-set! a i (array-ref a j)) + (array-set! a j x)))) + (vec->list (array->vec a))))) + +;; --------------------------------------------------------------------------- +;; Maps and sets +;; --------------------------------------------------------------------------- + +;; Each is made from a list of entries, so it shrinks as the list does. A +;; later entry wins over an earlier one with the same key. + +(: (-> (Gen %k) (Gen %v) (Gen (Map %k %v))) (where (Eq %k))) +(defun (gen-map-of gk gv) (gen-map list->map (gen-list (gen-tuple2 gk gv)))) + +(: (-> (Gen %a) (Gen (HashSet %a))) (where (Eq %a))) +(defun (gen-hashset g) (gen-map list->hashset (gen-list g))) + +(: (-> (Gen %k) (Gen %v) (Gen (OrderedMap %k %v))) (where (Ord %k))) +(defun (gen-orderedmap gk gv) (gen-map list->orderedmap (gen-list (gen-tuple2 gk gv)))) + +(: (-> (Gen %a) (Gen (OrderedSet %a))) (where (Ord %a))) +(defun (gen-orderedset g) (gen-map list->orderedset (gen-list g))) + +;; --------------------------------------------------------------------------- +;; Defaults by type +;; --------------------------------------------------------------------------- + +;; The generator a type has when a property does not name one. An instance is +;; only a generator: shrinking needs nothing more. +(def/trait (Arbitrary %a) + (: arbitrary (-> (Gen %a)))) + +(impl (Arbitrary bool) (defun (arbitrary) gen-bool)) +(impl (Arbitrary int) (defun (arbitrary) gen-int)) +(impl (Arbitrary long) (defun (arbitrary) gen-long)) +(impl (Arbitrary ulong) (defun (arbitrary) gen-ulong)) +(impl (Arbitrary bigint) (defun (arbitrary) gen-bigint)) +(impl (Arbitrary double) (defun (arbitrary) (gen-double))) +(impl (Arbitrary char) (defun (arbitrary) gen-char)) +(impl (Arbitrary string) (defun (arbitrary) gen-string)) +(impl (Arbitrary void) (defun (arbitrary) gen-unit)) + +(impl (Arbitrary (Option %a)) (where (Arbitrary %a)) + (defun (arbitrary) (gen-option (arbitrary)))) + +(impl (Arbitrary (Result %e %a)) (where (Arbitrary %e) (Arbitrary %a)) + (defun (arbitrary) (gen-result (arbitrary) (arbitrary)))) + +(impl (Arbitrary (Tuple %a %b)) (where (Arbitrary %a) (Arbitrary %b)) + (defun (arbitrary) (gen-tuple2 (arbitrary) (arbitrary)))) + +(impl (Arbitrary (Tuple %a %b %c)) (where (Arbitrary %a) (Arbitrary %b) (Arbitrary %c)) + (defun (arbitrary) (gen-tuple3 (arbitrary) (arbitrary) (arbitrary)))) + +(impl (Arbitrary (List %a)) (where (Arbitrary %a)) + (defun (arbitrary) (gen-list (arbitrary)))) + +(impl (Arbitrary (Vec %a)) (where (Arbitrary %a)) + (defun (arbitrary) (gen-vec (arbitrary)))) + +(impl (Arbitrary (Map %k %v)) (where (Arbitrary %k) (Eq %k) (Arbitrary %v)) + (defun (arbitrary) (gen-map-of (arbitrary) (arbitrary)))) + +(impl (Arbitrary (HashSet %a)) (where (Arbitrary %a) (Eq %a)) + (defun (arbitrary) (gen-hashset (arbitrary)))) + +(impl (Arbitrary (OrderedMap %k %v)) (where (Arbitrary %k) (Ord %k) (Arbitrary %v)) + (defun (arbitrary) (gen-orderedmap (arbitrary) (arbitrary)))) + +(impl (Arbitrary (OrderedSet %a)) (where (Arbitrary %a) (Ord %a)) + (defun (arbitrary) (gen-orderedset (arbitrary)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/test/prop.bjo — property tests. +;; +;; (deftest "reverse twice is the identity" +;; (for-all ((: xs (List int))) +;; (check (= (list-reverse (list-reverse xs)) xs)))) +;; +;; `for-all` runs its body many times with generated values. When a run +;; fails, it shrinks the run: it makes the recorded draws of the generators +;; smaller and replays them, and keeps each change that still fails. What is +;; left is reported, and stored beside the test file in NAME.failed.bjodat, +;; where the next run of the test tries it first. + +(export for-all label assume classify collect + ;; Named by the templates of the macros above, so they have to be + ;; reachable where those expand. + run-for-all! with-label prop-draw prop-arbitrary) + +(import (std prelude)) +(import (std syntax-match)) +(import (std random)) +(import (std test)) +(import (std test gen)) +(import (text bjodat-core)) + +(import/class + (InvalidOperationException (: System.InvalidOperationException (-> string InvalidOperationException)))) + +(:doc #:file "prop.bjodoc") + + +;; --------------------------------------------------------------------------- +;; Running one case +;; --------------------------------------------------------------------------- + +;; What `for-all` writes: given the choices, it draws the bindings and answers +;; their values as text, for a report, and the body as a function. +(type (: Property (-> Choices (Tuple (List string) (-> void))))) + +;; How one run came out. A failed run has its failures, as the test would +;; print them, the values of its bindings as text, and its signature: what +;; tells this failure apart from another (see `failure-signature`). +(type (: Outcome (Union Pass Discard (: Fail (List string) (List string) string)))) + +;; The choices of the run in progress, for `assume`. `None` outside a run. +(: (Box (Option Choices))) +(def choices-box (make-box None)) + +;; The labels of `classify` and `collect` in the run in progress, when the +;; run counts them. Only the random runs count them, not the replays of +;; shrinking. +(: (Box (Option (Box (List string))))) +(def labels-box (make-box None)) + +(: (-> (Gen %a) Choices %a)) +(defun (prop-draw g c) (gen-run g c)) + +(: (-> Choices %a) (where (Arbitrary %a))) +(defun (prop-arbitrary c) (gen-run (arbitrary) c)) + +;; What a failure is, without the values it saw: the place of the first +;; check that failed, or the type of the exception. Shrinking keeps only cases +;; that fail as the first one did. A smaller case that fails some other way is +;; a different bug, and reporting it would hide the one that was found. +;; +;; A check's report starts with its place and source; a label puts its name +;; on a line above. A resource left open is one signature, whatever the count. +(: (-> (List string) string)) +(defun (failure-signature failures) + (match failures + (Nil "") + ((Cons first _) + (def lines (vec->list (string-split first "\n"))) + (match lines + ((Cons head (Cons next _)) + (if (string-starts-with? head " in ") (str head "\n" next) head)) + ((Cons head Nil) + (if (string-starts-with? head " it left ") "a resource left open" head)) + (Nil ""))))) + +(: (-> Exception string)) +(defun (exception-name e) (.-FullName (.GetType e))) + +;; Runs the body of one case in a scope of its own: the scope waits for the +;; fibers the case started, so the next case does not share them. A case that +;; leaves a resource open fails, unless the test has `#:open-ok`. +(: (-> Property Choices bool Outcome)) +(defun (run-case prop c open-ok) + (def failures (make-box (list))) + (def saved (box-ref choices-box)) + (box-set! choices-box (Some c)) + (def outcome + (match (try (prop c) #:catch (System.Exception)) + ((Err e) + (if (choices-discarded? c) + Discard + (Fail (list #" the generator raised: ${(.-Message e)}") (list) + (str "the generator raised " (exception-name e))))) + ((Ok (Tuple shown body)) + (match (try (with-scope + (failures-into failures body) + (unless open-ok + (let ((left (scope-owned-count (current-scope)))) + (unless (= left 0) + (box-set! failures (cons #" it left ${left} resource(s) open" + (box-ref failures))))))) + #:catch (System.Exception)) + ((Err e) + (if (choices-discarded? c) + Discard + (Fail (list-reverse (cons #" it raised: ${(.-Message e)}" (box-ref failures))) shown + (if (list-empty? (box-ref failures)) + (str "it raised " (exception-name e)) + (failure-signature (list-reverse (box-ref failures))))))) + ((Ok _) + (if (list-empty? (box-ref failures)) + Pass + (Fail (list-reverse (box-ref failures)) shown + (failure-signature (list-reverse (box-ref failures)))))))))) + (box-set! choices-box saved) + outcome) + +;; --------------------------------------------------------------------------- +;; Shrinking +;; --------------------------------------------------------------------------- + +;; The order shrinking makes a case smaller in: a shorter vector is smaller, +;; and of two as long, the one smaller at the first place they differ. Every +;; kept change makes the vector smaller in this order, so shrinking ends. +(: (-> (Vec ulong) (Vec ulong) bool)) +(defun (shortlex< a b) + (def n (vec-length a)) + (cond + ((< n (vec-length b)) #t) + ((> n (vec-length b)) #f) + (else + (let go ((i 0)) + (cond + ((= i n) #f) + ((< (vec-ref a i) (vec-ref b i)) #t) + ((> (vec-ref a i) (vec-ref b i)) #f) + (else (go (+ i 1)))))))) + +;; The state of one shrink: the smallest failing case so far and what it +;; failed with, and what the shrink has spent. +(type (: Shrink (Record (: prop Property) + (: size int) + (: open-ok bool) + (: signature string) + (: best (Vec ulong) #:mutable) + (: failures (List string) #:mutable) + (: shown (List string) #:mutable) + (: replays int #:mutable) + (: shrinks int #:mutable) + (: limit int) + (: reserve long) + (: cut-short bool #:mutable)))) + +;; Whether the shrink may replay once more: within its limit, and with more +;; than a fifth of the test's deadline left. +(: (-> Shrink bool)) +(defun (may-replay? s) + (def ok (and (< (record-ref s replays) (record-ref s limit)) + (> (test-time-left) (record-ref s reserve)))) + (unless ok (record-set! s (cut-short #t))) + ok) + +;; Replays `candidate`, and keeps it when it fails as the first case did and +;; is smaller than the best so far. What is kept is the part of the vector the +;; replay used. +(: (-> Shrink (Vec ulong) bool)) +(defun (try! s candidate) + (cond + ((not (shortlex< candidate (record-ref s best))) #f) + ((not (may-replay? s)) #f) + (else + (record-set! s (replays (+ (record-ref s replays) 1))) + (def c (replaying-choices candidate (record-ref s size))) + (match (run-case (record-ref s prop) c (record-ref s open-ok)) + ((Fail failures shown signature) + (def used (choices-draws c)) + (if (and (= signature (record-ref s signature)) (shortlex< used (record-ref s best))) + (begin + (record-set! s (best used) (failures failures) (shown shown) + (shrinks (+ (record-ref s shrinks) 1))) + #t) + #f)) + (_ #f))))) + +(: (-> (Vec ulong) int int (Vec ulong))) +(defun (vec-without v i k) + (loop (:for j (range 0 (vec-length v))) + (:when (or (< j i) (>= j (+ i k)))) + (:acc out (vecing (vec-ref v j))))) + +(: (-> (Vec ulong) int int (Vec ulong))) +(defun (vec-zeroed v i k) + (loop (:for j (range 0 (vec-length v))) + (:acc out (vecing (if (and (>= j i) (< j (+ i k))) 0UL (vec-ref v j)))))) + +(: (-> (Vec ulong) int ulong (Vec ulong))) +(defun (vec-with v i x) (vec-set v i x)) + +(: (-> (Vec ulong) int (Vec ulong))) +(defun (vec-swapped v i) + (vec-set (vec-set v i (vec-ref v (+ i 1))) (+ i 1) (vec-ref v i))) + +;; Deletes blocks of k draws, from every place. +(: (-> Shrink int bool)) +(defun (delete-blocks! s k) + (let go ((i 0) (any #f)) + (if (> (+ i k) (vec-length (record-ref s best))) + any + (if (try! s (vec-without (record-ref s best) i k)) + (go i #t) + (go (+ i 1) any))))) + +;; Sets blocks of k draws to 0, from every place. +(: (-> Shrink int bool)) +(defun (zero-blocks! s k) + (let go ((i 0) (any #f)) + (if (> (+ i k) (vec-length (record-ref s best))) + any + (go (+ i 1) (or (try! s (vec-zeroed (record-ref s best) i k)) any))))) + +;; Makes each draw smaller: 0 if that fails, else the smallest value that +;; fails, found by halving the distance between a value that passed and one +;; that failed. A property need not fail for every value above the smallest, +;; so this finds a small failing value, not always the smallest. +(: (-> Shrink bool)) +(defun (minimize-draws! s) + (let go ((i 0) (any #f)) + (if (>= i (vec-length (record-ref s best))) + any + (let ((v (vec-ref (record-ref s best) i))) + (cond + ((= v 0UL) (go (+ i 1) any)) + ((try! s (vec-with (record-ref s best) i 0UL)) (go (+ i 1) #t)) + (else (go (+ i 1) (or (search-down! s i 0UL v) any)))))))) + +;; Draw i fails at `hi` and passed at `lo`. Tries the middle until the two +;; meet, and answers whether it kept a smaller value. +(: (-> Shrink int ulong ulong bool)) +(defun (search-down! s i lo hi) + (if (or (<= (- hi lo) 1UL) (>= i (vec-length (record-ref s best)))) + #f + (let ((mid (+ lo (/ (- hi lo) 2UL)))) + (if (try! s (vec-with (record-ref s best) i mid)) + (begin (ignore (search-down! s i lo mid)) #t) + (search-down! s i mid hi))))) + +;; The draws near i that a step pairs with draw i: up to 8 on each side. +(: (-> int int (List int))) +(defun (neighbours i n) + (loop (:for j (range (if (< i 8) 0 (- i 8)) (if (> (+ i 9) n) n (+ i 9)))) + (:when (not (= j i))) + (:acc out (listing j)))) + +;; Deletes a block of k draws and adds their sum to a draw near it. A list +;; whose elements must add up to something loses an element this way, which +;; deleting alone cannot do: the others have to grow to make up for it. +(: (-> Shrink int bool)) +(defun (delete-and-shift! s k) + (let go ((i 0) (any #f)) + (def best (record-ref s best)) + (if (> (+ i k) (vec-length best)) + any + (let* ((removed (loop (:for j (range i (+ i k))) (:acc t (summing (vec-ref best j))))) + (rest (vec-without best i k)) + (kept (any? (fun (j) (try! s (vec-with rest j (+ (vec-ref rest j) removed)))) + (neighbours (if (< i (vec-length rest)) i (- (vec-length rest) 1)) + (vec-length rest))))) + (if kept (go i #t) (go (+ i 1) any)))))) + +;; Moves value from a draw to a later one near it: the earlier draw gets +;; smaller, which makes the vector smaller, and the later one makes up for +;; it. Tries moving all of the earlier draw, then half of it. +(: (-> Shrink bool)) +(defun (redistribute! s) + (let go ((i 0) (any #f)) + (def best (record-ref s best)) + (if (>= i (vec-length best)) + any + (let* ((v (vec-ref best i)) + (moved (and (> v 0UL) + (any? (fun (j) + (and (> j i) + (let ((b (record-ref s best))) + (or (try! s (vec-with (vec-with b i 0UL) j (+ (vec-ref b j) v))) + (try! s (vec-with (vec-with b i (- v (/ v 2UL))) j + (+ (vec-ref b j) (/ v 2UL)))))))) + (neighbours i (vec-length best)))))) + (go (+ i 1) (or moved any)))))) + +;; Swaps adjacent draws that are out of order. +(: (-> Shrink bool)) +(defun (sort-pairs! s) + (let go ((i 0) (any #f)) + (if (>= (+ i 1) (vec-length (record-ref s best))) + any + (let ((b (record-ref s best))) + (if (and (> (vec-ref b i) (vec-ref b (+ i 1))) (try! s (vec-swapped b i))) + (go (+ i 1) #t) + (go (+ i 1) any)))))) + +;; Shrinks until a whole round of the steps changes nothing, or the shrink is +;; out of replays or time. +(: (-> Shrink void)) +(defun (shrink! s) + (let round () + (def deleted + (loop (:for k (list 8 4 2 1)) + (:acc any (folding #f (let ((c (delete-blocks! s k))) (or c any)))))) + (def zeroed + (loop (:for k (list 8 4 2 1)) + (:acc any (folding #f (let ((c (zero-blocks! s k))) (or c any)))))) + (def minimized (minimize-draws! s)) + (def sorted (sort-pairs! s)) + ;; The two steps that move value between draws cost many replays each, + ;; so they run only when the cheap steps above found nothing. + (def moved + (if (or deleted zeroed minimized sorted) + #f + (let* ((a (delete-and-shift! s 2)) + (b (delete-and-shift! s 1)) + (c (redistribute! s))) + (or a b c)))) + (def changed (or deleted zeroed minimized sorted moved)) + (when (and changed (not (record-ref s cut-short))) + (round)))) + +;; --------------------------------------------------------------------------- +;; The failure file +;; --------------------------------------------------------------------------- + +;; A stored case: which `for-all` it is for, the seed of the run that found +;; it, for a person to read, the size the draws were made at, and the draws. +;; In the file, a case is +;; +;; (case (test "name") (bindings "((: xs (List int)))") (index 0) +;; (seed "123") (size 37) (draws [1 0 5])) +;; +;; A `for-all` is found by its test, the source text of its bindings, and its +;; position among the `for-all`s of the test with the same bindings. Editing +;; the body keeps the case, since a case is most useful while the bug is being +;; fixed. Changing the generators drops it, since the draws no longer mean +;; what they meant. A `for-all` added to the test does not take over the cases +;; of another, unless its bindings are written the same. +;; +;; with a draw above long-max written as a ulong, 18446744073709551615UL, and +;; the others as plain numbers. +(type (: StoredCase (Record (: test string) + (: bindings string) + (: index int) + (: seed string) + (: size int) + (: draws (Vec ulong))))) + +(: (-> string Bjodat Bjodat)) +(defun (clause name value) (BjoList (list (BjoSym (string->symbol name)) value))) + +(: (-> StoredCase Bjodat)) +(defun (case->bjodat c) + (BjoList + (list (BjoSym (string->symbol "case")) + (clause "test" (BjoStr (record-ref c test))) + (clause "bindings" (BjoStr (record-ref c bindings))) + (clause "index" (BjoInt (cast System.Int64 (record-ref c index)))) + (clause "seed" (BjoStr (record-ref c seed))) + (clause "size" (BjoInt (cast System.Int64 (record-ref c size)))) + (clause "draws" (BjoVec (vec-map draw->bjodat (record-ref c draws))))))) + +(: (-> ulong Bjodat)) +(defun (draw->bjodat d) + (if (<= d 9223372036854775807UL) (BjoInt (cast System.Int64 d)) (BjoULong d))) + +;; The value of the clause `(name value)` in a form's clauses. +(: (-> string (List Bjodat) (Option Bjodat))) +(defun (clause-value name clauses) + (match clauses + (Nil None) + ((Cons (BjoList (Cons (BjoSym s) (Cons v Nil))) more) + (if (= (symbol->string s) name) (Some v) (clause-value name more))) + ((Cons _ more) (clause-value name more)))) + +(: (-> Bjodat (Option StoredCase))) +(defun (bjodat->case d) + (match d + ((BjoList (Cons (BjoSym head) clauses)) + (match (Tuple (clause-value "test" clauses) (clause-value "bindings" clauses) + (clause-value "index" clauses) (clause-value "seed" clauses) + (clause-value "size" clauses) (clause-value "draws" clauses)) + ((Tuple (Some (BjoStr test)) (Some (BjoStr bindings)) (Some (BjoInt index)) (Some (BjoStr seed)) + (Some (BjoInt size)) (Some (BjoVec draws))) + (if (= (symbol->string head) "case") + (Some (StoredCase (test test) (bindings bindings) (index (cast System.Int32 index)) (seed seed) + (size (cast System.Int32 size)) + (draws (vec-map (fun (x) (match x + ((BjoInt n) (cast System.UInt64 n)) + ((BjoULong n) n) + (_ 0UL))) + draws)))) + None)) + (_ None))) + (_ None))) + +;; The cases of a failure file: `(failed case ...)`. What does not read as a +;; case is left out. +(: (-> string (List StoredCase))) +(defun (parse-stored text) + (match (parse-bjodat text) + ((Ok (BjoList (Cons (BjoSym head) cases))) + (if (= (symbol->string head) "failed") + (loop (:for c cases) + (:when-let (Some sc) (bjodat->case c)) + (:acc out (listing sc))) + (list))) + (_ (list)))) + +;; The cases of each file that was read, by its path: the text it had, and the +;; cases now. +(: (Box (Map string (Tuple string (List StoredCase))))) +(def store-box (make-box (map-empty))) + +;; The failure file of a test file: beside it, NAME.failed.bjodat. +(: (-> string string)) +(defun (store-path test-file) + (if (string-ends-with? test-file ".bjo") + (str (substring/slice-prefix test-file) ".failed.bjodat") + (str test-file ".failed.bjodat"))) + +(: (-> string string)) +(defun (substring/slice-prefix path) + (list->string (list-reverse (list-tail (list-tail (list-tail (list-tail (list-reverse (string->list path))))))))) + +(: (-> (List StoredCase) string)) +(defun (stored-text cases) + (if (list-empty? cases) + "" + (str "(failed\n" + (string-join (list->vec (list-map (fun (c) (str " " (bjodat->string (case->bjodat c)))) cases)) "\n") + ")\n"))) + +;; The cases of a file, read on first use. A file that does not read is +;; treated as empty, and written over when the run changes it. +(: (-> string (List StoredCase))) +(defun (stored-cases path) + (match (map-try-ref (box-ref store-box) path) + ((Some (Tuple _ cases)) cases) + (None + (def text (if (file-exists? path) (file-read-text path) "")) + (def cases (parse-stored text)) + (box-set! store-box (map-set (box-ref store-box) path (Tuple text cases))) + (at-run-end! "std test prop: store" write-stores!) + cases))) + +(: (-> string (List StoredCase) void)) +(defun (set-stored-cases! path cases) + (ignore (stored-cases path)) + (match (map-try-ref (box-ref store-box) path) + ((Some (Tuple text _)) (box-set! store-box (map-set (box-ref store-box) path (Tuple text cases)))) + (None unit))) + +(: (-> string string int StoredCase bool)) +(defun (case-for? test bindings index c) + (and (= (record-ref c test) test) + (= (record-ref c bindings) bindings) + (= (record-ref c index) index))) + +;; Writes each file whose cases changed, and deletes it when none is left. A +;; run without filters also drops the cases of tests that no longer exist. +(: (-> void)) +(defun (write-stores!) + (def o (current-test-options)) + (when (record-ref o store) + (def names (list-map test-name (registered-tests))) + (map-for-each + (fun (path entry) + (match entry + ((Tuple text cases) + (def kept + (if (vec-empty? (record-ref o filters)) + (list-filter (fun (c) (any? (fun (n) (= n (record-ref c test))) names)) cases) + cases)) + (def new-text (stored-text kept)) + (unless (= new-text text) + (if (string-empty? new-text) + (when (file-exists? path) (file-delete path)) + (file-write-text path new-text)))))) + (box-ref store-box)))) + +;; --------------------------------------------------------------------------- +;; for-all +;; --------------------------------------------------------------------------- + +(: (-> (List string) (List string) (List string))) +(defun (shown-lines names shown) + (match (Tuple names shown) + ((Tuple (Cons n nm) (Cons v vm)) (cons #" ${n} = ${v}" (shown-lines nm vm))) + (_ (list)))) + +(: (-> string int string)) +(defun (where-text file line) + (def name (match (path-filename file) ((Some n) n) (None file))) + #"${name}:${line}") + +;; Shrinks a failing case and reports the smallest one found. Answers the +;; draws of that case, to store. +(: (-> Property int bool (List string) (Vec ulong) (List string) (List string) + string string (Vec ulong))) +(defun (shrink-and-report! prop size open-ok names draws failures shown signature heading) + (def o (current-test-options)) + (def limit (if (> (record-ref o shrink-limit) 0) (record-ref o shrink-limit) 2000)) + (def within (match (running-test) ((Some _) (test-time-left)) (None 86400000L))) + (def s (Shrink (prop prop) (size size) (open-ok open-ok) (signature signature) + (best draws) (failures failures) + (shown shown) (replays 0) (shrinks 0) (limit limit) + (reserve (/ within 5L)) (cut-short #f))) + (shrink! s) + (def shrinks (record-ref s shrinks)) + (def how + (cond + ((record-ref s cut-short) + #" shrunk ${shrinks} time(s), then stopped, out of time or replays, at:") + ((= shrinks 0) " with:") + (else #" shrunk ${shrinks} time(s) to:"))) + (test-failed! + (string-join + (list->vec (list-append (cons heading (cons how (shown-lines names (record-ref s shown)))) + (record-ref s failures))) + "\n")) + (record-ref s best)) + +(: (-> string int (Map string int) void)) +(defun (print-stats where runs counts) + (println #"for-all at ${where}: ${runs} run(s)") + (loop (:for (Tuple label n) (list-sort-by-count (map->list counts))) + (:do (println #" ${(/ (* n 100) (if (= runs 0) 1 runs))}% ${label}")))) + +(: (-> (List (Tuple string int)) (List (Tuple string int)))) +(defun (list-sort-by-count xs) + (defun (insert p sorted) + (match sorted + (Nil (list p)) + ((Cons q more) + (if (> (count-of p) (count-of q)) (cons p sorted) (cons q (insert p more)))))) + (list-foldl insert (list) xs)) + +(: (-> (Tuple string int) int)) +(defun (count-of p) (match p ((Tuple _ n) n))) + +(: (-> (Map string int) (List string) (Map string int))) +(defun (count-labels counts labels) + (list-foldl (fun (l m) (map-set m l (+ 1 (map-ref-or m l 0)))) counts labels)) + +;; Whether `sc` is one of the stored cases that still fail. +(: (-> (List (Tuple StoredCase (Tuple (List string) (Tuple (List string) string)))) StoredCase bool)) +(defun (failing-case? failing sc) + (any? (fun (f) (match f ((Tuple fc _) (eq? fc sc)))) failing)) + +;; What `for-all` expands to. `runs` is 0 when the form gives no `#:runs`. +(: (-> string int int string (List string) Property void)) +(defun (run-for-all! file line runs bindings names prop) + (def t + (match (running-test) + ((Some t) t) + (None (raise (cast Exception (InvalidOperationException. + #"for-all at ${(where-text file line)} runs only inside a deftest.")))))) + (def o (current-test-options)) + (def where (where-text file line)) + (def total (cond ((> runs 0) runs) ((> (record-ref o runs) 0) (record-ref o runs)) (else 100))) + (def index (next-test-index! bindings)) + (def test (test-name t)) + (def open-ok (record-ref t open-ok)) + (def path (store-path (test-file t))) + (def seed-text (displayed (record-ref o seed))) + (def stored + (if (record-ref o store) + (list-filter (fun (c) (case-for? test bindings index c)) (stored-cases path)) + (list))) + ;; Every stored case is tried first. Those that pass are dropped, and those + ;; that still fail stay. The first that fails is shrunk again and reported, + ;; and the random runs are skipped. + (def failing + (loop (:for c stored) + (:let outcome (run-case prop (replaying-choices (record-ref c draws) (record-ref c size)) open-ok)) + (:when-let (Fail failures shown signature) outcome) + (:acc out (listing (Tuple c (Tuple failures (Tuple shown signature))))))) + (unless (= (list-length failing) (list-length stored)) + (set-stored-cases! path (list-filter (fun (sc) (or (not (case-for? test bindings index sc)) (failing-case? failing sc))) + (stored-cases path)))) + (match failing + ((Cons (Tuple c (Tuple failures (Tuple shown signature))) _) + (def best (shrink-and-report! prop (record-ref c size) open-ok names (record-ref c draws) failures shown signature + #" for-all at ${where} failed on a case stored in ${(where-text path 1)} (seed ${(record-ref c seed)})")) + ;; The case keeps its place in the file, so a run that changes nothing + ;; writes nothing. + (set-stored-cases! path (list-map (fun (sc) (if (eq? sc c) (record-set c (draws best)) sc)) + (stored-cases path)))) + (Nil (random-runs! t o where total bindings index names prop open-ok path seed-text)))) + +(: (-> Test TestOptions string int string int (List string) Property bool string string void)) +(defun (random-runs! t o where total bindings index names prop open-ok path seed-text) + (def root (make-random/seed64 + (mix-seed (test-seed t) (mix-seed (string-hash bindings) (cast System.UInt64 index))))) + (def counts (make-box (map-empty))) + (def labels (make-box (list))) + (let go ((i 0) (ran 0) (discarded 0)) + (cond + ((= ran total) (finish-stats o where ran (box-ref counts))) + ((> discarded (* 10 total)) + (finish-stats o where ran (box-ref counts)) + (test-failed! #" for-all at ${where} ran only ${ran} of ${total} run(s): ${discarded} were discarded, by assume or gen-such-that, so the property was hardly tested")) + (else + (def size (gen-size-of-run ran total)) + (def c (recording-choices (random-split root) size)) + (box-set! labels (list)) + (def saved-labels (box-ref labels-box)) + (box-set! labels-box (Some labels)) + (def outcome (run-case prop c open-ok)) + (box-set! labels-box saved-labels) + (match outcome + (Pass + (box-set! counts (count-labels (box-ref counts) (box-ref labels))) + (go (+ i 1) (+ ran 1) discarded)) + (Discard (go (+ i 1) ran (+ discarded 1))) + ((Fail failures shown signature) + (finish-stats o where (+ ran 1) (box-ref counts)) + (def draws (choices-draws c)) + (def heading #" for-all at ${where} failed on run ${(+ ran 1)} of ${total} (seed ${seed-text})") + ;; A case that passes when replayed depends on something other than + ;; its draws, and shrinking it would chase noise. + (match (run-case prop (replaying-choices draws size) open-ok) + ((Fail _ _ _) + (def best (shrink-and-report! prop size open-ok names draws failures shown signature heading)) + (when (record-ref o store) + (set-stored-cases! path + (list-append (stored-cases path) + (list (StoredCase (test (test-name t)) (bindings bindings) (index index) + (seed seed-text) (size size) + (draws best))))))) + (_ + (test-failed! + (string-join + (list->vec (list-append (cons heading + (cons " and passed when run again with the same values, so it depends on something besides them:" + (shown-lines names shown))) + failures)) + "\n")))))))))) + +(: (-> TestOptions string int (Map string int) void)) +(defun (finish-stats o where ran counts) + (when (and (record-ref o stats) (not (map-empty? counts))) + (print-stats where ran counts))) + +;; The size of run i of n: 1 at the first run, 100 at the last. +(: (-> int int int)) +(defun (gen-size-of-run i n) + (if (<= n 1) 100 (+ 1 (/ (* i 99) (- n 1))))) + +;; --------------------------------------------------------------------------- +;; In a body +;; --------------------------------------------------------------------------- + +;; `discard!` at void, which is what a statement position wants. +(: (-> Choices void)) +(defun (discard-run! c) (discard! c "assume")) + +;; Discards the run when `ok` is #f. A discarded run is not counted. +(: (-> bool void)) +(defun (assume ok) + (unless ok + (match (box-ref choices-box) + ((Some c) (discard-run! c)) + (None (raise (cast Exception (InvalidOperationException. "assume is for the body of a for-all."))))))) + +;; Counts the run under `name` when `ok` is #t. `--stats` prints the counts. +(: (-> string bool void)) +(defun (classify name ok) + (when ok + (match (box-ref labels-box) + ((Some b) (box-set! b (cons name (box-ref b)))) + (None unit)))) + +;; Counts the run under the value as text. `--stats` prints the counts. +(: (-> %a void)) +(defun (collect value) (classify (displayed value) #t)) + +;; Runs `body`, and puts `name` above each failure in it. +(: (-> string (-> void) void)) +(defun (with-label name body) + (def failures (make-box (list))) + (match (try (failures-into failures body) #:catch (System.Exception)) + ((Ok _) unit) + ((Err e) (box-set! failures (cons #" it raised: ${(.-Message e)}" (box-ref failures))))) + (loop (:for f (list-reverse (box-ref failures))) + (:do (test-failed! #" in ${name}:\n${f}")))) + +;; `(label "name" body ...)` — a failure in the body is reported under the +;; name. +(def/macro (label form inject compare) + (syntax-match form + ((_ name first body ...) #'(with-label ,name (fun () ,first ,@body unit))) + (bad (syntax-error bad "label takes a name and a body: (label \"after insert\" body ...).")))) + +;; Whether a form is the bare `:` that a type declaration begins with. +(: (-> Syntax bool)) +(defun (colon? s) + (match s + ((SList _) #f) + (_ (= (syntax->string s) ":")))) + +;; The definition of one binding of a `for-all`: `(name gen)`, or +;; `(: name Type)` for the type's `arbitrary`, declared as a `def` is. +(: (-> Syntax Syntax Syntax)) +(defun (binding-def choices b) + (match b + ((SList (Cons colon (Cons name (Cons type Nil)))) + (cond + ((not (colon? colon)) + (if (colon? name) + (syntax-error b "A typed binding of for-all is declared as a def is: (: name Type), not (name : Type).") + (syntax-error b "A binding of for-all is (name generator) or (: name Type)."))) + ;; The compiler would say only that a dictionary is missing. + ((type-variable-in? type) + (syntax-error type "for-all draws values of a type it knows, and this type has a type variable, such as %a, in it. Write a type such as (List int), or give a generator: (name gen).")) + (else #'(def (: ,name ,type) (prop-arbitrary ,choices))))) + ((SList (Cons name (Cons g Nil))) #'(def ,name (prop-draw ,g ,choices))) + (_ (syntax-error b "A binding of for-all is (name generator) or (: name Type).")))) + +;; Whether a type, as written, names a type variable: %a, alone or inside. +(: (-> Syntax bool)) +(defun (type-variable-in? type) + (match type + ((SList items) (any? type-variable-in? items)) + (_ (let ((s (syntax->string type))) + (or (string-starts-with? s "%") (string-starts-with? s "'")))))) + +(: (-> Syntax Syntax)) +(defun (binding-name b) + (match b + ((SList (Cons colon (Cons name (Cons _ Nil)))) (if (colon? colon) name colon)) + ((SList (Cons name _)) name) + (_ b))) + +(: (-> Syntax Syntax (List Syntax) (List Syntax) Syntax)) +(defun (for-all-expansion form runs bindings body) + (when (list-empty? bindings) + (ignore (syntax-error form "for-all takes at least one binding: (for-all ((x gen-int)) body ...)."))) + (def names (list-map binding-name bindings)) + #'(run-for-all! ,(SStr (syntax-file form)) ,(SInt (int->string (syntax-line form))) ,runs + ,(SStr (syntax->string (SList bindings))) + (list ,@(list-map (fun (n) (SStr (syntax->string n))) names)) + (fun (choices) + ,@(list-map (fun (b) (binding-def #'choices b)) bindings) + (Tuple (list ,@(list-map (fun (n) #'(displayed ,n)) names)) + (fun () ,@body unit))))) + +;; `(for-all ((name gen) ...) body ...)`, or with `#:runs n` before the +;; bindings. A binding is `(name generator)` or `(: name Type)`. +(def/macro (for-all form inject compare) + (syntax-match form + ((_ #:runs n (binding ...) first body ...) + (for-all-expansion form n binding (cons first body))) + ((_ (binding ...) first body ...) + (for-all-expansion form #'0 binding (cons first body))) + (bad (syntax-error bad "for-all takes bindings and a body: (for-all ((xs (gen-list gen-int))) (check ...)), optionally #:runs n before the bindings.")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/text.bjo — persistent text: a B+ tree of UTF-8 chunks. Inserting, +;; deleting, splitting and joining are O(log n) and share what they do not +;; change. +;; +;; - Positions are cursors, as in a string; no function takes or returns a +;; byte offset. A cursor holds the byte offset internally, and a character +;; index becomes a cursor in O(log n) through the character counts. +;; - A text of at most `chunk-max` bytes is one string and no tree. +;; - The tree's nodes are C# (`Bjolang.Runtime.TextTree`), because they hold their +;; slots in inline arrays. Every operation is here, including those on one +;; node. +;; - A node has up to 16 children and keeps each one's length in bytes and its +;; number of newlines. The children of a leaves node (height 1) are chunks: +;; strings of valid UTF-8 of at most `chunk-max` bytes. The children of a +;; branch are nodes one level down. +;; - Every node other than a root has at least `min-kids` children. Chunks +;; are kept at `chunk-min` bytes or more where an edit can merge them with a +;; neighbour in the same node. This is not an invariant: reads do not +;; depend on it. +;; - An edit copies the nodes on its path. A transient's owner lets it edit +;; in place the nodes it copied. Owner 0 copies every node it edits. + +(import (std prelude)) +(import (std atomic)) +(import (std unicode grapheme)) + +(:doc #:file "text.bjodoc") + +(export + Text TextWalk TextBuilder TransientText TextCursor TextBackward TextBackwardStep Texting TextingInto + TextPiece textbuilder-add-piece! + ;; Texts + text-empty text string->text text->string text-length text-count text-empty? text-line-count + text-substring text-split-at text-append text-insert text-insert-text text-delete text-replace-range + ;; As a string has them + text-take text-drop text-take-right text-drop-right + text-contains? text-starts-with? text-ends-with? + text-replace text-split text-split-once text-join + text-search text-search-backward text-search-any text-find + text-trim text-trim-left text-trim-right text-pad-left text-pad-right + text-upcase text-downcase text-reverse + text-fold text-fold-right text-for-each text->list text->vec + ;; Lines + text-line + ;; Cursors + text-cursor-start text-cursor-end text-cursor-start? text-cursor-end? + text-cursor-ref text-cursor-next text-cursor-prev text-cursor-ref+next text-cursor-forward text-cursor-back + text-cursor-at-line text-cursor-index text-cursor-line text-cursor-column text-cursor-line-start text-cursor-line-end + text-cursor-search text-cursor-search-backward text-cursor-find text-cursor-find-backward + text-cursor=? text-cursor? text-cursor<=? text-cursor>=? + text-backward + ;; Grapheme clusters + text-cursor-next-grapheme text-cursor-prev-grapheme text-cursor-grapheme-boundary? + text-grapheme-count text->graphemes + ;; Chunks + text-fold-chunks text-for-each-chunk text-chunk-count + ;; Builders + textbuilder-empty textbuilder-add-string! textbuilder-add! textbuilder-add-text! textbuilder-length textbuilder->text + ;; Transients + text->transienttext transienttext->text transienttext-insert! transienttext-delete! + transienttext-replace-range! transienttext-append! transienttext-length transienttext-count + ;; Checks, for tests + text-check text-height) + +(import/class + (ArgumentException (: System.ArgumentException)) + (Utf8Builder (: BjoString.Utf8StringBuilder)) + (ArgumentOutOfRangeException (: System.ArgumentOutOfRangeException)) + (Exception (: System.Exception)) + (TextNode (: Bjolang.Runtime.TextTree.TextNode)) + (TextBranch (: Bjolang.Runtime.TextTree.TextBranch)) + (TextLeaves (: Bjolang.Runtime.TextTree.TextLeaves)) + (StringWalk (: Bjolang.Runtime.StringWalk))) + +(import/class ((Lanes %a) (: System.Runtime.Intrinsics.Vector128))) +(import/class ((Bytes32 %a) (: System.Runtime.Intrinsics.Vector256))) + +(import/extern + (new-branch (: Bjolang.Runtime.TextTree.TextNodes.NewBranch (-> long TextBranch))) + (new-leaves (: Bjolang.Runtime.TextTree.TextNodes.NewLeaves (-> long TextLeaves))) + (copy-branch (: Bjolang.Runtime.TextTree.TextNodes.CopyBranch (-> TextBranch long TextBranch))) + (copy-leaves (: Bjolang.Runtime.TextTree.TextNodes.CopyLeaves (-> TextLeaves long TextLeaves))) + (as-branch (: Bjolang.Runtime.TextTree.TextNodes.AsBranch (-> TextNode TextBranch))) + (as-leaves (: Bjolang.Runtime.TextTree.TextNodes.AsLeaves (-> TextNode TextLeaves))) + (branch-node (: Bjolang.Runtime.TextTree.TextNodes.BranchNode (-> TextBranch TextNode))) + (leaves-node (: Bjolang.Runtime.TextTree.TextNodes.LeavesNode (-> TextLeaves TextNode))) + (node-owner (: Bjolang.Runtime.TextTree.TextNodes.Owner (-> TextNode long))) + (node-count (: Bjolang.Runtime.TextTree.TextNodes.Count (-> TextNode int))) + (set-count! (: Bjolang.Runtime.TextTree.TextNodes.SetCount (-> TextNode int void))) + (slot-end (: Bjolang.Runtime.TextTree.TextNodes.End (-> TextNode int int))) + (set-slot-end! (: Bjolang.Runtime.TextTree.TextNodes.SetEnd (-> TextNode int int void))) + (slot-ends (: Bjolang.Runtime.TextTree.TextNodes.Ends (-> TextNode int (RoSpan int)))) + (slot-ends4 (: Bjolang.Runtime.TextTree.TextNodes.Ends4 (-> TextNode int (Lanes int)))) + (set-slot-ends4! (: Bjolang.Runtime.TextTree.TextNodes.SetEnds4 (-> TextNode int (Lanes int) void))) + (slot-lines (: Bjolang.Runtime.TextTree.TextNodes.Lines (-> TextNode int int))) + (set-slot-lines! (: Bjolang.Runtime.TextTree.TextNodes.SetLines (-> TextNode int int void))) + (slot-chars (: Bjolang.Runtime.TextTree.TextNodes.Chars (-> TextNode int int))) + (set-slot-chars! (: Bjolang.Runtime.TextTree.TextNodes.SetChars (-> TextNode int int void))) + (kid (: Bjolang.Runtime.TextTree.TextNodes.Kid (-> TextBranch int TextNode))) + (set-kid! (: Bjolang.Runtime.TextTree.TextNodes.SetKid (-> TextBranch int TextNode void))) + (chunk-at (: Bjolang.Runtime.TextTree.TextNodes.Chunk (-> TextLeaves int string))) + (set-chunk! (: Bjolang.Runtime.TextTree.TextNodes.SetChunk (-> TextLeaves int string void))) + (string-walk (: Bjolang.Runtime.StringWalks.Start (-> string StringWalk))) + (string-walk-done? (: Bjolang.Runtime.StringWalks.Done (-> StringWalk bool))) + (string-walk-current (: Bjolang.Runtime.StringWalks.Current (-> StringWalk char))) + (string-walk-next (: Bjolang.Runtime.StringWalks.Next (-> StringWalk StringWalk))) + ;; One allocation of the exact size, for a chunk with text put in it. + (concat2 (: BjoString.Utf8String.Concat (-> string string string))) + (concat3 (: BjoString.Utf8String.Concat (-> string string string string))) + (concat-all (: BjoString.Utf8String.Concat (-> (RoSpan string) string))) + ;; Checks the bytes and copies them. + (utf8->string (: BjoString.Utf8String.FromUtf8 (-> (RoSpan byte) string))) + ;; Vectorized in .NET. + (span-count (: System.MemoryExtensions.Count (-> (RoSpan byte) byte int))) + (span-index-of (: System.MemoryExtensions.IndexOf (-> (RoSpan byte) byte int))) + (span-find (: System.MemoryExtensions.IndexOf (-> (RoSpan byte) (RoSpan byte) int))) + (span-equal? (: System.MemoryExtensions.SequenceEqual (-> (RoSpan byte) (RoSpan byte) bool))) + (span-compare (: System.MemoryExtensions.SequenceCompareTo (-> (RoSpan byte) (RoSpan byte) int))) + (span-find-last (: System.MemoryExtensions.LastIndexOf (-> (RoSpan byte) (RoSpan byte) int))) + (span-last-index-of (: System.MemoryExtensions.LastIndexOf (-> (RoSpan byte) byte int))) + (span-index-of-any (: System.MemoryExtensions.IndexOfAny (-> (RoSpan byte) (RoSpan byte) int))) + (span-first-outside (: System.MemoryExtensions.IndexOfAnyExceptInRange (-> (RoSpan int) int int int))) + (pop-count32 (: System.Numerics.BitOperations.PopCount (-> uint int))) + ;; 32 bytes at once. + (bytes32-of (: System.Runtime.Intrinsics.Vector256.Create (-> (RoSpan byte) (Bytes32 byte)))) + (bytes32-splat (: System.Runtime.Intrinsics.Vector256.Create (-> byte (Bytes32 byte)))) + (bytes32-and (: System.Runtime.Intrinsics.Vector256.BitwiseAnd (-> (Bytes32 byte) (Bytes32 byte) (Bytes32 byte)))) + (bytes32-eq (: System.Runtime.Intrinsics.Vector256.Equals (-> (Bytes32 byte) (Bytes32 byte) (Bytes32 byte)))) + (bytes32-mask (: System.Runtime.Intrinsics.Vector256.ExtractMostSignificantBits (-> (Bytes32 byte) uint))) + ;; Four ints at once. + (lanes-splat (: System.Runtime.Intrinsics.Vector128.Create (-> int (Lanes int)))) + (lanes-of (: System.Runtime.Intrinsics.Vector128.Create (-> int int int int (Lanes int)))) + (lanes-add (: System.Runtime.Intrinsics.Vector128.Add (-> (Lanes int) (Lanes int) (Lanes int)))) + (lanes-and (: System.Runtime.Intrinsics.Vector128.BitwiseAnd (-> (Lanes int) (Lanes int) (Lanes int)))) + (lanes-ge (: System.Runtime.Intrinsics.Vector128.GreaterThanOrEqual (-> (Lanes int) (Lanes int) (Lanes int)))) + ;; The full 128-bit product, one mulx. + (big-mul (: System.Math.BigMul (-> ulong ulong uint128)))) + +(def cap 16) +(def min-kids 8) +(def chunk-max 1024) +(def chunk-min 256) +;; The longest text an edit puts into one chunk. A chunk that grows past +;; `chunk-max` then splits in two, each at most `chunk-max`. +(def small-insert 512) + +(def newline (cast byte 10)) + +(: (-> string %a)) +(defun (bad-argument why) (raise (cast Exception (ArgumentException. why)))) + +(: (-> string %a)) +(defun (out-of-range why) (raise (cast Exception (ArgumentOutOfRangeException. "offset" why)))) + +;; --------------------------------------------------------------------------- +;; Chunks +;; --------------------------------------------------------------------------- + +(: (-> string int)) +(defun (newlines-in s) #:inline (span-count (.AsSpan s) newline)) + +;; The newlines in bytes `from` to `to` of `s`. +(: (-> string int int int)) +(defun (newlines-between s from to) + (if (= from to) 0 (span-count (ro-span-slice (.AsSpan s) #:from from #:to to) newline))) + +;; The characters of `s`, counted with vectors. +(: (-> string int)) +(defun (chars-in s) #:inline (if (<= (string-length s) 1) (string-length s) (.Count s))) + +;; The characters in bytes `from` to `to` of `s`, boundaries of `s`. +;; +;; - Returns the bytes less those that continue a character (10xxxxxx), +;; counted 32 at a time. +;; - Allocates nothing. +(: (-> string int int int)) +(defun (chars-between s from to) + (def sp (ro-span-slice (.AsSpan s) #:from from #:to to)) + (def len (- to from)) + (def blocks (* 32 (/ len 32))) + (def top (bytes32-splat (cast byte 192))) + (def cont (bytes32-splat (cast byte 128))) + (def in-blocks + (let go ((j 0) (n 0)) + (if (>= j blocks) + n + (begin + (def v (bytes32-of (ro-span-slice sp #:from j #:to (+ j 32)))) + (go (+ j 32) (+ n (pop-count32 (bytes32-mask (bytes32-eq (bytes32-and v top) cont))))))))) + (let step ((j blocks) (n in-blocks)) + (cond + ((>= j len) (- len n)) + ((= (bitwise-and (cast int (ro-span-ref sp j)) 192) 128) (step (+ j 1) (+ n 1))) + (else (step (+ j 1) n))))) + +;; Whether byte `k` of `s` starts a character, or is its end. +(: (-> string int bool)) +(defun (boundary? s k) + (or (<= k 0) (>= k (string-length s)) + (not (= (bitwise-and (cast int (ro-span-ref (.AsSpan s) k)) 192) 128)))) + +;; The last character boundary at or before `k`. +(: (-> string int int)) +(defun (boundary-before s k) + (let go ((k k)) (if (boundary? s k) k (go (- k 1))))) + +;; Bytes `from` to `to` of `s`, sharing its array. +(: (-> string int int string)) +(defun (piece s from to) #:inline (.SliceBytes s from (- to from))) + +;; --------------------------------------------------------------------------- +;; Nodes +;; --------------------------------------------------------------------------- + +(def owners (make-atomic 0L)) + +(: (-> long)) +(defun (owner-next) (atomic-increment! owners)) + +;; A node that holds nothing, the root of a text that is one string. +(def no-node (leaves-node (new-leaves 0L))) + +(: (-> int long TextNode)) +(defun (new-node h o) + (if (= h 1) (leaves-node (new-leaves o)) (branch-node (new-branch o)))) + +;; `n` if `o` owns it, and otherwise a copy that `o` owns. +(: (-> int TextNode long TextNode)) +(defun (editable h n o) + (cond + ((and (not (= o 0L)) (= (node-owner n) o)) n) + ((= h 1) (leaves-node (copy-leaves (as-leaves n) o))) + (else (branch-node (copy-branch (as-branch n) o))))) + +;; The bytes before slot `i` of `n`. +(: (-> TextNode int int)) +(defun (slot-base n i) #:inline (if (= i 0) 0 (slot-end n (- i 1)))) + +(: (-> TextNode int int)) +(defun (slot-bytes n i) #:inline (- (slot-end n i) (slot-base n i))) + +;; The slot numbers of 4 lanes from `from`. +(def lanes-0 (lanes-of 0 1 2 3)) +(def lanes-4 (lanes-of 4 5 6 7)) +(def lanes-8 (lanes-of 8 9 10 11)) +(def lanes-12 (lanes-of 12 13 14 15)) + +;; Adds `d` to the 4 ends from slot `from` that are of slot `i` or after. +(: (-> TextNode int (Lanes int) (Lanes int) (Lanes int) void)) +(defun (add-ends4! n from slots i d) #:inline + (set-slot-ends4! n from (lanes-add (slot-ends4 n from) (lanes-and (lanes-ge slots i) d)))) + +;; Adds `d` to the ends of slot `i` and every slot after it, including those +;; past the count, 4 at a time with no branch. +(: (-> TextNode int int void)) +(defun (add-slot-bytes! n i d) + (def li (lanes-splat i)) + (def ld (lanes-splat d)) + (add-ends4! n 0 lanes-0 li ld) + (add-ends4! n 4 lanes-4 li ld) + (add-ends4! n 8 lanes-8 li ld) + (add-ends4! n 12 lanes-12 li ld)) + +;; Slot `i` of `n` holds `b` bytes; the slots after it, up to the count, keep +;; theirs. +(: (-> TextNode int int void)) +(defun (set-slot-bytes! n i b) #:inline + (add-slot-bytes! n i (- b (slot-bytes n i)))) + +(: (-> TextNode int)) +(defun (sum-bytes n) + (def count (node-count n)) + (if (= count 0) 0 (slot-end n (- count 1)))) + +(: (-> TextNode int)) +(defun (sum-lines n) + (def count (node-count n)) + (let go ((i 0) (acc 0)) (if (< i count) (go (+ i 1) (+ acc (slot-lines n i))) acc))) + +(: (-> TextNode int)) +(defun (sum-chars n) + (def count (node-count n)) + (let go ((i 0) (acc 0)) (if (< i count) (go (+ i 1) (+ acc (slot-chars n i))) acc))) + +;; Slot `i` of the leaves node `n` holds `c`. +(: (-> TextNode int string void)) +(defun (put-chunk! n i c) + (set-chunk! (as-leaves n) i c) + (set-slot-bytes! n i (string-length c)) + (set-slot-lines! n i (newlines-in c)) + (set-slot-chars! n i (chars-in c))) + +;; Slot `i` of the branch `n` holds the node `k`. +(: (-> TextNode int TextNode void)) +(defun (put-kid! n i k) + (set-kid! (as-branch n) i k) + (set-slot-bytes! n i (sum-bytes k)) + (set-slot-lines! n i (sum-lines k)) + (set-slot-chars! n i (sum-chars k))) + +;; Copies slot `from` of `src` to slot `to` of `dst`, both of height `h`. The +;; end of the slot is that of `src` plus `d`. +(: (-> int TextNode int TextNode int int void)) +(defun (copy-slot! h src from dst to d) + (set-slot-end! dst to (+ (slot-end src from) d)) + (set-slot-lines! dst to (slot-lines src from)) + (set-slot-chars! dst to (slot-chars src from)) + (if (= h 1) + (set-chunk! (as-leaves dst) to (chunk-at (as-leaves src) from)) + (set-kid! (as-branch dst) to (kid (as-branch src) from)))) + +;; Copies `n` slots of `src` from `from` to `dst` from `to`, adding `d` to +;; their ends. The ranges may overlap in one node. +(: (-> int TextNode int TextNode int int int void)) +(defun (shift-slots! h src from dst to n d) + (if (<= to from) + (loop (:for i (range 0 n)) (:do (copy-slot! h src (+ from i) dst (+ to i) d))) + (loop (:for i (down-from (- n 1) #:to 0)) (:do (copy-slot! h src (+ from i) dst (+ to i) d))))) + +;; Copies `n` slots of `src` from `from` to `dst` from `to`. The ranges may +;; overlap in one node. +;; +;; - The moved slots keep their bytes, after the bytes before slot `to` of +;; `dst`. +;; - The ends of the slots of `dst` after the moved ones are stale, until they +;; are overwritten or past the count. +(: (-> int TextNode int TextNode int int void)) +(defun (move-slots! h src from dst to n) + (shift-slots! h src from dst to n (- (slot-base dst to) (slot-base src from)))) + +;; Empties slots `from` to `to`, so that they keep nothing alive. +(: (-> int TextNode int int void)) +(defun (clear-slots! h n from to) + (loop (:for i (range from to)) + (:do (if (= h 1) + (set-chunk! (as-leaves n) i "") + (set-kid! (as-branch n) i no-node))))) + +;; Opens slot `i` of the editable node `n`, which has room, moving the slots +;; after it one to the right. +(: (-> int TextNode int void)) +(defun (open-slot! h n i) + (def count (node-count n)) + (move-slots! h n i n (+ i 1) (- count i)) + (set-count! n (+ count 1))) + +;; Removes slot `i` of the editable node `n`. +(: (-> int TextNode int void)) +(defun (remove-slot! h n i) + (def count (node-count n)) + (move-slots! h n (+ i 1) n i (- count i 1)) + (clear-slots! h n (- count 1) count) + (set-count! n (- count 1))) + +;; A new node with slots `from` to `to` of `n`. +(: (-> int TextNode int int long TextNode)) +(defun (node-part h n from to o) + (def part (new-node h o)) + (move-slots! h n from part 0 (- to from)) + (set-count! part (- to from)) + part) + +;; Moves the second half of the editable node `n` into a new node, and returns +;; that. +(: (-> int TextNode long TextNode)) +(defun (split-half! h n o) + (def count (node-count n)) + (def keep (/ count 2)) + (def right (node-part h n keep count o)) + (clear-slots! h n keep count) + (set-count! n keep) + right) + +;; Moves every slot of `r` to the end of the editable node `l`, which has room. +(: (-> int TextNode TextNode void)) +(defun (append-slots! h l r) + (def cl (node-count l)) + (def cr (node-count r)) + (move-slots! h r 0 l cl cr) + (set-count! l (+ cl cr))) + +;; Moves slots between the editable neighbours `l` and `r` until each has +;; half of them. Moving slots to the front of `r` first shifts its slots right +;; by the bytes that come. +(: (-> int TextNode TextNode void)) +(defun (even-out! h l r) + (def cl (node-count l)) + (def cr (node-count r)) + (def target (/ (+ cl cr) 2)) + (cond + ((> cl target) + (def k (- cl target)) + (shift-slots! h r 0 r k cr (- (slot-end l (- cl 1)) (slot-base l target))) + (move-slots! h l target r 0 k) + (clear-slots! h l target cl) + (set-count! l target) + (set-count! r (+ cr k))) + ((< cl target) + (def k (- target cl)) + (move-slots! h r 0 l cl k) + (move-slots! h r k r 0 (- cr k)) + (clear-slots! h r (- cr k) cr) + (set-count! l target) + (set-count! r (- cr k))) + (else unit))) + +;; As `slot-starting`, by comparing the ends in turn. For a step to the next +;; or previous chunk, whose slot is predictable: the loop is small enough for +;; .NET to inline into a per-character loop without taking its registers. +(: (-> TextNode int (Tuple int int))) +(defun (slot-scanning n at) + (def last (- (node-count n) 1)) + (let go ((i 0) (base 0)) + (def e (slot-end n i)) + (if (and (< i last) (<= e at)) (go (+ i 1) e) (Tuple i (- at base))))) + +;; The slot of `n` whose bytes hold offset `at`, and the offset in it. An +;; offset between two slots is the end of the first. +;; +;; - One vectorized search of the ends, with no branch on the slot it finds: +;; a lookup at an arbitrary offset costs no misprediction at each level. +(: (-> TextNode int (Tuple int int))) +(defun (slot-ending n at) + (def last (- (node-count n) 1)) + (def i (if (<= at 0) 0 (span-first-outside (slot-ends n last) 0 (- at 1)))) + (def j (if (< i 0) last i)) + (Tuple j (- at (slot-base n j)))) + +;; As `slot-ending`, but an offset between two slots is the start of the +;; second. +(: (-> TextNode int (Tuple int int))) +(defun (slot-starting n at) + (def last (- (node-count n) 1)) + (def i (span-first-outside (slot-ends n last) 0 at)) + (def j (if (< i 0) last i)) + (Tuple j (- at (slot-base n j)))) + +;; --------------------------------------------------------------------------- +;; The text +;; --------------------------------------------------------------------------- + +;; `small` is the text when `height` is 0, and `root` is unused. Otherwise +;; `root` is a node of that height and `small` is "". +(type (: Text #:opaque + (Record (: root TextNode) + (: height int) + (: bytes int) + (: lines int) + (: chars int) + (: small string)))) + +(: (-> string Text)) +(defun (small-text s) + (Text (root no-node) (height 0) (bytes (string-length s)) (lines (newlines-in s)) (chars (chars-in s)) (small s))) + +(def empty-text (small-text "")) + +(: (-> Text)) +(defun (text-empty) empty-text) + +(: (-> Text int)) +(defun (text-length r) (.bytes r)) + +(: (-> Text bool)) +(defun (text-empty? r) (= (.bytes r) 0)) + +;; Lines are separated by "\n": one more than the newlines. +(: (-> Text int)) +(defun (text-line-count r) (+ (.lines r) 1)) + +(: (-> Text int)) +(defun (text-height r) (.height r)) + +;; --------------------------------------------------------------------------- +;; Chunks of a tree +;; --------------------------------------------------------------------------- + +;; Folds `f` over the chunks of `n`, of height `h`, in order. +(: (-> (-?-> string %a %a) %a TextNode int %a)) +(defun (fold-node-chunks f acc n h) + (def count (node-count n)) + (if (= h 1) + (let go ((i 0) (acc acc)) + (if (< i count) (go (+ i 1) (f (chunk-at (as-leaves n) i) acc)) acc)) + (let go ((i 0) (acc acc)) + (if (< i count) (go (+ i 1) (fold-node-chunks f acc (kid (as-branch n) i) (- h 1))) acc)))) + +(: (-> (-?-> string %a %a) %a Text %a)) +(defun (text-fold-chunks f init r) + (cond + ((> (.height r) 0) (fold-node-chunks f init (.root r) (.height r))) + ((= (.bytes r) 0) init) + (else (f (.small r) init)))) + +(: (-> (-?-> string void) TextNode int void)) +(defun (each-node-chunk f n h) + (def count (node-count n)) + (if (= h 1) + (loop (:for i (range 0 count)) (:do (f (chunk-at (as-leaves n) i)))) + (loop (:for i (range 0 count)) (:do (each-node-chunk f (kid (as-branch n) i) (- h 1)))))) + +(: (-> (-?-> string void) Text void)) +(defun (text-for-each-chunk f r) + (cond + ((> (.height r) 0) (each-node-chunk f (.root r) (.height r))) + ((> (.bytes r) 0) (f (.small r))) + (else unit))) + +(: (-> Text int)) +(defun (text-chunk-count r) (text-fold-chunks (fun (_ n) (+ n 1)) 0 r)) + +;; The number of chunks under `n`. +(: (-> TextNode int int)) +(defun (node-chunks n h) + (def count (node-count n)) + (if (= h 1) + count + (let go ((i 0) (acc 0)) + (if (< i count) (go (+ i 1) (+ acc (node-chunks (kid (as-branch n) i) (- h 1)))) acc)))) + +;; Puts the chunks under `n` in `out` from `at`, and returns the slot after. +(: (-> TextNode int (Array string) int int)) +(defun (gather-chunks! n h out at) + (def count (node-count n)) + (if (= h 1) + (begin + (loop (:for i (range 0 count)) (:do (array-set! out (+ at i) (chunk-at (as-leaves n) i)))) + (+ at count)) + (let go ((i 0) (at at)) + (if (< i count) (go (+ i 1) (gather-chunks! (kid (as-branch n) i) (- h 1) out at)) at)))) + +;; The text under `n`, in one allocation of its size. +(: (-> TextNode int int string)) +(defun (node->string n h bytes) + (def (: out (Array string)) (make-array (node-chunks n h) #:fill "")) + (ignore (gather-chunks! n h out 0)) + (concat-all (array->ro-span out))) + +(: (-> Text string)) +(defun (text->string r) + (if (= (.height r) 0) (.small r) (node->string (.root r) (.height r) (.bytes r)))) + +;; The text of a tree, which may be empty (height 0), have a root with one +;; child, or hold no more than one chunk's worth. +(: (-> TextNode int Text)) +(defun (tree->text n h) + (cond + ((or (= h 0) (= (node-count n) 0)) empty-text) + ((and (> h 1) (= (node-count n) 1)) (tree->text (kid (as-branch n) 0) (- h 1))) + (else + (def bytes (sum-bytes n)) + (if (<= bytes chunk-max) + (small-text (node->string n h bytes)) + (Text (root n) (height h) (bytes bytes) (lines (sum-lines n)) (chars (sum-chars n)) (small "")))))) + +;; The tree of a text: its root and height, and height 0 for an empty one. +(: (-> Text long (Tuple TextNode int))) +(defun (tree-of r o) + (cond + ((> (.height r) 0) (Tuple (.root r) (.height r))) + ((= (.bytes r) 0) (Tuple no-node 0)) + (else + (def n (new-node 1 o)) + (set-count! n 1) + (put-chunk! n 0 (.small r)) + (Tuple n 1)))) + +;; The chunk that holds byte `at`, and the offset it starts at. The end of the +;; text is in its last chunk. +(: (-> Text int (Tuple string int))) +(defun (chunk-of r at) + (if (= (.height r) 0) + (Tuple (.small r) 0) + (let go ((n (.root r)) (h (.height r)) (k at)) + (def (Tuple i k2) (slot-starting n k)) + (if (= h 1) + (Tuple (chunk-at (as-leaves n) i) (- at k2)) + (go (kid (as-branch n) i) (- h 1) k2))))) + +;; As `chunk-of`, with `slot-scanning`: for the step to a neighbouring chunk. +(: (-> Text int (Tuple string int))) +(defun (chunk-scanning r at) + (if (= (.height r) 0) + (Tuple (.small r) 0) + (let go ((n (.root r)) (h (.height r)) (k at)) + (def (Tuple i k2) (slot-scanning n k)) + (if (= h 1) + (Tuple (chunk-at (as-leaves n) i) (- at k2)) + (go (kid (as-branch n) i) (- h 1) k2))))) + +;; As `chunk-of`, with the chunk after it when the same leaves node holds +;; it, and "" otherwise. +(: (-> Text int (Tuple string int string))) +(defun (chunk-and-next r at) + (if (= (.height r) 0) + (Tuple (.small r) 0 "") + (let go ((n (.root r)) (h (.height r)) (k at)) + (def (Tuple i k2) (slot-starting n k)) + (if (= h 1) + (Tuple (chunk-at (as-leaves n) i) (- at k2) + (if (< (+ i 1) (node-count n)) (chunk-at (as-leaves n) (+ i 1)) "")) + (go (kid (as-branch n) i) (- h 1) k2))))) + +;; An iteration over the chunks of a text in order, either way, from the +;; chunk with a given byte. It keeps the path: `nodes` and `slots` hold the +;; node and slot at each level, the leaves node first. A step moves to the +;; next slot of the lowest node that has one, and down its first or last +;; children, so a step costs O(1) amortized and not a search from the root. +(type (: ChunkIter + (Record (: nodes (Array TextNode)) + (: slots (Array int)) + (: height int) + (: chunk string #:mutable) + (: start int #:mutable)))) + +;; The iteration on the chunk that holds byte `at`, below the length of `r`. +(: (-> Text int ChunkIter)) +(defun (chunk-iter r at) + (def h (.height r)) + (if (= h 0) + (ChunkIter (nodes (make-array 0 #:fill no-node)) (slots (make-array 0 #:fill 0)) (height 0) + (chunk (.small r)) (start 0)) + (begin + (def nodes (make-array h #:fill no-node)) + (def slots (make-array h #:fill 0)) + (let go ((n (.root r)) (lvl (- h 1)) (k at) (start 0)) + (def (Tuple i k2) (slot-starting n k)) + (array-set! nodes lvl n) + (array-set! slots lvl i) + (if (= lvl 0) + (ChunkIter (nodes nodes) (slots slots) (height h) + (chunk (chunk-at (as-leaves n) i)) (start (+ start (- k k2)))) + (go (kid (as-branch n) i) (- lvl 1) k2 (+ start (- k k2)))))))) + +;; Moves `it` to the next chunk, and returns #f when there is none. +(: (-> ChunkIter bool)) +(defun (chunk-iter-next! it) + (def nodes (.nodes it)) + (def slots (.slots it)) + (def h (.height it)) + (def next-start (+ (.start it) (string-length (.chunk it)))) + (let up ((lvl 0)) + (cond + ((>= lvl h) #f) + ((< (+ (array-ref slots lvl) 1) (node-count (array-ref nodes lvl))) + (array-set! slots lvl (+ (array-ref slots lvl) 1)) + (let down ((l lvl)) + (when (> l 0) + (array-set! nodes (- l 1) (kid (as-branch (array-ref nodes l)) (array-ref slots l))) + (array-set! slots (- l 1) 0) + (down (- l 1)))) + (record-set! it (chunk (chunk-at (as-leaves (array-ref nodes 0)) (array-ref slots 0))) (start next-start)) + #t) + (else (up (+ lvl 1)))))) + +;; Moves `it` to the chunk before, and returns #f when there is none. +(: (-> ChunkIter bool)) +(defun (chunk-iter-prev! it) + (def nodes (.nodes it)) + (def slots (.slots it)) + (def h (.height it)) + (let up ((lvl 0)) + (cond + ((>= lvl h) #f) + ((> (array-ref slots lvl) 0) + (array-set! slots lvl (- (array-ref slots lvl) 1)) + (let down ((l lvl)) + (when (> l 0) + (def k (kid (as-branch (array-ref nodes l)) (array-ref slots l))) + (array-set! nodes (- l 1) k) + (array-set! slots (- l 1) (- (node-count k) 1)) + (down (- l 1)))) + (def c (chunk-at (as-leaves (array-ref nodes 0)) (array-ref slots 0))) + (record-set! it (chunk c) (start (- (.start it) (string-length c)))) + #t) + (else (up (+ lvl 1)))))) + +;; --------------------------------------------------------------------------- +;; Building +;; --------------------------------------------------------------------------- + +;; Builds a text from strings added at its end. +;; +;; - `levels` holds the open node of each level, the leaves node at 0. A full +;; node is closed, into the level above, when the next item comes. +;; - Text shorter than a chunk waits in `pending`. A longer string is cut into +;; chunks that share its array. +;; - `textbuilder->text` closes the open nodes. The last node of a level, if +;; it has fewer than `min-kids` children, is merged with or evened out with +;; the one before it. +(type (: TextBuilder #:opaque + (Record (: levels (Array TextNode)) + (: top int #:mutable) + (: pending StringBuilder) + (: owner long #:mutable) + (: bytes int #:mutable)))) + +(: (-> TextBuilder)) +(defun (textbuilder-empty) + (TextBuilder (levels (make-array 24 #:fill no-node)) (top -1) (pending (Utf8Builder. chunk-max)) + (owner (owner-next)) (bytes 0))) + +(: (-> TextBuilder int)) +(defun (textbuilder-length b) (.bytes b)) + +;; The open node of level `lvl`, made if there is none. +(: (-> TextBuilder int TextNode)) +(defun (open-node b lvl) + (when (> lvl (.top b)) + (array-set! (.levels b) lvl (new-node (+ lvl 1) (.owner b))) + (record-set! b (top lvl))) + (array-ref (.levels b) lvl)) + +;; Closes the full open node of level `lvl` into the level above. +(: (-> TextBuilder int void)) +(defun (close-level! b lvl) + (def full (array-ref (.levels b) lvl)) + (array-set! (.levels b) lvl (new-node (+ lvl 1) (.owner b))) + (add-node! b (+ lvl 1) full)) + +;; Adds `k`, a node of height `lvl`, to level `lvl`. +(: (-> TextBuilder int TextNode void)) +(defun (add-node! b lvl k) + (when (= (node-count (open-node b lvl)) cap) (close-level! b lvl)) + (def n (array-ref (.levels b) lvl)) + (def i (node-count n)) + (set-count! n (+ i 1)) + (put-kid! n i k)) + +(: (-> TextBuilder string void)) +(defun (add-chunk! b c) + (when (= (node-count (open-node b 0)) cap) (close-level! b 0)) + (def n (array-ref (.levels b) 0)) + (def i (node-count n)) + (set-count! n (+ i 1)) + (put-chunk! n i c)) + +(: (-> TextBuilder void)) +(defun (flush-pending! b) + (def p (.pending b)) + (when (> (stringbuilder-length p) 0) + (def c (stringbuilder->string p)) + (stringbuilder-clear! p) + (add-chunk! b c))) + +(: (-> TextBuilder string void)) +(defun (textbuilder-add-string! b s) + (def len (string-length s)) + (def p (.pending b)) + (def pl (stringbuilder-length p)) + (record-set! b (bytes (+ (.bytes b) len))) + (cond + ((= len 0) unit) + ((and (= pl 0) (>= len chunk-min) (<= len chunk-max)) (add-chunk! b s)) + ((<= (+ pl len) chunk-max) + (stringbuilder-add-string! p s) + (when (= (+ pl len) chunk-max) (flush-pending! b))) + (else + ;; The pending chunk filled up from the front of `s`, then whole chunks + ;; that share `s`, then the rest pending. + (def start + (if (= pl 0) + 0 + (begin + (def cut (boundary-before s (- chunk-max pl))) + (stringbuilder-add-string! p (piece s 0 cut)) + (flush-pending! b) + cut))) + (let go ((at start)) + (if (> (- len at) chunk-max) + (begin + (def cut (boundary-before s (+ at chunk-max))) + (add-chunk! b (piece s at cut)) + (go cut)) + (when (< at len) + (if (>= (- len at) chunk-min) + (add-chunk! b (piece s at len)) + (stringbuilder-add-string! p (piece s at len))))))))) + +;; Adds `ch`, in the pending chunk, which goes to the tree first when `ch` +;; would take it past `chunk-max`. +(: (-> TextBuilder char void)) +(defun (textbuilder-add! b ch) + (def cp (char->int ch)) + (def w (cond ((< cp 128) 1) ((< cp 2048) 2) ((< cp 65536) 3) (else 4))) + (def p (.pending b)) + (when (> (+ (stringbuilder-length p) w) chunk-max) (flush-pending! b)) + (stringbuilder-add! p ch) + (record-set! b (bytes (+ (.bytes b) w)))) + +(: (-> TextBuilder Text void)) +(defun (textbuilder-add-text! b r) + (text-for-each-chunk (fun (c) (textbuilder-add-string! b c)) r)) + +;; The pending text as the last chunk. A short one is merged with the chunk +;; before it, or evened out with it. +(: (-> TextBuilder void)) +(defun (flush-last! b) + (def p (.pending b)) + (def pl (stringbuilder-length p)) + (when (> pl 0) + (def c (stringbuilder->string p)) + (stringbuilder-clear! p) + (def n (if (>= (.top b) 0) (array-ref (.levels b) 0) no-node)) + (def count (node-count n)) + (if (or (>= pl chunk-min) (= count 0)) + (add-chunk! b c) + (begin + (def last (chunk-at (as-leaves n) (- count 1))) + (def t (string-append last c)) + (if (<= (string-length t) chunk-max) + (put-chunk! n (- count 1) t) + (begin + (def cut (boundary-before t (/ (string-length t) 2))) + (put-chunk! n (- count 1) (piece t 0 cut)) + (add-chunk! b (piece t cut (string-length t))))))))) + +(: (-> TextBuilder Text)) +(defun (textbuilder->text b) + (flush-last! b) + (def levels (.levels b)) + ;; Each level's open node goes into the level above, from the bottom. The + ;; one before it is always there: a level above is only opened by closing + ;; a node of this one. + (let go ((lvl 0)) + (when (< lvl (.top b)) + (def x (array-ref levels lvl)) + (def cx (node-count x)) + (when (> cx 0) + (def parent (array-ref levels (+ lvl 1))) + (def cp (node-count parent)) + (if (and (< cx min-kids) (> cp 0)) + (begin + (def h (+ lvl 1)) + (def y (editable h (kid (as-branch parent) (- cp 1)) (.owner b))) + (if (<= (+ (node-count y) cx) cap) + (begin (append-slots! h y x) (put-kid! parent (- cp 1) y)) + (begin + (even-out! h y x) + (put-kid! parent (- cp 1) y) + (add-node! b (+ lvl 1) x)))) + (add-node! b (+ lvl 1) x))) + (go (+ lvl 1)))) + (def top (.top b)) + (def result (if (< top 0) empty-text (tree->text (array-ref levels top) (+ top 1)))) + ;; Start over, with an owner that does not own what was given away. + (loop (:for i (range 0 (+ top 1))) (:do (array-set! levels i no-node))) + (record-set! b (top -1) (owner (owner-next)) (bytes 0)) + result) + +;; A text of the strings, in order. +(: (-> #:rest string Text)) +(defun (text #:rest parts) + (def b (textbuilder-empty)) + (loop (:for s parts) (:do (textbuilder-add-string! b s))) + (textbuilder->text b)) + +(: (-> string Text)) +(defun (string->text s) + (if (<= (string-length s) chunk-max) + (small-text s) + (begin + (def b (textbuilder-empty)) + (textbuilder-add-string! b s) + (textbuilder->text b)))) + +;; --------------------------------------------------------------------------- +;; Joining +;; --------------------------------------------------------------------------- + +;; A new node of height `h` with the children `a` and `b`. +(: (-> int TextNode TextNode long TextNode)) +(defun (pair-node h a b o) + (def n (new-node h o)) + (set-count! n 2) + (put-kid! n 0 a) + (put-kid! n 1 b) + n) + +;; Merges the chunks at slots `i - 1` and `i` of the editable leaves node `n` +;; when one is shorter than `chunk-min` and the two fit in one. +(: (-> int TextNode int void)) +(defun (tidy-seam! h n i) + (when (and (= h 1) (> i 0) (< i (node-count n))) + (def a (slot-bytes n (- i 1))) + (def b (slot-bytes n i)) + (when (and (or (< a chunk-min) (< b chunk-min)) (<= (+ a b) chunk-max)) + (def l (as-leaves n)) + (put-chunk! n (- i 1) (string-append (chunk-at l (- i 1)) (chunk-at l i))) + (remove-slot! 1 n i)))) + +;; Puts `k` in slot `j` of the editable node `n`. A full `n` is split first; +;; the result is then the two halves in order and #t. +(: (-> TextNode int int TextNode long (Tuple TextNode TextNode bool))) +(defun (add-kid-at n h j k o) + (if (< (node-count n) cap) + (begin + (open-slot! h n j) + (put-kid! n j k) + (Tuple n no-node #f)) + (begin + (def right (split-half! h n o)) + (def keep (node-count n)) + (if (<= j keep) + (begin (open-slot! h n j) (put-kid! n j k)) + (begin (open-slot! h right (- j keep)) (put-kid! right (- j keep) k))) + (Tuple n right #t)))) + +;; Joins `b`, of height `hb`, to the right of `a`, of the greater height `ha`. +;; The result is a node of height `ha`, and a second one after it when the +;; first had no room. +(: (-> TextNode int TextNode int long (Tuple TextNode TextNode bool))) +(defun (join-right a ha b hb o) + (def n (editable ha a o)) + (def last (- (node-count n) 1)) + (def y (kid (as-branch n) last)) + (if (= (- ha 1) hb) + (begin + (def cy (node-count y)) + (def cb (node-count b)) + (if (<= (+ cy cb) cap) + (begin + (def y2 (editable hb y o)) + (append-slots! hb y2 b) + (tidy-seam! hb y2 cy) + (put-kid! n last y2) + (Tuple n no-node #f)) + (if (or (< cy min-kids) (< cb min-kids)) + (begin + (def y2 (editable hb y o)) + (def b2 (editable hb b o)) + (even-out! hb y2 b2) + (put-kid! n last y2) + (add-kid-at n ha (+ last 1) b2 o)) + (add-kid-at n ha (+ last 1) b o)))) + (begin + (def (Tuple c extra split?) (join-right y (- ha 1) b hb o)) + (put-kid! n last c) + (if split? (add-kid-at n ha (+ last 1) extra o) (Tuple n no-node #f))))) + +;; Joins `a`, of height `ha`, to the left of `b`, of the greater height `hb`, +;; as `join-right` does. +(: (-> TextNode int TextNode int long (Tuple TextNode TextNode bool))) +(defun (join-left a ha b hb o) + (def n (editable hb b o)) + (def y (kid (as-branch n) 0)) + (if (= (- hb 1) ha) + (begin + (def cy (node-count y)) + (def ca (node-count a)) + (if (<= (+ cy ca) cap) + (begin + (def a2 (editable ha a o)) + (append-slots! ha a2 y) + (tidy-seam! ha a2 ca) + (put-kid! n 0 a2) + (Tuple n no-node #f)) + (if (or (< cy min-kids) (< ca min-kids)) + (begin + (def y2 (editable ha y o)) + (def a2 (editable ha a o)) + (even-out! ha a2 y2) + (put-kid! n 0 y2) + (add-kid-at n hb 0 a2 o)) + (add-kid-at n hb 0 a o)))) + (begin + (def (Tuple first second split?) (join-left a ha y (- hb 1) o)) + (put-kid! n 0 first) + (if split? (add-kid-at n hb 1 second o) (Tuple n no-node #f))))) + +;; The tree of `a` followed by `b`. Either may be empty (height 0) or have a +;; root with fewer than `min-kids` children; every other node is full enough. +(: (-> TextNode int TextNode int long (Tuple TextNode int))) +(defun (join a ha b hb o) + (cond + ((= ha 0) (Tuple b hb)) + ((= hb 0) (Tuple a ha)) + ((= ha hb) + (def ca (node-count a)) + (def cb (node-count b)) + (if (<= (+ ca cb) cap) + (begin + (def m (editable ha a o)) + (append-slots! ha m b) + (tidy-seam! ha m ca) + (Tuple m ha)) + (begin + (def a2 (editable ha a o)) + (def b2 (editable ha b o)) + (when (or (< ca min-kids) (< cb min-kids)) (even-out! ha a2 b2)) + (Tuple (pair-node (+ ha 1) a2 b2 o) (+ ha 1))))) + ((> ha hb) + (def (Tuple n extra split?) (join-right a ha b hb o)) + (if split? (Tuple (pair-node (+ ha 1) n extra o) (+ ha 1)) (Tuple n ha))) + (else + (def (Tuple first second split?) (join-left a ha b hb o)) + (if split? (Tuple (pair-node (+ hb 1) first second o) (+ hb 1)) (Tuple first hb))))) + +(: (-> Text Text long Text)) +(defun (concat-with a b o) + (cond + ((= (.bytes a) 0) b) + ((= (.bytes b) 0) a) + ((and (= (.height b) 0) (<= (.bytes b) small-insert)) (insert-with a (.bytes a) (.small b) o)) + ((and (= (.height a) 0) (<= (.bytes a) small-insert)) (insert-with b 0 (.small a) o)) + (else + (def (Tuple na ha) (tree-of a o)) + (def (Tuple nb hb) (tree-of b o)) + (def (Tuple n h) (join na ha nb hb o)) + (tree->text n h)))) + +(: (-> Text Text Text)) +(defun (text-append a b) (concat-with a b (owner-next))) + +;; --------------------------------------------------------------------------- +;; Splitting +;; --------------------------------------------------------------------------- + +(: (-> string int string void)) +(defun (check-boundary s k who) + (unless (boundary? s k) (inside-character who))) + +;; Raises that an offset is inside a character. Not inlined, as +;; `cursor-at-edge`. +(: (-> string void)) +(defun (inside-character who) #:no-inline + (bad-argument (str who ": the offset is in the middle of a character."))) + +;; Merges the chunk at slot `i` of the editable leaves node `n` with a +;; neighbour when it is shorter than `chunk-min`: into one chunk if they fit, +;; and otherwise into two of about the same length. +(: (-> TextNode int void)) +(defun (merge-small-chunk! n i) + (def count (node-count n)) + (when (and (> count 1) (< (slot-bytes n i) chunk-min)) + (def a (if (< (+ i 1) count) i (- i 1))) + (def l (as-leaves n)) + (def t (string-append (chunk-at l a) (chunk-at l (+ a 1)))) + (def tl (string-length t)) + (if (<= tl chunk-max) + (begin (put-chunk! n a t) (remove-slot! 1 n (+ a 1))) + (begin + (def cut (boundary-before t (/ tl 2))) + (put-chunk! n a (piece t 0 cut)) + (put-chunk! n (+ a 1) (piece t cut tl)))))) + +;; The trees of the bytes of `n`, of height `h`, before and after `at`, where +;; 0 < `at` < the bytes of `n`. +(: (-> TextNode int int string long (Tuple TextNode int TextNode int))) +(defun (split-tree n h at who o) + (def (Tuple i k) (slot-ending n at)) + (def count (node-count n)) + (if (= h 1) + (begin + (def c (chunk-at (as-leaves n) i)) + (def cl (string-length c)) + (check-boundary c k who) + (def left (node-part 1 n 0 i o)) + (set-count! left (+ i 1)) + (put-chunk! left i (piece c 0 k)) + (merge-small-chunk! left i) + (def rest (- count i 1)) + (def right + (if (< k cl) + (begin + (def r (new-node 1 o)) + (set-count! r 1) + (put-chunk! r 0 (piece c k cl)) + (move-slots! 1 n (+ i 1) r 1 rest) + (set-count! r (+ rest 1)) + (merge-small-chunk! r 0) + r) + (node-part 1 n (+ i 1) count o))) + (Tuple left 1 right (if (= (node-count right) 0) 0 1))) + (if (= k (slot-bytes n i)) + (Tuple (node-part h n 0 (+ i 1) o) h + (node-part h n (+ i 1) count o) (if (= (+ i 1) count) 0 h)) + (begin + (def (Tuple ln lh rn rh) (split-tree (kid (as-branch n) i) (- h 1) k who o)) + (def (Tuple l2 lh2) (join (node-part h n 0 i o) (if (= i 0) 0 h) ln lh o)) + (def (Tuple r2 rh2) (join rn rh (node-part h n (+ i 1) count o) (if (= (+ i 1) count) 0 h) o)) + (Tuple l2 lh2 r2 rh2))))) + +(: (-> Text int string long (Tuple Text Text))) +(defun (split-with r at who o) + (check-offset r at who) + (cond + ((= at 0) (Tuple empty-text r)) + ((= at (.bytes r)) (Tuple r empty-text)) + ((= (.height r) 0) + (def s (.small r)) + (check-boundary s at who) + (Tuple (small-text (piece s 0 at)) (small-text (piece s at (.bytes r))))) + (else + (def (Tuple ln lh rn rh) (split-tree (.root r) (.height r) at who o)) + (Tuple (tree->text ln lh) (tree->text rn rh))))) + +(: (-> Text int string void)) +(defun (check-offset r at who) + (when (or (< at 0) (> at (.bytes r))) (offset-outside who at (.bytes r)))) + +;; Raises that `at` is outside a text of `bytes` bytes. Not inlined, as +;; `cursor-at-edge`. +(: (-> string int int void)) +(defun (offset-outside who at bytes) #:no-inline + (out-of-range #"${who}: the offset ${at} is outside the text, of ${bytes} bytes.")) + +(: (-> Text TextCursor (Tuple Text Text))) +(defun (text-split-at r c) (split-with r (.at c) "text-split-at" (owner-next))) + +;; Bytes `from` to `to` of `r` as a text that shares its chunks. +(: (-> Text int int Text)) +(defun (slice r from to) + (check-range r from to "text-substring") + (def o (owner-next)) + (def (Tuple _ tail) (split-with r from "text-substring" o)) + (def (Tuple mid _) (split-with tail (- to from) "text-substring" o)) + mid) + +;; Bytes `from` to `to` of `r` as a text: one string when they fit in a chunk, +;; one leaves node of the pieces of the chunks when they are in at most `cap` +;; chunks, and a slice that shares the chunks otherwise. +(: (-> Text int int Text)) +(defun (range->text r from to) + (cond + ((<= (- to from) chunk-max) (small-text (substring-string r from to))) + ((or (> (- to from) (* cap chunk-min)) (= (.height r) 0)) (slice r from to)) + (else + (check-range r from to "text-substring") + (def it (chunk-iter r from)) + (def n (new-node 1 0L)) + (let go ((i 0)) + (def c (.chunk it)) + (def start (.start it)) + (def e (min to (+ start (string-length c)))) + (set-count! n (+ i 1)) + (put-chunk! n i (piece c (- (max from start) start) (- e start))) + (cond + ((>= e to) (tree->text n 1)) + ((or (= (+ i 1) cap) (not (chunk-iter-next! it))) (slice r from to)) + (else (go (+ i 1)))))))) + +(: (-> Text int int string void)) +(defun (check-range r from to who) + (check-offset r from who) + (check-offset r to who) + (when (> from to) (backward-range who))) + +;; Raises that a range ends before it starts. Not inlined, as +;; `cursor-at-edge`. +(: (-> string void)) +(defun (backward-range who) #:no-inline + (out-of-range (str who ": the end is before the start."))) + +;; The characters of `r` from cursor `a` to cursor `b`, as a text. +(: (-> Text TextCursor TextCursor Text)) +(defun (text-substring r a b) + (when (> (.at a) (.at b)) (backward-range "text-substring")) + (range->text r (.at a) (.at b))) + +;; --------------------------------------------------------------------------- +;; Inserting +;; --------------------------------------------------------------------------- + +;; Puts `s`, of `sl` newlines, `sc` characters and at most `small-insert` +;; bytes, at byte `at` of the editable node `n`, which has room for one more +;; slot. +(: (-> TextNode int int string int int long void)) +(defun (insert-into! n h at s sl sc o) + (def slen (string-length s)) + (def (Tuple i k) (slot-ending n at)) + (if (= h 1) + (begin + (def l (as-leaves n)) + (def c (chunk-at l i)) + (check-boundary c k "text-insert") + (def cl (string-length c)) + (def t (concat3 (piece c 0 k) s (piece c k cl))) + (if (<= (+ cl slen) chunk-max) + (begin + (set-chunk! l i t) + (set-slot-bytes! n i (+ cl slen)) + (set-slot-lines! n i (+ (slot-lines n i) sl)) + (set-slot-chars! n i (+ (slot-chars n i) sc))) + (begin + (def tl (string-length t)) + (def cut (boundary-before t (/ tl 2))) + (open-slot! 1 n (+ i 1)) + (put-chunk! n i (piece t 0 cut)) + (put-chunk! n (+ i 1) (piece t cut tl))))) + (begin + (def b (as-branch n)) + (def child (editable (- h 1) (kid b i) o)) + (set-kid! b i child) + (if (= (node-count child) cap) + (begin + (def right (split-half! (- h 1) child o)) + (open-slot! h n (+ i 1)) + (put-kid! n i child) + (put-kid! n (+ i 1) right) + (def lb (slot-bytes n i)) + (if (<= k lb) + (into-kid! n h i child k s sl sc o) + (into-kid! n h (+ i 1) right (- k lb) s sl sc o))) + (into-kid! n h i child k s sl sc o))))) + +(: (-> TextNode int int TextNode int string int int long void)) +(defun (into-kid! n h i child k s sl sc o) + (insert-into! child (- h 1) k s sl sc o) + (add-slot-bytes! n i (string-length s)) + (set-slot-lines! n i (+ (slot-lines n i) sl)) + (set-slot-chars! n i (+ (slot-chars n i) sc))) + +(: (-> Text int string long Text)) +(defun (insert-with r at s o) + (check-offset r at "text-insert") + (def slen (string-length s)) + (cond + ((= slen 0) r) + ((= (.height r) 0) + (def s0 (.small r)) + (def len0 (.bytes r)) + (check-boundary s0 at "text-insert") + (if (<= (+ len0 slen) chunk-max) + (small-text (concat3 (piece s0 0 at) s (piece s0 at len0))) + (text (piece s0 0 at) s (piece s0 at len0)))) + ((<= slen small-insert) + (def h (.height r)) + (def n (editable h (.root r) o)) + (def sl (newlines-in s)) + (def sc (chars-in s)) + (def (Tuple root height) + (if (= (node-count n) cap) + (begin + (def right (split-half! h n o)) + (Tuple (pair-node (+ h 1) n right o) (+ h 1))) + (Tuple n h))) + (insert-into! root height at s sl sc o) + (Text (root root) (height height) (bytes (+ (.bytes r) slen)) (lines (+ (.lines r) sl)) (chars (+ (.chars r) sc)) + (small ""))) + (else + (def (Tuple l rr) (split-with r at "text-insert" o)) + (concat-with (concat-with l (string->text s) o) rr o)))) + +(: (-> Text TextCursor string Text)) +(defun (text-insert r c s) (insert-with r (.at c) s (owner-next))) + +(: (-> Text TextCursor Text Text)) +(defun (text-insert-text r c x) + (def o (owner-next)) + (def (Tuple l rr) (split-with r (.at c) "text-insert-text" o)) + (concat-with (concat-with l x o) rr o)) + +;; --------------------------------------------------------------------------- +;; Deleting +;; --------------------------------------------------------------------------- + +;; Merges child `i` of the editable branch `n`, of height `h`, which has fewer +;; than `min-kids` children, with a neighbour, or evens the two out. +(: (-> TextNode int int long void)) +(defun (fix-underflow! n h i o) + (def count (node-count n)) + (when (> count 1) + (def a (if (< (+ i 1) count) i (- i 1))) + (def b (as-branch n)) + (def x (editable (- h 1) (kid b a) o)) + (def y (editable (- h 1) (kid b (+ a 1)) o)) + (def cx (node-count x)) + (if (<= (+ cx (node-count y)) cap) + (begin + (append-slots! (- h 1) x y) + (tidy-seam! (- h 1) x cx) + (put-kid! n a x) + (remove-slot! h n (+ a 1))) + (begin + (even-out! (- h 1) x y) + (put-kid! n a x) + (put-kid! n (+ a 1) y))))) + +;; Deletes bytes `at` to `at + len` of the editable node `n`, all in one +;; chunk, which hold `dl` newlines and `dc` characters. +(: (-> TextNode int int int int int long void)) +(defun (delete-into! n h at len dl dc o) + (def (Tuple i k) (slot-starting n at)) + (if (= h 1) + (begin + (def l (as-leaves n)) + (def c (chunk-at l i)) + (def cl (string-length c)) + (check-boundary c k "text-delete") + (check-boundary c (+ k len) "text-delete") + (if (= len cl) + (remove-slot! 1 n i) + (begin + (set-chunk! l i (concat3 (piece c 0 k) "" (piece c (+ k len) cl))) + (set-slot-bytes! n i (- cl len)) + (set-slot-lines! n i (- (slot-lines n i) dl)) + (set-slot-chars! n i (- (slot-chars n i) dc)) + (merge-small-chunk! n i)))) + (begin + (def b (as-branch n)) + (def child (editable (- h 1) (kid b i) o)) + (set-kid! b i child) + (delete-into! child (- h 1) k len dl dc o) + (add-slot-bytes! n i (- len)) + (set-slot-lines! n i (- (slot-lines n i) dl)) + (set-slot-chars! n i (- (slot-chars n i) dc)) + (when (< (node-count child) min-kids) (fix-underflow! n h i o))))) + +(: (-> Text int int long Text)) +(defun (delete-with r from to o) + (check-range r from to "text-delete") + (def len (- to from)) + (cond + ((= len 0) r) + ((= (.height r) 0) + (def s (.small r)) + (check-boundary s from "text-delete") + (check-boundary s to "text-delete") + (small-text (concat3 (piece s 0 from) "" (piece s to (.bytes r))))) + (else + (def (Tuple c start) (chunk-of r from)) + (if (<= to (+ start (string-length c))) + (begin + (def h (.height r)) + (def n (editable h (.root r) o)) + (def k (- from start)) + (check-boundary c k "text-delete") + (check-boundary c (+ k len) "text-delete") + (def dl (newlines-between c k (+ k len))) + (def dc (chars-between c k (+ k len))) + (delete-into! n h from len dl dc o) + (tree->text-sized n h (- (.bytes r) len) (- (.lines r) dl) (- (.chars r) dc))) + (begin + (def (Tuple l tail) (split-with r from "text-delete" o)) + (def (Tuple _ rr) (split-with tail len "text-delete" o)) + (concat-with l rr o)))))) + +;; `tree->text` for a root whose bytes, newlines and characters are known. +(: (-> TextNode int int int int Text)) +(defun (tree->text-sized n h bytes lines chars) + (if (or (<= bytes chunk-max) (and (> h 1) (= (node-count n) 1))) + (tree->text n h) + (Text (root n) (height h) (bytes bytes) (lines lines) (chars chars) (small "")))) + +(: (-> Text TextCursor TextCursor Text)) +(defun (text-delete r a b) (delete-with r (.at a) (.at b) (owner-next))) + +(: (-> Text TextCursor TextCursor string Text)) +(defun (text-replace-range r a b s) + (def o (owner-next)) + (insert-with (delete-with r (.at a) (.at b) o) (.at a) s o)) + +;; --------------------------------------------------------------------------- +;; Characters and text +;; --------------------------------------------------------------------------- + +;; The number of bytes of the character whose first byte is `b`. +(: (-> int int)) +(defun (char-width b) + (cond ((< b 128) 1) ((< b 224) 2) ((< b 240) 3) (else 4))) + +(: (-> (RoSpan byte) int int)) +(defun (byte-at sp i) #:inline (cast int (ro-span-ref sp i))) + +;; The character that starts at byte `k` of `c`. +(: (-> string int char)) +(defun (decode-at c k) + (def sp (.AsSpan c)) + (def b0 (byte-at sp k)) + (int->char + (cond + ((< b0 128) b0) + ((< b0 224) (bitwise-ior (shift-left (bitwise-and b0 31) 6) (bitwise-and (byte-at sp (+ k 1)) 63))) + ((< b0 240) (bitwise-ior (shift-left (bitwise-and b0 15) 12) + (bitwise-ior (shift-left (bitwise-and (byte-at sp (+ k 1)) 63) 6) + (bitwise-and (byte-at sp (+ k 2)) 63)))) + (else (bitwise-ior (shift-left (bitwise-and b0 7) 18) + (bitwise-ior (shift-left (bitwise-and (byte-at sp (+ k 1)) 63) 12) + (bitwise-ior (shift-left (bitwise-and (byte-at sp (+ k 2)) 63) 6) + (bitwise-and (byte-at sp (+ k 3)) 63)))))))) + +;; Bytes `from` to `to` as a string of their own. +(: (-> Text int int string)) +(defun (substring-string r from to) + (check-range r from to "text-substring") + (cond + ((= from to) "") + ((= (.height r) 0) + (check-boundary (.small r) from "text-substring") + (check-boundary (.small r) to "text-substring") + (string-copy (piece (.small r) from to))) + (else + ;; The pieces of the chunks from `from` to `to`, joined in one + ;; allocation of their size. + (def (Tuple c0 s0 after) (chunk-and-next r from)) + (check-boundary c0 (- from s0) "text-substring") + (def e0 (+ s0 (string-length c0))) + (if (<= to e0) + (begin + (check-boundary c0 (- to s0) "text-substring") + (string-copy (piece c0 (- from s0) (- to s0)))) + (begin + (def first (piece c0 (- from s0) (string-length c0))) + (def c1 (if (= (string-length after) 0) + (begin (def (Tuple c _) (chunk-of r e0)) c) + after)) + (def s1 e0) + (if (<= to (+ s1 (string-length c1))) + (begin + (check-boundary c1 (- to s1) "text-substring") + (concat2 first (piece c1 0 (- to s1)))) + (begin + ;; The chunks the range touches, counted on a second iteration. + (def it (chunk-iter r e0)) + (def ci (chunk-iter r e0)) + (def count (let go ((n 2)) + (if (>= (+ (.start ci) (string-length (.chunk ci))) to) + n + (begin (ignore (chunk-iter-next! ci)) (go (+ n 1)))))) + (def (: parts (Array string)) (make-array count #:fill "")) + (array-set! parts 0 first) + (let fill ((k 1)) + (def c (.chunk it)) + (def st (.start it)) + (if (= k (- count 1)) + (begin + (check-boundary c (- to st) "text-substring") + (array-set! parts k (piece c 0 (- to st)))) + (begin + (array-set! parts k c) + (ignore (chunk-iter-next! it)) + (fill (+ k 1))))) + (concat-all (array->ro-span parts))))))))) + +(: (-> Text int)) +(defun (text-count r) (.chars r)) + +;; --------------------------------------------------------------------------- +;; Lines +;; --------------------------------------------------------------------------- + +;; The offset after the `line`-th newline of `c`, counting from 1. +(: (-> string int int)) +(defun (nth-newline-end c line) + (def sp (.AsSpan c)) + (let go ((from 0) (left line)) + (def j (+ from (span-index-of (ro-span-slice sp #:from from) newline))) + (if (= left 1) (+ j 1) (go (+ j 1) (- left 1))))) + +;; The offset after the `line`-th newline of `n`, counting from 1. +(: (-> TextNode int int int)) +(defun (line-start-in n h line) + (let go ((i 0) (line line)) + (def l (slot-lines n i)) + (cond + ((> line l) (go (+ i 1) (- line l))) + ((= h 1) (+ (slot-base n i) (nth-newline-end (chunk-at (as-leaves n) i) line))) + (else (+ (slot-base n i) (line-start-in (kid (as-branch n) i) (- h 1) line)))))) + +(: (-> Text int string void)) +(defun (check-line r n who) + (when (or (< n 0) (> n (.lines r))) + (out-of-range #"${who}: there is no line ${n} in a text of ${(+ (.lines r) 1)} lines."))) + +;; The offset of the first byte of line `n`, counting from 0. +(: (-> Text int int)) +(defun (line-start r n) + (check-line r n "text-line") + (cond + ((= n 0) 0) + ((= (.height r) 0) (nth-newline-end (.small r) n)) + (else (line-start-in (.root r) (.height r) n)))) + +;; The offset of the newline that ends line `n`, or the end of the text for +;; the last line. +(: (-> Text int int)) +(defun (line-end r n) + (check-line r n "text-line") + (if (= n (.lines r)) (.bytes r) (- (line-start r (+ n 1)) 1))) + +;; The line that byte `at` is on: the newlines before it. +(: (-> Text int int)) +(defun (line-of r at) + (if (= (.height r) 0) + (newlines-between (.small r) 0 at) + (let go ((n (.root r)) (h (.height r)) (at at) (acc 0)) + (def (Tuple i k) (slot-starting n at)) + (def before (let sum ((j 0) (acc acc)) (if (< j i) (sum (+ j 1) (+ acc (slot-lines n j))) acc))) + (if (= h 1) + (+ before (newlines-between (chunk-at (as-leaves n) i) 0 k)) + (go (kid (as-branch n) i) (- h 1) k before))))) + +;; Line `n`, without its newline. +(: (-> Text int Text)) +(defun (text-line r n) + (range->text r (line-start r n) (line-end r n))) + +;; --------------------------------------------------------------------------- +;; Searching +;; --------------------------------------------------------------------------- + +;; Whether the bytes of `r` from `at` are those of `s`. +(: (-> Text int string bool)) +(defun (matches-at? r at s) + (def m (string-length s)) + (def ss (.AsSpan s)) + (and (<= (+ at m) (.bytes r)) + (let go ((p at)) + (if (>= p (+ at m)) + #t + (begin + (def (Tuple c start) (chunk-of r p)) + (def e (min (+ start (string-length c)) (+ at m))) + (and (span-equal? (ro-span-slice (.AsSpan c) #:from (- p start) #:to (- e start)) + (ro-span-slice ss #:from (- p at) #:to (- e at))) + (go e))))))) + +;; The offset of the first `s` at or after `at`, or -1, searched from the +;; chunks of `it`, which is on the chunk that holds `at` or one before it. +;; Leaves `it` on the chunk where the search stopped, at or before the chunk +;; of the match's end, so the next search can continue from it. +(: (-> Text ChunkIter string int int)) +(defun (index-from! r it s at) + (def m (string-length s)) + (def bytes (.bytes r)) + (def ss (.AsSpan s)) + (def first (cast byte (ro-span-ref ss 0))) + (let skip () + (when (and (<= (+ (.start it) (string-length (.chunk it))) at) (chunk-iter-next! it)) (skip))) + (let go ((at at)) + (if (> (+ at m) bytes) + -1 + (begin + (def c (.chunk it)) + (def start (.start it)) + (def cl (string-length c)) + (def sp (.AsSpan c)) + (def k (- at start)) + (def j (span-find (ro-span-slice sp #:from k) ss)) + (cond + ((>= j 0) (+ at j)) + ((not (chunk-iter-next! it)) -1) + (else + ;; A match that starts in this chunk and ends in the next one, + ;; or past it. + (def nc (.chunk it)) + (let straddle ((p (max k (- cl (- m 1))))) + (def q (if (< p cl) (span-index-of (ro-span-slice sp #:from p) first) -1)) + (if (< q 0) + (go (+ start cl)) + (begin + (def pp (+ p q)) + (def head (- cl pp)) + (if (and (span-equal? (ro-span-slice sp #:from pp) (ro-span-slice ss #:from 0 #:to head)) + (if (<= (- m head) (string-length nc)) + (span-equal? (ro-span-slice (.AsSpan nc) #:from 0 #:to (- m head)) + (ro-span-slice ss #:from head)) + (matches-at? r (+ start pp) s))) + (+ start pp) + (straddle (+ pp 1)))))))))))) + +;; The offset of the first `s` at or after `#:from`. +(: (-> Text string (#:from int) (Option int))) +(defun (index-of r s #:from 0) + (def m (string-length s)) + (cond + ((= m 0) (Some from)) + ((= (.height r) 0) + (def j (span-find (ro-span-slice (.AsSpan (.small r)) #:from from) (.AsSpan s))) + (if (>= j 0) (Some (+ from j)) None)) + ((> (+ from m) (.bytes r)) None) + (else + (def k (index-from! r (chunk-iter r from) s from)) + (if (>= k 0) (Some k) None)))) + +(: (-> Text string bool)) +(defun (text-contains? r s) (some? (index-of r s))) + +(: (-> Text string bool)) +(defun (text-starts-with? r s) (matches-at? r 0 s)) + +(: (-> Text string bool)) +(defun (text-ends-with? r s) + (def at (- (.bytes r) (string-length s))) + (and (>= at 0) (matches-at? r at s))) + +;; The offset of the last `s` that ends at or before `#:to`. +;; +;; - Each chunk, last first, is searched for the matches that start in it and +;; end in a later one, and then for those inside it. +(: (-> Text string (#:to int) (Option int))) +(defun (last-index-of r s #:to (text-length r)) + (def m (string-length s)) + (cond + ((= m 0) (Some to)) + ((> m to) None) + ((= (.height r) 0) + (def j (span-find-last (ro-span-slice (.AsSpan (.small r)) #:from 0 #:to to) (.AsSpan s))) + (if (>= j 0) (Some j) None)) + (else + (def first (cast byte (ro-span-ref (.AsSpan s) 0))) + (def it (chunk-iter r (- to 1))) + (def ss (.AsSpan s)) + ;; `after` is the chunk after this one, which the last step left. + (let go ((after "")) + (def c (.chunk it)) + (def start (.start it)) + (def cl (string-length c)) + (def sp (.AsSpan c)) + ;; A match from byte p of the chunk crosses its end when p > cl - m, + ;; and ends in time when start + p + m <= to. + (def lo (max 0 (+ (- cl m) 1))) + (def straddler + (let back ((hi (min (- cl 1) (- (- to start) m)))) + (if (< hi lo) + None + (begin + (def q (span-last-index-of (ro-span-slice sp #:from lo #:to (+ hi 1)) first)) + (def pp (+ lo q)) + (def head (- cl pp)) + (cond + ((< q 0) None) + ((and (span-equal? (ro-span-slice sp #:from pp) (ro-span-slice ss #:from 0 #:to head)) + (if (<= (- m head) (string-length after)) + (span-equal? (ro-span-slice (.AsSpan after) #:from 0 #:to (- m head)) + (ro-span-slice ss #:from head)) + (matches-at? r (+ start pp) s))) + (Some (+ start pp))) + (else (back (- pp 1)))))))) + (match straddler + ((Some k) (Some k)) + (None + (def e (min cl (- to start))) + (def j (if (>= e m) (span-find-last (ro-span-slice sp #:from 0 #:to e) (.AsSpan s)) -1)) + (cond + ((>= j 0) (Some (+ start j))) + ((= start 0) None) + (else (ignore (chunk-iter-prev! it)) (go c))))))))) + +;; --------------------------------------------------------------------------- +;; Equality, order and hashing +;; --------------------------------------------------------------------------- + +(: (-> (RoSpan byte) (RoSpan byte) bool int)) +(defun (compare-spans x y order?) #:inline + (cond (order? (span-compare x y)) ((span-equal? x y) 0) (else 1))) + +;; Compares the bytes of `a` and `b` from 0 to `len`, which both have, part by +;; part, where a part lies in one chunk of each. Returns 0 if they are all +;; equal. Otherwise the order of the first part that differs when `order?`, +;; and 1 when not. +(: (-> Text Text int bool int)) +(defun (compare-parts a b len order?) + (if (= len 0) + 0 + (begin + (def ia (chunk-iter a 0)) + (def ib (chunk-iter b 0)) + (let go ((p 0)) + (if (>= p len) + 0 + (begin + (def ca (.chunk ia)) + (def cb (.chunk ib)) + (def sa (.start ia)) + (def sb (.start ib)) + (def ea (+ sa (string-length ca))) + (def eb (+ sb (string-length cb))) + (def e (min len (min ea eb))) + (def d (compare-spans (ro-span-slice (.AsSpan ca) #:from (- p sa) #:to (- e sa)) + (ro-span-slice (.AsSpan cb) #:from (- p sb) #:to (- e sb)) + order?)) + (cond + ((not (= d 0)) d) + ((>= e len) 0) + (else + (when (= e ea) (ignore (chunk-iter-next! ia))) + (when (= e eb) (ignore (chunk-iter-next! ib))) + (go e))))))))) + +(: (-> Text Text bool)) +(defun (text=? a b) + (and (= (.bytes a) (.bytes b)) + (= (.lines a) (.lines b)) + (if (and (= (.height a) 0) (= (.height b) 0)) + (= (.small a) (.small b)) + (= (compare-parts a b (.bytes a) #f) 0)))) + +(: (-> Text Text int)) +(defun (text-compare a b) + (def d (compare-parts a b (min (.bytes a) (.bytes b)) #t)) + (cond + ((< d 0) -1) + ((> d 0) 1) + (else (compare (.bytes a) (.bytes b))))) + +;; A hash of the bytes in order, whatever chunks hold them: the folded +;; multiply of wyhash, as the string hash does. +;; +;; - The bytes are read as 16-byte blocks of two words. Block k of the text +;; goes to lane k mod 4, and the four lanes do not wait for each other. +;; - A block that spans two chunks is carried over in `w0` and `w1`, with `n` +;; of its bytes, and `lane` is the lane of the next block. +(type (: HashState (Struct (: s0 ulong) (: s1 ulong) (: s2 ulong) (: s3 ulong) + (: w0 ulong) (: w1 ulong) (: n int) (: lane int)))) + +(def hash-p0 (cast ulong 11562461410679940143UL)) +(def hash-p1 (cast ulong 16646288086500911323UL)) +(def hash-p2 (cast ulong 10285213230658275043UL)) +(def hash-p3 (cast ulong 6384245875588680899UL)) + +;; The high and low words of a * b, xored. +(: (-> ulong ulong ulong)) +(defun (fold-mul a b) #:inline + (def p (big-mul a b)) + (bitwise-xor (cast ulong (shift-right p 64)) (cast ulong p))) + +(: (-> HashState ulong ulong HashState)) +(defun (hash-feed st a b) + (def l (.lane st)) + (def st2 + (cond + ((= l 0) (record-set st (s0 (fold-mul (bitwise-xor a hash-p0) (bitwise-xor b (.s0 st)))))) + ((= l 1) (record-set st (s1 (fold-mul (bitwise-xor a hash-p1) (bitwise-xor b (.s1 st)))))) + ((= l 2) (record-set st (s2 (fold-mul (bitwise-xor a hash-p2) (bitwise-xor b (.s2 st)))))) + (else (record-set st (s3 (fold-mul (bitwise-xor a hash-p3) (bitwise-xor b (.s3 st)))))))) + (record-set st2 (lane (bitwise-and (+ l 1) 3)))) + +;; Bytes `from` to `to` of `sp`, at most 8, as the low bytes of a word, the +;; first lowest. +(: (-> (RoSpan byte) int int ulong)) +(defun (word-of sp from to) + (let go ((k (- to 1)) (w (cast ulong 0))) + (if (< k from) w (go (- k 1) (bitwise-ior (shift-left w 8) (cast ulong (ro-span-ref sp k))))))) + +;; Bytes `from` to `to` of `sp`, which fit in the carried block, put in it. +(: (-> HashState (RoSpan byte) int int HashState)) +(defun (carry-bytes st sp from to) + (def n (.n st)) + (def k (- to from)) + ;; The bytes that go to w0, from its byte n, and those after, to w1. + (def into0 (max 0 (min k (- 8 n)))) + (def w0 (bitwise-ior (.w0 st) (shift-left (word-of sp from (+ from into0)) (* 8 (min n 8))))) + (def w1 (if (= into0 k) + (.w1 st) + (bitwise-ior (.w1 st) (shift-left (word-of sp (+ from into0) to) (* 8 (max 0 (- n 8))))))) + (record-set st (w0 w0) (w1 w1) (n (+ n k)))) + +(: (-> string HashState HashState)) +(defun (hash-chunk c st0) + (def sp (.AsSpan c)) + (def len (string-length c)) + (def/mutable st st0) + (def/mutable i 0) + ;; The carried block, made whole and fed. + (when (> (.n st) 0) + (def take (min len (- 16 (.n st)))) + (set! st (carry-bytes st sp 0 take)) + (set! i take) + (when (= (.n st) 16) + (set! st (hash-feed (record-set st (w0 (cast ulong 0)) (w1 (cast ulong 0)) (n 0)) (.w0 st) (.w1 st))))) + (def blocks (/ (- len i) 16)) + (when (> blocks 0) + (def (: ws (RoSpan ulong)) (ro-span-> (ro-span-slice sp #:from i #:to (+ i (* 16 blocks))))) + (def/mutable k 0) + ;; Single blocks up to lane 0, then four at a time, then the rest. + (loop (:while (and (> (.lane st) 0) (< k blocks))) + (:do (set! st (hash-feed st (ro-span-ref ws (* 2 k)) (ro-span-ref ws (+ (* 2 k) 1)))) + (set! k (+ k 1)))) + (def/mutable s0 (.s0 st)) + (def/mutable s1 (.s1 st)) + (def/mutable s2 (.s2 st)) + (def/mutable s3 (.s3 st)) + (loop (:while (<= (+ k 4) blocks)) + (:do (def b (ro-span-slice ws #:from (* 2 k) #:to (+ (* 2 k) 8))) + (set! s0 (fold-mul (bitwise-xor (ro-span-ref b 0) hash-p0) (bitwise-xor (ro-span-ref b 1) s0))) + (set! s1 (fold-mul (bitwise-xor (ro-span-ref b 2) hash-p1) (bitwise-xor (ro-span-ref b 3) s1))) + (set! s2 (fold-mul (bitwise-xor (ro-span-ref b 4) hash-p2) (bitwise-xor (ro-span-ref b 5) s2))) + (set! s3 (fold-mul (bitwise-xor (ro-span-ref b 6) hash-p3) (bitwise-xor (ro-span-ref b 7) s3))) + (set! k (+ k 4)))) + (set! st (record-set st (s0 s0) (s1 s1) (s2 s2) (s3 s3))) + (loop (:while (< k blocks)) + (:do (set! st (hash-feed st (ro-span-ref ws (* 2 k)) (ro-span-ref ws (+ (* 2 k) 1)))) + (set! k (+ k 1)))) + (set! i (+ i (* 16 blocks)))) + (if (< i len) (carry-bytes st sp i len) st)) + +(: (-> Text int)) +(defun (text-hash r) + (def zero (cast ulong 0)) + (def seed (cast ulong 0x9E3779B97F4A7C15UL)) + (def st (text-fold-chunks hash-chunk + (HashState (s0 seed) (s1 (bitwise-xor seed hash-p1)) (s2 (bitwise-xor seed hash-p2)) + (s3 (bitwise-xor seed hash-p3)) (w0 zero) (w1 zero) (n 0) (lane 0)) + r)) + (def lanes (bitwise-xor (bitwise-xor (.s0 st) (.s1 st)) (bitwise-xor (.s2 st) (.s3 st)))) + (def h (fold-mul (bitwise-xor (.w0 st) hash-p0) + (bitwise-xor (.w1 st) (bitwise-xor lanes (cast ulong (+ (* 16 (.n st)) (.bytes r))))))) + (def fin (fold-mul (bitwise-xor h hash-p1) (bitwise-xor (cast ulong (.lane st)) hash-p2))) + (cast int (bitwise-xor fin (shift-right-logical fin 32)))) + +(impl (Eq Text) + (defun (= a b) (text=? a b)) + (defun (eq-hash r) (text-hash r))) + +(impl (Ord Text) + (defun (compare a b) (text-compare a b))) + +(impl (Written Text) + (defun (write-into r b) (write-into (text->string r) b))) + +(impl (Displayed Text) + (defun (display-into r b) (text-for-each-chunk (fun (c) (stringbuilder-add-string! b c)) r)) + (defun (displayed r) (text->string r))) + +;; --------------------------------------------------------------------------- +;; Iteration +;; --------------------------------------------------------------------------- + +;; A walk over the characters of a text, a struct: a walk over one chunk, and +;; the offset of the next. A step off the end of a chunk moves on to the +;; next one, so that only the last chunk's walk is ever done. +(type (: TextWalk #:opaque + (Struct (: text Text) + (: walk StringWalk) + (: next int)))) + +(: (-> Text TextWalk)) +(defun (text-walk r) + (if (= (.height r) 0) + (TextWalk (text r) (walk (string-walk (.small r))) (next (.bytes r))) + (text-walk-at r 0))) + +;; The walk over the chunk that starts at `at`. +(: (-> Text int TextWalk)) +(defun (text-walk-at r at) + (def (Tuple c _) (chunk-scanning r at)) + (TextWalk (text r) (walk (string-walk c)) (next (+ at (string-length c))))) + +(: (-> TextWalk TextWalk)) +(defun (text-walk-next w) + (def s (string-walk-next (.walk w))) + (if (and (string-walk-done? s) (< (.next w) (.bytes (.text w)))) + (text-walk-at (.text w) (.next w)) + (record-set w (walk s)))) + +(impl (Iterable Text) + (type %elem char) + (type %cursor TextWalk) + (defun (iterable-start r) (text-walk r)) + (defun (iterable-done? r w) (string-walk-done? (.walk w))) + (defun (iterable-current r w) (string-walk-current (.walk w))) + (defun (iterable-next r w) (text-walk-next w)) + (defun (count r) (.chars r))) + +(impl (Collection Text) + (type %content char) + (defun (length r) (.chars r)) + (defun (empty? r) (text-empty? r))) + +(impl (Refable Text) + (type %index TextCursor) + (type %out char) + (defun (ref r c) (text-cursor-ref r c)) + (defun (try-ref r c) (if (at-end? c) None (Some (char-at c))))) + +(impl (Cursor Text) + (type %elem char) + (type %cursor TextCursor) + (defun (cursor-start r) (text-cursor-start r)) + (defun (cursor-end r) (text-cursor-end r)) + (defun (cursor-end? r c) (at-end? c)) + (defun (cursor-ref r c) (text-cursor-ref r c)) + (defun (cursor-next r c) (text-cursor-next r c)) + (defun (cursor-prev r c) (text-cursor-prev r c))) + +;; --------------------------------------------------------------------------- +;; As a string has them +;; --------------------------------------------------------------------------- + +(: (-> string int void)) +(defun (check-count who n) + (when (< n 0) (bad-argument #"${who}: the count ${n} is negative."))) + + +;; The bytes of the first `n` characters of `c`, or all of it when it has +;; fewer. +;; +;; - Counts the characters of 32 bytes at a time, as `chars-between` does, +;; and scans the bytes of the block that holds the start of character `n`. +;; - Allocates nothing. +(: (-> string int int)) +(defun (bytes-of-first c n) + (def sp (.AsSpan c)) + (def blocks (* 32 (/ (string-length c) 32))) + (def top (bytes32-splat (cast byte 192))) + (def cont (bytes32-splat (cast byte 128))) + (let go ((j 0) (n n)) + (if (>= j blocks) + (nth-start c j n) + (begin + (def v (bytes32-of (ro-span-slice sp #:from j #:to (+ j 32)))) + (def k (- 32 (pop-count32 (bytes32-mask (bytes32-eq (bytes32-and v top) cont))))) + (if (< n k) (nth-start c j n) (go (+ j 32) (- n k))))))) + +;; The byte from `k` on where the `n`-th character after `k` that starts there +;; or later starts, counting from 0, or the length of `c` when there are fewer. +(: (-> string int int int)) +(defun (nth-start c k n) + (def sp (.AsSpan c)) + (def len (string-length c)) + (let step ((k k) (n n)) + (cond + ((>= k len) len) + ((= (bitwise-and (byte-at sp k) 192) 128) (step (+ k 1) n)) + ((= n 0) k) + (else (step (+ k 1) (- n 1)))))) + +;; The characters in the first `k` bytes of `c`, which has `cc` characters. +;; Counts from the nearer end: a chunk with as many characters as bytes is +;; ASCII, and needs no count. +(: (-> string int int int)) +(defun (chars-of-first c cc k) + (def cl (string-length c)) + (cond + ((= cc cl) k) + ((> (* 2 k) cl) (- cc (chars-between c k cl))) + (else (chars-between c 0 k)))) + +;; The offset after the first `n` characters of `r`, or its end when it has +;; fewer. +(: (-> Text int int)) +(defun (offset-after-chars r n) (byte-of-char r (min n (.chars r)))) + +;; The offset before the last `n` characters of `r`, or 0 when it has fewer. +(: (-> Text int int)) +(defun (offset-before-last-chars r n) (byte-of-char r (max 0 (- (.chars r) n)))) + +;; Adds bytes `from` to `to` of `r` to `b`, from the chunks of `it`, which is on +;; the chunk that holds `from` or one before it. Leaves `it` on the chunk that +;; holds `to`, or the last. +(: (-> TextBuilder Text ChunkIter int int void)) +(defun (add-range! b r it from to) + (let skip () + (when (and (<= (+ (.start it) (string-length (.chunk it))) from) (chunk-iter-next! it)) (skip))) + (let go () + (def c (.chunk it)) + (def start (.start it)) + (def e (min to (+ start (string-length c)))) + (when (< from e) + (textbuilder-add-string! b (piece c (- (max from start) start) (- e start)))) + (when (and (< e to) (chunk-iter-next! it)) (go)))) + +;; The first `n` characters of `r`, or all of it when it has fewer. +(: (-> Text int Text)) +(defun (text-take r n) + (check-count "text-take" n) + (def (Tuple l _) (split-with r (offset-after-chars r n) "text-take" (owner-next))) + l) + +;; `r` without its first `n` characters. +(: (-> Text int Text)) +(defun (text-drop r n) + (check-count "text-drop" n) + (def (Tuple _ rr) (split-with r (offset-after-chars r n) "text-drop" (owner-next))) + rr) + +;; The last `n` characters of `r`, or all of it when it has fewer. +(: (-> Text int Text)) +(defun (text-take-right r n) + (check-count "text-take-right" n) + (def (Tuple _ rr) (split-with r (offset-before-last-chars r n) "text-take-right" (owner-next))) + rr) + +;; `r` without its last `n` characters. +(: (-> Text int Text)) +(defun (text-drop-right r n) + (check-count "text-drop-right" n) + (def (Tuple l _) (split-with r (offset-before-last-chars r n) "text-drop-right" (owner-next))) + l) + +;; Bytes `from` to `to` of `r` as a text, from the chunks of `it`, which is on +;; the chunk that holds `from` or one before it. A range inside one chunk is +;; a string that shares the chunk's bytes; a longer one is copied when it is +;; at most `chunk-max` bytes, and a slice that shares the chunks otherwise. +(: (-> Text ChunkIter int int Text)) +(defun (range-from! r it from to) + (let skip () + (when (and (<= (+ (.start it) (string-length (.chunk it))) from) (chunk-iter-next! it)) (skip))) + (def c (.chunk it)) + (def start (.start it)) + (cond + ((<= to (+ start (string-length c))) (small-text (piece c (- from start) (- to start)))) + (else (range->text r from to)))) + +;; `r` with every `old` replaced by `new`, left to right without overlaps. +;; +;; - Searches through one chunk iteration, and adds the text between the +;; matches through another, sharing the arrays of the chunks it keeps whole. +;; - Raises when `old` is empty. +(: (-> Text string string Text)) +(defun (text-replace r old new) + (when (string-empty? old) (bad-argument "text-replace: the string to replace is empty.")) + (def m (string-length old)) + (cond + ((< (.bytes r) m) r) + ((= (.height r) 0) + (if (string-contains? (.small r) old) (string->text (string-replace (.small r) old new)) r)) + (else + (def si (chunk-iter r 0)) + (def first (index-from! r si old 0)) + (if (< first 0) + r + (begin + (def b (textbuilder-empty)) + (def it (chunk-iter r 0)) + (let go ((from 0) (at first)) + (add-range! b r it from at) + (textbuilder-add-string! b new) + (def next (+ at m)) + (def k (index-from! r si old next)) + (if (>= k 0) + (go next k) + (add-range! b r it next (.bytes r)))) + (textbuilder->text b)))))) + +;; The fields of `r` between the occurrences of `sep`, empty ones kept: one +;; more than the occurrences. An empty `sep` gives `r` as the one field. A +;; field inside one chunk shares the chunk's bytes. +(: (-> Text string (Vec Text))) +(defun (text-split r sep) + (def m (string-length sep)) + (cond + ((= m 0) (vec-add (vec-empty) r)) + ((= (.height r) 0) (vec-map small-text (string-split (.small r) sep))) + (else + (def si (chunk-iter r 0)) + (def ft (chunk-iter r 0)) + (def out (vecbuilder-empty)) + (let go ((from 0)) + (def k (index-from! r si sep from)) + (if (>= k 0) + (begin (vecbuilder-add! out (range-from! r ft from k)) (go (+ k m))) + (vecbuilder-add! out (range-from! r ft from (.bytes r))))) + (vecbuilder->vec out)))) + +;; The text before the first `sep` and the text after it, or None. +(: (-> Text string (Option (Tuple Text Text)))) +(defun (text-split-once r sep) + (match (index-of r sep) + ((Some k) + (def o (owner-next)) + (def (Tuple l rest) (split-with r k "text-split-once" o)) + (def (Tuple _ after) (split-with rest (string-length sep) "text-split-once" o)) + (Some (Tuple l after))) + (None None))) + +;; The texts of `parts` in order, with `sep` between each two. +(: (-> (Vec Text) string Text)) +(defun (text-join parts sep) + (def b (textbuilder-empty)) + (loop (:with first? #t #f) + (:for part parts) + (:do (unless first? (textbuilder-add-string! b sep)) + (textbuilder-add-text! b part))) + (textbuilder->text b)) + +;; The cursor on the first `needle` in `r`. An empty needle is at the start. +(: (-> Text string (Option TextCursor))) +(defun (text-search r needle) (text-cursor-search r (text-cursor-start r) needle)) + +;; The cursor on the last `needle` in `r`. An empty needle is at the end. +(: (-> Text string (Option TextCursor))) +(defun (text-search-backward r needle) (text-cursor-search-backward r (text-cursor-end r) needle)) + +;; The cursor on the first character of `r` that `pred` holds for. +(: (-> (-?-> char bool) Text (Option TextCursor))) +(defun (text-find pred r) (text-cursor-find pred r (text-cursor-start r))) + +;; The cursor on the first character at or after `start` that is in `chars`. +;; +;; - A set of ASCII characters is searched for in each chunk's bytes at once. +;; - An empty set is never found. +(: (-> Text string TextCursor (Option TextCursor))) +(defun (text-search-any r chars start) + (cond + ((or (string-empty? chars) (at-end? start)) None) + ((= (string-length chars) (string-count chars)) + (def set (.AsSpan chars)) + (def it (chunk-iter r (.at start))) + (let go ((from (- (.at start) (.start it)))) + (def k (span-index-of-any (ro-span-slice (.AsSpan (.chunk it)) #:from from) set)) + (cond + ((>= k 0) (Some (cursor-at-byte r (+ (.start it) from k)))) + ((chunk-iter-next! it) (go 0)) + (else None)))) + (else + (def set (loop (:for ch chars) (:acc (arraying ch)))) + (def n (array-length set)) + (text-cursor-find (fun (ch) (let go ((i 0)) (and (< i n) (or (char=? (array-ref set i) ch) (go (+ i 1)))))) r start)))) + +;; Whether `ch` is not white space. +(: (-> char bool)) +(defun (not-space? ch) (not (char-whitespace? ch))) + +;; `r` without the white space at its start. +(: (-> Text Text)) +(defun (text-trim-left r) + (match (text-find not-space? r) + ((Some c) (if (at-start? c) r (range->text r (.at c) (.bytes r)))) + (None empty-text))) + +;; `r` without the white space at its end. +(: (-> Text Text)) +(defun (text-trim-right r) + (match (text-cursor-find-backward not-space? r (text-cursor-end r)) + ((Some c) + (def e (step-next c)) + (if (at-end? e) r (range->text r 0 (.at e)))) + (None empty-text))) + +;; `r` without the white space at both ends. +(: (-> Text Text)) +(defun (text-trim r) (text-trim-right (text-trim-left r))) + +;; `n` copies of `ch`. +(: (-> char int string)) +(defun (repeated-char ch n) + (def b (stringbuilder-empty)) + (loop (:for _ (range 0 n)) (:do (stringbuilder-add! b ch))) + (stringbuilder->string b)) + +;; `r` with `#:with` before it until it is `width` characters long. +(: (-> Text int (#:with char) Text)) +(defun (text-pad-left r width #:with #\space) + (def k (- width (.chars r))) + (if (<= k 0) r (insert-with r 0 (repeated-char with k) (owner-next)))) + +;; `r` with `#:with` after it until it is `width` characters long. +(: (-> Text int (#:with char) Text)) +(defun (text-pad-right r width #:with #\space) + (def k (- width (.chars r))) + (if (<= k 0) r (insert-with r (.bytes r) (repeated-char with k) (owner-next)))) + +;; `r` with each character uppercased in the invariant culture, chunk by chunk. +(: (-> Text Text)) +(defun (text-upcase r) + (def b (textbuilder-empty)) + (text-for-each-chunk (fun (c) (textbuilder-add-string! b (.ToUpperInvariant c))) r) + (textbuilder->text b)) + +;; `r` with each character lowercased in the invariant culture, chunk by chunk. +(: (-> Text Text)) +(defun (text-downcase r) + (def b (textbuilder-empty)) + (text-for-each-chunk (fun (c) (textbuilder-add-string! b (.ToLowerInvariant c))) r) + (textbuilder->text b)) + +;; The characters of `r` in reverse order: its chunks from the last, each +;; reversed. +(: (-> Text Text)) +(defun (text-reverse r) + (if (= (.height r) 0) + (small-text (.Reverse (.small r))) + (begin + (def b (textbuilder-empty)) + (def it (chunk-iter r (- (.bytes r) 1))) + (let go () + (textbuilder-add-string! b (.Reverse (.chunk it))) + (when (chunk-iter-prev! it) (go))) + (textbuilder->text b)))) + +;; Folds `f` over the characters of `r`, first to last. +(: (-> (-?-> char %a %a) %a Text %a)) +(defun (text-fold f init r) + (loop (:for ch r) (:acc acc (folding init (f ch acc))) => acc)) + +;; Folds `f` over the characters of `r`, last to first. +(: (-> (-?-> char %a %a) %a Text %a)) +(defun (text-fold-right f init r) + (loop (:for ch (text-backward r (text-cursor-end r))) (:acc acc (folding init (f ch acc))) => acc)) + +(: (-> (-?-> char void) Text void)) +(defun (text-for-each f r) (loop (:for ch r) (:do (f ch)))) + +(: (-> Text (List char))) +(defun (text->list r) (text-fold-right (fun (ch acc) (Cons ch acc)) Nil r)) + +(: (-> Text (Vec char))) +(defun (text->vec r) (loop (:for ch r) (:acc (vecing ch)))) + +;; --------------------------------------------------------------------------- +;; Collecting and adding +;; --------------------------------------------------------------------------- + +;; What a builder takes a piece of text as: a string or a character. +(def/trait (TextPiece %a) + (: textbuilder-add-piece! (-> TextBuilder %a void))) + +(impl (TextPiece string) + (defun (textbuilder-add-piece! b s) (textbuilder-add-string! b s))) + +(impl (TextPiece char) + (defun (textbuilder-add-piece! b ch) (textbuilder-add! b ch))) + +;; The collector that gathers characters or strings into a text, through a +;; builder, as `stringing` gathers characters into a string. +(type (: (Texting %a) (Union texting))) + +(impl (Collector (Texting %a)) + (where (TextPiece %a)) + (type %elem %a) + (type %acc TextBuilder) + (type %out Text) + (defun (collector-init c) (textbuilder-empty)) + (defun (collector-step c acc x) (textbuilder-add-piece! acc x) acc) + (defun (collector-finish c acc) (textbuilder->text acc))) + +;; `(texting x #:into t)`: the builder starts with the chunks of `t`. +(type (: (TextingInto %a) (Struct (: seed Text)))) + +(impl (Collector (TextingInto %a)) + (where (TextPiece %a)) + (type %elem %a) + (type %acc TextBuilder) + (type %out Text) + (defun (collector-init c) + (def b (textbuilder-empty)) + (textbuilder-add-text! b (.seed c)) + b) + (defun (collector-step c acc x) (textbuilder-add-piece! acc x) acc) + (defun (collector-finish c acc) (textbuilder->text acc))) + +(impl (CollectorInto (Texting %a)) + (type %seed Text) + (type %into (TextingInto %a)) + (defun (collector-into c seed) (TextingInto (seed seed)))) + +(impl (Addable Text) + (type %elem string) + (defun (add r s) (insert-with r (.bytes r) s (owner-next)))) + +(impl (Addable! TextBuilder) + (type %elem char) + (defun (add! b ch) (textbuilder-add! b ch))) + +(impl (Addable! TransientText) + (type %elem string) + (defun (add! t s) (transienttext-append! t s))) + +;; --------------------------------------------------------------------------- +;; Cursors +;; --------------------------------------------------------------------------- + +;; A position in a text, a struct that holds the chunk it is in. +;; +;; - `at` is a byte offset on a character boundary. `chunk` starts at byte +;; `start` of the text, ends at byte `end`, and holds the character at `at`. +;; - At the end of the text, `chunk` is "" and `start` and `end` are `at`. +;; - A step inside the chunk reads its bytes only; a step out of it finds the +;; next chunk from the root. +(type (: TextCursor #:opaque + (Struct (: text Text) + (: chunk string) + (: start int) + (: end int) + (: at int)))) + +(: (-> Text TextCursor)) +(defun (end-cursor r) + (TextCursor (text r) (chunk "") (start (.bytes r)) (end (.bytes r)) (at (.bytes r)))) + +;; The cursor at `at`, a boundary below the length of `r`, found by +;; `chunk-of`. +;; +;; - Not inlined: inlined into a loop that starts with it, the descent takes +;; the room .NET would inline the loop's steps with. +(: (-> Text int TextCursor)) +(defun (cursor-in r at) #:no-inline + (def (Tuple c start) (chunk-of r at)) + (TextCursor (text r) (chunk c) (start start) (end (+ start (string-length c))) (at at))) + +;; As `cursor-in`, for a step into the next or previous chunk. +(: (-> Text int TextCursor)) +(defun (cursor-stepped r at) + (def (Tuple c start) (chunk-scanning r at)) + (TextCursor (text r) (chunk c) (start start) (end (+ start (string-length c))) (at at))) + +;; The cursor at byte `at`, a character boundary of `r`. +(: (-> Text int TextCursor)) +(defun (cursor-at-byte r at) + (if (= at (.bytes r)) (end-cursor r) (cursor-in r at))) + +;; The characters before slot `i` of `n`. +(: (-> TextNode int int)) +(defun (chars-before-slot n i) + (let go ((j 0) (acc 0)) (if (< j i) (go (+ j 1) (+ acc (slot-chars n j))) acc))) + +;; The cursor on character `n` of `r`, where 0 <= `n` <= its characters. +;; +;; - Descends by the character counts of the nodes, then steps through one +;; chunk; a chunk with as many characters as bytes is ASCII, and needs no +;; steps. +(: (-> Text int TextCursor)) +(defun (cursor-of-char r n) #:no-inline + (cond + ((>= n (.chars r)) (end-cursor r)) + ((= (.height r) 0) + (def c (.small r)) + (def at (if (= (.chars r) (.bytes r)) n (bytes-of-first c n))) + (TextCursor (text r) (chunk c) (start 0) (end (string-length c)) (at at))) + (else + (let go ((nd (.root r)) (h (.height r)) (n n) (start 0)) + (def last (- (node-count nd) 1)) + (def (Tuple i k) + (let scan ((i 0) (k n)) + (def ci (slot-chars nd i)) + (if (and (< i last) (>= k ci)) (scan (+ i 1) (- k ci)) (Tuple i k)))) + (def s2 (+ start (slot-base nd i))) + (if (= h 1) + (begin + (def c (chunk-at (as-leaves nd) i)) + (def cl (string-length c)) + (def off (if (= (slot-chars nd i) cl) k (bytes-of-first c k))) + (TextCursor (text r) (chunk c) (start s2) (end (+ s2 cl)) (at (+ s2 off)))) + (go (kid (as-branch nd) i) (- h 1) k s2)))))) + +;; The byte offset of character `n` of `r`, where 0 <= `n` <= its characters. +(: (-> Text int int)) +(defun (byte-of-char r n) (.at (cursor-of-char r n))) + +;; The characters before byte `at`, a boundary of `r`. +(: (-> Text int int)) +(defun (chars-before r at) + (cond + ((= at 0) 0) + ((= at (.bytes r)) (.chars r)) + ((= (.height r) 0) (chars-of-first (.small r) (.chars r) at)) + (else + (let go ((n (.root r)) (h (.height r)) (at at) (acc 0)) + (def (Tuple i k) (slot-starting n at)) + (def before (+ acc (chars-before-slot n i))) + (if (= h 1) + (+ before (chars-of-first (chunk-at (as-leaves n) i) (slot-chars n i) k)) + (go (kid (as-branch n) i) (- h 1) k before)))))) + +(: (-> Text TextCursor)) +(defun (text-cursor-start r) + (if (= (.bytes r) 0) (end-cursor r) (cursor-in r 0))) + +(: (-> Text TextCursor)) +(defun (text-cursor-end r) (end-cursor r)) + +(: (-> TextCursor bool)) +(defun (at-end? c) (= (.at c) (.bytes (.text c)))) + +(: (-> TextCursor bool)) +(defun (at-start? c) (= (.at c) 0)) + +;; Whether `c`, a cursor of `r`, is at its end. +(: (-> Text TextCursor bool)) +(defun (text-cursor-end? r c) (at-end? c)) + +;; Whether `c`, a cursor of `r`, is at its start. +(: (-> Text TextCursor bool)) +(defun (text-cursor-start? r c) (at-start? c)) + +(: (-> TextCursor string void)) +(defun (check-not-end c who) + (when (at-end? c) (cursor-at-edge who "end"))) + +(: (-> TextCursor string void)) +(defun (check-not-start c who) + (when (at-start? c) (cursor-at-edge who "start"))) + +;; Raises that the cursor is at the `edge` of the text. Not inlined, so that +;; the message is not built in the steps that check. +(: (-> string string void)) +(defun (cursor-at-edge who edge) #:no-inline + (out-of-range #"${who}: the cursor is at the ${edge} of the text.")) + +;; The character at `c`, which is not at the end. +(: (-> TextCursor char)) +(defun (char-at c) #:inline (decode-at (.chunk c) (- (.at c) (.start c)))) + +;; The cursor `to`, after the character at `c`. +(: (-> TextCursor int TextCursor)) +(defun (moved-to c to) + (cond + ((< to (.end c)) (record-set c (at to))) + ((< to (.bytes (.text c))) (cursor-stepped (.text c) to)) + (else (end-cursor (.text c))))) + +;; The cursor after the character at `c`, which is not at the end. +(: (-> TextCursor TextCursor)) +(defun (step-next c) + (moved-to c (+ (.at c) (char-width (byte-at (.AsSpan (.chunk c)) (- (.at c) (.start c))))))) + +;; The character at `c`, which is not at the end, and the cursor after it, +;; from one read of the chunk. +(: (-> TextCursor (Tuple char TextCursor))) +(defun (ref+next c) + (def sp (.AsSpan (.chunk c))) + (def k (- (.at c) (.start c))) + (def b0 (byte-at sp k)) + (def ch (if (< b0 128) (int->char b0) (decode-at (.chunk c) k))) + (Tuple ch (moved-to c (+ (.at c) (char-width b0))))) + +;; The cursor on the character before `c`, which is not at the start. +(: (-> TextCursor TextCursor)) +(defun (step-prev c) + (def p (if (> (.at c) (.start c)) c (cursor-stepped (.text c) (- (.at c) 1)))) + (def sp (.AsSpan (.chunk p))) + (let go ((k (- (- (.at c) 1) (.start p)))) + (if (and (> k 0) (= (bitwise-and (byte-at sp k) 192) 128)) + (go (- k 1)) + (record-set p (at (+ (.start p) k)))))) + +;; The character at `c`, a cursor of `r`. +(: (-> Text TextCursor char)) +(defun (text-cursor-ref r c) + (check-not-end c "text-cursor-ref") + (char-at c)) + +(: (-> Text TextCursor TextCursor)) +(defun (text-cursor-next r c) + (check-not-end c "text-cursor-next") + (step-next c)) + +(: (-> Text TextCursor TextCursor)) +(defun (text-cursor-prev r c) + (check-not-start c "text-cursor-prev") + (step-prev c)) + +(: (-> Text TextCursor (Tuple char TextCursor))) +(defun (text-cursor-ref+next r c) + (check-not-end c "text-cursor-ref+next") + (ref+next c)) + +;; Raises that a count is negative. Not inlined, as `cursor-at-edge`. +(: (-> string int void)) +(defun (negative-count who n) #:no-inline + (bad-argument #"${who}: the count ${n} is negative.")) + +;; Raises that there are fewer than `n` characters on the `side` of the +;; cursor. Not inlined, as `cursor-at-edge`. +(: (-> string int string void)) +(defun (too-few who n side) #:no-inline + (out-of-range #"${who}: there are fewer than ${n} characters ${side} the cursor.")) + +;; The steps a cursor takes one character at a time. A longer move finds its +;; character from the root, by the character counts. +(def few-steps 16) + +;; The cursor `n` characters after `c`, a cursor of `r`. +(: (-> Text TextCursor int TextCursor)) +(defun (text-cursor-forward r c n) + (when (< n 0) (negative-count "text-cursor-forward" n)) + (if (<= n few-steps) + (let go ((p c) (k n)) + (cond + ((= k 0) p) + ((at-end? p) (too-few "text-cursor-forward" n "after") p) + (else (go (step-next p) (- k 1))))) + (begin + (def t (.text c)) + (def target (+ (chars-before t (.at c)) n)) + (when (> target (.chars t)) (too-few "text-cursor-forward" n "after")) + (cursor-of-char t target)))) + +;; The cursor `n` characters before `c`, a cursor of `r`. +(: (-> Text TextCursor int TextCursor)) +(defun (text-cursor-back r c n) + (when (< n 0) (negative-count "text-cursor-back" n)) + (if (<= n few-steps) + (let go ((p c) (k n)) + (cond + ((= k 0) p) + ((at-start? p) (too-few "text-cursor-back" n "before") p) + (else (go (step-prev p) (- k 1))))) + (begin + (def t (.text c)) + (def target (- (chars-before t (.at c)) n)) + (when (< target 0) (too-few "text-cursor-back" n "before")) + (cursor-of-char t target)))) + +;; The cursor `column` characters into line `n`, or at the line's end when +;; it is shorter. +(: (-> Text int (#:column int) TextCursor)) +(defun (text-cursor-at-line r n #:column 0) + (def from (line-start r n)) + (def e (line-end r n)) + (let go ((p (cursor-at-byte r from)) (k column)) + (if (or (<= k 0) (>= (.at p) e)) p (go (step-next p) (- k 1))))) + +;; The line of `c`, a cursor of `r`, counting from 0. +(: (-> Text TextCursor int)) +(defun (text-cursor-line r c) (line-of (.text c) (.at c))) + +(: (-> Text TextCursor TextCursor)) +(defun (text-cursor-line-start r c) + (def t (.text c)) + (cursor-at-byte t (line-start t (line-of t (.at c))))) + +(: (-> Text TextCursor TextCursor)) +(defun (text-cursor-line-end r c) + (def t (.text c)) + (cursor-at-byte t (line-end t (line-of t (.at c))))) + +;; The characters before cursor `c` of `r`: its index, from 0 to +;; `text-count`. O(log n) time. +(: (-> Text TextCursor int)) +(defun (text-cursor-index r c) (chars-before (.text c) (.at c))) + +;; The characters between the start of the cursor's line and the cursor. +(: (-> Text TextCursor int)) +(defun (text-cursor-column r c) + (def t (.text c)) + (def from (line-start t (line-of t (.at c)))) + (- (chars-before t (.at c)) (chars-before t from))) + +;; The first `s` that starts at or after `c`, a cursor of `r`. +(: (-> Text TextCursor string (Option TextCursor))) +(defun (text-cursor-search r c s) + (def t (.text c)) + (option-map (fun (k) (cursor-at-byte t k)) (index-of t s #:from (.at c)))) + +;; The last `s` that ends at or before `c`, a cursor of `r`. +(: (-> Text TextCursor string (Option TextCursor))) +(defun (text-cursor-search-backward r c s) + (def t (.text c)) + (option-map (fun (k) (cursor-at-byte t k)) (last-index-of t s #:to (.at c)))) + +;; The first character at or after `c`, a cursor of `r`, that `pred` holds +;; for. +(: (-> (-?-> char bool) Text TextCursor (Option TextCursor))) +(defun (text-cursor-find pred r c) + (let go ((p c)) + (cond + ((at-end? p) None) + (else + (def (Tuple ch n) (ref+next p)) + (if (pred ch) (Some p) (go n)))))) + +;; The last character before `c`, a cursor of `r`, that `pred` holds for. +(: (-> (-?-> char bool) Text TextCursor (Option TextCursor))) +(defun (text-cursor-find-backward pred r c) + (let go ((p c)) + (if (at-start? p) + None + (begin + (def q (step-prev p)) + (if (pred (char-at q)) (Some q) (go q)))))) + +;; Comparisons of two cursors of one text. +(: (-> TextCursor TextCursor bool)) +(defun (text-cursor=? a b) (= (.at a) (.at b))) +(: (-> TextCursor TextCursor bool)) +(defun (text-cursor TextCursor TextCursor bool)) +(defun (text-cursor>? a b) (> (.at a) (.at b))) +(: (-> TextCursor TextCursor bool)) +(defun (text-cursor<=? a b) (<= (.at a) (.at b))) +(: (-> TextCursor TextCursor bool)) +(defun (text-cursor>=? a b) (>= (.at a) (.at b))) + +(impl (Eq TextCursor) + (defun (= a b) (= (.at a) (.at b))) + (defun (eq-hash c) (eq-hash (.at c)))) + +(impl (Ord TextCursor) + (defun (compare a b) (compare (.at a) (.at b)))) + +;; The characters from the cursor to the end. +(impl (Iterable TextCursor) + (type %elem char) + (type %cursor TextCursor) + (defun (iterable-start c) c) + (defun (iterable-done? c p) (at-end? p)) + (defun (iterable-current c p) (char-at p)) + (defun (iterable-next c p) (step-next p))) + +;; The characters before a cursor, last first. +(type (: TextBackward #:opaque (Struct (: from TextCursor)))) + +;; A step of a backward loop: the cursor on its character, or done. +(type (: TextBackwardStep #:opaque (Struct (: on TextCursor) (: done? bool)))) + +;; The characters before `c`, a cursor of `r`, last first. +(: (-> Text TextCursor TextBackward)) +(defun (text-backward r c) (TextBackward (from c))) + +(: (-> TextCursor TextBackwardStep)) +(defun (backward-from c) + (if (at-start? c) + (TextBackwardStep (on c) (done? #t)) + (TextBackwardStep (on (step-prev c)) (done? #f)))) + +(impl (Iterable TextBackward) + (type %elem char) + (type %cursor TextBackwardStep) + (defun (iterable-start b) (backward-from (.from b))) + (defun (iterable-done? b s) (.done? s)) + (defun (iterable-current b s) (char-at (.on s))) + (defun (iterable-next b s) (backward-from (.on s)))) + +;; --------------------------------------------------------------------------- +;; Grapheme clusters +;; --------------------------------------------------------------------------- + +;; The front end of (std unicode grapheme) for texts. Its cursors must be on +;; a cluster boundary, as the string front end's are. + +;; The cursor after the cluster that starts at `c`, a cluster boundary of `r`. +;; - Between two bytes of printable ASCII in the chunk, the next byte. +(: (-> Text TextCursor TextCursor)) +(defun (text-cursor-next-grapheme r c) + (check-not-end c "text-cursor-next-grapheme") + (def sp (.AsSpan (.chunk c))) + (def k (- (.at c) (.start c))) + (if (and (< (+ k 1) (ro-span-length sp)) (grapheme-plain-break? (byte-at sp k) (byte-at sp (+ k 1)))) + (record-set c (at (+ (.at c) 1))) + (next-grapheme-by-rules r c))) + +(: (-> Text TextCursor TextCursor)) +(defun (next-grapheme-by-rules r c) + (def (Tuple ch0 c1) (text-cursor-ref+next r c)) + (let go ((st (grapheme-start ch0)) (c c1)) + (if (at-end? c) + c + (begin + (def (Tuple ch nxt) (text-cursor-ref+next r c)) + (def st2 (grapheme-step st ch)) + (if (grapheme-break? st2) c (go st2 nxt)))))) + +;; The backward state `w` once known, reading back from `k`, the cursor of +;; the last character given. +(: (-> Text int TextCursor int)) +(defun (text-back-resolve r w k) + (let more ((w w) (k k)) + (cond + ((not (grapheme-back-wants? w)) w) + ((at-start? k) (grapheme-back-sot w)) + (else + (begin + (def k2 (step-prev k)) + (more (grapheme-back-more w (text-cursor-ref r k2)) k2)))))) + +;; The start of the cluster that ends at `c`, a cluster boundary of `r`. +;; - After two bytes of printable ASCII in the chunk, the byte before. +(: (-> Text TextCursor TextCursor)) +(defun (text-cursor-prev-grapheme r c) + (check-not-start c "text-cursor-prev-grapheme") + (def sp (.AsSpan (.chunk c))) + (def k (- (.at c) (.start c))) + (if (and (>= k 2) (grapheme-plain-break? (byte-at sp (- k 2)) (byte-at sp (- k 1)))) + (record-set c (at (- (.at c) 1))) + (prev-grapheme-by-rules r c))) + +(: (-> Text TextCursor TextCursor)) +(defun (prev-grapheme-by-rules r c) + (def c1 (step-prev c)) + (def b1 (text-cursor-ref r c1)) + (def w0 (if (and (grapheme-back-needs-after? b1) (not (at-end? c))) + (grapheme-back-start-before (text-cursor-ref r c)) + (grapheme-back-start))) + ;; `bc` is the cursor of the earliest character of the cluster so far, `b`. + (let go ((w w0) (bc c1) (b b1)) + (if (at-start? bc) + bc + (begin + (def ac (step-prev bc)) + (def a (text-cursor-ref r ac)) + (def w1 (grapheme-back-pair w a b)) + (def w2 (if (grapheme-back-wants? w1) (text-back-resolve r w1 ac) w1)) + (if (grapheme-back-break? w2) bc (go w2 ac a)))))) + +;; Whether `c` is on a cluster boundary of `r`. +(: (-> Text TextCursor bool)) +(defun (text-cursor-grapheme-boundary? r c) + (if (or (at-start? c) (at-end? c)) + #t + (begin + (def ac (step-prev c)) + (grapheme-back-break? + (text-back-resolve r (grapheme-back-pair (grapheme-back-start) (text-cursor-ref r ac) (text-cursor-ref r c)) ac))))) + +;; The clusters of `r`. +(: (-> Text int)) +(defun (text-grapheme-count r) + (def c0 (text-cursor-start r)) + (if (at-end? c0) + 0 + (begin + (def (Tuple ch0 c1) (text-cursor-ref+next r c0)) + (let go ((st (grapheme-start ch0)) (c c1) (n 1)) + (if (at-end? c) + n + (begin + (def (Tuple ch nxt) (text-cursor-ref+next r c)) + (def st2 (grapheme-step st ch)) + (go st2 nxt (if (grapheme-break? st2) (+ n 1) n)))))))) + +;; The clusters of `r`, in order, as texts. +(: (-> Text (Vec Text))) +(defun (text->graphemes r) + (let go ((c (text-cursor-start r)) (acc '())) + (if (at-end? c) + (list->vec (list-reverse acc)) + (begin + (def d (text-cursor-next-grapheme r c)) + (go d (cons (text-substring r c d) acc)))))) + +;; --------------------------------------------------------------------------- +;; Transients +;; --------------------------------------------------------------------------- + +;; A text that is edited in place: its edits copy only the nodes it does not +;; own yet. `transienttext->text` returns the text it holds and takes a new +;; owner, so that the text does not change after. +;; +;; - Positions are character indices, from 0 to the characters it holds. +;; - The hot chunk: the chunk of the last edit, held in `buf` as bytes 0 to +;; `hot-len`, with `hot-chars` characters, the first of which is character +;; `hot-first` of the text. `path` holds the nodes down to it, the leaves +;; node first, and `slots` the slot taken in each. `hot-len` is -1 when no +;; chunk is hot. +;; - `mark-char` and `mark-byte` are a character of the hot chunk and its byte +;; in `buf`, where the last edit in it ended: the next edit steps from there +;; to its own byte, and a chunk with as many bytes as characters needs no +;; steps. +;; - An edit inside the hot chunk moves bytes in `buf` and adds to the newline +;; and character counts on the path, and allocates nothing. The byte counts +;; on the path stay those of the old string until `cool!`. It must leave the +;; chunk at most `chunk-max` bytes, and not empty unless the text is one +;; string. +;; - While a chunk is hot, `bytes`, `lines` and `chars` are the text's, not +;; those in `text`, and when `dirty?` its slot holds an old string. `cool!` +;; puts the chunk back; every other operation cools first. +;; - An edit outside the hot chunk makes its own chunk hot only when it is +;; within `hot-reach` characters of `last`, where the edit before it ended. +;; Edits far apart edit the tree, which costs less than copying each chunk +;; in and out for one edit. +(def hot-reach 64) + +(type (: TransientText #:opaque + (Record (: text Text #:mutable) + (: owner long #:mutable) + (: buf (Array byte)) + (: path (Array TextNode)) + (: slots (Array int)) + (: hot-len int #:mutable) + (: hot-chars int #:mutable) + (: hot-first int #:mutable) + (: mark-char int #:mutable) + (: mark-byte int #:mutable) + (: dirty? bool #:mutable) + (: bytes int #:mutable) + (: lines int #:mutable) + (: chars int #:mutable) + (: last int #:mutable)))) + +(: (-> Text TransientText)) +(defun (text->transienttext r) + (TransientText (text r) (owner (owner-next)) + (buf (make-array chunk-max #:fill (cast byte 0))) + (path (make-array 24 #:fill no-node)) + (slots (make-array 24 #:fill 0)) + (hot-len -1) (hot-chars 0) (hot-first 0) (mark-char 0) (mark-byte 0) + (dirty? #f) (bytes 0) (lines 0) (chars 0) (last 0))) + +;; Copies the bytes of `s` into `buf` from `at`. +(: (-> string (Array byte) int void)) +(defun (copy-into! s buf at) #:inline + (.CopyTo (.AsSpan s) (span-slice (array->span buf) #:from at))) + +;; Whether byte `k` of `buf` continues a character. +(: (-> (Array byte) int bool)) +(defun (continues? buf k) #:inline + (= (bitwise-and (cast int (array-ref buf k)) 192) 128)) + +;; Makes the chunk with byte `at` hot, when no chunk is: the chunk that ends +;; at `at` when `left?`, and otherwise the one that starts there. +;; +;; - Copies the nodes on the path that the transient does not own. +;; - Returns #f, and leaves no chunk hot, when the chunk does not have `room` +;; bytes free before `chunk-max`. +(: (-> TransientText int bool int bool)) +(defun (heat! t at left? room) + (def r (.text t)) + (def h (.height r)) + (def o (.owner t)) + (def buf (.buf t)) + (if (= h 0) + (begin + (def len (.bytes r)) + (if (> (+ len room) chunk-max) + #f + (begin + (copy-into! (.small r) buf 0) + (record-set! t (hot-len len) (hot-chars (.chars r)) (hot-first 0) (mark-char 0) (mark-byte 0) + (dirty? #f) (bytes len) (lines (.lines r)) (chars (.chars r))) + #t))) + (begin + (def path (.path t)) + (def slots (.slots t)) + (def root (editable h (.root r) o)) + (unless (= (node-owner (.root r)) o) + (record-set! t (text (Text (root root) (height h) (bytes (.bytes r)) (lines (.lines r)) (chars (.chars r)) (small ""))))) + (let go ((n root) (lvl (- h 1)) (at at) (first 0)) + (def (Tuple i k) (if left? (slot-ending n at) (slot-starting n at))) + (def f (+ first (chars-before-slot n i))) + (array-set! path lvl n) + (array-set! slots lvl i) + (if (= lvl 0) + (begin + (def c (chunk-at (as-leaves n) i)) + (def len (string-length c)) + (if (> (+ len room) chunk-max) + #f + (begin + (copy-into! c buf 0) + (record-set! t (hot-len len) (hot-chars (slot-chars n i)) (hot-first f) (mark-char 0) (mark-byte 0) + (dirty? #f) (bytes (.bytes r)) (lines (.lines r)) (chars (.chars r))) + #t))) + (begin + (def b (as-branch n)) + (def child (editable lvl (kid b i) o)) + (set-kid! b i child) + (go child (- lvl 1) k f))))))) + +;; Adds `dl` newlines and `dc` characters to the slots on the path to the hot +;; chunk. +(: (-> TransientText int int void)) +(defun (adjust-path-counts! t dl dc) + (def path (.path t)) + (def slots (.slots t)) + (loop (:for lvl (range 0 (.height (.text t)))) + (:do (def n (array-ref path lvl)) + (def i (array-ref slots lvl)) + (set-slot-lines! n i (+ (slot-lines n i) dl)) + (set-slot-chars! n i (+ (slot-chars n i) dc))))) + +;; Adds `db` bytes to the slots on the path to the hot chunk. +(: (-> TransientText int void)) +(defun (adjust-path-bytes! t db) + (def path (.path t)) + (def slots (.slots t)) + (loop (:for lvl (range 0 (.height (.text t)))) + (:do (add-slot-bytes! (array-ref path lvl) (array-ref slots lvl) db)))) + +;; Whether an edit at character `n` is near enough the last one to make its +;; chunk hot. +(: (-> TransientText int bool)) +(defun (near? t n) #:inline + (<= (abs (- n (.last t))) hot-reach)) + +;; The byte in `buf` that `k` characters after byte `b` start at. +(: (-> (Array byte) int int int)) +(defun (buf-forward buf b k) + (let go ((b b) (k k)) + (if (= k 0) b (go (+ b (char-width (cast int (array-ref buf b)))) (- k 1))))) + +;; The byte in `buf` that `k` characters before byte `b` start at. +(: (-> (Array byte) int int int)) +(defun (buf-back buf b k) + (let go ((b b) (k k)) + (if (= k 0) + b + (go (let back ((p (- b 1))) (if (continues? buf p) (back (- p 1)) p)) (- k 1))))) + +;; The byte in `buf` where character `kc` of the hot chunk starts, stepped +;; from the mark. +(: (-> TransientText int int)) +(defun (hot-byte t kc) + (def m (.mark-char t)) + (cond + ((= (.hot-chars t) (.hot-len t)) kc) + ((>= kc m) (buf-forward (.buf t) (.mark-byte t) (- kc m))) + (else (buf-back (.buf t) (.mark-byte t) (- m kc))))) + +;; Puts `s`, of `sc` characters, at character `n` when that is in the hot +;; chunk and `s` fits in it, and returns whether it did. +(: (-> TransientText int string int bool)) +(defun (hot-insert! t n s sc) + (def len (.hot-len t)) + (def kc (- n (.hot-first t))) + (def slen (string-length s)) + (if (and (>= len 0) (>= kc 0) (<= kc (.hot-chars t)) (<= (+ len slen) chunk-max)) + (begin + (def buf (.buf t)) + (def k (hot-byte t kc)) + (array-copy! buf k buf (+ k slen) (- len k)) + (copy-into! s buf k) + (def sl (newlines-in s)) + (adjust-path-counts! t sl sc) + (record-set! t (hot-len (+ len slen)) (hot-chars (+ (.hot-chars t) sc)) + (mark-char (+ kc sc)) (mark-byte (+ k slen)) (dirty? #t) + (bytes (+ (.bytes t) slen)) (lines (+ (.lines t) sl)) (chars (+ (.chars t) sc))) + #t) + #f)) + +;; Deletes characters `from` to `to`, where `from` < `to`, when they are in +;; the hot chunk and leave some of it in a tree, and returns whether it did. +(: (-> TransientText int int bool)) +(defun (hot-delete! t from to) + (def len (.hot-len t)) + (def hc (.hot-chars t)) + (def kc (- from (.hot-first t))) + (def ec (- to (.hot-first t))) + (if (and (>= len 0) (>= kc 0) (<= ec hc) (or (< (- ec kc) hc) (= (.height (.text t)) 0))) + (begin + (def buf (.buf t)) + (def dc (- ec kc)) + (def k (hot-byte t kc)) + (def e (if (= hc len) (+ k dc) (buf-forward buf k dc))) + (def dl (span-count (ro-span-slice (array->ro-span buf) #:from k #:to e) newline)) + (array-copy! buf e buf k (- len e)) + (adjust-path-counts! t (- dl) (- dc)) + (record-set! t (hot-len (- len (- e k))) (hot-chars (- hc dc)) (mark-char kc) (mark-byte k) (dirty? #t) + (bytes (- (.bytes t) (- e k))) (lines (- (.lines t) dl)) (chars (- (.chars t) dc))) + #t) + #f)) + +;; Puts the hot chunk back in its slot, if there is one. +;; +;; - A chunk left shorter than `chunk-min` is merged with a neighbour, and +;; nodes left with fewer than `min-kids` children on the path then too, as +;; a delete does. +(: (-> TransientText void)) +(defun (cool! t) + (when (and (>= (.hot-len t) 0) (.dirty? t)) + (def r (.text t)) + (def h (.height r)) + (def len (.hot-len t)) + (def c (utf8->string (ro-span-slice (array->ro-span (.buf t)) #:from 0 #:to len))) + (if (= h 0) + (record-set! t (text (small-text c))) + (begin + (def path (.path t)) + (def slots (.slots t)) + (def o (.owner t)) + (def leaves (array-ref path 0)) + (adjust-path-bytes! t (- len (slot-bytes leaves (array-ref slots 0)))) + (set-chunk! (as-leaves leaves) (array-ref slots 0) c) + (when (< len chunk-min) + (merge-small-chunk! leaves (array-ref slots 0)) + (loop (:for lvl (range 1 h)) + (:do (when (< (node-count (array-ref path (- lvl 1))) min-kids) + (fix-underflow! (array-ref path lvl) (+ lvl 1) (array-ref slots lvl) o))))) + (record-set! t (text (tree->text-sized (.root r) h (.bytes t) (.lines t) (.chars t))))))) + (record-set! t (hot-len -1) (dirty? #f))) + +(: (-> TransientText Text)) +(defun (transienttext->text t) + (cool! t) + (record-set! t (owner (owner-next))) + (.text t)) + +;; The bytes of the text it holds. +(: (-> TransientText int)) +(defun (transienttext-length t) + (if (>= (.hot-len t) 0) (.bytes t) (.bytes (.text t)))) + +;; The characters of the text it holds. +(: (-> TransientText int)) +(defun (transienttext-count t) + (if (>= (.hot-len t) 0) (.chars t) (.chars (.text t)))) + +;; Raises that character index `n` is outside a text of `chars` characters. +;; Not inlined, as `cursor-at-edge`. +(: (-> string int int void)) +(defun (index-outside who n chars) #:no-inline + (out-of-range #"${who}: the index ${n} is outside the text, of ${chars} characters.")) + +(: (-> Text int string void)) +(defun (check-index r n who) + (when (or (< n 0) (> n (.chars r))) (index-outside who n (.chars r)))) + +;; Puts `s` before character `n`. +(: (-> TransientText int string void)) +(defun (transienttext-insert! t n s) + (def slen (string-length s)) + (def sc (chars-in s)) + (unless (hot-insert! t n s sc) + (cool! t) + (check-index (.text t) n "transienttext-insert!") + (def at (byte-of-char (.text t) n)) + (unless (and (> slen 0) (<= slen small-insert) (near? t n) (heat! t at #t slen) (hot-insert! t n s sc)) + (cool! t) + (record-set! t (text (insert-with (.text t) at s (.owner t)))))) + (record-set! t (last (+ n sc)))) + +(: (-> TransientText string void)) +(defun (transienttext-append! t s) + (transienttext-insert! t (transienttext-count t) s)) + +;; Deletes characters `from` to `to`. +(: (-> TransientText int int void)) +(defun (transienttext-delete! t from to) + (unless (and (< from to) (hot-delete! t from to)) + (cool! t) + (def r (.text t)) + (check-index r from "transienttext-delete!") + (check-index r to "transienttext-delete!") + (when (> from to) (backward-range "transienttext-delete!")) + (when (< from to) + (def a (byte-of-char r from)) + (unless (and (near? t from) (heat! t a #f 0) (hot-delete! t from to)) + (cool! t) + (def r2 (.text t)) + (record-set! t (text (delete-with r2 a (byte-of-char r2 to) (.owner t))))))) + (record-set! t (last from))) + +;; Replaces characters `from` to `to` with `s`. +(: (-> TransientText int int string void)) +(defun (transienttext-replace-range! t from to s) + (transienttext-delete! t from to) + (transienttext-insert! t from s)) + +;; --------------------------------------------------------------------------- +;; Checks +;; --------------------------------------------------------------------------- + +;; What is wrong with the tree under `n`, of height `h`, if anything: the +;; first problem found. +(: (-> TextNode int bool (Option string))) +(defun (check-node n h root?) + (def count (node-count n)) + (cond + ((> count cap) (Some #"a node of height ${h} has ${count} children")) + ((and (not root?) (< count min-kids)) (Some #"a node of height ${h} has ${count} children, fewer than ${min-kids}")) + ((= count 0) (Some #"a node of height ${h} is empty")) + (else + (let go ((i 0)) + (if (>= i count) + None + (if (= h 1) + (begin + (def c (chunk-at (as-leaves n) i)) + (cond + ((= (string-length c) 0) (Some "an empty chunk")) + ((> (string-length c) chunk-max) (Some #"a chunk of ${(string-length c)} bytes")) + ((not (= (slot-bytes n i) (string-length c))) (Some "a chunk's byte count is wrong")) + ((not (= (slot-lines n i) (newlines-in c))) (Some "a chunk's newline count is wrong")) + ((not (= (slot-chars n i) (chars-in c))) (Some "a chunk's character count is wrong")) + (else (go (+ i 1))))) + (begin + (def k (kid (as-branch n) i)) + (cond + ((not (= (slot-bytes n i) (sum-bytes k))) (Some #"a node of height ${h} has a wrong byte count")) + ((not (= (slot-lines n i) (sum-lines k))) (Some #"a node of height ${h} has a wrong newline count")) + ((not (= (slot-chars n i) (sum-chars k))) (Some #"a node of height ${h} has a wrong character count")) + (else + (match (check-node k (- h 1) #f) + ((Some why) (Some why)) + (None (go (+ i 1))))))))))))) + +;; What is wrong with `r`, if anything. For tests. +(: (-> Text (Option string))) +(defun (text-check r) + (cond + ((= (.height r) 0) + (cond + ((> (.bytes r) chunk-max) (Some "a text of one string is longer than a chunk")) + ((not (= (.bytes r) (string-length (.small r)))) (Some "the byte count is wrong")) + ((not (= (.lines r) (newlines-in (.small r)))) (Some "the newline count is wrong")) + ((not (= (.chars r) (chars-in (.small r)))) (Some "the character count is wrong")) + (else None))) + ((not (= (.bytes r) (sum-bytes (.root r)))) (Some "the byte count is wrong")) + ((not (= (.lines r) (sum-lines (.root r)))) (Some "the newline count is wrong")) + ((not (= (.chars r) (sum-chars (.root r)))) (Some "the character count is wrong")) + ((<= (.bytes r) chunk-max) (Some "a tree holds no more than one chunk")) + ((and (> (.height r) 1) (< (node-count (.root r)) 2)) (Some "a root branch has one child")) + (else (check-node (.root r) (.height r) #t)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/unicode/casefold.bjo — full Unicode case folding (CaseFolding.txt). +;; +;; - The fold is the default, culture-independent one: the mappings of status +;; C and F. The Turkic mappings (status T) are not used. +;; - The table is generated by gen_casefold.py into casefold-table.protobjo, +;; and decoded once when the module is initialized. +;; +;; Cost: +;; - ASCII is folded without the table. Other characters take a binary +;; search over the 1,606 mappings. +;; - A string that has nothing to fold is returned as it is, with no copy. + +(export + casefold-unicode-version + string-foldcase) + +(import (std prelude)) + +(:doc #:file "casefold.bjodoc") + +(include "casefold-table.protobjo") + +;; --------------------------------------------------------------------------- +;; The table +;; --------------------------------------------------------------------------- + +(: string) +(def casefold-unicode-version unicode-version) + +;; The value of a hex digit, as a byte of ASCII. +(: (-> byte int)) +(defun (hex-value b) + (def x (cast int b)) + (if (< x 97) (- x 48) (- x 87))) + +;; The bytes of `text` that are not white space: the generated text is +;; indented, and every character it holds is '0' or above. +(: (-> string (Array byte))) +(defun (table-bytes text) + (def src (string->bytes text)) + (def n (loop (:for b src) (:when (>= (cast int b) 48)) (:acc (summing 1)))) + (def (: out (Array byte)) (make-array n)) + (let go ((i 0) (j 0)) + (cond + ((>= i (array-length src)) out) + ((< (cast int (array-ref src i)) 48) (go (+ i 1) j)) + (else (array-set! out j (array-ref src i)) (go (+ i 1) (+ j 1)))))) + +(: (Array byte)) +(def fold-digits (table-bytes fold-text)) + +;; The value of the `n` hex digits at `i` in `fold-digits`. +(: (-> int int int)) +(defun (hex-at i n) + (let go ((k 0) (v 0)) + (if (= k n) + v + (go (+ k 1) (+ (* 16 v) (hex-value (array-ref fold-digits (+ i k)))))))) + +;; The code points that fold, in order. +(: (Array int)) +(def fold-keys (make-array fold-count)) + +;; Mapping `i` folds to `fold-targets` from `(array-ref fold-starts i)` up to +;; `(array-ref fold-starts (+ i 1))`. +(: (Array int)) +(def fold-starts (make-array (+ fold-count 1))) + +;; What the code points fold to. Filling it also fills `fold-keys` and +;; `fold-starts`. +(: (Array int)) +(def fold-targets + (begin + (def (: out (Array int)) (make-array (/ (- (array-length fold-digits) (* 7 fold-count)) 6))) + (let go ((m 0) (i 0) (t 0)) + (array-set! fold-starts m t) + (when (< m fold-count) + (array-set! fold-keys m (hex-at i 6)) + (def n (hex-at (+ i 6) 1)) + (loop (:for k (range 0 n)) + (:do (array-set! out (+ t k) (hex-at (+ i 7 (* 6 k)) 6)))) + (go (+ m 1) (+ i 7 (* 6 n)) (+ t n)))) + out)) + +;; The index of the mapping of `cp`, or -1 if `cp` does not fold. +(: (-> int int)) +(defun (fold-index cp) + (let go ((lo 0) (hi fold-count)) + (if (>= lo hi) + -1 + (let* ((mid (shift-right (+ lo hi) 1)) + (k (array-ref fold-keys mid))) + (cond + ((< cp k) (go lo mid)) + ((> cp k) (go (+ mid 1) hi)) + (else mid)))))) + +(: (-> int bool)) +(defun (ascii-upper? cp) #:inline (and (>= cp 65) (<= cp 90))) + +;; True if `ch` folds to something other than itself. +(: (-> char bool)) +(defun (folds? ch) #:inline + (def cp (char->int ch)) + (if (< cp 128) (ascii-upper? cp) (>= (fold-index cp) 0))) + +;; Adds the fold of `ch` to `sb`. +(: (-> StringBuilder char void)) +(defun (add-folded! sb ch) + (def cp (char->int ch)) + (if (< cp 128) + (stringbuilder-add-byte! sb (if (ascii-upper? cp) (+ cp 32) cp)) + (let ((i (fold-index cp))) + (if (< i 0) + (stringbuilder-add! sb ch) + (loop (:for k (range (array-ref fold-starts i) (array-ref fold-starts (+ i 1)))) + (:do (stringbuilder-add! sb (int->char (array-ref fold-targets k))))))))) + +;; --------------------------------------------------------------------------- +;; Strings +;; --------------------------------------------------------------------------- + +;; The fold of `s` from the cursor `from` on, after the part before it, which +;; has nothing to fold. +(: (-> string StringCursor string)) +(defun (fold-from s from) + (def sb (stringbuilder-empty)) + (stringbuilder-add-substring! sb s (string-cursor-start s) from) + (let go ((c from)) + (if (string-cursor-end? s c) + (stringbuilder->string sb) + (begin + (def (Tuple ch next) (string-cursor-ref+next s c)) + (add-folded! sb ch) + (go next))))) + +(: (-> string string)) +(defun (string-foldcase s) + (let scan ((c (string-cursor-start s))) + (if (string-cursor-end? s c) + s + (begin + (def (Tuple ch next) (string-cursor-ref+next s c)) + (if (folds? ch) + (fold-from s c) + (scan next)))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/unicode/grapheme.bjo — extended grapheme clusters (UAX #29). +;; +;; - The rules are two state machines over characters, which know nothing of +;; the container: a forward step, and a backward one that decides the +;; boundary between two characters and asks for the characters before them +;; when a rule needs them. A front end drives them with its own cursors. +;; - This module has the front end for strings. (std text) has the one for +;; texts. +;; - The property table is generated by gen_grapheme.py into +;; grapheme-table.protobjo. +;; +;; Cost: +;; - A step forward reads each character of the cluster once. +;; - A step back reads the cluster, and, for the three rules with context +;; (GB9c, GB11, GB12/13), the run of characters the rule looks back over. +;; A run of regional indicators is counted once: from a boundary followed by +;; one, the parity is known, so a loop back over a text takes O(n) time. + +(export + grapheme-unicode-version + ;; The forward state machine + grapheme-plain-break? + grapheme-start grapheme-step grapheme-break? + ;; The backward state machine + grapheme-back-start grapheme-back-needs-after? grapheme-back-start-before grapheme-back-pair + grapheme-back-wants? grapheme-back-more grapheme-back-sot grapheme-back-break? + ;; Strings + string-cursor-next-grapheme string-cursor-prev-grapheme string-cursor-grapheme-boundary? + string-grapheme-count string->graphemes) + +(import (std prelude)) + +(:doc #:file "grapheme.bjodoc") + +(import/class + (ArgumentOutOfRangeException (: System.ArgumentOutOfRangeException)) + (Exception (: System.Exception))) + +(include "grapheme-table.protobjo") + +;; --------------------------------------------------------------------------- +;; The property table +;; --------------------------------------------------------------------------- + +(: string) +(def grapheme-unicode-version unicode-version) + +;; The value of a hex digit, as a byte of ASCII. +(: (-> byte int)) +(defun (hex-value b) + (def x (cast int b)) + (if (< x 97) (- x 48) (- x 87))) + +;; The bytes of `text` that are not white space: the generated texts are +;; indented, and every character they hold is '0' or above. +(: (-> string (Array byte))) +(defun (table-bytes text) + (def src (string->bytes text)) + (def n (loop (:for b src) (:when (>= (cast int b) 48)) (:acc (summing 1)))) + (def (: out (Array byte)) (make-array n)) + (let go ((i 0) (j 0)) + (cond + ((>= i (array-length src)) out) + ((< (cast int (array-ref src i)) 48) (go (+ i 1) j)) + (else (array-set! out j (array-ref src i)) (go (+ i 1) (+ j 1)))))) + +;; Bytes written as two hex digits each. +(: (-> string (Array byte))) +(defun (hex-bytes text) + (def digits (table-bytes text)) + (def (: out (Array byte)) (make-array (/ (array-length digits) 2))) + (loop (:for i (range 0 (array-length out))) + (:do (array-set! out i (cast byte (+ (* 16 (hex-value (array-ref digits (* 2 i)))) + (hex-value (array-ref digits (+ (* 2 i) 1)))))))) + out) + +;; The block of each 256 code points. +(: (Array byte)) +(def stage1 (hex-bytes stage1-text)) + +;; The property bytes of the blocks, one a code point. +(: (Array byte)) +(def stage2 + (begin + (def kinds (hex-bytes kinds-text)) + (def codes (table-bytes stage2-text)) + (def (: out (Array byte)) (make-array (array-length codes))) + (loop (:for i (range 0 (array-length codes))) + (:do (array-set! out i (array-ref kinds (- (cast int (array-ref codes i)) 48))))) + out)) + +;; The property byte of a character: +;; - bits 0-3, Grapheme_Cluster_Break (the gcb- constants); +;; - bit 4, Extended_Pictographic; +;; - bits 5-6, Indic_Conjunct_Break (the incb- constants). +(: (-> char int)) +(defun (props-of ch) #:inline + (def cp (char->int ch)) + (cast int (array-ref stage2 (+ (shift-left (cast int (array-ref stage1 (shift-right cp 8))) 8) + (bitwise-and cp 255))))) + +(def gcb-cr 1) +(def gcb-lf 2) +(def gcb-control 3) +(def gcb-extend 4) +(def gcb-zwj 5) +(def gcb-ri 6) +(def gcb-prepend 7) +(def gcb-spacing-mark 8) +(def gcb-l 9) +(def gcb-v 10) +(def gcb-t 11) +(def gcb-lv 12) +(def gcb-lvt 13) + +(def incb-linker 1) +(def incb-consonant 2) +(def incb-extend 3) + +(: (-> int int)) +(defun (gcb p) #:inline (bitwise-and p 15)) + +(: (-> int bool)) +(defun (pictographic? p) #:inline (not (= (bitwise-and p 16) 0))) + +(: (-> int int)) +(defun (incb p) #:inline (bitwise-and (shift-right p 5) 3)) + +;; Whether a code point is printable ASCII: GCB=Other, and none of the +;; other properties. +(: (-> int bool)) +(defun (printable-ascii? cp) #:inline (and (>= cp 32) (< cp 127))) + +;; Whether the rules put a boundary between code points `a` and `b` with no +;; context: both are printable ASCII. A front end can test bytes with it. +(: (-> int int bool)) +(defun (grapheme-plain-break? a b) #:inline (and (printable-ascii? a) (printable-ascii? b))) + +;; --------------------------------------------------------------------------- +;; The rules +;; --------------------------------------------------------------------------- + +;; What the rules say of the boundary between two characters, by their +;; properties alone. +(def rule-join 0) +(def rule-break 1) +;; GB9c: a Linker, then InCB=Extend characters, then a Consonant. +(def rule-conjunct 2) +;; GB11: a pictograph, then Extend characters and a ZWJ, then a pictograph. +(def rule-emoji 3) +;; GB12/13: an odd number of regional indicators before the second. +(def rule-flag 4) + +;; The rule for the boundary between characters of properties `pa` and `pb`: +;; join, break, or which rule with context decides it. UAX #29 3.1.1, in +;; order. +(: (-> int int int)) +(defun (pair-rule pa pb) + (def a (gcb pa)) + (def b (gcb pb)) + (cond + ;; GB3, GB4, GB5 + ((and (= a gcb-cr) (= b gcb-lf)) rule-join) + ((or (= a gcb-cr) (= a gcb-lf) (= a gcb-control)) rule-break) + ((or (= b gcb-cr) (= b gcb-lf) (= b gcb-control)) rule-break) + ;; GB6, GB7, GB8 + ((and (= a gcb-l) (or (= b gcb-l) (= b gcb-v) (= b gcb-lv) (= b gcb-lvt))) rule-join) + ((and (or (= a gcb-lv) (= a gcb-v)) (or (= b gcb-v) (= b gcb-t))) rule-join) + ((and (or (= a gcb-lvt) (= a gcb-t)) (= b gcb-t)) rule-join) + ;; GB9, GB9a, GB9b + ((or (= b gcb-extend) (= b gcb-zwj) (= b gcb-spacing-mark)) rule-join) + ((= a gcb-prepend) rule-join) + ;; GB9c, GB11, GB12/13 + ((and (= (incb pb) incb-consonant) (or (= (incb pa) incb-linker) (= (incb pa) incb-extend))) rule-conjunct) + ((and (= a gcb-zwj) (pictographic? pb)) rule-emoji) + ((and (= a gcb-ri) (= b gcb-ri)) rule-flag) + ;; GB999 + (else rule-break))) + +;; --------------------------------------------------------------------------- +;; Forward +;; --------------------------------------------------------------------------- + +;; A forward state is an int: +;; - bits 0-6, the properties of the last character; +;; - bit 7, the text ends in a pictograph and Extend characters (GB11); +;; - bit 8, it ends in those and a ZWJ (GB11); +;; - bit 9, it ends in an odd number of regional indicators (GB12/13); +;; - bit 10, it ends in a Linker and InCB=Extend characters (GB9c); +;; - bit 11, there is a boundary before the last character. +(def fwd-pict 128) +(def fwd-pict-zwj 256) +(def fwd-odd-ri 512) +(def fwd-linker 1024) +(def fwd-break 2048) + +(: (-> int int bool)) +(defun (has? st bit) #:inline (not (= (bitwise-and st bit) 0))) + +;; The state after a character of properties `p`, from the state `st` before +;; it. +(: (-> int int bool int)) +(defun (state-after p st brk) + (def g (gcb p)) + (def i (incb p)) + (bitwise-ior + p + (if (or (pictographic? p) (and (= g gcb-extend) (has? st fwd-pict))) fwd-pict 0) + (if (and (= g gcb-zwj) (has? st fwd-pict)) fwd-pict-zwj 0) + (if (and (= g gcb-ri) (not (and (= (gcb st) gcb-ri) (has? st fwd-odd-ri)))) fwd-odd-ri 0) + (if (or (= i incb-linker) (and (= i incb-extend) (has? st fwd-linker))) fwd-linker 0) + (if brk fwd-break 0))) + +;; The state after the first character of a text, or of a cluster. +(: (-> char int)) +(defun (grapheme-start ch) (state-after (props-of ch) 0 #t)) + +;; The state after `ch`, from the state after the character before it. +;; - Before printable ASCII, the rules break unless the character before is +;; Prepend (GB9b), and the state after it is that of Other. +(: (-> int char int)) +(defun (grapheme-step st ch) #:inline + (if (printable-ascii? (char->int ch)) + (if (= (gcb st) gcb-prepend) 0 fwd-break) + (grapheme-step-table st ch))) + +(: (-> int char int)) +(defun (grapheme-step-table st ch) + (def p (props-of ch)) + (def r (pair-rule (bitwise-and st 127) p)) + (state-after p st + (cond + ((= r rule-join) #f) + ((= r rule-break) #t) + ((= r rule-conjunct) (not (has? st fwd-linker))) + ((= r rule-emoji) (not (has? st fwd-pict-zwj))) + (else (not (has? st fwd-odd-ri)))))) + +;; Whether there is a boundary before the last character of a state. +(: (-> int bool)) +(defun (grapheme-break? st) #:inline (has? st fwd-break)) + +;; --------------------------------------------------------------------------- +;; Backward +;; --------------------------------------------------------------------------- + +;; A backward state is an int: +;; - bits 0-1, the answer: 0 join, 1 break, 2 not known yet; +;; - bits 2-4, while not known, the rule that is looking back; +;; - bit 5, the parity of the regional indicators that end at the first +;; character given so far is known; bit 6, it is odd; +;; - bit 7, while counting regional indicators, an odd number so far. +(def back-join 0) +(def back-break 1) +(def back-wants 2) +(def back-known 32) +(def back-odd 64) +(def back-count-odd 128) + +;; The answer `answer`, where the regional indicators that end at the first +;; character given number `odd`: #t odd, #f even, or unknown with `known` #f. +(: (-> int bool bool int)) +(defun (back-answer answer known odd) + (bitwise-ior answer (if known back-known 0) (if odd back-odd 0))) + +;; The backward state of a cursor at the end of a text, or of one where +;; nothing is known of what follows. +(: (-> int)) +(defun (grapheme-back-start) 0) + +;; Whether the backward state of a cursor needs the character after it, given +;; `b`, the character before it: only when `b` is a regional indicator, as +;; the parity of those that end at `b` matters only then. +(: (-> char bool)) +(defun (grapheme-back-needs-after? b) #:inline + (and (not (printable-ascii? (char->int b))) (= (gcb (props-of b)) gcb-ri))) + +;; The backward state of a cursor on a cluster boundary, before `ch`. +;; - A boundary before a regional indicator has an even number of them +;; before it, which saves counting them. +(: (-> char int)) +(defun (grapheme-back-start-before ch) + (if (and (not (printable-ascii? (char->int ch))) (= (gcb (props-of ch)) gcb-ri)) back-known 0)) + +;; The state for the boundary between `a` and `b`, where `b` is the first +;; character given before, or the one before the cursor. +;; - Known at once, unless a rule needs the characters before `a`: then +;; grapheme-back-wants? is #t. +;; - Between two characters of printable ASCII, a break, and no regional +;; indicators end at `a`. +(: (-> int char char int)) +(defun (grapheme-back-pair w a b) #:inline + (if (grapheme-plain-break? (char->int a) (char->int b)) + (bitwise-ior back-break back-known) + (grapheme-back-pair-table w a b))) + +(: (-> int char char int)) +(defun (grapheme-back-pair-table w a b) + (def pa (props-of a)) + (def pb (props-of b)) + (def r (pair-rule pa pb)) + (def a-ri (= (gcb pa) gcb-ri)) + ;; The parity at `a`, from that at `b`, when it follows without counting. + (def known (or (not a-ri) (and (= (gcb pb) gcb-ri) (has? w back-known)))) + (def odd (and a-ri (= (gcb pb) gcb-ri) (not (has? w back-odd)))) + (cond + ((= r rule-join) (back-answer back-join known odd)) + ((= r rule-break) (back-answer back-break known odd)) + ((and (= r rule-conjunct) (= (incb pa) incb-linker)) (back-answer back-join known odd)) + ((and (= r rule-flag) known) (back-answer (if odd back-join back-break) known odd)) + ;; Count the indicators before `a`, starting with `a`. + ((= r rule-flag) (bitwise-ior back-wants (shift-left r 2) back-count-odd)) + (else (bitwise-ior back-wants (shift-left r 2))))) + +;; Whether the state needs the character before the last one given. +(: (-> int bool)) +(defun (grapheme-back-wants? w) #:inline (= (bitwise-and w 3) back-wants)) + +;; The state after `ch`, the character before the last one given. +(: (-> int char int)) +(defun (grapheme-back-more w ch) + (def p (props-of ch)) + (def r (bitwise-and (shift-right w 2) 7)) + (cond + ((= r rule-conjunct) + (cond + ((= (incb p) incb-extend) w) + ((= (incb p) incb-linker) (back-answer back-join #t #f)) + (else (back-answer back-break #t #f)))) + ((= r rule-emoji) + (cond + ((= (gcb p) gcb-extend) w) + ((pictographic? p) (back-answer back-join #t #f)) + (else (back-answer back-break #t #f)))) + ((= (gcb p) gcb-ri) (bitwise-xor w back-count-odd)) + (else (grapheme-back-sot w)))) + +;; The state when there is no character before the last one given. +(: (-> int int)) +(defun (grapheme-back-sot w) + (def r (bitwise-and (shift-right w 2) 7)) + (if (= r rule-flag) + (let ((odd (has? w back-count-odd))) + (back-answer (if odd back-join back-break) #t odd)) + (back-answer back-break #t #f))) + +;; Whether the state, once known, is a boundary. +(: (-> int bool)) +(defun (grapheme-back-break? w) #:inline (= (bitwise-and w 3) back-break)) + +;; --------------------------------------------------------------------------- +;; Strings +;; --------------------------------------------------------------------------- + +(: (-> string %a)) +(defun (out-of-range why) #:no-inline + (raise (cast Exception (ArgumentOutOfRangeException. "c" why)))) + +;; The cursor after the cluster that starts at `c`, a cluster boundary of `s`. +(: (-> string StringCursor StringCursor)) +(defun (string-cursor-next-grapheme s c) + (when (string-cursor-end? s c) (out-of-range "string-cursor-next-grapheme: the cursor is at the end.")) + (def (Tuple ch0 c1) (string-cursor-ref+next s c)) + (let go ((st (grapheme-start ch0)) (c c1)) + (if (string-cursor-end? s c) + c + (begin + (def (Tuple ch nxt) (string-cursor-ref+next s c)) + (def st2 (grapheme-step st ch)) + (if (grapheme-break? st2) c (go st2 nxt)))))) + +;; The state `w` once known, reading back from `k`, the cursor of the last +;; character given. +(: (-> string StringCursor int StringCursor int)) +(defun (string-back-resolve s start w k) + (let more ((w w) (k k)) + (cond + ((not (grapheme-back-wants? w)) w) + ((string-cursor=? k start) (grapheme-back-sot w)) + (else + (begin + (def k2 (string-cursor-prev s k)) + (more (grapheme-back-more w (string-cursor-ref s k2)) k2)))))) + +;; The start of the cluster that ends at `c`, a cluster boundary of `s`. +(: (-> string StringCursor StringCursor)) +(defun (string-cursor-prev-grapheme s c) + (def start (string-cursor-start s)) + (when (string-cursor=? c start) (out-of-range "string-cursor-prev-grapheme: the cursor is at the start.")) + (def c1 (string-cursor-prev s c)) + (def b1 (string-cursor-ref s c1)) + (def w0 (if (and (grapheme-back-needs-after? b1) (not (string-cursor-end? s c))) + (grapheme-back-start-before (string-cursor-ref s c)) + (grapheme-back-start))) + ;; `bc` is the cursor of the earliest character of the cluster so far, `b`. + (let go ((w w0) (bc c1) (b b1)) + (if (string-cursor=? bc start) + bc + (begin + (def ac (string-cursor-prev s bc)) + (def a (string-cursor-ref s ac)) + (def w1 (grapheme-back-pair w a b)) + (def w2 (if (grapheme-back-wants? w1) (string-back-resolve s start w1 ac) w1)) + (if (grapheme-back-break? w2) bc (go w2 ac a)))))) + +;; Whether `c` is on a cluster boundary of `s`. +(: (-> string StringCursor bool)) +(defun (string-cursor-grapheme-boundary? s c) + (def start (string-cursor-start s)) + (if (or (string-cursor=? c start) (string-cursor-end? s c)) + #t + (begin + (def ac (string-cursor-prev s c)) + (grapheme-back-break? + (string-back-resolve s start (grapheme-back-pair (grapheme-back-start) (string-cursor-ref s ac) (string-cursor-ref s c)) ac))))) + +;; The clusters of `s`. +(: (-> string int)) +(defun (string-grapheme-count s) + (def c0 (string-cursor-start s)) + (if (string-cursor-end? s c0) + 0 + (begin + (def (Tuple ch0 c1) (string-cursor-ref+next s c0)) + (let go ((st (grapheme-start ch0)) (c c1) (n 1)) + (if (string-cursor-end? s c) + n + (begin + (def (Tuple ch nxt) (string-cursor-ref+next s c)) + (def st2 (grapheme-step st ch)) + (go st2 nxt (if (grapheme-break? st2) (+ n 1) n)))))))) + +;; The clusters of `s`, in order. +(: (-> string (Vec string))) +(defun (string->graphemes s) + (let go ((c (string-cursor-start s)) (acc '())) + (if (string-cursor-end? s c) + (list->vec (list-reverse acc)) + (begin + (def d (string-cursor-next-grapheme s c)) + (go d (cons (substring s c d) acc)))))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/units.bjo — units of measure: unit tables, the unit expression parser, +;; and measurements at run time. + +(import (std prelude) (std maths) (std rational) (std orderedmap)) + +(:doc #:file "units.bjodoc") + +(export + ;; Factors + Factor factor factor-rational factor-pi factor-one factor-one? + factor* factor/ factor-expt factor->double + ;; Atoms and resolved unit expressions + Atom atom atom-name atom-dims atom-factor + Resolved resolved-coeff resolved-term resolved-atoms + resolved-dims resolved-factor + ;; Tables + UnitTable PrefixUse + empty-unit-table def-dimension def-unit def-prefix def-preferred + unit-table-names standard-units + ;; Resolving + resolve-unit conformable-units + ;; Measurements + Measurement measurement measurement-value measurement-dims + string->measurement measurement+ measurement- measurement* measurement/ + measurement-scale measurement-expt measurement-sqrt conformable? + measurement-in measurement->string measurement->string/mixed + convert-query dims->string + ;; Temperatures + Celsius Fahrenheit celsius->kelvin kelvin->celsius + fahrenheit->kelvin kelvin->fahrenheit + ;; Logarithmic + dBW dBm dBV dBSPL dBW->power dBm->power dBV->voltage dBSPL->pressure) + +(import/class + (Exception (: System.Exception)) + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +(import/extern + (clr-invariant (: System.Globalization.CultureInfo.InvariantCulture #:get))) + +(: (-> string %a)) +(defun (argument-error what) + (raise (cast Exception (ArgumentException. (str "(std units): " what))))) + +;; A Result's value, or ArgumentException with its message. +(: (-> (Result string %a) %a)) +(defun (ok-or-raise r) + (match r + ((Ok v) v) + ((Err e) (argument-error e)))) + +(: (-> int bigint)) +(defun (int->bigint n) (cast System.Numerics.BigInteger n)) + +(def rational-one (rational 1 1)) + +(: (-> (Tuple %a %b) %a)) +(defun (fst t) (match t ((Tuple a _) a))) + +(: (-> (Tuple %a %b) %b)) +(defun (snd t) (match t ((Tuple _ b) b))) + +;; --------------------------------------------------------------------------- +;; Factors +;; --------------------------------------------------------------------------- + +;; rat × pi^pi-power. +(type (: Factor #:opaque (Struct (: rat Rational) (: pi-power int)))) + +(: (-> Rational int Factor)) +(defun (factor r k) (Factor (rat r) (pi-power k))) + +(: (-> Factor Rational)) +(defun (factor-rational f) (struct-ref f rat)) + +(: (-> Factor int)) +(defun (factor-pi f) (struct-ref f pi-power)) + +(: factor-one Factor) +(def factor-one (factor rational-one 0)) + +(: (-> Factor bool)) +(defun (factor-one? f) (= f factor-one)) + +(: (-> Factor Factor Factor)) +(defun (factor* a b) + (factor (rational* (factor-rational a) (factor-rational b)) (+ (factor-pi a) (factor-pi b)))) + +(: (-> Factor Factor Factor)) +(defun (factor/ a b) + (factor (rational/ (factor-rational a) (factor-rational b)) (- (factor-pi a) (factor-pi b)))) + +(: (-> Factor double)) +(defun (factor->double f) + (* (rational->double (factor-rational f)) (pow pi (cast System.Double (factor-pi f))))) + +(: (-> bigint int)) +(defun (bit-count n) + (let go ((n n) (k 0)) + (if (zero? n) k (go (shift-right n 1) (+ k 1))))) + +;; The exact b-th root of n >= 0, or None. +(: (-> bigint int (Option bigint))) +(defun (exact-root n b) + (if (or (< n 2I) (= b 1)) + (Some n) + (let* ((bb (int->bigint b)) + (start (shift-left 1I (+ (/ (bit-count n) b) 1))) + (r (let go ((x start)) + (def y (/ (+ (* (- bb 1I) x) (/ n (expt x (- b 1)))) bb)) + (if (>= y x) x (go y))))) + (if (= (expt r b) n) (Some r) None)))) + +;; f^q, or None when the result is irrational, or q is negative and f zero. +(: (-> Factor Rational (Option Factor))) +(defun (factor-expt f q) + (def p (cast System.Int32 (rational-num q))) + (def s (cast System.Int32 (rational-den q))) + (def k (* (factor-pi f) p)) + (def n (rational-num (factor-rational f))) + (def d (rational-den (factor-rational f))) + (cond + ((not (= 0 (% k s))) None) + ((and (zero? n) (< p 0)) None) + ((and (< n 0I) (even? s)) None) + (else + (match (Tuple (exact-root (abs n) s) (exact-root d s)) + ((Tuple (Some a) (Some b)) + (Some (factor (rational-expt (rational (if (< n 0I) (- 0I a) a) b) p) (/ k s)))) + (_ None))))) + +;; --------------------------------------------------------------------------- +;; Exponent maps: dimensions and unit terms +;; --------------------------------------------------------------------------- +;; A map from a name to a nonzero rational exponent. Empty is 1. + +(: (-> (OrderedMap string Rational) (OrderedMap string Rational) (OrderedMap string Rational))) +(defun (exps* a b) + (orderedmap-filter (fun (k v) (not (rational-zero? v))) + (orderedmap-merge-with (fun (k x y) (rational+ x y)) a b))) + +(: (-> (OrderedMap string Rational) Rational (OrderedMap string Rational))) +(defun (exps-scale m q) + (if (rational-zero? q) + (orderedmap-empty) + (orderedmap-map-values (fun (v) (rational* v q)) m))) + +(: (-> (OrderedMap string Rational) (OrderedMap string Rational))) +(defun (exps-invert m) (exps-scale m (rational -1 1))) + +(: (-> string Rational string)) +(defun (exp-part name q) + (cond + ((= q rational-one) name) + ((rational-integer? q) (str name "^" (rational->string q))) + (else (str name "^(" (rational->string q) ")")))) + +;; "kg m^2/(s^3 A)", "1/s", "1" for the empty map. +(: (-> (OrderedMap string Rational) string)) +(defun (exps->string m) + (def entries (orderedmap->list m)) + (def pos (list-filter (fun (e) (> (rational-sign (snd e)) 0)) entries)) + (def neg (list-filter (fun (e) (< (rational-sign (snd e)) 0)) entries)) + (: (-> (List (Tuple string Rational)) bool string)) + (defun (parts es negate?) + (string-join (list->vec (list-map (fun (e) (exp-part (fst e) (if negate? (rational-negate (snd e)) (snd e)))) es)) + " ")) + (def numer (if (list-empty? pos) "1" (parts pos #f))) + (cond + ((list-empty? neg) numer) + ((= (list-length neg) 1) (str numer "/" (parts neg #t))) + (else (str numer "/(" (parts neg #t) ")")))) + +;; A dimension as text: "length/time", "mass/(length time^2)", "dimensionless". +(: (-> (OrderedMap string Rational) string)) +(defun (dims->string d) + (if (orderedmap-empty? d) "dimensionless" (exps->string d))) + +;; --------------------------------------------------------------------------- +;; Atoms and resolved expressions +;; --------------------------------------------------------------------------- + +;; A named unit with its dimension and its factor to the coherent unit of +;; that dimension. +(type (: Atom #:opaque (Struct (: name string) + (: dims (OrderedMap string Rational)) + (: factor Factor)))) + +(: (-> string (OrderedMap string Rational) Factor Atom)) +(defun (atom name dims f) (Atom (name name) (dims dims) (factor f))) + +(: (-> Atom string)) +(defun (atom-name a) (struct-ref a name)) + +(: (-> Atom (OrderedMap string Rational))) +(defun (atom-dims a) (struct-ref a dims)) + +(: (-> Atom Factor)) +(defun (atom-factor a) (struct-ref a factor)) + +;; coeff × the product of atoms to their exponents. atoms holds every atom +;; the term names. +(type (: Resolved #:opaque (Struct (: coeff Factor) + (: term (OrderedMap string Rational)) + (: atoms (OrderedMap string Atom))))) + +(: (-> Resolved Factor)) +(defun (resolved-coeff r) (struct-ref r coeff)) + +(: (-> Resolved (OrderedMap string Rational))) +(defun (resolved-term r) (struct-ref r term)) + +(: (-> Resolved (OrderedMap string Atom))) +(defun (resolved-atoms r) (struct-ref r atoms)) + +(: (-> Factor Resolved)) +(defun (resolved-number f) (Resolved (coeff f) (term (orderedmap-empty)) (atoms (orderedmap-empty)))) + +(: (-> Atom Resolved)) +(defun (resolved-atom a) + (Resolved (coeff factor-one) + (term (orderedmap-set (orderedmap-empty) (atom-name a) rational-one)) + (atoms (orderedmap-set (orderedmap-empty) (atom-name a) a)))) + +(: (-> Resolved Resolved Resolved)) +(defun (resolved* a b) + (Resolved (coeff (factor* (resolved-coeff a) (resolved-coeff b))) + (term (exps* (resolved-term a) (resolved-term b))) + (atoms (orderedmap-merge-with (fun (k x y) x) (resolved-atoms a) (resolved-atoms b))))) + +;; a/b, or None when b's coefficient is zero. +(: (-> Resolved Resolved (Option Resolved))) +(defun (resolved/ a b) + (option-map (fun (inv) + (resolved* a (Resolved (coeff inv) (term (exps-invert (resolved-term b))) (atoms (resolved-atoms b))))) + (factor-expt (resolved-coeff b) (rational -1 1)))) + +;; r^q, or None when the coefficient's power is irrational. +(: (-> Resolved Rational (Option Resolved))) +(defun (resolved-expt r q) + (option-map (fun (c) (Resolved (coeff c) (term (exps-scale (resolved-term r) q)) (atoms (resolved-atoms r)))) + (factor-expt (resolved-coeff r) q))) + +(: (-> Resolved (OrderedMap string Rational))) +(defun (resolved-dims r) + (orderedmap-fold (fun (name q acc) (exps* acc (exps-scale (atom-dims (orderedmap-ref (resolved-atoms r) name)) q))) + (orderedmap-empty) + (resolved-term r))) + +;; The exact factor from r to the coherent unit of its dimension. +(: (-> Resolved (Result string Factor))) +(defun (resolved-factor r) + (orderedmap-fold + (fun (name q acc) + (match acc + ((Ok f) + (match (factor-expt (atom-factor (orderedmap-ref (resolved-atoms r) name)) q) + ((Some g) (Ok (factor* f g))) + (None (Err (str "the factor of " (exp-part name q) " is irrational"))))) + (e e))) + (Ok (resolved-coeff r)) + (resolved-term r))) + +;; The factor as a double, rational powers included. +(: (-> Resolved double)) +(defun (resolved-double r) + (orderedmap-fold (fun (name q acc) + (* acc (pow (factor->double (atom-factor (orderedmap-ref (resolved-atoms r) name))) + (rational->double q)))) + (factor->double (resolved-coeff r)) + (resolved-term r))) + +;; --------------------------------------------------------------------------- +;; Tables +;; --------------------------------------------------------------------------- + +;; Which prefixes printing may put on a preferred unit. +(type (: PrefixUse (Union NoPrefix SmallPrefixes AllPrefixes))) + +(type (: UnitDef (Union (: UnitAtom Atom) (: UnitAlias Resolved)))) + +(type (: UnitTable #:opaque + (Record (: dimensions (OrderedMap string string)) + (: units (OrderedMap string UnitDef)) + (: prefixes (OrderedMap string Rational)) + (: preferred (OrderedMap string (Tuple string PrefixUse)))))) + +(: empty-unit-table UnitTable) +(def empty-unit-table + (UnitTable (dimensions (orderedmap-empty)) (units (orderedmap-empty)) + (prefixes (orderedmap-empty)) (preferred (orderedmap-empty)))) + +(: (-> UnitTable (List string))) +(defun (unit-table-names t) + (orderedmap-fold (fun (k v acc) (Cons k acc)) Nil (record-ref t units))) + +(: (-> char bool)) +(defun (name-start? c) + (or (char-alphabetic? c) (char=? c #\_) (char=? c #\%))) + +(: (-> char bool)) +(defun (name-char? c) + (or (name-start? c) (char-numeric? c))) + +(: (-> string bool)) +(defun (valid-name? s) + (match (string->list s) + ((Cons c rest) (and (name-start? c) (list-all? name-char? rest) (not (= s "pi")))) + (Nil #f))) + +(: (-> (-> char bool) (List char) bool)) +(defun (list-all? ok? cs) + (match cs + (Nil #t) + ((Cons c rest) (and (ok? c) (list-all? ok? rest))))) + +(: (-> UnitTable string (Result string UnitTable))) +(defun (check-new-name t name) + (cond + ((not (valid-name? name)) (Err (str "'" name "' is not a valid unit name"))) + ((orderedmap-has-key? (record-ref t units) name) (Err (str "the unit '" name "' is already defined"))) + (else (Ok t)))) + +;; A base dimension and its coherent unit: (def-dimension t "length" "m"). +(: (-> UnitTable string string (Result string UnitTable))) +(defun (def-dimension t dim unit) + (def (Ok _) (check-new-name t unit) :propagate) + (if (orderedmap-has-key? (record-ref t dimensions) dim) + (Err (str "the dimension '" dim "' is already defined")) + (Ok (record-set t + (dimensions (orderedmap-set (record-ref t dimensions) dim unit)) + (units (orderedmap-set (record-ref t units) unit + (UnitAtom (atom unit (orderedmap-set (orderedmap-empty) dim rational-one) factor-one)))))))) + +;; A unit defined by a unit expression: (def-unit t "ft" "12 in"). +;; A definition with coefficient 1 makes an alias: "N" is "kg m/s^2". Any +;; other makes a unit of its own. +(: (-> UnitTable string string (Result string UnitTable))) +(defun (def-unit t name definition) + (def (Ok _) (check-new-name t name) :propagate) + (def (Ok r) (result-map-err (fun (e) (str name ": " e)) (resolve-unit t definition)) :propagate) + (def (Ok f) (result-map-err (fun (e) (str name ": " e)) (resolved-factor r)) :propagate) + (def entry + (if (factor-one? (resolved-coeff r)) + (UnitAlias r) + (UnitAtom (atom name (resolved-dims r) f)))) + (Ok (record-set t (units (orderedmap-set (record-ref t units) name entry))))) + +;; A prefix and its value: (def-prefix t "k" "1000"), (def-prefix t "m" "1|1000"). +(: (-> UnitTable string string (Result string UnitTable))) +(defun (def-prefix t name value) + (match (string->rational value) + ((Some q) #:when (valid-name? name) + (Ok (record-set t (prefixes (orderedmap-set (record-ref t prefixes) name q))))) + ((Some q) (Err (str "'" name "' is not a valid prefix"))) + (None (Err (str "the value of the prefix '" name "' is not a number: " value))))) + +;; The unit printing uses for its dimension, and which prefixes it may take. +(: (-> UnitTable string PrefixUse (Result string UnitTable))) +(defun (def-preferred t unit use) + (def (Ok r) (resolve-unit t unit) :propagate) + (Ok (record-set t (preferred (orderedmap-set (record-ref t preferred) + (exps->string (resolved-dims r)) + (Tuple unit use)))))) + +;; --------------------------------------------------------------------------- +;; Lookup +;; --------------------------------------------------------------------------- + +(: (-> UnitDef Resolved)) +(defun (def->resolved d) + (match d + ((UnitAtom a) (resolved-atom a)) + ((UnitAlias r) r))) + +;; name as base and a trailing power: "m2" is (Tuple "m" 2). +(: (-> string (Option (Tuple string int)))) +(defun (split-power name) + (def cs (list-reverse (string->list name))) + (let go ((cs cs) (digits Nil)) + (match cs + ((Cons c rest) #:when (char-numeric? c) (go rest (Cons c digits))) + ((Cons c rest) #:when (not (list-empty? digits)) + (Some (Tuple (list->string (list-reverse cs)) (string->int (list->string digits))))) + (_ None)))) + +;; The longest prefix p with name = p ++ rest, rest a unit of the table. +(: (-> UnitTable string (Option (Tuple Rational string)))) +(defun (find-prefix t name) + (orderedmap-fold + (fun (p q best) + (if (and (string-starts-with? name p) (< (string-length p) (string-length name))) + (let ((rest (string-drop name (string-length p)))) + (if (and (orderedmap-has-key? (record-ref t units) rest) + (match best + ((Some (Tuple _ r)) (< (string-length rest) (string-length r))) + (None #t))) + (Some (Tuple q rest)) + best)) + best)) + None + (record-ref t prefixes))) + +;; A unit by name: the exact name, then a trailing power ("m2"), then a +;; prefix on a unit of the table ("km"). A prefixed unit is a new atom. +(: (-> UnitTable string (Option Resolved))) +(defun (lookup t name) + (match (orderedmap-try-ref (record-ref t units) name) + ((Some d) (Some (def->resolved d))) + (None + (match (split-power name) + ((Some (Tuple base n)) #:when (> n 0) + (match (lookup t base) + ((Some r) (resolved-expt r (int->rational n))) + (None None))) + (_ + (match (find-prefix t name) + ((Some (Tuple q rest)) + (def r (def->resolved (orderedmap-ref (record-ref t units) rest))) + (match (resolved-factor r) + ((Ok f) (Some (resolved-atom (atom name (resolved-dims r) (factor* (factor q 0) f))))) + ((Err _) None))) + (None None))))))) + +(: (-> char char int)) +(defun (char-cost a b) (if (char=? a b) 0 1)) + +;; Levenshtein distance. +(: (-> string string int)) +(defun (edit-distance a b) + (def bcs (string->list b)) + (def first-row (let go ((j (list-length bcs)) (acc Nil)) + (if (< j 0) acc (go (- j 1) (Cons j acc))))) + (: (-> char (List int) (List int))) + (defun (next-row ca prev) + (match prev + ((Cons p0 _) + (let go ((bs bcs) (prev prev) (left (+ p0 1)) (acc (Cons (+ p0 1) Nil))) + (match (Tuple bs prev) + ((Tuple (Cons cb more) (Cons diag (Cons up rest))) + (def v (min (+ up 1) (min (+ left 1) (+ diag (char-cost ca cb))))) + (go more (Cons up rest) v (Cons v acc))) + (_ (list-reverse acc))))) + (Nil Nil))) + (def last-row (list-foldl next-row first-row (string->list a))) + (match (list-reverse last-row) + ((Cons d _) d) + (Nil 0))) + +(: (-> UnitTable string string)) +(defun (unknown-unit t name) + (def limit (max 1 (/ (string-length name) 3))) + (def best + (list-foldl (fun (candidate best) + (def d (edit-distance name candidate)) + (match best + ((Some (Tuple bd _)) #:when (<= bd d) best) + (_ (if (<= d limit) (Some (Tuple d candidate)) best)))) + None + (list-reverse (unit-table-names t)))) + (match best + ((Some (Tuple _ c)) (str "unknown unit '" name "'. Did you mean '" c "'?")) + (None (str "unknown unit '" name "'")))) + +;; --------------------------------------------------------------------------- +;; The unit expression grammar +;; --------------------------------------------------------------------------- +;; Tightest first: +;; a|b a number divided by a number, inside the number token +;; ^ a power with a rational exponent: m^2, Hz^(-1/2), s^-1 +;; juxtaposition, a product: kg m +;; * / left to right: kg m/s^2 is kg·m/s², m/s kg is m/(s·kg) +;; + - measurements only +;; A trailing number on a name is a power: m2. pi is the number. + +(type (: Token (Union (: TNum Rational) (: TName string) (: TOp char)))) + +(type (: Expr (Union (: ENum Rational) + EPi + (: EUnit string) + (: EMul Expr Expr) + (: EDiv Expr Expr) + (: EPow Expr Rational) + (: ENeg Expr) + (: EAdd Expr Expr) + (: ESub Expr Expr) + (: ECall string Expr)))) + +(: (-> char bool)) +(defun (number-char? c) (or (char-numeric? c) (char=? c #\.))) + +;; Chars of a number from the front of cs: digits and points, an exponent +;; when digits follow the e, and a |-quotient. +(: (-> (List char) (Tuple (List char) (List char)))) +(defun (take-number cs) + (: (-> (List char) (List char) (Tuple (List char) (List char)))) + (defun (mantissa cs acc) + (match cs + ((Cons c rest) #:when (number-char? c) (mantissa rest (Cons c acc))) + ((Cons e rest) #:when (or (char=? e #\e) (char=? e #\E)) + (match rest + ((Cons s (Cons d more)) #:when (and (or (char=? s #\-) (char=? s #\+)) (char-numeric? d)) + (digits more (Cons d (Cons s (Cons e acc))))) + ((Cons d more) #:when (char-numeric? d) (digits more (Cons d (Cons e acc)))) + (_ (Tuple acc cs)))) + (_ (Tuple acc cs)))) + (: (-> (List char) (List char) (Tuple (List char) (List char)))) + (defun (digits cs acc) + (match cs + ((Cons c rest) #:when (char-numeric? c) (digits rest (Cons c acc))) + (_ (Tuple acc cs)))) + (def (acc rest) (mantissa cs Nil)) + (match rest + ((Cons bar (Cons c more)) #:when (and (char=? bar #\|) (number-char? c)) + (def (acc2 rest2) (mantissa (Cons c more) (Cons bar acc))) + (Tuple (list-reverse acc2) rest2)) + (_ (Tuple (list-reverse acc) rest)))) + +(: (-> (List char) (Tuple (List char) (List char)))) +(defun (take-name cs) + (let go ((cs cs) (acc Nil)) + (match cs + ((Cons c rest) #:when (name-char? c) (go rest (Cons c acc))) + (_ (Tuple (list-reverse acc) cs))))) + +(: (-> string (Result string (List Token)))) +(defun (tokenize text) + (let go ((cs (string->list text)) (acc Nil)) + (match cs + (Nil (Ok (list-reverse acc))) + ((Cons c rest) #:when (char-whitespace? c) (go rest acc)) + ((Cons #\* (Cons #\* rest)) (go rest (Cons (TOp #\^) acc))) + ((Cons c rest) #:when (number-char? c) + (def (chars more) (take-number cs)) + (match (string->rational (list->string chars)) + ((Some q) (go more (Cons (TNum q) acc))) + (None (Err (str "'" (list->string chars) "' is not a number"))))) + ((Cons c rest) #:when (name-start? c) + (def (chars more) (take-name cs)) + (go more (Cons (TName (list->string chars)) acc))) + ((Cons c rest) #:when (string-contains? "*/^+-()" (char->string c)) + (go rest (Cons (TOp c) acc))) + ((Cons c rest) (Err (str "unexpected '" (char->string c) "'")))))) + +(: (-> (List Token) bool)) +(defun (starts-factor? ts) + (match ts + ((Cons (TNum _) _) #t) + ((Cons (TName _) _) #t) + ((Cons (TOp #\() _) #t) + (_ #f))) + +(: (-> (List Token) string)) +(defun (describe-next ts) + (match ts + (Nil "the end") + ((Cons (TNum q) _) (str "'" (rational->string q) "'")) + ((Cons (TName n) _) (str "'" n "'")) + ((Cons (TOp c) _) (str "'" (char->string c) "'")))) + +(: (-> (List Token) (Result string (Tuple Expr (List Token))))) +(defun (parse-sum ts) + (def (Ok (Tuple first rest)) (parse-product ts) :propagate) + (let go ((e first) (ts rest)) + (match ts + ((Cons (TOp #\+) more) + (def (Ok (Tuple r after)) (parse-product more) :propagate) + (go (EAdd e r) after)) + ((Cons (TOp #\-) more) + (def (Ok (Tuple r after)) (parse-product more) :propagate) + (go (ESub e r) after)) + (_ (Ok (Tuple e ts)))))) + +(: (-> (List Token) (Result string (Tuple Expr (List Token))))) +(defun (parse-product ts) + (def (Ok (Tuple first rest)) (parse-juxt ts) :propagate) + (let go ((e first) (ts rest)) + (match ts + ((Cons (TOp #\*) more) + (def (Ok (Tuple r after)) (parse-juxt more) :propagate) + (go (EMul e r) after)) + ((Cons (TOp #\/) more) + (def (Ok (Tuple r after)) (parse-juxt more) :propagate) + (go (EDiv e r) after)) + (_ (Ok (Tuple e ts)))))) + +(: (-> (List Token) (Result string (Tuple Expr (List Token))))) +(defun (parse-juxt ts) + (def (Ok (Tuple first rest)) (parse-unary ts) :propagate) + (let go ((e first) (ts rest)) + (if (starts-factor? ts) + (let () + (def (Ok (Tuple r after)) (parse-power ts) :propagate) + (go (EMul e r) after)) + (Ok (Tuple e ts))))) + +(: (-> (List Token) (Result string (Tuple Expr (List Token))))) +(defun (parse-unary ts) + (match ts + ((Cons (TOp #\-) more) + (def (Ok (Tuple e after)) (parse-unary more) :propagate) + (Ok (Tuple (ENeg e) after))) + (_ (parse-power ts)))) + +(: (-> (List Token) (Result string (Tuple Expr (List Token))))) +(defun (parse-power ts) + (def (Ok (Tuple base rest)) (parse-atom ts) :propagate) + (match rest + ((Cons (TOp #\^) more) + (def (Ok (Tuple q after)) (parse-exponent more) :propagate) + (Ok (Tuple (EPow base q) after))) + (_ (Ok (Tuple base rest))))) + +(: (-> (List Token) (Result string (Tuple Rational (List Token))))) +(defun (parse-signed ts) + (match ts + ((Cons (TOp #\-) (Cons (TNum q) rest)) (Ok (Tuple (rational-negate q) rest))) + ((Cons (TOp #\+) (Cons (TNum q) rest)) (Ok (Tuple q rest))) + ((Cons (TNum q) rest) (Ok (Tuple q rest))) + (_ (Err (str "expected an exponent, got " (describe-next ts)))))) + +;; 2, -1, 0.5, (1/2), (-1|2). +(: (-> (List Token) (Result string (Tuple Rational (List Token))))) +(defun (parse-exponent ts) + (match ts + ((Cons (TOp #\() more) + (def (Ok (Tuple q rest)) (parse-signed more) :propagate) + (match rest + ((Cons (TOp #\)) after) (Ok (Tuple q after))) + ((Cons (TOp #\/) (Cons (TNum d) (Cons (TOp #\)) after))) #:when (not (rational-zero? d)) + (Ok (Tuple (rational/ q d) after))) + (_ (Err (str "expected ')' in an exponent, got " (describe-next rest)))))) + (_ (parse-signed ts)))) + +(: (-> (List Token) (Result string (Tuple Expr (List Token))))) +(defun (parse-atom ts) + (match ts + ((Cons (TNum q) rest) (Ok (Tuple (ENum q) rest))) + ((Cons (TName "pi") rest) (Ok (Tuple EPi rest))) + ((Cons (TName f) (Cons (TOp #\() rest)) #:when (or (= f "sqrt") (= f "cbrt")) + (def (Ok (Tuple e after)) (parse-sum rest) :propagate) + (match after + ((Cons (TOp #\)) more) (Ok (Tuple (ECall f e) more))) + (_ (Err (str "expected ')', got " (describe-next after)))))) + ((Cons (TName n) rest) (Ok (Tuple (EUnit n) rest))) + ((Cons (TOp #\() rest) + (def (Ok (Tuple e after)) (parse-sum rest) :propagate) + (match after + ((Cons (TOp #\)) more) (Ok (Tuple e more))) + (_ (Err (str "expected ')', got " (describe-next after)))))) + (_ (Err (str "expected a number or a unit, got " (describe-next ts)))))) + +(: (-> string (Result string Expr))) +(defun (parse-expression text) + (def (Ok ts) (tokenize text) :propagate) + (def (Ok (Tuple e rest)) (parse-sum ts) :propagate) + (match rest + (Nil (Ok e)) + (_ (Err (str "unexpected " (describe-next rest)))))) + +;; --------------------------------------------------------------------------- +;; Resolving: exact +;; --------------------------------------------------------------------------- + +(: (-> UnitTable Expr (Result string Resolved))) +(defun (resolve-expr t e) + (match e + ((ENum q) (Ok (resolved-number (factor q 0)))) + (EPi (Ok (resolved-number (factor rational-one 1)))) + ((EUnit n) + (match (lookup t n) + ((Some r) (Ok r)) + (None (Err (unknown-unit t n))))) + ((EMul a b) + (def (Ok x) (resolve-expr t a) :propagate) + (def (Ok y) (resolve-expr t b) :propagate) + (Ok (resolved* x y))) + ((EDiv a b) + (def (Ok x) (resolve-expr t a) :propagate) + (def (Ok y) (resolve-expr t b) :propagate) + (option->result (resolved/ x y) "division by zero")) + ((EPow a q) + (def (Ok x) (resolve-expr t a) :propagate) + (match (resolved-expt x q) + ((Some r) (Ok r)) + (None (Err (str "a power " (exp-part "x" q) " with an irrational factor"))))) + ((ENeg a) + (def (Ok x) (resolve-expr t a) :propagate) + (Ok (resolved* (resolved-number (factor (rational -1 1) 0)) x))) + ((EAdd _ _) (Err "+ is not allowed in a unit expression")) + ((ESub _ _) (Err "- is not allowed in a unit expression")) + ((ECall f _) (Err (str f " is not allowed in a unit expression"))))) + +;; A unit expression against a table: "km/h", "1|60 deg", "kg m^2/s^2". +(: (-> UnitTable string (Result string Resolved))) +(defun (resolve-unit t text) + (def (Ok e) (parse-expression text) :propagate) + (resolve-expr t e)) + +;; The names in the table whose dimension is that of text: GNU's ?. +(: (-> UnitTable string (Result string (List string)))) +(defun (conformable-units t text) + (def (Ok r) (resolve-unit t text) :propagate) + (def d (resolved-dims r)) + (Ok (list-filter (fun (n) (match (lookup t n) + ((Some u) (= (resolved-dims u) d)) + (None #f))) + (list-reverse (unit-table-names t))))) + +;; --------------------------------------------------------------------------- +;; The declarations: the standard table, for types and literals +;; --------------------------------------------------------------------------- + +;; The same table as `standard-definitions` below, which the run-time +;; functions use. A change to one is a change to the other. + +(def/dimension length #:base m) +(def/dimension mass #:base kg) +(def/dimension time #:base s) +(def/dimension current #:base A) +(def/dimension temperature #:base K) +(def/dimension amount #:base mol) +(def/dimension luminosity #:base cd) +(def/dimension information #:base b #:carrier long) + +(def/prefix Q "1e30") +(def/prefix R "1e27") +(def/prefix Y "1e24") +(def/prefix Z "1e21") +(def/prefix E "1e18") +(def/prefix P "1e15") +(def/prefix T "1e12") +(def/prefix G "1e9") +(def/prefix M "1e6") +(def/prefix k "1e3") +(def/prefix h "1e2") +(def/prefix da "1e1") +(def/prefix d "1e-1") +(def/prefix c "1e-2") +(def/prefix m "1e-3") +(def/prefix u "1e-6") +(def/prefix µ "1e-6") +(def/prefix n "1e-9") +(def/prefix p "1e-12") +(def/prefix f "1e-15") +(def/prefix a "1e-18") +(def/prefix z "1e-21") +(def/prefix y "1e-24") +(def/prefix r "1e-27") +(def/prefix q "1e-30") +(def/prefix Ki "1024") +(def/prefix ki "1024") +(def/prefix Mi "1048576") +(def/prefix Gi "1073741824") +(def/prefix Ti "1099511627776") +(def/prefix Pi "1125899906842624") +(def/prefix Ei "1152921504606846976") +(def/prefix Zi "1180591620717411303424") +(def/prefix Yi "1208925819614629174706176") + +(def/unit meter "1 m") +(def/unit meters "1 m") +(def/unit metre "1 m") +(def/unit metres "1 m") +(def/unit sec "1 s") +(def/unit second "1 s") +(def/unit seconds "1 s") +(def/unit bit "1 b") +(def/unit bits "1 b") +(def/unit amp "1 A") +(def/unit Amp "1 A") +(def/unit ampere "1 A") +(def/unit c "299792458 m/s") +(def/unit gn "9.80665 m/s^2") +(def/unit min "60 s") +(def/unit minute "1 min") +(def/unit minutes "1 min") +(def/unit h "60 min") +(def/unit hr "1 h") +(def/unit hour "1 h") +(def/unit hours "1 h") +(def/unit d "24 h") +(def/unit day "1 d") +(def/unit days "1 d") +(def/unit wk "7 d") +(def/unit week "1 wk") +(def/unit weeks "1 wk") +(def/unit fortnight "14 d") +(def/unit fortnights "1 fortnight") +(def/unit yr "31557600 s") +(def/unit year "1 yr") +(def/unit years "1 yr") +(def/unit in "0.0254 m") +(def/unit inch "1 in") +(def/unit inches "1 in") +(def/unit ft "12 in") +(def/unit foot "1 ft") +(def/unit feet "1 ft") +(def/unit yd "3 ft") +(def/unit yard "1 yd") +(def/unit yards "1 yd") +(def/unit mi "5280 ft") +(def/unit mile "1 mi") +(def/unit miles "1 mi") +(def/unit nmi "1852 m") +(def/unit fath "6 ft") +(def/unit fathom "1 fath") +(def/unit fathoms "1 fath") +(def/unit fur "660 ft") +(def/unit furlong "1 fur") +(def/unit furlongs "1 fur") +(def/unit au "149597870700 m") +(def/unit ly "c yr") +(def/unit lightyear "1 ly") +(def/unit lightyears "1 ly") +(def/unit pc "648000 au/pi") +(def/unit parsec "1 pc") +(def/unit parsecs "1 pc") +(def/unit ha "10000 m2") +(def/unit hectare "1 ha") +(def/unit hectares "1 ha") +(def/unit are "100 m2") +(def/unit ares "1 are") +(def/unit acre "4046.8564224 m2") +(def/unit acres "1 acre") +(def/unit l "0.001 m3") +(def/unit L "1 l") +(def/unit litre "1 l") +(def/unit litres "1 l") +(def/unit liter "1 l") +(def/unit liters "1 l") +(def/unit gal "3.785411784 l") +(def/unit gallon "1 gal") +(def/unit gallons "1 gal") +(def/unit qt "1|4 gal") +(def/unit quart "1 qt") +(def/unit quarts "1 qt") +(def/unit pt "1|2 qt") +(def/unit pint "1 pt") +(def/unit pints "1 pt") +(def/unit fl_oz "1|16 pt") +(def/unit fluid_ounce "1 fl_oz") +(def/unit fluid_ounces "1 fl_oz") +(def/unit fl_dr "1|8 fl_oz") +(def/unit gi "4 fl_oz") +(def/unit tsp "5 ml") +(def/unit teaspoon "1 tsp") +(def/unit teaspoons "1 tsp") +(def/unit tbsp "15 ml") +(def/unit tablespoon "1 tbsp") +(def/unit tablespoons "1 tbsp") +(def/unit cup "250 ml") +(def/unit cups "1 cup") +(def/unit us_tsp "4.92892159375 ml") +(def/unit us_tbsp "3 us_tsp") +(def/unit us_cup "16 us_tbsp") +(def/unit us_cups "1 us_cup") +(def/unit g "0.001 kg") +(def/unit gram "1 g") +(def/unit grams "1 g") +(def/unit t "1000 kg") +(def/unit ton "1 t") +(def/unit tons "1 t") +(def/unit tonne "1 t") +(def/unit tonnes "1 t") +(def/unit u "1.66053906660e-27 kg") +(def/unit amu "1 u") +(def/unit slug "14.5939029 kg") +(def/unit lb "0.45359237 kg") +(def/unit pound "1 lb") +(def/unit pounds "1 lb") +(def/unit oz "1|16 lb") +(def/unit ounce "1 oz") +(def/unit ounces "1 oz") +(def/unit gr "1|7000 lb") +(def/unit grain "1 gr") +(def/unit grains "1 gr") +(def/unit st "14 lb") +(def/unit stone "1 st") +(def/unit short_ton "2000 lb") +(def/unit short_tons "1 short_ton") +(def/unit long_ton "2240 lb") +(def/unit long_tons "1 long_ton") +(def/unit N "kg m/s^2") +(def/unit newton "1 N") +(def/unit newtons "1 N") +(def/unit lbf "lb gn") +(def/unit kgf "kg gn") +(def/unit dyn "1e-5 N") +(def/unit Pa "N/m^2") +(def/unit pascal "1 Pa") +(def/unit bar "100000 Pa") +(def/unit atm "101325 Pa") +(def/unit atmosphere "1 atm") +(def/unit atmospheres "1 atm") +(def/unit psi "lbf/in^2") +(def/unit mHg "133322.387415 Pa") +(def/unit mhg "1 mHg") +(def/unit mH2O "9806.65 Pa") +(def/unit mh2o "1 mH2O") +(def/unit J "N m") +(def/unit joule "1 J") +(def/unit joules "1 J") +(def/unit Nm "1 J") +(def/unit W "J/s") +(def/unit watt "1 W") +(def/unit watts "1 W") +(def/unit Watt "1 W") +(def/unit Watts "1 W") +(def/unit Wh "W h") +(def/unit hp "745.6998715822702 W") +(def/unit horsepower "1 hp") +(def/unit cal "4.184 J") +(def/unit calorie "1 cal") +(def/unit calories "1 cal") +(def/unit BTU "1055.05585262 J") +(def/unit eV "1.602176634e-19 J") +(def/unit erg "1e-7 J") +(def/unit C "A s") +(def/unit coulomb "1 C") +(def/unit V "W/A") +(def/unit volt "1 V") +(def/unit volts "1 V") +(def/unit ohm "V/A") +(def/unit ohms "1 ohm") +(def/unit S "1/ohm") +(def/unit siemens "1 S") +(def/unit F "C/V") +(def/unit farad "1 F") +(def/unit H "V s/A") +(def/unit henry "1 H") +(def/unit Wb "V s") +(def/unit weber "1 Wb") +(def/unit T "Wb/m^2") +(def/unit tesla "1 T") +(def/unit T_tesla "1 T") +(def/unit gauss "1e-4 T") +(def/unit Mx "1e-8 Wb") +(def/unit maxwell "1 Mx") +(def/unit Oe "1000/(4 pi) * A/m") +(def/unit oersted "1 Oe") +(def/unit poise "0.1 Pa s") +(def/unit St "1e-4 m^2/s") +(def/unit stokes "1 St") +(def/unit B "8 b") +(def/unit byte "1 B") +(def/unit bytes "1 B") +(def/unit bps "b/s") +(def/unit Bps "B/s") +(def/unit sr "1") +(def/unit steradian "1 sr") +(def/unit lm "cd sr") +(def/unit lumen "1 lm") +(def/unit lx "lm/m^2") +(def/unit lux "1 lx") +(def/unit Hz "1/s") +(def/unit hertz "1 Hz") +(def/unit Hertz "1 Hz") +(def/unit hz "1 Hz") +(def/unit Bq "1/s") +(def/unit becquerel "1 Bq") +(def/unit Ci "3.7e10 Bq") +(def/unit curie "1 Ci") +(def/unit Gy "J/kg") +(def/unit gray "1 Gy") +(def/unit Sv "J/kg") +(def/unit sievert "1 Sv") +(def/unit rem "0.01 Sv") +(def/unit rad_dose "0.01 Gy") +(def/unit R "2.58e-4 C/kg") +(def/unit roentgen "1 R") +(def/unit degR "5|9 K") +(def/unit rankine "1 degR") +(def/unit rad "1") +(def/unit radian "1 rad") +(def/unit radians "1 rad") +(def/unit deg "pi/180") +(def/unit degree "1 deg") +(def/unit degrees "1 deg") +(def/unit grad "pi/200") +(def/unit rev "2 pi") +(def/unit revolution "1 rev") +(def/unit revolutions "1 rev") +(def/unit rpm "rev/min") +(def/unit arcmin "1|60 deg") +(def/unit arcsec "1|3600 deg") +(def/unit % "0.01") +(def/unit percent "1 %") +(def/unit ppm "1e-6") +(def/unit ppb "1e-9") +(def/unit ppt "1e-12") +(def/unit planck "6.62607015e-34 J s") +(def/unit hbar "planck/(2 pi)") +(def/unit kph "km/h") +(def/unit mph "mi/h") +(def/unit kn "nmi/h") +(def/unit knot "1 kn") +(def/unit knots "1 kn") + +;; --------------------------------------------------------------------------- +;; The standard table +;; --------------------------------------------------------------------------- + +(def si-prefixes + (list (list "Q" "1e30") (list "R" "1e27") (list "Y" "1e24") (list "Z" "1e21") (list "E" "1e18") (list "P" "1e15") + (list "T" "1e12") (list "G" "1e9") (list "M" "1e6") (list "k" "1e3") (list "h" "1e2") (list "da" "1e1") + (list "d" "1e-1") (list "c" "1e-2") (list "m" "1e-3") (list "u" "1e-6") (list "µ" "1e-6") (list "n" "1e-9") + (list "p" "1e-12") (list "f" "1e-15") (list "a" "1e-18") (list "z" "1e-21") (list "y" "1e-24") + (list "r" "1e-27") (list "q" "1e-30") + (list "Ki" "1024") (list "ki" "1024") (list "Mi" "1048576") (list "Gi" "1073741824") + (list "Ti" "1099511627776") (list "Pi" "1125899906842624") (list "Ei" "1152921504606846976") + (list "Zi" "1180591620717411303424") (list "Yi" "1208925819614629174706176"))) + +(def si-dimensions + (list (list "length" "m") (list "mass" "kg") (list "time" "s") (list "current" "A") (list "temperature" "K") + (list "amount" "mol") (list "luminosity" "cd") (list "information" "b"))) + +;; (name definition alias ...), in order: a definition uses only names above it. +(def standard-definitions + (list ;; Aliases of the base units + (list "m" "m" "meter" "meters" "metre" "metres") + (list "s" "s" "sec" "second" "seconds") + (list "b" "b" "bit" "bits") + (list "A" "A" "amp" "Amp" "ampere") + ;; Constants + (list "c" "299792458 m/s") + (list "gn" "9.80665 m/s^2") + ;; Time + (list "min" "60 s" "minute" "minutes") + (list "h" "60 min" "hr" "hour" "hours") + (list "d" "24 h" "day" "days") + (list "wk" "7 d" "week" "weeks") + (list "fortnight" "14 d" "fortnights") + (list "yr" "31557600 s" "year" "years") + ;; Length + (list "in" "0.0254 m" "inch" "inches") + (list "ft" "12 in" "foot" "feet") + (list "yd" "3 ft" "yard" "yards") + (list "mi" "5280 ft" "mile" "miles") + (list "nmi" "1852 m") + (list "fath" "6 ft" "fathom" "fathoms") + (list "fur" "660 ft" "furlong" "furlongs") + (list "au" "149597870700 m") + (list "ly" "c yr" "lightyear" "lightyears") + (list "pc" "648000 au/pi" "parsec" "parsecs") + ;; Area + (list "ha" "10000 m2" "hectare" "hectares") + (list "are" "100 m2" "ares") + (list "acre" "4046.8564224 m2" "acres") + ;; Volume + (list "l" "0.001 m3" "L" "litre" "litres" "liter" "liters") + (list "gal" "3.785411784 l" "gallon" "gallons") + (list "qt" "1|4 gal" "quart" "quarts") + (list "pt" "1|2 qt" "pint" "pints") + (list "fl_oz" "1|16 pt" "fluid_ounce" "fluid_ounces") + (list "fl_dr" "1|8 fl_oz") + (list "gi" "4 fl_oz") + (list "tsp" "5 ml" "teaspoon" "teaspoons") + (list "tbsp" "15 ml" "tablespoon" "tablespoons") + (list "cup" "250 ml" "cups") + (list "us_tsp" "4.92892159375 ml") + (list "us_tbsp" "3 us_tsp") + (list "us_cup" "16 us_tbsp" "us_cups") + ;; Mass + (list "g" "0.001 kg" "gram" "grams") + (list "t" "1000 kg" "ton" "tons" "tonne" "tonnes") + (list "u" "1.66053906660e-27 kg" "amu") + (list "slug" "14.5939029 kg") + (list "lb" "0.45359237 kg" "pound" "pounds") + (list "oz" "1|16 lb" "ounce" "ounces") + (list "gr" "1|7000 lb" "grain" "grains") + (list "st" "14 lb" "stone") + (list "short_ton" "2000 lb" "short_tons") + (list "long_ton" "2240 lb" "long_tons") + ;; Force and pressure + (list "N" "kg m/s^2" "newton" "newtons") + (list "lbf" "lb gn") + (list "kgf" "kg gn") + (list "dyn" "1e-5 N") + (list "Pa" "N/m^2" "pascal") + (list "bar" "100000 Pa") + (list "atm" "101325 Pa" "atmosphere" "atmospheres") + (list "psi" "lbf/in^2") + (list "mHg" "133322.387415 Pa" "mhg") + (list "mH2O" "9806.65 Pa" "mh2o") + ;; Energy and power + (list "J" "N m" "joule" "joules" "Nm") + (list "W" "J/s" "watt" "watts" "Watt" "Watts") + (list "Wh" "W h") + (list "hp" "745.6998715822702 W" "horsepower") + (list "cal" "4.184 J" "calorie" "calories") + (list "BTU" "1055.05585262 J") + (list "eV" "1.602176634e-19 J") + (list "erg" "1e-7 J") + ;; Electricity and magnetism + (list "C" "A s" "coulomb") + (list "V" "W/A" "volt" "volts") + (list "ohm" "V/A" "ohms") + (list "S" "1/ohm" "siemens") + (list "F" "C/V" "farad") + (list "H" "V s/A" "henry") + (list "Wb" "V s" "weber") + (list "T" "Wb/m^2" "tesla" "T_tesla") + (list "gauss" "1e-4 T") + (list "Mx" "1e-8 Wb" "maxwell") + (list "Oe" "1000/(4 pi) * A/m" "oersted") + ;; Viscosity + (list "poise" "0.1 Pa s") + (list "St" "1e-4 m^2/s" "stokes") + ;; Information + (list "B" "8 b" "byte" "bytes") + (list "bps" "b/s") + (list "Bps" "B/s") + ;; Photometry + (list "sr" "1" "steradian") + (list "lm" "cd sr" "lumen") + (list "lx" "lm/m^2" "lux") + ;; Frequency and radioactivity + (list "Hz" "1/s" "hertz" "Hertz" "hz") + (list "Bq" "1/s" "becquerel") + (list "Ci" "3.7e10 Bq" "curie") + (list "Gy" "J/kg" "gray") + (list "Sv" "J/kg" "sievert") + (list "rem" "0.01 Sv") + (list "rad_dose" "0.01 Gy") + (list "R" "2.58e-4 C/kg" "roentgen") + ;; Temperature differences + (list "degR" "5|9 K" "rankine") + ;; Angles + (list "rad" "1" "radian" "radians") + (list "deg" "pi/180" "degree" "degrees") + (list "grad" "pi/200") + (list "rev" "2 pi" "revolution" "revolutions") + (list "rpm" "rev/min") + (list "arcmin" "1|60 deg") + (list "arcsec" "1|3600 deg") + ;; Ratios + (list "%" "0.01" "percent") + (list "ppm" "1e-6") + (list "ppb" "1e-9") + (list "ppt" "1e-12") + ;; Physical constants + (list "planck" "6.62607015e-34 J s") + (list "hbar" "planck/(2 pi)") + ;; Rates + (list "kph" "km/h") + (list "mph" "mi/h") + (list "kn" "nmi/h" "knot" "knots"))) + +(def standard-preferred + (list (Tuple "m" AllPrefixes) (Tuple "g" AllPrefixes) (Tuple "s" SmallPrefixes) + (Tuple "A" AllPrefixes) (Tuple "K" SmallPrefixes) (Tuple "mol" AllPrefixes) + (Tuple "cd" AllPrefixes) (Tuple "b" AllPrefixes) + (Tuple "N" AllPrefixes) (Tuple "Pa" AllPrefixes) (Tuple "J" AllPrefixes) + (Tuple "W" AllPrefixes) (Tuple "C" AllPrefixes) (Tuple "V" AllPrefixes) + (Tuple "ohm" AllPrefixes) (Tuple "S" AllPrefixes) (Tuple "F" AllPrefixes) + (Tuple "H" AllPrefixes) (Tuple "Wb" AllPrefixes) (Tuple "T" AllPrefixes) + (Tuple "lx" AllPrefixes) (Tuple "Hz" AllPrefixes) (Tuple "Gy" AllPrefixes))) + +(: (-> UnitTable (List string) UnitTable)) +(defun (add-definition t entry) + (match entry + ((Cons name (Cons definition aliases)) + (list-foldl (fun (alias t) (ok-or-raise (def-unit t alias (str "1 " name)))) + (if (orderedmap-has-key? (record-ref t units) name) + t + (ok-or-raise (def-unit t name definition))) + aliases)) + (_ (argument-error "a definition needs a name and a definition")))) + +(: standard-units UnitTable) +(def standard-units + (let* ((t (list-foldl (fun (d t) (ok-or-raise (def-dimension t (first-of-2 d) (second-of-2 d)))) + empty-unit-table si-dimensions)) + (t (list-foldl (fun (p t) (ok-or-raise (def-prefix t (first-of-2 p) (second-of-2 p)))) + t si-prefixes)) + (t (list-foldl (fun (e t) (add-definition t e)) t standard-definitions))) + (list-foldl (fun (p t) (ok-or-raise (def-preferred t (fst p) (snd p)))) + t standard-preferred))) + +(: (-> (List string) string)) +(defun (first-of-2 xs) + (match xs + ((Cons a _) a) + (Nil ""))) + +(: (-> (List string) string)) +(defun (second-of-2 xs) + (match xs + ((Cons _ (Cons b _)) b) + (_ ""))) + +;; --------------------------------------------------------------------------- +;; Measurements +;; --------------------------------------------------------------------------- + +;; A value in the coherent unit of its dimension. +(type (: Measurement (Struct (: value double) (: dims (OrderedMap string Rational))))) + +(: (-> double (OrderedMap string Rational) Measurement)) +(defun (measurement v d) (Measurement (value v) (dims d))) + +(: (-> Measurement double)) +(defun (measurement-value m) (struct-ref m value)) + +(: (-> Measurement (OrderedMap string Rational))) +(defun (measurement-dims m) (struct-ref m dims)) + +(def no-dims (orderedmap-empty)) + +(: (-> string Measurement Measurement (Result string Measurement))) +(defun (same-dims what a b) + (if (= (measurement-dims a) (measurement-dims b)) + (Ok a) + (Err (str "cannot " what " " (dims->string (measurement-dims a)) + " and " (dims->string (measurement-dims b)))))) + +(: (-> UnitTable Expr (Result string Measurement))) +(defun (eval-measure t e) + (match e + ((ENum q) (Ok (measurement (rational->double q) no-dims))) + (EPi (Ok (measurement pi no-dims))) + ((EUnit n) + (match (lookup t n) + ((Some r) (Ok (measurement (resolved-double r) (resolved-dims r)))) + (None (Err (unknown-unit t n))))) + ((EMul a b) + (def (Ok x) (eval-measure t a) :propagate) + (def (Ok y) (eval-measure t b) :propagate) + (Ok (measurement* x y))) + ((EDiv a b) + (def (Ok x) (eval-measure t a) :propagate) + (def (Ok y) (eval-measure t b) :propagate) + (Ok (measurement/ x y))) + ((EPow a q) + (def (Ok x) (eval-measure t a) :propagate) + (Ok (measurement-expt x q))) + ((ENeg a) + (def (Ok x) (eval-measure t a) :propagate) + (Ok (measurement-scale -1.0 x))) + ((EAdd a b) + (def (Ok x) (eval-measure t a) :propagate) + (def (Ok y) (eval-measure t b) :propagate) + (def (Ok _) (same-dims "add" x y) :propagate) + (Ok (measurement (+ (measurement-value x) (measurement-value y)) (measurement-dims x)))) + ((ESub a b) + (def (Ok x) (eval-measure t a) :propagate) + (def (Ok y) (eval-measure t b) :propagate) + (def (Ok _) (same-dims "subtract" x y) :propagate) + (Ok (measurement (- (measurement-value x) (measurement-value y)) (measurement-dims x)))) + ((ECall f a) + (def (Ok x) (eval-measure t a) :propagate) + (Ok (measurement-expt x (if (= f "sqrt") (rational 1 2) (rational 1 3))))))) + +;; "6000 m + 50 ft", "0.5 * 1.225 kg/m^3 * (120 km/h)^2", "sqrt(2 m^2)". +(: (-> string (#:table UnitTable) (Result string Measurement))) +(defun (string->measurement text #:table standard-units) + (def (Ok e) (parse-expression text) :propagate) + (eval-measure table e)) + +(: (-> string Measurement Measurement Measurement)) +(defun (checked what a b) + (match (same-dims what a b) + ((Ok _) a) + ((Err e) (argument-error e)))) + +;; Raises ArgumentException when the dimensions differ. +(: (-> Measurement Measurement Measurement)) +(defun (measurement+ a b) + (ignore (checked "add" a b)) + (measurement (+ (measurement-value a) (measurement-value b)) (measurement-dims a))) + +;; Raises ArgumentException when the dimensions differ. +(: (-> Measurement Measurement Measurement)) +(defun (measurement- a b) + (ignore (checked "subtract" a b)) + (measurement (- (measurement-value a) (measurement-value b)) (measurement-dims a))) + +(: (-> Measurement Measurement Measurement)) +(defun (measurement* a b) + (measurement (* (measurement-value a) (measurement-value b)) + (exps* (measurement-dims a) (measurement-dims b)))) + +(: (-> Measurement Measurement Measurement)) +(defun (measurement/ a b) + (measurement (/ (measurement-value a) (measurement-value b)) + (exps* (measurement-dims a) (exps-invert (measurement-dims b))))) + +(: (-> double Measurement Measurement)) +(defun (measurement-scale k m) + (measurement (* k (measurement-value m)) (measurement-dims m))) + +(: (-> Measurement Rational Measurement)) +(defun (measurement-expt m q) + (measurement (pow (measurement-value m) (rational->double q)) + (exps-scale (measurement-dims m) q))) + +(: (-> Measurement Measurement)) +(defun (measurement-sqrt m) (measurement-expt m (rational 1 2))) + +(: (-> Measurement Measurement bool)) +(defun (conformable? a b) (= (measurement-dims a) (measurement-dims b))) + +;; m as a number of target: (measurement-in m "ft"), (measurement-in m "12 mi/h"). +(: (-> Measurement string (#:table UnitTable) (Result string double))) +(defun (measurement-in m target #:table standard-units) + (def (Ok u) (string->measurement target #:table table) :propagate) + (if (conformable? m u) + (Ok (/ (measurement-value m) (measurement-value u))) + (Err (str "cannot convert " (dims->string (measurement-dims m)) + " to " target ", which is " (dims->string (measurement-dims u)))))) + +;; --------------------------------------------------------------------------- +;; Printing +;; --------------------------------------------------------------------------- + +;; 7 significant digits, invariant culture. +(: (-> double string)) +(defun (format-number x) + (string-replace (.ToString x "G7" clr-invariant) "−" "-")) + +(def print-prefixes + (list "Q" "R" "Y" "Z" "E" "P" "T" "G" "M" "k" "m" "u" "n" "p" "f" "a" "z" "y" "r" "q")) + +;; The prefix for x in a unit: the largest power of 1000 at most |x|, or the +;; smallest when |x| is below all. +(: (-> UnitTable double PrefixUse (Tuple string double))) +(defun (choose-prefix t x use) + (def a (abs x)) + (def candidates + (match use + (NoPrefix Nil) + (_ (list-filter (fun (p) (match use + (SmallPrefixes (< (snd p) 1.0)) + (_ #t))) + (list-map (fun (p) (Tuple p (rational->double (orderedmap-ref (record-ref t prefixes) p)))) + (list-filter (fun (p) (orderedmap-has-key? (record-ref t prefixes) p)) print-prefixes)))))) + (def all (Cons (Tuple "" 1.0) candidates)) + (def fitting (list-filter (fun (p) (<= (snd p) a)) all)) + (cond + ((= a 0.0) (Tuple "" 1.0)) + ((list-empty? fitting) + (list-foldl (fun (p best) (if (< (snd p) (snd best)) p best)) (Tuple "" 1.0) all)) + (else + (list-foldl (fun (p best) (if (> (snd p) (snd best)) p best)) (Tuple "" 0.0) fitting)))) + +;; In the dimension's preferred unit with a prefix by magnitude, or else in +;; the coherent unit written with the base units: "3.6 MJ", "33.33333 m/s". +(: (-> Measurement (#:table UnitTable) string)) +(defun (measurement->string m #:table standard-units) + (def d (measurement-dims m)) + (def v (measurement-value m)) + (cond + ((orderedmap-empty? d) (format-number v)) + (:match (Some (Tuple unit use)) (orderedmap-try-ref (record-ref table preferred) (exps->string d)) + (def u (match (lookup table unit) + ((Some r) (resolved-double r)) + (None 1.0))) + (def x (/ v u)) + (def (p pv) (choose-prefix table x use)) + (str (format-number (/ x pv)) " " p unit)) + (else + (def names (orderedmap-fold (fun (dim q acc) + (orderedmap-set acc (orderedmap-ref-or (record-ref table dimensions) dim dim) q)) + (orderedmap-empty) d)) + (str (format-number v) " " (exps->string names))))) + +;; Mixed output: (measurement->string/mixed m (list "h" "min")) is "5 h 25 min". +;; Each unit but the last takes a whole number, the last the rest. +(: (-> Measurement (List string) (#:table UnitTable) (Result string string))) +(defun (measurement->string/mixed m units #:table standard-units) + (def (Ok targets) + (list-foldl (fun (u acc) + (match acc + ((Ok xs) + (match (string->measurement u #:table table) + ((Ok v) #:when (conformable? v m) (Ok (Cons (Tuple u (measurement-value v)) xs))) + ((Ok v) (Err (str "cannot convert " (dims->string (measurement-dims m)) " to " u))) + ((Err e) (Err e)))) + (e e))) + (Ok Nil) units) + :propagate) + (let go ((ts (list-reverse targets)) (remaining (measurement-value m)) (parts Nil)) + (match ts + (Nil (Ok (string-join (list->vec (list-reverse parts)) " "))) + ((Cons (Tuple u scale) Nil) + (def x (/ remaining scale)) + (go Nil 0.0 (if (or (> (abs x) 1e-9) (list-empty? parts)) + (Cons (str (format-number x) " " u) parts) + parts))) + ((Cons (Tuple u scale) more) + (def whole (truncate (round-to (/ remaining scale) 9))) + (go more (- remaining (* whole scale)) + (if (= whole 0.0) parts (Cons (str (format-number whole) " " u) parts))))))) + +(import/extern + (clr-round-digits (: System.Math.Round (-> double int double)))) + +(: (-> double int double)) +(defun (round-to x digits) (clr-round-digits x digits)) + +;; A query as Umits takes it: "12 h in s", "65 mi/(12 mi/h) in h,min", +;; "6 ft, 2 in in m", or an expression alone, printed by measurement->string. +;; A single target of another dimension divides: "65 mi in 12 mi/h" is +;; "19500 s". +(: (-> string (#:table UnitTable) (Result string string))) +(defun (convert-query query #:table standard-units) + (def (lhs rhs) + (match (string-search-backward query " in ") + ((Some at) (Tuple (substring query (string-cursor-start query) at) + (Some (substring query (string-cursor-forward query at 4) (string-cursor-end query))))) + (None (Tuple query None)))) + (def (Ok m) (string->measurement (string-replace lhs "," " + ") #:table table) :propagate) + (match rhs + (None (Ok (measurement->string m #:table table))) + ((Some targets) + (def units (list-map string-trim (vec->list (string-split targets ",")))) + (match units + ((Cons u Nil) + (def (Ok target) (string->measurement u #:table table) :propagate) + (if (conformable? m target) + (Ok (str (format-number (/ (measurement-value m) (measurement-value target))) " " u)) + (Ok (measurement->string (measurement/ m target) #:table table)))) + (_ (measurement->string/mixed m units #:table table)))))) + +(impl (Written Measurement) + (defun (written m) (str "(string->measurement \"" (measurement->string m) "\")")) + (defun (write-into m b) (stringbuilder-add-string! b (written m)))) + +(impl (Displayed Measurement) + (defun (displayed m) (measurement->string m)) + (defun (display-into m b) (stringbuilder-add-string! b (measurement->string m)))) + +;; --------------------------------------------------------------------------- +;; Temperatures +;; --------------------------------------------------------------------------- +;; Absolute temperatures. Differences are measured in K or degR. + +(type (: Celsius (Newtype double))) +(type (: Fahrenheit (Newtype double))) + +(: (-> Celsius Measurement)) +(defun (celsius->kelvin c) + (def (Celsius x) c) + (measurement (+ x 273.15) (measurement-dims (ok-or-raise (string->measurement "K"))))) + +;; Raises ArgumentException when k is not a temperature. +(: (-> Measurement Celsius)) +(defun (kelvin->celsius k) + (Celsius (- (ok-or-raise (measurement-in k "K")) 273.15))) + +(: (-> Fahrenheit Measurement)) +(defun (fahrenheit->kelvin f) + (def (Fahrenheit x) f) + (measurement (* (+ x 459.67) (/ 5.0 9.0)) (measurement-dims (ok-or-raise (string->measurement "K"))))) + +;; Raises ArgumentException when k is not a temperature. +(: (-> Measurement Fahrenheit)) +(defun (kelvin->fahrenheit k) + (Fahrenheit (- (* (ok-or-raise (measurement-in k "K")) (/ 9.0 5.0)) 459.67))) + +(impl (Written Celsius) + (defun (written c) (def (Celsius x) c) (str "(Celsius " (written x) ")")) + (defun (write-into c b) (stringbuilder-add-string! b (written c)))) + +(impl (Displayed Celsius) + (defun (displayed c) (def (Celsius x) c) (str (format-number x) " °C")) + (defun (display-into c b) (stringbuilder-add-string! b (displayed c)))) + +(impl (Written Fahrenheit) + (defun (written f) (def (Fahrenheit x) f) (str "(Fahrenheit " (written x) ")")) + (defun (write-into f b) (stringbuilder-add-string! b (written f)))) + +(impl (Displayed Fahrenheit) + (defun (displayed f) (def (Fahrenheit x) f) (str (format-number x) " °F")) + (defun (display-into f b) (stringbuilder-add-string! b (displayed f)))) + +;; --------------------------------------------------------------------------- +;; Logarithmic +;; --------------------------------------------------------------------------- +;; Each raises ArgumentException when the measurement has the wrong dimension. + +(: (-> double Measurement string double)) +(defun (decibels k m reference) + (* k (log10 (ok-or-raise (measurement-in m reference))))) + +(: (-> double double string Measurement)) +(defun (from-decibels k x reference) + (measurement-scale (pow 10.0 (/ x k)) (ok-or-raise (string->measurement reference)))) + +(: (-> Measurement double)) +(defun (dBW p) (decibels 10.0 p "1 W")) + +(: (-> Measurement double)) +(defun (dBm p) (decibels 10.0 p "1 mW")) + +(: (-> Measurement double)) +(defun (dBV v) (decibels 20.0 v "1 V")) + +(: (-> Measurement double)) +(defun (dBSPL p) (decibels 20.0 p "20 uPa")) + +(: (-> double Measurement)) +(defun (dBW->power x) (from-decibels 10.0 x "1 W")) + +(: (-> double Measurement)) +(defun (dBm->power x) (from-decibels 10.0 x "1 mW")) + +(: (-> double Measurement)) +(defun (dBV->voltage x) (from-decibels 20.0 x "1 V")) + +(: (-> double Measurement)) +(defun (dBSPL->pressure x) (from-decibels 20.0 x "20 uPa")) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/url.bjo — URLs: parse, write, resolve, and percent-encoding. +;; +;; The grammar is RFC 3986's, read strictly. A URL holds each part as it is +;; written, percent-encoding included, so that a URL read and written again is +;; the same text. The decoded forms are functions over it. + +(import (std prelude)) + +;; Every export is documented there; the module doc is too. +(:doc #:file "url.bjodoc") + +(export + ;; The value. + Url Authority + url-scheme url-authority url-path url-query url-fragment + authority-userinfo authority-host authority-port + url-host url-port url-absolute? + + ;; To and from readable forms + parse-url url->string + + ;; A reference against a base. + url-resolve remove-dot-segments + + ;; The decoded parts. + url-segments url-params + query->params params->query param-ref param-ref-all + + ;; Percent-encoding, one function for each part. + encode-segment encode-query-part encode-fragment encode-userinfo encode-form + percent-decode form-decode + + ;; A URL written as a quoted literal. `Link` is a grammar. Its export also + ;; exports UrlParts, PathParts, QueryParam, UrlText and UrlPiece. + Link make-url link + + ;; The checked parts of a literal, and the checks that make them. + Scheme Host Port scheme host port scheme->string host->string port->int) + +;; =========================================================================== +;; Interop +;; =========================================================================== + +(import/class + (IPAddress (: System.Net.IPAddress)) + (IdnMapping (: System.Globalization.IdnMapping (-> IdnMapping)))) + +(import/extern + (clr-parse-ip (: System.Net.IPAddress.Parse (-> string IPAddress) + #:exceptions (System.FormatException))) + (clr-idn-ascii (: System.Globalization.IdnMapping.GetAscii (-> IdnMapping string string) + #:exceptions (System.ArgumentException)))) + +;; =========================================================================== +;; The value +;; =========================================================================== + +;; `//userinfo@host:port`. The user info and the host are as written: the host +;; of an IP literal keeps its brackets, `[::1]`. +(type (: Authority (Record (: userinfo (Option string)) + (: host string) + (: port (Option int))))) + +;; A URL, or a relative reference, as RFC 3986 calls one that has no scheme. +;; Each part is as written, percent-encoding included, and without the +;; punctuation that separates it: no `:` after the scheme, no `?` before the +;; query, no `#` before the fragment. `None` is a part that is not there, +;; which is not the same as one that is empty: `http://h/?` has a query, `""`. +(type (: Url (Record (: scheme (Option string)) + (: authority (Option Authority)) + (: path string) + (: query (Option string)) + (: fragment (Option string))))) + +(: (-> Url (Option string))) +(defun (url-scheme u) (record-ref u scheme)) + +(: (-> Url (Option Authority))) +(defun (url-authority u) (record-ref u authority)) + +(: (-> Url string)) +(defun (url-path u) (record-ref u path)) + +(: (-> Url (Option string))) +(defun (url-query u) (record-ref u query)) + +(: (-> Url (Option string))) +(defun (url-fragment u) (record-ref u fragment)) + +(: (-> Authority (Option string))) +(defun (authority-userinfo a) (record-ref a userinfo)) + +(: (-> Authority string)) +(defun (authority-host a) (record-ref a host)) + +(: (-> Authority (Option int))) +(defun (authority-port a) (record-ref a port)) + +(: (-> Url (Option string))) +(defun (url-host u) + (option-map (fun (a) (record-ref a host)) (record-ref u authority))) + +(: (-> Url (Option int))) +(defun (url-port u) + (match (record-ref u authority) + ((Some a) (record-ref a port)) + (None None))) + +;; Has a scheme: `https://example.com/`, `mailto:a@b`, and not `/a` or `//h/a`. +(: (-> Url bool)) +(defun (url-absolute? u) (some? (record-ref u scheme))) + +;; =========================================================================== +;; Bytes and their classes +;; =========================================================================== + +(: (-> (Array byte) int int)) +(defun (byte-at bytes i) (cast int (array-ref bytes i))) + +(: (-> int bool)) +(defun (alpha? b) (or (and (>= b 65) (<= b 90)) (and (>= b 97) (<= b 122)))) + +(: (-> int bool)) +(defun (digit? b) (and (>= b 48) (<= b 57))) + +(: (-> int int)) +(defun (hex-value b) + (cond + ((digit? b) (- b 48)) + ((and (>= b 65) (<= b 70)) (- b 55)) + ((and (>= b 97) (<= b 102)) (- b 87)) + (else -1))) + +;; ALPHA DIGIT - . _ ~ +(: (-> int bool)) +(defun (unreserved? b) + (or (alpha? b) (digit? b) (= b 45) (= b 46) (= b 95) (= b 126))) + +;; ! $ & ' ( ) * + , ; = +(: (-> int bool)) +(defun (sub-delim? b) + (case b + ((33 36 38 39 40 41 42 43 44 59 61) #t) + (else #f))) + +;; What a path segment holds as it is: unreserved, sub-delims, `:` and `@`. +(: (-> int bool)) +(defun (pchar? b) (or (unreserved? b) (sub-delim? b) (= b 58) (= b 64))) + +(: (-> int bool)) +(defun (path-char? b) (or (pchar? b) (= b 47))) + +;; A query and a fragment hold `/` and `?` too. +(: (-> int bool)) +(defun (query-char? b) (or (pchar? b) (= b 47) (= b 63))) + +(: (-> int bool)) +(defun (userinfo-char? b) (or (unreserved? b) (sub-delim? b) (= b 58))) + +(: (-> int bool)) +(defun (reg-name-char? b) (or (unreserved? b) (sub-delim? b))) + +(: (-> int bool)) +(defun (scheme-char? b) (or (alpha? b) (digit? b) (= b 43) (= b 45) (= b 46))) + +;; =========================================================================== +;; Percent-encoding +;; =========================================================================== + +(: (Array byte)) +(def hex-bytes (string->bytes "0123456789ABCDEF")) + +(: (-> StringBuilder int void)) +(defun (add-escape! sb b) + (stringbuilder-add-byte! sb 37) + (stringbuilder-add-byte! sb (byte-at hex-bytes (/ b 16))) + (stringbuilder-add-byte! sb (byte-at hex-bytes (% b 16)))) + +;; `s` as UTF-8, with each byte that `keep?` refuses written `%XX`. With `plus`, +;; a space is `+`, as a form writes it. +(: (-> (-> int bool) bool string string)) +(defun (encode-with keep? plus s) + (def bytes (string->bytes s)) + (def sb (stringbuilder-empty)) + (loop (:for i (range 0 (array-length bytes))) + (:do (let ((b (byte-at bytes i))) + (cond + ((keep? b) (stringbuilder-add-byte! sb b)) + ((and plus (= b 32)) (stringbuilder-add-byte! sb 43)) + (else (add-escape! sb b)))))) + (stringbuilder->string sb)) + +;; One segment of a path. A `/` is encoded, so it cannot begin a segment. +(: (-> string string)) +(defun (encode-segment s) (encode-with pchar? #f s)) + +;; A key or a value of a query. `&`, `=`, `+`, `;` and `#` are encoded, so a +;; value cannot begin another parameter or end the query, and a space is +;; `%20`, which every server reads as a space. +(: (-> int bool)) +(defun (query-part-char? b) + (or (unreserved? b) + (case b + ((33 36 39 40 41 42 44 58 64 47 63) #t) + (else #f)))) + +(: (-> string string)) +(defun (encode-query-part s) (encode-with query-part-char? #f s)) + +(: (-> string string)) +(defun (encode-fragment s) (encode-with query-char? #f s)) + +;; A user name or a password. `:` and `@` are encoded. +(: (-> string string)) +(defun (encode-userinfo s) (encode-with reg-name-char? #f s)) + +;; As `application/x-www-form-urlencoded` writes it, which is what a browser +;; sends for a form: letters, digits and `* - . _` as they are, a space as +;; `+`, and every other byte `%XX`. +(: (-> int bool)) +(defun (form-char? b) + (or (alpha? b) (digit? b) (= b 42) (= b 45) (= b 46) (= b 95))) + +(: (-> string string)) +(defun (encode-form s) (encode-with form-char? #t s)) + +;; The text that `s` encodes. With `plus`, a `+` is a space. Refused: a `%` +;; that two hex digits do not follow, and bytes that are not UTF-8. +(: (-> bool string (Result string string))) +(defun (decode-with plus s) + (def bytes (string->bytes s)) + (def n (array-length bytes)) + (def sb (stringbuilder-empty)) + (let go ((i 0)) + (cond + ((>= i n) + (result-map-err (fun (_) "the percent-encoded bytes are not UTF-8") + (try (stringbuilder->string sb) #:catch (System.InvalidOperationException)))) + (else + (def b (byte-at bytes i)) + (cond + ((= b 37) + (def hi (if (< (+ i 1) n) (hex-value (byte-at bytes (+ i 1))) -1)) + (def lo (if (< (+ i 2) n) (hex-value (byte-at bytes (+ i 2))) -1)) + (if (and (>= hi 0) (>= lo 0)) + (begin (stringbuilder-add-byte! sb (+ (* hi 16) lo)) (go (+ i 3))) + (Err (str "column " (int->string (+ i 1)) + ": a '%' begins an escape of two hex digits")))) + ((and plus (= b 43)) (stringbuilder-add-byte! sb 32) (go (+ i 1))) + (else (stringbuilder-add-byte! sb b) (go (+ i 1)))))))) + +(: (-> string (Result string string))) +(defun (percent-decode s) (decode-with #f s)) + +(: (-> string (Result string string))) +(defun (form-decode s) (decode-with #t s)) + +;; =========================================================================== +;; Reading a URL +;; =========================================================================== + +(: (-> (Array byte) int int string)) +(defun (slice bytes from to) (bytes->string bytes #:start from #:end to)) + +;; The first index from `i` before `end` whose byte is one of `stops`, or `end`. +(: (-> (Array byte) int int (-> int bool) int)) +(defun (find-any bytes i end stop?) + (cond + ((>= i end) end) + ((stop? (byte-at bytes i)) i) + (else (find-any bytes (+ i 1) end stop?)))) + +;; What a refusal calls the byte at `i`. +(: (-> int string)) +(defun (what-byte b) + (cond + ((= b 32) "a space") + ((>= b 128) "a character that is not ASCII") + ((< b 32) "a control character") + (else (str "'" (list->string (list (int->char b))) "'")))) + +(: (-> int string (Result string %a))) +(defun (refuse-at i what) (Err (str "column " (int->string (+ i 1)) ": " what))) + +;; Every byte from `i` to `end` is one `ok?` allows, or a `%XX`. +(: (-> (Array byte) int int (-> int bool) string (Result string bool))) +(defun (check-part bytes i end ok? part) + (if (>= i end) + (Ok #t) + (let ((b (byte-at bytes i))) + (cond + ((= b 37) + (if (and (< (+ i 2) end) + (>= (hex-value (byte-at bytes (+ i 1))) 0) + (>= (hex-value (byte-at bytes (+ i 2))) 0)) + (check-part bytes (+ i 3) end ok? part) + (refuse-at i "a '%' begins an escape of two hex digits"))) + ((ok? b) (check-part bytes (+ i 1) end ok? part)) + (else + (refuse-at i (str (what-byte b) " is not allowed in the " part + ", and has to be percent-encoded"))))))) + +;; Every byte from `i` to `end` is a hex digit. +(: (-> (Array byte) int int bool)) +(defun (all-hex? bytes i end) + (cond + ((>= i end) #t) + ((< (hex-value (byte-at bytes i)) 0) #f) + (else (all-hex? bytes (+ i 1) end)))) + +;; The digits of a port, `None` for none. +(: (-> (Array byte) int int (Result string (Option int)))) +(defun (read-port bytes i end) + (let go ((k i) (n 0)) + (cond + ((>= k end) (if (= k i) (Ok None) (Ok (Some n)))) + ((not (digit? (byte-at bytes k))) + (refuse-at k (str "a port is digits, and this is " (what-byte (byte-at bytes k))))) + ((> (+ (* n 10) (- (byte-at bytes k) 48)) 65535) + (refuse-at i "a port is at most 65535")) + (else (go (+ k 1) (+ (* n 10) (- (byte-at bytes k) 48))))))) + +;; `[...]`: an IPv6 address, or `v1.x`, the IPvFuture of RFC 3986. A zone, the +;; `%` of `fe80::1%eth0`, is not part of a URL's address. +(: (-> (Array byte) int int (Result string bool))) +(defun (check-ip-literal bytes i end) + (def first (if (< i end) (byte-at bytes i) -1)) + (cond + ((or (= first 118) (= first 86)) + (def dot (find-any bytes (+ i 1) end (fun (b) (= b 46)))) + (cond + ((or (= dot (+ i 1)) (>= dot end) (not (all-hex? bytes (+ i 1) dot))) + (refuse-at i "an IPvFuture address is v, hex digits, '.' and then its text")) + ((= (+ dot 1) end) (refuse-at dot "an IPvFuture address has text after its '.'")) + (else (check-part bytes (+ dot 1) end userinfo-char? "address")))) + ((< (find-any bytes i end (fun (b) (= b 37))) end) + (refuse-at (find-any bytes i end (fun (b) (= b 37))) "an IPv6 address in a URL has no zone")) + (else + (match (clr-parse-ip (slice bytes i end)) + ((Ok _) (Ok #t)) + ((Err _) (refuse-at i "this is not an IPv6 address")))))) + +;; `userinfo@host:port`, from `i` to `end`. +(: (-> (Array byte) int int (Result string Authority))) +(defun (read-authority bytes i end) + (def at (find-any bytes i end (fun (b) (= b 64)))) + (def host-start (if (< at end) (+ at 1) i)) + (def (Ok userinfo) + (if (< at end) + (result-map (fun (_) (Some (slice bytes i at))) + (check-part bytes i at userinfo-char? "user info")) + (Ok None)) + :propagate) + (def (Ok host-end) + (cond + ((and (< host-start end) (= (byte-at bytes host-start) 91)) + (def close (find-any bytes host-start end (fun (b) (= b 93)))) + (cond + ((>= close end) (refuse-at host-start "a '[' begins an IP address, and there is no ']'")) + (else + (def (Ok _) (check-ip-literal bytes (+ host-start 1) close) :propagate) + (if (or (= (+ close 1) end) (= (byte-at bytes (+ close 1)) 58)) + (Ok (+ close 1)) + (refuse-at (+ close 1) "after ']' comes ':' and the port, or the end of the host"))))) + (else + (def colon (find-any bytes host-start end (fun (b) (= b 58)))) + (result-map (fun (_) colon) + (check-part bytes host-start colon reg-name-char? "host")))) + :propagate) + (def (Ok port) + (if (< host-end end) (read-port bytes (+ host-end 1) end) (Ok None)) + :propagate) + (Ok (Authority (userinfo userinfo) (host (slice bytes host-start host-end)) (port port)))) + +;; A URL, or a relative reference, read by RFC 3986. What the grammar does not +;; allow is refused, with the column where it is: a space, a character that +;; is not ASCII, a `%` without two hex digits, a port that is not a number. +;; Such text has to be percent-encoded first. +(: (-> string (Result string Url))) +(defun (parse-url s) + (def bytes (string->bytes s)) + (def n (array-length bytes)) + ;; A scheme is a letter and scheme characters, and then a ':'. + (def scheme-end + (if (and (> n 0) (alpha? (byte-at bytes 0))) + (let ((k (find-any bytes 1 n (fun (b) (not (scheme-char? b)))))) + (if (and (< k n) (= (byte-at bytes k) 58)) k -1)) + -1)) + (def scheme (if (>= scheme-end 0) (Some (slice bytes 0 scheme-end)) None)) + (def after-scheme (if (>= scheme-end 0) (+ scheme-end 1) 0)) + (def has-authority? + (and (< (+ after-scheme 1) n) + (= (byte-at bytes after-scheme) 47) + (= (byte-at bytes (+ after-scheme 1)) 47))) + (def authority-end + (if has-authority? + (find-any bytes (+ after-scheme 2) n (fun (b) (or (= b 47) (= b 63) (= b 35)))) + after-scheme)) + (def (Ok authority) + (if has-authority? + (result-map Some (read-authority bytes (+ after-scheme 2) authority-end)) + (Ok None)) + :propagate) + (def path-end (find-any bytes authority-end n (fun (b) (or (= b 63) (= b 35))))) + (def (Ok _) (check-part bytes authority-end path-end path-char? "path") :propagate) + ;; `a:b` is the scheme `a`. A relative reference whose first segment has a + ;; ':' would read as one, so RFC 3986 does not allow it: `./a:b` does. + (def first-slash (find-any bytes authority-end path-end (fun (b) (= b 47)))) + (def first-colon (find-any bytes authority-end first-slash (fun (b) (= b 58)))) + (def (Ok _) + (if (and (none? scheme) (not has-authority?) (< first-colon first-slash)) + (refuse-at first-colon + "a relative reference cannot have a ':' in its first segment; write './' before it") + (Ok #t)) + :propagate) + (def has-query? (and (< path-end n) (= (byte-at bytes path-end) 63))) + (def query-end + (if has-query? (find-any bytes (+ path-end 1) n (fun (b) (= b 35))) path-end)) + (def (Ok query) + (if has-query? + (result-map (fun (_) (Some (slice bytes (+ path-end 1) query-end))) + (check-part bytes (+ path-end 1) query-end query-char? "query")) + (Ok None)) + :propagate) + (def (Ok fragment) + (if (< query-end n) + (result-map (fun (_) (Some (slice bytes (+ query-end 1) n))) + (check-part bytes (+ query-end 1) n query-char? "fragment")) + (Ok None)) + :propagate) + (Ok (Url (scheme scheme) + (authority authority) + (path (slice bytes authority-end path-end)) + (query query) + (fragment fragment)))) + +;; =========================================================================== +;; Writing a URL +;; =========================================================================== + +(: (-> Url string)) +(defun (url->string u) + (def sb (stringbuilder-empty)) + (match (record-ref u scheme) + ((Some s) (stringbuilder-add-string! sb s) (stringbuilder-add! sb #\:)) + (None unit)) + (match (record-ref u authority) + ((Some a) + (stringbuilder-add-string! sb "//") + (match (record-ref a userinfo) + ((Some ui) (stringbuilder-add-string! sb ui) (stringbuilder-add! sb #\@)) + (None unit)) + (stringbuilder-add-string! sb (record-ref a host)) + (match (record-ref a port) + ((Some p) (stringbuilder-add! sb #\:) (stringbuilder-add-string! sb (int->string p))) + (None unit))) + (None unit)) + (stringbuilder-add-string! sb (record-ref u path)) + (match (record-ref u query) + ((Some q) (stringbuilder-add! sb #\?) (stringbuilder-add-string! sb q)) + (None unit)) + (match (record-ref u fragment) + ((Some f) (stringbuilder-add! sb #\#) (stringbuilder-add-string! sb f)) + (None unit)) + (stringbuilder->string sb)) + +;; =========================================================================== +;; Resolving a reference +;; =========================================================================== + +;; RFC 3986 5.2.4: `.` and `..` segments taken out of a path. A `..` above the +;; root is dropped, so `/a/../../b` is `/b`. A path that ended in `.` or `..` +;; ends in `/`. +(: (-> string string)) +(defun (remove-dot-segments path) + (def absolute? (string-starts-with? path "/")) + (def parts (vec->list (string-split path "/"))) + (def segments (if absolute? (list-tail parts) parts)) + (def kept + (let go ((rest segments) (out (list))) + (match rest + (Nil (list-reverse out)) + ((Cons s more) + (def last? (list-empty? more)) + (cond + ((= s ".") (go more (if last? (cons "" out) out))) + ((= s "..") + (def popped (if (list-empty? out) out (list-tail out))) + (go more (if last? (cons "" popped) popped))) + (else (go more (cons s out)))))))) + (str (if absolute? "/" "") (string-join (list->vec kept) "/"))) + +;; RFC 3986 5.2.3: a relative path against the base's. +(: (-> Url string string)) +(defun (merge-paths base path) + (def base-path (record-ref base path)) + (cond + ((and (some? (record-ref base authority)) (string-empty? base-path)) (str "/" path)) + (else + (def bytes (string->bytes base-path)) + (def last-slash + (let go ((k (- (array-length bytes) 1))) + (cond ((< k 0) -1) + ((= (byte-at bytes k) 47) k) + (else (go (- k 1)))))) + (if (< last-slash 0) + path + (str (slice bytes 0 (+ last-slash 1)) path))))) + +;; `ref` read where `base` is, as a browser reads a link on a page: RFC 3986 +;; 5.2.2. A reference with a scheme stands for itself. `base` should be +;; absolute, as the RFC asks. +(: (-> Url Url Url)) +(defun (url-resolve base ref) + (def r-path (record-ref ref path)) + (cond + ((some? (record-ref ref scheme)) + (record-set ref (path (remove-dot-segments r-path)))) + ((some? (record-ref ref authority)) + (record-set ref (scheme (record-ref base scheme)) (path (remove-dot-segments r-path)))) + ((string-empty? r-path) + (Url (scheme (record-ref base scheme)) + (authority (record-ref base authority)) + (path (record-ref base path)) + (query (match (record-ref ref query) + ((Some q) (Some q)) + (None (record-ref base query)))) + (fragment (record-ref ref fragment)))) + (else + (Url (scheme (record-ref base scheme)) + (authority (record-ref base authority)) + (path (remove-dot-segments + (if (string-starts-with? r-path "/") r-path (merge-paths base r-path)))) + (query (record-ref ref query)) + (fragment (record-ref ref fragment)))))) + +;; =========================================================================== +;; The decoded parts +;; =========================================================================== + +;; The path's segments, decoded. `/a/b%2Fc/` is `("a" "b/c" "")`: the leading +;; `/` begins the first segment, a `/` that ends the path leaves an empty one, +;; and an encoded `/` is part of its segment. The empty path has none. +(: (-> Url (Result string (List string)))) +(defun (url-segments u) + (def path (record-ref u path)) + (cond + ((string-empty? path) (Ok (list))) + (else + (def parts (vec->list (string-split path "/"))) + (def raw (if (string-starts-with? path "/") (list-tail parts) parts)) + (result-map list-reverse + (list-foldl (fun (seg acc) + (def (Ok done) acc :propagate) + (result-map (fun (d) (Cons d done)) (percent-decode seg))) + (Ok (list)) + raw))))) + +;; `a=1&b=two+words&b` as pairs, in order, decoded, with `+` read as a space as +;; a form writes it. A parameter with no `=` has the value `""`, and an empty +;; one, `a=1&&b=2`, is no parameter. A key that comes twice is there twice. +(: (-> string (Result string (List (Tuple string string))))) +(defun (query->params q) + (def pieces (list-filter (fun (p) (not (string-empty? p))) (vec->list (string-split q "&")))) + (result-map list-reverse + (list-foldl (fun (piece acc) + (def (Ok done) acc :propagate) + (def bytes (string->bytes piece)) + (def n (array-length bytes)) + (def eq (find-any bytes 0 n (fun (b) (= b 61)))) + (def (Ok k) (form-decode (slice bytes 0 eq)) :propagate) + (def (Ok v) + (if (< eq n) (form-decode (slice bytes (+ eq 1) n)) (Ok "")) + :propagate) + (Ok (Cons (Tuple k v) done))) + (Ok (list)) + pieces))) + +;; The query's parameters, as `query->params` reads them. No query has none. +(: (-> Url (Result string (List (Tuple string string))))) +(defun (url-params u) + (match (record-ref u query) + ((Some q) (query->params q)) + (None (Ok (list))))) + +;; Pairs written as a query, without the `?`. Each key and value is encoded +;; with `encode-query-part`, so it cannot begin another parameter. +(: (-> (List (Tuple string string)) string)) +(defun (params->query params) + (string-join + (list->vec + (list-map (fun (p) + (match p + ((Tuple k v) (str (encode-query-part k) "=" (encode-query-part v))))) + params)) + "&")) + +;; The first value of `key`. +(: (-> (List (Tuple string string)) string (Option string))) +(defun (param-ref params key) + (match params + (Nil None) + ((Cons (Tuple k v) rest) (if (= k key) (Some v) (param-ref rest key))))) + +;; Every value of `key`, in order. +(: (-> (List (Tuple string string)) string (List string))) +(defun (param-ref-all params key) + (match params + (Nil (list)) + ((Cons (Tuple k v) rest) + (if (= k key) (Cons v (param-ref-all rest key)) (param-ref-all rest key))))) + +;; =========================================================================== +;; A URL written as a quoted literal +;; =========================================================================== +;; +;; '(url https "example.com" (port 8443) (path "users" ,name) +;; (query (page 2) (tag ,t)) (fragment "top")) +;; '(path "users" ,id "edit" (query (q ,text))) +;; +;; The grammar `Link` reads the literal. Each value is encoded for the part it +;; is in: a segment as a segment, a query key or value as one. So a `/` in a +;; computed segment does not make two segments, and a `&` in a computed value +;; does not begin a parameter. The compiler checks the keywords, the order and +;; the number of each part where you write the literal. It also runs the +;; checks `scheme`, `host` and `port` on a literal scheme, host and port. + +;; A scheme that `scheme` accepted, such as `https`. +(type (: Scheme #:opaque (Record (: name string)))) + +;; A host that `host` accepted, in the form a URL writes it. +(type (: Host #:opaque (Record (: name string)))) + +;; A port that `port` accepted: from 0 to 65535. +(type (: Port #:opaque (Record (: number int)))) + +;; A letter, then letters, digits, `+`, `-` and `.`. +(: (-> Symbol (Result string Scheme))) +(defun (scheme s) + (def text (symbol->string s)) + (def bytes (string->bytes text)) + (def n (array-length bytes)) + (if (and (> n 0) (alpha? (byte-at bytes 0)) + (= (find-any bytes 1 n (fun (b) (not (scheme-char? b)))) n)) + (Ok (Scheme (name text))) + (Err (str "'" text "' is not a scheme")))) + +;; A host as a URL writes it. An IPv6 address gets its brackets. A name that +;; is not ASCII becomes the IDNA form (`bücher.example` is +;; `xn--bcher-kva.example`). Text that is not a name or an address is an +;; error. +(: (-> string (Result string Host))) +(defun (host h) + (def bytes (string->bytes h)) + (def n (array-length bytes)) + (cond + ((and (> n 0) (= (byte-at bytes 0) 91)) + (match (if (= (byte-at bytes (- n 1)) 93) + (check-ip-literal bytes 1 (- n 1)) + (Err "it has no ']'")) + ((Ok _) (Ok (Host (name h)))) + ((Err e) (Err (str "'" h "' is not a host: " e))))) + ((< (find-any bytes 0 n (fun (b) (= b 58))) n) + (match (check-ip-literal bytes 0 n) + ((Ok _) (Ok (Host (name (str "[" h "]"))))) + ((Err e) (Err (str "'" h "' is not a host: " e))))) + ((= (find-any bytes 0 n (fun (b) (not (reg-name-char? b)))) n) (Ok (Host (name h)))) + ((< (find-any bytes 0 n (fun (b) (>= b 128))) n) + (match (clr-idn-ascii (IdnMapping.) h) + ((Ok ascii) (host ascii)) + ((Err _) (Err (str "'" h "' is not a host name"))))) + (else (Err (str "'" h "' is not a host name"))))) + +(: (-> int (Result string Port))) +(defun (port n) + (if (and (>= n 0) (<= n 65535)) + (Ok (Port (number n))) + (Err (str "a port is from 0 to 65535, and this is " (int->string n))))) + +(: (-> Scheme string)) +(defun (scheme->string s) (record-ref s name)) + +(: (-> Host string)) +(defun (host->string h) (record-ref h name)) + +(: (-> Port int)) +(defun (port->int p) (record-ref p number)) + +(def/grammar Link + ;; A URL with a scheme and a host, or a path on this site. + (Link (:or (: LinkUrl (url Scheme Host UrlParts)) + (: LinkPath (path PathParts)))) + + ;; The parts of a URL after the host, in any order, each at most once. + (UrlParts (:each (: port (:? (port Port))) + (: user (:? (user UrlText))) + (: path (:? (path UrlPiece ...))) + (: query (:? (query QueryParam ...))) + (: fragment (:? (fragment UrlText))))) + + ;; The segments of a path on this site, then its query and its fragment. + (PathParts (: segments UrlPiece ...) + (:each (: query (:? (query QueryParam ...))) + (: fragment (:? (fragment UrlText))))) + + ;; One parameter of a query: `(key value)`, `(key)` with no value, or a + ;; computed pair, `,(Tuple k v)`. + (QueryParam (:or (: ParamForm (UrlText (:? UrlPiece))) + (: ParamPair ,(Tuple string string)))) + + ;; One text: a string, a symbol as its name, or an int. + (UrlText (:or (: TextStr string) (: TextSym symbol) (: TextInt int))) + + ;; One text, or a computed `(Vec string)`: several segments, or several + ;; values of one key. + (UrlPiece (:or (: PieceOne UrlText) (: PieceMany ,(Vec string)))) + + (Scheme (symbol #:check scheme)) + (Host (string #:check host)) + (Port (int #:check port))) + +(: (-> UrlText string)) +(defun (text-of t) + (match t + ((TextStr s) s) + ((TextSym s) (symbol->string s)) + ((TextInt n) (int->string n)))) + +(: (-> UrlPiece (List string))) +(defun (piece-texts p) + (match p + ((PieceOne t) (list (text-of t))) + ((PieceMany v) (vec->list v)))) + +(: (-> (List UrlPiece) (List string))) +(defun (pieces-texts ps) + (list-foldr (fun (p acc) (list-append (piece-texts p) acc)) (list) ps)) + +;; `key=value` texts, encoded, for one parameter. +(: (-> QueryParam (List string))) +(defun (param-texts p) + (match p + ((ParamPair (Tuple k v)) (list (str (encode-query-part k) "=" (encode-query-part v)))) + ((ParamForm key None) (list (encode-query-part (text-of key)))) + ((ParamForm key (Some value)) + (def k (encode-query-part (text-of key))) + (list-map (fun (v) (str k "=" (encode-query-part v))) (piece-texts value))))) + +(: (-> (List QueryParam) string)) +(defun (query-text params) + (string-join (list->vec (list-foldr (fun (p acc) (list-append (param-texts p) acc)) + (list) params)) + "&")) + +(: (-> (List string) string)) +(defun (path-text segs) + (if (list-empty? segs) + "/" + (string-join (list->vec (list-map (fun (s) (str "/" (encode-segment s))) segs)) ""))) + +(: (-> (Option UrlText) (Option string))) +(defun (fragment-text f) (option-map (fun (t) (encode-fragment (text-of t))) f)) + +;; The `Url` that a literal stands for. The grammar and the checks refused +;; each bad literal, so this does not fail. +(: (-> Link Url)) +(defun (make-url l) + (match l + ((LinkPath parts) + (Url (scheme None) (authority None) + (path (path-text (pieces-texts (record-ref parts segments)))) + (query (option-map query-text (record-ref parts query))) + (fragment (fragment-text (record-ref parts fragment))))) + ((LinkUrl s h parts) + (Url (scheme (Some (scheme->string s))) + (authority (Some (Authority + (userinfo (option-map (fun (t) (encode-userinfo (text-of t))) + (record-ref parts user))) + (host (host->string h)) + (port (option-map port->int (record-ref parts port)))))) + (path (match (record-ref parts path) + ((Some pieces) (path-text (pieces-texts pieces))) + (None ""))) + (query (option-map query-text (record-ref parts query))) + (fragment (fragment-text (record-ref parts fragment))))))) + +;; The text of a literal: `(url->string (make-url l))`. +(: (-> Link string)) +(defun (link l) (url->string (make-url l))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; std/uuid.bjo — uuids, as RFC 9562 specifies them. +;; +;; A `Uuid` is 16 bytes, held as two `ulong`s in network byte order. Two uuids +;; compare as their bytes compare, which is also how their text compares and +;; how Postgres orders its `uuid` type. A version 7 uuid starts with the time, +;; so version 7 uuids sort by the time they were made. +;; +;; The random bits come from `(std random)`, and the time of a version 7 uuid +;; from `current-clock`. Both are parameters, so a test can fix them. + +(export + Uuid + ;; Making one + uuid-v4 uuid-v7 uuid-v5 uuid-v3 + uuid-nil uuid-max + uuid-ns-dns uuid-ns-url uuid-ns-oid uuid-ns-x500 + ;; Text and bytes + uuid->string string->uuid uuid-string? + uuid->bytes bytes->uuid + ;; What is in one + uuid-version uuid-v7-time + ;; .NET + uuid->guid guid->uuid) + +(import (std prelude)) +(import (only (std random) current-random random-u64)) +(import (only (std datetime) Instant now instant->unix-milliseconds unix-milliseconds->instant)) +(import (only (std crypto hash) HashAlgorithm make-hasher hasher-add! hasher-add-string! hasher-finish!)) +(import (only (std atomic) Atomic make-atomic atomic-update!)) + +(:doc #:file "uuid.bjodoc") + +;; The constructor that takes the fields in the order of the bytes: +;; `(uint, ushort, ushort, byte x 8)` is bytes 0-3, 4-5, 6-7 and 8-15, +;; big-endian. +(import/class + (Guid (: System.Guid (-> uint ushort ushort byte byte byte byte byte byte byte byte Guid)))) + +(import/extern + ;; With `#t`: the 16 bytes in the order of the text. + (clr-guid-bytes (: System.Guid.ToByteArray (-> Guid bool (Array byte))))) + +;; `hi` is the first 8 bytes, `lo` the last 8. Opaque, so every `Uuid` is one +;; that a function here made. `type/derive` does not take `#:opaque`, so `Eq` +;; and `Ord` are written out: `hi` first, then `lo`, which is the order of the +;; bytes. +(type (: Uuid #:opaque (Struct (: hi ulong) (: lo ulong)))) + +(impl (Eq Uuid) + (defun (= a b) (and (= (struct-ref a hi) (struct-ref b hi)) (= (struct-ref a lo) (struct-ref b lo)))) + (defun (eq-hash u) (bitwise-xor (eq-hash (struct-ref u hi)) (* 31 (eq-hash (struct-ref u lo)))))) + +(impl (Ord Uuid) + (defun (compare a b) + (def by-hi (compare (struct-ref a hi) (struct-ref b hi))) + (if (= by-hi 0) (compare (struct-ref a lo) (struct-ref b lo)) by-hi))) + +(: (-> Uuid ulong)) +(defun (hi-of u) (struct-ref u hi)) + +(: (-> Uuid ulong)) +(defun (lo-of u) (struct-ref u lo)) + +;; --------------------------------------------------------------------------- +;; Bits and bytes +;; --------------------------------------------------------------------------- + +;; Everything in `hi` except the 4 version bits, bits 12 to 15. +(: ulong) +(def version-mask 0xFFFFFFFFFFFF0FFF) + +;; Everything in `lo` except the 2 variant bits at the top. +(: ulong) +(def variant-mask 0x3FFFFFFFFFFFFFFF) + +;; The variant of RFC 9562: the top two bits of `lo` are 10. +(: ulong) +(def rfc-variant 0x8000000000000000) + +(: (-> ulong int ulong)) +(defun (with-version hi version) + (bitwise-ior (bitwise-and hi version-mask) (shift-left (cast ulong version) 12))) + +(: (-> ulong ulong)) +(defun (with-variant lo) (bitwise-ior (bitwise-and lo variant-mask) rfc-variant)) + +;; The 8 bytes from `start`, big-endian. +(: (-> (Array byte) int ulong)) +(defun (read-u64 bytes start) + (let go ((i start) (v 0UL)) + (if (= i (+ start 8)) + v + (go (+ i 1) (bitwise-ior (shift-left v 8) (cast ulong (array-ref bytes i))))))) + +;; Writes `v` into the 8 bytes from `start`, big-endian. +(: (-> (Array byte) int ulong void)) +(defun (write-u64! bytes start v) + (loop (:for i (range 0 8)) + (:do (array-set! bytes (+ start i) (cast byte (shift-right-logical v (* 8 (- 7 i)))))))) + +;; --------------------------------------------------------------------------- +;; Making one +;; --------------------------------------------------------------------------- + +;; 122 random bits. +(: (-> (#:rng RandomSource) Uuid)) +(defun (uuid-v4 #:rng (parameter-ref current-random)) + (def r1 (random-u64 #:rng rng)) + (def r2 (random-u64 #:rng rng)) + (Uuid (hi (with-version r1 4)) (lo (with-variant r2)))) + +;; A version 7 uuid without its version and variant bits: the 48 bits of the +;; time and then the 74 random bits, as one 122-bit number. `high` is its top +;; 58 bits and `low` its bottom 64. +(type (: V7Bits (Struct (: high ulong) (: low ulong)))) + +(: (-> V7Bits V7Bits bool)) +(defun (v7-greater? a b) + (or (> (struct-ref a high) (struct-ref b high)) + (and (= (struct-ref a high) (struct-ref b high)) + (> (struct-ref a low) (struct-ref b low))))) + +;; `b` plus 1. +(: (-> V7Bits V7Bits)) +(defun (v7-next b) + (def bottom (+ (struct-ref b low) 1UL)) + (V7Bits (high (if (= bottom 0UL) (+ (struct-ref b high) 1UL) (struct-ref b high))) + (low bottom))) + +;; The last version 7 value that this process made. +(: (Atomic V7Bits)) +(def last-v7 (make-atomic (V7Bits (high 0UL) (low 0UL)))) + +;; The Unix time in milliseconds, then 74 random bits. The uuids that this +;; process makes always increase, also when two are made in one millisecond +;; and when the clock steps back. +;; +;; - A value that is not greater than the last one is the last one plus 1, as +;; RFC 9562 section 6.2 allows. +;; - `last-v7` changes in one atomic step, so this holds across threads. +(: (-> (#:clock Clock) (#:rng RandomSource) Uuid)) +(defun (uuid-v7 #:clock (parameter-ref current-clock) #:rng (parameter-ref current-random)) + (def ms (bitwise-and (cast ulong (instant->unix-milliseconds (now #:clock clock))) 0xFFFFFFFFFFFF)) + (def r1 (random-u64 #:rng rng)) + (def r2 (random-u64 #:rng rng)) + (def candidate (V7Bits (high (bitwise-ior (shift-left ms 10) (bitwise-and r1 0x3FF))) + (low r2))) + (def chosen + (atomic-update! last-v7 (fun (last) (if (v7-greater? candidate last) candidate (v7-next last))))) + (def top (struct-ref chosen high)) + (def bottom (struct-ref chosen low)) + ;; 48 bits of time, the version, then the top 12 random bits. + (def rand-a (bitwise-and (bitwise-ior (shift-left top 2) (shift-right-logical bottom 62)) 0xFFF)) + (Uuid (hi (bitwise-ior (shift-left (shift-right-logical top 10) 16) (bitwise-ior 0x7000 rand-a))) + (lo (with-variant bottom)))) + +;; A digest of the namespace's 16 bytes and the name's UTF-8 bytes. The uuid +;; is the first 16 bytes of the digest, with the version and the variant set. +;; SHA-1 and MD5 digests are 20 and 16 bytes. +(: (-> HashAlgorithm int Uuid string Uuid)) +(defun (name-based algorithm version namespace name) + (def h (make-hasher #:algorithm algorithm)) + (hasher-add! h (uuid->bytes namespace)) + (hasher-add-string! h name) + (def digest (hasher-finish! h)) + (Uuid (hi (with-version (read-u64 digest 0) version)) + (lo (with-variant (read-u64 digest 8))))) + +;; The same namespace and name always give the same uuid. SHA-1. +(: (-> Uuid string Uuid)) +(defun (uuid-v5 namespace name) (name-based Sha1 5 namespace name)) + +;; As `uuid-v5`, with MD5. For a system that already uses version 3. +(: (-> Uuid string Uuid)) +(defun (uuid-v3 namespace name) (name-based Md5 3 namespace name)) + +(: Uuid) +(def uuid-nil (Uuid (hi 0UL) (lo 0UL))) + +(: Uuid) +(def uuid-max (Uuid (hi 18446744073709551615UL) (lo 18446744073709551615UL))) + +;; The namespaces of RFC 9562, section 6.6. +(: (-> string Uuid)) +(defun (known text) + (match (string->uuid text) + ((Some u) u) + (None (panic! (str "(std uuid): not a uuid: " text))))) + +(: Uuid) +(def uuid-ns-dns (known "6ba7b810-9dad-11d1-80b4-00c04fd430c8")) + +(: Uuid) +(def uuid-ns-url (known "6ba7b811-9dad-11d1-80b4-00c04fd430c8")) + +(: Uuid) +(def uuid-ns-oid (known "6ba7b812-9dad-11d1-80b4-00c04fd430c8")) + +(: Uuid) +(def uuid-ns-x500 (known "6ba7b814-9dad-11d1-80b4-00c04fd430c8")) + +;; --------------------------------------------------------------------------- +;; Text and bytes +;; --------------------------------------------------------------------------- + +(: (Array byte)) +(def lower-digits (string->bytes "0123456789abcdef")) + +(: (Array byte)) +(def upper-digits (string->bytes "0123456789ABCDEF")) + +;; Whether the 32 hex digits have a hyphen before digit `i`. +(: (-> int bool)) +(defun (hyphen-before? i) (or (= i 8) (= i 12) (= i 16) (= i 20))) + +;; 8-4-4-4-12 hex digits, lower case unless `#:upper #t`. +(: (-> Uuid (#:upper bool) string)) +(defun (uuid->string u #:upper #f) + (def digits (if upper upper-digits lower-digits)) + (def sb (stringbuilder-empty)) + (loop (:for i (range 0 32)) + (:do (when (hyphen-before? i) (stringbuilder-add-byte! sb 45)) + (let ((word (if (< i 16) (hi-of u) (lo-of u))) + (shift (* 4 (- 15 (% i 16))))) + (stringbuilder-add-byte! + sb (cast int (array-ref digits (cast int (bitwise-and (shift-right-logical word shift) 15UL)))))))) + (stringbuilder->string sb)) + +;; 0 to 15 for an ASCII hex digit in either case, -1 for any other byte. +(: (-> byte int)) +(defun (hex-value b) + (def c (cast int b)) + (cond + ((and (>= c 48) (<= c 57)) (- c 48)) + ((and (>= c 65) (<= c 70)) (- c 55)) + ((and (>= c 97) (<= c 102)) (- c 87)) + (else -1))) + +;; Only the 36-character form, in upper or lower case: no braces, no +;; `urn:uuid:`, and no form without hyphens, so that one uuid has one text in +;; each case. +(: (-> string (Option Uuid))) +(defun (string->uuid text) + (def bytes (string->bytes text)) + (if (not (= (array-length bytes) 36)) + None + ;; `i` indexes the text, `digit` counts the hex digits read. + (let go ((i 0) (digit 0) (h 0UL) (l 0UL)) + (cond + ((= i 36) (Some (Uuid (hi h) (lo l)))) + ((or (= i 8) (= i 13) (= i 18) (= i 23)) + (if (= (array-ref bytes i) 45uy) (go (+ i 1) digit h l) None)) + (else + (def v (hex-value (array-ref bytes i))) + (cond + ((< v 0) None) + ((< digit 16) (go (+ i 1) (+ digit 1) (bitwise-ior (shift-left h 4) (cast ulong v)) l)) + (else (go (+ i 1) (+ digit 1) h (bitwise-ior (shift-left l 4) (cast ulong v)))))))))) + +(: (-> string bool)) +(defun (uuid-string? text) (some? (string->uuid text))) + +;; 16 bytes, in network byte order. +(: (-> Uuid (Array byte))) +(defun (uuid->bytes u) + (def (: bytes (Array byte)) (make-array 16)) + (write-u64! bytes 0 (hi-of u)) + (write-u64! bytes 8 (lo-of u)) + bytes) + +;; `None` unless there are exactly 16 bytes. +(: (-> (Array byte) (Option Uuid))) +(defun (bytes->uuid bytes) + (if (= (array-length bytes) 16) + (Some (Uuid (hi (read-u64 bytes 0)) (lo (read-u64 bytes 8)))) + None)) + +(impl (Written Uuid) + (defun (written u) (uuid->string u)) + (defun (write-into u b) (stringbuilder-add-string! b (written u)))) + +;; --------------------------------------------------------------------------- +;; What is in one +;; --------------------------------------------------------------------------- + +;; The 4 version bits: 4 for `uuid-v4`, 7 for `uuid-v7`. 0 for `uuid-nil` and +;; 15 for `uuid-max`. +(: (-> Uuid int)) +(defun (uuid-version u) + (cast int (bitwise-and (shift-right-logical (hi-of u) 12) 15UL))) + +;; When a version 7 uuid was made, to the millisecond. `None` for any other +;; version. +(: (-> Uuid (Option Instant))) +(defun (uuid-v7-time u) + (if (= (uuid-version u) 7) + (unix-milliseconds->instant (cast long (shift-right-logical (hi-of u) 16))) + None)) + +;; --------------------------------------------------------------------------- +;; .NET +;; --------------------------------------------------------------------------- + +;; A `System.Guid` with the same 16 bytes, for a .NET library such as Npgsql. +;; A Guid keeps its first three fields in the machine's byte order, and its +;; `CompareTo` does not compare the bytes in order. Compare the `Uuid`. +(: (-> Uuid Guid)) +(defun (uuid->guid u) + (def b (uuid->bytes u)) + (def hi (hi-of u)) + (Guid. (cast uint (shift-right-logical hi 32)) + (cast ushort (shift-right-logical hi 16)) + (cast ushort hi) + (array-ref b 8) (array-ref b 9) (array-ref b 10) (array-ref b 11) + (array-ref b 12) (array-ref b 13) (array-ref b 14) (array-ref b 15))) + +(: (-> Guid Uuid)) +(defun (guid->uuid g) + (def b (clr-guid-bytes g #t)) + (Uuid (hi (read-u64 b 0)) (lo (read-u64 b 8)))) +;; text/bjodat-core.bjo +;; +;; The bjodat value, the reader and the writer. The reader reads the document's +;; UTF-8 bytes. +;; +;; **This is the machinery, not the interface.** `(text bjodat)` is what a +;; program imports: it declares a record and generates a decoder that reads the +;; text straight into it, with no `Bjodat` value in between. What is here is +;; what that decoder is built out of, and what a program reaches for directly +;; only when the shape of its data is not known until run time. +;; +;; A program that declares records with `def/bjodat-type` does not need this +;; module: `(text bjodat)` re-exports `Bjodat`, `Reader` and the one-pass +;; driver, so one import covers the generator, the traits and the writer. What +;; is left here, and the reason to import it by name, is the *tree* — +;; `parse-bjodat`, `bjodat-ref` and the entry accessors. +;; +;; It used to be imported alongside `(text bjodat)` always, and the comment +;; here said a facade could not spare you the second line. It can; I was wrong +;; about what `re-export` could be made to do. See `re-export` in +;; `Docs/MODULES.org`. +;; +;; Source's shapes, read as data: a bare identifier is a Symbol rather than a +;; reference, and `{...}` is a map rather than a comprehension. A file is a +;; stream of values. +;; +;; A map is held as a *flat* vec of alternating key and value, in the order it +;; was written. Not a trie, for two reasons that a benchmark settled: building +;; one is about seventy percent of what reading a keyed document costs, and a +;; record decoded from it reads every field once and then throws it away. A +;; linear scan of reference comparisons over a handful of interned keywords is +;; cheaper than hashing them. Keeping the order is what lets a value be written +;; back out looking like what was read. + +(export + Bjodat + bjodat-max-depth + read-bjodat parse-bjodat + read-bjodat-all parse-bjodat-all + bjodat->string + bjodat-entry-count bjodat-entry-key bjodat-entry-value bjodat-ref + ;; What a generated one-pass decoder drives. Not an API for people: see the + ;; section it is defined in. + Reader + open-map! next-key! end-entry! key=? field-value skip-value! + ;; The same, for a record written as a tagged form: `(tag (field value) ...)`. + open-form! next-field! end-clause! clause-value skip-clause! clause-values! + read-one-pass parse-one-pass read-one-pass-all parse-one-pass-all) + +(import (std prelude)) +(import (only (std ports) read-bytes)) +(import (std maths)) +(import (std simd)) + +;; Every export is documented there; the module doc is too. +(:doc #:file "bjodat-core.bjodoc") + +(import/extern + (clr-to-long (: Bjolang.Runtime.BjoNum.ParseLong)) + (clr-to-double (: Bjolang.Runtime.BjoNum.ParseDouble))) + +(import/extern + (clr-nan? (: System.Double.IsNaN (-> double bool))) + (clr-positive-infinity? (: System.Double.IsPositiveInfinity (-> double bool))) + (clr-negative-infinity? (: System.Double.IsNegativeInfinity (-> double bool)))) + +;; Hex and binary, which have no reader of their own. +(import/extern + (clr-to-ulong (: System.UInt64.Parse (-> string ulong))) + (clr-radix-ulong (: System.Convert.ToUInt64 (-> string int ulong)))) + +;; --------------------------------------------------------------------------- +;; The value +;; --------------------------------------------------------------------------- + +(type + (: Bjodat (Union + (: BjoBool bool) + (: BjoInt long) + (: BjoFloat double) + ;; Written with the suffixes of Bjolang's own literals: 12UL, + ;; 1.50M, 123I. + (: BjoULong ulong) + (: BjoDecimal decimal) + (: BjoBigInt bigint) + (: BjoStr string) + (: BjoChar char) + (: BjoSym Symbol) + (: BjoKey Keyword) + (: BjoList (List Bjodat)) + (: BjoVec (Vec Bjodat)) + ;; key, value, key, value — an even length, in written order. + (: BjoMap (Vec Bjodat))))) + +(: int) +(def bjodat-max-depth 200) + +;; --- what a map is, for anyone holding one --------------------------------- + +(: (-> (Vec Bjodat) int)) +(defun (bjodat-entry-count entries) (/ (vec-length entries) 2)) + +(: (-> (Vec Bjodat) int Bjodat)) +(defun (bjodat-entry-key entries i) (vec-ref entries (* i 2))) + +(: (-> (Vec Bjodat) int Bjodat)) +(defun (bjodat-entry-value entries i) (vec-ref entries (+ (* i 2) 1))) + +;; A keyword key, found by reference. Written order, so the first wins. +(: (-> (Vec Bjodat) Keyword (Option Bjodat))) +(defun (bjodat-ref entries k) + (def n (vec-length entries)) + (let go ((i 0)) + (if (>= i n) + None + (match (vec-ref entries i) + ((BjoKey kk) (if (eq? kk k) (Some (vec-ref entries (+ i 1))) (go (+ i 2)))) + (_ (go (+ i 2))))))) + +;; --------------------------------------------------------------------------- +;; The name cache +;; --------------------------------------------------------------------------- +;; A document repeats its names. A manifest of twenty thousand records writes +;; the same five keys twenty thousand times, and interning one costs a string +;; built off the scratch buffer and then a dictionary probe — so almost every +;; one of those allocations is of a string that already exists. +;; +;; This holds the last name seen in each slot and compares the buffer against it +;; without building a string, which is what `stringbuilder=?` is for. A hit +;; costs that comparison and nothing else. A miss costs what interning +;; always cost, plus the failed comparison. +;; +;; Direct-mapped rather than associative: a probe is one comparison, and two +;; names colliding costs a miss rather than a wrong answer. Sized for the number +;; of distinct names a *record* has, not the number of records. +;; +;; This was a class in `BjolangRuntime/BjoText.cs` — a hundred lines of C# that +;; existed for one library. It is here instead because a library in the standard +;; library should not need a file in the runtime; what the runtime owes it is +;; primitives, and `stringbuilder-byte-ref` and `stringbuilder=?` are the two it +;; was missing. + +(: int) +(def name-slots 64) + +;; Empty slots hold `""`. No name of any other length can equal it, and a name +;; that *is* empty answers `""`, which is the right answer anyway — so there is +;; no empty case to test for. `make-array` would leave nulls, which +;; `string-length` would not survive. +(: (-> (Array string))) +(defun (new-name-cache) + (def a (make-array name-slots)) + (loop (:for i (range 0 name-slots)) + (:do (array-set! a i ""))) + a) + +;; The first, middle and last unit and the length: three reads whatever the +;; length, and enough to part the names of one record. With the first unit and +;; the length alone, `name` and `note` shared a slot, and so did `name` and +;; `pinned`, and each made the other a string every time. +(: (-> StringBuilder int int)) +(defun (name-slot sb n) + (if (= n 0) + 0 + (bitwise-and (+ (* (+ (* (+ (* (stringbuilder-byte-ref sb 0) 31) + (stringbuilder-byte-ref sb (shift-right n 1))) + 31) + (stringbuilder-byte-ref sb (- n 1))) + 31) + n) + (- name-slots 1)))) + +;; The buffer's contents as a string, reusing the one already handed out for +;; those same units. +;; +;; This stops short of answering the interned `Keyword` itself, which would save +;; the probe inside `string->keyword` as well. Nothing here can: interning is +;; the runtime's, and a name read from a port is not known until it is read. +(: (-> (Array string) StringBuilder string)) +(defun (cached-name cache sb) + (def n (stringbuilder-length sb)) + (def i (name-slot sb n)) + (def name (array-ref cache i)) + (cond + ((stringbuilder=? sb name) name) + (else + (def s (stringbuilder->string sb)) + (array-set! cache i s) + s))) + +;; --------------------------------------------------------------------------- +;; The reader +;; --------------------------------------------------------------------------- +;; The reader works on bytes: a chunk of the document in `bytes`, the one it is +;; looking at at `next`, and `limit` where the chunk ends. Looking is an array +;; load and stepping is an increment. A chunk comes from the port with +;; `read-bytes` (see `ensure!`); the `parse-` readers hand over the string's +;; own bytes as the one chunk there is, and no port. +;; +;; All the format's punctuation is ASCII. A character of several bytes - in a +;; string, a symbol, a keyword or a character literal - is checked to be UTF-8 +;; and copied whole; one that is not UTF-8 is refused. A column counts +;; characters: only a byte that starts one moves it. +;; +;; The byte is handled as a number, so that the end of the document is a +;; value, `spent`, rather than a second case. + +(: int) +(def spent -1) + +;; How much a read from the port asks for: the first one what `read-some` +;; gives a file, each one after it twice the one before, up to `read-most`. +;; Each read costs a sync on the port, a new array and the reader's slow path +;; at the chunk's end, so a big document is read in big chunks. `read-bytes` +;; makes the port's buffer as big as what it asks for, so a small one is +;; still read in a small one. +(: int) +(def read-first 16384) + +(: int) +(def read-most 262144) + +(type + (: Reader (Record (: port (Option InputPort) #:mutable) + ;; How many bytes the next read from the port asks for. + (: ask int #:mutable) + (: bytes (Array byte) #:mutable) + (: next int #:mutable) + (: limit int #:mutable) + (: row int #:mutable) + (: col int #:mutable) + ;; One buffer for every string, symbol and number. + (: scratch StringBuilder) + ;; One array for every list, vec and map in the document. + (: stack (Array Bjodat) #:mutable) + (: top int #:mutable) + ;; One cache for every name in the document. + (: names (Array string)) + ;; The keywords read, by the slot of their name: the name + ;; in `keys`, and the value made for it in `key-values`. + ;; See `cached-keyword`. + (: keys (Array string)) + (: key-values (Array Bjodat))))) + +;; A reader over `bytes`, and then whatever `port` has after them. +(: (-> (Option InputPort) (Array byte) Reader)) +(defun (reader port bytes) + (Reader (port port) + (ask read-first) + (bytes bytes) + (next 0) + (limit (array-length bytes)) + (row 1) + (col 1) + (scratch (stringbuilder-empty)) + (stack (make-array 16)) + (top 0) + (names (new-name-cache)) + (keys (new-name-cache)) + (key-values (make-array name-slots)))) + +(: (-> Reader Bjodat void)) +(defun (push! rd v) + (def a (record-ref rd stack)) + (def n (record-ref rd top)) + (cond + ((< n (array-length a)) (array-set! a n v)) + (else + (def bigger (array-resize a (* n 2))) + (array-set! bigger n v) + (record-set! rd (stack bigger)))) + (record-set! rd (top (+ n 1)))) + +(: (-> Reader int (Vec Bjodat))) +(defun (harvest! rd mark) + (def v (subarray->vec (record-ref rd stack) mark (- (record-ref rd top) mark))) + (record-set! rd (top mark)) + v) + +(: (-> Reader StringBuilder)) +(defun (buffer rd) + (def sb (record-ref rd scratch)) + (stringbuilder-clear! sb) + sb) + +;; `count` bytes of `old` from `from`, then `more`: what is left of one chunk +;; and the next. Almost always nothing is left, and `more` is the answer. +(: (-> (Array byte) int int (Array byte) (Array byte))) +(defun (join-bytes old from count more) + (if (= count 0) + more + (let ((joined (make-array (+ count (array-length more))))) + (array-copy! old from joined 0 count) + (array-copy! more 0 joined count (array-length more)) + joined))) + +;; Whether there are at least `n` bytes from `next` on, reading from the port +;; until there are or it ends. Bytes not yet looked at are kept, so a +;; character cut in two by the end of a chunk is whole afterwards. +(: (-> Reader int bool)) +(defun (ensure! rd n) + (let go () + (def have (- (record-ref rd limit) (record-ref rd next))) + (cond + ((>= have n) #t) + (else + (match (record-ref rd port) + (None #f) + ((Some p) + (def ask (record-ref rd ask)) + (match (read-bytes p (if (> n ask) n ask)) + (None + (record-set! rd (port None)) + #f) + ((Some more) + (record-set! rd + (bytes (join-bytes (record-ref rd bytes) (record-ref rd next) have more)) + (next 0) + (limit (+ have (array-length more))) + (ask (if (< ask read-most) (* ask 2) read-most))) + (go))))))))) + +;; The byte the reader is looking at, or `spent` at the end of the document. +(: (-> Reader int)) +(defun (held-code rd) + (def i (record-ref rd next)) + (cond + ((< i (record-ref rd limit)) (cast int (array-ref (record-ref rd bytes) i))) + ((ensure! rd 1) (cast int (array-ref (record-ref rd bytes) (record-ref rd next)))) + (else spent))) + +(: (-> Reader bool)) +(defun (at-end? rd) (= (held-code rd) spent)) + +;; Anything that is not ASCII answers #\nul, because every character this is +;; asked about is one of the format's own punctuation. +(: (-> Reader char)) +(defun (peek rd) + (def c (held-code rd)) + (if (and (>= c 0) (< c 128)) (int->char c) #\nul)) + +;; Moves past the byte being looked at, which is a character of its own: every +;; caller has just looked at it and found it ASCII. A character of several +;; bytes is taken by `take-character!`. +(: (-> Reader void)) +(defun (step! rd) + (record-set! rd + (next (+ (record-ref rd next) 1)) + (col (+ (record-ref rd col) 1)))) + +(: (-> Reader void)) +(defun (newline! rd) + (record-set! rd + (next (+ (record-ref rd next) 1)) + (row (+ (record-ref rd row) 1)) + (col 1))) + +;; --- characters of several bytes -------------------------------------------- + +(: string) +(def not-utf-8 "a byte that is not UTF-8") + +;; How many bytes a character starting with `lead` has, by RFC 3629; 0 for a +;; byte no character starts with: a continuation byte, C0 and C1 (which could +;; only start an overlong form), and F5 and up. +(: (-> int int)) +(defun (utf8-length lead) + (cond + ((< lead 128) 1) + ((< lead 194) 0) + ((< lead 224) 2) + ((< lead 240) 3) + ((< lead 245) 4) + (else 0))) + +(: (-> int bool)) +(defun (continuation? b) (= (bitwise-and b 192) 128)) + +;; How many bytes the character the reader is looking at has, when they are +;; UTF-8; 0 when they are not, or the document ends inside them. The second +;; byte is narrower after E0 and F0, which would otherwise start an overlong +;; form, after ED, which would start a surrogate, and after F4, which would +;; go past U+10FFFF. +(: (-> Reader int)) +(defun (sequence-length rd) + (def lead (held-code rd)) + (def n (utf8-length lead)) + (cond + ((<= n 1) n) + ((not (ensure! rd n)) 0) + (else (sequence-length-at (record-ref rd bytes) (record-ref rd next) (record-ref rd limit))))) + +;; The same for the character at `i` of `b`, whose bytes end at `limit`: 0 +;; also when the character runs past `limit`, where the reader may have more +;; of it to fetch. +(: (-> (Array byte) int int int)) +(defun (sequence-length-at b i limit) + (def lead (cast int (array-ref b i))) + (def n (utf8-length lead)) + (cond + ((<= n 1) n) + ((> (+ i n) limit) 0) + (else + (def second (cast int (array-ref b (+ i 1)))) + (def low (cond ((= lead 224) 160) ((= lead 240) 144) (else 128))) + (def high (cond ((= lead 237) 159) ((= lead 244) 143) (else 191))) + (if (and (>= second low) + (<= second high) + (let check ((k 2)) + (cond + ((>= k n) #t) + ((continuation? (cast int (array-ref b (+ i k)))) (check (+ k 1))) + (else #f)))) + n + 0)))) + +;; Copies the `n` bytes of the character being looked at into `sb`, and moves +;; past them: one column. +(: (-> Reader StringBuilder int void)) +(defun (take-character! rd sb n) + (def b (record-ref rd bytes)) + (def i (record-ref rd next)) + (loop (:for k (range 0 n)) + (:do (stringbuilder-add-byte! sb (cast int (array-ref b (+ i k)))))) + (record-set! rd + (next (+ i n)) + (col (+ (record-ref rd col) 1)))) + +;; The character being looked at, of `n` bytes known to be UTF-8. +(: (-> Reader int char)) +(defun (held-character rd n) + (def i (record-ref rd next)) + (vec-ref (string->vec (bytes->string (record-ref rd bytes) #:start i #:end (+ i n))) 0)) + +;; The character being looked at, `c`, copied into `sb` and stepped past: an +;; ASCII byte as it is, a character of several bytes once it is known to be +;; UTF-8. False, having taken nothing, when it is not. +(: (-> Reader StringBuilder int bool)) +(defun (add-held! rd sb c) + (cond + ((< c 128) + (stringbuilder-add-unit! sb c) + (step! rd) + #t) + (else + (def n (sequence-length rd)) + (cond + ((= n 0) #f) + (else + (take-character! rd sb n) + #t))))) + +;; --- character classes ------------------------------------------------------ + +;; A `case` rather than a chain of `=`: it is a C# switch, which Roslyn turns +;; into a jump table, and this is asked about every character of the document. +(: (-> int bool)) +(defun (space-code? c) + (case c + ((32 10 9 13) #t) + (else #f))) + +;; What ends a name: whitespace, the end of the port (`spent`, -1), or one of +;; the delimiters the lexer refuses inside a symbol — ( ) [ ] { } , : " ; ' +;; +;; One case rather than three calls and fifteen comparisons, because +;; `take-name!` asks this per character of every symbol, keyword and number. +;; +;; A `.` and a `/` are deliberately absent: they are symbol characters, which +;; is what makes `net8.0` and `bjolang/http` one symbol each. +(: (-> int bool)) +(defun (symbol-code? c) + (case c + ((-1 32 10 9 13 40 41 91 93 123 125 44 58 34 59 39) #f) + (else #t))) + +(: (-> int bool)) +(defun (digit-code? c) (and (>= c 48) (<= c 57))) + +;; White space from `start`, 32 or 16 bytes at a time, to the first byte that +;; is not white space or the last whole block before `limit`: where it ends, +;; the line and column there, and whether a byte that is not white space was +;; found. The line moves by the newlines; after one, the column is counted +;; from it. 256 bits where the CPU has them, and 128 where it has not, which +;; is every ARM64 CPU. +(: (-> (Array byte) int int int int (Tuple int int int bool))) +(defun (scan-space b start limit row col) + (if simd256-accelerated? + (scan-space/256 b start limit row col) + (scan-space/128 b start limit row col))) + +;; The line and column after `n` bytes of white space whose newlines are the +;; set bits of `nls`. +(: (-> int int int int (Tuple int int))) +(defun (after-space nls n row col) + (if (= nls 0) + (Tuple row (+ col n)) + ;; The bytes after the last newline are the column, less one. + (Tuple (+ row (popcount nls)) + (- n (- 31 (leading-zeros nls)))))) + +(: (-> (Array byte) int int int int (Tuple int int int bool))) +(defun (scan-space/256 b start limit row col) + (let go ((i start) (row row) (col col)) + (if (> (+ i 32) limit) + (Tuple i row col #f) + (let () + (def v (simd256-load b i)) + (def nl (simd-eq v (simd256-splat 10uy))) + (def spaces (simd-mask-bits (simd-or (simd-or nl (simd-eq v (simd256-splat 32uy))) + (simd-or (simd-eq v (simd256-splat 9uy)) + (simd-eq v (simd256-splat 13uy)))))) + (def nls (simd-mask-bits nl)) + (if (= spaces -1) + (let () + (def (Tuple r c) (after-space nls 32 row col)) + (go (+ i 32) r c)) + (let () + (def t (trailing-zeros (bitwise-not spaces))) + (def (Tuple r c) (after-space (bitwise-and nls (- (shift-left 1 t) 1)) t row col)) + (Tuple (+ i t) r c #t))))))) + +(: (-> (Array byte) int int int int (Tuple int int int bool))) +(defun (scan-space/128 b start limit row col) + (let go ((i start) (row row) (col col)) + (if (> (+ i 16) limit) + (Tuple i row col #f) + (let () + (def v (simd128-load b i)) + (def nl (simd-eq v (simd128-splat 10uy))) + (def spaces (simd-mask-bits (simd-or (simd-or nl (simd-eq v (simd128-splat 32uy))) + (simd-or (simd-eq v (simd128-splat 9uy)) + (simd-eq v (simd128-splat 13uy)))))) + (def nls (simd-mask-bits nl)) + (if (= spaces 65535) + (let () + (def (Tuple r c) (after-space nls 16 row col)) + (go (+ i 16) r c)) + (let () + (def t (trailing-zeros (bitwise-xor spaces 65535))) + (def (Tuple r c) (after-space (bitwise-and nls (- (shift-left 1 t) 1)) t row col)) + (Tuple (+ i t) r c #t))))))) + +;; White space a chunk at a time: the chunk is scanned with `scan-space` and +;; then a byte at a time, and the reader is moved once for it. A chunk that +;; ends in white space is followed by the next. +(: (-> Reader void)) +(defun (skip-space-run! rd) + (def b (record-ref rd bytes)) + (def limit (record-ref rd limit)) + (def (Tuple fast-end fast-row fast-col stopped) + (scan-space b (record-ref rd next) limit (record-ref rd row) (record-ref rd col))) + (def (Tuple end row col) + (let go ((i fast-end) (row fast-row) (col fast-col)) + (if (or stopped (>= i limit)) + (Tuple i row col) + (let ((c (cast int (array-ref b i)))) + (cond + ((= c 10) (go (+ i 1) (+ row 1) 1)) + ((or (= c 32) (= c 9) (= c 13)) (go (+ i 1) row (+ col 1))) + (else (Tuple i row col))))))) + (record-set! rd (next end) (row row) (col col)) + (when (and (>= end limit) (ensure! rd 1)) + (skip-space-run! rd))) + +;; Whitespace, and `;` to the end of the line. +(: (-> Reader void)) +(defun (skip-space! rd) + (let go () + (def c (held-code rd)) + (cond + ((space-code? c) + (skip-space-run! rd) + (go)) + ((= c 59) + (let skip () + (def k (held-code rd)) + (when (and (not (= k spent)) (not (= k 10))) + (step! rd) + (skip))) + (go)) + (else unit)))) + +;; --------------------------------------------------------------------------- +;; Refusals +;; --------------------------------------------------------------------------- + +(: (-> Reader string string)) +(defun (at rd what) + (str "line " (int->string (record-ref rd row)) + ", column " (int->string (record-ref rd col)) + ": " what)) + +(: (-> Reader string)) +(defun (found rd) + (def c (held-code rd)) + (cond + ((= c spent) "the document ended") + ((< c 128) (str "'" (list->string (list (int->char c))) "'")) + (else + (def n (sequence-length rd)) + (if (= n 0) + not-utf-8 + (str "'" (list->string (list (held-character rd n))) "'"))))) + +;; The refusal of a string or a name with bytes in it that are not UTF-8. +(: (-> Reader string)) +(defun (not-utf-8-here rd) (at rd "text has to be UTF-8, and this byte is not")) + +(: (-> Reader string string)) +(defun (too-deep rd what) + (at rd (str "this " what " nests deeper than " (int->string bjodat-max-depth) + ", which is as far as reading goes"))) + +;; --------------------------------------------------------------------------- +;; Strings +;; --------------------------------------------------------------------------- + +(: (-> char int)) +(defun (hex-digit c) + (cond + ((and (char>=? c #\0) (char<=? c #\9)) (- (char->int c) 48)) + ((and (char>=? c #\a) (char<=? c #\f)) (- (char->int c) 87)) + ((and (char>=? c #\A) (char<=? c #\F)) (- (char->int c) 55)) + (else -1))) + +;; The escapes of a Bjolang string, read by the rules `readEscape` in the +;; compiler's Lexer.fs reads them by: +;; +;; - `\n`, `\t`, `\r`, `\"` and `\\`. +;; - `\u{H}`: the character of 1 to 6 hex digits, a Unicode scalar value. +;; - A backslash that ends its line, spaces and tabs after it aside, continues +;; the string on the next line. That line's spaces and tabs up to +;; `text-col`, the column the string's text starts at, stand for nothing. +;; - Any other escape is refused, at its backslash. +(: (-> Reader StringBuilder int (Result string bool))) +(defun (read-escape! rd sb text-col) + (def row (record-ref rd row)) + (def col (record-ref rd col)) + (def refuse (fun (what) (Err (str "line " (int->string row) ", column " (int->string col) ": " what)))) + (step! rd) + (def c (held-code rd)) + (def add (fun (code) (stringbuilder-add-byte! sb code) (step! rd) (Ok #t))) + (cond + ((= c spent) (Err (at rd "the document ended inside a string"))) + ((= c 110) (add 10)) + ((= c 116) (add 9)) + ((= c 114) (add 13)) + ((= c 34) (add 34)) + ((= c 92) (add 92)) + ((= c 117) (read-code-point! rd sb refuse)) + ((or (= c 32) (= c 9) (= c 10) (= c 13)) (read-continuation! rd refuse text-col)) + (else + (refuse (str (if (< c 128) (str "\\" (list->string (list (int->char c)))) (str "a backslash before " (found rd))) + " is not an escape. A string's escapes are \\n, \\t, \\r, \\\", \\\\ and \\u{...}. " + "For a backslash, write \\\\, or write the text as a raw string"))))) + +;; `\u{H}`, at its `u`. +(: (-> Reader StringBuilder (-> string (Result string bool)) (Result string bool))) +(defun (read-code-point! rd sb refuse) + (def shape "\\u is written \\u{...}, with 1 to 6 hex digits between the braces, as in \\u{E9} or \\u{1F600}") + (step! rd) + (cond + ((not (= (held-code rd) 123)) (refuse shape)) + (else + (step! rd) + (let go ((digits "") (n 0)) + (def d (hex-digit (peek rd))) + (cond + ((and (= (held-code rd) 125) (> (string-length digits) 0)) + (step! rd) + (cond + ((or (> n 1114111) (and (>= n 55296) (<= n 57343))) + (refuse (str "\\u{" digits "} is not a Unicode scalar value. A surrogate, D800 to DFFF, " + "is half of a UTF-16 pair, and nothing is above 10FFFF"))) + (else (stringbuilder-add! sb (int->char n)) (Ok #t)))) + ((and (>= d 0) (< (string-length digits) 6)) + (def digit (peek rd)) + (step! rd) + (go (str digits (list->string (list digit))) (+ (* n 16) d))) + (else (refuse shape))))))) + +;; A backslash before a space, a tab or a line break, at that character: the +;; end of the line, and the margin of the next, stand for nothing. +(: (-> Reader (-> string (Result string bool)) int (Result string bool))) +(defun (read-continuation! rd refuse text-col) + (let blanks () + (def c (held-code rd)) + (when (or (= c 32) (= c 9)) + (step! rd) + (blanks))) + (def broke + (cond + ((= (held-code rd) 10) (newline! rd) #t) + ((= (held-code rd) 13) + (step! rd) + (cond + ((= (held-code rd) 10) (newline! rd) #t) + (else #f))) + (else #f))) + (cond + ((not broke) + (refuse (str "a backslash before a space is an escape only at the end of a line, where it " + "continues the string on the next. For a backslash, write \\\\"))) + (else + (let margin () + (def c (held-code rd)) + (when (and (< (record-ref rd col) text-col) (or (= c 32) (= c 9))) + (step! rd) + (margin))) + (Ok #t)))) + +;; A string, raw or escaped. `""` is the empty string, and a third quote after +;; it opens a raw one. +(: (-> Reader (Result string string))) +(defun (read-text rd) + (step! rd) + (cond + ((not (= (held-code rd) 34)) (read-escaped-text rd)) + (else + (step! rd) + (if (= (held-code rd) 34) + (read-raw-text rd) + (Ok ""))))) + +;; --------------------------------------------------------------------------- +;; Raw strings +;; --------------------------------------------------------------------------- +;; `"""..."""`, read by the rules `readRawString` in the compiler's Lexer.fs +;; reads it by, which Docs/Syntax.org describes. The two have to agree, and +;; TestFiles/263_raw_strings.bjo reads one file with both to show that they do. + +;; Steps over a run of quotes, and answers how long it was. +(: (-> Reader int)) +(defun (skip-quotes! rd) + (let go ((k 0)) + (cond + ((= (held-code rd) 34) (step! rd) (go (+ k 1))) + (else k)))) + +(: (-> StringBuilder int void)) +(defun (add-quotes! sb k) + (loop (:for _ (range 0 k)) + (:do (stringbuilder-add-byte! sb 34)))) + +(: (-> Reader int int string)) +(defun (too-many-quotes rd n k) + (at rd (str "a raw string opened with " (int->string n) " quotes has " (int->string k) + " in a row. Open and close it with more quotes than it holds in a row"))) + +;; The held quote is the third of the opening ones. +(: (-> Reader (Result string string))) +(defun (read-raw-text rd) + (def n (+ 2 (skip-quotes! rd))) + (def sb (buffer rd)) + (let spaces () + (def c (held-code rd)) + (when (or (= c 32) (= c 9)) + (stringbuilder-add-unit! sb c) + (step! rd) + (spaces))) + (cond + ((= (held-code rd) 10) (newline! rd) (read-raw-lines rd n)) + ((= (held-code rd) 13) + (step! rd) + (cond + ((= (held-code rd) 10) (newline! rd) (read-raw-lines rd n)) + (else (stringbuilder-add-byte! sb 13) (read-raw-line rd n sb)))) + (else (read-raw-line rd n sb)))) + +;; A raw string on the line of its opening quotes, which ends on that line. +(: (-> Reader int StringBuilder (Result string string))) +(defun (read-raw-line rd n sb) + (let go () + (def c (held-code rd)) + (cond + ((= c spent) (Err (at rd "the document ended inside a string"))) + ((= c 10) + (Err (at rd (str "a raw string that begins on the line of its opening quotes ends on that line. " + "One over several lines begins on the line after them")))) + ((= c 34) + (def k (skip-quotes! rd)) + (cond + ((= k n) (Ok (stringbuilder->string sb))) + ((> k n) (Err (too-many-quotes rd n k))) + (else (add-quotes! sb k) (go)))) + ((add-held! rd sb c) (go)) + (else (Err (not-utf-8-here rd)))))) + +;; A raw string over several lines. Each line is kept as it was written, with +;; the row it is on, until the closing quotes say what the margin is. +(: (-> Reader int (Result string string))) +(defun (read-raw-lines rd n) + (let next-line ((lines Nil)) + (def row (record-ref rd row)) + (def sb (stringbuilder-empty)) + (let spaces () + (def c (held-code rd)) + (when (or (= c 32) (= c 9)) + (stringbuilder-add-unit! sb c) + (step! rd) + (spaces))) + (def lead (skip-quotes! rd)) + (cond + ((= lead n) (strip-margin (stringbuilder->string sb) (list-reverse lines))) + ((> lead n) (Err (too-many-quotes rd n lead))) + (else + (add-quotes! sb lead) + (let go () + (def c (held-code rd)) + (cond + ((= c spent) + (Err (at rd (str "the document ended inside a raw string, which closes with " + (int->string n) " quotes on a line of their own")))) + ((= c 10) + (newline! rd) + (next-line (Cons (Tuple row (stringbuilder->string sb)) lines))) + ((= c 13) + (step! rd) + (cond + ((= (held-code rd) 10) + (newline! rd) + (next-line (Cons (Tuple row (stringbuilder->string sb)) lines))) + (else (stringbuilder-add-byte! sb 13) (go)))) + ((= c 34) + (def k (skip-quotes! rd)) + (cond + ((> k n) (Err (too-many-quotes rd n k))) + ((= k n) + (Err (at rd (str "a raw string closes only on a line of its own. For " (int->string n) + " quotes in the text, open and close it with more")))) + (else (add-quotes! sb k) (go)))) + ((add-held! rd sb c) (go)) + (else (Err (not-utf-8-here rd))))))))) + +(: (-> string bool)) +(defun (blank? s) + (none? (string-find #(not (or (char=? &1 #\space) (char=? &1 #\tab))) s))) + +;; What follows `prefix` in `s`, which starts with it. +(: (-> string string string)) +(defun (after-prefix s prefix) + (let go ((c (string-cursor-start s)) (n (string-count prefix))) + (if (= n 0) + (substring s c (string-cursor-end s)) + (go (string-cursor-next s c) (- n 1))))) + +;; The margin is the whitespace before the closing quotes. It comes off every +;; line, and a line with text that does not begin with it is refused, since +;; cutting it anyway would cut text. +(: (-> string (List (Tuple int string)) (Result string string))) +(defun (strip-margin margin lines) + (def out (stringbuilder-empty)) + (let go ((left lines) (first #t)) + (match left + (Nil (Ok (stringbuilder->string out))) + ((Cons (Tuple row s) more) + (unless first (stringbuilder-add-byte! out 10)) + (cond + ((string-starts-with? s margin) + (stringbuilder-add-string! out (after-prefix s margin)) + (go more #f)) + ((blank? s) (go more #f)) + (else + (Err (str "line " (int->string row) ", column 1: this line of a raw string does not " + "begin with its margin. The whitespace before the closing quotes is removed " + "from every line, so every line with text in it begins with that whitespace")))))))) + +;; Where a run of a string from `start` stops, 32 or 16 bytes at a time: at +;; the first `"`, `\` or newline, or at the last whole block before `limit`. +;; Also how many characters are before it, which is the bytes that are not a +;; continuation byte (10xxxxxx), whether one of them is 128 or more, and +;; whether a stop was found. The count is the characters' count only when the +;; bytes are UTF-8, which `take-run!` checks when one is 128 or more. +(: (-> (Array byte) int int (Tuple int int bool bool))) +(defun (scan-run b start limit) + (if simd256-accelerated? + (scan-run/256 b start limit) + (scan-run/128 b start limit))) + +(: (-> (Array byte) int int (Tuple int int bool bool))) +(defun (scan-run/256 b start limit) + (let go ((i start) (chars 0) (high #f)) + (if (> (+ i 32) limit) + (Tuple i chars high #f) + (let () + (def v (simd256-load b i)) + (def stops (simd-mask-bits (simd-or (simd-or (simd-eq v (simd256-splat 34uy)) + (simd-eq v (simd256-splat 92uy))) + (simd-eq v (simd256-splat 10uy))))) + (def conts (simd-mask-bits (simd-eq (simd-and v (simd256-splat 192uy)) (simd256-splat 128uy)))) + (def highs (simd-mask-bits (simd-ge v (simd256-splat 128uy)))) + (if (= stops 0) + (go (+ i 32) (+ chars (- 32 (popcount conts))) (or high (not (= highs 0)))) + (let () + ;; The lanes before the first stop. + (def t (trailing-zeros stops)) + (def before (- (shift-left 1 t) 1)) + (Tuple (+ i t) + (+ chars (- t (popcount (bitwise-and conts before)))) + (or high (not (= (bitwise-and highs before) 0))) + #t))))))) + +(: (-> (Array byte) int int (Tuple int int bool bool))) +(defun (scan-run/128 b start limit) + (let go ((i start) (chars 0) (high #f)) + (if (> (+ i 16) limit) + (Tuple i chars high #f) + (let () + (def v (simd128-load b i)) + (def stops (simd-mask-bits (simd-or (simd-or (simd-eq v (simd128-splat 34uy)) + (simd-eq v (simd128-splat 92uy))) + (simd-eq v (simd128-splat 10uy))))) + (def conts (simd-mask-bits (simd-eq (simd-and v (simd128-splat 192uy)) (simd128-splat 128uy)))) + (def highs (simd-mask-bits (simd-ge v (simd128-splat 128uy)))) + (if (= stops 0) + (go (+ i 16) (+ chars (- 16 (popcount conts))) (or high (not (= highs 0)))) + (let () + (def t (trailing-zeros stops)) + (def before (- (shift-left 1 t) 1)) + (Tuple (+ i t) + (+ chars (- t (popcount (bitwise-and conts before)))) + (or high (not (= (bitwise-and highs before) 0))) + #t))))))) + +;; Copies the run of a string that needs no attention, from where the reader +;; is to the first `"`, `\`, newline, byte that is not UTF-8 or end of the +;; chunk, into `sb` in one piece, and moves past it. Whatever stopped the run +;; is left for `read-escaped-text` to look at. +;; +;; The run is found a block at a time by `scan-run`, and the bytes left +;; before `limit` one at a time. A run with a byte of 128 or more is checked +;; as UTF-8 in one call. A run that is not UTF-8, or that the end of the chunk +;; cuts inside a character, is read again by `take-run/checked!`, which stops +;; before the byte at fault, so that the error and its column are where they +;; were. +(: (-> Reader StringBuilder void)) +(defun (take-run! rd sb) + (def b (record-ref rd bytes)) + (def start (record-ref rd next)) + (def limit (record-ref rd limit)) + (def (Tuple fast-end fast-chars fast-high stopped) (scan-run b start limit)) + (def (Tuple end chars high) + (if stopped + (Tuple fast-end fast-chars fast-high) + (let go ((i fast-end) (chars fast-chars) (high fast-high)) + (if (>= i limit) + (Tuple i chars high) + (let ((c (cast int (array-ref b i)))) + (cond + ((or (= c 34) (= c 92) (= c 10)) (Tuple i chars high)) + ((< c 128) (go (+ i 1) (+ chars 1) high)) + ((continuation? c) (go (+ i 1) chars #t)) + (else (go (+ i 1) (+ chars 1) #t)))))))) + (cond + ((and high (not (bytes-utf-8? b start end))) (take-run/checked! rd sb)) + ((> end start) + (stringbuilder-add-bytes! sb b start end) + (record-set! rd + (next end) + (col (+ (record-ref rd col) chars)))) + (else unit))) + +;; `take-run!` a byte at a time, each character checked as UTF-8: the run +;; stops before a byte that is not, and before a character the end of the +;; chunk cuts. +(: (-> Reader StringBuilder void)) +(defun (take-run/checked! rd sb) + (def b (record-ref rd bytes)) + (def start (record-ref rd next)) + (def limit (record-ref rd limit)) + (def (Tuple end chars) + (let go ((i start) (chars 0)) + (if (>= i limit) + (Tuple i chars) + (let ((c (cast int (array-ref b i)))) + (cond + ((or (= c 34) (= c 92) (= c 10)) (Tuple i chars)) + ((< c 128) (go (+ i 1) (+ chars 1))) + (else + (let ((n (sequence-length-at b i limit))) + (if (= n 0) (Tuple i chars) (go (+ i n) (+ chars 1)))))))))) + (when (> end start) + (stringbuilder-add-bytes! sb b start end) + (record-set! rd + (next end) + (col (+ (record-ref rd col) chars))))) + +;; An escaped string, after its opening quote. The bytes between the ones +;; that need a look are copied in runs by `take-run!`. +(: (-> Reader (Result string string))) +(defun (read-escaped-text rd) + (def sb (buffer rd)) + ;; The column after the opening quote, which a continued line lines up with. + (def text-col (record-ref rd col)) + (let go () + (take-run! rd sb) + (cond + ((at-end? rd) (Err (at rd "the document ended inside a string"))) + (else + (def c (held-code rd)) + (cond + ((= c 34) (step! rd) (Ok (stringbuilder->string sb))) + ((= c 92) (match (read-escape! rd sb text-col) + ((Err e) (Err e)) + ((Ok _) (go)))) + ((= c 10) + (stringbuilder-add-unit! sb c) + (newline! rd) + (go)) + ((add-held! rd sb c) (go)) + (else (Err (not-utf-8-here rd)))))))) + +;; --------------------------------------------------------------------------- +;; Numbers +;; --------------------------------------------------------------------------- + +(: (-> Reader StringBuilder int)) +(defun (take-digits! rd sb) + (let go ((n 0)) + (if (digit-code? (held-code rd)) + (begin + (stringbuilder-add-unit! sb (held-code rd)) + (step! rd) + (go (+ n 1))) + n))) + +;; The largest magnitude a long holds, and the one a negative long holds. +(: ulong) +(def long-max-magnitude 9223372036854775807UL) + +(: ulong) +(def long-min-magnitude 9223372036854775808UL) + +;; A digit of `radix`, or -1. +(: (-> int char int)) +(defun (radix-digit radix c) + (def d (hex-digit c)) + (if (< d radix) d -1)) + +;; `0x` and `0b`. The `0` has been stepped over and the marker is held. +;; +;; The digits are a magnitude, read as a ulong. `Convert.ToInt64` reads 16 hex +;; digits as a bit pattern, so `0xFFFFFFFFFFFFFFFF` was -1. Here a magnitude +;; above `long-max-magnitude` does not fit in a long, except as the magnitude +;; of `-0x8000000000000000`. +(: (-> Reader bool int (Result string Bjodat))) +(defun (read-radix rd neg radix) + (step! rd) + (def sb (buffer rd)) + (let go ((n 0)) + (if (and (not (at-end? rd)) (>= (radix-digit radix (peek rd)) 0)) + (begin (stringbuilder-add-unit! sb (held-code rd)) (step! rd) (go (+ n 1))) + (cond + ((= n 0) + (Err (at rd (str "this number has no digits after its radix, and there is " + (found rd))))) + ((and (= radix 2) (digit-code? (held-code rd))) + (Err (at rd (str "a binary number has the digits 0 and 1, and this is " + (found rd))))) + (else + (def unsigned? (ulong-suffix? rd)) + (match (try (clr-radix-ulong (stringbuilder->string sb) radix) + #:catch (System.OverflowException)) + ((Err _) + (Err (at rd (if unsigned? + "this number does not fit in a ulong" + "this number does not fit in a long")))) + ((Ok v) + (cond + ((and unsigned? neg) (Err (at rd "a ulong is not negative"))) + (unsigned? (Ok (BjoULong v))) + ((and neg (= v long-min-magnitude)) (Ok (BjoInt (- -9223372036854775807L 1L)))) + ((> v long-max-magnitude) (Err (at rd "this number does not fit in a long"))) + (neg (Ok (BjoInt (- 0L (cast long v))))) + (else (Ok (BjoInt (cast long v)))))))))))) + +;; Steps over `UL` when it is held. A `U` without the `L` is refused by what +;; reads next, as any other character after a number is. +(: (-> Reader bool)) +(defun (ulong-suffix? rd) + (cond + ((and (= (held-code rd) 85) (ensure! rd 2) + (= (cast int (array-ref (record-ref rd bytes) (+ (record-ref rd next) 1))) 76)) + (step! rd) + (step! rd) + #t) + (else #f))) + +(: (-> Reader StringBuilder (Result string bool))) +(defun (scan-fraction! rd sb) + (if (not (= (held-code rd) 46)) + (Ok #f) + (begin + (stringbuilder-add-byte! sb 46) + (step! rd) + (if (= (take-digits! rd sb) 0) + (Err (at rd (str "a number needs a digit after its decimal point, and this is " + (found rd)))) + (Ok #t))))) + +(: (-> Reader StringBuilder (Result string bool))) +(defun (scan-exponent! rd sb) + (def c (held-code rd)) + (cond + ((not (or (= c 101) (= c 69))) (Ok #f)) + (else + (stringbuilder-add-byte! sb 101) + (step! rd) + ;; `s` is bound rather than read three times, and it is not only the + ;; calls: with a name, `(or (= s 43) (= s 45))` is emitted as a + ;; ternary, and with the calls inlined the operands stop being simple + ;; and it becomes an if/else over a temporary, to keep the short + ;; circuit in the right order. + (def s (held-code rd)) + (when (or (= s 43) (= s 45)) + (stringbuilder-add-unit! sb s) + (step! rd)) + (if (= (take-digits! rd sb) 0) + (Err (at rd (str "a number needs a digit after its exponent, and this is " + (found rd)))) + (Ok #t))))) + +;; A digit is held. The sign, if there was one, has been stepped over already — +;; which is what tells `-` the symbol apart from `-1` the number, since deciding +;; needs a character of lookahead the reader does not have until it has moved. +;; +;; A number ends at whitespace, at a delimiter or at the end, as source +;; tokenises it. `1abc`, `1.2.3` and `0x1g` are refused. Before, each was read +;; as two values. +(: (-> Reader bool (Result string Bjodat))) +(defun (read-magnitude rd neg) + (def (Ok v) (read-number rd neg) :propagate) + (if (symbol-code? (held-code rd)) + (Err (at rd (str "a number ends at a space or a delimiter, and this is " (found rd)))) + (Ok v))) + +(: (-> Reader bool (Result string Bjodat))) +(defun (read-number rd neg) + (cond + ((= (held-code rd) 48) + (step! rd) + (def c (held-code rd)) + (cond + ((or (= c 120) (= c 88)) (read-radix rd neg 16)) + ((or (= c 98) (= c 66)) (read-radix rd neg 2)) + (else + (def sb (signed-buffer rd neg)) + (stringbuilder-add-byte! sb 48) + (finish-number rd neg sb)))) + (else (finish-number rd neg (signed-buffer rd neg))))) + +;; The buffer, with the `-` in it for a negative number. The sign is parsed +;; with the digits: `-9223372036854775808` fits in a long and its magnitude +;; does not, and `-0.0` is a zero with a sign, which subtracting from 0.0 loses. +(: (-> Reader bool StringBuilder)) +(defun (signed-buffer rd neg) + (def sb (buffer rd)) + (when neg (stringbuilder-add-byte! sb 45)) + sb) + +(: (-> Reader bool StringBuilder (Result string Bjodat))) +(defun (finish-number rd neg sb) + (ignore (take-digits! rd sb)) + (def (Ok frac?) (scan-fraction! rd sb) :propagate) + (def (Ok exp?) (scan-exponent! rd sb) :propagate) + (cond + ;; 1.50M: a decimal keeps the scale it was written with. + ((= (held-code rd) 77) + (step! rd) + (if exp? + (Err (at rd "a decimal is written without an exponent")) + (match (try (string->decimal (stringbuilder->string sb)) + #:catch (System.OverflowException)) + ((Ok d) (Ok (BjoDecimal d))) + ((Err _) (Err (at rd "this number does not fit in a decimal")))))) + ((= (held-code rd) 73) + (step! rd) + (if (or frac? exp?) + (Err (at rd "a bigint is a whole number: it has no decimal point and no exponent")) + (Ok (BjoBigInt (string->bigint (stringbuilder->string sb)))))) + ((ulong-suffix? rd) + (cond + ((or frac? exp?) (Err (at rd "a ulong is a whole number: it has no decimal point and no exponent"))) + (neg (Err (at rd "a ulong is not negative"))) + (else + (match (try (clr-to-ulong (stringbuilder->string sb)) #:catch (System.OverflowException)) + ((Ok n) (Ok (BjoULong n))) + ((Err _) (Err (at rd "this number does not fit in a ulong"))))))) + ((or frac? exp?) (Ok (BjoFloat (clr-to-double sb)))) + ;; Wider than a long is still a number. + (else + (match (try (clr-to-long sb) #:catch (System.OverflowException)) + ((Ok n) (Ok (BjoInt n))) + ((Err _) (Ok (BjoFloat (clr-to-double sb)))))))) + +;; --------------------------------------------------------------------------- +;; Symbols, keywords, and what a `#` begins +;; --------------------------------------------------------------------------- + +(: (-> Reader StringBuilder int)) +;; A byte that is not UTF-8 ends the name, and is refused by what reads next. +(defun (take-name! rd sb) + (let go ((n 0)) + (def c (held-code rd)) + (if (and (symbol-code? c) (add-held! rd sb c)) + (go (+ n 1)) + n))) + +;; NaN and the infinities, spelled as Scheme spells them. They read as names +;; do, so a name is checked for them before it becomes a symbol. +(: (-> string (Option double))) +(defun (special-double name) + (cond + ((= name "+inf.0") (Some (/ 1.0 0.0))) + ((= name "-inf.0") (Some (/ -1.0 0.0))) + ((or (= name "+nan.0") (= name "-nan.0")) (Some (/ 0.0 0.0))) + (else None))) + +;; A name as a symbol, unless it is one of the doubles above. +(: (-> Reader StringBuilder Bjodat)) +(defun (name-value rd sb) + (def name (cached-name (record-ref rd names) sb)) + (match (special-double name) + ((Some d) (BjoFloat d)) + (None (BjoSym (string->symbol name))))) + +(: (-> Reader (Result string Bjodat))) +(defun (read-symbol rd) + (def sb (buffer rd)) + (if (= (take-name! rd sb) 0) + (Err (at rd (str (found rd) " does not begin a value"))) + (Ok (name-value rd sb)))) + +;; `:name`. The colon is held. +(: (-> Reader (Result string Bjodat))) +(defun (read-keyword rd) + (step! rd) + (def sb (buffer rd)) + (if (= (take-name! rd sb) 0) + (Err (at rd (str "a keyword needs a name after its colon, and this is " + (found rd)))) + (Ok (cached-keyword rd sb)))) + +;; The keyword in the buffer, as a value. A map repeats its keys, and a key read +;; before is the value already made for it: no string, no lookup in the table +;; of interned keywords, and no allocation. `string->keyword` makes a .NET +;; string of the name and looks that up, each time it is called. Two keywords +;; that share a slot take turns in it and are made each time. +(: (-> Reader StringBuilder Bjodat)) +(defun (cached-keyword rd sb) + (def keys (record-ref rd keys)) + (def i (name-slot sb (stringbuilder-length sb))) + (cond + ((stringbuilder=? sb (array-ref keys i)) (array-ref (record-ref rd key-values) i)) + (else + (def name (stringbuilder->string sb)) + (def v (BjoKey (string->keyword name))) + (array-set! keys i name) + (array-set! (record-ref rd key-values) i v) + v))) + +;; The characters source spells with a word. +(: (-> string int)) +(defun (named-char name) + (cond + ((= name "newline") 10) + ((= name "space") 32) + ((= name "tab") 9) + ((= name "return") 13) + ((= name "nul") 0) + (else -1))) + +;; `#\c`, `#\newline`. The backslash is held. +(: (-> Reader (Result string Bjodat))) +(defun (read-char-literal rd) + (step! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended inside a character"))) + ((= (sequence-length rd) 0) (Err (not-utf-8-here rd))) + (else + (def lead (held-code rd)) + (def first (if (< lead 128) (int->char lead) (held-character rd (sequence-length rd)))) + (def sb (buffer rd)) + ;; `#\` and a line feed is the line feed, and the next line begins. + (if (= lead 10) + (begin (stringbuilder-add-unit! sb lead) (newline! rd)) + (ignore (add-held! rd sb lead))) + ;; A name is a run of symbol characters, as in the compiler's lexer. After + ;; `#\(`, `#\ ` or `#\` and a line feed the literal is that one + ;; character, and what follows is the next value. + ;; `n` is bound rather than inlined into the test: a name compares + ;; straight into the `if`, where the call needs a temporary for its + ;; result and then a second one for the comparison. + (def n (if (symbol-code? lead) (take-name! rd sb) 0)) + (cond + ((= n 0) (Ok (BjoChar first))) + (else + (def name (stringbuilder->string sb)) + (def code (named-char name)) + (if (< code 0) + (Err (at rd (str "#\\" name " is not a character"))) + (Ok (BjoChar (int->char code))))))))) + +;; `#t`, `#f`, `#\c`, `#:keyword`. The `#` is held. +(: (-> Reader (Result string Bjodat))) +(defun (read-hash rd) + (step! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended after '#'"))) + (else + (def c (peek rd)) + (cond + ((char=? c #\t) (step! rd) (Ok (BjoBool #t))) + ((char=? c #\f) (step! rd) (Ok (BjoBool #f))) + ((char=? c #\\) (read-char-literal rd)) + ((char=? c #\:) (read-keyword rd)) + (else (Err (at rd (str "'#' begins #t, #f, #\\c or #:keyword, and this is " + (found rd))))))))) + +;; --------------------------------------------------------------------------- +;; Lists, vecs and maps +;; --------------------------------------------------------------------------- + +(: (-> Reader int int char string (Result string (Vec Bjodat)))) +(defun (read-sequence rd depth closer-code closer what) + (step! rd) + (def mark (record-ref rd top)) + (let go () + (skip-space! rd) + (cond + ((at-end? rd) + (Err (at rd (str "the document ended inside a " what)))) + ((= (held-code rd) closer-code) + (step! rd) + (Ok (harvest! rd mark))) + (else + (match (read-value rd (+ depth 1)) + ((Err e) (Err e)) + ((Ok v) (push! rd v) (go))))))) + +;; A key is any value but a list. Refusing one is what makes `{(` unambiguous: +;; a `(` where a key would go means the map is written in the parenthesised +;; form, and nothing else. +(: (-> Reader int (Result string Bjodat))) +(defun (read-key rd depth) + (match (read-value rd depth) + ((Err e) (Err e)) + ((Ok (BjoList _)) (Err (at rd "a list is not a key"))) + ((Ok v) (Ok v)))) + +;; `{(k v) (k v)}`. The opening brace has been stepped over and a `(` is held. +(: (-> Reader int int (Result string Bjodat))) +(defun (read-paired-map rd depth mark) + (let go () + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended inside a map"))) + ((= (held-code rd) 125) + (step! rd) + (Ok (BjoMap (harvest! rd mark)))) + ((not (= (held-code rd) 40)) + (Err (at rd (str "this map is written (key value), so its entries all are, " + "and here there is " (found rd))))) + (else + (step! rd) + (def (Ok k) (read-key rd (+ depth 1)) :propagate) + (push! rd k) + (def (Ok v) (read-value rd (+ depth 1)) :propagate) + (push! rd v) + (skip-space! rd) + (if (= (held-code rd) 41) + (begin (step! rd) (go)) + (Err (at rd (str "an entry holds a key and a value, and then there is " + (found rd))))))))) + +;; `{k v k v}`. The opening brace has been stepped over before reaching this fun. +(: (-> Reader int int (Result string Bjodat))) +(defun (read-flat-map rd depth mark) + (let go () + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended inside a map"))) + ((= (held-code rd) 125) + (step! rd) + (Ok (BjoMap (harvest! rd mark)))) + ((= (held-code rd) 40) + (Err (at rd "this map is written flat, so a '(' here is an entry of the other form"))) + (else + (def (Ok k) (read-key rd (+ depth 1)) :propagate) + (push! rd k) + (skip-space! rd) + (if (or (at-end? rd) (= (held-code rd) 125)) + (Err (at rd "this map has a key with no value")) + (match (read-value rd (+ depth 1)) + ((Err e) (Err e)) + ((Ok v) (push! rd v) (go)))))))) + +;; Which form a map is written in is decided by the first thing inside it, and +;; then every entry has to agree. +(: (-> Reader int (Result string Bjodat))) +(defun (read-map rd depth) + (step! rd) + (def mark (record-ref rd top)) + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended inside a map"))) + ((= (held-code rd) 125) + (step! rd) + (Ok (BjoMap (harvest! rd mark)))) + ((= (held-code rd) 40) (read-paired-map rd depth mark)) + (else (read-flat-map rd depth mark)))) + +;; --------------------------------------------------------------------------- +;; A value +;; --------------------------------------------------------------------------- + +(: (-> Reader int (Result string Bjodat))) +(defun (read-value rd depth) + (skip-space! rd) + (cond + ((> depth bjodat-max-depth) (Err (too-deep rd "value"))) + ((at-end? rd) (Err (at rd "the document ended where a value was expected"))) + (else + (def c (peek rd)) + (cond + ((char=? c #\() + (match (read-sequence rd depth 41 #\) "list") + ((Err e) (Err e)) + ((Ok v) (Ok (BjoList (vec->list v)))))) + ((char=? c #\[) + (match (read-sequence rd depth 93 #\] "vec") + ((Err e) (Err e)) + ((Ok v) (Ok (BjoVec v))))) + ((char=? c #\{) (read-map rd depth)) + ((char=? c #\") + (match (read-text rd) + ((Err e) (Err e)) + ((Ok s) (Ok (BjoStr s))))) + ((char=? c #\#) (read-hash rd)) + ((char=? c #\:) (read-keyword rd)) + ((char=? c #\)) (Err (at rd "')' closes nothing here"))) + ((char=? c #\]) (Err (at rd "']' closes nothing here"))) + ((char=? c #\}) (Err (at rd "'}' closes nothing here"))) + ((char=? c #\,) (Err (at rd "a comma is not a separator here"))) + ((char=? c #\') (Err (at rd "a symbol in bjodat is written without a quote"))) + ((digit-code? (held-code rd)) (read-magnitude rd #f)) + ;; `-` is a symbol of its own, and the start of a number when a digit + ;; follows it. Stepping over it first is what makes the difference + ;; visible; if no digit came, what was stepped over is the first + ;; character of a symbol. + ((char=? c #\-) + (step! rd) + (cond + ((digit-code? (held-code rd)) (read-magnitude rd #t)) + (else + (def sb (buffer rd)) + (stringbuilder-add-byte! sb 45) + (ignore (take-name! rd sb)) + (Ok (name-value rd sb))))) + (else (read-symbol rd)))))) + +;; --------------------------------------------------------------------------- +;; Reading a document +;; --------------------------------------------------------------------------- + +;; A byte order mark, EF BB BF, is skipped and is not a column. +(: (-> Reader void)) +(defun (skip-bom! rd) + (when (and (ensure! rd 3) + (let ((b (record-ref rd bytes)) (i (record-ref rd next))) + (and (= (cast int (array-ref b i)) 239) + (= (cast int (array-ref b (+ i 1))) 187) + (= (cast int (array-ref b (+ i 2))) 191)))) + (record-set! rd (next (+ (record-ref rd next) 3))))) + +(: (Array byte)) +(def no-bytes (make-array 0)) + +;; A reader for a port, and one for a string's own bytes. +(: (-> InputPort Reader)) +(defun (port-reader port) (reader (Some port) no-bytes)) + +(: (-> string Reader)) +(defun (string-reader text) (reader None (string->bytes text))) + +;; Exactly one value, and then the end. +(: (-> InputPort (Result string Bjodat))) +(defun (read-bjodat port) (one-value (port-reader port))) + +(: (-> Reader (Result string Bjodat))) +(defun (one-value rd) + (skip-bom! rd) + (match (read-value rd 1) + ((Err e) (Err e)) + ((Ok v) + (skip-space! rd) + (if (at-end? rd) + (Ok v) + (Err (at rd (str "the value ended here, and then there is " (found rd)))))))) + +(: (-> string (Result string Bjodat))) +(defun (parse-bjodat text) (one-value (string-reader text))) + +;; Every value in the file, in order. A file is a stream, so an empty one is an +;; empty list rather than a refusal. +(: (-> InputPort (Result string (List Bjodat)))) +(defun (read-bjodat-all port) (all-values (port-reader port))) + +(: (-> Reader (Result string (List Bjodat)))) +(defun (all-values rd) + (skip-bom! rd) + ;; Built front to back, so a value is one cons and there is no reverse. + (def out (listbuilder-empty)) + (let go () + (skip-space! rd) + (if (at-end? rd) + (Ok (listbuilder->list out)) + (match (read-value rd 1) + ((Err e) (Err e)) + ((Ok v) (listbuilder-add! out v) (go)))))) + +(: (-> string (Result string (List Bjodat)))) +(defun (parse-bjodat-all text) (all-values (string-reader text))) + +;; --------------------------------------------------------------------------- +;; Reading a record in one pass +;; --------------------------------------------------------------------------- +;; What `(text bjodat-codec)`'s generated decoders drive, so that a record can be +;; read without a `Bjodat` tree standing between the text and the fields. +;; +;; The tree is not free. A map costs, per entry, a name string built off the +;; buffer, an interned `Keyword`, a `BjoKey` node and two slots in a vec — and +;; then the decoder scans that vec once per field, and the whole of it is thrown +;; away the moment the record exists. A generated decoder knows its field names +;; when it is *compiled*, so it can compare the key against them in the buffer +;; and build none of it. +;; +;; The caller drives the loop rather than handing in a table, because the caller +;; is generated code that already knows what it is looking for, and a table +;; would be the thing this is trying not to build. So these are small and +;; awkward on purpose: they are not an API for people. +;; +;; What is *not* here is a typed value reader. A field's value still goes +;; through `read-value` and the field type's own `bjodat->` instance, which is +;; what keeps every instance — including one a user wrote — working, and every +;; error message identical to the tree decoder's. The entry is what this saves, +;; not the value. + +;; Steps over `{`, and answers whether this map's entries are parenthesised. +;; Which form it is in is decided by the first thing inside it, exactly as +;; `read-map` decides it, and then every entry has to agree. +(: (-> Reader string (Result string bool))) +(defun (open-map! rd owner) + (skip-space! rd) + (cond + ((at-end? rd) + (Err (at rd (str owner ": the document ended where a map was expected")))) + ((not (= (held-code rd) 123)) + (Err (at rd (str owner ": expected a map, and here there is " (found rd))))) + (else + (step! rd) + (skip-space! rd) + (Ok (= (held-code rd) 40))))) + +;; A key's name, taken into the buffer without being interned. +;; +;; A key that is not a keyword is read and dropped, and leaves the buffer empty +;; — which no field name can equal, so it takes the same path an unknown key +;; does. That matches the tree decoder, where `bjodat-ref` only ever matches a +;; `BjoKey` and a key of any other shape is simply never found. +;; +;; A key written `#:name` is the same keyword as `:name`, as `read-keyword` +;; reads it, so it takes the same path. +(: (-> Reader string (Result string bool))) +(defun (take-key! rd owner) + (skip-space! rd) + (when (and (= (held-code rd) 35) (ensure! rd 2) + (= (cast int (array-ref (record-ref rd bytes) (+ (record-ref rd next) 1))) 58)) + (step! rd)) + (cond + ((= (held-code rd) 58) + (step! rd) + (def sb (buffer rd)) + (if (= (take-name! rd sb) 0) + (Err (at rd (str owner ": a keyword needs a name after its colon, and this is " + (found rd)))) + (Ok #t))) + (else + (match (read-key rd 1) + ((Err e) (Err e)) + ((Ok _) + (stringbuilder-clear! (record-ref rd scratch)) + (Ok #t)))))) + +;; `#t` when a key is in the buffer and the reader stands at its value; `#f` +;; when the map closed. +(: (-> Reader string bool (Result string bool))) +(defun (next-key! rd owner paired) + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd (str owner ": the document ended inside a map")))) + ((= (held-code rd) 125) (step! rd) (Ok #f)) + ((and paired (not (= (held-code rd) 40))) + (Err (at rd (str owner ": this map is written (key value), so its entries all are, " + "and here there is " (found rd))))) + ((and (not paired) (= (held-code rd) 40)) + (Err (at rd (str owner ": this map is written flat, so a '(' here is an entry of the " + "other form")))) + (else + (when paired (step! rd)) + (match (take-key! rd owner) + ((Err e) (Err e)) + ((Ok _) + (skip-space! rd) + (if (or (at-end? rd) (= (held-code rd) 125)) + (Err (at rd (str owner ": this map has a key with no value"))) + (Ok #t))))))) + +;; The `)` of a parenthesised entry. Nothing at all in the flat form. +(: (-> Reader string bool (Result string bool))) +(defun (end-entry! rd owner paired) + (if (not paired) + (Ok #t) + (begin + (skip-space! rd) + (if (= (held-code rd) 41) + (begin (step! rd) (Ok #t)) + (Err (at rd (str owner ": an entry holds a key and a value, and then there is " + (found rd)))))))) + +;; Is the key in the buffer this name? The comparison the whole thing is for: +;; no string is built, and nothing is interned. +(: (-> Reader string bool)) +(defun (key=? rd name) + (def sb (record-ref rd scratch)) + (stringbuilder=? sb name)) + +;; A field's value, as a `Bjodat` for its own instance to convert. +(: (-> Reader (Result string Bjodat))) +(defun (field-value rd) (read-value rd 1)) + +;; An unknown key's value, read and dropped. It has to be *read*: a value is not +;; a token, and finding where it ends means parsing it. +(: (-> Reader (Result string bool))) +(defun (skip-value! rd) + (match (read-value rd 1) + ((Err e) (Err e)) + ((Ok _) (Ok #t)))) + +;; --------------------------------------------------------------------------- +;; Reading a record written as a tagged form +;; --------------------------------------------------------------------------- +;; `(package (name bjolang/http) (version "1.2.3"))` — the head symbol names the +;; type, and each clause is a field. +;; +;; This is the *parenthesised map* with two differences: a `(` opens it rather +;; than a `{`, and it carries a tag. So the loop a generated decoder runs is the +;; one `next-key!` and `end-entry!` already drive, and what is new here is only +;; the opener, the clause name being a bare symbol rather than a keyword, and +;; what an unknown clause costs to get past. +;; +;; **The shape never picks the type.** A `BjoList` headed by a symbol is a +;; tagged record where a tagged record is expected and a union case where a +;; union is, exactly as a `{...}` is a record in one field and a lookup table in +;; the next. The reader commits to neither and there is nothing here that could: +;; these functions are called by a decoder that already knows what it wants. + +;; Steps over `(` and the head symbol, and refuses a head that is not `tag`. +(: (-> Reader string string (Result string bool))) +(defun (open-form! rd owner tag) + (skip-space! rd) + (cond + ((at-end? rd) + (Err (at rd (str owner ": the document ended where (" tag " ...) was expected")))) + ((not (= (held-code rd) 40)) + (Err (at rd (str owner ": expected (" tag " ...), and here there is " (found rd))))) + (else + (step! rd) + (skip-space! rd) + (def sb (buffer rd)) + (cond + ((= (take-name! rd sb) 0) + (Err (at rd (str owner ": a form begins with the name of its type, and this is " + (found rd))))) + ;; The comparison the whole thing is for: the tag is matched against + ;; the buffer, so a form costs no string and interns nothing for its + ;; own name any more than it does for a field's. + ((key=? rd tag) (Ok #t)) + ;; Only here, on the way to refusing, is a string built. + (else + (Err (at rd (str owner ": expected (" tag " ...), and this is (" + (stringbuilder->string sb) " ...)")))))))) + +;; `#t` when a clause's field name sits in the buffer and the reader stands at +;; its first value; `#f` when the form closed. +(: (-> Reader string (Result string bool))) +(defun (next-field! rd owner) + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd (str owner ": the document ended inside a form")))) + ((= (held-code rd) 41) (step! rd) (Ok #f)) + ((not (= (held-code rd) 40)) + (Err (at rd (str owner ": a clause is written (name value), and here there is " + (found rd))))) + (else + (step! rd) + (skip-space! rd) + ;; Where the head begins, which is where a refusal points. + (def row (record-ref rd row)) + (def col (record-ref rd col)) + (if (field-name! rd) + (Ok #t) + ;; The tree decoder's sentence, `bjodat-clause-shape`'s, so that the + ;; two decoders refuse the same document in the same words. + (begin + (record-set! rd (row row) (col col)) + (Err (at rd (str owner ": a clause begins with a field name")))))))) + +;; Takes the name of a clause into the buffer, and answers whether it is one: a +;; name that `read-value` reads as a symbol. `#t`, `123`, `-1` and `+inf.0` are +;; runs of symbol characters too, and they are not symbols. The tree decoder +;; refuses a clause that does not begin with a symbol, and so does this. +(: (-> Reader bool)) +(defun (field-name! rd) + (def c (held-code rd)) + (def sb (buffer rd)) + (cond + ((or (= c 35) (digit-code? c)) #f) + ((and (= c 45) (ensure! rd 2) + (digit-code? (cast int (array-ref (record-ref rd bytes) (+ (record-ref rd next) 1))))) + #f) + ((= (take-name! rd sb) 0) #f) + ((or (= c 43) (= c 45)) + (not (or (stringbuilder=? sb "+inf.0") (stringbuilder=? sb "-inf.0") + (stringbuilder=? sb "+nan.0") (stringbuilder=? sb "-nan.0")))) + (else #t))) + +;; The `)` that closes a clause holding one value. +;; +;; The owner and the field name arrive apart and are joined only on the way to +;; refusing. Joined by the caller they would be one string built per field per +;; record — which is the cost this whole path exists to avoid — and dropped they +;; would leave a message that does not say which clause was wrong. The tree +;; decoder's `bjodat-one-value` says the same sentence under the same name. +(: (-> Reader string string (Result string bool))) +(defun (end-clause! rd owner name) + (skip-space! rd) + (if (= (held-code rd) 41) + (begin (step! rd) (Ok #t)) + (Err (at rd (str owner "." name ": a clause holds a field name and one value"))))) + +;; The value of a clause that holds one. A clause with no value, `(name)`, is +;; refused in `bjodat-one-value`'s words: `read-value` would refuse its `)` as +;; closing nothing. +(: (-> Reader string string (Result string Bjodat))) +(defun (clause-value rd owner name) + (skip-space! rd) + (if (= (held-code rd) 41) + (Err (at rd (str owner "." name ": a clause holds a field name and one value"))) + (read-value rd 1))) + +;; An unknown clause, read and dropped: the name has been taken, the values have +;; not. **Any arity**, and that is the point of it. A flat map's unknown key is +;; one value and done, but a clause may hold none, one or five, and a reader +;; that insisted on one would make every new clause a breaking change for every +;; older tool. This is what keeps a manifest forward-compatible. +(: (-> Reader (Result string bool))) +(defun (skip-clause! rd) + (let go () + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended inside a clause"))) + ((= (held-code rd) 41) (step! rd) (Ok #t)) + (else + (match (read-value rd 1) + ((Err e) (Err e)) + ((Ok _) (go))))))) + +;; Every value of a clause, in written order, for a `#:rest` field that takes +;; them all. The closing `)` is stepped over. +;; +;; The values arrive as `Bjodat` and are converted by the element type's own +;; instance, exactly as the single-value case does through `field-value`. That +;; is not laziness: `read-value` carries the depth budget, so a manifest nesting +;; `(package (depends (package (depends ...))))` five hundred deep is refused by +;; `bjodat-max-depth` rather than by the stack. A `#:rest` that recursed +;; straight back into a generated decoder would have no budget at all. +(: (-> Reader (Result string (List Bjodat)))) +(defun (clause-values! rd) + (def out (listbuilder-empty)) + (let go () + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended inside a clause"))) + ((= (held-code rd) 41) (step! rd) (Ok (listbuilder->list out))) + (else + (match (read-value rd 1) + ((Err e) (Err e)) + ((Ok v) (listbuilder-add! out v) (go))))))) + +;; One record, and then the end, for a decoder generated by `def/bjodat-type`. +;; +;; The decoder is declared `-?->` rather than `->`, and has to be: a written +;; arrow outranks the default, so a plain one would pin the generated decoder to +;; its *sync* copy — and then a bjoroutine reading a socket would call the copy +;; that parks the thread. The blocking lint says so in as many words, which is +;; how this was found. +(: (-> InputPort (-?-> Reader (Result string %a)) (Result string %a))) +(defun (read-one-pass port decode) (one-record (port-reader port) decode)) + +(: (-> Reader (-?-> Reader (Result string %a)) (Result string %a))) +(defun (one-record rd decode) + (skip-bom! rd) + (match (decode rd) + ((Err e) (Err e)) + ((Ok v) + (skip-space! rd) + (if (at-end? rd) + (Ok v) + (Err (at rd (str "the value ended here, and then there is " (found rd)))))))) + +(: (-> string (-?-> Reader (Result string %a)) (Result string %a))) +(defun (parse-one-pass text decode) (one-record (string-reader text) decode)) + +;; A `[...]` of them, which is what a manifest's list of dependencies is. +;; +;; A `List` rather than a `Vec` because it is built by consing and turned round +;; once, where a vec would be an RRB append per element. A caller that wants a +;; vec can say so in one call; a caller that folds over it once — which is most +;; of them — should not pay for a tree it does not index into. +(: (-> InputPort (-?-> Reader (Result string %a)) (Result string (List %a)))) +(defun (read-one-pass-all port decode) (all-records (port-reader port) decode)) + +(: (-> Reader (-?-> Reader (Result string %a)) (Result string (List %a)))) +(defun (all-records rd decode) + (skip-bom! rd) + (skip-space! rd) + (if (not (= (held-code rd) 91)) + (Err (at rd (str "expected a vec of values, and here there is " (found rd)))) + (begin + (step! rd) + ;; Built front to back, so a record is one cons and there is no reverse. + (def out (listbuilder-empty)) + (let go () + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended inside a vec"))) + ((= (held-code rd) 93) (step! rd) (Ok (listbuilder->list out))) + (else + (match (decode rd) + ((Err e) (Err e)) + ((Ok v) (listbuilder-add! out v) (go))))))))) + +(: (-> string (-?-> Reader (Result string %a)) (Result string (List %a)))) +(defun (parse-one-pass-all text decode) (all-records (string-reader text) decode)) + +;; --------------------------------------------------------------------------- +;; Writing +;; --------------------------------------------------------------------------- +;; The canonical spelling: maps flat, in the order they are held. + +(: (-> StringBuilder string void)) +(defun (escape-into! sb s) + (stringbuilder-add! sb #\") + (loop (:for c s) + (:do (let* ((n (char->int c))) + (cond + ((= n 34) (stringbuilder-add-string! sb "\\\"")) + ((= n 92) (stringbuilder-add-string! sb "\\\\")) + ((= n 10) (stringbuilder-add-string! sb "\\n")) + ((= n 9) (stringbuilder-add-string! sb "\\t")) + ((= n 13) (stringbuilder-add-string! sb "\\r")) + (else (stringbuilder-add! sb c)))))) + (stringbuilder-add! sb #\")) + +(: (-> StringBuilder char void)) +(defun (char-into! sb c) + (def n (char->int c)) + (stringbuilder-add-string! sb "#\\") + (cond + ((= n 10) (stringbuilder-add-string! sb "newline")) + ((= n 32) (stringbuilder-add-string! sb "space")) + ((= n 9) (stringbuilder-add-string! sb "tab")) + ((= n 13) (stringbuilder-add-string! sb "return")) + ((= n 0) (stringbuilder-add-string! sb "nul")) + (else (stringbuilder-add! sb c)))) + +;; A double with a fraction or an exponent, so that it reads back as a +;; `BjoFloat`: .NET writes 1.0 as "1", which the reader takes for an integer. +;; "1E+20" already has its exponent. NaN and the infinities are written as +;; Scheme writes them, which the reader reads back. +(: (-> double string)) +(defun (double-text d) + (cond + ((clr-nan? d) "+nan.0") + ((clr-positive-infinity? d) "+inf.0") + ((clr-negative-infinity? d) "-inf.0") + (else + (def s (double->string d)) + (if (or (string-contains? s ".") (string-contains? s "E") (string-contains? s "e")) + s + (str s ".0"))))) + +(: (-> StringBuilder Bjodat void)) +(defun (write-into! sb v) + (match v + ((BjoBool b) (stringbuilder-add-string! sb (if b "#t" "#f"))) + ((BjoInt n) (stringbuilder-add-string! sb (long->string n))) + ((BjoFloat d) (stringbuilder-add-string! sb (double-text d))) + ((BjoULong n) (stringbuilder-add-string! sb (str (displayed n) "UL"))) + ((BjoDecimal d) (stringbuilder-add-string! sb (str (decimal->string d) "M"))) + ((BjoBigInt n) (stringbuilder-add-string! sb (str (bigint->string n) "I"))) + ((BjoStr s) (escape-into! sb s)) + ((BjoChar c) (char-into! sb c)) + ((BjoSym s) (stringbuilder-add-string! sb (symbol->string s))) + ((BjoKey k) + (stringbuilder-add! sb #\:) + (stringbuilder-add-string! sb (keyword->string k))) + ((BjoList xs) + (stringbuilder-add! sb #\() + (loop (:with first #t #f) + (:for x xs) + (:do (unless first (stringbuilder-add! sb #\space)) + (write-into! sb x))) + (stringbuilder-add! sb #\))) + ((BjoVec xs) + (stringbuilder-add! sb #\[) + (loop (:with first #t #f) + (:for x xs) + (:do (unless first (stringbuilder-add! sb #\space)) + (write-into! sb x))) + (stringbuilder-add! sb #\])) + ((BjoMap entries) + (stringbuilder-add! sb #\{) + (loop (:with first #t #f) + (:for x entries) + (:do (unless first (stringbuilder-add! sb #\space)) + (write-into! sb x))) + (stringbuilder-add! sb #\})))) + +(: (-> Bjodat string)) +(defun (bjodat->string v) + (def sb (stringbuilder-empty)) + (write-into! sb v) + (stringbuilder->string sb)) +;; text/bjodat.bjo +;; +;; The bjodat data format: declare a record, get a reader and a writer. +;; +;; (import (text bjodat)) +;; +;; (def/bjodat-type +;; (: Dep (Record (: name string) +;; (: version string #:default "*") +;; (: pinned bool #:optional)))) +;; +;; (bjodat-parse-Dep "{:name \"http\" :version \"1.2\"}") ;; -> (Ok Dep) +;; (Dep->bjodat d) ;; -> Bjodat +;; +;; `bjodat-parse-` reads the text **straight into the record**. There is +;; no intermediate tree: the key is compared against the field names in the +;; reader's own buffer, so a record costs no key string, no interned `Keyword`, +;; no node and no map. That is the interface to reach for, and it is about a +;; fifth faster than going through a tree. +;; +;; The tree is still here and still useful — for data whose shape is not known +;; until run time, and as the thing a field's own decoder converts. It lives in +;; `(text bjodat-core)`, and the names this module re-exports below are enough +;; that most programs never need to name it. +;; +;; One import is enough. That was not true when this module was split out, and +;; the reason it was not is worth recording rather than leaving as a silence: I +;; believed a type could not be republished through a facade, wrote that in +;; three places, and was wrong. `re-export` takes a type now — it publishes the +;; declaring module's declaration under a spelling of this module's, so there +;; is one `Bjodat` and not two — and a union comes with its cases, so a +;; hand-written `->bjodat` instance can name `BjoInt` here as readily as it +;; could over there. See `re-export` in `Docs/MODULES.org`. +;; +;; A record is a map whose keys are keywords naming its fields. The keys are +;; emitted as keyword *literals* at expansion time, so decoding a field through +;; a tree compares interned references and never interns anything at run time. +;; +;; There is no `null`. If a field is allowed to be missing, mark it with `#:optional` +;; (which is just shorthand for `#:default None`). Since this format maps directly +;; to Bjolang values, missing fields are strictly represented as `None`. If an +;; `(Option %a)` must be explicitly written out in the file, `None` is encoded as +;; an empty list `()`. + +(export + ;; The generator. Everything else here exists to serve what it writes. + def/bjodat-type + ;; The traits, for a type of your own that the generator did not declare. + ->bjodat bjodat-> + ;; What generated field code calls. Exported because a macro template's names + ;; resolve against this module's scope. + bjodat-map bjodat-field bjodat-field/default + bjodat-read-field bjodat-need-field + bjodat-key bjodat-entries + ;; And the same, for a record written as a tagged form. + bjodat-clauses bjodat-clause-field bjodat-clause-field/default + bjodat-clause-rest bjodat-clause-rest/default + bjodat-read-clause bjodat-read-rest bjodat-need-clause + bjodat-clause bjodat-clause-many bjodat-clause-values bjodat-form-value) + +(import (std prelude)) +(import (std syntax-match)) +(import (text bjodat-core)) + +;; Every export is documented there, but the re-exports, whose docs are in +;; bjodat-core; the module doc is too. +(:doc #:file "bjodat.bjodoc") + +;; What `def/bjodat-type` writes has to resolve where it lands, and where it +;; lands is somebody else's module. So everything the template names travels +;; from here, whether or not this module wrote it. +;; +;; The two types are the reason the split used to cost every consumer a second +;; import: a generated signature reads `(-> Name Bjodat)` and `(-> Reader ...)`. +;; They are *this* module's spellings of `bjodat-core`'s declarations, not +;; copies of them — the key is still `bjodat_core`'s, so a value built through +;; either name is one type. `Bjodat`'s cases come with it, which is what lets a +;; hand-written instance match `BjoInt` without naming the other module. +;; +;; The rest is the one-pass driver the generated decoder calls, and the entry +;; points that drive a generated `bjodat-read-`. `bjodat->string` is here +;; because writing a record back out is the other half of the round trip, and +;; nothing else about the tree is: a program that wants to read data of unknown +;; shape imports `(text bjodat-core)` and gets the reader, the writer and the +;; accessors together. +;; +;; What does *not* travel is an implementation. A re-export publishes a +;; spelling and the declaration behind it, and an impl reaches a consumer +;; through the module that wrote it — see `re-export` in `Docs/MODULES.org`. +;; `bjodat-core` writes none, so there is nothing here to miss. +(re-export + Bjodat Reader + open-map! next-key! end-entry! key=? skip-value! + open-form! next-field! skip-clause! + read-one-pass parse-one-pass read-one-pass-all parse-one-pass-all + bjodat->string) + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- + +(def/trait (->bjodat %a) + (: ->bjodat (-> %a Bjodat))) + +(def/trait (bjodat-> %a) + (: bjodat-> (-> Bjodat (Result string %a)))) + +;; --------------------------------------------------------------------------- +;; Scalars +;; --------------------------------------------------------------------------- + +(impl (->bjodat Bjodat) (defun (->bjodat v) v)) +(impl (->bjodat long) (defun (->bjodat n) (BjoInt n))) +(impl (->bjodat int) (defun (->bjodat n) (BjoInt (cast System.Int64 n)))) +(impl (->bjodat double) (defun (->bjodat n) (BjoFloat n))) +(impl (->bjodat ulong) (defun (->bjodat n) (BjoULong n))) +(impl (->bjodat decimal) (defun (->bjodat n) (BjoDecimal n))) +(impl (->bjodat bigint) (defun (->bjodat n) (BjoBigInt n))) +(impl (->bjodat bool) (defun (->bjodat b) (BjoBool b))) +(impl (->bjodat string) (defun (->bjodat s) (BjoStr s))) +(impl (->bjodat char) (defun (->bjodat c) (BjoChar c))) +(impl (->bjodat Symbol) (defun (->bjodat s) (BjoSym s))) +(impl (->bjodat Keyword) (defun (->bjodat k) (BjoKey k))) + +(impl (bjodat-> Bjodat) + (defun (bjodat-> v) (Ok v))) + +(impl (bjodat-> long) + (defun (bjodat-> v) + (match v + ((BjoInt n) (Ok n)) + (_ (Err "expected an integer"))))) + +;; A cast would keep the low 32 bits, and a number out of range would read as +;; a different one. +(impl (bjodat-> int) + (defun (bjodat-> v) + (match v + ((BjoInt n) + (if (and (>= n -2147483648L) (<= n 2147483647L)) + (Ok (cast System.Int32 n)) + (Err "expected an integer that fits in an int"))) + (_ (Err "expected an integer"))))) + +(impl (bjodat-> double) + (defun (bjodat-> v) + (match v + ((BjoFloat d) (Ok d)) + ;; An integer is a perfectly good double. + ((BjoInt n) (Ok (cast System.Double n))) + (_ (Err "expected a number"))))) + +;; A plain integer that is not negative is a ulong too, so a file need not +;; write the suffix. +(impl (bjodat-> ulong) + (defun (bjodat-> v) + (match v + ((BjoULong n) (Ok n)) + ((BjoInt n) (if (>= n 0L) (Ok (cast System.UInt64 n)) (Err "expected an integer that is not negative"))) + (_ (Err "expected an integer"))))) + +;; An integer is a decimal exactly. A double is not: 0.1 has no exact binary +;; form, so it is refused rather than rounded. +(impl (bjodat-> decimal) + (defun (bjodat-> v) + (match v + ((BjoDecimal d) (Ok d)) + ((BjoInt n) (Ok (cast System.Decimal n))) + (_ (Err "expected a decimal, such as 1.50M"))))) + +(impl (bjodat-> bigint) + (defun (bjodat-> v) + (match v + ((BjoBigInt n) (Ok n)) + ((BjoInt n) (Ok (cast System.Numerics.BigInteger n))) + ((BjoULong n) (Ok (cast System.Numerics.BigInteger n))) + (_ (Err "expected an integer"))))) + +(impl (bjodat-> bool) + (defun (bjodat-> v) + (match v + ((BjoBool b) (Ok b)) + (_ (Err "expected #t or #f"))))) + +(impl (bjodat-> string) + (defun (bjodat-> v) + (match v + ((BjoStr s) (Ok s)) + (_ (Err "expected a string"))))) + +(impl (bjodat-> char) + (defun (bjodat-> v) + (match v + ((BjoChar c) (Ok c)) + (_ (Err "expected a character"))))) + +;; A symbol and a keyword are different values, and a field asking for one does +;; not take the other: that distinction is the reason both are in the format. +(impl (bjodat-> Symbol) + (defun (bjodat-> v) + (match v + ((BjoSym s) (Ok s)) + (_ (Err "expected a symbol"))))) + +(impl (bjodat-> Keyword) + (defun (bjodat-> v) + (match v + ((BjoKey k) (Ok k)) + (_ (Err "expected a keyword"))))) + +;; --------------------------------------------------------------------------- +;; Absence +;; --------------------------------------------------------------------------- + +(impl (->bjodat (Option %a)) + (where (->bjodat %a)) + (defun (->bjodat o) + (def (Some x) o :leave-with (BjoList Nil)) + (->bjodat x))) + +(impl (bjodat-> (Option %a)) + (where (bjodat-> %a)) + (defun (bjodat-> v) + (match v + ((BjoList Nil) (Ok None)) + (_ (match (bjodat-> v) + ((Err e) (Err e)) + ((Ok x) (Ok (Some x)))))))) + +;; --------------------------------------------------------------------------- +;; Sequences +;; --------------------------------------------------------------------------- + +(impl (->bjodat (Vec %a)) + (where (->bjodat %a)) + (defun (->bjodat v) + (BjoVec (vec-map (fun (x) (->bjodat x)) v)))) + +(impl (bjodat-> (Vec %a)) + (where (bjodat-> %a)) + (defun (bjodat-> v) + (match v + ;; Into one builder, which the vec takes at the end, and not a + ;; `vec-add` per element, which made a vec for each. The first element + ;; that does not decode ends the walk. + ((BjoVec items) + (def b (vecbuilder-empty/small)) + (let go ((c (vec-cursor items))) + (if (vec-cursor-done? c) + (Ok (vecbuilder->vec b)) + (match (bjodat-> (vec-cursor-current c)) + ((Err e) (Err e)) + ((Ok x) + (vecbuilder-add! b x) + (go (vec-cursor-next c))))))) + (_ (Err "expected a vec"))))) + +;; A list is the other sequence, and it is a different shape on the page: a +;; `(...)` rather than a `[...]`. A field takes the one its type names. +(impl (->bjodat (List %a)) + (where (->bjodat %a)) + (defun (->bjodat xs) + (BjoList (list-map (fun (x) (->bjodat x)) xs)))) + +(impl (bjodat-> (List %a)) + (where (bjodat-> %a)) + (defun (bjodat-> v) + (def (BjoList items) v :leave-with (Err "expected a list")) + (def (Ok b)(list-foldl (fun (el acc) + (def* ((Ok b) acc :propagate) + ((Ok x) (bjodat-> el) :propagate)) + (Ok (Cons x b))) + (Ok Nil) + items) :propagate) + (Ok (list-reverse b)))) + +;; --------------------------------------------------------------------------- +;; What generated field code calls +;; --------------------------------------------------------------------------- + +(: (-> string Bjodat (Result string (Vec Bjodat)))) +(defun (bjodat-map owner v) + (match v + ((BjoMap entries) (Ok entries)) + (_ (Err (str owner ": expected a map"))))) + +;; The owner and the key are prefixed onto whatever the value's own decoder +;; said, so a wrong type reads "Package.version: expected a string" rather than +;; naming neither. +(: (-> string (Vec Bjodat) Keyword (Result string %a)) (where (bjodat-> %a))) +(defun (bjodat-field owner entries key) + (match (bjodat-ref entries key) + (None (Err (str owner ": the map has no :" (keyword->string key) " entry"))) + ((Some v) + (result-map-err (fun (e) (str owner "." (keyword->string key) ": " e)) (bjodat-> v))))) + +;; Absent is the default. Unlike JSON there is no null to mean it as well. +(: (-> string (Vec Bjodat) Keyword %a (Result string %a)) (where (bjodat-> %a))) +(defun (bjodat-field/default owner entries key fallback) + (match (bjodat-ref entries key) + (None (Ok fallback)) + ((Some v) + (result-map-err (fun (e) (str owner "." (keyword->string key) ": " e)) (bjodat-> v))))) + +;; --- and what a one-pass decoder calls -------------------------------------- +;; +;; Wait, if this is a "one-pass" decoder, why are we still reading values into +;; intermediate `Bjodat` trees? This is deliberate! It allows us to reuse all the +;; existing `bjodat->` decoders (including custom ones written by users) and keeps +;; error messages perfectly consistent. +;; +;; The performance savings of "one-pass" come entirely from the keys: we avoid +;; allocating the map, interning the keywords, creating `BjoKey` nodes, and doing +;; map lookups. + +;; --- the two cases a generated encoder builds, as functions ----------------- +;; +;; `(BjoKey k)` written directly into the template would not compile in the +;; importing module. A union's *cases* do not cross a second module boundary: +;; this module can publish the `Bjodat` type it imported, but not `BjoKey`, and +;; the encoder is spliced into the module that wrote `def/bjodat-type`. A +;; function is an ordinary binding and re-exports cleanly, so these two are what +;; make one import enough. +;; +;; Writing an instance by hand for a type of your own still wants the cases, and +;; that is what `(import (text bjodat-core))` is for. It is the rarer thing and +;; it is the one that should say so. + +(: (-> Keyword Bjodat)) +(defun (bjodat-key k) (BjoKey k)) + +;; The entries were pushed in reverse, and are turned round once here. +(: (-> (List Bjodat) Bjodat)) +(defun (bjodat-entries acc) (BjoMap (list->vec (list-reverse acc)))) + +(: (-> string Keyword Reader (Result string %a)) (where (bjodat-> %a))) +(defun (bjodat-read-field owner key rd) + (match (field-value rd) + ((Err e) (Err e)) + ((Ok v) + (result-map-err (fun (e) (str owner "." (keyword->string key) ": " e)) (bjodat-> v))))) + +;; A field that was never met. The message is `bjodat-field`'s, word for word, +;; so which decoder read the document is not something an error report reveals. +(: (-> (Option %a) string Keyword (Result string %a))) +(defun (bjodat-need-field slot owner key) + (match slot + ((Some x) (Ok x)) + (None (Err (str owner ": the map has no :" (keyword->string key) " entry"))))) + +;; --------------------------------------------------------------------------- +;; The same, for a record written as a tagged form +;; --------------------------------------------------------------------------- +;; `(package (name bjolang/http) (version "1.2.3"))`. A clause names a field and +;; holds its value, and the head symbol names the type. +;; +;; The clauses are scanned where a map's entries are scanned, and for the same +;; reason: a record has a handful of fields, and walking a handful beats +;; building a trie to throw away. What is deliberately *not* done is flattening +;; the clauses into a keyword map first and reusing `bjodat-ref`. That was the +;; shorter implementation and it would have interned a `Keyword` per field — +;; which is the one cost the tagged form has no reason to pay, since its names +;; arrive as symbols that are interned already. + +;; Every clause has to be `(name ...)`. Checked up front rather than as each +;; field is looked for, because the one-pass reader refuses a malformed clause +;; whether or not any field wanted it, and two decoders that disagree about the +;; same document are worse than either rule. +(: (-> string (List Bjodat) (Result string (List Bjodat)))) +(defun (bjodat-clause-shape owner cs) + (let go ((rest cs)) + (match rest + (Nil (Ok cs)) + ((Cons c more) + (match c + ((BjoList (Cons (BjoSym _) _)) (go more)) + ;; `next-field!`'s sentence, so that the two decoders refuse the same + ;; document in the same words. + (_ (Err (str owner ": a clause begins with a field name")))))))) + +;; The clauses of `(tag (name value) ...)`, with the tag checked. +(: (-> string string Bjodat (Result string (List Bjodat)))) +(defun (bjodat-clauses owner tag v) + (match v + ((BjoList (Cons (BjoSym h) cs)) + (if (= (symbol->string h) tag) + (bjodat-clause-shape owner cs) + (Err (str owner ": expected (" tag " ...), and this is (" + (symbol->string h) " ...)")))) + (_ (Err (str owner ": expected (" tag " ...)"))))) + +;; The values of the first clause named `name`. **The first wins**, which is +;; what `bjodat-ref` does for a map and what the one-pass decoder does for a +;; repeated clause, so a document read any of the three ways answers the same. +(: (-> (List Bjodat) string (Option (List Bjodat)))) +(defun (bjodat-clause-ref cs name) + (let go ((rest cs)) + (def* ((Cons c more) rest :leave-with None) + ((BjoList (Cons (BjoSym h) vs)) c :leave-with (go more))) + (if (= (symbol->string h) name) + (Some vs) + (go more)))) + +;; A clause holding exactly one value, which is every field but a `#:rest` one. +(: (-> string string (List Bjodat) (Result string %a)) (where (bjodat-> %a))) +(defun (bjodat-one-value owner name vs) + (match vs + ((Cons v Nil) + (result-map-err (fun (e) (str owner "." name ": " e)) (bjodat-> v))) + (_ (Err (str owner "." name ": a clause holds a field name and one value"))))) + +;; Every value of a clause, for a `#:rest` field. +(: (-> string string (List Bjodat) (Result string (List %a))) (where (bjodat-> %a))) +(defun (bjodat-all-values owner name vs) + (result-map list-reverse + (list-foldl (fun (el acc) + (def (Ok b) acc :propagate) + (match (bjodat-> el) + ((Err e) (Err (str owner "." name ": " e))) + ((Ok x) (Ok (Cons x b))))) + (Ok Nil) + vs))) + +(: (-> string (List Bjodat) string (Result string %a)) (where (bjodat-> %a))) +(defun (bjodat-clause-field owner cs name) + (match (bjodat-clause-ref cs name) + (None (Err (str owner ": there is no (" name " ...) clause"))) + ((Some vs) (bjodat-one-value owner name vs)))) + +(: (-> string (List Bjodat) string %a (Result string %a)) (where (bjodat-> %a))) +(defun (bjodat-clause-field/default owner cs name fallback) + (match (bjodat-clause-ref cs name) + (None (Ok fallback)) + ((Some vs) (bjodat-one-value owner name vs)))) + +(: (-> string (List Bjodat) string (Result string (List %a))) (where (bjodat-> %a))) +(defun (bjodat-clause-rest owner cs name) + (match (bjodat-clause-ref cs name) + (None (Err (str owner ": there is no (" name " ...) clause"))) + ((Some vs) (bjodat-all-values owner name vs)))) + +(: (-> string (List Bjodat) string (List %a) (Result string (List %a))) (where (bjodat-> %a))) +(defun (bjodat-clause-rest/default owner cs name fallback) + (match (bjodat-clause-ref cs name) + (None (Ok fallback)) + ((Some vs) (bjodat-all-values owner name vs)))) + +;; --- and what a one-pass decoder calls -------------------------------------- +;; +;; Both of these leave the clause fully consumed, closing paren and all, so the +;; generated loop has one shape whether the field took one value or every value. + +(: (-> string string Reader (Result string %a)) (where (bjodat-> %a))) +(defun (bjodat-read-clause owner name rd) + (def* ((Ok v) (clause-value rd owner name) :propagate) + ((Ok x) (bjodat-> v) :leave ((Err e) (Err (str owner "." name ": " e)))) + ((Ok y) (end-clause! rd owner name) :propagate)) + (Ok x)) + + +(: (-> string string Reader (Result string (List %a))) (where (bjodat-> %a))) +(defun (bjodat-read-rest owner name rd) + (match (clause-values! rd) + ((Err e) (Err e)) + ((Ok vs) (bjodat-all-values owner name vs)))) + +;; A clause that was never met. The message is `bjodat-clause-field`'s, word for +;; word, so which decoder read the document is not something a report reveals. +(: (-> (Option %a) string string (Result string %a))) +(defun (bjodat-need-clause slot owner name) + (match slot + ((Some x) (Ok x)) + (None (Err (str owner ": there is no (" name " ...) clause"))))) + +;; --- the three shapes a generated encoder builds, as functions -------------- +;; +;; Here for the reason `bjodat-key` and `bjodat-entries` are: a union's *cases* +;; do not cross a second module boundary, and the encoder is spliced into +;; whichever module wrote `def/bjodat-type`. + +(: (-> string Bjodat Bjodat)) +(defun (bjodat-clause name v) + (BjoList (Cons (BjoSym (string->symbol name)) (Cons v Nil)))) + +(: (-> string (List Bjodat) Bjodat)) +(defun (bjodat-clause-many name vs) + (BjoList (Cons (BjoSym (string->symbol name)) vs))) + +(: (-> (List %a) (List Bjodat)) (where (->bjodat %a))) +(defun (bjodat-clause-values xs) + (list-map (fun (x) (->bjodat x)) xs)) + +;; The clauses were pushed in reverse, and are turned round once here. +(: (-> string (List Bjodat) Bjodat)) +(defun (bjodat-form-value tag acc) + (BjoList (Cons (BjoSym (string->symbol tag)) (list-reverse acc)))) + +;; --------------------------------------------------------------------------- +;; The generator +;; --------------------------------------------------------------------------- +;; A runtime macro. Field types are never inspected: what a field encodes as is +;; settled later by instance resolution, and only the markers are read here. + +(: (-> string Syntax)) +(defun (jb-sym name) (SSym (string->symbol name))) + +(: (-> Syntax Syntax)) +(defun (jb-str s) (SStr (syntax->string s))) + +;; `(Box %a)` names the type `Box`; a bare `Package` names itself. +(: (-> Syntax Syntax)) +(defun (jb-head-name head) + (match head + ((SList parts) (list-head parts)) + (_ head))) + +(: (-> Syntax (List Syntax))) +(defun (jb-head-vars head) + (match head + ((SList parts) (list-tail parts)) + (_ Nil))) + +(: (-> (-> Symbol Syntax) Syntax string (List Syntax))) +(defun (jb-where inject head trait) + (def vars (jb-head-vars head)) + (if (list-empty? vars) + Nil + (list #'(,(inject 'where) + ,@(list-map (fun (v) #'(,(inject (string->symbol trait)) ,v)) vars))))) + +;; --- markers --------------------------------------------------------------- + +(: (-> Syntax string bool)) +(defun (jb-marker=? m name) (= (syntax->string m) name)) + +;; The key this field is written under. `#:key :other-name` renames it; without +;; one the key is the field's own name. Built here, at expansion time, so the +;; generated code holds a keyword literal. +;; +;; Note: We generate a keyword literal directly in the decoder for every field. +;; You might worry this is slow (e.g. hashing and interning the string every time), +;; and historically, this macro used to generate a static variable to cache it. +;; +;; We don't do that anymore because the compiler's `Codegen` phase is now smart +;; enough to automatically hoist all keyword literals into static fields behind +;; the scenes. It's just as fast, and the code is cleaner. +(: (-> Syntax (List Syntax) Syntax)) +(defun (jb-key name markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) (SKey (string->keyword (syntax->string name)))) + ((jb-marker=? (list-head ms) "#:key") (list-head (list-tail ms))) + (else (go (list-tail ms)))))) + +;; The fallback expression, or None if the field is required. +(: (-> (List Syntax) (Option Syntax))) +(defun (jb-fallback markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) None) + (:def m (list-head ms)) + ((jb-marker=? m "#:optional") (Some (jb-sym "None"))) + ((jb-marker=? m "#:default") (Some (list-head (list-tail ms)))) + (else (go (list-tail ms)))))) + +(: (-> (List Syntax) bool)) +(defun (jb-optional? markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) #f) + ((jb-marker=? (list-head ms) "#:optional") #t) + (else (go (list-tail ms)))))) + +;; `#:rest` — the field takes *every* value of its clause rather than one, so +;; `(depends (package ...) (package ...))` fills a `(List Dep)`. +;; +;; This is the answer to the question a tagged list otherwise leaves open, which +;; is which clauses may repeat. It is answered per *field*, by the type, because +;; that is the only place that knows: `version` holds one and `depends` holds +;; many, and nothing about the syntax could tell them apart. +;; +;; A field is marked, not a clause repeated. `(depends a) (depends b)` does not +;; accumulate — the first clause wins, as the first entry under a key does. +;; Accumulating would have meant one rule for maps and another for forms. +(: (-> (List Syntax) bool)) +(defun (jb-rest? markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) #f) + ((jb-marker=? (list-head ms) "#:rest") #t) + (else (go (list-tail ms)))))) + +;; `(Record #:tag package (: name ...) ...)` — the marker, if it is there, and +;; the fields with it taken off. +;; +;; There is no default. `Package` spelled `package` is a casing guess, and which +;; tag a type is written under belongs in the file's vocabulary rather than in a +;; rule about capital letters. +(: (-> (List Syntax) (Option Syntax))) +(defun (jb-tag body) + (cond + ((list-empty? body) None) + ((jb-marker=? (list-head body) "#:tag") + (Some (jb-str (list-head (list-tail body))))) + (else None))) + +(: (-> (List Syntax) (List Syntax))) +(defun (jb-fields body) + (if (some? (jb-tag body)) + (list-tail (list-tail body)) + body)) + +;; What the record declaration keeps: everything that is not ours. +(: (-> (List Syntax) (List Syntax))) +(defun (jb-keep markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) Nil) + (:def m (list-head ms)) + ((jb-marker=? m "#:default") (go (list-tail (list-tail ms)))) + ((jb-marker=? m "#:key") (go (list-tail (list-tail ms)))) + ((jb-marker=? m "#:optional") (go (list-tail ms))) + ((jb-marker=? m "#:rest") (go (list-tail ms))) + (else (Cons m (go (list-tail ms))))))) + +(: string) +(def jb-shape + "a field is written (: name type), then any of #:mutable, #:optional, #:default , #:key :name, #:rest.") + +(: (-> Syntax string (List Syntax))) +(defun (jb-fail-list form message) + (ignore (syntax-error form message)) + Nil) + +(: (-> Syntax Syntax)) +(defun (jb-field-name f) + (syntax-match f + ((_ name _ marker ...) name) + (bad (syntax-error bad jb-shape)))) + +(: (-> Syntax (List Syntax))) +(defun (jb-field-markers f) + (syntax-match f + ((_ _ _ marker ...) marker) + (bad (jb-fail-list bad jb-shape)))) + +(: (-> Syntax Syntax)) +(defun (jb-strip f) + (syntax-match f + ((colon name ty marker ...) #'(,colon ,name ,ty ,@(jb-keep marker))) + (bad (syntax-error bad jb-shape)))) + +;; --- encode ---------------------------------------------------------------- +;; +;; The entries are built up in reverse and turned round once, so a map is one +;; vec and the order is the order the fields were declared in. An optional +;; field that is None is not written at all. + +(: (-> (List Syntax) Syntax Syntax)) +(defun (jb-enc-chain fields acc) + (cond + ((list-empty? fields) #'(bjodat-entries ,acc)) + (else + (def f (list-head fields)) + (def name (jb-field-name f)) + (def markers (jb-field-markers f)) + (def key (jb-key name markers)) + (def push #'(Cons (->bjodat (record-ref v ,name)) + (Cons (bjodat-key ,key) ,acc))) + (def step (if (jb-optional? markers) + #'(if (some? (record-ref v ,name)) ,push ,acc) + push)) + #'(let ((,acc ,step)) + ,(jb-enc-chain (list-tail fields) acc))))) + +(: (-> (List Syntax) Syntax)) +(defun (jb-encode fields) + (def acc (jb-sym "__jb-acc")) + #'(let ((,acc Nil)) + ,(jb-enc-chain fields acc))) + +;; The same, as a tagged form. An optional field that is `None` is not written, +;; and neither is a `#:rest` field that is empty *and* has a fallback to come +;; back as — `(authors)` is noise, and a field with no fallback has to be +;; written or the document would not read back. + +(: (-> (List Syntax) Syntax Syntax Syntax)) +(defun (jb-form-enc-chain fields tag acc) + (cond + ((list-empty? fields) #'(bjodat-form-value ,tag ,acc)) + (else + (def f (list-head fields)) + (def name (jb-field-name f)) + (def markers (jb-field-markers f)) + (def nm (jb-key-name name markers)) + (def push (if (jb-rest? markers) + #'(Cons (bjodat-clause-many + ,nm (bjodat-clause-values (record-ref v ,name))) + ,acc) + #'(Cons (bjodat-clause ,nm (->bjodat (record-ref v ,name))) + ,acc))) + (def step (cond + ((jb-optional? markers) + #'(if (some? (record-ref v ,name)) ,push ,acc)) + ((and (jb-rest? markers) (some? (jb-fallback markers))) + #'(if (list-empty? (record-ref v ,name)) ,acc ,push)) + (else push))) + #'(let ((,acc ,step)) + ,(jb-form-enc-chain (list-tail fields) tag acc))))) + +(: (-> (List Syntax) Syntax Syntax)) +(defun (jb-form-encode fields tag) + (def acc (jb-sym "__jb-acc")) + #'(let ((,acc Nil)) + ,(jb-form-enc-chain fields tag acc))) + +;; --- decode ---------------------------------------------------------------- +;; +;; One `match` per field wrapping the next, with the constructor innermost. + +(: (-> Syntax Syntax (List Syntax) int Syntax Syntax)) +(defun (jb-chain owner entries fields i ctor) + (cond + ((list-empty? fields) ctor) + (else + (def f (list-head fields)) + (def v (jb-sym (str "__jb-v" (int->string i)))) + (def e (jb-sym (str "__jb-e" (int->string i)))) + (def markers (jb-field-markers f)) + (def key (jb-key (jb-field-name f) markers)) + (def get (match (jb-fallback markers) + (None #'(bjodat-field ,owner ,entries ,key)) + ((Some d) #'(bjodat-field/default ,owner ,entries ,key ,d)))) + #'(match ,get + ((Err ,e) (Err ,e)) + ((Ok ,v) ,(jb-chain owner entries (list-tail fields) (+ i 1) ctor)))))) + +(: (-> Syntax (List Syntax) Syntax)) +(defun (jb-ctor head fields) + #'(Ok (,(jb-head-name head) + ,@(let go ((fs fields) (i 0)) + (if (list-empty? fs) + Nil + (Cons #'(,(jb-field-name (list-head fs)) + ,(jb-sym (str "__jb-v" (int->string i)))) + (go (list-tail fs) (+ i 1)))))))) + +(: (-> Syntax (List Syntax) Syntax)) +(defun (jb-decode head fields) + (def owner (jb-str (jb-head-name head))) + (def entries (jb-sym "__jb-entries")) + (def e (jb-sym "__jb-e")) + #'(match (bjodat-map ,owner v) + ((Err ,e) (Err ,e)) + ((Ok ,entries) ,(jb-chain owner entries fields 0 (jb-ctor head fields))))) + +;; The same, over the clauses of a tagged form. `jb-ctor` is shared, so both +;; decoders end in one constructor call written once. + +(: (-> Syntax Syntax (List Syntax) int Syntax Syntax)) +(defun (jb-form-chain owner cs fields i ctor) + (cond + ((list-empty? fields) ctor) + (else + (def f (list-head fields)) + (def v (jb-var i)) + (def e (jb-sym "__jb-e")) + (def markers (jb-field-markers f)) + (def nm (jb-key-name (jb-field-name f) markers)) + (def get (if (jb-rest? markers) + (match (jb-fallback markers) + (None #'(bjodat-clause-rest ,owner ,cs ,nm)) + ((Some d) #'(bjodat-clause-rest/default ,owner ,cs ,nm ,d))) + (match (jb-fallback markers) + (None #'(bjodat-clause-field ,owner ,cs ,nm)) + ((Some d) #'(bjodat-clause-field/default ,owner ,cs ,nm ,d))))) + #'(match ,get + ((Err ,e) (Err ,e)) + ((Ok ,v) ,(jb-form-chain owner cs (list-tail fields) (+ i 1) ctor)))))) + +(: (-> Syntax Syntax (List Syntax) Syntax)) +(defun (jb-form-decode head tag fields) + (def owner (jb-str (jb-head-name head))) + (def cs (jb-sym "__jb-clauses")) + (def e (jb-sym "__jb-e")) + #'(match (bjodat-clauses ,owner ,tag v) + ((Err ,e) (Err ,e)) + ((Ok ,cs) ,(jb-form-chain owner cs fields 0 (jb-ctor head fields))))) + +;; --- decode in one pass ----------------------------------------------------- +;; +;; The same record, read straight off the reader with no `Bjodat` map in +;; between. The shape is a loop carrying one slot per field: +;; +;; (let go ((s0 None) (s1 None)) +;; (match (next-key! rd owner paired) +;; ((Ok #f) ) +;; ((Ok _) (cond ((key=? rd "name") ... (go (Some x) s1)) +;; ((key=? rd "git") ... (go s0 (Some x))) +;; (else (go s0 s1)))))) +;; +;; We use an `(Option %a)` variable to track each field during the loop, rather +;; than a separate value and a boolean 'seen' flag. Since `Option` is a lightweight +;; struct, this adds no extra memory overhead and elegantly tracks both "has it +;; been seen?" and "what is the value?" simultaneously. +;; +;; We compare incoming keys directly against the expected string in the reader's +;; raw buffer using `key=?`. This is the core optimization of the one-pass +;; decoder: we never allocate memory or intern strings for keys we encounter in +;; the file. (The standard tree decoder had to intern every single key just to +;; look it up in the map). + +(: (-> int Syntax)) +(defun (jb-slot i) (jb-sym (str "__jb-s" (int->string i)))) + +(: (-> int Syntax)) +(defun (jb-var i) (jb-sym (str "__jb-v" (int->string i)))) + +;; The key's name as a string, for `key=?`. `#:key :other-name` renames it here +;; exactly as it renames the keyword literal, so the two cannot drift. +(: (-> Syntax (List Syntax) Syntax)) +(defun (jb-key-name name markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) (SStr (syntax->string name))) + ((jb-marker=? (list-head ms) "#:key") + (match (list-head (list-tail ms)) + ((SKey k) (SStr (keyword->string k))) + (other (SStr (syntax->string other))))) + (else (go (list-tail ms)))))) + +(: (-> (List Syntax) int (List Syntax))) +(defun (jb-slot-inits fields i) + (if (list-empty? fields) + Nil + (Cons #'(,(jb-slot i) None) + (jb-slot-inits (list-tail fields) (+ i 1))))) + +;; The slots a recursive call passes on, unchanged. +(: (-> (List Syntax) int (List Syntax))) +(defun (jb-slot-args fields i) + (if (list-empty? fields) + Nil + (Cons (jb-slot i) (jb-slot-args (list-tail fields) (+ i 1))))) + +;; The same, with slot `hit` replaced by the value just read. +(: (-> (List Syntax) int int Syntax (List Syntax))) +(defun (jb-slot-args/hit fields i hit v) + (match fields + (Nil Nil) + (_ (Cons (if (= i hit) v (jb-slot i)) + (jb-slot-args/hit (list-tail fields) (+ i 1) hit v))))) + +(: (-> Syntax Syntax Syntax (List Syntax) (List Syntax) int (List Syntax))) +(defun (jb-clauses owner paired go all fields i) + (cond + ((list-empty? fields) Nil) + (else + (def f (list-head fields)) + (def markers (jb-field-markers f)) + (def name (jb-field-name f)) + (def x (jb-sym "__jb-x")) + (def e (jb-sym "__jb-e")) + ;; The *first* entry under a key wins, which is what `bjodat-ref` does + ;; for the tree decoder: it answers the first match and stops. So a key + ;; whose slot is filled is not this field's any more. It goes to the + ;; `else` branch, which skips its value as it skips an unknown key's. + ;; Decoded, a later value of the wrong type was refused here and not by + ;; the tree decoder, which never looks at it. + (Cons #'((and (none? ,(jb-slot i)) (key=? rd ,(jb-key-name name markers))) + (match (bjodat-read-field ,owner ,(jb-key name markers) rd) + ((Err ,e) (Err ,e)) + ((Ok ,x) + (match (end-entry! rd ,owner ,paired) + ((Err ,e) (Err ,e)) + ((Ok _) + (,go ,@(jb-slot-args/hit all 0 i #'(Some ,x)))))))) + (jb-clauses owner paired go all (list-tail fields) (+ i 1)))))) + +;; Every slot answered for, innermost first, ending in the constructor. A +;; required field that was never met is the one refusal; a field with a fallback +;; takes it here rather than at the entry, because absence is only known once +;; the map has closed. +(: (-> Syntax Syntax (List Syntax) (List Syntax) int Syntax)) +(defun (jb-settle head owner all fields i) + (cond + ((list-empty? fields) (jb-ctor head all)) + (else + (def f (list-head fields)) + (def markers (jb-field-markers f)) + (def key (jb-key (jb-field-name f) markers)) + (def e (jb-sym "__jb-e")) + (def p (jb-sym "__jb-p")) + (def rest (jb-settle head owner all (list-tail fields) (+ i 1))) + (match (jb-fallback markers) + (None #'(match (bjodat-need-field ,(jb-slot i) ,owner ,key) + ((Err ,e) (Err ,e)) + ((Ok ,(jb-var i)) ,rest))) + ((Some d) #'(let ((,(jb-var i) (match ,(jb-slot i) + ((Some ,p) ,p) + (None ,d)))) + ,rest)))))) + +(: (-> Syntax (List Syntax) Syntax)) +(defun (jb-one-pass head fields) + (def owner (jb-str (jb-head-name head))) + (def paired (jb-sym "__jb-paired")) + (def go (jb-sym "__jb-go")) + (def e (jb-sym "__jb-e")) + #'(match (open-map! rd ,owner) + ((Err ,e) (Err ,e)) + ((Ok ,paired) + (let ,go (,@(jb-slot-inits fields 0)) + (match (next-key! rd ,owner ,paired) + ((Err ,e) (Err ,e)) + ((Ok #f) ,(jb-settle head owner fields fields 0)) + ((Ok _) + (cond + ,@(jb-clauses owner paired go fields fields 0) + (else + (match (skip-value! rd) + ((Err ,e) (Err ,e)) + ((Ok _) + (match (end-entry! rd ,owner ,paired) + ((Err ,e) (Err ,e)) + ((Ok _) (,go ,@(jb-slot-args fields 0)))))))))))))) + +;; --- reading a tagged form in one pass -------------------------------------- +;; +;; The same loop, over `(package (name "x") (version "1.2"))`. Three things +;; differ from the map above and nothing else does: `open-form!` checks the tag +;; where `open-map!` only found a brace, a clause name is a bare symbol rather +;; than a keyword, and an unknown clause costs `skip-clause!` rather than +;; `skip-value!` because it may hold any number of values. +;; +;; The closing `)` of a clause is consumed by whichever of `bjodat-read-clause` +;; and `bjodat-read-rest` read it, which is why there is no `end-entry!` here to +;; match the map decoder's. That keeps the two branches one shape. + +(: (-> Syntax Syntax (List Syntax) (List Syntax) int (List Syntax))) +(defun (jb-form-clauses owner go all fields i) + (cond + ((list-empty? fields) Nil) + (else + (def f (list-head fields)) + (def markers (jb-field-markers f)) + (def nm (jb-key-name (jb-field-name f) markers)) + (def x (jb-sym "__jb-x")) + (def e (jb-sym "__jb-e")) + (def read (if (jb-rest? markers) + #'(bjodat-read-rest ,owner ,nm rd) + #'(bjodat-read-clause ,owner ,nm rd))) + ;; The first clause under a name wins, as the first entry under a key + ;; does. A later one goes to the `else` branch and is skipped as an + ;; unknown clause is, at any arity, as the tree decoder ignores it. + (Cons #'((and (none? ,(jb-slot i)) (key=? rd ,nm)) + (match ,read + ((Err ,e) (Err ,e)) + ((Ok ,x) + (,go ,@(jb-slot-args/hit all 0 i #'(Some ,x)))))) + (jb-form-clauses owner go all (list-tail fields) (+ i 1)))))) + +(: (-> Syntax Syntax (List Syntax) (List Syntax) int Syntax)) +(defun (jb-form-settle head owner all fields i) + (cond + ((list-empty? fields) (jb-ctor head all)) + (else + (def f (list-head fields)) + (def markers (jb-field-markers f)) + (def nm (jb-key-name (jb-field-name f) markers)) + (def e (jb-sym "__jb-e")) + (def p (jb-sym "__jb-p")) + (def rest (jb-form-settle head owner all (list-tail fields) (+ i 1))) + (match (jb-fallback markers) + (None #'(match (bjodat-need-clause ,(jb-slot i) ,owner ,nm) + ((Err ,e) (Err ,e)) + ((Ok ,(jb-var i)) ,rest))) + ((Some d) #'(let ((,(jb-var i) (match ,(jb-slot i) + ((Some ,p) ,p) + (None ,d)))) + ,rest)))))) + +(: (-> Syntax Syntax (List Syntax) Syntax)) +(defun (jb-form-one-pass head tag fields) + (def owner (jb-str (jb-head-name head))) + (def go (jb-sym "__jb-go")) + (def e (jb-sym "__jb-e")) + #'(match (open-form! rd ,owner ,tag) + ((Err ,e) (Err ,e)) + ((Ok _) + (let ,go (,@(jb-slot-inits fields 0)) + (match (next-field! rd ,owner) + ((Err ,e) (Err ,e)) + ((Ok #f) ,(jb-form-settle head owner fields fields 0)) + ((Ok _) + (cond + ,@(jb-form-clauses owner go fields fields 0) + (else + (match (skip-clause! rd) + ((Err ,e) (Err ,e)) + ((Ok _) (,go ,@(jb-slot-args fields 0)))))))))))) + +;; --- the form --------------------------------------------------------------- + +(: (-> (-> Symbol Syntax) Syntax Syntax (List Syntax) Syntax)) +(defun (jb-expand inject colon head body) + (def tag (jb-tag body)) + (def fields (jb-fields body)) + (def name (syntax->string (jb-head-name head))) + ;; Which of the two spellings this record is written in is settled + ;; once, here, and the three generated bodies agree by construction. + ;; A type has one encoding: a value that read as `{...}` writes back as + ;; `{...}`, and one that read as `(tag ...)` writes back as that. + (def encode (match tag + (None (jb-encode fields)) + ((Some t) (jb-form-encode fields t)))) + (def decode (match tag + (None (jb-decode head fields)) + ((Some t) (jb-form-decode head t fields)))) + (def one-pass (match tag + (None (jb-one-pass head fields)) + ((Some t) (jb-form-one-pass head t fields)))) + (def enc (inject (string->symbol (str name "->bjodat")))) + (def dec (inject (string->symbol (str "bjodat->" name)))) + (def rdr (inject (string->symbol (str "bjodat-read-" name)))) + (def prs (inject (string->symbol (str "bjodat-parse-" name)))) + #'(begin + (type (,colon ,head (Record ,@(list-map jb-strip fields)))) + + (impl (->bjodat ,head) + ,@(jb-where inject head "->bjodat") + (defun (->bjodat v) ,encode)) + + (impl (bjodat-> ,head) + ,@(jb-where inject head "bjodat->") + (defun (bjodat-> v) ,decode)) + + ;; The named pair. `bjodat->` alone would be ambiguous at a call site; + ;; this signature is what fixes the instance, once per type. + (: ,enc (-> ,head ,(inject 'Bjodat)) ,@(jb-where inject head "->bjodat")) + (defun (,enc v) (->bjodat v)) + (: ,dec (-> ,(inject 'Bjodat) (Result string ,head)) ,@(jb-where inject head "bjodat->")) + (defun (,dec v) (bjodat-> v)) + + ;; And the one-pass pair, which reads the text itself rather than a + ;; map someone else built. Same record, same messages; no tree. + (: ,rdr (-> ,(inject 'Reader) (Result string ,head)) ,@(jb-where inject head "bjodat->")) + (defun (,rdr rd) ,one-pass) + (: ,prs (-> string (Result string ,head)) ,@(jb-where inject head "bjodat->")) + ;; The reader is passed by *name*, not wrapped in a `(fun (rd) ...)`. + ;; A lambda body is a member of its own and is sealed to the sync copy + ;; of whatever it calls, so the wrapped form made the suspending copy + ;; of this function park the thread — which the blocking lint said in + ;; as many words. A bare name carries an effect variable instead, and + ;; `parse-one-pass` declares its parameter `-?->`, so the caller's + ;; colour picks the copy. + (defun (,prs text) (parse-one-pass text ,rdr)))) + +(def/macro (def/bjodat-type form inject compare) + (syntax-match form + ((_ typedef) + (syntax-match typedef + ((colon head ('Record field ...)) (jb-expand inject colon head field)) + (bad (syntax-error bad "def/bjodat-type takes a record: (def/bjodat-type (: Name (Record (: field type) ...)))")))) + (bad (syntax-error bad "def/bjodat-type takes exactly one record declaration.")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; text/csv-codec.bjo — records to CSV rows and back. +;; +;; Two traits for a field, a trait for a record, and a macro that writes the +;; record's instance, as (text json-codec) has them for JSON. The readers and +;; writers here are built on (text csv), and an error is its CsvError. +;; +;; A record's columns are found by name in the header row, so the order of +;; the columns does not matter and columns the record does not have are not +;; read. Without a header, the fields are the columns in order. +;; +;; Its reference is its docs, in csv-codec.bjodoc. + +(export + ;; Fields + ->csv-field csv-field-> + ;; Records + CsvRecord csv-columns csv-record-> ->csv-record + CsvColumns CsvColumn + ;; What generated code calls. Exported because a macro template's names + ;; resolve against this module's exports. + csv-columns-of csv-required csv-optional csv-decode-field csv-decode-field/default csv-row-of + ;; The generator + def/csv-type + ;; Reading and writing records + csv-fold-records csv-for-each-record parse-csv-records + csv-records->string write-csv-records) + +(import (std prelude)) +(import (std syntax-match)) +(import (text csv)) + +;; Every export is documented there; the module doc is too. +(:doc #:file "csv-codec.bjodoc") + +;; =========================================================================== +;; Fields +;; =========================================================================== + +(def/trait (->csv-field %a) + (: ->csv-field (-> %a string))) + +;; The error is what is wrong with the text, without where: the reader adds +;; the line, the row and the column. +(def/trait (csv-field-> %a) + (: csv-field-> (-> string (Result string %a)))) + +(impl (->csv-field string) (defun (->csv-field s) s)) +(impl (->csv-field int) (defun (->csv-field n) (int->string n))) +(impl (->csv-field long) (defun (->csv-field n) (long->string n))) +(impl (->csv-field double) (defun (->csv-field n) (double->string n))) +(impl (->csv-field decimal) (defun (->csv-field n) (decimal->string n))) +(impl (->csv-field bool) (defun (->csv-field b) (if b "true" "false"))) + +;; `parse` raises on text that is not a number, as the prelude's string->int +;; does. The text is quoted in the error: "12,5" is not a decimal. +(: (-> string string (-> string %a) (Result string %a))) +(defun (parsed s what parse) + (result-map-err (fun (_) (str (written s) " is not " what)) + (try (parse s) #:catch (System.FormatException System.OverflowException)))) + +(impl (csv-field-> string) (defun (csv-field-> s) (Ok s))) +(impl (csv-field-> int) (defun (csv-field-> s) (parsed s "an int" string->int))) +(impl (csv-field-> long) (defun (csv-field-> s) (parsed s "a long" string->long))) +(impl (csv-field-> double) (defun (csv-field-> s) (parsed s "a number" string->double))) +(impl (csv-field-> decimal) (defun (csv-field-> s) (parsed s "a decimal" string->decimal))) + +;; true or false, in any case, with space around it or not. +(impl (csv-field-> bool) + (defun (csv-field-> s) + (def t (string-downcase (string-trim s))) + (cond + ((= t "true") (Ok #t)) + ((= t "false") (Ok #f)) + (else (Err (str (written s) " is not true or false")))))) + +;; An empty field is None, and is written for None. +(impl (->csv-field (Option %a)) + (where (->csv-field %a)) + (defun (->csv-field o) + (match o + ((Some x) (->csv-field x)) + (None "")))) + +(impl (csv-field-> (Option %a)) + (where (csv-field-> %a)) + (defun (csv-field-> s) + (if (string-empty? s) + (Ok None) + (match (csv-field-> s) + ((Ok v) (Ok (Some v))) + ((Err e) (Err e)))))) + +;; =========================================================================== +;; Records +;; =========================================================================== + +;; A column a record reads: its name in the header, and whether the header +;; has to have it. +(type (: CsvColumn (Record (: name string) (: required bool)))) + +;; A record's columns, one for each field, in the order of the fields. `%a` +;; is the record: it says whose columns they are, and nothing else. +(type (: (CsvColumns %a) (Newtype (IArray CsvColumn)))) + +;; The error of csv-record-> is the field it is about, counted from 0 in the +;; record's order, and what is wrong. +(def/trait (CsvRecord %a) + (: csv-columns (-> (CsvColumns %a))) + ;; `at` holds, for each field, where its column is in the row: -1 for a + ;; column the header does not have. + (: csv-record-> (-> (IArray int) (IArray string) (Result (Tuple int string) %a))) + (: ->csv-record (-> %a (IArray string)))) + +;; --------------------------------------------------------------------------- +;; What generated code calls +;; --------------------------------------------------------------------------- + +(: (-> string CsvColumn)) +(defun (csv-required name) (CsvColumn (name name) (required #t))) + +(: (-> string CsvColumn)) +(defun (csv-optional name) (CsvColumn (name name) (required #f))) + +(: (-> (List CsvColumn) (CsvColumns %a))) +(defun (csv-columns-of cols) (CsvColumns (list->iarray cols))) + +(: (-> (List string) (IArray string))) +(defun (csv-row-of fields) (list->iarray fields)) + +;; Field `k`, whose column is at `pos` in the row. +(: (-> (IArray int) int (IArray string) (Result (Tuple int string) %a)) (where (csv-field-> %a))) +(defun (csv-decode-field at k row) + (def pos (iarray-ref at k)) + (if (>= pos (iarray-length row)) + (Err (Tuple k "the row ends before this column")) + (result-map-err (fun (e) (Tuple k e)) (csv-field-> (iarray-ref row pos))))) + +;; The same, with `fallback` when the header has no such column, or the row +;; ends before it, or the field is empty. +(: (-> (IArray int) int (IArray string) %a (Result (Tuple int string) %a)) (where (csv-field-> %a))) +(defun (csv-decode-field/default at k row fallback) + (def pos (iarray-ref at k)) + (cond + ((or (< pos 0) (>= pos (iarray-length row))) (Ok fallback)) + (:def s (iarray-ref row pos)) + ((string-empty? s) (Ok fallback)) + (else + (result-map-err (fun (e) (Tuple k e)) (csv-field-> s))))) + +;; =========================================================================== +;; Reading +;; =========================================================================== + +;; Where each column is in the row without a header: the fields in order. +(: (-> (IArray CsvColumn) (IArray int))) +(defun (in-order cols) + (list->iarray (loop (:for i (range 0 (iarray-length cols))) (:acc (listing i))))) + +;; Where each column is in the row: the first column in the header with its +;; name, or -1 for one the header need not have. +(: (-> (IArray CsvColumn) (IArray string) (Result string (IArray int)))) +(defun (layout cols names) + (def found (transientmap-empty)) + (loop (:for (Tuple i name) (enumerate names)) + (:do (unless (transientmap-contains? found name) + (transientmap-set! found name i)))) + (let go ((k 0) (out (listbuilder-empty))) + (cond + ((= k (iarray-length cols)) (Ok (list->iarray (listbuilder->list out)))) + (:def col (iarray-ref cols k)) + (:match (Some i) (transientmap-try-ref found (record-ref col name)) + (listbuilder-add! out i) + (go (+ k 1) out)) + ((record-ref col required) + (Err (str "the header has no column " (written (record-ref col name))))) + (else + (listbuilder-add! out -1) + (go (+ k 1) out))))) + +;; A field that does not decode, as a CsvError: at the line its row starts +;; on, in the field the column is in, and named by the column. +(: (-> CsvReader (IArray CsvColumn) (IArray int) int string CsvError)) +(defun (field-error rd cols at k message) + (CsvError (message (str (record-ref (iarray-ref cols k) name) ": " message)) + (line (csv-row-line rd)) + (row (csv-row-number rd)) + (field (+ (iarray-ref at k) 1)))) + +;; With a header, the first row is it, and an input with no rows has no +;; records. An error in the header is about all of it, so its field is 0. +(: (-> CsvReader bool %seed (-> %a %seed %seed) (Result CsvError %seed)) (where (CsvRecord %a))) +(defun (fold-records rd header? seed f) + (: columns (CsvColumns %a)) + (def columns (csv-columns)) + (def (CsvColumns cols) columns) + (def placed + (cond + ((not header?) (Ok (Some (in-order cols)))) + (else + (match (csv-next! rd) + ((Err e) (Err e)) + ((Ok None) (Ok None)) + ((Ok (Some names)) + (match (layout cols names) + ((Ok at) (Ok (Some at))) + ((Err message) + (Err (CsvError (message message) (line (csv-row-line rd)) (row 1) (field 0)))))))))) + (match placed + ((Err e) (Err e)) + ((Ok None) (Ok seed)) + ((Ok (Some at)) + (let loop ((seed seed)) + (match (csv-next! rd) + ((Err e) (Err e)) + ((Ok None) (Ok seed)) + ((Ok (Some row)) + (match (csv-record-> at row) + ((Ok v) (loop (f v seed))) + ((Err (Tuple k message)) (Err (field-error rd cols at k message)))))))))) + +;; Each record and the seed so far give the next seed. The records before an +;; error have been folded when the error is answered. +(: (-> %s %seed (-> %a %seed %seed) + (#:header? bool) (#:separator char) (#:quote-char (Option char)) (#:strict? bool) + (Result CsvError %seed)) + (where (CsvSource %s) (CsvRecord %a))) +(defun (csv-fold-records source seed f + #:header? #t #:separator #\, #:quote-char (Some #\") #:strict? #t) + (call-with-csv-reader source + (fun (rd) (fold-records rd header? seed f)) + #:separator separator #:quote-char quote-char #:strict? strict?)) + +(: (-> %s (-> %a void) + (#:header? bool) (#:separator char) (#:quote-char (Option char)) (#:strict? bool) + (Result CsvError Unit)) + (where (CsvSource %s) (CsvRecord %a))) +(defun (csv-for-each-record source f + #:header? #t #:separator #\, #:quote-char (Some #\") #:strict? #t) + (call-with-csv-reader source + (fun (rd) (fold-records rd header? unit (fun (v u) (f v) u))) + #:separator separator #:quote-char quote-char #:strict? strict?)) + +(: (-> %s + (#:header? bool) (#:separator char) (#:quote-char (Option char)) (#:strict? bool) + (Result CsvError (Vec %a))) + (where (CsvSource %s) (CsvRecord %a))) +(defun (parse-csv-records source + #:header? #t #:separator #\, #:quote-char (Some #\") #:strict? #t) + (def out (transientvec-empty)) + (match (call-with-csv-reader source + (fun (rd) (fold-records rd header? unit (fun (v u) (transientvec-add! out v) u))) + #:separator separator #:quote-char quote-char #:strict? strict?) + ((Ok _) (Ok (transientvec->vec out))) + ((Err e) (Err e)))) + +;; =========================================================================== +;; Writing +;; =========================================================================== + +;; The column names, in the order of the fields. +(: (-> (CsvColumns %a) (IArray string))) +(defun (column-names columns) + (def (CsvColumns cols) columns) + (iarray-map (fun (c) (record-ref c name)) cols)) + +;; The header, then a row for each record, each ended by a line end. +(: (-> %rs + (#:header? bool) (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool) + (#:crlf? bool) + string) + (where (CsvRecord %a) (Iterable %rs #:elem %a))) +(defun (csv-records->string records + #:header? #t #:separator #\, #:quote-char (Some #\") #:quote-all? #f + #:crlf? #f) + (: columns (CsvColumns %a)) + (def columns (csv-columns)) + (def head (if header? + (csv->string (list (column-names columns)) + #:separator separator #:quote-char quote-char #:quote-all? quote-all? + #:crlf? crlf?) + "")) + (str head + (csv->string (map (fun (r) (->csv-record r)) records) + #:separator separator #:quote-char quote-char #:quote-all? quote-all? + #:crlf? crlf?))) + +(: (-> OutputPort %rs + (#:header? bool) (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool) + (#:crlf? bool) + void) + (where (CsvRecord %a) (Iterable %rs #:elem %a))) +(defun (write-csv-records port records + #:header? #t #:separator #\, #:quote-char (Some #\") #:quote-all? #f + #:crlf? #f) + (: columns (CsvColumns %a)) + (def columns (csv-columns)) + (when header? + (write-csv-row port (column-names columns) + #:separator separator #:quote-char quote-char #:quote-all? quote-all? + #:crlf? crlf?)) + (write-csv port (map (fun (r) (->csv-record r)) records) + #:separator separator #:quote-char quote-char #:quote-all? quote-all? + #:crlf? crlf?)) + +;; =========================================================================== +;; The generator +;; =========================================================================== +;; A runtime macro, as def/json-type is. Field types are never inspected: +;; how a field reads and writes is settled later by instance resolution, and +;; only the markers are read here. + +(: (-> string Syntax)) +(defun (ct-sym name) (SSym (string->symbol name))) + +(: (-> Syntax Syntax)) +(defun (ct-str s) (SStr (syntax->string s))) + +(: (-> int Syntax)) +(defun (ct-int n) (SInt (int->string n))) + +;; `(Box %a)` names the type `Box`; a bare `Point` names itself. +(: (-> Syntax Syntax)) +(defun (ct-head-name head) + (match head + ((SList parts) (list-head parts)) + (_ head))) + +(: (-> Syntax (List Syntax))) +(defun (ct-head-vars head) + (match head + ((SList parts) (list-tail parts)) + (_ Nil))) + +;; Each type parameter has to read and write as a field. +(: (-> (-> Symbol Syntax) Syntax (List Syntax))) +(defun (ct-where inject head) + (def vars (ct-head-vars head)) + (if (list-empty? vars) + Nil + (list #'(,(inject 'where) + ,@(list-append + (list-map (fun (v) #'(,(inject (string->symbol "csv-field->")) ,v)) vars) + (list-map (fun (v) #'(,(inject (string->symbol "->csv-field")) ,v)) vars)))))) + +;; --- fields ---------------------------------------------------------------- + +(: (-> Syntax string bool)) +(defun (ct-marker=? m name) (= (syntax->string m) name)) + +;; The column's name: `#:csv "Name"`, or the field's own. +(: (-> Syntax (List Syntax) Syntax)) +(defun (ct-column name markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) (ct-str name)) + ((ct-marker=? (list-head ms) "#:csv") (list-head (list-tail ms))) + (else (go (list-tail ms)))))) + +;; The fallback expression, or None if the field is required. `#:optional` is +;; `#:default None`. +(: (-> (List Syntax) (Option Syntax))) +(defun (ct-fallback markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) None) + (else + (def m (list-head ms)) + (cond + ((ct-marker=? m "#:optional") (Some (ct-sym "None"))) + ((ct-marker=? m "#:default") (Some (list-head (list-tail ms)))) + (else (go (list-tail ms)))))))) + +;; The markers the record type keeps: all but #:csv, #:default and #:optional. +(: (-> (List Syntax) (List Syntax))) +(defun (ct-keep markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) Nil) + (else + (def m (list-head ms)) + (cond + ((ct-marker=? m "#:default") (go (list-tail (list-tail ms)))) + ((ct-marker=? m "#:csv") (go (list-tail (list-tail ms)))) + ((ct-marker=? m "#:optional") (go (list-tail ms))) + (else (Cons m (go (list-tail ms))))))))) + +(: string) +(def ct-shape + "a field is written (: name type), then any of #:mutable, #:optional, #:default , #:csv \"column\".") + +;; `syntax-error` has the type Syntax, so this is how a list is answered. +(: (-> Syntax string (List Syntax))) +(defun (ct-fail-list form message) + (ignore (syntax-error form message)) + Nil) + +(: (-> Syntax Syntax)) +(defun (ct-field-name f) + (syntax-match f + ((_ name _ marker ...) name) + (bad (syntax-error bad ct-shape)))) + +(: (-> Syntax (List Syntax))) +(defun (ct-field-markers f) + (syntax-match f + ((_ _ _ marker ...) marker) + (bad (ct-fail-list bad ct-shape)))) + +(: (-> Syntax Syntax)) +(defun (ct-strip f) + (syntax-match f + ((colon name ty marker ...) #'(,colon ,name ,ty ,@(ct-keep marker))) + (bad (syntax-error bad ct-shape)))) + +;; --- columns, encode, decode ------------------------------------------------ + +;; A column is required unless the field has a fallback. +(: (-> (List Syntax) Syntax)) +(defun (ct-columns fields) + #'(csv-columns-of + (list ,@(list-map (fun (f) + (def column (ct-column (ct-field-name f) (ct-field-markers f))) + (match (ct-fallback (ct-field-markers f)) + (None #'(csv-required ,column)) + ((Some _) #'(csv-optional ,column)))) + fields)))) + +(: (-> (List Syntax) Syntax)) +(defun (ct-encode fields) + #'(csv-row-of + (list ,@(list-map (fun (f) #'(->csv-field (record-ref v ,(ct-field-name f)))) fields)))) + +;; One `match` per field wrapping the next, with the constructor innermost. +(: (-> (List Syntax) int Syntax Syntax)) +(defun (ct-chain fields i ctor) + (cond + ((list-empty? fields) ctor) + (else + (def f (list-head fields)) + (def v (ct-sym (str "__ct-v" (int->string i)))) + (def e (ct-sym (str "__ct-e" (int->string i)))) + (def get (match (ct-fallback (ct-field-markers f)) + (None #'(csv-decode-field at ,(ct-int i) row)) + ((Some d) #'(csv-decode-field/default at ,(ct-int i) row ,d)))) + #'(match ,get + ((Err ,e) (Err ,e)) + ((Ok ,v) ,(ct-chain (list-tail fields) (+ i 1) ctor)))))) + +(: (-> Syntax (List Syntax) Syntax)) +(defun (ct-ctor head fields) + #'(Ok (,(ct-head-name head) + ,@(let go ((fs fields) (i 0)) + (if (list-empty? fs) + Nil + (Cons #'(,(ct-field-name (list-head fs)) ,(ct-sym (str "__ct-v" (int->string i)))) + (go (list-tail fs) (+ i 1)))))))) + +;; --- the form --------------------------------------------------------------- + +(: (-> (-> Symbol Syntax) Syntax Syntax (List Syntax) Syntax)) +(defun (ct-expand inject colon head fields) + #'(begin + (type (,colon ,head (Record ,@(list-map ct-strip fields)))) + + (impl (CsvRecord ,head) + ,@(ct-where inject head) + (defun (csv-columns) ,(ct-columns fields)) + (defun (csv-record-> at row) ,(ct-chain fields 0 (ct-ctor head fields))) + (defun (->csv-record v) ,(ct-encode fields))))) + +(def/macro (def/csv-type form inject compare) + (syntax-match form + ((_ typedef) + (syntax-match typedef + ((colon head ('Record field ...)) (ct-expand inject colon head field)) + (bad (syntax-error bad "def/csv-type takes a record: (def/csv-type (: Name (Record (: field type) ...)))")))) + (bad (syntax-error bad "def/csv-type takes exactly one record declaration.")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; text/csv.bjo — the module (text csv): RFC 4180, read and written. +;; +;; Three layers for reading, each built on the one below: +;; +;; 1. csv-next!: a pull reader, one row at a time, as OCaml's Csv.next +;; 2. csv-fold and csv-for-each: every row, never more than one held +;; 3. parse-csv: the whole document, as a vec of rows +;; +;; The reader works on bytes, as (text json) does: a chunk of the input in +;; `bytes`, the next byte at `next`, and the end of the chunk at `limit`. The +;; separator, the quote, CR and LF are all ASCII, and in UTF-8 an ASCII byte +;; is never part of a longer character. So the scan looks at bytes only, and a +;; field is checked to be UTF-8 once, when it becomes a string. +;; +;; A field is always whole in one chunk. When the scan reaches the end of the +;; chunk, the bytes from the start of the field are kept, the next chunk is +;; added after them, and the scan continues where it stopped. +;; +;; Its reference is its docs, in csv.bjodoc. +;; +;; A row is an IArray: the reader keeps one array for the fields of the row +;; it is reading, and copies them out once, into the row's own array. +;; +;; Each chunk is checked to be UTF-8 once, as it arrives, by making a +;; Utf8Chunk of it. A field is then taken from the chunk and not checked +;; again; the chunk itself makes sure that its offsets are between +;; characters. +;; +;; Open: +;; - The scan is a byte loop. A SIMD search for the four special bytes, as +;; (text json) skips white space, would be faster for long fields. +;; - A header row, and rows decoded to records by column name, are still to +;; come. + +(export + ;; Errors + CsvError + ;; Sources + CsvInput CsvSource csv-input CsvFile csv-file + ;; Pulling rows + CsvReader call-with-csv-reader csv-next! csv-row-line csv-row-number + ;; Folds + csv-fold csv-for-each + ;; The whole document + parse-csv + ;; Rows of bytes, cut into pieces + csv-row-end csv-cut-rows + ;; Writing + csv-row->string csv->string write-csv-row write-csv) + +(import (std prelude)) +(import (only (std ports) read-bytes)) +(import (std maths)) +(import (std simd)) + +(import/class + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +;; Every export is documented there; the module doc is too. +(:doc #:file "csv.bjodoc") + +;; An ArgumentException, which is what a programming error raises here. +(: (-> string %a)) +(defun (argument-error message) + (raise (cast Exception (ArgumentException. (str "(text csv): " message))))) + +;; =========================================================================== +;; Errors +;; =========================================================================== + +;; `row` counts the rows read, from 1, and blank lines are not rows. `field` +;; counts from 1 in that row. +(type (: CsvError (Record (: message string) (: line int) (: row int) (: field int)))) + +(impl (Displayed CsvError) + (defun (displayed e) + (str "line " (int->string (record-ref e line)) + ", row " (int->string (record-ref e row)) + ", field " (int->string (record-ref e field)) + ": " (record-ref e message))) + (defun (display-into e b) (stringbuilder-add-string! b (displayed e)))) + +;; =========================================================================== +;; Options +;; =========================================================================== + +;; The end of the input, as a byte the reader looks at. +(: int) +(def spent -1) + +;; The quote byte when there is no quoting. It is not `spent`, so that the end +;; of the input does not start a quoted field. +(: int) +(def no-quote -2) + +(type (: Options (Record (: sep-code int) (: quote-code int) (: strict bool)))) + +(: (-> char (Option char) bool Options)) +(defun (options separator quote-char strict) + (def sep (char->int separator)) + (def q (match quote-char ((Some c) (char->int c)) (None no-quote))) + (when (or (>= sep 128) (= sep 10) (= sep 13)) + (argument-error "#:separator has to be an ASCII character that does not end a line")) + (when (or (>= q 128) (= q 10) (= q 13) (= q sep)) + (argument-error + "#:quote-char has to be an ASCII character that does not end a line and is not the separator")) + (Options (sep-code sep) (quote-code q) (strict strict))) + +;; =========================================================================== +;; The reader +;; =========================================================================== + +;; How much a read from the port asks for: 16 KiB first, and each read after +;; it twice the one before, up to 64 KiB. Each chunk is copied into a +;; Utf8Chunk, and an array over 85 000 bytes goes on .NET's large object +;; heap, which is slow to allocate and freed only by a full collection. +(: int) +(def read-first 16384) + +(: int) +(def read-most 65536) + +;; `chunk` holds a checked copy of `bytes`, from 0 to `limit`; see +;; check-chunk!. +;; `line` is the line the reader is on, from 1. `rows` is the number of rows +;; read. `started` is set once the byte order mark check is done. After an +;; error, `failed` holds it, and every later read gives it again. +(type (: CsvReader #:opaque (Record (: port (Option InputPort) #:mutable) + (: ask int #:mutable) + (: bytes (Array byte) #:mutable) + (: next int #:mutable) + (: limit int #:mutable) + (: chunk Utf8Chunk #:mutable) + (: line int #:mutable) + ;; The line the row being read, or last read, starts on. + (: row-line int #:mutable) + (: rows int #:mutable) + (: options Options) + ;; One buffer for every field with a doubled quote. + (: scratch StringBuilder) + ;; The fields of the row being read, and how many. + (: fields (Array string) #:mutable) + (: count int #:mutable) + (: started bool #:mutable) + (: failed (Option CsvError) #:mutable)))) + +(: (Array byte)) +(def no-bytes (make-array 0)) + +(: Utf8Chunk) +(def no-chunk (bytes->utf8-chunk no-bytes)) + +;; A reader over `bytes`, and then whatever `port` has after them. +(: (-> (Option InputPort) (Array byte) Options CsvReader)) +(defun (make-reader port bytes opts) + (def rd (CsvReader (port port) + (ask read-first) + (bytes bytes) + (next 0) + (limit (array-length bytes)) + (chunk no-chunk) + (line 1) + (row-line 1) + (rows 0) + (options opts) + (scratch (stringbuilder-empty)) + (fields (make-array 16)) + (count 0) + (started #f) + (failed None))) + (check-chunk! rd) + rd) + +(: (-> CsvReader int int string CsvError)) +(defun (error-at rd field line message) + (CsvError (message message) (line line) (row (+ (record-ref rd rows) 1)) (field field))) + +(: (-> CsvReader int string CsvError)) +(defun (csv-error rd field message) + (error-at rd field (record-ref rd line) message)) + +;; --------------------------------------------------------------------------- +;; Chunks +;; --------------------------------------------------------------------------- + +;; `count` bytes of `old` from `from`, then `more`: what is left of one chunk +;; and the next. +(: (-> (Array byte) int int (Array byte) (Array byte))) +(defun (join-bytes old from count more) + (if (= count 0) + more + (let ((joined (make-array (+ count (array-length more))))) + (array-copy! old from joined 0 count) + (array-copy! more 0 joined count (array-length more)) + joined))) + +;; A Utf8Chunk of the bytes the reader has now, so that its offsets are the +;; reader's. A field before its valid end is taken from it without being +;; checked again. One after it is checked by itself, so that the error can +;; say which field it is. +(: (-> CsvReader void)) +(defun (check-chunk! rd) + (set! rd .chunk (bytes->utf8-chunk (get rd .bytes) #:end (get rd .limit)))) + +;; Whether there are at least `n` bytes from `next` on. Reads from the port +;; until there are or the port ends. The bytes from `next` on are kept, and +;; `next` becomes 0. +(: (-> CsvReader int bool)) +(defun (ensure! rd n) + (let go () + (def have (- (get rd .limit) (get rd .next))) + (cond + ((>= have n) #t) + (else + (def (Some p) (get rd .port) :leave-with #f) + (def ask (get rd .ask)) + (match (read-bytes p (if (> n ask) n ask)) + (None + (set! rd .port None) + #f) + ((Some more) + (record-set! rd + (bytes (join-bytes (get rd .bytes) (get rd .next) have more)) + (next 0) + (limit (+ have (length more))) + (ask (if (< ask read-most) (* ask 2) read-most))) + (check-chunk! rd) + (go))))))) + +;; At least one byte more than the chunk has now, with the bytes from `next` +;; on kept. #f when the input has no more. A scan at index `i` continues at +;; `(+ next (- i start))` afterwards, where `start` was `next` before the call. +(: (-> CsvReader bool)) +(defun (more! rd) + (ensure! rd (+ (- (get rd .limit) (get rd .next)) 1))) + +;; The byte at `next`, or `spent` at the end of the input. +(: (-> CsvReader int)) +(defun (held-code rd) + (def i (get rd .next)) + (cond + ((< i (get rd .limit)) (cast int (get (get rd .bytes) i))) + ((ensure! rd 1) (cast int (get (get rd .bytes) (get rd .next)))) + (else spent))) + +(: (-> CsvReader void)) +(defun (step! rd) + (set! rd .next (+ (get rd .next) 1))) + +;; Past one line end: LF, CR LF, or CR alone. +(: (-> CsvReader void)) +(defun (end-line! rd) + (def c (held-code rd)) + (step! rd) + (when (and (= c 13) (= (held-code rd) 10)) + (step! rd)) + (record-set! rd (line (+ (record-ref rd line) 1)))) + +;; Whether the reader is at a byte order mark: EF BB BF. +(: (-> CsvReader bool)) +(defun (at-mark? rd) + (and (ensure! rd 3) + (let ((b (record-ref rd bytes)) (i (record-ref rd next))) + (and (= (cast int (array-ref b i)) 239) + (= (cast int (array-ref b (+ i 1))) 187) + (= (cast int (array-ref b (+ i 2))) 191))))) + +;; The byte at `next`, for an error message. +(: (-> CsvReader string)) +(defun (found rd) + (def c (held-code rd)) + (if (and (>= c 32) (< c 127)) + (str "'" (list->string (list (int->char c))) "'") + (str "the byte " (int->string c)))) + +;; --------------------------------------------------------------------------- +;; Fields +;; --------------------------------------------------------------------------- + +(: string) +(def not-utf-8 "the field has bytes that are not UTF-8") + +;; Whether the bytes from `start` to `end` of the chunk are UTF-8. +(: (-> CsvReader (Array byte) int int bool)) +(defun (utf-8? rd b start end) + (or (<= end (utf8-chunk-valid-end (record-ref rd chunk))) (bytes-utf-8? b start end))) + +;; The bytes from `start` to `end` as a string, or None when they are not +;; UTF-8. A field starts and ends next to an ASCII byte or at the end of the +;; chunk, so both offsets are between characters. +(: (-> CsvReader (Array byte) int int (Option string))) +(defun (text-of rd b start end) + (def c (record-ref rd chunk)) + (cond + ((= start end) (Some "")) + ((<= end (utf8-chunk-valid-end c)) (Some (utf8-chunk-substring c start end))) + ((bytes-utf-8? b start end) (Some (bytes->string b #:start start #:end end))) + (else None))) + +(: (-> CsvReader int (Array byte) int int (Result CsvError string))) +(defun (field-string rd field b start end) + (match (text-of rd b start end) + ((Some s) (Ok s)) + (None (Err (csv-error rd field not-utf-8))))) + +;; The first index from `from` that holds the separator, the quote, CR or LF, +;; or `limit` if there is none. +;; +;; 32 or 16 bytes at a time, as (text json) skips white space: each block is +;; compared with the four bytes at once, and the first match is the lowest set +;; bit of the mask. The bytes after the last whole block are looked at one by +;; one. Without a quote, `q` is not a byte, and the separator takes its lane. +(: (-> (Array byte) int int int int int)) +(defun (plain-end b from limit sep q) + (def qb (if (< q 0) sep q)) + (if simd256-accelerated? + (plain-end/256 b from limit sep qb) + (plain-end/128 b from limit sep qb))) + +(: (-> (Array byte) int int int int int)) +(defun (plain-end/256 b from limit sep q) + (def vs (simd256-splat (cast byte sep))) + (def vq (simd256-splat (cast byte q))) + (def lf (simd256-splat 10uy)) + (def cr (simd256-splat 13uy)) + (let go ((i from)) + (cond + ((> (+ i 32) limit) (plain-end/bytes b i limit sep q)) + (:def v (simd256-load b i)) + (:def m (simd-mask-bits (simd-or (simd-or (simd-eq v vs) (simd-eq v vq)) + (simd-or (simd-eq v lf) (simd-eq v cr))))) + ((= m 0) (go (+ i 32))) + (else (+ i (trailing-zeros m)))))) + +(: (-> (Array byte) int int int int int)) +(defun (plain-end/128 b from limit sep q) + (def vs (simd128-splat (cast byte sep))) + (def vq (simd128-splat (cast byte q))) + (def lf (simd128-splat 10uy)) + (def cr (simd128-splat 13uy)) + (let go ((i from)) + (cond + ((> (+ i 16) limit) (plain-end/bytes b i limit sep q)) + (:def v (simd128-load b i)) + (:def m (simd-mask-bits (simd-or (simd-or (simd-eq v vs) (simd-eq v vq)) + (simd-or (simd-eq v lf) (simd-eq v cr))))) + ((= m 0) (go (+ i 16))) + (else (+ i (trailing-zeros m)))))) + +(: (-> (Array byte) int int int int int)) +(defun (plain-end/bytes b from limit sep q) + (let go ((i from)) + (cond + ((>= i limit) i) + (:def c (cast int (array-ref b i))) + ((or (= c sep) (= c 10) (= c 13) (= c q)) i) + (else (go (+ i 1)))))) + +;; A field that does not start with the quote. It ends before the separator, +;; a line end or the end of the input. When strict, a quote in it is an error; +;; else the quote is part of the field. +(: (-> CsvReader int (Result CsvError string))) +(defun (read-plain rd field) + (def* (opts (.options rd)) + (sep (.sep-code opts)) + (q (.quote-code opts))) + (let go ((from (get rd .next))) + (def* (b (.bytes rd)) + (start (.next rd)) + (limit (.limit rd)) + (i (plain-end b from limit sep q))) + (cond + ((and (< i limit) (= (cast int (array-ref b i)) q)) + (if (get opts .strict) + (Err (csv-error rd field "a quote inside a field that does not start with one")) + (go (+ i 1)))) + ((< i limit) + (set! rd .next i) + (field-string rd field b start i)) + ((more! rd) (go (+ (get rd .next) (- i start)))) + (else + (set! rd .next i) + (field-string rd field b start i))))) + +;; The first index from `from` that holds the quote, CR or LF, or `limit` if +;; there is none. A block at a time, as plain-end; `q` is always a byte here, +;; since a field is quoted only when there is a quote. +(: (-> (Array byte) int int int int)) +(defun (quoted-end b from limit q) + (if simd256-accelerated? + (quoted-end/256 b from limit q) + (quoted-end/128 b from limit q))) + +(: (-> (Array byte) int int int int)) +(defun (quoted-end/256 b from limit q) + (def vq (simd256-splat (cast byte q))) + (def lf (simd256-splat 10uy)) + (def cr (simd256-splat 13uy)) + (let go ((i from)) + (cond + ((> (+ i 32) limit) (quoted-end/bytes b i limit q)) + (:def v (simd256-load b i)) + (:def m (simd-mask-bits (simd-or (simd-eq v vq) (simd-or (simd-eq v lf) (simd-eq v cr))))) + ((= m 0) (go (+ i 32))) + (else (+ i (trailing-zeros m)))))) + +(: (-> (Array byte) int int int int)) +(defun (quoted-end/128 b from limit q) + (def vq (simd128-splat (cast byte q))) + (def lf (simd128-splat 10uy)) + (def cr (simd128-splat 13uy)) + (let go ((i from)) + (cond + ((> (+ i 16) limit) (quoted-end/bytes b i limit q)) + (:def v (simd128-load b i)) + (:def m (simd-mask-bits (simd-or (simd-eq v vq) (simd-or (simd-eq v lf) (simd-eq v cr))))) + ((= m 0) (go (+ i 16))) + (else (+ i (trailing-zeros m)))))) + +(: (-> (Array byte) int int int int)) +(defun (quoted-end/bytes b from limit q) + (let go ((i from)) + (cond + ((>= i limit) i) + (:def c (cast int (array-ref b i))) + ((or (= c q) (= c 10) (= c 13)) i) + (else (go (+ i 1)))))) + +;; The text between the quotes, from `start` to `end`. With `escaped`, each +;; quote in it is the first of a doubled pair, and the second is dropped. +(: (-> CsvReader int (Array byte) int int bool (Result CsvError string))) +(defun (quoted-string rd field b start end escaped) + (def q (get rd .options .quote-code)) + (cond + ((not escaped) (field-string rd field b start end)) + ((not (utf-8? rd b start end)) (Err (csv-error rd field not-utf-8))) + (else + (def sb (record-ref rd scratch)) + (stringbuilder-clear! sb) + (let go ((from start)) + (def i (quoted-end b from end q)) + (cond + ;; A line end inside the quotes is part of the text. + ((and (< i end) (not (= (cast int (array-ref b i)) q))) + (stringbuilder-add-bytes! sb b from (+ i 1)) + (go (+ i 1))) + ((< i end) + (stringbuilder-add-bytes! sb b from (+ i 1)) + (go (+ i 2))) + (else + (stringbuilder-add-bytes! sb b from end) + (Ok (stringbuilder->string sb)))))))) + +;; After the closing quote: the separator, a line end or the end of the input. +;; When not strict, anything else up to one of those is added to the field. +(: (-> CsvReader int string (Result CsvError string))) +(defun (after-quote rd field content) + (def opts (record-ref rd options)) + (def c (held-code rd)) + (cond + ((or (= c (record-ref opts sep-code)) (= c 10) (= c 13) (= c spent)) (Ok content)) + ((record-ref opts strict) + (Err (csv-error rd field (str "a quoted field ends at its closing quote, and then there is " + (found rd))))) + (else + (match (read-plain rd field) + ((Ok tail) (Ok (str content tail))) + ((Err e) (Err e)))))) + +;; A field that starts with the quote, at `next`. `from` is where the search +;; for the closing quote continues, `escaped` whether a doubled quote was +;; passed, and `lines` how many line ends are inside the quotes. +;; +;; A quote or a CR is only known after the byte that follows it is seen, so +;; when that byte is not in the chunk, the chunk is made longer first. +(: (-> CsvReader int (Result CsvError string))) +(defun (read-quoted rd field) + (def* (opts (.options rd)) + (q (.quote-code opts)) + (first-line (.line rd))) + (let go ((from (+ (.next rd) 1)) (escaped #f) (lines 0)) + (def* (b (.bytes rd)) + (start (.next rd)) + (limit (.limit rd)) + (i (quoted-end b from limit q)) + (c (if (< i limit) + (cast int (array-ref b i)) + spent)) + (after (if (< (+ i 1) limit) + (cast int (array-ref b (+ i 1))) + spent))) + (cond + ((and (or (= c spent) (and (or (= c q) (= c 13)) (= after spent))) + (more! rd)) + (go (+ (.next rd) (- i start)) escaped lines)) + ((= c 10) (go (+ i 1) escaped (+ lines 1))) + ((= c 13) (go (if (= after 10) (+ i 2) (+ i 1)) escaped (+ lines 1))) + ((and (= c q) (= after q)) (go (+ i 2) #t lines)) + ((= c q) + (record-set! rd (next (+ i 1)) (line (+ (.line rd) lines))) + (match (quoted-string rd field b (+ start 1) i escaped) + ((Ok content) (after-quote rd field content)) + ((Err e) (Err e)))) + ;; The input ended inside the quotes. + ((.strict opts) + (Err (error-at rd field first-line "the input ends inside a quoted field that starts on this line"))) + (else + (record-set! rd (next i) (line (+ (.line rd) lines))) + (quoted-string rd field b (+ start 1) i escaped))))) + +(: (-> CsvReader int (Result CsvError string))) +(defun (read-field rd field) + (if (= (held-code rd) (get rd .options .quote-code)) + (read-quoted rd field) + (read-plain rd field))) + +;; --------------------------------------------------------------------------- +;; Rows +;; --------------------------------------------------------------------------- + +;; The next row, or None at the end of the input. Blank lines are skipped. A +;; line end after the last row is not an empty row. +(: (-> CsvReader (Result CsvError (Option (IArray string))))) +(defun (read-row rd) + (unless (record-ref rd started) + (record-set! rd (started #t)) + (when (at-mark? rd) + (record-set! rd (next (+ (record-ref rd next) 3))))) + (def sep (get rd .options .sep-code)) + (let skip () + (def c (held-code rd)) + (cond + ((= c spent) (Ok None)) + ((or (= c 10) (= c 13)) + (end-line! rd) + (skip)) + (else + (record-set! rd (row-line (record-ref rd line))) + (def q (record-ref (record-ref rd options) quote-code)) + (let go ((field 1)) + (def* (b (.bytes rd)) + (start (.next rd)) + (limit (.limit rd)) + (i (plain-end b start limit sep q))) + ;; At the end of the chunk, `c` is the quote, so that read-field + ;; reads the field and the chunk after it. + (def c (if (< i limit) (cast int (array-ref b i)) q)) + (cond + ;; The usual field: no quote, and its end in the chunk. It is read + ;; here, without read-field. + ((not (= c q)) + (match (text-of rd b start i) + (None (Err (csv-error rd field not-utf-8))) + ((Some s) + (push-field! rd s) + (cond + ((= c sep) + (record-set! rd (next (+ i 1))) + (go (+ field 1))) + (else + (record-set! rd (next i)) + (end-row! rd)))))) + (else + (match (read-field rd field) + ((Err e) (Err e)) + ((Ok s) + (push-field! rd s) + ;; A field ends only at the separator, a line end or the end + ;; of the input. + (cond + ((= (held-code rd) sep) + (step! rd) + (go (+ field 1))) + (else (end-row! rd)))))))))))) + +;; At the end of a row: past its line end, if it has one, and the row. +(: (-> CsvReader (Result CsvError (Option (IArray string))))) +(defun (end-row! rd) + (unless (= (held-code rd) spent) (end-line! rd)) + (record-set! rd (rows (+ (record-ref rd rows) 1))) + (Ok (Some (take-row! rd)))) + +;; Adds a field to the row being read, growing the array when it is full. +(: (-> CsvReader string void)) +(defun (push-field! rd s) + (def a (.fields rd)) + (def n (.count rd)) + (cond + ((< n (array-length a)) (array-set! a n s)) + (else + (def bigger (array-resize a (* n 2))) + (array-set! bigger n s) + (record-set! rd (fields bigger)))) + (record-set! rd (count (+ n 1)))) + +;; The row read, as an IArray of its own: one copy of the fields, and no +;; object but the array. The array is used again for the next. +(: (-> CsvReader (IArray string))) +(defun (take-row! rd) + (def row (subarray->iarray (record-ref rd fields) 0 (record-ref rd count))) + (record-set! rd (count 0)) + row) + +;; The next row, or None at the end of the input. After an error, the same +;; error again. +(: (-> CsvReader (Result CsvError (Option (IArray string))))) +(defun (csv-next! rd) + (match (record-ref rd failed) + ((Some e) (Err e)) + (None + (match (read-row rd) + ((Err e) + (record-set! rd (failed (Some e))) + (Err e)) + (r r))))) + +;; The line the row last read starts on, from 1: where a row's own errors, +;; such as a field that does not decode, are. +(: (-> CsvReader int)) +(defun (csv-row-line rd) (record-ref rd row-line)) + +;; How many rows have been read, the last one included. Blank lines are not +;; rows. +(: (-> CsvReader int)) +(defun (csv-row-number rd) (record-ref rd rows)) + +;; =========================================================================== +;; Sources +;; =========================================================================== + +;; Declared before the trait that names it. +(type (: CsvInput #:opaque (Union (: CsvInText string) + (: CsvInPort InputPort) + (: CsvInFile string)))) + +;; What can be read. A type of your own gets an implementation by handing +;; csv-input something that already has one. +(def/trait (CsvSource %s) + (: csv-input (-> %s CsvInput))) + +;; A path, which the reader opens and closes. A string on its own is CSV. +(type (: CsvFile #:opaque (Record (: path string)))) + +(: (-> string CsvFile)) +(defun (csv-file path) (CsvFile (path path))) + +(impl (CsvSource string) + (defun (csv-input s) (CsvInText s))) + +(impl (CsvSource InputPort) + (defun (csv-input p) (CsvInPort p))) + +(impl (CsvSource CsvFile) + (defun (csv-input f) (CsvInFile (record-ref f path)))) + +;; A port that was handed over stays open. A file is opened here, through the +;; FS effect, and closed here; a file that does not open raises. +(: (-> CsvInput Options (-> CsvReader %a) %a)) +(defun (with-reader input opts proc) + (match input + ((CsvInText text) (proc (make-reader None (string->bytes text) opts))) + ((CsvInPort p) (proc (make-reader (Some p) no-bytes opts))) + ((CsvInFile path) + (match (call-with-input-file path (fun (p) (proc (make-reader (Some p) no-bytes opts)))) + ((Ok a) a) + ((Err e) (raise e)))))) + +;; The reader is for proc's use, and its file is closed when proc returns. +(: (-> %s (-> CsvReader %a) + (#:separator char) (#:quote-char (Option char)) (#:strict? bool) + %a) + (where (CsvSource %s))) +(defun (call-with-csv-reader source proc #:separator #\, #:quote-char (Some #\") #:strict? #t) + (with-reader (csv-input source) (options separator quote-char strict?) proc)) + +;; =========================================================================== +;; Folds +;; =========================================================================== + +(: (-> CsvReader %seed (-> (IArray string) %seed %seed) (Result CsvError %seed))) +(defun (fold-rows rd seed f) + (let loop ((seed seed)) + (match (csv-next! rd) + ((Err e) (Err e)) + ((Ok None) (Ok seed)) + ((Ok (Some row)) (loop (f row seed)))))) + +;; Each row and the seed so far give the next seed. The rows before an error +;; have been folded when the error is answered. +(: (-> %s %seed (-> (IArray string) %seed %seed) + (#:separator char) (#:quote-char (Option char)) (#:strict? bool) + (Result CsvError %seed)) + (where (CsvSource %s))) +(defun (csv-fold source seed f #:separator #\, #:quote-char (Some #\") #:strict? #t) + (with-reader (csv-input source) (options separator quote-char strict?) + (fun (rd) (fold-rows rd seed f)))) + +(: (-> %s (-> (IArray string) void) + (#:separator char) (#:quote-char (Option char)) (#:strict? bool) + (Result CsvError Unit)) + (where (CsvSource %s))) +(defun (csv-for-each source f #:separator #\, #:quote-char (Some #\") #:strict? #t) + (with-reader (csv-input source) (options separator quote-char strict?) + (fun (rd) (fold-rows rd unit (fun (row u) (f row) u))))) + +;; =========================================================================== +;; The whole document +;; =========================================================================== + +(: (-> %s + (#:separator char) (#:quote-char (Option char)) (#:strict? bool) + (Result CsvError (Vec (IArray string)))) + (where (CsvSource %s))) +(defun (parse-csv source #:separator #\, #:quote-char (Some #\") #:strict? #t) + (with-reader (csv-input source) (options separator quote-char strict?) + (fun (rd) + (def rows (transientvec-empty)) + (match (fold-rows rd unit (fun (row u) (transientvec-add! rows row) u)) + ((Ok _) (Ok (transientvec->vec rows))) + ((Err e) (Err e)))))) + +;; =========================================================================== +;; Rows of bytes, cut into pieces +;; =========================================================================== +;; +;; For a reader that reads the pieces of a large text on several threads. A +;; cut is just after a line feed that is not inside quotes. Whether a line +;; feed is inside quotes is the parity of the quotes before it, since a quote +;; in a quoted field is written as two quotes. So the quotes of each piece +;; are counted with one vectorized call, and only the bytes from each cut's +;; target to the next line feed are looked at one by one. + +(import/extern + (count-byte (: System.MemoryExtensions.Count (-> (RoSpan byte) byte int))) + (index-of-byte (: System.MemoryExtensions.IndexOf (-> (RoSpan byte) byte int))) + (index-of-either (: System.MemoryExtensions.IndexOfAny (-> (RoSpan byte) byte byte int)))) + +;; The quote's byte, or -1 without quoting. +(: (-> (Option char) int)) +(defun (quote-byte quote-char) + (match quote-char + (None -1) + ((Some c) + (def q (char->int c)) + (when (or (>= q 128) (= q 10) (= q 13)) + (argument-error "#:quote-char has to be an ASCII character that does not end a line")) + q))) + +;; The index just after the first line feed from `from` that is not inside +;; quotes, when `from` is inside quotes or not; None when there is none. +(: (-> (Array byte) int int bool (Option int))) +(defun (row-end-from b from q inside) + (let go ((i from) (inside inside)) + (def rest (array->ro-span b #:from i)) + (def j (if (< q 0) (index-of-byte rest 10uy) (index-of-either rest 10uy (cast byte q)))) + (cond + ((< j 0) None) + ((not (= (array-ref b (+ i j)) 10uy)) (go (+ i j 1) (not inside))) + (inside (go (+ i j 1) inside)) + (else (Some (+ i j 1)))))) + +;; The number of quotes from `from` to `to`. +(: (-> (Array byte) int int int int)) +(defun (quotes-in b from to q) + (if (< q 0) 0 (count-byte (array->ro-span b #:from from #:to to) (cast byte q)))) + +;; The index just after the line feed that ends the row starting at `from`, a +;; line feed inside quotes being part of the row; None when no line feed ends +;; it, as for the last row of a text without a final line feed. +(: (-> (Array byte) int (#:quote-char (Option char)) (Option int))) +(defun (csv-row-end bytes from #:quote-char (Some #\")) + (when (or (< from 0) (> from (array-length bytes))) + (argument-error "csv-row-end: the index is not in the bytes")) + (row-end-from bytes from (quote-byte quote-char) #f)) + +;; The bytes from `#:from`, a row's start, to the end cut into at most +;; `parts` pieces of whole rows of about one size: the index where each +;; piece starts, and the end. No piece is empty. None when the quotes do not +;; close. +(: (-> (Array byte) int (#:from int) (#:quote-char (Option char)) (Option (Array int)))) +(defun (csv-cut-rows bytes parts #:from 0 #:quote-char (Some #\")) + (def n (array-length bytes)) + (when (< parts 1) (argument-error "csv-cut-rows: the number of pieces has to be 1 or more")) + (when (or (< from 0) (> from n)) (argument-error "csv-cut-rows: #:from is not in the bytes")) + (def q (quote-byte quote-char)) + ;; Where each piece would start if all were of one size, and the end. + (def targets + {arraying (+ from (cast int (quotient (* (cast long (- n from)) (cast long k)) (cast long parts)))) + (:for k (range 0 (+ parts 1)))}) + (def counts {arraying (quotes-in bytes (array-ref targets k) (array-ref targets (+ k 1)) q) + (:for k (range 0 parts))}) + (if (odd? (loop (:for c counts) (:acc s (folding 0 (+ s c))))) + None + (Some (cuts-at bytes q targets counts from n)))) + +;; The cuts: from the byte before each target, the end of the row it is in, +;; so that a target just after a line feed is a cut. The quotes before the +;; target are the sum of the counts before it; the byte before the target, +;; when it is a quote, is not before that byte. +(: (-> (Array byte) int (Array int) (Array int) int int (Array int))) +(defun (cuts-at b q targets counts from n) + (def parts (array-length counts)) + (def out (: (make-array (+ parts 1)) (Array int))) + (array-set! out 0 from) + (let go ((k 1) (before (array-ref counts 0)) (found 1)) + (if (= k parts) + (finish-cuts out found n) + (match (let* ((target (array-ref targets k)) + (start (max from (- target 1))) + (quote-before? (and (< start target) (>= q 0) (= (cast int (array-ref b start)) q)))) + (row-end-from b start q (odd? (if quote-before? (- before 1) before)))) + (None (finish-cuts out found n)) + ((Some c) + (def last (array-ref out (- found 1))) + (if (and (> c last) (< c n)) + (begin + (array-set! out found c) + (go (+ k 1) (+ before (array-ref counts k)) (+ found 1))) + (go (+ k 1) (+ before (array-ref counts k)) found))))))) + +;; The `found` cuts and then the end, when the last cut is not the end. +(: (-> (Array int) int int (Array int))) +(defun (finish-cuts out found n) + (if (< (array-ref out (- found 1)) n) + (begin + (array-set! out found n) + (array-resize out (+ found 1))) + (array-resize out found))) + +;; =========================================================================== +;; The writer +;; =========================================================================== +;; +;; A field gets quotes only when it holds the separator, the quote, CR or LF, +;; or when it is the only field of its row and empty: an empty line is not a +;; row when it is read back, and `""` is. + +;; `specials` holds the characters that make a field need quotes. +(type (: Writer (Record (: separator char) + (: quote-char (Option char)) + (: specials string) + (: quote-all bool) + (: newline string)))) + +(: (-> char (Option char) bool bool Writer)) +(defun (writer separator quote-char quote-all crlf) + (ignore (options separator quote-char #t)) + (def specials (match quote-char + ((Some q) (list->string (list separator q #\return #\newline))) + (None (list->string (list separator #\return #\newline))))) + (Writer (separator separator) + (quote-char quote-char) + (specials specials) + (quote-all quote-all) + (newline (if crlf "\r\n" "\n")))) + +;; `s` between quotes, each quote in it doubled. +(: (-> StringBuilder string char void)) +(defun (add-quoted! sb s q) + (def qs (list->string (list q))) + (stringbuilder-add! sb q) + (let go ((from (string-cursor-start s))) + (match (string-search-any s qs from) + (None (stringbuilder-add-substring! sb s from (string-cursor-end s))) + ((Some at) + (def after (string-cursor-next s at)) + (stringbuilder-add-substring! sb s from after) + (stringbuilder-add! sb q) + (go after)))) + (stringbuilder-add! sb q)) + +(: (-> StringBuilder Writer string void)) +(defun (add-field! sb w s) + (def needs (or (record-ref w quote-all) + (some? (string-search-any s (record-ref w specials) (string-cursor-start s))))) + (cond + ((not needs) (stringbuilder-add-string! sb s)) + (:match (Some q) (record-ref w quote-char) (add-quoted! sb s q)) + ((string-empty? s) unit) + (else + (argument-error + (str "the field " (written s) " needs quotes, and there is no #:quote-char"))))) + +;; Without the line end. The fields are walked once, so the row can be any +;; Iterable; a row of one empty field is known only after it. +(: (-> StringBuilder Writer %row void) (where (Iterable %row #:elem string))) +(defun (add-row! sb w row) + (def mark (stringbuilder-length sb)) + (def n (loop (:for s row) + (:with first? #t #f) + (:do (unless first? (stringbuilder-add! sb (record-ref w separator))) + (add-field! sb w s)) + (:acc (counting s)))) + (when (and (= n 1) (= (stringbuilder-length sb) mark)) + (match (record-ref w quote-char) + ((Some q) + (stringbuilder-add! sb q) + (stringbuilder-add! sb q)) + (None unit)))) + +;; One row as CSV text, without a line end. +(: (-> %row (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool) string) + (where (Iterable %row #:elem string))) +(defun (csv-row->string row #:separator #\, #:quote-char (Some #\") #:quote-all? #f) + (def sb (stringbuilder-empty)) + (add-row! sb (writer separator quote-char quote-all? #f) row) + (stringbuilder->string sb)) + +;; Every row, each ended by a line end. +(: (-> %rows (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool) (#:crlf? bool) + string) + (where (Iterable %rows #:elem %row) (Iterable %row #:elem string))) +(defun (csv->string rows #:separator #\, #:quote-char (Some #\") #:quote-all? #f #:crlf? #f) + (def w (writer separator quote-char quote-all? crlf?)) + (def sb (stringbuilder-empty)) + (loop (:for row rows) + (:do (add-row! sb w row) + (stringbuilder-add-string! sb (record-ref w newline)))) + (stringbuilder->string sb)) + +;; One row and its line end. +(: (-> OutputPort %row (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool) + (#:crlf? bool) + void) + (where (Iterable %row #:elem string))) +(defun (write-csv-row port row #:separator #\, #:quote-char (Some #\") #:quote-all? #f #:crlf? #f) + (def w (writer separator quote-char quote-all? crlf?)) + (def sb (stringbuilder-empty)) + (add-row! sb w row) + (stringbuilder-add-string! sb (record-ref w newline)) + (write-string port (stringbuilder->string sb))) + +;; Text is written to the port each time this much has been made, so a long +;; document is never one string. +(: int) +(def write-chunk 65536) + +;; Every row, each ended by a line end. The rows are read once, in order, so +;; a Seq is written as it is made. +(: (-> OutputPort %rows (#:separator char) (#:quote-char (Option char)) (#:quote-all? bool) + (#:crlf? bool) + void) + (where (Iterable %rows #:elem %row) (Iterable %row #:elem string))) +(defun (write-csv port rows #:separator #\, #:quote-char (Some #\") #:quote-all? #f #:crlf? #f) + (def w (writer separator quote-char quote-all? crlf?)) + (def sb (stringbuilder-empty)) + (loop (:for row rows) + (:do (add-row! sb w row) + (stringbuilder-add-string! sb (record-ref w newline)) + (when (>= (stringbuilder-length sb) write-chunk) + (write-string port (stringbuilder->string sb)) + (stringbuilder-clear! sb)))) + (when (> (stringbuilder-length sb) 0) + (write-string port (stringbuilder->string sb)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; text/encodings.bjo +;; +;; Every character encoding .NET knows, as a value to hand to +;; `bytes->string`, `string->bytes` and `reencode-input-port`, and finding +;; one by name or code page. +;; +;; The names are .NET's, lower-cased: `utf-8`, `windows-1252`, `dos-862`, +;; `shift_jis`. Two kinds differ. .NET's `utf-16` and `utf-32` are +;; little-endian and are `utf-16le` and `utf-32le` here, next to `utf-16be` +;; and `utf-32be`. Where .NET gives one name to two code pages, the less +;; common one is `cp` and its number: `euc-jp` is 51932 and `cp20932` the +;; other, `iso-2022-jp` is 50220 and `cp50222` the other. +;; +;; Each value is a property of `Bjolang.Runtime.Encodings` that makes its +;; encoding the first time it is read, so importing the module makes none of +;; them and a program pays only for the ones it uses. + +(export + encoding-named encoding-of-code-page encoding-name + asmo-708 big5 cp1025 cp20932 cp50222 cp866 cp875 csiso2022jp dos-720 + dos-862 euc-cn euc-jp euc-kr gb18030 gb2312 hz-gb-2312 ibm-thai ibm00858 + ibm00924 ibm01047 ibm01140 ibm01141 ibm01142 ibm01143 ibm01144 ibm01145 + ibm01146 ibm01147 ibm01148 ibm01149 ibm037 ibm1026 ibm273 ibm277 ibm278 + ibm280 ibm284 ibm285 ibm290 ibm297 ibm420 ibm423 ibm424 ibm437 ibm500 + ibm737 ibm775 ibm850 ibm852 ibm855 ibm857 ibm860 ibm861 ibm863 ibm864 + ibm865 ibm869 ibm870 ibm871 ibm880 ibm905 iso-2022-jp iso-2022-kr + iso-8859-1 iso-8859-13 iso-8859-15 iso-8859-2 iso-8859-3 iso-8859-4 + iso-8859-5 iso-8859-6 iso-8859-7 iso-8859-8 iso-8859-8-i iso-8859-9 johab + koi8-r koi8-u ks_c_5601-1987 macintosh shift_jis us-ascii utf-16be + utf-16le utf-32be utf-32le utf-8 windows-1250 windows-1251 windows-1252 + windows-1253 windows-1254 windows-1255 windows-1256 windows-1257 + windows-1258 windows-874 x-chinese-cns x-chinese-eten x-cp20001 x-cp20003 + x-cp20004 x-cp20005 x-cp20261 x-cp20269 x-cp20936 x-cp20949 x-cp50227 + x-ebcdic-koreanextended x-europa x-ia5 x-ia5-german x-ia5-norwegian + x-ia5-swedish x-iscii-as x-iscii-be x-iscii-de x-iscii-gu x-iscii-ka + x-iscii-ma x-iscii-or x-iscii-pa x-iscii-ta x-iscii-te x-mac-arabic + x-mac-ce x-mac-chinesesimp x-mac-chinesetrad x-mac-croatian x-mac-cyrillic + x-mac-greek x-mac-hebrew x-mac-icelandic x-mac-japanese x-mac-korean + x-mac-romanian x-mac-thai x-mac-turkish x-mac-ukrainian) + +(import (std prelude)) + +(:doc #:file "encodings.bjodoc") + +(import/extern + (clr-named (: Bjolang.Runtime.Encodings.Named (-> string (Option Encoding)))) + (clr-of-code-page (: Bjolang.Runtime.Encodings.OfCodePage (-> int (Option Encoding)))) + (clr-name-of (: Bjolang.Runtime.Encodings.NameOf (-> Encoding string)))) + +(: (-> string (Option Encoding))) +(defun (encoding-named name) (clr-named name)) + +(: (-> int (Option Encoding))) +(defun (encoding-of-code-page code-page) (clr-of-code-page code-page)) + +(: (-> Encoding string)) +(defun (encoding-name encoding) (clr-name-of encoding)) + +;; The encodings. +(import/extern + (asmo-708 (: Bjolang.Runtime.Encodings.Cp708 #:get)) + (big5 (: Bjolang.Runtime.Encodings.Cp950 #:get)) + (cp1025 (: Bjolang.Runtime.Encodings.Cp21025 #:get)) + (cp20932 (: Bjolang.Runtime.Encodings.Cp20932 #:get)) + (cp50222 (: Bjolang.Runtime.Encodings.Cp50222 #:get)) + (cp866 (: Bjolang.Runtime.Encodings.Cp866 #:get)) + (cp875 (: Bjolang.Runtime.Encodings.Cp875 #:get)) + (csiso2022jp (: Bjolang.Runtime.Encodings.Cp50221 #:get)) + (dos-720 (: Bjolang.Runtime.Encodings.Cp720 #:get)) + (dos-862 (: Bjolang.Runtime.Encodings.Cp862 #:get)) + (euc-cn (: Bjolang.Runtime.Encodings.Cp51936 #:get)) + (euc-jp (: Bjolang.Runtime.Encodings.Cp51932 #:get)) + (euc-kr (: Bjolang.Runtime.Encodings.Cp51949 #:get)) + (gb18030 (: Bjolang.Runtime.Encodings.Cp54936 #:get)) + (gb2312 (: Bjolang.Runtime.Encodings.Cp936 #:get)) + (hz-gb-2312 (: Bjolang.Runtime.Encodings.Cp52936 #:get)) + (ibm-thai (: Bjolang.Runtime.Encodings.Cp20838 #:get)) + (ibm00858 (: Bjolang.Runtime.Encodings.Cp858 #:get)) + (ibm00924 (: Bjolang.Runtime.Encodings.Cp20924 #:get)) + (ibm01047 (: Bjolang.Runtime.Encodings.Cp1047 #:get)) + (ibm01140 (: Bjolang.Runtime.Encodings.Cp1140 #:get)) + (ibm01141 (: Bjolang.Runtime.Encodings.Cp1141 #:get)) + (ibm01142 (: Bjolang.Runtime.Encodings.Cp1142 #:get)) + (ibm01143 (: Bjolang.Runtime.Encodings.Cp1143 #:get)) + (ibm01144 (: Bjolang.Runtime.Encodings.Cp1144 #:get)) + (ibm01145 (: Bjolang.Runtime.Encodings.Cp1145 #:get)) + (ibm01146 (: Bjolang.Runtime.Encodings.Cp1146 #:get)) + (ibm01147 (: Bjolang.Runtime.Encodings.Cp1147 #:get)) + (ibm01148 (: Bjolang.Runtime.Encodings.Cp1148 #:get)) + (ibm01149 (: Bjolang.Runtime.Encodings.Cp1149 #:get)) + (ibm037 (: Bjolang.Runtime.Encodings.Cp37 #:get)) + (ibm1026 (: Bjolang.Runtime.Encodings.Cp1026 #:get)) + (ibm273 (: Bjolang.Runtime.Encodings.Cp20273 #:get)) + (ibm277 (: Bjolang.Runtime.Encodings.Cp20277 #:get)) + (ibm278 (: Bjolang.Runtime.Encodings.Cp20278 #:get)) + (ibm280 (: Bjolang.Runtime.Encodings.Cp20280 #:get)) + (ibm284 (: Bjolang.Runtime.Encodings.Cp20284 #:get)) + (ibm285 (: Bjolang.Runtime.Encodings.Cp20285 #:get)) + (ibm290 (: Bjolang.Runtime.Encodings.Cp20290 #:get)) + (ibm297 (: Bjolang.Runtime.Encodings.Cp20297 #:get)) + (ibm420 (: Bjolang.Runtime.Encodings.Cp20420 #:get)) + (ibm423 (: Bjolang.Runtime.Encodings.Cp20423 #:get)) + (ibm424 (: Bjolang.Runtime.Encodings.Cp20424 #:get)) + (ibm437 (: Bjolang.Runtime.Encodings.Cp437 #:get)) + (ibm500 (: Bjolang.Runtime.Encodings.Cp500 #:get)) + (ibm737 (: Bjolang.Runtime.Encodings.Cp737 #:get)) + (ibm775 (: Bjolang.Runtime.Encodings.Cp775 #:get)) + (ibm850 (: Bjolang.Runtime.Encodings.Cp850 #:get)) + (ibm852 (: Bjolang.Runtime.Encodings.Cp852 #:get)) + (ibm855 (: Bjolang.Runtime.Encodings.Cp855 #:get)) + (ibm857 (: Bjolang.Runtime.Encodings.Cp857 #:get)) + (ibm860 (: Bjolang.Runtime.Encodings.Cp860 #:get)) + (ibm861 (: Bjolang.Runtime.Encodings.Cp861 #:get)) + (ibm863 (: Bjolang.Runtime.Encodings.Cp863 #:get)) + (ibm864 (: Bjolang.Runtime.Encodings.Cp864 #:get)) + (ibm865 (: Bjolang.Runtime.Encodings.Cp865 #:get)) + (ibm869 (: Bjolang.Runtime.Encodings.Cp869 #:get)) + (ibm870 (: Bjolang.Runtime.Encodings.Cp870 #:get)) + (ibm871 (: Bjolang.Runtime.Encodings.Cp20871 #:get)) + (ibm880 (: Bjolang.Runtime.Encodings.Cp20880 #:get)) + (ibm905 (: Bjolang.Runtime.Encodings.Cp20905 #:get)) + (iso-2022-jp (: Bjolang.Runtime.Encodings.Cp50220 #:get)) + (iso-2022-kr (: Bjolang.Runtime.Encodings.Cp50225 #:get)) + (iso-8859-1 (: Bjolang.Runtime.Encodings.Cp28591 #:get)) + (iso-8859-13 (: Bjolang.Runtime.Encodings.Cp28603 #:get)) + (iso-8859-15 (: Bjolang.Runtime.Encodings.Cp28605 #:get)) + (iso-8859-2 (: Bjolang.Runtime.Encodings.Cp28592 #:get)) + (iso-8859-3 (: Bjolang.Runtime.Encodings.Cp28593 #:get)) + (iso-8859-4 (: Bjolang.Runtime.Encodings.Cp28594 #:get)) + (iso-8859-5 (: Bjolang.Runtime.Encodings.Cp28595 #:get)) + (iso-8859-6 (: Bjolang.Runtime.Encodings.Cp28596 #:get)) + (iso-8859-7 (: Bjolang.Runtime.Encodings.Cp28597 #:get)) + (iso-8859-8 (: Bjolang.Runtime.Encodings.Cp28598 #:get)) + (iso-8859-8-i (: Bjolang.Runtime.Encodings.Cp38598 #:get)) + (iso-8859-9 (: Bjolang.Runtime.Encodings.Cp28599 #:get)) + (johab (: Bjolang.Runtime.Encodings.Cp1361 #:get)) + (koi8-r (: Bjolang.Runtime.Encodings.Cp20866 #:get)) + (koi8-u (: Bjolang.Runtime.Encodings.Cp21866 #:get)) + (ks_c_5601-1987 (: Bjolang.Runtime.Encodings.Cp949 #:get)) + (macintosh (: Bjolang.Runtime.Encodings.Cp10000 #:get)) + (shift_jis (: Bjolang.Runtime.Encodings.Cp932 #:get)) + (us-ascii (: Bjolang.Runtime.Encodings.Cp20127 #:get)) + (utf-16be (: Bjolang.Runtime.Encodings.Cp1201 #:get)) + (utf-16le (: Bjolang.Runtime.Encodings.Cp1200 #:get)) + (utf-32be (: Bjolang.Runtime.Encodings.Cp12001 #:get)) + (utf-32le (: Bjolang.Runtime.Encodings.Cp12000 #:get)) + (utf-8 (: Bjolang.Runtime.Encodings.Cp65001 #:get)) + (windows-1250 (: Bjolang.Runtime.Encodings.Cp1250 #:get)) + (windows-1251 (: Bjolang.Runtime.Encodings.Cp1251 #:get)) + (windows-1252 (: Bjolang.Runtime.Encodings.Cp1252 #:get)) + (windows-1253 (: Bjolang.Runtime.Encodings.Cp1253 #:get)) + (windows-1254 (: Bjolang.Runtime.Encodings.Cp1254 #:get)) + (windows-1255 (: Bjolang.Runtime.Encodings.Cp1255 #:get)) + (windows-1256 (: Bjolang.Runtime.Encodings.Cp1256 #:get)) + (windows-1257 (: Bjolang.Runtime.Encodings.Cp1257 #:get)) + (windows-1258 (: Bjolang.Runtime.Encodings.Cp1258 #:get)) + (windows-874 (: Bjolang.Runtime.Encodings.Cp874 #:get)) + (x-chinese-cns (: Bjolang.Runtime.Encodings.Cp20000 #:get)) + (x-chinese-eten (: Bjolang.Runtime.Encodings.Cp20002 #:get)) + (x-cp20001 (: Bjolang.Runtime.Encodings.Cp20001 #:get)) + (x-cp20003 (: Bjolang.Runtime.Encodings.Cp20003 #:get)) + (x-cp20004 (: Bjolang.Runtime.Encodings.Cp20004 #:get)) + (x-cp20005 (: Bjolang.Runtime.Encodings.Cp20005 #:get)) + (x-cp20261 (: Bjolang.Runtime.Encodings.Cp20261 #:get)) + (x-cp20269 (: Bjolang.Runtime.Encodings.Cp20269 #:get)) + (x-cp20936 (: Bjolang.Runtime.Encodings.Cp20936 #:get)) + (x-cp20949 (: Bjolang.Runtime.Encodings.Cp20949 #:get)) + (x-cp50227 (: Bjolang.Runtime.Encodings.Cp50227 #:get)) + (x-ebcdic-koreanextended (: Bjolang.Runtime.Encodings.Cp20833 #:get)) + (x-europa (: Bjolang.Runtime.Encodings.Cp29001 #:get)) + (x-ia5 (: Bjolang.Runtime.Encodings.Cp20105 #:get)) + (x-ia5-german (: Bjolang.Runtime.Encodings.Cp20106 #:get)) + (x-ia5-norwegian (: Bjolang.Runtime.Encodings.Cp20108 #:get)) + (x-ia5-swedish (: Bjolang.Runtime.Encodings.Cp20107 #:get)) + (x-iscii-as (: Bjolang.Runtime.Encodings.Cp57006 #:get)) + (x-iscii-be (: Bjolang.Runtime.Encodings.Cp57003 #:get)) + (x-iscii-de (: Bjolang.Runtime.Encodings.Cp57002 #:get)) + (x-iscii-gu (: Bjolang.Runtime.Encodings.Cp57010 #:get)) + (x-iscii-ka (: Bjolang.Runtime.Encodings.Cp57008 #:get)) + (x-iscii-ma (: Bjolang.Runtime.Encodings.Cp57009 #:get)) + (x-iscii-or (: Bjolang.Runtime.Encodings.Cp57007 #:get)) + (x-iscii-pa (: Bjolang.Runtime.Encodings.Cp57011 #:get)) + (x-iscii-ta (: Bjolang.Runtime.Encodings.Cp57004 #:get)) + (x-iscii-te (: Bjolang.Runtime.Encodings.Cp57005 #:get)) + (x-mac-arabic (: Bjolang.Runtime.Encodings.Cp10004 #:get)) + (x-mac-ce (: Bjolang.Runtime.Encodings.Cp10029 #:get)) + (x-mac-chinesesimp (: Bjolang.Runtime.Encodings.Cp10008 #:get)) + (x-mac-chinesetrad (: Bjolang.Runtime.Encodings.Cp10002 #:get)) + (x-mac-croatian (: Bjolang.Runtime.Encodings.Cp10082 #:get)) + (x-mac-cyrillic (: Bjolang.Runtime.Encodings.Cp10007 #:get)) + (x-mac-greek (: Bjolang.Runtime.Encodings.Cp10006 #:get)) + (x-mac-hebrew (: Bjolang.Runtime.Encodings.Cp10005 #:get)) + (x-mac-icelandic (: Bjolang.Runtime.Encodings.Cp10079 #:get)) + (x-mac-japanese (: Bjolang.Runtime.Encodings.Cp10001 #:get)) + (x-mac-korean (: Bjolang.Runtime.Encodings.Cp10003 #:get)) + (x-mac-romanian (: Bjolang.Runtime.Encodings.Cp10010 #:get)) + (x-mac-thai (: Bjolang.Runtime.Encodings.Cp10021 #:get)) + (x-mac-turkish (: Bjolang.Runtime.Encodings.Cp10081 #:get)) + (x-mac-ukrainian (: Bjolang.Runtime.Encodings.Cp10017 #:get))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; text/json-codec.bjo — records to JSON and back. +;; +;; Two traits and a macro that writes the instances for a record. +;; +;; Nullable types are Options, keys that may be absent just specify a default +;; value + +(export + ;; The traits + ->json json-> + ;; What generated field code calls. Exported because a macro template's names + ;; resolve against this module's exports. + json-object json-field json-field/default + ;; The generator + def/json-type) + +(import (std prelude)) +(import (std syntax-match)) +(import (text json)) + +;; Every export is documented there; the module doc is too. +(:doc #:file "json-codec.bjodoc") + +;; --------------------------------------------------------------------------- +;; The traits +;; --------------------------------------------------------------------------- + +(def/trait (->json %a) + (: ->json (-> %a Json))) + + +(def/trait (json-> %a) + (: json-> (-> Json (Result string %a)))) + +;; --------------------------------------------------------------------------- +;; Scalars +;; --------------------------------------------------------------------------- + +(impl (->json Json) (defun (->json j) j)) +(impl (->json long) (defun (->json n) (JsonInt n))) +(impl (->json int) (defun (->json n) (JsonInt (cast System.Int64 n)))) +(impl (->json double) (defun (->json n) (JsonFloat n))) +(impl (->json bool) (defun (->json b) (JsonBool b))) +(impl (->json string) (defun (->json s) (JsonStr s))) + +(impl (json-> Json) + (defun (json-> j) (Ok j))) + +(impl (json-> long) + (defun (json-> j) + (match j + ((JsonInt n) (Ok n)) + (_ (Err "expected an integer"))))) + +(impl (json-> int) + (defun (json-> j) + (match j + ;; A JsonInt is a long; one outside an int's range is refused rather + ;; than wrapped by the cast. + ((JsonInt n) + (if (and (>= n -2147483648L) (<= n 2147483647L)) + (Ok (cast System.Int32 n)) + (Err "expected an integer that fits in an int"))) + (_ (Err "expected an integer"))))) + + +(impl (json-> double) + (defun (json-> j) + (match j + ((JsonFloat d) (Ok d)) + ;; an int is a valid double. + ((JsonInt n) (Ok (cast System.Double n))) + (_ (Err "expected a number"))))) + +(impl (json-> bool) + (defun (json-> j) + (match j + ((JsonBool b) (Ok b)) + (_ (Err "expected a boolean"))))) + +(impl (json-> string) + (defun (json-> j) + (match j + ((JsonStr s) (Ok s)) + (_ (Err "expected a string"))))) + +;; --------------------------------------------------------------------------- +;; Nullability +;; --------------------------------------------------------------------------- + +(impl (->json (Option %a)) + (where (->json %a)) + (defun (->json o) + (match o + ((Some x) (->json x)) + (None JsonNull)))) + +(impl (json-> (Option %a)) + (where (json-> %a)) + (defun (json-> j) + (match j + (JsonNull (Ok None)) + (_ (match (json-> j) + ((Err e) (Err e)) + ((Ok v) (Ok (Some v)))))))) + +(impl (->json (Vec %a)) + (where (->json %a)) + (defun (->json v) + (JsonArr (vec->iarray (vec-map (fun (x) (->json x)) v))))) + +(impl (json-> (Vec %a)) + (where (json-> %a)) + (defun (json-> j) + (match j + ((JsonArr items) + (fold (fun (el acc) + (def (Ok b) acc :propagate) + (def (Ok v) (json-> el) :propagate) + (Ok (vec-add b v))) + (Ok (vec-empty)) + items)) + (_ (Err "expected an array"))))) + +;; A JSON object with values all of one type. +(impl (->json (Map string %a)) + (where (->json %a)) + (defun (->json m) + (JsonObj (map-map-values (fun (v) (->json v)) m)))) + +(impl (json-> (Map string %a)) + (where (json-> %a)) + (defun (json-> j) + (match j + ((JsonObj m) + (map-fold (fun (k v acc) + (match acc + ((Err e) (Err e)) + ((Ok b) + (match (json-> v) + ((Err e) (Err (str "\"" k "\": " e))) + ((Ok d) (Ok (map-set b k d))))))) + (Ok (map-empty)) + m)) + ;; In order, so that a key written twice keeps its last value here too. + ((JsonFields fields) + (fold (fun (p acc) + (def (Tuple k v) p) + (match acc + ((Err e) (Err e)) + ((Ok b) + (match (json-> v) + ((Err e) (Err (str "\"" k "\": " e))) + ((Ok d) (Ok (map-set b k d))))))) + (Ok (map-empty)) + fields)) + (_ (Err "expected an object"))))) + +;; --------------------------------------------------------------------------- +;; What generated field code calls +;; --------------------------------------------------------------------------- + +;; The object itself, a JsonObj or a JsonFields: json-field looks in either. +(: (-> string Json (Result string Json))) +(defun (json-object owner j) + (match j + ((JsonObj _) (Ok j)) + ((JsonFields _) (Ok j)) + (_ (Err (str owner ": expected an object"))))) + +;; The owner and key are prefixed onto whatever the value's own decoder said, +;; so a wrong type reads "Point.x: expected an integer" rather than naming +;; neither. Paths down the tree are not provided +(: (-> string Json string (Result string %a)) (where (json-> %a))) +(defun (json-field owner m key) + (match (json-object-ref m key) + (None (Err (str owner ": the object has no \"" key "\" field"))) + ((Some j) + (result-map-err (fun (e) (str owner "." key ": " e)) (json-> j))))) + +;; Absent and null ar the same in the sense that a nullable has the default value None +(: (-> string Json string %a (Result string %a)) (where (json-> %a))) +(defun (json-field/default owner m key fallback) + (match (json-object-ref m key) + (None (Ok fallback)) + ((Some JsonNull) (Ok fallback)) + ((Some j) + (result-map-err (fun (e) (str owner "." key ": " e)) (json-> j))))) + +;; --------------------------------------------------------------------------- +;; The generator +;; --------------------------------------------------------------------------- +;; This is a runtime macro. Field types are never inspected: what a field encodes as is +;; settled later by instance resolution, and only the markers are read here. + +(: (-> string Syntax)) +(defun (jt-sym name) (SSym (string->symbol name))) + +(: (-> Syntax Syntax)) +(defun (jt-str s) (SStr (syntax->string s))) + +;; `(Box %a)` names the type `Box`; a bare `Point` names itself. +(: (-> Syntax Syntax)) +(defun (jt-head-name head) + (match head + ((SList parts) (list-head parts)) + (_ head))) + +(: (-> Syntax (List Syntax))) +(defun (jt-head-vars head) + (match head + ((SList parts) (list-tail parts)) + (_ Nil))) + + +(: (-> (-> Symbol Syntax) Syntax (List Syntax))) +(defun (jt-where-enc inject head) + (def vars (jt-head-vars head)) + (if (list-empty? vars) + Nil + (list #'(,(inject 'where) + ,@(list-map (fun (v) #'(,(inject (string->symbol "->json")) ,v)) vars))))) + +(: (-> (-> Symbol Syntax) Syntax (List Syntax))) +(defun (jt-where-dec inject head) + (def vars (jt-head-vars head)) + (if (list-empty? vars) + Nil + (list #'(,(inject 'where) + ,@(list-map (fun (v) #'(,(inject (string->symbol "json->")) ,v)) vars))))) + +;; --- fields ---------------------------------------------------------------- + +(: (-> Syntax string bool)) +(defun (jt-marker=? m name) (= (syntax->string m) name)) + +;; The JSON key: `#:json "name"` means it uses the "name" as the key for the field +;; instead of the defined name +(: (-> Syntax (List Syntax) Syntax)) +(defun (jt-key name markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) (jt-str name)) + ((jt-marker=? (list-head ms) "#:json") (list-head (list-tail ms))) + (else (go (list-tail ms)))))) + +;; The fallback expression, or None if the field is required. `#:optional` is +;; `#:default None`. +(: (-> (List Syntax) (Option Syntax))) +(defun (jt-fallback markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) None) + (else + (def m (list-head ms)) + (cond + ((jt-marker=? m "#:optional") (Some (jt-sym "None"))) + ((jt-marker=? m "#:default") (Some (list-head (list-tail ms)))) + (else (go (list-tail ms)))))))) + +;; strip json markers (default, json, optional) +(: (-> (List Syntax) (List Syntax))) +(defun (jt-keep markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) Nil) + (else + (def m (list-head ms)) + (cond + ((jt-marker=? m "#:default") (go (list-tail (list-tail ms)))) + ((jt-marker=? m "#:json") (go (list-tail (list-tail ms)))) + ((jt-marker=? m "#:optional") (go (list-tail ms))) + (else (Cons m (go (list-tail ms))))))))) + +(: string) +(def jt-shape + "a field is written (: name type), then any of #:mutable, #:optional, #:default , #:json \"key\".") + +;; `syntax-error` has the type Syntax, so this is how we work around it. +(: (-> Syntax string (List Syntax))) +(defun (jt-fail-list form message) + (ignore (syntax-error form message)) + Nil) + +(: (-> Syntax Syntax)) +(defun (jt-field-name f) + (syntax-match f + ((_ name _ marker ...) name) + (bad (syntax-error bad jt-shape)))) + +(: (-> Syntax (List Syntax))) +(defun (jt-field-markers f) + (syntax-match f + ((_ _ _ marker ...) marker) + (bad (jt-fail-list bad jt-shape)))) + +(: (-> Syntax Syntax)) +(defun (jt-strip f) + (syntax-match f + ((colon name ty marker ...) #'(,colon ,name ,ty ,@(jt-keep marker))) + (bad (syntax-error bad jt-shape)))) + +;; --- encode ---------------------------------------------------------------- + +;; An optional field is dropped when it is None; everything else is written. +(: (-> Syntax Syntax Syntax)) +(defun (jt-put buf f) + (def name (jt-field-name f)) + (def markers (jt-field-markers f)) + (def write #'(transientmap-set! ,buf ,(jt-key name markers) (->json (record-ref v ,name)))) + (if (jt-optional? markers) + #'(when (some? (record-ref v ,name)) ,write) + write)) + +(: (-> (List Syntax) bool)) +(defun (jt-optional? markers) + (let go ((ms markers)) + (cond + ((list-empty? ms) #f) + ((jt-marker=? (list-head ms) "#:optional") #t) + (else (go (list-tail ms)))))) + +(: (-> (List Syntax) Syntax)) +(defun (jt-encode fields) + (def buf (jt-sym "__jt-buf")) + #'(let ((,buf (transientmap-empty))) + ,@(list-map (fun (f) (jt-put buf f)) fields) + (JsonObj (transientmap->map ,buf)))) + +;; --- decode ---------------------------------------------------------------- + +;; One `match` per field wrapping the next, with the constructor innermost. +;; Verbose. Macrology. +(: (-> Syntax Syntax (List Syntax) int Syntax Syntax)) +(defun (jt-chain owner m fields i ctor) + (cond + ((list-empty? fields) ctor) + (else + (def f (list-head fields)) + (def v (jt-sym (str "__jt-v" (int->string i)))) + (def e (jt-sym (str "__jt-e" (int->string i)))) + (def name (jt-field-name f)) + (def markers (jt-field-markers f)) + (def get (match (jt-fallback markers) + (None #'(json-field ,owner ,m ,(jt-key name markers))) + ((Some d) #'(json-field/default ,owner ,m ,(jt-key name markers) ,d)))) + #'(match ,get + ((Err ,e) (Err ,e)) + ((Ok ,v) ,(jt-chain owner m (list-tail fields) (+ i 1) ctor)))))) + +(: (-> Syntax (List Syntax) Syntax)) +(defun (jt-ctor head fields) + #'(Ok (,(jt-head-name head) + ,@(let go ((fs fields) (i 0)) + (if (list-empty? fs) + Nil + (Cons #'(,(jt-field-name (list-head fs)) ,(jt-sym (str "__jt-v" (int->string i)))) + (go (list-tail fs) (+ i 1)))))))) + +(: (-> Syntax (List Syntax) Syntax)) +(defun (jt-decode head fields) + (def owner (jt-str (jt-head-name head))) + (def m (jt-sym "__jt-m")) + (def e (jt-sym "__jt-e")) + #'(match (json-object ,owner j) + ((Err ,e) (Err ,e)) + ((Ok ,m) ,(jt-chain owner m fields 0 (jt-ctor head fields))))) + +;; --- the form --------------------------------------------------------------- + +(: (-> (-> Symbol Syntax) Syntax Syntax (List Syntax) Syntax)) +(defun (jt-expand inject colon head fields) + (def name (syntax->string (jt-head-name head))) + (def enc (inject (string->symbol (str name "->json")))) + (def dec (inject (string->symbol (str "json->" name)))) + #'(begin + (type (,colon ,head (Record ,@(list-map jt-strip fields)))) + + (impl (->json ,head) + ,@(jt-where-enc inject head) + (defun (->json v) ,(jt-encode fields))) + + (impl (json-> ,head) + ,@(jt-where-dec inject head) + (defun (json-> j) ,(jt-decode head fields))) + + ;; The named pair. `json->` alone would be ambiguous at a call site; + ;; this signature is what fixes the instance, once per type. + (: ,enc (-> ,head Json) ,@(jt-where-enc inject head)) + (defun (,enc v) (->json v)) + (: ,dec (-> Json (Result string ,head)) ,@(jt-where-dec inject head)) + (defun (,dec j) (json-> j)))) + +(def/macro (def/json-type form inject compare) + (syntax-match form + ((_ typedef) + (syntax-match typedef + ((colon head ('Record field ...)) (jt-expand inject colon head field)) + (bad (syntax-error bad "def/json-type takes a record: (def/json-type (: Name (Record (: field type) ...)))")))) + (bad (syntax-error bad "def/json-type takes exactly one record declaration.")))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; text/json.bjo +;; +;; RFC 8259, read off the document's UTF-8 bytes. + +(export + Json JsonObjects + json-max-depth + read-json parse-json + json-object-ref + json->string write-json) + +(import (std prelude)) +(import (only (std ports) read-bytes)) +(import (std maths)) +(import (std simd)) + +;; Every export is documented there; the module doc is too. +(:doc #:file "json.bjodoc") + +;; TODO: use bjolangs builtins: bjolangs builtins uses the invariant culture now. +(import/extern + (clr-to-long (: Bjolang.Runtime.BjoNum.ParseLong)) + (clr-to-double (: Bjolang.Runtime.BjoNum.ParseDouble)) + ;; The same parsers, on the bytes in the reader's buffer: a number that is + ;; whole in the chunk is parsed where it is, without a copy. + (span->long (: BjoString.Utf8Number.ParseLong (-> (RoSpan byte) long))) + (span->double (: BjoString.Utf8Number.ParseDouble (-> (RoSpan byte) double)))) + +(type + (: Json (Union + JsonNull + (: JsonBool bool) + (: JsonInt long) + (: JsonFloat double) + (: JsonStr string) + ;; An `IArray`: a parsed array is read and not changed, and an + ;; `IArray` is its elements' array and no object of its own. + (: JsonArr (IArray Json)) + (: JsonObj (Map string Json)) + ;; An object read with `#:objects 'fields`: its keys and values in + ;; the order the document has them, each key once. + (: JsonFields (IArray (Tuple string Json)))))) + +;; How a reader stores an object: as a Map, or as its fields in an IArray, +;; which is quicker to build and keeps their order. +(def/grammar JsonObjects + (:or (: ObjectsAsMap 'map) (: ObjectsAsFields 'fields))) + +;; The value of `key` in an object of either kind, or None when `j` is not an +;; object or has no such key. In a JsonFields this is a walk over the fields. +(: (-> Json string (Option Json))) +(defun (json-object-ref j key) + (match j + ((JsonObj m) (map-try-ref m key)) + ((JsonFields fields) + (def n (iarray-length fields)) + (let go ((i 0)) + (cond + ((= i n) None) + (:def (Tuple k v) (iarray-ref fields i)) + ((= k key) (Some v)) + (else (go (+ i 1)))))) + (_ None))) + +;; Max stack depth. This should probably be a parameter. +(: int) +(def json-max-depth 1000) + +;; --------------------------------------------------------------------------- +;; The reader +;; --------------------------------------------------------------------------- +;; The reader works on bytes: a chunk of the document in `bytes`, the one it is +;; looking at at `next`, and `limit` where the chunk ends. Looking is an array +;; load and stepping is an increment. A chunk comes from the port with +;; `read-bytes` (see `ensure!`); `parse-json` hands over the string's own bytes as the one +;; chunk there is, and no port. +;; +;; Everything JSON says outside a string is ASCII, so outside a string a byte +;; is a character. Inside one, a character of several bytes is checked to be +;; UTF-8 and copied as it is; one that is not UTF-8 is refused. A column +;; counts characters: only a byte that starts one moves it. +;; +;; The byte is handled as a number, so that the end of the document is a +;; value, `spent`, rather than a second case. + +(: int) +(def spent -1) + +;; How much a read from the port asks for: the first one what `read-some` +;; gives a file, each one after it twice the one before, up to `read-most`. +;; Each read costs a sync on the port, a new array and a pass of the reader's +;; slow path at the chunk's end, so a big document is read in big chunks. +;; `read-bytes` makes the port's buffer as big as what it asks for, so a small +;; one is still read in a small one. +(: int) +(def read-first 16384) + +(: int) +(def read-most 262144) + +(type + (: Reader (Record (: port (Option InputPort) #:mutable) + ;; How many bytes the next read from the port asks for. + (: ask int #:mutable) + (: bytes (Array byte) #:mutable) + (: next int #:mutable) + (: limit int #:mutable) + (: row int #:mutable) + (: col int #:mutable) + ;; One buffer for every string and every number in the + ;; document. + (: scratch StringBuilder) + ;; One array for every array in the document, and how much + ;; of it is in use. + (: stack (Array Json) #:mutable) + (: top int #:mutable) + ;; The keys last read, by slot; empty until the first + ;; object. See `cached-key`. + (: keys (Array string) #:mutable) + ;; Whether objects are read as JsonFields. + (: as-fields bool) + ;; One array for every JsonFields' pairs, as `stack` is + ;; for arrays. + (: pairs (Array (Tuple string Json)) #:mutable) + (: ptop int #:mutable) + ;; A hash table of pair indices, for the check for a + ;; repeated key; see repeated-key/hashed?. A slot is in + ;; use when its stamp is the check's, so the table is never + ;; cleared. + (: slot-stamps (Array int) #:mutable) + (: slot-pairs (Array int) #:mutable) + (: stamp int #:mutable)))) + +;; What a reader's key cache is until the first key. +(: (Array string)) +(def no-keys (make-array 0)) + +;; What a reader's pair stack and hash table are until the first JsonFields +;; that needs them. +(: (Array (Tuple string Json))) +(def no-pairs (make-array 0)) + +(: (Array int)) +(def no-slots (make-array 0)) + +;; A reader over `bytes`, and then whatever `port` has after them. +(: (-> (Option InputPort) (Array byte) JsonObjects Reader)) +(defun (reader port bytes objects) + (Reader (port port) + (as-fields (match objects (ObjectsAsMap #f) (ObjectsAsFields #t))) + (pairs no-pairs) + (ptop 0) + (slot-stamps no-slots) + (slot-pairs no-slots) + (stamp 0) + (ask read-first) + (bytes bytes) + (next 0) + (limit (array-length bytes)) + (row 1) + (col 1) + (scratch (stringbuilder-empty)) + (stack (make-array 16)) + (top 0) + (keys no-keys))) + +;; --------------------------------------------------------------------------- +;; The key cache +;; --------------------------------------------------------------------------- +;; An array of objects repeats its keys: twenty thousand records write the same +;; five keys twenty thousand times. This holds the last key read in each slot +;; and compares the buffer with it, without making a string, which is what +;; `stringbuilder=?` is for. A key read before costs that comparison and no +;; allocation, and the objects share one string for it. `(text bjodat-core)` +;; caches its names the same way, with the same slots. +;; +;; 64 slots, because an object may have many keys: a tweet has forty. Keys that +;; share a slot take turns in it and are made each time, as without a cache. + +(: int) +(def key-slots 64) + +;; Empty slots hold `""`, which no key but the empty one equals, and that one +;; is `""` anyway. +(: (-> (Array string))) +(defun (new-key-cache) + (def a (make-array key-slots)) + (loop (:for i (range 0 key-slots)) + (:do (array-set! a i ""))) + a) + +;; The first, middle and last byte and the length: three reads whatever the +;; length, and enough to part the keys of one object. With the first byte and +;; the length alone, `name` and `note` shared a slot, and so did `name` and +;; `pinned`, and each made the other a string every time. +(: (-> StringBuilder int int)) +(defun (key-slot sb n) + (if (= n 0) + 0 + (bitwise-and (+ (* (+ (* (+ (* (stringbuilder-byte-ref sb 0) 31) + (stringbuilder-byte-ref sb (shift-right n 1))) + 31) + (stringbuilder-byte-ref sb (- n 1))) + 31) + n) + (- key-slots 1)))) + +;; The buffer as a string, the one already made for the same key if there is +;; one. +(: (-> Reader StringBuilder string)) +(defun (cached-key rd sb) + (when (= (array-length (record-ref rd keys)) 0) + (record-set! rd (keys (new-key-cache)))) + (def cache (record-ref rd keys)) + (def i (key-slot sb (stringbuilder-length sb))) + (def known (array-ref cache i)) + (cond + ((stringbuilder=? sb known) known) + (else + (def s (stringbuilder->string sb)) + (array-set! cache i s) + s))) + +;; Pushes a new element into the array or grows it +(: (-> Reader Json void)) +(defun (push! rd v) + (def a (record-ref rd stack)) + (def n (record-ref rd top)) + (cond + ((< n (array-length a)) (array-set! a n v)) + (else + (def bigger (array-resize a (* n 2))) + (array-set! bigger n v) + (record-set! rd (stack bigger)))) + (record-set! rd (top (+ n 1)))) + +;; Harvests the array, by copying it out of the stack +(: (-> Reader int (IArray Json))) +(defun (harvest! rd mark) + (def v (subarray->iarray (record-ref rd stack) mark (- (record-ref rd top) mark))) + (record-set! rd (top mark)) + v) + +;; The buffer, emptied and ready. +(: (-> Reader StringBuilder)) +(defun (buffer rd) + (def sb (record-ref rd scratch)) + (stringbuilder-clear! sb) + sb) + +;; `count` bytes of `old` from `from`, then `more`: what is left of one chunk +;; and the next. Almost always nothing is left, and `more` is the answer. +(: (-> (Array byte) int int (Array byte) (Array byte))) +(defun (join-bytes old from count more) + (if (= count 0) + more + (let ((joined (make-array (+ count (array-length more))))) + (array-copy! old from joined 0 count) + (array-copy! more 0 joined count (array-length more)) + joined))) + +;; Whether there are at least `n` bytes from `next` on, reading from the port +;; until there are or it ends. Bytes not yet looked at are kept, so a +;; character cut in two by the end of a chunk is whole afterwards. +(: (-> Reader int bool)) +(defun (ensure! rd n) + (let go () + (def have (- (record-ref rd limit) (record-ref rd next))) + (cond + ((>= have n) #t) + (else + (def (Some p) (get rd .port) :leave-with #f) + (def ask (record-ref rd ask)) + (match (read-bytes p (if (> n ask) n ask)) + (None + (record-set! rd (port None)) + #f) + ((Some more) + (record-set! rd + (bytes (join-bytes (record-ref rd bytes) (record-ref rd next) have more)) + (next 0) + (limit (+ have (array-length more))) + (ask (if (< ask read-most) (* ask 2) read-most))) + (go))))))) + +;; The byte the reader is looking at, or `spent` at the end of the document. +(: (-> Reader int)) +(defun (held-code rd) + (def i (record-ref rd next)) + (cond + ((< i (record-ref rd limit)) (cast int (array-ref (record-ref rd bytes) i))) + ((ensure! rd 1) (cast int (array-ref (record-ref rd bytes) (record-ref rd next)))) + (else spent))) + +(: (-> Reader bool)) +(defun (at-end? rd) + (= (held-code rd) spent)) + +;; Every character this is asked about is a json char meaning it is ascii. +;; Anything else — the end of the document, the first byte of a letter from a +;; language — answers #\nul. +(: (-> Reader char)) +(defun (peek rd) + (def c (held-code rd)) + (if (and (>= c 0) (< c 128)) (int->char c) #\nul)) + +;; Moves past the byte being looked at, which is a character of its own. +;; +;; It does not look at what it is stepping over, because every caller just has: +;; a digit, an ASCII character inside a string, a bracket. A character of +;; several bytes is taken by `take-character!`. +(: (-> Reader void)) +(defun (step! rd) + (record-set! rd + (next (+ (record-ref rd next) 1)) + (col (+ (record-ref rd col) 1)))) + + +(: (-> int bool)) +(defun (space? c) (or (= c 32) (= c 10) (= c 9) (= c 13))) + +;; Moves past the white space. Usually there is none, and that costs one look +;; at the next byte. +(: (-> Reader void)) +(defun (skip-space! rd) + (when (space? (held-code rd)) + (skip-space-run! rd))) + +;; White space from `start`, 32 or 16 bytes at a time, to the first byte that +;; is not white space or the last whole block before `limit`: where it ends, +;; the line and column there, and whether a byte that is not white space was +;; found. The line moves by the newlines; after one, the column is counted +;; from it. 256 bits where the CPU has them, as `scan-run` has. +(: (-> (Array byte) int int int int (Tuple int int int bool))) +(defun (scan-space b start limit row col) + (if simd256-accelerated? + (scan-space/256 b start limit row col) + (scan-space/128 b start limit row col))) + +;; The line and column after `n` bytes of white space whose newlines are the +;; set bits of `nls`. +(: (-> int int int int (Tuple int int))) +(defun (after-space nls n row col) + (if (= nls 0) + (Tuple row (+ col n)) + ;; The bytes after the last newline are the column, less one. + (Tuple (+ row (popcount nls)) + (- n (- 31 (leading-zeros nls)))))) + +(: (-> (Array byte) int int int int (Tuple int int int bool))) +(defun (scan-space/256 b start limit row col) + (let go ((i start) (row row) (col col)) + (cond + ((> (+ i 32) limit) + (Tuple i row col #f)) + (:def v (simd256-load b i)) + (:def nl (simd-eq v (simd256-splat 10uy))) + (:def spaces (simd-mask-bits (simd-or (simd-or nl (simd-eq v (simd256-splat 32uy))) + (simd-or (simd-eq v (simd256-splat 9uy)) + (simd-eq v (simd256-splat 13uy)))))) + (:def nls (simd-mask-bits nl)) + ((= spaces -1) + (def (Tuple r c) (after-space nls 32 row col)) + (go (+ i 32) r c)) + (else + (def t (trailing-zeros (bitwise-not spaces))) + (def (Tuple r c) (after-space (bitwise-and nls (- (shift-left 1 t) 1)) t row col)) + (Tuple (+ i t) r c #t))))) + +(: (-> (Array byte) int int int int (Tuple int int int bool))) +(defun (scan-space/128 b start limit row col) + (let go ((i start) (row row) (col col)) + (cond + ((> (+ i 16) limit) (Tuple i row col #f)) + (:def v (simd128-load b i)) + (:def nl (simd-eq v (simd128-splat 10uy))) + (:def spaces (simd-mask-bits (simd-or (simd-or nl (simd-eq v (simd128-splat 32uy))) + (simd-or (simd-eq v (simd128-splat 9uy)) + (simd-eq v (simd128-splat 13uy)))))) + (:def nls (simd-mask-bits nl)) + ((= spaces 65535) + (def (Tuple r c) (after-space nls 16 row col)) + (go (+ i 16) r c)) + (else + (def t (trailing-zeros (bitwise-xor spaces 65535))) + (def (Tuple r c) (after-space (bitwise-and nls (- (shift-left 1 t) 1)) t row col)) + (Tuple (+ i t) r c #t))))) + +;; White space a chunk at a time: the chunk is scanned with a local index, and +;; the reader is moved once for it. A chunk that ends in white space is +;; followed by the next. +(: (-> Reader void)) +(defun (skip-space-run! rd) + (def b (record-ref rd bytes)) + (def limit (record-ref rd limit)) + (def (Tuple fast-end fast-row fast-col stopped) + (scan-space b (record-ref rd next) limit (record-ref rd row) (record-ref rd col))) + (def (Tuple end row col) + (let go ((i fast-end) (row fast-row) (col fast-col)) + (cond + ((or stopped (>= i limit)) + (Tuple i row col)) + (:def c (cast int (array-ref b i))) + ((= c 10) (go (+ i 1) (+ row 1) 1)) + ((or (= c 32) (= c 9) (= c 13)) (go (+ i 1) row (+ col 1))) + (else (Tuple i row col))))) + (record-set! rd (next end) (row row) (col col)) + (when (and (>= end limit) (ensure! rd 1)) + (skip-space-run! rd))) + + + +;; --------------------------------------------------------------------------- +;; Characters of several bytes +;; --------------------------------------------------------------------------- + +(: string) +(def not-utf-8 "a byte that is not UTF-8") + +;; How many bytes a character starting with `lead` has, by RFC 3629; 0 for a +;; byte no character starts with: a continuation byte, C0 and C1 (which could +;; only start an overlong form), and F5 and up. +(: (-> int int)) +(defun (utf8-length lead) + (cond + ((< lead 128) 1) + ((< lead 194) 0) + ((< lead 224) 2) + ((< lead 240) 3) + ((< lead 245) 4) + (else 0))) + +(: (-> int bool)) +(defun (continuation? b) (= (bitwise-and b 192) 128)) + +;; How many bytes the character the reader is looking at has, when they are +;; UTF-8; 0 when they are not, or the document ends inside them. The second +;; byte is narrower after E0 and F0, which would otherwise start an overlong +;; form, after ED, which would start a surrogate, and after F4, which would +;; go past U+10FFFF. +(: (-> Reader int)) +(defun (sequence-length rd) + (def lead (held-code rd)) + (def n (utf8-length lead)) + (cond + ((<= n 1) n) + ((not (ensure! rd n)) 0) + (else (sequence-length-at (record-ref rd bytes) (record-ref rd next) (record-ref rd limit))))) + +;; The same for the character at `i` of `b`, whose bytes end at `limit`: 0 +;; also when the character runs past `limit`, where the reader may have more +;; of it to fetch. +(: (-> (Array byte) int int int)) +(defun (sequence-length-at b i limit) + (def lead (cast int (array-ref b i))) + (def n (utf8-length lead)) + (cond + ((<= n 1) n) + ((> (+ i n) limit) 0) + (:def second (cast int (array-ref b (+ i 1)))) + (:def low (cond ((= lead 224) 160) ((= lead 240) 144) (else 128))) + (:def high (cond ((= lead 237) 159) ((= lead 244) 143) (else 191))) + ((and (>= second low) + (<= second high) + (let check ((k 2)) + (cond + ((>= k n) #t) + ((continuation? (cast int (array-ref b (+ i k)))) (check (+ k 1))) + (else #f)))) + n) + (else 0))) + +;; Copies the `n` bytes of the character being looked at into `sb`, and moves +;; past them: one column. +(: (-> Reader StringBuilder int void)) +(defun (take-character! rd sb n) + (def b (record-ref rd bytes)) + (def i (record-ref rd next)) + (loop (:for k (range 0 n)) + (:do (stringbuilder-add-byte! sb (cast int (array-ref b (+ i k)))))) + (record-set! rd + (next (+ i n)) + (col (+ (record-ref rd col) 1)))) + +;; --------------------------------------------------------------------------- +;; Refusals +;; --------------------------------------------------------------------------- +;; Functions to raise errors + +(: (-> Reader string string)) +(defun (at rd what) + (str "line " (int->string (record-ref rd row)) + ", column " (int->string (record-ref rd col)) + ": " what)) + +(: (-> Reader string)) +(defun (found rd) + (def c (held-code rd)) + (cond + ((= c spent) "the document ended") + ((< c 128) (str "'" (list->string (list (int->char c))) "'")) + (else + (def n (sequence-length rd)) + (def i (record-ref rd next)) + (if (= n 0) + not-utf-8 + (str "'" (bytes->string (record-ref rd bytes) #:start i #:end (+ i n)) "'"))))) + +(: (-> Reader string string)) +(defun (too-deep rd what) + (at rd (str "this " what " nests deeper than " (int->string json-max-depth) + ", which is as far as reading goes"))) + +;; --------------------------------------------------------------------------- +;; Strings +;; --------------------------------------------------------------------------- + +;; The Bjolang lexer has no \b or \f of its own. +(: string) +(def escaped-backspace (list->string (list (int->char 8)))) + +(: string) +(def escaped-formfeed (list->string (list (int->char 12)))) + +(: (-> char int)) +(defun (hex-digit c) + (cond + ((and (char>=? c #\0) (char<=? c #\9)) (- (char->int c) 48)) + ((and (char>=? c #\a) (char<=? c #\f)) (- (char->int c) 87)) + ((and (char>=? c #\A) (char<=? c #\F)) (- (char->int c) 55)) + (else -1))) + +(: (-> int bool)) +(defun (high-surrogate? n) (and (>= n 55296) (<= n 56319))) + +(: (-> int bool)) +(defun (low-surrogate? n) (and (>= n 56320) (<= n 57343))) + +;; The scalar a surrogate pair stands for. +(: (-> int int int)) +(defun (scalar-of hi lo) + (+ 65536 (* (- hi 55296) 1024) (- lo 56320))) + +(: (-> Reader (Result string int))) +(defun (read-hex4 rd) + (let go ((n 0) (left 4)) + (cond + ((= left 0) (Ok n)) + ((at-end? rd) (Err (at rd "the document ended inside a \\u escape"))) + (:def d (hex-digit (peek rd))) + ((< d 0) (Err (at rd (str "a \\u escape takes four hex digits, and this is " (found rd))))) + (else + (step! rd) + (go (+ (* n 16) d) (- left 1)))))) + +;; The pair is one character. A Bjolang char is a Unicode scalar... Not a match made in heaven +(: (-> Reader int (Result string string))) +(defun (read-low-surrogate rd hi) + (def incomplete "a high surrogate has to be followed by \\u and its low surrogate") + (cond + ((or (at-end? rd) (not (char=? (peek rd) #\\))) (Err (at rd incomplete))) + (:do (step! rd)) + ((or (at-end? rd) (not (char=? (peek rd) #\u))) (Err (at rd incomplete))) + (:do (step! rd)) + (:def (Ok lo) (read-hex4 rd) + :propagate) + ((low-surrogate? lo) (Ok (list->string (list (int->char (scalar-of hi lo)))))) + (else (Err (at rd incomplete))))) + +(: (-> Reader (Result string string))) +(defun (read-unicode rd) + (def (Ok n) (read-hex4 rd) + :propagate) + (cond + ((low-surrogate? n) (Err (at rd "a low surrogate with no high surrogate before it"))) + ((high-surrogate? n) (read-low-surrogate rd n)) + (else (Ok (list->string (list (int->char n))))))) + +(: (-> Reader (Result string string))) +(defun (read-escape rd) + (if (at-end? rd) + (Err (at rd "the document ended after a backslash")) + (case (peek rd) + ((#\") (step! rd) (Ok "\"")) + ((#\\) (step! rd) (Ok "\\")) + ((#\/) (step! rd) (Ok "/")) + ((#\b) (step! rd) (Ok escaped-backspace)) + ((#\f) (step! rd) (Ok escaped-formfeed)) + ((#\n) (step! rd) (Ok "\n")) + ((#\r) (step! rd) (Ok "\r")) + ((#\t) (step! rd) (Ok "\t")) + ((#\u) (step! rd) (read-unicode rd)) + (else + (Err (at rd (str "\\" (list->string (list (peek rd))) " is not an escape"))))))) + +;; Where a run from `start` stops, 32 or 16 bytes at a time: at the first `"`, `\` +;; or control character, or at the last whole block before `limit`. +;; Also how many characters are before it, which is the bytes that are not a +;; continuation byte (10xxxxxx), whether one of them is 128 or more, and +;; whether a stop was found. The count is the characters' count only when the +;; bytes are UTF-8, which `take-run!` checks when one is 128 or more. +;; +;; 256 bits where the CPU has them, and 128 where it has not, which is every +;; ARM64 CPU: there a 256-bit vector is two 128-bit ones done in software. +(: (-> (Array byte) int int (Tuple int int bool bool))) +(defun (scan-run b start limit) + (if simd256-accelerated? + (scan-run/256 b start limit) + (scan-run/128 b start limit))) + +(: (-> (Array byte) int int (Tuple int int bool bool))) +(defun (scan-run/128 b start limit) + (let go ((i start) (chars 0) (high #f)) + (if (> (+ i 16) limit) + (Tuple i chars high #f) + (let () + (def v (simd128-load b i)) + (def stops (simd-mask-bits (simd-or (simd-or (simd-eq v (simd128-splat 34uy)) + (simd-eq v (simd128-splat 92uy))) + (simd-lt v (simd128-splat 32uy))))) + (def conts (simd-mask-bits (simd-eq (simd-and v (simd128-splat 192uy)) (simd128-splat 128uy)))) + (def highs (simd-mask-bits (simd-ge v (simd128-splat 128uy)))) + (if (= stops 0) + (go (+ i 16) (+ chars (- 16 (popcount conts))) (or high (not (= highs 0)))) + (let () + (def t (trailing-zeros stops)) + (def before (- (shift-left 1 t) 1)) + (Tuple (+ i t) + (+ chars (- t (popcount (bitwise-and conts before)))) + (or high (not (= (bitwise-and highs before) 0))) + #t))))))) + +(: (-> (Array byte) int int (Tuple int int bool bool))) +(defun (scan-run/256 b start limit) + (let go ((i start) (chars 0) (high #f)) + (if (> (+ i 32) limit) + (Tuple i chars high #f) + (let () + (def v (simd256-load b i)) + (def stops (simd-mask-bits (simd-or (simd-or (simd-eq v (simd256-splat 34uy)) + (simd-eq v (simd256-splat 92uy))) + (simd-lt v (simd256-splat 32uy))))) + (def conts (simd-mask-bits (simd-eq (simd-and v (simd256-splat 192uy)) (simd256-splat 128uy)))) + (def highs (simd-mask-bits (simd-ge v (simd256-splat 128uy)))) + (if (= stops 0) + (go (+ i 32) (+ chars (- 32 (popcount conts))) (or high (not (= highs 0)))) + (let () + ;; The lanes before the first stop. + (def t (trailing-zeros stops)) + (def before (- (shift-left 1 t) 1)) + (Tuple (+ i t) + (+ chars (- t (popcount (bitwise-and conts before)))) + (or high (not (= (bitwise-and highs before) 0))) + #t))))))) + +;; Copies the run of characters that need no attention, from where the reader +;; is to the first `"`, `\`, control character, byte that is not UTF-8 or end +;; of the chunk, into `sb` in one piece, and moves past it. Whatever stopped +;; the run is left for `read-text` to look at. +;; +;; The run is found a block at a time by `scan-run`, and the bytes left +;; before `limit` one at a time. A run with a byte of 128 or more is checked +;; as UTF-8 in one call. A run that is not UTF-8, or that the end of the chunk +;; cuts inside a character, is read again by `take-run/checked!`, which stops +;; before the byte at fault, so that the error and its column are where they +;; were. +(: (-> Reader StringBuilder void)) +(defun (take-run! rd sb) + (def b (record-ref rd bytes)) + (def start (record-ref rd next)) + (def limit (record-ref rd limit)) + (def (Tuple fast-end fast-chars fast-high stopped) (scan-run b start limit)) + (def (Tuple end chars high) + (if stopped + (Tuple fast-end fast-chars fast-high) + (let go ((i fast-end) (chars fast-chars) (high fast-high)) + (if (>= i limit) + (Tuple i chars high) + (let ((c (cast int (array-ref b i)))) + (cond + ((or (= c 34) (= c 92) (< c 32)) (Tuple i chars high)) + ((< c 128) (go (+ i 1) (+ chars 1) high)) + ((continuation? c) (go (+ i 1) chars #t)) + (else (go (+ i 1) (+ chars 1) #t)))))))) + (cond + ((and high (not (bytes-utf-8? b start end))) (take-run/checked! rd sb)) + ((> end start) + (stringbuilder-add-bytes! sb b start end) + (record-set! rd + (next end) + (col (+ (record-ref rd col) chars)))) + (else unit))) + +;; `take-run!` a byte at a time, each character checked as UTF-8: the run +;; stops before a byte that is not, and before a character the end of the +;; chunk cuts. +(: (-> Reader StringBuilder void)) +(defun (take-run/checked! rd sb) + (def b (record-ref rd bytes)) + (def start (record-ref rd next)) + (def limit (record-ref rd limit)) + (def (Tuple end chars) + (let go ((i start) (chars 0)) + (if (>= i limit) + (Tuple i chars) + (let ((c (cast int (array-ref b i)))) + (cond + ((or (= c 34) (= c 92) (< c 32)) (Tuple i chars)) + ((< c 128) (go (+ i 1) (+ chars 1))) + (else + (let ((n (sequence-length-at b i limit))) + (if (= n 0) (Tuple i chars) (go (+ i n) (+ chars 1)))))))))) + (when (> end start) + (stringbuilder-add-bytes! sb b start end) + (record-set! rd + (next end) + (col (+ (record-ref rd col) chars))))) + +;; Reads a json string. An object's key goes through the key cache. +(: (-> Reader bool (Result string string))) +(defun (read-text rd key?) + (step! rd) + (def sb (buffer rd)) + (let go () + (take-run! rd sb) + (def c (held-code rd)) + (cond + ((= c spent) (Err (at rd "the document ended inside a string"))) + ((= c 34) (step! rd) (Ok (if key? (cached-key rd sb) (stringbuilder->string sb)))) + ((= c 92) + (step! rd) + (def (Ok piece) (read-escape rd) + :propagate) + (stringbuilder-add-string! sb piece) + (go)) + ((< c 32) + (Err (at rd "a control character in a string has to be written as an escape"))) + ((< c 128) + (stringbuilder-add-unit! sb c) + (step! rd) + (go)) + ;; The first byte of a character of several, all copied once they are + ;; known to be UTF-8. + (:def n (sequence-length rd)) + ((= n 0) (Err (at rd "a string has to be UTF-8, and this byte is not"))) + (else + (take-character! rd sb n) + (go))))) + +;; --------------------------------------------------------------------------- +;; Numbers +;; --------------------------------------------------------------------------- + +(: (-> char bool)) +(defun (digit? c) (and (char>=? c #\0) (char<=? c #\9))) + + +(: (-> Reader StringBuilder int)) +;; Reads a digit into the string builder and advances until +;; whatever is in the reader is no longer a digit. Returns the number of digits taken +(defun (take-digits! rd sb) + (let go ((n 0)) + (def c (held-code rd)) + ;; ascii codes for 0-9 + (cond + ((and (>= c 48) (<= c 57)) + (stringbuilder-add-unit! sb c) + (step! rd) + (go (+ n 1))) + (else n)))) + +;; Answers whether what was scanned has to be read as a double. +(: (-> Reader StringBuilder (Result string bool))) +(defun (scan-exponent rd sb) + (cond + ((and (not (at-end? rd)) (or (char=? (peek rd) #\e) (char=? (peek rd) #\E))) + (stringbuilder-add! sb #\e) + (step! rd) + (when (and (not (at-end? rd)) (or (char=? (peek rd) #\+) (char=? (peek rd) #\-))) + (stringbuilder-add! sb (peek rd)) + (step! rd)) + (if (= (take-digits! rd sb) 0) + (Err (at rd (str "a number needs a digit after its exponent, and this is " (found rd)))) + (Ok #t))) + (else (Ok #f)))) + +(: (-> Reader StringBuilder (Result string bool))) +(defun (scan-fraction rd sb) + (cond + ((and (not (at-end? rd)) (char=? (peek rd) #\.)) + (stringbuilder-add! sb #\.) + (step! rd) + (if (= (take-digits! rd sb) 0) + (Err (at rd (str "a number needs a digit after its decimal point, and this is " + (found rd)))) + (match (scan-exponent rd sb) + ((Err e) (Err e)) + ((Ok _) (Ok #t))))) + (else (scan-exponent rd sb)))) + +(: (-> Reader StringBuilder (Result string bool))) +(defun (scan-number rd sb) + (when (and (not (at-end? rd)) (char=? (peek rd) #\-)) + (stringbuilder-add! sb #\-) + (step! rd)) + (cond + ((or (at-end? rd) (not (digit? (peek rd)))) + (Err (at rd (str "a number needs a digit here, and this is " (found rd))))) + + ;; Check whether a zero is a lone zero or if it is a number with a leading zero + ;; which is not allowed. + ((char=? (peek rd) #\0) + (stringbuilder-add! sb #\0) + (step! rd) + (if (and (not (at-end? rd)) (digit? (peek rd))) + (Err (at rd "a number may not have a leading zero")) + (scan-fraction rd sb))) + (else + (ignore (take-digits! rd sb)) + (scan-fraction rd sb)))) + +;; A number byte by byte, fetching more of the document as it goes: for a +;; number that the end of a chunk cuts. +(: (-> Reader (Result string Json))) +(defun (read-number/slow rd) + (def sb (buffer rd)) + (def (Ok fractional?) (scan-number rd sb) + :propagate) + (if fractional? + (Ok (JsonFloat (clr-to-double sb))) + ;; Wider than a long is still a number, and a double is what every + ;; other reader answers with. + (match (try (clr-to-long sb) #:catch (System.OverflowException)) + ((Ok n) (Ok (JsonInt n))) + ((Err _) (Ok (JsonFloat (clr-to-double sb))))))) + +;; --------------------------------------------------------------------------- +;; Numbers off the chunk +;; --------------------------------------------------------------------------- +;; A number is read off the chunk's bytes with a local index, and the reader +;; moves past it once at the end, rather than once a byte. A number is ASCII, +;; so the column moves by as many bytes as the number has. +;; +;; A number that reaches the end of the chunk, when the port may have more, +;; could be longer than the chunk shows. That one is read by +;; `read-number/slow`, which fetches the rest. + +;; What a scan of a number found, with a position: the end of the number, or +;; for a refusal the byte that is wrong. A tuple of two ints rather than a +;; union, because a tuple is a value and a union case with a field is an +;; allocation for every number in the document. +(type (: NumberScan (Tuple int int))) + +(: int) +(def num-int 0) +(: int) +(def num-frac 1) +(: int) +(def num-no-digit 2) +(: int) +(def num-leading-zero 3) +(: int) +(def num-no-fraction-digit 4) +(: int) +(def num-no-exponent-digit 5) + +(: (-> (Array byte) int int bool)) +(defun (digit-at? b i limit) + (and (< i limit) + (let ((c (cast int (array-ref b i)))) + (and (>= c 48) (<= c 57))))) + +(: (-> (Array byte) int int int bool)) +(defun (byte-at? b i limit code) + (and (< i limit) (= (cast int (array-ref b i)) code))) + +;; The position after the digits from `i`. +(: (-> (Array byte) int int int)) +(defun (digits-end b i limit) + (let go ((i i)) + (if (digit-at? b i limit) (go (+ i 1)) i))) + +;; The exponent, if there is one, from `i`, after a number that is fractional +;; when `frac?` is. +(: (-> (Array byte) int int bool NumberScan)) +(defun (scan-exponent-at b i limit frac?) + (cond + ((or (byte-at? b i limit 101) (byte-at? b i limit 69)) + (def signed (if (or (byte-at? b (+ i 1) limit 43) (byte-at? b (+ i 1) limit 45)) (+ i 2) (+ i 1))) + (def end (digits-end b signed limit)) + (if (= end signed) (Tuple num-no-exponent-digit end) (Tuple num-frac end))) + (frac? (Tuple num-frac i)) + (else (Tuple num-int i)))) + +;; The fraction, if there is one, from `i`. +(: (-> (Array byte) int int NumberScan)) +(defun (scan-fraction-at b i limit) + (cond + ((byte-at? b i limit 46) + (def end (digits-end b (+ i 1) limit)) + (if (= end (+ i 1)) + (Tuple num-no-fraction-digit end) + (scan-exponent-at b end limit #t))) + (else (scan-exponent-at b i limit #f)))) + +;; The number from `pos`, whose digits start at `first`, by RFC 8259. +(: (-> (Array byte) int int int NumberScan)) +(defun (scan-number-at b pos first limit) + (cond + ((not (digit-at? b first limit)) (Tuple num-no-digit first)) + ((byte-at? b first limit 48) + (if (digit-at? b (+ first 1) limit) + (Tuple num-leading-zero (+ first 1)) + (scan-fraction-at b (+ first 1) limit))) + (else (scan-fraction-at b (digits-end b first limit) limit)))) + +;; Up to 18 digits cannot overflow a long, so they are added up as they are. +;; Longer ones go through the parser, which answers a double when the number +;; is too wide for a long. +(: (-> Reader (Array byte) int int int Json)) +(defun (integer-of rd b start first end) + (cond + ((<= (- end first) 18) + (def n (let go ((i first) (acc 0L)) + (if (< i end) + (go (+ i 1) (+ (* acc 10L) (cast System.Int64 (- (cast int (array-ref b i)) 48)))) + acc))) + (JsonInt (if (= first start) n (- 0L n)))) + (else + (match (try (long-at b start end) #:catch (System.OverflowException)) + ((Ok n) (JsonInt n)) + ((Err _) (JsonFloat (double-at b start end))))))) + +;; The number from `start` to `end` of the reader's bytes, parsed where it is. +;; In functions of their own, so that the span is not a local of the readers +;; that call them. +(: (-> (Array byte) int int long)) +(defun (long-at b start end) (span->long (array->ro-span b #:from start #:to end))) + +(: (-> (Array byte) int int double)) +(defun (double-at b start end) (span->double (array->ro-span b #:from start #:to end))) + +(: (-> Reader (Result string Json))) +(defun (read-number rd) + (def* (b (record-ref rd bytes)) + (start (record-ref rd next)) + (limit (record-ref rd limit)) + (first (if (byte-at? b start limit 45) (+ start 1) start)) + ((status . end) (scan-number-at b start first limit)) + (more? (match (record-ref rd port) (None #f) (_ #t)))) + (cond + ((and more? (>= end limit)) (read-number/slow rd)) + (else + (record-set! rd + (next end) + (col (+ (record-ref rd col) (- end start)))) + (cond + ((= status num-int) (Ok (integer-of rd b start first end))) + ((= status num-frac) (Ok (JsonFloat (double-at b start end)))) + ((= status num-no-digit) + (Err (at rd (str "a number needs a digit here, and this is " (found rd))))) + ((= status num-leading-zero) + (Err (at rd "a number may not have a leading zero"))) + ((= status num-no-fraction-digit) + (Err (at rd (str "a number needs a digit after its decimal point, and this is " + (found rd))))) + (else + (Err (at rd (str "a number needs a digit after its exponent, and this is " + (found rd))))))))) + +;; --------------------------------------------------------------------------- +;; Values +;; --------------------------------------------------------------------------- + +;; only one json null, true or false value. +(: Json) +(def json-true (JsonBool #t)) + +(: Json) +(def json-false (JsonBool #f)) + +(: Json) +(def json-null JsonNull) + +(: (-> Reader string Json (Result string Json))) +(defun (read-word rd word value) + (let go ((k (string-cursor-start word))) + (cond + ((string-cursor-end? word k) + (Ok value)) + ((or (at-end? rd) (not (char=? (peek rd) (string-cursor-ref word k)))) + (Err (at rd (str "'" word "' is spelled out to here, and then there is " (found rd))))) + (else + (step! rd) + (go (string-cursor-next word k)))))) + +(: (-> Reader int (Result string Json))) +(defun (read-array rd depth) + (cond + ((> depth json-max-depth) (Err (too-deep rd "array"))) + (:do + (step! rd) + (skip-space! rd)) + (:def mark (record-ref rd top)) + ((at-end? rd) (Err (at rd "the document ended inside an array"))) + ((char=? (peek rd) #\]) (step! rd) (Ok (JsonArr (harvest! rd mark)))) + (else + (let go () + (def (Ok v) (read-value rd (+ depth 1)) + :propagate) + (push! rd v) + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended inside an array"))) + ((char=? (peek rd) #\,) (step! rd) (go)) + ((char=? (peek rd) #\]) (step! rd) (Ok (JsonArr (harvest! rd mark)))) + (else + (Err (at rd (str "an array wants ',' or ']' here, and this is " + (found rd)))))))))) + +;; --------------------------------------------------------------------------- +;; Objects as fields +;; --------------------------------------------------------------------------- +;; An object's pairs go on the reader's pair stack, as an array's elements go +;; on `stack`, and are copied out once into the object's own IArray. A key +;; written twice is found when the object closes: by comparing every two +;; keys of a small object, and through a hash table of a bigger one, which +;; the reader keeps, so that the check allocates nothing. Only an object that +;; does repeat a key is built again. + +;; Pushes a pair onto the pair stack, growing it. +(: (-> Reader (Tuple string Json) void)) +(defun (push-pair! rd p) + (def a (record-ref rd pairs)) + (def n (record-ref rd ptop)) + (cond + ((< n (array-length a)) (array-set! a n p)) + (else + (def bigger (array-resize a (if (< n 8) 16 (* n 2)))) + (array-set! bigger n p) + (record-set! rd (pairs bigger)))) + (record-set! rd (ptop (+ n 1)))) + +;; Up to this many keys, every two are compared. +(: int) +(def few-keys 8) + +;; Whether a key is in the pairs from `from` to `to` twice. +(: (-> Reader (Array (Tuple string Json)) int int bool)) +(defun (repeated-key? rd a from to) + (if (<= (- to from) few-keys) + (let outer ((i from)) + (cond + ((>= i to) #f) + ((let inner ((j (+ i 1))) + (cond + ((>= j to) #f) + ((= (fst (array-ref a i)) (fst (array-ref a j))) #t) + (else (inner (+ j 1))))) + #t) + (else (outer (+ i 1))))) + (repeated-key/hashed? rd a from to))) + +;; The table at least twice as big as `n`, and a power of two. +(: (-> Reader int void)) +(defun (fit-table! rd n) + (def size (array-length (record-ref rd slot-stamps))) + (when (< size (* 2 n)) + (def bigger (let grow ((s 64)) (if (< s (* 2 n)) (grow (* s 2)) s))) + (record-set! rd (slot-stamps (make-array bigger)) (slot-pairs (make-array bigger)) (stamp 0)))) + +;; Open addressing over the key's hash. Keys are compared only when their +;; slots collide. +(: (-> Reader (Array (Tuple string Json)) int int bool)) +(defun (repeated-key/hashed? rd a from to) + (fit-table! rd (- to from)) + (def stamps (record-ref rd slot-stamps)) + (def index (record-ref rd slot-pairs)) + (def mask (- (array-length stamps) 1)) + (def stamp (+ (record-ref rd stamp) 1)) + (record-set! rd (stamp stamp)) + (let go ((i from)) + (cond + ((>= i to) #f) + (else + (def k (fst (array-ref a i))) + (let probe ((s (bitwise-and (eq-hash k) mask))) + (cond + ((not (= (array-ref stamps s) stamp)) + (array-set! stamps s stamp) + (array-set! index s i) + (go (+ i 1))) + ((= (fst (array-ref a (array-ref index s))) k) #t) + (else (probe (bitwise-and (+ s 1) mask))))))))) + +;; The pairs from `from` to `to` with each key once: where it was last, with +;; its last value, as a Map keeps the last value. +(: (-> (Array (Tuple string Json)) int int (IArray (Tuple string Json)))) +(defun (last-of-each a from to) + (def out (listbuilder-empty)) + (loop (:for i (range from to)) + (:do (def k (fst (array-ref a i))) + (unless (let later ((j (+ i 1))) + (cond ((>= j to) #f) + ((= (fst (array-ref a j)) k) #t) + (else (later (+ j 1))))) + (listbuilder-add! out (array-ref a i))))) + (list->iarray (listbuilder->list out))) + +;; The object's pairs, copied out of the stack. +(: (-> Reader int (IArray (Tuple string Json)))) +(defun (harvest-pairs! rd mark) + (def a (record-ref rd pairs)) + (def top (record-ref rd ptop)) + (def v (if (repeated-key? rd a mark top) + (last-of-each a mark top) + (subarray->iarray a mark (- top mark)))) + (record-set! rd (ptop mark)) + v) + +;; Reads an object as a JsonFields. On a duplicate key, the last value wins. +(: (-> Reader int (Result string Json))) +(defun (read-object/fields rd depth) + (cond + ((> depth json-max-depth) (Err (too-deep rd "object"))) + (:do + (step! rd) + (skip-space! rd)) + (:def mark (record-ref rd ptop)) + ((at-end? rd) (Err (at rd "the document ended inside an object"))) + ((char=? (peek rd) #\}) (step! rd) (Ok (JsonFields (harvest-pairs! rd mark)))) + (else + (let go () + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended inside an object"))) + ((not (char=? (peek rd) #\")) + (Err (at rd (str "an object's key has to be a string, and this is " (found rd))))) + (:def (Ok key) (read-text rd #t) + :propagate) + (:do (skip-space! rd)) + ((or (at-end? rd) (not (char=? (peek rd) #\:))) + (Err (at rd (str "an object wants ':' after a key, and this is " + (found rd))))) + (:do (step! rd)) + (:def (Ok v) (read-value rd (+ depth 1)) + :propagate) + (:do (push-pair! rd (Tuple key v)) + (skip-space! rd)) + ((at-end? rd) (Err (at rd "the document ended inside an object"))) + ((char=? (peek rd) #\,) (step! rd) (go)) + ((char=? (peek rd) #\}) + (step! rd) + (Ok (JsonFields (harvest-pairs! rd mark)))) + (else + (Err (at rd (str "an object wants ',' or '}' here, and this is " + (found rd)))))))))) + +;; Reads an object as `#:objects` says. +(: (-> Reader int (Result string Json))) +(defun (read-object rd depth) + (if (record-ref rd as-fields) + (read-object/fields rd depth) + (read-object/map rd depth))) + +;; Reads an object as a JsonObj. On a duplicate key, the last value wins. +(: (-> Reader int (Result string Json))) +(defun (read-object/map rd depth) + (cond + ((> depth json-max-depth) (Err (too-deep rd "object"))) + (:do + (step! rd) + (skip-space! rd)) + (:def b (transientmap-empty)) + ((at-end? rd) (Err (at rd "the document ended inside an object"))) + ((char=? (peek rd) #\}) (step! rd) (Ok (JsonObj (transientmap->map b)))) + (else + (let go () + (skip-space! rd) + (cond + ((at-end? rd) (Err (at rd "the document ended inside an object"))) + ((not (char=? (peek rd) #\")) + (Err (at rd (str "an object's key has to be a string, and this is " (found rd))))) + (:def (Ok key) (read-text rd #t) + :propagate) + (:do (skip-space! rd)) + ((or (at-end? rd) (not (char=? (peek rd) #\:))) + (Err (at rd (str "an object wants ':' after a key, and this is " + (found rd))))) + (:do (step! rd)) + (:def (Ok v) (read-value rd (+ depth 1)) + :propagate) + (:do (transientmap-set! b key v) + (skip-space! rd)) + ((at-end? rd) (Err (at rd "the document ended inside an object"))) + ((char=? (peek rd) #\,) (step! rd) (go)) + ((char=? (peek rd) #\}) + (step! rd) + (Ok (JsonObj (transientmap->map b)))) + (else + (Err (at rd (str "an object wants ',' or '}' here, and this is " + (found rd)))))))))) + +(: (-> Reader int (Result string Json))) +(defun (read-value rd depth) + (skip-space! rd) + (if (at-end? rd) + (Err (at rd "the document ended where a value was expected")) + (case (peek rd) + ((#\{) (read-object rd depth)) + ((#\[) (read-array rd depth)) + ((#\") (match (read-text rd #f) + ((Err e) (Err e)) + ((Ok s) (Ok (JsonStr s))))) + ((#\t) (read-word rd "true" json-true)) + ((#\f) (read-word rd "false" json-false)) + ((#\n) (read-word rd "null" json-null)) + ((#\- #\0 #\1 #\2 #\3 #\4 #\5 #\6 #\7 #\8 #\9) (read-number rd)) + (else (Err (at rd (str (found rd) " does not begin a value"))))))) + +;; --------------------------------------------------------------------------- +;; Reading a document +;; --------------------------------------------------------------------------- +;; Whether the reader is looking at a byte order mark: EF BB BF. +(: (-> Reader bool)) +(defun (at-mark? rd) + (and (ensure! rd 3) + (let ((b (record-ref rd bytes)) (i (record-ref rd next))) + (and (= (cast int (array-ref b i)) 239) + (= (cast int (array-ref b (+ i 1))) 187) + (= (cast int (array-ref b (+ i 2))) 191))))) + +(: (-> Reader (Result string Json))) +(defun (read-document rd) + ;; A byte order mark ahead of the value is skipped, as RFC 8259 lets a + ;; parser do, and does not count as a column. + (when (at-mark? rd) + (record-set! rd (next (+ (record-ref rd next) 3)))) + (def (Ok v) (read-value rd 1) + :propagate) + (skip-space! rd) + (if (at-end? rd) + (Ok v) + (Err (at rd (str "the value ended here, and then there is " (found rd)))))) + +(: (Array byte)) +(def no-bytes (make-array 0)) + +(: (-> InputPort (#:objects JsonObjects) (Result string Json))) +(defun (read-json port #:objects 'map) + (read-document (reader (Some port) no-bytes objects))) + +;; The string's bytes are the document: no port in between. +(: (-> string (#:objects JsonObjects) (Result string Json))) +(defun (parse-json text #:objects 'map) + (read-document (reader None (string->bytes text) objects))) + +;; --------------------------------------------------------------------------- +;; The writer +;; --------------------------------------------------------------------------- +;; One `StringBuilder` for the whole document, which `write-json` sends to +;; its port and empties each time it is full. The keys of a JsonObj are +;; written in sorted order: a `Map` does not keep the order they were read in, +;; and sorted keys make the same value give the same text in every run. A +;; JsonFields has an order of its own, and is written in it. + +(import/class + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +(import/extern + (clr-finite? (: System.Double.IsFinite (-> double bool)))) + +(: (Vec char)) +(def hex-digits (string->vec "0123456789abcdef")) + +(: (-> StringBuilder int void)) +(defun (add-hex4! sb n) + (stringbuilder-add-string! sb "\\u") + (loop (:for shift (list 12 8 4 0)) + (:do (stringbuilder-add! sb (vec-ref hex-digits (bitwise-and (shift-right n shift) 15)))))) + +;; One character as JSON writes it in a string: an escape for `"`, `\` and the +;; control characters, as RFC 8259 requires, and with `ascii` for every +;; character outside ASCII too, as one `\u` escape or, outside the BMP, as a +;; surrogate pair of them. +(: (-> StringBuilder char bool void)) +(defun (add-char! sb c ascii) + (def n (char->int c)) + (cond + ((= n 34) (stringbuilder-add-string! sb "\\\"")) + ((= n 92) (stringbuilder-add-string! sb "\\\\")) + ((= n 10) (stringbuilder-add-string! sb "\\n")) + ((= n 13) (stringbuilder-add-string! sb "\\r")) + ((= n 9) (stringbuilder-add-string! sb "\\t")) + ((= n 8) (stringbuilder-add-string! sb "\\b")) + ((= n 12) (stringbuilder-add-string! sb "\\f")) + ((< n 32) (add-hex4! sb n)) + ((and ascii (> n 126) (< n 65536)) (add-hex4! sb n)) + ((and ascii (>= n 65536)) + (def v (- n 65536)) + (add-hex4! sb (+ 55296 (shift-right v 10))) + (add-hex4! sb (+ 56320 (bitwise-and v 1023)))) + (else (stringbuilder-add! sb c)))) + +;; The characters a string has to have escaped: `"`, `\` and the control +;; characters. All ASCII, so `string-search-any` finds them with a byte search. +(: string) +(def must-escape + (str "\"\\" (list->string (loop (:for n (range 0 32)) (:acc (listing (int->char n))))))) + +;; A string, quoted. The runs between the characters that have to be escaped +;; are copied whole. With `ascii` every character outside ASCII has to be +;; escaped too, so each character is looked at. +(: (-> StringBuilder string bool void)) +(defun (add-string! sb s ascii) + (stringbuilder-add! sb #\") + (if ascii + (string-for-each (fun (c) (add-char! sb c ascii)) s) + (let go ((from (string-cursor-start s))) + (match (string-search-any s must-escape from) + (None (stringbuilder-add-substring! sb s from (string-cursor-end s))) + ((Some at) + (stringbuilder-add-substring! sb s from at) + (add-char! sb (string-cursor-ref s at) ascii) + (go (string-cursor-next s at)))))) + (stringbuilder-add! sb #\")) + +;; The shortest text that reads back as the same double, with `.0` added when +;; it has neither a point nor an exponent, so that it reads back as a +;; `JsonFloat` and not a `JsonInt`. +(: (-> StringBuilder double void)) +(defun (add-float! sb d) + (unless (clr-finite? d) + (raise (cast Exception (ArgumentException. (str "JSON has no number for " (double->string d)))))) + (def from (stringbuilder-length sb)) + (stringbuilder-add-double! sb d) + ;; The bytes just written: `.` is 46, `E` 69 and `e` 101. + (def fraction? + (let go ((i from)) + (cond ((= i (stringbuilder-length sb)) #f) + ((let ((b (stringbuilder-byte-ref sb i))) (or (= b 46) (= b 69) (= b 101))) #t) + (else (go (+ i 1)))))) + (unless fraction? + (stringbuilder-add-string! sb ".0"))) + +;; An object's entries in an array, in the order of their keys. An insertion +;; sort, in place: an object has a few keys, and a sort that allocates as it +;; merges costs more than the comparisons for them. One with many keys is +;; sorted by `vec-sort-by`, which takes n log n comparisons. +(: (-> (Map string Json) (Array (Tuple string Json)))) +(defun (sorted-entries fields) + (def n (map-length fields)) + (if (> n 16) + (vec->array (vec-sort-by (fun (a b) (compare (fst a) (fst b))) (map->vec fields))) + (let () + (def entries (make-array n)) + (loop (:for e (map->seq fields)) + (:for i (range 0 n)) + (:do (array-set! entries i e))) + (loop (:for i (range 1 n)) + (:do (def e (array-ref entries i)) + (let shift ((j (- i 1))) + (if (and (>= j 0) (> (compare (fst (array-ref entries j)) (fst e)) 0)) + (let () + (array-set! entries (+ j 1) (array-ref entries j)) + (shift (- j 1))) + (array-set! entries (+ j 1) e))))) + entries))) + +(: (-> StringBuilder int int void)) +(defun (add-newline! sb indent level) + (when (> indent 0) + (stringbuilder-add! sb #\newline) + (loop (:for _ (range 0 (* indent level))) + (:do (stringbuilder-add! sb #\space))))) + +;; How full the builder may get before `write-json` sends it to its port. +;; Below the builder's chunk size, so the builder stays one buffer. +(: int) +(def json-write-part 32768) + +;; Hands the builder to `flush` once it holds `json-write-part` bytes. Called +;; between two elements or two fields. +(: (-> StringBuilder (-?-> StringBuilder void) void)) +(defun (maybe-flush! sb flush) #:inline + (when (>= (stringbuilder-length sb) json-write-part) + (flush sb))) + +;; `j` added to `sb`. `flush` is given the builder when it is full, and +;; empties it; `json->string` gives one that does nothing. +(: (-> StringBuilder Json int bool int (-?-> StringBuilder void) void)) +(defun (add-value! sb j indent ascii level flush) + (match j + (JsonNull (stringbuilder-add-string! sb "null")) + ((JsonBool b) (stringbuilder-add-string! sb (if b "true" "false"))) + ((JsonInt n) (stringbuilder-add-long! sb n)) + ((JsonFloat d) (add-float! sb d)) + ((JsonStr s) (add-string! sb s ascii)) + ((JsonArr items) + (cond + ((iarray-empty? items) (stringbuilder-add-string! sb "[]")) + (else + (stringbuilder-add! sb #\[) + (loop (:for item items) + (:with first? #t #f) + (:do (unless first? (stringbuilder-add! sb #\,)) + (add-newline! sb indent (+ level 1)) + (add-value! sb item indent ascii (+ level 1) flush) + (maybe-flush! sb flush))) + (add-newline! sb indent level) + (stringbuilder-add! sb #\])))) + ((JsonObj fields) + (cond + ((map-empty? fields) (stringbuilder-add-string! sb "{}")) + (else + ;; The entries, sorted by key once: each is written with the value it + ;; already holds, and no key is looked up again. + (stringbuilder-add! sb #\{) + (loop (:for (Tuple k v) (sorted-entries fields)) + (:with first? #t #f) + (:do (add-field! sb k v first? indent ascii level flush))) + (add-newline! sb indent level) + (stringbuilder-add! sb #\})))) + ;; In their own order, which for a document read is the document's. The + ;; same loop as a JsonObj's, written again so that each walks a type it + ;; knows: a function generic in the collection would have a suspending + ;; copy, which is not specialised. + ((JsonFields fields) + (cond + ((iarray-empty? fields) (stringbuilder-add-string! sb "{}")) + (else + (stringbuilder-add! sb #\{) + (loop (:for (Tuple k v) fields) + (:with first? #t #f) + (:do (add-field! sb k v first? indent ascii level flush))) + (add-newline! sb indent level) + (stringbuilder-add! sb #\})))))) + +;; One field of an object at `level`, and the comma before it unless it is +;; the first. +(: (-> StringBuilder string Json bool int bool int (-?-> StringBuilder void) void)) +(defun (add-field! sb k v first? indent ascii level flush) #:inline + (unless first? (stringbuilder-add! sb #\,)) + (add-newline! sb indent (+ level 1)) + (add-string! sb k ascii) + (stringbuilder-add-string! sb (if (> indent 0) ": " ":")) + (add-value! sb v indent ascii (+ level 1) flush) + (maybe-flush! sb flush)) + +;; JSON text for a value. Compact unless `#:indent` is more than 0, which puts +;; each element and field on a line of its own, indented that many spaces a +;; level. With `#:ascii #t`, the text is ASCII. +(: (-> Json (#:indent int) (#:ascii bool) string)) +(defun (json->string j #:indent 0 #:ascii #f) + (def sb (stringbuilder-empty)) + (add-value! sb j indent ascii 0 (fun (_) unit)) + (stringbuilder->string sb)) + +(import/extern + (clr-write-builder (: Bjolang.Runtime.BjoOutputPort.WriteBuilder + (-> OutputPort StringBuilder void) #:blocking)) + (clr-write-builder-await (: Bjolang.Runtime.BjoOutputPort.WriteBuilderAsync + (-> OutputPort StringBuilder void) #:async))) + +;; A builder's bytes to `p`, and the builder emptied. +(: (-> OutputPort StringBuilder void)) +(defbjouble (write-builder! p sb) + (#:sync (clr-write-builder p sb) (stringbuilder-clear! sb)) + (#:bjo (clr-write-builder-await p sb) (stringbuilder-clear! sb))) + +;; The text `json->string` gives, written to `port`. It goes out in parts of +;; about `json-write-part` bytes as it is made, so no string of the whole +;; text is made. From a bjoroutine, each part's write suspends. +(: (-> OutputPort Json (#:indent int) (#:ascii bool) void)) +(defun (write-json port j #:indent 0 #:ascii #f) + (def sb (stringbuilder-empty)) + ;; A local function, not a `fun`, so that it takes the colour of the copy + ;; it is in: in the suspending copy of `write-json`, its write suspends. + (defun (flush b) (write-builder! port b)) + (add-value! sb j indent ascii 0 flush) + (write-builder! port sb)) + +(impl (Written Json) + (defun (written j) (json->string j)) + (defun (write-into j b) (stringbuilder-add-string! b (written j)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; text/xml.bjo — the module (text xml): names, nodes and documents. +;; +;; The tree is opaque, and its constructors enforce what every tree obeys, +;; whoever built it: no two text children side by side, no empty text child, +;; no duplicate attribute, and no namespace declaration among the attributes. +;; The reader, the writer and the modules still to come — matching, paths, +;; decoding, HTML — use only what is exported here, so the views below are +;; the whole of the access there is. +;; +;; A tree can be built by hand as deep as memory allows, and a stack overflow +;; cannot be caught in .NET. So nothing here recurses per nesting level: every +;; walk keeps its own stack, a list of the child lists still to visit. +;; +;; Its reference is its docs, in xml.bjodoc. +;; +;; Open: +;; - The printed form does not read back. SXML literals and patterns for this +;; tree are (text xml match)'s, still to come. +;; - Only an element carries a position; a text, a comment or a PI has none, +;; so a decoder can point no closer than the element around it. + +(export + ;; Names + XmlName xml-name xml-name-local xml-name-ns xml-namespace + ;; Small types + XmlPos XmlAttrs Doctype + ;; Nodes + Node xml-element xml-text xml-comment xml-pi + element? text? comment? pi? + element-name element-attrs element-children element-pos element-view + text-value comment-value pi-target pi-data + node-case + ;; Helpers + attr-ref element-children-only text-content descendants node=? + ;; Documents + Document xml-document document-root document-doctype document-prolog document-epilog + document=?) + +(import (std prelude)) + +(import/class + (XName (: System.Xml.Linq.XName)) + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +(import/extern + (clr-xname (: System.Xml.Linq.XName.Get (-> string string XName))) + (clr-xname-local (: System.Xml.Linq.XName.LocalName (-> XName string) #:get)) + (clr-xname-ns (: System.Xml.Linq.XName.NamespaceName (-> XName string) #:get)) + (clr-xname-hash (: System.Xml.Linq.XName.GetHashCode (-> XName int))) + (clr-xname-clark (: System.Xml.Linq.XName.ToString (-> XName string))) + (clr-verify-ncname (: System.Xml.XmlConvert.VerifyNCName (-> string string) + #:exceptions (System.Xml.XmlException System.ArgumentNullException)))) + +;; Every export is documented there; the module doc is too. +(:doc #:file "xml.bjodoc") + +;; An ArgumentException, which is what a programming error raises here. Never +;; an XmlException: the reader treats those as the document's errors. +(: (-> string %a)) +(defun (argument-error message) + (raise (cast Exception (ArgumentException. (str "(text xml): " message))))) + +;; =========================================================================== +;; Names +;; =========================================================================== + +;; An XName, which .NET interns, so two names are equal exactly when they are +;; the same object. No prefix: a prefix belongs to the text a name was read +;; from or is written to, not to the name. +(type (: XmlName #:opaque (Struct (: v XName)))) + +(impl (Eq XmlName) + (defun (= a b) (eq? (struct-ref a v) (struct-ref b v))) + (defun (eq-hash n) (clr-xname-hash (struct-ref n v)))) + +;; Clark notation: {uri}local, or local alone in no namespace. +(impl (Written XmlName) + (defun (written n) (clr-xname-clark (struct-ref n v))) + (defun (write-into n b) (stringbuilder-add-string! b (written n)))) + +(: string) +(def xml-namespace "http://www.w3.org/XML/1998/namespace") + +;; What xmlns and xmlns:p are in. Nothing is ever named in it. +(: string) +(def xmlns-namespace "http://www.w3.org/2000/xmlns/") + +(: (-> string (#:ns string) XmlName)) +(defun (xml-name local #:ns "") + (match (clr-verify-ncname local) + ((Ok _) (XmlName (v (clr-xname local ns)))) + ((Err e) (argument-error (str "xml-name: " (.-Message e)))))) + +(: (-> XmlName string)) +(defun (xml-name-local n) (clr-xname-local (struct-ref n v))) + +;; "" when the name is in no namespace. +(: (-> XmlName string)) +(defun (xml-name-ns n) (clr-xname-ns (struct-ref n v))) + +;; =========================================================================== +;; Small types +;; =========================================================================== + +;; As .NET reports it: 1-based, and the column of the name after the '<'. +(type (: XmlPos (Struct (: line int) (: column int)))) + +;; In document order. +(type (: XmlAttrs (Alias (List (Tuple XmlName string))))) + +;; Kept so that a document can be written back with it. Nothing validates +;; against it. +(type (: Doctype (Record (: name string) + (: public-id (Option string)) + (: system-id (Option string))))) + +;; =========================================================================== +;; Nodes +;; =========================================================================== + +;; The case names are long because an importer receives them along with the +;; type, although only this module may use them. +(type-rec + (: Node #:opaque (Union (: XmlNodeElement XmlName XmlAttrs (List Node) (Option XmlPos)) + (: XmlNodeText string) + (: XmlNodeComment string) + (: XmlNodePI string string)))) + +(: (-> Node string)) +(defun (variant-name n) + (match n + ((XmlNodeElement _ _ _ _) "an element") + ((XmlNodeText _) "a text") + ((XmlNodeComment _) "a comment") + ((XmlNodePI _ _) "a processing instruction"))) + +(: (-> string Node string %a)) +(defun (wrong-variant who n wanted) + (argument-error (str who ": the node is " (variant-name n) ", not " wanted))) + +;; ---- Attributes ----------------------------------------------------------- + +;; Above this many attributes, duplicates are found with a map rather than by +;; comparing every pair. +(: int) +(def few-attributes 16) + +(: (-> XmlName bool)) +(defun (namespace-declaration? name) + (or (= (xml-name-ns name) xmlns-namespace) + (and (= (xml-name-ns name) "") (= (xml-name-local name) "xmlns")))) + +(: (-> XmlName XmlAttrs (Option string))) +(defun (attr-value name attrs) + (let loop ((xs attrs)) + (match xs + (Nil None) + ((Cons (Tuple n value) rest) (if (= n name) (Some value) (loop rest)))))) + +(: (-> XmlName void)) +(defun (check-attr-name name) + (when (namespace-declaration? name) + (argument-error + (str "xml-element: " (displayed name) " is a namespace declaration, which is not an attribute. " + "The writer declares namespaces itself.")))) + +(: (-> XmlName %a)) +(defun (duplicate-attr name) + (argument-error (str "xml-element: the attribute " (displayed name) " is given twice"))) + +(: (-> XmlAttrs void)) +(defun (check-few-attrs attrs) + (let loop ((xs attrs)) + (match xs + (Nil unit) + ((Cons (Tuple name _) rest) + (check-attr-name name) + (when (some? (attr-value name rest)) (duplicate-attr name)) + (loop rest))))) + +(: (-> XmlAttrs void)) +(defun (check-many-attrs attrs) + (def seen (transientmap-empty)) + (let loop ((xs attrs)) + (match xs + (Nil unit) + ((Cons (Tuple name _) rest) + (check-attr-name name) + (when (transientmap-contains? seen name) (duplicate-attr name)) + (transientmap-set! seen name #t) + (loop rest))))) + +(: (-> XmlAttrs void)) +(defun (check-attrs attrs) + (if (> (list-length attrs) few-attributes) + (check-many-attrs attrs) + (check-few-attrs attrs))) + +;; ---- Children ------------------------------------------------------------- + +;; Whether the children already obey the invariants, in which case the list +;; is kept as it is. The parser's always do. +(: (-> (List Node) bool)) +(defun (children-normal? kids) + (let loop ((xs kids) (after-text #f)) + (match xs + (Nil #t) + ((Cons (XmlNodeText s) rest) + (if (or after-text (string-empty? s)) #f (loop rest #t))) + ((Cons _ rest) (loop rest #f))))) + +(: (-> (List Node) (List Node))) +(defun (normalize-children kids) + (def out (listbuilder-empty)) + (def pending (stringbuilder-empty)) + (defun (flush) + (when (> (stringbuilder-length pending) 0) + (listbuilder-add! out (XmlNodeText (stringbuilder->string pending))) + (stringbuilder-clear! pending))) + (let loop ((xs kids)) + (match xs + (Nil + (flush) + (listbuilder->list out)) + ((Cons (XmlNodeText s) rest) + (stringbuilder-add-string! pending s) + (loop rest)) + ((Cons n rest) + (flush) + (listbuilder-add! out n) + (loop rest))))) + +;; ---- Construction --------------------------------------------------------- + +;; Merges adjacent text children and drops empty ones. Raises on a duplicate +;; attribute or a namespace declaration. +(: (-> XmlName XmlAttrs (List Node) (#:pos (Option XmlPos)) Node)) +(defun (xml-element name attrs children #:pos None) + (check-attrs attrs) + (XmlNodeElement name attrs (if (children-normal? children) children (normalize-children children)) pos)) + +;; Empty text is allowed on its own; an element drops it. +(: (-> string Node)) +(defun (xml-text s) (XmlNodeText s)) + +;; Not checked here: a comment with "--" in it is refused when it is written. +(: (-> string Node)) +(defun (xml-comment s) (XmlNodeComment s)) + +;; Target, then data. Checked when it is written, as a comment is. +(: (-> string string Node)) +(defun (xml-pi target data) (XmlNodePI target data)) + +;; ---- Views ---------------------------------------------------------------- + +(: (-> Node bool)) +(defun (element? n) (match n ((XmlNodeElement _ _ _ _) #t) (_ #f))) + +(: (-> Node bool)) +(defun (text? n) (match n ((XmlNodeText _) #t) (_ #f))) + +(: (-> Node bool)) +(defun (comment? n) (match n ((XmlNodeComment _) #t) (_ #f))) + +(: (-> Node bool)) +(defun (pi? n) (match n ((XmlNodePI _ _) #t) (_ #f))) + +(: (-> Node XmlName)) +(defun (element-name n) + (match n + ((XmlNodeElement name _ _ _) name) + (_ (wrong-variant "element-name" n "an element")))) + +(: (-> Node XmlAttrs)) +(defun (element-attrs n) + (match n + ((XmlNodeElement _ attrs _ _) attrs) + (_ (wrong-variant "element-attrs" n "an element")))) + +(: (-> Node (List Node))) +(defun (element-children n) + (match n + ((XmlNodeElement _ _ kids _) kids) + (_ (wrong-variant "element-children" n "an element")))) + +(: (-> Node (Option XmlPos))) +(defun (element-pos n) + (match n + ((XmlNodeElement _ _ _ pos) pos) + (_ (wrong-variant "element-pos" n "an element")))) + +;; The whole element in one call. Option is a struct and Tuple a value tuple, +;; so this allocates nothing. +(: (-> Node (Option (Tuple XmlName XmlAttrs (List Node))))) +(defun (element-view n) + (match n + ((XmlNodeElement name attrs kids _) (Some (Tuple name attrs kids))) + (_ None))) + +(: (-> Node string)) +(defun (text-value n) + (match n + ((XmlNodeText s) s) + (_ (wrong-variant "text-value" n "a text")))) + +(: (-> Node string)) +(defun (comment-value n) + (match n + ((XmlNodeComment s) s) + (_ (wrong-variant "comment-value" n "a comment")))) + +(: (-> Node string)) +(defun (pi-target n) + (match n + ((XmlNodePI target _) target) + (_ (wrong-variant "pi-target" n "a processing instruction")))) + +(: (-> Node string)) +(defun (pi-data n) + (match n + ((XmlNodePI _ data) data) + (_ (wrong-variant "pi-data" n "a processing instruction")))) + +(: (-> Node %r)) +(defun (no-clause n) + (argument-error (str "node-case: no clause for " (variant-name n) ", and no #:else"))) + +;; #:else is declared first because the other defaults call it. +(: (-> Node + (#:else (-> Node %r)) + (#:element (-> XmlName XmlAttrs (List Node) (Option XmlPos) %r)) + (#:text (-> string %r)) + (#:comment (-> string %r)) + (#:pi (-> string string %r)) + %r)) +(defun (node-case n + #:else (fun (m) (no-clause m)) + #:element (fun (name attrs kids pos) (else n)) + #:text (fun (s) (else n)) + #:comment (fun (s) (else n)) + #:pi (fun (target data) (else n))) + (match n + ((XmlNodeElement name attrs kids pos) (element name attrs kids pos)) + ((XmlNodeText s) (text s)) + ((XmlNodeComment s) (comment s)) + ((XmlNodePI target data) (pi target data)))) + +;; =========================================================================== +;; Helpers +;; =========================================================================== + +(: (-> Node XmlName (Option string))) +(defun (attr-ref n name) + (match n + ((XmlNodeElement _ attrs _ _) (attr-value name attrs)) + (_ None))) + +(: (-> Node (List Node))) +(defun (element-children-only n) + (match n + ((XmlNodeElement _ _ kids _) (list-filter element? kids)) + (_ Nil))) + +;; The text of every text node below, in document order. A text node's own +;; value; "" for a comment or a processing instruction. +(: (-> Node string)) +(defun (text-content n) + (match n + ((XmlNodeText s) s) + ((XmlNodeElement _ _ kids _) + (def sb (stringbuilder-empty)) + (let loop ((stack (list kids))) + (match stack + (Nil (stringbuilder->string sb)) + ((Cons Nil rest) (loop rest)) + ((Cons (Cons (XmlNodeText s) more) rest) + (stringbuilder-add-string! sb s) + (loop (Cons more rest))) + ((Cons (Cons (XmlNodeElement _ _ inner _) more) rest) + (loop (Cons inner (Cons more rest)))) + ((Cons (Cons _ more) rest) + (loop (Cons more rest)))))) + (_ ""))) + +;; Every node below, in document order, not the node itself. +(: (-> Node (Seq Node))) +(defun (descendants n) + (def top (match n ((XmlNodeElement _ _ kids _) kids) (_ Nil))) + (seq + (let loop ((stack (list top))) + (match stack + (Nil unit) + ((Cons Nil rest) (loop rest)) + ((Cons (Cons k more) rest) + (yield k) + (match k + ((XmlNodeElement _ _ inner _) (loop (Cons inner (Cons more rest)))) + (_ (loop (Cons more rest))))))))) + +;; ---- Equality ------------------------------------------------------------- + +;; Pair by pair, which is the usual case when the lists are equal at all. +(: (-> XmlAttrs XmlAttrs bool)) +(defun (attrs-in-order=? xs ys) + (let loop ((xs xs) (ys ys)) + (match (Tuple xs ys) + ((Tuple Nil Nil) #t) + ((Tuple (Cons (Tuple n1 v1) r1) (Cons (Tuple n2 v2) r2)) + (if (and (= n1 n2) (= v1 v2)) (loop r1 r2) #f)) + (_ #f)))) + +;; Both lists are free of duplicates, so the same length and every attribute +;; of one found in the other is the same set. +(: (-> XmlAttrs XmlAttrs bool)) +(defun (attrs-found? xs ys) + (let loop ((xs xs)) + (match xs + (Nil #t) + ((Cons (Tuple name value) rest) + (if (= (attr-value name ys) (Some value)) (loop rest) #f))))) + +(: (-> XmlAttrs XmlAttrs bool)) +(defun (attrs-mapped? xs ys) + (def m (transientmap-empty)) + (let fill ((ys ys)) + (match ys + (Nil unit) + ((Cons (Tuple name value) rest) + (transientmap-set! m name value) + (fill rest)))) + (let loop ((xs xs)) + (match xs + (Nil #t) + ((Cons (Tuple name value) rest) + (if (= (transientmap-try-ref m name) (Some value)) (loop rest) #f))))) + +;; In any order, as XML has them. +(: (-> XmlAttrs XmlAttrs bool)) +(defun (attrs=? xs ys) + (cond + ((attrs-in-order=? xs ys) #t) + (:def n (list-length xs)) + ((not (= n (list-length ys))) #f) + ((> n few-attributes) (attrs-mapped? xs ys)) + (else (attrs-found? xs ys)))) + +;; Structure only: positions are not compared, and attributes in any order. +;; The stack holds the pairs of child lists still to compare. +(: (-> Node Node bool)) +(defun (node=? a b) + (let loop ((stack (list (Tuple (list a) (list b))))) + (match stack + (Nil #t) + ((Cons (Tuple Nil Nil) rest) (loop rest)) + ((Cons (Tuple (Cons x xs) (Cons y ys)) rest) + (match (Tuple x y) + ((Tuple (XmlNodeElement n1 a1 k1 _) (XmlNodeElement n2 a2 k2 _)) + (if (and (= n1 n2) (attrs=? a1 a2)) + (loop (Cons (Tuple k1 k2) (Cons (Tuple xs ys) rest))) + #f)) + ((Tuple (XmlNodeText s1) (XmlNodeText s2)) + (if (= s1 s2) (loop (Cons (Tuple xs ys) rest)) #f)) + ((Tuple (XmlNodeComment s1) (XmlNodeComment s2)) + (if (= s1 s2) (loop (Cons (Tuple xs ys) rest)) #f)) + ((Tuple (XmlNodePI t1 d1) (XmlNodePI t2 d2)) + (if (and (= t1 t2) (= d1 d2)) (loop (Cons (Tuple xs ys) rest)) #f)) + (_ #f))) + (_ #f)))) + +;; Summed, so that the order of the attributes does not change it. +(: (-> XmlAttrs int)) +(defun (attrs-hash attrs) + (list-foldl (fun (a h) (def (Tuple name value) a) (+ h (hash-combine (eq-hash name) (eq-hash value)))) + 0 + attrs)) + +;; Agrees with node=?: positions and attribute order leave it alone. The end +;; of each child list is hashed too, so that nesting is part of it. +(: (-> Node int)) +(defun (node-hash n) + (let loop ((stack (list (list n))) (h 17)) + (match stack + (Nil h) + ((Cons Nil rest) (loop rest (hash-combine h 1))) + ((Cons (Cons k more) rest) + (match k + ((XmlNodeElement name attrs kids _) + (loop (Cons kids (Cons more rest)) + (hash-combine (hash-combine (hash-combine h 2) (eq-hash name)) (attrs-hash attrs)))) + ((XmlNodeText s) (loop (Cons more rest) (hash-combine (hash-combine h 3) (eq-hash s)))) + ((XmlNodeComment s) (loop (Cons more rest) (hash-combine (hash-combine h 4) (eq-hash s)))) + ((XmlNodePI target data) + (loop (Cons more rest) + (hash-combine (hash-combine (hash-combine h 5) (eq-hash target)) (eq-hash data))))))))) + +;; Without this, = on a Node would be the C# record's own Equals, which +;; recurses per nesting level and compares positions. +(impl (Eq Node) + (defun (= a b) (node=? a b)) + (defun (eq-hash n) (node-hash n))) + +;; =========================================================================== +;; Documents +;; =========================================================================== + +(type (: Document #:opaque (Record (: doctype (Option Doctype)) + (: prolog (List Node)) + (: root Node) + (: epilog (List Node))))) + +(: (-> string (List Node) void)) +(defun (check-misc where nodes) + (list-for-each + (fun (n) + (unless (or (comment? n) (pi? n)) + (argument-error + (str "xml-document: the " where " may hold comments and processing instructions, and this is " + (variant-name n))))) + nodes)) + +;; Raises if the root is not an element, or the prolog or epilog holds +;; anything but comments and processing instructions. +(: (-> Node (#:doctype (Option Doctype)) (#:prolog (List Node)) (#:epilog (List Node)) + Document)) +(defun (xml-document root #:doctype None #:prolog Nil #:epilog Nil) + (unless (element? root) + (argument-error (str "xml-document: the root has to be an element, and this is " (variant-name root)))) + (check-misc "prolog" prolog) + (check-misc "epilog" epilog) + (Document (doctype doctype) (prolog prolog) (root root) (epilog epilog))) + +(: (-> Document Node)) +(defun (document-root d) (record-ref d root)) + +(: (-> Document (Option Doctype))) +(defun (document-doctype d) (record-ref d doctype)) + +(: (-> Document (List Node))) +(defun (document-prolog d) (record-ref d prolog)) + +(: (-> Document (List Node))) +(defun (document-epilog d) (record-ref d epilog)) + +(: (-> (List Node) (List Node) bool)) +(defun (nodes=? xs ys) + (match (Tuple xs ys) + ((Tuple Nil Nil) #t) + ((Tuple (Cons x xr) (Cons y yr)) (if (node=? x y) (nodes=? xr yr) #f)) + (_ #f))) + +(: (-> Document Document bool)) +(defun (document=? a b) + (and (= (record-ref a doctype) (record-ref b doctype)) + (nodes=? (record-ref a prolog) (record-ref b prolog)) + (node=? (record-ref a root) (record-ref b root)) + (nodes=? (record-ref a epilog) (record-ref b epilog)))) + +(impl (Eq Document) + (defun (= a b) (document=? a b)) + (defun (eq-hash d) + (list-foldl (fun (n h) (hash-combine h (node-hash n))) + (node-hash (record-ref d root)) + (list-append (record-ref d prolog) (record-ref d epilog))))) + +;; =========================================================================== +;; The printer +;; =========================================================================== +;; +;; SXML's look, for the REPL and for debugging: +;; +;; (p (@ (class "x")) "hi" (b "!")) +;; (*COMMENT* " c ") (*PI* target "data") +;; (*TOP* (*DOCTYPE* html SYSTEM "a.dtd") ... (root ...)) +;; +;; A name in a namespace is written in Clark notation, {uri}local. Nothing +;; promises yet that this reads back. + +(: (-> StringBuilder string void)) +(defun (add-quoted! sb s) + (stringbuilder-add! sb #\") + (loop (:for c s) + (:do (case c + ((#\") (stringbuilder-add-string! sb "\\\"")) + ((#\\) (stringbuilder-add-string! sb "\\\\")) + ((#\newline) (stringbuilder-add-string! sb "\\n")) + ((#\tab) (stringbuilder-add-string! sb "\\t")) + (else (if (= (char->int c) 13) + (stringbuilder-add-string! sb "\\r") + (stringbuilder-add! sb c)))))) + (stringbuilder-add! sb #\")) + +;; A node still to print, or text that closes one. +(type (: PrintItem (Union (: PrintNode Node) (: PrintText string)))) + +(: (-> StringBuilder Node void)) +(defun (print-node! sb n) + (let loop ((items (list (PrintNode n)))) + (match items + (Nil unit) + ((Cons (PrintText s) rest) + (stringbuilder-add-string! sb s) + (loop rest)) + ((Cons (PrintNode (XmlNodeText s)) rest) + (add-quoted! sb s) + (loop rest)) + ((Cons (PrintNode (XmlNodeComment s)) rest) + (stringbuilder-add-string! sb "(*COMMENT* ") + (add-quoted! sb s) + (stringbuilder-add! sb #\)) + (loop rest)) + ((Cons (PrintNode (XmlNodePI target data)) rest) + (stringbuilder-add-string! sb (str "(*PI* " target " ")) + (add-quoted! sb data) + (stringbuilder-add! sb #\)) + (loop rest)) + ((Cons (PrintNode (XmlNodeElement name attrs kids _)) rest) + (stringbuilder-add! sb #\() + (write-into name sb) + (unless (list-empty? attrs) + (stringbuilder-add-string! sb " (@") + ;; todo: rewrite as loop. + (let each ((as attrs)) + (match as + (Nil unit) + ((Cons (Tuple an value) more) + (stringbuilder-add-string! sb " (") + (write-into an sb) + (stringbuilder-add! sb #\space) + (add-quoted! sb value) + (stringbuilder-add! sb #\)) + (each more)))) + (stringbuilder-add! sb #\))) + (loop (list-foldr (fun (k acc) (Cons (PrintText " ") (Cons (PrintNode k) acc))) + (Cons (PrintText ")") rest) + kids)))))) + +(impl (Written Node) + (defun (write-into n b) (print-node! b n))) + +(: (-> StringBuilder Doctype void)) +(defun (print-doctype! sb dt) + (stringbuilder-add-string! sb (str "(*DOCTYPE* " (get dt .name))) + (match (Tuple (get dt .public-id) (get dt .system-id)) + ((Tuple (Some pub) (Some sys)) + (stringbuilder-add-string! sb " PUBLIC ") + (add-quoted! sb pub) + (stringbuilder-add! sb #\space) + (add-quoted! sb sys)) + ((Tuple (Some pub) None) + (stringbuilder-add-string! sb " PUBLIC ") + (add-quoted! sb pub)) + ((Tuple None (Some sys)) + (stringbuilder-add-string! sb " SYSTEM ") + (add-quoted! sb sys)) + ((Tuple None None) unit)) + (stringbuilder-add! sb #\))) + +(impl (Written Document) + (defun (write-into d sb) + (stringbuilder-add-string! sb "(*TOP*") + (match (get d .doctype) + ((Some dt) + (stringbuilder-add! sb #\space) + (print-doctype! sb dt)) + (None unit)) + (list-for-each (fun (n) + (stringbuilder-add! sb #\space) + (print-node! sb n)) + (list-append (get d .prolog) (Cons (get d .root) (get d .epilog)))) + (stringbuilder-add! sb #\)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; text/xml/read.bjo — the module (text xml read). +;; +;; Four layers, each built on the one below: +;; +;; 1. signals: a pull stream, as OCaml's Xmlm has it +;; 2. xml-fold: Kiselyov's SSAX fold, generic in its seed +;; 3. xml-tree-fold: a tree of the caller's own type +;; 4. parse-xml: a Document; and xml-for-each, one element at a time +;; +;; Only the signal layer calls XmlReader, so its quirks stay there: an empty +;; element's missing end tag, text in several pieces, namespace declarations +;; among the attributes. It is also where XmlException is caught and becomes an +;; XmlError, around the reader calls and nothing else, so an exception a +;; handler raises passes through as it was. +;; +;; A fold that fails at a node has already run its handlers for every node +;; before it. That is what streaming means. +;; +;; Its reference is its docs, in read.bjodoc. +;; +;; Open: +;; - #:dtd 'parse refuses a DOCTYPE that names an external DTD, because +;; XmlResolver.ThrowingResolver refuses to open it. So an XHTML document +;; reads only under 'ignore, where XmlReader does not report the DOCTYPE at +;; all and the Document has no doctype. .NET 10's default resolver would skip +;; an external DTD silently instead, and expand an external entity to +;; nothing, which keeps the doctype but quietly changes content. +;; - A DOCTYPE refused and a limit reached have line and column 0: .NET throws +;; them without a position, and has reset the reader's own by then. +;; - xml-skip-element! does not check what it skips against #:max-depth. +;; XmlReader's own node stack grows with that depth, bounded only by +;; #:max-characters. +;; - #:whitespace 'drop drops whitespace-only text inside xml:space="preserve" +;; as well. +;; - A try compiles to a closure and a delegate, so each signal costs two small +;; allocations. A try emitted as a statement would remove them. +;; - Every element and attribute read makes its name again: a VerifyNCName and +;; an XName.Get. XmlReader hands out the same string objects for a name it +;; has seen, so a small cache keyed on them would save both. +;; - Reading is synchronous. An async variant over XmlReader.ReadAsync belongs +;; in the signal layer, where the reader is, and nowhere above it. + +(export + ;; Options + DtdMode WhitespaceMode + ;; Errors + XmlError XmlErrorKind + ;; Sources + XmlInput XmlSource xml-input XmlFile xml-file + ;; Signals + XmlSignal XmlSignals call-with-xml-signals xml-next! xml-skip-element! + ;; Folds + xml-fold xml-tree-fold + ;; Documents + parse-xml xml-for-each) + +(import (std prelude)) +(import (text xml)) + +(import/class + (XmlReader (: System.Xml.XmlReader)) + (XmlReaderSettings (: System.Xml.XmlReaderSettings (-> XmlReaderSettings))) + (XmlResolver (: System.Xml.XmlResolver)) + (DtdProcessing (: System.Xml.DtdProcessing)) + (XmlNodeType (: System.Xml.XmlNodeType)) + (IXmlLineInfo (: System.Xml.IXmlLineInfo)) + (XmlException (: System.Xml.XmlException)) + (Stream (: System.IO.Stream)) + (TextReader (: System.IO.TextReader)) + (Object (: System.Object)) + (InvalidOperationException (: System.InvalidOperationException (-> string InvalidOperationException))) + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +(import/extern + ;; For a string, read as text: there is no encoding to declare. + (clr-reader-over-text (: System.Xml.XmlReader.Create (-> TextReader XmlReaderSettings XmlReader))) + (clr-reader-over-stream (: System.Xml.XmlReader.Create (-> Stream XmlReaderSettings XmlReader))) + (clr-settings-dtd! (: System.Xml.XmlReaderSettings.DtdProcessing (-> XmlReaderSettings DtdProcessing void) #:set)) + (clr-settings-resolver! (: System.Xml.XmlReaderSettings.XmlResolver (-> XmlReaderSettings XmlResolver void) #:set)) + (clr-settings-entity-characters! (: System.Xml.XmlReaderSettings.MaxCharactersFromEntities + (-> XmlReaderSettings long void) #:set)) + (clr-settings-document-characters! (: System.Xml.XmlReaderSettings.MaxCharactersInDocument + (-> XmlReaderSettings long void) #:set)) + (clr-settings-close-input! (: System.Xml.XmlReaderSettings.CloseInput (-> XmlReaderSettings bool void) #:set)) + (clr-settings-ignore-comments! (: System.Xml.XmlReaderSettings.IgnoreComments + (-> XmlReaderSettings bool void) #:set)) + (clr-throwing-resolver (: System.Xml.XmlResolver.ThrowingResolver #:get)) + (clr-dtd-prohibit (: System.Xml.DtdProcessing.Prohibit #:get)) + (clr-dtd-ignore (: System.Xml.DtdProcessing.Ignore #:get)) + (clr-dtd-parse (: System.Xml.DtdProcessing.Parse #:get)) + (clr-read (: System.Xml.XmlReader.Read (-> XmlReader bool))) + (clr-skip (: System.Xml.XmlReader.Skip (-> XmlReader void))) + (clr-node-type (: System.Xml.XmlReader.NodeType (-> XmlReader XmlNodeType) #:get)) + (clr-local-name (: System.Xml.XmlReader.LocalName (-> XmlReader string) #:get)) + (clr-namespace (: System.Xml.XmlReader.NamespaceURI (-> XmlReader string) #:get)) + (clr-name (: System.Xml.XmlReader.Name (-> XmlReader string) #:get)) + (clr-value (: System.Xml.XmlReader.Value (-> XmlReader string) #:get)) + (clr-depth (: System.Xml.XmlReader.Depth (-> XmlReader int) #:get)) + (clr-empty-element? (: System.Xml.XmlReader.IsEmptyElement (-> XmlReader bool) #:get)) + (clr-next-attribute (: System.Xml.XmlReader.MoveToNextAttribute (-> XmlReader bool))) + (clr-to-element (: System.Xml.XmlReader.MoveToElement (-> XmlReader bool))) + ;; Kept a .NET string, whose null says the attribute is absent. + (clr-attribute (: System.Xml.XmlReader.GetAttribute (-> XmlReader string System.String))) + (clr-has-line-info (: System.Xml.IXmlLineInfo.HasLineInfo (-> IXmlLineInfo bool))) + (clr-line-number (: System.Xml.IXmlLineInfo.LineNumber (-> IXmlLineInfo int) #:get)) + (clr-line-position (: System.Xml.IXmlLineInfo.LinePosition (-> IXmlLineInfo int) #:get)) + (clr-exception-line (: System.Xml.XmlException.LineNumber (-> XmlException int) #:get)) + (clr-exception-column (: System.Xml.XmlException.LinePosition (-> XmlException int) #:get)) + (clr-inner-exception (: System.Exception.InnerException (-> Exception Exception) #:get)) + (clr-port-stream (: Bjolang.Runtime.BjoInputPort.AsStream (-> InputPort Stream)))) + +;; Every export is documented there; the module doc is too. +(:doc #:file "read.bjodoc") + +;; =========================================================================== +;; Options +;; =========================================================================== + +;; 'prohibit refuses a DOCTYPE. 'ignore skips it, and XmlReader then does not +;; report it at all, so the document has no doctype. 'parse reads it, never +;; resolving anything external, and expands at most 10 000 000 characters of +;; entities. +(def/grammar DtdMode + (:or (: DtdProhibit 'prohibit) (: DtdIgnore 'ignore) (: DtdParse 'parse))) + +;; 'drop leaves out text that is only whitespace, which includes the space in +;; x y. +(def/grammar WhitespaceMode + (:or (: KeepWhitespace 'keep) (: DropWhitespace 'drop))) + +(type (: Options (Record (: dtd DtdMode) + (: max-depth int) + (: max-characters long) + (: drop-whitespace bool) + (: comments bool)))) + +(: long) +(def entity-characters 10000000L) + +(: (-> DtdMode int long WhitespaceMode bool Options)) +(defun (options dtd max-depth max-characters whitespace comments) + (when (< max-depth 1) + (raise (cast Exception (ArgumentException. "(text xml read): #:max-depth has to be at least 1")))) + (when (< max-characters 0L) + (raise (cast Exception (ArgumentException. "(text xml read): #:max-characters cannot be negative")))) + (Options (dtd dtd) + (max-depth max-depth) + (max-characters max-characters) + (drop-whitespace (match whitespace (KeepWhitespace #f) (DropWhitespace #t))) + (comments comments))) + +(: (-> Options XmlReaderSettings)) +(defun (reader-settings opts) + (def settings (XmlReaderSettings.)) + (clr-settings-close-input! settings #f) + (clr-settings-ignore-comments! settings (not (record-ref opts comments))) + (clr-settings-document-characters! settings (record-ref opts max-characters)) + (clr-settings-resolver! settings clr-throwing-resolver) + (match (record-ref opts dtd) + (DtdProhibit (clr-settings-dtd! settings clr-dtd-prohibit)) + (DtdIgnore (clr-settings-dtd! settings clr-dtd-ignore)) + (DtdParse + (clr-settings-dtd! settings clr-dtd-parse) + (clr-settings-entity-characters! settings entity-characters))) + settings) + +;; =========================================================================== +;; Errors +;; =========================================================================== + +(def/grammar XmlErrorKind + (:or (: Malformed 'malformed) + (: DtdProhibited 'dtd-prohibited) + (: TooDeep 'too-deep) + (: LimitExceeded 'limit-exceeded))) + +;; line and column are 0 when neither the exception nor the reader knew them. +(type (: XmlError (Record (: kind XmlErrorKind) (: message string) (: line int) (: column int)))) + +(: (-> XmlErrorKind string)) +(defun (kind-name k) + (match k + (Malformed "malformed") + (DtdProhibited "dtd-prohibited") + (TooDeep "too-deep") + (LimitExceeded "limit-exceeded"))) + +(impl (Displayed XmlError) + (defun (displayed e) + (str (kind-name (record-ref e kind)) + " at line " (int->string (record-ref e line)) + ", column " (int->string (record-ref e column)) + ": " (record-ref e message))) + (defun (display-into e b) (stringbuilder-add-string! b (displayed e)))) + +(: (-> XmlErrorKind string (Option XmlPos) XmlError)) +(defun (xml-error kind message pos) + (match pos + ((Some at) (XmlError (kind kind) (message message) (line (struct-ref at line)) (column (struct-ref at column)))) + (None (XmlError (kind kind) (message message) (line 0) (column 0))))) + +;; The message .NET gives for `text` read with `settings`, or "" if it reads. +(: (-> string XmlReaderSettings string)) +(defun (reference-message text settings) + (match (try (with-open ((r (clr-reader-over-text (cast TextReader (open-input-string text)) settings))) + (let loop () (when (clr-read r) (loop)))) + #:catch (System.Xml.XmlException)) + ((Ok _) "") + ((Err e) (.-Message e)))) + +;; A DOCTYPE, a document longer than its limit, and entities longer than +;; theirs are XmlExceptions like any other, told apart only by their message +;; and thrown with no position. So their messages are compared with the ones +;; .NET gives for three tiny documents, made to fail the same way. Exactly, and +;; not by searching for a word: a malformed document can quote any word back. +(: (-> string)) +(defun (dtd-prohibited-message) + (def settings (XmlReaderSettings.)) + (clr-settings-dtd! settings clr-dtd-prohibit) + (reference-message "" settings)) + +(: (-> string)) +(defun (document-limit-message) + (def settings (XmlReaderSettings.)) + (clr-settings-document-characters! settings 2L) + (reference-message "0123456789" settings)) + +(: (-> string)) +(defun (entity-limit-message) + (def settings (XmlReaderSettings.)) + (clr-settings-dtd! settings clr-dtd-parse) + (clr-settings-resolver! settings clr-throwing-resolver) + (clr-settings-entity-characters! settings 2L) + (reference-message "]>&e;" settings)) + +;; ThrowingResolver's refusal arrives wrapped in the reader's own exception. +(: (-> XmlException bool)) +(defun (refused-resource? e) + (match (cast Object (clr-inner-exception (cast Exception e))) + ((:is System.Xml.XmlException _) #t) + (_ #f))) + +(: (-> DtdMode XmlException XmlErrorKind)) +(defun (exception-kind dtd e) + (def message (.-Message e)) + (cond + ((= message (dtd-prohibited-message)) DtdProhibited) + ((= message (document-limit-message)) LimitExceeded) + ((= message (entity-limit-message)) LimitExceeded) + ((and (match dtd (DtdParse #t) (_ #f)) (refused-resource? e)) DtdProhibited) + (else Malformed))) + +(: (-> (Option IXmlLineInfo) (Option XmlPos))) +(defun (line-info-pos info) + (match info + ((Some li) + (if (clr-has-line-info li) + (Some (XmlPos (line (clr-line-number li)) (column (clr-line-position li)))) + None)) + (None None))) + +;; Where the exception says. A DOCTYPE refused, a limit reached and a resource +;; refused are thrown with no position, and by then the reader has reset its +;; own to 0 as well, so those have none. +(: (-> DtdMode XmlException XmlError)) +(defun (exception-error dtd e) + (def line (clr-exception-line e)) + (xml-error (exception-kind dtd e) + (.-Message e) + (if (> line 0) + (Some (XmlPos (line line) (column (clr-exception-column e)))) + None))) + +;; =========================================================================== +;; Sources +;; =========================================================================== + +;; Declared before the trait that names it. +(type (: XmlInput #:opaque (Union (: InText string) + (: InStream Stream) + (: InFile string)))) + +;; What can be read. A type of your own gets an implementation by handing +;; xml-input something that already has one. +(def/trait (XmlSource %s) + (: xml-input (-> %s XmlInput))) + +;; A path, which the reader opens and closes. A string on its own is XML. +(type (: XmlFile #:opaque (Record (: path string)))) + +(: (-> string XmlFile)) +(defun (xml-file path) (XmlFile (path path))) + +(impl (XmlSource string) + (defun (xml-input s) (InText s))) + +;; Bytes, whose encoding XmlReader works out: a byte order mark, or the +;; encoding the XML declaration names, or UTF-8. +(impl (XmlSource Stream) + (defun (xml-input s) (InStream s))) + +;; A port is bytes too, read the same way. Through the port's own stream view, +;; so bytes it has already buffered are not lost. +(impl (XmlSource InputPort) + (defun (xml-input p) (InStream (clr-port-stream p)))) + +(impl (XmlSource XmlFile) + (defun (xml-input f) (InFile (record-ref f path)))) + +;; =========================================================================== +;; 1. Signals +;; =========================================================================== + +(type (: XmlSignal (Union (: XmlStart XmlName XmlAttrs (Option XmlPos)) + XmlEnd + (: XmlData string) + (: XmlComment string) + (: XmlPI string string) + (: XmlDoctype Doctype)))) + +;; `held` means the reader stands on a node that has not been reported yet, +;; because merging text had to read one node past it. `owed-end` is the end of +;; an empty element, which XmlReader never reports. +(type (: XmlSignals #:opaque (Record (: reader (Option XmlReader)) + (: line-info (Option IXmlLineInfo)) + (: options Options) + (: held bool #:mutable) + (: owed-end bool #:mutable) + (: after-start bool #:mutable) + (: finished bool #:mutable) + (: failed (Option XmlError) #:mutable)))) + +(: (-> (Option XmlReader) Options (Option XmlError) XmlSignals)) +(defun (signals reader opts failed) + (XmlSignals (reader reader) + (line-info (match reader + ((Some r) (match (cast Object r) + ((:is System.Xml.IXmlLineInfo li) (Some li)) + (_ None))) + (None None))) + (options opts) + (held #f) + (owed-end #f) + (after-start #f) + (finished #f) + (failed failed))) + +;; XmlNodeType, as numbers. +(: int) +(def element-node 1) +(: int) +(def text-node 3) +(: int) +(def cdata-node 4) +(: int) +(def pi-node 7) +(: int) +(def comment-node 8) +(: int) +(def doctype-node 10) +(: int) +(def whitespace-node 13) +(: int) +(def significant-whitespace-node 14) +(: int) +(def end-element-node 15) + +(: (-> XmlReader int)) +(defun (node-code r) (cast int (clr-node-type r))) + +(: (-> int bool)) +(defun (text-code? code) + (or (= code text-node) (= code cdata-node) (= code whitespace-node) (= code significant-whitespace-node))) + +(: string) +(def xmlns-namespace "http://www.w3.org/2000/xmlns/") + +;; Space, tab, line feed and carriage return: XML's whitespace, and no other. +(: (-> string bool)) +(defun (whitespace-only? s) + (string-fold (fun (c all) (and all (let ((n (char->int c))) (or (= n 32) (= n 9) (= n 10) (= n 13))))) + #t + s)) + +(: (-> XmlReader string (Option string))) +(defun (attribute-opt r name) + (match (cast Object (clr-attribute r name)) + ((:is System.String s) (Some (clr-string->string s))) + (_ None))) + +;; In document order, without the namespace declarations, and back on the +;; element afterwards. +(: (-> XmlReader XmlAttrs)) +(defun (read-attrs r) + (def out (listbuilder-empty)) + (let loop () + (when (clr-next-attribute r) + (def ns (clr-namespace r)) + (unless (= ns xmlns-namespace) + (listbuilder-add! out (Tuple (xml-name (clr-local-name r) #:ns ns) (clr-value r)))) + (loop))) + (ignore (clr-to-element r)) + (listbuilder->list out)) + +(: (-> XmlSignals XmlReader (Result XmlError (Option XmlSignal)))) +(defun (start-element s r) + (def pos (line-info-pos (record-ref s line-info))) + (def max-depth (record-ref (record-ref s options) max-depth)) + (cond + ((>= (clr-depth r) max-depth) + (Err (xml-error TooDeep + (str "the elements nest deeper than #:max-depth, " (int->string max-depth)) + pos))) + (else + ;; Before the attributes: on an attribute, IsEmptyElement is false. + (def empty (clr-empty-element? r)) + (def name (xml-name (clr-local-name r) #:ns (clr-namespace r))) + (def attrs (read-attrs r)) + (record-set! s (owed-end empty) (after-start #t)) + (Ok (Some (XmlStart name attrs pos)))))) + +;; Text, CDATA and whitespace, from the current node to the next node that is +;; none of them, as one string. The reader is left on that node. +(: (-> XmlSignals XmlReader string)) +(defun (read-data s r) + (def first (clr-value r)) + (cond + ((not (clr-read r)) (record-set! s (finished #t)) first) + ((not (text-code? (node-code r))) (record-set! s (held #t)) first) + (else + (def sb (stringbuilder-empty)) + (stringbuilder-add-string! sb first) + (let loop () + (stringbuilder-add-string! sb (clr-value r)) + (cond + ((not (clr-read r)) (record-set! s (finished #t))) + ((text-code? (node-code r)) (loop)) + (else (record-set! s (held #t))))) + (stringbuilder->string sb)))) + +(: (-> XmlSignals XmlReader (Result XmlError (Option XmlSignal)))) +(defun (read-signal s r) + (let loop () + (cond + ((and (not (record-ref s held)) (not (clr-read r))) + (record-set! s (finished #t)) + (Ok None)) + (:do (record-set! s (held #f))) + (:def code (node-code r)) + ((= code element-node) (start-element s r)) + ((= code end-element-node) (Ok (Some XmlEnd))) + ;; Outside the root there is only whitespace, and it is not data. + ((and (text-code? code) (= (clr-depth r) 0)) (loop)) + ((text-code? code) + (def data (read-data s r)) + (if (and (record-ref (record-ref s options) drop-whitespace) (whitespace-only? data)) + (loop) + (Ok (Some (XmlData data))))) + ((= code comment-node) (Ok (Some (XmlComment (clr-value r))))) + ((= code pi-node) (Ok (Some (XmlPI (clr-name r) (clr-value r))))) + ((= code doctype-node) + (Ok (Some (XmlDoctype (Doctype (name (clr-name r)) + (public-id (attribute-opt r "PUBLIC")) + (system-id (attribute-opt r "SYSTEM"))))))) + ;; The XML declaration. + (else (loop))))) + +(: (-> XmlSignals XmlError (Result XmlError %a))) +(defun (fail s e) + (record-set! s (failed (Some e))) + (Err e)) + +(: (-> XmlSignals Exception XmlError)) +(defun (caught s e) + (match (cast Object e) + ((:is System.Xml.XmlException x) + (exception-error (record-ref (record-ref s options) dtd) x)) + (_ (raise e)))) + +;; The next signal, or None at the end of the document. After an error, the +;; same error again. +(: (-> XmlSignals (Result XmlError (Option XmlSignal)))) +(defun (xml-next! s) + (cond + (:match (Some e) (record-ref s failed) (Err e)) + ((record-ref s owed-end) + (record-set! s (owed-end #f) (after-start #f)) + (Ok (Some XmlEnd))) + (:do (record-set! s (after-start #f))) + ((record-ref s finished) (Ok None)) + (:match (Some r) (record-ref s reader) + (match (try (read-signal s r) #:catch (System.Xml.XmlException)) + ((Ok (Err e)) (fail s e)) + ((Ok signal) signal) + ((Err e) (fail s (caught s e))))) + (else (Ok None)))) + +;; Right after an XmlStart: skips the rest of that element, its XmlEnd +;; included. Anywhere else it raises. The skipped part is not checked against +;; #:max-depth. +(: (-> XmlSignals (Result XmlError Unit))) +(defun (xml-skip-element! s) + (cond + (:match (Some e) (record-ref s failed) (Err e)) + ((not (record-ref s after-start)) + (raise (cast Exception (InvalidOperationException. + "(text xml read): xml-skip-element! is for right after an XmlStart")))) + (:do (record-set! s (after-start #f))) + ((record-ref s owed-end) (record-set! s (owed-end #f)) (Ok unit)) + (:match (Some r) (record-ref s reader) + (match (try (let () + (clr-skip r) + (if (= (node-code r) 0) + (record-set! s (finished #t)) + (record-set! s (held #t)))) + #:catch (System.Xml.XmlException)) + ((Ok _) (Ok unit)) + ((Err e) (fail s (caught s e))))) + (else (Ok unit)))) + +;; XmlReader.Create reads ahead, so it can fail too. Then the first xml-next! +;; answers with the error. +(: (-> (-> XmlReaderSettings XmlReader) Options (-> XmlSignals %a) %a)) +(defun (run-signals create opts proc) + (def settings (reader-settings opts)) + (def s (match (try (create settings) #:catch (System.Xml.XmlException)) + ((Ok r) (signals (Some r) opts None)) + ((Err e) (match (cast Object e) + ((:is System.Xml.XmlException x) + (signals None opts (Some (exception-error (record-ref opts dtd) x)))) + (_ (raise e)))))) + (try (proc s) + #:finally (match (record-ref s reader) + ((Some r) (.Dispose r)) + (None unit)))) + +;; A port or stream that was handed over stays open. A file is opened here, +;; through the FS effect, and closed here. +(: (-> XmlInput Options (-> XmlSignals %a) %a)) +(defun (with-signals input opts proc) + (match input + ((InText text) + (def port (open-input-string text)) + (run-signals (fun (settings) (clr-reader-over-text (cast TextReader port) settings)) opts proc)) + ((InStream stream) + (run-signals (fun (settings) (clr-reader-over-stream stream settings)) opts proc)) + ((InFile path) + (match (open-read-stream path) + ((Ok opened) + (with-open ((stream opened)) + (run-signals (fun (settings) (clr-reader-over-stream stream settings)) opts proc))) + ((Err e) (raise e)))))) + +;; The XmlSignals is for proc's use, and closed when proc returns. +(: (-> %s (-> XmlSignals %a) + (#:dtd DtdMode) (#:max-depth int) (#:max-characters long) + (#:whitespace WhitespaceMode) (#:comments? bool) + %a) + (where (XmlSource %s))) +(defun (call-with-xml-signals source proc + #:dtd 'prohibit #:max-depth 512 #:max-characters 0L + #:whitespace 'keep #:comments? #f) + (with-signals (xml-input source) (options dtd max-depth max-characters whitespace comments?) proc)) + +;; =========================================================================== +;; 2. The SSAX fold +;; =========================================================================== + +;; An element still open: what finish is called with when it closes. +(type (: (Frame %seed) (Record (: name XmlName) + (: attrs XmlAttrs) + (: pos (Option XmlPos)) + (: parent %seed)))) + +;; One iteration per signal, as a loop rather than a call: a flat document of a +;; million elements is a million iterations. The open elements are the list. +(: (-> XmlSignals %seed + (-> XmlName XmlAttrs (Option XmlPos) %seed %seed) + (-> XmlName XmlAttrs (Option XmlPos) %seed %seed %seed) + (-> string %seed %seed) + (-> string %seed %seed) + (-> string string %seed %seed) + (-> Doctype %seed %seed) + (Result XmlError %seed))) +(defun (fold-signals sig seed new-level finish text comment pi doctype) + (let loop ((seed seed) (open Nil)) + (match (xml-next! sig) + ((Err e) (Err e)) + ((Ok None) (Ok seed)) + ((Ok (Some (XmlStart name attrs pos))) + (loop (new-level name attrs pos seed) + (Cons (Frame (name name) (attrs attrs) (pos pos) (parent seed)) open))) + ((Ok (Some XmlEnd)) + (match open + ((Cons f rest) + (loop (finish (record-ref f name) (record-ref f attrs) (record-ref f pos) (record-ref f parent) seed) + rest)) + (Nil (Ok seed)))) + ((Ok (Some (XmlData s))) (loop (text s seed) open)) + ((Ok (Some (XmlComment s))) (loop (comment s seed) open)) + ((Ok (Some (XmlPI target data))) (loop (pi target data seed) open)) + ((Ok (Some (XmlDoctype dt))) (loop (doctype dt seed) open))))) + +;; Kiselyov's SSAX: the seed is threaded through the document. Descending into +;; an element, new-level turns the seed into the element's own; leaving it, +;; finish gets the seed from before the element and the element's final seed, +;; and answers the seed to carry on with. Every handler passes the seed +;; through unless given. +(: (-> %s %seed + (#:new-level (-> XmlName XmlAttrs (Option XmlPos) %seed %seed)) + (#:finish (-> XmlName XmlAttrs (Option XmlPos) %seed %seed %seed)) + (#:text (-> string %seed %seed)) + (#:comment (-> string %seed %seed)) + (#:pi (-> string string %seed %seed)) + (#:doctype (-> Doctype %seed %seed)) + (#:dtd DtdMode) (#:max-depth int) (#:max-characters long) + (#:whitespace WhitespaceMode) + (Result XmlError %seed)) + (where (XmlSource %s))) +(defun (xml-fold source seed + #:new-level (fun (name attrs pos seed) seed) + #:finish (fun (name attrs pos parent seed) seed) + #:text (fun (s seed) seed) + #:comment (fun (s seed) seed) + #:pi (fun (target data seed) seed) + #:doctype (fun (dt seed) seed) + #:dtd 'prohibit #:max-depth 512 #:max-characters 0L #:whitespace 'keep) + (with-signals (xml-input source) (options dtd max-depth max-characters whitespace #t) + (fun (sig) (fold-signals sig seed new-level finish text comment pi doctype)))) + +;; =========================================================================== +;; 3. The tree fold +;; =========================================================================== + +;; The seed is the children so far, last first, and whether this is the level +;; outside the root, where only the root is kept. +(: (-> (Option %a) (Tuple (List %a) bool) (Tuple (List %a) bool))) +(defun (keep-child value seed) + (def (Tuple kids top) seed) + (match value + ((Some a) #:when (not top) (Tuple (Cons a kids) top)) + (_ seed))) + +(: (-> XmlInput + (-> XmlName XmlAttrs (Option XmlPos) (List %a) %a) + (-> string %a) + (-> string (Option %a)) + (-> string string (Option %a)) + Options + (Result XmlError %a))) +(defun (tree-fold input element text comment pi opts) + (def folded + (with-signals input opts + (fun (sig) + (fold-signals sig (Tuple Nil #t) + (fun (name attrs pos seed) (Tuple Nil #f)) + (fun (name attrs pos parent seed) + (def (Tuple kids _) seed) + (def (Tuple siblings top) parent) + (Tuple (Cons (element name attrs pos (list-reverse kids)) siblings) top)) + (fun (s seed) + (def (Tuple kids top) seed) + (Tuple (Cons (text s) kids) top)) + (fun (s seed) (keep-child (comment s) seed)) + (fun (target data seed) (keep-child (pi target data) seed)) + (fun (dt seed) seed))))) + (match folded + ((Err e) (Err e)) + ((Ok (Tuple (Cons root _) _)) (Ok root)) + ((Ok _) (Err (xml-error Malformed "the document has no root element" None))))) + +;; A tree of the caller's own type, as OCaml's Xmlm builds with input_tree: +;; element gets an element's name, attributes, position and folded children, +;; and text a piece of text. A comment or PI handler answers None to leave the +;; node out, which is what the defaults do. The answer is the root element's; +;; what is outside the root is not folded. +(: (-> %s + (-> XmlName XmlAttrs (Option XmlPos) (List %a) %a) + (-> string %a) + (#:comment (-> string (Option %a))) + (#:pi (-> string string (Option %a))) + (#:dtd DtdMode) (#:max-depth int) (#:max-characters long) + (#:whitespace WhitespaceMode) + (Result XmlError %a)) + (where (XmlSource %s))) +(defun (xml-tree-fold source element text + #:comment (fun (s) None) + #:pi (fun (target data) None) + #:dtd 'prohibit #:max-depth 512 #:max-characters 0L #:whitespace 'keep) + (tree-fold (xml-input source) element text comment pi + (options dtd max-depth max-characters whitespace #t))) + +;; =========================================================================== +;; 4. The whole document +;; =========================================================================== + +;; Outside the root: what the Document is made of, last first. +(type (: DocumentParts (Record (: doctype (Option Doctype)) + (: prolog (List Node)) + (: root (Option Node)) + (: epilog (List Node))))) + +(type (: DocumentSeed (Union (: AtTop DocumentParts) + (: InElement (List Node))))) + +(: (-> Node DocumentSeed DocumentSeed)) +(defun (add-node n seed) + (match seed + ((InElement kids) (InElement (Cons n kids))) + ((AtTop parts) + (match (record-ref parts root) + (None (AtTop (record-set parts (prolog (Cons n (record-ref parts prolog)))))) + ((Some _) (AtTop (record-set parts (epilog (Cons n (record-ref parts epilog)))))))))) + +(: (-> XmlName XmlAttrs (Option XmlPos) DocumentSeed DocumentSeed DocumentSeed)) +(defun (finish-element name attrs pos parent seed) + (def kids (match seed ((InElement kids) (list-reverse kids)) ((AtTop _) Nil))) + (def element (xml-element name attrs kids #:pos pos)) + (match parent + ((InElement siblings) (InElement (Cons element siblings))) + ((AtTop parts) (AtTop (record-set parts (root (Some element))))))) + +(: (-> DocumentSeed (Result XmlError Document))) +(defun (document-of seed) + (match seed + ((AtTop (DocumentParts doctype prolog (root (Some root)) epilog)) + (Ok (xml-document root + #:doctype doctype + #:prolog (list-reverse prolog) + #:epilog (list-reverse epilog)))) + (_ (Err (xml-error Malformed "the document has no root element" None))))) + +(: (-> XmlInput Options (Result XmlError Document))) +(defun (parse-document input opts) + (def start (AtTop (DocumentParts (doctype None) (prolog Nil) (root None) (epilog Nil)))) + (def folded + (with-signals input opts + (fun (sig) + (fold-signals sig start + (fun (name attrs pos seed) (InElement Nil)) + finish-element + (fun (s seed) (add-node (xml-text s) seed)) + (fun (s seed) (add-node (xml-comment s) seed)) + (fun (target data seed) (add-node (xml-pi target data) seed)) + (fun (dt seed) + (match seed + ((AtTop parts) (AtTop (record-set parts (doctype (Some dt))))) + (_ seed))))))) + (match folded + ((Err e) (Err e)) + ((Ok seed) (document-of seed)))) + +;; Processing instructions are always kept, and comments when #:comments? +;; says so, in the prolog and epilog as in the tree. +(: (-> %s + (#:dtd DtdMode) (#:max-depth int) (#:max-characters long) + (#:whitespace WhitespaceMode) (#:comments? bool) + (Result XmlError Document)) + (where (XmlSource %s))) +(defun (parse-xml source + #:dtd 'prohibit #:max-depth 512 #:max-characters 0L + #:whitespace 'keep #:comments? #f) + (parse-document (xml-input source) (options dtd max-depth max-characters whitespace comments?))) + +;; =========================================================================== +;; 5. One element at a time +;; =========================================================================== + +;; An element still open while one is being built, and its children so far, +;; last first. +(type (: Building (Record (: name XmlName) + (: attrs XmlAttrs) + (: pos (Option XmlPos)) + (: kids (List Node))))) + +(: (-> Node (List Building) (List Building))) +(defun (add-child n open) + (match open + ((Cons top rest) (Cons (record-set top (kids (Cons n (record-ref top kids)))) rest)) + (Nil Nil))) + +;; The rest of the element whose XmlStart was just read, as a Node. The open +;; elements are the list, so this is a loop however deep the element goes. +(: (-> XmlSignals XmlName XmlAttrs (Option XmlPos) (Result XmlError Node))) +(defun (build-element sig name attrs pos) + (let loop ((open (list (Building (name name) (attrs attrs) (pos pos) (kids Nil))))) + (match (xml-next! sig) + ((Err e) (Err e)) + ((Ok None) (Err (xml-error Malformed "the document ended inside an element" None))) + ((Ok (Some (XmlStart n a p))) + (loop (Cons (Building (name n) (attrs a) (pos p) (kids Nil)) open))) + ((Ok (Some XmlEnd)) + (match open + ((Cons top rest) + (def element (xml-element (record-ref top name) (record-ref top attrs) + (list-reverse (record-ref top kids)) + #:pos (record-ref top pos))) + (match rest + (Nil (Ok element)) + (_ (loop (add-child element rest))))) + (Nil (Err (xml-error Malformed "an end with no start" None))))) + ((Ok (Some (XmlData s))) (loop (add-child (xml-text s) open))) + ((Ok (Some (XmlComment s))) (loop (add-child (xml-comment s) open))) + ((Ok (Some (XmlPI target data))) (loop (add-child (xml-pi target data) open))) + ((Ok (Some (XmlDoctype _))) (loop open))))) + +(: (-> XmlSignals XmlName (-> Node void) (Result XmlError Unit))) +(defun (for-each-element sig name f) + (let loop () + (match (xml-next! sig) + ((Err e) (Err e)) + ((Ok None) (Ok unit)) + ((Ok (Some (XmlStart n attrs pos))) + (cond + ((not (= n name)) (loop)) + (:def (Ok element) (build-element sig n attrs pos) :propagate) + (else + (f element) + (loop)))) + ((Ok (Some _)) (loop))))) + +;; Burst mode: a Node for each outermost element with this name, handed to f +;; and then dropped, so what is held is one such element and never the whole +;; document. An element with this name inside one is part of it, and not +;; visited on its own. +(: (-> %s XmlName (-> Node void) + (#:dtd DtdMode) (#:max-depth int) (#:max-characters long) + (#:whitespace WhitespaceMode) (#:comments? bool) + (Result XmlError Unit)) + (where (XmlSource %s))) +(defun (xml-for-each source name f + #:dtd 'prohibit #:max-depth 512 #:max-characters 0L + #:whitespace 'keep #:comments? #f) + (with-signals (xml-input source) (options dtd max-depth max-characters whitespace comments?) + (fun (sig) (for-each-element sig name f)))) +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; As a special exception to the Mozilla Public License, version 2.0, if you +;; compile your application source code and portions of this software are +;; embedded into the generated object code or executable form as a normal +;; consequence of the compilation process (such as inline functions, +;; templates, generics, or macros), you may redistribute such embedded portions +;; in such object code or executable form without complying with the source code +;; availability requirements or notice obligations of Section 3 of the MPL 2.0. + +;; text/xml/write.bjo — the module (text xml write). +;; +;; System.Xml.XmlWriter does the escaping. What it does not do is refuse all +;; content it cannot write: a comment with "--" in it, PI data with "?>" in it +;; and a PI named xml are silently changed or dropped rather than refused, so +;; those are checked here first and are an XmlWriteError like the rest. +;; +;; A name carries no prefix, so the prefixes are chosen here: one already in +;; scope for the namespace, then one the caller hinted at, then a default +;; namespace declaration for an element, or a generated ns1, ns2 ... for an +;; attribute, which the default namespace does not apply to. XmlWriter writes +;; the declarations the chosen prefixes need. +;; +;; The tree is walked with a stack of its own, and indentation is written by +;; hand: see `indents?`. +;; +;; Its reference is its docs, in write.bjodoc. +;; +;; Open: +;; - The doctype is written with WriteRaw, after checking its parts, because +;; WriteDocType takes a null for an id that is not there and there is no +;; null to pass it. +;; - A doctype's internal subset is not kept by the reader, so it is not +;; written back. Its entities were expanded while reading, but an attribute +;; default it declared is then an ordinary attribute of every element it +;; applied to. +;; - What was written to a port before an error stays written, and nothing +;; says how far it got. +;; - #:prefixes may give one prefix twice. The first one is used, and the +;; namespace of the second gets a generated prefix where the two meet. + +(export + XmlWriteError + node->xml node->xml-string + document->xml document->xml-string) + +(import (std prelude)) +(import (text xml)) + +(import/class + (XmlWriter (: System.Xml.XmlWriter)) + ;; What `XmlWriter.Create` takes, and what an output port also is. + (TextWriter (: System.IO.TextWriter)) + (XmlWriterSettings (: System.Xml.XmlWriterSettings (-> XmlWriterSettings))) + (ConformanceLevel (: System.Xml.ConformanceLevel)) + (ClrBuilder (: System.Text.StringBuilder (-> ClrBuilder))) + (ArgumentException (: System.ArgumentException (-> string ArgumentException)))) + +(import/extern + (clr-writer-over-text (: System.Xml.XmlWriter.Create (-> TextWriter XmlWriterSettings XmlWriter))) + (clr-writer-over-builder (: System.Xml.XmlWriter.Create (-> ClrBuilder XmlWriterSettings XmlWriter))) + (clr-settings-omit-declaration! (: System.Xml.XmlWriterSettings.OmitXmlDeclaration + (-> XmlWriterSettings bool void) #:set)) + (clr-settings-conformance! (: System.Xml.XmlWriterSettings.ConformanceLevel + (-> XmlWriterSettings ConformanceLevel void) #:set)) + (clr-settings-close-output! (: System.Xml.XmlWriterSettings.CloseOutput (-> XmlWriterSettings bool void) #:set)) + (clr-settings-end-on-close! (: System.Xml.XmlWriterSettings.WriteEndDocumentOnClose + (-> XmlWriterSettings bool void) #:set)) + (clr-conformance-document (: System.Xml.ConformanceLevel.Document #:get)) + (clr-conformance-fragment (: System.Xml.ConformanceLevel.Fragment #:get)) + (clr-start-element (: System.Xml.XmlWriter.WriteStartElement (-> XmlWriter string string string void))) + (clr-end-element (: System.Xml.XmlWriter.WriteEndElement (-> XmlWriter void))) + (clr-attribute (: System.Xml.XmlWriter.WriteAttributeString (-> XmlWriter string string string string void))) + (clr-string (: System.Xml.XmlWriter.WriteString (-> XmlWriter string void))) + (clr-comment (: System.Xml.XmlWriter.WriteComment (-> XmlWriter string void))) + (clr-pi (: System.Xml.XmlWriter.WriteProcessingInstruction (-> XmlWriter string string void))) + (clr-whitespace (: System.Xml.XmlWriter.WriteWhitespace (-> XmlWriter string void))) + (clr-raw (: System.Xml.XmlWriter.WriteRaw (-> XmlWriter string void))) + (clr-flush (: System.Xml.XmlWriter.Flush (-> XmlWriter void))) + (clr-verify-ncname (: System.Xml.XmlConvert.VerifyNCName (-> string string) + #:exceptions (System.Xml.XmlException System.ArgumentNullException))) + (clr-verify-name (: System.Xml.XmlConvert.VerifyName (-> string string) + #:exceptions (System.Xml.XmlException System.ArgumentNullException))) + (clr-verify-public-id (: System.Xml.XmlConvert.VerifyPublicId (-> string string) + #:exceptions (System.Xml.XmlException System.ArgumentNullException))) + (clr-verify-xml-chars (: System.Xml.XmlConvert.VerifyXmlChars (-> string string) + #:exceptions (System.Xml.XmlException System.ArgumentNullException)))) + +;; Every export is documented there; the module doc is too. +(:doc #:file "write.bjodoc") + +;; =========================================================================== +;; Errors +;; =========================================================================== + +(type (: XmlWriteError (Record (: message string)))) + +(impl (Displayed XmlWriteError) + (defun (displayed e) (record-ref e message)) + (defun (display-into e b) (stringbuilder-add-string! b (record-ref e message)))) + +(: (-> string (Result XmlWriteError %a))) +(defun (write-error message) + (Err (XmlWriteError (message message)))) + +(: (Result XmlWriteError Unit)) +(def done (Ok unit)) + +;; =========================================================================== +;; Content XmlWriter would change rather than refuse +;; =========================================================================== + +(: (-> string (Option string))) +(defun (comment-problem text) + (cond + ((string-contains? text "--") (Some "a comment may not contain \"--\"")) + ((string-ends-with? text "-") (Some "a comment may not end in \"-\"")) + (else None))) + +(: (-> string string (Option string))) +(defun (pi-problem target data) + (cond + ((err? (clr-verify-ncname target)) + (Some (str "\"" target "\" is not a processing instruction target"))) + ((= (string-downcase target) "xml") + (Some "a processing instruction may not be named xml; the XML declaration is #:declaration?")) + ((string-contains? data "?>") (Some "a processing instruction's data may not contain \"?>\"")) + (else None))) + +;; =========================================================================== +;; Namespaces +;; =========================================================================== + +(: string) +(def xmlns-namespace "http://www.w3.org/2000/xmlns/") + +;; The declarations in scope, innermost first, as (prefix . uri): one list per +;; open element that declared anything, so that looking a prefix up costs the +;; number of declaring elements around it rather than the depth. The default +;; namespace is the prefix "". +(type (: Scope (Alias (List (List (Tuple string string)))))) + +;; Prefix to URI, as #:prefixes gives them. +(type (: Hints (Alias (List (Tuple string string))))) + +(: (-> (List (Tuple string string)) string (Option string))) +(defun (bound-in frame prefix) + (match frame + (Nil None) + ((Cons (Tuple p u) rest) (if (= p prefix) (Some u) (bound-in rest prefix))))) + +;; What a prefix means here. "xml" always means the xml namespace, and the +;; default namespace is none until something declares one. +(: (-> Scope string (Option string))) +(defun (scope-uri scope prefix) + (if (= prefix "xml") + (Some xml-namespace) + (let loop ((frames scope)) + (match frames + (Nil (if (= prefix "") (Some "") None)) + ((Cons frame rest) + (match (bound-in frame prefix) + ((Some u) (Some u)) + (None (loop rest)))))))) + +;; A prefix that means `uri` here, and is not hidden by a nearer declaration of +;; the same prefix. The default namespace only when `default-ok`. +(: (-> Scope string bool (Option string))) +(defun (scope-prefix scope uri default-ok) + (let frames ((fs scope)) + (match fs + (Nil None) + ((Cons frame rest) + (let bindings ((bs frame)) + (match bs + (Nil (frames rest)) + ((Cons (Tuple p u) more) + (if (and (= u uri) + (or default-ok (not (string-empty? p))) + (= (scope-uri scope p) (Some uri))) + (Some p) + (bindings more))))))))) + +(: (-> Hints string (Option string))) +(defun (hint-prefix hints uri) + (match hints + (Nil None) + ((Cons (Tuple p u) rest) (if (= u uri) (Some p) (hint-prefix rest uri))))) + +(: (-> Hints string bool)) +(defun (hinted? hints prefix) + (match hints + (Nil #f) + ((Cons (Tuple p _) rest) (or (= p prefix) (hinted? rest prefix))))) + +;; A prefix that nothing here and nothing hinted uses: ns1, ns2 and so on. +(: (-> Scope Hints string)) +(defun (fresh-prefix scope hints) + (let loop ((n 1)) + (def p (str "ns" (int->string n))) + (if (or (some? (scope-uri scope p)) (hinted? hints p)) + (loop (+ n 1)) + p))) + +;; An element's prefix, and the declaration it needs, if any. An element in no +;; namespace inside a default namespace needs xmlns="". +(: (-> Scope Hints string (Tuple string (List (Tuple string string))))) +(defun (element-prefix scope hints uri) + (cond + ((string-empty? uri) + (Tuple "" (if (= (scope-uri scope "") (Some "")) Nil (list (Tuple "" ""))))) + (:match (Some p) (scope-prefix scope uri #t) (Tuple p Nil)) + (:match (Some p) (hint-prefix hints uri) (Tuple p (list (Tuple p uri)))) + (else (Tuple "" (list (Tuple "" uri)))))) + +;; An attribute's prefix, given the declarations the element has made so far. +;; A hinted prefix this element already uses for another namespace cannot be +;; declared again on it, so a generated one is used instead. +(: (-> Scope (List (Tuple string string)) Hints string + (Tuple string (List (Tuple string string))))) +(defun (attribute-prefix scope frame hints uri) + (def here (Cons frame scope)) + (cond + ((string-empty? uri) (Tuple "" frame)) + ((= uri xml-namespace) (Tuple "xml" frame)) + (:match (Some p) (scope-prefix here uri #f) (Tuple p frame)) + (:match (Some p) (hint-prefix hints uri) #:when (none? (bound-in frame p)) + (Tuple p (Cons (Tuple p uri) frame))) + (else + (def p (fresh-prefix here hints)) + (Tuple p (Cons (Tuple p uri) frame))))) + +;; Every namespace a name in the tree is in. +(: (-> Node (TransientMap string bool))) +(defun (used-namespaces root) + (def used (transientmap-empty)) + (let loop ((stack (list (list root)))) + (match stack + (Nil used) + ((Cons Nil rest) (loop rest)) + ((Cons (Cons n more) rest) + (match (element-view n) + ((Some (Tuple name attrs kids)) + (transientmap-set! used (xml-name-ns name) #t) + (list-for-each (fun (a) (def (Tuple an _) a) (transientmap-set! used (xml-name-ns an) #t)) attrs) + (loop (Cons kids (Cons more rest)))) + (None (loop (Cons more rest)))))))) + +;; A bad hint is a programming error, as a bad name is. +(: (-> Hints void)) +(defun (check-hints hints) + (list-for-each + (fun (h) + (def (Tuple p u) h) + (when (or (err? (clr-verify-ncname p)) (= p "xml") (= p "xmlns")) + (raise (cast Exception (ArgumentException. (str "(text xml write): #:prefixes: \"" p "\" cannot be a prefix"))))) + (when (or (string-empty? u) (= u xml-namespace) (= u xmlns-namespace)) + (raise (cast Exception (ArgumentException. (str "(text xml write): #:prefixes: \"" p "\" cannot name \"" u "\"")))))) + hints)) + +;; =========================================================================== +;; The walk +;; =========================================================================== + +;; A node to write, or an element to close. `newline` is whether an indent +;; goes before it, and `mixed` whether its subtree may have whitespace added. +;; An element closing says whether it put declarations in scope, which are +;; then taken out again. +(type (: Step (Union (: Visit Node int bool bool) + (: Leave int bool bool)))) + +(: XmlName) +(def xml-space (xml-name "space" #:ns xml-namespace)) + +(: (-> (List Node) bool)) +(defun (has-text? kids) + (match kids + (Nil #f) + ((Cons k rest) (or (text? k) (has-text? rest))))) + +;; Whether an element's children go on lines of their own. Never inside a +;; subtree where whitespace would be content: an element with text among its +;; children, what is below one, and xml:space="preserve". +(: (-> Node (List Node) bool bool)) +(defun (indents? n kids mixed) + (and (not mixed) + (not (list-empty? kids)) + (not (has-text? kids)) + (not (= (attr-ref n xml-space) (Some "preserve"))))) + +(: (-> int string)) +(defun (newline-indent depth) + (str "\n" (string-pad-left "" (* 2 depth)))) + +;; The start tag and the attributes, and the declarations it made. The root +;; also declares the hinted prefixes for the namespaces the tree uses. +(: (-> XmlWriter Scope Hints Hints XmlName XmlAttrs (List (Tuple string string)))) +(defun (write-start w scope hints root-hints name attrs) + (def uri (xml-name-ns name)) + (def (Tuple prefix declared) (element-prefix scope hints uri)) + (clr-start-element w prefix (xml-name-local name) uri) + (def with-hints + (list-foldl (fun (h frame) + (def (Tuple p u) h) + (cond + ((some? (bound-in frame p)) frame) + (else + (clr-attribute w "xmlns" p xmlns-namespace u) + (Cons (Tuple p u) frame)))) + declared + root-hints)) + (list-foldl (fun (a frame) + (def (Tuple an value) a) + (def auri (xml-name-ns an)) + (def (Tuple ap frame2) (attribute-prefix scope frame hints auri)) + (clr-attribute w ap (xml-name-local an) auri value) + frame2) + with-hints + attrs)) + +(: (-> XmlWriter Node bool Hints (Result XmlWriteError Unit))) +(defun (write-tree w root indent hints) + (def root-hints + (if (list-empty? hints) + Nil + (let ((used (used-namespaces root))) + (list-filter (fun (h) (def (Tuple _ u) h) (transientmap-contains? used u)) hints)))) + (let loop ((steps (list (Visit root 0 #f (not indent)))) (scope Nil) (first #t)) + (match steps + (Nil done) + ((Cons (Leave depth newline declared) rest) + (when newline (clr-whitespace w (newline-indent depth))) + (clr-end-element w) + (loop rest (match scope ((Cons _ outer) #:when declared outer) (_ scope)) #f)) + ((Cons (Visit n depth newline mixed) rest) + (when newline (clr-whitespace w (newline-indent depth))) + (cond + ((text? n) + (clr-string w (text-value n)) + (loop rest scope #f)) + ((comment? n) + (match (comment-problem (comment-value n)) + ((Some problem) (write-error problem)) + (None (clr-comment w (comment-value n)) (loop rest scope #f)))) + ((pi? n) + (match (pi-problem (pi-target n) (pi-data n)) + ((Some problem) (write-error problem)) + (None (clr-pi w (pi-target n) (pi-data n)) (loop rest scope #f)))) + (else + (def kids (element-children n)) + (def frame (write-start w scope hints (if first root-hints Nil) (element-name n) (element-attrs n))) + (def inside (indents? n kids mixed)) + (def declared (not (list-empty? frame))) + (loop (list-append (list-map (fun (k) (Visit k (+ depth 1) inside (not inside))) kids) + (Cons (Leave depth inside declared) rest)) + (if declared (Cons frame scope) scope) + #f))))))) + +;; =========================================================================== +;; Documents +;; =========================================================================== + +;; Written raw, because WriteDocType takes null for an id that is not there and +;; there is no null to pass. Nothing is escaped: each part is checked instead, +;; and the system id quoted with whichever quote it does not contain. +(: (-> Doctype (Result XmlWriteError string))) +(defun (doctype-markup dt) + (def name (record-ref dt name)) + (def system-literal + (match (record-ref dt system-id) + (None (Ok None)) + ((Some sys) + (cond + ((err? (clr-verify-xml-chars sys)) (write-error "the doctype's system id has a character XML does not allow")) + ((not (string-contains? sys "\"")) (Ok (Some (str "\"" sys "\"")))) + ((not (string-contains? sys "'")) (Ok (Some (str "'" sys "'")))) + (else (write-error "the doctype's system id has both kinds of quote in it")))))) + (cond + ((err? (clr-verify-name name)) (write-error (str "\"" name "\" is not a doctype name"))) + (:def (Ok sys) system-literal :propagate) + (else + (match (Tuple (record-ref dt public-id) sys) + ((Tuple None None) (Ok (str ""))) + ((Tuple None (Some s)) (Ok (str ""))) + ((Tuple (Some pub) (Some s)) + (if (err? (clr-verify-public-id pub)) + (write-error "the doctype's public id has a character a public id may not") + (Ok (str "")))) + ((Tuple (Some _) None) (write-error "a doctype with a public id needs a system id too")))))) + +(type (: TopItem (Union (: TopDoctype Doctype) (: TopNode Node)))) + +(: (-> Document (List TopItem))) +(defun (top-items doc) + (def nodes (list-map TopNode (list-append (document-prolog doc) + (Cons (document-root doc) (document-epilog doc))))) + (match (document-doctype doc) + ((Some dt) (Cons (TopDoctype dt) nodes)) + (None nodes))) + +(: (-> XmlWriter Document bool Hints bool (Result XmlWriteError Unit))) +(defun (write-document w doc indent hints declaration) + (when declaration (clr-pi w "xml" "version=\"1.0\"")) + (let loop ((items (top-items doc)) (first (not declaration))) + (match items + (Nil done) + ((Cons item rest) + (when (and indent (not first)) (clr-whitespace w "\n")) + (def (Ok _) + (match item + ((TopDoctype dt) + (def (Ok markup) (doctype-markup dt) :propagate) + (clr-raw w markup) + done) + ((TopNode n) (write-tree w n indent (if (element? n) hints Nil)))) + :propagate) + (loop rest #f))))) + +;; =========================================================================== +;; Running a writer +;; =========================================================================== + +(: (-> bool bool XmlWriterSettings)) +(defun (writer-settings document declaration) + (def settings (XmlWriterSettings.)) + (clr-settings-omit-declaration! settings (not declaration)) + (clr-settings-conformance! settings (if document clr-conformance-document clr-conformance-fragment)) + ;; The port is the caller's, and a write that failed half-way is not ended + ;; with end tags nobody asked for. + (clr-settings-close-output! settings #f) + (clr-settings-end-on-close! settings #f) + settings) + +;; What XmlWriter refuses is an ArgumentException, an XmlException or an +;; InvalidOperationException. Anything else, an IOException from the port say, +;; passes through. What was written before the error stays written. +(: (-> XmlWriter (-> XmlWriter (Result XmlWriteError Unit)) (Result XmlWriteError Unit))) +(defun (run-writer writer body) + (match (try (with-open ((w writer)) + (def result (body w)) + (when (ok? result) (clr-flush w)) + result) + #:catch (System.ArgumentException System.Xml.XmlException System.InvalidOperationException)) + ((Ok result) result) + ((Err e) (write-error (.-Message e))))) + +(: (-> Node OutputPort (#:indent? bool) (#:prefixes (List (Tuple string string))) + (Result XmlWriteError Unit))) +(defun (node->xml node port #:indent? #f #:prefixes Nil) + (check-hints prefixes) + (run-writer (clr-writer-over-text (cast TextWriter port) (writer-settings #f #f)) + (fun (w) (write-tree w node indent? prefixes)))) + +(: (-> Node (#:indent? bool) (#:prefixes (List (Tuple string string))) + (Result XmlWriteError string))) +(defun (node->xml-string node #:indent? #f #:prefixes Nil) + (check-hints prefixes) + (def sb (ClrBuilder.)) + (match (run-writer (clr-writer-over-builder sb (writer-settings #f #f)) + (fun (w) (write-tree w node indent? prefixes))) + ((Ok _) (Ok (.ToString sb))) + ((Err e) (Err e)))) + +;; #:declaration? writes , with no encoding: which one the +;; text ends up in is the port's business. +(: (-> Document OutputPort (#:indent? bool) (#:prefixes (List (Tuple string string))) + (#:declaration? bool) + (Result XmlWriteError Unit))) +(defun (document->xml doc port #:indent? #f #:prefixes Nil #:declaration? #t) + (check-hints prefixes) + (run-writer (clr-writer-over-text (cast TextWriter port) (writer-settings #t declaration?)) + (fun (w) (write-document w doc indent? prefixes declaration?)))) + +(: (-> Document (#:indent? bool) (#:prefixes (List (Tuple string string))) + (#:declaration? bool) + (Result XmlWriteError string))) +(defun (document->xml-string doc #:indent? #f #:prefixes Nil #:declaration? #t) + (check-hints prefixes) + (def sb (ClrBuilder.)) + (match (run-writer (clr-writer-over-builder sb (writer-settings #t declaration?)) + (fun (w) (write-document w doc indent? prefixes declaration?))) + ((Ok _) (Ok (.ToString sb))) + ((Err e) (Err e)))) diff --git a/corpus/en.txt b/corpus/en.txt new file mode 100644 index 0000000..7bde370 --- /dev/null +++ b/corpus/en.txt @@ -0,0 +1,2882 @@ +Georg Ferdinand Ludwig Philipp Cantor ( KAN-tor; German: [ˈɡeːɔʁk ˈfɛʁdinant ˈluːtvɪç ˈfɪlɪp ˈkantoːɐ̯]; 3 March [O.S. 19 February] 1845 – 6 January 1918) was a mathematician who played a pivotal role in the creation of set theory, which has become a fundamental theory in mathematics. Cantor established the importance of one-to-one correspondence between the members of two sets, defined infinite and well-ordered sets, and proved that the real numbers are more numerous than the natural numbers. Cantor's method of proof of this theorem implies the existence of an infinity of infinities. He defined the cardinal and ordinal numbers and their arithmetic. Cantor's work is of great philosophical interest, a fact of which he was well aware. He also repeatedly plagiarized the work of Richard Dedekind. +Originally, Cantor's theory of transfinite numbers was regarded as counter-intuitive – even shocking. This caused it to encounter resistance from mathematical contemporaries such as Leopold Kronecker and Henri Poincaré and later from Hermann Weyl and L. E. J. Brouwer, while Ludwig Wittgenstein raised philosophical objections; see Controversy over Cantor's theory. Cantor, a devout Lutheran Christian, believed the theory had been communicated to him by God. Some Christian theologians (particularly neo-Scholastics) saw Cantor's work as a challenge to the uniqueness of the absolute infinity in the nature of God – on one occasion equating the theory of transfinite numbers with pantheism – a proposition that Cantor vigorously rejected. Not all theologians were against Cantor's theory; prominent neo-scholastic philosopher Konstantin Gutberlet was in favor of it, and Cardinal Johann Baptist Franzelin accepted it as a valid theory (after Cantor made some important clarifications). +The objections to Cantor's work were occasionally fierce: Leopold Kronecker's public opposition and personal attacks included describing Cantor as a "scientific charlatan", a "renegade" and a "corrupter of youth". Kronecker objected to Cantor's proofs that the algebraic numbers are countable, and that the transcendental numbers are uncountable, results now included in a standard mathematics curriculum. Writing decades after Cantor's death, Wittgenstein lamented that mathematics is "ridden through and through with the pernicious idioms of set theory", which he dismissed as "utter nonsense" that is "laughable" and "wrong". Cantor's recurring bouts of depression from 1884 to the end of his life have been blamed on the hostile attitude of many of his contemporaries, though some have explained these episodes as probable manifestations of a bipolar disorder. +The harsh criticism has been matched by later accolades. In 1904, the Royal Society awarded Cantor its Sylvester Medal, the highest honor it can confer for work in mathematics. David Hilbert defended Cantor's work from its critics by declaring, "No one shall expel us from the paradise that Cantor has created." +Georg Cantor was born on March 3, 1845, in Saint Petersburg, Russian Empire, and was brought up in the city until the age of eleven. The eldest of six children, he was regarded as an outstanding violinist. His father, Georg Waldemar Cantor, was Danish, but fled with his family to Russia during the Napoleonic Wars; his mother, Maria Anna Böhm, was an Austro-Hungarian born in Saint Petersburg. Cantor's family was Lutheran, though his mother was raised Catholic. He described his paternal grandparents as having belonged to the Portuguese Jewish community of Copenhagen. His maternal grandfather Franz Böhm (1788–1846; the violinist Joseph Böhm's brother) was a well-known musician and a soloist in a Russian imperial orchestra. +Cantor's father worked as a broker in the Saint Petersburg stock exchange until an illness in 1856, which forced the family to seek out a more temperate climate. They moved to Germany, first to Wiesbaden, then to Frankfurt. In 1860, Cantor graduated with distinction from the Realschule in Darmstadt; his exceptional skills in mathematics, trigonometry in particular, were noted. In August 1862, he then graduated from the "Höhere Gewerbeschule Darmstadt", now the Technische Universität Darmstadt. In 1862 Cantor entered the Swiss Federal Polytechnic in Zurich. After receiving a substantial inheritance upon his father's death in June 1863, Cantor transferred to the University of Berlin, attending lectures by Leopold Kronecker, Karl Weierstrass and Ernst Kummer. He spent the summer of 1866 at the University of Göttingen, then and later a center for mathematical research. Cantor was a good student, and he received his doctoral degree in 1867. +Cantor submitted his dissertation on number theory at the University of Berlin in 1867. After teaching briefly in a Berlin girls' school, he took up a position at the University of Halle, where he spent his entire career. He was awarded the requisite habilitation for his thesis, also on number theory, which he presented in 1869 upon his appointment at Halle. +In 1874, Cantor married Vally Guttmann. They had six children, the last (Rudolph) born in 1886. Cantor was able to support a family despite his modest academic pay, thanks to his inheritance from his father. During his honeymoon in the Harz mountains, Cantor spent much time in mathematical discussions with Richard Dedekind, whom he had met two years earlier while on holiday in Gersau in Switzerland, and with whom a lively mathematical correspondence had developed. +Cantor was promoted to extraordinary professor in 1872 and made full professor in 1879. To attain the latter rank at the age of 34 was a notable accomplishment, but Cantor desired a chair at a more prestigious university, in particular at Berlin, at that time the leading German university. However, his work encountered too much opposition for that to be possible. Kronecker, who headed mathematics at Berlin until his death in 1891, became increasingly uncomfortable with the prospect of having Cantor as a colleague, perceiving him as a "corrupter of youth" for teaching his ideas to a younger generation of mathematicians. Worse yet, Kronecker, a well-established figure within the mathematical community and Cantor's former professor, disagreed fundamentally with the thrust of Cantor's work ever since he had intentionally delayed the publication of Cantor's first major publication in 1874. Kronecker, now seen as one of the founders of the constructive viewpoint in mathematics, disliked much of Cantor's set theory because it asserted the existence of sets satisfying certain properties, without giving specific examples of sets whose members did indeed satisfy those properties. Whenever Cantor applied for a post in Berlin, he was declined, and the process usually involved Kronecker, so Cantor came to believe that Kronecker's stance would make it impossible for him ever to leave Halle. +In 1881, Cantor's Halle colleague Eduard Heine died. Halle accepted Cantor's suggestion that Heine's vacant chair be offered to Dedekind, Heinrich M. Weber and Franz Mertens, in that order, but each declined the chair after being offered it. Friedrich Wangerin was eventually appointed, but he was never close to Cantor. +In 1882, the mathematical correspondence between Cantor and Dedekind came to an end, apparently as a result of Dedekind's declining the chair at Halle. Cantor also began another important correspondence, with Gösta Mittag-Leffler in Sweden, and soon began to publish in Mittag-Leffler's journal Acta Mathematica. But in 1885, Mittag-Leffler was concerned about the philosophical nature and new terminology in a paper Cantor had submitted to Acta. He asked Cantor to withdraw the paper from Acta while it was in proof, writing that it was "... about one hundred years too soon." Cantor complied, but then curtailed his relationship and correspondence with Mittag-Leffler, writing to a third party, "Had Mittag-Leffler had his way, I should have to wait until the year 1984, which to me seemed too great a demand! ... But of course I never want to know anything again about Acta Mathematica." +Cantor suffered his first known bout of depression in May 1884. Criticism of his work weighed on his mind: every one of the fifty-two letters he wrote to Mittag-Leffler in 1884 mentioned Kronecker. A passage from one of these letters is revealing of the damage to Cantor's self-confidence: +... I don't know when I shall return to the continuation of my scientific work. At the moment I can do absolutely nothing with it, and limit myself to the most necessary duty of my lectures; how much happier I would be to be scientifically active, if only I had the necessary mental freshness. +This crisis led him to apply to lecture on philosophy rather than on mathematics. He also began an intense study of Elizabethan literature, thinking there might be evidence that Francis Bacon wrote the plays attributed to William Shakespeare (see Shakespearean authorship question); this ultimately resulted in two pamphlets, published in 1896 and 1897. +Cantor recovered soon thereafter, and subsequently made further important contributions, including his diagonal argument and theorem. However, he never again attained the high level of his remarkable papers of 1874–84, even after Kronecker's death on 29 December 1891. He eventually sought, and achieved, a reconciliation with Kronecker. Nevertheless, the philosophical disagreements and difficulties dividing them persisted. +In 1889, Cantor was instrumental in founding the German Mathematical Society, and he chaired its first meeting in Halle in 1891, where he first introduced his diagonal argument; his reputation was strong enough, despite Kronecker's opposition to his work, to ensure he was elected as the first president of this society. Setting aside the animosity Kronecker had displayed towards him, Cantor invited him to address the meeting, but Kronecker was unable to do so because his wife was dying from injuries sustained in a skiing accident at the time. Georg Cantor was also instrumental in the establishment of the first International Congress of Mathematicians, which took place in Zürich, Switzerland, in 1897. +After Cantor's 1884 hospitalization there is no record that he was in any sanatorium again until 1899. Soon after that second hospitalization, Cantor's youngest son Rudolph died suddenly on 16 December (Cantor was delivering a lecture on his views on Baconian theory and William Shakespeare), and this tragedy drained Cantor of much of his passion for mathematics. Cantor was again hospitalized in 1903. One year later, he was outraged and agitated by a paper presented by Julius König at the Third International Congress of Mathematicians. The paper attempted to prove that the basic tenets of transfinite set theory were false. Since the paper had been read in front of his daughters and colleagues, Cantor perceived himself as having been publicly humiliated. Although Ernst Zermelo demonstrated less than a day later that König's proof had failed, Cantor remained shaken, and momentarily questioning God. Cantor suffered from chronic depression for the rest of his life, for which he was excused from teaching on several occasions and repeatedly confined to various sanatoria. The events of 1904 preceded a series of hospitalizations at intervals of two or three years. He did not abandon mathematics completely, however, lecturing on the paradoxes of set theory (Burali-Forti paradox, Cantor's paradox, and Russell's paradox) to a meeting of the Deutsche Mathematiker-Vereinigung in 1903, and attending the International Congress of Mathematicians at Heidelberg in 1904. +In 1911, Cantor was one of the distinguished foreign scholars invited to the 500th anniversary of the founding of the University of St. Andrews in Scotland. Cantor attended, hoping to meet Bertrand Russell, whose newly published Principia Mathematica repeatedly cited Cantor's work, but the encounter did not come about. The following year, St. Andrews awarded Cantor an honorary doctorate, but illness precluded his receiving the degree in person. +Cantor retired in 1913, and lived in poverty and suffered from malnourishment during World War I. The public celebration of his 70th birthday was canceled because of the war. In June 1917, he entered a sanatorium for the last time and continually wrote to his wife asking to be allowed to go home. Georg Cantor had a fatal heart attack on 6 January 1918, in the sanatorium where he had spent the last year of his life. +In 1970, he was memorialized by the naming of Cantor crater on the Moon. +Cantor's work between 1874 and 1884 is the origin of set theory. Prior to this work, the concept of a set was a rather elementary one that had been used implicitly since the beginning of mathematics, dating back to the ideas of Aristotle. No one had realized that set theory had any nontrivial content. Before Cantor, there were only finite sets (which are easy to understand) and "the infinite" (which was considered a topic for philosophical, rather than mathematical, discussion). By proving that there are (infinitely) many possible sizes for infinite sets, Cantor established that set theory was not trivial, and it needed to be studied. Set theory has come to play the role of a foundational theory in modern mathematics, in the sense that it interprets propositions about mathematical objects (for example, numbers and functions) from all the traditional areas of mathematics (such as algebra, analysis, and topology) in a single theory, and provides a standard set of axioms to prove or disprove them. The basic concepts of set theory are now used throughout mathematics. +In one of his earliest papers, Cantor proved that the set of real numbers is "more numerous" than the set of natural numbers; this showed, for the first time, that there exist infinite sets of different sizes. He was the first to appreciate the importance of one-to-one correspondences (hereinafter denoted "1-to-1 correspondence") in set theory: two sets are said to have the same "size" if there exists a 1-to-1 correspondence between them. He used this concept to define finite and infinite sets, subdividing the latter into denumerable (or countably infinite) sets and nondenumerable sets (uncountably infinite sets). +Cantor developed important concepts in topology and their relation to cardinality. For example, he showed that the Cantor set, discovered by Henry John Stephen Smith in 1875, is nowhere dense, but has the same cardinality as the set of all real numbers, whereas the rationals are everywhere dense, but countable. He also showed that all countable dense linear orders without end points are order-isomorphic to the rational numbers. +Cantor introduced fundamental constructions in set theory, such as the power set of a set A, which is the set of all possible subsets of A. He later proved that the size of the power set of A is strictly larger than the size of A, even when A is an infinite set; this result soon became known as Cantor's theorem. Cantor developed an entire theory and arithmetic of infinite sets, called cardinals and ordinals, which extended the arithmetic of the natural numbers. His notation for the cardinal numbers was the Hebrew letter ℵ (ℵ, aleph) with a natural number subscript; for the ordinals he employed the Greek letter ω (ω, omega). This notation is still in use today. +The Continuum hypothesis, introduced by Cantor, was presented by David Hilbert as the first of his twenty-three open problems in his address at the 1900 International Congress of Mathematicians in Paris. Cantor's work also attracted favorable notice beyond Hilbert's celebrated encomium. The US philosopher Charles Sanders Peirce praised Cantor's set theory and, following public lectures delivered by Cantor at the first International Congress of Mathematicians, held in Zürich in 1897, Adolf Hurwitz and Jacques Hadamard also both expressed their admiration. At that Congress, Cantor renewed his friendship and correspondence with Dedekind. From 1905, Cantor corresponded with his British admirer and translator Philip Jourdain on the history of set theory and on Cantor's religious ideas. This was later published, as were several of his expository works. +Cantor's first ten papers were on number theory, his thesis topic. At the suggestion of Eduard Heine, the Professor at Halle, Cantor turned to analysis. Heine proposed that Cantor solve an open problem that had eluded Peter Gustav Lejeune Dirichlet, Rudolf Lipschitz, Bernhard Riemann, and Heine himself: the uniqueness of the representation of a function by trigonometric series. Cantor solved this problem in 1869. It was while working on this problem that he discovered transfinite ordinals, which occurred as indices n in the nth derived set Sn of a set S of zeros of a trigonometric series. Given a trigonometric series f(x) with S as its set of zeros, Cantor had discovered a procedure that produced another trigonometric series that had S1 as its set of zeros, where S1 is the set of limit points of S. If Sk+1 is the set of limit points of Sk, then he could construct a trigonometric series whose zeros are Sk+1. Because the sets Sk were closed, they contained their limit points, and the intersection of the infinite decreasing sequence of sets S, S1, S2, S3,... formed a limit set, which we would now call Sω, and then he noticed that Sω would also have to have a set of limit points Sω+1, and so on. He had examples that went on forever, and so here was a naturally occurring infinite sequence of infinite numbers ω, ω + 1, ω + 2, ... +Between 1870 and 1872, Cantor published more papers on trigonometric series, and also a paper defining irrational numbers as convergent sequences of rational numbers. Dedekind, whom Cantor befriended in 1872, cited this paper later that year, in the paper where he first set out his celebrated definition of real numbers by Dedekind cuts. While extending the notion of number by means of his revolutionary concept of infinite cardinality, Cantor was paradoxically opposed to theories of infinitesimals of his contemporaries Otto Stolz and Paul du Bois-Reymond, describing them as both "an abomination" and "a cholera bacillus of mathematics". Cantor also published an erroneous "proof" of the inconsistency of infinitesimals. +The publication of Cantor's 1874 paper "Ueber eine Eigenschaft des Inbegriffes aller reellen algebraischen Zahlen" ("On a Property of the Collection of All Real Algebraic Numbers") is often considered to mark the beginning of set theory as a branch of mathematics. This paper was the first to provide a rigorous proof that there is more than one kind of infinity. Previously, all infinite collections had been implicitly assumed to be equinumerous (that is, of "the same size", or having the same number of elements). Cantor proved that real numbers and the positive integers cannot be put in 1-to-1 correspondence and are thus not equinumerous. In other words, the real numbers are not countable. His proof differs from the diagonal argument that he gave in 1891. Cantor's article also contains a new method of constructing transcendental numbers. Transcendental numbers were first constructed by Joseph Liouville in 1844. +Cantor established these results using two constructions. His first construction shows how to write the real algebraic numbers as a sequence a1, a2, a3.... In other words, the real algebraic numbers are countable. Cantor starts his second construction with any sequence of real numbers. Using this sequence, he constructs nested intervals whose intersection contains a real number not in the sequence. Since every sequence of real numbers can be used to construct a real not in the sequence, the real numbers cannot be written as a sequence – that is, the real numbers are not countable. By applying his construction to the sequence of real algebraic numbers, Cantor produces a transcendental number. Cantor adds that his constructions provide more – namely, a new proof of Liouville's theorem: Every interval contains infinitely many transcendental numbers. Cantor's next article contains a construction that proves the set of transcendental numbers has the same "power" (see below) as the set of real numbers. +Analysis of the correspondence between Cantor and Dedekind later showed that the proof for the countability of the algebraic numbers originated with Dedekind, who also substantially simplified Cantor's original proof of the uncountability of the reals. Cantor failed to acknowledge Dedekind's contributions in his 1874 article. +Between 1879 and 1884, Cantor published a series of six articles in Mathematische Annalen that together formed an introduction to his set theory. At the same time, opposition to Cantor's ideas grew, led by Leopold Kronecker, who admitted mathematical concepts only if they could be constructed in a finite number of steps from the natural numbers, which he took as intuitively given. For Kronecker, Cantor's hierarchy of infinities was inadmissible, since accepting the concept of actual infinity would open the door to paradoxes that would challenge the validity of mathematics as a whole. Cantor also introduced the Cantor set during this period. +The fifth paper in this series, "Grundlagen einer allgemeinen Mannigfaltigkeitslehre" ("Foundations of a General Theory of Aggregates"), published in 1883, was the most important of the six and was also published as a separate monograph. It contained Cantor's reply to his critics and showed how the transfinite numbers are a systematic extension of the natural numbers. It begins by defining well-ordered sets. Ordinal numbers are then introduced as the order types of well-ordered sets. Cantor then defines the addition and multiplication of the cardinal and ordinal numbers. In 1885, he extended his theory of order types so that the ordinal numbers simply became a special case of order types. +In 1891, Cantor published a paper containing his "diagonal argument" for the existence of an uncountable set. He applied the same idea to prove Cantor's theorem: the cardinality of the power set of a set A is strictly larger than the cardinality of A. This established the richness of the hierarchy of infinite sets, and of the cardinal and ordinal arithmetic Cantor had defined. His argument is fundamental in the solution of the Halting problem and the proof of Gödel's first incompleteness theorem. Cantor wrote on the Goldbach conjecture in 1894. +In 1895 and 1897, Cantor published a two-part paper in Mathematische Annalen under Felix Klein's editorship; these were his last significant papers on set theory. The first paper begins by defining set, subset, etc., in ways that would be largely acceptable now. The cardinal and ordinal arithmetic are reviewed. Cantor wanted the second paper to include a proof of the continuum hypothesis, but had to settle for laying out his theory of well-ordered sets and ordinal numbers. Cantor attempts to prove that if A and B are sets with A equivalent to a subset of B and B equivalent to a subset of A, then A and B are equivalent. Ernst Schröder had stated this theorem a bit earlier, but his proof, as well as Cantor's, was flawed. Felix Bernstein supplied a correct proof in his 1898 PhD thesis; hence the name Cantor–Bernstein–Schröder theorem. +Cantor's 1874 Crelle paper was the first to invoke the notion of 1-to-1 correspondence, though he did not use that term. He then began looking for 1-to-1 correspondence between the points of the unit square and the points of a unit line segment. In an 1877 letter to Richard Dedekind, Cantor proved a far stronger result: for any positive integer n, there exists a 1-to-1 correspondence between the points on the unit line segment and all the points in an n-dimensional space. Of this discovery, Cantor wrote to Dedekind: "Je le vois, mais je ne le crois pas!" ("I see it, but I don't believe it!") The result he found so astonishing has implications for geometry and the notion of dimension. +In 1878, Cantor submitted another paper to Crelle's Journal in which he precisely defined the concept of a 1-to-1 correspondence and introduced the notion of "power" (a term he took from Jakob Steiner) or "equivalence" of sets: two sets are equivalent (have the same power) if there is a 1-to-1 correspondence between them. Cantor defined countable sets (or denumerable sets) as sets that can be put into 1-to-1 correspondence with the natural numbers, and proved that the rational numbers are denumerable. He also proved that n-dimensional Euclidean space Rn has the same power as the real numbers R, as does a countably infinite product of copies of R. While he made free use of countability as a concept, he did not write the word "countable" until 1883. Cantor also discussed his thinking about dimension, stressing that his mapping between the unit interval and the unit square was not a continuous one. +This paper displeased Kronecker and Cantor wanted to withdraw it, but Dedekind persuaded him not to do so and Karl Weierstrass supported its publication. Nevertheless, Cantor never again submitted anything to Crelle. +Cantor was the first to formulate what later came to be known as the continuum hypothesis or CH: there exists no set whose power is greater than that of the naturals and less than that of the reals (or equivalently, the cardinality of the reals is exactly aleph-one, rather than just at least aleph-one). Cantor believed the continuum hypothesis to be true and tried in vain for many years to prove it. His inability to prove the continuum hypothesis caused him considerable anxiety. +The difficulty Cantor had in proving the continuum hypothesis has been underscored by later developments in mathematics: a 1940 result by Kurt Gödel and a 1963 one by Paul Cohen together imply that the continuum hypothesis can be neither proved nor disproved using standard Zermelo–Fraenkel set theory plus the axiom of choice (the combination referred to as "ZFC"). +In 1883, Cantor divided the infinite into the transfinite and the absolute. +The transfinite is increasable in magnitude, while the absolute is unincreasable. For example, an ordinal α is transfinite because it can be increased to α+1. On the other hand, the ordinals form an absolutely infinite sequence that cannot be increased in magnitude because there are no larger ordinals to add to it. In 1883, Cantor also introduced the well-ordering principle "every set can be well-ordered" and called it a "law of thought". +Cantor extended his work on the absolute infinite by using it in a proof. Around 1895, he began to regard his well-ordering principle as a theorem and attempted to prove it. In 1899, he sent Dedekind a proof of the equivalent aleph theorem: the cardinality of every infinite set is an aleph. First he defined two types of multiplicities: consistent multiplicities (sets) and inconsistent multiplicities (absolutely infinite multiplicities). Next he assumed that the ordinals form a set, proved that this leads to a contradiction, and concluded that the ordinals form an inconsistent multiplicity. He used this inconsistent multiplicity to prove the aleph theorem. In 1932, Zermelo criticized the construction in Cantor's proof. +Cantor avoided paradoxes by recognizing that there are two types of multiplicities. In his set theory, when it is assumed that the ordinals form a set, the resulting contradiction implies only that the ordinals form an inconsistent multiplicity. In contrast, Bertrand Russell treated all collections as sets, which leads to paradoxes. In Russell's set theory, the ordinals form a set, so the resulting contradiction implies that the theory is inconsistent. From 1901 to 1903, Russell discovered three paradoxes implying that his set theory is inconsistent: the Burali-Forti paradox, Cantor's paradox, and Russell's paradox. Russell named paradoxes after Cesare Burali-Forti and Cantor even though neither of them believed they had found paradoxes. +In 1908, Zermelo published his axiom system for set theory. He had two motivations for developing the axiom system: eliminating the paradoxes and securing his proof of the well-ordering theorem. Zermelo had proved this theorem in 1904 using the axiom of choice, but his proof was criticized for a variety of reasons. His response to the criticism included his axiom system and a new proof of the well-ordering theorem. His axioms support this new proof, and they eliminate the paradoxes by restricting the formation of sets. +In 1923, John von Neumann developed an axiom system that eliminates the paradoxes with an approach similar to Cantor's—namely, by identifying collections that are not sets and treating them differently. Von Neumann wrote that a class is too big to be a set if it can be put into one-to-one correspondence with the class of all sets. He defined a set as a class that is a member of some class and postulated the axiom "A class is not a set if and only if there is a one-to-one correspondence between it and the class of all sets". This axiom implies that these big classes are not sets, which eliminates the paradoxes since they cannot be members of any class. Von Neumann also used his axiom to prove the well-ordering theorem: Like Cantor, he assumed that the ordinals form a set. The resulting contradiction implies that the class of all ordinals is not a set. Then his axiom provides a one-to-one correspondence between this class and the class of all sets. This correspondence well-orders the class of all sets, which implies the well-ordering theorem. In 1930, Zermelo defined models of set theory that satisfy von Neumann's axiom. +The concept of the existence of actual infinity was an important shared concern of mathematics, philosophy, and religion. Preserving the orthodoxy of the relationship between God and mathematics, though not in the same form as held by his critics, was long a concern of Cantor's. He directly addressed this intersection between these disciplines in the introduction to his Grundlagen einer allgemeinen Mannigfaltigkeitslehre, where he stressed the connection between his view of the infinite and the philosophical one. To Cantor, his mathematical views were intrinsically linked to their philosophical and theological implications: he identified the absolute infinite with God and considered his work on transfinite numbers to have been directly communicated to him by God, who had chosen Cantor to reveal them to the world. He was a devout Lutheran whose explicit Christian beliefs shaped his philosophy of science. Joseph Dauben has traced the effect Cantor's Christian faith had on the development of transfinite set theory. +Debate among mathematicians grew out of opposing views in the philosophy of mathematics about the nature of actual infinity. Some held that infinity was an abstraction that was not mathematically legitimate, and denied its existence. Mathematicians from three major schools of thought (constructivism and its two offshoots, intuitionism and finitism) opposed Cantor's theories in this matter. For constructivists such as Kronecker, this rejection of actual infinity stems from fundamental disagreement with the idea that nonconstructive proofs such as Cantor's diagonal argument are sufficient proof that something exists, holding instead that constructive proofs are required. Intuitionism also rejects that actual infinity is an expression of any sort of reality, but arrives at that view differently than constructivism. First, Cantor's argument rests on logic to prove the existence of transfinite numbers as an actual mathematical entity, whereas intuitionists hold that mathematical entities cannot be reduced to logical propositions, originating instead in the intuitions of the mind. Second, the notion of infinity as an expression of reality is itself disallowed in intuitionism, since the human mind cannot intuitively construct an infinite set. Mathematicians such as L. E. J. Brouwer and especially Henri Poincaré adopted an intuitionist stance against Cantor's work. Finally, Wittgenstein's attacks were finitist: he believed that Cantor's diagonal argument conflated the intension of a set of cardinal or real numbers with its extension, thus conflating the concept of rules for generating a set with an actual set. +Some Christian theologians saw Cantor's work as a challenge to the uniqueness of the absolute infinity in the nature of God. In particular, neo-Thomist thinkers saw the existence of an actual infinity that consisted of something other than God as jeopardizing "God's exclusive claim to supreme infinity". Cantor strongly believed that this was a misinterpretation of infinity and was convinced that set theory could help correct this mistake: "the transfinite species are just as much at the disposal of the intentions of the Creator and His absolute boundless will as are the finite numbers". Prominent neo-scholastic German philosopher Konstantin Gutberlet was in favor of such theory, holding that it did not contradict the nature of God. +Cantor also believed his theory of transfinite numbers ran counter to both materialism and determinism and was shocked when he realized that he was the only faculty member at Halle who did not hold deterministic philosophical beliefs. +It was important to Cantor that his philosophy provide an "organic explanation" of nature, and in his 1883 Grundlagen, he said that such an explanation could arise only by drawing on the resources of the philosophy of Spinoza and Leibniz. In making these claims, Cantor may have been influenced by F. A. Trendelenburg, whose lecture courses he attended at Berlin. Cantor produced a Latin commentary on Book 1 of Spinoza's Ethica. Trendelenburg was also the examiner of Cantor's Habilitationsschrift. +In 1888, Cantor published his correspondence with several philosophers on set theory's philosophical implications. In an extensive attempt to persuade other Christian thinkers and authorities of his views, Cantor had corresponded with Christian philosophers such as Tilman Pesch and Joseph Hontheim, as well as theologians such as Cardinal Johann Baptist Franzelin, who once replied by equating the theory of transfinite numbers with pantheism. Franzelin accepted the theory after Cantor made some clarifications. Cantor even sent one letter directly to Pope Leo XIII, and addressed several pamphlets to him. +Cantor's philosophy of the nature of numbers led him to affirm the freedom of mathematics to posit and prove concepts apart from the realm of physical phenomena, as expressions within an internal reality. The only restrictions on this metaphysical system are that all mathematical concepts be devoid of internal contradiction, and that they follow from existing definitions, axioms, and theorems. This belief is summarized in his assertion that "the essence of mathematics is its freedom." These ideas parallel those of Edmund Husserl, whom Cantor had met in Halle. +Meanwhile, Cantor fiercely opposed infinitesimals, calling them an "abomination" and "the cholera bacillus of mathematics". +Cantor's 1883 paper reveals that he was well aware of the opposition his ideas were encountering: "I realize that in this undertaking I place myself in a certain opposition to views widely held concerning the mathematical infinite and to opinions frequently defended on the nature of numbers." Hence he devotes much space to justifying his earlier work, asserting that mathematical concepts may be freely introduced as long as they are free of contradiction and defined in terms of previously accepted concepts. He also cites Aristotle, René Descartes, George Berkeley, Gottfried Leibniz, and Bernard Bolzano on infinity. He always strongly rejected Immanuel Kant's views in both the philosophy of mathematics and metaphysics. He shared Russell's motto "Kant or Cantor", and called Kant "yonder sophistical Philistine who knew so little mathematics". +Until the 1970s, the chief academic publications on Cantor were two short monographs by Arthur Moritz Schönflies (1927) – largely the correspondence with Mittag-Leffler – and Fraenkel (1930). Both were at second and third hand; neither had much on his personal life. The gap was largely filled by Eric Temple Bell's Men of Mathematics (1937), which one of Cantor's modern biographers called "perhaps the most widely read modern book on the history of mathematics" and as "one of the worst". Bell presents Cantor's relationship with his father as Oedipal, Cantor's differences with Kronecker as a quarrel between two Jews, and Cantor's madness as Romantic despair over his failure to win acceptance for his mathematics. Grattan-Guinness found that none of these claims were true, but they are found in many books of the intervening period, owing to the absence of any other narrative. There are other legends, independent of Bell – including one that calls Cantor's father a foundling, shipped to Saint Petersburg by unknown parents. Joseph Dauben's biography contains a critique of Bell's book. Dauben writes: +Cantor devoted some of his most vituperative correspondence, as well as a portion of the Beiträge, to attacking what he described at one point as the "infinitesimal Cholera bacillus of mathematics", which had spread from Germany through the work of Thomae, du Bois Reymond and Stolz, to infect Italian mathematics ... Any acceptance of infinitesimals necessarily meant that his own theory of number was incomplete. Thus to accept the work of Thomae, du Bois-Reymond, Stolz and Veronese was to deny the perfection of Cantor's own creation. Understandably, Cantor launched a thorough campaign to discredit Veronese's work in every way possible. +Absolute infinite +Aleph number +Cardinality of the continuum +Cantor medal – award by the Deutsche Mathematiker-Vereinigung in honor of Cantor +Continuum hypothesis +Countable set +Derived set (mathematics) +Epsilon numbers (mathematics) +Factorial number system +Pairing function +List of things named after Georg Cantor +Dauben, Joseph W. (1977). "Georg Cantor and Pope Leo XIII: Mathematics, Theology, and the Infinite". Journal of the History of Ideas. 38 (1): 85–108. doi:10.2307/2708842. JSTOR 2708842. +Dauben, Joseph W. (1979). Georg Cantor: His Mathematics and Philosophy of the Infinite. Boston: Harvard University Press. ISBN 978-0-691-02447-9. +Dauben, Joseph (2004) [1993]. Georg Cantor and the Battle for Transfinite Set Theory (PDF). Proceedings of the 9th ACMS Conference (Westmont College, Santa Barbara, Calif.). pp. 1–22. Archived (PDF) from the original on 23 January 2018. Internet version published in Journal of the ACMS 2004. Note, though, that Cantor's Latin quotation described in this article as "a familiar passage from the Bible" is actually from the works of Seneca and has no implication of divine revelation. +Ewald, William B., ed. (1996). From Immanuel Kant to David Hilbert: A Source Book in the Foundations of Mathematics. New York: Oxford University Press. ISBN 978-0-19-853271-2. +Grattan-Guinness, Ivor (1971). "Towards a Biography of Georg Cantor". Annals of Science. 27 (4): 345–391. doi:10.1080/00033797100203837. +Grattan-Guinness, Ivor (2000). The Search for Mathematical Roots: 1870–1940. Princeton University Press. ISBN 978-0-691-05858-0. +Hallett, Michael (1986). Cantorian Set Theory and Limitation of Size. New York: Oxford University Press. ISBN 978-0-19-853283-5. +Moore, Gregory H. (1982). Zermelo's Axiom of Choice: Its Origins, Development & Influence. Springer. ISBN 978-1-4613-9480-8. +Moore, Gregory H. (19 April 2023). "The roots of Russell's paradox". Russell: The Journal of Bertrand Russell Archives. 8 (1). Johns Hopkins University Press: 46–56. ISSN 1913-8032. Retrieved 28 March 2025. +Moore, Gregory H.; Garciadiego, Alejandro (1981). "Burali-Forti's Paradox: A Reappraisal of Its Origins". Historia Mathematica. 8 (3): 319–350. doi:10.1016/0315-0860(81)90070-7. +Purkert, Walter (1989). "Cantor's Views on the Foundations of Mathematics". In Rowe, David E.; McCleary, John (eds.). The History of Modern Mathematics, Volume 1. Academic Press. pp. 49–65. ISBN 978-0-12-599662-4. +Purkert, Walter; Ilgauds, Hans Joachim (1985). Georg Cantor: 1845–1918. Birkhäuser. ISBN 978-0-8176-1770-7. +Suppes, Patrick (1972) [1960]. Axiomatic Set Theory. New York: Dover. ISBN 978-0-486-61630-8. Although the presentation is axiomatic rather than naive, Suppes proves and discusses many of Cantor's results, which demonstrates Cantor's continued importance for the edifice of foundational mathematics. +Zermelo, Ernst (1908). "Untersuchungen über die Grundlagen der Mengenlehre I". Mathematische Annalen. 65 (2): 261–281. doi:10.1007/bf01449999. S2CID 120085563. +Zermelo, Ernst (1930). "Über Grenzzahlen und Mengenbereiche: neue Untersuchungen über die Grundlagen der Mengenlehre" (PDF). Fundamenta Mathematicae. 16: 29–47. doi:10.4064/fm-16-1-29-47. Archived (PDF) from the original on 28 June 2004. +van Heijenoort, Jean (1967). From Frege to Godel: A Source Book in Mathematical Logic, 1879–1931. Harvard University Press. ISBN 978-0-674-32449-7. +Older sources on Cantor's life should be treated with caution. See section § Biographies above. +Cantor, Georg (1955) [1915]. Philip Jourdain (ed.). Contributions to the Founding of the Theory of Transfinite Numbers. New York: Dover Publications.. ISBN 978-0-486-60045-1 +Cantor, Georg (1874). "Ueber eine Eigenschaft des Inbegriffes aller reellen algebraischen Zahlen" (PDF). Journal für die Reine und Angewandte Mathematik. 1874 (77): 258–262. doi:10.1515/crll.1874.77.258. S2CID 199545885. Archived (PDF) from the original on 7 October 2017. +Cantor, Georg (1878). "Ein Beitrag zur Mannigfaltigkeitslehre". Journal für die Reine und Angewandte Mathematik. 1878 (84): 242–258. doi:10.1515/crelle-1878-18788413. +Georg Cantor (1879). "Ueber unendliche, lineare Punktmannichfaltigkeiten (1)". Mathematische Annalen. 15 (1): 1–7. doi:10.1007/bf01444101. S2CID 179177510. +Georg Cantor (1880). "Ueber unendliche, lineare Punktmannichfaltigkeiten (2)". Mathematische Annalen. 17 (3): 355–358. doi:10.1007/bf01446232. S2CID 179177438. +Georg Cantor (1882). "Ueber unendliche, lineare Punktmannichfaltigkeiten (3)". Mathematische Annalen. 20 (1): 113–121. doi:10.1007/bf01443330. S2CID 177809016. +Georg Cantor (1883). "Ueber unendliche, lineare Punktmannichfaltigkeiten (4)". Mathematische Annalen. 21 (1): 51–58. doi:10.1007/bf01442612. S2CID 179177480. +Georg Cantor (1883). "Ueber unendliche, lineare Punktmannichfaltigkeiten (5)". Mathematische Annalen. 21 (4): 545–591. doi:10.1007/bf01446819. S2CID 121930608. Published separately as: Grundlagen einer allgemeinen Mannigfaltigkeitslehre. +Georg Cantor (1884). "Ueber unendliche, lineare Punktmannichfaltigkeiten (6)". Mathematische Annalen. 23 (4): 453–488. doi:10.1007/BF01446598. S2CID 179178052. +Georg Cantor (1891). "Ueber eine elementare Frage der Mannigfaltigkeitslehre" (PDF). Jahresbericht der Deutschen Mathematiker-Vereinigung. 1: 75–78. Archived (PDF) from the original on 1 January 2018. +Cantor, Georg (1895). "Beiträge zur Begründung der transfiniten Mengenlehre (1)". Mathematische Annalen. 46 (4): 481–512. doi:10.1007/bf02124929. S2CID 177801164. Archived from the original on 23 April 2014. +Cantor, Georg (1897). "Beiträge zur Begründung der transfiniten Mengenlehre (2)". Mathematische Annalen. 49 (2): 207–246. doi:10.1007/bf01444205. S2CID 121665994. +Cantor, Georg (1932). Ernst Zermelo (ed.). "Gesammelte Abhandlungen mathematischen und philosophischen inhalts". Berlin: Springer. Archived from the original on 3 February 2014.. Almost everything that Cantor wrote. Includes excerpts of his correspondence with Dedekind (p. 443–451) and Fraenkel's Cantor biography (p. 452–483) in the appendix. +Aczel, Amir D. (2000). The Mystery of the Aleph: Mathematics, the Kabbala, and the Search for Infinity. New York: Four Walls Eight Windows Publishing.. ISBN 0-7607-7778-0. A popular treatment of infinity, in which Cantor is frequently mentioned. +Dauben, Joseph W. (June 1983). "Georg Cantor and the Origins of Transfinite Set Theory". Scientific American. 248 (6): 122–131. Bibcode:1983SciAm.248f.122D. doi:10.1038/scientificamerican0683-122. +Ferreirós, José (2007). Labyrinth of Thought: A History of Set Theory and Its Role in Mathematical Thought. Basel, Switzerland: Birkhäuser.. ISBN 3-7643-8349-6 Contains a detailed treatment of both Cantor's and Dedekind's contributions to set theory. +Halmos, Paul (1998) [1960]. Naive Set Theory. New York & Berlin: Springer.. ISBN 3-540-90092-6 +Hilbert, David (1926). "Über das Unendliche". Mathematische Annalen. 95: 161–190. doi:10.1007/BF01206605. S2CID 121888793. +Hill, C. O.; Rosado Haddock, G. E. (2000). Husserl or Frege? Meaning, Objectivity, and Mathematics. Chicago: Open Court.. ISBN 0-8126-9538-0 Three chapters and 18 index entries on Cantor. +Meschkowski, Herbert (1983). Georg Cantor, Leben, Werk und Wirkung (Georg Cantor, Life, Work and Influence, in German). Vieweg, Braunschweig. +Newstead, Anne (2009). "Cantor on Infinity in Nature, Number, and the Divine Mind"[2], American Catholic Philosophical Quarterly, 83 (4): 532–553, https://doi.org/10.5840/acpq200983444. With acknowledgement of Dauben's pioneering historical work, this article further discusses Cantor's relation to the philosophy of Spinoza and Leibniz in depth, and his engagement in the Pantheismusstreit. Brief mention is made of Cantor's learning from F.A.Trendelenburg. +Penrose, Roger (2004). The Road to Reality. Alfred A. Knopf.. ISBN 0-679-77631-1 Chapter 16 illustrates how Cantorian thinking intrigues a leading contemporary theoretical physicist. +Rucker, Rudy (2005) [1982]. Infinity and the Mind. Princeton University Press.. ISBN 0-553-25531-2 Deals with similar topics to Aczel, but in more depth. +Rodych, Victor (2007). "Wittgenstein's Philosophy of Mathematics". In Edward N. Zalta (ed.). The Stanford Encyclopedia of Philosophy. Metaphysics Research Lab, Stanford University.. +Leonida Lazzari, L'infinito di Cantor. Editrice Pitagora, Bologna, 2008. +Quotations related to Georg Cantor at Wikiquote +Media related to Georg Cantor at Wikimedia Commons +Works by or about Georg Cantor at the Internet Archive +O'Connor, John J.; Robertson, Edmund F., "Georg Cantor", MacTutor History of Mathematics Archive, University of St Andrews +O'Connor, John J.; Robertson, Edmund F., "A history of set theory", MacTutor History of Mathematics Archive, University of St Andrews Mainly devoted to Cantor's accomplishment. +Georg Cantor, britannica.com +Stanford Encyclopedia of Philosophy: Set theory by Thomas Jech. The Early Development of Set Theory by José Ferreirós. +"Cantor infinities", analysis of Cantor's 1874 article, BibNum (for English version, click 'à télécharger'). +The Infinite Heist: Story of "the hard-fought journey to embed the concept of infinity into math's foundations" involving Cantor, Dedekind, Leopold Kronecker and others +Stede Bonnet (c. 1688 – 10 December 1718), known as the Gentleman Pirate, was an English pirate and landowner. Bonnet was born into a wealthy English family on the island of Barbados, and inherited the family estate after his father's death in 1694. Despite his lack of sailing experience, Bonnet decided he should turn to piracy and crime in the spring of 1717. He bought a sailing vessel, the Revenge, and travelled with his paid crew along the Eastern Seaboard of what is now the Eastern United States, capturing other vessels and burning other Barbadian ships. +Bonnet set sail for Nassau in the Bahamas, to the haven for pirates known as the "Republic of Pirates", but he was seriously wounded en route during an encounter with a Spanish warship. After arriving in Nassau, Bonnet met the infamous pirate Blackbeard, with whom he developed a close union despite initially being enemies. Incapable of leading his crew, Bonnet temporarily ceded his ship's command to Blackbeard. Before separating in December 1717, Blackbeard and Bonnet plundered and captured merchant ships along the East Coast. After Bonnet failed to capture the Protestant Caesar, his crew abandoned him to join Blackbeard aboard the Queen Anne's Revenge. Bonnet stayed on Blackbeard's ship as a guest, and did not command a crew again until summer 1718, when he was pardoned by North Carolina governor Charles Eden and received clearance to undertake privateering against Spanish shipping interests. Bonnet was tempted to resume his piracy but did not want to lose his pardon, so he adopted the alias "Captain Thomas" and changed his ship's name to Royal James. He had returned to piracy by July of 1718. +In August, Bonnet anchored Royal James on an estuary of the Cape Fear River to careen and repair the ship. In late August and September, Colonel William Rhett, with the authorisation of South Carolina's governor Robert Johnson, led a naval expedition against pirates on the river. Rhett's and Bonnet's men engaged in combat for hours, but the outnumbered pirates ultimately surrendered. Rhett arrested the pirates and brought them to Charles Town (now Charleston, South Carolina) in early October. Bonnet escaped on 24 October, but was soon recaptured on Sullivan's Island. On 10 November, Bonnet was brought to trial and charged with two acts of piracy. Judge Nicholas Trott sentenced him to death. Bonnet wrote to Johnson to ask for clemency, but Johnson endorsed the judge's decision, and Bonnet was hanged in Charles Town on 10 December. +Stede Bonnet was born in 1688, and he was christened at Christ Church parish on 29 July 1688. His parents, Edward and Sarah Bonnet, owned an estate of over 400 acres (160 ha) southeast of Bridgetown, Barbados, which was bequeathed to Bonnet upon his father's death in 1694. Bonnet received a good education, and lived as one of the richest and highest members of Bridgetown society. After his parents’ death, Bonnet was raised by his maternal grandfather and deputy secretary of Barbados, John Whetstone, who was son of Thomas Whetstone and grandnephew of Oliver Cromwell. Bonnet married Mary Allamby in Bridgetown on 21 November 1709. They had three sons – Allamby (who died by 1715), Edward, and Stede Jr – and a daughter, Mary. The three surviving children were all younger than five when their father abandoned them for piracy. +In the Oxford Dictionary of National Biography, historian Robert C. Ritchie suggests that Bonnet was driven to piracy to escape his bad marriage, or to "recapture his more adventurous youth". Details of Bonnet's military service are unclear, but he held the rank of major in the Barbados militia. Bonnet's militia service coincided with the War of the Spanish Succession, but there is no record that he took part in the fighting. +During the spring of 1717, Bonnet decided to become a pirate, despite having no knowledge of shipboard life. He contracted a local shipyard to build him a sloop, which he equipped with ten guns and named the Revenge. This was unusual, as most pirates seized their ships by mutiny, by boarding them, or by converting a privateer vessel to a pirate ship. Bonnet enlisted a crew of more than seventy men. He relied on his quartermaster and officer for their knowledge of sailing, and as a result, he was not highly respected by his crew. In another break from tradition, Bonnet paid his crew wages, not shares of plunder as most pirates did. After staying at anchor for several days, telling inquirers that he was going to use the Revenge for island trading, Bonnet departed Carlisle Bay, Barbados, under the cover of darkness. +Bonnet's initial cruise took him to the coast of the Colony of Virginia, near the entrance of the Chesapeake Bay, where he captured and plundered four vessels, and burned the Barbadian ship Turbes to keep news of his crimes from his home island. He sailed north to New York City, taking two more ships, and picking up naval supplies and releasing captives at Gardiners Island. Finding that New York was a less profitable hunting ground, by August Bonnet had returned to the Carolinas, where he attacked two more ships, a brigantine from Boston and a Barbadian sloop. He stripped the brigantine, but brought the cargo-filled Barbadian sloop to an inlet off North Carolina to use for careening and repairing the Revenge. After the Barbadian sloop's tackle was used to careen the Revenge, the ship was dismantled for timber, and the remains were burned. In September, Bonnet set course for Nassau, which was an infamous pirate den on the island of New Providence in the Bahamas. En route, he encountered, fought, and escaped from a Spanish man-of-war. The Revenge was badly damaged, Bonnet was seriously wounded, and half the crew of the sloop was killed or wounded in the encounter. Putting in at Nassau, Bonnet replaced his casualties and refitted the Revenge, increasing the sloop's armament to twelve guns. +While at Nassau, Bonnet met Captain Benjamin Hornigold and Blackbeard for the first time. Blackbeard and Bonnet struck up an unlikely friendship, and decided to cruise together, with Blackbeard taking command while Bonnet kept to his cabin, recovering from his wounds. The ship sailed northward to Delaware Bay, where they plundered eleven ships. On 29 September, the Revenge, captained by Blackbeard, plundered the sloop Betty, which had a cargo full of Madeira wine. Captain Codd, whose merchant ship was taken on 12 October, described Bonnet as walking the deck in his nightshirt, lacking any command and still unwell from his wounds. The Revenge later captured and looted the Spofford and Sea Nymph, which were leaving Philadelphia. On 22 October, the Revenge stopped and robbed the Robert and Good Intent of their supplies. On 17 November Blackbeard captured the French slave ship La Concorde which he took as his own, naming her Queen Anne's Revenge. Bonnet took back command of Revenge. +Some time after 19 December, Bonnet and Blackbeard separated. Bonnet sailed into the western Caribbean. On 28 March 1718, he encountered the 400-ton merchant vessel Protestant Caesar off Honduras. The ship escaped him, and his frustrated crew became restive. When Bonnet encountered Blackbeard again shortly afterwards at Turneffe Atoll, Bonnet's crew deserted him to join Blackbeard. Calling Bonnet on board his own ship, Blackbeard essentially imprisoned him, suggesting that as a gentleman he would prefer a life of leisure to that of command. Blackbeard put a henchman named Richards in command of the Revenge. Bonnet, surprised that his colleague had betrayed him, found himself as a guest aboard Blackbeard's ship, the Queen Anne's Revenge. Bonnet confided in a few loyal crew members that he was ready to give up his criminal life if he could exile himself in Spain or Portugal. +Under Richards, the Revenge captured a Jamaican sloop, the Adventure, captained by David Herriot, who joined the pirates. Bonnet accompanied Blackbeard to South Carolina, where Blackbeard's four vessels blockaded the port of Charles Town in the late spring. Needing a place to rest and refit his vessels, Blackbeard headed north to Topsail Island, where the Queen Anne's Revenge ran aground and was lost in June. With dissent rising amongst the pirates, Blackbeard allowed Bonnet to take command of the Revenge again. Leaving the remaining vessels at Topsail Island, Blackbeard and Bonnet went ashore and journeyed to Bath, which was the capital of North Carolina. Once there, both men accepted pardons from Governor Charles Eden under King George's Act of Grace, putatively on condition of their renouncing piracy forever. While Blackbeard quietly returned to Topsail Island, Bonnet stayed in Bath to get a "clearance" to sail the Revenge to Denmark's Caribbean colony of St. Thomas, where he planned to buy a letter of marque and go privateering against Spanish shipping. Eden granted Bonnet this clearance. +Bonnet returned to Topsail Island to find that Blackbeard had sunk several of his pirate ships, taken all the loot, and robbed the Revenge and two other vessels of the squadron of most of their supplies, before sailing away. Bonnet then rescued seventeen pirates who had been marooned on a sand bar by Blackbeard for disagreeing with his plans. +Bonnet began to resupply the ravaged Revenge, having bumboats bring food on board from ashore. One of these informed Bonnet that Blackbeard was moored in Ocracoke Inlet with only eighteen pirates. Wanting to take revenge on Blackbeard, Bonnet set sail for Ocracoke, but missed Blackbeard's departure by several hours. Bonnet gave chase but failed to catch the other pirate. Although Bonnet apparently never discarded his hopes of reaching St. Thomas and getting his letter of marque, Blackbeard had stolen the food and supplies he and his men needed to subsist. +Hoping to preserve his pardon while returning to his pirate ways, Bonnet adopted the alias "Captain Thomas" and changed the Revenge's name to the Royal James. The name Royal James that Bonnet conferred on his sloop was possibly a reference to James Francis Edward Stuart, the son of the deposed James II of England who was at the time plotting to retake the throne of England from George I of Great Britain. +Bonnet further tried to disguise his return to piracy by engaging in a pretence of trade with the next two vessels he robbed. Soon afterward, Bonnet quit the charade of trading and reverted to naked piracy, encouraged by his crew. Cruising north to Delaware Bay, in July Bonnet pillaged several more vessels, totalling eleven since he had received his pardon. He took several prisoners, some of whom joined his pirate crew. While Bonnet set loose most of his prizes after looting them, he retained control of the last two sloops he captured. +During these operations the Royal James had begun to leak badly and was in need of careening. Bonnet sailed the ship to the Cape Fear River, a well-known pirate rendezvous, to undertake repairs which were estimated to require two months of work. Shortly afterwards, a small shallop entered the river and was captured. Bonnet had the shallop broken up to help repair the Royal James. The work of careening was done, in whole or in part, by the prisoners Bonnet had captured. Bonnet threatened at least one man with marooning if he did not work with the slaves to repair the Royal James. Bonnet remained in the Cape Fear River, intending to wait out the hurricane season there. +By the end of August, news had reached Charles Town that a pirate vessel was moored in the Cape Fear River. Robert Johnson, governor of South Carolina, authorised Colonel William Rhett to lead a naval expedition against the pirates, believing that North Carolina was in league with the pirates and would not do anything if left to combat them alone. After a false start due to the appearance of another pirate ship near Charles Town, Rhett arrived at the mouth of the Cape Fear River on 26 September with two eight-gun sloops, the Henry and the Sea Nymph, and a force of 130 militia men. Bonnet sent three canoes out to Rhett's ships and was disconcerted to learn that they were warships. +Bonnet planned to fight his way out to sea in the morning with his crew of 45. He also wrote a letter to Johnson, threatening to burn all the ships in Charles Town Harbour. At daybreak, on 27 September 1718, Bonnet set sail toward Rhett's force, and all three sloops opened fire, initiating the Battle of Cape Fear River. Rhett sailed his two ships towards the Royal James, forcing the pirate to stay in the shallow waters along the river. Royal James soon ran aground, but in the chase so did the South Carolinian vessels. The Sea Nymph was stuck too far away to be of use, leaving only the Henry in range of the Royal James. +The battle was at a stalemate for the next five or six hours, with all the participants immobilized. Bonnet's men had the advantage that their deck was heeled away from their opponents, giving them cover, while the Henry's deck was tilted toward the pirates, thus exposing Rhett's men to punishing musket volleys. Bonnet's force suffered twelve casualties while killing ten and wounding fourteen of Rhett's 70-man crew. Most of Bonnet's men fought enthusiastically, challenging their enemies to board and fight hand to hand, and tying a knot in their flag as a mock signal to come aboard and render aid. Nevertheless, some of the prisoners who had been forced to join the pirate crew refused to fire on Rhett's men, and one narrowly escaped death at Bonnet's hands in the confusion of the engagement. +The battle was ultimately decided when the rising tide lifted Rhett's sloops free while temporarily leaving the Royal James stranded. As the Henry closed with Bonnet's ship, his crew demanded that he surrender. Bonnet at first refused, saying that he would blow up the ship before surrendering, and threatening to shoot any man who did. Despite this, the crew won Bonnet over and a white flag was raised. After negotiations were shouted between the two ships, Bonnet agreed to surrender unconditionally. Rhett returned to Charles Town with his prisoners and the Royal James on 3 October. +In Charles Town, Bonnet was separated from the bulk of his crew and held for three weeks along with his boatswain, Pell, and his sailing master, Herriot, at the home of Town Marshal Nathaniel Partridge. The remaining crew were held outside of Charles Town at the public watchhouse, due to the town lacking an actual jail. On 24 October, Bonnet and Herriot escaped, probably by colluding with local merchant Richard Tookerman. Johnson at once placed a £700 bounty on Bonnet's head and dispatched search teams to track him down. Bonnet and Herriot, accompanied by a slave and a Native American who wanted to become pirates, obtained a boat and made for the north shore of Charles Town Harbour, but foul winds and lack of supplies forced the four of them onto Sullivan's Island. Johnson sent a posse under Rhett to Sullivan's Island to hunt for Bonnet. +The posse discovered Bonnet after an extensive search, and opened fire, killing Herriot and wounding the other two of Bonnet's followers. Bonnet surrendered and was returned to Charles Town, imprisoned in a safer location. While awaiting trial, some sort of civil uprising in his support took place within the city, an event authorities would later describe as having nearly resulted in the burning of the town and the overthrow of the government. Bonnet possibly awaited his execution in the Court of Guard, a militia guardhouse in the city where the Exchange and Provost stood as of 2007. +The Vice-Admiralty judge Nicholas Trott sat in judgment on Bonnet's crew and sentenced most of them to hang on 5 November, with Pell turning King's evidence against his fellow pirates. They were executed on 8 November. Bonnet was brought to trial before Trott two days later, and formally charged with two acts of piracy, against the Francis and the Fortune, whose commanders were on hand to testify against Bonnet in person. Bonnet pleaded not guilty to the offences but provided little defence for his actions. On 12 November, Trott sentenced Bonnet to death by hanging, using his final speech to recite Bonnet's crimes and suggest that in the afterlife he would be "in the lake which burneth with fire and brimstone, which is the second death". +Johnson decreed that Bonnet would be executed on 10 December. While awaiting this, Bonnet wrote to Johnson, begging abjectly for clemency and promising to become a "menial servant" to Johnson and his government if he was spared. He sent several other letters, including to Rhett who offered to take Bonnet to England for a re-trial if Johnson allowed it. Bonnet's visibly disintegrating mind moved many Carolinians to pity, particularly the female population, and London papers later reported that the governor delayed his execution seven times. Johnson stayed firm, and Bonnet was hanged at White Point, in Charles Town, on 10 December. After being cut down, his body was tossed alongside those of his crew into a nearby marshland below the low water mark. No marker was provided for the graves. +The actual degree of authority any pirate captain exercised over his crew was questionable, as he had no access to the procedures and sanctions of Admiralty law that supported government captains. Many pirate captains were elected by their crews and could be deposed in the same manner. At his trial, Bonnet downplayed his own authority over his pirate crew. He told the court that his crew engaged in piracy against his will, and said he had not given them consent to rob any vessels. The court did not accept these protestations. Pell testified that Bonnet's quartermaster, Robert Tucker, had more power than Bonnet. Despite this, at Delaware Bay Bonnet ordered two of his crew to be flogged for breaches of discipline. Pirates did not lightly submit to flogging, as they resented the frequent use of this punishment in the naval and merchant services from which most of them came. +In Wilmington, North Carolina, a local legend states that Bonnet allegedly captured a former crewmate’s wife. During routine maintenance at Cape Fear, the crew decided to visit the plantation of a retired crewmate. The crewmate allowed them to partake in food, shelter and alcohol, but he had to travel away for business. In a drunken stupor, Bonnet ordered his men to capture the young wife and refused to listen to arguments against the act. She was bound and taken aboard, where the pirates immediately set sail to the mouth of the Cape Fear River. The crew released her bindings and allowed her free rein on deck, so they could focus their attentions on sailing. She jumped off the ship and swam to the Old Dram Tree to wait for her husband. The legend states that Bonnet and his crew never returned to the plantation in case of retaliation. +Bonnet is alleged to have been one of the few pirates to make his prisoners walk the plank. The detailed record of his trial, however, makes no mention of this method of execution, and the historian Hugh F. Rankin argues that the idea is a modern fabrication. Marcus Rediker, Professor of History at the University of Pittsburgh, agrees that pirates were unlikely to make captives walk the plank, preferring either bloodless assaults or far more violent methods of murder. +Portrayals of Bonnet extend to video games, such as Sid Meier's Pirates! and Assassin's Creed IV: Black Flag. In the 1941 film The Devil and Daniel Webster, the Devil summons Bonnet to join a jury of the damned, announcing Bonnet and fellow seaman Floyd Ireson as "the fiendish butchers". +The period comedy television series Our Flag Means Death, created by David Jenkins, stars Rhys Darby as Bonnet and Taika Waititi as Blackbeard. Jenkins depicts a romantic relationship between the two. +Botting, Douglas (1978). The Pirates. Time-Life Books. ISBN 0-8094-2652-8. +Bowden, Oliver (1 November 2013). Assassin's Creed: Black Flag. Penguin UK. ISBN 9780718193768. +Butler, Lindley S. (2000). Pirates, Privateers, & Rebel Raiders of the Carolina Coast. Chapel Hill, North Carolina: UNC Press. ISBN 0-87169-240-6. +Converse, Cris (2016). Assassin's Creed IV Black Flag Game Guide. Booksmango. ISBN 9781633235014. +Cordingly, David (1996). Under the Black Flag: The Romance and Reality of Life Among the Pirates. New York: Random House. ISBN 0-679-42560-8. +Fox, Edward Theophilus (2013). Piratical Schemes and Contracts: Pirate Articles and Their Society 1660-1730 (PhD). Exeter: University of Exeter. +Geltzer, Jeremy (2017). Film Censorship in America: A State-By-State History. McFarland & Company. ISBN 9781476669526. +Gosse, Philip (1924). The Pirates' Who's Who. New York: Burt Franklin. ISBN 1-60303-284-3. {{cite book}}: ISBN / Date incompatibility (help) +Johnson, Charles; Wetmore, Alexander (1724). A General History of the Pyrates, from their First Rise and Settlement in the Island of Providence, to the Present Time... (2 ed.). London: T. Warner. +Pringle, Patrick (2001). Jolly Roger: The Story of the Great Age of Piracy. Mineola, New York: Dover Publications. ISBN 0-486-41823-5. +Rankin, Hugh F. (1965). The Pirates of Colonial North Carolina. Raleigh: State Department of Archives and History. +Rankin, Hugh F. (1969). The Golden Age of Piracy. New York: Holt, Rinehart, and Winston. SBN 03-081018-3. +Ritchie, Robert C. (2008). "Bonnet, Stede". Oxford Dictionary of National Biography (online ed.). Oxford University Press. doi:10.1093/ref:odnb/61696. (Subscription, Wikipedia Library access or UK public library membership required.) +Sanders, Joanne McRee, ed. (1982). Barbados Records: Marriages, 1693–1800. Vol. 1. Houston: Sanders Historical Publications. +Sanders, Joanne McRee, ed. (1984). Barbados Records: Baptisms, 1637–1800. Baltimore: Genealogical Publications. +Seitz, Don Carlos; Gospel, Howard F.; Wood, Stephen (2002). Under the Black Flag: Exploits of the Most Notorious Pirates (2 ed.). Mineola, New York: Courier Dover Publications. ISBN 0-486-42131-7. +Woodard, Colin (2007). The Republic of Pirates: Being the True and Surprising Story of the Caribbean Pirates and the Man who Brought Them Down. New York: Harcourt. ISBN 978-0-15-603462-3. +Cowse, Benjamin, ed. (1719). The Tryals of Major Stede Bonnet, and Other Pirates (PDF). London: Rose and Crown in St Paul's Church-Yard. +Brown, Paul (2015). Sins Dyed In Blood: The Lost Pirate of Blackbeard's Golden Age. Superelastic. ISBN 9781311947765. Retrieved 16 February 2022. +Proceedings of Stede Bonnet's Trial from A Complete Collection of State Trials and Proceedings for High Treason and Other Crimes and Misdemeanors from the Earliest Period to the Year 1783 by Thomas Bayly Howell and William Cobbett, from Google Books +Aftermath is the fourth studio album by the English rock band the Rolling Stones. The group recorded the album at RCA Studios in California in December 1965 and March 1966, during breaks between their international tours. It was released in the United Kingdom on 15 April 1966 by Decca Records and in the United States in late June by London Records. It is the band's fourth British and sixth American studio album, and closely follows a series of international hit singles that helped bring the Stones newfound wealth and fame rivalling that of their contemporaries the Beatles. +Aftermath is considered by music scholars to be an artistic breakthrough for the Rolling Stones. It is their first album to consist entirely of original compositions, all of which were credited to Mick Jagger and Keith Richards. The band's original leader Brian Jones reemerged as a key contributor and experimented with instruments not usually associated with popular music, including the sitar, Appalachian dulcimer, Japanese koto and marimbas, as well as playing guitar and harmonica. Along with Jones' instrumental textures, the Stones incorporated a wider range of chords and stylistic elements beyond their Chicago blues and R&B influences, such as pop, folk, country, psychedelia, Baroque and Middle Eastern music. Influenced by intense love affairs, tensions within the group and a demanding touring itinerary, Jagger and Richards wrote the album around psychodramatic themes of love, sex, desire, power and dominance, hate, obsession, modern society and rock stardom. Women feature as prominent characters in their often dark, sarcastic, casually offensive lyrics. +The album's release was briefly delayed by controversy over the original packaging idea and title – Could You Walk on the Water? – due to the London label's fear of offending Christians in the US with its allusion to Jesus walking on water. In response to the lack of creative control, and without another idea for the title, the Stones bitterly settled on Aftermath, and two different photos of the band were used for the cover to each edition of the album. The UK release featured a run-time of more than 52 minutes, the longest for a popular music LP up to that point. The American edition was issued with a shorter track listing, substituting the single "Paint It Black" in place of four of the British version's songs, in keeping with the industry preference for shorter LPs in the US market at the time. +Aftermath was an immediate commercial success in both the UK and the US, topping the British albums chart for eight consecutive weeks and eventually achieving platinum certification from the Recording Industry Association of America. An inaugural release of the album era and a rival to the contemporaneous impact of the Beatles' Rubber Soul (1965), it reflected the youth culture and values of 1960s Swinging London and the burgeoning counterculture while attracting thousands of new fans to the Rolling Stones. The album was also highly successful with critics, although some listeners were offended by the derisive attitudes towards female characters in certain songs. Its subversive music solidified the band's rebellious rock image while pioneering the darker psychological and social content that glam rock and British punk rock would explore in the 1970s. Aftermath has since been considered the most important of the Stones' early, formative music and their first classic album, frequently ranking on professional lists of the greatest albums. +In 1965, the Rolling Stones' popularity increased markedly with a series of international hit singles written by the band's lead singer Mick Jagger and their guitarist Keith Richards. This success attracted the attention of Allen Klein, an American businessman who became their US representative in August while Andrew Loog Oldham, the group's manager, continued in the role of promoter and record producer. One of Klein's first actions on the band's behalf was to force Decca Records to grant a $1.2 million royalty advance to the group ($12,182,819 in 2025), bringing the members their first signs of financial wealth and allowing them to purchase country houses and new cars. Their October–December 1965 tour of North America was the group's fourth and largest tour there up to that point. According to the biographer Victor Bockris, through Klein's involvement, the concerts afforded the band "more publicity, more protection and higher fees than ever before". +By this time, the Rolling Stones had begun to respond to the increasingly sophisticated music of the Beatles, in comparison to whom they had long been promoted by Oldham as a rougher alternative. With the success of the Jagger-Richards-penned singles "(I Can't Get No) Satisfaction" (1965), "Get Off of My Cloud" (1965) and "19th Nervous Breakdown" (1966), the band increasingly rivalled the Beatles' musical and cultural influence. The Stones' outspoken, surly attitude on songs like "Satisfaction" alienated the Establishment detractors of rock music, which, as the music historian Colin King explains, "only made the group more appealing to those sons and daughters who found themselves estranged from the hypocrisies of the adult world – an element that would solidify into an increasingly militant and disenchanted counterculture as the decade wore on." Like other contemporary British and American rock acts, with Aftermath the Stones sought to create an album as an artistic statement, inspired by the Beatles' achievements with their December 1965 release Rubber Soul – an LP that Oldham later described as having "changed the musical world we lived in then to the one we still live in today". +Within the Stones, tensions were rife as Brian Jones continued to be viewed by fans and the press as the band's leader, a situation that Jagger and Oldham resented. The group dynamics were also affected by some of the band members' romantic entanglements. Jones' new relationship with the German model Anita Pallenberg, which had taken on sadomasochistic aspects, helped renew his confidence and encourage him to experiment musically, while her intelligence and sophistication both intimidated and elicited envy from the other Stones. Jagger came to view his girlfriend, Chrissie Shrimpton, as inadequate by comparison; while Jagger sought a more glamorous companion commensurate with his newfound wealth, the aura surrounding Jones and Pallenberg contributed to the end of his and Shrimpton's increasingly acrimonious relationship. Richards' relationship with Linda Keith also deteriorated as her drug use escalated to include Mandrax and heroin. The band's biographer Stephen Davis describes these entanglements as a "revolution under way within the Stones", adding that "Anita Pallenberg restored the faltering Brian Jones to his place in the band and in the Rolling Stones mythos. Keith Richards fell in love with her too, and their romantic triad realigned the precarious political axis within the Stones." +Aftermath is the first Stones LP to be composed entirely of original material by the group. All the songs were, at Oldham's instigation, written by and credited to the songwriting partnership of Jagger and Richards. The pair wrote much of the material during the October–December 1965 tour and recording began immediately after the tour ended. According to the band's bassist Bill Wyman in his book Rolling with the Stones, they originally conceived Aftermath as the soundtrack for a planned film, Back, Behind and in Front. The plan was abandoned after Jagger met the potential director, Nicholas Ray, and disliked him. The recording sessions took place at RCA Studios in Los Angeles on 8–10 December 1965 and, following promotion for their "19th Nervous Breakdown" single and an Australasian tour, on 6–9 March 1966. Charlie Watts, the group's drummer, told the press that they had completed 10 songs during the first block of sessions; according to Wyman's book, at least 20 were recorded in March. Among the songs were four tracks issued on singles by the Rolling Stones in the first half of 1966, the A-sides of which were "19th Nervous Breakdown" and "Paint It Black". "Ride On, Baby" and "Sittin' on a Fence" were also recorded during the sessions but were not released until the 1967 US album Flowers. +Referring to the atmosphere at RCA, Richards told Beat Instrumental magazine in February 1966: "Our previous sessions have always been rush jobs. This time we were able to relax a little, take our time." The main engineer for the album, Dave Hassinger, was pivotal in making the group feel comfortable during the sessions, as he let them experiment with instrumentals and team up with session musicians like Jack Nitzsche to variegate their sound. Wyman recalled that Nitzsche and Jones would pick up instruments that were in the studio and experiment with sounds for each song. According to Jagger, Richards was writing a lot of melodies and the group would perform them in different ways, which were mainly thought out in the studio. In the recollection of the engineer Denny Bruce, the songs often developed through Nitzsche organising the musical ideas on piano. Wyman was later critical of Oldham for nurturing Jagger and Richards as songwriters to the exclusion of the rest of the band. The bassist also complained that "Paint It Black" should have been credited to the band's collective pseudonym, Nanker Phelge, rather than Jagger–Richards, since the song originated from a studio improvisation by himself, Jones and Watts, with Jones providing the melody line. +Jones proved important in shaping the album's tone and arrangements, as he experimented with instruments that were unusual in popular music, such as the marimba, sitar and Appalachian dulcimer. Davis cites the "acid imagery and exotic influences" on Rubber Soul, particularly George Harrison's use of the Indian sitar on "Norwegian Wood", as the inspiration for Jones' experimentation with the instrument in January 1966: "One night George put the massive sitar in Brian's hands, and within an hour Brian was working out little melodies." According to Nitzsche, Jones deserved a co-writing credit for "Under My Thumb", which Nitzsche recalled as being an unoriginal-sounding three-chord sequence until Jones discovered a Mexican marimba left behind from a previous session, and transformed the piece by providing its central riff. Wyman agreed, saying, "Well, without the marimba part, it's not really a song, is it?" +During the recording sessions, Richards and Oldham dismissed Jones' interest in exotic instrumentation as an affectation. According to the music journalist Barbara Charone, writing in 1979, everyone connected with the Stones credited Jones for "literally transforming certain records with some odd magical instrument". While Nitzsche was shocked at how cruelly they treated Jones, he later said that Jones was sometimes absent or incapacitated by drugs. Hassinger recalled seeing Jones often "laying on the floor, stoned or on some trip" and unable to play, but that his bandmates would wait for him to leave rather than entering into an argument as other bands would. +Because of Jones' distractions, Richards ended up playing most of the guitar parts for Aftermath, making it one of the first albums to have him do so. Richards later said he found the challenge musically rewarding but resented Jones for his unprofessional attitude when the band were under extreme pressure to record and maintain a hectic touring schedule. On some songs, Richards supported Wyman's bass lines with a fuzz bass part, which the music historians Philippe Margotin and Jean-Michel Guesdon suggest was influenced by Paul McCartney's use on the track "Think for Yourself" (from Rubber Soul). Aftermath was also the first Stones LP to be released with the majority of its tracks in true stereo, as opposed to electronically created stereo. +According to the musicologist David Malvinni, Aftermath is the culmination of the Rolling Stones' stylistic development dating back to 1964, a synthesis of previously explored sounds from the blues, rock and roll, rhythm and blues, soul, folk rock and pop ballads. Margotin and Guesdon go further in saying the album shows the Stones to be free from influences that had overwhelmed their earlier music, specifically the band's Chicago blues roots. Instead, they say, the record features an original style of art rock that resulted from Jones' musical experimentation and draws not only on the blues and rock but also pop, R&B, country, Baroque, classical and world music. Musical tones and scales from English lute song and Middle Eastern music feature among Aftermath's riff-based rock and blues (in both its country and urban forms). While still considering it a blues rock effort, Rob Sheffield likens the music to a collaboration between the art rock band the Velvet Underground and the Stax house band. Neil McCormick of The Telegraph described the album as "R’n’B with a bracingly modern attitude", noting the expanded instrumental repertoire of Jones. In a 1995 interview for Rolling Stone, Jagger regarded it as a stylistically diverse work and milestone for him that "finally laid to rest the ghost of having to do these very nice and interesting, no doubt, but still, cover versions of old R&B songs – which we didn't really feel we were doing justice, to be perfectly honest". +Along with their 1967 follow-up, Between the Buttons, Aftermath is cited by Malvinni as part of the Rolling Stones' pop-rock period as it features a chordal range more diverse and inclusive of minor chords than their blues-based recordings. Pat Gilbert of Mojo stated that the album is defined by its "thoughtfully crafted pop songs, fleshed out with exotic instruments including sitar, dulcimer, harpsichord, marimbas and bells". According to Kevin Courrier, the Stones use "softly intricate" arrangements that lend the record a "seductive ambience" similar to Rubber Soul, particularly on "Lady Jane", "I Am Waiting", "Under My Thumb" and "Out of Time". The latter two songs, among Aftermath's more standard pop-rock titles, are often-cited examples of Jones interweaving unconventional instruments and quirky sounds into the album's sonic character, his use of the marimba featured on both. In the opinion of Philip Norman, Jones' varied contributions give Aftermath both the "chameleon colours" associated with Swinging London fashion and a "visual quality" unlike any other Stones album. Robert Christgau says the texture of the Stones' blues-derived hard rock is "permanently enriched" as Jones "daub[s] on occult instrumental [colours]", Watts "mold[s] jazz chops to rock forms", Richards "rock[s] roughly on" and the band "as a whole learn[s] to respect and exploit (never revere) studio nuance"; Wyman's playing here is described by Sheffield as the "funkiest" on a Stones LP. +Citing individual songs, Rolling Stone describes Aftermath as "an expansive collection of tough riffs ('It's Not Easy') and tougher acoustic blues ('High and Dry'); of zooming psychedelia ('Paint It Black'), baroque-folk gallantry ('I Am Waiting') and epic groove (the eleven minutes of 'Goin' Home')". Jon Savage also highlights the stylistic diversity of the album, saying that it "range[s] from modern madrigals ('Lady Jane'), music-hall ragas ('Mother's Little Helper'), strange, curse-like dirges ('I Am Waiting') and uptempo pop ('Think') to several bone-dry blues mutations ('High and Dry', 'Flight 505' [and] 'Going Home')". The first four songs of Aftermath's US edition – "Paint It Black", "Stupid Girl", "Lady Jane" and "Under My Thumb" – are identified by the music academic James Perone as its most explicit attempts to transcend the blues-based rock and roll conventions of the Stones' past. He also notes how Richards' guitar riff and solo on the latter track are "minimalistic, in a fairly low tessitura and relatively emotionless", compared to previous Stones hits like "(I Can't Get No) Satisfaction", "Get Off of My Cloud" and "19th Nervous Breakdown". +Overall, the darker themes lead Margotin and Guesdon to call Aftermath "a sombre album in which desolation, paranoia, despair and frustration are echoed as track succeeds track". According to Steven Hyden, Jagger's songwriting explores "sex as pleasure, sex as power, love disguised as hate and hate disguised as love". Sheffield believes the time period's flower power ideology is recast in a dark light on "these tough, lean, desperately lonely songs", while Norman calls them "songs of callow male triumph" in which Jagger alternately displays childlike charm and misogynistic scorn. While songs such as "Stupid Girl" and "Under My Thumb" may be misogynistic, they are also interpreted as dark representations of the narrator's hateful masculinity. Misogyny, as on "Under My Thumb", "may be just a tool for restoring the fragile narcissism and arrogance of the male narrator", muses the music scholar Norma Coates. +Referring to the American version of the LP, Perone identifies numerous musical and lyrical features that lend Aftermath a conceptual unity which, although not sufficient for it to be considered a concept album, allows for the record to be understood "as a psychodrama around the theme of love, desire and obsession that never quite turns out right". It may also be read "as part of a dark male fantasy world, perhaps constructed as a means of dealing with loneliness caused by a broken relationship or a series of broken relationships with women." As Perone explains: +The individual songs seem to ping-pong back and forth between themes of love/desire for women and the desire to control women and out-and-out misogyny. However, the band uses musical connections between songs as well as the subtheme of travel, the use of feline metaphors for women and other lyrical connections to suggest that the characters whom lead singer Mick Jagger portrays throughout the album are really one and perhaps stem from the deep recesses of his psyche. +Aftermath's diverse musical style contrasts the dark themes explored in Jagger and Richards' lyrics, which often scorn female lovers. Margotin and Guesdon say that Jagger, who had been accused of misogyny before the album, is avenging real-life grievances with the songs, using "language and imagery that had the power to hurt". "Stupid Girl", which assails the "supposed greed and facile certitudes of women", is speculated by the writers to indirectly criticise Shrimpton. "High and Dry" expresses a cynical outlook on a lost romantic connection, while "Under My Thumb", "Out of Time" and "Think" show how "a man's revenge on his mistress (or perhaps wife) becomes a source of real pleasure". Shrimpton was devastated by the lyrics to "Out of Time", in which Jagger sings, "You're obsolete, my baby, my poor old-fashioned baby". Savage views such songs as evoking "the nastiness of the Rolling Stones' constructed image" in lyrical form by capturing Jagger's antipathy towards Shrimpton, whom he describes as a "feisty upper-middle-class girl who gave as good as she got". Conceding that male chauvinism became a key theme of the Stones' lyrics from late 1965 onwards, Richards later told Bockris: "It was all a spin-off from our environment ... hotels and too many dumb chicks. Not all dumb, not by any means, but that's how one got. You got really cut off." +In Guesdon and Margotin's view, the Stones express a more compassionate attitude towards women in "Mother's Little Helper", which examines a housewife's reliance on pharmaceutical drugs to cope with her daily life, and in "Lady Jane"'s story of romantic courtship. By contrast, Davis writes of Aftermath containing a "blatant attack on motherhood" and says that "Mother's Little Helper" addresses "tranquilised suburban housewives". According to Hassinger, his wife Marie provided the inspiration for "Mother's Little Helper" when she supplied some downers in response to a request from one of the studio staff. Davis likens "Lady Jane" to a Tudor love song with lyrics apparently inspired by Henry VIII's love letters to Lady Jane Seymour. Some listeners assumed the song was about Jagger's high-society friend Jane Ormsby-Gore, daughter of David Ormsby-Gore, 5th Baron Harlech. In what the music journalist Chris Salewicz terms a "disingenuous" claim, Jagger told Shrimpton that "Lady Jane" was written for her. +According to the music historian Simon Philo, like all the Stones' 1966 releases, Aftermath also reflects the band's "engagement" with Swinging London, a scene in which their decadent image afforded them a pre-eminent role by capturing the meritocratic ideals of youth, looks and wealth over social class. Author Ian MacDonald says that, as on Between the Buttons, the Stones perform here as storytellers of the scene and produce a "subversive" kind of pop music comparable to their contemporaries the Kinks. As Greil Marcus observes, the songs' protagonists can be interpreted as London bohemians severely disdainful of bourgeois comfort, positing "a duel between the sexes" and weaponizing humour and derision. Courrier adds that, as the "evil twin" of Rubber Soul, Aftermath takes that album's "romantic scepticism" and reframes it into a narrative of "underclass revolt". +Both "Mother's Little Helper" and "What to Do" connect modern society to feelings of unhappiness. The band's misgivings about their rock stardom are also touched on, including relentless concert tours in "Goin' Home" and fans who imitate them in "Doncha Bother Me", in which Jagger sings, "The lines around my eyes are protected by copyright law". Savage views the same lyric, preceded by the lines "All the clubs and the bars / And the little red cars / Not knowing why, but trying to get high", as the Stones' cynical take on Swinging London at a time when the phenomenon was receiving international attention and being presented as a tourist attraction. According to Perone, "I Am Waiting" suggests paranoia on the narrator's part and that societal forces are the cause, yet the song presents a degree of resignation in comparison to the album's other commentaries on class- and consumer-focused society. +During the recording, Oldham wanted Could You Walk on the Water? to be the album's title. In mid-January 1966, the British press announced that a new Rolling Stones LP carrying that title would be released on 10 March. In Rolling with the Stones, Wyman refers to the announcement as "audacity" on Oldham's part, although he supposes that Could You Walk on the Water? was their manager's proposed title for the band's March compilation album Big Hits (High Tide and Green Grass), rather than for Aftermath. At the time, Richards complained that Oldham was continually trying to "[get] in on the act" and "the Stones have practically become a projection of his own ego." A Decca spokesman said the company would not issue an album with such a title "at any price"; Oldham's idea upset executives at the company's American distributor, London Records, who feared the allusion to Jesus walking on water would provoke a negative response from Christians. +The title controversy embroiled the Stones in a conflict with Decca, delaying Aftermath's release from March to April 1966. Oldham had also proposed the idea of producing a deluxe gatefold featuring six pages of colour photos from the Stones' recent American tour and a cover depicting the band walking atop a California reservoir in the manner of "pop messiahs on the Sea of Galilee", as Davis describes. Rejected by Decca, the packaging was used instead for the US version of Big Hits, albeit with a cover showing the band standing on the shore of the reservoir. According to Davis, "in the bitterness (over lack of control of their work) that followed, the album was called Aftermath for want of another concept." Rolling Stone discerns a connection between the final title and themes explored in the music: "Aftermath of what? of the whirlwind fame that had resulted from releasing five albums in two years, for one thing ... And of hypocritical women". In Norman's view, an "aftermath" of the earlier title's "sacrilegious reference to the most spectacular of Christ's miracles" is "the very thing from which their God-fearing bosses may well have saved them", effectively avoiding the international furore that John Lennon created with his remark, in March, that the Beatles are "more popular than Jesus". +The front cover photo for Aftermath's British release was taken by Guy Webster and the cover design was done by Oldham, credited as "Sandy Beach". Instead of the elaborate essay that Oldham usually supplied for the Stones' albums, the liner notes were written by Hassinger and were a straight commentary on the music. Hassinger wrote in part: "It's been great working with the Stones, who, contrary to the countless jibes of mediocre comedians all over the world, are real professionals, and a gas to work with." For the cover image, close-ups of the band members' faces were diagonally aligned against a pale-pink and black coloured background, and the album title was cut in half across a line break. The back of the LP featured four black-and-white photos of the group taken by Jerry Schatzberg at his photographic studio in New York in February 1966. Jones was vocal in his dislike of Oldham's design when interviewed by Melody Maker in April. +For the American edition's cover, David Bailey took a colour photo of Jones and Richards in front of Jagger, Watts and Wyman, and set it against a blurred black background. According to Margotin and Guesdon, the photo was intentionally blurred as "an allusion to the psychedelic movement" and "corresponds better to the Stones' new artistic direction". +Aftermath's release was preceded by the Rolling Stones' two-week tour of Europe, which began on 25 March 1966. Decca issued the album in the United Kingdom on 15 April and an accompanying press release that declared: "We look to Shakespeare and Dickens and Chaucer for accounts of other times in our history, and we feel that tomorrow we will on many occasions look to the gramophone records of the Rolling Stones ... who act as a mirror for today's mind, action and happenings." On the same day, Time magazine published a feature titled "London: A Swinging City", belatedly recognising the Swinging London phenomenon a year after its peak. The British edition of Aftermath featured a run-time of 52 minutes and 23 seconds, the longest for a popular music LP at that time. The record was pressed with reduced volume to allow for its unusual length. In the Netherlands, Phonogram Records rush-released the album during the week of 14 May in response to high demand from Dutch music retailers. +In the US, London delayed the album's release to market the Big Hits compilation first but issued "Paint It Black" as a single in May. The song was originally released as "Paint It, Black", the comma being an error by Decca, which stirred controversy over a purported racial connotation. The band began their fifth North American tour on 24 June in support of Aftermath; it was their highest-grossing tour yet and, according to Richards, the start of a period of rapprochement between Jones, Jagger and himself. In late June, London released the American edition of the album with "Paint It Black" replacing "Mother's Little Helper", which was released in the same period in the US as a single with "Lady Jane" as the B-side. "Out of Time", "Take It or Leave It" and "What to Do" were similarly cut from the US LP's running order in an effort to significantly reduce its length, in keeping with the industry policy of issuing shorter albums and maximising the amount of LP releases for popular artists. Aftermath was the band's fourth British and sixth American studio album. +In the UK, Aftermath topped the Record Retailer LPs chart (subsequently adopted as the UK Albums Chart) for eight consecutive weeks, replacing the soundtrack album for The Sound of Music (1965) at number 1. It stayed on the chart for 28 weeks. Aftermath proved the fourth-highest-selling album of 1966 in the UK, and it also became a top-10 best-seller in the Netherlands. In the US, the album entered the Billboard Top LPs at number 117 on 2 July, making it the chart's highest new entry that week. By 13 August, it had risen to number 2 behind the Beatles' Yesterday and Today. That month, the Recording Industry Association of America awarded Aftermath a Gold certification for shipments of 500,000 copies; in 1989 it was certified Platinum for one million copies. +According to the pop historian Richard Havers, Aftermath's 1966 US chart run was assisted by the success of "Paint It Black", which topped the Billboard Hot 100 for two weeks in June. "Mother's Little Helper" was a Hot 100 hit as well, peaking at number 8 on the chart. The album's songs also proved popular among other recording artists, "Mother's Little Helper", "Take It or Leave It", "Under My Thumb" and "Lady Jane" all being covered within a month of Aftermath's release. Adding to Jagger and Richards' success as writers, Chris Farlowe topped the UK charts with his Jagger-produced recording of "Out of Time" in August. +Aftermath received highly favourable reviews in the music press. It was released just months before Bob Dylan's Blonde on Blonde and the Beatles' Revolver, albums by artists that Jagger and Richards had received comparisons to while Oldham was promoting the band's artistic maturation to the press. Among British critics, Richard Green of Record Mirror, in April 1966, began his review by saying: "Whether they realise it or not – and I think Andrew Oldham does – the Rolling Stones have on their hands the smash LP of the year with Aftermath", adding that it would take much effort to surpass their achievement. Green said the music is unmistakably rock and roll and was especially impressed by Watts' drumming. Keith Altham of the New Musical Express (NME) hailed the Stones as "masterminds behind the electric machines" who have recorded an LP of "the finest value for money ever". He described "Goin' Home" as a "fantastic R&B improvisation" and said that "Lady Jane", "Under My Thumb" and "Mother's Little Helper" have the potential to be great singles. Aftermath was regarded in Melody Maker as the group's best LP to date and one that would "effortlessly take Britain by storm". The magazine's reviewer applauded its focus "on big beat, power and interesting 'sounds'", noting how the use of dulcimer, sitar, organ, harpsichord, marimba and fuzz boxes creates an "overwhelming variety of atmospherics and tones". +While the lyrics' derisive attitude to women offended some listeners, this aspect received little attention in the British pop press or complaints from female fans. In the cultural journal New Left Review, Alan Beckett wrote that the band's lyrics could only be fully appreciated by an audience familiar with modern city life, particularly London. He said that the Stones' "archetypal girl", as first introduced in their 1965 song "Play with Fire", was "rich, spoiled, confused, weak, using drugs, etc.", adding that: "Anyone who has been around Chelsea or Kensington can put at least one name to this character." Responding in the same publication, the intellectual historian Perry Anderson (using the pseudonym of Richard Merton) defended the band's message as an audacious and satirical exposé on sexual inequality. He said that in songs such as "Stupid Girl" and "Under My Thumb", the Stones had "defied a central taboo of the social system" and that "they have done so in the most radical and unacceptable way possible: by celebrating it." +Some feminist writers defended "Under My Thumb". Camille Paglia considered the song "a work of art", despite its sexist lyrics, and Aftermath a "great album" with "rich sonorities". In a 1973 piece for Creem, Patti Smith recounted her response to the album in 1966: "The Aftermath album was the real move. two faced woman. doncha bother me. the singer displays contempt for his lady. he's on top and that's what I like. then he raises her as queen. his obsession is her. 'goin home.' What a song ... stones music is screwing music." +Among US commentators, Bryan Gray wrote in the Deseret News: "This album does the best job yet of alienating the over-twenties. The reason – they attempt to sing." Record World's review panel selected the album as one of their three "Albums of the Week", predicting a major seller while highlighting "Paint It Black" as "only the first of a series of hot [tracks]". Billboard's reviewer predicted that Aftermath would become another hit for the Stones, citing "Paint It Black" as the focal point of the hard rock album and revering Oldham for his production. Cash Box was extremely impressed by the LP and also predicted immediate chart success, saying "Lady Jane" and "Goin' Home" in particular are likely to attract considerable notice. Writing in Esquire in 1967, Robert Christgau said that the Stones' records present the only possible challenge to Rubber Soul's place as "an album that for innovation, tightness and lyrical intelligence" far surpassed any previous work in popular music. About two years later, in Stereo Review, he included the American Aftermath in his basic rock "library" of 25 albums and attributed the Stones' artistic identity largely to Jagger, "whose power, subtlety and wit are unparalleled in contemporary popular music". While suggesting Jagger and Richards rank second behind John Lennon and Paul McCartney as composers of melody in rock, Christgau still considered it the best album in any category and wrote: +Rock aficionados class the Stones with the Beatles, but perhaps they haven't impressed a wider audience because their devotion to the music is pure: the Hollyridge Strings will never record an album of Jagger–Richard melodies. But for anyone willing to discard his preconceptions, Aftermath is a great experience, a distillation of everything that rock and blues are about. +Aftermath is often considered one of the most important of the Rolling Stones' early albums. It was an inaugural release of the album era, during which the LP replaced the single as the primary product and form of artistic expression in popular music. As with Rubber Soul, the extent of Aftermath's commercial success foiled the music industry's attempts to re-establish the LP market as the domain of wealthier, adult record-buyers – a plan that had been driven by the industry's disapproval of the uncouth image associated with Jagger and their belief that young record-buyers were more concerned with singles. In Malvinni's opinion, Aftermath was "the crucial step for the Stones' conquering of the pop world and their much-needed answer" to Rubber Soul, which had similarly embodied the emergence of youth culture in popular music during the mid-1960s. With their continued commercial success, the Stones joined the Beatles and the Who as one of the few rock acts who were able to follow their own artistic direction and align themselves with London's elite bohemian scene without alienating the wider youth audience or appearing to compromise their working-class values. Speaking on the cultural impact of Aftermath's British release in 1966, Margotin and Guesdon say it was, "in a sense, the soundtrack of Swinging London, a gift to hip young people" and "one of the brightest stars of the new culture (or counterculture) that was to reach its zenith the following year in the Summer of Love". +Aftermath is regarded as the most artistically formative of the Rolling Stones' early work. Their new sound on the album helped expand their following by the thousands, while its content solidified their dark image. As Ritchie Unterberger observes, its contemptuous perspective about society and women contributed significantly to the group's reputation as "the bad boys" of rock music. According to John Mendelsohn from PopMatters, the social commentary of "Mother's Little Helper" in particular "cemented their reputation as a subversive cultural force", as it exposed the hypocrisy of mainstream culture's exclusive association of psychoactive drug use with addicts and rock stars. The NME's Jazz Monroe writes that Aftermath simultaneously disowned and reimagined rock tradition and forever elevated the Stones as equals to the Beatles. Writing for The A.V. Club, Hyden describes it as "a template for every classic Stones album that came afterward", crediting its "sarcastic, dark and casually shocking" songs with introducing themes Jagger would explore further in the future through a "complex, slippery persona" that allowed him to "be good and evil, man and woman, tough and tender, victim and victimiser". This deliberately "confounding, complicated image" helped make Jagger one of the most captivating lead musicians in rock, Hyden concludes. +The album proved influential in the development of rock music. Its dark content pioneered the darker psychological and social themes of glam rock and British punk rock in the 1970s. The music historian Nicholas Schaffner, in The British Invasion: From the First Wave to the New Wave (1982), acknowledges the Stones on the album for being the first recording act to engage themes of sex, drugs and rock culture "with both a measure of intelligence and a corresponding lack of sentimentality or even romanticism". The attitude of songs like "Paint It Black" in particular influenced punk's nihilistic outlook. Elvis Costello called his album This Year's Model (1978) "a ghost version of Aftermath" and called "This Year's Girl" an answer song to "Stupid Girl". +Some of Aftermath's blues-oriented rock elements foreshadowed the blues-rock music of the late 1960s. Schaffner suggests "Goin' Home" anticipated the trend of extended musical improvisations by professional rock bands, while Rob Young of Uncut says it heralded "the approaching psychedelic tide" in the manner of Rubber Soul. Summarising Aftermath's impact in 2017, the pop culture writer Judy Berman describes "Paint It Black" as "rock's most nihilistic hit to date" and concludes that, "with Jones ditching his guitar for a closetful of exotic instruments and the band channelling their touring musicians' homesickness on the record's 11-minute culminating blues jam, 'Goin' Home,' they also pushed rock forward." +Aftermath is often considered the Rolling Stones' first classic album. According to Stephen Davis, its standing as the first wholly Jagger-Richards collection makes it, "for serious fans, the first real Rolling Stones album". Schaffner says it is "the most creative" and possibly the best of their albums "in the first five years", while Hyden cites it as their "first full-fledged masterpiece". Writing for Uncut, Ian MacDonald recognises it as an "early peak" in the Stones' career, and Jody Rosen, in a "Back Catalogue" feature for Blender, includes it as the first of the group's "essential" albums. The Guardian's Alexis Petridis names Aftermath the Stones' fifth-best record, while Graeme Ross of The Independent ranks it sixth and suggests it stands on a level with other benchmark LPs from 1966, including Blonde on Blonde, Revolver and the Beach Boys' Pet Sounds. In The Rolling Stone Illustrated History of Rock & Roll (1976), Christgau names Aftermath the first in a series of Stones LPs – including Between the Buttons, Beggars Banquet (1968) and Let It Bleed (1969) – that stand "among the greatest rock albums". In MusicHound Rock (1999), Greg Kot highlights Jones' "canny" instrumental contributions while identifying Aftermath as the album that transformed the Stones from British blues "traditionalists" into canonical artists of the album-rock era, alongside the Beatles and Bob Dylan. In a retrospective review for AllMusic, Unterberger applauds the band's use of influences from Dylan and psychedelia on "Paint It Black", and similarly praises "Under My Thumb", "Lady Jane" and "I Am Waiting" as masterpieces. +In 2002, both versions of Aftermath were digitally remastered as part of ABKCO Records' reissue campaign of the Rolling Stones' 1960s albums. Reviewing the reissues for Entertainment Weekly, David Browne recommends the UK version over the US, while Rob Sheffield, in his appraisal in The Rolling Stone Album Guide (2004), prefers the US edition for its replacement of "Mother's Little Helper" with "Paint It Black" and highlights the clever lyrics of Jagger. Colin Larkin, who rates the British version higher in his Encyclopedia of Popular Music (2011), describes Aftermath as "a breakthrough work in a crucial year" and an album that demonstrates a flexibility in the group's writing and musical styles as well as "signs of the band's inveterate misogyny". In their book The Beatles vs. The Rolling Stones: Sound Opinions on the Great Rock 'n' Roll Rivalry (2010), Jim DeRogatis and Greg Kot agree that Aftermath is "the first really great Stones album beginning to end", with DeRogatis especially impressed by the British edition's first half of songs. +The pop culture author Shawn Levy, in his 2002 book Ready, Steady, Go!: Swinging London and the Invention of Cool, says that, unlike the three previous Stones albums, Aftermath displayed "purpose" in its sequencing and "a real sense that a coherent vision was at work" in the manner of the Beatles' Rubber Soul. However, he adds that with the August 1966 release of Revolver, Aftermath appeared "limp, tame, dated". Young believes its reputation as a work on-par with Rubber Soul is undeserved since the quality of its songs is inconsistent, the production is "relatively straight" and the assorted stylistic approach ensures it lacks the unifying aspect of the period's other major LPs. Discussing the album's critical legacy for PopMatters, Mendelsohn and Eric Klinger echo this sentiment while agreeing that it is more of a transitional work for the Stones and not up to the level of the albums from their subsequent "golden years" – Beggars Banquet, Let It Bleed, Sticky Fingers (1971) and Exile on Main St. (1972). In an article for Clash celebrating Aftermath's 40th anniversary, Simon Harper concedes that its artistic standing alongside the Beatles' contemporaneous works may be debatable but, "as the rebirth of the world's greatest rock and roll band, its importance is undisputed." +Some retrospective appraisals are critical towards the harsh treatment of female characters on the album. As Schaffner remarks, "the brutal thrust of such ditties as 'Stupid Girl,' 'Under My Thumb' and 'Out of Time' has since, of course, induced paroxysms of rage among feminists." Young infers that the album's principal lyrical theme now evokes a "rather old-fashioned sensation of brattish, spiky misogyny", presenting female characters as "pill-popping housewives ... the idiotic hussy ... the 'obsolete' fashion dummy ... or the subjugated arm candy". Berman also singles out this aspect in her otherwise positive estimation of Aftermath, saying it "indulged the Stones' misogyny on the bitchy diss track 'Stupid Girl' and tamed a shrew on 'Under My Thumb,' a nasty piece of work". Unterberger expresses similar reservations about the substance behind songs like "Goin' Home" and "Stupid Girl", finding the latter particularly callow. +Aftermath frequently appears on professional rankings of the best albums. In 1987, it was voted 68th in Paul Gambaccini's book Critics' Choice: The Top 100 Rock 'n' Roll Albums of All Time, based on submissions from an international panel of 81 critics, writers and broadcasters. In contemporaneous rankings of the greatest albums, the Dutch OOR, the British Sounds and the Irish Hot Press placed it as 17th, 61st and 85th, respectively. In 2000, it was voted number 387 in Colin Larkin's All Time Top 1000 Albums. In 2003, Rolling Stone ranked the American edition at number 108 on the magazine's "500 Greatest Albums of All Time" list. In 2017, Pitchfork listed Aftermath at number 98 on the website's "200 Best Albums of the 1960s". +The album is also highlighted in popular record guides. It is named in Greil Marcus' 1979 anthology Stranded as one of his "Treasure Island" albums, comprising a personal discography of rock music's first 25 years. The American edition of the album is included in "A Basic Record Library" of 1950s and 1960s recordings published in Christgau's Record Guide: Rock Albums of the Seventies (1981). The same version appears in James Perone's book The Album: A Guide to Pop Music's Most Provocative, Influential and Important Creations (2012) and in Chris Smith's 101 Albums That Changed Popular Music (2009), albeit in the latter's appendix "Ten Albums That Almost Made It". In addition, Aftermath features in Bill Shapiro's 1991 Rock & Roll Review: A Guide to Good Rock on CD (listed in its section on "The Top 100 Rock Compact Discs"), Chuck Eddy's The Accidental Evolution of Rock'n'roll (1997), the 2006 Greenwood Encyclopedia of Rock History's "Most Significant Rock Albums", and Robert Dimery's 1001 Albums You Must Hear Before You Die (2010). +All tracks are written by Mick Jagger and Keith Richards. +ABKCO's 2002 SACD remaster of the UK edition was released with an otherwise unavailable stereo mix of "Mother's Little Helper". +All tracks are written by Mick Jagger and Keith Richards. +Notes +The timings of "Paint It Black" and "Goin' Home" on the CD reissue are incorrect. +"Paint It Black" was originally mistitled "Paint It, Black" +Credits are from the 2002 CD booklet and contributions listed in Philippe Margotin and Jean-Michel Guesdon's book All the Songs, except where noted otherwise. +The Rolling Stones +Mick Jagger – lead and backing vocals, percussion; harmonica ("Doncha Bother Me") +Keith Richards – harmony and backing vocals, electric and acoustic guitars; fuzz bass ("Under My Thumb", "Flight 505", "It's Not Easy") +Brian Jones – electric and acoustic guitars; sitar ("Paint It Black"), dulcimer ("Lady Jane", "I Am Waiting"), harmonica ("Goin' Home", "High and Dry"), marimba ("Under My Thumb", "Out of Time"), vibraphone ("Out of Time"), koto ("Take It or Leave It") +Bill Wyman – bass guitar, fuzz bass ("Mother's Little Helper"); organ ("Paint It Black"), marimba, bells +Charlie Watts – drums, percussion, marimba, bells +Additional musicians +Jack Nitzsche – piano, organ, harpsichord, percussion +Ian Stewart – piano, organ +Additional personnel +David Bailey – photography (US edition) +Dave Hassinger – engineering +Andrew Loog Oldham – production, cover design (UK edition) +Jerry Schatzberg – photography +Guy Webster – photography (UK edition) +British Invasion +Classic rock +List of rock albums +Merton, Richard (January–February 1968). "Comment on Beckett's 'Stones'". New Left Review. 1 (47) 981: 29–31. doi:10.64590/rcy – via newleftreview.org. +Aftermath at Discogs (list of releases) +Aftermath (2002 ABKCO remaster of UK edition) (Adobe Flash) at Spotify (streamed copy where licensed) +Aftermath (2002 ABKCO remaster of US edition) (Adobe Flash) at Spotify (streamed copy where licensed) +"all things" is the seventeenth episode of the seventh season of the American science fiction television series The X-Files. Written and directed by lead actress Gillian Anderson, it first aired on April 9, 2000, on the Fox network. The episode is unconnected to the wider mythology of The X-Files and functions as a "Monster-of-the-Week" story. Watched by 12.18 million people, the initial broadcast had a Nielsen household rating of 7.1. The episode received mixed reviews from critics; many called the dialogue pretentious and criticized the characterization of Scully. However, viewer response was generally positive. +The series centers on Federal Bureau of Investigation (FBI) special agents Fox Mulder (David Duchovny) and Dana Scully (Anderson) who work on cases linked to the paranormal, called "X-Files". Mulder is a believer in the paranormal. The skeptical Scully was initially assigned to debunk his work, but the two have developed a deep friendship. In this episode, a series of coincidences lead Scully to meet Dr. Daniel Waterston (Nicolas Surovy), a married man with whom she had an affair while at medical school. After Waterston slips into a coma, Scully puts aside her skepticism and seeks out alternative medicine to save Waterston. +"all things" is the only episode of the series written and directed by Anderson, as well as the first episode of The X-Files to be directed by a woman. The episode makes heavy use of "The Sky Is Broken", a song from Moby's 1999 album Play, as well as a gong. The episode has been analyzed for its themes of pragmatism and feminist philosophy. +FBI special agent Dana Scully (Gillian Anderson) is getting dressed in front of a mirror. As she leaves, her colleague Fox Mulder (David Duchovny) lies in his bed, half of his body covered by bedsheets. The narrative flashes back to a few days earlier: Scully arrives at a hospital and, after a series of coincidences, meets her former professor, Daniel Waterston (Nicolas Surovy), with whom she had an affair while attending medical school. He is ill and suffering from an undiagnosed heart condition. She questions whether she made the right decision to leave him and abandon her medical career to pursue a career in the FBI. She meets Waterston's daughter, Maggie (Stacy Haiduk), who is extremely resentful of Scully for the effect she had on Waterston's family. +Mulder—on his way to England investigating heart chakra-shaped crop circles—calls Scully and asks her to meet a contact of his, Colleen Azar (Colleen Flynn), to obtain some information. As Scully speaks to Mulder on her cellphone while driving her car, a woman appears on a crosswalk. Scully brakes hard to avoid hitting the woman. As she does so, she narrowly avoids colliding with a semi-truck. She realizes that, had the woman not stepped in her path, the truck would have killed her. When she later arrives at the house of Azar, she observes that Scully is going through a personal crisis and tries to offer her guidance, but Scully is dismissive. +Later, Scully returns to apologize to Azar and agrees to listen to her ideas. Azar shares her knowledge of Buddhism, the concept of the collective unconscious, and the idea of personal auras. Azar believes these concepts might explain these strange occurrences. While visiting Waterston, he nearly dies but Scully saves him using a defibrillator; however, this also puts him in a coma. After a confrontation with Maggie at the hospital over what happened to her father, Scully walks through Chinatown. Seeing the woman who appeared earlier at the crosswalk, she follows her to a small Buddhist temple before the mysterious woman seemingly vanishes. Inside the temple, Scully has a vision of what is ailing Waterston. She returns to the hospital with Azar to visit Waterston. +Azar and a healer provide alternative treatment for Waterston, who fully recovers. He announces that he still wants a relationship with Scully, but she realizes she is no longer the same person she was those many years ago and rejects him. As she sits outside the hospital on a bench, Scully thinks that she sees the mysterious woman again, but it turns out to be Mulder. Later, the two agents sit in Mulder's apartment talking about the events of the last few days. Mulder begins to speak more existentially about what transpired, implying that fate has brought them together but, when he turns to look at Scully, he sees that she has fallen asleep. +Sometime during the sixth season of The X-Files, Anderson approached series creator Chris Carter and asked if she could write a script for an episode that explored her own interest in "Buddhism and the power of spiritual healing"; ultimately, she wanted to write a script in which Scully pursued a "deeply personal X-File, one in which [she] is taken down a spiritual path when logic fails her". She wrote the basic outline of what became "all things" in one sitting, which Carter approved due to the "personal and quiet" nature of the story. Anderson's first draft of "all things" was 15 pages too long, and it did not feature a concluding fourth act. Carter and executive producer Frank Spotnitz thus began to work with Anderson to finish the episode, although Carter and Spotnitz later acknowledged that the majority of the script "was all Gillian". +Despite her satisfaction with the final version, Anderson regrets a handful of the "necessary" script changes, most notably, the addition that Scully and Waterston's affair was intimate. In the original script, the two came "close to having an affair", but Scully ended the relationship when she discovered that Waterston was married. In the commentary for the episode, Anderson elaborated on Scully and Waterston's original backstory: after Scully and Waterston came close to having an affair, Scully left to study at Quantico to become an FBI agent. After she left, Waterston become depressed, and his family began to suspect the affair. The emotional turmoil was too much for Waterston's wife, who killed herself, which made Waterston's daughter, Maggie, resent Scully, as shown in the finished episode. Anderson believed that the removal of this backstory made it hard for the audience to understand Maggie's disgust with Scully. +When Anderson first wrote the episode, she did not intend to imply that Mulder and Scully had had sex; Spotnitz and the production crew, however, felt it was natural to suggest that their relationship had evolved into a romantic one. The idea of heart chakra crop circles was included because Anderson wanted "whatever Mulder was involved in that took him away from me, away from Washington, to somehow tie into what it was that I was going through—the journey that I was going through". As such, Anderson dedicated much of her time researching both crop circles and heart chakras, but she later gave additional credit to Spotnitz, who aided her in the research process. +Around the same time that she approached Carter about writing an episode, Anderson was being solicited by television networks, who were interested in having her direct shows. She, however, had never directed before and decided that she would first helm an episode of The X-Files before working on other series. Consequently, when Anderson pitched her initial script idea, she also expressed her desire to direct the episode. Carter accepted her story, but did not appoint her as director until all the revisions and rewrites had been completed. Anderson worked with series director Kim Manners for the majority of the episode, and he assigned Anderson directing exercises—such as making a list of shots for every scene—to get her familiar with the demands of directing. This episode of The X-Files was the first to be directed by a woman. +Anderson's directing helped to energize the production, and the crew worked harder than usual to ensure that everything was in order for her: Production designer Corey Kaplan went out of his way to find a Buddhist temple at Anderson's request, and casting director Rick Millikan helped Anderson choose the appropriate actors. Millikan later said that he particularly enjoyed working with Anderson, because "it was fun for [him] to watch her go through the casting process because it was all new to her." On set, Anderson's directing style was described as "right on the money" by Marc Shapiro in his book all things: The Official Guide to The X-Files, Volume 6. He later wrote that "Anderson wielded a deft hand in her directorial debut, prodding the actors to her will, making decisions on the fly, and handling the complex special effects sequences". Fans of the show later wrote to express their appreciation of Anderson's directing abilities. Anderson was also involved in post-production editing, during which she was forced to cut the final conversation scene between Scully and Daniel Waterston down by about 10 minutes. +The meditation scene required clips from previous episodes to appear in flashback. Initially, Paul Rabwin and the special effects crew arranged the necessary scenes and placed them in animated bubbles. However, the crew was unhappy with the bubbles and felt that they were too "hokey", so they adopted a more standard slit-scan effect. In order to create the sequence of Scully visualizing Waterston's heart condition, Nicolas Surovy had to lie naked on a platform surrounded by a blue screen. A shot of a prosthetic beating heart was then crafted and filmed separately, and via motion control, the two shots were combined into a composite whole. +Anderson wanted to include "The Sky Is Broken", a song from Moby's 1999 album Play in the episode, as she felt that the song's lyrics "fit with [the] idea that was unfolding for the script". Anderson crafted the first shot after the opening credits, which involved Scully getting ready while water dripped from a sink, to create a "continuation of sound, rhythmic sound", because it was important to the show's musical aspect. Anderson and series composer Mark Snow worked together in post-production; after filming, she sent Snow several CDs of music and asked him for compositions that were similar in style and feel. A certain melody that the two worked on later became "Scully's Theme", which was not broadcast until the eighth season episode "Within". "all things" also featured the use of the gong, an instrument that Anderson called "very Tibetan" and "appropriate for this episode". +In the chapter "Scully as a Pragmatist Feminist" of the book The Philosophy of The X-Files, Erin McKenna argues that "all things" represents an "important shift" in Scully's approach to science, knowledge acquisition, and the pursuit of the truth. She reasons that the events of the episode open Scully's mind to new ways of knowing, specifically citing "auras, chakras, visions ... and the importance of coincidence". McKenna notes that Scully's shift in perspective is a shift to American pragmatism, a belief that reality is ever-changing. Pragmatists believe "the truth is out there"—the motto of the series—in a manner similar to Mulder's. In "all things", Scully begins to embrace pragmatism, although she still clings to her skeptic roots. Mixing the two, Scully evolves from a mere skeptic who demands proof to validate a truth, to an empiricist who wants proof, but is open to other perspectives. +In addition, McKenna reasons that "all things" is heavily influenced by feminist philosophy and epistemology, schools of thought that try to criticize or re-evaluate the ideas of traditional philosophy and epistemology from within a feminist framework. According to McKenna, feminism rejects dualistic ways of thinking, especially "typical male/female dualism". Feminist philosophy, instead, calls for a pluralistic way of thinking, noting that there are many consistent sets of truths about the world. In the episode, Scully starts out "sure of her more rational scientific view and approach". As the episode progresses, however, she decides to branch out. Eventually, she brings in a spirit healer to "corroborate or nullify the new beliefs she is encountering". Despite dabbling in mysticism, a field generally stereotyped as feminine by the patriarchy, Scully engages in "protracted inquiry", examining all sides of the issue, in order to return Waterston to health. +When Mulder and Scully talk at the end of the episode, Mulder questions the fact that he left "town for two days and [Scully] spoke to God in a Buddhist temple and God spoke back". Scully retorts that, "I didn't say God spoke back". McKenna proposes that this is an example of Scully's rational scientific approach meshing with her newer, feministic pragmatism. The two modes of understanding are not "to be seen ... as competing systems, but as complementary, as are Scully and Mulder themselves". McKenna concludes that this is represented in the opening scene, in which Mulder and Scully are implied to have had sex. This is meant as a metaphor, showing the full merging of Scully's and Mulder's different philosophies into pragmatic feminism. +In her academic monograph on the series, Theresa L. Geller considers "all things" at length while discussing the show's sexual politics. Geller contends that Anderson uses her directorial debut to make a feminist intervention into the character of Scully, noting that the episode's title is "possibly a comment on the earlier Scully episode, 'All Souls' that cast Scully yet again as self-sacrificing mother" while "all things" "allowed Anderson to create a story that shifted Scully's arc from the immaterial religiosity of Catholicism to a more material, embodied spirituality, from souls to things". Geller analyzes the episode in the context of Anderson's long-standing engagement with female fans, arguing that it "offers a narrative that acknowledges shippers identification with Scully not as an object of seduction, but as a figure directly impacted by women and changed by what she learns when she listens to them". +Geller sees this intervention as directly tied to the episode's narrative peculiarity: "Although neither FBI-related nor paranormal, 'all things' suggests—and models—ways we can learn from women's knowledge, even when it is intuitive, 'irrational', and embodied. To do so, however, meant a shift in form, in genre." "all things" expands the generic vocabulary of The X-Files by including a scene from Vittorio De Sica's 1953 woman-centered melodrama Indiscretion of an American Wife, a reference that "underscores the generic conventions to the 'pathetic impulses behind this bargaining' women do in the romance", but does so only to highlight that, "in this episode, Scully's desires are catalyzed, ultimately, not by men in her life, but by women she does not know—much like the women fans of The X-Files that have shaped the show". +The episode originally aired in the United States on the Fox network on April 9, 2000, and was first broadcast in the United Kingdom on Sky1 on July 9, 2000. In the U.S., "all things" was watched by 12.18 million viewers. It earned a Nielsen household rating of 7.5, with an 11 share, meaning that roughly 7.5 percent of all television-equipped households, and 11 percent of households watching television, tuned into the episode. In the U.K., "all things" was seen by 580,000 viewers, making it the seventh-most watched program on Sky1 for that week. On May 13, 2003, the episode was released on DVD as part of the complete seventh season. +Emily St. James of The A.V. Club awarded the episode a "C" and called it "a curious failure". She felt that the writing was "pretentious" and composed of "some weird, weird bullshit". St. James wrote that, although the episode was unsuccessful, there was something so "pure and unadorned at its center that I can't outright hate it". Furthermore, she admired the show and Anderson for "making the attempt". Kevin Silber of Space.com gave the episode a negative review, critical of the script and characterization. He said, "nothing much seems to happen, and what does occur is substantially driven by coincidence and arbitrariness". He did not like Azar and disapproved of Scully's philosophical "reverie", calling it "facile, and hard to reconcile with the determined rationalism she's displayed over the years in the face of events no less strange than those that occur here". +In their book Wanting to Believe: A Critical Guide to The X-Files, Millennium & The Lone Gunmen, Robert Shearman rated the episode one star out of five, calling the premise and characters dull. The author criticized Anderson for looking at the "minutiae of life too intensely", which made many of the actors and actresses come off as ciphers. Furthermore, Shearman was critical of Anderson's directing style, calling it "pretentious", noting that the plot's significance was drowned out by unnecessary artistic flourishes and needless pizzazz. Paula Vitaris from CFQ gave the episode a negative review, awarding it one star out of four. She called Anderson's directing "heavy-handed" and bemoaned the storyline because it "plays havoc with Scully's motivations and character as established in the past seven years". +Not all reviews were negative. Kinney Littlefield of the Orange County Register wrote that the "wistful, meditative episode" was "not bad for Anderson's first directing effort". He did, however, comment that it was not as "sly as the episode about an alien baseball player that Duchovny directed". The Michigan Daily writer Melissa Runstrom, in a review of the seventh season, called the episode "interesting". +While the episode received lukewarm reviews from critics, fans of the show reacted generally positively to "all things", and the show's producers received calls and letters from viewers stating that they "loved the vulnerability and quiet determination that Scully revealed in the unusual episode". +"all things" at IMDb +The Battle of Big Black River Bridge was fought on May 17, 1863, as part of the Vicksburg Campaign of the American Civil War. During the war, the city of Vicksburg, Mississippi, was a key point on the Mississippi River. On April 30, 1863, a Union army commanded by Major General Ulysses S. Grant began crossing onto the east side of the Mississippi River as part of a campaign against Vicksburg. After engaging and defeating Confederate forces in several intermediate battles, Grant's army defeated Lieutenant General John C. Pemberton's troops at the decisive Battle of Champion Hill on May 16. During the retreat from Champion Hill battlefield, one division of Pemberton's army, commanded by Major General William W. Loring, was cut off from Pemberton's main body. Pemberton, retreating westwards towards Vicksburg, did not know the location of Loring's division, and he held a bridgehead on the east side of the Big Black River to cover Loring's anticipated withdrawal across the river on the morning of May 17. +The Confederate force east of the Big Black River was commanded by Brigadier General John S. Bowen and consisted of the two brigades of his division, the brigade of Brigadier General John C. Vaughn, and one additional regiment. Bowen's soldiers had taken heavy casualties at Champion Hill, and most of Vaughn's troops were conscripts of uncertain loyalty. On the morning of May 17, the Union XIII Corps commanded by Major General John A. McClernand approached the Confederate position. Two divisions and another brigade from a third deployed to face the Confederates. Union Brigadier General Michael Kelly Lawler advanced his troops into an meander scar in advance of the main Union line on the north end of the battlefield. From this sheltered position, Lawler's troops charged, striking the Confederate line in the region held by Vaughn's brigade. +Vaughn's troops broke, with the soldiers either running away or surrendering. The collapse of this portion of the line forced the rest of the Confederate troops to withdraw. The withdrawal became chaotic and 1,751 Confederate soldiers and 18 cannons were captured. The cannons were captured because their teams of horses were erroneously positioned on the other side of the Big Black River. The Union army suffered fewer than 300 casualties. The retreating Confederates burned both the railroad bridge over the Big Black River and a steamboat that had been serving as a bridge. Loring had already given up on rejoining his division to Pemberton's army and his troops marched to Jackson, Mississippi. On May 18, the Union army crossed the Big Black River. The surviving Confederate soldiers with Pemberton entered the fortifications at Vicksburg, and the siege of Vicksburg began the next day; it ended in the Confederate surrender on July 4, 1863. +Early in the American Civil War, the Union military leadership developed the Anaconda Plan, which was a strategy to defeat the Confederate States of America by controlling its coastline and major rivers. A significant component of this strategy was controlling the Mississippi River. Much of the Mississippi Valley fell under Union control in early 1862 after the capture of New Orleans, Louisiana, and several land victories. The strategically important city of Vicksburg, Mississippi, was still in Confederate hands, and it served as a strong defensive position that commanded the river and prevented the Union from separating the two halves of the Confederacy. Union Navy elements were sent upriver from New Orleans in May to try to take the city, a move that was unsuccessful. In late June, a joint army–navy expedition returned to make another campaign against Vicksburg. Union Navy leadership decided that the city could not be taken without more infantry, who were not forthcoming. An attempt to construct Williams's Canal across a meander of the river in June and July, bypassing Vicksburg, failed. +In late November, about 40,000 Union infantry commanded by Major General Ulysses S. Grant began moving south towards Vicksburg from a starting point in Tennessee. After a supply depot and part of his supply line were destroyed during the Holly Springs Raid and Forrest's West Tennessee Raid in December, Grant ordered a retreat. Meanwhile, another arm of the expedition under the command of Major General William T. Sherman left Memphis, Tennessee, on the same day as the Holly Springs Raid and traveled down the Mississippi River. After diverting up the Yazoo River, Sherman's troops began skirmishing with Confederate soldiers who were defending a line of hills above the Chickasaw Bayou. A Union attack on December 29 was defeated decisively at the Battle of Chickasaw Bayou, and Sherman's soldiers withdrew on January 1, 1863. +By late March, further attempts to bypass Vicksburg had failed. Grant then considered three plans: to withdraw to Memphis and retry the overland route through northern Mississippi; to move south along the west side of the Mississippi River, cross below Vicksburg, and then strike for the city; or to make an amphibious assault across the river directly against Vicksburg. An assault across the river risked heavy casualties, and a withdrawal to Memphis could be politically disastrous if the public perceived such a movement as a retreat. Grant then decided upon the downstream crossing. The advance along the west bank of the Mississippi began on March 29, spearheaded by Major General John A. McClernand's XIII Corps. The movement down the river was masked by decoy operations such as Steele's Greenville expedition, Streight's Raid, and Grierson's Raid. Confederate regional commander John C. Pemberton fell for the Union decoys (especially Grierson's Raid), and lost touch with the true tactical situation, believing Grant was withdrawing. +On April 29, the Union Navy's Mississippi Squadron, commanded by David Dixon Porter, bombarded the Confederate defenses at Grand Gulf, Mississippi, but the resulting Battle of Grand Gulf failed to drive the Confederates away. With Grand Gulf still in enemy hands, Grant decided to cross further downriver. McClernand's corps, which formed the lead element of Grant's army, crossed the river at Bruinsburg, Mississippi, beginning on the morning of April 30. Brigadier General John S. Bowen, the Confederate commander at Grand Gulf, would be flanked out of the position there if Grant crossed Bayou Pierre. Bowen moved a portion of his force to Port Gibson to block the Union movements, but he did not have enough troops to destroy Grant's bridgehead and had to hold out for further reinforcements. The two armies collided on May 1, and the Battle of Port Gibson began. After a day of fighting, the Confederates were defeated; they abandoned Grand Gulf on May 3. +After Port Gibson, Grant moved his troops to the northeast. McClernand advanced on the Union left with his XIII corps, Sherman and his XV Corps advanced in the center, and Major General James B. McPherson's XVII Corps moved on the right. On the morning of May 12, McPherson's corps encountered Confederate troops near Raymond, Mississippi, bringing on the Battle of Raymond. The Union won the battle, but the fighting at Raymond led Grant to change his plans; instead of moving directly north, he swung over towards Jackson, Mississippi. This was intended to disperse a Confederate force gathering there, under the command of General Joseph E. Johnston, who decided to abandon Jackson in the face of the Union pressure. A delaying action was fought on May 14 as the Confederates bought time to evacuate the city. The Union took the city and destroyed military facilities within it. +After he had already withdrawn from Jackson, Johnston sent Pemberton orders to move east, stating that Johnston's army would move west and catch Grant's command between the two Confederate forces. However, Johnston then marched his army away from the area in which a combination with Pemberton could have been easily be made. Pemberton decided that Johnston's orders were not compatible with a previous directive that he had received from the Confederate President Jefferson Davis that Vicksburg and Port Hudson, Louisiana, should be held. While Pemberton favored making a stand behind the Big Black River, he was convinced by some of his subordinate officers to make an offensive strike where Grant's supply line was believed to be. When planning this offensive, Pemberton did not know that Grant had forgone a traditional line of communications during his movement inland. Beginning the offensive, the Confederates started a difficult march slowed by poor road conditions and a washed-out bridge, while Grant moved west in three columns towards Edwards Station. On the morning of May 16, elements of the Union and Confederate armies made contact, and Pemberton ordered his force to march back to Edwards. The ensuing Battle of Champion Hill was a decisive Confederate defeat. Major General William W. Loring's Confederate division was cut off during the retreat from the field and withdrew using a different route, separated from the rest of the army. Pemberton did not know the location of Loring's division. +In preparation for what was likely to transpire, Pemberton had a portion of his army hold the line east of the Big Black River to prevent Loring from being cut off from the main Confederate body at the river crossing. In early May, the Confederate defensive line was laid out by Samuel H. Lockett. The fortifications were made of cotton bales and dirt. To the south lay Gin Lake: the Confederate right flank was at the lake with the line running north to the Big Black River, which made a bend east of the bridge across it. Both the Jackson Road and the Southern Railroad of Mississippi ran through the center of the Confederate line and crossed the river. The tracks of the railroad ran on a raised roadbed. While the area west of the bridge contained 60-foot (18 m) high hills, the area east of the river was flat. A bayou ran about 0.75 miles (1.21 km) east of the railroad bridge, cutting its course in front of most of the Confederate line and emptying into the Big Black River at the north end. The Confederate line was just west of the bayou, and trees were felled at the bayou to form an abatis. The southern portion of the Confederate line contained several artillery positions and faced flat, open fields to the east, while the northern part of the line contained only one artillery position and its field of fire was obstructed by woods. At the Confederate far left, a smaller set of works had been built east of the bayou, in a position where it could enfilade a Union advance. This set of works could contain only one regiment. Behind the Confederate line but east of the river were a pair of blockhouses that had been built earlier in the year to protect the bridge against cavalry raids. A secondary bridge was created by turning a small steamboat named Dot crossways in the river where a ferry had existed; the boat's machinery was removed and planks were placed on the decks. This position was 11 miles (18 km) east of Vicksburg. +Manning the position was Pemberton's rear guard, which consisted of Brigadier General John C. Vaughn's brigade, Bowen's division, and the 4th Mississippi Infantry Regiment. Bowen commanded this 5,000-strong force, which he deemed insufficient to strongly defend the entire Confederate defensive works. The Confederates had 18 cannons. Vaughn's soldiers were inexperienced, mostly conscripts from the East Tennessee region, whose populace was heavily pro-Union. Although they were fresh, their loyalty to the Confederacy was uncertain. Bowen's division was the elite unit of Pemberton's army, but they were exhausted after hard fighting at Port Gibson and Champion Hill and had suffered nearly 1,000 casualties in the latter battle, out of about 4,500 troops engaged. Pemberton selected Bowen's division for the rear guard as Major General Carter L. Stevenson's division had already passed through the area and the rest of Pemberton's army was either at Vicksburg or with Loring. Colonel Francis M. Cockrell's Missouri brigade held the southern portion of the Confederate line and the area where the Jackson Road crossed it, as Bowen considered this unit to be his best and he expected the Union attack to fall there. Vaughn's soldiers and the 4th Mississippi were positioned in an area north of the railroad where the Union was least expected to attack, and Brigadier General Martin E. Green's brigade held the far left. If the Confederates were forced to retreat, they would have to cross open ground to the bridge and Dot, which would become bottlenecks in a retreat. +Early on the morning of May 17, McClernand's troops advanced through Edwards and then encountered the Confederate line. His advance was led by the division of Brigadier General Eugene Carr. Carr's lead brigade was commanded by Brigadier General William P. Benton, with the 33rd Illinois Infantry Regiment in front as skirmishers. The Illinoisans encountered the Confederate lines, and they then took up a position in the woods facing the north end of the Confederate line. Carr was informed of the encounter and brought up more troops. Benton's troops took up a position in the fields east of the woods on the Union right, and Brigadier General Michael Kelly Lawler's brigade formed south of the road. One of Green's regiments, the 1st Missouri Cavalry Regiment (dismounted), had remained on the far side of the Big Black River, but Bowen now ordered it across the river and into the works on the east side of the bayou. This movement worried Carr, who shifted Lawler to Benton's right, while Brigadier General Peter J. Osterhaus's division deployed to the south. Brigadier General Theophilus Garrard's brigade was the left portion of Osterhaus's line, and that of Colonel Daniel W. Lindsey was to the right. +The Union artillery opened fire, and an artillery duel developed. While the Confederate soldiers took cover behind their defenses, the Union soldiers lay flat on the ground, as their position did not provide any cover. Lindsey advanced his brigade west along the railroad 300 yards (270 m), placing his troops ahead of the rest of the Union line. Osterhaus wanted to position Garrard's troops behind Lindsey's, but it was believed that Lawler needed assistance, so two of Garrard's regiments, the 49th Indiana Infantry and 69th Indiana Infantry, were sent to support the Union right. Osterhaus suffered a leg wound during the exchange when Confederate fire exploded an artillery limber; Osterhaus was replaced by Brigadier General Albert Lee. Grant was confident of victory, and at 8:00 am sent a message to Sherman stating that he believed that the Confederates would be forced to retreat in such a rapid fashion that they could not destroy the bridge. The Confederates, meanwhile, had such poor morale that Lockett believed the position could not be held; he requested and received permission to prepare the bridge and Dot for burning. +Confederate attention was drawn to the center, near the railroad, by the Union artillery fire. Lawler believed that the remains of a meander scar near the Big Black River would shelter his brigade, and had three regiments – the 11th Wisconsin Infantry, the 21st Iowa Infantry, and the 23rd Iowa Infantry – dash across open ground to reach the depression. Two cannons were positioned in a small clearing between the right of the woods and the Big Black River, and with the 22nd Iowa Infantry Regiment in support. This position allowed Lawler to enfilade the Confederate position east of the bayou, as well as part of the primary defensive works. This advance was accomplished without many casualties. While these movements were occurring, Brigadier General Stephen G. Burbridge's brigade of Brigadier General A. J. Smith's division arrived and was positioned on the Union far left. +The two regiments sent from Garrard's brigade took the position previously occupied by the 22nd Iowa, and they also moved into the meander scar. Colonel William Kinsman, the commander of the 23rd Iowa, proposed to Lawler that his regiment should attack the Confederates. Kinsman's reasoning was that the Confederates would only have time to fire one volley before the Union soldiers reached the defenses, and the Confederates might not put up a stiff fight after the Champion Hill defeat. Lawler ordered a charge by his whole brigade; with the 21st and 23rd Iowa in the front rank while the other two regiments charged behind. His soldiers were ordered not to fire before reaching the Confederate line. The attack quickly moved across the front of the 1st Missouri Cavalry (dismounted) and out of the regiment's field of fire, although not before Kinsman was killed and the commander of the 21st Iowa, Colonel Samuel Merrill, was wounded. The 49th and 69th Indiana joined the attack; while Lawler's soldiers advanced at an angle across Green's front and struck one of Vaughn's regiments, the 61st Tennessee Infantry Regiment. Green's troops fired a volley into Lawler's brigade that the historian Timothy B. Smith describes as "wild and ragged". Lawler's soldiers stopped to fire once they reached the abatis; the Tennesseans were routed and the defenders either ran away or surrendered. Bearss describes the attack, which took three minutes, as one of the shortest of the war. +The hole in the Confederate line reached to the railroad, and Green's troops withdrew as well, although some were captured. In the 1st Missouri Cavalry (dismounted), 90 soldiers and the unit's commander, Colonel Elijah Gates, were captured. Cockrell watched the Confederate left run away and ordered his troops to withdraw, as their position was no longer tenable. Cockrell's retreat became chaotic as well. Confederate artillery west of the river provided covering fire for the retreat. Benton's brigade attacked around the time Lawler broke through the line. Burbridge's and Albert Lee's forces charged the Confederate defenses, only to meet no resistance. Once almost all of the fleeing Confederates crossed the bridge, Lockett had the bridge and Dot set on fire. He then formed a new line west of the river by using the brigades of Brigadier Generals Stephen D. Lee and William E. Baldwin, who had arrived from Bovina, Mississippi, and part of Landis's Missouri Battery that had been positioned on the west bank before the battle. Two other Confederate steamboats, Charm and Paul Jones, which had been located downstream from the bridge were also burned. +Albert Lee's soldiers spent the afternoon after the battle in low-intensity fighting across the river against a Confederate force while Carr and Smith's troops patrolled the field. The Union reported the capture of 1,751 Confederates as well as 18 cannons. The Confederate artillery losses came about because the horse teams for the cannons had been erroneously moved across the river before the battle for unclear reasons. Green reported having suffered 485 casualties while two of Vaughn's regiments combined for 546 losses. Most of the casualties were prisoners or those missing in action. Few of the casualties occurred before the line fell to pieces. One of Vaughn's regiments, Cockrell's brigade, and the 4th Mississippi did not file casualty reports, but they were known to have suffered heavily in soldiers captured. Union casualties were 279 soldiers killed, wounded, or missing, of which 39 were killed, 237 wounded, and 3 missing. About 200 of the casualties were in Lawler's brigade. Grant's 12-year old son Fred was wounded in the leg while following the pursuit of the routed Confederates; Fred was present with his father for the entire campaign. Sergeant William Wesley Kendall of the 49th Indiana Infantry Regiment was awarded the Medal of Honor for leading a company in the main Union charge; he was among the first Union soldiers to enter the Confederate fortifications. +The Confederates withdrew into the Vicksburg fortifications. Pemberton ordered several outlying positions withdrawn into the main lines and the Vicksburg defenses were also physically improved. There was much outrage against Pemberton within the Confederate army due to the outcome of the movements and battles of the last several days. Loring had noticed light from fires in Union-occupied Edwards on the morning of May 17, and with the way blocked he marched his troops to Jackson, where they joined forces with Johnston on May 19. McClernand had a replacement bridge built over the Big Black River on the morning of May 18, while McPherson's and Sherman's corps crossed at other points. Sherman sent a cavalry regiment towards Snyder's Bluff, where the Confederate fortifications were found to have been abandoned, while Grant's army had regained a connection to the Union Navy elements on the Yazoo River. Sherman's main force came into contact with the Confederate defenders of Vicksburg, and McClernand and McPherson's forces arrived late that day. The Siege of Vicksburg began. Believing that the Confederates in Vicksburg could be easily defeated, Grant launched major attacks on May 19 and 22. Both attacks were repulsed bloodily, and Grant settled in for siege operations. The Confederates ran low on supplies, and Pemberton surrendered on July 4. The fall of Vicksburg was one of the key events of the war. +The site of the battle was listed on the National Register of Historic Places in 1971 as the Big Black River Battlefield. As of 2023, portions of the piers of the railroad bridge existing during the battle still remain at the crossing of the Big Black River. A trail runs along the river bank, and a historical marker is placed in the vicinity of the battlefield, although the battlefield itself is privately owned. +Ballard, Michael B. (2004). Vicksburg: The Campaign that Opened the Mississippi. Chapel Hill, North Carolina: University of North Carolina Press. ISBN 0-8078-2893-9. +Bearss, Edwin C. (1991) [1986]. The Campaign for Vicksburg. Vol. II: Grant Strikes a Fatal Blow. Dayton, Ohio: Morningside Bookshop. ISBN 0-89029-313-9. +Bearss, Edwin C. (2007) [2006]. Fields of Honor. Washington, D.C.: National Geographic. ISBN 978-1-4262-0093-9. +Grabau, Warren (2000). Ninety-eight Days: A Geographer's View of the Vicksburg Campaign. Knoxville, Tennessee: University of Tennessee Press. ISBN 978-1-57233-068-9. +Kennedy, Frances H. (1998). The Civil War Battlefield Guide (2nd ed.). Boston/New York: Houghton Mifflin. ISBN 978-0-395-74012-5. +Miller, Donald L. (2019). Vicksburg: Grant's Campaign that Broke the Confederacy. New York, New York: Simon and Schuster. ISBN 978-1-4516-4139-4. +Shea, William L.; Winschel, Terrence J. (2003). Vicksburg Is the Key: The Struggle for the Mississippi River. Lincoln, Nebraska: University of Nebraska Press. ISBN 978-0-8032-9344-1. +Smith, Timothy B. (2006) [2004]. Champion Hill: Decisive Battle for Vicksburg. El Dorado Hills, California: Savas Beatie. ISBN 1-932714-19-7. +Smith, Timothy B. (2013). ""A Victory Could Hardly Have Been More Complete": The Battle of Big Black River Bridge". In Woodworth, Steven E.; Grear, Charles D. (eds.). The Vicksburg Campaign: March 29 – May 18, 1863. Carbondale, Illinois: Southern Illinois University Press. pp. 173–193. ISBN 978-0-8093-3270-0. +Smith, Timothy B. (2024). The Inland Campaign for Vicksburg: Five Battles in Seventeen Days, May 1–17, 1863. Lawrence, Kansas: University Press of Kansas. ISBN 978-0-7006-3655-6. +Symonds, Craig L. (2008). Lincoln and His Admirals. Oxford: Oxford University Press. ISBN 978-0-19-975157-0. +Winschel, Terrence J. (1998). "Chickasaw Bayou, Mississippi". In Kennedy, Frances H. (ed.). The Civil War Battlefield Guide (2nd ed.). Boston, Massachusetts/New York, New York: Houghton Mifflin. pp. 154–156. ISBN 978-0-395-74012-5. +Woodworth, Steven E. (2005). Nothing But Victory: The Army of the Tennessee, 1861–1865. New York City: Alfred A. Knopf. ISBN 0-375-41218-2. +Fullenkamp, Leonard; Bowman, Stephen; Luvaas, Jay (1998). Guide to the Vicksburg Campaign. Lawrence, Kansas: University Press of Kansas. ISBN 0-7006-0922-9. +National Park Service, Battle of Big Black River Bridge +Henry Clifford, 10th Baron Clifford (c. 1454 – 23 April 1523) was an English nobleman. His father, John Clifford, 9th Baron Clifford, was killed in the Wars of the Roses fighting for the House of Lancaster when Henry was around five years old. A local legend later developed that—on account of John Clifford having killed one of the House of York's royal princes in battle, and the new Yorkist King Edward IV seeking revenge—Henry was spirited away by his mother. As a result, it was said, he grew up ill-educated, living a pastoral life in the care of a shepherd family. Thus, ran the story, Clifford was known as the "shepherd lord". More recently, historians have questioned this narrative, noting that for a supposedly ill-educated man, he was signing charters only a few years after his father's death, and that in any case, Clifford was officially pardoned by King Edward in 1472. It may be that he deliberately avoided attracting Yorkist attention in his early years, although probably not to the extent portrayed in the local mythology. +The Yorkist regime came to an end in 1485 with the invasion of Henry Tudor, who defeated Edward's brother, Richard III, at the Battle of Bosworth Field. Henry's victory meant that he needed men to control the North of England for him, and Clifford's career as a loyal Tudor servant began. Soon after Bosworth, the King gave him responsibility for crushing the last remnants of rebellion in the north. Clifford was not always successful in this, and his actions were not always popular. On more than one occasion, he found himself at loggerheads with the city of York, the civic leadership of which was particularly independently minded. When another Yorkist rebellion broke out in 1487, Clifford suffered an embarrassing military defeat by the rebels outside the city walls. Generally, however, royal service was extremely profitable for him: King Henry needed trustworthy men in the region and was willing to build up their authority in order to protect his own. +Although Clifford's later years were devoted to service in the north and fighting the Scots (he took part in the decisive English victory at Flodden in 1513) he fell out with the King on numerous occasions. Clifford was not an easy-going personality; his abrasiveness caused trouble with his neighbours, occasionally breaking out in violent feuds. This was not the behaviour the King expected from his lords. Furthermore, Clifford had married a cousin of the King, yet Clifford's infidelity to her was notorious among his contemporaries. This also drew the King's ire, to the extent that the couple's separation was mooted. Clifford's first wife had died by 1511, and Clifford remarried. This was also a tempestuous match, and on one occasion he and his wife ended up in court accusing each other of adultery. Clifford's relations with his eldest son and heir, the eventual Henry Clifford, 1st Earl of Cumberland, were equally turbulent. Clifford rarely attended the royal court himself, but sent his son to be raised with the King's heir, Prince Arthur. Clifford later complained that young Henry not only lived above his station, he consorted with men of bad influence; Clifford also accused his son of regularly beating up his father's servants on his return to Yorkshire. +Clifford outlived the King and attended the coronation of Henry VIII in 1509. While continuing to serve as the King's man in the north, Clifford carried on his feuds with the local gentry. He also indulged his interests in astronomy, for which he built a small castle for observation purposes. Clifford grew ill in 1522 and died in April of the following year; his widow later remarried. Young Henry inherited the title as 11th Baron Clifford as well as a large fortune and estate, the result of his father's policy of frugality and avoiding the royal court for most of his life. +The Clifford family, originally from Normandy, settled in England after the conquest of 1066. The family was elevated to the peerage in 1299 as Barons Clifford, and also held the minor baronies of Skipton in North Yorkshire and of Appleby in Westmoreland. The historian Chris Given-Wilson has described the Clifford family as one of the greatest 15th-century families never to receive an earldom. By the time of Clifford's birth, the King, Henry VI, was politically weak and occasionally incapacitated, which prevented him from ruling effectively. His failure to control his nobility, combined with the loss of England's French territories during the latter years of the Hundred Years' War had seen the political situation in England deteriorate into what the scholar David Loades has called a "chaos of factional quarrels". Civil war (known to historians as the Wars of the Roses) broke out in 1455. By 1461 a number of battles had been fought between nobles loyal to the Lancastrian King and those of the Yorkists, led by Richard, Duke of York, who had claimed the throne in 1460. +These engagements became increasingly bloody, comments the author Robin Neillands, "either in the actual battle or the subsequent rout". At the Battle of Wakefield in December 1460 Clifford's father supposedly encountered York's second son Edmund, Earl of Rutland, on Wakefield Bridge, as the latter was attempting to flee the destruction of his father's army. John, Lord Clifford, crying "by God's blood, thy father slew mine and so shall I slay thee", stabbed Rutland to death. Lord Clifford himself died on 28 March the following year during another clash at Ferrybridge, North Yorkshire. Tradition states that he was killed by a headless arrow to the throat and buried, along with those who died with him, in a common burial pit. +The next day, the bulk of the Yorkist and Lancastrian armies faced each other at the Battle of Towton. After what is believed to be the biggest and possibly bloodiest battle ever to take place on English soil, the Lancastrians were routed, and the son of the Duke of York was crowned King Edward IV. On 4 November 1461, at Edward's first parliament, the dead Lord Clifford was attainted and his estates and barony forfeited to the Crown. The bulk of the Clifford lands were granted to Richard, Earl of Warwick, while Richard, Duke of Gloucester, and Sir William Stanley received the Lordship of Westmorland and the Barony of Skipton respectively. The latter included the Clifford caput baroniae, Skipton Castle. +Henry Clifford was born around 1454, the eldest son and heir of John Clifford and Margaret Bromflete. In the view of the medievalist A. G. Dickens, Margaret, as sole heiress to her father Henry, brought Clifford's father a "questionable claim" to the title Lord Vescy. She also brought Clifford extensive lands in the East Riding. +Popular belief later held that as a boy of seven, Clifford was spirited away from his home in Skipton Castle following his father's death. For his own protection, so it went, his mother sent him to live in Londesborough on the property of a trusted family nurse where he employed himself tending the family's sheep. Whenever his mother believed him likely to be discovered he would be moved. Precisely where to is unknown, but both Yorkshire and Cumberland are possible; in the latter case, for example, Clifford's father-in-law held estates in Threlkeld. This supposedly gave Clifford the soubriquet "shepherd lord". The story seems to have originated with the 16th-century antiquarian Edward Hall and been reiterated by Lady Anne Clifford, in her 17th-century family history. The early modern historian Jessica Malay, argues that "with Edward IV on the throne (elder brother of the Earl of Rutland) and the Clifford hereditary lands forfeit, the Clifford dynasty was threatened with extinction". Lady Anne was, she says, "keen to emphasise the role of women in the survival of the Clifford dynasty", and as such created a "dramatic narrative" in which Margaret deliberately defies the crown for the sake of her dead husband's heir. Anne clearly believed that King Edward sought revenge for the murder of his younger brother, which put young Clifford's life in danger. Malay suggests that, while Anne Clifford believed the story of the shepherd's family taking her ancestor in, modern historians generally discount it as folklore, to greater or lesser degrees. It has received some traction; the 19th-century genealogist George Edward Cokayne accepted the story of Clifford's being "(for security against the disfavour with which his family was viewed by the reigning house) concealed by his mother" and raised as a shepherd, as did the antiquarian J. W. Clay in a 1905 article for the Yorkshire Archaeological Journal. The scholar R. T. Spence also repeated the story in his 1959 University of London PhD thesis on the later Cliffords (writing that Clifford was "brought up as a Shepherd boy to escape the fate of his father's victim"). Three years later Dickens (in his edition of the Clifford Papers) described how Clifford "aged about seven, lay in real danger and was brought up first as a shepherd". +The topographer Thomas Dunham Whitaker expressed doubt as to the 'shepherd lord' story's veracity in 1821. More recently, the historian K. B. McFarlane has gone further, arguing that it was probably "apocryphal", and J. R. Lander calls it "very dubious indeed". James Ross has pointed out that Clifford was pardoned by Edward IV in 1472 and could hardly have been in danger from the King thereafter. Further, he notes, as early as 1466 Clifford was named publicly as receiving a bequest of a sword and a silver bowl by Henry Harlington of Craven. This argues that the young lord could not have been difficult to find, comments Ross. He also, though, suggests that Clifford may well have kept a low profile after Towton, if only temporarily: "it may not have been with a shepherd, but surely Clifford was in hiding in secret somewhere". Malay also suggests that "in all likelihood, he spent only a few years in rural retreat" in Cumberland. Clifford's biographer Henry Summerson, writing in the Oxford Dictionary of National Biography, also refutes the theory, "later stories to the contrary notwithstanding, that the seven-year-old Henry Clifford was ever pursued by vengeful Yorkists". Summerson notes, for example, that Hall wrote that Clifford—due to his upbringing by remote shepherds—was illiterate. In reality, says Summerson, Clifford "was later to be not just literate but even bookish, owning volumes on law and medicine". Summerson agrees that "it may be that the Clifford heir thought it prudent to keep a low profile" in the early years of the new regime. While the medievalist Vivienne Rock subscribes to the theory that Clifford grew up ill-educated, she agrees that in later life "he did become an able administrator for his substantial estates". +Ross described the Clifford estates—centred on Cumberland, Westmorland, Durham and Yorkshire—as "valuable and strategically important in the troubled north". The 9th Baron had never, though, been as wealthy as some of the neighbouring families, such as the Darcys. His 1461 attainder prevented his son from inheriting, but in 1470 King Edward was forced from the throne and into exile, and Henry VI was returned to the throne. The Earl of Warwick—now aligned with the House of Lancaster against Edward—was in charge of the government, and his brother, John, Marquess Montagu, was granted the Henry Clifford's wardship during his minority. Summerson posits that this was a chance for Clifford to regain his inheritance. There was probably insufficient time to press his claim, however, as both Nevilles were killed at the Battle of Barnet on 14 April the following year. Edward IV's victory at Barnet, and at the Battle of Tewkesbury a few weeks later, destroyed the remnants of Lancastrian resistance and returned Edward to the throne. Despite Clifford's Lancastrian connections, he seems never to have been in any danger at this time, as on 16 March 1472 Edward granted him a royal pardon. This was despite an attempt by Clifford's brother Thomas to raise an—albeit unsuccessful—pro-Lancastrian rebellion in Hartlepool. Henry Clifford was duly allowed to inherit the estates of his maternal grandfather, Henry Bromflete, Lord Vescy—who had died in 1469—but not yet his Clifford patrimony. Further, as his mother was still alive, a third of his inheritance—her dower—remained out of his control until her death in 1493. +Edward IV died in April 1483 and his son Edward V was intended to succeed to the throne. However, he and his brother were declared illegitimate by their uncle, Richard of Gloucester, who took the throne himself as Richard III. Richard's reign was brief; in 1485 the heir of Lancaster, Henry Tudor, Earl of Richmond invaded England and defeated Richard at the Battle of Bosworth Field on 22 August 1485. Nothing is known of Clifford's career between his pardon in 1472 and the end of the Yorkist regime, except that he had remained in the country. Michael Hicks has suggested that his presence in the north, even though still attainted, made Gloucester's hold on the Clifford lands more fragile than was comfortable for the Duke: "no doubt Gloucester himself could keep what he had, but could his heirs?" Clifford had been one of a number of stalwart Lancastrian lords excluded from local power in the region during Gloucester's hegemony, first as Duke and then King. +Henry Tudor took the throne as Henry VII and from that point Clifford's position swiftly, and radically, improved. He received a number of local offices and sat on commissions in Westmorland and Yorkshire, although he was not to be appointed justice of the peace in the West Riding, until 1497. Following Bosworth, the new King's biggest priority was securing the north, where it was suspected that the Earls of Northumberland and of Westmorland were planning an insurrection. On 18 August Clifford was commissioned to raise a force to crush dissent in the region. He sent the earls to London under arrest and received into the King's grace those who wished to make peace with the new regime ("for all", notes A. J. Pollard, "but a number of named men"). On 24 October 1486, Clifford wrote to the city of York (at the time, the capital of the north) warning them not to sell arms or armour to non-residents. +Clifford was present at King Henry's first parliament on 15 September 1485, at which time he was legally still attainted. He attended every parliament until 23 November 1514, being summoned as Henrico Clifford de Clifford ch'r. During his first parliament Clifford successfully petitioned for the overturning of his father's attainder, which restored Clifford's patrimony to him. He was knighted on 9 November 1485. +Clifford made a natural ally for King Henry, and soon became one of his most trusted men in the north. Summerson suggests that Henry had little choice in restoring Clifford to his traditional regional position, as Northern England had been firmly Yorkist for over 20 years, first under the Nevilles and then under Gloucester. The latter had made Yorkshire his power base. Clifford, already loyal to Lancaster and then Tudor, was an obvious choice to act as the King's man, and Henry gradually increased Clifford's power. On 2 May 1486 Clifford received the stewardship of the Lordship of Middleham and bailiwick of the Honour of Richmond. The former had been one of Richard of Gloucester's most important headquarters. After Richard took the throne, he granted it to Sir John Conyers, one of Gloucester's closest advisers; both Middleham and Richmond had been Neville strongholds before that. Conyers seems to have been placed in Clifford's custody around this time, although relations between the two men seem to have improved: Clifford later jointly shared in a £1,000 bond to the King for Conyers's good behaviour. In October 1486 Clifford sat on a commission to "levy for the King, all profits arising from the King's manors and lands in the counties of Westmorland and Cumberland, the lordship of Penrith and the forest of Inglewood" in expectation of an invasion by Scotland. +The city of York jealously guarded its liberties, and traditionally rejected all interference from the outside unless it was perceived as absolutely warranted. This resistance troubled Clifford throughout his career. During the Yorkist rebellion of 1487, which attempted to place Lambert Simnel on the throne (as a pretender for Edward IV's second son, Richard of Shrewsbury) Clifford was responsible for guarding the city. He reinforced the garrison with 200 of his men at arms; when the rebel army passed close by, Clifford followed it to Braham. He attempted to engage it on 10 June, but was beaten off. He camped in Tadcaster overnight, where word was brought to him that a small force of rebels, led by Lords Scrope of Masham and of Bolton had launched an assault on Bootham Bar. This forced Clifford to withdraw back to York and face the rebels on 13 June. The subsequent encounter was not an unqualified success, notes Summerson; Clifford was defeated in a scuffle outside the gates, and lost all his baggage. The military historian Philip A. Haigh writes that Clifford was "utterly disgraced" and R. W. Hoyle describes his efforts as a "fiasco". The city scribes "laconically recorded the disastrous outcome", writes Anthony Goodman, and emphasised how the King's man in the north "had signally failed" to contain the rising. +Meanwhile, the King's army under John de Vere, Earl of Oxford, had won a decisive victory over the rebels at the Battle of Stoke 16 June 1487. Clifford was again given responsibility for the safety of York, and he claimed "captenship" over the city, an assertion the city rejected. In 1488 Clifford and Lady Anne both joined the city's Corpus Christi Guild. This does not seem to have restored Clifford in the eyes of the city officialdom, as the following year they again refused him entry, claiming that his intentions threatened the city's liberties. This may well have been prescient, suggests Summerson, as in 1513 Clifford attempted to claim the city's troops for his own army. In 1489 the townspeople, "denyed the entrie of the Lords Clifford and othre, that in nowise noon othre gentilman of what degreor condiconhe he of be suffred to enter this the Kyngs Chaumbre and so all to be excludet and noon to have reule bot the Maiour, Aldermen and the Shireffs". The city's statement came just before rebellion again broke out in Yorkshire, this time against heavy taxes. The commons overran the city and refused to allow Clifford or the sheriff, Marmaduke Constable entry. Instead, the citizens not only allowed the rebels to enter, they provided them a degree of military assistance. The medievalist David Grummitt comments that the city's reluctance to allow Clifford either office or military assistance is in stark contrast to the fervour with which they served "our ful gode and gracious lorde the duc of Gloucestre" as both Duke and King. +Clifford was in London in 1494 when he and the King's second son, Prince Henry, among others, were made Knights of the Bath. Clifford spent much of the remainder of the decade on service in the north. Although he never held office on the border, he led a major campaign in 1497, besieging and capturing Norham Castle from the Scots. Clifford was probably a member of the Council of the North around the turn of the century. This body was under the nominal leadership of Prince Arthur and managed by the Archbishop of York, Thomas Savage. Clifford's lordship of the north, posits Summerson, was reciprocal: Henry extended royal power in the region by strengthening Clifford, and likewise, Clifford strengthened and augmented his own position through royal service. +Clifford, although a figure of political and social influence, only ever had regional interests. His approach to his estates was generally positive, suggests Summerson. Clifford regularly travelled between Westmorland and Yorkshire (visiting manors "where no Clifford had been seen for a quarter of a century") and took the opportunity to rebuild and repair castles and other properties as he did. These he funded with traditional feudal dues, such as offices, wardships and marriages that were within his purview. His determined augmentation of his estates occasionally led to summonses before the royal council for enclosing land. Conversely, Clifford attempted to build good relations with his tenants and neighbours through financial generosity and hospitality, such as in 1521, when he held a "great Christmas" at Brough Castle. +On occasion, Clifford made the enmity of his neighbours as a direct result of his royal service. For example, it was often to the Crown's advantage that, where possible, it influenced civic elections in favour of royal candidates. A particularly important such office was that of the city recorder. In the early years of Henry's reign the administration of York, as the capital of the north, keenly interested the King. Its regional position, combined with a history of Yorkist loyalism, made it, the scholar James Lee suggests, a "touchstone for loyalty to Henry". The King attempted to impose his own man, but the city council disagreed. Clifford then attempted to intercede for the King, but to no avail, and in the end, a compromise candidate, John Vavasour, was elected. Summerson notes that Clifford's attempts to insert himself into local politics were "not always well-received". Summerson highlights Clifford's declaration in 1486 to the Mayor and Common Council that he intended "to mynistre as myn auncistres haith done here to fore in all thinges that accordith to my dewtie". In response, York's officials "firmly" informed Clifford that he had no such duty as his ancestors had never wielded such authority. Clifford also attempted, unsuccessfully, to influence the civic celebrations the city organised for the King's first visit to York later the same year. He wished, says Lee, to show the King the degree to which he was in control now that he had been returned to his family's traditional position; he was told by Vavasour that the city would do as it saw fit. +In 1487 the Earl of Oxford had been granted the wardship and marriage of the 17-year-old Elizabeth Greystoke, granddaughter and sole heiress of Ralph, Baron Greystoke. Oxford soon sold the rights (worth nearly £300 per annum) to Clifford. Within a short time, though, Elizabeth was taken from Clifford's custody ("without leave asking, and not without peril to his person") by Thomas, Lord Dacre. By 1491, relations between the two men had deteriorated to the extent that the King personally prosecuted them both in the Star Chamber for rioting; they were each fined £20. King Henry was more likely to have been concerned, in cases such as these, with bending his tenants-in-chief to his political will than the revenue these forfeits added to his Exchequer. Hicks has suggested that this behaviour made Clifford less trustworthy in Henry's eyes as a crown agent. In 1496 the Captain of Carlisle, Henry Wyatt, wrote to the King expressing, as Agnes Conway calls it, his "poor opinion" of Clifford. Wyatt considered Clifford's wife, Lady Anne St John, to be a more able administrator than her husband, whom he considered inefficient, and told the King so plainly. +Clifford's success at improving his finances eventually placed him in the top third of the English nobility and enabled him to successfully create new connections and strengthen existing ones. This he achieved through both marriage alliances with, and retaining among, the local gentry. Clifford was also a major patron to local abbeys, monasteries and priories. To Bolton Priory, for example, he donated a manuscript now known as A Treatise of Natural Philosophy in Old French. Other houses included Gisborough, Mount Grace and Shap; Mount Grace was particularly favoured. Clifford was a regular correspondent with the heads of other houses, including Byland, Carlisle, Furness, Holmcultram and St. Mary's, York. His extensive patronage did not always bring him success in his political negotiations with them. In 1518, for example, the Dean of York, Brian Higton wrote to Clifford explaining why he had refused to accept Clifford's favoured nominee as parish priest of Conisbrough Church:Where ye dide of laite presente your clerk unto the church of Conesburgh of your patronege, surely I cane nott (of my conscience) admytte hym to itt, fore his connyng is mervyllus slendure. I haue scyne few prestis so symple lernede in my life. If itt please you to commande some of your lernede chapplens to oppoise hym in your presence, I dowte not butte ye shall perceyue the truth. And fore the lakk of his lernynge (Which is manifesteo) I do putte hym bakk, ande fore noyne oder cause, nor at no mannys desire or motlon. +In the later years of the 15th century, Clifford was frequently the target of the King's displeasure. He often failed to act as the stabilising force in the north that Henry had intended. A feud with Christopher Moresby, an important member of the local gentry, had started in the 1470s and continued well into Henry's reign. Another time, Clifford led local resistance to a royal tax. In retaliation, Henry challenged Clifford's hereditary right to the shrievalty of Westmorland with quo warranto proceedings in 1505. Clifford's goods were sequestered until he could show by what authority he held the office, and he also had to provide a number of large obligations for his good behaviour. These included a £1,000 bond in May that year, £200 if he departed the council without permission and £2,000 on condition that he, his servants, tenants and "part-takers" kept the peace with Roger Tempest. Clifford had an ongoing feud with Tempest and had attacked and pulled down Tempest's house in Broughton. Although Clifford's shrieval rights were in the event upheld, the case took over a year to be decided, during which time the profits of the office went to the King. On 14 June 1506 Edmund Dudley delivered Clifford his general pardon. By this time Clifford had paid another £100 in cash ("redie money") to the King and had been pressured for £120 more. +King Henry died on 21 April 1509, and Clifford attended his funeral in Westminster. He stayed to attend the coronation of King Henry VIII on 23 June, when he was made a knight banneret. Shortly after, Dudley—by then imprisoned in the Tower of London on charges of constructive treason—petitioned Henry VIII over what he believed were grave injustices carried out by the King's father against members of his nobility, including Clifford. The period Clifford spent in the south was one of the few occasions in Clifford's life where he spent a lengthy period away from his northern heartlands. According to Cokayne—possibly citing an unnamed contemporary—Clifford "seldom 'came to court, or London'", spending much of his time in Barden Tower, Bolton, from where most of his extant charters and letters are signed. +War with Scotland broke out again in 1513 when the Scottish King, James IV, declared war on England. James intended to honour the Auld Alliance with France by diverting Henry VIII's English troops from their campaign against the French, against whom England was a member of the Catholic League in the War of the League of Cambrai, supporting the Pope. Henry VIII had also opened old wounds by claiming to be the overlord of Scotland, further angering the Scots. The first—and as it turned out, the only—engagement of the Scottish campaign was fought at Flodden on 9 September. Clifford brought 207 archers and 116 billmen from Yorkshire under his banner of the Red Wyvern and commanded the vanguard. King James was killed in battle, and Clifford captured three Scottish cannons which he took to "decorate" Skipton Castle; the contemporary Ballad of Flodden Field refers to "Lord Clifford with his clapping guns". +In 1521, the Emperor Charles V resumed war with Francis I. King Henry offered to mediate, but this achieved little and by the end of the year England and the Empire were aligned together against France. Clifford provided 1,000 marks towards funding the campaign, one of the highest sums the crown received. +Clifford is known to have married twice. Possibly at the end of 1486—and certainly by 1493—he had wed Anne St. John of Bletsoe Castle. She was the daughter of Sir John St John and Alice, daughter of Sir Thomas Bradshaigh of Haigh. Anne's grandmother was Margaret Beauchamp, the mother of Margaret Beaufort, making Anne half-cousin to King Henry VII. It is probable that the King and his mother had a hand in arranging Anne's marriage to Clifford. Their relationship does not seem to have been peaceful, and this probably exacerbated the King's disfavour of Clifford. Clifford's marriage problems were in part due to his conspicuous infidelity, which caused sufficient tension between him and Anne that their separation was suggested. Anne's chaplain began negotiating this with the King and Lady Margaret Beaufort, who went as far as to offer Anne and her daughters a position in Margaret's household expressing the wish that Anne "shall come up and attend upon my Lady". In the event, the crisis passed and Clifford and Anne stayed together until her death in 1508. She was buried in Skipton Church. +By July 1511, Clifford had married Florence Pudsey, widow of Thomas Talbot. She was the daughter of Henry Pudsey of Berforth and Margaret Conyers, daughter of Christopher Conyers of Hornby. Clifford and Lady Florence were enjoined to the confraternity of Guisborough Abbey. Their marriage, too, was fraught with difficulties, and Florence sued her husband in York consistory court for the restitution of conjugal rights. In doing so, suggest the scholars Tim Thornton and Katherine Carlton, "she did not perhaps expect her own conduct to be brought into question". Clifford, though, in his turn, accused her of adultery with a member of his household, one Roger Wharton. Wharton, under examination in court, confessed that "I will never denye ffor a man may be in bedd with a woman and yett do noo hurte". Thornton and Carlton continue, "in one simple statement, Wharton shed light upon the sexual mores of the Clifford household". Wharton also accused Clifford of having an extra-marital relationship with one Jane Browne, also of his household. +Clifford had several illegitimate children by a number of mistresses, including two sons, Thomas and Anthony. They both later received positions within the family, Thomas becoming deputy-governor of Carlisle Castle in 1537, and Anthony being appointed steward of Cowling, Grassington and Sutton. Both were also made master foresters of Craven. Thomas and Anthony may have been illegitimate, but Clifford considered them men of "substance, education and experience [and] gentlemen", and provided for them in his will. +From his first marriage to Anne, he left two sons, his heir Henry, and Thomas. With Anne, he also had four daughters, and by Florence, another daughter. A number of these married into the Bowes family of Streatlam, Co. Durham. Clifford's heir and namesake was born around 1493, and was raised at court with the King's son, the future Henry VIII. The relationship between father and son appears to have been as turbulent as that between Clifford and his wives, with a relationship "strained to breaking point", suggests Dickens. In 1511, Clifford complained that young Henry was both wild and a wastrel, who dressed flamboyantly in cloth of gold, "more lyk a duke than a pore baron's sonne as hee is". He protested about "the ungodly and ungudely disposition of my son Henrie Clifforde, in such wise as yt was abominable to heare it". Among his complaints was that Henry had threatened Clifford's servants and disobeyed his father. Clifford also alleged that his son had assaulted Clifford's old servant Henry Popely, had damaged and stolen Clifford's possessions and had sought to retain important men from Clifford's "countree" for himself. He had also harmed Clifford's close relations with local religious institutions, said Clifford, by stealing tithes and beating their tenants and servants. The King, meanwhile, had ordered Clifford to pay £40 to his son towards his upkeep at court, which Clifford had done. Clifford had urged his son "to forsake the dangerous counsels of certain evilly-disposed young gentlemen". Clifford's exhortations were not wholly successful, as on at least one occasion his son was incarcerated in the Fleet Prison. +Summerson suggests that Clifford was to a degree culpable for his son's behaviour, considering that if he "had ideas above his station, the responsibility was largely his father's, who not only placed him at court but also set about marrying him into the high aristocracy". It is also probable, suggests Dickens, that Clifford's own frugality towards his son's expenses encouraged his heir's behaviour, perhaps combined with irritation at his father's longevity. Furthermore, Dickens asserts, young Henry's sojourn at court forced a great distance between him and his father, which prevented him from learning at first-hand the responsibilities he would at some point be expected to take up in the north. Young Henry also appears to have fallen out with his stepmother Florence. It was intended that he marry Margaret, daughter of George, Earl of Shrewsbury, but she died before the betrothal. In 1512 young Henry married Margaret Percy, daughter of the Earl of Northumberland, which further augmented the Clifford family's wealth and influence in the northeast. +Historians have speculated on Clifford's personality. Summerson, for example, suggests that Clifford was often an abrasive individual, particularly to his tenants and regularly caused the very kind of social disorder that he was expected to suppress. Ross has speculated that Clifford's early years, particularly "the impact of Towton ... must have been profoundly shocking and traumatic", while Goodman has suggested that Clifford's solo attack on the 1487 rebels at Brougham indicates a chivalrous streak, as personal bravery was a highly prized quality. Micheal K. Jones and Malcolm G. Underwood have described Clifford as "eccentric", possibly on account of his upbringing. +Clifford is known to have had an interest in astrology, astronomy and alchemy. A major eclipse crossed England in 1502, for which occasion Clifford is supposed to have built Barden Tower as an observatory. The astronomer S. J. Johnson has speculated that it was his witnessing the eclipse that sparked Clifford's interest in the subject, "in which he did greatly delight". It is likely that Clifford's obsession with the skies—which led him to spend most of his time as a recluse in Barden Tower—was the cause of his wife's consistory suit for her conjugal rights. In Barden, says Jones and Underwood, Clifford led a "strange, reclusive existence". +Clifford had religious interests also and in 1515 spent a large sum on a new chapel, which was intended to be as extravagant as possible. +By September 1522 Clifford was described as "feebled with sickness". The Scottish war was ongoing, and it had been planned that Clifford would again lead an army; in the event, he was too ill to do so, and his son took his place. Clifford died on 23 April 1523. His widow, Florence, later remarried to Richard Grey, son of Thomas Grey, Marquis of Dorset; she died in 1558. Clifford was buried in either Bolton Priory or that of Shap. Following his death, inquisition post mortems assessed his annual income at £1332 2s. 4d, and Lady Anne Clifford later reported him rich "in money, chattells, goods and great stocks of land". His son Henry—no longer a minor—gained livery of his patrimony on 18 July 1523. He was summoned two years later to parliament and created Earl of Cumberland. The elevation of the Clifford family to the upper peerage, suggests Summerson, "owed much to Henry Clifford [the elder]'s labours to revive the fortunes of his family". Spence explains Clifford's wealth as resulting from "the prudence and economy of a lifetime's residence on his estates", combined with abstinence of court and its expense, except when made unavoidable by summonses to parliament. Spence also notes, though, that the first Earl was to go on to both waste and neglect his estates in favour of extravagant court living. +The Romantic poet William Wordsworth wrote two pieces—Song at the Feast at Brougham Castle and White Doe of Rylstone—romanticising Clifford's career. The White Doe, written between 1806 and 1807 describes Clifford as being "most happy in the shy recess / of Barden's lowly quietness". Wordsworth depicts various aspects of Clifford's life: the loss of his estates in 1461, his rustic upbringing—and the role his father-in-law, Sir Lancelot Threlkeld played—his post-Bosworth revival and his castle building. Wordsworth also imagines the Christmas celebration at Brough Castle "and the peculiarly Wordsworthian results" of Clifford's early life. The poem, suggests the scholar Curtis Bradford, indicates that Wordsworth "was not entirely uninterested in the antiquarian romanticism so characteristic of his time". Charlotte Mary Yonge compares Clifford in his shepherd hut to the roaming of the deposed King Henry VI—now supposedly a hermit—around the north, and casts them together: "both are in hiding: each is content with his lot. The boy does not dream that the hermit is really a king. That he is a man of God is clear, and young Clifford loves him, for his goodness, and most willingly places himself under Henry's tutelage". +The life and career of Henry Clifford was fictionalised by Isaac Albéniz and Francis Money-Coutts—the former writing the music, the latter the libretto—in their opera Henry Clifford, which premiered in 1895. +Airport Central railway station is an underground Transperth commuter railway station at terminals one and two of Perth Airport in Western Australia. The station is located on the Airport line and is one of three stations that were built as part of the Forrestfield–Airport Link project. +The contract for the Forrestfield–Airport Link, which consists of 8 kilometres (5 mi) of twin bored tunnels and three new stations, was awarded to Salini Impregilo and NRW Pty Ltd in April 2016. Construction on Airport Central station began in March 2017 following preparatory work. By January 2018, excavation was complete and, in May 2018, the two tunnel boring machines (TBMs) reached the station after tunnelling from High Wycombe. The TBMs left the station tunnelling north-west in July, and construction of the rest of the station started. As well as the station, a 280-metre (920 ft) elevated walkway was built by Georgiou Group, linking the station to the airport's terminal T1. +Originally planned to open in 2020, the line officially opened on 9 October 2022. It is served by trains every twelve minutes during peak hour and every fifteen minutes outside peak and on weekends and public holidays. At night, trains are half-hourly or hourly. The journey to Perth station takes eighteen minutes. +Airport Central station is located adjacent to the air traffic control tower at Perth Airport terminals one and two (T1 and T2). To the east, the adjacent station is High Wycombe station. To the north-west, the adjacent station is Redcliffe station, which leads to Perth station and connections to the other lines on the Transperth system. +The station has three levels: a below-ground platform level, a below-ground concourse level above the platform level, and an above-ground entrance level which connects to a 280-metre-long (920 ft) elevated walkway called the Skybridge. The Skybridge crosses over a car park towards T1, with lifts and stairs connecting the Skybridge to external ground level approximately 200 metres (660 ft) east of T2. Linking the entrance level to the concourse level are two lifts and three escalators, which were the longest operational escalators in the southern hemisphere at 35 metres (115 ft) long and 15 metres (49 ft) high, until they were overtaken by 45-metre (148 ft) escalators at Sydney Central station in 2024. On the concourse are fare gates and toilets. Linking the concourse to the platform are two lifts, two pairs of escalators, and two sets of stairs. The platform level consists of an island platform 12.5 metres (41 ft) wide and 150 metres (490 ft) long at a depth of 17 metres (56 ft) below ground level. Each of the two platform edges can accommodate a six-car Transperth train. +The station was designed by Weston Williamson and GHD Woodhead. In June 2023, the station won the Award for Public Architecture at the Australian Institute of Architects's WA Architect Awards. +The station takes its name from the Airport Central precinct, which encompasses the area around T1 and T2. Rather than be located directly next to T1, the station is placed centrally within the precinct so that it is as close as possible to any future terminals. Under the Perth Airport master plan, terminals three and four (T3 and T4) near Redcliffe station will be replaced by new terminals in the Airport Central precinct. The master plan states that new terminals will be connected to the station via an extension of the Skybridge or by a new underground walkway. A proposed hotel next to the station may also be connected to the Skybridge. As of 2024, T3 and T4 are planned to be replaced by new terminals at Airport Central from 2031. +The design of the roof is based on the contours of aircraft. There is a two-part artwork along the northern and southern interior walls of the station by Anne Neil and John Walley entitled Journeys. The artwork along the northern wall consists of multi-coloured petal-shaped objects designed to evoke birds and flight. On the southern wall are 50 glass panels with an artwork designed to "describe the Noongar connection to the river and estuary waters, and the journeys traditionally taken in accordance with the six seasons." Within the Skybridge, audio plays reflecting upon the significance of the Swan River and the Whadjuk country to the Noongar people. The audio consists of a narrative voiceover by a traditional custodian, animal sounds, and an original musical score. The audio changes throughout the day, mimicking the journey along the Swan River. There is also artwork along the Skybridge walls and travellators designed by Jade Dolman and Crispian Warrell of Nani Creative. +During initial planning, the station was called Consolidated Airport station due to the planned consolidation of all passenger terminals to the precinct. The station was built by the Public Transport Authority (PTA) as part of the Forrestfield–Airport Link project, which involved the construction of 8 kilometres (5 mi) of twin bored tunnels from High Wycombe to Bayswater and two other stations: Forrestfield (later renamed High Wycombe) and Belmont (later renamed Redcliffe) stations. The station was renamed to its present name in April 2016 upon the awarding of the main contract, worth A$1.176 billion, to a joint venture of Salini Impregilo and NRW Pty Ltd (SI/NRW). At the time the contract was signed, the station was expected to be in operation by 2020. Weston Williamson and GHD Woodhead were appointed by SI/NRW as the designers of the three stations. The construction of the Skybridge was under a different contract, awarded to Georgiou Group in late 2018 for $31 million. The Skybridge was funded partly by an $8.6 million contribution by Perth Airport and partly by the Forrestfield–Airport Link budget; its construction was managed by Perth Airport. +In late 2016, work began on reorganising the airport's car park and car rental booths to make way for the Airport Central station construction site. The station box was planned to be excavated and constructed between March 2017 and February 2018. During this time a headstone from the 1890s was discovered. Construction on the station began in May 2017, starting with the construction of diaphragm walls. The station's design was released in May 2017. The diaphragm walls were completed in July 2017, allowing excavation to begin the same month. Excavation was completed in January 2018 and construction of the concrete base slab commenced the following month. The base slab was completed by April 2018. +The two tunnel boring machines (TBMs) were planned to arrive at Airport Central station in late February and late March 2018, having tunnelled from High Wycombe. This was delayed after the first TBM, Grace, was stopped on 14 February following a ground disturbance issue. This also caused the second TBM, Sandy, to stop on 28 March so that it would not be tunnelling next to Grace. The two TBMs resumed tunnelling in April. Grace arrived at the station on 8 May, and Sandy arrived on 19 May. After undergoing maintenance, both TBMs left the station in July, tunnelling towards Redcliffe. Construction on other elements of the station commenced after that, including staircases and infrastructure for the escalators and lifts. +In November 2018, construction on the Skybridge started. Construction on the station's steel structure and the concrete concourse slab began in early 2019. The 35-metre-long (115 ft) escalators were installed in May and June 2019. In mid-2019, construction of the platform and ventilation structures at the eastern and western ends of the station began and the modules for the Skybridge began to be lifted into place. By November 2019, the concourse slab and the steel lift frames were complete and the Skybridge was connected to the station. The Skybridge achieved practical completion in early 2020. By March 2020, the station was 70 percent complete and the roof was beginning to be installed. The roof was mostly complete by June 2020, allowing work such as the fit-out of escalators and lifts and the installation of cladding along the walls to commence. +In December 2018, State Transport Minister Rita Saffioti announced that the opening date of the project had been delayed from 2020 to 2021. In May 2021, she announced that the project had been delayed again, the new opening date being the first half of 2022. Following the state budget in May 2022, the government changed its position on the line's opening date, saying it would open some time later in the year. On 16 August, the opening date was revealed to be 9 October 2022, which is when the station did open. +Airport Central station is served by the Airport line on the Transperth network. These services are operated by the PTA. To the east, the line terminates at the next station, High Wycombe. To the west, the line joins the Midland line and Ellenbrook line two stations along at Bayswater, running along that line to Perth station, before running along the Fremantle line to terminate at Claremont station. Airport line trains stop at the station every twelve minutes during peak on weekdays and every fifteen minutes outside peak and on weekends and public holidays. At night, trains are half-hourly or hourly. The last train leaves at about 2 am with the next train arriving approximately four to five hours later – 5:30 am on weekdays in 2022 – which has been criticised for not being early enough for fly-in fly-out workers, whose flights typically leave in the early morning. The Mining and Energy Union called for the government to tweak the train schedule. The transport minister has said that the line needs to be closed for maintenance overnight but that she would look at what could be done. The Airport line caters to about 80 percent of flights. +The station was predicted to have average daily boardings of 6,100 upon opening, rising to 11,000 in 2031. The journey to Perth station takes eighteen minutes. The station is in fare zone two, and there is no surcharge. +South-west of the station is a bus stop which is used by rail replacement bus services to Perth and High Wycombe as well as bus routes 37 and 224, which travel to Cannington and Oats Street stations respectively. +Airport Central station information page on the Transperth website +Airport Central station information page on the Forrestfield–Airport Link website +Airport Central station animation on the State Library of Western Australia website +Sir John Barbirolli (né Giovanni Battista Barbirolli; 2 December 1899 – 29 July 1970) was a British conductor and cellist. He is remembered above all as conductor of the Hallé Orchestra in Manchester, which he helped save from dissolution in 1943 and conducted for the rest of his life. Earlier in his career he was Arturo Toscanini's successor as music director of the New York Philharmonic, serving from 1936 to 1943. He was also chief conductor of the Houston Symphony from 1961 to 1967, and was a guest conductor of many other orchestras, including the BBC Symphony Orchestra, London Symphony Orchestra, the Philharmonia, the Berlin Philharmonic and the Vienna Philharmonic, with all of which he made recordings. +Born in London of Italian and French parentage, Barbirolli grew up in a family of professional musicians. After starting out as a cellist, he was given the chance to conduct, from 1926 with the British National Opera Company, and then with Covent Garden's touring company. On taking up the conductorship of the Hallé he had less opportunity to work in the opera house, but in the 1950s he conducted productions of works by Verdi, Wagner, Gluck, and Puccini at Covent Garden with such success that he was invited to become the company's permanent musical director, an invitation he declined. Late in his career he made several recordings of operas, of which his 1967 set of Puccini's Madama Butterfly for EMI is probably the best known. +Both in the concert hall and on record, Barbirolli was particularly associated with the music of English composers such as Elgar, Delius and Vaughan Williams. His interpretations of other late Romantic composers, such as Mahler and Sibelius, as well as of earlier classical composers, including Schubert, are also still admired. +Giovanni Battista Barbirolli was born on 2 December 1899 in Southampton Row, Holborn, London, the second child and eldest son of an Italian father and a French mother. He was a British national from birth, and as Southampton Row is within the sound of Bow Bells, Barbirolli always regarded himself as a Cockney. His father, Lorenzo Barbirolli (1864–1929), was a Venetian violinist who had settled in London with his wife, Louise Marie, née Ribeyrol (1870–1962). Lorenzo and his father had played in the orchestra at La Scala, Milan, where they had taken part in the première of Otello in 1887. In London they played in West End theatre orchestras, principally that of the Empire, Leicester Square. +The young Barbirolli began to play the violin when he was four, but soon changed to the cello. He later said that this was at the instigation of his grandfather who, exasperated at the child's habit of wandering around while practising the violin, bought him a small cello to stop him from "getting in everybody's way". His education at St Clement Danes Grammar School overlapped, from 1910, with a scholarship at Trinity College of Music. As a Trinity student, he made his concert debut in a cello concerto in the Queen's Hall in 1911. +The following year he won the Ada Lewis Scholarship to study at the Royal Academy of Music, which he attended from 1912 to 1916, studying harmony, counterpoint and theory under J. B. McEwen and the cello with Herbert Walenn. In 1914 he was joint winner of the academy's Charles Rube Prize for ensemble playing, and in 1916 The Musical Times singled him out as "that excellent young 'cello player, Mr Giovanni Barbirolli." The principal of the academy, Sir Alexander Mackenzie, had forbidden students to play the chamber music of Ravel, which he regarded as "a pernicious influence". Barbirolli was keenly interested in modern music, and he and three colleagues secretly rehearsed Ravel's String Quartet in the privacy of a men's lavatory in the academy. +From 1916 to 1918 Barbirolli was a freelance cellist in London. He recalled, "My first orchestral engagement was with the Queen's Hall Orchestra – I was probably the youngest orchestral musician ever, joining them in 1916. We had an enormous repertory – six concerts a week, three hours or more rehearsal a day. In those days we were happy if we began and finished together". While playing in the Queen's Hall Orchestra, Barbirolli also played in the opera pit for the Beecham and Carl Rosa opera companies, in recitals with the pianist Ethel Bartlett, with orchestras in theatres, cinemas, hotels and dance-halls, and, as he said, "everywhere except the street". During the last year of the First World War, Barbirolli enlisted in the army and became a lance-corporal in the Suffolk Regiment. Here he had his first opportunity to conduct, when an orchestra of volunteers was formed. He later described the experience: +I was stationed on the Isle of Grain – a ghastly place but the first line of defence against invasion – and in our battalion of the Suffolks we had a number of professional musicians. So we formed an orchestra and played in the equivalent of the NAAFI during our spare time. I was the principal cello and we were conducted by the bandmaster, one Lieutenant Bonham. The other boys knew that I was longing to conduct and one day when Bonham fell ill with 'flu, they thought "old Barby" – as I was known – should have a go. It was really rather romantic – I was scrubbing the floor in the Officers' Mess when they came and invited me to take over. We did the Light Cavalry overture and Coleridge-Taylor's Petite Suite de Concert but I can't say I recall the rest of the programme. +While in the army, Barbirolli adopted the anglicised form of his first name for the sake of simplicity: "The sergeant-major had great difficulty in reading my name on the roll-call. 'Who is this Guy Vanni?' he used to ask. So I chose John." After demobilisation he reverted to the original form of his name, using it until 1922. +On re-entering civilian life, Barbirolli resumed his career as a cellist. His association with Edward Elgar's Cello Concerto began with its première in 1919, when he played as a rank and file member of the London Symphony Orchestra. He was the soloist at another performance of the concerto just over a year later. The Musical Times commented, "Signor Giovanni Barbirolli was not entirely equal to the demands of the solo music, but his playing unquestionably gave a considerable amount of pleasure." At the Three Choirs Festival of 1920 he took part in his first Dream of Gerontius, under Elgar's baton, in the LSO cellos. He joined two newly founded string quartets as cellist: the Kutcher Quartet, led by his former fellow student at Trinity, Samuel Kutcher, and the Music Society Quartet (later called the International Quartet) led by André Mangeot. He also made several early broadcasts with Mangeot's quartet. +Barbirolli's ambition was to conduct. He was the prime mover in establishing the Guild of Singers and Players Chamber Orchestra in 1924, and in 1926 he was invited to conduct a new ensemble at the Chenil Gallery in Chelsea, initially called the "Chenil Chamber Orchestra" but later renamed "John Barbirolli's Chamber Orchestra". Barbirolli's concerts impressed Frederic Austin, director of the British National Opera Company (BNOC), who in the same year invited him to conduct some performances with the company. Barbirolli had never conducted a chorus or a large orchestra, but had the confidence to accept. He made his operatic debut directing Gounod's Roméo et Juliette at Newcastle, followed within days by performances of Aida and Madama Butterfly. He conducted the BNOC frequently over the next two years, and made his debut at the Royal Opera House, Covent Garden, with Madama Butterfly in 1928. The following year he was invited to conduct the opening work in Covent Garden's international season, Don Giovanni, with a cast that included Mariano Stabile, Elisabeth Schumann and Heddle Nash. +In 1929, after financial problems had forced the BNOC to disband, the Covent Garden management set up a touring company to fill the gap, and appointed Barbirolli as its musical director and conductor. The operas in the company's first provincial tour included Die Meistersinger, Lohengrin, La bohème, Madama Butterfly, The Barber of Seville, Tosca, Falstaff, Faust, Cavalleria rusticana, Pagliacci, Il trovatore, and the first performances in English of Turandot. In later tours with the company Barbirolli had the chance to conduct more of the German opera repertory, including Der Rosenkavalier, Tristan und Isolde, and Die Walküre. During his years with the touring opera companies Barbirolli did not neglect the concert hall. In 1927, deputising at short notice for Sir Thomas Beecham, he conducted the London Symphony Orchestra in a performance of Elgar's Symphony No. 2, winning the thanks of the composer. Barbirolli also won warm praise from Pablo Casals, whom he had accompanied in Haydn's D major cello concerto at the same concert. He conducted a Royal Philharmonic Society concert at which Ralph Vaughan Williams was presented with the society's Gold Medal, and another RPS concert at which Gustav Mahler's music, rarely heard at that time, was given – Kindertotenlieder, with Elena Gerhardt as soloist. Although Barbirolli later came to love Mahler's music, in the 1930s he thought it sounded thin. +When the Hallé Orchestra announced in 1932 that its regular conductor, Hamilton Harty, was to spend some time conducting overseas, Barbirolli was one of four guest conductors named to direct the orchestra in Harty's absence: the other three were Elgar, Beecham and Pierre Monteux. Barbirolli's programmes included works by composers as diverse as Purcell, Delius, Mozart and Franck. In June 1932, Barbirolli married the singer Marjorie Parry, a member of the BNOC. In 1933 he was invited to become conductor of the Scottish Orchestra. It was not then, as its successor the Scottish National Orchestra was later to be, a permanent ensemble, but gave a season lasting about six months of each year. Barbirolli remained with the Scottish Orchestra for three seasons, "rejuvenating the playing and programmes and winning most favourable opinions". Notwithstanding his growing reputation in Britain, Barbirolli's name was little known internationally, and most of the musical world was taken by surprise in 1936 when he was invited to conduct the New York Philharmonic Orchestra in succession to Arturo Toscanini. +By the spring of 1936, the management of the New York Philharmonic was confronted with a problem. Toscanini had left in search of higher fees with the NBC Symphony Orchestra. Wilhelm Furtwängler had accepted the orchestra's invitation to fill the post, but he was politically unacceptable to a section of the philharmonic's audience because he continued to live and work in Germany under the Nazi government. Following a campaign of protest in New York he felt unable to take up the appointment. For want of any available conductor of comparable fame the management of the orchestra invited five guest conductors to divide the season among them. Barbirolli was allotted the first ten weeks of the season, comprising 26 concerts. He was followed by the composer-conductors Igor Stravinsky, Georges Enescu and Carlos Chávez, each conducting for two weeks, and finally by Artur Rodziński of the Cleveland Orchestra, for eight weeks. +Barbirolli's first concert in New York was on 5 November 1936. The programme consisted of short pieces by Berlioz and Arnold Bax, and symphonies by Mozart (the Linz) and Brahms (the Fourth). During his ten weeks, he programmed several American novelties including Charles Martin Loeffler's tone-poem Memories of My Childhood, a symphony by Anis Fuleihan, and Philip James's Bret Harte overture. He also conducted Serge Koussevitzky's Double Bass Concerto. The players told the philharmonic management that they would be happy for Barbirolli to be appointed to a permanent position. The outcome of this was an invitation to him to become music director and permanent conductor for three years starting with the 1937–38 season. At the same time as this great change in his professional life, Barbirolli's personal life was also transformed. His marriage had not lasted; within four years he and Marjorie Barbirolli had been living apart. In 1938 she sued for divorce on the grounds of his desertion. The suit was undefended, and the divorce was granted in December 1938. In 1939, Barbirolli married the British oboist Evelyn Rothwell. The marriage lasted for the rest of Barbirolli's life. +One of the features of Barbirolli's time in New York was his regular programming of modern works. He gave the world premières of Walton's second Façade Suite, and Britten's Sinfonia da Requiem and Violin Concerto; he also introduced pieces by Jacques Ibert, Eugene Goossens, and Arthur Bliss and by many American composers including Samuel Barber, Deems Taylor and Daniel Gregory Mason. The new works he presented were not avant-garde, but they nevertheless alienated the conservative subscription audience, and after an initial increase in ticket sales in his early years sales declined. Barbirolli also had to cope with what The Gramophone described as "a rough press campaign in New York from interested parties who wished to evict him from his post". The influential critic Olin Downes had opposed Barbirolli's appointment from the outset, insisting that, though "we abhor chauvinism", preference should have been given to "native conductors". Downes had a grudge against the philharmonic: shortly before Barbirolli's appointment Downes was sacked as the commentator for the orchestra's prestigious Sunday broadcasts. He and the composer Virgil Thomson continually wrote disparagingly about Barbirolli, comparing him unfavourably with Toscanini. The management of the orchestra nevertheless renewed Barbirolli's appointment in 1940. In 1942, when his second contract was reaching its expiry, he was offered 18 concerts for the 1943–44 season, and the Los Angeles Philharmonic invited him to become its conductor, but he accepted neither offer as he had decided to return to England. +Barbirolli's first reason for leaving was local musical politics. He later said, "The Musicians Union there ... brought out a new regulation saying that everyone, even soloists and conductors, must become members. Horowitz, Heifetz and the rest were shocked by this but there was little they could do about it. They also said that conductors must become American citizens. I couldn't do that during the war, or at any time for that matter." His second reason for leaving was that he felt strongly that he was needed in England. In the spring of 1942 he made a hazardous Atlantic crossing: +I was in America when the war broke out, as conductor of the New York Philharmonic. A. V. Alexander, who was First Sea Lord, wrote to me to say that, contrary to expectations, music was flourishing and would I come back as I was missed. I was longing to return and it was just a question of how it was to be managed. A.V. went to Churchill, who apparently said, "If he's fool enough to come, let him come". It took us 23 days to cross on a fruit trader and, of our convoy of 75, only 32 ships arrived in Liverpool. I played here for ten weeks with the LSO and LPO for the benefit of the musicians, and then went back on a Fyffe banana boat of 5,000 tons. We were spotted by U-boats the moment we left Northern Ireland but that kind of thing never worries me as I'm something of a fatalist. It had been wonderful anyhow to be back, to see England at its greatest, and to visit my old mother. +Barbirolli returned to New York to complete his contractual obligations to the philharmonic. Shortly after his return he received an appeal from the Hallé Orchestra to become its conductor. The orchestra was in danger of extinction for lack of players, and Barbirolli seized the opportunity to help it. +In 1943 Barbirolli made another Atlantic crossing, avoiding death by a fluke: he changed flights from Lisbon with the actor Leslie Howard when the latter wished to postpone his own flight for a few days. Barbirolli's plane landed safely; Howard's was shot down. In Manchester, Barbirolli immediately set about reviving the Hallé. The number of players in the orchestra was down to about 30. Most younger players were serving in the armed forces, and to compound the shortage the management of the orchestra had ended the arrangement by which many of its players were also members of the BBC Northern Orchestra. The Hallé board resolved that its orchestra must follow the example of the Liverpool Philharmonic, which the Hallé's former conductor Malcolm Sargent had transformed into a full-time, permanent orchestra. Only four of the players shared with the BBC chose to join the Hallé. +The Times later wrote of Barbirolli's first actions for the orchestra: "In a couple of months of endless auditions, he rebuilt the Hallé, accepting any good player, whatever his musical background – he found himself with a schoolboy first flute, a schoolmistress hornist, and various brass players recruited from brass and military bands in the Manchester area ... The reborn Hallé's first concert somehow lived up to the Hallé's great reputation." The Musical Times also noted, "From his earliest days with the orchestra it was the string tone that commanded immediate attention and respect. There was a fiery intensity and glowing warmth that proclaimed the born string coach". Barbirolli retained his reputation for training orchestras: after his death, one of his former players commented, "If you wanted orchestral experience you'd be set for life, starting in the Hallé with John Barbirolli." Further afield, critics, audiences and players in Europe and the United States commented on the improvement in the playing of their orchestras when Barbirolli was in charge. Later he extended his teaching skills to the Royal Academy of Music, where he took charge of the student orchestra from 1961. +Barbirolli refused invitations to take up more prestigious and lucrative conductorships. Shortly after he took over the Hallé he received an offer from the sponsors of an ambitious scheme that would have put him in charge of the London Symphony Orchestra, and in the early 1950s the BBC sought to recruit him for the BBC Symphony Orchestra. Also in the early 1950s the head of the Royal Opera House, David Webster, wanted him to become the musical director there. Barbirolli conducted six operas for Webster, Turandot, Aida, Orfeo ed Euridice, Tristan und Isolde, La bohème and Madama Butterfly, 1951–53, but he declined to be wooed away from the Hallé. His biographer Charles Reid wrote, "His Manchester kingdom is a kingdom indeed. He is not manacled or chivied in his choice of programmes. Broadly speaking he conducts only what he loves ... His kingdom approximates to a conductor's paradise." Nevertheless, in 1958, after building the orchestra up and touring continually, conducting up to 75 concerts a year, he arranged a less onerous schedule, allowing him more time to appear as a guest conductor with other orchestras. He also appeared at the Vienna State Opera, and Rome Opera House, where he conducted Aida in 1969. In 1960 he accepted an invitation to succeed Leopold Stokowski as chief conductor of the Houston Symphony in Texas, a post he held until 1967, conducting an annual total of 12 weeks there in early spring and late autumn between Hallé engagements. In 1961 he began a regular association with the Berlin Philharmonic Orchestra, which lasted for the rest of his life. +From 1953 onwards, Barbirolli and the Hallé appeared regularly at the Henry Wood Promenade Concerts at the Royal Albert Hall in London. As well as major works from the mainstream repertory they gave an annual concert of music by Viennese composers, including Franz Lehár and Johann Strauss, which, like Sir Malcolm Sargent's annual Gilbert and Sullivan nights, rapidly became a firm favourite with the promenaders. At one 1958 promenade concert Barbirolli and the Hallé played a replica of Charles Hallé's first concert with the orchestra in 1858. +Barbirolli's interest in new music waned in post-war years, but he and the Hallé appeared regularly at the Cheltenham Festival, where he premiered new works of a mostly traditional style by William Alwyn, Richard Arnell, Arthur Benjamin, Peter Racine Fricker, Gordon Jacob, Alan Rawsthorne, Kenneth Leighton and others. For its hundredth anniversary in 1958 the Hallé commissioned several new works, and gave the British premiere of Walton's virtuosic divertimento Partita. Increasingly, Barbirolli concentrated on his core repertory of the standard symphonic classics, the works of English composers, and late-romantic music, particularly that of Mahler. In the 1960s he made a series of international tours with the Philharmonia (Latin America, 1963), BBC Symphony Orchestra (Czechoslovakia, Poland and the USSR, 1967) and the Hallé (Latin America and West Indies, 1968). It was a lasting disappointment to him that it never proved possible to take the Hallé on a tour of the United States. +In 1968, after 25 years with the Hallé, Barbirolli retired from the principal conductorship; no successor was appointed in his lifetime. He was appointed the orchestra's conductor laureate. He reduced the number of his appearances with the Hallé, but nevertheless took it on another European tour in 1968, this time to Switzerland, Austria and Germany. In his last years a propensity to concentrate on detail at the expense of the whole of a piece became marked. His loyal friend and admirer the critic Neville Cardus wrote privately in 1969, "he seems so much to love a single phrase that he lingers over it, caressing it; meanwhile the general momentum is lost." His final year, 1970, was dogged by heart trouble; he suffered collapses in April, May, June and July. His last two concerts were with the Hallé at the 1970 King's Lynn Festival. He produced "inspired" renderings of Elgar's Symphony No. 1 and Sea Pictures. The last work he conducted in public was Beethoven's Symphony No. 7 on the Saturday before his death. On the day he died, 29 July 1970, he spent several hours rehearsing the New Philharmonia Orchestra for a forthcoming tour of Japan that he was scheduled to lead. +Barbirolli died at his London home of a heart attack, aged 70. Among planned engagements forestalled by his death were a production of Otello at the Royal Opera House, which would have been his first appearance there for nearly 20 years, and opera recordings for EMI, including Puccini's Manon Lescaut and Verdi's Falstaff. +Among Barbirolli's state awards were a British knighthood in 1949 and appointment as a Member of the Order of the Companions of Honour in 1969; the Finnish Grand Star and Collar of Commander 1st Class of the Order of the White Rose in 1963; from Italy the Order of Merit in 1964; and from France, Officier de l'Ordre des Arts et des Lettres, 1966, and Officier de l'Ordre national du Mérite, 1968. Awards from musical institutions included the Freedom of the Worshipful Company of Musicians, 1966; Honorary Academician of the Accademia Nazionale di Santa Cecilia, 1960; Gold Medal of the Royal Philharmonic Society, 1950; Bruckner Medal, Bruckner Society of America, 1959; and the Mahler Medal, Mahler-Bruckner Society of America, 1965. He was also awarded the title of Doctor of Music honoris causa (DMus h.c.) from the National University of Ireland in 1952. +There are memorials to Barbirolli in Manchester and London. Barbirolli Square in Manchester is named in his honour and features a sculpture of him by Byron Howard (2000). The square includes the present base of the Hallé Orchestra, the Bridgewater Hall, in which the Barbirolli Room commemorates the conductor. At his old school, St Clement Danes, now relocated in Chorleywood, the main hall is named in his honour. A commemorative blue plaque was placed on the wall of the Bloomsbury Park Hotel in Southampton Row in May 1993 to mark Barbirolli's birthplace. The Sir John Barbirolli Memorial Foundation of the Royal Philharmonic Society was instituted after his death to assist young musicians with the purchase of instruments. In 1972 the Barbirolli Society was set up with the principal aim of promoting the continued release of Barbirolli's recorded performances. Its honorary officers have included Evelyn Barbirolli, Daniel Barenboim and Michael Kennedy. In April 2012, he was voted into the inaugural Gramophone "Hall of Fame". +Barbirolli is remembered as an interpreter of Elgar, Vaughan Williams and Mahler, as well as Schubert, Beethoven, Sibelius, Verdi and Puccini, and as a staunch supporter of new works by British composers. Vaughan Williams dedicated his Eighth Symphony to Barbirolli, whose nickname, "Glorious John", comes from the inscription Vaughan Williams wrote at the head of the score: "For glorious John, with love and admiration from Ralph." Barbirolli did not disdain lighter repertoire. The music critic Richard Osborne wrote that, if all Barbirolli's recordings were to be lost except that of Lehár's Gold and Silver Waltz, "there would be reason enough to say, 'Now, there was a conductor!'" +Barbirolli's repertoire was not as wide as that of many of his colleagues because he insisted on exhaustive preparation for any work he conducted. His colleague Sir Adrian Boult liked and admired Barbirolli but teased him for his meticulousness: "We can't all be like you and spend months studying these things and then have days of rehearsals before we conduct them. For some of us they're only sporting events." Barbirolli was shocked by such levity. His approach was illustrated by the care he took with Mahler's symphonies. His biographer Michael Kennedy commented, "it is ironical that the effort of composing the symphonies shortened Mahler's life; interpreting them certainly put an enormous strain on Barbirolli in his last decade." He found that mastering a Mahler symphony took between 18 months and two years, and he would spend hours meticulously bowing all the string parts in preparation for his performances. His first performance of Mahler's Ninth took nearly 50 hours of rehearsal. +From almost the start of his career Barbirolli was a frequent recording artist. As a young cellist he made four records for Edison Bell in 1911, with piano accompaniment by his sister Rosa, and as part of the Kutcher and the Music Society string quartets he recorded music by Mozart, Purcell, Vaughan Williams and others in 1925 and 1926. As a conductor he began recording in 1927 for the National Gramophonic Society (an offshoot of The Gramophone). Among his records from that period was the first to be made of Elgar's Introduction and Allegro for Strings. On hearing it, the composer said, "I'd never realised it was such a big work." Elgar, despite an extensive discography as a conductor, never recorded the work himself, and some have speculated that "the breadth, nobility and lyrical poetry" of Barbirolli's interpretation left the composer disinclined to compete. In 1928 Barbirolli made some recordings for the Edison Bell label. The same year, he began his long association with the His Master's Voice (HMV) label. Immediately after the LSO concert at which he had stood in for Beecham, he was approached by Fred Gaisberg, the chief recording producer for His Master's Voice who signed him for his company shortly afterwards. An HMV colleague of Gaisberg described Barbirolli as "a treasure", because he "could accompany Chaliapin without provoking an uproar, win golden opinions from Jascha Heifetz, Artur Rubinstein, Fritz Kreisler and Pablo Casals, and conduct one of the finest recorded performances of the Quintet from Meistersinger". +Many of Barbirolli's pre-war recordings for HMV were of concertos. His reputation as an accompanist tended to obscure his talents as a symphonic conductor, and later, his detractors in New York "damned him with faint praise by exalting his powers as an accompanist and then implying that that was where it all stopped." Barbirolli became very sensitive on this point, and for many years after the war he was reluctant to accompany anyone in the recording studio. Among his early His Master's Voice records are works, mainly concertos, by Brahms, Bruch, Chopin, Dvořák, Glazunov, Mendelssohn, Mozart, Schumann, Sibelius, Tchaikovsky and Vieuxtemps. From the 1990s onwards, archive recordings of Barbirolli's early concerts in New York have been issued on CD. Kennedy wrote in 2004 that they "prove that the orchestra played superbly for him and that the criticism of him was largely unjustified." Recordings from this period include symphonies by Beethoven, Mendelssohn, Mozart, Schubert, Schumann, Sibelius and Tchaikovsky, and other orchestral music by Berlioz, Debussy, Menotti, Purcell, Ravel, Respighi, and Rimsky-Korsakov. +Within six months of his return to Britain in 1943, Barbirolli resumed his contract with HMV, conducting the Hallé in the Third Symphony of Bax and the Fifth of Vaughan Williams, followed by works by a wide range of composers from Corelli to Stravinsky. In 1955 he signed a contract with Pye Records, with whom he and the Hallé recorded a wide repertoire, and made their first stereophonic recordings. These records were distributed in the US by Vanguard Records. A company was formed, named Pye-Barbirolli, of which he was a director: the arrangement was designed to ensure an equal partnership between the company and the musicians. They made many recordings, including symphonies by Beethoven, Dvořák, Elgar, Mozart, Nielsen, Sibelius, Mahler, Tchaikovsky and Vaughan Williams, as well as a few concertos, short orchestral pieces and operatic excerpts. +In 1962, HMV persuaded Barbirolli to return. With the Hallé he recorded a Sibelius symphony cycle, Elgar's Second Symphony, Falstaff and The Dream of Gerontius, Schubert's Ninth Symphony, Vaughan Williams's A London Symphony, and works by Grieg and Delius. With other orchestras, Barbirolli recorded a wide range of his repertoire, including many recordings still in the catalogues in 2025. Of these, his Elgar recordings include the Cello Concerto with Jacqueline du Pré, Sea Pictures with Janet Baker, and orchestral music including the First Symphony, Enigma Variations and many of the shorter works. His Mahler recordings include the Fifth and Sixth Symphonies (with the New Philharmonia) and Ninth Symphony (with the Berlin Philharmonic). With the Vienna Philharmonic, he recorded a Brahms symphony cycle, and with Daniel Barenboim, the two Brahms Piano Concertos. He made three operatic sets for HMV: Purcell's Dido and Aeneas with Victoria de los Ángeles (1966), Verdi's Otello with James McCracken, Gwyneth Jones and Dietrich Fischer-Dieskau (1969), and a set of Madama Butterfly with Renata Scotto, Carlo Bergonzi and Rome Opera forces that has remained in the catalogues since its first issue in 1967. The impact of the last was such that the head of the Rome Opera invited him to come and conduct "any opera you care to name with as much rehearsal as you wish." HMV planned to record Die Meistersinger with Barbirolli in Dresden in 1970, but following the Warsaw Pact invasion of Czechoslovakia in 1968 he refused to conduct in the Soviet bloc, and his place was taken by Herbert von Karajan. +Notes +References +Ayre, Leslie (1966). The Wit of Music: Introduction by Sir John Barbirolli. London: Leslie Frewin. OCLC 857354. +Brookes, Christopher (1985). His Own Man: The Life of Neville Cardus. London: Methuen. ISBN 0-413-50940-0. +Cox, David (1980). The Henry Wood Proms. London: British Broadcasting Corporation. ISBN 0-563-17697-0. +Haltrecht, Montague (1975). The Quiet Showman: Sir David Webster and the Royal Opera House. London: Collins. ISBN 0-00-211163-2. +Horowitz, Joseph (1997). Understanding Toscanini. Berkeley: University of California Press. ISBN 0-520-08542-6. +Kennedy, Michael (1987). Adrian Boult. London: Papermac. ISBN 0-333-48752-4. +Kennedy, Michael (1971). Barbirolli, Conductor Laureate: The Authorised Biography. London: MacGibbon and Key. ISBN 0-261-63336-8. +Kennedy, Michael (1982). The Hallé, 1858–1983: A History of the Orchestra. London: Macmillan. ISBN 0-7190-0921-9. +Kennedy, Michael (1989). William Walton. Oxford: Oxford University Press. ISBN 0-19-315418-8. +Lindsay, Maurice (1951). "Northern Diary". In Ralph Hill (ed.). Music 1951. Harmondsworth: Penguin Books. OCLC 26147349. +Osborne, Richard (1998). Herbert von Karajan: A Life in Music. London: Chatto and Windus. ISBN 1-85619-763-8. +Previn, André (1979). Orchestra. London: Macdonald and Jane's. ISBN 0-354-04420-6. +Reid, Charles (1957). "John Barbirolli". In Milein Cosman (ed.). Musical Sketchbook. Oxford: Bruno Cassirer. OCLC 3225493. +Reid, Charles (1971). John Barbirolli – A Biography. London: Hamish Hamilton. ISBN 0-241-01819-6. +Reid, Charles (1968). Malcolm Sargent. London: Hamish Hamilton. OCLC 500687986. +Rennison, Nick (2003). The London Blue Plaque Guide. Stroud: Sutton Publishing. ISBN 0-7509-3388-7. +Rigby, Charles (1948). John Barbirolli. Altrincham: John Sherratt and Son. OCLC 500687986. +Rothwell, Evelyn (2002). Life with Glorious John. London: Robson Books. ISBN 1-86105-474-2. +John Barbirolli at AllMusic +Portraits of John Barbirolli at the National Portrait Gallery, London +Photos of Barbirolli throughout his career +Shellac recordings with John Barbirolli in the Online Archive of the Österreichische Mediathek. Retrieved 7 August 2026 +The 2012–13 Michigan Wolverines men's basketball team represented the University of Michigan during the 2012–13 NCAA Division I men's basketball season. The team played its home games in Ann Arbor, Michigan, at the Crisler Center. This season marked the team's 96th consecutive year as a member of the Big Ten Conference, and it is occasionally referred to as "Team 96". The team was led by sixth-year head coach John Beilein. +As the defending 2011–12 Big Ten Conference men's basketball season regular season co-champions, the Wolverines finished fourth in the conference in 2012–13 and as National Runner-up in the 2013 NCAA Division I men's basketball tournament after losing in the championship game to Louisville (Louisville's championship was later vacated due to a sex scandal, which ruled multiple players ineligible). The team achieved a 31–8 record, the most wins by the program in 20 seasons. +Following the 2011–12 season, the team lost graduating senior captains Zack Novak and Stu Douglass, who moved on to professional basketball careers in Europe. The incoming class of Mitch McGary, Glenn Robinson III and Nik Stauskas was ranked among the best classes in the nation by the media. With its new lineup, the team matched the greatest starts in school history. Starting the season with 11 consecutive wins matched the best start since Michigan's 1989 national champions, the 1988–89 team. At 16–0, Michigan matched its best start since the last repeat Big Ten Regular season championship, the 1985–86 team, tying a school record. Reaching 19–1 set a record for the best start in school history. The team also reached the number one position in the AP Poll for the first time since the Fab Five 1992–93 team. The team entered February with a 20–1 record (7–1 Big Ten), but with an injury to eventual B1G All-Defensive selection Jordan Morgan and a stretch of games against its strongest conference opponents, Michigan lost three out of four games. The team closed the season with a 5–5 run to finish tied for fourth in the conference and won one game in the Big Ten tournament before being eliminated. +The team was led by 2013 national player of the year, Big Ten Player of the Year and 2013 Consensus All-American Trey Burke and three additional All-Conference honorees. Tim Hardaway Jr. was named to the 2012–13 All-Big Ten first team by the coaches and to the second team by the media; Robinson was named an honorable mention All-Big Ten by the media, and Morgan was an All-Big Ten Defensive team selection. Stauskas and McGary earned multiple Big Ten Freshman of the Week recognitions during the season. Burke was the second National Player of the Year and eighth first-team consensus All-American in Michigan basketball history. +Michigan entered the 2013 NCAA Division I men's basketball tournament as the youngest team in the field. The team made its first Sweet Sixteen appearance since the 1993–94 team did so. The Wolverines made their first appearance in the Final Four and the national title game since 1993 (which was vacated along with their 1992 appearance due to a scandal involving the eligibility of Chris Webber). Following the season, at the 2013 NBA draft, Burke and Hardaway were selected ninth and twenty-fourth, respectively, becoming the first pair of first-round NBA draft choices from Michigan since the 1994 NBA draft. +The team lost both of its captains from the prior season to graduation after they completed their eligibility. Senior two-time captain Douglass turned pro, joining Basket Navarra Club of the Liga Española de Baloncesto. Senior three-time captain Novak graduated to play for Landstede Basketbal in the Dutch Basketball League. In addition to the loss of senior co-captains, the team lost three players who decided to transfer: Evan Smotrycz, Carlton Brundidge and Colton Christian. Smotrycz, who transferred to Maryland, had started in 42 of the 69 games he played during his first two years. He left the program as the fifth-best all-time three-point shooter with a percentage of 40.5. Brundidge transferred to Detroit and Christian transferred to Florida International. The program also announced the departure of walk-on Sai Tummala and the return of walk-on Corey Person for a fifth year. +Glenn Robinson III, son of former first overall NBA draft pick Glenn Robinson, verbally committed on September 14, 2010, as the first commitment in Michigan's class of 2012. On March 26, 2011, Michigan received its second verbal commitment of the class of 2012 from Canadian wing guard Nik Stauskas. On November 3, Mitch McGary, who was ranked as the number two prospect in the nation at the time, announced his verbal commitment to Michigan. Within hours of the commitment, ESPN announced that Michigan ranked as the fifth best class in the nation. Each of the three signed a National Letter of Intent with the team on November 9. After several other schools announced their commitments, the signing of McGary moved Michigan from outside the top 25 at the end of October to the number 7 class in the nation by mid-November, according to ESPN. That November, McGary, Robinson and Stauskas ranked 3, 34, and 79, respectively, in the Rivals.com Class of 2012 Top 150. +All three recruits had playoff success: McGary was a member of Brewster Academy's National Prep Championship team. Stauskas earned the championship game MVP of the NEPSAC Class AA Championship after leading St. Mark's to victory. Robinson led Lake Central to its first Indiana High School Athletic Association sectional title in 15 years. He earned the Post-Tribune's Boys basketball player of the year award for Northwest Indiana and the state MVP from the Indiana Basketball News. +McGary was invited to participate in the four-team April 1, 2012 All-American Championship, along with future teammate Robinson in New Orleans. Robinson was awarded the ESPNHS All-American Championship Player of the Game for his 16-point, 4-rebound performance, which included 5 dunks among his 6-for-7 shooting night. McGary was also selected to represent USA Basketball at the fifteenth annual Nike Hoops Summit on April 7 as part of the 2012 USA Junior National Select Team. +Michael "Spike" Albrecht committed to Michigan on April 6, 2012. Albrecht led Northfield Mount Hermon School to the 2012 NEPSAC Class AAA Boys' Basketball Tournament, defeating McGary's Brewster Academy in the semifinals. Albrecht was MVP of the tournament. Albrecht was a former AAU teammate of McGary and Robinson. Albrecht's and outgoing captain Novak's fathers were collegiate teammates. Albrecht was a 2011 graduate of Crown Point High School. After most of the big signings announced their decisions on the April 11 signing day, Michigan was ranked as the early number 5 preseason selection as a team by ESPN. +Zak Irvin was named 2013 Indiana Mr. Basketball, giving Hamilton Southeastern High School the state's first back-to-back winners (Gary Harris) from the same high school. Derrick Walton was runner up in the 2013 Mr. Basketball of Michigan by a 2130–2086 margin to Iowa State signee Monte Morris despite having one more first place vote. Both Walton and Irvin were 2013 Parade All-American honorees and were named state Boys Basketball Gatorade Player of the Year. +Michigan basketball hosted its on campus media day press conference on October 10, 2012, at Crisler Center. The team was represented by Burke, Hardaway, Morgan and Beilein at the Big Ten Conference Basketball Media Day press conference on October 24 in Rosemont, Illinois. When the 24-person Big Ten Media poll was announced, Michigan was predicted to finish second to Indiana. Burke was a preseason All-Conference selection by the same media. +Michigan enjoyed high expectations from the national media as well during the preseason. Sporting News named them to the number 5 position in its preseason poll, while conference foes Indiana and Ohio State were ranked number 1 and 3, respectively. Michigan also earned the number 5 ranking in both the preseason Coaches' Poll and preseason Associated Press poll, but behind Indiana (#1) and Ohio State (#4) in both. This was Michigan's highest preseason ranking since the 1992–93 team. Sports Illustrated ranked Michigan number 9 in its October preseason power rankings, again behind Indiana (#1) and Ohio State (#6), but when it printed its November preseason poll listed Michigan at number 6 ahead of Ohio State (#7), but behind Indiana (#1). Prior to the first exhibition game on November 1, senior Josh Bartelstein was named team captain. +ESPN's Eamonn Brennan noted that Michigan was a perimeter oriented team with possibly the best one-two guard combination in the country. Burke and Hardaway were both preseason John R. Wooden Award top 50 selections. Burke was a preseason All-American (Associated Press first team, CBS Sports second team, Sports Illustrated second team, ESPN second team, Sporting News third team) as well as a preseason Naismith Award top 50 selection. +The acronym "WE ON" was the slogan for the whole team for the 2012–13 season. The team and coaching staff selected it at the beginning of the season as another way for Michigan to say Bo Schembechler's famous quote, "the Team, the Team, the Team." The acronym "WE ON" stands for "When Everyone Operates N'sync." Michigan assistant coach Bacari Alexander even tweeted that the acronym summarizes this team-focused mindset. According to Dave Brandon, assistant coach LaVall Jordan and director of basketball operations Travis Conlan guided the team to develop the slogan. +Adidas warm up shirts were created and were first worn on November 21, 2012, before Michigan's NIT Season Tip-Off game against the Pittsburgh Panthers. The shirts use the school colors with blue with maize lettering. On February 12, 2013, when Michigan played the first of two regularly scheduled rivalry games against Michigan State, students in the Izzone at the Breslin Center wore T-shirts with "YOU OFF" printed in Spartan colors (green and white). Michigan State went on to win the game by 23 points. +"WE" – WE is also an acronym for "Wolverine Excellence", which is chiefly used by the team. Wolverine Excellence calls for players and managers to focus on its five core values: integrity, unity, passion, diligence and appreciation. +"ON" – ON represented each player's individual commitment to season-long improvement. There was a "U-M Fuse Box" (mock-University of Michigan set of switches) in the Men's Basketball locker room in Crisler Center. Each player had selected one word on which to focus during the season and before every practice and game, each player flipped his switch "on". This represented each player's commitment to improvement every day. +Prior to the season, the five freshmen were dubbed the "Fresh Five", a moniker hearkening remembrance of the incoming freshman class of 1991, known as the Fab Five, that led Michigan to back-to-back Championship games while starting the majority of games. However, the nickname and its expectations were downplayed most of the season by the team. Nonetheless, during the final two weeks of the regular season The New York Times ran a feature story on the team that reinvigorated the nickname while noting that the current five were the supporting cast rather than the stars of the team like the original five. +ESPN's Chantel Jennings tweeted a picture of a Flag of Canada redone in Michigan's team colors of maize and blue at Crisler Arena on December 4. One of her followers noted that the big version on the wall was a general Stauskas fan flag and that a little version of the flag was added next to it for each three-pointer Canadian Stauskas made in the game. By mid-December of his freshman season, the students in the Maize Rage (Michigan student section at Crisler Arena) had begun cheering Stauskas on with such a flag. The Michigan Public & Media Relations office started mentioning the flag before Christmas. The flag was mentioned widely in the press as he continued his hot shooting into January. +Through the first week of January, Stauskas was shooting 53.7% on three-point shots as the team held a 15–0 record. +The team, which represented Michigan Basketball for the ninety-sixth season in the Big Ten Conference, is sometimes referred to as Team 96 or Squad 96. Although this team represented Michigan for the ninety-sixth consecutive year, there was controversy as to whether the team should in fact be marketed as Team 96 because the 1908–09 Michigan Wolverines men's basketball team was not properly accounted for among the prior teams. The subsequent team was marketed as Team 98. +Hardaway began the season with his fourth career double-double by scoring 25 points (including 5-for-5 three-point shooting) and adding 10 rebounds in a 100–62 win over Slippery Rock; this earned him his second career Big Ten Player of the Week award. The 100-point performance of November 9 was Michigan's first since the 2007–08 team posted 103 points against Oakland on December 12, 2007. +Michigan hosted games for the first two rounds of the 2012 NIT Season Tip-Off. The team's first game of the tournament, a November 12 contest against the IUPUI Jaguars, resulted in a 91–54 Michigan victory. The win also gave Michigan its first consecutive 90-point performances since the 2000–01 team recorded three consecutive 90-point games. The following night, Michigan defeated Cleveland State by a 77–47 margin, marking the first time the school opened the season with three consecutive 30-point victories. The win also marked the first time Michigan had posted three consecutive 30-point wins since the 1988–89 Wolverines team (the 1989 national champion) won five consecutive 30-point games. +In the championship rounds of the NIT Season Tip-Off tournament at Madison Square Garden on November 21 and 23, Michigan defeated Pittsburgh and Kansas State, respectively, to win the tournament. Hardaway, who totaled 39 points at the Garden, was the tournament MVP and was joined on the all-tournament team by Burke, who contributed 27 points and 10 assists. Stauskas earned Big Ten Freshman of the Week honors for his NIT tournament performance in which he averaged 12.5 points and 4.5 rebounds. On November 27, Michigan raised its Big Ten championship banner from the season before. Then, the team defeated its first ranked opponent of the season – NC State (#18 AP/#18 Coaches) – in the ACC–Big Ten Challenge. The win was supported by Burke's first career double-double which included a career-high 11 assists, no turnovers and 18 points as well as a career-high 20 points from Stauskas in a 79–72 victory. +Ranked third in both polls, Michigan established a record as the highest ranked visiting team in the history of Carver Arena on December 1. Giving Stauskas his first career start, Michigan emerged victorious over Bradley. It was the third game of the season that Michigan never trailed in. Stauskas repeated as Big Ten Freshman of the Week on December 3 for his back-to-back 20-point performances. +On December 4, Michigan defeated Western Michigan 73–41, giving the team its first 8–0 start since the 1996–97 team; the team never trailed in the game. Michigan defeated Arkansas 80–67 in its December 8 matchup. It marked the fourth 9–0 start in school history (1988–89, 1985–86 and 1926–27) and the third consecutive game that Michigan never trailed. Michigan went to 10–0 on December 11 by defeating Binghamton 67–39. +Michigan traveled to play in the December 15 Brooklyn Hoops Winter Festival at the Barclays Center, where they defeated West Virginia (coach Beilein's previous employer) 81–66 to move to 11–0 for the third time in school history (1985–86 and 1988–89). The team never trailed in the contest. For averaging 23.0 points, 6.5 assists, 4.5 rebounds and 2.0 steals with only 1.0 turnover in the games against Binghamton and West Virginia, Burke earned Player of the Week on December 17. +On December 20, the team became the second in school in history (1985–86 went 16–0) to reach 12–0 when it beat Eastern Michigan. After recording his first career double-double in the game with only 18 minutes of play, McGary was recognized as Big Ten Freshman of the Week. Hardaway suffered an ankle injury that caused him to miss the December 29 game against Central Michigan and that broke his 81 consecutive games played streak that went back to the beginning of his Michigan career. The game also marked the team's second game without Jon Horford, who dislocated his knee during the West Virginia game. In Hardaway's absence, Burke posted his second career double-double with 22 points and a career-high tying 11 assists and Robinson posted his second career 20-point game. Stauskas added career highs with 5 three-pointers and 7 rebounds and earned his third Big Ten Freshman of the Week award on December 31. The thirteenth win clinched Michigan's ninth perfect non-conference regular season record. The team never trailed in the contest. +On January 3, Michigan opened the 2012–13 Big Ten conference schedule with a 94–66 win over Northwestern to move to 14–0. Burke had 23 points, 5 assists and a career high 4 steals, while Hardaway added 21 points, Morgan posted a double-double with 12 points and 13 rebounds, while Stauskas and Robinson added 10 points each. Burke and Robinson both posted their third and first career double-doubles, respectively, in a 95–67 victory against Iowa on January 6. It marked the first time that the team scored 90 points in back-to-back conference games since the 1999–2000 team and tied for the most points in a conference game since the 1997–98 team. On January 7, Burke was recognized as Big Ten Conference Player of the Week and Robinson was recognized as Conference Freshman of the Week. +The team tied the school record for best start by defeating Nebraska for its 16th straight victory to start the season on January 9. The victory also marked Beilein's 400th Division I coaching victory. On January 13, the team lost to Ohio State (#15 AP/#14 Coaches) snapping their winning streak. Horford returned to the lineup after missing 5 games. Michigan had been the only remaining unbeaten team and was expected to be ranked #1 if they had won. The loss was Michigan's 9th straight in Columbus. +Michigan defeated Minnesota (#9 AP/#12 Coaches) at Williams Arena on January 17, marking the first time Michigan defeated a top-10 team on the road since a December 6, 1996, victory by the 1996–97 team over Duke. After his performance, in which he scored 21 points while making 7 of 8 shots and 4 of 5 three-point shots, in addition to recording 5 rebounds, 3 assists, 3 steals and 2 blocks, Hardaway earned a second Big Ten Player of the Week Award. +Michigan defeated Purdue 68–53 on January 24 and Illinois 74–60 on January 27. This gave Michigan the first 19–1 start in school history. Robinson earned his second Big Ten Freshman of the Week award on January 28 for a pair of 12-point performances during a week in which he averaged 8 rebounds while maintaining 71.4% field goal percentage. Later that day, Michigan was ranked number one in the AP Poll with 51 of the 65 first place votes. It marked the first time Michigan ranked atop the AP Poll since the Fab Five 1992–93 team did so on December 5, 1992. +Michigan concluded January by beating Northwestern a second time on January 30, becoming the first team in the country to reach 20 wins. After enduring an injured ankle against Illinois, starting center Jordan Morgan missed his first game since redshirting a full season for the 2009–10 Wolverines when he sat out the Northwestern game. Horford started in his place. +On February 2, 2013, Michigan (#1 AP/#2 Coaches) appeared on ESPN's College GameDay against Indiana (#3 AP/#3 Coaches) who hosted the game at Assembly Hall. Michigan lost 81–73, but the television broadcast of the game on ESPN set a Big Ten record for viewership with 4.035 million viewers. Morgan only played two minutes behind replacement starter Horford. Michigan then defeated Ohio State (#10 AP/#10 Coaches) in overtime in the rematch at home on February 5. Starting center Morgan only played 4 minutes. Coach Beilein noted that Morgan's absence affected the team's ability to match up defensively and substitute as it desired. +Michigan lost to Wisconsin on February 9 in overtime following a half-court buzzer beater by Wisconsin that tied the game in regulation It marked Michigan's eleventh consecutive loss against Wisconsin on the road. Morgan did not play, but McGary played 32 minutes, totaling 12 points, 3 steals and 8 rebounds. However, dramatically increased play in Morgan's absence enabled McGary to earn his second Big Ten Freshman of the Week honor on February 11. +On February 12, Michigan lost 75–52 in its rivalry game against (#8 AP/#8 Coaches) Michigan State. Among those in attendance were Rick Snyder, Matthew Stafford, Mark Dantonio, Brady Hoke, and Lloyd Carr. It marked the first time both teams were ranked in the top ten of the AP Poll, and resulted in Michigan losing back-to-back games and three consecutive road games for the first time since the 2010–11 team did so. Burke's 18 points, 4 assists and 3 steals were one of Michigan's few bright spots in their February 12 contest. +On February 17 against Penn State, Burke posted a season-high 29 points along with 5 assists, 3 rebounds and two steals. As a result of his efforts, Burke won his third player of the week award of the season and fourth of his career on February 18. Morgan returned to the starting lineup in the Penn State contest, but he only played 7 minutes. +After playing only 22 total minutes since January 27, Morgan played more minutes (17) than the other low post players (McGary, Horford, and Max Bielfeldt) for the first time on February 24 against Illinois. Michigan defeated Illinois 71–58 behind 26 points and 8 assists from Burke. In the game, Burke became the seventh Wolverine sophomore to reach 1000 career points. On February 27, Michigan surrendered a 15-point second-half lead to give Penn State its only conference win of the season. Following the game, the team called a rare players-only meeting at the Pizza House, with fifth-year reserve Corey Person doing most of the talking. +On March 3, Michigan defeated Michigan State (#9 AP/#10 Coaches) in the Ann Arbor rematch of their rivalry series as Trey Burke made 2 steals in the final 30 seconds to secure the 58–57 victory. Michigan snapped a 453-game streak with at least one made three-point shot and became the first team since February 2009 to defeat a top ten opponent without making one. On March 6, with its Big Ten Conference Championship destiny in its own hands, Michigan defeated Purdue to set up a championship showdown with Indiana on March 10. Michigan concluded its road schedule with a 5–5 record. +Michigan closed out the season by losing to Indiana (#2 AP/#2 Coaches) on March 10, failing its pursuit of defense of its conference co-championship. Michigan led by 5 with 52 seconds to go, but then missed 3 free throws, including the first attempt during 2 one-and-one situations. Burke and Morgan also missed last-second shots at the rim. Michigan wound down its regular season with its fifteenth sellout and five seniors on the roster: Eso Akunne, Josh Bartelstein, Blake McLimans, Corey Person, and Matt Vogrich. The loss kept Michigan from its first undefeated home season since the 1976–77 team. +Michigan participated in the 2013 Big Ten Conference men's basketball tournament at the United Center in Chicago. As the #5 seed, they defeated Penn State in the opening round on March 14 by an 83–66 margin. In the second round, the team lost to Wisconsin, 68–59. Prior to the 2013 NCAA Division I men's basketball tournament, Jeff Goodman of CBSSports.com named Michigan first among tournament teams in terms of having the most future NBA talent on its roster (in the absence of Kentucky, which was left to accept an invitation to the 2013 National Invitation Tournament). Among Michigan's starting lineup, Burke, Robinson, Hardaway, and McGary were all expected to be likely NBA draft choices, while Stauskas had the potential to be one. +Despite its highly touted lineup, Michigan entered the tournament as the sixth youngest team in the country and the youngest team in the field of 68 teams according to Sports Illustrated, based on weighted minutes played. The game marked Michigan's first NCAA Championship Monday appearance since 1989. Some sources claim 1993 as the school's last appearance, but those results have been vacated by the NCAA due to the University of Michigan basketball scandal. +Michigan was entered in the tournament's South region where it would play its first two games at The Palace of Auburn Hills in nearby Auburn Hills, Michigan. As a number four seed, Michigan opened the tournament by defeating South Dakota State 71–56, giving the team its most wins in 20 years and matching Beilein's career high as it raised its record to 27–7. Michigan then ousted 5th-seeded VCU by a 78–53 margin. That gave Michigan its first Sweet Sixteen appearance since the 1993–94 team went to the 1994 NCAA Division I men's basketball tournament. +The regional championship rounds for Michigan were at AT&T Stadium in Arlington, Texas. On March 29 against Kansas, Michigan overcame a 14-point deficit with 6:50 remaining and a 10-point deficit with 2:52 remaining to force overtime before an eventual victory. Burke scored eight points in the final 1:15 of regulation time, including a game-tying long three-pointer with 4.2 seconds remaining. McGary (25 points and 14 rebounds) and Burke (23 points and 10 assists) both posted double-doubles. +In the regional finals on March 31 against Florida, Michigan built a 13–0 lead and never led by less than 10 the rest of the game. Several players had career-highs in the game: Stauskas with 6 three-point shots, Burke 8 rebounds, McGary 5 steals and off the bench Albrecht 7 points and 3 steals. Eight of McGary's 11 points came as Michigan built the initial 13–0 lead and 5 of Stauskas' 6 three-pointers came as Michigan built a 41–17 lead and closed the half with a 47–30 lead. Burke was named South Regional Tournament Most Outstanding Player. McGary and Stauskas joined Burke on the five-man South All-Regional team. Following the regional championship postgame prayer and with Mrs. Beilein's consent, McGary and Hardaway gave Beilein a Gatorade shower. +The national championship rounds were held at the Georgia Dome in Atlanta, Georgia. In the April 6 national semifinal against Syracuse, Michigan emerged with its thirty-first victory, the most since the 1992–93 team went 31–5. McGary posted 10 points, 12 rebounds and 6 assists, while the bench contributed 22 points, including 6 from Albrecht, who raised his NCAA tournament total to 5-for-5 on three-point shots. +Michigan lost the April 8 national championship game against Midwest number one seed Louisville by an 82–76 score. Albrecht scored 17 first-half points on 4-for-4 three-point shooting. Burke scored 24 points in the championship game and made the seven-man All-Tournament team (which was revised multiple times) along with teammates McGary and Albrecht. The turning point of the game was described as a missed call by the referees when, as Michigan trailed 67–64 with 5 minutes left, Burke pinned Peyton Siva's dunk attempt with a clean, all-ball block, but was called for a foul, resulting in two made free throws by Siva. Michigan never got closer than 4 points the rest of the game. Louisville's championship has since been vacated by the NCAA, as well as all its wins from 2011 to 2015. This followed an escort recruitment scandal at the university. +Statistics come from the team's website: +Burke's 260 assists set a school single-season record. He also finished his sophomore season with 1,231 career points, surpassing the former school record for sophomore season career point total of 1,218 by Chris Webber. Stauskas reached a total of 80 made three-point shots, which surpassed Hardaway's 2-year-old school freshman season record of 76. +Stauskas also earned recognition from Sports Illustrated's Seth Davis as National Freshman of the Week on December 3. On January 9, Burke earned the Oscar Robertson National Player of the Week from the United States Basketball Writers Association (USBWA). Burke also earned ESPN.com Player of the Week recognition on April 1. +The Wooden Award midseason top 25 list, which included Burke, was announced on January 10. On January 31, Burke was named to the Oscar Robertson Trophy (USBWA National Player of the Year) midseason top 12 list, while Stuaskas and Robinson were named to the Wayman Tisdale Award (USBWA National Freshman of the Year) top 12 midseason list. Burke was one of six Big Ten players named among the top 30 finalists for the Naismith Award when the list was announced on February 26. Burke was among four Big Ten Players on the March 4, 14-man Robertson watchlist. On March 9, Burke was named among the top 15 Wooden Award finalists. On March 11, Burke was named one of five finalists for the Cousy Award. Burke was named one of four finalists for the Naismith Award on March 24. +Burke was named a first-team All-American by Sporting News on March 11, the USBWA on March 18, Sports Illustrated on March 19, CBSSports.com on March 20, the National Association of Basketball Coaches (NABC) on March 28, and the Associated Press on April 1. Burke was one of four Big Ten players named to the 10-man Wooden All-American team of finalists for the Wooden Award on April 1. On April 1, Robinson was one of two Big Ten players (Harris) named to the 21-man 2013 Kyle Macy Freshman All-America team. Burke became the fifth Consensus All-American (Cazzie Russell, Rickey Green, Gary Grant and Webber) in school history. +SI also named Burke National Player of the Year, making him the second (Russell, 1966) National Player of the Year in school history. On April 4, Burke won the Bob Cousy Award and was named Associated Press College Basketball Player of the Year. +Burke won both the Oscar Robertson Trophy from the USBWA as well as the John R. Wooden Award on April 5. On April 7, he won the NABC Player of the Year and Naismith College Player of the Year awards, giving him a sweep of the four major player of the year awards. +Burke earned Big Ten Conference Men's Basketball Player of the Year in 2013 and was a unanimous 2012–13 All-Big Ten 1st team selection. Hardaway was a 1st team selection by the coaches and second team by the media. Robinson was an honorable mention All-Conference selection and All-freshman honoree by the coaches. Morgan was a coaches All-defensive team selection. On March 12, the USBWA named Burke and Hardaway to its 2012–13 Men's All-District V (OH, IN, IL, MI, MN, WI) Team, based upon voting from its national membership. Burke was recognized as District V Player of the Year. Burke and Hardaway were selected to the NABC Division I All-District 7 first team on March 26, as selected and voted on by member coaches of the NABC, making them eligible for the State Farm Coaches' Division I All-America team. On that same day, Burke was selected to the 21-man 2013 Lute Olson All-America Team. +John Beilein was selected as an assistant coach for the 2013 World University Games. Seniors Josh Bartelstein and Matt Vogrich were among the 38 Big Ten men's basketball players recognized as Winter Academic All-Big Ten for maintaining 3.0 averages. Trey Burke was named team MVP. +Prior to the final four, McGary stated that he would not enter the 2013 NBA draft, but a few days later said he had been caught off guard and would prefer to respond after he had time to reflect on his season. +At 12:30 PM ET on April 4, Forbes sports business reporter Darren Heitner tweeted that Burke and teammate Hardaway would declare for the 2013 NBA Draft. According to Heiter, one of Burke's potential sports agents was Alonzo Shavers, who had known Burke since his birth. Burke's mother responded at 2:15 PM that same day that "He has not made any decision" about going pro, according to Yahoo! Sports reporter Eric Adelson. At 2:23 PM, Heitner tweeted that Hardaway Sr. said his son was undecided. +On April 9 before boarding the airplane to return from the NCAA Final Four, Beilein met with Burke, Hardaway, Robinson and McGary to direct them to seek the advice of the NBA advisory committee. The draft board had until April 15 to develop each individual report and the players had until April 28 to enter the draft. On that same date, ESPN's Jason King predicted that if all four players had left for the NBA draft, the 2013–14 team would have begun the season unranked. USA Today projected on April 9 that if one of the four possible 2013 NBA draft entrants returned, Michigan could have been ranked number twenty-four, and that if they all returned, Michigan would have been preseason number one. ESPN journalist Myron Medcalf predicted on April 12 that Burke and Hardaway would enter the draft and that McGary and Robinson were on the borderline of doing so. +On April 13, reports surfaced that Burke would announce that he would enter the NBA draft at a press conference the following day. He entered the 2013 NBA Draft on April 14. On April 17, Hardaway declared for the NBA draft. McGary and Robinson announced on April 18 that they had decided not to enter their names in the NBA draft. +On June 27, 2013, Burke was selected ninth in the 2013 NBA Draft by the Milwaukee Bucks and then traded to the Utah Jazz for the fourteenth and twenty-first picks, which were used to select Shabazz Muhammad and Gorgui Dieng. Hardaway was drafted twenty-fourth by the New York Knicks. Burke and Hardaway became the first Michigan duo selected in the first round since Juwan Howard and Jalen Rose in the 1994 NBA draft. Burke became the first top-10 Wolverine selection since Jamal Crawford in the 2000 NBA draft, and joined Victor Oladipo and Cody Zeller, who were selected second and fourth, respectively, to give the Big Ten its first top ten trio since the 1990 NBA draft. Hardaway joined his father, who was picked fourteenth in 1989 NBA draft, as a first round selection. +Every player that started in the 2013 national championship game was drafted either in the 2013 or 2014 NBA draft. Four players from Michigan's 2012 entering class were eventually drafted in the NBA draft (Stauskas, McGary, Robinson and LeVert). +Sources: +Following the season the team lost five seniors from the roster: guard Eso Akunne, guard Josh Bartelstein (captain), forward Blake McLimans, guard Corey Person and guard Matt Vogrich. The team's required 10-year dissociation with players implicated in the University of Michigan basketball scandal ended on May 8, 2013. Bartelstein blogged for the team from October 15, 2010, until April 11, 2013. His blogs from this season were turned into an ebook about the season entitled We On: Behind the Scenes of Michigan's Final Four Run. +Stats @ ESPN +Acamptonectes is a genus of ichthyosaurs, a type of dolphin-like marine reptiles, that lived during the Early Cretaceous around 130 million years ago. The first specimen, a partial adult skeleton, was discovered in Speeton, England, in 1958, but was not formally described until 2012 by Valentin Fischer and colleagues. They also recognised a partial subadult skeleton belonging to the genus from Cremlingen, Germany, and specimens from other localities in England. The genus contains the single species Acamptonectes densus; the generic name means "rigid swimmer" and the specific name means "compact" or "tightly packed". +A small ichthyosaur, Acamptonectes is estimated to have been 3 metres (9.8 ft) long. The generic name refers to unusual adaptations in the body of Acamptonectes that made its trunk rigid, including tightly fitting bones in the occiput (back and lower part of the skull) and interlocking vertebral centra ("bodies" of the vertebrae), which were likely adaptations that enabled it to swim at high speeds with a tuna-like form of locomotion. Other distinguishing characteristics include an extremely slender snout and unique ridges on the basioccipital bone of the braincase. As an ichthyosaur, Acamptonectes had large eye sockets and a tail fluke. Acamptonectes was similar in morphology to the related but earlier fellow ophthalmosaurines Ophthalmosaurus and Mollesaurus. +The discovery of Acamptonectes had significant implications for the evolutionary history of ichthyosaurs. The generalised platypterygiine ophthalmosaurids were long believed to be the only lineage of ichthyosaurs that survived into the Early Cretaceous following a mass extinction of ichthyosaurs across the Jurassic–Cretaceous boundary. As one of the first-known ophthalmosaurine ophthalmosaurids from the Early Cretaceous, the discovery of Acamptonectes provided evidence against such a mass extinction. Although the larger eyes of Acamptonectes would have made it better adapted than platypterygiines to depth diving, it was probably a generalist predator. Its teeth, which were slender and textured with longitudinal ridges, were adapted for impaling prey, which suggests it likely fed on soft, fleshy prey such as fish and squid. +Over a series of weekends in 1958, four students and a technician from Hull University's geology department collected a fossil specimen from the Speeton Clay Formation at Speeton in northern England. The fossil belonged to an ichthyosaur or "fish lizard", a Mesozoic group of marine reptiles; it consists of a partial adult skeleton that includes a fragmentary skull roof, a mandible, vertebrae, ribs, and the scapular girdle (the shoulder area). In 1991, it was transferred to the Hunterian Museum of the University of Glasgow (GLAHM) when the geology department of Hull University was closed. It was catalogued under the specimen number GLAHM 132855, and was also known as the "Speeton Clay ichthyosaur". Palaeontologist Robert M. Appleby described the specimen and assigned it to the genus Platypterygius as the species "P. speetoni" (which he considered primitive within that genus), in a monograph that remained unpublished at the time of his death in 2003. A second specimen of the species was found in 1985, also in the Speeton Clay, and is catalogued as NHMUK R11185 at the Natural History Museum, London (NHMUK). This specimen consists of a partial rostrum (snout) and mandible, fragmentary ribs, and a complete right humerus (upper-arm bone). +Palaeontologist Jeff Liston recognised the significance of the Speeton Clay ichthyosaur while working at the Hunterian Museum, and Appleby's widow Valerie asked him to help finish Appleby's unpublished monograph. Liston approached ichthyosaur specialist Valentin Fischer about writing a description of the animal. Fischer examined the specimen in 2011 and realised it represented the same ichthyosaur as a specimen from Cremlingen in northern Germany about which he had recently written a draft paper with several colleagues. This German specimen was discovered in 2005 when private fossil collector Hans-Dieter Macht found some vertebrae in a construction area. Macht notified the director of the State Natural History Museum of Braunschweig (SNHM), whereafter excavation began; the specimen was collected within three days because construction work had to continue. It was prepared and mounted in 2005 at the museum, where it is catalogued as SNHM1284-R. It consists of a partial skeleton of a subadult and includes a fragmentary skull roof, a complete mandible, a partial axial skeleton, and a partial scapular girdle. It was assigned to the genus Platypterygius in a 2008 article. +Historically, the genus Platypterygius has been treated as a catch-all wastebasket taxon for Cretaceous ichthyosaurs which contained multiple distantly-related species. Liston and Fischer recognised the specimens were distinct from other species referred to Platypterygius and belonged to a new species and genus. After determining the Speeton Clay specimen is much larger than the Cremlingen specimen and thus likely more mature, Liston and Fischer decided to make it the holotype of the new species because juvenile specimens often have characteristics absent in adults. The Cremlingen specimen and the other Speeton Clay specimen became paratypes (additional specimens in the series of type specimens). In 2012, a team of palaeontologists led by Fischer formally named the new genus and species Acamptonectes densus. The generic name is derived from the Greek words akamptos and nektes, which together mean "rigid swimmer"; the specific name means "compact" or "tightly packed". In full, the scientific name refers to the robust, tightly fitting bones of the occiput (back and lower part of the skull) and the tightly interlocking centra ("bodies") of the cervical (neck) and dorsal (back) vertebrae. The holotype, GLAHM 132855, was listed under the incorrect specimen number GLAHM 132588 in the original version of the description. +Multiple basioccipitals (a bone at the lower part of the occiput), stapes (one of the ear bones), and a basisphenoid (a bone within the lower part of the braincase) from the Cambridge Greensand Formation of Cambridge, England, were also assigned to Acamptonectes sp. (i.e., an uncertain species within the genus Acamptonectes) by Fischer and colleagues in 2012. This assumed that Acamptonectes was the only Cretaceous ichthyosaur of the subfamily Ophthalmosaurinae from Eurasia. Some of the specimens are housed at the Sedgwick Museum of Earth Sciences, University of Cambridge (CAMSM), and several others are housed at the Hunterian Museum and the Natural History Museum. Several of the bones are essentially identical to those of A. densus, while others differ in some details. The bones are generally small so their differences from A. densus were considered to be related to either the ages of the animals or evolutionary changes. In 2014, Fischer and colleagues identified a basioccipital and humerus belonging to Ophthalmosaurus (or a closely related ichthyosaur) from Berriasian-aged rocks (dating to between 145 and 139.8 million years old) near Nettleton, Lincolnshire. Therefore, since their prior assumption no longer held, Fischer and colleagues re-listed the Cambridge Greensand specimens as belonging to indeterminate ophthalmosaurines that are not identifiable below the subfamily level. +Additional Acamptonectes remains known may also include an ichthyosaur specimen from Hannover, Germany. In 1909, German palaeontologist Ferdinand Broili named it as a new species of the genus Ichthyosaurus, Ichthyosaurus brunsvicensis, but considered its assignment to the genus tentative. Broili's specimen consisted of an incomplete basicranium (skull base) and an incomplete interclavicle (a bone between the clavicles, or collar bones); however, the specimen was destroyed during World War II. Palaeontologist Christopher McGowan regarded it as a member of Platypterygius in 1972 and 2003 but Fischer and colleagues assigned it to cf. Acamptonectes (i.e., possibly Acamptonectes or a related animal). They found it similar in several features to Acamptonectes but also different in others; they suggested the specimen was a juvenile because of the size and shape of its basicranium. Due to its fragmentary and inaccessible nature, they considered I. brunsvicensis a nomen dubium (dubious name). +Acamptonectes was a small ichthyosaur, with an estimated body length of 3 metres (9.8 ft). Like other ichthyosaurs, it had a long, thin snout, large eye sockets, and a tail fluke that was supported by vertebrae in the lower half. Ichthyosaurs were superficially similar to dolphins and had flippers rather than legs, and most (except for early species) had dorsal fins. Although the colour of Acamptonectes is unknown, at least some ichthyosaurs may have been uniformly dark-coloured in life, which is evidenced by the discovery of high concentrations of eumelanin pigments in the preserved skin of an early ichthyosaur fossil. +Acamptonectes was similar in morphology to the related but earlier ophthalmosaurines Ophthalmosaurus and Mollesaurus. Features of the humerus in specimen SNHM1284-R are indicative of its immaturity; however, the humerus lacks the sandpaper-like texture of the humeral shafts (shafts of the upper arm bones) of juvenile ichthyosaurs and is thus thought to represent a subadult. The holotype and specimen NHMUK R11185 are large compared to other members of the wider family Ophthalmosauridae; the holotype is thought to have been an adult because of the extensive fusion of its bones, including within the occiput, and the smooth texture of the humerus. +The snout of Acamptonectes was elongated and extremely slender; in the holotype, it is only 45 mm (1.8 in) wide in front of the bony nostrils. The snout was also only 0.044 times as deep as it was long, one of the lowest ratios among ophthalmosaurids. Much of the snout was formed by the premaxillae, which formed the front portion of the upper jaw. The fossa praemaxillaris, a groove that ran parallel to the tooth row of the upper jaw, was deep and continuous, and ended in a series of aligned foramina (depressions). Behind and above the premaxillae were the nasals, which the holotype preserves in three dimensions, documenting the shape of the upper side of the snout. The back part of the nasal had a downward-extending bulge that was similar to that of related genera such as Ophthalmosaurus. This bulge gave rise to a short, robust, wing-like extension that formed an overhang over the rear of the bony nostril; this feature was also present in Ophthalmosaurus and Platypterygius australis. The edge of this overhang was roughened, indicating this was probably the attachment site for a soft tissue structure. +The back part of the skull roof is incompletely known from the hind part of the lacrimal bone (in front of the eye opening), the postfrontal (above and behind the eye opening), the parietal (at the rear of the skull roof), and parts of a supratemporal that formed the rear corners of the skull roof. A forward-directed extension of the supratemporal formed the internal rear edge of the supratemporal fenestra, an opening in the skull roof situated behind the eyes. The parietal, which would have formed the inner margin of the supratemporal fenestra, had a convex front margin that would have interdigitated (interlocked) with either the frontal or postfrontal bones, which are not preserved in the known specimens. +When viewed from the side, the quadrate bone, which connected to the lower jaw to form the jaw joint, was C-shaped. Two probable hyoid bones (tongue bones) are preserved in specimen SNHM1284-R; these bones were rod-like with one spatula-shaped end. The stapes had a shaft that was more slender than in any other ichthyosaur, and its head was large and square; these features are regarded as an autapomorphy—a characteristic that distinguishes the genus from related genera. The basisphenoid had a well-developed crest on its upper surface; this is considered another autapomorphy because this surface was a wide, flat plateau in other ichthyosaur species. At its front end, the basisphenoid was fused to the parasphenoid (another bone within the lower part of the braincase) and no suture (border between the two bones) can be seen. +The supraoccipital at the upper rear of the braincase (part of the skull which encloses the brain) was only weakly arched; it thus differed from those of Platypterygius and Baptanodon, which were U-shaped. Below the supraoccipital were the two exoccipitals, which formed the sides of the foramen magnum (the canal for the spinal cord). Located further below was the basioccipital, which formed the floor of the foramen magnum. The midline canal that formed this floor was bordered by ridges, giving a bilobed appearance when seen from above; this is also regarded as an autapomorphy of the genus. Below the foramen magnum, the basioccipital formed the occipital condyle, which connected with the first vertebra of the neck to form the head joint. The occipital condyle was well-demarcated from the remainder of the bone by a constricted band, unlike most other ophthalmosaurids. The condyle was rounded and had visible growth rings, as in related genera. The opisthotics, which are on either side of the basioccipital, possessed extensions called the paroccipital processes which pointed backwards and upwards. These processes were elongated and slender in Acamptonectes and Ophthalmosaurus, but short and stout in other ophthalmosaurids. +The dentary (the tooth-bearing bone at the front of the lower jaw) was elongated, straight, and had a blunt front tip; this contrasts with the down-turned and beak-like tips of some ichthyosaurs in the subfamily Platypterygiinae. The splenial bones expanded in depth at their rear, forming the lower margin of the mandible and much of its midline surface. A groove similar in morphology to the fossa premaxilliaris of the upper jaw, termed the fossa dentalis, ran parallel to the dentary. Two Acamptonectes specimens lack the "3"-shaped upper surface of the angular bones that are otherwise typical of ophthalmosaurids; instead, the surface in these specimens consists of a simple, flat groove bordered by two walls. Because the "3" shape is present in the holotype specimen, however, this feature may have varied between individuals or growth stages. The articular bone in one specimen was stouter than those in other ophthalmosaurids, which were nearly as thick as it was long. +The teeth of Acamptonectes had striated bases and quadrangular roots, as in numerous ophthalmosaurids, but they were not square like those of Platypterygius. Some of the roots of SNHM1284-R had resorption pits, indicating its teeth were still growing. The only-known complete tooth crown was small compared to those of other ophthalmosaurids; it was also slender and sharply pointed, and similar to the teeth from the rear of the jaw in Baptanodon. The bottom two-thirds of the crown had subtle, longitudinal ridges and was covered in a coarse texture that was nevertheless finer than those in Aegirosaurus and some Platypterygius specimens. The base of the crown was slightly bulbous and almost smooth, unlike those in other ophthalmosaurids. +As was typical for ichthyosaurs, the vertebral centra of Acamptonectes were disc-shaped and deeply concave on both ends. The processes (bony projections that serve as muscle and rib attachments) projecting from the centra were greatly reduced as an adaptation for its fully aquatic lifestyle. In Acamptonectes, the front-most cervical (neck) centra were high and short, and the following cervical and dorsal (trunk) centra become progressively longer. In the rear dorsal vertebral column, the centra became shorter and higher; this trend peaked at the first caudal (tail) vertebra, which was 3.12 times as high as it was long. The remaining caudals became longer and lower again; the caudals, which comprised the fin, were as long as they were high, a feature that was previously identified only in P. platydactylus. +The first two cervicals—the atlas and axis—were fused into a single complex that was wide when viewed from the rear. The front dorsal vertebrae have diapophyses (sideways-protruding processes to which ribs attach) fused to the centra; this feature was shared with several other ophthalmosaurids. The centra of the dorsal vertebrae were autapomorphic, being tightly interlocking and having extensive posterolateral lamellae (ridges lining the rear surfaces of the centra). This interlocking stiffened the front section of the vertebral column in conjunction with the strong occiput of the skull. Such stiffening can be observed in other ichthyosaurs belonging to the wider group Thunnosauria, though not to the degree as seen in Acamptonectes. +The neural arches of the vertebrae had narrow pre- and postzygapophyses (articular processes projecting forward and backward from the centra) that were unpaired (fused into a single element) in all vertebrae. In contrast, in P. hercynicus and Sveltonectes, these processes were paired in the front of the vertebral column. The neural spines (large upward-projecting processes) were of variable height within each specimen; they were markedly longer in some dorsals than others, reaching 1.25 times the height of the largest centrum. These long spines may be bony extensions that are analogous to the extraneural processes, a row of bones located above the tops of the neural spines that are preserved in two juvenile Stenopterygius specimens. The top surfaces of the neural spines were often pitted, indicating they had a cartilage covering. The ribs were distinct in being robust with a round cross-section; this contrasted with the "8"-shaped cross-section that is seen in other thunnosaurian ichthyosaurs. +The coracoid (a paired bone in the scapular girdle) was roughly hexagonal, contrasting with the rounded shape in Platypterygius, and had outer and midline edges that were straight and parallel. The upper and lower surfaces were slightly paddle-shaped, and the mid-line surface was eye-shaped as in Ophthalmosaurus, although it was not as thick as those in Sveltonectes and P. australis. The mid-line surface was unfinished and had deep pits, indicating the presence of a thick layer of cartilage. At the front, the mid-line margin was strongly deflected outward, forming the rugose (roughened and wrinkled) edge of a wide, sheet-like process similar to that in Ophthalmosaurus. The process was separated from the scapular facet (articulation with the scapula) by a deep, wide notch, as in many Ophthalmosaurus specimens. The scapular facet of the coracoid was small, deeply pitted, and triangular while the glenoid facet (articulation with the humerus) was large and eye-shaped. These facets were not markedly separated, unlike those in Sveltonectes, where they were set at an angle of 100°. The hind margin of the coracoid was sheet-like and lacked a notch. +As with its coracoid, the scapula (shoulder blade) of Acamptonectes was similar to that of Ophthalmosaurus. It was strongly compressed from side to side, unlike that in P. hercynicus, in which the shaft was thick and rod-like. The lower part of the scapula was expanded from front to back, forming a wide, rugose, articular, tear-drop-shaped surface that articulated with the coracoid and glenoid facets. It had a large, flat, fan-like acromial process at the front (which connected with the clavicle), like those in Ophthalmosaurus and P. americanus. The coracoid facet of the scapula was triangular and continuous with the larger glenoid facet, as in Ophthalmosaurus but unlike P. australis. The side and mid-line surfaces of the acromial process were slightly concave. +The deltopectoral crest (to where the deltoid muscle attached) on the upper-front part of the humerus was more prominent in Acamptonectes than in Ophthalmosaurus and Arthropterygius, but less so than in Sveltonectes and Platypterygius. On the opposite side of the upper humerus, the trochanter dorsalis (a tubercle or protrusion where muscles attached) was tall and narrow, as in Sveltonectes and many species of Platypterygius. The humerus had three facets on its lower side, including a facet for a bone at the front and a backward-deflected facet for the ulna, which was also similar to that of Ophthalmosaurus, and had five articular processes. The expanded upper surface that articulated with the humerus was slightly concave and pitted, unlike that of Arthropterygius, in which the ulna's humerus facet formed a pyramidal-shaped process. +The facet for the radius was straight and trapezoidal, and merged with facets for two wrist bones, the intermedium and the ulnare. The facet for the pisiform, another wrist bone, was small and triangular, and was located at the back of the ulna's lower side. The ulna had a concave and edge-like hind margin. The phalanx bones (finger bones within the flipper) were oval as in Ophthalmosaurus, Arthropterygius, and some species of Brachypterygius; they tapered away from the body, and the edges of the peripheral phalanx bones were irregular and slightly concave. +In 2012, a phylogenetic analysis conducted by Fischer and colleagues found Acamptonectes to be a member of the family Ophthalmosauridae based on several characteristics. These include: the reduced extracondylar area (a band of bone surrounding the occipital condyle), the plate-like dorsal trochanter of the humerus, the presence of a facet at the front of the humerus' bottom end for a paddle bone, and the lack of notching in the paddle bones that was considered to be homoplastic (independently acquired). It was also found to be more closely related to other ophthalmosaurids than Arthropterygius based on the large processes of the basipterygoids (bones at the base of the braincase), the lack of a peg on the basioccipital, and the large trochanters of the femur. +Relationships within Ophthalmosauridae have historically been unstable in analyses due to the fragmentary nature of many ophthalmosaurid specimens; furthermore, many ophthalmosaurid genera are known from a single specimen. Removal of these fragmentary genera, however, has degraded the resolution of analyses even further. The phylogenetic analysis conducted by Fischer and colleagues in 2012 recovered two novel clades (groups) within Ophthalmosauridae; the Ophthalmosaurinae and Platypterygiinae, the existence of which had long been suspected by ichthyosaur researchers—Maxim Arkhangelsky had named the clades as subfamilies as early as 2001—but had not yet been supported robustly by the results of phylogenetic analyses. Fischer and colleagues placed Acamptonectes was placed in the former clade, although its placement there represented a secondary reversal of the group's only uniting characteristic; a notch on the bottom of the basioccipital. +Within the Ophthalmosaurinae, various positions have been recovered for Acamptonectes due to the same issues. In 2012, Fischer and colleagues found that it grouped closest with "Ophthalmosaurus" natans, with Ophthalmosaurus icenicus and Mollesaurus being successively less-closely related. The relationship with "O." natans was formed on account of the reduced presence of striations on the teeth, although Fischer and colleagues indicated this characteristic was homoplastic so they did not consider it sufficient to resurrect the previously used genus name Baptanodon for "O." natans. In 2013, they recovered the same arrangement in a derivative analysis for the description of Malawania, as did Nikolay Zverkov and colleagues in a 2015 analysis focusing on Grendelius—albeit with a clade consisting of Cryopterygius, Undorosaurus, and Paraophthalmosaurus being closer to Acamptonectes than Mollesaurus. +Arkhangelsky and Zverkov previously recovered all of these species with the exception of Mollesaurus in a polytomy (unresolved clade) in 2014. A 2019 analysis by Zverkov and Vladimir Efimov found an otherwise identical arrangement, in which the positions of Mollesaurus and Acamptonectes were exchanged, which was also found in another 2019 analysis by Zverkov and Natalya Prilepskaya, and in the 2020 description of a new specimen of Muiscasaurus by María Páramo-Fonseca and colleagues, in which Muiscasaurus was the next-closest relative of these species. In their description of Acuetzpalin, a 2020 analysis by Jair Barrientos-Lara and Jesús Alvarado-Ortega found "O." natans and O. icenicus to form a clade with the exclusion of Mollesaurus and then Acamptonectes, which was also recovered by Megan Jacobs and David Martill in their 2020 description of Thalassodraco. +A 2014 analysis of the description of Janusaurus conducted by Aubrey Roberts and colleagues found Acamptonectes to be the sister group to a clade consisting of O. icenicus and Leninia, which collectively constituted one branch of the Ophthalmosaurinae. The same arrangement was recovered by a 2017 analysis of the description of Keilhauia conducted by Lene Delsett and colleagues. In 2019, another analysis by the same authors found Acamptonectes closer to Janusaurus, Keilhauia, and Palvennia than to Paraophthalmosaurus, "O." natans (as Baptanodon), O. icenicus, or Gengasaurus in successive order of closeness to the base of the Ophthalmosaurinae. In each case, however, the Bremer support—a measure of the likelihood of a phylogenetic tree's arrangement over alternatives—of the groupings was low. +Other analyses also found Acamptonectes within unresolved polytomies. For the 2016 description of Muiscasaurus, Erin Maxwell and colleagues found O. icenicus, "O." natans, Undorosaurus, and Acamptonectes in a polytomy at the base of the Ophthalmosauridae. Contrary to most analyses, they did not recover a distinct Ophthalmosaurinae. Also in 2016, Fischer and colleagues found Ophthalmosaurinae to consist of Mollesaurus as the sister group to a polytomy including O. icenicus, "O." natans, Leninia, Acamptonectes, and a group containing Cryopterygius, Janusaurus, and Palvennia. In 2019, Maxwell, Dirley Cortés, Pedro Patarroyo, and Parra Ruge recovered a poorly-resolved Ophthalmosauridae containing Acamptonectes in a large polytomy. In their 2020 description of Arthropterygius thalassonotus, Lisandro Campos and colleagues placed Acamptonectes in a polytomy with O. icenicus, Leninia, and Athabascasaurus, which formed the sister group to a clade of Keilhauia and Undorosaurus; the base of the Ophthalmosaurinae was formed by a polytomy of those species, and Baptanodon and Gengasaurus. +The phylogenetic tree from the analysis of Páramo-Fonseca and colleagues in 2020 is reproduced below. +Ichthyosaurs were traditionally thought to have been affected by three extinction events; one at the Triassic–Jurassic boundary, one at the Jurassic–Cretaceous boundary, and a final extinction in the Cretaceous at the boundary of the Cenomanian and Turonian ages that left no survivors. Some researchers suggested their species diversity declined after the mid-Jurassic, with the ichthyosaurs continuing until they disappeared at the end of the Cenomanian. This decline was thought to have been associated with a transition in the dominant ichthyosaur lineage; the large-eyed, thunniform (tuna-like) ophthalmosaurines, which were successful and widespread notwithstanding their hyper-specialisation, would have been replaced by the more generalised platypterygiines, which had smaller eyes and longer bodies. +Acamptonectes is a significant find because it is an ophthalmosaurine from the Early Cretaceous, demonstrating the ophthalmosaurines were not entirely wiped out at the Jurassic–Cretaceous boundary. Fischer and colleagues also found evidence of other ophthalmosaurines in the Early Cretaceous by reanalyzing known material, including the Nettleton Ophthalmosaurus specimens. They also cited reports of the Late Jurassic-aged platypterygiines Brachypterygius, Aegirosaurus, Caypullisaurus, and Yasykovia—which has been synonymised with Nannopterygius—from the Early Cretaceous. +By tabulating the number of genera that disappeared in each age, Fischer and colleagues found no clear boundary between individual ages from the Late Jurassic (Oxfordian) to Early Cretaceous (Aptian) that could be considered an extinction event for ophthalmosaurids. The Jurassic–Cretaceous boundary had a net extinction rate of 0 and even the highest survival rates. By counting the number of new clades that emerged, however, they computed the cladogenesis (clade formation) rate to have been lower in the Cretaceous. They concluded—contrary to traditional thinking—the Jurassic-Cretaceous extinction event had a negligible impact on ichthyosaurs compared to its impact on other marine reptiles, and that ophthalmosaurids remained diverse until their final extinction. +With their dolphin-like bodies, ichthyosaurs were better adapted to their aquatic environment than any other group of marine reptiles. They were viviparous that gave birth to live young and were likely incapable of leaving the water. As homeotherms ("warm-blooded") with high metabolic rates, ichthyosaurs would have been active swimmers. Jurassic and Cretaceous ichthyosaurs, including Acamptonectes, had evolved a thunniform method of swimming rather than the anguilliform (undulating or eel-like) methods of earlier species. Thunniform ichthyosaurs were able to swim faster and more efficiently than other marine reptiles of similar sizes, and were better adapted to a pelagic (open-ocean) lifestyle. Their swimming was aided by their compact bodies and crescent-shaped tail fins. +Most of the skeleton of Acamptonectes appears to have been unusually rigid, which would have severely limited the extent of side-to-side motion in the front part of the skeleton. Its snout was also shallower than those in related species, and its ribs were more rounded in cross-section. According to palaeontologist Darren Naish, one of the describers of the genus, these may have been further adaptations to increase the stiffness of the animal's body by making these body parts more resistant to bending. The tightly packed occipital bones and cervical vertebrae would have allowed limited movement in the neck, suggesting Acamptonectes must have "shot through the water like a dart", according to fellow describer Ulrich Joger. +As an ophthalmosaurine, Acamptonectes would likely have been an opportunistic generalist predator that fed on fish and squid. Adaptations for speed suggest that Acamptonectes and other ophthalmosaurines were likely also pursuit predators. In 1987, paleontologist Judy A. Massare proposed "feeding guilds" as a way to group marine reptiles; some ichthyosaur species were thought to have "pierced" small prey using needle-like teeth, and others to have "crunched" hard-shelled prey using robust teeth. In 2012, palaeontologist Maria Zammit suggested that the slender tooth crowns with longitudinal ridges seen in Acamptonectes (which placed it in the "pierce II/generalist" guild) were likely used to impale rather than grasp prey, and its diet may thus have consisted of fleshy prey that did not have a hard exterior. Nevertheless, she noted that its shallow snout and unique tooth morphology may indicate a different diet and lifestyle from other known Cretaceous ichthyosaurs. +Ichthyosaurs had the largest eyes of any known vertebrate group, which can be inferred from bones in the eye sockets known as scleral rings, and would therefore have possessed sensitive low-light vision that would have aided prey capture at great depths. In the related genus Ophthalmosaurus, the maximum diameter of the eyeball would have been 23 centimetres (9.1 in), allowing movement to be detected at depths of 300 metres (980 ft) in the mesopelagic zone. Ophthalmosaurus could likely dive for around 20 minutes and reach depths of at least 600 metres (2,000 ft). In addition to good eyesight, the enlarged olfactory region of the brain indicates ichthyosaurs had a sensitive sense of smell. +Acamptonectes is known from rocks dating to the Hauterivian stage of the Lower Cretaceous (approximately 133 to 129 million years old) in the Speeton Clay Formation of England, which is composed of claystone and mudrock, and is generally about 100–130 metres (330–430 ft) thick. The Acamptonectes holotype came from the D2D beds and the specimen NHMUK R11185 came from the slightly older D2C beds. Material preserved in these sediments is sometimes reworked from the underlying older Valanginian rocks rather than originating from the Hauterivian. The holotype of Acamptonectes is partially articulated, as were some nearby crinoid fossils, indicating the specimen was not reworked and genuinely came from the Hauterivian. +Carbon-13 concentration (δ13C) levels in the Speeton Clay Formation increased during the Valanginian and the early Hauterivian. This may have occurred when land submerged by rising sea levels released carbon-13 into the oceans. Concentrations of oxygen 18 (δ18O), however, increased during this time, indicating an episode of cooling; specifically, δ18O levels in belemnite fossils indicate the temperature of the Speeton Clay was about 11 °C (52 °F) at the beginning of the Hauterivian, rose to 15 °C (59 °F) during the middle part of this stage, and reverted to 11 °C (52 °F) by its end. Evidence of photosynthetic organisms indicate the Speeton Clay environment was at least partially located in the photic zone (the layer in the ocean that light reaches). +Numerous other organisms have been recovered from the Speeton Clay Formation; many of these were borers, including foraminiferans, fungi, chlorophyte algae, and various animals such as sponges, polychaetes, brachiopods, barnacles, bivalves, and echinoids. In addition to the crinoids, other invertebrates in the Speeton Clay Formation are represented by a wide variety of ammonites and belemnites. While both bony fish and cartilaginous fish—the latter group represented by sharks and rays of various types—are known from the Speeton Clay, they are poorly preserved and not very abundant. Marine reptiles are uncommon in this formation; other than Acamptonectes, they are represented by some fragmentary plesiosaur remains. +SNHM1284-R, the German specimen of Acamptonectes, comes from late Hauterivian rocks of the Lower Saxony Basin near Cremlingen in eastern Lower Saxony. The Lower Cretaceous sediments of this basin are rich in siliclastic rocks that were deposited in the southern region of the proto-North Sea, an epicontinental sea covering much of Northwest Germany during the Lower Cretaceous. Since this region linked the warmer Tethys Sea and the colder Boreal Sea, its environment was very susceptible to change. The late Hauterivian rocks of the region were deposited in the neritic zone (shallow seas) during a time of alternating marine transgression and regression (rising and falling sea levels). The surface waters were generally cool, although they sometimes warmed when warmer water from the Tethys Sea entered the region. Sedimentation rates were high and the bottom waters were somewhat anoxic (oxygen deprived). Organisms that inhabited this sea include dinoflagellates, ammonites, and belemnites. +List of ichthyosaur genera +List of ichthyosaur type specimens +Timeline of ichthyosaur research +Babakotia is an extinct genus of medium-sized lemur, or strepsirrhine primate, from Madagascar that contains a single species, Babakotia radofilai. Together with Palaeopropithecus, Archaeoindris, and Mesopropithecus, it forms the family Palaeopropithecidae, commonly known as the sloth lemurs. The name Babakotia comes from the Malagasy name for the indri, babakoto, to which it and all other sloth lemurs are closely related. Due to its mix of morphological traits that show intermediate stages between the slow-moving smaller sloth lemurs and the suspensory large sloth lemurs, it has helped determine the relationship between both groups and the closely related and extinct monkey lemurs. +Babakotia radofilai and all other sloth lemurs share many traits with living sloths, demonstrating convergent evolution. It had long forearms, curved digits, and highly mobile hip and ankle joints. Its skull was more heavily built than that of indriids, but not as much as in the larger sloth lemurs. Its dentition is similar to that of all other indriids and sloth lemurs. It lived in the northern part of Madagascar and shared its range with at least two other sloth lemur species, Palaeopropithecus ingens and Mesopropithecus dolichobrachion. Babakotia radofilai was primarily a leaf-eater (folivore), though it also ate fruit and hard seeds. It is known only from subfossil remains and may have died out shortly after the arrival of humans on the island, but not enough radiocarbon dating has been done with this species to know for certain. +The name of the genus Babakotia derives from the Malagasy common name for the Indri, babakoto, a close relative of Babakotia. The species name, radofilai, was chosen in honor of French mathematician and expatriate Jean Radofilao, an avid spelunker who mapped the caves where remains of Babakotia radofilai were first found. +Babakotia radofilai is the sole member of the genus Babakotia and belongs to the family Palaeopropithecidae, which includes three other genera of sloth lemurs: Palaeopropithecus, Archaeoindris, and Mesopropithecus. This family in turn belongs to the infraorder Lemuriformes, which includes all the Malagasy lemurs. +The first subfossil remains of Babakotia radofilai were discovered as part of a series of expeditions following upon discoveries of Jean Radofilao and two Anglo-Malagasy reconnaissance expeditions in 1981 and 1986–7. The second wave of research was launched in the 1980s by biological anthropologist Elwyn L. Simons who unearthed in 1988 at a cave known as Antsiroandoha in the Ankarana Massif, northern Madagascar a nearly complete skeleton and skull in addition to the remains of roughly a dozen other individuals. Identified immediately as a sloth lemur (palaeopropithecid) upon its discovery, Babakotia along with Mesopropithecus helped to settle a debate about the relationship between the sloth lemurs, the monkey lemurs (family Archaeolemuridae) and the living indriids. The monkey lemurs had skulls that more closely resembled the indriids, but their teeth were very specialized and unlike those of the indriids. The larger sloth lemurs, on the other hand, retained a dentition similar to living indriids, yet differed by having more robust and specialized skulls. Babakotia and Mesopropithecus not only shared the indriid dentition, but also the indriid-like skulls, providing evidence that sloth lemurs were most closely related to living indriids, with monkey lemurs as a sister group to both. Furthermore, the discovery of Babakotia helped to demonstrate that the ancestral indriids were not "ricochetal leapers" (bouncing rapidly from tree to tree) like living indriids, but vertical climbers and hanging feeders, and possibly occasional leapers. +Weighing between 16 and 20 kg (35 and 44 lb), Babakotia radofilai was a medium-sized lemur and noticeably smaller than the large sloth lemurs (Archaeoindris and Palaeopropithecus), but larger than the small sloth lemurs (Mesopropithecus). In many ways, it had an intermediate level of adaptations for suspensory behavior between the large sloth lemurs and the small sloth lemurs. This includes its highly mobile hip and ankle joints, as well as other specializations in the vertebral column, pelvis, and limbs. Its forelimbs were 20% longer than its hind limbs, giving it a higher intermembral index (~119) than Mesopropithecus (~97 to 113), suggesting that it was convergently similar to arboreal sloths. It had a reduced tarsus and curved, elongated digits, adapted for grasping and suggesting suspensory behavior. Its hind feet were reduced, making it well-adapted for climbing and hanging (like in other palaeopropithecids), but not leaping (like in indriids). Wrist bones found in 1999 further demonstrated that this species was a vertical climber. Additionally, analysis of its semicircular canals, lumbar vertebrae and its spinous processes indicate slow movement and climbing (antipronograde) adaptations, but not necessarily sloth-like hanging, vertical clinging, or leaping. Therefore, it was likely a slow climber like a loris and also exhibited some suspensory behavior like a sloth. +All sloth lemurs have relatively robust skulls compared to the indriids, yet despite shared cranial features with the larger sloth lemurs, its skull still resembles that of an indri. The cranial traits shared with the other sloth lemurs include relatively small orbits, robust zygomatic arches, and a mostly rectangular hard palate. The small orbits taken into consideration with the relative size of the optic canal suggest that Babakotia had low visual acuity, which is typical for lemurs. The skull length averages 144 mm (5.7 in). +The dental formula of Babakotia radofilai was the same as the other sloth lemurs and indriids: either 2.1.2.31.1.2.3 or 2.1.2.32.0.2.3 × 2 = 30. It is unclear whether one of the teeth in the permanent dentition is an incisor or canine, resulting in these two conflicting dental formulae. Regardless, the lack of either a lower canine or incisor results in a four-tooth toothcomb instead of the more typical six-tooth strepsirrhine toothcomb. Babakotia radofilai differed slightly from indriids in having somewhat elongated premolars. Its cheek teeth had broad shearing crests and crenulated enamel. +Like all other lemurs, Babakotia radofilai was endemic to Madagascar. Its remains have only been found in limestone caves at the Ankarana Massif within the Ankarana Reserve and at Anjohibe, indicating a range across the extreme north and northwest of the island. The restricted range of this arboreal primate, particularly during a time when much of the island was blanketed in forest, might have been due to habitat specificity, competitive exclusion, or some other unknown factor. It was sympatric (occurred together) with Palaeopropithecus maximus and Mesopropithecus dolichobrachion. +Based on its size, the morphology of its molars, and microwear analysis on its teeth, Babakotia radofilai was likely a folivore, while supplementing its diet with fruit and hard seeds. In all sloth lemurs, including Babakotia radofilai, the permanent teeth erupted early, a trait seen in indriids that improves survivability of juveniles during the first dry season following weaning. +Because it died out relatively recently and is only known from subfossil remains, it is considered to be a modern form of Malagasy lemur. Babakotia radofilai lived during the Holocene epoch and is thought to have disappeared shortly after the arrival of humans to the island, possibly within the last 1,000 years. The only radiocarbon date that has been reported for it dates back to 3100–2800 BCE. +Diffuse panbronchiolitis (DPB) is an inflammatory lung disease of unknown cause. It is a severe, progressive form of bronchiolitis, an inflammatory condition of the bronchioles (small air passages in the lungs). The term diffuse signifies that lesions appear throughout both lungs, while panbronchiolitis refers to inflammation found in all layers of the respiratory bronchioles (those involved in gas exchange). DPB causes severe inflammation and nodule-like lesions of terminal bronchioles, chronic sinusitis, and intense coughing with large amounts of sputum production. +The disease is believed to occur when there is susceptibility, or a lack of immune system resistance, to DPB-causing bacteria or viruses, caused by several genes that are found predominantly in individuals of East Asian descent. The highest incidence occurs among Japanese people, followed by Koreans. DPB occurs more often in males and usually begins around age 40. It was recognized as a distinct new disease in the early 1960s and was formally named diffuse panbronchiolitis in 1969. +If left untreated, DPB progresses to bronchiectasis, an irreversible lung condition that involves enlargement of the bronchioles, and pooling of mucus in the bronchiolar passages. Daily treatment of DPB with macrolide antibiotics such as erythromycin eases symptoms and increases survival time, but the disease currently has no known cure. The eventual result of DPB can be respiratory failure and heart problems. +The term "bronchiolitis" generally refers to inflammation of the bronchioles. DPB is classified as a form of "primary bronchiolitis", which means that the underlying cause of bronchiolitis is originating from or is confined to the bronchioles. Along with DPB, additional forms of primary bronchiolitis include bronchiolitis obliterans, follicular bronchiolitis, respiratory bronchiolitis, mineral dust airway disease, and a number of others. Unlike DPB, bronchiolitis that is not considered "primary" would be associated with diseases of the larger airways, such as chronic bronchitis. +Symptoms of DPB include chronic sinusitis (inflamed paranasal sinuses), wheezing, crackles (respiratory sounds made by obstructions such as phlegm and secretions in the lungs), dyspnea (shortness of breath), and a severe cough that yields large amounts of sputum (coughed-up phlegm). There may be pus in the sputum, and affected individuals may have fever. Typical signs of DPB progression include dilation (enlargement) of the bronchiolar passages and hypoxemia (low levels of oxygen in the blood). If DPB is left untreated, bronchiectasis will occur; it is characterized by dilation and thickening of the walls of the bronchioles, inflammatory damage to respiratory and terminal bronchioles, and pooling of mucus in the lungs. DPB is associated with progressive respiratory failure, hypercapnia (increased levels of carbon dioxide in the blood), and can eventually lead to pulmonary hypertension (high blood pressure in the pulmonary vein and artery) and cor pulmonale (dilation of the right ventricle of the heart, or "right heart failure"). +DPB is idiopathic, which means an exact physiological, environmental, or pathogenic cause of the disease is unknown. However, several factors are suspected to be involved with its pathogenesis (the way in which the disease works). +The major histocompatibility complex (MHC) is a large genomic region found in most vertebrates that is associated with the immune system. It is located on chromosome 6 in humans. A subset of MHC in humans is human leukocyte antigen (HLA), which controls the antigen-presenting system, as part of adaptive immunity against pathogens such as bacteria and viruses. When human cells are infected by a pathogen, some of them can present parts of the pathogen's proteins on their surfaces; this is called "antigen presentation". The infected cells then become targets for types of cytotoxic T-cells, which kill the infected cells so they can be removed from the body. +Genetic predisposition for DPB susceptibility has been localized to two HLA haplotypes (a nucleotide or gene sequence difference between paired chromosomes, that is more likely to occur among a common ethnicity or trait) common to people of East Asian descent. HLA-B54 is associated with DPB in the Japanese, while HLA-A11 is associated with the disease in Koreans. Several genes within this region of class I HLA are believed to be responsible for DPB, by allowing increased susceptibility to the disease. The common genetic background and similarities in the HLA profile of affected Japanese and Korean individuals were considered in the search for a DPB gene. It was suggested that a mutation of a suspected disease-susceptibility gene located somewhere between HLA-B and HLA-A had occurred on an ancestral chromosome carrying both HLA-B54 and HLA-A11. Further, it is possible that a number of genetic recombination events around the disease locus (location on a chromosome) could have resulted in the disease being associated with HLA-B54 in the Japanese and HLA-A11 in Koreans. After further study, it was concluded that a DPB susceptibility gene is located near the HLA-B locus at chromosome 6p21.3. Within this area, the search for a genetic cause of the disease has continued. +Because many genes belonging to HLA remain unidentified, positional cloning (a method used to identify a specific gene, when only its location on a chromosome is known) has been used to determine that a mucin-like gene is associated with DPB. In addition, diseases caused by identified HLA genes in the DPB-susceptibility region have been investigated. One of these, bare lymphocyte syndrome I (BLS I), exhibits a number of similarities with DPB in those affected, including chronic sinusitis, bronchiolar inflammation and nodules, and the presence of H. influenzae. Also like DPB, BLS I responds favorably to erythromycin therapy by showing a resolution of symptoms. The similarities between these two diseases, the corresponding success with the same mode of treatment, and the fact that the gene responsible for BLS I is located within the DPB-causing area of HLA narrows the establishment of a gene responsible for DPB. Environmental factors such as inhaling toxic fumes and cigarette smoking are not believed to play a role in DPB, and unknown environmental and other non-genetic causes—such as unidentified bacteria or viruses—have not been ruled out. +Cystic fibrosis (CF), a progressive multi-system lung disease, has been considered in the search for a genetic cause of DPB. This is for a number of reasons. CF, like DPB, causes severe lung inflammation, abundant mucus production, infection, and shows a genetic predominance among Caucasians of one geographic group to the rarity of others; whereas DPB dominates among East Asians, CF mainly affects individuals of European descent. While no gene has been implicated as the cause of DPB, mutation in a specific gene—much more likely to occur in Europeans—causes CF. This mutation in the CF-causing gene is not a factor in DPB, but a unique polymorphism (variation) in this gene is known to occur in many Asians not necessarily affected by either disease. It is being investigated whether this gene in any state of mutation could contribute to DPB. +Inflammation is a normal part of the human immune response, whereby leukocytes (white blood cells), including neutrophils (white blood cells that specialize in causing inflammation), gather, and chemokines (proteins released from certain cells, which activate or elicit a response from other cells) accumulate at any location in the body where bacterial or viral infections occur. Inflammation interferes with the activity of bacteria and viruses, and serves to clear them from the body. In DPB, bacteria such as Haemophilus influenzae and Pseudomonas aeruginosa can cause the proliferation of inflammatory cells into the bronchiolar tissues. However, when neither bacteria are present with DPB, the inflammation continues for an as yet unknown reason. In either circumstance, inflammation in DPB can be so severe that nodules containing inflammatory cells form in the walls of the bronchioles. The presence of inflammation and infection in the airways also results in the production of excess mucus, which must be coughed up as sputum. The combination of inflammation, nodule development, infection, mucus, and frequent cough contributes to the breathing difficulties in DPB. +The fact that inflammation in DPB persists with or without the presence of P. aeruginosa and H. influenzae provides a means to determine several mechanisms of DPB pathogenesis. Leukotrienes are eicosanoids, signaling molecules made from essential fatty acids, which play a role in many lung diseases by causing the proliferation of inflammatory cells and excess mucus production in the airways. In DPB and other lung diseases, the predominant mediator of neutrophil-related inflammation is leukotriene B4, which specializes in neutrophil proliferation via chemotaxis (the movement of some types of cells toward or away from certain molecules). +Inflammation in DPB is also caused by the chemokine MIP-1alpha and its involvement with CD8+ T cells. Beta defensins, a family of antimicrobial peptides found in the respiratory tract, are responsible for further inflammation in DPB when a pathogen such as P. aeruginosa is present. If present with DPB, the human T-lymphotropic virus, type I, a retrovirus, modifies DPB pathogenesis by infecting T helper cells and altering their effectiveness in recognizing the presence of known or unknown pathogens involved with DPB. +The diagnosis of DPB requires analysis of the lungs and bronchiolar tissues, which can require a lung biopsy, or the more preferred high resolution computed tomography (HRCT) scan of the lungs. The diagnostic criteria include severe inflammation in all layers of the respiratory bronchioles and lung tissue lesions that appear as nodules within the terminal and respiratory bronchioles in both lungs. The nodules in DPB appear as opaque lumps when viewed on X-rays of the lung, and can cause airway obstruction, which is evaluated by a pulmonary function test, or PFT. Lung X-rays can also reveal dilation of the bronchiolar passages, another sign of DPB. HRCT scans often show blockages of some bronchiolar passages with mucus, which is referred to as the "tree-in-bud" pattern. Hypoxemia, another sign of breathing difficulty, is revealed by measuring the oxygen and carbon dioxide content of the blood, using a blood test called arterial blood gas. Other findings observed with DPB include the proliferation of lymphocytes (white blood cells that fight infection), neutrophils, and foamy histiocytes (tissue macrophages) in the lung lining. Bacteria such as H. influenzae and P. aeruginosa are also detectable, with the latter becoming more prominent as the disease progresses. The white blood, bacterial and other cellular content of the blood can be measured by taking a complete blood count (CBC). Elevated levels of IgG and IgA (classes of immunoglobulins) may be seen, as well as the presence of rheumatoid factor (an indicator of autoimmunity). Hemagglutination, a clumping of red blood cells in response to the presence of antibodies in the blood, may also occur. Neutrophils, beta-defensins, leukotrienes, and chemokines can also be detected in bronchoalveolar lavage fluid injected then removed from the bronchiolar airways of individuals with DPB, for evaluation. +In the differential diagnosis (finding the correct diagnosis between diseases that have overlapping features) of some obstructive lung diseases, DPB is often considered. A number of DPB symptoms resemble those found with other obstructive lung diseases such as asthma, chronic bronchitis, and emphysema. Wheezing, coughing with sputum production, and shortness of breath are common symptoms in such diseases, and obstructive respiratory functional impairment is found on pulmonary function testing. Cystic fibrosis, like DPB, causes severe lung inflammation, excess mucus production, and infection; but DPB does not cause disturbances of the pancreas nor the electrolytes, as does CF, so the two diseases are different and probably unrelated. DPB is distinguished by the presence of lesions that appear on X-rays as nodules in the bronchioles of both lungs; inflammation in all tissue layers of the respiratory bronchioles; and its higher prevalence among individuals with East Asian lineage. +DPB and bronchiolitis obliterans are two forms of primary bronchiolitis. Specific overlapping features of both diseases include strong cough with large amounts of often pus-filled sputum; nodules viewable on lung X-rays in the lower bronchi and bronchiolar area; and chronic sinusitis. In DPB, the nodules are more restricted to the respiratory bronchioles, while in OB they are often found in the membranous bronchioles (the initial non-cartilaginous section of the bronchiole, that divides from the tertiary bronchus) up to the secondary bronchus. OB is a bronchiolar disease with worldwide prevalence, while DPB has more localized prevalence, predominantly in Japan. Prior to clinical recognition of DPB in recent years, it was often misdiagnosed as bronchiectasia, COPD, IPF, phthisis miliaris, sarcoidosis or alveolar cell carcinoma. +Macrolide antibiotics, such as erythromycin, are an effective treatment for DPB when taken regularly over an extended period of time. Clarithromycin or roxithromycin are also commonly used. The successful results of macrolides in DPB and similar lung diseases stems from managing certain symptoms through immunomodulation (adjusting the immune response), which can be achieved by taking the antibiotics in low doses. Treatment consists of daily oral administration of erythromycin for two to three years, an extended period that has been shown to dramatically improve the effects of DPB. This is apparent when an individual undergoing treatment for DPB, among a number of disease-related remission criteria, has a normal neutrophil count detected in BAL fluid, and blood gas (an arterial blood test that measures the amount of oxygen and carbon dioxide in the blood) readings show that free oxygen in the blood is within the normal range. Allowing a temporary break from erythromycin therapy in these instances has been suggested, to reduce the formation of macrolide-resistant P. aeruginosa. However, DPB symptoms usually return, and treatment would need to be resumed. Although highly effective, erythromycin may not prove successful in all individuals with the disease, particularly if macrolide-resistant P. aeruginosa is present or previously untreated DPB has progressed to the point where respiratory failure is occurring. +With erythromycin therapy in DPB, great reduction in bronchiolar inflammation and damage is achieved through suppression of not only neutrophil proliferation, but also lymphocyte activity and obstructive mucus and water secretions in airways. The antibiotic effects of macrolides are not involved in their beneficial effects toward reducing inflammation in DPB. This is evident because the treatment dosage is much too low to fight infection, and in DPB cases with the occurrence of macrolide-resistant P. aeruginosa, erythromycin therapy still reduces inflammation. +A number of factors are involved in suppression of inflammation by erythromycin and other macrolides. They are especially effective at inhibiting the proliferation of neutrophils, by diminishing the ability of interleukin 8 and leukotriene B4 to attract them. Macrolides also reduce the efficiency of adhesion molecules that allow neutrophils to stick to bronchiolar tissue linings. Mucus production in the airways is a major culprit in the morbidity and mortality of DPB and other respiratory diseases. The significant reduction of inflammation in DPB attributed to erythromycin therapy also helps to inhibit the production of excess mucus. +Untreated DPB leads to bronchiectasis, respiratory failure, and death. A journal report from 1983 indicated that untreated DPB had a five-year survival rate of 62.1%, while the 10-year survival rate was 33.2%. With erythromycin treatment, individuals with DPB now have a much longer life expectancy due to better management of symptoms, delay of progression, and prevention of associated infections like P. aeruginosa. The 10-year survival rate for treated DPB is about 90%. In DPB cases where treatment has resulted in significant improvement, which sometimes happens after about two years, treatment has been allowed to end for a while. However, individuals allowed to stop treatment during this time are closely monitored. As DPB has been proven to recur, erythromycin therapy must be promptly resumed once disease symptoms begin to reappear. In spite of the improved prognosis when treated, DPB currently has no known cure. +DPB has its highest prevalence among the Japanese, at 11 per 100,000 population. Korean, Chinese, and Thai individuals with the disease have been reported as well. A genetic predisposition among East Asians is suggested. The disease is more common in males, with the male to female ratio at 1.4–2:1 (or about 5 men to 3 women). The average onset of the disease is around age 40, and two-thirds of those affected are non-smokers, although smoking is not believed to be a cause. The presence of HLA-Bw54 increases the risk of diffuse panbronchiolitis 13.3-fold. +In Europe and the Americas, a relatively small number of DPB cases have been reported in Asian immigrants and residents, as well as in individuals of non-Asian ancestry. Misdiagnosis has occurred in the West owing to less recognition of the disease than in Asian countries. Relative to the large number of Asians living in the west, the small number of them thought to be affected by DPB suggests non-genetic factors may play some role in its cause. This rarity seen in Western Asians may also be partly associated with misdiagnosis. +In the early 1960s, a relatively new chronic lung disease was being observed and described by physicians in Japan. In 1969, the name "diffuse panbronchiolitis" was introduced to distinguish it from chronic bronchitis, emphysema, alveolitis, and other obstructive lung disease with inflammation. Between 1978 and 1980, the results of a nationwide survey initiated by the Ministry of Health and Welfare of Japan revealed more than 1,000 probable cases of DPB, with 82 histologically confirmed. By the 1980s, it was internationally recognized as a distinct disease of the lungs. +Before the 1980s, the prognosis or expected outcome of DPB was poor, especially in cases with superinfection (the emergence of a new viral or bacterial infection, in addition to the currently occurring infection) by P. aeruginosa. DPB continued to have a very high mortality rate before generalized antibiotic treatment and oxygen therapy were beginning to be used routinely in the effort to manage symptoms. Around 1985, when long-term treatment with the antibiotic erythromycin became the standard for managing DPB, the prognosis significantly improved. In 1990, the association of DPB with HLA was initially asserted. +Borscht (English: ) is a sour soup, made with meat stock, vegetables and seasonings, common in Eastern Europe, Central Europe and Northern Asia. In English, the word borscht, borrowed via Yiddish, is most often associated with the variant of the soup originating in Ukraine, made with red beetroots as one of the main ingredients, which give the dish its distinctive red color. The same name, however, is also used for a wide selection of sour-tasting soups without beetroots, such as sorrel-based green borscht, rye-based white borscht, and cabbage borscht. +Borscht derives from an ancient soup originally cooked from pickled stems, leaves and umbels of common hogweed (Heracleum sphondylium), an herbaceous plant growing in damp meadows, which lent the dish its Slavic name. With time, it evolved into a diverse array of tart soups, among which the Ukrainian beet-based red borscht has become the most popular. It is typically made by combining meat or bone stock with sautéed vegetables, which—as well as beetroots—usually include cabbage, carrots, onions, potatoes, and tomatoes. Depending on the recipe, borscht may include meat or fish, or be purely vegetarian; it may be served either hot or cold, and it may range from a hearty one-pot meal to a clear broth or a smooth drink. It is often served with smetana or sour cream, hard-boiled eggs or potatoes, but there exists an ample choice of more involved garnishes and side dishes, such as uszka or pampushky, that can be served with the soup. +Its popularity has spread throughout Eastern Europe and—by way of migration away from the Russian Empire—to other continents. In North America, borscht is often linked with either Jews or Mennonites, the groups who first brought it there from Europe. Several ethnic groups claim borscht, in its various local implementations, as their own national dish consumed as part of ritual meals within Orthodox, Greek Catholic, Roman Catholic, and Jewish religious traditions. In 2022, shortly after Russia's invasion of Ukraine, UNESCO recognized the "Culture of Ukrainian borscht cooking" on its List of Intangible Cultural Heritage in Need of Urgent Safeguarding. UNESCO noted that several countries in the broader region have also practiced borscht cooking and their inscription does not imply exclusivity or ownership of the dish for any particular country. +The English name derives, through Yiddish, from Ukrainian and Russian бо́рщ (borshch, Ukrainian: [ˈbɔrʃt͡ʃ] , Russian: [ˈborɕː] ). Together with cognates in other Slavic languages, such as Belarusian: бо́ршч (borshch), Polish: barszcz, and others, it comes from Proto-Slavic *bъ̃rščь, 'hogweed', and ultimately from Proto-Indo-European *bʰr̥stis, 'point', 'stubble'. +The English form borscht comes from Yiddish באָרשט (borsht), as the dish was first popularized in North America by Yiddish-speaking Ashkenazi Jews from Eastern Europe. +The stock is typically made by boiling meat, bones, or both. Beef, pork or a combination of both are most commonly used, with brisket, ribs, shank and chuck considered to give the most flavorful results, especially if cooked on a high flame. Marrow bones are considered best for the bone stock. Meat stock is usually cooked for about two hours, whereas bone stock takes four to six hours to prepare. Meat and bones are usually removed afterwards and the meat is only added back into the soup about 10–15 minutes before the borscht is done. Some recipes call for smoked meats, resulting in a distinctively smoky borscht, while others use poultry or mutton stock. Fasting varieties are typically made with fish stock to avoid the use of meat, while purely vegetarian recipes often substitute forest mushroom broth for the stock. +The vegetables most commonly added to borscht are beetroots, white cabbage, carrots, parsley root, potatoes, onions and tomatoes. Some recipes may also call for beans, tart apples, turnip, swede, celeriac, zucchini or bell peppers. Parsnip may be used as a substitute for parsley root, and tomato paste is often used as well as or instead of fresh tomatoes. Onions, carrots, parsley root, turnip and other root vegetables are sautéed (traditionally in animal fat, especially lard or butter) and then mixed with tomatoes or tomato paste. Dry beans are boiled separately. Potatoes and cabbage are boiled in the stock for about 15 minutes before the precooked vegetables are added. +The traditional technique of preparing the soup is to precook the vegetables—by sautéing, braising, boiling or baking—separately from the meat and only then to combine them with the stock. This distinctive feature of borscht derives from the practice of slow cooking in the Russian oven (traditional masonry stove, used for both cooking and heating), wherein the differences in cooking times of individual ingredients had to be taken into account in order to ensure that all components reach doneness at the same time. The importance of this method is reflected in the Russian language, where a variant in which all vegetables are added raw directly into the stock is referred to by the diminutive form borshchok rather than borshch. +The soup is typically flavored with a wide selection of herbs, spices and condiments. Salt, black pepper, garlic, bay leaves and dill are among the most commonly used. Other aromatics often added to borscht include allspice, celery stalks, parsley, marjoram, hot peppers, saffron, horseradish, ginger and prunes. Some recipes require flour or roux to further thicken the borscht. A common opinion is that a good borscht should be thick enough for a spoon to stand upright in it. +The dominant tastes in borscht are sweet and sour. This combination is traditionally obtained by adding beet sour. The sour is made by covering sliced beetroots with lukewarm preboiled water and allowing bacteria to ferment some of the sugars present in beetroots into dextran (which gives the liquid a slightly viscous consistency), mannitol, acetic acid and lactic acid. Stale rye bread is often added to hasten the process, but usually omitted in Jewish recipes, as chametz (leavened bread) would make the sour unfit for Passover meals. Sugar, salt and lemon juice may also be added to balance the flavor. After about 2–5 days (or 2–3 weeks without the bread), the deep red, sweet and sour liquid may be strained and is ready to use. It is added to borscht shortly before the soup is done, as prolonged boiling would cause the tart flavor to dissipate. +The beet sour is known in Slavic languages as kvas (lit. 'sour, acid'; compare kvass) and in Yiddish as rosl (from a Slavic word originally referring to any brine obtained by steeping salted meat or vegetables in water; compare Russian rassol, 'pickle juice', Polish rosół, 'broth'). Apart from its employment in borscht, it may also be added to prepared horseradish or used as pot roast marinade. +As the traditional method of making borscht with beet sour often requires planning at least several days ahead, many recipes for quicker borscht replace the beet sour with fresh beetroot juice, while the sour taste is imparted by other ingredients. Vinegar, tomato products, lemon juice or citric acid may be used, as well as dry red wine, dill pickle juice, murături juice, sauerkraut juice, tart apples, Mirabelle plums, apricots, or a fermented rye flour and water mixture. +As early as the 19th century, it was recommended for fever and sore throat. It was believed to purify the blood, improve the complexion, and restore appetite. Beets, which are the base of borscht, are a rich source of folic acid, vitamin C, and B vitamins. They contain powerful antioxidants—betaine and pigments—that support the circulatory system and help fight free radicals. People with insulin resistance, type 2 diabetes, rheumatism, gout, or kidney stones should avoid borscht due to its high glycemic index and the presence of oxalates. +There are multiple examples of the soup in Ukrainian cuisine. Virtually every oblast has its own version. Differences between particular varieties may regard the type of stock used (meat, bone, or both), the type of meat (beef, pork, poultry, etc.), the choice of vegetables and the method of cutting and cooking them. For example, although the typical recipe calls for beef and pork, the Kyiv variant uses mutton or lamb as well as beef, while in the Poltava region, the stock for borscht is cooked on poultry meat, that is, chicken, duck or goose. The use of zucchini, beans and apples is characteristic of the Chernihiv borscht; in this variant, beetroots are sautéed in vegetable oil rather than lard, and the sour taste comes solely from tomatoes and tart apples. The Lviv borscht is based on bone stock and is served with chunks of Vienna sausages. In Southern Ukraine borshch typically includes legumes. Historically, borshch with meat was reserved as a holiday dish, meanwhile varieties consumed during lent would include fish, such as dried crucian carp. +Numerous seasonal varieties of borshch exist in Ukrainian tradition: in spring and during summer, green borshch is traditionally cooked, meanwhile borshch with pork would be traditionally reserved for Christmas season, and in autumn borshch with mutton and chicken was also popular. The traditional variety of borshch consumed during Lent contained fish, wheat flour, hemp oil, beans and potatoes. Other popular traditional ingredients of borshch are salo, fried sausages. During the late 19th century a dish known as panskyi borshch ("lords' borshch") would be cooked with the addition of duck, chicken and pork meat. +Borscht is symbolic of hospitality in Ukraine and is part of multiple traditional celebrations and rituals. In some parts of Ukraine, the third day of a wedding celebration is called do nevistky na borshch, which translates to "visit daughter-in-law to eat borscht". In 2022, UNESCO added "Culture of Ukrainian borscht cooking" to the List of Intangible Cultural Heritage in Need of Urgent Safeguarding, citing the Russian invasion of Ukraine. +As well as the thick borschts described above, Polish cuisine offers a ruby-colored beetroot bouillon known as barszcz czysty czerwony, or clear red borscht. It is made by combining strained meat-and-vegetable stock with wild mushroom broth and beet sour. In some versions, smoked meat may be used for the stock and the tartness may be obtained or enhanced by adding lemon juice, dill pickle brine, or dry red wine. It may be served either in a soup bowl or—especially at dinner parties—as a hot beverage in a twin-handled cup, with a croquette or a filled pastry on the side. Unlike other types of borscht, it is not whitened with sour cream. +Barszcz wigilijny, or Christmas Eve borscht, is a variant of the clear borscht that is traditionally served during the Polish Christmas Eve supper. In this version, meat stock is either omitted or replaced with fish broth, usually made by boiling the heads cut off from fish used in other Christmas Eve dishes. The mushrooms used for cooking the mushroom broth are reserved for uszka (small filled dumplings), which are then served with the borscht. +Ashkenazi Jews living in Eastern Europe adopted beetroot borscht from their Slavic neighbors and adapted it to their taste and religious requirements. As combining meat with milk is proscribed by kosher dietary laws, Jews have developed two variants of the soup: meat (fleischik) and dairy (milchik). The meat variant is typically made from beef brisket (pork is never used) and cabbage, while the dairy one is vegetarian, blended with sour cream or a mixture of milk and egg yolks. Both variants typically contain beetroots and onions, and are flavored with beet sour, vinegar or citric acid for tartness and beet sugar for sweetness. Galician Jews traditionally liked their borscht particularly sweet. Jewish borscht may be served either hot or cold, typically with a hot boiled potato on the side. In prewar Eastern Europe it was traditionally put up to ferment around Purim so that it would be ready four weeks later for the Passover holiday. +Russian variants include a Siberian style borscht, characterized by meatballs; Pskov borscht with dried smelt from the local lakes; monastic Lenten borscht with marinated kelp instead of cabbage and the Russian Navy borscht (flotsky borshch), the defining characteristic of which is that the vegetables are cut into square or diamond-shaped chunks rather than julienned. +Kuban borscht is made from local beets, Kuban borschevaya 43 cultivar, which is distinguished by its less bright color. As a result, the borscht turns out golden or orange in color. +In summer, cold borscht (also known as cold beet soup) is a popular alternative to borscht variants that are normally served hot. It consists of beet sour or beet juice blended with sour cream, buttermilk, soured milk, kefir or yogurt. The mixture has a distinctive pink or magenta color. It is served refrigerated, typically over finely chopped beetroot, cucumbers, radishes and green onion together with halves of a hard-boiled egg, and sprinkled with fresh dill. Chopped veal, ham, or crawfish tails may be added as well. +The dish originates from the shared culinary traditions of the Polish–Lithuanian Commonwealth. Originally a common everyday food among the local peasantry of the Grand Duchy of Lithuania, the cold soup eventually became a delicacy on noble tables. Early 19th-century accounts, such as by physician Johann Peter Frank who was served the icy soup in Nyasvizh in 1805, describe it as an already established local dish. The first known recipe for the dish, simply called chłodnik, was recorded in Warsaw at the end of the 18th century by Paul Tremo, the chef to the Commonwealth's last ruler. +The Polish–Lithuanian poet Adam Mickiewicz mentioned the Lithuanian cold beet soup in his 1834 epic poem Pan Tadeusz. It was a daily summer staple food of enjoyed by many social classes, ranging from peasants to nobles. Today, it is an exceptionally popular summer dish in Lithuania, in whose capital Vilnius, an annual festival (the Vilnius Pink Soup Fest) has been organised since 2023 to celebrate its cultural heritage. +Although borscht is mostly used to describe a beet-based soup, there are soups in some culinary traditions with the same or similar names, but with sometimes wide variations in ingredients and preparation methods. In such soups, beetroots are not used or merely optional. The principal common trait among such borschts is a tart flavor from sour-tasting ingredients. According to A Gift to Young Housewives, a book from the 19th century, "borscht" may or may not include beets (depending from recipe to recipe in the book). +In Polish cuisine, white borscht (barszcz biały, also known as żur or żurek, 'sour soup') is made from a fermented mixture of rye flour or oatmeal and water. It is typically flavored with garlic and marjoram, and served over eggs and boiled fresh sausage; the water in which the sausage was boiled is often used instead of meat stock. +In the Carpathian Mountains of southern Poland, variants of borscht are also made in which the tart taste comes from dairy products, such as whey or buttermilk. Although the deep red color of beetroot borscht may remind those unfamiliar with Polish cuisine of blood, the type of borscht that does contain animal (usually poultry) blood mixed with vinegar is dark brownish-gray in color and aptly called "gray borscht" (barszcz szary), which is a regional name of the Polish blood soup better known as czernina. +Green borscht (zeleny borshch), a light soup made from leaf vegetables, is an example common in Ukrainian and Russian cuisines. The naturally tart-tasting sorrel is most commonly used, but spinach, chard, nettle, garden orache and occasionally dandelion, goutweed or ramsons, may be added as well, especially after the spring season for sorrel has passed. Like beetroot borscht, it is based on meat or vegetable broth and is typically served with boiled potatoes and hard-boiled eggs, sprinkled with dill. There is also a variety of Ukrainian green borscht which includes both sorrel and beetroots. +In Romanian and Moldovan cuisines, a mixture of wheat bran or cornmeal with water that has been left to ferment, similar to, but less cloudy than that used in Polish white borscht, is called borș. It is used to impart a sour taste to a variety of tangy Romanian soups, known as either also borș or ciorbă. Variants include ciorbă de perișoare (with meatballs), ciorbă de burtă (with tripe), borș de pește (with fish) and borș de sfeclă roșie (with beetroots). +The Armenian, Azerbaijani and Georgian version of borscht is a hot soup made with beef stock, green peppers and other vegetables, which may or may not include beetroots, and flavored with chopped red chili and fresh cilantro. In ethnic Mennonite cuisine, borscht refers to a whole range of seasonal vegetable soups based on beef or chicken stock—from spring borscht made with spinach, sorrel and chard to summer borscht with cabbage, tomatoes, maize and squash to fall and winter borscht with cabbage, beets and potatoes. +In Chinese cuisine, a soup known as Luosong tang, a translation based on the term "Russian soup" from the Russian diaspora in China, is based on red cabbage and tomatoes, and lacks beetroots altogether; also known as "Chinese borscht", it originated in Harbin, close to the Russian border in northeast China, and has spread as far as Hong Kong. In Shanghai's Haipai cuisine, tomatoes are the main ingredient; beef and its broth, onions and cabbages are also added; while flour, rather than sour cream, is used for thickening. +The diversity of borscht styles is matched by the wide choice of garnishes and side dishes with which various types of borscht may be served. Most often, borscht is served with smetana, a soured dairy product similar to the French crème fraîche. The smetana may be served in a separate pitcher for the diners to add the desired amount themselves or the borscht may come already "whitened" with the smetana already added. The cream can also be thickened with flour before being added to the soup. Yogurt and a mixture of milk and yolks are possible substitutes. +Chopped herbs are often sprinkled on the surface of the soup; dill is most common, but parsley, chives or scallion are often added as well. Individual helpings may be spiced up with minced hot peppers or garlic. Many types of borscht are served over halves or quarters of hard-boiled chicken or quail eggs. Navy beans, broad beans or string beans are also a common addition. +Meat, removed from the stock on which the borscht was based, may be cut into smaller chunks and either added back into the soup or served on the side with horseradish or mustard. Bacon and sausages are also commonly used as borscht garnishes. Borscht based on bone stock may be served Old Polish-style, with marrow from the bones. +Some types of soup, such as Poltava borscht, may be served with halushky, or thick noodles of wheat or buckwheat flour. Siberian borscht is eaten with boiled meatballs (frikadelki) of minced beef and onion. In Poland and parts of western Ukraine, borscht is typically ladled over uszka, or bite-sized ear-shaped dumplings made from pasta dough wrapped around mushroom, buckwheat or meat filling. Mushroom-filled uszka are particularly associated with Polish Christmas Eve borscht. +Borscht, like any other soup in East Slavic cuisines, is seldom eaten by itself, but rather accompanied by a side dish. At a minimum, spoonfuls of borscht are alternated with bites of a slice of bread. Buckwheat groats or boiled potatoes, often topped with pork cracklings, are other simple possibilities. +In Ukraine, borscht is often accompanied with pampushky, or savory, puffy yeast-raised rolls glazed with oil and crushed garlic. In Russian cuisine, borscht may be served with assorted side dishes based on tvorog, or the East European variant of farmer cheese, such as vatrushki, syrniki or krupeniki. Vatrushki are baked round cheese-filled tarts; syrniki are small pancakes wherein the cheese is mixed into the batter; and a krupenik is a casserole of buckwheat groats baked with cheese. +Pirozhki, or baked dumplings with fillings as for uszka, are another common side for both thick and clear variants of borscht. Polish clear borscht may also be served with a croquette or paszteciki. A typical Polish croquette (krokiet) is made by wrapping a crêpe (thin pancake) around a filling and coating it in breadcrumbs before refrying; paszteciki (lit. 'little pâtés') are variously shaped filled hand-held pastries of yeast-raised or flaky dough. Another way to serve borscht is with a coulibiac, or a large loaf-shaped pie. Possible fillings for croquettes, paszteciki and coulibiacs include mushrooms, sauerkraut and minced meat. +Borscht derives from a soup originally made by the Slavs from common hogweed (Heracleum sphondylium, also known as cow parsnip), which gave the dish its Slavic name. Growing commonly in damp meadows throughout the north temperate zone, hogweed was used not only as fodder (as its English names suggest), but also for human consumption—from Eastern Europe to Siberia, to northwestern North America. +The Slavs collected hogweed in May and used its roots for stewing with meat. As for the stems, leaves, and umbels; these would be chopped, covered with water and left in a warm place to ferment. After a few days, lactic and alcoholic fermentation produced a mixture described as "something between beer and sauerkraut". +The said soup—with aforementioned fermented hogweed concoction used—was characterized by a mouth-puckering amount of sourness in its taste, while its smell was described as pungent As the Polish ethnographer Łukasz Gołębiowski wrote in 1830, "Poles have been always partial to tart dishes, which are somewhat peculiar to their homeland and vital to their health." Simon Syrenius (Szymon Syreński), a 17th century Polish botanist, described "our Polish hogweed" as a vegetable that was well known throughout Poland, Ruthenia, Lithuania and Samogitia (that is, most of the northern part of Eastern Europe), typically used for cooking a "tasty and graceful soup" with capon stock, eggs, sour cream and millet. More interested in the plant's medicinal properties than its culinary use, he also recommended pickled hogweed juice as a cure for fever or hangover. +One of the earliest possible mentions of borscht as a soup is found in the diary of German merchant Martin Gruneweg, who visited Kyiv in 1584. After Gruneweg reached river Borshchahivka in Kyiv's vicinity on 17 October 1584, he wrote down a local legend saying that the river was so named because there was a borscht market. However, he doubted the story noting that "Ruthenians buy borscht rarely or never, because everyone cooks their own at home as it's their staple food and drink". +Another early written reference to the Slavic hogweed soup can be found in Domostroy (Domestic Order), a 16th century Russian compendium of moral rules and homemaking advice. It recommends growing the plant "by the fence, around the whole garden, where the nettle grows", to cook a soup of it in springtime and reminds the reader to, "for the Lord's sake, share it with those in need". +Hogweed borscht was mostly a poor man's food. The soup's humble beginnings are still reflected in Polish fixed expressions, where "cheap like borscht" is the equivalent of "dirt cheap" (also attested as a calque in Yiddish and Canadian English), whereas adding "two mushrooms into borscht" is synonymous with excess. For the professors of the University of Kraków, who led a monastic way of life in the 17th century, hogweed borscht was a fasting dish which they ate regularly from Lent till Rogation days. It was uncommon on the royal table, although according to the 16th century Polish botanist Marcin of Urzędów—citing Giovanni Manardo, a court physician to the Jagiellonian kings of Hungary—the Polish-born King Vladislaus II used to have a Polish hogweed-based dish prepared for him at his court in Buda. +With time, other ingredients were added to the soup, eventually replacing hogweed altogether, and the names borshch or barszcz became generic terms for any sour-tasting soup. In 19th century rural Poland, this term included soups made from barberries, currants, gooseberries, cranberries, celery or plums. +When describing the uses of common hogweed, John Gerard, a 17th century English botanist, observed that "the people of [Poland] and Lithuania [used] to make [a] drink with the decoction of this herb and leaven or some other thing made of meal, which is used instead of beer and other ordinary drink". It may suggest that hogweed soup was on some occasions combined with a fermented mixture of water and barley flour, oatmeal or rye flour. Such soured, gelatinous flour-and-water mixture, originally known as kissel (from the Proto-Slavic root *kyslŭ, 'sour') had been already mentioned in The Tale of Bygone Years, a 12th century chronicle of Kievan Rus', and continued to be a staple of Ukrainian and Russian cooking until the middle of the 19th century. In Poland, a soup based on diluted kissel became known as either żur (from Middle High German sur, 'sour') or barszcz and later—to distinguish it from the red beetroot borscht—as barszcz biały, 'white borscht'. +The earliest known Polish recipes for borscht, written by chefs catering to Polish magnates (aristocrats), are from the late 17th century. Stanisław Czerniecki, head chef to Prince Aleksander Michał Lubomirski, included several borscht recipes in his Compendium ferculorum (A Collection of Dishes), the first cookbook published originally in Polish, in 1682. They include such sour soups as lemon borscht and "royal borscht", the latter made from assorted dried, smoked or fresh fish and fermented rye bran. A manuscript recipe collection from the Radziwiłł family court, dating back to c. 1686, contains an instruction for making hogweed borscht mixed with poppy seeds or ground almonds. As this was a Lenten dish, it was garnished, in a trompe-l'œil fashion typical of Baroque cuisine, with mock eggs made from finely chopped pike that was partly dyed with saffron and formed into oval balls. An alternative recipe for the almond borscht replaced pickled hogweed with vinegar.. In the 18th century, borscht made from fermented beetroot appeared on tables, and it was this version that gained the most popularity. It was served at the famous Thursday dinners of King Stanisław August Poniatowski, as well as during Easter breakfast at the Czartoryski princes home. In the 18th century, the term borscht man referred to someone clumsy and awkward. However, thanks to culinary experiments and increasingly sophisticated recipes, borscht gained recognition and became a permanent part of Polish tradition. The 19th-century historian Cezary Biernacki wrote: Borscht was and is the most commonplace, and with the addition of spices, the most accurate, truly Polish soup, received with great taste, indeed, and respect. The 19th century was a turning point; it was then that red borscht with dumplings began to appear on Christmas Eve tables. +Borscht also evolved into a variety of sour soups to the east of Poland. Examples include onion borscht, a recipe for which was included in a 1905 Russian cookbook, and sorrel-based green borscht, which is still a popular summer soup in Ukraine and Russia. A Gift to Young Housewives by Elena Molokhovets, the best-selling Russian cookbook of the 19th century, first published in 1861, contains nine recipes for borscht, some of which are based on kvass, a traditional Slavic fermented beverage made from rye bread. Kvass-based variants were also known in Ukraine at that time; some of them were types of green borscht, while others were similar to the Russian okroshka. +Before the advent of beet-based borscht, cabbage borscht was of particular importance. Made from either fresh cabbage or sauerkraut, it could be indistinguishable from the Russian shchi. Indeed, the mid-19th century Explanatory Dictionary of the Living Great Russian Language defines borshch as sour beet or "a kind of shchi" with sour beet base. The significance of cabbage as an essential ingredient of borscht is manifest in the Ukrainian proverb, "without bread, it's no lunch; without cabbage, it's no borscht." +Beet (Beta vulgaris), a plant native to the Mediterranean Basin, was already grown in antiquity. Only the leaves were of culinary use, as the tapered, tough, whitish and bitter-tasting root was considered unfit for human consumption. It is probably that beet greens were used in variants of green borscht long before the invention of the beetroot-based red borscht. Beet varieties with round, red, sweet taproots, known as beetroots, were not reliably reported until the 12th century and did not spread to Eastern Europe before the 16th century. +Mikołaj Rej, a Polish Renaissance poet and moralist, included the earliest known Polish recipe for pickled beetroots in his 1568 book, Life of an Honest Man. It would later evolve into ćwikła, or chrain mit burik, a beet-and-horseradish relish popular in Polish and Jewish cuisines. Rej also recommended the "very tasty brine" left over from beetroot pickling, which was an early version of beet sour. The sour found some applications in Polish folk medicine as a cure for hangover and—mixed with honey—as a sore throat remedy. +It may never be known who first thought of using beet sour to flavor borscht, which also gave the soup its now-familiar red color. One of the earliest mentions of borscht with pickled beets comes from Russian ethnographer Andrey Meyer, who wrote in his 1781 book that people in Ukraine make fermented red beets with Acanthus, which they in turn use to cook their borscht. The book "Description of the Kharkiv Governorate" of 1785, which describes the food culture of the Ukrainians, says that borscht was the most consumed food, cooked from beets and cabbage with various other herbal spices and millet, on sour kvass; it was always made with pork lard or beef lard, on holidays with lamb or poultry, and sometimes with game. Jerzy Samuel Bandtkie's Polish-German dictionary published in 1806 was the first to define barszcz as a tart soup made from pickled beetroots. +The fact that certain 19th century Russian and Polish cookbooks, such as Handbook of the Experienced Russian Housewife (1842) by Yekaterina Avdeyeva and The Lithuanian Cook (1854) by Wincenta Zawadzka, refer to beetroot-based borscht as "Little Russian borscht" (where "Little Russian" is a term used at the time for ethnic Ukrainians under imperial Russian rule) suggests that this innovation took place in Ukraine, whose soils and climate are particularly well suited to beet cultivation. Ukrainian legends, probably of 19th century origin, attribute the invention of beetroot borscht either to Zaporozhian Cossacks, serving in the Polish army, on their way to break the siege of Vienna in 1683, or to Don Cossacks, serving in the Russian army, while laying siege to Azov in 1695. +Spanish conquistadors brought potatoes and tomatoes from the Americas to Europe in the 16th century, but these vegetables only became commonly grown and consumed in Eastern Europe in the 19th century. Eventually, both became staples of peasant diet and essential ingredients of Ukrainian and Russian borscht. Potatoes replaced turnips in borscht recipes, and tomatoes—fresh, canned or paste—took over from beet sour as the source of tartness. The turnip is rarely found in modern recipes, and even then, together with potatoes. In Ukraine, beet sour and tomatoes were both used for some time until the latter ultimately prevailed during the last third of the 19th century. Other innovation during the 19th century included the addition of powdered white mushrooms to borshch, which reportedly stemmed from Greek merchants from Nizhyn. +In Ukraine borshch was popularized as a national dish by the 19th century, when many recipes for cooking borshch had become known, some of which would contain more than 20 ingredients. Among the rich it was not unusual to have several varieties of the dish during one meal. One recipe preserved from that time includes ingredients such as cardamom, eggs, mushrooms and cherry. +Russian and Polish aristocrats used to employ celebrated French chefs, who later presented their dishes as foreign curiosity back in France. One of the first French chefs to do so was Marie-Antoine Carême, who worked briefly for Emperor Alexander I in 1819. In his take on borscht, the original Russian soup served only as inspiration for an extravagant haute cuisine dish with an air of eastern exoticism. Apart from vegetables and beet sour, his recipe calls for a roast chicken, a fried chicken, a duck, a piece of veal, an oxtail, a marrow bone, one pound of bacon, and six large sausages, and suggests serving with beef quenelles, deviled eggs and croûtons. +Auguste Escoffier, Carême's apprentice, who was mostly fascinated by the soup's vivid ruby-red color, simplified his master's recipe, while also securing the place of potage bortsch (lit. 'borscht soup') in French cuisine. Urbain Dubois and Émile Bernard, both of whom had been employed at Polish aristocratic courts, presented borscht to the French public as a Polish soup; their cookbook, La cuisine classique, published in 1856, contains a borscht recipe under the descriptive name, potage au jus de betteraves à la polonaise (lit. 'Polish-style beet-juice soup'), which had been changed to potage barsch à la polonaise by the third edition in 1868. In 1867, beetroot borscht was served, along with herrings, sturgeon, coulibiac, Pozharsky cutlets and vinaigrette salad, at a Russian-themed dinner at the International Exposition in Paris, strengthening its international association with Russian culture. +Over the course of the 19th and 20th centuries, borscht's popularity spread beyond its Slavic homeland, largely due to such factors as territorial expansion of the Russian Empire, Russia's growing political clout and cultural stature, and waves of emigration out of the country. As Russia grew to cover most of northern and central Eurasia, borscht was introduced to the cuisines of various peoples inhabiting the territories both within and adjacent to the empire, from Finland to the Caucasus and Iran, to Central Asia and China, to Alaska (Russian America). +Borscht's westward expansion was less successful; Germans used to scoff at the soup along with other East European fare. +Mass migration from the Russian Empire to North America—initially mostly by members of persecuted religious minorities—was instrumental in bringing borscht across the Atlantic. Jews from the Pale of Settlement, an area that stretched along the western edges of the Russian Empire and included much of present-day Ukraine, brought with them the Ukrainian variety of borscht with beetroot. +The earliest waves of migration, however, occurred at a time when cabbage-based borscht was still the dominant variant of the soup in at least parts of Russia. The Mennonites, who began arriving in Canada and the United States from Russia's Volga region in the 1870s, still eschew beetroots in their borscht; instead, Mennonite varieties include Komst Borscht (with cabbage or sauerkraut) and Somma Borscht (sorrel-based "summer borscht"). According to the Jewish Encyclopedia published in 1906, cabbage-based kraut borscht was also more popular than the beet-based variant in American Jewish cuisine at the time. +In the 1930s, when most American hotels refused to accept Jewish guests due to widespread anti-Semitism, New York Jews began flocking to Jewish-owned resorts in the Catskill Mountains for their summer vacations. The area grew into a major center of Jewish entertainment, with restaurants offering all-you-can-eat Ashkenazi Jewish fare, including copious amounts of borscht. Grossinger's, one of the largest resorts, served borscht throughout the day, every day of the year. The region became known, initially in derision, as the "Borscht Belt", reinforcing the popular association between borscht and American Jewish culture. As most visitors arrived in the summertime, the borscht was typically served cold. Marc Gold was one of its largest suppliers, producing 1,750 short tons (1,590 tonnes) a year in his business's heyday. Gold's borscht consists of puréed beetroots seasoned with sugar, salt and citric acid; it is usually blended with sour cream and served as a refreshing beverage, more aptly described as a "beet smoothie". Such type of "purplish, watery broth" is, according to Nikolai Burlakoff, author of The World of Russian Borsch, "associated in America with borsch, in general, and Jewish borsch in particular." +In the Soviet Union, borscht was one of the most popular everyday dishes. It was described in 2008 by journalist James Meek as "the common denominator of the Soviet kitchen, the dish that tied together ... the high table of the Kremlin and the meanest canteen in the boondocks of the Urals, ... the beetroot soup that pumped like the main artery through the kitchens of the east Slav lands." Among Soviet leaders, the Ukrainian-born Leonid Brezhnev was especially partial to borscht, which his wife continued to personally cook for him even after they had moved into the Kremlin. +The soup has even played a role in the Soviet space program. In March 1961, as part of a communications equipment test, a pre-recorded recipe for borscht was broadcast from the Korabl-Sputnik 4 spacecraft. The craft, carrying animals and a mannequin, had been launched into low Earth orbit in preparation for crewed space flights. +All ingredients for the space borscht (which include beef, beetroots, cabbage, potatoes, carrots, onions, parsley root, and tomato paste) were cooked separately, then combined one by one in strictly controlled order, sterilized, packed into tubes, sealed airtight and autoclaved. In the 1970s, the tubes were replaced with packages of rehydratable freeze-dried borscht with regular-size bits of cooked vegetables. +An article on borscht in the Soviet-era book Entsyclopedia Domashnego Hozyaystva (lit. 'Encyclopedia of Housekeeping') suggests to make a soup with beets, other vegetables, and tomato purée as a "borscht" in general. Its recipe of meat borscht also suggests adding vinegar to one's taste. +Borscht is often associated with its role in religious traditions of various denominations (Eastern Orthodox, Greek and Roman Catholic, and Jewish) that are common in Eastern Europe. In East Slavic countries, "memorial borscht" is served as the first course at a post-funeral wake. According to a traditional belief, the soul of the departed either feeds on or is carried up to heaven by puffs of steam rising from bowls of borscht and other hot dishes, such as blini, porridge, boiled potatoes or freshly baked bread. In the region of Polesye, straddling the Belarusian-Ukrainian border, the same steaming-hot dishes, including borscht, are given as an offering to the souls of deceased ancestors during the annual semi-pagan remembrance ceremony known as Dzyady or Forefathers' Night. +In Poland and Ukraine, borscht is usually one of the dishes served at a Christmas Eve dinner. Celebrated after the first star has appeared in the sky on December 24 (Roman Catholic) or January 6 (Greek Catholic), it is a meal which is at the same time festive and fasting, a multicourse affair (traditionally, with twelve distinct dishes) that excludes ingredients of land-animal origin. Christmas Eve borscht is, therefore, either vegetarian or based on fish stock and is not typically mixed with sour cream. In Ukraine, the soup contains vegetables that are sautéed in vegetable oil rather than lard, as well as beans and mushrooms. It may also be thickened with wheat flour dry-roasted in a pan instead of the usual roux. The Polish version of Christmas Eve borscht is a clear ruby-red broth. Both Ukrainian and Polish variants are often served with uszka. +While Christmas in Poland is traditionally linked to red borscht, Lent—the fasting period that leads up to Easter—is associated with a meatless version of white borscht, or żur. Youths used to celebrate Holy Saturday, the last day of the fast, with a mock "funeral" of the white borscht, in which a pot of the soup was either buried in the ground or broken, sometimes—to the crowd's amusement—while being carried by an unsuspecting boy on his head. On the next day, the white borscht would reappear on the Easter table, but this time, in its more coveted, meat-based guise with sausage, bacon and eggs. +In Eastern European Ashkenazi Jewish tradition, vegetarian borscht served with sour cream and boiled potatoes on the side, known as peysakhdiker borsht, is considered an essential dish during the Passover period. As the holiday is observed in spring (March or April), the preparation of Passover borscht used to provide an opportunity to use up the beet sour left over from pickled beetroots that had been consumed during winter, remaining potatoes that had been stored throughout the winter and sour cream that was readily available in the new calving season. Cold borscht blended with sour cream is also popular on Shavuot (Feast of Weeks), a holiday customarily associated with dairy foods, observed in late May or early June. Seudah Shlishit, or the third meal of the Shabbat, often includes borscht as well. +In 2022, the United Nations Educational, Scientific, and Cultural Organization (UNESCO) announced that it had placed "Culture of Ukrainian borscht cooking" on the List of Intangible Cultural Heritage in Need of Urgent Safeguarding. UNESCO noted that Borscht cooking was "also practised in communities in the broader region", and its designation "does not imply exclusivity, nor ownership, of the heritage concerned". Instead it recognized the significant cultural importance of Borscht to Ukrainians and the need to safeguard this culture, particularly in light of Russia's invasion of Ukraine. According to the festival blog of the Smithsonian Institution, "The designation by the international cultural authority was widely seen as a landmark decision in the ongoing cultural dispute between the two countries on borshch’s true country of origin." +In its currently most popular, beet-based version, borscht most probably originated in what is now Ukraine. Borscht's role as a staple of everyday Ukrainian diet is reflected in the Ukrainian saying, "borscht and porridge are our food" (compare the equivalent Russian saying, where borscht is replaced with shchi). The hearty soup in which the beetroot is just one of sundry vegetables, as opposed to the typically Polish clear beet broth, is still known in Poland as "Ukrainian borscht". +Borscht is associated with and claimed by several ethnic groups, especially Ukrainians, Russians, Poles, Lithuanians and Ashkenazi Jews, as their own national or ethnic dish and cultural icon. Such gastronationalistic claims are not necessarily mutually exclusive, as the soup's history predates the emergence in Eastern Europe of modern nation states with their ever-shifting borders. Borscht, in the words of Burlakoff, "is perfectly suited to a global culture". He describes it as "a global phenomenon", in which "local variants are so numerous and diverse that it is hard sometimes for a non-specialist to grasp that any single example of it is something that is part of a unified tradition". In his view, borscht "is an almost perfect example of ... 'glocalization'—a phenomenon that is global in distribution but reflective of local needs and ways in its variants and adaptation; ... a highly localized product that became globalized, and in the process adapted to conditions other than the original ones." +However, according to Irina Perianova, a Russian linguist and anthropologist, "people tend to be very proprietal about their food and proud of it." Perianova offers competing Russian and Ukrainian views on the origin and ingredients of borscht as an example of "a common connection between culinary and territorial claims", which results in the culinary area turning into "a battlefield generating and proliferating all kinds of myths". In 2020 Ukraine began the process to have borscht recognised as an element of the country's intangible cultural heritage, an initiative supported by chefs and food writers such as Marianna Dushar. +In the Soviet Union, government-sponsored cookbooks, such as The Book of Tasty and Healthy Food (1939) curated by Anastas Mikoyan, and Cookery and Directory of Recipes and Culinary Production, promoted a unified Soviet cuisine with standardized and nutritionally "rational" versions of traditional dishes. The same cooking techniques and recipes were taught in culinary vocational schools throughout the country, establishing a common cooking style in Soviet cafés and restaurants. Though inspired by the cuisines of the country's various ethnic groups, many recipes were presented as part of an overall Soviet heritage, disassociated from their individual geographic origins. +By many people both inside and outside the Soviet Union, borscht was increasingly seen not as an ethnic Ukrainian soup, but as a Soviet or—metonymically—Russian dish. This approach was criticized by William Pokhlyobkin, a Russian food writer, who unequivocally described beet-based borscht as one of the "dishes of Ukrainian cookery" which "have entered the menu of international cuisine". "One could understand", he wrote, "and forgive foreigners for calling borscht or varenyky Russian national dishes, but when it turns out that they gleaned this information from Soviet cookbooks or from restaurant menus, one becomes embarrassed for our authors and chefs, who popularize the national cuisines of our peoples [that is, the ethnic groups of the Soviet Union] with such ignorance." +Three grand soups – Classification of soups in Japan +Shchi – Russian-style cabbage soup, some variants of the dish may contain beets +Cabbage soup – Soup dish, kapusniak/kapustnica variants of cabbage soup are made sour +Borscht Belt – Cultural region of New York, US – a popular vacation spot for New York City Jews from the 1920s through the 1960s +Edward Rudolph Bradley Jr. (June 22, 1941 – November 9, 2006) was an American broadcast journalist and news anchor who is best known for reporting with 60 Minutes and CBS News. +After graduating from Cheyney State College, Bradley became a teacher and part-time radio disc jockey and reporter in Philadelphia, where his first major story was covering the 1964 Philadelphia race riot. He moved to New York City in 1967 and worked for WCBS as a radio news reporter. Four years later, Bradley moved to Paris, France, where he covered the Paris Peace Accords as a stringer for CBS News. In 1972, he transferred to Vietnam and covered the Vietnam War and the Cambodian Civil War, coverage for which he won Alfred I. duPont and George Polk awards. Bradley moved to Washington, D.C. following the wars and covered Jimmy Carter's first presidential campaign. He became CBS News' first African American White House correspondent, holding the position from 1976 to 1978. During this time, Bradley also anchored the Sunday night broadcast of the CBS Evening News, a position he held until 1981. +In 1981, Bradley joined 60 Minutes. While working for CBS News and 60 Minutes, he reported on approximately 500 stories and won numerous Peabody and Emmy awards for his work. He covered a wide range of topics, including the rescue of Vietnamese refugees, segregation in the United States, the AIDS epidemic in Africa, and sexual abuse within the Catholic Church. Bradley died in 2006 of leukemia. +Edward Rudolph Bradley Jr. was born on June 22, 1941, in Philadelphia, Pennsylvania. His parents divorced when he was young and he was raised in a poor household by his mother, Gladys Gaston Bradley, and spent summers with his father, Edward Sr., in Detroit. Bradley attended high school at Mount Saint Charles Academy in Rhode Island and Saint Thomas More Catholic Boys School in Philadelphia, graduating from the latter in 1959. He received a Bachelor of Science degree in education from Cheyney State College in 1964. While at Cheyney State, Bradley played offensive tackle for the school's football team. +Bradley began his career as a math teacher in Philadelphia in 1964. While working as a teacher, he also worked at WDAS as a disc jockey. While working for WDAS, Bradley covered the 1964 Philadelphia race riot and interviewed Martin Luther King Jr. Those experiences led him to pursue a career as a journalist, with Bradley later saying, "I knew that God put me on this Earth to be on the radio." Bradley moved to New York City in 1967, working for WCBS. While there, Bradley found he was primarily assigned stories most relevant to African American listeners. After confronting his editor about those assignments, Bradley received assignments on a broader array of topics. Bradley left WCBS in 1971. +Bradley moved to Paris, France, in 1971. He was fluent in French, and while there was hired by CBS News as a stringer. He transferred to Saigon in 1972 to report on the Vietnam War and Cambodian Civil War, as well as reporting on the Paris Peace Accords. While in Cambodia, Bradley was wounded by a mortar round. He transferred to CBS's Washington bureau in 1974, returning to Asia the following year to continue reporting on both wars. Bradley was one of the last American journalists to be evacuated in 1975 during the Fall of Saigon. He was awarded Alfred I. duPont and George Polk awards for his coverage in Vietnam and Cambodia. +In 1976, Bradley was assigned to cover Jimmy Carter's presidential campaign, as well as the Republican and Democratic national conventions, covering them until 1996. Following Carter's victory, Bradley became CBS's first African American White House correspondent, a position he held from 1976 to 1978. Bradley disliked the position as it tied him to the movements of the president. Also in 1976, Bradley began anchoring the Sunday night broadcasts of the CBS Evening News, holding the post until 1981. In 1978, he became one of the principal correspondents for the documentary program CBS Reports, also leaving in 1981. +Bradley won the first of 20 News and Documentary Emmy Awards in his career for his 1979 documentary "The Boat People", reporting on Vietnamese refugees escaping the country via boat or ship, at one point wading into the water to assist in the rescue of the refugees. "The Boat People" also earned Bradley an Edward Murrow Award, a duPont citation, and a commendation from the British Academy of Film and Television Arts. The same year, another Bradley documentary, "Blacks in America: With All Deliberate Speed?", aired. The documentary detailed segregation in the United States and how the treatment of African Americans in the U.S. had changed since Brown v. Board of Education. The two-hour program also won duPont and Emmy awards. +Following Dan Rather's move to the CBS Evening News, Bradley joined the news magazine program 60 Minutes. According to producer Don Hewitt, Bradley's "calm, cool, and collected" reporting style was the right fit for the program. His interview style has drawn comparisons to television detective Columbo and been described as "disarming", "confident", and "streetwise". He was noted for his ability to get interview subjects to divulge information on camera with his body language. In his first decade on 60 Minutes, Bradley reported numerous high-profile stories on a variety of topics, including with Lena Horne, convicted criminal and author Jack Henry Abbott, and on schizophrenia. He won Emmys for all three stories. +In 1986, Bradley interviewed singer Liza Minnelli and expressed interest in wearing an earring. Minnelli gave him a diamond stud after the interview, which Bradley began wearing on air. He was the first male reporter to consistently wear an earring on air, "challenging the notions of journalistic propriety", according to Robb Report writer Kristopher Fraser. He became known for bucking fashion trends for newscasters. His iconic style included an array of patterns, a short beard, and the earring worn in his left ear. Mike Wallace said after Bradley's death that he thought Bradley's decision to wear an earring inspired others to do the same. +Bradley repeatedly turned down offers to anchor the CBS Evening News in the late 1980s, preferring instead to continue working on 60 Minutes. His reporting in the 1990s included such topics as Chinese forced labor camps, Russian military installations, and the effects of nuclear weapons testing near Semey, Kazakhstan. He also profiled numerous people, including Thomas Quasthoff, Muhammad Ali, and Mike Tyson. He won a series of awards for his reporting that decade, including Emmys, duPont citations, and a Peabody Award. Bradley also anchored CBS's Street Stories from 1992 to 1993. In 1995, he was awarded the grand prize Robert F. Kennedy Journalism Award for the CBS Reports documentary "In the Killing Fields of America". +Throughout the 2000s until his death in 2006, Bradley continued to cover a variety of topics, including the AIDS epidemic in Africa, sexual abuse within the Catholic Church, and the 1955 murder of Emmett Till. He also interviewed high-profile people, such as Bob Dylan and Neil Armstrong, and conducted the only television interview with Timothy McVeigh. Bradley reported approximately 500 stories for 60 Minutes over his 25-year tenure with the program, more than any other correspondent over the same time period. In 2005, Bradley was awarded a Lifetime Achievement Award by the National Association of Black Journalists. +Bradley's reporting was not without criticism. Washington Post columnist Brock Yates questioned the completeness of Bradley's 1986 report on acceleration systems failures with the Audi 5000 sedan and why Audi engineers reportedly could not reproduce the problem. In 1989, Bradley reported on daminozide, a chemical used on apples, as well as seven pesticides used on the fruit. His report called daminozide a carcinogen particularly dangerous to children and sparked a national panic. Scientists with the American Society of Toxiocology noted a lack of scientific evidence in Bradley's report and the United States Environmental Protection Agency and Food and Drug Administration issued a joint statement 18 days after Bradley's story aired, declaring apples safe to eat. A trade group of apple growers from Washington sued 60 Minutes after the story aired, but had their claims dismissed after the United States Supreme Court upheld an appeals court ruling that the association failed to disprove Bradley's story. His coverage of Kathleen Willey, who accused Bill Clinton of sexual misconduct in the 1990s, drew criticism for not pushing Willey in his interview, giving her a disproportionate amount of airtime, and leaving out important information from Clinton's attorney Robert Bennett. The integrity of his December 2003 interview with Michael Jackson was also called into question after CBS refused to air a music special unless Jackson discussed molestation charges with CBS News. Jackson was paid an undisclosed sum for the special by CBS's entertainment division. +Bradley was diagnosed with lymphocytic leukemia in his later years, keeping the illness secret from many, including colleagues such as Wallace. His health rapidly declined after contracting an infection, but Bradley continued to work, saying that he preferred to die "with [his] boots on". Bradley filed 20 stories in his final year with 60 Minutes, conducting his last interviews with members of the Duke University lacrosse team accused of rape weeks before his death. Bradley died at Mount Sinai Hospital in Manhattan on November 9, 2006, at the age of 65. +More than 2,000 people attended Bradley's funeral service at Riverside Church in New York. Among the attendees were the Reverends Al Sharpton and Jesse Jackson, musicians Jimmy Buffett and Wynton Marsalis, journalists Dan Rather, Walter Cronkite, and Charlayne Hunter-Gault, and former U.S. president Bill Clinton. In April 2007, Bradley was honored with a jazz funeral mass and procession at St. Augustine Church during the New Orleans Jazz & Heritage Festival. +Morley Safer described the themes of Bradley's reporting as "justice, justice served and justice denied". Bradley's reporting on the AIDS epidemic in Africa has been credited with convincing drug companies to donate and discount drugs to treat the disease. His reporting on psychiatric hospitals in the U.S. prompted federal investigations into the largest chains, and his reporting on the Duke lacrosse team has been credited with ensuring the accused had a fair trial. Broadcast on October 15, 2006, the 60 Minutes edition that had Bradley's interview with the Duke lacrosse players had nearly 17 million viewers. It was the ninth-most watched show that week and one of the highest rated episodes of the year. Bradley was seen as an inspiration for Black Americans, with columnist Clarence Page writing: +Mr. Bradley challenged the system. He worked hard and prepared himself. He opened himself to the world and dared the world to turn him away. He wanted to be a lot, and he succeeded. Thanks to him, the rest of us know that we can too. +Salim Muwakkil wrote for The Progressive about Bradley's impact on Black journalists, noting that Bradley "proved blacks not only could do the job, but they could do it with panache". Matt Zoller Seitz, writing for Slant Magazine, said Bradley forced audiences and the television news industry to "accept him on his own terms" and that he "annihilate[d] received wisdom about what it meant to be a professional journalist, a black man and an American". +In 1994, Bradley and the Radio Television News Directors Association Foundation started a scholarship program in his name for journalists of color. It awards $10,000 annually. In 2007, he was inducted into the Broadcast Pioneers of Philadelphia Hall of Fame. +Bradley was named one of the "100 Outstanding American Journalists in the last 100 years" in 2012 by faculty at New York University. In 2015, the Pennsylvania General Assembly renamed City Avenue in Philadelphia "Ed Bradley Way". A mural of Bradley was completed in the city in 2018, and a historical marker was installed in 2021. +Bradley was fond of jazz and hosted Jazz at Lincoln Center on National Public Radio. He performed with Jimmy Buffett and the Neville Brothers and was referred to as "the fifth Neville brother" by the group. He was an outdoorsman, and often hiked or skied in his free time. +Bradley married three times, to Diane Jefferson, Priscilla Coolidge, and Patricia Blanchet. He split his time between homes in New York and Colorado. +History of African Americans in Philadelphia +The Interviews: An Oral History of Television +List of Cheyney University of Pennsylvania alumni +List of people from Philadelphia +National Association of Black Journalists Hall of Fame +Ed Bradley at IMDb +Appearances on C-SPAN +Ed Bradley at The Interviews: An Oral History of Television +"All-American Bitch" (stylized in all lowercase) is a song by American singer-songwriter Olivia Rodrigo, released in 2023 from her second studio album, Guts. Rodrigo wrote the song with its producer, Dan Nigro. It became available as the album's first track on September 8, 2023, when the album was released by Geffen Records. "All-American Bitch" is a pop-punk and rock track begins with a folk and folk-pop verse and transitions into pop-punk during the chorus, incorporating influences of punk rock, grunge, and pop rock. Lyrically, it is a satirical song that explores Rodrigo's concerns about society's double standards and contradictory expectations for women. +Music critics believe "All-American Bitch" was a successful opening track and appealed to Generation Z. They described Rodrigo's vocal performance in the verses as angelic and compared the song to the work of other rock artists. In the United States, "All-American Bitch" debuted at number 13 on the Billboard Hot 100. The song reached the top 10 in Australia, Ireland, and New Zealand, receiving a double platinum certification in Brazil, platinum certifications in Australia and Canada, and a gold certification in New Zealand. +"All-American Bitch" was promoted with a rehearsal video and a Vevo Live performance in an abandoned theater. Rodrigo also performed the song on Saturday Night Live, where she stabbed a red-colored cake at a tea party and splattered a white-colored cake on her face; the performance received positive reviews from critics. She included the song on the set list of her 2024–2025 concert tour, the Guts World Tour. +Olivia Rodrigo's debut studio album, Sour (2021), was released in May 2021, following which she took a break from songwriting for six months. She conceived the follow-up album, Guts (2023), at the age of 19, while experiencing "lots of confusion, mistakes, awkwardness & good old fashioned teen angst". Sour's producer, Dan Nigro, returned to produce every track on it. Rodrigo listened to one of her favorite bands, Rage Against the Machine, on the way to studio sessions. She wrote over 100 songs with Nigro, and included the more rock-oriented tracks on the album because they drew a bigger reaction from Rodrigo's audiences during live shows. +Rodrigo and Nigro had completed two disparate sets of songs for Guts, some that were fun and uptempo and others that were serious. Nigro wondered how to "bridge this together", worried that the two halves sounded too different from each other. Rodrigo conceived the idea for a song titled "All-American Bitch" while reading the titular essay in Joan Didion's 1968 book Slouching Towards Bethlehem, wherein one of the runaway hippies describes his mother as an "all-American bitch", later reflecting: "I was like, 'Wow, that's so cool.' It's such a provocative set of words." After the song was written on a piano, Nigro was thrilled and envisioned it as the album's opening track. It was turned into a rock song with a live band. Rodrigo lost her voice after screaming for 15 minutes in Nigro's garage, which he layered to create what he described as a "choir of screams" for the song. She viewed it as one of her best-written songs and believed it captured repressed feelings she had wanted to express since the age of 15. Rodrigo had always experienced emotional turmoil over her rage and dissatisfaction, which she could not express as she felt pressured to portray gratitude, and she struggled wanting to be the "perfect American girl" but not always feeling that way. +Rodrigo announced the album title on June 26, 2023, and its lead single, "Vampire", was released four days later. On August 1, 2023, she revealed Guts' tracklist, which featured "All-American Bitch" as the first track. The song became available for digital download on the album, which was released on September 8, 2023. A rehearsal video accompanied its release, in which Rodrigo sports a shirt with a picture of Fiona Apple. +"All-American Bitch" is two minutes and 45 seconds long. It was recorded at Amusement Studios and East West Studios in Los Angeles. Nigro engineered the song with Sam Stewart, Sterling Laws, Dave Schiffman, and Jasmine Chen and provided production and vocal production. He played acoustic guitar, electric guitar, percussion, bass, synthesizer, and drum programming; Stewart played acoustic guitar and electric guitar; Laws played drums; and Ryan Linvill played bass. Spike Stent mixed the song at SLS Studios in London, and Randy Merrill mastered it. +"All-American Bitch" is a pop-punk and rock song that begins with a folk and folk-pop verse, and transitions into pop-punk during the chorus, incorporating influences of punk rock, grunge, and pop rock. Spin's Ilana Kaplan believed the song combines Hole's grunge sound with the pop-punk essence of Avril Lavigne and the fictional band Pink Slip from the 2003 movie Freaky Friday. It begins with gently strummed acoustic guitars and Rodrigo delivers falsetto vocals with a soft coo. She assumes a gentle tone in the verses and a furious one in the choruses. A full band plays electric guitars and drums in the chorus as Rodrigo's delivery gets more aggressive. She screams loudly after singing the song's lyric about "scream[ing] inside". Hannah Dailey of Billboard believed it shifts from "ethereal choral moments to screamed grungey punk sections", and Heather Phares of AllMusic compared the transition to the work of Phoebe Bridgers and Courtney Love. Rodrigo credited the transition to the influence of Rage Against the Machine, which American Songwriter's Patrick Hosken also observed. Writing for The Forty-Five, Rhian Daly thought Rodrigo's delivery went from "pretty to blood-curdling", and Pitchfork's Cat Zhang likened her jeering tone to Lavigne and the All-American Rejects. The song's production received comparisons to Bikini Kill, Sleater-Kinney, and Paramore. +"All-American Bitch" is a satirical song, in which Rodrigo expresses concerns about society's difficult expectations for women by sarcastically describing herself as someone who satisfies them. She uses pop culture references to suggest that the expectations are contradictory in nature, as women are expected to be alluring yet innocent, selfless yet ambitious, and also constantly grateful: "I am light as a feather, I'm as stiff as a board." Among other inconsistencies, Rodrigo perceives that they are expected to allow crude jokes to pass unchallenged, be empathetic but never to the point of making others uncomfortable, never show their anger, and internalize their emotions. In the chorus, Rodrigo announces that she is aware of her age and acts accordingly, reflecting a common motif in most of her music. She references the Kennedy family while declaring her desire to meet ideals: "I got class and integrity, just like a goddamn Kennedy". Some lyrics focus specifically on Rodrigo's experiences as a famous woman and how prominent women face pressure to not appear too emotional. The song closes out with Rodrigo alluding to her award acceptance speeches in the final lines. Writing for The Guardian, Laura Snapes described the song as "a satirical diatribe against the expectations and double standards she still feels bound by", and PopMatters' Jeffrey Davies called it a 2023 version of Meredith Brooks's 1997 single "Bitch". +Music critics believed "All-American Bitch" fulfilled its job as the opening track of Guts. Elle's Erica Gonzales thought it was a "killer opener", and The Australian's Jules LeFevre opined the song was an ideal way to begin the album. Rob Sheffield of Rolling Stone thought it started Guts off with "a fantastic pop-punk angst rant", and Sowing of Sputnikmusic believed that the album was irrefutably successful and this began with the song. Writing for Billboard, Jason Lipshutz ranked "All-American Bitch" as Guts' second best track and thought it proved that Rodrigo had a talent for "genre refraction". He, Sheffield, and The New York Times' Lindsay Zoladz each drew parallels between the song and Sour's opener "Brutal". +Reviewers praised Rodrigo's vocal performance and the production of "All-American Bitch". Kaplan and The A.V. Club's Mary Kate Carr opined that the song would be suitable to soundtrack the 1999 film 10 Things I Hate About You. Punch Liwanag of the Manila Bulletin believed the production and Rodrigo's vocals on "All-American Bitch" came across as effortless, and Gabriel Saulog of Billboard Philippines thought the guitar riff flawlessly captured her turmoil and inner struggles. Gonzales opined the drums in the chorus of "All-American Bitch" hit "like a tantrum" and the melody oscillated from "sweet to sour" and "angelic to chaotic". Several critics described Rodrigo's vocals in the verses as angelic, and MusicOMH's John Murphy believed her screams in the latter half were attention-grabbing and could compete with Black Francis. +Critics were also positive about the lyrical themes of "All-American Bitch" and believed it captured feelings experienced by Generation Z. Beats Per Minute's Lucas Martins thought the song combined youthful drama and introspection to create an earnest expression of frustration, which embodied Rodrigo's generation and resonated perfectly with the zeitgeist. Chris Willman of Variety opined that it tapped into a significant aspect of Rodrigo's appeal, embracing the essence of adolescence without attempting to sound "too seasoned" prematurely. The Line of Best Fit's Matthew Kim viewed "All-American Bitch" as a compelling work of social commentary, and Lipshutz stated his respect for the lyrics about Rodrigo's award acceptance speeches. Writing for the Chicago Tribune, Bob Gendron called the song a potential "generational anthem". +Reviewers highlighted specific lyrics and some included "All-American Bitch" on best-of-2023 lists. Sheffield ranked it as Rodrigo's fourth-best song in September 2023, describing it as a display of her attitude and the "perfect theme song" for Guts. He picked his favorite lyric: "I know my age and I act like it/I got what you can't resist/I'm a perfect all-American bitch." GQ placed the Kennedy reference in its list of the album's standout and "gutsiest" lyrics, and Nylon included it alongside the titular lines in its list of Guts' impeccable lyrics. "All-American Bitch" was placed on critical lists of the best songs of 2023 at number five by BBC News and in the top twenty by Esquire. The latter magazine's Bria McNeal believed that Rodrigo traipsed through an age-old story, but she transformed it into something completely modern and displayed "perfect, unfiltered rage against the machine". +"All-American Bitch" debuted at number 13 on the US Billboard Hot 100 issued for September 23, 2023. In Canada, the song entered at number 15 on the Canadian Hot 100 issued for the same date and was certified double platinum by Music Canada. It debuted at number 78 on the UK Singles Chart. "All-American Bitch" received a gold certification in the United Kingdom from the British Phonographic Industry, and the Official Charts Company declared it her 17th-biggest song in the country in February 2024. +In Australia, "All-American Bitch" entered at number 10 and became Rodrigo's eighth top-10 song. The song was certified platinum by the Australian Recording Industry Association (ARIA) for selling 70,000 equivalent units. It debuted at number seven in New Zealand and became her 10th top-10 song. "All-American Bitch" received a gold certification from Recorded Music NZ. The song charted at number nine on the Billboard Global 200. Elsewhere, it reached national record charts at number 7 on the Sweden Heatseeker chart, number 8 in Ireland, number 22 in Singapore, number 29 in Greece, number 34 in Portugal, number 96 in Poland, and number 171 in France. "All-American Bitch" received a double platinum certification in Brazil. +Rodrigo performed "All-American Bitch" at the Ace Hotel Los Angeles on October 9, 2023, in an exclusive concert for American Express cardholders. On December 1, 2023, she reprised the song at the KIIS-FM Jingle Ball. +Rodrigo sang "All-American Bitch" on Saturday Night Live on December 9. The performance began with her innocently sitting at a tea party in a high-neck pink dress, drinking tea behind a table covered with cakes and other items and humming soft vocals. During the chorus, Rodrigo jumped up and laid down on the table, stabbing one cake with a knife and dancing on the others, as the lights changed from white to quick red flashes to reflect the change in her mood. As the song approached a climax, she screamed while splattering red-colored cake on her face. Rodrigo concluded by sticking her tongue out for the camera. The performance received positive reviews, with critics describing it as stunning, stellar, scorching, and memorable. Exclaim!'s Vish Khanna called it "a blast of theatrical mall punk" and believed the props were used creatively and Rodrigo was mesmerizing. The bakery that provided 20 cakes for the performance received attention on social media. On December 13, the singer Noelle Denton alleged that the concept was "stolen" from the music video for her 2021 song "Your Mom Calls Me" and the creative director might have seen her video when it was shared by a mutual friend. +On December 15, Rodrigo uploaded a Vevo Live performance of "All-American Bitch", in which she performed in an abandoned theater in a shiny blue dress. The song was included on the set list of her 2024–2025 concert tour, the Guts World Tour, as part of a set of rock-oriented songs, alongside "Brutal", "Obsessed" (2024), "Good 4 U" (2021), and "Get Him Back!" (2023). The set recalls 1990s rock artists like Alanis Morissette and Gwen Stefani according to Chicago Sun-Times' Selena Fragassi. Rodrigo performed it in a red bodysuit while red talking lips, pictures, and news stories symbolizing misogyny and unattainable beauty ideals were projected on a screen behind her. She replaced the word "hips" and sang the original lyric: "Perfect all-american tits". Midway, she asked the crowd to think of a person or thing that really upset them and scream, to which the audience obliged; Rania Aniftos of Billboard described this as "a hilarious and healing moment of emotion" and included it among the show's five best moments, with Carly May Gravley of Dallas Observer commented that it led to an intense emotional release and left everyone's hearing permanently damaged. Rodrigo reprised "All-American Bitch" during her Lollapalooza Chile set in March 2025. The song appeared in the trailer for the Apple TV+ series The Buccaneers (2023). +In November 2025, the song was featured in a joint Instagram video by the White House and the Department of Homeland Security that showed Immigration and Customs Enforcement officers detaining and deporting people to promote self-deportation as an alternative. Rodrigo commented on the post, "don't ever use my songs to promote your racist, hateful propaganda." +Credits are adapted from the liner notes of Guts. +Dan Nigro – producer, songwriter, engineer, acoustic guitar, electric guitar, percussion, vocal producer, bass, synthesizer, drum programming, background vocals +Olivia Rodrigo – vocals, background vocals, songwriter +Sam Stewart – engineer, acoustic guitar, electric guitar +Sterling Laws – drums, engineer +Dave Schiffman – engineer +Jasmine Chen – engineer +Ryan Linvill – bass +Randy Merrill – mastering +Spike Stent – mixing +During the 2009–10 English football season, Notts County competed in Football League Two, the fourth tier of the English football league system. Shortly before the season began, the club was subject to a high-profile takeover by Munto Finance, purportedly a wealthy Middle East-based consortium with ambitions to take the club to the Premier League. The former England manager Sven-Göran Eriksson was appointed director of football, and lavish spending began in apparent early efforts to achieve these ambitions. In reality, Munto Finance was controlled by the convicted fraudster Russell King, and the club had been acquired as part of an elaborate scheme to list a fake mining company on the stock exchange. The promised money did not exist, King fled when the scheme collapsed and Notts County were left deeply in debt. Eriksson resigned following a further takeover by Ray Trew, who prevented bankruptcy and oversaw a successful conclusion to the season, with the team winning the League Two championship and promotion to Football League One. The team also fared well in the FA Cup, reaching the last sixteen of the competition. +The season saw four different owners, three permanent first-team managers and two spells of interim management. In total, the team played 54 competitive matches, winning 31, drawing 14 and losing nine. Lee Hughes scored 33 goals across all competitions, becoming the first Notts County player to score 30 goals in a season since Tommy Lawton sixty years earlier, while Ben Davies made the most appearances, featuring in 51 games. Munto Finance's takeover and the wider scheme of which it formed a part were investigated by journalists from the beginning. It was the subject of a 2011 episode of the BBC One show Panorama, a BBC Sounds podcast series in 2022, and a documentary for Sky Documentaries in 2025. Notts County continued to experience off-field problems over the following years, culminating in a four-year spell in non-League football between 2019 and 2023; coverage of the club's plight and subsequent attempts to recover often refer to the events of 2009. +In the 2009–10 season, Notts County (nicknamed the Magpies) competed in League Two, the fourth tier of the English football league system, their sixth successive year at this level. The club's time in the fourth tier had been preceded by a serious financial crisis. Between 2002 and 2003, Notts County had spent a record 534 days in administration. Bankruptcy was prevented in a takeover largely financed by supporter Haydn Green; as part of the deal, the Notts County Supporters' Trust also gained representation on the board of directors. In 2007, shortly before his death, Green sold his stake in the club to the Trust for £75,000, making the latter the majority shareholder. The Trust were not required to pay Green upfront; instead, the £75,000 would only be paid to Green (or his estate after his death) if the Trust sold their shares in the future. +The years after exiting administration saw the team consistently struggle. It was relegated to League Two in 2004, and was in danger of relegation to non-League football before its final game of the 2005–06 season. In 2007–08, League Two safety was assured only in a 1–0 win over Wycombe Wanderers in the penultimate game of the season, while the following season the team finished 19th. The sustained poor on-field performance resulted in supporters becoming progressively more unhappy under Trust ownership. There were rumours that the club had taken on large loans from unknown individuals in response to financial problems, and the efforts of supporters to find out more were resisted by officials. In April 2009, John Armstrong-Holmes (the club's chairman) and Eric Kerry, the Trust's representatives on the club's board, survived a vote of no confidence, with Armstrong-Holmes arguing that his and Kerry's removal would mean "potential investment may be jeopardised". +The Munto Finance takeover of Notts County was orchestrated by Russell King, who had first been convicted of fraud in 1991 after falsely reporting his Aston Martin had been stolen to claim £600,000 in insurance. Before his involvement in Notts County, King had stolen £671,000 from Belgravia Financial Services Group, and had acquired a 49% stake in the investment bank First London by falsely claiming he was managing billions of dollars for the Bahraini royal family. The others involved were Nathan Willett and his father Peter, of whom little is known; in 2022, the makers of a BBC podcast series about the takeover were unable to locate even a forwarding address for the pair. Once the takeover was completed, Peter Trembling acted as executive chairman of Notts County; Matt Scott of The Guardian would later describe him as Munto Finance's "mouthpiece". Following the takeover's collapse, Trembling maintained that he had been deceived, and that he had lost much of his own money as a result of the affair. +While preparing for the 2009–10 season, Notts County were approached by Trembling, who indicated that Munto Finance was interested in buying the club. At a meeting, Trembling told Armstrong-Holmes that Munto Finance were a Middle East-based consortium, and that unspecified royalty was involved. A further meeting took place in Bahrain between Armstrong-Holmes and Roy Parker (a shareholder in the club and its vice chairman) and representatives of the consortium. According to Armstrong-Holmes, King was in attendance, but played no part in the talks until Armstrong-Holmes demanded a bank guarantee as a condition of a sale, which King soon produced. Unbeknown to Notts County's representatives, the guarantee, which appeared to be from First London, was worthless. It had not been approved by the bank's board and came from a part of the bank that no longer existed. A deal was agreed, with the takeover announced in July 2009. Trembling was appointed as chief executive, and Munto Finance announced its ambition to see Notts County promoted to the Championship within five years. +As part of the deal, Munto Finance insisted that it be given ownership of the club for nothing, and the Trust pushed its members to approve the "gift" of the Trust's shares to the consortium. As this would not be a sale, no money would need to be paid to the estate of Haydn Green. Some supporters protested, questioning whether paying Green's estate would not be the "honourable thing" to do, but Armstrong-Holmes urged Trust members to back the deal, saying that "Haydn Green's position in the club's history is acknowledged and will be honoured by Munto". Trust members voted overwhelmingly in favour of the transfer of the shares, and Munto Finance consequently acquired the club for free. Chris Porter, who discussed the takeover in a study of fan ownership of football clubs, says that the sale "amounted to an up-front outlay from the would-be recipient of promised future riches", and that the Trust failed to ask why supposedly wealthy investors were unwilling to make a relatively modest initial outlay to buy the club. The identity of those who would be financing the club was shrouded in secrecy, but the journalist and Notts County supporter Simon Goodley was soon tipped off about the involvement of King, and he began to investigate. Also soon investigating was Matt Scott, who was encouraged to do so by a source after noting in his column that the takeover had not yet been approved. +Notts County had already begun signing players for the 2009–10 season before the takeover, with these players either being from League Two or non-League clubs or joining on loan. Midfielders Neal Bishop, Ben Davies and Ricky Ravenhill joined from Barnet, Shrewsbury Town and Darlington respectively, while Kettering Town striker Craig Westcarr had joined on a one-year deal. Also signing pre-takeover were defenders Graeme Lee, who joined the club for an undisclosed fee from Bradford City, and Brendan Maloney, who joined on a six-month loan from Nottingham Forest. In the week after Munto Finance took control of the club, the Magpies announced the signing of two strikers, Lee Hughes and Luke Rodgers, on free transfers from Oldham Athletic and Yeovil Town respectively. +On 22 July, Notts announced the appointment of the former England manager Sven-Göran Eriksson as director of football in what Stuart James of The Guardian described as "one of the game's most staggering moves of recent times". Despite Munto Finance's initially stated target of seeing Notts County promoted to the Championship, Eriksson said he wanted to see the club promoted to the Premier League. Eriksson said he was "particularly attracted to this role and the unique opportunity to help build a club over the longer term". With Eriksson's appointment, Notts began to target, or were linked in the press with, much more high-profile players. At one point, reports suggested the Brazil international and World Cup winner Roberto Carlos might join the club. On another occasion, David Beckham was asked by a radio journalist about the possibility of working with his old England manager. The club's final pre-season signing was that of striker Karl Hawley from Preston North End for an undisclosed fee. +To prepare for the new season, the Magpies played a series of friendlies. The first match after Eriksson's arrival, the game against Nottingham Forest at Notts County's Meadow Lane stadium, attracted nearly 13,000 spectators, and saw Eriksson "paraded" before the crowd prior to kick off. +Key +Results +Notts started the season strongly. Hughes scored a hat-trick on the opening day as the Magpies defeated Bradford 5–0 at home with over 9,000 spectators in attendance. The team won 4–0 at Macclesfield Town in its first away game, but then suffered its first defeat, losing 2–1 at Chesterfield. During August, the club used its apparent new-found wealth to strengthen its squad. The signing of goalkeeper Kasper Schmeichel from Manchester City for an undisclosed fee believed to surpass the club record was announced on 14 August. That of Johnnie Jackson from Colchester United (also for an undisclosed fee) followed two days later. Schmeichel made his debut in a 3–0 home win over Dagenham & Redbridge on 22 August. Three days later, Notts announced the signing of England international Sol Campbell on a five-year contract: "there were offers from the Premier League but that would have been short-term. I'm thinking of the future", he told the BBC. In their final match of the month, Notts were beaten 1–0 at Barnet. +Early in September, Notts County strengthened their squad further, bringing in winger Matt Ritchie on loan from Portsmouth until the end of December. The month began with two home games, a 1–1 draw with Burton Albion and a 5–2 win over Northampton Town in which Hughes scored his second hat-trick of the season. Campbell made his debut the following week at Morecambe, but he could not prevent the Magpies from losing 2–1. It proved to be Campbell's only game for the club, as he left by mutual consent four days later. He had grown suspicious that the club's supposed wealth was not real; defender Mike Edwards later said that, shortly before leaving, Campbell had warned his teammates of his doubts. Despite losing Campbell, Notts ended the month with two victories, a 3–1 home win over Port Vale and a 3–0 win at Lincoln City in which Rodgers scored a hat-trick. +On 21 September, the Football League confirmed they were investigating the takeover. It had still not been approved by the league, who were demanding to know the identity of the new owners. The same week, King's involvement in the takeover was made public by Goodley in The Sun, though King maintained he had acted as an advisor on only "strategic and media issues" and had relinquished his role after Notts County had appointed a new communications manager. On 27 September, in an effort to satisfy the league as to the identity of the new owners, Notts named two Middle East-based families as among the club's new investors, a move soon undermined when a member of one of the families denied involvement. Scott also questioned whether King really had ceased to be involved, noting the latter's continued close connections with the companies and individuals linked with Munto Finance. +October began with two draws, the first 1–1 at Cheltenham Town, the second 2–2 at home to Torquay United, a result which left the Magpies in fifth place, four points from top spot. The day after the latter, Ian McParland was sacked as manager, replaced on an interim basis by Dave Kevan and Michael Johnson. Kevan and Johnson ultimately took charge for two matches, a 0–0 draw at Rotherham United and a 2–0 home win over Crewe Alexandra. In the meantime, on 20 October, the Football League approved the takeover, accepting Notts County's new owners as fit-and proper persons despite still not knowing who they were. With the takeover now sanctioned, Notts moved to appoint their new manager. Eriksson and Trembling held talks with Roberto Mancini, who was keen on the job, but Eriksson was by now growing concerned about the lack of investment being made in the club, and warned Mancini to wait. Ultimately, Hans Backe was appointed instead. Backe took charge of Notts County's final match of October, a 1–1 draw at home to Shrewsbury Town. +In early November it was revealed that Notts County's parent company was subject to a winding-up petition due to £400,000 of unpaid debt. This was despite assurances to the Notts County Supporters' Trust, made when the takeover was agreed, that the debts would be settled quickly. The club narrowly avoided a court hearing on the issue on 12 November, two days before the team's first league match of November, a 3–3 draw at Bury. This match saw an unusual first-half incident, when Daniel Nardiello of Bury beat Schmeichel and fired the ball towards an empty net, only to see the ball become stuck in mud before it had crossed the goal line, allowing Schmeichel enough time to run back and gather. After a 0–0 home draw with Aldershot Town, Notts travelled to Rochdale in their final league match of the month, where they fell to a 2–1 loss. +Shortly after the Rochdale match, it was revealed that the Football League had renewed its investigation into Notts County's ownership. The club's off-field position was by now deteriorating rapidly, with King seemingly having vanished, and with creditors visiting the club frequently to demand payment. Despite the mounting problems, the Magpies won their first two games of December, 4–0 at home to Darlington and 2–0 at Hereford United. In the days following the latter, it was reported that Trembling was planning a management buyout of the club, and that Eriksson was on the verge of resigning. Armstrong-Holmes also told The Guardian that he now felt he had been "hoodwinked" into handing the club over to Munto Finance. Trembling completed his takeover on 12 December, promising to look for fresh investment for the club. The same day, Notts were beaten 2–1 at home by Accrington Stanley. This proved to be Backe's final game; he resigned shortly afterwards, with Kevan resuming temporary charge. In their final match of the month, the Magpies won 4–1 at Burton with Hughes scoring a hat-trick. +Notts County's financial problems continued in the new year. Early in January, the club was served with a new winding-up petition issued by HMRC. The situation was not helped by the unusually cold winter forcing the postponement of matches and denying the club much-needed income. Trembling claimed Notts needed investment of £25 million to retain Eriksson and sustain the ambition to reach the Premier League, but the required investment had not been found by the week the club was due in court in connection with the winding-up petition. The day before the hearing, the Magpies played their first league match of the month, a 3–0 win at Dagenham. On 27 January came news that the court had been granted a 28-day extension to the club to settle its debt with HMRC, though the latter requested that this be marked as final; no further extensions would be forthcoming. The team played its final match of the month on 30 January, defeating Barnet 2–0 at home. +In their first match of February, the Magpies won 1–0 at Grimsby Town. A 2–1 defeat at AFC Bournemouth followed in the next game, before the team played Grimsby again, this time a 1–1 draw at Meadow Lane. On 11 February, news came that Ray Trew had agreed to buy the club. In a statement, Trembling said Trew was "not of the £25m+ ilk", but did have sufficient funds to prevent the Magpies from going bankrupt. As a result of the takeover, Trembling and Eriksson both departed their posts, though Eriksson was given the "non-active" role of joint life president of the club. Though Notts had been saved, Trew warned supporters that the club's position remained precarious and administration could not be ruled out. Notts County's first league match following the takeover was a 1–1 draw at Aldershot, a result which left the Magpies in seventh place, fourteen points from top spot. Three days later, on 23 February, Trew announced that administration would not be necessary, and that he had appointed Steve Cotterill as manager for the remainder of the season. Cotterill took charge for the first time in the final game of February, a 5–0 home win over Hereford in which Westcarr scored a hat-trick and Rodgers a brace. +March began with consecutive wins over Macclesfield, Accrington, and Chesterfield; the last of these saw Notts County move up to fourth place, three points behind Bournemouth, their next opponents. A Hughes brace gave Notts 1–0 and 2–1 leads, but an injury-time equaliser for the away team earned Bournemouth a 2–2 draw. Despite the concession of a late goal, the result moved the Magpies into the top three. There then followed two away matches, the first a 1–0 win at Crewe, the second a 0–0 draw with Bradford which extended the team's unbeaten run to nine. Now in second place, Notts hosted third place Rotherham in their final match of the month. The game appeared to be heading for a goalless draw entering the final minute, but a 90th minute goal from Rodgers gave the Magpies a 1–0 win, allowing them to end March in second place, with 68 points from 36 matches, 10 points behind top spot. +April began with a 5–0 home win over Bury, with five different goal scorers for the Magpies. That same afternoon, the league leaders Rochdale were beaten 2–0 at Chesterfield; these results meant Notts had reduced the gap between themselves and first place to seven points, having played a game less. Rochdale were held to a draw in their next game, allowing Notts to move within five points with a 1–0 win at Shrewsbury. Rochdale would then lose their next two matches, and Notts took first place after a 1–0 win at Northampton and a 3–1 home win over Lincoln. Promotion to League One was confirmed on 17 April with a 4–1 home win over Morecambe, before Notts played Rochdale at Meadow Lane on 20 April. The match attracted 10,536 spectators, and was won 1–0 by the home team through a 30th minute goal by Hughes. The victory put the Magpies four points clear of second with a game in hand, and was described by Paul Fletcher of the BBC as "a huge step towards sealing the League Two title". Notts lost their next match at Port Vale, but then clinched the title on 27 April with a 5–0 win at Darlington. The team were presented the trophy after their final home game, a 5–0 win over Cheltenham, before the season concluded with a 0–0 draw at Torquay. +Key +Results +Notts County's FA Cup campaign began in the first round with a home match against fellow League Two club Bradford. Karl Hawley and Johnnie Jackson scored shortly before and after half time, and the Magpies won 2–1. Notts defeated League Two opposition again in the second round, this time coming from 0–1 down to win 2–1 at Bournemouth, and so earned a home tie with Forest Green Rovers of the Football Conference in the third round. As the match was due to be played in the midst of Trembling's efforts to find a new buyer for the club, he had reportedly hoped to use the tie as an opportunity to host potential investors; the match, however, was postponed due to freezing weather conditions. The tie eventually went ahead on 20 January and was won 2–1 by the Magpies, who consequently reached the FA Cup's fourth round for the first time in 15 years. +Notts County's next opponents were Premier League side Wigan Athletic, who visited Meadow Lane on 23 January. A crowd of 9,073 saw the Magpies take a 2–0 first half lead through goals from Hughes and Davies, but Wigan fought back in the second half, and a late goal from Ben Watson meant the match finished 2–2, meaning a replay would be required. With speculation ongoing about the club's future, Davies welcomed the replay as a "chance to get in the headlines for the right reasons". In the replay, a 75th minute goal from Stephen Hunt, followed shortly afterwards by an own goal from Wigan's Gary Caldwell, gave the Magpies a 2–0 win over their Premier League opponents. The victory meant Notts County progressed to the fifth round of the FA Cup for the first time since 1992. Here, the Magpies were drawn away to another Premier League club, this time Fulham. The match, coming shortly after Trew's purchase of the club, resulted in a 4–0 defeat for Notts, who were consequently eliminated from the competition. +Key +Results +As a League Two team, Notts County entered the League Cup in the first round, where they were drawn at home to Doncaster Rovers of the Championship. The Magpies lost 1–0 and were thus eliminated from the competition. +Key +Results +In the first round of the Football League Trophy, a competition for League One and Two teams, Notts County were drawn away to Bradford. The match finished 2–2, requiring a penalty shootout to determine the winner. Bradford won this 3–2, and the Magpies were eliminated from the competition. +Key +Results +Before Trew's takeover, Notts County had been penalised for breaking League Two's salary cap, which required wages exceed no more than 60% of revenue. Though Trew claimed to have rectified this, the club nevertheless spent much of the season under a transfer embargo, unable to sign any further players. A total of 24 players made at least one appearance for Notts County during the 2009–10 season. Of these, Ben Davies played in the most games, featuring in 51 of 54 matches. Eight other players – Neal Bishop, Mike Edwards, Lee Hughes, Ricky Ravenhill, Luke Rodgers, Kasper Schmeichel, John Thompson and Craig Westcarr – made a minimum of 40 appearances, while three players (Sol Campbell, Sean Canham and Nathan Fox) played only once, the latter two's appearances coming as substitutes. +Hughes scored 33 goals across all competitions during the campaign. In doing so, he became the first Notts County player to score 30 goals in a season since Tommy Lawton in 1949–50. Davies (16), Rodgers (13) and Westcarr (11) were the other players to reach a minimum of 10 goals. Goalkeeper Kasper Schmeichel kept 24 clean sheets in his 43 league appearances, setting a new club record for the most in a season. +Key +Numbers in parentheses denote appearances as substitute. +Source: +Investigations of the Munto Finance takeover, and the broader scheme of which it was part, continued after its collapse. One was undertaken by the BBC journalist Peter Marshall, whose investigation was broadcast in an episode of Panorama shown on BBC One in April 2011. A BBC Sounds podcast series produced by the journalist (and Notts County supporter) Ben Robinson in 2022, and a documentary produced for Sky Documentaries in 2025, uncovered further information. These investigations, together with the reporting of Matt Scott, confirmed King's central role. Eriksson gave an interview to Scott shortly after the former's departure from Notts, in which Eriksson said that King had played a pivotal part in convincing him to join the club. Meanwhile, Marshall found evidence of King giving orders and approving finances at Meadow Lane, and Trembling told Marshall that King had been his boss. +The investigations revealed a complicated network of companies. Munto Finance was a subsidiary of Qadbak Investments, which was in turn closely connected to a third company, Swiss Commodity Holding (SCH), the sole directors of which were Nathan and Peter Willett. Eriksson had been promised a shareholding in SCH, and Campbell handed a £33,000 per week "ambassadorial" contract (in addition to his contract with Notts County). The company's logo was even incorporated into Notts County's badge. King was preparing a fraudulent stock market floatation of SCH, which presented itself as a mining company with reserves of $1.9 trillion and the rights to extract the mineral wealth of North Korea (despite lacking the means of doing so). The takeover of Notts County, together with a similar attempted takeover of the BMW Sauber Formula One team, was intended to lend positive publicity to SCH and drive interest in its floatation. Had King succeeded in getting SCH listed, it would have resulted in "a fraud on a colossal scale". +King and Nathan Willett visited North Korea in October 2009, taking with them Eriksson and Abid Hyat Khan, a man introduced to Eriksson as a Bahraini prince, but in fact an alleged fraudster hiding from British police. During the visit, Eriksson was asked by North Korean officials to fix a favourable 2010 FIFA World Cup group for the North Korea team. King and his SCH colleagues held meetings with North Korean officials during the visit, during which he handed out certificates to his hosts for shares in SCH supposedly worth $2 billion. However, the delegation's departure from Pyongyang International Airport was considerably delayed, with the North Koreans apparently having grown suspicious. Eriksson described a tense wait before the flight was allowed to leave, with King and Willett "panicking and sweating". King's scheme effectively collapsed, and he fled to Bahrain. Marshall would later track him down and confront him there, but King closed the door on Marshall without speaking. Nathan Willett also disappeared. +In 2012, while still in Bahrain, King, posing as "Jack Bartholemew", provided a dossier of information to the journalists and authors Anthony Summers and Robyn Swann, purporting to demonstrate North Korea's nuclear capability; King would have received an advance from a book Summers and Swann wrote with the material, but the latter were able to establish his identity. King then launched another fraudulent scheme, this time creating a magazine which he passed off as a Middle East-based edition of the Financial Times to obtain the advertising revenue and barter deals. In 2018, King was extradited from Bahrain to Jersey to stand trial in connection with still another fraud, this one perpetrated in Jersey in 2008. He was convicted in 2019 and sentenced to six years imprisonment. At a later hearing, Jersey authorities ordered that he pay £322,212 in connection with the case or face additional time in prison. Released in 2021, he was confronted again at his home by Alice Levine, the presenter of the BBC Sounds series, in the course of her and Robinson's investigation. He refused to come to the door, and even denied that he was Russell King. +"We expected it to be bad, but probably not this bad", Jim Rodwell, Notts County's new chief executive, said of the club's finances a month after Trew's takeover. At that point, the club was thought to be £6.9 million in debt. Trew later said that the final debts were in fact £7.3 million, including a six figure sum owed to a law firm and £250,000 owed to Campbell for his solitary appearance. The situation would have been worse had Eriksson not agreed to forgo £2.5 million owed to him by the club. There was considerable anger towards Notts County from other League Two clubs, who felt that the Magpies had not been punished sufficiently for breaking the salary cap. Trew dismissed this in an open letter, claiming the club's FA Cup run and his investment had brought it within the salary cap, though he acknowledged this had not been the case before his arrival. Nevertheless, Notts were required to release Schmeichel, reportedly earning £15,000 per week, following the season's conclusion, with Trew explaining that retaining and attempting to gain a transfer fee for the goalkeeper would have been a huge risk for a club with such precarious finances. +Notts County remained in League One for five years until returning to League Two in 2015. Interest in the Munto Finance takeover has continued in the years since, particularly as the club continued to experience off-field problems. In 2016, with the club once again subject to winding up petitions, and Trew looking to sell, Harry Reardon of These Football Times wrote of the "bizarre" takeover, hoping that a proposed sale to overseas investors would "go a little better than last time". Trew sold the club to Alan Hardy, a Nottingham-based businessman, in December 2016, but by 2019 the Magpies were not only once again in the midst of a severe financial crisis but also relegated to non-League football for the first time in their history. Coverage of the club's plight at that time referenced the events of 2009. During the 2022–23 season, some journalists framed the club's efforts to win promotion back to the league as an attempt to recover from the off-field problems of the previous two decades, the 2009 takeover among them. +Infobox kits +Home colours: "Notts Strike Deal With Nike". Notts County F.C. 7 August 2009. Archived from the original on 10 August 2009. Retrieved 30 October 2024. +Away colours: Macaskill, Sandy (23 September 2009). "Sol Campbell leaves Notts County". The Telegraph. Archived from the original on 30 August 2015. Retrieved 30 October 2024. +General +Conn, David (2004). The Beautiful Game? Searching for the Soul of Football. London: Yellow Jersey Press. ISBN 0-224-06435-5. +Porter, Chris (2019). Supporter Ownership in English Football: Class, Culture and Politics. Cham: Palgrave Macmillan. ISBN 978-3030054373. +Robinson, Ben (2024). The Trillion Dollar Conman. London: Icon Books. ISBN 978-183773-142-8. +Warsop, Keith; Brown, Tony (2007). The Definitive Notts County F.C. Nottingham: Tony Brown. ISBN 978-1-899468-99-7. +Specific +Cloud is a 2005 puzzle video game developed by a team of students in the University of Southern California's (USC) Interactive Media Program. The team began development of the game for Microsoft Windows in January 2005 with a US$20,000 grant from the USC Game Innovation Lab; the game was released as a free download that October. By July 2006, the hosting website had received 6 million visits, and the game had been downloaded 600,000 times. +The game centers on a boy who dreams of flying while asleep in a hospital bed. The concept was partially based on lead designer Jenova Chen's childhood; he was often hospitalized for asthma and would daydream while alone in his room. Assuming the role of the boy, the player flies through a dream world and manipulates clouds to solve puzzles. The game was intended to spark emotions in the player that the video game industry usually ignored. +Cloud won the Best Student Philosophy award at the 2006 Slamdance Guerrilla Games Competition, and a Student Showcase award at the 2006 Independent Games Festival. The game was received well by critics, who cited its visuals, music, and relaxing atmosphere as high points. Chen and producer Kellee Santiago went on to co-found the studio Thatgamecompany, which has considered remaking Cloud as a commercial video game. +Cloud, a single-player video game for Microsoft Windows, centers on a boy who dreams of flying through the sky while asleep in a hospital bed. The player assumes control of the sleeping boy's avatar—the projection of the boy into his dream world—and guides him through his dream of a small group of islands with a light gathering of clouds. The avatar's direction and speed are controlled with a mouse; movement is generally on a horizontal plane, but vertical flight can be attained by holding down the third button of the mouse. The player may interact with clouds only while flying horizontally. +The game contains three types of clouds: white clouds, which follow the avatar; gray, neutral clouds, which become white when touched; and black clouds, which may be combined with white clouds to cause rain, dissolving both clouds. A large number of white clouds more easily dissolves a small number of dark clouds than an equal number, and vice versa. White clouds stop following the avatar if the player moves too quickly and they resume following if approached again. Each of the four missions in Cloud has a different objective, including forming patterns in the sky with clouds, eliminating black clouds, and creating rain over each of the islands. +Cloud was designed and released in 2005 by a team of seven students in the Interactive Media master's degree program at the USC School of Cinematic Arts. The game was made as a funded research project in the USC Game Innovation Lab. Development began in January 2005 and the game was released in late October, receiving its final update in December. The group was headed by Jenova Chen and included Stephen Dinehart, Erik Nelson, Aaron Meyers, Glenn Song, composer Vincent Diamante, producer Kellee Santiago and advisor Tracy Fullerton, director of the Game Innovation Lab. The game won the 2005 Game Innovation Grant of $20,000 from the lab, which was intended to support the production of experimental games. +Chen had the first idea for the game; while walking to school one day he looked up at the sky, noticing the difference between the fluffy clouds there and the "polluted and gray" clouds of Shanghai where he was born, and thought about making a game about clouds. It was given a story to "create the premise and help player to be emotionally invested"; however, the team avoided making the story too strong, as it would "distract the player from the core experience" of flying freely and shaping clouds. In the early stages of development, the game had an involved backstory about an alien who attempts to clean up the environment. The group's managing professor, however, challenged them to have a child as the protagonist rather than an alien or mad scientist, to which Chen, who needed the funding to continue at school, quickly agreed. +The story was then cut down to "a simple 'poetic' introduction to the cloud child trapped in a hospital bed". The child was partially based on Chen's childhood experience, as he was often hospitalized for asthma and would daydream while waiting for the doctors. According to Chen, Cloud was designed to "expand the spectrum of emotions video games evoke". The team intended Cloud to "communicate a feeling of youthfulness, freedom, and the wonder of imagination". It was built on a modified version of a game engine developed by several team members for their previous game, Dyadin. At the 2006 Game Developers Conference student showcase, Chen and Santiago pitched Cloud to Sony representative John Hight as the first game in the "Zen" genre. Hight was interested, but Sony declined to publish the game. +Cloud won the Best Student Philosophy award for artistic achievement at the Slamdance Guerrilla Games Competition and a Student Showcase award at the Independent Games Festival. It was featured on Spike TV, G4TV, and CBS Sunday. The game immediately received a great deal of attention when it was released; site traffic overran the server that hosted the website, and then crashed those of the school. By February 2006, just over three months after the game's release, the website had been viewed over one million times and the game downloaded over 300,000 times. By July 2006, it had reached six million visits and 600,000 downloads. +The game received positive reviews from critics. Joel Durham, Jr. of GameSpy claimed that "everything about Cloud is simply jaw-dropping", and cited its music, visuals, and sensation of flying as high points. William Usher of Game Tunnel also praised the visuals and audio: he believed that its graphics created a relaxing atmosphere, and that the "touching musical score" would emotionally move any player. A reviewer for Game Informer said that the game pointed "to a bright future" in which games would inspire a wider range of emotions; however, the writer called Cloud more of an "experience" than a game. Ron White of PC World similarly called it "the most relaxing experience I've ever had that involved a computer". Durham concluded that Cloud "would set your mind free". +Chen and Santiago went on to co-found the video game studio Thatgamecompany. Composer Vincent Diamante and Erik Nelson worked for the company on its second game, Flower. Thatgamecompany considered remaking Cloud as a commercial video game, but said in 2008 that it would only do so if the staff could not conceive any other game ideas. +Official website +"Thatgamecompany | TGC » Cloud". Archived from the original on 16 March 2011. Retrieved 29 July 2009. +Benjamin Géza Affleck (born August 15, 1972) is an American actor and filmmaker. His accolades include two Academy Awards, two BAFTA Awards, and three Golden Globes. Affleck began his career as a child when he starred in the PBS educational series The Voyage of the Mimi (1984–1988). He later appeared in the independent comedy Dazed and Confused (1993) and several Kevin Smith comedies, including Mallrats (1995), Chasing Amy (1997), and Dogma (1999). +Affleck gained wider recognition when he and Matt Damon won the Academy Award for Best Original Screenplay for writing Good Will Hunting (1997), which they also starred in. He established himself as a leading man in studio films, including the disaster film Armageddon (1998), the action crime thriller Reindeer Games (2000), the war drama Pearl Harbor (2001), and the thriller The Sum of All Fears (2002). After a career downturn, Affleck made a comeback by portraying George Reeves in the biopic Hollywoodland (2006), winning the Volpi Cup for Best Actor. +His directorial debut, Gone Baby Gone (2007), which he also co-wrote, was well received. He then directed and starred in the crime drama The Town (2010) and the political thriller Argo (2012), both of which were critical and commercial successes. For the latter, Affleck won the BAFTA Award for Best Director, and the BAFTA and Academy Award for Best Picture. Affleck then starred in the psychological thriller Gone Girl (2014) and played the superhero Batman in the DC Extended Universe (2016–2023). He starred in the thriller The Accountant (2016) and the sports drama The Way Back (2020). Affleck had supporting roles in the dramas The Last Duel (2021), The Tender Bar (2021) and Air (2023), the third of which he also directed. +Affleck is the co-founder of the Eastern Congo Initiative, a grantmaking and advocacy-based nonprofit organization. He is a supporter of the Democratic Party. Affleck and Damon are co-owners of the production company Artists Equity and were also co-owners of Pearl Street Films. +Affleck was born Benjamin Géza Affleck-Boldt on August 15, 1972, in Berkeley, California. His family moved to Massachusetts when he was three, living in Falmouth, where his brother Casey was born, before settling in Cambridge. His mother, Christopher Anne "Chris" Boldt, was a Harvard-educated elementary school teacher. His father, Timothy Byers Affleck, was an aspiring playwright who was "mostly unemployed". Ben's father worked sporadically as a carpenter, auto mechanic, bookie, electrician, bartender, and janitor at Harvard. In the mid-1960s, he had been an actor and stage manager with the Theater Company of Boston. +During Affleck's childhood, his father had a self-described "severe, chronic problem with alcoholism", and Affleck has recalled him drinking "all day ... every day". His father was "very difficult", and Affleck felt a sense of "relief" at the age of 11 when his parents divorced, and his father exited the family home. His father continued to drink heavily and eventually became homeless, spending two years living on the streets of Cambridge. When Affleck was 16, his father entered a rehabilitation facility in Indio, California. He lived at the facility for 12 years to maintain his sobriety and worked there as an addiction counselor. +Affleck was raised in a politically active, liberal household. He and his brother, Casey, were surrounded by people who worked in the arts; the boys regularly attended theater performances with their mother and were encouraged to make their own home movies. David Wheeler, a family friend, later remembered Affleck as a "very bright and intensely curious" child. The brothers auditioned for roles in local commercials and film productions because of their mother's friendship with a Cambridge-area casting director, and Affleck first acted professionally at the age of seven. His mother saved his wages in a college trust fund and hoped her son would ultimately become a teacher, worrying that acting was an insecure and "frivolous" profession. When Affleck was 13, he filmed a children's television program in Mexico. He learned to speak Spanish during a year spent traveling around the country with his mother and brother. +As a Cambridge Rindge and Latin high school student, Affleck acted in theater productions and was inspired by drama teacher Gerry Speca. He became close friends with fellow student Matt Damon, whom he had known since the age of eight. Although Damon was two years older, the pair had "identical interests" and both wanted to pursue acting careers. They traveled to New York together for acting auditions and saved money for train and airline tickets in a joint bank account. While Affleck had high SAT scores, he was largely an unfocused student with poor attendance. He spent a few months studying Spanish at the University of Vermont, chosen because of its proximity to his then-girlfriend, but he left after fracturing his hip while playing basketball. At 18, Affleck moved to Los Angeles, studying Middle Eastern affairs at Occidental College for a year and a half. +Affleck acted professionally throughout his childhood but, in his own words, "not in the sense that I had a mom that wanted to take me to Hollywood or a family that wanted to make money from me ... I kind of chanced into something." He first appeared, at the age of seven, in an independent film, The Dark End of the Street (1981), directed by a family friend. His biggest success as a child actor was as the star of the PBS children's series The Voyage of the Mimi (1984) and The Second Voyage of the Mimi (1988), produced for sixth-grade science classes. Affleck worked "sporadically" on Mimi from the age of eight to fifteen in both Massachusetts and Mexico. As a teenager, he appeared in the ABC after school special Wanted: A Perfect Man (1986), the television film Hands of a Stranger (1987), and a 1989 Burger King commercial. +After high school, Affleck moved briefly to New York in search of acting work. Later, while studying at Occidental College in Los Angeles, Affleck directed student films. As an actor, he had a series of "knock-around parts, one to the next". He played Patrick Duffy's son in the television film Daddy (1991), made an uncredited appearance as a basketball player in the Buffy the Vampire Slayer film (1992), and had a supporting role as a prep school student in School Ties (1992). He played a high school quarterback in the NBC television series Against the Grain (1993), and a steroid-abusing high school football player in Body to Die For: The Aaron Henry Story (1994). Affleck's most notable role during this period was as a high school bully in Richard Linklater's cult classic Dazed and Confused (1993). Linklater wanted a likable actor for the villainous role, and, while Affleck was "big and imposing", he was "so smart and full of life ... I just liked him." Affleck later said Linklater was instrumental in demystifying the filmmaking process for him. +Affleck's first starring film role was as an aimless art student in the college drama Glory Daze (1995), with Stephen Holden of The New York Times remarking that his "affably mopey performance finds just the right balance between obnoxious and sad sack". He then played a bully in filmmaker Kevin Smith's comedy Mallrats (1995) and became friends with Smith during the filming. Affleck had begun to worry that he would be relegated to a career of "throwing people into their lockers", but Smith wrote him a lead role in the romantic comedy Chasing Amy (1997). The film was Affleck's breakthrough. Janet Maslin of The New York Times praised Affleck's "wonderful ease" playing the role, combining "suave good looks with cool comic timing". Owen Gleiberman of Entertainment Weekly described it as a "wholesome and quick-witted" performance. When Affleck starred as a recently returned Korean War veteran in the coming-of-age drama Going All the Way (1997), Todd McCarthy of Variety found him "excellent", while Janet Maslin of The New York Times noted that his "flair for comic self-doubt made a strong impression." +The success of 1997's Good Will Hunting, which Affleck co-wrote and acted in, marked a turning point in his career. The screenplay originated in 1992 when Damon wrote a 40-page script for a playwriting class at Harvard University. He asked Affleck to act out the scenes with him in front of the class and, when Damon later moved into Affleck's Los Angeles apartment, they began working on the script in earnest. The film, which they wrote mainly during improvisation sessions, was set partly in their hometown of Cambridge and drew from their own experiences. The screenplay was borne of their desire to create "an acting reel" for themselves. They sold the screenplay to Castle Rock in 1994 when Affleck was 22 years old. During the development process, they received notes from industry figures, including Rob Reiner and William Goldman. Following a lengthy dispute with Castle Rock about a suitable director, Affleck and Damon persuaded Miramax to purchase the screenplay. The two friends moved back to Boston for a year before the film finally went into production, directed by Gus Van Sant, and co-starring Damon, Affleck, Minnie Driver, and Robin Williams. +On its release, Janet Maslin of The New York Times praised the "smart and touching screenplay", while Emanuel Levy of Variety found it "funny, nonchalant, moving and angry". Jay Carr of The Boston Globe wrote that Affleck brought "a beautifully nuanced tenderness" to his role as the working-class friend of Damon's mathematical prodigy character. Affleck and Damon eventually won the Academy Award for Best Original Screenplay. Affleck has described this period of his life as "dreamlike": "It was like one of those scenes in an old movie when a newspaper comes spinning out of the black on to the screen. You know, '$100 Million Box Office! Awards!'" He remains the youngest writer (at age 25) ever to win an Oscar for screenwriting. +Armageddon, released in 1998, established Affleck as a viable leading man for Hollywood studio films. Good Will Hunting had not yet been released during the casting process and, after Affleck's screen test, director Michael Bay dismissed him as “a geek". Producer Jerry Bruckheimer convinced Bay that Affleck would be a star, but he was required to lose weight, become tanned, and get his teeth capped before filming began. The film, where he starred opposite Bruce Willis as a blue-collar driller tasked by NASA with stopping an asteroid from colliding with Earth, was a box office success. Daphne Merkin of The New Yorker remarked: "Affleck demonstrates a sexy Paul Newmanish charm and is clearly bound for stardom." +Later in 1998, Affleck had a supporting role as an arrogant English actor in the period romantic comedy Shakespeare in Love, starring his then-girlfriend Gwyneth Paltrow. Lael Loewenstein of Variety remarked that Affleck "does some of his very best work, suggesting that comedy may be his true calling," while Janet Maslin of The New York Times found him "very funny". Shakespeare in Love won seven Academy Awards, including Best Picture, while the cast won the Screen Actors Guild Award for Outstanding Performance by a Cast. Affleck then appeared as a small-town sheriff in the supernatural horror film Phantoms. Stephen Holden of The New York Times wondered why actors like Affleck and Peter O'Toole had agreed to appear in the "junky" film: "Affleck's thudding performance suggests he is reading his dialogue for the first time, directly from cue cards." +Affleck and Damon had an on-screen reunion in Kevin Smith's religious satire Dogma, which premiered at the 1999 Cannes Film Festival. Janet Maslin of The New York Times remarked that the pair, playing fallen angels, "bring great, understandable enthusiasm to Mr. Smith's smart talk and wild imaginings". Affleck starred opposite Sandra Bullock in the romantic comedy Forces of Nature (1999), playing a groom whose attempts to get to his wedding are complicated by his free-spirited traveling companion. Joe Leydon of Variety praised "his winning ability to play against his good looks in a self-effacing comic turn." Affleck then appeared opposite Courtney Love in the little-seen ensemble comedy 200 Cigarettes (1999). +Interested in a directorial career, Affleck shadowed John Frankenheimer throughout pre-production of the action thriller Reindeer Games (2000). Frankenheimer, directing his last feature film, described Affleck as having "a very winning, likable quality about him. I've been doing this for a long time and he's really one of the nicest." He starred opposite Charlize Theron as a hardened criminal, with Elvis Mitchell of The New York Times enjoying the unexpected casting choice: "Affleck often suggests one of the Kennedys playing Clark Kent ... He looks as if he has never missed a party or a night's sleep. He's game, though, and his slight dislocation works to the advantage of Reindeer Games." He then had a supporting role as a ruthless stockbroker in the crime drama Boiler Room (2000). A.O. Scott of The New York Times felt Affleck had merely "traced over" Alec Baldwin's performance in Glengarry Glen Ross, while Peter Rainer of New York Magazine said he "does a series of riffs on Baldwin's aria, and each one is funnier and crueler than the next". He provided the voice of Joseph in the animated Joseph: King of Dreams. In his last film role of 2000, Affleck starred opposite his girlfriend Paltrow in the romantic drama Bounce. Stephen Holden of The New York Times praised the "understated intensity and exquisite detail" of his performance: "His portrait of a young, sarcastically self-defined 'people person' who isn't half as confident as he would like to appear is close to definitive." +Affleck reunited with director Michael Bay for the critically derided war drama Pearl Harbor (2001). Todd McCarthy of Variety wrote "the blandly handsome Affleck couldn't convince that he'd ever so much as been turned down for a date, much less lost the love of his life to his best friend". Affleck then parodied Good Will Hunting with Damon and Van Sant in Kevin Smith's Jay and Silent Bob Strike Back (2001), made a cameo in the comedy Daddy and Them (2001), and had a supporting role in the little-seen The Third Wheel (2002). He portrayed the CIA analyst Jack Ryan in the thriller The Sum of All Fears (2002). Stephen Holden of The New York Times felt he was miscast in a role previously played by both Harrison Ford and Alec Baldwin: "Although Mr. Affleck can be appealing when playing earnest young men groping toward maturity, he simply lacks the gravitas for the role." Affleck had an "amazing experience" making the thriller Changing Lanes (2002), and later cited Roger Michell as someone he learned from as a director. Robert Koehler of Variety described it as one of his "most thoroughly wrought" performances: "The journey into a moral fog compels him to play more inwardly and thoughtfully than he ever has before." +Affleck became more involved with television and film production in the early 2000s. He and Damon had set up Pearl Street Films in 1998, named after the street that ran between their childhood homes. Their next production company LivePlanet, co-founded in 2000 with Sean Bailey and Chris Moore, sought to integrate the Internet into mainstream television and film production. LivePlanet's biggest success was the documentary series Project Greenlight, aired on HBO and later Bravo, which focused on first-time filmmakers being given the chance to direct a feature film. Project Greenlight was nominated for the Primetime Emmy Award for Outstanding Reality Program in 2002, 2004, and 2005. Push, Nevada (2002), created, written, and produced by Affleck and Bailey, was an ABC mystery drama series that placed a viewer-participation game within the frame of the show. Caryn James of The New York Times praised the show's "nerve, imagination, and clever writing", but Robert Bianco of USA Today described it as a "knock-off" of Twin Peaks. The show was canceled by ABC after seven episodes due to low ratings. Over time, LivePlanet's focus shifted from multimedia projects to more traditional film production. Affleck and his partners signed a film production deal with Disney in 2002; it expired in 2007. +While Affleck had been a tabloid figure for much of his career, he was the subject of increased media attention in 2003 because of his relationship with Jennifer Lopez. By the end of the year, Affleck had become, in the words of GQ, the "world's most over-exposed actor". His tabloid fame coincided with a series of poorly received films. +The first of these was Daredevil (2003), in which Affleck starred as the blind superhero. Affleck was a longtime comic book fan, and, in 1999, had written a foreword for Kevin Smith's Guardian Devil about his love for the character of Daredevil. The film was a commercial success, but received a mixed response from critics. Elvis Mitchell of The New York Times said Affleck was "lost" in the role: "Affleck is shriveled by the one-dimensional role ... [Only his scenes with Jon Favreau have] a playful side that allows Mr. Affleck to show his generosity as an actor." In 2014, Affleck described Daredevil as the only film he regretted making. He next appeared as a low-ranking mobster in the romantic comedy Gigli (2003), co-starring Lopez. The film was almost uniformly panned, with Manohla Dargis of the Los Angeles Times remarking that "Affleck doesn't have the chops or the charm to maneuver around (or past) bad material." Rex Reed of The Observer criticized the co-stars, writing that the film reminds the world how "pathetically incompetent they both are in the only two things that matter in career longevity—craft and talent." Yet Affleck has repeatedly defended director Martin Brest since the film's release, describing him as "one of the really great directors." In his last film role of 2003, Affleck starred as a reverse engineer in the sci-fi thriller Paycheck (2003). Peter Bradshaw of The Guardian remarked on Affleck's "self-deprecating charm" and wondered why he could not find better scripts. Manohla Dargis of the Los Angeles Times found it "almost unfair" to critique Affleck, given that he “had such a rough year.” +Affleck's poor critical notices continued in 2004 when he starred as a bereaved husband in the romantic comedy Jersey Girl, directed by longtime collaborator Smith. Stephen Holden of The New York Times described Affleck as an actor "whose talent has curdled as his tabloid notoriety has spread." Later that year, he starred opposite James Gandolfini in the holiday comedy Surviving Christmas. Holden noted in The New York Times that the film "found a clever way to use Ben Affleck's disagreeable qualities. The actor's shark-like grin, cocky petulance and bullying frat-boy swagger befit his character." At this point, the quality of scripts offered to Affleck "was just getting worse and worse", the negative press coverage "really started to affect" him, and he decided to take a career break. The Los Angeles Times published a piece on the downfall of Affleck's career in late 2004. The article noted that, unlike film critics and tabloid journalists, "few industry professionals seem to be gloating over Affleck's travails.” +Affleck began a career comeback in 2006 with his acclaimed performance as Superman actor George Reeves in the noir biopic Hollywoodland. Peter Travers of Rolling Stone praised "an award-caliber performance ... This is feeling, nuanced work from an actor some of us had prematurely written off." Geoffrey Macnab of The Guardian said he "beautifully" captured "the character's curious mix of charm, vulnerability and fatalism". He was awarded the Volpi Cup at the Venice Film Festival and was nominated for a Golden Globe for Best Supporting Actor. Also in 2006, he made a cameo in Smith's Clerks II, starred in the little-seen Man About Town and had a minor role in the crime drama Smokin' Aces. +In 2007, Affleck made his feature film directorial debut with Gone Baby Gone, a crime drama set in a working-class Boston neighborhood, starring his brother Casey as a private investigator searching for a young abductee. Affleck co‑wrote the screenplay, based on the book by Dennis Lehane, with childhood friend Aaron Stockard, having first mentioned his intention to adapt the story in 2003. It opened to enthusiastic reviews. Manohla Dargis of The New York Times praised the film's "sensitivity to real struggle," while Stephen Farber of The Hollywood Reporter described it as "thoughtful, deeply poignant, [and] splendidly executed." +While Affleck intended to "keep a primary emphasis on directing" going forward in his career, he acted in three films in 2009. In the ensemble romantic comedy He's Just Not That Into You, the chemistry between Affleck and Jennifer Aniston was praised. Affleck played a congressman in the political thriller State of Play. Wesley Morris of The Boston Globe found him "very good in the film's silliest role," but David Edelstein of New York Magazine remarked of Affleck: "He might be smart and thoughtful in life [but] as an actor his wheels turn too slowly." He had a supporting role as a bartender in the little-seen comedy film Extract. Peter Travers of Rolling Stone described his performance as "a goofball delight", while Manohla Dargis of The New York Times declared it "a real performance". In 2010, Affleck starred in The Company Men as a mid-level sales executive who is made redundant during the 2008 financial crisis. David Denby of The New Yorker declared that Affleck "gives his best performance yet", while Richard Corliss of Time found he "nails Bobby's plunge from hubris to humiliation". +Following the modest commercial success of Gone Baby Gone, Warner Bros. developed a close working relationship with Affleck and offered him his choice of the studio's scripts. He decided to direct the crime drama The Town (2010), an adaptation of Chuck Hogan's novel Prince of Thieves. He also re-wrote the screenplay and starred in the film as a bank robber. The film became a surprise box office hit, and gained further critical acclaim for Affleck. A.O. Scott of The New York Times praised his "skill and self-confidence as a serious director," while Roger Ebert of the Chicago Sun-Times noted: "Affleck has the stuff of a real director. Everything is here. It's an effective thriller, he works closely with actors, he has a feel for pacing." Also in 2010, Affleck and Damon's production company, Pearl Street Films, signed a first-look producing deal at Warner Bros. +Affleck soon began work on his next directorial project, Argo (2012), for Warner Bros. Written by Chris Terrio and starring Affleck as a CIA operative, the film tells the story of the CIA plan to save six U.S. diplomats during the 1979 Iran hostage crisis by faking a production for a large-scale science fiction film. Anthony Lane of The New Yorker said the film offered "further proof that we were wrong about Ben Affleck". Peter Travers of Rolling Stone remarked: "Affleck takes the next step in what looks like a major directing career ... He directs the hell out of it, nailing the quickening pace, the wayward humor, the nerve-frying suspense." A major critical and commercial success, Argo won the Academy Award, Golden Globe Award, and BAFTA Award for Best Picture. The cast won the Screen Actors Guild Award for Outstanding Performance by a Cast. Affleck himself won the Golden Globe Award, Directors Guild of America Award, and BAFTA Award for Best Director, becoming the first director to win these awards without a nomination for the Academy Award for Best Director. +The following year, Affleck played a romantic lead in Terrence Malick's experimental drama To the Wonder. Malick, a close friend of Affleck's godfather, first met Affleck in the 1990s to offer advice about the plot of Good Will Hunting. Peter Bradshaw of The Guardian enjoyed "a performance of dignity and sensitivity," while The New Yorker's Richard Brody described Affleck as "a solid and muscular performer" who "conveys a sense of thoughtful and willful individuality". Affleck's performance as a poker boss was considered a highlight of the poorly reviewed thriller Runner Runner (2013). Betsy Sharkey of the Los Angeles Times remarked that it was "one killer of a character, and Affleck plays him like a Bach concerto – every note perfectly played." +He then pushed back production on his own directorial project to star as a husband accused of murder in David Fincher's psychological thriller Gone Girl (2014). Fincher cast him partly because he understood what it felt like to be misrepresented by tabloid media: "What many people don't know is that he's crazy smart, but since he doesn't want that to get awkward, he downplays it. I think he learned how to skate on charm." David Edelstein of New York Magazine noted that Fincher's controlled style of directing had a "remarkable" effect on Affleck's acting: "I never thought I'd write these words, but he carries the movie. He's terrific." Justin Chang of Variety found Affleck "perfectly cast": "It's a tricky turn, requiring a measure of careful underplaying and emotional aloofness, and he nails it completely." In 2015, Affleck and Damon's Project Greenlight was resurrected by HBO for one season. +Given Affleck's growing reputation as a filmmaker, his decision to star as Batman in the 2016 superhero film Batman v Superman: Dawn of Justice was regarded by Dave Itzkoff of The New York Times as "a somewhat bewildering choice". Affleck reprised his role as Batman later that year, making a brief cameo appearance in Suicide Squad (2016). He starred as an autistic accountant in the action thriller The Accountant (2016), which was an unexpected commercial success. Peter Debruge of Variety felt Affleck's "boy-next-door" demeanor – "so normal and non-actorly that most of his performances feel like watching one of your buddies up on screen" – was "a terrific fit" for the role. Stephen Holden of The New York Times wondered why Affleck, "looking appropriately dead-eyed and miserable," committed himself to the film. +Live by Night, which Affleck wrote, directed, co-produced, and starred in, was released in late 2016. Adapted from Dennis Lehane's novel of the same name, the Prohibition-era gangster drama received largely unenthusiastic reviews and failed to recoup its $65 million production budget. David Sims of The Atlantic described it as "a fascinating mess of a movie" and criticized Affleck's "stiff, uncomfortable" performance. He noted that one of the last action scenes "is so wonderfully staged, its action crisp and easy to follow, that it reminds you what skill Affleck has with the camera". In October 2016, Affleck and Damon made a one-off stage appearance for a live reading of the Good Will Hunting screenplay at New York's Skirball Theater. The superhero film Justice League, in which Affleck returned as Batman, was released in 2017. He later described it as a "difficult" on-set experience. Zack Snyder stepped down during filming due to the death of his daughter, the replacement director Joss Whedon's treatment of actors was the subject of complaints, and Affleck himself was struggling with addiction issues. The film drew mixed opinions from critics; Todd McCarthy of The Hollywood Reporter wrote that Affleck "looks like he'd rather be almost anywhere else but here." +Amidst an alcoholism relapse, Affleck did not work in 2017. He stepped down as director, writer and, ultimately, as the star of The Batman, saying he "couldn't crack it" and no longer felt "passionate" about the story. Filming of the drug-trafficking thriller Triple Frontier was postponed by six months in order to accommodate his treatment for "health issues". Upon Triple Frontier's release in 2019, Rodrigo Perez of The Playlist remarked that director J.C. Chandor "gets a lot of mileage out of the Sad Affleck narrative and perhaps both director and actor lean into the idea." Later in 2019, Affleck made a cameo appearance in Jay and Silent Bob Reboot, having had little contact with Kevin Smith since the making of Clerks II in 2006. Affleck played a supporting role as a diplomat in Dee Rees's political thriller The Last Thing He Wanted (2020). The Netflix movie, filmed in mid-2018, received negative reviews from critics, with Tomris Laffly of Variety describing Affleck's performance as "oddly removed". +Affleck's starring role as a recovering alcoholic in the sports drama The Way Back (2020) was widely praised. The themes of the movie were "close to home" for Affleck. He relapsed during pre-production in 2018 and the movie was shot in the days after he left rehab; Affleck agreed to put his salary in escrow and was accompanied to set by a sober coach. Richard Lawson of Vanity Fair said it was hard to avoid the movie's "meta angle": "Affleck handles his self-conscious task with a generous humility—giving a performance built not out of histrionics or big actor moments, but instead from the messy details of a man in a plateaued distress". David Sims of The Atlantic praised the "subtlety", "vulnerability" and "lumbering physicality" of his performance, describing it as "the rawest and most natural" work of his career. Due to the COVID-19 pandemic, cinemas closed in the second week of the movie's release and Warner Bros. made it available to view-on-demand earlier than scheduled. He received a Best Actor nomination at the 2021 Critics' Choice Awards. Also in 2021, Snyder's director's cut of Justice League featured a newly filmed scene with Affleck. +Affleck had supporting roles in two 2021 releases. He played Peter II, Count of Alençon, a hedonistic aristocrat in Ridley Scott's medieval drama The Last Duel; he also co-wrote the movie's screenplay with Damon and Nicole Holofcener. Bilge Ebiri of New York Magazine was impressed by Affleck's "imperious" performance as the "wonderfully skeezy Pierre, a marvelously out-there creation who shouldn't work at all and yet becomes an engine of uneasy delights." Brian Truitt of USA Today said he "steals the movie": "He seems to have written the most entertaining character in the movie for himself, but we'll allow it." Later that year, Affleck appeared as a substitute father figure in George Clooney's coming-of-age drama The Tender Bar. The film premiered at the London Film Festival, with Clooney remarking that he cast Affleck because he is "very intelligent, and he can also come off as the big goomba." Pete Hammond of Deadline described it as "a part he was born to play", writing that he "beautifully and knowingly" delivered "an unforgettable portrait of the uncle you wish you had." Kevin Maher of The Times said he played the role with "extraordinary subtly and depth": "He dominates his every scene, with deftly delivered quips, comedy reaction shots or sheer hang-dog charisma alone." For the role, Affleck was nominated for the Golden Globe Award and Screen Actors Guild Award for Best Supporting Actor. +In 2022, Affleck and Ana de Armas starred in Adrian Lyne's thriller Deep Water, an adaptation of Patricia Highsmith's novel. It was released on streaming service Hulu and received largely negative reviews. Justin Chang of the Los Angeles Times said Affleck "excels at playing the emasculated dreamboat, the golden boy gone to seed." Adam Nayman of The Ringer said the film "serves as an example of his formidable skill set within a very specific range": "What used to seem like the callow cockiness of a handsome front-runner has hardened—and deepened—into a kind of grizzled charisma, the gravitas of a frat boy facing down his own expiration date." He also made a cameo appearance in Smith's Clerks III. +Affleck's fifth directorial project, Air, about Nike's creation of the Air Jordan, premiered in 2023 and was well-reviewed. The movie starred Matt Damon and Viola Davis, with Affleck playing a supporting role as Phil Knight. It marked the first release from Affleck and Damon's independent production company Artists Equity, which aims to share profits with all stakeholders. Affleck serves as the company's CEO and intends to work exclusively for Artists Equity as a filmmaker. In a review of Air for The Ringer, Adam Nayman wrote that Affleck, while not a "masterful" filmmaker, "has a gift for pacing and working with actors ... He and Damon spend most of the time pumping each other up, and the chemistry between them is undeniable." David Sims of The Atlantic said Air was "a great return to Affleck's original impulses as a director: It's a fun, well-made film for grown-ups that gives its actors room to flesh out their characters." Also in 2023, Affleck starred as a detective in Robert Rodriguez's action thriller Hypnotic and made his final appearance as Batman in the film The Flash, appearing in a cameo. +In 2025, Affleck starred in the action thriller The Accountant 2. In January 2026, he alongside Damon starred in the crime thriller The Rip. He also directed and starred opposite Gillian Anderson in the kidnapping thriller Animals, which was debuted on Netflix on October 9, 2026. +In March 2026, Netflix acquired Affleck's AI-focused post-production startup, InterPositive, for an estimated $600 million, at which time he joined the company as a senior advisor to oversee the technology's integration. +After traveling in the region between 2007 and early 2010, Affleck and Whitney Williams co-founded the nonprofit organization Eastern Congo Initiative in 2010. ECI acts as a grant maker for Congolese-led, community-based charities. It offers training and resources to cooperatives of Congolese farmers while leveraging partnerships with companies including Theo Chocolate and Starbucks. ECI also aims to raise public awareness and drive policy change in the United States. +Affleck has written op-eds about issues facing eastern Congo for the Washington Post, Politico, the Huffington Post, Time, The New York Times and the Los Angeles Times. He has appeared as a discussion panelist at many events, including at the Center for Strategic and International Studies, the Global Philanthropy Forum, and the Clinton Global Initiative. During visits to Washington D.C., Affleck has testified before the House Subcommittee on Africa, Global Health and Human Rights, the House Armed Services Committee, the Senate Foreign Relations Committee, and the Senate Appropriations Subcommittee on State, Foreign Operations, and Related Projects. +Affleck is a supporter of the A-T Children's Project. While filming Forces of Nature in 1998, Affleck befriended ten-year-old Joe Kindregan (1988–2015), who had the rare disease ataxia-telangiectasia (A-T), and his family. He became actively involved in fundraising for A-T, and he and Kindregan testified before the House Appropriations Subcommittee on Labor, Health & Human Services, and Education in 2001, asking senators to support stem-cell research and to double the budget of the National Institutes of Health. In 2007, Affleck was the keynote speaker at Kindregan's high school graduation ceremony in Fairfax, Virginia. Kindregan appeared as an extra in Argo (2012). In 2013, in celebration of Kindregan's 25th birthday and "15 years of friendship with Joe and his family," Affleck and Garner matched donations made to the A-T Children's Project. Affleck appeared in CinemAbility (2013), a documentary film which explores Hollywood's portrayals of people with disabilities. +Affleck is a supporter of Paralyzed Veterans of America. He filmed public service announcements for the organization in both 2009 and 2014. He has also volunteered on behalf of Operation Gratitude. +Affleck is a member of Feeding America's Entertainment Council. He made an appearance at the Greater Boston Food Bank in 2007, and at a Denver food bank in 2008. Affleck spoke at a Feeding America rally in Washington D.C. in 2009, and filmed a public service announcement for the charity in 2010. Affleck and Ellen DeGeneres launched Feeding America's Small Change Campaign in 2011. Also that year, he and Howard Graham Buffett co-wrote an article for The Huffington Post, highlighting the "growing percentage of the food insecure population that is not eligible for federal nutrition programs". During the COVID-19 pandemic, Affleck organized an online celebrity poker tournament to benefit the charity, made a personal donation and urged others to support the cause. +Affleck is a supporter of the Los Angeles-based homelessness charity Midnight Mission, having volunteered at and donated to the charity. He has also volunteered at Atlanta Mission. +Affleck has described himself as "moderately liberal." He was raised in "a very strong union household". In 2000, he spoke at a rally at Harvard University in support of an increased living wage for all workers on campus; his father worked as a janitor at the university. He later narrated a documentary, Occupation (2002), about a sit-in organized by the Harvard Living Wage Campaign. Affleck and Senator Ted Kennedy held a press conference on Capitol Hill in 2004, pushing for an increase in the minimum wage. He spoke at a 2007 press conference at Boston's City Hall in support of SEIU's unionization efforts for the city's low-paid hospital workers. During the Writers' Strike in 2008, Affleck voiced support for the picketers. +Affleck is pro-choice. In a 2000 interview, he stated that he believes "very strongly in a woman's right to choose". In 2012, he supported the Draw the Line campaign, describing reproductive rights as "fundamental". +Affleck was a longtime supporter of legalizing gay marriage, saying in 2004 that he hoped to look back on the marriage debate "with some degree of embarrassment for how antiquated it was". Also that year, he remarked that it was "outrageous and offensive" to suggest members of the transgender community were not entitled to equal rights. He appeared alongside his gay cousin in a 2005 Parents and Friends of Lesbians and Gays print advertising campaign. +In 2008, Affleck expressed concern about conspiracy theories claiming Barack Obama was an Arab or a Muslim: "This prejudice that we have allowed to fester in this campaign ... the acceptance of both of those things as a legitimate slur is really a problem." A reporter for The Washington Post overheard Affleck denouncing the Israeli invasion of Gaza at a Washington party in 2009. Steven Clemons, a participant in the conversation, said Affleck listened "to alternative takes ... What Affleck spoke about that night was reasoned, complex and made a lot of sense." Later that year, in a New York Times interview, Affleck remarked that his views were closer to those of the Israeli Labor Party than Likud. +In 2012, he praised Senator John McCain's leadership in defending Huma Abedin against anti-Muslim attacks. Affleck engaged in a widely shared discussion about the relationship between liberal principles and Islam during a 2014 appearance on Real Time with Bill Maher. In a 2017 Guardian interview, he reflected on the viral moment: "I strongly believe that no one should be stereotyped on the basis of their race or religion. It's one of the most fundamental tenets of liberal thought." In 2023, Affleck signed the Artists4Ceasefire open letter calling for a ceasefire during the Israeli bombardment of Gaza. +Affleck appeared at a press conference with New York Senator Chuck Schumer in 2002, in support of a proposed Anti-Nuclear Terrorism Prevention Act. In 2003, he criticized the "questionable and aggressive" use of the Patriot Act and the resulting "encroachments on civil liberties". Affleck criticized the Bush tax cuts on many occasions. In 2007, he filmed a public service announcement for Divided We Fail, a nonpartisan AARP campaign seeking affordable, quality healthcare for all Americans. +Affleck expressed support for the Second Amendment in 2004, and said in 2012 that he owned several guns, both for skeet shooting and for the protection of his family. In 2020, he said trips to gun ranges as a young adult made him "uncomfortable to remember ... given the subsequent tragedies with young people and guns." +Affleck appeared alongside then-Senator Barack Obama at a 2006 rally in support of Proposition 87, which sought to reduce petroleum consumption in favor of alternative energy. He appeared in a global warming awareness video produced by the Center for American Progress Action Fund in 2007. Also that year, Affleck admitted he was not "particularly good at being green" while, in 2014, he named "a 1966 Chevelle" as his guilty pleasure. In 2016, Affleck filmed an endorsement for Rezpect Our Water, an online petition to stop construction of the Dakota Access Pipeline. +Affleck registered to vote as a Democrat in 1992, and has campaigned on behalf of several Democratic presidential nominees. He supported Al Gore in the final weeks of the 2000 presidential campaign, attending rallies in California, Pennsylvania, and Florida. However, Affleck was unable to vote due to a registration issue in New York City, where he was residing at the time, and later joked, "I'm going to vote twice next time, in true Boston fashion." +Affleck was involved in the 2004 presidential campaign of John Kerry. During the Democratic National Convention in Boston, he spoke to many delegations, appeared on political discussion shows, and attended fundraising events. Affleck took part in a voter registration public service announcement, and traveled with Kerry during the opening weekend of his Believe in America Tour, making speeches at rallies in Pennsylvania, West Virginia, and Ohio. +Affleck campaigned for President Barack Obama. He appeared alongside the then-Senator at a 2006 rally, introducing him as "the most galvanizing leader to come out of either party, in my opinion, in at least a decade". He donated to Obama's presidential campaign in 2007, and hosted two fundraisers for the candidate during the 2008 Democratic Primary. Affleck urged voters to "help make history" in a MoveOn.org campaign, and made several appearances during the 2008 Democratic National Convention. In the week of the presidential election, he appeared on Saturday Night Live to jokingly endorse Senator John McCain. Affleck did not actively campaign for Obama's reelection in 2012, though he still supported him. +Affleck supported Hillary Clinton's 2016 presidential election campaign. He first met Clinton at Camp David in 1998, and, when she was a Senate candidate in 2000, he introduced her at a Cornell University rally and helped fundraise for her campaign. Affleck pointed to the First Lady's work with children, women and "working families". He supported Obama during the 2008 Democratic Primary, noting that Clinton had "moved toward the center" during the campaign. Affleck donated to Clinton's campaign fund during the 2016 Democratic Primary. During the 2016 presidential election, Affleck recorded a New Hampshire voter public service announcement, and was named by the Clinton campaign as a "Hillblazer" – one of 1,100 individuals who had contributed or raised at least $100,000. OpenSecrets reported that he raised $149,028. +During the latter stages of the 2020 Democratic Party presidential primaries, Affleck said during an interview conducted in Spanish: "I like Bernie, I like Biden, I like Warren but the thing is that most of all; I don't like Trump." When Biden became the Democratic candidate, he donated to his campaign fund. +In 2002, Affleck donated to Dick Gephardt's Congressional campaign, Robert Reich's campaign for Governor of Massachusetts, and appeared in campaign literature for former classmate Marjorie Decker, running as a city councilor in Massachusetts. He made donations to the presidential campaigns of both Dennis Kucinich and Wesley Clark in 2003. In 2005, he donated to the campaign fund of Deval Patrick, a candidate for Governor of Massachusetts. In 2006, Affleck contributed to Cory Booker's Newark mayoral campaign, to Eliot Spitzer's New York gubernatorial campaign, and introduced Congressmen Joe Courtney and Chris Murphy at rallies in Connecticut. He donated to the 2008 Congressional campaign of Pennsylvania's Patrick Murphy, and to the 2010 Senate campaign of Kirsten Gillibrand. Affleck hosted a 2012 fundraiser for Senate candidate Elizabeth Warren, endorsed her in a Progressive Change Campaign Committee video, and made a campaign donation. In 2013, he hosted a fundraiser for Cory Booker, and made donations to the Senate campaigns of both Booker and Alison Lundergan Grimes. In 2014, he donated to Bobby Shriver's Los Angeles County Supervisor campaign, and to Prophet La'omar Walker's California State Assembly campaign. He donated to the campaign of Senate candidate Kamala Harris in 2015 and, in 2016, he donated to the Congressional campaign of Melissa Gilbert and the West Virginia Senate campaign of Corey Palumbo. In 2017, he donated to the Senate reelection campaigns of Elizabeth Warren and Chris Coons, and to Adam Schiff's Congressional campaign. In 2018, he made contributions to the Congressional campaigns of Alexandria Ocasio-Cortez, Sharice Davids and Leann Jacobsen, and to Michigan gubernatorial candidate Abdul El-Sayed and Georgia gubernatorial candidate Stacey Abrams. In 2019, Affleck donated to the presidential campaigns funds of both Cory Booker and Kamala Harris, and hosted a fundraiser for Booker. Also in 2019, he contributed to the congressional campaign funds of both Alexandria Ocasio-Cortez and Ilhan Omar. In 2020, he spoke at a virtual rally in support of Whitney Williams, a candidate in the 2020 Montana gubernatorial election, and donated to her campaign fund. In 2023, he donated to the Senate campaign of Barbara Lee. +In the early 2000s, Affleck often expressed an interest in running for political office one day, but since 2007, he has denied any political ambitions and spoken repeatedly about the need for campaign finance reform. In 2005, The Washington Post reported that Virginia Democrats were trying to persuade Affleck to run as a Senate candidate. His publicist dismissed the rumor. In 2012, political pundits and Democratic strategists including Bob Shrum and Tad Devine speculated that Affleck was considering running for a Massachusetts Senate seat. Affleck denied the rumor, and joked that he "also won't be throwing my hat in the ring to run the UN". +Affleck was married to actress Jennifer Garner from 2005 to 2018. They began dating in August 2004, having established a friendship on the sets of Pearl Harbor (2001) and Daredevil (2003). They were married on June 29, 2005, in a private Turks and Caicos ceremony. Victor Garber, who officiated the ceremony, and his partner, Rainer Andreesen, were the only guests. Affleck and Garner have three children together. +They publicly announced their separation in June 2015, with Affleck continuing to live in a guesthouse at the family home until mid-2017. They jointly filed for divorce in April 2017, seeking joint physical and legal custody of their children. They jointly hired a mediator to help them negotiate a financial settlement and the divorce was finalized in October 2018. In 2020, Affleck described the divorce as "the biggest regret of my life" and "a painful experience, even if you're on the best possible terms and you agree it's the best choice." +While Affleck believes paparazzi attention is "part of the deal" of stardom, he has spoken out against photographs taken at his private residence and paparazzi interest in his children specifically, who he says became the "big money" draw for photographers waiting outside his home. In 2013, Affleck and Garner hosted an event for lawmakers at their home in support of a bill that would protect celebrities' children from harassment by photographers; their six-year-old daughter made a speech about her personal experiences. Garner also testified before the California Assembly Judiciary Committee in support of the bill, which later became law. Despite the legislation, photographers still regularly waited outside their children's school and police assistance was sometimes required. In 2014, Affleck argued in favor of United Kingdom-style legislation that requires media outlets to blur children's faces in published photos. He has said there are "real practical dangers" involved in paparazzi attention, citing Garner's longtime stalker, Steven Burky, who was arrested in December 2009 while trying to blend in with paparazzi outside their daughter's preschool. Burky was charged with two counts of stalking, to which he pleaded not guilty by reason of insanity. In March 2010, he was ruled insane, sent to California's state mental hospital, and ordered to stay away from the Affleck-Garner family for 10 years if released. +Affleck first dated Jennifer Lopez from 2002 to 2004. They became friends on the set of Gigli in December 2001, having previously encountered each other at industry parties. They began a romantic relationship in July 2002 when Lopez filed for divorce from her second husband, Cris Judd. Their relationship was extensively publicized, with tabloids referring to the couple as Bennifer, the first celebrity portmanteau of its kind. They appeared together in the music video for her song "Jenny from the Block" and the film Jersey Girl. Lopez's album This Is Me... Then was dedicated to and inspired by Affleck. They became engaged in November 2002 but their planned wedding on September 14, 2003, was postponed with four days' notice because of "excessive media attention". They called off the engagement in January 2004. +Affleck and Lopez remained in occasional contact in the years after their breakup and spoke highly of each other in public. According to Lopez, Affleck's discomfort with the media scrutiny was one reason for their split and, years later, she described it as her "first real heartbreak": "I think different time, different thing, who knows what could've happened but there was a genuine love there." +Both during the relationship and in the months after the breakup, Affleck characterized some of the media commentary as rooted in racism, classism, and sexism: "We were thought of as two different kinds of people." In subsequent years, he pushed back against the "curious notion" that he should view the relationship as a mistake, and reflected on the prevailing tabloid culture at the time. Affleck acknowledged that "there were ways I did contribute to it", citing the "Jenny from the Block" music video and a joint promotional interview for Gigli. He said neither of them "anticipated" the degree of attention they would receive: "I think Jen and I made a mistake in that we fell in love, we were excited and maybe too accessible." +Affleck and Lopez began dating again in April 2021, 20 years after they had first met. They announced their second engagement in April 2022 and were married in a Las Vegas ceremony on July 16, 2022, hosting a larger celebration for family and friends at Affleck's Georgia house later that summer. The couple separated in April 2024, with Lopez filing for divorce in August of that year. The divorce was finalized in February 2025. Affleck later said there was "no scandal, no soap opera, no intrigue" involved in their divorce. +Affleck was in a relationship with Cheyenne Rothman from 1990 to 1997. Affleck began dating actress Gwyneth Paltrow in October 1997 after they met at a Miramax dinner, and they later worked together on Shakespeare in Love (1998). Although they first broke up in January 1999, months later, Paltrow persuaded Affleck to co-star with her in Bounce (2000) and they soon resumed their relationship. They separated again in October 2000. In 2015, Paltrow said they were friends. +Affleck had a long-distance relationship with New York-based television producer Lindsay Shookus from mid-2017 to mid-2018; they briefly dated again in early 2019. Affleck dated Cuban actress Ana de Armas, whom he met on the set of Deep Water in the fall of 2019, from early 2020 to early 2021. +Affleck has both anxiety and depression, and has taken antidepressants since age 26. He is a recovering alcoholic and follows a 12-step program. He said he used alcohol to alleviate a constant feeling of "discomfort" and remarked that it "took me a long time to fundamentally, deeply, without a hint of doubt, admit to myself that I am an alcoholic." +Affleck became sober in his mid-twenties, stating in a 1998 interview that alcohol was "dangerous" for him. He received residential treatment for addiction in 2001 and maintained his sobriety for a "couple of years" afterward. In subsequent years, he refused to discuss his alcoholism in press interviews and later described it as a period where he "drank relatively normally". "I thought, 'I want to just drink like a normal person. I want to have wine at dinner.' And I was able to for about eight years." Affleck gradually began to drink "more and more" and, eventually, was drinking until he "passed out" on a nightly basis. Garner supported Affleck's struggles with alcoholism during and after their marriage and said in 2020 that attending Al-Anon meetings empowered her to change "the dance" of their relationship. Affleck received residential treatment in 2017 and, following a publicly documented relapse and intervention, again in 2018. He completed further outpatient treatment over the following year. In late 2019, TMZ filmed him stumbling on a Los Angeles street; he acknowledged the following day that he had a brief "slip" after over a year of sobriety. He later described the incident as "embarrassing": "I wish it didn't happen. I really wish it wasn't on the internet for my kids to see." +During press for The Way Back (2020), in which he plays an alcoholic, Affleck stressed that he felt "vulnerable" when talking about his addiction and did not intend to "go on talking about this issue forever": "I think the value, if there is value in me talking about being a recovering alcoholic, is that that doesn't have to be who you are. That doesn't have to be the label on your head." In 2023, he said he often advises fellow actors to avoid speaking publicly about their addiction issues where possible: "You don't need to be anybody's poster child." +Affleck won the 2004 California State Poker Championship, taking home the first prize of $356,400 and qualifying for the 2004 World Poker Tour final tournament. He was one of many celebrities, along with Leonardo DiCaprio and Tobey Maguire, who took part in Molly Bloom's high-stakes poker games in the mid-2000s. Affleck has repeatedly denied tabloid reports of a gambling addiction. +Affleck came from an Episcopalian family, but was not raised in a religious household. In 2008, he listed the Gospel of Matthew as one of the books that made a difference in his life. As infants, each of his three children were baptized as members of the United Methodist Church and, from 2015 to 2020, Affleck, Garner and their children were regularly photographed at Methodist church services in Los Angeles. +Affleck is agnostic and has described faith as a "struggle": "But the twelve-step program is faith-based ... Faith has served me well in recovery as an alcoholic." He identifies more with the teachings of Buddhism: "I have a difficult time with the theistic aspects of AA ... One of the things I like about Buddhism is it's like, believe what you believe." +Affleck's ancestry is English, Welsh, Scottish and German, along with some Irish. +Affleck appeared on the PBS genealogy series Finding Your Roots in 2014. When told that an ancestor had been a slave owner in Georgia, Affleck responded: "God. It gives me kind of a sagging feeling to see a biological relationship to that. But, you know, there it is, part of our history ... We tend to separate ourselves from these things by going like, 'It's just dry history, and it's all over now'." Leaked emails from the 2015 Sony email hacking scandal showed that, after filming, Affleck felt uncomfortable about the segment, which was not included in the final broadcast. The show's host, professor Henry Louis Gates Jr., stated: "We focused on what we felt were the most interesting aspects of his ancestry." An internal investigation by PBS concluded that Affleck had exerted "improper influence" over the editorial process and that the producers of the show, including Gates, had violated PBS standards by improperly withholding information. The show was temporarily postponed, resuming after a fact-checker and an "independent genealogist" were added to the show's staff. Affleck's episode was removed from the show's online archive. +During the MeToo movement in 2017, Affleck was accused by two women of inappropriate behavior. Actress Hilarie Burton stated that, during an on-air appearance on TRL Uncensored in 2003, Affleck "wraps his arm around me, and comes over and tweaks my left boob". Affleck responded on Twitter: "I acted inappropriately toward Ms. Burton and I sincerely apologize." Annamarie Tendler, a makeup artist, said that Affleck "grabbed my ass at a Golden Globes party in 2014 ... He tried to play it like he was politely moving me out of the way." +In response to the allegations of sexual assault against Harvey Weinstein, Affleck pledged to donate any future profits from his early Miramax films to charities supporting victims of sexual assault, and said he had only been aware that Weinstein "was sleazy and a bully". In a tweet, actress Rose McGowan responded: "You lie." She said she met Affleck after being sexually assaulted by Weinstein during the Sundance Film Festival in 1997 and told him, while crying, that she had "just come from Harvey's and he said, 'Goddamnit, I told him to stop doing that.'" In a leaked email regarding McGowan's case, Affleck stated: "I never saw Rose at any hotel in Sundance. She never told me nor did I ever infer that she was attacked by anyone." In a 2019 interview, Affleck said: "I don't really want to get into other people's individual stories because I feel like those are their stories and they're entitled to tell as much or as little of those as they want. I believe Rose. I support her. I really like and admire her tenacity and I wish her the best." In 2020, McGowan clarified her comments: "It's not like I'm raging at Ben Affleck. I never said to him, 'I was just raped.' It's just more to illustrate the point of this continual thing of everybody knowing and everybody being part of it, unwittingly or proactively." +Affleck has appeared in more than 50 films and won many accolades throughout his career as an actor, writer, and director. He first gained recognition as a writer when he won the Golden Globe and the Academy Award for Best Original Screenplay for Good Will Hunting (1997), which he co-wrote with Matt Damon; Affleck remains the youngest person ever to win a Best Original Screenplay Oscar, at 25 years old. As an actor, he received Golden Globe nominations for his performances in Hollywoodland (2006) and The Tender Bar (2021). The film Argo (2012), which he directed, co-produced, and starred in, won him the Golden Globe Award, BAFTA, and Directors Guild Award for Best Director, as well as the Golden Globe Award, BAFTA, the Producers Guild Award, and the Academy Award for Best Picture. +Ben Affleck at IMDb +Appearances on C-SPAN +The 1920–21 season was Burnley's 29th season in the Football League, and their 4th consecutive campaign in the Football League First Division, the top tier of English football. Burnley were confident of success ahead of the season, having finished as First Division runners-up in 1919–20. After losing their first three games, Burnley embarked on a 30-match unbeaten league run from 4 September 1920 until 26 March 1921, winning the First Division and becoming English champions for the first time in their history. Burnley's unbeaten run stood as a single-season Football League record for over 80 years, until it was bettered by Arsenal in the 2003–04 season. Burnley ended the 1920–21 season on 59 points, having won 23 games, drawn 13, and lost 6. +The team reached the third round of the FA Cup, defeating Leicester City away and Queens Park Rangers at home, before unexpectedly losing away to Second Division side Hull City. Burnley won the East Lancashire Charity Cup, beating Blackburn Rovers 8–2 over two legs, but were eliminated from the Lancashire Senior Cup at the semi-final stage by Manchester City. As league champions, Burnley qualified for the Charity Shield, in which they were beaten 2–0 by FA Cup winners Tottenham Hotspur. Burnley also played two friendly matches during the season. The first, against Blackburn Rovers, marked the opening of Accrington Stanley's new stadium; the other was a benefit match for Patsy Gallacher, against a representative team from the Scottish Football League. +Burnley used 23 players during the season and had nine goalscorers. Their top scorer was Scottish forward Joe Anderson, with 31 competitive goals, including 25 in the league. Eight new players were signed by Burnley during the season, and eleven were released or transferred. Match attendances were the highest they had been at the club's home ground, Turf Moor, with an average gate of over 30,000. The highest attendance of the campaign was 42,653, who saw Burnley beat Bolton Wanderers 3–1 on 26 February 1921; the lowest was 22,000, for the match against Sunderland on the final day of the season. +The 1920–21 campaign was the second season of competitive football in England after the First World War. It was Burnley's 29th season in the Football League, and their 4th consecutive season in the Football League First Division, since promotion from the Football League Second Division in 1912–13. Burnley's chairman, Harry Windle, had been elected to the position in 1909, and manager John Haworth was marking his 11th consecutive year in charge. After finishing as runners-up to West Bromwich Albion in the First Division the previous season, there was a sense of eager anticipation within the club before the season began. Team captain Tommy Boyle claimed that his side was capable of building on its success of the previous season and winning the league championship, despite Burnley not having won a trophy since their FA Cup victory in 1914. +Burnley's only pre-season friendly games were two intramural practice matches, between the first team and the reserves, the second of which was watched by a crowd of around 10,000 at the club's home ground, Turf Moor. The team's last competitive match had ended in a 2–0 defeat against Manchester United in the Lancashire Senior Cup on 8 May 1920, almost four months earlier. The strip for 1920–21 was very little changed from that of the previous season; the claret jersey with light blue sleeves and a light blue stripe around the collar was kept along with the white shorts, but the claret socks were replaced by black. Minor works were carried out on the Turf Moor stadium, with improvements to the pavilion, club offices and referee's changing room. +The nucleus of the Burnley team remained unchanged from the previous campaign. Eight new players signed for the club, and eleven left during the season. New signings included goalkeeper Frank Birchenough from West Ham United and full-back Bob McGrory from Dumbarton. Also brought in were George Richardson from non-League side Horden Athletic and Tom Brophy from St Helens Town. Attackers Richard Cragg, Billy Clarkson and Patrick Norris were among those who left the club in pre-season. +Transfer activity continued after the season began. Full back Tom Bamford, who had not played a match for Burnley since before the First World War thanks to the emergence of Len Smelt, left the club and joined Rochdale in September 1920. West Bromwich Albion's Len Moorwood was signed in October 1920 to provide further goalkeeping backup. Inside forward Jack Lane was brought in from Cradley Heath in December 1920, followed by full back John Pearson from the same club two months later. Winger George Douglas was signed from Leicester City in February 1921. Thomas Jackson, who had made only one first-team appearance for Burnley, left the club in January 1921 to sign for Scottish side Dundee. Two players who had been signed at the beginning of the season left Burnley in April 1921; McGrory moved to Stoke City on a free transfer and Birchenough was released after playing two league matches. Bert Freeman left Burnley at the end of the season after nine years service, by which time he had become the club's all-time leading goalscorer. +Burnley's league campaign began on 28 August 1920 with a home match against Bradford City, who had finished 15th in the league in 1919–20. The season began inauspiciously for Burnley as they lost 4–1, but the match did see eventual top scorer Joe Anderson score the first of his 25 league goals of the campaign. Two further away defeats followed, 1–0 at newly promoted Huddersfield Town and 2–0 at Bradford City, leaving Burnley at the bottom of the league table. This run of defeats was Burnley's worst start to a league season since 1906–07, when they also lost their opening three matches. Haworth subsequently made several changes to the team; goalkeeper Jerry Dawson and full back Tommy Boyle were reinstated, and Freeman and James Lindsay were dropped in favour of Billy Nesbitt and Benny Cross respectively. Burnley picked up their first win on 6 September 1920, beating Huddersfield Town 3–0 at Turf Moor with goals from Bob Kelly, Boyle, and Nesbitt. The Burnley News praised the team's "brilliant forward work" and singled out Boyle for particular praise, while noting that Cross had made an "excellent impression" on his debut. Burnley went on to defeat Middlesbrough 2–1 at home, and to draw with them 0–0 at Ayresome Park. On 25 September 1920, four players scored in a 4–0 victory against a weakened Chelsea team, giving Burnley their third successive home win. +October began with a 1–1 draw with Chelsea at Stamford Bridge, after which the team recorded four straight wins throughout the remainder of the month. Burnley achieved home and away victories against Bradford (Park Avenue), and repeated that feat in the next two matches against Tottenham Hotspur. The 2–0 victory on 30 October 1920 was the first time in the season that any team had been able to prevent Tottenham from scoring in a league match, but Burnley's performance at White Hart Lane was met with disapproval from the correspondent from the Daily Mail, who considered they had set out to stifle their opponents and in doing so "spoilt the match". Burnley's winning streak carried on into November, with goals from Kelly, Boyle and Cross helping the side to defeat Newcastle United 3–1 at home and 2–1 away to lift them to second place in the league. The home fixture was marred by tragedy when a charabanc transporting Burnley supporters from Grassington overturned, killing five people. After the next game, a 2–2 draw with Oldham Athletic at Boundary Park on 20 November 1920, Burnley moved to the top of the table on goal average, 11 weeks after having been at the bottom. In the return match at Turf Moor a week later, Oldham were comfortably beaten 7–1; Kelly scored four goals, and the others were added by Cross (two goals) and Boyle. +A win and a draw against Liverpool, followed by a 2–0 victory over Preston North End, took Burnley into the Christmas period three points clear at the top of the league. The convincing 6–0 win over Sheffield United on Christmas Day, in which Anderson scored four times, set a new club record unbeaten streak of 17 games, beating the record set during the 1897–98 campaign. Burnley continued their good form into 1921, beating Preston North End away before achieving two victories against local rivals Blackburn Rovers. The first of these wins, a 4–1 success, was watched by 41,534 spectators, the biggest home crowd of the season until then. The victory was followed by a 3–1 away win a week later. On 5 February 1921, Anderson scored five goals in a "brilliant" performance against Aston Villa as Burnley recorded their second 7–1 win of the season. The result saw Burnley equal the Football League record of 22 matches unbeaten, held by Sheffield United and Preston North End. A new league unbeaten record was set with a 0–0 draw with Aston Villa four days later. Despite losing George Halley to an illness which forced him to miss the remainder of the season, Burnley secured a late home win over Derby County in the following game, sending Derby to the bottom of the league table. February ended with a 3–1 victory against Bolton Wanderers in front of a crowd of 42,653, the largest league attendance ever at Turf Moor at the time. The team took four points from the next three matches. Burnley firstly drew 1–1 away against a Bolton Wanderers team who were unbeaten at home, in front of a then-record crowd of 54,609 at Burnden Park, before beating Arsenal 1–0 at Turf Moor. The following week Burnley secured an away draw at Arsenal, despite their opponents attacking for much of the game. +A late Cross goal gave Burnley a 1–0 win over Manchester United at Turf Moor, extending the team's unbeaten record to 30 matches. In the next match, Burnley lost a league fixture for the first time since 4 September 1920 when they were beaten 3–0 by Manchester City at Hyde Road. City were also challenging for the league title and eventually finished as runners-up. More than 37,000 spectators attended the match and several people were injured as the stadium became overcrowded. Burnley followed up the setback with a 3–0 win away at Manchester United; strong winds and a boggy playing surface led Burnley to adapt their usual short passing game, with good effect. A win over Manchester City the following weekend took the total number of league wins in the season to 23. Beset by injuries to the forward line, the team suffered its fifth league defeat of the campaign on 9 April 1921, losing 2–0 away at West Bromwich Albion. +Burnley played West Brom again seven days later, drawing 1–1. The team went into the match away at Everton on 23 April 1921 needing a draw to clinch the league championship with three games remaining. Everton took the lead 15 minutes into the game, but Cross scored the equalising goal three minutes later, and Burnley held on to become English champions for the first time in their history. Local newspapers were effusive in their praise of the Burnley side, calling them "the greatest team that ever was". The return fixture at Turf Moor the following week also finished 1–1, Cross again scoring the Burnley goal. In the penultimate game, Burnley were beaten 1–0 away at Sunderland, their sixth and final league defeat of the season. The campaign ended with a 2–2 draw against Sunderland on 7 May 1921 in front of a season-lowest crowd of 22,000. The draw took Burnley to a tally of 59 points, five points clear of second-placed Manchester City, and one short of West Bromwich Albion's then-record total of 60 points set in 1919–20. +At half time during the final match of the season, the championship trophy was paraded around the Turf Moor pitch accompanied by a marching band. After the full-time whistle was blown, supporters swarmed the pitch to celebrate the team's success. The Football League president, John McKenna, made the official presentation of the trophy to the Burnley captain Boyle and congratulated the side on their achievement, particularly praising "their splendid training and their beautiful football". Medals were awarded to the manager Haworth, the club trainer Charlie Bates and the eleven players who had featured in the match against Sunderland. Three more medals were later awarded to Eddie Mosscrop, David Taylor and Alf Basnett. +Key +Results +Source: +Burnley's first match outside the league in the 1920–21 season was a friendly on 22 September 1920 against a Blackburn Rovers XI to mark the opening of Accrington Stanley's new stadium at Peel Park. Burnley won the game 10–1, with seven goals from Anderson as well as strikes from Lindsay, Walter Weaver and Jackson. Burnley's opening game in the FA Cup was an away tie at Leicester City on 8 January 1921. Kelly scored for the fourth game in succession and Anderson scored four goals as Burnley won the match 7–3, the first time the team had scored seven goals in a competitive match away from home. After the match, the Athletic News described Burnley as the best team in the country. +Following the win over Leicester City, Burnley were drawn against Queens Park Rangers at Turf Moor in the second round. Despite a good performance by their opponents, Burnley progressed to the third round with a 4–2 win, in which Kelly struck twice in the first half and Anderson scored either side of half time. In the third round, Burnley were handed an away tie at Hull City, who were struggling in the Second Division and had won only two league matches in the previous five months. Despite being without first-team regulars Halley and Anderson through injury, Burnley were expected to win comfortably, not having lost since 4 September 1920. Hull City played above all expectations and Burnley suffered a 3–0 defeat. +In April 1921, Burnley won the East Lancashire Charity Cup for the second consecutive season, beating Blackburn Rovers 8–2 on aggregate over two legs. The first leg was won 6–2 at Turf Moor thanks to goals from Mosscrop, Anderson, Cross and a hat-trick from Kelly, before strikes from Anderson and Lindsay gave Burnley a 2–0 win at Ewood Park. This success was followed by a 2–1 friendly victory over a Scottish Football League XI in a benefit match for Celtic winger Patsy Gallacher. Burnley also participated in the Lancashire Senior Cup; the league matches against Blackburn Rovers and Preston North End also counted as group stage matches in the competition. The team won all four of these games to qualify for the semi-finals, where they were drawn against Manchester City. The game took place on 9 May 1921, two days after the end of the league season. This meant Burnley were without captain Boyle, whose contract had expired following the match against Sunderland. Other changes were also made to the starting line-up, including the selection of back-up goalkeeper Moorwood in place of Dawson. Burnley lost the game 2–0, and their opponents went on to win the trophy that year. +As champions of the Football League, Burnley qualified for the Charity Shield. Burnley's opponents were Tottenham Hotspur, who had finished sixth in the First Division and beaten Wolverhampton Wanderers in the FA Cup final. In what was the last match of the 1920–21 campaign, Burnley fell to a 2–0 defeat at White Hart Lane on 16 May 1921. The game was not without controversy, as the Burnley players claimed that the second goal should have been disallowed despite the Tottenham goalscorer being onside, maintaining that the whistle had already been blown by the referee. +Key +Results +Burnley manager Haworth used 24 different players during the 1920–21 season, and there were nine different goalscorers. The team played in a 2–3–5 formation throughout the campaign, with two fullbacks, three halfbacks, two outside forwards, two inside forwards and a centre forward. Billy Watson played the highest number of games, appearing in all 45 First Division and FA Cup matches. Nesbitt and Anderson each played 43 times. Anderson was the top goalscorer for Burnley in the campaign with 33 competitive goals, including 25 in the league, the highest total since Freeman's 36 goals in 1912–13. With a tally of 26 goals, Kelly was the second-highest scorer, followed by Cross with 16, including the goal that won the title for Burnley. Club captain Boyle was the highest-scoring full back, with seven goals in 38 league appearances. Winger Nesbitt scored five times during the season. +England international goalkeeper Dawson missed three games, two because of an injury sustained in the opening match. Centre forward Freeman, who had become the first ever Burnley player to score 100 league goals, and held the club record for most goals in a season, played only four matches. His final appearance for Burnley came in the FA Cup third round defeat to Hull City. Several players made bit-part contributions to the campaign; Brophy and McGrory made four and three first-team appearances respectively and Birchenough, Douglas, Lane, Moorwood, Bill Taylor and Billy Morgan played in two matches or fewer. Richardson and Pearson, both new signings in the 1920–21 season, failed to make a first-team appearance for Burnley during the campaign. Jackson was a squad member until January 1921, but did not play any league or cup games for Burnley in the 1920–21 campaign. +In a review of the season, the correspondent from the Burnley News described Burnley's play as "some of the finest football ever served up to any crowd". He attributed the team's success partly to their thorough training, noting that the Burnley players were often able to outlast their opponents in stamina, and partly to luck and a lack of serious injuries during the season. The team's successes on the playing field led to a marked increase in attendance figures; the total number of paying spectators was over 250,000 higher than the 1919–20 season, and gate receipts for league matches totalled a club record £34,000 (equivalent to £1,440,000 in 2025). +The Burnley board had planned a tour of Spain to take place shortly after the culmination of the league campaign; the trip had to be cancelled when the club received notice from the Spanish Football Federation that one of their intended opponents, Barcelona, had been suspended from all matches. Further tours of Norway and France were then arranged, but these also had to be abandoned after the Football Association refused to grant permission. At the end of the season five players were given free transfers by the club; Smelt and Birchenough, who had both played for Burnley during the campaign, and three reserve players. Moreover, two players—Freeman and George Thompson—were placed on the transfer list by the manager. New terms were agreed with top goalscorer Anderson and wing-half Watson to keep them at the club. +Burnley's 30-game unbeaten streak during the 1920–21 season stood as a Football League record for 83 years until it was surpassed by Arsenal, who completed the entire 2003–04 season without losing a league match. After the defeat to Bradford City on 28 August 1920, Burnley did not lose again at Turf Moor until 11 February 1922, when they were beaten 2–1 by Blackburn Rovers. In the same match, the halfback line of Boyle, Halley and Watson—used in 25 first-team games during 1920–21—was seen for the final time, having played together for the first time in September 1913. The majority of the championship-winning team remained intact going into the 1921–22 season, although players such as David Taylor and Weaver found themselves less involved in first-team matches. +Works cited +Lee, Edward; Whalley, Phil (2002). The Pride and Glory: 120 Years of Burnley Football Club. Burnley Football Club. ASIN B003GE002K. +Simpson, Ray (2007). The Clarets Chronicles: The Definitive History of Burnley Football Club 1882–2007. Burnley Football Club. ISBN 978-0-9557468-0-2. +Specific +Archimedes of Syracuse ( AR-kih-MEE-deez; c. 287 – c. 212 BC) was an Ancient Greek mathematician, physicist, engineer, astronomer, and inventor from the city of Syracuse in Sicily. Although few details of his life are known, based on his surviving work, he is considered one of the leading scientists in classical antiquity, and one of the greatest mathematicians of all time. Archimedes anticipated modern calculus and analysis by applying the concept of the infinitesimals and the method of exhaustion to derive and rigorously prove many geometrical theorems, including the area of a circle, the surface area and volume of a sphere, the area of an ellipse, the area under a parabola, the volume of a segment of a paraboloid of revolution, the volume of a segment of a hyperboloid of revolution, and the area of a spiral. +Archimedes' other mathematical achievements include deriving an approximation of pi (π), defining and investigating the Archimedean spiral, and devising a system using exponentiation for expressing very large numbers. He was also one of the first to apply mathematics to physical phenomena, working on statics and hydrostatics. Archimedes' achievements in this area include a proof of the law of the lever, the widespread use of the concept of center of gravity, and the enunciation of the law of buoyancy known as Archimedes' principle. In astronomy, he made measurements of the apparent diameter of the Sun and the size of the universe. He is also said to have built a planetarium device that demonstrated the movements of the known celestial bodies, and may have been a precursor to the Antikythera mechanism. He is also credited with designing innovative machines, such as his screw pump, compound pulleys, and defensive war machines to protect his native Syracuse from invasion. +Archimedes died during the siege of Syracuse, when he was killed by a Roman soldier despite orders that he should not be harmed. Cicero describes visiting Archimedes' tomb, which was surmounted by a sphere and a cylinder that Archimedes requested be placed there to represent his most valued mathematical discovery. +Unlike his inventions, Archimedes' mathematical writings were little known in antiquity. Alexandrian mathematicians read and quoted him, but the first comprehensive compilation was not made until c. 530 AD by Isidore of Miletus in Byzantine Constantinople, while Eutocius' commentaries on Archimedes' works in the same century opened them to wider readership for the first time. In the Middle Ages, Archimedes' work was translated into Arabic in the 9th century and then into Latin in the 12th century, and were an influential source of ideas for scientists during the Renaissance and in the Scientific Revolution. The discovery in 1906 of works by Archimedes in the Archimedes Palimpsest has provided new insights into how he obtained mathematical results. +The details of Archimedes's life are obscure; a biography of Archimedes mentioned by Eutocius was allegedly written by his friend Heraclides Lembus, but this work has been lost, and modern scholarship is doubtful that it was written by Heraclides to begin with. +Based on a statement by the Byzantine Greek scholar John Tzetzes that Archimedes lived for 75 years before his death in 212 BC, Archimedes is estimated to have been born c. 287 BC in the seaport city of Syracuse, Sicily, at that time a self-governing colony in Magna Graecia. In the Sand-Reckoner, Archimedes gives his father's name as Phidias, an astronomer about whom nothing else is known; Plutarch wrote in his Parallel Lives that Archimedes was related to King Hiero II, the ruler of Syracuse, although Cicero and Silius Italicus suggest he was of humble origin. It is also unknown whether he ever married or had children, or if he ever visited Alexandria, Egypt, during his youth; though his surviving written works, addressed to Dositheus of Pelusium, a student of the Alexandrian astronomer Conon of Samos, and to the head librarian Eratosthenes of Cyrene, suggested that he maintained collegial relations with scholars based there. In the preface to On Spirals addressed to Dositheus, Archimedes says that "many years have elapsed since Conon's death." Conon of Samos lived c. 280 – c. 220 BC, suggesting that Archimedes may have been an older man when writing some of his works. +Another story of a problem that Archimedes is credited solving with in service of Hiero II is the "wreath problem". According to Vitruvius, writing about two centuries after Archimedes' death, King Hiero II of Syracuse had commissioned a golden wreath for a temple to the immortal gods, and had supplied pure gold to be used by the goldsmith. However, the king had begun to suspect that the goldsmith had substituted some cheaper silver and kept some of the pure gold for himself, and, unable to make the smith confess, asked Archimedes to investigate. Later, while stepping into a bath, Archimedes allegedly noticed that the level of the water in the tub rose more the lower he sank in the tub and, realizing that this effect could be used to determine the golden crown's volume, was so excited that he took to the streets naked, having forgotten to dress, crying "Eureka!", meaning "I have found [it]!" According to Vitruvius, Archimedes then took a lump of gold and a lump of silver that were each equal in weight to the wreath, and, placing each in the bathtub, showed that the wreath displaced more water than the gold and less than the silver, demonstrating that the wreath was gold mixed with silver. +A different account is given in the Carmen de Ponderibus, an anonymous 5th-century Latin didactic poem on weights and measures once attributed to the grammarian Priscian. In this poem, the lumps of gold and silver were placed on the scales of a balance, and then the entire apparatus was immersed in water; the difference in density between the gold and the silver, or between the gold and the crown, causes the scale to tip accordingly. Unlike the more famous bathtub account given by Vitruvius, this poetic account uses the hydrostatics principle now known as Archimedes' principle that is found in his treatise On Floating Bodies, where a body immersed in a fluid experiences a buoyant force equal to the weight of the fluid it displaces. Galileo Galilei, who invented a hydrostatic balance in 1586 inspired by Archimedes' work, considered it "probable that this method is the same that Archimedes followed, since, besides being very accurate, it is based on demonstrations found by Archimedes himself." +A large part of Archimedes' work in engineering probably arose from fulfilling the needs of his home city of Syracuse. Athenaeus of Naucratis in his Deipnosophistae quotes a certain Moschion for a description on how King Hiero II commissioned the design of a huge ship, the Syracusia, which is said to have been the largest ship built in classical antiquity and, according to Moschion's account, it was launched by Archimedes. Plutarch tells a slightly different account, relating that Archimedes boasted to Hiero that he was able to move any large weight, at which point Hiero challenged him to move a ship. These accounts contain many fantastic details that are historically implausible, and the authors of these stories provide conflicting ideas about how this task was accomplished: Plutarch states that Archimedes constructed a block-and-tackle pulley system, while Hero of Alexandria attributed the same boast to Archimedes' invention of the baroulkos, a kind of windlass. Pappus of Alexandria attributed this feat, instead, to Archimedes' use of mechanical advantage, the principle of leverage to lift objects that would otherwise have been too heavy to move, attributing to him the oft-quoted remark: "Give me a place to stand on, and I will move the Earth." +Athenaeus, likely garbling the details of Hero's account of the baroulkos, also mentions that Archimedes used a "screw" in order to remove any potential water leaking through the hull of the Syracusia. Although this device is sometimes referred to as Archimedes' screw, it likely predates him by a significant amount, and none of his closest contemporaries who describe its use (Philo of Byzantium, Strabo, and Vitruvius) credit him with its use. +The greatest reputation Archimedes earned during antiquity was for the defense of his city from the Romans during the Siege of Syracuse. According to Plutarch, Archimedes had constructed war machines for Hiero II, but had never been given an opportunity to use them during Hiero's lifetime. In 214 BC, however, during the Second Punic War, when Syracuse switched allegiances from Rome to Carthage, the Roman army under Marcus Claudius Marcellus attempted to take the city, Archimedes allegedly personally oversaw the use of these war machines in the defense of the city, greatly delaying the Romans, who were able to capture the city only after a long siege. Three different historians, Plutarch, Livy, and Polybius provide testimony about these war machines, describing improved catapults, cranes that dropped heavy pieces of lead on the Roman ships or which used an iron claw to lift them out of the water, dropping them back in so that they sank. +A much more improbable account, not found in any of the three earliest accounts (Plutarch, Polybius, or Livy) describes how Archimedes used "burning mirrors" to focus the sun's rays onto the attacking Roman ships, setting them on fire. The earliest account to mention ships being set on fire, by the 2nd century CE satirist Lucian of Samosata, does not mention mirrors, and only says the ships were set on fire by artificial means, which may imply that burning projectiles were used. The first author to mention mirrors is Galen, writing later in the same century. Nearly four hundred years after Lucian and Galen, Anthemius, despite skepticism, tried to reconstruct Archimedes' hypothetical reflector geometry. The purported device, sometimes called "Archimedes' heat ray", has been the subject of an ongoing debate about its credibility since the Renaissance. René Descartes rejected it as false, while modern researchers have attempted to recreate the effect using only the means that would have been available to Archimedes, with mixed results. +There are several divergent accounts of Archimedes' death during the sack of Syracuse after it fell to the Romans: The oldest account, from Livy, says that, while drawing figures in the dust, Archimedes was killed by a Roman soldier who did not know he was Archimedes. According to Plutarch, the soldier demanded that Archimedes come with him, but Archimedes declined, saying that he had to finish working on the problem, and the soldier killed Archimedes with his sword. Another story from Plutarch has Archimedes carrying mathematical instruments before being killed because a soldier thought they were valuable items. Another Roman writer, Valerius Maximus (fl. 30 AD), wrote in Memorable Doings and Sayings that Archimedes' last words as the soldier killed him were "... but protecting the dust with his hands, said 'I beg of you, do not disturb this.'" which is similar to the last words now commonly attributed to him, "Do not disturb my circles", which do not appear in any ancient sources. +Marcellus was reportedly angered by Archimedes' death, as he considered him a valuable scientific asset (he called Archimedes "a geometrical Briareus") and had ordered that he should not be harmed. Cicero (106–43 BC) mentions that Marcellus brought to Rome two planetariums Archimedes built, which were constructed by Archimedes and which showed the motion of the Sun, Moon and five planets, one of which he donated to the Temple of Virtue in Rome, and the other he allegedly kept as his only personal loot from Syracuse." Pappus of Alexandria reports on a now lost treatise by Archimedes On Sphere-Making, which may have dealt with the construction of these mechanisms. Constructing mechanisms of this kind would have required a sophisticated knowledge of differential gearing, which was once thought to have been beyond the range of the technology available in ancient times, but the discovery in 1902 of the Antikythera mechanism, another device built c. 100 BC designed with a similar purpose, has confirmed that devices of this kind were known to the ancient Greeks, with some scholars regarding Archimedes' device as a precursor. +While serving as a quaestor in Sicily, Cicero himself found what was presumed to be Archimedes' tomb near the Agrigentine gate in Syracuse, in a neglected condition and overgrown with bushes. Cicero had the tomb cleaned up and was able to see the carving and read some of the verses that had been added as an inscription. The tomb carried a sculpture illustrating Archimedes' favorite mathematical proof, that the volume and surface area of the sphere are two-thirds that of an enclosing cylinder including its bases. +While he is often regarded as a designer of mechanical devices, Archimedes also made contributions to the field of mathematics, both in applying the techniques of his predecessors to obtain new results, and developing new methods of his own. +In Quadrature of the Parabola, Archimedes states that a certain proposition in Euclid's Elements demonstrating that the area of a circle is proportional to its diameter was proven using a lemma now known as the Archimedean property, that “the excess by which the greater of two unequal regions exceed the lesser, if added to itself, can exceed any given bounded region.” Prior to Archimedes, Eudoxus of Cnidus and other earlier mathematicians applied this lemma, a technique now referred to as the "method of exhaustion," to find the volume of a tetrahedron, cylinder, cone, and sphere, for which proofs are given in book XII of Euclid's Elements. +In Measurement of a Circle, Archimedes employed this method to show that the area of a circle is the same as a right triangle whose base and height are equal to its radius and circumference. He then approximated the ratio between the radius and the circumference, the value of π, by drawing a larger regular hexagon outside a circle then a smaller regular hexagon inside the circle, and progressively doubling the number of sides of each regular polygon, calculating the length of a side of each polygon at each step. As the number of sides increases, it becomes a more accurate approximation of a circle. After four such steps, when the polygons had 96 sides each, he was able to determine that the value of π lay between 3⁠1/7⁠ (approx. 3.1429) and 3⁠10/71⁠ (approx. 3.1408), consistent with its actual value of approximately 3.1416. In the same treatise, he also asserts that the value of the square root of 3 as lying between ⁠265/153⁠ (approximately 1.7320261) and ⁠1351/780⁠ (approximately 1.7320512), which he may have derived from a similar method. +In Quadrature of the Parabola, Archimedes used this technique to prove that the area enclosed by a parabola and a straight line is ⁠4/3⁠ times the area of a corresponding inscribed triangle as shown in the figure at right, expressing the solution to the problem as an infinite geometric series with the common ratio ⁠1/4⁠: +∑n=0∞4−n=1+4−1+4−2+4−3+⋯=43. +If the first term in this series is the area of the triangle, then the second is the sum of the areas of two triangles whose bases are the two smaller secant lines, and whose third vertex is where the line that is parallel to the parabola's axis and that passes through the midpoint of the base intersects the parabola, and so on. This proof uses a variation of the series 1/4 + 1/16 + 1/64 + 1/256 + · · · which sums to ⁠1/3⁠. +He also used this technique in order to measure the surface areas of a sphere and cone, to calculate the area of an ellipse, and to find the area contained within an Archimedean spiral. +For it is more feasible, having already in one’s possession, through the method, of a knowledge of some sort of the matters under investigation, to provide the proof – rather than investigating it, knowing nothing. +In addition to developing on the works of earlier mathematicians with the method of exhaustion, Archimedes also pioneered a novel technique using the law of the lever in order to measure the area and volume of shapes using physical means. He first gives an outline of this proof in Quadrature of the Parabola alongside the geometric proof, but he gives a fuller explanation in The Method of Mechanical Theorems. According to Archimedes, he proved the results in his mathematical treatises first using this method, and then worked backwards, applying the method of exhaustion only after he had already calculated an approximate value for the answer. +Archimedes also developed methods for representing large numbers. +In The Sand Reckoner, Archimedes devised a system of counting based on the myriad, the Greek term for the number 10,000, in order to calculate a number that was greater than the grains of sand needed to fill the universe. He proposed a number system using powers of a myriad of myriads (100 million, i.e., 10,000 x 10,000) and concluded that the number of grains of sand required to fill the universe would be 8 vigintillion, or 8×1063. In doing so, he demonstrated that mathematics could represent arbitrarily large numbers. +In the Cattle Problem, Archimedes challenges the mathematicians at the Library of Alexandria to count the numbers of cattle in the Herd of the Sun, which involves solving a number of simultaneous Diophantine equations. A more difficult version of the problem in which some of the answers are required to be square numbers, and the answer is a very large number, approximately 7.760271×10206544. +In a lost work described by Pappus of Alexandria, Archimedes proved that there are exactly thirteen semiregular polyhedra. +Archimedes made his work known through correspondence with mathematicians in Alexandria, which were originally written in Doric Greek, the dialect of ancient Syracuse. +The following are ordered chronologically based on new terminological and historical criteria set by Knorr (1978) and Sato (1986). +This is a short work consisting of three propositions. It is written in the form of a correspondence with Dositheus of Pelusium, who was a student of Conon of Samos. In Proposition II, Archimedes gives an approximation of the value of pi (π), showing that it is greater than ⁠223/71⁠ (3.1408...) and less than ⁠22/7⁠ (3.1428...). +In this treatise, also known as Psammites, Archimedes finds a number that is greater than the grains of sand needed to fill the universe. This book mentions the heliocentric theory of the Solar System proposed by Aristarchus of Samos, as well as contemporary ideas about the size of the Earth and the distance between various celestial bodies, and attempts to measure the apparent diameter of the Sun. By using a system of numbers based on powers of the myriad, Archimedes concludes that the number of grains of sand required to fill the universe is 8×1063 in modern notation. The introductory letter states that Archimedes' father was an astronomer named Phidias. The Sand Reckoner is the only surviving work in which Archimedes discusses his views on astronomy. +Archimedes discusses astronomical measurements of the Earth, Sun, and Moon, as well as Aristarchus' heliocentric model of the universe, in the Sand-Reckoner. Without the use of either trigonometry or a table of chords, Archimedes determines the Sun's apparent diameter by first describing the procedure and instrument used to make observations (a straight rod with pegs or grooves), applying correction factors to these measurements, and finally giving the result in the form of upper and lower bounds to account for observational error. +Ptolemy, quoting Hipparchus, also references Archimedes' solstice observations in the Almagest. This would make Archimedes the first known Greek to have recorded multiple solstice dates and times in successive years. +There are two books to On the Equilibrium of Planes: the first contains seven postulates and fifteen propositions, while the second book contains ten propositions. In the first book, Archimedes proves the law of the lever, which states that: +Magnitudes are in equilibrium at distances reciprocally proportional to their weights. +Earlier descriptions of the principle of the lever are found in a work by Euclid and in the Mechanical Problems, belonging to the Peripatetic school of the followers of Aristotle, the authorship of which has been attributed by some to Archytas. +Archimedes uses the principles derived to calculate the areas and centers of gravity of various geometric figures including triangles, parallelograms and parabolas. +In this work of 24 propositions addressed to Dositheus, Archimedes proves by two methods that the area enclosed by a parabola and a straight line is 4/3 the area of a triangle with equal base and height. He achieves this by two different methods: first by applying the law of the lever, and by calculating the value of a geometric series that sums to infinity with the ratio 1/4. +In this two-volume treatise addressed to Dositheus, Archimedes obtains the result of which he was most proud, namely the relationship between a sphere and a circumscribed cylinder of the same height and diameter. The volume is ⁠4/3⁠πr3 for the sphere, and 2πr3 for the cylinder. The surface area is 4πr2 for the sphere, and 6πr2 for the cylinder (including its two bases), where r is the radius of the sphere and cylinder. +This work of 28 propositions is also addressed to Dositheus. The treatise defines what is now called the Archimedean spiral. It is the locus of points corresponding to the locations over time of a point moving away from a fixed point with a constant speed along a line which rotates with constant angular velocity. Equivalently, in modern polar coordinates (r, θ), it can be described by the equation r=a+bθ with real numbers a and b. +This is an early example of a mechanical curve (a curve traced by a moving point) considered by a Greek mathematician. +This is a work in 32 propositions addressed to Dositheus. In this treatise Archimedes calculates the areas and volumes of sections of cones, spheres, and paraboloids. +There are two books of On Floating Bodies. In the first book, Archimedes spells out the law of equilibrium of fluids and proves that water will adopt a spherical form around a center of gravity. +Archimedes' principle of buoyancy is given in this work, stated as follows: +Any body wholly or partially immersed in fluid experiences an upthrust equal to, but opposite in direction to, the weight of the fluid displaced. +In the second part, he calculates the equilibrium positions of sections of paraboloids. This was probably an idealization of the shapes of ships' hulls. Some of his sections float with the base under water and the summit above water, similar to the way that icebergs float. +Also known as Loculus of Archimedes or Archimedes' Box, this is a dissection puzzle similar to a Tangram, and the treatise describing it was found in more complete form in the Archimedes Palimpsest. Archimedes calculates the areas of the 14 pieces, which can be assembled to form a square. Reviel Netz of Stanford University argued in 2003 that Archimedes was attempting to determine how many ways the pieces could be assembled into the shape of a square. Netz calculates that the pieces can be made into a square 17,152 ways. The number of arrangements is 536 when solutions that are equivalent by rotation and reflection are excluded. The puzzle represents an example of an early problem in combinatorics. +The origin of the puzzle's name is unclear, and it has been suggested that it is taken from the Ancient Greek word for "throat" or "gullet", stomachos (στόμαχος). Ausonius calls the puzzle Ostomachion, a Greek compound word formed from the roots of osteon (ὀστέον, 'bone') and machē (μάχη, 'fight'). +In this work, addressed to Eratosthenes and the mathematicians in Alexandria, Archimedes challenges them to count the numbers of cattle in the Herd of the Sun, which involves solving a number of simultaneous Diophantine equations. Gotthold Ephraim Lessing discovered this work in a Greek manuscript consisting of a 44-line poem in the Herzog August Library in Wolfenbüttel, Germany, in 1773. There is a more difficult version of the problem in which some of the answers are required to be square numbers. A. Amthor first solved this version of the problem in 1880, and the answer is a very large number, approximately 7.760271×10206544. +As with The Cattle Problem, The Method of Mechanical Theorems was written in the form of a letter to Eratosthenes in Alexandria. +In this work Archimedes uses a novel method, an early form of Cavalieri's principle, to rederive the results from the treatises sent to Dositheus (Quadrature of the Parabola, On the Sphere and Cylinder, On Spirals, On Conoids and Spheroids) that he had previously used the method of exhaustion to prove, using the law of the lever he applied in On the Equilbrium of Planes in order to find the center of gravity of an object first, and reasoning geometrically from there in order to more easily derive the volume of an object. Archimedes states that he used this method to derive the results in the treatises sent to Dositheus before he proved them more rigorously with the method of exhaustion, stating that it is useful to know that a result is true before proving it rigorously, much as Eudoxus of Cnidus was aided in proving that the volume of a cone is one-third the volume of cylinder by knowing that Democritus had already asserted it to be true on the argument that this is true by the fact that the pyramid has one-third the rectangular prism of the same base. +This treatise was thought lost until the discovery of the Archimedes Palimpsest in 1906. +Archimedes' Book of Lemmas or Liber Assumptorum is a treatise with 15 propositions on the nature of circles. The earliest known copy of the text is in Arabic. T. L. Heath argued that it cannot have been written by Archimedes in its current form, since it quotes Archimedes by name, suggesting the lemmas were compiled by a later author. He thought its ideas, like the Arbelos, were Archimedean in origin. Tannery supposes that if such a work once existed, it had already been lost by the time of Pappus, who put similar lemmas in the Collection, calling them, "ancient and from various works". +Other questionable attributions to Archimedes' work include the Latin poem Carmen de ponderibus et mensuris (4th or 5th century), which describes the use of a hydrostatic balance, to solve the problem of the crown, and the 12th-century text Mappae clavicula, which contains instructions on how to perform assaying of metals by calculating their specific gravities. +Many written works by Archimedes have not survived or are only extant in heavily edited fragments: Pappus of Alexandria mentions On Sphere-Making, as well as a work on semiregular polyhedra, and another work on spirals, while Theon of Alexandria quotes a remark about refraction from the now-lost Catoptrica. Principles, addressed to Zeuxippus, explained the number system used in The Sand Reckoner; there are also On Balances; On Centers of Gravity. +Scholars in the medieval Islamic world also attribute to Archimedes a formula for calculating the area of a triangle from the length of its sides, which today is known as Heron's formula due to its first known appearance in the work of Heron of Alexandria in the 1st century AD, and may have been proven in a lost work of Archimedes that is no longer extant. +In 1906, the Danish professor Johan Ludvig Heiberg visited Constantinople to examine a 174-page goatskin parchment of prayers, written in the 13th century, after reading a short transcription published seven years earlier (in 1899) by Papadopoulos-Kerameus. He confirmed that it was indeed a palimpsest, a document with text that had been written over an erased older work. Palimpsests were created by scraping the ink from existing works and reusing them, a common practice in the Middle Ages, as vellum was expensive. The older works in the palimpsest were identified by scholars as 10th-century copies of previously lost treatises by Archimedes. The palimpsest holds seven treatises, including the only surviving copy of On Floating Bodies in the original Greek. It is the only known source of The Method of Mechanical Theorems, referred to by Suidas and thought to have been lost forever. Stomachion was also discovered in the palimpsest, with a more complete analysis of the puzzle than had been found in previous texts. +The treatises in the Archimedes Palimpsest include: +On the Equilibrium of Planes +On Spirals +Measurement of a Circle +On the Sphere and Cylinder +On Floating Bodies +The Method of Mechanical Theorems +Stomachion +Speeches by the 4th century BC politician Hypereides +A commentary on Aristotle's Categories +Other works +The parchment spent hundreds of years in a monastery library in Constantinople before being sold to a private collector in the 1920s. On 29 October 1998, it was sold at auction to an anonymous buyer for a total of $2.2 million. The palimpsest was stored at the Walters Art Museum in Baltimore, Maryland, where it was subjected to a range of modern tests including the use of ultraviolet and X-ray light to read the overwritten text. It has since returned to its anonymous owner. +Sometimes called the father of mathematics and mathematical physics, historians of science and mathematics almost universally agree that Archimedes was the finest mathematician from antiquity. +The reputation that Archimedes had for mechanical inventions in classical antiquity is well-documented; Athenaeus recounts in his Deipnosophistae how Archimedes launched the largest known ship in antiquity, the Syracusia, while Apuleius talks about his work in catoptrics. Plutarch had claimed that Archimedes disdained mechanics and focused primarily on pure geometry, but this is generally considered to be a mischaracterization by modern scholarship, fabricated to bolster Plutarch's own Platonist values rather than to an accurate presentation of Archimedes. +Archimedes' work was translated into Arabic by Thābit ibn Qurra, and into Latin via Arabic by Gerard of Cremona. Direct Greek to Latin translations were later done by William of Moerbeke and Iacobus Cremonensis. +During the Renaissance, the Editio princeps (First Edition) was published in Basel in 1544 by Johann Herwagen with the works of Archimedes in Greek and Latin, which were an influential source of ideas for scientists during the Renaissance and again in the 17th century. +Leonardo da Vinci repeatedly expressed admiration for Archimedes, and attributed his invention Architonnerre to Archimedes. Galileo Galilei called him "superhuman" and "my master", while Christiaan Huygens said, "I think Archimedes is comparable to no one", consciously emulating him in his early work. Gottfried Wilhelm Leibniz said, "He who understands Archimedes and Apollonius will admire less the achievements of the foremost men of later times". +Italian numismatist and archaeologist Filippo Paruta and Leonardo Agostini reported on a bronze coin in Sicily with the portrait of Archimedes on the obverse and a cylinder and sphere with the monogram ARMD in Latin on the reverse. Although the coin is now lost and its date is not precisely known, Ivo Schneider described the reverse as "a sphere resting on a base – probably a rough image of one of the planetaria created by Archimedes," and suggested it might have been minted in Rome for Marcellus who "according to ancient reports, brought two spheres of Archimedes with him to Rome". +Carl Friedrich Gauss's heroes were Archimedes and Newton, and Moritz Cantor, who studied under Gauss in the University of Göttingen, reported that he once remarked in conversation that "there had been only three epoch-making mathematicians: Archimedes, Newton, and Eisenstein". Likewise, Alfred North Whitehead said that "in the year 1500 Europe knew less than Archimedes who died in the year 212 BC." The historian of mathematics Reviel Netz, echoing Whitehead's proclamation on Plato and philosophy, said that "Western science is but a series of footnotes to Archimedes," calling him "the most important scientist who ever lived." and Eric Temple Bell, wrote that "Any list of the three 'greatest' mathematicians of all history would include the name of Archimedes. The other two usually associated with him are Newton and Gauss. Some, considering the relative wealth—or poverty—of mathematics and physical science in the respective ages in which these giants lived, and estimating their achievements against the background of their times, would put Archimedes first." +The discovery in 1906 of previously lost works by Archimedes in the Archimedes Palimpsest has provided new insights into how he obtained mathematical results. +The Fields Medal for outstanding achievement in mathematics carries a portrait of Archimedes, along with a carving illustrating his proof on the sphere and the cylinder. The inscription around the head of Archimedes is a quote attributed to 1st century AD poet Manilius, which reads in Latin: Transire suum pectus mundoque potiri ("Rise above oneself and grasp the world"). +The world's first seagoing steamship with a screw propeller was the SS Archimedes, which was launched in 1839 and named in honor of Archimedes and his work on the screw. +Archimedes has also appeared on postage stamps issued by East Germany (1973), Greece (1983), Italy (1983), Nicaragua (1971), San Marino (1982), and Spain (1963). +The exclamation of Eureka! attributed to Archimedes is the state motto of California. In this instance, the word refers to the discovery of gold near Sutter's Mill in 1848 which sparked the California gold rush. +There is a crater on the Moon named Archimedes (29.7°N 4.0°W / 29.7; -4.0) in his honor, as well as a lunar mountain range, the Montes Archimedes (25.3°N 4.6°W / 25.3; -4.6). +List of things named after Archimedes +Arbelos +Archimedean point +Archimedes number +Archimedes paradox +Methods of computing square roots +Salinon +Steam cannon +Twin circles +Zhang Heng +Plutarch, Life of Marcellus +"Athenaeus, Deipnosophistae". perseus.tufts.edu. Retrieved 7 March 2023. +Acerbi, Fabio (2018). "Hellenistic Mathematics". In Keyser, Paul T; Scarborough, John (eds.). Oxford Handbook of Science and Medicine in the Classical World. pp. 268–292. doi:10.1093/oxfordhb/9780199734146.013.69. ISBN 978-0-19-973414-6. Retrieved 26 May 2021. +Dijksterhuis, E. J. (Eduard Jan) (1987). Archimedes. Princeton, N.J. : Princeton University Press. ISBN 978-0-691-08421-3. Retrieved 30 April 2025. +Netz, Reviel (2022). A New History of Greek Mathematics. Cambridge University Press. ISBN 978-1-108-83384-4. +Clagett, Marshall. 1964–1984. Archimedes in the Middle Ages 5 volumes. Madison, WI: University of Wisconsin Press. +Clagett, Marshall. 1970. "Archimedes". In Charles Coulston Gillispie, ed. Dictionary of Scientific Biography. Vol. 1 (Abailard–Berg). New York: Charles Scribner's Sons. pp. 213–231. +Gow, Mary. 2005. Archimedes: Mathematical Genius of the Ancient World. Enslow Publishing. ISBN 978-0-7660-2502-8. +Hasan, Heather. 2005. Archimedes: The Father of Mathematics. Rosen Central. ISBN 978-1-4042-0774-5. +Netz, Reviel. 2004–2017. The Works of Archimedes: Translation and Commentary. 1–2. Cambridge University Press. Vol. 1: "The Two Books on the Sphere and the Cylinder". ISBN 978-0-521-66160-7. Vol. 2: "On Spirals". ISBN 978-0-521-66145-4. +Netz, Reviel, and William Noel. 2007. The Archimedes Codex. Orion Publishing Group. ISBN 978-0-297-64547-4. +Pickover, Clifford A. 2008. Archimedes to Hawking: Laws of Science and the Great Minds Behind Them. Oxford University Press. ISBN 978-0-19-533611-5. +Simms, Dennis L. 1995. Archimedes the Engineer. Continuum International Publishing Group. ISBN 978-0-7201-2284-8. +Stein, Sherman. 1999. Archimedes: What Did He Do Besides Cry Eureka?. Mathematical Association of America. ISBN 978-0-88385-718-2. +Heiberg's Edition of Archimedes. Texts in Classical Greek, with some in English. +Archimedes on In Our Time at the BBC +Works by Archimedes at Project Gutenberg +Works by or about Archimedes at the Internet Archive +Archimedes at the Indiana Philosophy Ontology Project +Archimedes at PhilPapers +The Archimedes Palimpsest project at The Walters Art Museum in Baltimore, Maryland +"Archimedes and the Square Root of 3". MathPages.com. +"Archimedes on Spheres and Cylinders". MathPages.com. +Testing the Archimedes steam cannon Archived 29 March 2010 at the Wayback Machine +Christgau's Record Guide: Rock Albums of the Seventies is a music reference book by American music journalist and essayist Robert Christgau. It was first published in October 1981 by Ticknor & Fields. The book compiles approximately 3,000 of Christgau's capsule album reviews, most of which were originally written for his "Consumer Guide" column in The Village Voice throughout the 1970s. The entries feature annotated details about each record's release and cover a variety of genres related to rock music. +Christgau's reviews are informed by an interest in the aesthetic and political dimensions of popular music, a belief that it could be consumed intelligently, and a desire to communicate his ideas to readers in an entertaining, provocative, and compact way. Many of the older reviews were rewritten for the guide to reflect his changed perspective and matured stylistic approach. He undertook an intense preparation process for the book during 1979 and 1980, which temporarily hindered both his awareness of current music and his marriage to fellow writer Carola Dibbell, whom he later credited as an influence on his work. +The guide was critically well received, earning praise for its extensive discography, Christgau's judgment, and his colorful writing. Reviewers also noted his opinionated tastes, analytical commentary, pithy language, and critical quips. A staple of rock-era reference works, Christgau's Record Guide became widely popular in libraries as a source for popular music studies and as an authoritative guide for fellow critics, record collectors, and music shops, influencing the development of critical standards for evaluating music. It later appeared on several expert lists of the best popular music literature. +Christgau's Record Guide has been reprinted several times in book form and later on Christgau's website in its entirety. Two more "Consumer Guide" collections have been published, compiling his capsule reviews from the 1980s and the 1990s, respectively. +In 1969, Robert Christgau began reviewing contemporary album releases in his "Consumer Guide" column, which was published more-or-less monthly in The Village Voice – an alternative weekly newspaper local to New York City – and for brief periods in Newsday and Creem magazine during the 1970s. His method was to select about 20 albums to review in capsule format, averaging approximately 50 words each, and to assign each album a letter grade rating on a scale from A-plus to E-minus. The column was a product of The Village Voice's deal with Christgau – allotting him one 2,500-word piece per month – and his desire to provide prospective buyers with ratings of albums, including those that did not receive significant radio airplay. Some of Christgau's early columns were reprinted in his first book, Any Old Way You Choose It, a 1973 anthology of essays previously published in the Voice and Newsday. +Among the most revered and influential of the early rock critics, Christgau wrote the "Consumer Guide" with a conviction that popular music could be consumed intelligently and discussed in a manner similar to books in literary criticism. His opinions and enthusiasm for music were informed by left-leaning politics, principles of humanism and secularism, and an interest in finding new understandings of the aesthetic and political dimensions in popular culture's intersection with the avant-garde. As a journalist, he wanted to convey his findings confidently and in a way that would entertain and provoke his readership. As such, his writing took on a densely compressed style featuring insulting language, slang, personal asides, and highbrow allusions meant to engage readers with an extensive knowledge of culture and music history, including popular music's canon and the metanarratives of specific musicians. +Over the course of the 1970s, Christgau broadened The Village Voice's readership nationally with his writing and editorial leadership, transforming the newspaper into a premier venue for popular music criticism at a time when the field began to peak in cultural influence. His own reputation developed as the field's leading American writer, with a cult following of the "Consumer Guide" column. +In the late 1970s, Christgau conceived of a book that would collect reviews from his columns through that decade. He began to pitch Christgau's Record Guide to publishers in early 1979 and received a publishing deal shortly thereafter. He soon realized that the proposed book would not adequately represent the decade unless he significantly revised and expanded his existing columns. He believed his existing body of reviews overlooked important musical artists and would likely comprise less than two-thirds of the needed material for the book. In July of that year, he took a vacation from The Village Voice and left New York for Maine with his wife, fellow writer Carola Dibbell, to work on the book. They chartered a boathouse and brought with them a stereo system and numerous LP records. As Christgau recalled in his memoir Going Into the City (2015), "I had hundreds of records to find out about, hundreds to find, hundreds to re-review, hundreds to touch up." +Christgau continued working on the book after his return to New York. He was aided significantly by access to the record library of his neighbor, fellow journalist Vince Aletti, who owned all of James Brown's scarcely catalogued Polydor LPs from the 1970s. Beginning with Brown, Christgau re-examined the discographies of major artists in a chronological manner to curtail a sense of hindsight in the writing. "When possible", he said, "I piled on the changer artists I actually felt like hearing that day in a ploy intended to scare up the excited little feeling in the pit of my stomach without which I am loath to give any album an A." The work intensified in 1980; from the beginning of February to the end of July he spent every day preparing the book. Recounting in his memoir, he said he worked 14 hours daily while "in book mode", which "was so grueling that for most of 1980 I was barely aware of the music of the moment, the only such hiatus in what is now fifty years." +Christgau's intense immersion into preparing the book also put a strain on his marriage to Dibbell, as did their efforts to overcome infertility. In his words, the guide almost ruined his personal life: "We postponed further parenthood strategization. We hardly ever went out. The apartment sank to new depths of disarray as LPs and paper migrated into the dining room. And since I was home every minute with the stereo on, my life partner could never be alone, with herself or with her work." Dibbell's admission of an affair at the time led to a brief separation before she and Christgau reunited with a stronger commitment to each other, reflected in the book's dedication: "TO CAROLA – NEVER AGAIN." As they mended their relationship, Christgau slowed down his work pace in August 1980 while allowing Dibbell to "provide the tough edits I needed". In his memoir, he paid tribute to her influence on his work: "Her aesthetic responsiveness was unending ... no one affected my writing like Carola". Christgau finished the guide in mid-September, submitting the manuscript a few weeks past his publisher's deadline. +Christgau's Record Guide collects approximately 3,000 "Consumer Guide" reviews of albums through the 1970s. The reviews are arranged alphabetically by artist name and accompanied by annotations for each record. Christgau regraded some older albums to reflect his changed perspective, while omitting other records and text from the original columns in favor of new material. Much of the material was previously unpublished, as more than half of the original reviews were expanded and extensively revised by Christgau for the guide, especially those covering the decade's first half. "Much of the early CG material was rewritten for the book for a reason", Christgau explained: "I didn't evolve my current high-density stylistic approach until 1975 or so". He wanted to retain parts of the original text as much as possible, "not just to poach language but to inject what sense of the moment I could". +The book covers albums from a variety of rock-related genres including hard rock, heavy metal, punk rock, funk, disco, soul, blues, country, and reggae. With regard to its scope, Christgau said he "tried to grade every '70s rock album worth owning" and what he called "semipopular" music, while maintaining a perspective of being "equal" with rock listeners. The reviews often feature analytical commentary on the aesthetic or cultural significance of the music, as well as critical one-liner jokes. For example, his review of the Leonard Cohen album Live Songs (1973) states Cohen "risks turning into the Pete Seeger of romantic existentialism", while the Doobie Brothers' Takin' It to the Streets (1976) is panned in a single sentence: "You can lead a Doobie to the recording studio, but you can't make him think." +The book is appended with introductory essays by Christgau, including a historical overview of rock and an explanation of his grading system; an A-plus record is defined as "an organically conceived masterpiece that repays prolonged listening", while "E records are frequently cited as proof that there is no God." Other unworthy music of note is relegated to two pages of lists toward the end of the guide, titled "Who Cares?" and "Meltdown". The last section of the book, titled "A Basic Record Library", lists the albums he regarded as the essential records of the 1950s, 1960s, and 1970s, respectively. +Christgau's Record Guide: Rock Albums of the Seventies was first published in October 1981 by Ticknor & Fields (an imprint of Houghton Mifflin) in New Haven, Connecticut. It was released at a time when bookstores saw an influx of rock music reference books as publishers competed with one another for the market. The book was published in the United Kingdom the following year through the London-based Vermilion imprint, and was reprinted in 1985 by Houghton Mifflin in New York. +In 1990, Christgau's Record Guide was reprinted by Da Capo Press under the title Rock Albums of the Seventies: A Critical Guide. In the reprint's introduction, Christgau said he had revised some of the content. It was followed by Christgau's Record Guide: The '80s that same year and Christgau's Consumer Guide: Albums of the '90s in 2000, forming a three-volume series of books compiling the capsule reviews. The contents of all three "Consumer Guide" collections were made freely available on Christgau's website when it went online in 2001 with the help of fellow critic and web designer Tom Hull. +Christgau's Record Guide was well received in the popular music press during the 1980s. Reviewing the book in 1982 for The New York Times, Robert Palmer found it notable among the recent slew of rock reference publications "because it is both obsessively complete and caustically outspoken". He identified qualities of maturity, intelligence, and humor in Christgau's music criticism, and recommended the book as a valuable resource for those with a serious interest in contemporary rock, even if some of the opinions may prove divisive. "It is a bit too cantankerously opinionated to deserve an A-plus," Palmer wrote, "but it certainly repays reading and rereading." Liam Lacey of The Globe and Mail called the guide "archly entertaining", while a reviewer for The Washington Post said Christgau "writes beautifully, and his book is a provocative capsule history of '70s pop". In The Harvard Crimson, David M. Handelman gave the guide an "A" grade and deemed Christgau "everything a rock critic should be" and without the pretensions of contemporaries like John Rockwell, Kit Rachlis, and Dave Marsh: "There's non [sic] of the tortured delving into non existent souls. He trashes trash, and embraces fun, genius, and things between." Handelman also supported his brevity of style and the capsule-style format, arguing that anything longer is often unreasonable for a review of an album; "how many people read beyond the good/bad label?", he questioned. Herb Hendler named Christgau's Record Guide among the books and magazines "relevant to youth and the rock era" in Year by Year in the Rock Era (1983), a chronicle of rock music's cultural impact from 1954 to 1982. In The Boston Phoenix, Mark Moses found Christgau's critiques to be "funny, accurate, and revelatory." While noting that "Anyone can be enriched by this book, and anyone can be enervated by it," Moses still rated it "A Minus." +Steve Simels, who had been inspired to pursue a journalistic career by Christgau's early 1970s writing, ended up reviewing the book for Stereo Review and offered qualified praise. He regarded Christgau's reviews as indisputably well-written with fair-minded critical judgment, impressive witticisms and one-liners, and a deeper interest in black music than most other white critics. But Simels expressed reservations about what he perceived to be a mawkish sense of feminist conscience, "knee-jerk" liberal politics, and a predilection for "conceptual music", exemplified by "A" grades for all four Ramones albums. After laboring through the entire guide, he found himself "surprised to discover how even-handed [Christgau]'s been over the long haul", and wrote in sum that it is worth reading, even if not entirely reliable. In High Fidelity, David Browne said some of Christgau's musings would be too complicated for newcomers to rock journalism, but concluded that "he remains one of pop's most astute critics" and the guide functions best as a way of discovering good records – such as Bill Withers's Still Bill (1972) – otherwise obscured by the complex discography of 1970s popular music. +Some reviewers were more critical of the book. Marsh, who had cited the "Consumer Guide" concept as an influence on his contemporaneous Rolling Stone Record Guide, gave the collection a B-plus in Musician. He found Christgau "concise, contentious, condescending, provocative and pedagogic" with a shrewd sense of judgment and sharp insight, but complained of gratuitous, off-topic commentary and possible attempts to keep up with fashionable consensus, as there were no clear indications which reviews had been rewritten in retrospect for the book. Marsh ultimately questioned whether Christgau's tough-mindedness and ideological rigidity made him ideal for a guide book and asked what it means if "the most influential rock critic has never written a book that wasn't an anthology". A response by Christgau was published in the magazine a few months later, in which he expressed appreciation for Marsh's "kind words" about the book and referred back to its introductory essay to answer questions posed in the review: +Though I don't indicate which reviews are newly written, I do state that I've reconsidered every record I had doubts about and stand by every judgment. As the co-editor of a competing consumer guide, Dave knows that the most important thing to do when you're reviewing records is to listen to them first. +British music scholar Paul Taylor issued different complaints about the book in his 1985 guide to literature on popular music, Popular Music Since 1955. He called Christgau's Record Guide "an odd collection" for several reasons. "Included are certainly the best albums", he observed, "but it is the way in which the bad examples are selected that is dubious, and simply mediocre records are avoided." +Beyond the popular press, the book also garnered positive notice among academic journals focusing on reference works and curation of library collections. A review in Choice magazine highly recommended the guide, arguing that it functions superbly as a read for spontaneous pleasure and a reference to rock's "major and minor classics" while highlighting the last section for librarians assembling a rock record collection. Illinois Libraries, the journal publication of the Illinois Library Association, advised AV librarians to consult the guide for help in selecting music recordings to archive. The journal's reviewer called Christgau "a senior critic if rock music has such a thing" and suggested readers focus less on his dislikes of artists like John Denver and more on his "well-founded and meaningful" enthusiasm for Terry Garthwaite, Brian Eno, and the Ramones. +Lee Ash, general editor of the Haworth Press's Special Collections journal, reviewed Christgau's Record Guide as among books recommended to library special collections. The guide's "quality, content, scope, and evaluative notes" impressed Ash, who had a greater familiarity with discographies of early chamber music and only a mild enthusiasm for rock music. He concluded it is an essential guide for music collection and providing "critical material for argument". In Library Journal, P. G. Feehan regarded Christgau as a rigorous, smart critic and the book an excellent companion to The Rolling Stone Record Guide, particularly because of his extensive coverage of albums from fringe and import record labels. Feehan's one cavil was "his four-letter-word-studded, hip/smart style, which may turn off readers from west of the Hudson River". +Christgau's Record Guide and similar review collections played a role in the rise of rock critics as tastemakers, promoters, and cultural historians in the music industry, whose standards were being reinvented by rock music. These critics constructed their own versions of what popular music academic Roy Shuker called "the traditional high-low culture split, usually around notions of artistic integrity, authenticity, and the nature of commercialism". As with The Rolling Stone Record Guide, Christgau's Record Guide became popular with music aficionados, collectors, and both secondhand and specialty record stores, who kept copies of different volumes on hand. Christgau's guides to the 1970s and 1980s were "bibles in the field", as Shuker described, "establishing orthodoxies as to the relative value of various styles or genres and pantheons of artists". While reviewing the 1980s volume for the Chicago Tribune in 1990, Greg Kot said Christgau's Record Guide and Ira Robbins's Trouser Press Record Guide had been "the bibles of my existence as a rock critic". Fellow critic Rob Sheffield named it among his six favorite books in a list published by The Week, believing other "obsessive music freaks" likely own it too. "This book is the all-time rock-'n'-roll argument starter", he said, "and I'll be arguing with it for the rest of my life." It was also read by novelists Dylan Hicks and Jonathan Lethem when they were young adults; Lethem later revealed that "for years, I calibrated my record collection against the grades ... jotting dissenting views in pencil in the margins". In retrospect, Christgau said the book "greatly enhance[d] my personal profile as well as reaching readers I'm proud are still out there." +The guide was published in an era when popular music studies were the domain of non-academic sources such as journalists rather than music departments and classical academics. It became widely popular in library catalogues by the late 1980s, along with other anthologized works of rock journalism by critics like Ellen Willis and Lester Bangs. The "Consumer Guide" reviews collected in the book were cited by Weisbard as part of rock and popular music's reinvention of critical standards. The end of the 20th century saw another paradigm shift, as paid journalism declined and academic departments gradually embraced popular music studies. In 1997, the Music Library Association used both 1970s and 1980s volumes as references to prepare select rock recordings for A Basic Music Library: Essential Scores and Sound Recordings, published by the American Library Association as a guide for librarians and other specialist collectors. +Christgau believed the 1970s guide was the most "authoritative" in the "Consumer Guide" book series because the decade's smaller music market was easier to process. He characterized the first volume as "a kind of canon-defining work, making the case for Van Morrison and, say, the McGarrigle sisters, and against Black Sabbath and, say, Donny Hathaway." In comparison to The Rolling Stone Record Guide, Christgau said "I think mine is better, but that's not worth arguing. What's undeniable is that mine was written by one person, and was therefore an act of sensibility rather than cultural fiat, with no canonizing intention albeit some canonizing effect (which was soon nullified by the Rock and Roll Hall of Fame [in 1983], a status grab far more effective than any book)." In the two decades that followed, he felt the canon grew "far murkier, vaster, and more various", while his personal taste became more idiosyncratic and divergent from general critical consensus. According to The Boston Globe's Ivan Kreilkamp, the "audacious, canon-defining work" remained the critic's most impactful work. In Eric Weisbard's opinion, Christgau "wasn't so much canonizing as using the endless listening party to find new wrinkles in his, and popular music's, unsummarizable aesthetic". +Christgau's Record Guide has been regarded as "essential rock writing" by literary academic Cornel Bonca. Chuck Eddy included it in his buying guide to books on rock music, while Jon Savage recommended it as a useful discography of 1970s punk rock. In 2006, all three "Consumer Guide" books were collectively ranked fifth on The A.V. Club's list of the 17 essential books about popular music; in the opinion of the list's writers, Christgau "made a sublime art of the capsule review, packing pithy observations and heartfelt appreciation into 150-word boxes". Michaelangelo Matos, a staff writer for the website, was greatly influenced by Christgau and said the first two volumes were books he had read most frequently as an adult. He highlighted Christgau's humor, ability to extract the essence of an album in a few sentences, and punctuating of reviews with letter grades, "a cunning rhetorical device as much as simple judgment". In 2016, Billboard placed the first volume at number 47 on a list of the 100 greatest music books; an accompanying blurb read: "His '70s collection offers a fantastic primer on rock and soul's most fruitful decade. Whether or not you share Christgau's passion for Al Green's 'Let's Get Married' or his disdain for all things Eagles, you'll love his pith and wit." +1970s in music +Album era +Classic rock +Music criticism +Music journalism +Rockism and poptimism +Spin Alternative Record Guide +Marsh, Dave (January 13, 1977). "The Critics' Critic II". Rolling Stone – via Rock's Backpages. (subscription required) +Nugent, Stephen L. (January 1984). "Christgau's Guide: Rock Albums of the 70's by Robert Christgau; The Rock Who's Who: A Complete Guide to the Great Artists and Albums of 30 Years by Brock Helander; Dictionary of American Pop/Rock by Arnold Shaw". Popular Music. 4: 307–311. doi:10.1017/S0261143000006280. JSTOR 853369. S2CID 162564741. +Official website +Christgau's Record Guide: Rock Albums of the Seventies at the Internet Archive +Christgau's Record Guide: Rock Albums of the Seventies in libraries (WorldCat catalog) +The 1998 National Football Conference (NFC) Championship Game was the 29th title game of the NFC. This National Football League (NFL) playoff game was played on January 17, 1999, to determine the NFC champion for the 1998 NFL season. The visiting Atlanta Falcons defeated the heavily favored Minnesota Vikings 30–27 in sudden death overtime to win their first conference championship and advance to the franchise's first Super Bowl appearance. As a result of their loss, the Vikings were eliminated from the playoffs and became the first team in NFL history to have at least 15 wins in the regular season and not win the Super Bowl, a feat that was also later accomplished by the 2004 Pittsburgh Steelers, 2007 New England Patriots (who won 16 games), 2011 Green Bay Packers, 2015 Carolina Panthers, and 2024 Detroit Lions and Kansas City Chiefs. +The game is considered one of the most memorable conference championship games in NFL history. Entering the playoffs, the Vikings were the favorite to win the Super Bowl, as they had set the NFL record for most points scored by a team in a single season. They had gone undefeated in their home stadium, the Hubert H. Humphrey Metrodome, during the regular season, and their placekicker, Gary Anderson, had become the first kicker in NFL history to convert every field goal and extra point attempt in a season. At a critical moment late in the game, Anderson missed a field goal for the first time that year, which, if converted, would have given the Vikings a nearly insurmountable 10-point lead. Instead, the Falcons scored a touchdown to tie the game on their ensuing drive and subsequently won by a field goal in overtime. Due to its impact on the game's outcome, Anderson's missed field goal has since become the focal point of the loss. +The Falcons lost 34–19 to the Denver Broncos two weeks later in Super Bowl XXXIII. The Falcons would not return to the Super Bowl until the 2016 NFL season, when the Falcons lost in overtime to the New England Patriots in Super Bowl LI. Although the game long stood as the proudest moment in the history of the Falcons franchise, the 1998 NFC Championship Game has been remembered for the effect it had on the Vikings players and their fan base, as it is seen by some sportswriters as one of the most devastating losses in NFL history. +Entering the 1998 season, the Minnesota Vikings had accumulated a history of disappointing losses despite their elite level of play. Although they were the first NFL franchise to play in four Super Bowls, they also became the first to lose four Super Bowls after their final appearance in Super Bowl XI. In other seasons, they had come within seconds of winning playoff games only to lose in dramatic fashion. During a 1975 Divisional round playoff game, the Vikings lost to the Dallas Cowboys on a 50-yard touchdown pass from Roger Staubach to Drew Pearson that Vikings fans, media personalities, and former players claim should have been nullified by offensive pass interference. Twelve years later, the Vikings lost to the Washington Redskins in the 1987 NFC Championship Game after running back Darrin Nelson dropped a game-tying touchdown pass on fourth down in the game's final minute. This history of misfortune led the NFL Network to rank the Vikings as the second most "snake-bitten" franchise of all-time, behind only the Cleveland Browns for their playoff losses in the late 1980s. +Under head coach Dennis Green, the Vikings were perennial playoff contenders throughout the 1990s, but they experienced little success once they reached the postseason. In the first round of the 1998 NFL draft, the Vikings selected wide receiver Randy Moss, who, despite his talent, was passed by several teams, even those in need of a wide receiver, due to concerns surrounding Moss's misbehavior and multiple arrests during high school and college. Moss used this as motivation to make teams who passed on him regret their decision. That year, Moss set the NFL record for most touchdown receptions by a rookie with 17, and combined with future Hall of Fame wide receiver Cris Carter and quarterback Randall Cunningham, he formed the centerpiece of the Vikings' offensive attack, which also set an NFL record by scoring 556 points during the season. The Vikings' defense was led by future Hall of Fame defensive tackle John Randle and was ranked sixth overall in points allowed during the season. +The Vikings finished the regular season with a record of 15–1 and held the first overall seed in the NFC playoffs; the two previous NFL teams to finish the regular season with 15 wins, the 1984 San Francisco 49ers and the 1985 Chicago Bears, had each won the Super Bowl. Former player turned analyst Brian Baldinger claimed that "They were easily the best team in football," and Pro Football Hall of Fame writer Ray Didinger observed, "It seems like this is the unstoppable team." Dan Barreiro, a sports radio host in the Minneapolis area, noted that for the Vikings franchise, "'98 was the season. All the stars had aligned." +Vikings placekicker Gary Anderson had joined the team that off-season after playing for three different teams in his 16-year NFL career. In 1998, he became the first placekicker in NFL history to convert every field goal and extra point attempted, scoring a regular season record of 164 points in the process. He finished the regular season 35/35 on field goals, with a long of 53 yards, and 59/59 on extra points. As a result, he was voted to the 1998 NFC Pro Bowl team, the fourth Pro Bowl invitation of his career, and was voted to the AP All-Pro team for the first time. He also converted every field goal and extra point attempt in a Divisional playoff round victory against the Arizona Cardinals the week before the NFC Championship. Entering the NFC Championship Game, his last miss was on December 15, 1997, against the Denver Broncos, when he was a member of the San Francisco 49ers. +The Atlanta Falcons had a "frustrating" team history, as described by Atlanta sports journalist Terence Moore. Moore singled out the 1980 divisional playoff game against the Dallas Cowboys, in which the Falcons gave up a 14-point lead en route to a defeat, as well as a 6-turnover performance during their 1991 divisional playoff defeat against the Washington Redskins as notable examples of frustration. Prior to the 1998 season, the Falcons had never advanced to the NFC Championship game, let alone to a Super Bowl. +The 1998 season was not expected to be any different, as the Falcons had failed to reach the playoffs the previous two seasons and only made the playoffs twice in the previous fifteen seasons. However, head coach Dan Reeves, who was hired before the previous season, had overhauled the roster in an attempt to reverse the team's fortunes. Thirty-eight of the fifty-three players on the 1998 Falcons team had been brought in by Reeves over the previous year, including journeyman quarterback Chris Chandler, who had a career-best season in 1998. Running back Jamal Anderson also posted a career-high 1,846 rushing yards, which led the NFC that year, and the defense finished fourth overall in points allowed. As a result, the Falcons won the NFC West with a record of 14–2 and clinched the second overall seed in the NFC playoffs, behind only the Vikings. The team was nicknamed the "Dirty Birds" after a touchdown dance created by tight end O.J. Santiago but popularized by Anderson. +After experiencing chest pains following the team's 27–17 victory over the New Orleans Saints on December 13, Reeves underwent quadruple bypass surgery. The team was coached by defensive coordinator Rich Brooks for the final two regular season games. Reeves was able to return to the team in time for their first playoff game, in which the Falcons beat their division rival, the San Francisco 49ers, to clinch a spot in the Conference Championship. Despite an impressive season, they were not expected to match up well against the Vikings, who had beaten teams by an average of 23.22 points at home that year and were installed as 11-point favorites for the Championship Game. +The Falcons won the coin toss before the game and elected to receive the opening kickoff. They drove down the field and scored first on a five-yard touchdown pass to Jamal Anderson. On the next drive, the Vikings answered the score with a 31-yard touchdown pass from Cunningham to Randy Moss, tying the game at 7. Neither team scored in the remainder of the first quarter. In the second quarter, Gary Anderson kicked a field goal after the Vikings recovered a Falcons fumble to make the score 10–7. After forcing the Falcons to punt on the next drive, the Vikings scored another touchdown on a one-yard run by Cunningham, increasing the lead to 17–7 with five minutes remaining in the first half. The Falcons then lost another fumble, which gave possession back to the Vikings. On the ensuing drive, Moss dropped what would have been a touchdown pass in the end zone, leaving Gary Anderson to kick another field goal to make the score 20–7. After forcing another Falcons punt, the Vikings attempted to increase their lead before halftime, but Falcons lineman Chuck Smith forced a fumble on Cunningham, and the Falcons recovered the ball deep in Vikings territory. The Falcons subsequently scored on a 14-yard touchdown pass from Chris Chandler to Terance Mathis to cut their deficit to 20–14 by the end of the half. +The Falcons forced the Vikings to punt on the opening drive of the second half, and two long plays by wide receiver Tim Dwight set up a 27-yard field goal by Morten Andersen, which cut the Falcons' deficit to three points. The Vikings answered the score on their ensuing possession, driving 82 yards in 15 plays to score a touchdown on a five-yard Matthew Hatchette reception, which made the score 27–17 with just over 13 minutes left in the fourth quarter. The Falcons responded with a 70-yard completion to Tony Martin, which set up a 24-yard field goal by Morten Andersen to make the score 27–20. On the ensuing Vikings drive, Cunningham fumbled a snap and the Falcons recovered the ball on the Vikings' 30-yard line; the Falcons failed to score after an incomplete pass turned the ball over on downs with six minutes left in the game. The Vikings then drove down to the Falcons' 22-yard line, where an incomplete pass on 3rd down set up a field goal attempt by Gary Anderson. +Here's the snap, the kick is up, and it is ... no good! No good! Gary Anderson has missed a field goal for the first time in two years! +The incompletion on 3rd down stopped the clock with 2:11 left in the game. The field goal attempt was placed from 39 yards away, which is not considered a particularly difficult field goal distance by NFL standards. Because the game was played indoors at the Metrodome, there were no adverse weather conditions that might have affected the kick. The ball was snapped on 4th down with punter Mitch Berger holding from the left hash mark. The Vikings faced a heavy rush from cornerback Michael Booker on the left side of the line of scrimmage and cornerbacks Ronnie Bradford and Ray Buchanan on the right, who ran into Anderson and knocked him to the ground after the kick. The ball sailed about a foot wide left of the upright with 2:07 left on the game clock. Afterward, Anderson stood on the field momentarily with arms akimbo before heading to the sideline, as Falcons players celebrated around him. +A converted field goal would have given the Vikings a 10-point lead, which would have almost certainly clinched victory, according to Pro Football Prospectus and the NFL Network documentary The Missing Rings. Citing a mathematical algorithm by sports analytics company numberFire, The New York Post reported that the Vikings had a 95.23% chance of winning the game had Anderson converted the field goal. +The Falcons took possession of the football at their own 29-yard line and quickly drove down the field. With just over a minute left in the game, Vikings safety Robert Griffith dropped an interception off a deflected pass, which would have also almost certainly clinched victory. Instead, Chandler threw a touchdown pass to Mathis on the next play, tying the game 27–27 with 49 seconds remaining in regulation. On the ensuing possession, the Vikings managed only seven yards and then kneeled on third down, which ran out the clock and forced overtime. Under the NFL overtime rules then in force, the first team to score in the overtime period would win. +The Vikings won the coin toss and started overtime with possession of the football on their own 29-yard line, but managed to convert only one first down and punted to Atlanta. The Falcons drove to their own 41-yard line before being stopped on 3rd down, and they punted the ball back to Vikings. On the ensuing drive, Cunningham attempted a deep pass to Moss that was narrowly broken up by Eugene Robinson; had the pass been completed, it most likely would have resulted in a touchdown and victory for the Vikings. Instead, the Vikings had to punt on 4th down again. The Falcons then drove to the Vikings' 21-yard line, where Morten Andersen converted a 38-yard field goal for the win. +After the game, Gary Anderson was described as "inconsolable". Although the Vikings still led by seven points at the time, his missed kick had a demoralizing effect on the team. ESPN contributor Ben Goessling noted "how swiftly it pulled the bottom out from under a team that had an air of inevitability about it to that point." Writing for the website Sporting News, Jeff Diamond, the Vikings' general manager at the time, observed that, "Our team played the rest of the game as if it was in shock that our automatic kicker had missed at the most critical time." Randle concurred, describing his reaction to the kick as, "I was standing there like someone just punched me in my stomach, and was like, 'Oh my God, oh my God.'" Brian Billick, the offensive coordinator of the 1998 Vikings, went as far as stating, "I'm not sure the city had ever rebounded from it." Cunningham concurred by claiming, "With that kick, it just seemed like the whole franchise went wide left." +Carter openly wept in the locker room after the game and was affected so badly by the loss that he considered retirement. "Walking off that field and losing like that," Carter reflected, "I didn't even know if I wanted to play football anymore. Because I just, I felt like that I would never win after that." He went on to call the game, "The most devastating loss that I've ever been a part of." Both Carter and fellow Hall of Fame member Randle believed that the 1998 Vikings team was their best chance at winning a Super Bowl; neither player ever won the Super Bowl in their careers. +Randle described the feeling after the game as, "It's like driving down a street and getting every green light for the next ten miles, and you're just cruising along, and it's just smooth. And all of a sudden, you're getting there, you're almost there, and all of a sudden, the fucking light turns red, and you get sideswiped." Randle further laments the loss due to the background of many of his teammates, whom he described as "misfits, guys who just got their second chance." Beyond Moss's legal troubles, Carter had been released by the Philadelphia Eagles early in his career for substance abuse issues, and Cunningham was released by the Eagles and remained unsigned to a team two seasons prior. Randle himself was considered undersized coming out of college and was not heavily pursued by NFL teams. +Cunningham drew on his religious faith to persevere through the loss, believing that God had a reason for everything to happen. Years later, he reflected, "It just wasn't our destiny to be in the Super Bowl. That's my conclusion. Because if it was, we would have gone." Immediately after the game, he expressed this belief in an attempt to console Carter. The following season, Cunningham was benched for poor play after six games and was released by the Vikings at the end of the year. He also would never play in a Super Bowl. +Overjoyed with victory, Falcons players also wept after the game, and the team ripped open pillows and threw feathers in celebration on their charter flight back to Atlanta. Buchanan stated that the win "feels like a miracle" due to the negative perception of the Falcons in years past. "This team was dirt. People stepped on us and wiped their feet on the doormat. Now we feel like a bunch of Michael Jordans." Chandler joked that due to his game-winning field goal, Morten Andersen "gets to keep his green card", referring to Andersen's Danish nationality, while running back Jamal Anderson felt vindicated for the media's continuous focus on the Vikings in the days leading up to the game, which made him feel disrespected. +Falcons defensive end Chuck Smith questioned the Vikings' toughness because of the ease with which they had won during the season. "It's one thing to beat up on people," said Smith, "but how do you react when someone's finally hitting you back? We've been slugging it out all year." Years later, Smith went on to criticize Gary Anderson in particular for his missed field goal and its contribution to the Vikings' loss. +By virtue of their loss, the 1998 Vikings became the first team in NFL history to compile a regular season record of 15–1 and not win the Super Bowl. Fox Sports, NBC Sports, and the NFL Network each named the team as one of the five greatest not to win the Super Bowl, and coach Dennis Green believed that the 1998 Vikings would have been considered the best NFL team of their generation had they gone on to win. As part of a series celebrating the 100th anniversary of the league, the NFL ranked the 1998 Vikings as the 38th greatest team in league history, the third highest ranking for a team that didn't win a championship and the highest for a team that failed to play in either the Super Bowl or the NFL Championship Game. In a 2018 retrospective, Sports Illustrated called the 1998 Vikings "The Greatest Team Never to Make It". +The loss had a dramatic effect on Minnesota sports culture, as the 1998 Vikings were considered the team most likely to deliver a championship to a franchise that had already suffered multiple heartbreaking defeats. Both Damon Amendolara of CBS Radio and Don Banks of Sports Illustrated consider the Vikings' loss as one of the most devastating in NFL history and criticized the television program NFL Top 10 for not including it on a list of such losses. Banks further noted that "The Vikings have never completely recovered from that game," a sentiment shared by Bill Simmons and NFL Films. This perception has extended into popular culture, as the game became a plot point in the episode "Little Minnesota" of the television show How I Met Your Mother when Robin, a Canadian character, asks the significance of a banner in a Minnesota-themed bar that reads, "I'm drinking till I forget the 1999 [sic] NFC Championship." +Gary Anderson's missed kick has been singled out as the main contributing factor to the Vikings' loss, as the Falcons were able to capitalize on the late shift in momentum produced by an unexpected opportunity to tie and eventually win the game. Considering this impact on the game's outcome and the historic performances of Anderson and the 1998 Vikings team, the miss has since been noted as a memorable moment in the greater history of the NFL. Paul Allen, the play-by-play radio announcer for the Vikings, and Dan Barreiro both consider the miss as one of the most devastating moments in the history of Minnesota sports. According to Chad Hartman, another sports radio host based in Minneapolis, "[Anderson] will always be known as the guy who was a part of screwing up the Vikings' trip to the Super Bowl, even though he had this magnificent season." ESPN voted the miss as the most memorable play in Vikings history, and ESPN contributor Ben Goessling believes that the miss influenced misfortunes that the franchise faced in subsequent years, including two additional conference championship losses following the 2000 and 2009 seasons. +Anderson has claimed that in greater context, the miss was not particularly notable, and it is only remembered because the Falcons won the game. Buchanan, who was attempting to block Anderson's kick, believes that Anderson would not have been able to convert the field goal regardless, since if the kick was on target, it would have "hit [Buchanan] in the face mask." Vikings backup quarterback Brad Johnson echoed this sentiment, stating, "I think someone got loose on the left end and they almost blocked it, and [Anderson] tried to slice it in there." Several sportswriters, as well as Randle, have defended Anderson, pointing out that the Vikings' defense deserved the real blame for the loss for allowing Atlanta to tie the game after Anderson's miss. Barreiro also criticized the performance of Vikings starting quarterback Cunningham, who he felt did not handle the pressure of the game well and called "dreadful down the stretch". +Vikings head coach Dennis Green was criticized for his decision to kneel on 3rd down and play for overtime instead of attempting to score before the end of regulation. Radio host Bob Sansevere, author of The Best Minnesota Sports Arguments, called it "one of the all-time boneheaded decisions a coach has ever made in any sport". Of Green's play call, Peter King of Sports Illustrated wrote, "Minnesota coach Dennis Green did a great job this year, but if he doesn't wake up and stare at the ceiling in the next few days and say out loud: 'Boy, I screwed that one up,' then he's not being honest with himself." Nonetheless, local Minneapolis newspaper Star Tribune has contended that taking a knee was the correct decision due to the performance of the Vikings' offense, whose struggles that day were also noted by Carter. +Falcons head coach Dan Reeves was praised for his ability to lead the team after having heart surgery only weeks prior, as Reeves' comeback proved to be an emotional rallying point for the team. Austin Murphy of Sports Illustrated noted that Reeves' return "[galvanized] the already close-knit Falcons", and Falcons linebacker Jessie Tuggle agreed: "Dan has really inspired us all. ... He walked in the meeting room four days after having had surgery, and you could have heard a pin drop. We wanted to hear every last word he had to say." Reeves described his time during the season as "more rewarding than any other teams I've been involved in", and to Falcons special teams coach Joe DeCamillis, Reeves' leadership that season was "his best coaching job ever". CBS called Reeves' return "an amazing comeback that people will be talking about for years". +Chandler was considered to be the offensive hero of the game, despite the attention that the Vikings' offense received during the season. Media analysis before the game noted that the Vikings' defense needed to focus on shutting down Falcons running back Jamal Anderson; feeling overlooked, Chandler used the coverage as extra motivation. In a retrospective for the website SportsGrid, Geoff Magliocchetti detailed the end of the game, in which he claims that Chandler "embarked on the drive of his life, an 8-play, 71-yard masterpiece, finding Mathis from 16 yards out for the tying score." +The Falcons missed the playoffs the following season (while their home stadium hosted the Super Bowl) and would not return to the postseason until 2002. Following the 2001 season, Chandler was released by the team, and Jamal Anderson retired due to a knee injury. As a result, their win in the 1998 NFC Championship game stood as the franchise's proudest moment for years, particularly due to the atmosphere of pessimism that surrounded the franchise at the time. An NFL Films retrospective on the Falcons' season noted that "years of pain were wiped away in one unforgettable afternoon," and Dan Weiner stated that, "The 1998 season was a dream come true ... For once, the Atlanta Falcons made believers of us all." In 2010, ESPN named the 1998 team as the greatest Falcons team of all-time, and the network also voted Morton Andersen's game-winning field goal as the top play in Falcons' history. +In the hours before the AFC Championship Game, the Denver Broncos were watching the NFC Championship Game on the JumboTron at Mile High Stadium to see who they would play in Super Bowl XXXIII should they defeat the New York Jets. Broncos head coach Mike Shanahan expressed surprise at Anderson's missed kick, and running back Terrell Davis said that the Falcons' subsequent victory put the Broncos in the mindset that they had "won the Super Bowl already," as their greatest potential challenge had been eliminated. This distraction nearly cost the Broncos, as they played a poor game. Tim Connolly, the Vikings team president at the time, recalled that Shanahan insinuated relief that the Broncos were not playing the Vikings when the two met at a reception after the conference championship games. +Two weeks later in the Super Bowl, the Falcons played the Broncos, the game pitting coach Dan Reeves against his former team and their star quarterback, John Elway. Reeves had led the Broncos to three Super Bowl appearances, all losses, and he was fired as the Broncos' head coach after the 1992 NFL season. Afterward, Elway was quoted as saying, "These last three years have been hell. I know I would not have been back here if Dan Reeves had been here. It wasn't worth it to me. I didn't enjoy it. It wasn't any fun, and I got tired of working with him." Reeves responded by saying, "Just tell him it wasn't exactly heaven for me either. One of these days I hope he grows up. Maybe he'll mature sometime." During his tenure, Reeves had also fired Shanahan, who was an assistant coach on the team, and these points of contention became a media storyline entering the game. Nonetheless, all parties involved stated that any lingering animosity had long since passed. +The Falcons lost Super Bowl XXXIII by a score of 34–19, which earned the Broncos their second consecutive Super Bowl victory. The game was anticipated as a match up between the Vikings and the Broncos, and the Falcons' presence in the game was noted as an anticlimax. The night before the game, safety Eugene Robinson was arrested for soliciting a prostitute. Although Robinson played in the game, the distraction contributed to a poor performance by the Falcons team, who managed only six points in six drives deep into Denver territory and surrendered a season-high point total. +Per Pro Football Reference: +Per Pro Football Reference: +1Completions/attempts +2Carries +3Long gain +4Receptions +5Times targeted +As credited during the Fox Sports broadcast of the 1998 NFC Championship game: +Hall of Fame‡ +As credited during the Fox Sports broadcast of the 1998 NFC Championship game: +Referee: Walt Coleman (#65) +Umpire: Ron Botchan (#110) +Head linesman: Mark Baltz (#26) +Line judge: Byron Boston (#18) +Field judge: Bill Lovett (#98) +Side judge: Neely Dunn (#89) +Back judge: Billy Smith (#2) +2015 Minnesota Vikings season (a missed field goal by Blair Walsh in the NFC Wild Card) +Double Doink +Wide Right (Buffalo Bills) +1998 NFC Championship Game: Atlanta Falcons vs. Minnesota Vikings NFL Full Game on YouTube +Video of Anderson's miss from NFL.com +Segment of The Missing Rings that focuses on the 1998 NFC Championship Game from Vikings.com +Al-Altan (c. 1196 – 1246), also known as Altalun and Altaluqan, was the youngest child and favourite daughter of Genghis Khan, the founder of the Mongol Empire, and Börte, his primary wife. As part of Genghis's policy of marrying his daughters to powerful rulers in exchange for their submission to him, she married Barchuq Art Tegin, the ruler of the wealthy Uyghur people to the southwest, around 1211. +After Genghis died in 1227 and Ögedei Khan, his third son by Börte, ascended to the Mongol throne, it is likely that the Mongol imperial government began to appropriate the territory and taxes of the Uyghurs for themselves. When Ögedei died after an extended drinking binge in 1241, Al-Altan was present—she had probably travelled to her brother's court to defend her Uyghur subjects against the encroachment. She was rumoured to have poisoned Ögedei and remained under suspicion until the accession of her nephew, Güyük Khan, five years later. Shortly afterwards, Al-Altan was put on trial and executed by the general Eljigidei. Accounts of Al-Altan's life and death were heavily suppressed, with official chronicles compelled to excise or obscure potentially troublesome details. The injustice of her death became a major contention during the Toluid Revolution in 1251, during which Eljigidei was executed by Al-Altan's sympathisers in revenge. +Al-Altan's mother, Börte, was born into the Onggirat tribe, who lived along the Greater Khingan mountain range south of the Ergüne river, in modern-day Inner Mongolia. She married a Mongol leader named Temüjin c. 1178 after a seven-year betrothal. Over the next twenty or so years, Börte gave birth to nine children: four sons named Jochi, Chagatai, Ögedei, and Tolui, and five daughters named Qojin, Checheyigen, Alaqa, Tümelün, and Al-Altan, the youngest. After this last birth c. 1196, Temüjin continued to father offspring with other women whom he had married, but they always remained inferior in status to Börte and her children. +In the decade after Al-Altan's birth, Temüjin steadily increased his power and subjugated rival tribes, a process which culminated in his coronation as Genghis Khan, ruler of the newly-founded Mongol Empire, in 1206. Before and after this acclamation, Genghis employed his daughters with Börte in a crucial role: they were married to important male rulers, who would then submit to Genghis in exchange for rank and power in the new empire. On the other side, Genghis gained the loyalties of large steppe populations without unnecessary bloodshed, and Al-Altan and her sisters took important administrative roles in large tribes. In addition, they served as the link between their father and his new son-in-law vassals. Al-Altan's eldest sister Qojin married Butu of the Ikires tribe, Checheyigen and Alaqa married into the ruling families of the Oirats and Ongud respectively, and Tümelün married back into Börte's Onggirat tribe. +In 1209, Barchuq Art Tegin, the ruler or idiqut (lit. 'lord of fortune') of the wealthy Uyghur people to the southwest of the Mongol heartland, rejected the authority of his suzerain, the Qara Khitai state in Central Asia. Barchuq thereafter sought the friendship and protection of Genghis Khan by sending gifts of gold and jewellery and helping the Mongols pursue some of their Merkit enemies. In 1211, his efforts were rewarded when Genghis named him a "fifth son" and betrothed Al-Altan, then around fifteen, to him. This was a high honour, as Al-Altan was considered Genghis's favourite daughter. In the years thereafter, Barchuq and his 18,000 warriors campaigned as auxiliaries to the main Mongol military in various campaigns, most notably the invasion of the Khwarazmian Empire between 1218 and 1223, where the Uyghurs took part in the sieges of Otrar, Taliqan, and Nishapur. It is unknown if Al-Altan joined the expedition. They also participated in the conquest of the Western Xia state in 1226–27 and in the invasion of Europe in 1236–42. Barchuq died before 1241. +Genghis died in August 1227 and was succeeded by his chosen heir, Al-Altan's brother Ögedei, after a two-year interregnum. A generous ruler, Ögedei became increasingly addicted to alcohol and died in December 1241 after a wine-fuelled binge resulted in alcohol poisoning or organ failure. Rumours that he had been poisoned began to circulate, directed at two important women who had attended the fatal party. One of them, Ibaqa Beki, a former wife of Genghis, had served as a cupbearer and was immediately under suspicion; she was, however, cleared after being defended by the prominent general Eljigidei. The other was Al-Altan. +Al-Altan must have attended the party to be accused of the crime, but why she was present there and not in the Uyghur lands is not certain. One theory, favoured by historians such as Thomas T. Allsen and Anne F. Broadbridge, hypothesizes that under Ögedei, the Mongols began to interfere with the administration and taxation of the previously semi-autonomous Uyghur state. This process would have increased imperial control at the expense of Al-Altan and her family. Broadbridge argues that Al-Altan may have travelled to Ögedei's court to argue against the imperial encroachment; following her brother's death, his wife Töregene accused Al-Altan of poisoning him in retribution. +Unlike Ibaqa Beki, Al-Altan was never cleared of suspicion. Having survived Töregene's regency between 1241 and 1246, Al-Altan was probably present when her nephew Güyük Khan, son of Ögedei and Töregene, was crowned in the latter year. After his coronation, she was put on trial and executed; all details were heavily suppressed due to the taboo against killing a member of the royal family. An oversight in a medieval chronicle reveals that her executioner was Eljigidei, who had decisively exonerated Ibaqa Beki five years earlier. In return for committing this taboo act, Güyük rewarded Eljigidei with a senior military position in West Asia, distant from any enemies he gained through killing Al-Altan. +In 1251, Möngke Khan acceded to the throne as part of the Toluid Revolution, in which the family of Genghis's youngest son Tolui seized power from the descendants of Ögedei. Möngke's faction put forward numerous arguments that the house of Ögedei had contravened Mongol law and custom and were thus unfit to rule. The most provocative of these allegations was that they had unlawfully executed Al-Altan, Genghis's favourite daughter, without consulting the wider family. In the purges that began after Möngke foiled an Ögedeyid coup attempt, Eljigidei was singled out as a target, accused of murder by prominent figures such as Batu Khan and Möngke's brother Kublai Khan. He attempted to escape capture but was caught near the city of Herat and soon executed, allegedly being boiled alive on Batu's orders. +During Töregene's regency, Barchuq's successor Kesmes, who was either a son or stepson of Al-Altan, had died of unknown causes. He was replaced with a brother named Salindi, who was personally selected by Töregene and remained extremely loyal to her and Güyük. Salindi remained loyal to the Ögedeyids after the Toluid Revolution but was eventually captured, tortured for a confession, and executed by his brother Ögünch, who replaced him as idiqut. +Because of the taboo nature of her death and the possibility that the Ögedeyids unlawfully interfered in the Uyghur administration, numerous aspects of Al-Altan's life were censored in official chronicles, such as the Secret History of the Mongols. One passage which described the inheritances of Genghis Khan's daughters was excised from the Secret History. This was likely intended to obscure the injustice of the Ögedeyid actions in encroaching on Uyghur territory. Persian sources such as the Jami al-tawarikh, by the 14th-century historian Rashid al-Din, go further by denying that she married Barchuq at all; they instead provide contradictory statements. Suggestions that she married a member of another tribe and then a Uyghur are unlikely, as if a widow remarried, it was normally within her dead husband's tribe. In other places, the Persian sources state that Barchuq and Al-Altan were betrothed but did not marry before Genghis's death because Barchuq had a cherished wife at home—Broadbridge notes that the Mongols would have been deeply insulted. Rashid al-Din then says that Al-Altan died when travelling to marry Barchuq during Ögedei's reign, but later contradicts himself by unintentionally revealing that Eljigidei killed her. Because of these contradictions and improbabilities, Broadbridge considers the Persian narratives untrustworthy. +Women in the Mongol Empire +Allsen, Thomas T. (1983). "The Yüan Dynasty and the Uighurs of Turfan in the 13th Century". In Rossabi, Morris (ed.). China Among Equals: The Middle Kingdom and Its Neighbors, 10th–14th Centuries. Berkeley: University of California Press. pp. 243–280. ISBN 978-0-5200-4562-0. +Atwood, Christopher P. (2004). Encyclopedia of Mongolia and the Mongol Empire. New York: Facts on File. ISBN 978-0-8160-4671-3. +Broadbridge, Anne F. (2018). Women and the Making of the Mongol Empire. Cambridge Studies in Islamic Civilization. Great Barrington: Cambridge University Press. ISBN 978-1-1086-3662-9. +Broadbridge, Anne F. (2022). "Daughters, Consort Families, and the Military". In May, Timothy; Hope, Michael (eds.). The Mongol World. Abingdon: Routledge. pp. 341–350. ISBN 978-1-3151-6517-2. +Dunnell, Ruth W. (2023). "The Rise of Chinggis Khan and the United Empire". In Biran, Michal; Kim, Hodong (eds.). The Cambridge History of the Mongol Empire. Cambridge: Cambridge University Press. pp. 19–106. ISBN 978-1-3163-3742-4. +May, Timothy (2018). The Mongol Empire. Edinburgh: Edinburgh University Press. ISBN 978-0-7486-4237-3. JSTOR 10.3366/j.ctv1kz4g68. +Ratchnevsky, Paul (1991). Genghis Khan: His Life and Legacy. Translated by Thomas Haining. Oxford: Blackwell Publishing. ISBN 978-0-6311-6785-3. +Zhao, George Q. (2008). Marriage as Political Strategy and Cultural Expression: Mongolian Royal Marriages from World Empire to Yuan Dynasty. New York: Peter Lang. ISBN 978-1-4331-0275-2. +The cerebellum (pl.: cerebella or cerebellums; Latin for 'little brain') is a major feature of the hindbrain of all vertebrates. Although usually smaller than the cerebrum, in some animals such as the mormyrid fishes it may be as large as it or even larger. In humans, the cerebellum plays an important role in motor control and cognitive functions such as attention and language as well as emotional control such as regulating fear and pleasure responses, but its movement-related functions are the most solidly established. The human cerebellum does not initiate movement, but contributes to motor coordination, precision, and accurate timing: it receives input from sensory systems of the spinal cord and from other parts of the brain, and integrates these inputs to fine-tune motor activity. Cerebellar damage produces disorders in fine movement, equilibrium, posture, and motor learning in humans. +Anatomically, the human cerebellum has the appearance of a separate structure attached to the bottom of the brain, tucked underneath the cerebral hemispheres. Its cortical surface is covered with finely spaced parallel grooves, in striking contrast to the broad irregular convolutions of the cerebral cortex. These parallel grooves conceal the fact that the cerebellar cortex is actually a thin, continuous layer of tissue tightly folded in the style of an accordion. Within this thin layer are several types of neurons with a highly regular arrangement, the most important being Purkinje cells and granule cells. This complex neural organization gives rise to a massive signal-processing capability, but almost all of the output from the cerebellar cortex passes through a set of small deep nuclei lying in the white matter interior of the cerebellum. +In addition to its direct role in motor control, the cerebellum is necessary for several types of motor learning, most notably in sensorimotor relationships. Several theoretical models have been developed to explain sensorimotor calibration in terms of synaptic plasticity within the cerebellum. These models derive from those formulated by David Marr and James Albus, based on the observation that each cerebellar Purkinje cell receives two dramatically different types of input: one comprises thousands of weak inputs from the parallel fibers of the granule cells; the other is an extremely strong input from a single climbing fiber. The basic concept of the Marr–Albus theory is that the climbing fiber serves as a "teaching signal", which induces a long-lasting change in the strength of parallel fiber inputs. Observations of long-term depression in parallel fiber inputs have provided some support for theories of this type, but their validity remains controversial. +At the level of gross anatomy, the cerebellum consists of a tightly folded layer of cortex, with white matter underneath and a fluid-filled ventricle at the base. Four deep cerebellar nuclei are embedded in the white matter. Each part of the cortex consists of the same small set of neuronal elements, laid out in a highly stereotyped geometry. At an intermediate level, the cerebellum and its auxiliary structures can be separated into several hundred or thousand independently functioning modules called "microzones" or "microcompartments". +The cerebellum is located in the posterior cranial fossa. The fourth ventricle, pons and medulla are in front of the cerebellum. It is separated from the overlying cerebrum by a layer of leathery dura mater, the cerebellar tentorium; all of its connections with other parts of the brain travel through the pons. Anatomists classify the cerebellum as part of the metencephalon, which also includes the pons; the metencephalon is the upper part of the rhombencephalon or "hindbrain". Like the cerebral cortex, the cerebellum is divided into two cerebellar hemispheres; it also contains a narrow midline zone (the vermis). A set of large folds is, by convention, used to divide the overall structure into 10 smaller "lobules". Because of its large number of tiny granule cells, the cerebellum contains more neurons than the total from the rest of the brain, but takes up only 10% of the total brain volume. The number of neurons in the cerebellum is related to the number of neurons in the neocortex. There are about 3.6 times as many neurons in the cerebellum as in the neocortex, a ratio that is conserved across many different mammalian species. +The unusual surface appearance of the cerebellum conceals the fact that most of its volume is made up of a very tightly folded layer of gray matter: the cerebellar cortex. Each ridge or gyrus in this layer is called a folium. High‑resolution MRI finds the adult human cerebellar cortex has an area of 730 square cm, packed within a volume of dimensions 6 cm × 5 cm × 10 cm. Underneath the gray matter of the cortex lies white matter, made up largely of myelinated nerve fibers running to and from the cortex. Embedded within the white matter—which is sometimes called the arbor vitae (tree of life) because of its branched, tree-like appearance in cross-section—are four deep cerebellar nuclei, composed of gray matter. +Connecting the cerebellum to different parts of the nervous system are three paired cerebellar peduncles. These are the superior cerebellar peduncle, the middle cerebellar peduncle and the inferior cerebellar peduncle, named by their position relative to the vermis. The superior cerebellar peduncle is mainly an output to the cerebral cortex, carrying efferent fibers via thalamic nuclei to upper motor neurons in the cerebral cortex. The fibers arise from the deep cerebellar nuclei. The middle cerebellar peduncle is connected to the pons and receives all of its input from the pons mainly from the pontine nuclei. The input to the pons is from the cerebral cortex and is relayed from the pontine nuclei via transverse pontine fibers to the cerebellum. The middle peduncle is the largest of the three and its afferent fibers are grouped into three separate fascicles taking their inputs to different parts of the cerebellum. The inferior cerebellar peduncle receives input from afferent fibers from the vestibular nuclei, spinal cord and the tegmentum. Output from the inferior peduncle is via efferent fibers to the vestibular nuclei and the reticular formation. The whole of the cerebellum receives modulatory input from the inferior olivary nucleus via the inferior cerebellar peduncle. +Based on the surface appearance, three lobes can be distinguished within the cerebellum: the anterior lobe (above the primary fissure), the posterior lobe (below the primary fissure), and the flocculonodular lobe (below the posterior fissure). These lobes divide the cerebellum from rostral to caudal (in humans, top to bottom). In terms of function, however, there is a more important distinction along the medial-to-lateral dimension. Leaving out the flocculonodular lobe, which has distinct connections and functions, the cerebellum can be parsed functionally into a medial sector called the spinocerebellum and a larger lateral sector called the cerebrocerebellum. A narrow strip of protruding tissue along the midline is called the cerebellar vermis. (Vermis is Latin for "worm".) +The smallest region, the flocculonodular lobe, is often called the vestibulocerebellum. It is the oldest part in evolutionary terms (archicerebellum) and participates mainly in balance and spatial orientation; its primary connections are with the vestibular nuclei, although it also receives visual and other sensory input. Damage to this region causes disturbances of balance and gait. +The medial zone of the anterior and posterior lobes constitutes the spinocerebellum, also known as paleocerebellum. This sector of the cerebellum functions mainly to fine-tune body and limb movements. It receives proprioceptive input from the dorsal columns of the spinal cord (including the spinocerebellar tract) and from the cranial trigeminal nerve, as well as from visual and auditory systems. It sends fibers to deep cerebellar nuclei that, in turn, project to both the cerebral cortex and the brain stem, thus providing modulation of descending motor systems. +The lateral zone, which in humans is by far the largest part, constitutes the cerebrocerebellum, also known as neocerebellum. It receives input exclusively from the cerebral cortex (especially the parietal lobe) via the pontine nuclei (forming cortico-ponto-cerebellar pathways), and sends output mainly to the ventrolateral thalamus (in turn connected to motor areas of the premotor cortex and primary motor area of the cerebral cortex) and to the red nucleus. There is disagreement about the best way to describe the functions of the lateral cerebellum: It is thought to be involved in planning movement that is about to occur, in evaluating sensory information for action, and in a number of purely cognitive functions, such as determining the verb which best fits with a certain noun (as in "sit" for "chair"). +Two types of neuron play dominant roles in the cerebellar circuit: Purkinje cells and granule cells. Three types of axons also play dominant roles: mossy fibers and climbing fibers (which enter the cerebellum from outside), and parallel fibers (which are the axons of granule cells). There are two main pathways through the cerebellar circuit, originating from mossy fibers and climbing fibers, both eventually terminating in the deep cerebellar nuclei. +Mossy fibers project directly to the deep nuclei, but also give rise to the following pathway: mossy fibers → granule cells → parallel fibers → Purkinje cells → deep nuclei. Climbing fibers project to Purkinje cells and also send collaterals directly to the deep nuclei. The mossy fiber and climbing fiber inputs each carry fiber-specific information; the cerebellum also receives dopaminergic, serotonergic, noradrenergic, and cholinergic inputs that presumably perform global modulation. +The cerebellar cortex is divided into three layers. At the bottom lies the thick granular layer, densely packed with granule cells, along with interneurons, mainly Golgi cells but also including Lugaro cells and unipolar brush cells. In the middle lies the Purkinje layer, a narrow zone that contains the cell bodies of Purkinje cells and Bergmann glial cells. At the top lies the molecular layer, which contains the flattened dendritic trees of Purkinje cells, along with the huge array of parallel fibers penetrating the Purkinje cell dendritic trees at right angles. This outermost layer of the cerebellar cortex also contains two types of inhibitory interneuron: stellate cells and basket cells. Both stellate and basket cells form GABAergic synapses onto Purkinje cell dendrites. +The top, outermost layer of the cerebellar cortex is the molecular layer. This layer contains the flattened dendritic trees of Purkinje cells, and the huge array of parallel fibers, from the granular layer, that penetrate the Purkinje cell dendritic trees at right angles. The molecular layer also contains two types of inhibitory interneuron: stellate cells and basket cells. Both stellate and basket cells form GABAergic synapses onto Purkinje cell dendrites. +Purkinje cells are among the most distinctive neurons in the brain, and one of the earliest types to be recognized—they were first described by the Czech anatomist Jan Evangelista Purkyně in 1837. They are distinguished by the shape of their dendritic tree: the dendrites branch very profusely, but are severely flattened in a plane perpendicular to the cerebellar folds. Thus, the dendrites of a Purkinje cell form a dense planar net, through which parallel fibers pass at right angles. The dendrites are covered with dendritic spines, each of which receives synaptic input from a parallel fiber. Purkinje cells receive more synaptic inputs than any other type of cell in the brain—estimates of the number of spines on a single human Purkinje cell run as high as 200,000. The large, spherical cell bodies of Purkinje cells are packed into a narrow layer (one cell thick) of the cerebellar cortex, called the Purkinje layer. After emitting collaterals that affect nearby parts of the cortex, their axons travel into the deep cerebellar nuclei, where they make on the order of 1,000 contacts each with several types of nuclear cells, all within a small domain. Purkinje cells use GABA as their neurotransmitter, and therefore exert inhibitory effects on their targets. +Purkinje cells form the heart of the cerebellar circuit, and their large size and distinctive activity patterns have made it relatively easy to study their response patterns in behaving animals using extracellular recording techniques. Purkinje cells normally emit action potentials at a high rate even in the absence of the synaptic input. In awake, behaving animals, mean rates averaging around 40 Hz are typical. The spike trains show a mixture of what are called simple and complex spikes. A simple spike is a single action potential followed by a refractory period of about 10 ms; a complex spike is a stereotyped sequence of action potentials with very short inter-spike intervals and declining amplitudes. Physiological studies have shown that complex spikes (which occur at baseline rates around 1 Hz and never at rates much higher than 10 Hz) are reliably associated with climbing fiber activation, while simple spikes are produced by a combination of baseline activity and parallel fiber input. Complex spikes are often followed by a pause of several hundred milliseconds during which simple spike activity is suppressed. +A specific, recognizable feature of Purkinje neurons is the expression of calbindin. Calbindin staining of rat brain after unilateral chronic sciatic nerve injury suggests that Purkinje neurons may be newly generated in the adult brain, initiating the organization of new cerebellar lobules. +Cerebellar granule cells, in contrast to Purkinje cells, are among the smallest neurons in the brain. They are also the most numerous neurons in the brain: In humans, estimates of their total number average around 50 billion, which means that about 3/4 of the brain's neurons are cerebellar granule cells. Their cell bodies are packed into a thick layer at the bottom of the cerebellar cortex. A granule cell emits only four to five dendrites, each of which ends in an enlargement called a dendritic claw. These enlargements are sites of excitatory input from mossy fibers and inhibitory input from Golgi cells. +The thin, unmyelinated axons of granule cells rise vertically to the upper (molecular) layer of the cortex, where they split in two, with each branch traveling horizontally to form a parallel fiber; the splitting of the vertical branch into two horizontal branches gives rise to a distinctive "T" shape. A human parallel fiber runs for an average of 3 mm in each direction from the split, for a total length of about 6 mm (about 1/10 of the total width of the cortical layer). As they run along, the parallel fibers pass through the dendritic trees of Purkinje cells, contacting one of every 3–5 that they pass, making a total of 80–100 synaptic connections with Purkinje cell dendritic spines. Granule cells use glutamate as their neurotransmitter, and therefore exert excitatory effects on their targets. +Granule cells receive all of their input from mossy fibers, but outnumber them by 200 to 1 (in humans). Thus, the information in the granule cell population activity state is the same as the information in the mossy fibers, but recoded in a much more expansive way. Because granule cells are so small and so densely packed, it is difficult to record their spike activity in behaving animals, so there is little data to use as a basis for theorizing. The most popular concept of their function was proposed in 1969 by David Marr, who suggested that they could encode combinations of mossy fiber inputs. The idea is that with each granule cell receiving input from only 4–5 mossy fibers, a granule cell would not respond if only a single one of its inputs were active, but would respond if more than one were active. This combinatorial coding scheme would potentially allow the cerebellum to make much finer distinctions between input patterns than the mossy fibers alone would permit. +Mossy fibers enter the granular layer from their points of origin, many arising from the pontine nuclei, others from the spinal cord, vestibular nuclei etc. In the human cerebellum, the total number of mossy fibers has been estimated at 200 million. These fibers form excitatory synapses with the granule cells and the cells of the deep cerebellar nuclei. Within the granular layer, a mossy fiber generates a series of enlargements called rosettes. The contacts between mossy fibers and granule cell dendrites take place within structures called glomeruli. Each glomerulus has a mossy fiber rosette at its center, and up to 20 granule cell dendritic claws contacting it. Terminals from Golgi cells infiltrate the structure and make inhibitory synapses onto the granule cell dendrites. The entire assemblage is surrounded by a sheath of glial cells. Each mossy fiber sends collateral branches to several cerebellar folia, generating a total of 20–30 rosettes; thus a single mossy fiber makes contact with an estimated 400–600 granule cells. +Purkinje cells also receive input from the inferior olivary nucleus on the contralateral side of the brainstem via climbing fibers. Although the inferior olive lies in the medulla oblongata and receives input from the spinal cord, brainstem and cerebral cortex, its output goes entirely to the cerebellum. A climbing fiber gives off collaterals to the deep cerebellar nuclei before entering the cerebellar cortex, where it splits into about 10 terminal branches, each of which gives input to a single Purkinje cell. In striking contrast to the 100,000-plus inputs from parallel fibers, each Purkinje cell receives input from exactly one climbing fiber; but this single fiber "climbs" the dendrites of the Purkinje cell, winding around them and making a total of up to 300 synapses as it goes. The net input is so strong that a single action potential from a climbing fiber is capable of producing an extended complex spike in the Purkinje cell: a burst of several spikes in a row, with diminishing amplitude, followed by a pause during which activity is suppressed. The climbing fiber synapses cover the cell body and proximal dendrites; this zone is devoid of parallel fiber inputs. +Climbing fibers fire at low rates, but a single climbing fiber action potential induces a burst of several action potentials in a target Purkinje cell (a complex spike). The contrast between parallel fiber and climbing fiber inputs to Purkinje cells (over 100,000 of one type versus exactly one of the other type) is perhaps the most provocative feature of cerebellar anatomy, and has motivated much of the theorizing. In fact, the function of climbing fibers is the most controversial topic concerning the cerebellum. There are two schools of thought, one following Marr and Albus in holding that climbing fiber input serves primarily as a teaching signal, the other holding that its function is to shape cerebellar output directly. Both views have been defended in great length in numerous publications. In the words of one review, "In trying to synthesize the various hypotheses on the function of the climbing fibers, one has the sense of looking at a drawing by Escher. Each point of view seems to account for a certain collection of findings, but when one attempts to put the different views together, a coherent picture of what the climbing fibers are doing does not appear. For the majority of researchers, the climbing fibers signal errors in motor performance, either in the usual manner of discharge frequency modulation or as a single announcement of an 'unexpected event'. For other investigators, the message lies in the degree of ensemble synchrony and rhythmicity among a population of climbing fibers." +The deep nuclei of the cerebellum are clusters of gray matter lying within the white matter at the core of the cerebellum. They are, with the minor exception of the nearby vestibular nuclei, the sole sources of output from the cerebellum. These nuclei receive collateral projections from mossy fibers and climbing fibers as well as inhibitory input from the Purkinje cells of the cerebellar cortex. The four nuclei (dentate, globose, emboliform, and fastigial) each communicate with different parts of the brain and cerebellar cortex. (The globose and the emboliform nuclei are also referred to as combined in the interposed nucleus). The fastigial and interposed nuclei belong to the spinocerebellum. The dentate nucleus, which in mammals is much larger than the others, is formed as a thin, convoluted layer of gray matter, and communicates exclusively with the lateral parts of the cerebellar cortex. The flocculus of the flocculonodular lobe is the only part of the cerebellar cortex that does not project to the deep nuclei—its output goes to the vestibular nuclei instead. +The majority of neurons in the deep nuclei have large cell bodies and spherical dendritic trees with a radius of about 400 μm, and use glutamate as their neurotransmitter. These cells project to a variety of targets outside the cerebellum. Intermixed with them are a lesser number of small cells, which use GABA as a neurotransmitter and project exclusively to the inferior olivary nucleus, the source of climbing fibers. Thus, the nucleo-olivary projection provides an inhibitory feedback to match the excitatory projection of climbing fibers to the nuclei. There is evidence that each small cluster of nuclear cells projects to the same cluster of olivary cells that send climbing fibers to it; there is strong and matching topography in both directions. +When a Purkinje cell axon enters one of the deep nuclei, it branches to make contact with both large and small nuclear cells, but the total number of cells contacted is only about 35 (in cats). Conversely, a single deep nuclear cell receives input from approximately 860 Purkinje cells (again in cats). +From the viewpoint of gross anatomy, the cerebellar cortex appears to be a homogeneous sheet of tissue, and, from the viewpoint of microanatomy, all parts of this sheet appear to have the same internal structure. There are, however, a number of respects in which the structure of the cerebellum is compartmentalized. There are large compartments that are generally known as zones; these can be divided into smaller compartments known as microzones. +The first indications of compartmental structure came from studies of the receptive fields of cells in various parts of the cerebellar cortex. Each body part maps to specific points in the cerebellum, but there are numerous repetitions of the basic map, forming an arrangement that has been called "fractured somatotopy". A clearer indication of compartmentalization is obtained by immunostaining the cerebellum for certain types of protein. The best-known of these markers are called "zebrins", because staining for them gives rise to a complex pattern reminiscent of the stripes on a zebra. The stripes generated by zebrins and other compartmentalization markers are oriented perpendicular to the cerebellar folds—that is, they are narrow in the mediolateral direction, but much more extended in the longitudinal direction. Different markers generate different sets of stripes, the widths and lengths vary as a function of location, but they all have the same general shape. +Oscarsson in the late 1970s proposed that these cortical zones can be partitioned into smaller units called microzones. A microzone is defined as a group of Purkinje cells all having the same somatotopic receptive field. Microzones were found to contain on the order of 1000 Purkinje cells each, arranged in a long, narrow strip, oriented perpendicular to the cortical folds. Thus, as the adjoining diagram illustrates, Purkinje cell dendrites are flattened in the same direction as the microzones extend, while parallel fibers cross them at right angles. +It is not only receptive fields that define the microzone structure: The climbing fiber input from the inferior olivary nucleus is equally important. The branches of a climbing fiber (usually numbering about 10) usually activate Purkinje cells belonging to the same microzone. Moreover, olivary neurons that send climbing fibers to the same microzone tend to be coupled by gap junctions, which synchronize their activity, causing Purkinje cells within a microzone to show correlated complex spike activity on a millisecond time scale. Also, the Purkinje cells belonging to a microzone all send their axons to the same small cluster of output cells within the deep cerebellar nuclei. Finally, the axons of basket cells are much longer in the longitudinal direction than in the mediolateral direction, causing them to be confined largely to a single microzone. The consequence of all this structure is that cellular interactions within a microzone are much stronger than interactions between different microzones. +In 2005, Richard Apps and Martin Garwicz summarized evidence that microzones themselves form part of a larger entity they call a multizonal microcomplex. Such a microcomplex includes several spatially separated cortical microzones, all of which project to the same group of deep cerebellar neurons, plus a group of coupled olivary neurons that project to all of the included microzones as well as to the deep nuclear area. +The cerebellum is provided with blood from three paired major arteries: the superior cerebellar artery (SCA), the anterior inferior cerebellar artery (AICA), and the posterior inferior cerebellar artery (PICA). The SCA supplies the upper region of the cerebellum. It divides at the upper surface and branches into the pia mater where the branches anastomose with those of the anterior and posterior inferior cerebellar arteries. The AICA supplies the front part of the undersurface of the cerebellum. The PICA arrives at the undersurface, where it divides into a medial branch and a lateral branch. The medial branch continues backward to the cerebellar notch between the two hemispheres of the cerebellum; while the lateral branch supplies the under surface of the cerebellum, as far as its lateral border, where it anastomoses with the AICA and the SCA. +The strongest clues to the function of the cerebellum have come from examining the consequences of damage to it. Animals and humans with cerebellar dysfunction show, above all, problems with motor control, on the same side of the body as the damaged part of the cerebellum. They continue to be able to generate motor activity but lose precision, producing erratic, uncoordinated, or incorrectly timed movements. A standard test of cerebellar function is to reach with the tip of the finger for a target at arm's length: A healthy person will move the fingertip in a rapid straight trajectory, whereas a person with cerebellar damage will reach slowly and erratically, with many mid-course corrections. Deficits in non-motor functions are more difficult to detect. Thus, the general conclusion reached decades ago is that the basic function of the cerebellum is to calibrate the detailed form of a movement, not to initiate movements or to decide which movements to execute. +Prior to the 1990s the function of the cerebellum was mostly believed to be purely motor-related, but later research has pointed to an expanded role of cerebellar connectivity beyond basic motoric functions. Functional imaging studies have shown cerebellar activation in relation to language, attention, and mental imagery; correlation studies have shown interactions between the cerebellum and non-motor areas of the cerebral cortex; and a variety of non-motor symptoms have been recognized in people with damage that appears to be confined to the cerebellum. In particular, the cerebellar cognitive affective syndrome or Schmahmann's syndrome has been described in adults and children. Estimates based on functional mapping of the cerebellum using functional MRI suggest that more than half of the cerebellar cortex is interconnected with association zones of the cerebral cortex. +Kenji Doya has argued that the cerebellum's function is best understood not in terms of the behaviors it affects, but the neural computations it performs; the cerebellum consists of a large number of more or less independent modules, all with the same geometrically regular internal structure, and therefore all, it is presumed, performing the same computation. If the input and output connections of a module are with motor areas (as many are), then the module will be involved in motor behavior; but, if the connections are with areas involved in non-motor cognition, the module will show other types of behavioral correlates. Thus the cerebellum has been implicated in the regulation of many differing functional traits such as affection, emotion including emotional body language perception and behavior. The cerebellum, Doya proposes, is best understood as predictive action selection based on "internal models" of the environment or a device for supervised learning, in contrast to the basal ganglia, which perform reinforcement learning, and the cerebral cortex, which performs unsupervised learning. Three decades of brain research have led to the proposal that the cerebellum generates optimized mental models and interacts closely with the cerebral cortex, where updated internal models are experienced as creative intuition ("a ha") in working memory. It has also been proposed that the cerebellum and neocortex construct predictive world models through prediction-error learning and reuse these models for sensory comprehension and motor output generation, drawing parallels with autoregressive generative AI.In this framework, cerebellar prediction-error learning, traditionally classified as supervised learning, can instead be understood as a form of unsupervised or self-supervised learning, analogous to that used in autoregressive generative AI, in which future sensory inputs serve as the prediction targets rather than externally supplied ground-truth labels. +The comparative simplicity and regularity of the cerebellar anatomy led to an early hope that it might imply a similar simplicity of computational function, as expressed in one of the first books on cerebellar electrophysiology, The Cerebellum as a Neuronal Machine by John C. Eccles, Masao Ito, and János Szentágothai. Although a full understanding of cerebellar function has remained elusive, at least four principles have been identified as important: (1) feedforward processing, (2) divergence and convergence, (3) modularity, and (4) plasticity. +Feedforward processing: The cerebellum differs from most other parts of the brain (especially the cerebral cortex) in that the signal processing is almost entirely feedforward—that is, signals move unidirectionally through the system from input to output, with very little recurrent internal transmission. The small amount of recurrence that does exist consists of mutual inhibition; there are no mutually excitatory circuits. This feedforward mode of operation means that the cerebellum, in contrast to the cerebral cortex, cannot generate self-sustaining patterns of neural activity. Signals enter the circuit, are processed by each stage in sequential order, and then leave. As Eccles, Ito, and Szentágothai wrote, "This elimination in the design of all possibility of reverberatory chains of neuronal excitation is undoubtedly a great advantage in the performance of the cerebellum as a computer, because what the rest of the nervous system requires from the cerebellum is presumably not some output expressing the operation of complex reverberatory circuits in the cerebellum but rather a quick and clear response to the input of any particular set of information." +Divergence and convergence: In the human cerebellum, information from 200 million mossy fiber inputs is expanded to 40 billion granule cells, whose parallel fiber outputs then converge onto 15 million Purkinje cells. Because of the way that they are lined up longitudinally, the 1000 or so Purkinje cells belonging to a microzone may receive input from as many as 100 million parallel fibers, and focus their own output down to a group of fewer still deep nuclear cells. Thus, the cerebellar network receives a modest number of inputs, processes them very extensively through its rigorously structured internal network, and sends out the results via a very limited number of output cells. +Modularity: The cerebellar system is functionally divided into more or less independent modules, which probably number in the hundreds to thousands. All modules have a similar internal structure, but different inputs and outputs. A module (a multizonal microcompartment in the terminology of Apps and Garwicz) consists of a small cluster of neurons in the inferior olivary nucleus, a set of long narrow strips of Purkinje cells in the cerebellar cortex (microzones), and a small cluster of neurons in one of the deep cerebellar nuclei. Different modules share input from mossy fibers and parallel fibers, but in other respects they appear to function independently—the output of one module does not appear to significantly influence the activity of other modules. +Plasticity: The synapses between parallel fibers and Purkinje cells, and the synapses between mossy fibers and deep nuclear cells, are both susceptible to modification of their strength. In a single cerebellar module, input from as many as a billion parallel fibers converges onto a group of less than 50 deep nuclear cells, and the influence of each parallel fiber on those nuclear cells is adjustable. This arrangement gives tremendous flexibility for fine-tuning the relationship between the cerebellar inputs and outputs. +There is considerable evidence that the cerebellum plays an essential role in some types of motor learning. The tasks where the cerebellum most clearly comes into play are those in which it is necessary to make fine adjustments to the way an action is performed. There has, however, been much dispute about whether learning takes place within the cerebellum itself, or whether it merely serves to provide signals that promote learning in other brain structures. Most theories that assign learning to the circuitry of the cerebellum are derived from the ideas of David Marr and James Albus, who postulated that climbing fibers provide a teaching signal that induces synaptic modification in parallel fiber–Purkinje cell synapses. Marr assumed that climbing fiber input would cause synchronously activated parallel fiber inputs to be strengthened. Most subsequent cerebellar-learning models, however, have followed Albus in assuming that climbing fiber activity would be an error signal, and would cause synchronously activated parallel fiber inputs to be weakened. Some of these later models, such as the Adaptive Filter model of Fujita made attempts to understand cerebellar function in terms of optimal control theory. +The idea that climbing fiber activity functions as an error signal has been examined in many experimental studies, with some supporting it but others casting doubt. In a pioneering study by Gilbert and Thach from 1977, Purkinje cells from monkeys learning a reaching task showed increased complex spike activity—which is known to reliably indicate activity of the cell's climbing fiber input—during periods when performance was poor. Several studies of motor learning in cats observed complex spike activity when there was a mismatch between an intended movement and the movement that was actually executed. Studies of the vestibulo-ocular reflex (which stabilizes the visual image on the retina when the head turns) found that climbing fiber activity indicated "retinal slip", although not in a very straightforward way. +One of the most extensively studied cerebellar learning tasks is the eyeblink conditioning paradigm, in which a neutral conditioned stimulus (CS) such as a tone or a light is repeatedly paired with an unconditioned stimulus (US), such as an air puff, that elicits a blink response. After such repeated presentations of the CS and US, the CS will eventually elicit a blink before the US, a conditioned response or CR. Experiments showed that lesions localized either to a specific part of the interposed nucleus (one of the deep cerebellar nuclei) or to a few specific points in the cerebellar cortex would abolish learning of a conditionally timed blink response. If cerebellar outputs are pharmacologically inactivated while leaving the inputs and intracellular circuits intact, learning takes place even while the animal fails to show any response, whereas, if intracerebellar circuits are disrupted, no learning takes place—these facts taken together make a strong case that the learning, indeed, occurs inside the cerebellum. +The large base of knowledge about the anatomical structure and behavioral functions of the cerebellum have made it a fertile ground for theorizing—there are perhaps more theories of the function of the cerebellum than of any other part of the brain. The most basic distinction among them is between "learning theories" and "performance theories"—that is, theories that make use of synaptic plasticity within the cerebellum to account for its role in learning, versus theories that account for aspects of ongoing behavior on the basis of cerebellar signal processing. Several theories of both types have been formulated as mathematical models and simulated using computers. A Kalman filter model has been proposed that fits with two major requirements: the cerebellum as involved in predictions and in sequencing. A biologically constrained cerebellar neural network incorporating a recurrent pathway from output to input neurons has also been shown to acquire word prediction and develop representations of syntactic roles in its Purkinje-cell layer; the recurrent loop was required for both functions. +Perhaps the earliest "performance" theory was the "delay line" hypothesis of Valentino Braitenberg. The original theory put forth by Braitenberg and Roger Atwood in 1958 proposed that slow propagation of signals along parallel fibers imposes predictable delays that allow the cerebellum to detect time relationships within a certain window. Experimental data did not support the original form of the theory, but Braitenberg continued to argue for modified versions. The hypothesis that the cerebellum functions essentially as a timing system has also been advocated by Richard Ivry. Another influential "performance" theory is the Tensor network theory of Pellionisz and Llinás, which provided an advanced mathematical formulation of the idea that the fundamental computation performed by the cerebellum is to transform sensory into motor coordinates. +Theories in the "learning" category almost all derive from publications by Marr and Albus. Marr's 1969 paper proposed that the cerebellum is a device for learning to associate elemental movements encoded by climbing fibers with mossy fiber inputs that encode the sensory context. Albus proposed in 1971 that a cerebellar Purkinje cell functions as a perceptron, a neurally inspired abstract learning device. The most basic difference between the Marr and Albus theories is that Marr assumed that climbing fiber activity would cause parallel fiber synapses to be strengthened, whereas Albus proposed that they would be weakened. Albus also formulated his version as a software algorithm he called a CMAC (Cerebellar Model Articulation Controller), which has been tested in a number of applications. +Damage to the cerebellum often causes motor-related symptoms, the details of which depend on the part of the cerebellum involved and how it is damaged. Damage to the flocculonodular lobe may show up as a loss of equilibrium and in particular an altered, irregular walking gait, with a wide stance caused by difficulty in balancing. Damage to the lateral zone typically causes problems in skilled voluntary and planned movements which can cause errors in the force, direction, speed and amplitude of movements. Other manifestations include hypotonia (decreased muscle tone), dysarthria (problems with speech articulation), dysmetria (problems judging distances or ranges of movement), dysdiadochokinesia (inability to perform rapid alternating movements such as walking), impaired check reflex or rebound phenomenon, and intention tremor (involuntary movement caused by alternating contractions of opposing muscle groups). Damage to the midline portion may disrupt whole-body movements, whereas damage localized more laterally is more likely to disrupt fine movements of the hands or limbs. Damage to the upper part of the cerebellum tends to cause gait impairments and other problems with leg coordination; damage to the lower part is more likely to cause uncoordinated or poorly aimed movements of the arms and hands, as well as difficulties in speed. This complex of motor symptoms is called ataxia. +To identify cerebellar problems, neurological examination includes assessment of gait (a broad-based gait being indicative of ataxia), finger-pointing tests and assessment of posture. If cerebellar dysfunction is indicated, a magnetic resonance imaging scan can be used to obtain a detailed picture of any structural alterations that may exist. +The list of medical problems that can produce cerebellar damage is long, including stroke, hemorrhage, swelling of the brain (cerebral edema), tumors, alcoholism, physical trauma such as gunshot wounds or explosives, and chronic degenerative conditions such as olivopontocerebellar atrophy. Some forms of migraine headache may also produce temporary dysfunction of the cerebellum, of variable severity. Infection can result in cerebellar damage in such conditions as the prion diseases and Miller Fisher syndrome, a variant of Guillain–Barré syndrome. +The human cerebellum changes with age. These changes may differ from those of other parts of the brain. +The cerebellum is the youngest brain region (and body part) according to an epigenetic biomarker of tissue age known as an epigenetic clock: it is about 15 years younger than expected in a centenarian. Further, gene expression patterns in the human cerebellum show less age-related alteration than that in the cerebral cortex. +Some studies have reported reductions in numbers of cells or volume of tissue, but the amount of data relating to this question is not very large. +Congenital malformation, hereditary disorders, and acquired conditions can affect cerebellar structure and, consequently, cerebellar function. Unless the causative condition is reversible, the only possible treatment is to help people live with their problems. Visualization of the fetal cerebellum by ultrasound scan at 18 to 20 weeks of pregnancy can be used to screen for fetal neural tube defects with a sensitivity rate of up to 99%. +In normal development, endogenous sonic hedgehog signaling stimulates rapid proliferation of cerebellar granule neuron progenitors (CGNPs) in the external granule layer (EGL). Cerebellar development occurs during late embryogenesis and the early postnatal period, with CGNP proliferation in the EGL peaking during early development (postnatal day 7 in the mouse). As CGNPs terminally differentiate into cerebellar granule cells (also called cerebellar granule neurons, CGNs), they migrate to the internal granule layer (IGL), forming the mature cerebellum (by post-natal day 20 in the mouse). Mutations that abnormally activate sonic hedgehog signaling predispose to cancer of the cerebellum (medulloblastoma) in humans with Gorlin Syndrome and in genetically engineered mouse models. +Congenital malformation or underdevelopment (hypoplasia) of the cerebellar vermis is a characteristic of both Dandy–Walker syndrome and Joubert syndrome. In very rare cases, the entire cerebellum may be absent. The inherited neurological disorders Machado–Joseph disease, ataxia telangiectasia, and Friedreich's ataxia cause progressive neurodegeneration linked to cerebellar loss. Congenital brain malformations outside the cerebellum can, in turn, cause herniation of cerebellar tissue, as seen in some forms of Arnold–Chiari malformation. +Other conditions that are closely linked to cerebellar degeneration include the idiopathic progressive neurological disorders multiple system atrophy and Ramsay Hunt syndrome type I, and paraneoplastic cerebellar degeneration an autoimmune disorder, in which tumors elsewhere in the body elicit an autoimmune response that causes neuronal loss in the cerebellum. Cerebellar atrophy can result from chronic thiamine (vitamin B1) deficiency as seen in dry beriberi and in Wernicke–Korsakoff syndrome, or from vitamin E deficiency. +Cerebellar atrophy has been observed in many other neurological disorders including Huntington's disease, multiple sclerosis, essential tremor, progressive myoclonus epilepsy, and Niemann–Pick disease. Cerebellar atrophy can also occur as a result of exposure to toxins including heavy metals or pharmaceutical or recreational drugs. +There is a general consensus that the cerebellum is involved in pain processing. The cerebellum receives pain input from both descending cortico-cerebellar pathways and ascending spino-cerebellar pathways, through the pontine nuclei and inferior olives. Some of this information is transferred to the motor system inducing a conscious motor avoidance of pain, graded according to pain intensity. +These direct pain inputs, as well as indirect inputs, are thought to induce long-term pain avoidance behavior that results in chronic posture changes and consequently, in functional and anatomical remodeling of vestibular and proprioceptive nuclei. As a result, chronic neuropathic pain can induce macroscopic anatomical remodeling of the hindbrain, including the cerebellum. The magnitude of this remodeling and the induction of neuron progenitor markers suggest the contribution of adult neurogenesis to these changes. +The circuits in the cerebellum are similar across all classes of vertebrates, including fish, reptiles, birds, and mammals. There is also an analogous brain structure in cephalopods with well-developed brains, such as octopuses. This has been taken as evidence that the cerebellum performs functions important to all animal species with a brain. +There is considerable variation in the size and shape of the cerebellum in different vertebrate species. In amphibians, it is little developed, and in lampreys, and hagfish, the cerebellum is barely distinguishable from the brain-stem. Although the spinocerebellum is present in these groups, the primary structures are small, paired-nuclei corresponding to the vestibulocerebellum. The cerebellum is a bit larger in reptiles, considerably larger in birds, and larger still in mammals. The large paired and convoluted lobes found in humans are typical of mammals, but the cerebellum is, in general, a single median lobe in other groups, and is either smooth or only slightly grooved. In mammals, the neocerebellum is the major part of the cerebellum by mass, but, in other vertebrates, it is typically the spinocerebellum. +The cerebellum of cartilaginous and bony fishes is extraordinarily large and complex. In at least one important respect, it differs in internal structure from the mammalian cerebellum: The fish cerebellum does not contain discrete deep cerebellar nuclei. Instead, the primary targets of Purkinje cells are a distinct type of cell distributed across the cerebellar cortex, a type not seen in mammals. In mormyrid fish (a family of weakly electrosensitive freshwater fish), the cerebellum is considerably larger than the rest of the brain. The largest part of it is a special structure called the valvula, which has an unusually regular architecture and receives much of its input from the electrosensory system. +The hallmark of the mammalian cerebellum is an expansion of the lateral lobes, whose main interactions are with the neocortex. As monkeys evolved into great apes, the expansion of the lateral lobes continued, in tandem with the expansion of the frontal lobes of the neocortex. In ancestral hominids, and in Homo sapiens until the middle Pleistocene period, the cerebellum continued to expand, but the frontal lobes expanded more rapidly. The most recent period of human evolution, however, may actually have been associated with an increase in the relative size of the cerebellum, as the neocortex reduced its size somewhat while the cerebellum expanded. The size of the human cerebellum, compared to the rest of the brain, has been increasing in size while the cerebrum decreased in size. With both the development and implementation of motor tasks, visual-spatial skills and learning taking place in the cerebellum, the growth of the cerebellum is thought to have some form of correlation to greater human cognitive abilities. The lateral hemispheres of the cerebellum are now 2.7 times greater in both humans and apes than they are in monkeys. These changes in the cerebellum size cannot be explained by greater muscle mass. They show that either the development of the cerebellum is tightly linked to that of the rest of the brain or that neural activities taking place in the cerebellum were important during Hominidae evolution. Due to the cerebellum's role in cognitive functions, the increase in its size may have played a role in cognitive expansion. +Most vertebrate species have a cerebellum and one or more cerebellum-like structures, brain areas that resemble the cerebellum in terms of cytoarchitecture and neurochemistry. The only cerebellum-like structure found in mammals is the dorsal cochlear nucleus (DCN), one of the two primary sensory nuclei that receive input directly from the auditory nerve. The DCN is a layered structure, with the bottom layer containing granule cells similar to those of the cerebellum, giving rise to parallel fibers that rise to the superficial layer and travel across it horizontally. The superficial layer contains a set of GABAergic neurons called cartwheel cells that resemble Purkinje cells anatomically and chemically—they receive parallel fiber input, but do not have any inputs that resemble climbing fibers. The output neurons of the DCN are pyramidal cells. They are glutamatergic, but also resemble Purkinje cells in some respects—they have spiny, flattened superficial dendritic trees that receive parallel fiber input, but they also have basal dendrites that receive input from auditory nerve fibers, which travel across the DCN in a direction at right angles to the parallel fibers. The DCN is most highly developed in rodents and other small animals, and is considerably reduced in primates. Its function is not well understood; the most popular speculations relate it to spatial hearing in one way or another. +Most species of fish and amphibians possess a lateral line system that senses pressure waves in water. One of the brain areas that receives primary input from the lateral line organ, the medial octavolateral nucleus, has a cerebellum-like structure, with granule cells and parallel fibers. In electrosensitive fish, the input from the electrosensory system goes to the dorsal octavolateral nucleus, which also has a cerebellum-like structure. In ray-finned fishes (by far the largest group), the optic tectum has a layer—the marginal layer—that is cerebellum-like. +All of these cerebellum-like structures appear to be primarily sensory-related rather than motor-related. All of them have granule cells that give rise to parallel fibers that connect to Purkinje-like neurons with modifiable synapses, but none have climbing fibers comparable to those of the cerebellum—instead they receive direct input from peripheral sensory organs. None has a demonstrated function, but the most influential speculation is that they serve to transform sensory inputs in some sophisticated way, perhaps to compensate for changes in body posture. In fact, James M. Bower and others have argued, partly on the basis of these structures and partly on the basis of cerebellar studies, that the cerebellum itself is fundamentally a sensory structure, and that it contributes to motor control by moving the body in a way that controls the resulting sensory signals. Despite Bower's viewpoint, there is also strong evidence that the cerebellum directly influences motor output in mammals. +Even the earliest anatomists were able to recognize the cerebellum by its distinctive appearance. Aristotle and Herophilus (quoted in Galen) called it the παρεγκεφαλίς (parenkephalis), as opposed to the ἐγκέφαλος (enkephalos) or brain proper. Galen's extensive description is the earliest that survives. He speculated that the cerebellum was the source of motor nerves. +Further significant developments did not come until the Renaissance. Vesalius discussed the cerebellum briefly, and the anatomy was described more thoroughly by Thomas Willis in 1664. More anatomical work was done during the 18th century, but it was not until early in the 19th century that the first insights into the function of the cerebellum were obtained. Luigi Rolando in 1809 established the key finding that damage to the cerebellum results in motor disturbances. Jean Pierre Flourens in the first half of the 19th century carried out detailed experimental work, which revealed that animals with cerebellar damage can still move, but with a loss of coordination (strange movements, awkward gait, and muscular weakness), and that recovery after the lesion can be nearly complete unless the lesion is very extensive. By the beginning of the 20th century, it was widely accepted that the primary function of the cerebellum relates to motor control; the first half of the 20th century produced several detailed descriptions of the clinical symptoms associated with cerebellar disease in humans. +The name cerebellum is a diminutive of cerebrum (brain); it can be translated literally as little brain. The Latin name is a direct translation of the Ancient Greek παρεγκεφαλίς (parenkephalis), which was used in the works of Aristotle, the first known writer to describe the structure. No other name is used in the English-language literature, but historically a variety of Greek or Latin-derived names have been used, including cerebrum parvum, encephalion, encranion, cerebrum posterius, and parencephalis. +This article was submitted to WikiJournal of Medicine for external academic peer review in 2016 (reviewer reports). The updated content was reintegrated into the Wikipedia page under a CC-BY-SA-3.0 license (2016). The version of record as reviewed is: Marion Wright; Skaggs, William; Nielsen, Finn; et al. (2016). "The Cerebellum" (PDF). WikiJournal of Medicine. 3 (1). doi:10.15347/wjm/2016.001. Wikidata Q44001486. +Media related to Cerebellum at Wikimedia Commons +Llinas R, Negrello MN (2015). "Cerebellum". Scholarpedia. 10 (1): 4606. Bibcode:2015SchpJ..10.4606L. doi:10.4249/scholarpedia.4606. +Cerebellum–Cell Centered Database +Handbook of the Cerebellum and Cerebellar Disorders – Manto, M., Gruol, D.L., Schmahmann, J., Koibuchi, N., Rossi, F. (Eds.) – Springer – New York +Stained brain slice images which include the "cerebellum" at the BrainMaps project +A Man's Incomplete Brain Reveals Cerebellum's Role In Thought And Emotion +Woman living without cerebellum +Essentials of Cerebellum and Cerebellar Disorders. Gruol, D.L., Koibuchi, N., Manto, M., Molinari, M., Schmahmann, J.D., Shen, Y. (Eds.). Springer, New York, 2016 +Cerebellum histology images +The Cerebellum – Journal (Springer Nature) +Cerebellum and Ataxias – Journal (BioMed Central) +The 1995 Japanese Grand Prix (formally the XXI Fuji Television Japanese Grand Prix) was a Formula One motor race held at the Suzuka Circuit, Suzuka on 29 October 1995. It was the sixteenth and penultimate race of the 1995 Formula One World Championship. The 53-lap race was won from pole position by German Michael Schumacher, driving a Benetton-Renault, with Finn Mika Häkkinen second in a McLaren-Mercedes and Schumacher's British teammate Johnny Herbert third. +Jean Alesi, driving for Ferrari, started second, alongside Schumacher. However, Alesi was forced to serve a 10-second stop-and-go penalty because his car moved forward before the start. Alesi climbed back up to second, before retiring on lap 25. Schumacher's rival in the Drivers' Championship, Damon Hill, started fourth amidst pressure from the British media after poor performances at previous races. Hill moved up to second because of Alesi's retirement, but spun off the track on lap 40. +Schumacher's win was his ninth of the season, matching the record set in 1992 by Nigel Mansell. Benetton was confirmed Constructors' Champions as Williams could not pass its points total in the one remaining race. +This race marked that last time until 2010 Bahrain Grand Prix that 24 drivers and cars started a Grand Prix. +Heading into the penultimate race of the season, Benetton driver Michael Schumacher had already won the season's Drivers' Championship, having clinched the title at the previous race, the Pacific Grand Prix. Schumacher led the championship with 92 points; Damon Hill was second with 59 points. A maximum of 20 points were available for the remaining two races, which meant that Hill could not catch Schumacher. Although the Drivers' Championship was decided, the Constructors' Championship was not. Benetton were leading on 123 points and Williams were second with 102 points heading into the 16th race, with a maximum of 32 points available. In the week leading up to the race, Hill was criticised by the British media after poor performances in previous races; there was continued speculation that Williams were going to replace him with Heinz-Harald Frentzen or Gerhard Berger for the 1996 season. Despite the rumours, Williams team boss Frank Williams gave Hill "an unequivocal vote of confidence" heading into the race. +There were two driver changes heading into the race. Having been in one of the two Sauber cars since the fifth race of the season at Monaco, Jean-Christophe Boullion was released from the team and replaced by Karl Wendlinger. The Austrian was given another chance to prove himself after suffering an accident at the 1994 Monaco Grand Prix, which left him in a coma for weeks. The second driver change was Mika Häkkinen's return to McLaren after missing the Pacific Grand Prix because of an operation for appendicitis. +Two practice sessions were held before the race; the first was held on Friday morning and the second on Saturday morning. Both sessions lasted 1 hour and 45 minutes with weather conditions dry throughout. Schumacher was fastest in the first session, posting a time of 1:40.410, two-tenths of a second quicker than Häkkinen. The Williams and Ferrari cars occupied the remaining top six positions; Williams drivers Hill and David Coulthard third and fifth respectively. The Ferrari cars were fourth and sixth fastest; Jean Alesi ahead of Berger. Häkkinen lapped faster than Schumacher in the second practice session with a time of 1:40.389. Eddie Irvine took second place in the Jordan car, three-tenths of a second behind Häkkinen. Hill was third in the Williams, two-tenths behind Häkkinen, with Schumacher fourth behind Hill. The Ferrari cars were fifth and eighth; Alesi in front of Berger. Frentzen's Sauber and Coulthard's Williams split the Ferrari drivers. Despite both the Williams cars going off into the gravel, Hill and Coulthard made the top 10. +The qualifying session was split into two one-hour sessions; the first was held on Friday afternoon with the second held on Saturday afternoon. The fastest time from either sessions counted towards their final grid position. Schumacher clinched his tenth career pole position, in his Benetton B195, with a time of 1:38.023. He was joined on the front row by Alesi, who was eight-tenths of a second behind. Schumacher was particularly pleased with the performance of his Benetton, saying that "I have rarely had such a good car ... I think I can be confident for the race". Alesi was satisfied about his performance, but worried about a mechanical problem which had caused him to crash on Friday, accusing the Ferrari team of withholding information from him. Alesi was scheduled to leave Ferrari for Benetton in a swap with Schumacher at the end of the season, and the relationship between him and the team was becoming increasingly strained. Häkkinen was third in the McLaren, with Hill fourth, a second slower than Schumacher. Despite Häkkinen's best qualifying effort of the season alongside his Belgium third place, his teammate, Mark Blundell, had a disappointing qualifying session. In the first part of qualifying, Blundell crashed into the wall, meaning he could not set a time as his car was too badly damaged. Blundell had his second crash of the weekend at the 130R corner in Saturday practice, which was more serious than the first. Following medical advice, Blundell did not participate in the second qualifying session. He was unable to set a time, leaving him at the back of the grid. Aguri Suzuki crashed his Ligier during Saturday qualifying; he was unable to start the race since he was in a hospital with a broken rib. +The drivers took to the track at 09:30 JST (GMT +9) for a 30-minute warmup session. Despite underperforming in qualifying, both Williams cars performed better in the wet weather warmup session; Hill had the fastest time of 2:00.025. Coulthard was third in the other Williams car; Schumacher split them in second position. Alesi completed the top four, eight-tenths of a second behind Hill. +The track surface was damp for most of the race, which meant that lap times were slower than the previous days' qualifying sessions. Though 24 cars qualified for the race, only 22 took the start: Suzuki was unable to start because of his crash in qualifying and Roberto Moreno's Forti car suffered a gearbox problem. For the first time since the Japanese Grand Prix was held at Suzuka in 1987, tickets for the race did not sell out, despite the fact that three Japanese drivers entered the race. +The race started at 14:00 JST. All of the drivers opted to start on wet-weather tyres as the track was damp from the morning rain. Schumacher, from pole position on the grid, held onto the lead into the first corner. Alesi, who started alongside Schumacher, was judged to have jumped the start, and served a 10-second stop-and-go penalty on lap three, from which he returned to the race in tenth place. Alesi's teammate Berger also jumped the start and received the same penalty. Gianni Morbidelli, near the back of the field in one of the Footwork cars, spun at the first corner on lap one after being hit from behind by Wendlinger's Sauber. Morbidelli stalled his car in the process, forcing him to retire from the race. On lap seven, Alesi stopped at the pits to change to dry weather slick tyres, as the track was beginning to dry. On returning to the race, he began to make his way through the field constantly recording fastest laps; the first of which was 1:54.416, five seconds faster than the remainder of the field. Schumacher made a pit stop on lap 10 for slicks, handing the lead to Häkkinen for a lap before he too pitted. Alesi's progress was interrupted when he spun attempting to pass Pedro Lamy's Minardi for 15th place, but he made his way up to second by lap 10, overtaking Hill around the outside in the final chicane to take the place. Alerted by Alesi's pace on the slick tyres, the other drivers came into the pits to change to slick tyres. +The two Jordan cars collided on lap 15. Rubens Barrichello spun in the final chicane when he attempted to brake later than his teammate Irvine. Barrichello hit a wall, which damaged his car's rear wing and caused him to retire from the race. Irvine was involved in another collision at the chicane on lap 20 when Frentzen hit him from behind. Irvine continued without damage, but Frentzen had to pit for a new front wing. At the front, Alesi was lapping faster than Schumacher, even though Schumacher was on dry tyres. Alesi was only six seconds behind Schumacher when his Ferrari 412T2 suffered an apparent differential failure on lap 25. It was later discovered that the problem was a driveshaft failure, possibly as a result of his earlier spin. Schumacher made a pit stop for a second time on lap 31, returning to the race in second place behind Hill. Schumacher set the fastest lap of the race on lap 33, and regained his lead on the next lap when Hill made his pit stop. Behind them, Häkkinen and Coulthard were third and fourth respectively before their pit stops, but Coulthard pitted six laps later than Häkkinen and returned to the track in third place, one place ahead of the Finn. Johnny Herbert was fifth in the second Benetton car after the second round of pit stops, with Irvine rounding out the point-scoring places in sixth. +At this stage, the rain began to fall again, but only at the Spoon Curve end of the track. The Williams drivers were second and third until Hill ran off the track at Spoon Curve two laps after his pit stop. He damaged his front wing in the process and returned to the track in fourth. Hill returned to the pits to let his pit crew replace the damaged wing. He rejoined fifth, but was then given a ten-second stop-and-go penalty for speeding in the pitlane. Coulthard made the same mistake as his teammate by running through the gravel trap at the Spoon Curve but looked like he was going to escape with only minor damage. However, as he braked for 130R, the next corner, the gravel which had entered his sidepods flew out, causing him to lose control and get his car stuck in the gravel trap. Hill was told by his team on the radio to speed up as he had not yet taken his stop-and-go penalty, but later that lap he spun off at Spoon Curve and retired from the race without having taken the penalty. Blundell, Irvine and Frentzen also left the track at Spoon Curve but all finished the race. With his closest challenger out, Schumacher won the race after 53 laps to secure his ninth victory of the season in a time of 1:36:52.930. The win, along with Herbert's third place and the retirements of Hill and Coulthard, gave Benetton the 1995 Constructors' Championship. Häkkinen finished second in his McLaren, 20 seconds behind Schumacher. Irvine was fourth in his Jordan with Olivier Panis fifth in his Ligier. Mika Salo took sixth place and the final point in his Tyrrell. Despite starting last, Blundell finished in seventh, just 1.6 seconds behind Salo. The delayed Frentzen, Luca Badoer, Wendlinger, Lamy and Taki Inoue completed the finishers. +This was Schumacher's last win for Benetton, as he moved to the Ferrari team for the 1996 season. Herbert reiterated Schumacher's opinion by stating that Benetton did "a fantastic job". Hill was disappointed about the race and the season as a whole; he said afterwards: +Just when you think that it couldn't get any worse, it does. There is no easy way out of this, you just have to keep pressing on. The easiest thing to do is to give up, and it would probably be less painful that way, but that is not an option. While we were in the race we were competitive and I was in with a shout, I suppose, all the time I was on the track. But things took a massive turn for the worse, I am afraid. I drove through the rain and the second time I spun off I think it was oil rather than rain. It is not a glorious end to the season but the ingredients are all there and there is no reason why we should not get into the winning habit again. +As a result of Hill not taking his 10-second stop-and-go penalty because of his retirement, Williams were fined $10,000 by Formula One's governing body, the Fédération Internationale de l'Automobile (FIA). In an interview with Motor Sport magazine in 2008, Hill said that the 1995 season, as a whole, "went down, mentally, and it all just got to me". He also said he believed that it was in 1995 that Frank Williams and Patrick Head decided to replace him for the 1997 season. +1980 Formula One World Champion Alan Jones praised Alesi's performance, saying that it "will go down as one of the great drives in Grand Prix racing". Alesi stated that if his driveshaft had not failed, he would "have fought for it, all the way to the end". Alesi added that he believed he did not jump the start, but admitted that "the car crept forwards by a few centimetres" because of the downhill slope of the grid. In an interview with Autosport magazine in 2009, Alesi said that he went to see the race director before the race to see how he could avoid a penalty at the downhill start and that he was "totally fed up" with the penalty decision given. Berger also questioned his penalty, claiming that his car did not move before the green light went on. Alesi has mentioned several times that this was "the race of my life". +Bold text indicates the World Champions. +Note: Only the top five positions are included for both sets of standings. +A team of approximately 1,000 people developed Grand Theft Auto V over several years. Rockstar Games released the action-adventure game in September 2013 for PlayStation 3 and Xbox 360, in November 2014 for PlayStation 4 and Xbox One, in April 2015 for Windows, and in March 2022 for PlayStation 5 and Xbox Series X/S. The first main Grand Theft Auto series entry since Grand Theft Auto IV, its development was led by Rockstar North's core 360-person team, who collaborated with several other international Rockstar studios. The team considered the game a spiritual successor to many of their previous projects like Red Dead Redemption and Max Payne 3. After its unexpected announcement in 2011, the game was promoted with press showings, cinematic trailers, viral marketing strategies and special editions. Its release date, though subject to several delays, was widely anticipated. +The open world setting, modelled on Los Angeles and other areas of Southern California, constituted much of the development effort. Key team members conducted field trips around Southern California to gather research and footage, and Google Maps projections of Los Angeles were used to help design the city's road networks. The proprietary Rockstar Advanced Game Engine (RAGE) was overhauled to increase its draw distance rendering capabilities. For the first time in the series, players control three protagonists throughout the single-player mode. The team found the multiple-protagonist design a fundamental change to the story and gameplay devices. They refined the shooting and driving mechanics and tightened the narrative's pacing and scope. +The actors selected to portray the protagonists invested much time and research into character development. Motion capture was used to record the characters' facial and body movements. Like its predecessors, the game features an in-game radio that plays a selection of licensed music tracks. An original score was composed over several years by a team of five music producers. They worked in close collaboration, sampling and incorporating different influences into each other's ideas. The game's 2014 re-release added a first-person view option along with the traditional third-person view. To accommodate first-person, the game received a major visual and technical upgrade, as well as new gameplay features like a replay editor that lets players create gameplay videos. +Preliminary work on Grand Theft Auto V began around Grand Theft Auto IV's release in April 2008; full development lasted approximately three years. Rockstar North's core 360-person team co-opted studios around the world owned by parent company Rockstar Games to facilitate development between a full team of over 1,000. These included Rockstar's Leeds, Lincoln, London, New England, San Diego and Toronto studios. Technical director Adam Fowler said that while development was shared between studios in different countries, the process involved close collaboration between the core team and others. This was necessary to avoid difficulties if studios did not communicate with each other as many game mechanics work in tandem. Game development ceased by 25 August 2013, when it was submitted for manufacturing. Media analyst Arvind Bhatia estimated the game's development budget exceeded US$137 million, and The Scotsman reporter Marty McLaughlin estimated that the combined development and marketing efforts exceeded £170 million (US$265 million), making it the most expensive video game ever made at its time. +The proprietary Rockstar Advanced Game Engine (RAGE) was overhauled for the game to improve its draw distance rendering capabilities, and the Euphoria and Bullet engines handle further animation and environment rendering tasks. The team found they could render the game world with greater detail than in Grand Theft Auto IV because they had become familiar with the PlayStation 3 and Xbox 360's hardware. Art director Aaron Garbut said that while the consoles' ageing hardware was tiring to work with, the team could still render detailed lighting and shadows and "maintain a consistent look". Vice president Dan Houser felt working on Grand Theft Auto IV with relatively new hardware was a challenge, but the team had since learnt to develop for the consoles more efficiently. The PlayStation 3, PlayStation 4 and Xbox One versions fit onto one Blu-ray Disc; Xbox 360 copies are distributed on two DVDs and require an 8 GB installation on the HDD or external storage device; while the Windows (PC) version takes up seven DVDs. The team asserted any differences between the PlayStation 3 and Xbox 360 versions would be negligible. +Initial work on Grand Theft Auto V constituted the open world creation, where preliminary models were constructed in-engine during pre-production. The game's setting is the fictional US state of San Andreas and city of Los Santos, based on Southern California and Los Angeles respectively. San Andreas was first used as Grand Theft Auto: San Andreas' setting, which featured three cities separated by open countryside. The team thought the ambition of including three cities in San Andreas was too high, as it did not emulate the cities as well as they had hoped. Houser felt an effective portrayal of Los Angeles needs to emulate its urban sprawl, and that dividing the workforce between multiple cities would have detracted from capturing "what L.A. is". Garbut said PlayStation 2 era technology lacked the technical capabilities to capture Los Angeles adequately, such that San Andreas' rendition of Los Santos looked like a "backdrop or a game level with pedestrians randomly milling about". The team disregarded San Andreas as a departure point for Grand Theft Auto V because they had moved on to a new generation of consoles and wanted to build the city from scratch. According to Garbut, game hardware had "evolved so much from San Andreas" that using it as a model would have been redundant. The team's focus on one city instead of three meant they could produce Los Santos in higher quality and at a grander scale than in the previous game. +Los Angeles was extensively researched for the game. The team organised field research trips with tour guides and architectural historians, and captured around 250,000 photographs and many hours of video footage. Houser said, "We spoke to FBI agents that have been undercover, experts in the Mafia, street gangsters who know the slang—we even went to see a proper prison". He considered the open world's research and creation the most challenging aspects of the game's production. Google Maps and Street View projections of Los Angeles were used by the team to help design Los Santos' road networks. The team studied virtual globe models, census data and documentaries to reproduce the city's geographical and demographic spread. The team opted to condense the city's spread into an area players could comfortably traverse to capture "the essence of what's really there in a city, but in a far smaller area", according to Houser. The New Yorker's Sam Sweet opined that the "exhaustive field work ... wasn't conducted to document a living space. Rather, it was collected to create an extremely realistic version of a Los Angeles that doesn't actually exist". Garbut noted that Los Angeles was used merely as a starting point and that the team were not "dictated by reality" while building Los Santos. +The open world includes vast tracts of countryside around the city proper. Research took the team to California's rural regions; Garbut recalled a visit he took with Houser to Bombay Beach that inspired them to set Trevor's initial story against the Salton Sea. The team wanted a large world without open, empty areas and condensed Southern California's countryside into a detailed play space. The game world covers 49 square miles (130 km2)—about an eightieth of Los Angeles County. Its scale is greater than Rockstar's previous open world games; Garbut estimated it is large enough to fit San Andreas, Grand Theft Auto IV and Red Dead Redemption's worlds combined inside. To accommodate the world's size, the team overhauled the RAGE to improve its draw distance rendering capabilities. The large, open space permitted the re-introduction of fixed-wing aircraft, omitted from Grand Theft Auto IV because of its relatively smaller scale. "We wanted somewhere big [to let players] fly properly", Houser explained. Lead producer Leslie Benzies noted that to avoid a "hollow" countryside area, the team populated the open world with wildlife. +A single-player story revolving around three lead protagonists was one of Grand Theft Auto V's earliest design objectives. Garbut felt such a deviation from the gameplay's core structure was a risk, and recalled team concern that a departure from Grand Theft Auto's traditional, single lead character set-up "might backfire". Early game conceptualisations would have told three separate stories through different protagonists. Later, Grand Theft Auto IV's stories inspired the concept that story trajectories would meet throughout the game. Eventually, the concept evolved into three interconnected stories that intertwined through the missions. According to Benzies, the team made the multiple character formula "integral to the structure of the gameplay as well as the narrative". Houser opined that Grand Theft Auto V is their "strongest plotted game because the characters are so intertwined" and that the "meeting points [between the characters' stories] are very exciting". +The central story theme is the "pursuit of the almighty dollar". Missions follow the lead characters' efforts to plan and execute complicated heists to accrue wealth for themselves. The team focused on money as the central theme in response to the 2008 financial crisis, as its effects turn the main characters back to a life of crime. "We wanted this post-crash feeling, because it works thematically in this game about bank robbers", Houser explained. The positive reaction to Grand Theft Auto IV's "Three Leaf Clover" mission—an elaborate heist executed by lead protagonist Niko Bellic and accomplices—encouraged the team to develop the story around the heists. Houser said while "Three Leaf Clover" was well-received, the team had not captured the thrill of the robbery to their best abilities and wanted Grand Theft Auto V to achieve it. He felt a strong bank robbery mission "was a good device that we'd never used in the past. Repeating ourselves is a fear when we're doing games where part of the evolution is just technological". +The game has players control three characters: Michael De Santa, Franklin Clinton and Trevor Philips. The team wrote each character to embody a game protagonist archetype; Michael represents greed, Franklin ambition and Trevor insanity. Houser felt Michael and Trevor were written to juxtapose each other, with Michael "like the criminal who wants to compartmentalise and be a good guy some of the time" and Trevor "the maniac who isn't a hypocrite". He considered that the three lead characters helped move the game's story into more original territory than its predecessors, which traditionally followed a single protagonist rising through the ranks of a criminal underworld. Ned Luke portrayed Michael, Shawn "Solo" Fonteno portrayed Franklin, and Steven Ogg portrayed Trevor. Fonteno first became aware of the acting job through his friend DJ Pooh, who worked on San Andreas and was involved in Grand Theft Auto V's music production. +When Luke's agent advised him of the casting call, he initially did not want to audition for the part because it was in a video game. After reading the audition material and learning more about the project, he became interested. He reflected, "I went immediately after reading the material from 'I'm not doing it' to 'nobody else is doing it'. It was just brilliant". During the initial audition process, Ogg noticed on-set chemistry between him and Luke, which he felt helped secure them the roles. "When [Luke] and I went in the room together we immediately had something", he explained. While the actors knew their auditions were for Rockstar Games, it was not until they signed contracts that they learnt it was a Grand Theft Auto title. +Work for the actors began in 2010. Their performances were mostly recorded using motion capture technology. Dialogue for scenes with characters seated in vehicles was recorded in studios. Because the actors had their dialogue and movements recorded on-set, they found their performances no different to film or television roles. Their dialogue was scripted such that they could not ad-lib; however, with directorial approval, they sometimes made small changes to their performances. To prepare for his role as Michael, Luke gained 25 pounds and studied Rockstar's previous games, starting with Grand Theft Auto IV. He considered Michael's characterisation to be an amalgamation of Hugh Beaumont's portrayal of Ward Cleaver in the American sitcom Leave It to Beaver (1957–63) and Al Pacino's portrayal of Tony Montana in the 1983 film Scarface. +Ogg felt Trevor's characterisation developed over time. He said, "Nuances and character traits that began to appear—his walk, his manner of speech, his reactions, definitely informed his development throughout the game". Ogg cites Tom Hardy's portrayal of English criminal Charles Bronson in the 2008 biopic Bronson as a strong stylistic influence. He opined that while Trevor embodies the violent, psychopathic Grand Theft Auto anti-hero archetype, he wanted to evoke player sympathy for Trevor's story. "To elicit other emotions was tough, and it was the biggest challenge and it's something that meant a lot to me", Ogg explained. Fonteno felt growing up in South Los Angeles and being exposed to drug trafficking and gang culture authenticated his portrayal of Franklin. "I lived his life before ... He's been surrounded by drugs, the crime, living with his aunt—I lived with my grandmother—so there was a lot of familiarity", Fonteno said. Having not worked as an actor since portraying Face in the 2001 film The Wash, he sought counsel from Luke and Ogg to refine his acting skills. +Grand Theft Auto V's multiple protagonist design was envisioned to improve the series' core mechanics. The team sought to innovate game storytelling and negate stale familiarity by not evolving the gameplay's core structure. "We didn't want to do the same thing over again", said Houser. The multiple protagonist idea was first raised during San Andreas' development but contemporaneous hardware restrictions made it infeasible. Garbut explained, "It didn't work from a tech point of view because the three characters need three times as much memory, three types of animation and so on". After Grand Theft Auto IV's release, the team developed The Lost and Damned and The Ballad of Gay Tony, episodic content packages that followed new protagonists. The three interwoven stories received positive remarks, so the team structured Grand Theft Auto V around this model. +The development team found that players experienced greater freedom when controlling three characters in missions. Lead mission designer Imran Sarwar felt they opened up more strategic manoeuvres. He cited a combat scenario where Michael sets up at a sniper outpost to cover Trevor, who makes a frontal assault on the enemy position while Franklin manipulates flank points. Benzies felt character switching streamlines the interplay between free roam and linear mission gameplay, as it eliminates San Andreas' cumbersome long-distance drives to mission start points. Players may "explore the whole map without having to worry about the long drive back", according to Benzies. Houser noted the mechanic's use during missions negated long drives as well. The team implemented dynamic mission content throughout the open world, a feature borrowed from Red Dead Redemption. Dynamic missions present themselves while players explore the open world and may be accepted or ignored. In Los Santos, for example, players may encounter an armoured van and try to intercept it to steal its contents. +The team overhauled the game's shooting and driving mechanics to match the standards of its contemporaries. Public reception to the team's previous games (such as Grand Theft Auto IV, Red Dead Redemption and Max Payne 3) was considered during the process. To increase the pace of shootouts, the team removed hard locking—a central mechanic in Grand Theft Auto IV that instantly locks onto the enemy nearest to the crosshair. Associate technical director Phil Hooker found hard locking "too disorientating" and immersion-breaking, "as you didn't have to think about enemy locations". He said Grand Theft Auto IV players "just rely on holding and shooting until a target is dead", so Grand Theft Auto V introduces a timer that breaks the lock on a target after a few seconds. The team refined Red Dead Redemption's cover system for the game, with increased fluidity moving into and out of cover. Regarding the reworked vehicle mechanics, Houser felt the game took influence from racing games and corrected Grand Theft Auto IV's "boat-like" driving controls. +Grand Theft Auto V is the first game in its series to use an original score. Music supervisor Ivan Pavlovich summarised the original score idea as "daunting", because it was unprecedented for a Grand Theft Auto game. Like most previous series entries, the game uses licensed music tracks provided by an in-game radio as well. Pavlovich hoped the original score would enhance the licensed music use, not detract from it. He further noted the balancing act between the score's "ambient subtext and tensions" and the game's on-screen action. To work on the score, Rockstar engaged The Alchemist, Oh No and Tangerine Dream with Woody Jackson, who had previously worked on Red Dead Redemption, L.A. Noire and Max Payne 3's music. The team of producers collaborated over several years to create more than twenty hours of music that scores both the game's missions and dynamic gameplay throughout the single-player and multiplayer modes. +Early in the game's development, the music team were shown an early build before starting work on the score. Their work was mostly completed later in development, but they continued composing until its final build was submitted for manufacturing. Edgar Froese, Tangerine Dream's founding member, initially rejected the offer of producing music for a video game. After he was flown to the studio and shown the game, he was impressed by its scale and cinematic nature and changed his mind. Froese's first eight months of work on the score produced 62 hours of music. He recorded with Tangerine Dream in Austria, but further work was conducted at Jackson's United States studio, which The Alchemist and Oh No used as well. +Jackson's initial role was to provide the score for Trevor's missions, and he took influence from artists such as The Mars Volta and Queens of the Stone Age. When he learnt the team would be building off each other's work, he voiced concern that the finished product could be disjointed. After sharing his work with the team, however, he was particularly impressed by Froese's contributions. "Edgar evolved the music, made it into a whole other thing", Jackson said. Froese had interpolated funk sounds with Jackson's hip-hop influences. Froese and Jackson then sent their work between The Alchemist and Oh No, who heavily sampled it. The Alchemist opined, "We were sampling, taking a piece form [sic] here, a piece from there ... We pitched stuff up, chopped it, tweaked it. Then we chose the tracks that worked and everyone came in and layered on that". DJ Shadow then mixed the team's creations and matched them to the gameplay. Pavlovich considered "how to make the hip-hop and rock score not sound like they were instrumentals of songs on the radio, but rather something unique to the score" a challenge. +Pavlovich found that while Rockstar assigned the team missions to write music for, some of their random creations influenced other missions and sparked inspiration for further score development. He discussed a "stem-based" system used to make music fit dynamic game factors where the team would compose music to underscore outcomes players could make immediately after completing missions. Each of these stems, Froese reflected, included up to 62 five-minute WAV files, which were sent to Pavlovich in New York. "He then created, very professionally, a mix down for each of the eight stems needed for a mission and sent out the material to the other artists involved", Froese elaborated. Oh No drew from scenes within the game to make his work feel contextually pertinent with the action on-screen. The iconographic introduction of Los Santos early in the game, for example, inspired him to "create a smooth West Coast vibe that embodied" the city. He supplied horns, electric and bass guitars, and percussion parts to fit with the car chase scenes. "We wanted everything to set the right tone", he explained. +The Rockstar team wanted to synergise the game world's depiction of California with the radio stations by licensing tracks that imparted an appropriate "Cali feel". Pavlovich noted that Los Angeles' cultural saturation of pop music necessitated the Non-Stop-Pop FM station; he said "the first time you get off an airplane in L.A. and you hear the radio and the pop just seeps out ... We wanted that. It really connects you to the world". He felt that greater discernment was required for licensed music choices than in Grand Theft Auto IV because Grand Theft Auto V's music plays a pivotal role in generating Californian atmosphere. Music "reflects the environment in which the game is set", he said. Initially, the team planned to license over 900 tracks for the radio, but they refined the number to 241. The tracks are shared between fifteen stations, with an additional two talk-back stations and a radio station for custom audio files on the PC version. Some tracks were written specifically for the game, such as rapper and producer Flying Lotus' original work composed for the FlyLo FM station he hosts. Pavlovich noted how the team would first develop an idea of what each station would sound like, and then select a DJ to match the station's genre, such as Kenny Loggins who hosts the classic rock station Los Santos Rock Radio. He felt that to strike a balance between the radio and the score was a meticulous process, and cited a scenario where players would drive to a mission objective while listening to the radio, with the score taking over once players left the vehicle and proceeded to the mission's next stage. +During a September 2009 earnings call, Take-Two Interactive chief executive officer (CEO) Strauss Zelnick was asked about Grand Theft Auto V, the purported next game in the series. He replied, "We're not going to announce it, we're not going to announce when we are going to announce it, and we are not going to announce a strategy about announcing it or about when we are going to announce it either, or about the announcement strategy surrounding the announcement of the strategy". That November, Houser discussed with The Times his work on the series and the process that would be undertaken for the next Grand Theft Auto game. He expressed plans to co-write a thousand-page script and said that, when developing a new game, the team typically created a city and then developed the lead cast. In July 2010, Rockstar North posted seven job advertisements related to a new title. The company wanted to recruit environment artists, physics programmers and character animators—the latter advertisement asked for recruits with "professional experience developing a third person action game". Journalists wrote that the job listing was indicative of Grand Theft Auto V's existence. In June 2011, anonymous sources allegedly close to the developer told GameSpot the title was "well under way", with a 2012 release date likely. +Rockstar Games first confirmed the game's existence on 25 October 2011 in an announcement on its official website and Twitter feed. Take-Two Interactive's share price subsequently increased by seven per cent. Journalists said the announcement ignited significant anticipation within the gaming industry, which they owed to the series' cultural significance. The game did not meet its original projected March–May 2013 release date. By 30 October 2012, promotional posters had spread to the Internet, and a listing by the retailer Game had leaked the projected release date. Rockstar announced a scheduled Q2 2013 release that day and began accepting pre-orders on 5 November. +On 31 January 2013, the company announced the release date had been postponed until 17 September of that year. "It simply needs a little more polish to be of the standard we and, more importantly, you require", Rockstar stated in a press release. On 23 August, reports stated that some European PlayStation 3 users who had pre-ordered Grand Theft Auto V were able to download parts of the game, including its soundtrack and some character dialogue. Details of the game were leaked later that day and on following days before Sony removed the pre-order file from the European PlayStation Network and released an official apology to Rockstar and its fans. In response, Rockstar stated it was "deeply disappointed by leaks and spoilers being spread in advance of the game's launch". The game was released in Japan on 10 October. At E3 2014, a re-release of the game was announced for PC, PlayStation 4, and Xbox One. The PC version, initially scheduled for simultaneous release with the console versions, was delayed three times: first to 27 January 2015, later to 24 March and again to 14 April. According to Rockstar, the game required extra development time for "polishing". PlayStation 5 and Xbox Series X/S versions were announced in June 2020; originally scheduled for release on 11 November 2021, they were later pushed to March 2022, specified in February 2022 to 15 March. They feature additional technical enhancements and performance upgrades. +The game was extensively marketed through video trailers and press demonstrations. On 2 November 2011, a week after the announcement, the debut trailer was released. It is narrated by Michael and depicts the open world accompanied by the song "Ogdens' Nut Gone Flake" by English rock band Small Faces. A press release published the same day confirmed the game's open world recreation of Southern California and Los Angeles. Almost a year later, Game Informer ran a Grand Theft Auto V cover story for their December 2012 issue. Rockstar intended to release the second promotional trailer on 2 November. However, these plans were hampered by Hurricane Sandy, which cut power supplies to Rockstar's New York offices. The trailer was eventually released on 14 November; it introduces the lead protagonists' back-stories and features the song "Skeletons" by American musician Stevie Wonder. +To unveil the cover art, Rockstar contracted artists to paint a mural on a wall in Manhattan on 31 March 2013, followed by the artwork's digital release on 2 April. It showed English model Shelby Welinder portraying a blonde beach-goer. Three trailers were released on 26 April, each focusing on one of the protagonists. The songs "Radio Ga Ga" by English band Queen, "Hood Gone Love It" by American rapper Jay Rock, and "Are You Sure Hank Done It This Way" by American musician Waylon Jennings are used in the Michael, Franklin and Trevor trailers respectively. +A trailer released on 9 July features the first gameplay footage. It demonstrates the shooting and driving mechanics, and the ability to swap between characters instantaneously. Grand Theft Auto Online was unveiled in a trailer released on 15 August. The video displayed activities like bank heists, small robberies, "traditional" game modes, purchase of property and bicycle riding. The final pre-launch trailer was released on 29 August, intended to be a television advertisement. The song "Sleepwalking" by American band The Chain Gang of 1974 was used in this trailer. +Viral marketing strategies were used to promote the game. Visitors to the website of The Epsilon Program—a fictional religious cult within the Grand Theft Auto universe—were offered a chance to register for that group. After filling in an online membership form, the terms and conditions revealed that the site was a casting call for five people to appear in the game as members of the fictional cult. The official Grand Theft Auto V website was redesigned on 13 August 2013 to show a preview of activities and locales within the open world and an examination of the lead protagonists' stories. More information was released on the website on 24 August, 6 September, and 13 September. +To spur pre-order sales, Rockstar collaborated with several retail outlets to provide special edition versions of the game. The "Special Edition" includes a unique case packaging, a game map and unlock codes for additional content in the single-player and multiplayer modes. The publisher collaborated with Sony to release a 500 GB PlayStation 3 console, which includes a copy of the game, 30-day trial membership for the PlayStation Plus service, and set of Grand Theft Auto V-branded headphones. All game pre-orders granted the purchaser an access code for the in-game Atomic Blimp aircraft. GameStop held a promotional raffle with the chance to win a real-life Bravado Banshee sports car (the game's counterpart of the Dodge Viper). Rockstar collaborated with West Coast Customs to build the vehicle. +Shortly after the game's release, the iFruit application was released for iOS devices. It let players customise vehicles, create custom license plates and teach Franklin's dog Chop new tricks, which unlocked additional in-game abilities. Upon its launch, some users reported problems connecting to the application's servers; these problems were resolved with an update on 25 September 2013. iFruit was released for Android on 29 October, and for Windows Phone devices on 19 November. A PlayStation Vita port was released on 1 April 2014. Rockstar stopped supporting the application on 12 December 2022. +The enhanced version for PC, PlayStation 4, and Xbox One features an increased draw distance, finer texture details, denser traffic, upgraded weather effects, and new wildlife and vegetation. It features more than 162 new songs across the game's radio stations. Players could transfer Grand Theft Auto Online characters and progression between some platforms and gain exclusive activities and in-game discounts on weapons and vehicles. The re-release features a new on-foot first-person view option that players may configure to personal preference (for example, by making the view toggle to third-person when taking cover). Animation director Rob Nelson said that a first-person option was raised during PlayStation 3 and Xbox 360 development, but their relatively smaller banks of memory were already being pushed, such that adding new first-person animations would have inhibited the open-world render. According to Nelson, the first-person view required more development effort than simply repositioning the camera, because of the need to adapt combat to a different view. The weapons were upgraded to a higher resolution, and new animations including weapon recoil, reload and switch were added. "I think we created 3,000 animations on weapons alone", said Nelson. +The PlayStation 4 version uses the DualShock 4's touchpad to navigate camera options and speaker to play smartphone calls, while the Xbox One Controller's "Impulse Triggers" may rumble while players use vehicles. The PC version features the "Rockstar Editor", a replay editor that lets players create video clips of their gameplay. It features a "Director Mode" that lets players record footage with various characters that speak and perform contextual actions at will. Players can adjust the time of day and weather settings, and use cheat codes to access more cinematic effects. An editing suite lets players add music from the game's soundtrack and score, and access various depth of field and filter effects. Finished works may be uploaded directly to YouTube and entered into Rockstar Games Social Club contests. Later, the PlayStation 4 and Xbox One versions received the Rockstar Editor through a free title update, which added various features such as expanded sound effects and text styles libraries across all three platforms. +Art director Aaron Garbut said the addition of first-person inspired the enhanced version's graphical upgrade. Remodelled cars feature interior effects including functional speedometers, fuel gauges and dashboard handbrake lights. The team added new particle and lighting effects, "like fireflies at night in the countryside, or ambient light pollution over Los Santos at night", according to Garbut. Red Dead Redemption inspired the team to add more vegetation to "break up the hard edges [and] straight lines" of the open world. The original version's vegetation was replaced with more detailed equivalents in the enhanced version. An upgraded weather system lets tree branches and leaves blow realistically in the wind. The team hand-placed weeds along fences and walls and placed grass over many of the open world's terrains. They then layered foliage, rocks and litter over the grass. An upgraded screen space ambient occlusion (SSAO) system renders new particle, shadow and weather effects, such as volumetric fog or neon reflections in cars at night. The ambient light pollution over nighttime Los Santos may dissipate in poor weather. A dynamic depth of field system sharpens and softens images to emulate camera autofocus, and improved shaders produce new colours in skin and terrain textures. +Initial PC version development began in parallel with PlayStation 3 and Xbox 360. PC development later gave way as the focus shifted to the console releases but eventually resumed. Because the team had planned a PC version from early on, they made technical decisions in advance to facilitate later development, like support for 64-bit computing and DirectX 11. The art team authored their source art at high resolutions even though they needed to be compressed on the PlayStation 3 and Xbox 360, with the foresight that PC versions would display these assets uncompressed. These early decisions aided the PlayStation 4 and Xbox One versions' development as well, due to their similar system architectures to the PC. Their similarities helped the team "ramp up" to the PC version, where they could push the visuals and technology further than before. The PC development team consisted of members of the original team and PC specialists from Rockstar's other studios who had brought Grand Theft Auto IV, Max Payne 3 and L.A. Noire to the platform. The PC's recommended specifications are based on the game running a native 1080p resolution at 60 frames per second (fps); the team suggested 60 fps as the optimal performance benchmark. The PC build also supports 4K resolution and uncapped framerates. The team opted to give players the choice to configure the game according to their system specifications. Players may configure LOD draw distances, anisotropic filtering, graphics effects and so on. A population density slider affects the density of street-walking pedestrians and cars on the roads. +General elections were held in Liechtenstein on 3 and 16 February 1936 to elect the 15 members of the Landtag. The Progressive Citizens' Party (FBP) won eleven seats and retained its majority in the Landtag. The Patriotic Union (VU), which had been founded the previous month as a merger of the Christian-Social People's Party (VP) and Liechtenstein Homeland Service (LHD), won four seats. Voter turnout was 95.5%. +The election campaign was marked by attacks and accusations by both parties, including that of antisemitism and connections to Nazi Germany. Following the elections, the majority government led by prime minister Josef Hoop remained. The VU had received around 48% of the vote but won only four seats, meaning that previous calls for the introduction of proportional representation were heightened, leading to implementation of a new electoral system in 1939. +The 1932 elections had been held under a new electoral system that replaced the majority system with direct mandates from Liechtenstein's municipalities, intended to ensure that the interests of the municipalities were directly represented in the Landtag. The Progressive Citizens' Party (FBP) won a majority of thirteen seats, and the Christian-Social People's Party (VP) won two. The majority government under the leadership of prime minister Josef Hoop remained, which had been in power since 1928. +Hoop's second term in office was marked by the Rotter kidnapping, when Liechtenstein became the target of press attacks from Nazi Germany for naturalizing German-Jewish theatre managers Fritz and Alfred Rotter. The attacks caused four Liechtensteiner Nazis to unsuccessfully attempt to kidnap the two men and forcefully extradite them to Germany, which resulted in the deaths of Alfred and his wife Gertrud. Afterwards, Hoop's government successfully negotiated the end of German press attacks by agreeing to end naturalizations in Liechtenstein and releasing the perpetrators from prison early. +A new political party, the Liechtenstein Homeland Service (LHD), was founded in 1933. The party initially supported the creation of an authoritarian corporatist state similar to that of Austria under Engelbert Dollfuss, but it quickly moved towards that of Nazism. From 1935, the LHD and VP formed an alliance known as the National Opposition designed to push for the introduction of proportional representation, but the subsequent referendum was narrowly rejected by voters. Motivated by a desire to unite the opposition before the 1936 elections, the two parties merged to form the Patriotic Union (VU) on 5 January 1936. The LHD, despite being the smaller of the two parties, disproportionately dominated the new party; LHD leader Otto Schaedler became party president and prominent LHD member Alois Vogt became party secretary of the VU. +A by-election was held on 3 February 1935 in Triesenberg following the resignation of Wilhelm Beck of the VP. The result was a victory for Johann Beck of the FBP. +Ten members of the Landtag (parliament) were elected via direct mandates for one member representing each of Liechtenstein's eleven municipalities with a population equal to or greater than 300 (all except for Planken). The remaining five members were elected via a national vote using a majority system, wherein the five candidates with the most votes were elected. Landtag members sit for a four-year term. Once formed, the Landtag elects the prime minister and two government councillors who govern in a cabinet. The prime minister is appointed for a six-year term, and government councillors sit for a four-year term. Only men aged 21 and above were eligible to vote. +Both the FBP and VU campaigned primarily in their respective party newspapers, the Liechtensteiner Volksblatt and the newly formed Liechtensteiner Vaterland. The election campaign was marked by polemics and accusations by both parties, resulting in complaints of "slander" and "incitement" on both sides. +The economy was a major issue of the election; the FBP campaigned on the achievements of Hoop's government, highlighting the building of asphalt roads, the beginning of construction of the Liechtenstein inland canal, expansion of public welfare, and subsidies for agriculture and business. It also attacked the VU for the former VP's failure in the 1928 embezzlement scandal, arguing that the VU could not be trusted with Liechtenstein's finances again. Similarly, the FBP attacked the VU on the LHD's influence in the party, calling the LHD the "root of evil" within the VU. The FBP accused it of seeking to introduce a corporatist Ständestaat, similar to that in Austria under Dollfuss, and "antisemitism in the style of Der Stürmer", a Nazi newspaper. In particular, the Vaterland's openly antisemitic editor, Carl Freiherr von Vogelsang, was particularly attacked by the FBP and was referred to as a "vile political agitator". +The VU fiercely criticized the FBP, focusing on poor economic conditions and rising unemployment, and continued to advocate for the introduction of proportional representation. The party also accused the FBP of distributing welfare and jobs unfairly, and of supporting large retail stores such as Migros instead of domestic businesses. The VU campaigned vaguely on supporting groups such as workers, farmers, and businesses, as well as lowering interest rates, and distributing work more justly, all without cutting spending in other areas. During the campaign, the VU focused particularly on getting Schaedler elected in the national vote, thus neglecting their other candidates. +The Liechtenstein Workers' Association asked both parties to place its president, Franz Hemmerle, as a candidate on their party lists. Similarly, the chamber of commerce (Gewerbeverband) asked both parties to include one or two business representatives on their candidate lists. The VU agreed to both proposals in principle, but as the FBP refused both, neither was carried out. +The FBP won eleven seats, a two-seat decrease from their 1932 performance, and retained its majority in the Landtag. The VU won four, an increase of two from the previous VP. Voter turnout was 95.5%. +The election of the Landtag members in the municipalities took place on 3 February. +The national vote to elect the remaining five Landtag members took place on 16 February. +Immediately following the election, president of the Landtag, Anton Frommelt of the FBP, appeared to have lost re-election, but a bipartisan recount gave him three more votes and won him the seat by a narrow margin. In response, the VU postulated that electoral fraud had taken place and stoked rumours of vote manipulation. VU leaders claimed to have calmed angry crowds in Triesenberg and Balzers, which had been prepared to march on the capital of Vaduz. The VU formally accepted the result of the election in the first session of the Landtag. +The VU had received nearly 47% of the second-round national vote, but only won four seats; the party would have won six or seven seats under a proportional system. Upon the opening of the newly elected Landtag, the VU demanded voluntary proportional representation from the FBP by having several posts, including the vice president of the Landtag and one government councillor, be filled by the VU. Ultimately, as a peace gesture, the FBP agreed to elect Schaedler as vice president in return for "constructive cooperation". This broke from the previous tradition of the vice president being a member of the ruling party from the Unterland. The FBP also elected Alois Schädler to government and generally ensured near-equal representation for the VU in other committees and posts. The new government was elected on 28 February, and Hoop remained as prime minister. +After the election, the VU repeated its calls for the introduction of proportional representation and announced a new popular initiative for 1937. The party ultimately achieved proportional representation with the formation of a coalition government between the FBP and VU in March 1938, and proportional representation was formally introduced to Liechtenstein in 1939. +A by-election was held on 14 June in Triesenberg following the death of Josef Beck in an accident on 17 May. The result was a victory for Wendelin Beck of the VU. +1936 Liechtenstein local elections +Nohlen, Dieter; Stöver, Philip (2010). Elections in Europe: A data handbook. Nomos. ISBN 978-3-8329-5609-7. +Geiger, Peter (1997a). Krisenzeit: Liechtenstein in den Dreissigerjahren 1928–1939 [Time of crisis: Liechtenstein in the 1930s 1928–1939] (in German). Vol. 1. Zürich: Liechtenstein Institute, Chronos Verlag. ISBN 3-906393-28-3. +Geiger, Peter (1997b). Krisenzeit: Liechtenstein in den Dreissigerjahren 1928–1939 [Time of crisis: Liechtenstein in the 1930s 1928–1939] (in German). Vol. 2. Zürich: Liechtenstein Institute, Chronos Verlag. ISBN 3-906393-28-3. +Vogt, Paul (1987). 125 Jahre Landtag [125 Years of the State Parliament] (in German). Vaduz: Landtag of the Principality of Liechtenstein. +The 13th Waffen Mountain Division of the SS Handschar (1st Croatian) was a mountain infantry division of the Waffen-SS, an armed branch of the German Nazi Party that served alongside but was never formally part of the Wehrmacht during World War II. At the post-war Nuremberg trials, the Waffen-SS was declared to be a criminal organisation due to its major involvement in war crimes and crimes against humanity. From March to December 1944, the division fought a counter-insurgency campaign against communist-led Yugoslav Partisan resistance forces in the Independent State of Croatia, a fascist puppet state of Germany that encompassed almost all of modern-day Croatia, all of modern-day Bosnia and Herzegovina, and parts of modern-day Serbia. +The division was named Handschar (Bosnian: handžar), after a local fighting knife or scimitar carried by Ottoman policemen during the centuries that the region was part of the Ottoman Empire. It was the first non-Germanic Waffen-SS division, and its formation marked the expansion of the Waffen-SS into a multi-ethnic military force. Composed mainly of Bosnian Muslims with some Catholic Croats, and mostly German and Yugoslav Volksdeutsche officers and non-commissioned officers, the members of the division took an oath of allegiance to the German Führer Adolf Hitler and the Croatian Poglavnik Ante Pavelić. +The division fought briefly in the Syrmia region north of the Sava river before crossing into northeastern Bosnia. After crossing the Sava, it established a designated "security zone" in northeastern Bosnia between the Sava, Bosna, Drina, and Spreča rivers. It also fought outside the security zone on several occasions, and earned a reputation for brutality and savagery, not only during combat operations but also for atrocities committed against Serb and Jewish civilians. In late 1944, parts of the division were transferred briefly to the Zagreb area, and non-German members began to desert in large numbers. Over the winter of 1944–45, the unit was sent to the Baranja region, where it fought against the Red Army and Bulgarians throughout southern Hungary, falling back via a series of defensive lines until they were inside the Reich frontier. +Most of the remaining Bosnian Muslims left at this point and attempted to return to Bosnia. The rest retreated further west, hoping to surrender to the Western Allies. Most of the remaining members became prisoners of the British Army. Subsequently, 38 officers were extradited to Yugoslavia to face criminal charges, and ten were executed. Hundreds of former members of the division fought in the 1947–48 civil war in Mandatory Palestine and the 1948 Arab–Israeli War. +After the invasion of the Kingdom of Yugoslavia by the Axis powers on 6 April 1941, the extreme Croat nationalist and fascist Ante Pavelić, who had been in exile in Benito Mussolini's Italy, was appointed Poglavnik (leader) of an Ustaše-led Croatian state – the Independent State of Croatia (often called the NDH, from the Croatian: Nezavisna Država Hrvatska). The NDH combined almost all of modern-day Croatia, all of modern-day Bosnia and Herzegovina and parts of modern-day Serbia into an "Italian-German quasi-protectorate". NDH authorities, led by the Ustaše Militia, subsequently implemented genocidal policies against the Serb, Jewish and Roma population living within the borders of the new state. +To secure the loyalty of the Bosnian Muslims, Pavelić ordered that property in Zagreb be converted into a mosque that he named the "Poglavnik's Mosque". Despite Pavelić's assurances of equality with the Croats, many Muslims quickly became dissatisfied with Croatian rule. A Muslim leader reported that not one Muslim occupied an influential post in the administration. Although this was an overstatement, Muslims were underrepresented in government positions, with only two of 20 ministerial positions. None of the six state secretaries were Muslim, and there were only 13 Muslim "people representatives" in a total of 206. Fierce fighting broke out between the Ustaše, the Serb-chauvinist Chetniks and pan-Yugoslav Partisans in NDH territory. Some Ustaše militia units became convinced that the Muslims were communist sympathisers, and burned their villages and murdered many civilians. The Chetniks accused the Muslims of taking part in the Ustaše violence against Serbs and perpetrated similar atrocities against the Muslim population. The Muslims received little protection from the Croatian Home Guard, the regular army of the NDH, whom the Germans described as "of minimal combat value". Local militias were raised, but these were also of limited value and only one, the Tuzla-based Home Guard Hadžiefendić Legion, led by Muhamed Hadžiefendić, was of any significance. +The Bosnian Muslim elite and notables in various cities and towns issued resolutions or memoranda to the NDH and German authorities that publicly denounced the genocide of the Serbs and the NDH laws targeting them. These were issued in: Prijedor (23 September 1941), Sarajevo (12 October), Mostar (21 October), Banja Luka (12 November), Bijeljina (2 December) and Tuzla (11 December). The resolutions condemned the Ustaše in Bosnia and Herzegovina, both for their mistreatment of Muslims and for their attempts to turn Muslims and Serbs against one another. One memorandum declared that since the beginning of the Ustaše regime, the Muslims had dreaded the lawless activities that the Ustaše, Croatian government authorities and various illegal groups had been perpetrating against the Serbs. +The Bosnian Muslims' dissatisfaction with the Ustaše rule of the NDH and their need for protection were combined with nostalgia for the period of Habsburg rule in Bosnia and a generally friendly attitude towards Germany among prominent Bosnian Muslims. These factors led to a push towards autonomy for the Bosnian Muslim community which was strongly opposed by Pavelić as counter to the territorial integrity of the NDH. By November 1942, the autonomists were desperate to protect the Muslim people and wrote to Adolf Hitler asking that he authorise the creation of an autonomous "political-administrative authority" in Bosnia within the structure of the NDH, led by a Hitler appointee. +On 6 December 1942, Reichsführer-SS Heinrich Himmler and key Waffen-SS recruiting officer SS-Obergruppenführer und General der Waffen-SS Gottlob Berger approached Hitler with the proposal to raise a Bosnian Muslim SS division. Both the Wehrmacht and the Waffen-SS were concerned about the rapidly deteriorating security situation in the NDH that tied down German military personnel needed elsewhere. A German source noted that by 1943 over 100,000 Bosnian Muslims had been killed and 250,000 had become refugees. In addition, a serious food shortage threatened the region. "The Muslims," remarked SS-Gruppenführer und Generalleutnant der Waffen-SS Artur Phleps, "bear the special status of being persecuted by all others". +The romantic notions that Himmler had about the Bosnian Muslims were probably significant in the division's genesis. Nonetheless, a memorandum dated 1 November 1942 also indicates that leading Muslim autonomists had already suggested the creation of a volunteer Waffen-SS unit under German command. Himmler was personally fascinated by the Islamic faith and believed that Islam created fearless soldiers. He found their ferocity preferable to the gentility of Christians and believed their martial qualities should be further developed and put to use. He thought that Muslim men would make perfect SS soldiers as Islam "promises them Heaven if they fight and are killed in action." As for their ethnic background and SS requirements, it appears that Himmler accepted the theories advanced by both Croatian and German nationalists that the Croatian people, including the Muslims, were not ethnic Slavs but pure Aryans of either Gothic or Iranian descent. +Himmler was inspired by the noted successes of the Bosnian-Herzegovinian Infantry in World War I. He "endeavoured to restore what he called 'an old Austrian' tradition by reviving the Bosnian regiments of the former Austro-Hungarian Army in the form of a Bosnian Muslim SS Division". Once raised, the division was to engage and destroy Josip Broz Tito's Partisan forces operating in north-eastern Bosnia, thus restoring local "order". Himmler's primary concern in the region was not the security of the local Muslim population, but the welfare of ethnic German settlers to the north in Syrmia. "Srem (Syrmia) is the breadbasket of Croatia, and hopefully it and our beloved German settlements will be secured. I hope that the area south of Srem will be liberated by ... the Bosnian division ... so that we can at least restore partial order in this ridiculous (Croatian) state." +Hitler formally approved the project in mid-February 1943 and Himmler put Phleps, commander of the 7th SS Volunteer Mountain Division Prinz Eugen, in charge of raising the first SS division to be recruited from a non-Germanic people. +On 18 February 1943, Phleps travelled to Zagreb to begin formal negotiations with the NDH government. He met with German foreign ministry envoy Siegfried Kasche and NDH Foreign Minister Dr. Mladen Lorković who represented Pavelić. Pavelić had already agreed to raise the division, but the Waffen-SS and NDH governments had very different ideas of how it would be recruited and controlled. Lorković suggested that it be named the SS Ustaša Division, a Croatian unit raised with SS assistance, with familiar geographically based regimental names such as Bosna, Krajina and Una. This reflected the concerns shared by Pavelić and Kasche that an exclusively Muslim division might aid a Muslim bid for independence. As a compromise, the word "Croatian" was included in its official title and Catholic Croatian officers were recruited. Himmler and Phleps largely prevailed and created the division as they saw fit, leaving the NDH very unhappy with the outcome, particularly regarding its ethnic composition. +SS-Standartenführer Karl von Krempler, a specialist in Islam who spoke Serbo-Croatian, was charged by Himmler and Phleps with organising the division. On 3 March 1943, Phleps met with von Krempler, who was to work with NDH government representative Alija Šuljak. The campaign began on 20 March 1943, when von Krempler and Šuljak began an 18-day recruiting tour through 11 Bosnian districts. With assistance from the SS, recruiting rallies were held in the towns of Živinice and Gračanica. In the meantime the Germans began raising the divisional headquarters staff in Berlin, including SS-Standartenführer der Reserve Herbert von Oberwurzer, who was transferred from the 6th SS Mountain Division Nord to command the division. Outside the NDH, recruiting was conducted in the southern parts of the German-occupied territory of Serbia that had significant Muslim populations, namely the Sandžak region along the border with the Italian governorate of Montenegro, and the parts of the Kosovo region occupied by Germany. Despite high German expectations and considerable efforts from local collaborators, recruitment from these areas was low. Due to the low number of recruits, to raise an Albanian SS regiment the Germans mobilized the population of Novi Pazar in the Kosovo region on 10 April and 30 May. Eligible males who did not volunteer were rounded up by the Albanian gendarmerie to serve in the regiment. Some Muslims escaped serving in the regiment by fleeing the city. +Šuljak and von Krempler soon fell out over the aims and purposes of the proposed division. Šuljak, an entirely political appointee, criticised von Krempler's Serbian dialect and his use of traditional Islamic colours and emblems (green flags and crescent moons) during the recruitment drive rather than the Ustaše symbols. When he reached Tuzla in central Bosnia, von Krempler met with the militia leader Hadžiefendić. On 28 March, Hadžiefendić escorted von Krempler to Sarajevo, where he introduced him to the leader of Bosnia's Islamic clergy, Hafiz Muhamed Pandža, the reis-ul-ulema, as well as other leading Muslim politicians not involved with the Ustaše. The NDH government and Kasche were furious, demanding von Krempler's immediate removal. The SS ignored the demand and von Krempler continued recruiting, including deserters from the NDH armed forces. +In March–April 1943, both Himmler and a group of Muslim leaders within the NDH requested that the Mufti of Jerusalem, Haj Amin al-Husseini, then resident in Berlin, assist in organising and recruiting Muslims into the Waffen-SS and other units. He was escorted by Krempler, who spoke Turkish. From 30 March to 10 April, the Mufti visited Zagreb, Sarajevo and Banja Luka to meet senior Muslim leaders and agitate in support of the new division. The Mufti also blessed and inspected the division, during which he used the Nazi salute. The Mufti insisted, "The most important task of this division must be to protect the homeland and families [of the Bosnian volunteers]; the division must not be permitted to leave Bosnia", but the Germans paid no attention. +Despite the support of al-Husseini, recruitment of Muslims for the division fell well short of the numbers needed. Himmler then allowed a 10 per cent Christian component, but the recruitment of sufficient Muslims continued to prove difficult, resulting in the induction of 2,800 Catholic Croats into the division. To Himmler's dismay, this was greater than the ratio of Catholics to Muslims that he had wanted. +Husejin Biščević (Husejin Biščević or Biščević-beg; born 28 July 1884) was the highest ranking (and perhaps the oldest) Bosnian military officer to volunteer. Bišcević had served in the Austro–Hungarian army and in August 1943 was appointed as an SS-Obersturmbannführer to command the divisional anti-aircraft battalion. He was eventually judged unsuitable, and replaced with a German just before the division went into combat. +Sources differ regarding the division's initial composition. Pavlowitch states that sixty percent of its recruits were Muslims and the rest were Yugoslav Volksdeutsche, who made up the majority of its officers and non-commissioned officers (NCOs). Tomasevich states that its planned complement was 23,200 Muslims and 2,800 Croats, with mostly German officers. Lepre indicates that the division's prescribed strength was reduced from 26,000 to 21,000, and Cohen states that the division achieved a maximum strength of 17,000 in April 1944. The division had a Muslim imam for each battalion other than the all-German signal battalion. For about six months the division included about 1,000 ethnic Albanians from Kosovo and the Sandžak region who made up the 1st Battalion of the 2nd Regiment, which later became the 1st Battalion of the 28th Regiment (I/28). +By the time the division had completed its training, it was still about one-third below its designated strength in both officers and NCOs, and its officer corps remained almost entirely German. Most of the officers and NCOs were drawn from replacement units of other Waffen-SS divisions. +The division was initially sent to southern France for formation and training, where it was accommodated mainly in towns and villages in the Aveyron and Lozère départements. For a long period after its official formation, the division was unnamed and was referred to as the "Kroatische SS-Freiwilligen-Division" (Croatian SS-Volunteer Division) or the "Muselmanen-Division" (Muslim Division). The decision by the Waffen-SS to form and train the division outside Bosnia was contrary to the advice given by the NDH's German plenipotentiary general, Edmund Glaise von Horstenau. This advice soon proved prophetic. +On 9 August 1943, Oberst Karl-Gustav Sauberzweig took command of the division from von Oberwurzer. Sauberzweig transferred to the Waffen-SS and was appointed to the rank of SS-Oberführer. He was a Prussian who had been decorated as an eighteen-year-old company commander during World War I and had served as a regimental commander during the early stages of Operation Barbarossa before being wounded. A "proven leader of men", he spoke no Serbo-Croatian but quickly gained the lasting respect and affection of the men of the division. +On the night of 16/17 September 1943, while the 13th SS Division was training in Villefranche-de-Rouergue in France, a group of pro-Partisan soldiers led by Muslim and Catholic junior officers staged a mutiny within the pioneer battalion. Led by Ferid Džanić, Božo Jelinek (aka Eduard Matutinović), Nikola Vukelić and Lutfija Dizdarević, they captured most of the German personnel and executed five German officers, including battalion commander SS-Obersturmbannführer Oskar Kirchbaum. The mutineers believed that many of the enlisted men would join them and they could reach the Western Allies. +The revolt was put down with the assistance of the unit imam, Halim Malkoč, and unit physician Dr. Willfried Schweiger. Malkoč told the Bosnian enlisted men of 1st Company that they were being deceived, released the German NCOs and rallied the company to hunt down the ringleaders. Schweiger did the same with the 2nd Company. Dizdarević and Džanić were shot and killed during the fighting, and Vukelić was captured, while Jelinek escaped. +Sources vary on the number of mutineers killed after the revolt was suppressed. Tomasevich states that 78 of the worst offenders were executed, but Lepre lists only 14 executions while four more deserters were located and shot in late September. Cohen states that about 150 mutineers were killed immediately, while Noel Malcolm writes that 15 of the mutineers died and a further 141 were killed in a subsequent "clean-up operation." The Germans attributed the infiltration to Tito's suggestion that his Partisan followers enlist for police duty to receive superior weapons, uniforms and training. Enlistees who were deemed "unsuitable for service" or "politically unreliable" were subsequently purged. Eventually, 825 Bosnians were removed from the division and sent to Germany for labour service with Organisation Todt. Of these, 265 refused and were sent to the Neuengamme concentration camp where dozens of them died. +Speaking of the Bosnian Muslim troops who had served in the Austro–Hungarian army, Himmler later said, "I knew there was a chance that a few traitors might be smuggled into the division, but I haven't the slightest doubt concerning the loyalty of the Bosnians. These troops were loyal to their supreme commander twenty years ago, so why shouldn't they be so today." Himmler awarded both Malkoč and Schweiger the Iron Cross Second Class for thwarting the mutiny. Five soldiers were also decorated. +When Villefranche-de-Rouergue was liberated in 1944, the local population decided to pay tribute to the mutineers by naming one of its streets Avenue des Croates (The local population saw Bosnian Muslims as Croats of Islamic faith). It still commemorates "the revolt of the Croats" every 17 September. Cohen states that after the war, the Yugoslav government requested a name change to "the Revolt of the Yugoslavs" to obscure the mutineers' ethnicity, but the French refused. The Villefranche-de-Rouergue uprising was also commemorated in the city with a monument designed by Croatian sculptor Vanja Radauš. +As a result of the mutiny, the division was moved to the Neuhammer training grounds in the Silesian region of Germany (present-day Poland) to complete its training. During the training phase, the German officers, pleased with its progress, coined the term Mujo for the Bosnian Muslims. The members of the division swore an oath of allegiance to both Hitler and Pavelić. +On 9 October 1943, SS headquarters officially named the division the 13. SS-Freiwilligen b.h. Gebirgs-Division (Kroatien), but a short time later a change was made to differentiate it from those that were composed of Germans and it became the 13th Waffen Mountain Division of the SS Handschar (1st Croatian) (German: 13. Waffen-Gebirgsdivision der SS "Handschar" (kroat. Nr. 1), Serbo-Croatian Latin: 13. oružana brdska divizija SS-a Handžar, također i hrvatska br. 1). The division was named Handschar, after a local fighting knife or sword carried by Turkish policemen during the centuries that the region was part of the Ottoman Empire. On 15 February 1944, the division completed its training and returned to the NDH by rail. +The division was formed for the primary role of securing around 6,000 square kilometres (2,300 sq mi) of territory (the designated security zone) in north-eastern Bosnia within the NDH. The security zone encompassed the Posavina, Semberija and Majevica regions between the Sava, Bosna, Drina and Spreča rivers. It roughly corresponded with the area of operations of the Partisan 3rd Corps. Himmler saw this task as critical to the safeguarding of important agricultural areas and Volksdeutsche settlements in the Syrmia region to the north. +The division participated in what may have been the largest anti-Partisan sweep of World War II: Operation Maibaum. The 13th SS Division also participated in other divisional and corps-sized anti-Partisan operations between March and May 1944. +The division first saw action during Operation Wegweiser (signpost) from 9 to 12 March 1944. Operation Wegweiser aimed to clear part of the Syrmia region which was occupied by Partisans who threatened the Zagreb-Belgrade railway. The Partisans were operating from forests around Bosut and villages along the Sava. As the division entered the area, the Partisans withdrew to the southeast, avoiding a decisive engagement. Sauberzweig claimed the Partisans suffered 573 killed and 82 captured. Hoare writes that the division massacred hundreds of Serb civilians during the operation. According to Vladimir Dedijer and Antun Miletić, 223 civilians were killed in Bosut, 352 in Sremska Rača, and 70 in Jamena, mostly old men, women and children. The operation was a success in clearing the Bosut forests, but the Partisans returned to the area soon after it concluded. +On 15 March 1944, Operation Save was launched to clear Partisans from the Semberija region. Sauberzweig wrote an open letter to the division: "We have now reached the Bosnian frontier and will (soon) begin the march into the homeland. ... The Führer has provided you with his best weapons. Not only do you (have these) in your hands, but above all you have an idea in your hearts – to liberate the homeland. ... Before long, each of you shall be standing in the place that you call home, as a soldier and a gentleman; standing firm as a defender of the idea of saving the culture of Europe – the idea of Adolf Hitler." +Sauberzweig also ordered that each commander read a prepared message as his unit crossed the Sava River, which emphasised that the "liberation of Bosnia" and ultimately the liberation of "Muslim Albania" was their goal. This was a direct appeal to the Albanian troops as well as the Bosnians. The 27th Regiment crossed the Sava at dawn at Bosanska Rača near the confluence with the Drina. The rest crossed at Brčko covered by an intense artillery bombardment. Contact was immediately made with Partisan forces, who quickly withdrew into the forests. The service support units remained north of the Sava in Vinkovci, which became their permanent garrison area. The 27th Regiment advanced easily across the Pannonian Plain through Velino Selo to Brodac and then on to Bijeljina which was taken against light Partisan resistance late on 16 March. +The 27th Regiment then consolidated its position in Bijeljina while the 28th Regiment and the divisional reconnaissance battalion (German: Aufklärungsabteilung) bore the brunt of the fighting as they advanced through Pukis, Čelić and Koraj at the foot of the Majevica mountains. Sauberzweig later recorded that the 2nd battalion of the 28th Regiment (II/28) "at Čelić stormed the Partisan defences with the (new) battalion commander Hans Hanke at the point" and that enemy forces withdrew after a hard battle with heavy losses, low on ammunition. +Once the area was secured, defensive positions were established along the Čelić–Lopare road and company strength units were sent out to conduct reconnaissance. On the night of 17–18 March, elements of the Partisan 16th Vojvodina Division and 36th Vojvodina Division mounted unsuccessful attacks against the 28th Regiment's positions at Koraj and Zabrđe, losing over 200. +After Operation Save, the division remained relatively static for more than three weeks, mopping up and repelling local Partisan attacks. The Muslims were impatient to push further into Bosnia, but the reconnaissance battalion was heavily engaged on the divisional western flank, attacking positions held by the Partisan 3rd Vojvodina Brigade of the 36th Vojvodina Division at Gornji Rahić on 26 March, killing 124 Partisans and capturing 14. The battalion captured several more Partisan positions in the following week. In early April, 200 Partisans of the 16th Muslim Brigade surrendered to the division. They were mostly former members of various Muslim militias who had been conscripted into the ranks of the Partisans. +Operation Osterei (Easter Egg) began on 12 April 1944 intending to clear the Majevica mountain, which was held by elements of the 3rd Corps led by General Kosta Nađ. The 27th Regiment quickly captured Janja and drove through Donja Trnova to reach the Ugljevik coal mines, an important economic objective for the German war machine. Following fighting which continued into the evening of 13 April, the 27th Regiment reported Partisan casualties of 106 dead, 45 captured and two deserters along with large amounts of weapons and ammunition. The regiment also seized a huge amount of medical supplies from aid stations in the area of Donja Trnova. +The 28th Regiment drove south through Mačkovac and during fighting around Priboj, its 1st Battalion (I/28), made up of Albanians, incurred heavy casualties. The Partisan 3rd Corps then withdrew the 16th and 36th Vojvodina Divisions south across the Tuzla–Zvornik road. The reconnaissance battalion continued the advance, driving into the western Majevicas and capturing Srebrnik and Gradačac then linking up with the Croatian Home Guard 1st Mountain Brigade. The Germans considered Operation Osterei a major success, achieving all objectives with minimal losses. +During the final phase of Operation Osterei, I/28 was withdrawn from the fighting and transferred to Pristina in Kosovo to form part of the 21st Waffen Mountain Division of the SS Skanderbeg (1st Albanian) being raised by Himmler's order. A new I/28 was raised from other divisional units and recruits. +In the latter part of Operation Osterei, Jagdkommandos, lightly armed and mobile "hunter teams" of company or battalion strength, were used to break up and harass Partisans still operating on the flanks. These teams killed over 380 Partisans and captured over 200 between 21 and 23 April. By mid-April, half of the security zone had been cleared of Partisans. +The ambitious goal of Operation Maibaum (Maypole) was to destroy the Partisan 3rd Corps. Army Group F ordered V SS Mountain Corps to form a blockade along the Drina to prevent the Partisan force from crossing into Serbia. Operation Maibaum was one of the largest counter-insurgency operations of World War II and included 7th SS Division and NDH forces. The 13th SS Division was under the command of V SS Corps, and the primary tasks of the division were to capture Tuzla and Zvornik, then drive south parallel with the Drina to meet other Corps elements. The original plan included the parachute insertion of the 500th SS Parachute Battalion into the Vlasenica area, but this was cancelled due to unsuitable weather. Flank security was to be provided by the reconnaissance battalion in the Srebrnik area. The deployment south of the Spreča, and therefore outside the security zone, during Operation Maibaum was ordered by the corps commander, Phleps, despite opposition from Sauberzweig. This caused friction between the two that eventually required Himmler's intervention. +On 23 April, the 28th Regiment pushed south along mountain roads through Tuzla. The following day it continued as far as Stupari. On 25 April, the 27th Regiment advanced south to capture Zvornik. At the same time, the 28th Regiment sent I/28 east towards Vlasenica and II/28 continuing south towards Kladanj, capturing the town on 27 April. Due to the level of the Drinjača at Kladanj, instead of fording the river and pushing east towards Vlasenica, II/28 continued south-east towards Han Pijesak, where they met elements of 7th SS Division advancing north. +I/28 captured Vlasenica on 28 April but was almost immediately attacked from the east by two Partisan divisions. A third Partisan division surrounded the headquarters of the 28th Regiment at Šekovići, 30 kilometres north-west of Vlasenica. Both II/28 and the reconnaissance battalion were rushed to Vlasenica, where II/28 relieved the battered I/28 and then surrounded Šekovići. After a 48-hour battle, during which II/28 was decimated, the town was taken. +While the battle of Šekovići raged, the 27th Regiment extended the Drina blockade further south, ambushing a Partisan column and reaching Nova Kasaba on 30 April. After the situation at Šekovići improved on 1 May, the 27th Regiment returned to patrolling the Tuzla–Zvornik road. The 28th Regiment moved to the Simin Han-Lopare area on 5 May, while the 7th SS Division pursued the Partisans withdrawing south. +Operation Maibaum had not only stopped the Partisan 3rd Corps from crossing the Drina into Serbia, it had scattered the Partisan formation. On 6 May, V SS Mountain Corps ordered the division to return to the security zone north of the Spreča. +On 17–18 May 1944, the division, along with the local Majevica-Tuzla Chetnik unit commanded by Radivoje Kerović, commenced Operation Maiglöckchen (May Bell) to destroy several Partisan brigades in the Majevicas. The Partisans were encircled in the Stolice heights. An attempt by the Partisan 16th Vojvodina Division to relieve the surrounded force was defeated by the reconnaissance battalion and elements of the 28th Regiment. The relieving column was driven back across the Spreća. After heavy bombardment by the artillery regiment, the trapped Partisan force escaped south out of the pocket under cover of darkness on 18 May. The Partisans suffered considerable casualties, for example, the 17th Majevica Brigade of the 27th East Bosnia Division lost 16 killed and 60 missing. At the conclusion of Operation Maiglöckchen, the 27th Regiment remained near Zvornik and the 28th Regiment deployed near Srebrenik. +After Operation Maiglöckchen, the division shifted to a largely defensive posture aimed at denying the Partisans access to the security zone. Since its arrival in the zone, the division had been assisted in this task by local forces of varying reliability. These included four groups of Chetniks numbering 13,000, Nešad Topčić's Zeleni kadar (or Green Cadres, a Muslim nationalist militia) most of whom eventually joined the Partisans, and both Ustaše militia and the Croatian Home Guard, neither of which was effective. These same groups, along with the Partisans, had simultaneously been trying to encourage Bosnian and Croat members to defect. Between March and June 1944, these attempts were largely fruitless, producing fewer than 200 deserters. +Following Operation Rösselsprung (Knight's Move), the German attempt to kill or capture Tito at Drvar in western Bosnia on 25 May 1944, the Partisan leader ordered a general uprising. The Partisan 3rd Corps planned an offensive that involved three parallel columns of divisional strength thrusting north into the zone to engage and destroy German and local allied forces. The columns comprised: +"Western Column" – 16th Vojvodina Division commanded by Danilo Lekić +"Centre Column" – 38th East Bosnia Division commanded by Miloš Zekić +"Eastern Column" – 36th Vojvodina Division commanded by Marko Perić +Operation Vollmond (Full Moon) was devised quickly after the reconnaissance battalion observed Partisan forces crossing the Tuzla–Zvornik road on the evening of 6 June 1944. Sauberzweig's aim was to assault from the east and north, pushing the advancing Partisan forces against the Drina. The German plan underestimated the strength of the Partisan "Western Column" and had placed only one battalion (I/28) on high ground in the Partisans' path. This battalion included many raw recruits and was protecting two batteries of the artillery regiment, one of which (7/AR13) was deployed just east of Lopare. +Late on 7 June, the Partisans scattered I/28 and although II/28 was sent from Srebrenik to assist, the 16th Vojvodina Division surged forward against the positions of 7/AR13, which consisted of 80 men with four 150 mm guns and only one machine gun. After four hours of fighting, and with half the battery's personnel dead, the artillerymen ran out of small arms ammunition and scattered into the forest. As a result of a counterattack by II/28 on the afternoon of 9 June and throughout 10 June, the 16th Vojvodina Division withdrew that afternoon and the battalion pursued both the "Western Column" and "Centre Column" south. The guns and vehicles of 7/AR13 had been destroyed by the Partisans before they withdrew and there were reports that the German dead at Lopare had been mutilated. The "Eastern Column" was stopped by the 27th Regiment and the last of the Partisan 3rd Corps was pushed back across the Spreča on 12 June. +According to German accounts, Operation Vollmond cost the division 205 dead, 528 wounded and 89 missing. I/28 had been decimated, with only 180 men remaining. Sauberzweig claimed 3,000 Partisans were killed in this operation, but Phleps dismissed this as "a large exaggeration". According to one German after-action report, they had killed over 1,500 Partisans and captured large amounts of arms and ammunition. According to Partisan sources, the losses of the Partisan 3rd Corps were: +"Western Column" – 58 dead, 198 wounded and 29 missing +"Central Column" – 12 dead, 19 wounded, 17 missing +"Eastern Column" – 72 dead, 142 wounded and 9 missing +After Operation Vollmond, the commander of the 27th Regiment, Desiderius Hampel, was appointed division commander at his substantive rank of Standartenführer and Sauberzweig was tasked with forming a new corps headquarters, the IX Waffen Mountain Corps of the SS (Croatian). The commander of the 28th Regiment, Helmuth Raithel, was tasked to raise the new 23rd Waffen Mountain Division of the SS Kama (2nd Croatian). Both the new corps headquarters and new division were to be formed in southern Hungary. Three NCOs from every company of the 13th SS Division, along with other personnel, were transferred to form the core of the 23rd SS Division. New commanders were appointed to the two mountain infantry regiments, notably Hanke to the 28th. +Soon after Hampel assumed command, he became aware that local Chetniks were scavenging the Operation Vollmond battlefields for divisional equipment. He met with the Chetnik leader Kerović and arranged for the return of the equipment in exchange for small arms ammunition and boxes of hand grenades. +Throughout the remainder of June 1944 and into the first week of July, the division was engaged in fighting off attempted Partisan incursions into the security zone and reinforcing local NDH and Chetniks who were under attack from the Partisans. During the summer of 1944, elements of the division were tasked with guarding Hungarian Jewish forced labourers building fortifications in Tuzla. While guarding the labourers, they subjected them to cruel treatment and shot 22 who were unable to continue working. +Operation Fliegenfänger (Flypaper) was launched on 14 July 1944. It aimed to destroy a makeshift Partisan runway and the Partisan forces guarding it in the Osmaci area about 26 kilometres (16 mi) southeast of Tuzla, south of the Tuzla–Zvornik road. The airstrip was just north of the southern boundary of the security zone and was being used by Allied aircraft to bring in supplies and evacuate wounded Partisans to Italy. It had been built by the 19th Birač Brigade of the 27th East Bosnia Division between 3 and 6 July, and the first flight into the airfield occurred on the night of 6/7 July. The 19th Birač Brigade was also responsible for its defence. Two battalions of the 27th Regiment, along with a battalion of Chetniks from Majevica launched the operation from the line of Caparde–Memići–Prnjavor, and captured the towns of Osmaci and Memići and damaged the airfield despite stiff resistance. That afternoon, the 19th Birač Brigade counter-attacked and pushed the German and Chetniks back across the Tuzla–Zvornik road. At the same time as the counter-attack was underway, the headquarters of the Partisan 3rd Corps ordered the 36th Vojvodina Division to clear the enemy from the area so that the airfield could be used. During the night of 14/15 July, elements of the 36th Vojvodina Division arrived in the Osmaci area and relieved the 19th Birač Brigade, which moved towards Srebrenica. During the night of 16/17 July, Allied aircraft delivered equipment to Osmaci and about 100 wounded Partisans were airlifted to Italy. +According to German reports, 42 Partisans were killed, while the division's losses were four dead and seven wounded. The Partisan force withdrew south to the Vlasenica–Rajići area. The cooperation with the Chetnik battalion was described in the IX SS Mountain Corps war diary as "effective". +While the division was conducting Operation Fliegenfänger, the Second Panzer Army was developing a plan to block a large Partisan force from moving from central Bosnia into western Serbia to reinforce the Partisans. Due to issues with the availability of various units of V SS Mountain Corps, the planned Operation Rose was cancelled. +In the interim, since most of the division was deployed in the southern part of the security zone, Hampel planned a divisional-level operation, named Operation Heiderose (Wild Rose) targeting Partisan positions north-west of Šekovići, south of the Spreča. The 27th Regiment, reinforced by the same Chetnik battalion that cooperated in Operation Fliegenfänger, would attack east towards Šekovići and the 28th Regiment would push south. A detached battalion of the 7th SS Division would act as a blocking force driving north. Hampel was unhappy with the performance of the new commander of the 27th Regiment (SS-Obersturmbannführer Hermann Peter) and put the divisional chief of staff, SS-Sturmbannführer Erich Braun, in charge of the 27th Regiment. +On 17 July, the operation began. Fierce resistance was immediately encountered by the 28th Regiment, while the 27th Regiment climbed the heights to Udrč before dusk without serious opposition. Early on 18 July, the Chetniks attacked from Matkovac towards Šekovići. The 27th Regiment reached Bačkovac and seized the high ground south of Šekovići on 19 July 1944. The 28th Regiment forced the Partisan 36th Vojvodina Division back. The battalion from the 7th SS Division encountered serious resistance around Vlasenica but pushed through. On 19 July, the 28th Regiment drove north towards Gornje Petrovice to attack the Partisan 12th Corps at Živinice. The reconnaissance battalion of the 7th SS Division pushed in from Vareš as a blocking force to stop the Partisans from withdrawing via Kladanj. +The Partisans then committed the 16th Vojvodina Division to assist the 36th Vojvodina Division which had borne the brunt of Operation Heiderose to this point. On 20 July, both divisions attacked the 27th Regiment, which counterattacked the following day after the Partisans were rebuffed with heavy casualties. On 23 July, the Partisans began to withdraw south out of the area. The division began to search for the hidden Partisan bases that German intelligence had indicated were located there. After a day of searching, the division uncovered more than ten Partisan bases, which the cooperating Chetniks began to clear despite having shown little interest in actually fighting the Partisans in previous days. +Operation Heiderose was a significant success for the division, inflicting serious losses on the Partisans. More than 900 Partisans were killed and a large amount of equipment was captured: one anti-tank gun, two mortars, 22 machine guns, over 800 rifles and nearly 500,000 rounds of small arms ammunition. The division suffered 24 killed and over 150 wounded. Communist sources hold that the Partisans suffered serious losses, with the 12th Corps alone having 250 dead, wounded and missing. However, Partisan reports estimate that German losses were significantly larger than their own. Erich Braun was recommended for the Knight's Cross of the Iron Cross but the recommendation was not supported, apparently due to disagreements between the divisional staff and Phleps that had occurred during the division's formation. +In the first week of August 1944, the Second Panzer Army was finally ready to move to thwart the Partisan advance from Bosnia into Serbia, originally planned as Operation Rose. Renamed Operation Rübezahl (Mountain Spirit), the revamped plan required the 7th and 13th SS Divisions to form the Bosnian phase of the operation, named Operation Hackfleisch (Minced Meat). Hackfleisch aimed to drive out the Partisans occupying the area between the towns of Kladanj, Vlasenica, Sokolac and Olovo, south of the security zone. The overall plan involved columns that were to force the Partisans eastwards into the pincers of the other units, thereby destroying them. +The columns were allocated tasks and composed as follows: +the reconnaissance battalion of the 7th SS Division was to drive west from Vareš and attack the Partisans around the town of Olovo, pushing them east; +I/28 from the 13th SS Division was to attack south and south-east from Ribnica towards Olovo; +III/28 from the 13th SS Division was to thrust south and south-east from Kladanj towards the village of Petrović; +The 27th Regiment of the 13th SS Division was to drive southwards from Šekovići; +parts of the 14th Regiment of the 7th SS Division were to push north-west between Olovo and Sokolac; and +parts of the 13th Regiment of the 7th SS Division were to attack northwards through Sokolac. +The 27th Regiment became locked in fierce fighting with the Partisan 27th East Bosnia Division and 38th Bosnia Division, which had launched an offensive near Vlasenica. The inability of the 27th Regiment to push forward meant that the plan to encircle the Partisan force was not achieved and the Partisans escaped by crossing the Vlasenica–Han Pijesak road to the east. Other Partisan forces withdrew in the direction of Goražde. +As far as the Germans were concerned, Operation Hackfleisch was a moderate success, with 227 Partisans killed and over 50 prisoners taken. It delayed, but did not prevent, the Partisans' advance into Serbia. To allow the 7th SS Division to pursue Partisan forces withdrawing towards central Bosnia, both I/28 and III/28 were placed under the command of that division for the period of 8–17 August 1944. During the remainder of August 1944, the rump of the division fought hard to keep the Partisans out of the security zone. The Partisan 11th Krajina Division and 38th Bosnia Division quickly infiltrated into the southern part of the security zone and the 27th East Bosnia Division crossed into the area north-west of Srebrenica and drove towards Bratunac. After a series of rapid redeployments and battalion and regimental attacks, the division scattered the 11th Krajina Division and mauled the 27th East Bosnia Division. +The division had by now been fighting almost continuously throughout the summer. According to divisional commander Hampel, it had been exhausted even before Operation Hackfleisch began. The cumulative effect of this exhaustion, the deteriorating situation that the Germans faced on all fronts and rumours probably spread among members by the Partisans and Ustaše, was that the division began to disintegrate in early September 1944. +In early September 1944, the division returned north to the security zone, basing the infantry battalions in the villages of Kurukaja, Vukovije (south of Foča), Osmaci and Srebrnik. Almost immediately, the Partisan 3rd Corps commenced an offensive, attacking II/28 at Srebrnik. It held on despite being pressed hard during two days of fighting against the 11th Krajina Division. By the end of the first week in September, divisional supply columns were being attacked by Chetniks, sensing German weakness and looking for arms and supplies. Three members of the division were killed in these attacks. Over the same week, the Allies conducted Operation Ratweek throughout the Balkans, which involved Allied air power pounding Axis troop concentrations and key infrastructure to impede the German withdrawal from Greece. The local effect was to complicate the division's logistics through the destruction of the Sarajevo-Brod railway and mass desertions of NDH troops tasked with securing supply lines. +On 8 September, II/27 was assaulted unsuccessfully by the Partisans at Matkovac. After the first week of September, most of the division's fighting power was shifted to the western boundary of the security zone to meet Partisan incursions. The division succeeded in dislodging Partisan units from Slatna and Međeđa and allied Chetniks captured Skurgić. After relief by NDH forces, the division's fighting regiments withdrew to Brčko for rest and refit. As a result of the shift to the western area of the security zone, the zone's southern area was quickly overrun by Partisan forces. Zvornik and Tuzla had fallen by mid-September. +On 17 August 1944, Tito offered a general amnesty and many in the division took advantage of this opportunity. During the first three weeks of September, while hard fighting continued, over 2,000 Bosnians deserted, many taking their weapons with them. They went home, joined the Green Cadres militia or went over to the Ustaše. Many defected to the Partisans, with over 700 having joined the Partisan 3rd Corps by early October. +Due to high rates of desertion from the 13th SS Division, Sauberzweig proposed to disarm the Bosnians in both the 13th SS Division and the 23rd SS Division, but Himmler instead opted to transport the 2,000 Bosnians of the 23rd SS Division from Hungary to Bosnia and re-organise the remaining troops of both divisions there, with key support units from 13th SS Division centralised under IX SS Mountain Corps, which would also move to Bosnia from Hungary. +In the early morning of 3 October 1944, the Partisan 28th Slavonia Division assaulted a squadron of the reconnaissance battalion at Janja close to the Drina on the eastern boundary of the security zone. As they broke out of the encirclement to the north, the rest of the reconnaissance battalion drove south from Bijeljina and stopped the Partisan advance at heavy cost. Rushing towards Janja from the east, III/27 came into contact with Partisans around Modran, reaching the Janja garrison at 10 pm that night and receiving artillery reinforcement by 3/AR 13 during the night. At dawn the following day, an additional four Partisan brigades attacked the garrison in Janja, with fighting continuing throughout the day before the Partisans withdrew to the south. Jagdkommandos were sent after the fleeing enemy but were not able to inflict significant losses on them as they had already crossed the Drina into the German-occupied territory of Serbia. Following this battle, Army Group F concluded that the division's overall combat value was minimal. +A few days later, the 9th Company of the 28th Regiment (9/28) displayed what could still be achieved by the Bosnians under determined leadership when Leutnant Hans König ambushed the Partisan 17th Majevica Brigade near Vukosavci, killing at least 67 and capturing orders for future operations. König was awarded the German Cross in Gold for his fanatical leadership. +While 9/28 was fighting near Vukosavci, several units, consisting mainly of artillery, were temporarily detached for duty with other formations of the Second Panzer Army fighting Soviet troops within the German-occupied territory of Serbia. The division was not re-united until January 1945. +After a request from Army Group F to Himmler, a decision was made to move IX SS Mountain Corps and the division to perform road and railway security duties near Zagreb. This would relieve the LXIX Army Corps from those duties so that it could reinforce a defensive line along the Drina facing the advancing Red Army. The 28th Regiment, I/27 and III/AR 13 were to remain behind at the Brčko bridgehead to keep the Sava river bridge open. The heavy desertions from the division were the main reason for the move from north-east Bosnia to northern Croatia, but it made matters worse: the Bosnians were very reluctant to leave Bosnia and the already serious desertion rate became a flood when the move began on 16 October 1944. Many took their weapons with them and hundreds joined the Partisans. In mid-October, 700 members of the division stationed at Orašje joined the Partisans, and were distributed between the 17th Majevica Brigade and 21st East Bosnian Brigade. +On 20 October, the Red Army and Partisan forces captured Belgrade and the following day the divisional staff imam, Abdulah Muhasilović, incited a mutiny and led 100 men back to Bosnia. Himmler finally agreed to the disarming of the "unreliable" Bosnians under Operation Herbstlaub (Autumn Leaves) on 25 October. Between 900 and 1,000 Bosnians in the Brčko bridgehead and over 2,300 in Zagreb were allocated to labour battalions and similar non-combatant auxiliary duties, although the disarming operation was not completed in the Zagreb area until mid-November. By the beginning of November the division, which had been ninety-five percent non-German in January 1944, became fifty percent German. The plans to re-organise the division were abandoned and Sauberzweig was relieved of command. In addition, Sauberzweig's IX SS Mountain Corps staff was dissolved and the remnants were placed under the command of LXVIII Army Corps, under which they remained for the rest of the war. +Accounts differ on the fate of the division: author George Lepre says the division fought until the end of the war with Bosnians, while author George H. Stein states that in November, before fighting the Soviets, all Muslim personnel were disbanded and the associated German personnel and equipment were deployed elsewhere. Meanwhile, the advance of the Red Army through Belgrade and north into the formerly Hungarian-occupied Yugoslav region of Baranja made holding the line of the Danube critical for the Germans. In early November the Soviet troops established a bridgehead over the Danube from Apatin (in modern-day Serbia). From 9 November onward, the division was committed in several stages to the Eastern Front, starting with the reconnaissance battalion which went into action at Dárda west of the Apatin bridgehead on 10 November. They were followed by a grouping of three battalions (I/27, II/28 and II/28), an artillery battalion (III/AR 13) and pioneer support. Known as Kampfgruppe Hanke after their commander Hans Hanke, they moved from the Brčko bridgehead and joined a blocking position at Pélmonostor on 14 November south-west of a second Soviet bridgehead that had been established at Batina (in modern-day Croatia). As a result, the last Muslim SS troops left Bosnia. Within a week they were joined by the reconnaissance battalion, which had acquitted itself well in fighting west of Apatin. By 20 November, the Red Army was across the Danube in force at Batina. The following day Kampfgruppe Hanke was driven out of its positions when the remaining 200 troops withdrew. By 25 November the rest of the division was on its way from the Zagreb area. The remnants of Kampfgruppe Hanke were placed under command of Reichsgrenadier Division Hoch und Deutschmeister, withdrawing as far as Siklós in southern Hungary by 29 November. +Within days Kampfgruppe Hanke was withdrawn from the front line to rejoin the division and re-fit, and they were moved to Barcs on the Drava river for that purpose. On 2 December the division was reunited, except for the rest of the pioneer battalion, which remained stranded in Bosnia by damaged bridges and railways. As a result of continued desertions and the catastrophic casualties suffered by Kampfgruppe Hanke, the division retained little of its original Bosnian character. Despite the return of disarmed Bosnians from labour units, the attachment of Hungarian infantry and artillery units and the arrival of German replacements meant the division looked almost the same as any other unit in the Second Panzer Army. +After a short break, the division was allocated to a sector of the Margarethe Line, a series of fixed defensive works south of Lake Balaton which held up the Soviet advance. After several Red Army assaults were repulsed, static trench warfare developed between December 1944 and January 1945. In early March 1945, a few units took part in Operation Frühlingserwachen (Spring Awakening), which was to be Germany's last major offensive. Meanwhile, from December 1944 to March 1945, the rest of the division remained at Barcs rebuilding. In February, they had been joined by the divisional pioneer battalion, which had finally managed to withdraw from Bosnia. On 16 March, the Soviets launched the Vienna Offensive, bypassing the Barcs area. That night, the division attacked the flanks of the Soviet drive at Heresznye, where it undertook its last offensive operation of the war. +On 29 March, the Soviet 57th Army and the Bulgarian First Army assaulted the entire Second Panzer Army front, quickly breaching the line near Nagybajom. The division, holding positions just south of the penetration, began to withdraw to the northwest. It fell back to the Mura and barely managed to cross under constant air and ground attacks, with grievous casualties. Unable to take up effective positions, the division crossed the Reich frontier on 6 April and took up positions at Pettau in the so-called "Reich Defence Line", where it remained until 5 May. Its last fighting was around Kiesmanndorff on 19 April. +On 5 May, the division's remaining men, both German and Bosnian, began to retreat northwards towards Austria. On 8 May, an order was sent to retreat to Wolfsberg, Carinthia. On learning of the orders, unit imams of the 28th Regiment "approached their commander, Hans Hanke and requested that they and their men be discharged and be allowed to attempt to return to their homeland ... Soon, all of the Bosnians remaining in the division were asked if they wished to remain." There are two reports of large-scale reprisal killings of these members of the division by the Partisans, one of which included the mass shooting of 1,400 soldiers. +The retreat continued until 11 May. On 12 May, Hampel carried out surrender negotiations with the British Army at Sankt Veit an der Glan. On 15 May most of the men were transported to Rimini in Italy, where they were incarcerated with other prisoners of war from 7th SS Division and 16th SS Panzergrenadier Division Reichsführer-SS. Scattered remnants did not surrender until 18 May and many attempted to obliterate their SS blood group tattoo. Hampel escaped from a prisoner-of-war camp in Fallingbostel. +While it achieved successes and proved itself competent in counter-insurgency operations against the Partisans in eastern Bosnia, the division earned a reputation for brutality and savagery, not only during combat operations, but also through atrocities committed against Serb and Jewish civilians in the security zone. Its reprisal attacks in northern and eastern Bosnia left many hundreds and possibly as many as several thousand Serb civilians dead by the spring and summer of 1944. The post-war Nuremberg trials made the declaratory judgement that the Waffen-SS was a criminal organisation due to its major involvement in war crimes and crimes against humanity, including the killing of prisoners-of-war and atrocities committed in occupied countries. Excluded from this judgement were those who were conscripted into the Waffen-SS and had not personally committed war crimes and crimes against humanity. +Thirty-eight members of the division were extradited to Yugoslavia to face trial following the war. Some committed suicide, including Generalleutnant Sauberzweig on 20 October 1946 and SS-Obersturmführer Hans König. Trials took place at a military court in Sarajevo between 22 and 30 August 1947. Only seven of the 38 defendants were charged with specific offences, although the indictment accused the division of murdering some 5,000 people. The accused were defended by three Yugoslav lawyers: two civilians and one military officer. All were found guilty; 10 were sentenced to death, and 28 received prison terms of between five years and life. SS-Obersturmführer Imam Halim Malkoč had already been executed in Bihać on 7 March 1947. All of those who were executed were junior officers and non-commissioned officers. Almost all the prisoners were released early and by 1952 all had been freed, except for one who had died. Having escaped, Hampel never faced trial and lived in Graz, Austria until his death on 11 January 1981. +Hundreds of members of the 13th and 23rd SS Divisions volunteered to fight in the 1948–1949 Arab–Israeli War. The Syrian government requested the transfer of 8,000 Bosnian Muslim refugees to Syria, many for recruitment into the Syrian Armed Forces. Iraq sent representatives to Europe and invited 2,500 Bosnian Muslims to settle there. Frantzman and Culibrk estimate that approximately 1,000 former Bosnian Muslim SS members fought in Palestine. Many of the volunteers served in the Arab Liberation Army's Ajnaddin Battalion. +The following officers commanded the division. The first two raised and trained the division in its early stages of development and Sauberzweig and Hampel commanded it on active operations against the Partisans and later against the Red Army and Bulgarians: +SS-Gruppenführer und Generalleutnant der Waffen-SS Artur Phleps (in charge of raising the division, from 13 February 1943) +SS-Standartenführer der Reserve Herbert Von Obwurzer (9 March 1943 – 1 August 1943) +SS-Brigadeführer und Generalmajor der Waffen-SS Karl-Gustav Sauberzweig (1 August 1943 – 1 June 1944) +SS-Brigadeführer und Generalmajor der Waffen-SS Desiderius Hampel (1 June 1944 – 12 May 1945) +Several members were decorated with high German military awards, with five awarded the Knight's Cross of the Iron Cross (including Hampel and Hanke), five awarded the German Cross in Gold (including Hanke) and one awarded the German Cross in Silver. All five presentations of the Knight's Cross to members of the division are disputed and cannot be verified at the German National Archive. According to Gerhard von Seemen, Hampel, Karl Liecke and Hanke presumably received the Knight's Cross on the same day from the commander of the 2nd Panzer Army, General der Artillerie Maximilian de Angelis. +The divisional order of battle was: +27th Waffen Gebirgsjäger (Mountain Infantry) Regiment of the SS (1st Croatian) +28th Waffen Gebirgsjäger (Mountain Infantry) Regiment of the SS (2nd Croatian) +13th SS Mountain Artillery Regiment +13th SS Mountain Reconnaissance Battalion +13th SS Panzerjäger (Anti-tank) Battalion +13th SS Mountain Pioneer Battalion +13th SS Anti-aircraft Battalion +13th SS Mountain Signals Battalion +13th SS Division Supply Battalion +13th SS Logistics Battalion +13th SS Medical Battalion +13th SS Replacement Battalion +From 24 September 1944, the supply and logistics battalions were combined to form the 13th SS Service Support/Supply Regiment. +The division's identification symbol, used on its vehicles, was the Handschar sword. The uniform was a regular SS M43 field jacket issue, with a divisional collar patch showing an arm holding a scimitar over a swastika. On the left arm was a Croatian armshield (red-white chessboard) and on the right, the Edelweiss flower patch of the German mountain troops. The chessboard armshield was controversial, especially with the imams, who, after crossing the Sava river, removed them. Former SS personnel serving in the division were entitled to wear a Sig Rune badge that was attached to the left breast pocket of the tunic. +Headgear was either the SS M43 fez which was to be worn by all ranks, while German officers had the option to wear the mountain cap (Bergmütze). The fez was chosen by Himmler because it had been worn by the Bosnian-Herzegovinian Infantry regiments of the Austro-Hungarian army from 1894 to 1918, as well as by the Austro-Hungarian Albanian Legion from 1916 to 1918. There were two versions of the fez: a field grey model to be worn in combat and while on duty and a red model for parades, marching exercises and while off duty. Both the fez and mountain cap bore the death's head and eagle of the SS. The mountain cap was also adorned with an Edelweiss flower patch, worn on the left side of the cap. +The divisional song was set to the melody of "Wir fahren gegen Engelland" by Herms Niel. +List of Waffen-SS units +Relations between Nazi Germany and the Arab world +Table of ranks and insignia of the Waffen-SS +Waffen-SS foreign volunteers and conscripts +Michael Lee Capel (born October 13, 1961) is an American professional baseball player who was a pitcher in Major League Baseball (MLB) for the Chicago Cubs, the Milwaukee Brewers, and the Houston Astros. In 49 career games, Capel pitched 62+1⁄3 innings, struck out 43 batters, and had a career win–loss record of 3–4 with a 4.62 earned run average (ERA). While he played in MLB, Capel stood at 6 feet 1 inch (185 cm) and weighed 175 pounds (79 kg). A starting pitcher in college and parts of his Minor League Baseball career, he converted to relief pitching while in Chicago's minor league system. +The Philadelphia Phillies chose Capel in the 24th round of the 1980 MLB draft, but the 18-year-old did not sign with the team; instead, he opted to attend the University of Texas. Capel played on the 1982 USA College All-Star Team, which competed in the Amateur World Series in Seoul and placed third. The next year, Capel and the Texas Longhorns won the College World Series. After he was drafted by the Cubs, Capel left Texas and signed to play professional baseball; he played in six seasons of Minor League Baseball before he made his MLB debut in 1988. Capel spent the entire 1989 season in Triple-A, one level below the majors, but the Cubs released him at the end of the year. He agreed to terms with the Brewers and played in MLB after an injury opened a spot on Milwaukee's roster, but was again released at the end of the season. A free agent, the Astros signed Capel, and over the course of the season he pitched in 25 games for the team. He spent the final part of his career in the Astros farm system, and after he made the 1992 Triple-A All-Star team, Capel played his last season in 1993. After retirement, Capel worked as the general manager of a car dealership in Houston, Texas. +Capel was born on October 13, 1961, in Marshall, Texas, and attended Spring High School in Harris County. As a child, he watched the Astros play on weekends. During his senior year in 1979, the Spring Lions won the AAAA conference state championship, and Capel was named to the All-State team, composed of the best high school players in the state. He set several pitching records at Spring High School. Future teammate Calvin Schiraldi called Capel a "hard thrower when he came out of high school" and "the top guy out of the state in 1980"; Roger Clemens said he was "probably the best pitcher in the state at that time". The Philadelphia Phillies drafted Capel with the 605th overall pick in the 1980 MLB Draft, and offered him $45,000 (equivalent to $150,000 in 2025) to sign. Instead of signing with the Phillies, he chose to attend the University of Texas. +Aside from intrateam scrimmages, Capel did not play baseball his freshman season due to a torn ligament (or stretched ligaments) in his elbow. Capel pitched sidearm for the remainder of his collegiate career to compensate for a loss in the velocity of his fastball. In 1982, he was named to the All-Southwest Conference team and pitched to a 9–0 win–loss record with a 3.68 ERA, as the Longhorns finished 59–6. In the 1982 College World Series, Texas defeated the Oklahoma State Cowboys and the Stanford Cardinal, but losses to the Miami Hurricanes and the Wichita State Shockers eliminated them from play, per the tournament's double-elimination format. Capel played on the United States team in the 1982 Amateur World Series, overseen by the International Baseball Federation. Starting against Australia, he pitched a 14–4 complete game victory, with the match ending after seven innings per the championship's ten-run rule. The United States eventually placed third in the competition. +The 1983 Longhorns finished the regular season with a 61–14 record and were ranked as the number one team in the nation by Collegiate Baseball Magazine. That season, the Texas team featured four future MLB pitchers: Capel, Clemens, Bruce Ruffin, and Schiraldi. In the 1983 College World Series, Capel allowed four hits in a complete game against the Michigan Wolverines as the Longhorns advanced to the World Series finals. Facing the Alabama Crimson Tide, the Longhorns made Clemens their starting pitcher and won 4–3. The championship was the fourth World Series title in University of Texas history, and their first since 1975. Capel finished the season with a 13–1 record and a 2.98 ERA. As of 2018, his career winning percentage (.957) ties him with Rick Burley for the fifth-best in Texas Longhorns history (with a minimum of ten decisions). In June, before the College World Series, he was drafted by the Cubs with the 320th overall pick in the 1983 MLB draft. +After he signed a professional contract with the Cubs, Capel no longer threw sidearmed. Capel began professional baseball with the Double-A Midland Cubs, but was demoted to the Class A Quad Cities Cubs after pitching to a 1–1 record with a 6.91 ERA in Double-A. Now in Single-A, Capel recorded a 3–2 record with a 2.42 ERA and a 1.03 walks plus hits per inning pitched rate. In 1984, Capel split playing time between the Class A-Advanced Lodi Crushers and Midland; he led Midland in losses (10) and wild pitches (11) and had the second-worst ERA (6.31) on the team. With Midland, Capel started 11 games out of the 16 in which he appeared. On his 0–7 record for the Crushers, Tom Alexander of the Lodi News-Sentinel wrote that Capel's luck "has been all bad when it comes to wins". Both Lodi and Midland finished their seasons with losing records. +From 1985 to 1986, he played for the Double-A Pittsfield Cubs of the Eastern League. In 1986, Capel only pitched in relief, had a career-best 1.87 ERA over 62+2⁄3 innings pitched, and led his team with 13 saves. After he was promoted to the Triple-A Iowa Cubs, Capel split time between starting and relieving roles in 1987. At 7–10, Capel tied for Iowa's lead in wins and losses, and led the team in strikeouts, with 75; his seven wins became a career high, and his loss total matched a career high set in Midland three years earlier. After the season, Ray Sons of the Chicago Sun-Times said Capel "may be ready for middle relief in Chicago", and Dave van Dyck listed him as a prospect for the 1988 season. During winter, Capel, Greg Maddux, Manny Trillo, Damon Berryhill, and several other Cubs played for teams in the Caribbean. Capel played for the Águilas del Zulia of Venezuela and led the team with five saves. +Capel played in spring training ball as he tried to make Chicago's opening day roster. After the Cubs acquired Mike Bielecki from the Pittsburgh Pirates on March 31, 1988, they assigned Capel to Triple-A despite a Sun-Times prediction that he would be on the MLB roster. Prior to the acquisition, when Al Nipper was supposed to become the Cubs' fifth starter, Capel was to take his spot in the bullpen. The Cubs recalled Capel and Mark Grace to the majors on May 3, while Drew Hall and Rolando Roomes were optioned to Iowa; at the time, he had a 1.54 ERA and two saves in Triple-A ball. On May 7, Capel made his MLB debut: he pitched the final 1+2⁄3 innings of a game against the San Francisco Giants, allowing no runs and no hits but issuing a walk in a 2–1 Cubs loss. His first career win came the following day against the Giants, although he blew a save by giving up a two-run home run to Bob Brenly and lost a 5–4 lead. The Cubs ultimately won 13–7. Capel continued to pitch for the club until June 30, when the Cubs optioned him to Iowa to open a roster spot for Rich "Goose" Gossage to come off the 15-day disabled list (DL). +He returned to the MLB club on August 8 to replace Schiraldi, but was demoted on August 13 before making an appearance, as Chicago activated Les Lancaster from the DL. Capel rejoined the club on August 31, and pitched in five more games before the season's end. Over his first MLB season, Capel pitched in 20 games and allowed 16 earned runs over 29+1⁄3 innings for a 4.91 ERA. He pitched the entire 1989 season for Iowa, appearing in 64 games for the team. On October 15, Capel was granted free agency by the Cubs; he signed with the Brewers two months later. +Capel competed against 22 other pitchers for an MLB roster spot in the Brewers' spring training camp. The Brewers optioned Capel to their Triple-A affiliate, the Denver Zephyrs, on April 5, before the start of Milwaukee's 1990 season. On May 17, Denver placed Capel on the DL with a strained Achilles tendon, which opened a roster spot on the Zephyrs and allowed the Brewers to send Jaime Navarro back to Denver. When the Brewers needed a replacement for the injured Bill Wegman, Capel was called up to the major leagues. For his Brewers debut, Capel faced five batters and allowed four runs (two charged to Capel) against the Baltimore Orioles. His strikeout of Billy Ripken was the only MLB out Capel recorded in 1990. On June 8, he made his second and final outing with the Brewers and allowed four runs (three charged to Capel) against the Toronto Blue Jays; combined, batters hit .857 against Capel as he finished the season with a 135.00 ERA. When Greg Vaughn, returning from a turf toe injury and a sprained left ankle, was activated from the DL on June 11, Capel was sent to Triple-A. He finished the season with Denver where he had a 4–3 record with a 4.26 ERA. On October 4, the Brewers granted Capel free agency; he signed with the Astros on January 5, 1991. +By 1991, Capel threw four pitches: a fastball, a forkball, a slider, and occasionally a curveball. He began his 1991 season with the Triple-A Tucson Toros; the Astros added Capel to their MLB roster on June 7. Two days later, against the New York Mets, Capel allowed the game-winning home run to Howard Johnson in the top of the 11th inning. Against the Mets on June 14, Capel, starting pitcher Pete Harnisch, and closer Jim Clancy combined for a four-hitter. His final MLB appearance came on August 14 against the San Diego Padres, when he pitched one inning in relief of Jim Corsi in a 4–1 loss. Capel said pitching for Houston was "the high point of [his] career". Over 25 games pitched, Capel finished the season with a 1–3 record and a 3.03 ERA. +To reduce their roster to 35 players, the Astros assigned Capel and seven others to the Toros prior to the 1992 MLB season. Capel was selected to the Triple-A All-Star Game to replace Tim Scott, whom the Padres promoted to MLB. He told Javier Morales of the Arizona Daily Star "this [was] one of the greatest thrills of [his] career," and that he hoped it showed the Astros he could pitch in MLB. Doug Jones worked as Houston's closer in 1992, however, and Capel never played for the MLB club. For the Toros, Capel had a 6–6 record and a 2.19 ERA with a team-leading 18 saves. +The following season, the Montreal Expos invited Capel to their spring camp as a non-roster invitee. Capel elected to remain with the Astros organization where he allowed 26 earned runs in 32+2⁄3 innings and pitched the entire season in Triple-A. The Toros, managed by Rick Sweet, finished first in the Pacific Coast League (PCL) South Division with an 83–60 record and made the playoffs. Unlike the 1993 MLB playoffs, in which four teams competed, only two teams could make the PCL playoffs. In the championship series, the Toros faced the Portland Beavers, champions of the North Division and holders of a PCL-best 87–56 record. In a best-of-seven series, the Toros defeated the Beavers four games to two. After the season, the Astros did not re-sign Capel. While playing in MLB, Capel stood at 6 feet 1 inch (185 cm) and weighed 175 pounds (79 kg). +In February 1986, Capel pitched in the University of Texas Alumni v. Varsity game, which ended in a 14–14 tie. Capel started to use dipping tobacco in 1976 and in March 1988 had a non-cancerous lesion removed from his lip; he tried to quit the product in March 1988. Capel met his wife, Elizabeth, at Roger Clemens' wedding. Outside of baseball, he is a close friend of Clemens, and during Clemens' 2012 perjury trial, Capel testified on his work ethic and character. In 2012, Mike worked as the general manager of a car dealership in Houston. Capel's son, Conner, played baseball as an outfielder for Seven Lakes High School in Katy, Texas, where he had a .456 batting average with 23 stolen bases and 36 runs scored in his senior year. The Cleveland Indians drafted Conner in the fifth round of the 2016 MLB draft. +Career statistics from MLB · ESPN · Baseball Reference · Baseball Reference (Minors) · Retrosheet · Baseball Almanac +"Dance in the Dark" is a song by American singer Lady Gaga from her third extended play (EP), The Fame Monster (2009)—the reissue of her debut studio album, The Fame (2008). She co-wrote the track with its producer Fernando Garibay. The lyrics are about a woman who prefers to have sex in the dark as she is insecure in her body. Having met such women while working on the MAC AIDS Fund, Gaga said that the song is not about freedom, but rather the assurance that she understood their feelings. A Europop track, it contains retro and new wave music influences, and begins with a stuttering introduction. It includes a spoken interlude, where Gaga lists famous dead people. +Interscope Records intended "Dance in the Dark" to be the third single from The Fame Monster, but Gaga insisted on "Alejandro", which was released in April 2010. Some months later, "Dance in the Dark" was issued to Australian and French radio, making it the EP's fourth and final single. Critics praised the song for its chorus and theme, although some found it a formulaic production. Retrospective reviewers ranked the song as one of Gaga's best. "Dance in the Dark" reached the top ten of charts in Poland, Hungary and Czech Republic, as well as the US Dance/Electronic Digital Song. It was nominated for a Grammy Award for Best Dance Recording. +Gaga performed "Dance in the Dark" as the opening song of the Monster Ball Tour. In the first leg of the tour, Gaga appeared behind a scrim-lit screen, and in the revamped shows, she performed it on a set reminiscent of a New York City night scene. Other events where she performed the song include the 2010 BRIT Awards—dedicated to Gaga's close friend, Alexander McQueen, who had committed suicide a few days earlier—and her Las Vegas residency, Enigma. A remix of the song appeared on her album The Remix (2010). +Lady Gaga and Fernando Garibay wrote and arranged "Dance in the Dark"; they were also responsible for instrumentation. Garibay solely handled programming, and served as the main producer, with Gaga co-producing with him. Jonas Westling, Dan Parry and Christian Delano were the recording and tracking engineers. The song was recorded at Metropolis Studios in London. +Gaga told the Los Angeles Times that the inspiration behind "Dance in the Dark" came from an intimate experience between two people in a bedroom. The song is about a girl who prefers to have sex in the dark as she feels self-conscious about her body. Gaga explained, "She doesn't want her man to see her naked. She will be free, and she will let her inner animal out, but only when the lights are out." While working on the Mac AIDS Fund, she realized that women her age do not express such insecurities, fearing that their boyfriends would not love them if they did so. Gaga added that she herself struggled with issues of body image and self-doubt. +All of these new things entering my life are changing the way I view my purpose, but 'Dance in the Dark' in particular is about me wanting to live—but also, the song isn't called 'Dance in the Light'. I'm not a gospel singer trying to cross people over. What I'm saying is, 'I get it. I feel you, I feel the same way, and it's OK.' I hope and pray that I can inspire some sort of change in people subliminally through the show. They're singing 'Dance in the Dark,' but they're dancing and they're free, they're letting it out. But the songs are not about freedom, they're about [the fact that] I get it. I feel the way you feel. +"Dance in the Dark" was released as a promotional single from the extended play (EP) The Fame Monster—the reissue of Gaga's debut studio album, The Fame (2008)—on November 9, 2009, when it became available to purchase on iTunes Store in Belgium. According to MTV, Interscope Records initially planned "Dance in the Dark" as the EP's third single, but Gaga preferred "Alejandro", which was released on April 20, 2010. Universal Music Group issued "Dance in the Dark" to Australian and French radio, on July 26 and August 25, 2010, respectively, making it the EP's fourth and final single. In March of that year, Gaga released her first remix album entitled The Remix, which included the Monarchy Stylites rendition of the song. +"Dance in the Dark" is a Europop track which infuses influences of retro and new wave music in its composition. Author Robin James in the book Resilience & Melancholy: Pop Music, Feminism, Neoliberalism recognized synth-pop elements in the song, which was influenced by Depeche Mode's "Strangelove" (1987). Paul Lester of the BBC and Tony Hardy of Consequence called it an R&B production; Hardy also found dance elements in the song. +"Dance in the Dark" begins with a stuttering introduction and orgasmic groans, followed by Gaga singing the song. Musicologist Alexandra Apolloni found that "Dance in the Dark" was one of many songs where Gaga uses a "technologically-imposed vocal stutter". She cited author Daniel Goldmark's opinion that using stutter in American popular music was a way to demonstrate abnormality. For Apolloni, such vocal instabilities complement moments where "bodily vocal labor is audible", a result of Gaga physically manipulating her voice. Apolloni saw moments of "vocal ugliness and abjection" in the song, which she interpreted as Gaga vocally expressing pain and pleasure, eliciting a "'real' (albeit totally performed) non-technological body". +Sal Cinquemani from Slant Magazine noted influences from 1980s music in "Dance in the Dark", writing, "The song isn't a cautionary tale per se, but a call to arms to misfits everywhere." Analyzing the song's music, Michael Cragg of The Guardian found it "a glorious juxtaposition of textures: the harsh, stuttering vocal meshing with that glacial riff, while the processed vocals of the verses burst into the sunshine of the chorus". He wrote about its "robotic" spoken interlude, which he believed is "eventually washed away by a warm synthetic breeze". The interlude references famous dead people: Marilyn Monroe, Judy Garland, Sylvia Plath, JonBenét Ramsey, Liberace, Jesus, Stanley Kubrick, and Diana, Princess of Wales. +The lyrics of the song refer to vampires and werewolves: "Run run her kiss is a vampire grin/The moon lights her way while she's howlin' at him". Gaga explained that the lines conveyed how people rely on external motivations to cope with internal anxiety: "She doesn't feel free without the moon". Gaga said the song discusses self-insecurity common among men and women at any age. Sociologist Mathieu Deflem believed the lyrics "I'm a free bitch, baby!" exemplified Gaga's response to sexism in her music where she confirms her strength as a woman. +"Dance in the Dark" received mixed to positive reviews from music critics. Some called it "campy", single-worthy and an album highlight. Praise focused on the song's suitability for workout, and extravagant and club-friendly production. Reviewers also positively compared Gaga to Madonna and noted the references to famous people. Michael Hubbard of MusicOMH highlighted the spoken-word lyrics referencing Princess Diana: "You will never fall apart Diana, you're still in our hearts / Never let you fall apart / Together we'll dance in the dark." Nick Levine from Digital Spy found that the song "makes you want to dance with your top off in a grotty German bondage basement". +"Dance in the Dark" received a nomination for Best Dance Recording at the 53rd Annual Grammy Awards. The song was ranked as one of Gaga's best by the Official Charts Company, Rolling Stone, The Guardian, Vulture and Glamour. It was cited as a "cult favorite" by Vulture. The song's chorus—called "one of Gaga's most lustrous" by Rolling Stone and "deliciously outrageous, sky-scraping" by Vulture—was praised for "pierc[ing] through the mesh of synths like a laser". Other plaudits focused on the song's theme. Glamour argued that its topic about body insecurities and sound (describing it as "one of Gaga's brightest efforts: booming and industrial yet incredibly haunting") made it her most emotionally impactful song to date. Elena Gorgan of Softpedia believed Gaga, through the song, was unafraid of pushing boundaries and changing what people view as "normal". +Some reviews were less enthusiastic, calling the song "slightly too disposable" and finding the dance and R&B production formulaic. Adam White of The Independent liked the beginning, but found that the song "gradually declines into something more middle-of-the-road than it first appears". Hubbard added that "after a stuttering intro of orgasmic groans it becomes a little bit Gaga-by-numbers, which is a shame given the standard of the early tracks". +In late 2009, "Dance in the Dark" charted on the Hungarian Mahasz Single Top 10 lista, the UK Singles Chart and the Canadian Hot 100 at number 9, number 89 and number 88, respectively, for one week. The song debuted at number 43 on the Australian Singles Chart, later peaking at number 24. It entered the French Digital Singles Chart at number 40 and peaked at number 30. In the US, "Dance in the Dark" peaked at number 22 on the Bubbling Under Hot 100 Singles chart in August 2010 and number 9 on Hot Dance/Electronic Digital Songs in October 2010. It reached number two on the Polish Dance Top 50 in October 2010. +Gaga performed "Dance in the Dark" as the opening song of the Monster Ball Tour (2009–2011). During the original 2009 leg of the tour, the show began as Gaga appeared behind a giant, green laser-lit video screen featuring scrim lights, in a futuristic silver jeweled jumpsuit with bulbs on it. Wearing matching eye makeup and a mask, she sang "Dance in the Dark" while surrounded by dancers in white balaclavas and jumpsuits. The scrim was lifted during the performance. +"Dance in the Dark" was again the opening song in the revamped 2010–2011 shows of the Monster Ball Tour. Gaga performed the song in a set reminiscent of a New York City night scene, with flickering neon signs displaying the words "ugly", "sexy" and "liquor", fire escape stairwells and a broken yellow taxi. For Katrin Horn, a postdoctoral fellow in American studies, the "Dance in the Dark" performance helped highlight a recurring theme in Gaga's work from the tour's beginning—"pop culture's obsession with both decay and beauty, or more precisely the decay of beauty"—as the song is dedicated to famous dead people who had struggled with their reputation when they were alive. +Gaga sang "Telephone", the second single from The Fame Monster, and "Dance in the Dark" at the Brit Awards on February 16, 2010, at Earls Court Exhibition Centre. Inspired by the recent death of her friend, fashion designer Alexander McQueen, she changed the concept of her performance at the last minute to pay tribute to him. Gaga said, "I wanted to do a very, very forward performance, something that I felt was a true representation of the future." Her performance of "Dance in the Dark", described as more restrained compared to her previous ones by The Wall Street Journal, began as she got up from her piano while the disco groove of the song was heard. Wearing a lace bodysuit and a large bouffant wig, Gaga took to a giant keytar and performed a techno rendition of the track. In 2015, Liv Moss of Official Charts Company called it one of the "biggest, best and weirdest" performances in the history of Brit Awards. The following year, Daniel Welsh of HuffPost UK cited it as one of Gaga's 15 most memorable performances on television. +Gaga sang "Dance in the Dark" on her 2018–2020 Las Vegas residency show, Enigma. The song was preceded by an interlude, which introduced her alien counterpart, named Enigma. Gaga was sent through a future simulation by the character, and reappeared on stage in a fluorescent green jacket to perform "Dance in the Dark". The song was performed on piano at select dates of The Mayhem Ball tour (2025–2026). The Arts Desk's Thomas H. Green, who found the song's album version cheesy, opined that the live, stripped-down "Dance in the Dark" became "emotionally alive", and a fierce middle finger to gaslighting. +Credits adapted from the liner notes of The Fame Monster. +Recording and postproduction locations +Recorded at Metropolis Studios, London, Englandand Paradise Studios, Hollywood, Los Angeles, California +Audio mixing at Sarm Studios, London, England +Audio mastering at Oasis Mastering, Burbank, California +Personnel +Lady Gaga – producer, instrumentation and arrangement +Fernando Garibay – co-producer, instrumentation, programming and arrangement +Jonas Westling – recording engineer +Dan Parry – recording engineer +Christian Delano – recording engineer +Robert Orton – mixer +Gene Grimaldi – mastering engineer +On June 29, 2020, Japanese-British singer-songwriter Rina Sawayama released a cover of "Dance in the Dark" as part of Spotify's Spotify Singles series. Produced by Clarence Clarity, the song was included in Sawayama's remix EP Sawayama Remixed (2020). Due to the COVID-19 pandemic, Sawayama recorded her cover at her home studio, instead of Electric Lady Studios in New York City, where Spotify Singles are generally recorded. It is an electropop track with influences of nu metal. +Robin Murray of Clash and Brendan Wetmore of Paper Mag reviewed the song positively. Murray described Sawayama's take on the original as "seismic". Wetmore opined that "Sawayama's voice somehow rivals even Lady Gaga's level of theatricality". He felt that her rendition was "almost painting a picture of an animated Disney love story", because of "how much the track evokes the feeling of listening to pop's greatest hitmakers of the late '90s, the golden era of the Disney ballad." +Niels Henrik David Bohr (; Danish: [ˈne̝ls ˈpoɐ̯ˀ]; 7 October 1885 – 18 November 1962) was a Danish theoretical physicist who made foundational contributions to understanding atomic structure and quantum theory, for which he received the Nobel Prize in Physics in 1922. He was also a philosopher and a promoter of scientific research. +Bohr developed the Bohr model of the atom, in which he proposed that energy levels of electrons are discrete and that the electrons revolve in stable orbits around the atomic nucleus but can jump from one energy level (or orbit) to another. Although the Bohr model has been supplanted by other models, its underlying principles remain valid. He conceived the principle of complementarity: that items could be separately analysed in terms of contradictory properties, like behaving as a wave or a stream of particles. The notion of complementarity dominated Bohr's thinking in both science and philosophy. +Bohr founded the Institute of Theoretical Physics at the University of Copenhagen, now known as the Niels Bohr Institute, which opened in 1920. Bohr mentored and collaborated with physicists including Hans Kramers, Oskar Klein, George de Hevesy, and Werner Heisenberg. He predicted the properties of a new zirconium-like element, which was named hafnium, after the Latin name for Copenhagen, where it was discovered. Later, the synthetic element bohrium was named after him because of his groundbreaking work on the structure of atoms. +During the 1930s, Bohr helped refugees from Nazism. After Denmark was occupied by the Germans, he met with Heisenberg, who had become the head of the German nuclear weapon project. In September 1943 word reached Bohr that he was about to be arrested by the Germans, so he fled to Sweden. From there, he was flown to Britain, where he joined the British Tube Alloys nuclear weapons project, and was part of the British mission to the Manhattan Project. After the war, Bohr called for international cooperation on nuclear energy. He was involved with the establishment of CERN and the Research Establishment Risø of the Danish Atomic Energy Commission and became the first chairman of the Nordic Institute for Theoretical Physics in 1957. In 1999, he was named the fourth greatest physicist of all time by Physics World. +Niels Henrik David Bohr was born on 7 October 1885 in Copenhagen, the second of three children of Christian Bohr, Professor of Physiology at the University of Copenhagen, and Ellen Adler, the daughter of Danish Jewish banker David Baruch Adler. He had an elder sister, Jenny, and a younger brother, Harald. Jenny became a teacher, while Harald became a mathematician and footballer who played for the Danish national team at the 1908 Summer Olympics in London. Niels was a passionate footballer as well, and the two brothers played several matches for the Copenhagen-based Akademisk Boldklub (Academic Football Club), with Niels as goalkeeper. +Bohr was educated at Gammelholm Latin School, starting when he was seven. In 1903, Bohr enrolled as an undergraduate at the University of Copenhagen. His major was physics, which he studied under Christian Christiansen, the university's only professor of physics at that time. He also studied astronomy and mathematics under Thorvald Thiele, and philosophy under Harald Høffding, a friend of his father. +In 1905, a gold medal competition was sponsored by the Royal Danish Academy of Sciences and Letters to investigate a method for measuring the surface tension of liquids that had been proposed by Lord Rayleigh in 1879. This involved measuring the frequency of oscillation of the radius of a water jet. Bohr conducted a series of experiments using his father's laboratory in the university; the university itself had no physics laboratory. To complete his experiments, he had to make his own glassware, creating test tubes with the required elliptical cross-sections. He went beyond the original task, incorporating improvements into both Rayleigh's theory and his method, by taking into account the viscosity of the water, and by working with finite amplitudes instead of just infinitesimal ones. His essay, which he submitted at the last minute, won the prize. He later submitted an improved version of the paper to the Royal Society in London for publication in the Philosophical Transactions of the Royal Society. +In 1909, Bohr received a master's degree in Physics for a thesis on the electron theory of metals, a topic assigned by his supervisor, Christian Christiansen. Bohr subsequently elaborated his master's thesis into his much-larger doctoral thesis. He surveyed the literature on the subject, settling on a model developed by Paul Drude and elaborated by Hendrik Lorentz, in which the electrons in a metal are considered to behave like a gas. Bohr extended Lorentz's model, but was still unable to account for phenomena like the Hall effect, and concluded that electron theory could not fully explain the magnetic properties of metals. The thesis was accepted in April 1911, and Bohr conducted his formal defence on 13 May. Harald had received his doctorate the previous year. Bohr's thesis was groundbreaking, but attracted little interest outside Scandinavia because it was written in Danish, a Copenhagen University requirement at the time. In 1921, the Dutch physicist Hendrika Johanna van Leeuwen would independently derive a theorem in Bohr's thesis that is today known as the Bohr–Van Leeuwen theorem. +In September 1911, Bohr, supported by a fellowship from the Carlsberg Foundation, travelled to England, where most of the theoretical work on the structure of atoms and molecules was being done. He went to the University of Cambridge, where he met J. J. Thomson, Director of the Cavendish Laboratory, and attended lectures on electromagnetism given by James Jeans and Joseph Larmor. He did some research on cathode rays, but failed to impress Thomson. He had more success with younger physicists like the Australian William Lawrence Bragg, and New Zealand's Ernest Rutherford, whose 1911 small central nucleus Rutherford model of the atom had challenged Thomson's 1904 plum pudding model. Bohr received an invitation from Rutherford to conduct post-doctoral work at the Victoria University of Manchester, where Bohr met George de Hevesy and Charles Galton Darwin (whom Bohr referred to as "the grandson of the real Darwin"). +Bohr returned to Denmark in July 1912 for his wedding, and travelled around England and Scotland on his honeymoon. On his return, he became a Privatdocent at the University of Copenhagen, giving lectures on thermodynamics. Martin Knudsen put Bohr's name forward for a docent, which was approved in July 1913, and Bohr then began teaching medical students. His three papers, which later became famous as "the trilogy", were published in Philosophical Magazine in July, September and November of that year. He adapted Rutherford's nuclear structure to Max Planck's quantum theory and so created his Bohr model of the atom. +Planetary models of atoms were not new, but Bohr's treatment was. Taking the 1912 paper by Darwin on the role of electrons in the interaction of alpha particles with a nucleus as his starting point, he advanced the theory of electrons travelling in orbits of quantised "stationary states" around the atom's nucleus in order to stabilise the atom, but it wasn't until his 1921 paper that he showed that the chemical properties of each element were largely determined by the number of electrons in the outer orbits of its atoms. He introduced the idea that an electron could drop from a higher-energy orbit to a lower one, in the process emitting a quantum of discrete energy. This became a basis for what is now known as the old quantum theory. +In 1885, Johann Balmer had come up with his Balmer series to describe the visible spectral lines of a hydrogen atom: +1λ=RH(122−1n2)for n=3,4,5,... +where λ is the wavelength of the absorbed or emitted light and RH is the Rydberg constant. Balmer's formula was corroborated by the discovery of additional spectral lines, but for thirty years, no one could explain why it worked. In the first paper of his trilogy, Bohr was able to derive it from his model: +RZ=2π2meZ2e4h3 +where me is the electron's mass, e is its charge, h is the Planck constant and Z is the atom's atomic number (1 for hydrogen). +The model's first hurdle was the Pickering series, lines that did not fit Balmer's formula. When challenged on this by Alfred Fowler, Bohr replied that they were caused by ionised helium, helium atoms with only one electron. The Bohr model was found to work for such ions. Many older physicists, like Thomson, Rayleigh and Hendrik Lorentz, did not like the trilogy, but the younger generation, including Rutherford, David Hilbert, Albert Einstein, Enrico Fermi, Max Born and Arnold Sommerfeld saw it as a breakthrough. Einstein called Bohr's model "the highest form of musicality in the sphere of thought." The trilogy's acceptance was entirely due to its ability to explain phenomena that stymied other models, and to predict results that were subsequently verified by experiments. Today, the Bohr model of the atom has been superseded, but is still the best known model of the atom, as it often appears in high school physics and chemistry texts. +Bohr did not enjoy teaching medical students. He later admitted that he was not a good lecturer, because he needed a balance between clarity and truth, between "Klarheit und Wahrheit". He decided to return to Manchester, where Rutherford had offered him a job as a reader in place of Darwin, whose tenure had expired. Bohr accepted. He took a leave of absence from the University of Copenhagen, which he started by taking a holiday in Tyrol with his brother Harald and aunt Hanna Adler. There, he visited the University of Göttingen and the Ludwig-Maximilians-Universität München, where he met Sommerfeld and conducted seminars on the trilogy. The First World War broke out while they were in Tyrol, greatly complicating the trip back to Denmark and Bohr's subsequent voyage with Margrethe to England, where he arrived in October 1914. They stayed until July 1916, by which time he had been appointed to the Chair of Theoretical Physics at the University of Copenhagen, a position created especially for him. His docentship was abolished at the same time, so he still had to teach physics to medical students. New professors were formally introduced to King Christian X, who expressed his delight at meeting such a famous football player. +In April 1917, Bohr began a campaign to establish an Institute of Theoretical Physics. He gained the support of the Danish government and the Carlsberg Foundation, and sizeable contributions were also made by industry and private donors, many of them Jewish. Legislation establishing the institute was passed in November 1918. Now known as the Niels Bohr Institute, it opened on 3 March 1921, with Bohr as its director. His family moved into an apartment on the first floor. Bohr's institute served as a focal point for researchers into quantum mechanics and related subjects in the 1920s and 1930s, when most of the world's best-known theoretical physicists spent some time in his company. Early arrivals included Hans Kramers from the Netherlands, Oskar Klein from Sweden, George de Hevesy from Hungary, Wojciech Rubinowicz from Poland, and Svein Rosseland from Norway. Bohr became widely appreciated as their congenial host and eminent colleague. Klein and Rosseland produced the institute's first publication even before it opened. +The Bohr model worked well for hydrogen and ionized single-electron helium, which impressed Einstein but could not explain more complex elements. By 1919, Bohr was moving away from the idea that electrons orbited the nucleus and developed heuristics to describe them. The rare-earth elements posed a particular classification problem for chemists because they were so chemically similar. An important development came in 1924 with Wolfgang Pauli's discovery of the Pauli exclusion principle, which put Bohr's models on a firm theoretical footing. Bohr was then able to declare that the as-yet-undiscovered element 72 was not a rare-earth element but an element with chemical properties similar to those of zirconium. (Elements had been predicted and discovered since 1871 by chemical properties), and Bohr was immediately challenged by the French chemist Georges Urbain, who claimed to have discovered a rare-earth element 72, which he called "celtium". At the Institute in Copenhagen, Dirk Coster and George de Hevesy took up the challenge of proving Bohr right and Urbain wrong. Starting with a clear idea of the chemical properties of the unknown element greatly simplified the search process. They went through samples from Copenhagen's Museum of Mineralogy looking for a zirconium-like element and soon found it. The element, which they named hafnium (hafnia being the Latin name for Copenhagen), turned out to be more common than gold. +The Bohr Festival (German: Bohrfestspiele) was a series of seven lectures given by Bohr from 12 to 22 June 1922 at the Institute of Theoretical Physics in Göttingen. These were the Wolfskehl Lectures, funded by the Wolfskehl Foundation. Taking place in the fortnight leading up to the Göttingen International Handel Festival, it became known as the Bohr Festival. In 1991, Friedrich Hund suggested that James Franck was responsible for the comparison. In the lectures, Bohr outlined the current development of the Bohr-Sommerfeld theory, remarking "how incomplete and uncertain everything still is". +In 1922, Bohr was awarded the Nobel Prize in Physics "for his services in the investigation of the structure of atoms and of the radiation emanating from them". The award thus recognised both the trilogy and his early leading work in the emerging field of quantum mechanics. For his Nobel lecture, Bohr gave his audience a comprehensive survey of what was then known about the structure of the atom, including the correspondence principle, which he had formulated. This states that the behaviour of systems described by quantum theory reproduces classical physics in the limit of large quantum numbers. +The discovery of Compton scattering by Arthur Holly Compton in 1923 convinced most physicists that light was composed of photons and that energy and momentum were conserved in collisions between electrons and photons. In 1924, Bohr, Kramers, and John C. Slater, an American physicist working at the Institute in Copenhagen, proposed the Bohr–Kramers–Slater theory (BKS). It was more of a program than a full physical theory, as the ideas it developed were not worked out quantitatively. The BKS theory became the final attempt at understanding the interaction of matter and electromagnetic radiation on the basis of the old quantum theory, in which quantum phenomena were treated by imposing quantum restrictions on a classical wave description of the electromagnetic field. +Modelling atomic behaviour under incident electromagnetic radiation using "virtual oscillators" at the absorption and emission frequencies, rather than the (different) apparent frequencies of the Bohr orbits, led Max Born, Werner Heisenberg and Kramers to explore different mathematical models. They led to the development of matrix mechanics, the first form of modern quantum mechanics. The BKS theory also generated discussion of, and renewed attention to, difficulties in the foundations of the old quantum theory. The most provocative element of BKS – that momentum and energy would not necessarily be conserved in each interaction, but only statistically – was soon shown to be in conflict with experiments conducted by Walther Bothe and Hans Geiger. In light of these results, Bohr informed Darwin that "there is nothing else to do than to give our revolutionary efforts as honourable a funeral as possible". +The introduction of spin by George Uhlenbeck and Samuel Goudsmit in November 1925 was a milestone. The next month, Bohr travelled to Leiden to attend celebrations of the 50th anniversary of Hendrick Lorentz receiving his doctorate. When his train stopped in Hamburg, he was met by Wolfgang Pauli and Otto Stern, who asked for his opinion of the spin theory. Bohr pointed out that he had concerns about the interaction between electrons and magnetic fields. When he arrived in Leiden, Paul Ehrenfest and Albert Einstein informed Bohr that Einstein had resolved this problem using relativity. Bohr then had Uhlenbeck and Goudsmit incorporate this into their paper. Thus, when he met Werner Heisenberg and Pascual Jordan in Göttingen on the way back, he had become, in his own words, "a prophet of the electron magnet gospel". +Heisenberg first came to Copenhagen in 1924, then returned to Göttingen in June 1925, shortly thereafter developing the mathematical foundations of quantum mechanics. When he showed his results to Max Born in Göttingen, Born realised that they could best be expressed using matrices. This work attracted the attention of the British physicist Paul Dirac, who came to Copenhagen for six months in September 1926. Austrian physicist Erwin Schrödinger also visited in 1926. His attempt at explaining quantum physics in classical terms using wave mechanics impressed Bohr, who believed it contributed "so much to mathematical clarity and simplicity that it represents a gigantic advance over all previous forms of quantum mechanics". +When Kramers left the institute in 1926 to take up a chair as professor of theoretical physics at the Utrecht University, Bohr arranged for Heisenberg to return and take Kramers's place as a lektor at the University of Copenhagen. Heisenberg worked in Copenhagen as a university lecturer and assistant to Bohr from 1926 to 1927. +Bohr became convinced that light behaved like both waves and particles and, in 1927, experiments confirmed the de Broglie hypothesis that matter (like electrons) also behaved like waves. He conceived the philosophical principle of complementarity: that items could have apparently mutually exclusive properties, such as being a wave or a stream of particles, depending on the experimental framework. He felt that it was not fully understood by professional philosophers. +In February 1927, Heisenberg developed the first version of the uncertainty principle, presenting it using a thought experiment where an electron was observed through a gamma-ray microscope. Bohr was dissatisfied with Heisenberg's argument, since it required only that a measurement disturb properties that already existed, rather than the more radical idea that the electron's properties could not be discussed at all apart from the context they were measured in. In a paper presented at the Como Conference in September 1927, Bohr emphasised that Heisenberg's uncertainty relations could be derived from classical considerations about the resolving power of optical instruments. Understanding the true meaning of complementarity would, Bohr believed, require "closer investigation". Einstein preferred the determinism of classical physics over the probabilistic new quantum physics to which he himself had contributed. Philosophical issues that arose from the novel aspects of quantum mechanics became widely celebrated subjects of discussion. Einstein and Bohr had good-natured arguments over such issues throughout their lives. +In 1914, Carl Jacobsen, the heir to Carlsberg breweries, bequeathed his mansion (the Carlsberg Honorary Residence, currently known as Carlsberg Academy) to be used for life by the Dane who had made the most prominent contribution to science, literature or the arts, as an honorary residence (Danish: Æresbolig). Harald Høffding had been the first occupant, and upon his death in July 1931, the Royal Danish Academy of Sciences and Letters gave Bohr occupancy. He and his family moved there in 1932. He was elected president of the Academy on 17 March 1939. +By 1929, the phenomenon of beta decay prompted Bohr to again suggest that the law of conservation of energy be abandoned, but Wolfgang Pauli's hypothetical neutrino and the subsequent 1932 discovery of the neutron provided another explanation. This prompted Bohr to create a new theory of the compound nucleus in 1936, which explained how neutrons could be captured by the nucleus. In this model, the nucleus could be deformed like a drop of liquid. He worked on this with a new collaborator, the Danish physicist Fritz Kalckar, who died suddenly in 1938. +The discovery of nuclear fission by Otto Hahn in December 1938 (and its theoretical explanation by Lise Meitner) generated intense interest among physicists. Bohr brought the news to the United States where he opened the fifth Washington Conference on Theoretical Physics with Fermi on 26 January 1939. When Bohr told George Placzek that this resolved all the mysteries of transuranic elements, Placzek told him that one remained: the neutron capture energies of uranium did not match those of its decay. Bohr thought about it for a few minutes and then announced to Placzek, Léon Rosenfeld and John Wheeler that "I have understood everything." Based on his liquid drop model of the nucleus, Bohr concluded that it was the uranium-235 isotope and not the more abundant uranium-238 that was primarily responsible for fission with thermal neutrons. In April 1940, John R. Dunning demonstrated that Bohr was correct. In the meantime, Bohr and Wheeler developed a theoretical treatment, which they published in a September 1939 paper on "The Mechanism of Nuclear Fission". +Heisenberg said of Bohr that he was "primarily a philosopher, not a physicist". Bohr read the 19th-century Danish Christian existentialist philosopher Søren Kierkegaard. Richard Rhodes argued in The Making of the Atomic Bomb that Bohr was influenced by Kierkegaard through Høffding. In 1909, Bohr sent his brother Kierkegaard's Stages on Life's Way as a birthday gift. In the enclosed letter, Bohr wrote, "It is the only thing I have to send home; but I do not believe that it would be very easy to find anything better ... I even think it is one of the most delightful things I have ever read." Bohr enjoyed Kierkegaard's language and literary style, but mentioned that he had some disagreement with Kierkegaard's philosophy. Some of Bohr's biographers suggested that this disagreement stemmed from Kierkegaard's advocacy of Christianity, while Bohr was an atheist. +There has been some dispute over the extent to which Kierkegaard influenced Bohr's philosophy and science. David Favrholdt argued that Kierkegaard had minimal influence over Bohr's work, taking Bohr's statement about disagreeing with Kierkegaard at face value, while Jan Faye argued that one can disagree with the content of a theory while accepting its general premises and structure. +Bohr sat on the Board of Editors of the book series World Perspectives which published a variety of books on philosophy. +There has been much subsequent debate and discussion about Bohr's views and philosophy of quantum mechanics. Regarding his ontological interpretation of the quantum world, Bohr has been seen as an anti-realist, an instrumentalist, a phenomenological realist or some other kind of realist. Furthermore, though some have seen Bohr as being a subjectivist or a positivist, most philosophers agree that this is a misunderstanding of Bohr as he never argued for verificationism or for the idea that the subject had a direct impact on the outcome of a measurement. +Bohr has often been quoted saying that there is "no quantum world" but only an "abstract quantum physical description". This was not publicly said by Bohr, but rather a private statement attributed to Bohr by Aage Petersen in a reminiscence after his death. N. David Mermin recalled Victor Weisskopf declaring that Bohr wouldn't have said anything of the sort and exclaiming, "Shame on Aage Petersen for putting those ridiculous words in Bohr's mouth!" +Numerous scholars have argued that the philosophy of Immanuel Kant had a strong influence on Bohr. Like Kant, Bohr thought distinguishing between the subject's experience and the object was an important condition for attaining knowledge. This can only be done through the use of causal and spatial-temporal concepts to describe the subject's experience. Thus, according to Jan Faye, Bohr thought that it is because of "classical" concepts like "space", "position", "time", "causation", and "momentum" that one can talk about objects and their objective existence. Bohr held that basic concepts like "time" are built in to our ordinary language and that the concepts of classical physics are merely a refinement of them. Therefore, for Bohr, classical concepts need to be used to describe experiments that deal with the quantum world. Bohr writes: +[T]he account of all evidence must be expressed in classical terms. The argument is simply that by the word 'experiment' we refer to a situation where we can tell to others what we have done and what we have learned and that, therefore, the account of the experimental arrangement and of the results of the observations must be expressed in unambiguous language with suitable application of the terminology of classical physics (APHK, p. 39). +According to Faye, there are various explanations for why Bohr believed that classical concepts were necessary for describing quantum phenomena. Faye groups explanations into five frameworks: empiricism (i.e. logical positivism); Kantianism (or Neo-Kantian models of epistemology); Pragmatism (which focus on how human beings experientially interact with atomic systems according to their needs and interests); Darwinianism (i.e. we are adapted to use classical type concepts, which Léon Rosenfeld said that we evolved to use); and Experimentalism (which focuses strictly on the function and outcome of experiments that thus must be described classically). These explanations are not mutually exclusive, and at times Bohr seems to emphasise some of these aspects while at other times he focuses on other elements. +According to Faye "Bohr thought of the atom as real. Atoms are neither heuristic nor logical constructions." However, according to Faye, he did not believe "that the quantum mechanical formalism was true in the sense that it gave us a literal ('pictorial') rather than a symbolic representation of the quantum world." Therefore, Bohr's theory of complementarity "is first and foremost a semantic and epistemological reading of quantum mechanics that carries certain ontological implications". As Faye explains, Bohr's indefinability thesis is that +[T]he truth conditions of sentences ascribing a certain kinematic or dynamic value to an atomic object are dependent on the apparatus involved, in such a way that these truth conditions have to include reference to the experimental setup as well as the actual outcome of the experiment. +Faye notes that Bohr's interpretation makes no reference to a "collapse of the wave function during measurements" (and indeed, he never mentioned this idea). Instead, Bohr "accepted the Born statistical interpretation because he believed that the ψ-function has only a symbolic meaning and does not represent anything real". Since for Bohr, the ψ-function is not a literal pictorial representation of reality, there can be no real collapse of the wavefunction. +A much debated point in recent literature is what Bohr believed about atoms and their reality and whether they are something else than what they seem to be. Some like Henry Folse argue that Bohr saw a distinction between observed phenomena and a transcendental reality. Jan Faye disagrees with this position and holds that for Bohr, the quantum formalism and complementarity was the only thing we could say about the quantum world and that "there is no further evidence in Bohr's writings indicating that Bohr would attribute intrinsic and measurement-independent state properties to atomic objects [...] in addition to the classical ones being manifested in measurement." +The rise of Nazism in Germany prompted many scholars to flee their countries, either because they were Jewish or because they were political opponents of the Nazi regime. In 1933, the Rockefeller Foundation created a fund to help support refugee academics, and Bohr discussed this programme with the President of the Rockefeller Foundation, Max Mason, in May 1933 during a visit to the United States. Bohr offered the refugees temporary jobs at the institute, provided them with financial support, arranged for them to be awarded fellowships from the Rockefeller Foundation, and ultimately found them places at institutions around the world. Those that he helped included Guido Beck, Felix Bloch, James Franck, George de Hevesy, Otto Frisch, Hilde Levi, Lise Meitner, George Placzek, Eugene Rabinowitch, Stefan Rozental, Erich Ernst Schneider, Edward Teller, Arthur von Hippel and Victor Weisskopf. +In April 1940, early in the Second World War, Nazi Germany invaded and occupied Denmark. To prevent the Germans from discovering Max von Laue's and James Franck's gold Nobel medals, Bohr had de Hevesy dissolve them in aqua regia. In this form, they were stored on a shelf at the Institute until after the war, when the gold was precipitated and the medals re-struck by the Nobel Foundation. Bohr's own medal had been donated to an auction to the Finnish Relief Fund, and was auctioned off in March 1940, along with the medal of August Krogh. The buyer later donated the two medals to the Danish Historical Museum in Frederiksborg Castle, where they are still kept, although Bohr's medal temporarily went to space with Andreas Mogensen on ISS Expedition 70 in 2023–2024. +Bohr kept the Institute running, but all the foreign scholars departed. +Bohr was aware of the possibility of using uranium-235 to construct an atomic bomb, referring to it in lectures in Britain and Denmark shortly before and after the war started, but he did not believe that it was technically feasible to extract a sufficient quantity of uranium-235. In September 1941, Heisenberg, who had become head of the German nuclear energy project, visited Bohr in Copenhagen. During this meeting the two men took a private moment outside, the content of which has caused much speculation, as both gave differing accounts. According to Heisenberg, he began to address nuclear energy, morality and the war, to which Bohr seems to have reacted by terminating the conversation abruptly while not giving Heisenberg hints about his own opinions. Ivan Supek, one of Heisenberg's students and friends, claimed that the main subject of the meeting was Carl Friedrich von Weizsäcker, who had proposed trying to persuade Bohr to mediate peace between Britain and Germany. +In 1957, Heisenberg wrote to Robert Jungk, who was then working on the book Brighter than a Thousand Suns: A Personal History of the Atomic Scientists. Heisenberg explained that he had visited Copenhagen to communicate to Bohr the views of several German scientists, that production of a nuclear weapon was possible with great efforts, and this raised enormous responsibilities on the world's scientists on both sides. When Bohr saw Jungk's depiction in the Danish translation of the book, he drafted (but never sent) a letter to Heisenberg, stating that he deeply disagreed with Heisenberg's account of the meeting, that he recalled Heisenberg's visit as being to encourage cooperation with the inevitably victorious Nazis and that he was shocked that Germany was pursuing nuclear weapons under Heisenberg's leadership. +Michael Frayn's 1998 play Copenhagen explores what might have happened at the 1941 meeting between Heisenberg and Bohr. A television film version of the play by the BBC was first screened on 26 September 2002, with Stephen Rea as Bohr. With the subsequent release of Bohr's letters, the play has been criticised by historians as being a "grotesque oversimplification and perversion of the actual moral balance" due to adopting a pro-Heisenberg perspective. +The same meeting had previously been dramatised by the BBC's Horizon science documentary series in 1992, with Anthony Bate as Bohr, and Philip Anthony as Heisenberg. The meeting is also dramatised in the Norwegian/Danish/British miniseries The Heavy Water War. +In September 1943, word reached Bohr and his brother Harald that the Nazis considered their family to be Jewish, since their mother was Jewish, and that they were therefore in danger of being arrested. The Danish resistance helped Bohr and his wife escape by sea to Sweden on 29 September. The next day, Bohr persuaded King Gustaf V of Sweden to make public Sweden's willingness to provide asylum to Jewish refugees. On 2 October 1943, Swedish radio broadcast that Sweden was ready to offer asylum, and the mass rescue of the Danish Jews by their countrymen followed swiftly thereafter. Some historians claim that Bohr's actions led directly to the mass rescue, while others say that, though Bohr did all that he could for his countrymen, his actions were not a decisive influence on the wider events. Eventually, over 7,000 Danish Jews escaped to Sweden. +When the news of Bohr's escape reached Britain, Lord Cherwell sent a telegram to Bohr asking him to come to Britain. Bohr arrived in Scotland on 6 October in a de Havilland Mosquito operated by the British Overseas Airways Corporation (BOAC). The Mosquitos were unarmed high-speed bomber aircraft that had been converted to carry small, valuable cargoes or important passengers. By flying at high speed and high altitude, they could cross German-occupied Norway and avoid German fighters. Bohr, equipped with parachute, flying suit and oxygen mask, spent the three-hour flight lying on a mattress in the aircraft's bomb bay. During the flight, Bohr did not wear his flying helmet as it was too small, and consequently did not hear the pilot's intercom instruction to turn on his oxygen supply when the aircraft climbed to high altitude to overfly Norway. He passed out from oxygen starvation and only revived when the aircraft descended to lower altitude over the North Sea. Bohr's son Aage followed his father to Britain on another flight a week later, and became his personal assistant. +Bohr was warmly received by James Chadwick and Sir John Anderson, but for security reasons Bohr was kept out of sight. He was given an apartment at St James's Palace and an office with the British Tube Alloys nuclear weapons development team. Bohr was astonished at the amount of progress that had been made. Chadwick arranged for Bohr to visit the United States as a Tube Alloys consultant, with Aage as his assistant. On 8 December 1943, Bohr arrived in Washington, D.C., where he met with the director of the Manhattan Project, Brigadier General Leslie R. Groves Jr. He visited Einstein and Pauli at the Institute for Advanced Study in Princeton, New Jersey, and went to Los Alamos in New Mexico, where the nuclear weapons were being designed. For security reasons, he went under the name of "Nicholas Baker" in the United States, while Aage became "James Baker". In May 1944 the Danish resistance newspaper De frie Danske reported that they had learned that 'the famous son of Denmark Professor Niels Bohr' in October the previous year had fled his country via Sweden to London and from there travelled to Moscow from where he could be assumed to support the war effort. +Bohr did not remain at Los Alamos, but paid a series of extended visits over the course of the next two years. Robert Oppenheimer credited Bohr with acting "as a scientific father figure to the younger men", most notably Richard Feynman. Bohr is quoted as saying, "They didn't need my help in making the atom bomb." Oppenheimer gave Bohr credit for an important contribution to the work on modulated neutron initiators. "This device remained a stubborn puzzle", Oppenheimer noted, "but in early February 1945 Niels Bohr clarified what had to be done". +Bohr recognised early that nuclear weapons would change international relations. In April 1944, he received a letter from Peter Kapitza, written some months before when Bohr was in Sweden, inviting him to come to the Soviet Union. The letter convinced Bohr that the Soviets were aware of the Anglo-American project, and would strive to catch up. He sent Kapitza a non-committal response, which he showed to the authorities in Britain before posting. Bohr met Churchill on 16 May 1944, but found that "we did not speak the same language". Churchill disagreed with the idea of openness towards the Russians to the point that he wrote in a letter: "It seems to me Bohr ought to be confined or at any rate made to see that he is very near the edge of mortal crimes." +Oppenheimer suggested that Bohr visit President Franklin D. Roosevelt to convince him that the Manhattan Project should be shared with the Soviets in the hope of speeding up its results. Bohr's friend, Supreme Court Justice Felix Frankfurter, informed President Roosevelt about Bohr's opinions, and a meeting between them took place on 26 August 1944. Roosevelt suggested that Bohr return to the United Kingdom to try to win British approval. When Churchill and Roosevelt met at Hyde Park on 19 September 1944, they rejected the idea of informing the world about the project, and the aide-mémoire of their conversation contained a rider that "enquiries should be made regarding the activities of Professor Bohr and steps taken to ensure that he is responsible for no leakage of information, particularly to the Russians". +In June 1950, Bohr addressed an "Open Letter" to the United Nations calling for international cooperation on nuclear energy. In the 1950s, after the Soviet Union's first nuclear weapon test in 1949, the International Atomic Energy Agency was created along the lines of Bohr's suggestion. In 1957, he received the first ever Atoms for Peace Award. +Following the ending of the war, Bohr returned to Copenhagen on 25 August 1945, and was re-elected President of the Royal Danish Academy of Arts and Sciences on 21 September. At a memorial meeting of the Academy on 17 October 1947 for King Christian X, who had died in April, the new king, Frederik IX, announced that he was conferring the Order of the Elephant on Bohr. This award was normally awarded only to royalty and heads of state, but the king said that it honoured not just Bohr personally, but Danish science. Bohr designed his own coat of arms, which featured a taijitu (symbol of yin and yang) and a motto in Latin: contraria sunt complementa, "opposites are complementary". +The Second World War demonstrated that science, and physics in particular, now required considerable financial and material resources. To avoid a brain drain to the United States, twelve European countries banded together to create CERN, a research organisation along the lines of the national laboratories in the United States, designed to undertake Big Science projects beyond the resources of any one of them alone. Questions soon arose regarding the best location for the facilities. Bohr and Kramers felt that the Institute in Copenhagen would be the ideal site. Pierre Auger, who organised the preliminary discussions, disagreed; he felt that both Bohr and his Institute were past their prime, and that Bohr's presence would overshadow others. After a long debate, Bohr pledged his support to CERN in February 1952, and Geneva was chosen as the site in October. The CERN Theory Group was based in Copenhagen until their new accommodation in Geneva was ready in 1957. Victor Weisskopf, who later became the Director General of CERN, summed up Bohr's role, saying that "there were other personalities who started and conceived the idea of CERN. The enthusiasm and ideas of the other people would not have been enough, however, if a man of his stature had not supported it." +Meanwhile, Scandinavian countries formed the Nordic Institute for Theoretical Physics in 1957, with Bohr as its chairman. He was also involved with the founding of the Research Establishment Risø of the Danish Atomic Energy Commission, and served as its first chairman from February 1956. +Bohr died of heart failure on 18 November 1962 at his home in Carlsberg, Copenhagen. He was cremated, and his ashes were buried in the family plot in the Assistens Cemetery in the Nørrebro section of Copenhagen, along with those of his parents, his brother Harald, and his son Christian. Years later, his wife's ashes were also interred there. On 7 October 1965, on what would have been his 80th birthday, the Institute for Theoretical Physics at the University of Copenhagen was officially renamed to what it had been called unofficially for many years: the Niels Bohr Institute. +In 1910, Bohr met Margrethe Nørlund, the sister of mathematician Niels Erik Nørlund. Bohr resigned his membership in the Church of Denmark on 16 April 1912, and he and Margrethe were married in a civil ceremony at the town hall in Slagelse on 1 August. Years later, his brother, Harald, similarly left the church before getting married. Bohr and Margrethe had six sons. The eldest, Christian, died in a boating accident in 1934. Another son, Harald, was severely mentally disabled, and was placed in an institution away from his family's home at the age of 4 and died of childhood meningitis six years later. Aage Bohr became a successful physicist, and in 1975 was awarded the Nobel Prize in Physics, like his father. A son of Aage, Vilhelm A. Bohr, is a scientist affiliated with the University of Copenhagen and the National Institute on Aging in the U.S. Hans became a physician; Erik, a chemical engineer; and Ernest, a lawyer. Like his uncle Harald, Ernest Bohr became an Olympic athlete, playing field hockey for Denmark at the 1948 Summer Olympics in London. +The Bohr model's semicentennial was commemorated in Denmark on 21 November 1963 with a postage stamp depicting Bohr, the hydrogen atom and the formula for the difference of any two hydrogen energy levels: hν=ϵ2−ϵ1. Several other countries have also issued postage stamps depicting Bohr. In 1997, the Danish National Bank began circulating the 500-krone banknote with the portrait of Bohr smoking a pipe. On 7 October 2012, Bohr's birthday was celebrated in a Google Doodle depicting the Bohr model of the hydrogen atom. An asteroid, 3948 Bohr, was named after him, as was the Bohr lunar crater, and bohrium, the chemical element with atomic number 107, in acknowledgement of his work on the structure of atoms. +Einstein–Podolsky–Rosen paradox – Historical critique of quantum mechanics +Niels Bohr Archive +Author profile in the database zbMATH +Works by Niels Bohr at Project Gutenberg +Niels Bohr at IMDb +Newspaper clippings about Niels Bohr in the 20th Century Press Archives of the ZBW +Niels Bohr on Nobelprize.org including the Nobel Lecture, 11 December 1922 The Structure of the Atom +Oral history interview transcript for Niels Bohr on 31 October 1962, American Institute of Physics, Niels Bohr Library & Archives – interviews conducted by Thomas S. Kuhn, Leon Rosenfeld, Erik Rudinger, and Aage Petersen +1 November 1962 +7 November 1962 +14 November 1962 +17 November 1962 +"The Bohr-Heisenberg meeting in September 1941". American Institute of Physics. Archived from the original on 4 July 2011. Retrieved 2 March 2013. +"Resources for Frayn's Copenhagen: Niels Bohr". Massachusetts Institute of Technology. Retrieved 9 October 2013. +"Video – Niels Bohr (1962): Atomic Physics and Human Knowledge". Lindau Nobel Laureate Meetings. Archived from the original on 21 December 2014. Retrieved 9 July 2014. +The Battle of Dyrrhachium took place on October 18, 1081 between the Byzantine Empire, led by the Emperor Alexios I Komnenos (r. 1081–1118), and the Normans of southern Italy under Robert Guiscard, Duke of Apulia and Calabria. The battle was fought outside the city of Dyrrhachium (present-day Durrës in Albania), the major Byzantine stronghold in the western Balkans, and ended in a Norman victory. +Following the Norman conquest of Byzantine Italy and Saracen Sicily, the Byzantine emperor, Michael VII Doukas (r. 1071–1078), betrothed his son to Robert Guiscard's daughter. When Michael was deposed, Robert took this as an excuse to invade the Byzantine Empire in 1081. His army laid siege to Dyrrhachium, but his fleet was defeated by the Venetians. On October 18, the Normans engaged a Byzantine army under Alexios I Komnenos outside Dyrrhachium. The battle began with the Byzantine right wing routing the Norman left wing, which broke and fled. Varangian troops joined in the pursuit of the fleeing Normans, but became separated from the main force and were massacred. Norman knights attacked the Byzantine centre and routed it, causing the bulk of the Byzantine army to rout. +After this victory, the Normans took Dyrrhachium in February 1082 and advanced inland, capturing most of Macedonia and Thessaly. Robert was then forced to leave Greece to deal with an attack on his ally, the Pope, by the Holy Roman Emperor, Henry IV (r. 1084–1105). Robert left his son Bohemond in charge of the army in Greece. Bohemond was initially successful, defeating Alexios in several battles, but was defeated by Alexios outside Larissa in 1083. Forced to retreat to Italy, Bohemond lost all the territory gained by the Normans in the campaign. The Byzantine recovery began the Komnenian restoration. +The Normans first arrived in Southern Italy in 1015 from northern France and served local Lombard lords as mercenaries against the Byzantine Empire. As they were paid with lands, soon they were powerful enough to challenge Papal authority; in 1054, they defeated the Papal States at the Battle of Civitate, forcing the Holy See to acknowledge their authority. In 1059, Pope Nicholas II made Robert Guiscard, of the Hauteville family, Duke of Apulia, Calabria, and Sicily. However, most of Apulia and Calabria were in Byzantine hands, and Sicily was in Saracen hands. +By 1071, Robert, together with his brother Roger, had taken over the last Byzantine stronghold in Italy, Bari. By the next year, they conquered all of Sicily, ending the Islamic Emirate of Sicily. In 1074, the Byzantine Emperor Michael VII sent an envoy to Robert offering the hand of his son Constantine to Robert's daughter Helena. Guiscard accepted the offer and sent his daughter to Constantinople. However, in 1078, Michael was overthrown by Nicephorus Botaneiates, an event that destroyed any chances Helena had for the throne. This gave Robert a pretext to invade the empire claiming his daughter had been mistreated; however, his intervention was delayed by a revolt in Italy. +Robert conscripted all men of a fighting age into the army, which he refitted. Meanwhile, he sent an ambassador to the Byzantine court with orders to demand proper treatment for Helena and to win over the Domestic of the Schools, Alexios. The results of these attempts remain unknown, but the ambassador was favorably impressed by Alexios and as he was returning to Italy, he heard of the latters's successful coup against Botaneiates, by which he became Alexios I Komnenos. +When the ambassador returned, he urged Robert to make peace, claiming that Alexios wanted nothing but friendship with the Normans. However, Robert had already committed himself to war; he sent his son Bohemond with an advance force towards Greece and Bohemond landed at Aulon, with Robert following shortly after. +The Norman fleet of 150 ships including 60 horse transports set off towards the Byzantine Empire at the end of May 1081. The army numbered 15,000 men, including about 1,300 Norman knights. The fleet sailed to Avalona in Byzantine territory; they were joined by several ships from Ragusa, a republic in the Balkans who were enemies of the Byzantines. Pope Gregory VII called the Croatian king Demetrius Zvonimir to help the Normans, therefore the Croats and the Normans made a anti-byzantine pact. +Robert soon left Avalona and sailed to the island of Corfu, which surrendered because of a small garrison. Having won a bridgehead and a clear path for reinforcements from Italy, he advanced on the city of Dyrrhachium, the capital and chief port of Illyria. The city was well defended on a long, narrow peninsula running parallel to the coast, but separated by marshlands. Guiscard brought his army onto the peninsula and pitched camp outside the city walls. However, as Robert's fleet sailed to Dyrrhachium, it was hit by a storm and lost several ships. +Meanwhile, when Alexios heard that the Normans were preparing to invade Byzantine territory, he sent an ambassador to the Doge of Venice, Domenico Selvo, requesting aid and offering trading rights in return. The Doge, alarmed by Norman control of the Strait of Otranto, took command of the Venetian fleet and sailed at once, surprising the Norman fleet under the command of Bohemond as night was falling. The Normans counter-attacked tenaciously, but their inexperience in naval combat betrayed them. The experienced Venetian navy attacked in a close formation known as "sea harbour" and together with their use of Greek fire "bombs", the Norman line scattered, and the Venetian fleet sailed into Dyrrhachium's harbour. +Robert was not discouraged by this naval defeat, and began his siege of Dyrrhachium. In command of the garrison at Dyrrhachium was the experienced general George Palaiologos, sent by Alexios with orders to hold out at all costs while Alexios himself mustered an army to relieve the city. +Meanwhile, a Byzantine fleet arrived and – after joining with the Venetian fleet – attacked the Norman fleet, which was again defeated. The Byzantine garrison at Dyrrhachium managed to hold out all summer, despite Robert's catapults, ballistae and siege tower. The garrison made continuous sallies from the city; on one occasion, Palaiologos fought all day with an arrowhead in his skull. Another sally succeeded in destroying Robert's siege tower. +Robert's camp was struck by disease; according to contemporary historian Anna Comnena up to 10,000 men died, including 500 knights. Even so, the situation of the Dyrrhachium garrison grew desperate because of the effects of Norman siege weapons. Alexios learned of this while he was in Salonica with his army so he advanced in full force against the Normans. According to Comnena, Alexios had about 20,000 men; historian John Haldon puts the army's size between 18,000 and 20,000 men, while John Birkenmeier estimates it between 20–25,000 men. It consisted of Thracian and Macedonian tagmata, which numbered about 5,000 men; the elite excubitors and vestiaritai units, which numbered around 1,000 men; a force of Manichaeans which comprised 2,800 men, Thessalian cavalry, Balkan conscripts, Armenian infantry and other light troops. Aside from the Byzantine regulars, Alexios was bolstered by 2,000 Turkish and 1,000 Frankish mercenaries, about 1,000 Varangians and 7,000 Turkish auxiliaries sent by the Seljuk Sultanate of Rûm. Alexios also withdrew the tagmas from Heraclea Pontica and the remaining Byzantine holdings in Asia Minor and by doing so, he effectively left them undefended. +Alexios advanced from Salonica and pitched camp on the river Charzanes near Dyrrhachium on October 15. He held a war council there and sought advice from his senior officers; among them was George Palaiologos, who had managed to sneak out of the city. A majority of the senior officers, including Palaiologos, urged caution, noting that time was with the Emperor. Alexios, however, favoured an immediate assault, hoping to catch Guiscard's army from the rear, while they were still besieging the city. Alexios moved his army to the hills opposite the city, planning to attack the Normans the next day. +Guiscard, however, had been informed of Alexios' arrival by his scouts and on the night of October 17, he moved his army from the peninsula to the mainland. Upon learning of Guiscard's move, Alexios revised his battle plan. He split his army into three divisions, with the left wing under the command of Gregory Pakourianos, the right wing under the command of Nikephoros Melissenos, and himself in command of the centre. Guiscard formed his battle line opposite Alexios's, with the right wing under the command of the Count of Giovinazzo, the left under Bohemond and Guiscard facing Alexios in the centre. +The Varangians had been ordered to march just in front of the main line with a strong division of archers a little behind them. The archers had been commanded to move in front of the Varangians and loose a volley before retreating behind them. The archers continued this tactic until the army neared contact. +As the opposing armies closed in, Guiscard sent a detachment of cavalry positioned in the centre to feint an attack on the Byzantine positions. Guiscard hoped the feint would draw up the Varangians; however, this plan failed when the cavalry was forced back by the archers. The Norman right wing suddenly charged forward to the point where the Byzantine left and centre met, directing its attack against the Varangian left flank. The Varangians stood their ground while the Byzantine left, including some of Alexios' elite troops, attacked the Normans. The Norman formation disintegrated and the routed Normans fled towards the beach. There, according to Comnena, they were rallied by Guiscard's wife, Sikelgaita, described as "like another Pallas, if not a second Athena". +In the meantime, the Byzantine right and centre had been engaging in skirmishes with the Normans opposite them. However, with the collapse of the Norman right, the knights were in danger of being outflanked. At this point, the Varangians (mainly Anglo-Saxons who had left England after the Norman Conquest) joined in the pursuit of the Norman right. With their massive battle axes, the Varangians attacked the Norman knights, who were driven away after their horses panicked. The Varangians soon became separated from the main force and exhausted so they were in no position to resist an assault. Guiscard sent a strong force of spearmen and crossbowmen against the Varangian flank and inflicted heavy casualties on them. A few Varangians, including the Varangian commander Nabites, escaped. The few remaining Varangians fled into the church of the Archangel Michael. The Normans immediately set the church on fire, and all Varangians perished in the blaze. +Meanwhile, George Palaiologos sortied out of Dyrrhachium, but failed to save the situation. Alexios's ally, Serbian King Constantine Bodin stayed aside with his army, intending to await the outcome of the battle. When the Byzantines were defeated and started to flee, Bodin retreated with his army. The Turks who had been lent to him by the Seljuk Sultan Suleyman I followed Constantine's example. +Deprived of his left wing (still in pursuit of the Norman right), Alexios was exposed in the centre. Guiscard sent his heavy cavalry against the Byzantine centre. They first routed the Byzantine skirmishers before breaking into small detachments and smashing into various points of the Byzantine line. This charge broke the Byzantine lines and caused them to rout. The imperial camp, which had been left unguarded, fell to the Normans. +Alexios and his guards resisted as long as they could before retreating. As they retreated, Alexios was separated from his guard and was attacked by Norman soldiers. While escaping, he was wounded in his forehead and lost a lot of blood, but eventually made it back to Ohrid, where he regrouped his army. +The battle was a heavy defeat for Alexios. Historian Jonathan Harris states that the defeat was "every bit as severe as that at Manzikert." He lost about 5,000 of his men, including most of the Varangians. Norman losses are unknown, but John Haldon claims they are substantial as both wings broke and fled. Historian Robert Holmes states: "The new knightly tactic of charging with the lance couched – tucked firmly under the arm to unite the impact of man and horse – proved a battle-winner." +George Palaiologos had not been able to re-enter the city after the battle and left with the main force. The defense of the citadel was left to the Venetians, while the city itself was left to the Count of the Tent mobilizing from Arbanon (Ancient Greek: ἐξ Ἀρβάνων ὁρμωμένω Κομισκόρτη). +In February 1082, Dyrrhachium fell after a Venetian or Amalfian citizen opened the gates to the Normans. The Norman army proceeded to take most of northern Greece without facing much resistance. While Guiscard was in Kastoria, messengers arrived from Italy, bearing news that Apulia, Calabria, and Campania were in revolt. He also learned that the Holy Roman Emperor, Henry IV, was at the gates of Rome and besieging Pope Gregory VII, a Norman ally. Alexios had negotiated with Henry and given him 360,000 gold pieces in return for an alliance. Henry responded by invading Italy and attacking the Pope. Guiscard rushed to Italy, leaving Bohemond in command of the army in Greece. +Alexios, desperate for money, ordered the confiscation of all the church's treasure. With this money, Alexios mustered an army near Thessalonica and went to fight Bohemond. However, Bohemond defeated Alexios in two battles: one near Arta and the other near Ioannina. This left Bohemond in control of Macedonia and nearly all of Thessaly. Bohemond advanced with his army against the city of Larissa. Meanwhile, Alexios had mustered a new army and with 7,000 Seljuk Turks sent by the Sultan, he advanced on the Normans at Larissa and defeated them. The demoralised and unpaid Norman army returned to the coast and sailed back to Italy. Meanwhile, Alexios granted the Venetians a commercial colony in Constantinople, as well as exemption from trading duties in return for their renewed aid. They responded by recapturing Dyrrhachium and Corfu and returning them to the Byzantine Empire. These victories returned the Empire to its previous status quo and marked the beginning of the Komnenian restoration. +Comnena, Anna (1996). The Alexiad. Translated by Sewter, E. R. A. London: Penguin Books. ISBN 0-14-044215-4. +John Neville Crawford (1 December 1886 – 2 May 1963) was an English first-class cricketer who played mainly for Surrey County Cricket Club and South Australia. An amateur, he played as an all-rounder. As a right-handed batsman, Crawford had a reputation for scoring quickly and hitting powerful shots. He bowled medium-paced off spin and was noted for his accuracy and his ability to make the ball turn sharply from the pitch. Unusually for a first-class cricketer, Crawford wore spectacles while playing. +Crawford established a reputation as an outstanding cricketer while still a schoolboy. He played Test cricket for England before he was 20 years old, and successfully toured Australia with the Marylebone Cricket Club (MCC) in 1907–08. He played only 12 matches for England, although critics believed he had a great future in the sport and was a potential future England captain. In two successive English seasons, he completed the double of 1,000 runs and 100 wickets in first-class games. +A dispute over the composition of Surrey's team chosen to play a high-profile game in 1909, after several professional players were omitted for disciplinary reasons, led to an increasingly bitter disagreement between Crawford and the Surrey authorities. Crawford was told he had no future with the club, and moved to Australia. There, he worked as a teacher and continued his cricket career with South Australia. This arrangement had a controversial end, when he clashed with the South Australian Cricket Association over money and moved to New Zealand to play for Otago. +That relationship also ended badly, and he left Otago before being conscripted into the New Zealand armed forces near the end of the First World War. When he was demobilised, he returned to England and made his peace with Surrey. He played a handful of games between 1919 and 1921 but faded out of first-class cricket to pursue a career in industry. In all first-class cricket, Crawford scored 9,488 runs at an average of 32.60 and took 815 wickets at an average of 20.66. Although he continued to play cricket at a lower level, the remainder of Crawford's life passed in relative obscurity. +Jack Crawford was born on 1 December 1886 at Cane Hill, Coulsdon, Surrey, the youngest of three sons of the Rev John Charles Crawford and his wife Alice; the couple also had three daughters. Crawford senior was the chaplain at the recently opened Cane Hill Asylum, in the grounds of which Crawford was born. He grew up in a cricketing environment. His father and uncle, Frank Crawford, played first-class cricket for Kent; his brothers Vivian and Reginald were also first-class cricketers. The whole family played cricket and encouraged Crawford from a young age, and from the age of eleven he regularly played with adults. +After attending Glengrove School in Eastbourne, Crawford went to St Winifred's School in Henley-on-Thames where, in his two years in the cricket team, he scored 2,093 runs and took 366 wickets. In 1902, Crawford moved to Repton School. Reaching the cricket team in his first year, he remained in the eleven until he left the school in 1905. His impact was considerable. A 1906 report in Wisden Cricketers' Almanack rated him as one of the best three schoolboy cricketers in the previous 40 years, matched only by A. G. Steel and Stanley Jackson. Cricket historians similarly praised his cricket at Repton. Benny Green notes that his prolific achievements "created ... chaos among schoolboy cricketers." Gerald Brodribb describes him as "probably the best ever" schoolboy cricketer. +By 1904, Crawford dominated the Repton team. He scored 759 runs and his 75 wickets were more than the combined total of all the other bowlers in the team. He was particularly effective in the school's most important fixtures. The report in Wisden described him as possibly the best amateur bowler in England that year: he bowled medium-paced off spin, although he varied the speed of his delivery from slow to fast. Surrey County Cricket Club took an interest in Crawford almost immediately, calling him to a trial in 1903. Following his achievements in 1904, he was invited to play for the county. The county club was in the midst of a spell of uncertainty; several men captained the team, but only for a handful of matches each. The composition of the team continually changed, and the team performed poorly, causing unrest among supporters accustomed to success. Crawford was just one of many players brought in as an experiment, albeit one of the most successful. He made his first-class debut against Kent. Taking three wickets and top-scoring in Surrey's first innings with 54, Crawford did well enough to retain his place for another seven games, and was praised in the press for his performances. Against Gloucestershire, he took seven wickets for 43 runs in the second innings, and a total of ten wickets in the match. In the season as a whole he took 44 first-class wickets at an average of 16.93 to top the county's bowling averages, and scored 229 runs at an average of 16.35. +Although hampered by injuries during the 1905 season for Repton, his last at the school, Crawford scored 766 runs with a batting average of 85. In the five matches in which he was fit to bowl he took 55 wickets at an average under 13. In the August holidays, he returned to play for Surrey. In his second game, he took seven for 90 against Yorkshire and in his third, he scored his maiden first-class century—119 not out against Derbyshire—to become the youngest centurion for the county, a record that was not broken until 2013. Later, he took eight for 24 against Northamptonshire and scored 142 not out against Leicestershire. At the end of the season, he played in the Hastings Festival, appearing in several representative games for teams representing the South of England and played for the Rest of England against the County Champions Yorkshire. Crawford finished second in Surrey's batting averages for 1905; in all first-class games he scored 543 runs at an average of 33.93 and took 47 wickets at an average of 18.46. As the season ended, he was invited by the Marylebone Cricket Club (MCC) to join their tour of South Africa that winter. +Having finished his school career, Crawford joined the first MCC tour of South Africa in the 1905–06 season. The team was not particularly strong—before the tour, critics judged it to possess the equivalent strength of a moderate county team. The MCC was criticised by the press both in England and in South Africa for omitting many of the strongest players. As the youngest member of the team, Crawford was given the traditional role of writing press reports to be sent back home during the tour. When the team played in Worcester during the tour, the local press carried a report that Crawford planned to remain in South Africa. The 1906 Wisden carried the same story; in a comment on his success in 1905, it suggested that it was uncertain for how long he would play, and questioned whether, if he played Test cricket, he would do so for England or South Africa. The cricket historian Nigel Hart queries why Wisden mentioned South Africa; Crawford's only known connection came through his uncle, who had left South Africa five years before. +In all first-class matches on the tour, Crawford scored 531 runs at an average of 31.23, coming third in the tour batting averages, and took 34 wickets at an average of 18.44, placing him fifth among the regular bowlers in the team. In the early tour matches, Crawford recorded two five-wicket hauls and scored 98. He made his Test match debut for England against South Africa in the first Test at the age of 19 years and 32 days to become England's youngest Test cricketer, a record he held until Brian Close made his debut in 1949. Batting at number six, Crawford scored 44 runs in the first innings and 43 in the second; he took a wicket with his first delivery, bowling Bert Vogler. He was wicketless in the second innings, and bowled inaccurately as the home team recorded a one-wicket win. Wisden noted that Crawford batted well in both innings. After a month of cricket in between the Tests, during which Crawford scored 212 in a minor game, the English team were suffering from fatigue when the remaining matches were played. In the second Test, Crawford was promoted to open the batting, a position he retained for most of the series. In the final match, he scored 74, his first Test fifty and the highest score of his Test career. His best bowling also came in the final match, where he took three for 69. +In the Test series, Crawford scored 281 runs at an average of 31.22 and took nine wickets at 35.77. England lost the series 4–1, finding it difficult to bat against the South African googly bowlers. Wisden's tour report described Crawford's batting as one of the few positives for his team. Writing home during the tour, the MCC captain Pelham Warner frequently praised Crawford and the extent of his ability at a young age. The English press even parodied what Crawford's biographer, Michael Burns, describes as Warner's "near-obsession" with Crawford. Other critics believed that Crawford should have been more successful as a bowler but was distracted by the matting surfaces used in South Africa; because he was able to make the ball bounce and turn extravagantly from the surface, he tried to spin the ball too much and lost accuracy as a result. +Returning to England for the 1906 season, Crawford completed the double of 1,000 runs and 100 wickets in first-class cricket, the youngest player at the time to accomplish this feat. Against Gloucestershire, he scored 148 and then took seven for 85 and four for 63 with the ball; he also took ten wickets in his next game. In total, he compiled 1,174 runs at an average of 30.10, and took 118 wickets at an average of 20.28. These performances earned him selection for the representative Gentlemen versus Players matches at both Lord's and The Oval, in which he played for the amateur Gentlemen. At the end of the season, he was chosen as one of Wisden's Cricketers of the Year. The citation said: "Few cricketers have won equal fame at so early an age", and noted the unusual ease with which he had moved into first-class cricket. Crawford's powerful batting and straight hitting were praised, as was his accurate bowling. The report also stated: "That he should have done all this is the more astonishing from the fact that he invariably plays in glasses. No one handicapped in this way has ever been so consistently successful both as batsman and bowler." +By this stage, Crawford had a reputation as a batsman who favoured aggressive, powerful shots and his performances drew in crowds. In 1907, in the space of a few days, he hit deliveries in two different matches through the windows of both the home and the visitors' dressing rooms at The Oval. Towards the end of the season, he scored 103 runs in 90 minutes against Kent, his only century that year, hitting several deliveries into the crowd; The Times described the ball in this innings as "soaring away like a bird". Crawford's overall batting record was similar to the previous season: he scored 1,158 runs at an average of 30.47. With the ball, he took 124 wickets at 16.95, completing his second double. Once again, he was selected in both Gentlemen v Players matches, taking six for 54 in the second, and was chosen to play two of the three Test matches against South Africa. These were his only Test matches in England; he scored 26 runs in three innings and was wicketless after bowling 29 overs. He missed England's victory in the second match, the only match in the series not to be a draw, but Wisden's correspondent believed that the pitch conditions for that game made it a mistake to leave Crawford out. Despite his lack of success in the Tests, Crawford was invited to tour Australia that winter even before the South African series was complete; he suggested to the press that he might remain in Australia after the tour. +The MCC team which toured Australia in 1907–08, like that which toured South Africa, faced questions about its strength. Because at least four first-choice players chose not to tour, and another was not selected despite strong claims, the team was less representative than usual for an Ashes series. The tourists lost the Tests 4–1, although Wisden suggested that the series was more competitive than suggested by the results. +Crawford began the tour well, taking seven wickets in the opening first-class match against Western Australia. In the second game, he scored 114 runs in 58 minutes against South Australia, the fastest first-class century scored in Australia until then, followed by bowling figures of five for 40 in the second innings. Crawford contributed little more with bat or ball in the lead-up to the Test series; in England, the cricketer Gilbert Jessop wrote an article suggesting that Crawford had become an "unsound" batsman. He scarcely bowled in the first Test, won by Australia, and both the English and the Australian press were critical of his bowling. Frederick Fane, the acting-captain of England, seemed to share this view in the second Test, withholding Crawford from the attack for some time. However, Crawford took five for 79 in the first innings helped to restrict Australia on a good batting pitch and took eight wickets in the match as England levelled the series. +Australia won the remaining games to win the series. In the third match, Crawford scored his only half-century of the series, hitting 62 out of England's 363 to help his team build up a first innings lead. However, Australia scored 506 at the second attempt, during which Crawford bowled nearly 46 overs to take three for 113. The series was settled in the fourth Test; Australia recorded a big victory after rain affected the pitch and created difficult batting conditions during England's first innings. On the first day, Crawford took five for 48, his best Test figures, as Australia were bowled out for 214 on a very good pitch. Wisden noted Crawford "[mixed] up his pace with remarkable skill". By this stage of the tour, the press looked on him much more favourably, and he received praise for his performances. However, his heavy workload with the ball affected his health; he lost a stone in weight during the tour, and before the final Test a Melbourne doctor diagnosed that he had "strained the right side of his heart" and advised that he see a specialist. Crawford played in that game, won by Australia, despite his ill health, with several other players unfit, and the tour manager later wrote that he looked poorly throughout and should not have played. Nevertheless, he bowled 54 overs and took eight wickets in the match. +Wisden considered Crawford's bowling to be one of the most successful features of the tour and praised his ability to spin the ball, reporting: "It was said of Crawford that even on the most perfect wickets he could at times make the ball break back." He led the English Test bowling averages with 30 wickets at an average of 24.73. His batting was less productive than expected; in five Tests, he scored 162 runs at an average of 18.00, batting usually at number eight. In all first-class matches on the tour, he scored 610 runs at 26.52 and took 66 wickets at 25.19. Crawford's contributions enhanced his growing reputation and critics expected his cricket to go from strength to strength. Australian batsman Clem Hill commented: "There are grand cricketers in this game, and then there is Jack Crawford." The final game in the series turned out to be Crawford's last Test. In 12 Test matches, he had scored 469 runs at an average of 22.33 and took 39 wickets at 29.48. +As the tour neared its completion, the Australian press reported that Crawford planned to remain in Australia, that he had attracted the attention of several "society" women, and that he had become engaged. Even so, he returned to England when the tour ended. In May 1909, he published a book about the tour, JN Crawford's Trip to "Kangaroo" land. +During the 1908 season, Crawford narrowly failed to complete his third double. He scored 1,371 runs at an average of 37.05 and took 98 wickets at 21.48. His season began with a probable dispute over the Surrey captaincy. H. D. G. Leveson Gower had been appointed captain for 1908 but the combination of an injury and his marriage left him unavailable for four matches near the beginning of the season. Crawford also missed the beginning of the season; a newspaper article by Albert Trott, a former Australian Test all-rounder then playing for Middlesex, suggested that Crawford withdrew from the team because he was not appointed captain in Leveson Gower's absence. Instead Harry Bush, who had not played first-class cricket for five years, led the team. Trott was sympathetic to Crawford, stating that the Surrey committee went out of their way "to inflict a most undeserved slight" on Crawford. +In the remainder of the season, Crawford scored 232 against Somerset, his highest first-class score, as well as centuries against Derbyshire and Hampshire. The increased strength and variety of Surrey's bowling attack restricted his opportunities; he was used less as a strike bowler and bowled fewer overs than in the previous season. Contemporary reports suggested that his performances tailed off as the season progressed, possibly through tiredness from having played too much cricket, and that he became a less accurate bowler through trying too hard to spin the ball. The Surrey captaincy remained unsettled; Crawford led the team on several occasions, including Surrey's final game of the season, when his brother Vivian captained Leicestershire, their opponents. +Crawford, who played as an amateur but was not independently wealthy, received an increased expenses allowance during 1908, but other financial dealings with the committee were less successful: they paid his laundry bills in 1905, but refused to do so afterwards; additionally, they refused his request in 1907 to have part of his expenses paid for matches that he missed, as happened with the wages of professionals. His financial problems were compounded by his inability to find a suitable job, and the Surrey committee threatened that he "ought not to be played on the same terms" unless he "[commenced] to earn his livelihood". +Crawford's performances in the 1909 season were less effective than in previous years. He began well, but was later hampered by an injury which prevented him from bowling. His batting average also fell. Surrey experienced disciplinary problems with several of their professional players during the season. The club president, Lord Alverstone, favoured amateurs, arguing that a losing amateur team was preferable to a professional team that won; several professionals were left out in favour of amateurs. Apart from adversely affecting some players' careers, this preference had destabilised the team over several seasons. Crawford disagreed with Alverstone, but nevertheless frequently assumed the leadership in the absence of the regular captain Leveson Gower for much of the season. He captained Surrey to a win over the touring Australian team early in 1909 after a strong performance by the professional bowler Tom Rushby. However, his captaincy was strongly criticised in the press, particularly over his management of the bowling. Around this time, Crawford's form declined. With the bat, he was often dismissed attempting aggressive shots, and he lost his effectiveness as a bowler. As a result, he was not chosen for the Gentlemen and missed selection in the Ashes series. The press continued to criticise his captaincy, despite Surrey's good results under his leadership. Burns notes: "For a young man with no experience of prolonged failure on the cricket field, this was almost certainly a traumatic time." +Surrey had other problems at this time. When the team arrived in Chesterfield to play Derbyshire, eight of the professionals were arrested following an incident in the street; the matter was cleared up and the press suggested that the police had been over-eager. The county subsequently defeated Derbyshire by an innings in early July; Rushby and his fellow professionals Walter Lees and W. C. Smith were very successful with the ball. Shortly after this, Rushby and Lees were involved in an incident of some kind—the exact details are unknown. Leveson Gower suspended them, making them unavailable for Surrey's second match against the Australians. Crawford was asked to lead the team for that game, but with Rushby and Lees omitted, another bowler missing and himself unable to bowl owing to a shoulder injury, he considered the attack too weak. Consequently, he refused the captaincy, and apologised to the Australians for the selection of what he thought a substandard team. In Crawford's absence, Surrey were led by the professional Tom Hayward; the match, affected by rain, was drawn. +The cricket press noticed the absence of Crawford and the professionals from Surrey's team over the following games, and noted that Crawford was now playing club cricket. Meanwhile, Alverstone wrote to Crawford that the Surrey committee fully supported Leveson Gower's decision to omit Rushby and Lees, and told him that Surrey would not select him again unless he apologised to Leveson Gower. Crawford refused, writing to Alverstone: "I do not know who was responsible for the selection of the second eleven sort of team furnished up for such an important match ... There seems to be some impression amongst a few of the Surrey committee that I am some young professional instead of being a young fellow who has had an experience of cricket that has seldom fallen to the lot of anyone, and my request for an alteration of the team should have had some weight". In his History of Cricket, Benny Green describes the letter as evidence of a "literate and quietly self-confident young man who will not easily be manipulated." +Alverstone replied that he "regretted" Crawford's views; the committee respected Crawford as a "brilliant amateur with great experience", but Crawford had not supported his captain—a prime duty for an amateur. Crawford replied that he had been unaware of the reason for the suspensions of Rushby and Lees—which Alverstone and the committee disputed—and would have supported Leveson Gower, but suggested it was unfair to ask him to captain a weak bowling team and then demand he apologise for preferring not to. He also observed that the players omitted from the team against the Australians had since been restored to the county team, making their exclusion harder to understand in the first place. +At this point Leveson Gower withdrew his invitation for Crawford to appear at the Scarborough Festival; Leveson Gower also prevented Crawford's selection for the MCC winter tour of South Africa. Crawford's father wrote to the Surrey committee in support of his son. By the beginning of August 1909, two weeks after the match against the Australians, the Surrey committee severed their connection with Crawford; Leveson Gower felt that he had shown the "deepest ingratitude". Wisden reported: "The committee were much incensed and passed a resolution that Crawford be not again asked to play for the county." Informed of his expulsion by letter, Crawford replied to the committee a final time: "I fail to see why I should practically be branded as a criminal because, as acting captain, I declined the responsibility of skippering a team which did not include three essential players, an independence which I trust will remain in spite of the awful example made of me to every amateur in the United Kingdom." +The Surrey committee initially attempted to keep the dispute private, but Crawford sent copies of the letters to the newspapers, explaining in a letter that he wished to end speculation about his absence from Surrey. Burns notes that this "[generated] a strong response from the public, mainly unsympathetic to the young amateur" in the letters pages of many newspapers. Many commentators felt that the argument could have been resolved easily had either team made concessions. Green comments that the committee probably either expected Crawford to back down, or were happy to sacrifice him to establish their authority. Another of the players involved, Rushby, left Surrey at the end of the season to play league cricket, but later returned to the team. Crawford's father made a further attempt to end the dispute between Surrey and his son in 1910, asking the committee to reverse their decision. Wisden reported that Alverstone declined on the grounds that it would suggest a lack of confidence in the committee, but that if Crawford "came forward in a sportsmanlike way [Alverstone] would be proud to give his personal support to the step proposed. This of course meant that an apology was expected." +In his History of Cricket, Green contended that the "Surrey committee must be held accountable for a degree of idiocy rarely met with even in the realms of cricket administration" for the way they dealt with Crawford, "one of the world's most prodigious all-rounders." Hart observes that Crawford was challenging figures high up in the cricket establishment on the Surrey committee; he also suggests that Crawford's actions in his later career, and the complaints made against him by other cricket authorities, reveal a stubbornness in his character and suggest he was not merely a victim of the Surrey committee. The Times speculated that factors other than the dispute may have contributed to Surrey's decision. Burns believes that Crawford was feeling pressure from several directions: his inability to secure a job, possible frustration at the controlling influence of his father, a desire to be independent, his poor run of form and criticism of his captaincy. Even though, according to Burns, it was an "unwinnable battle", he suggests simply: "Young Jack was in the mood for a fight." +At the end of the season, Crawford played his last matches in England for 10 years, appearing for an "England XI" against the Australians and for the Gentlemen of the South. He had previously discussed emigrating to Australia with the Australian Test player Victor Trumper, who had asked Clem Hill to investigate a potential teaching post at St Peter's College, Adelaide. Originally, assuming that he would be part of the MCC tour to South Africa, he planned to take up his position in March 1910. When the dispute arose, Crawford sent a telegram to inform the college he would take up the post of "Resident Master" early in 1910, for a salary of £160 per year. Burns suggests that this knowledge that he had the offer of work may have prompted him to take a stand against the Surrey committee. In October 1909 he was offered the position of "Ordinary Master" at the college, and £50 towards his travel expenses if he departed immediately. He left England by boat two days later, seen off by his family. No cricket figures saw his departure, but there was considerable interest from the press. +At St Peter's School, Crawford combined his teaching role with supervision of sports, including acting as coach for the cricket team. On several occasions, he had to request leave from the headmaster to play first-class cricket. After arriving in Australia in December 1909, he was playing district cricket for East Torrens within a week, and played in South Australia's last three matches in the Sheffield Shield competition; the team went on to win the trophy for the first time since 1893–94. Crawford played a large part in this success. There was some controversy over his eligibility to play; New South Wales initially protested, but there was a precedent for ignoring the requirement of a three-month qualification period, and the New South Wales Cricket Association (NSWCA) Executive Committee accepted this in the case of Crawford, to the displeasure of the full NSWCA. Over four seasons, Crawford played 22 matches for South Australia, scoring 1,512 runs at an average of 40.86 and taking 120 wickets at 23.86. Green suggests that such a performance would have earned him Test selection if he had been Australian, and Wisden noted that his record in Australia was impressive. +In the 1910–11 Australian season, Crawford scored a half-century in every first-class game he played but was less successful with the ball. His success against the touring South Africans, perhaps helped by his experience facing their googly bowlers on previous occasions, brought him close to selection for the Australian Test team that year. The Australian Board of Control eventually decided that, as he had already appeared for England, they could not choose him; instead they picked Charlie Kelleway as the all-rounder. Crawford resigned from his position as St Peter's in March 1911, suggesting to the press that he might move into farming. He may have considered leaving South Australia, but the Cricket Association appointed him as a clerk at the Adelaide Oval, a position which earned £200 per annum, with additional responsibility for coaching and youth scouting. Meanwhile, in England, after unsuccessful attempts by his father to change the mind of the Surrey committee, including a failed bid to secure his own election to the committee, Crawford sent a written apology to Surrey at some time in 1910. Consequently, the committee passed a motion in March 1911 which ended the ban on Crawford playing for the county. +Press rumours over the following 12 months cast doubt on Crawford's future, suggestions including a return to England for the 1912 season. Instead, he remained with South Australia in the 1911–12 season. Although less successful generally, he played for a non-representative Australian XI against the MCC touring team which contested the Ashes that season and scored 110 in as many minutes against bowlers including Sydney Barnes, at the time regarded as the greatest bowler in the world. The following season, in which South Australia again won the Shield, Crawford took seven for 31 against Western Australia, including a hat-trick, and scored 163 in 177 minutes against Victoria before taking eight for 66 in their first innings. +In 1913, Crawford was included in an Australian team which toured North America. The team was organised by Edgar Mayne, a South Australian batsman who tried to secure backing from the Australian Board of Control to make it an official representative team, but the Board refused to do so. In all matches, Crawford scored over 1,000 runs and took over 200 wickets, and he was particularly effective as a bowler in the matches designated first-class. He later stated that this was the best tour he had been on; more press rumours suggested that he would return to England at the conclusion of the tour, but he returned to Australia with the rest of the team. +In December 1913, Crawford wrote to the South Australian Cricket Association (SACA) asking for a six-month leave of absence and a guaranteed renewal of his contract. The SACA described his letter as "arrogant" before eventually granting his request and offering him a three-year contract worth around £300 per year, including coaching fees. In effect, he wanted to double his salary or leave to seek a position in New Zealand. Describing these events, Hart comments: "Crawford's financial dealings with the SACA reveal him both as mercenary and as an awkward 'cuss'. The latter aspect of his personality needs to be taken into account lest he be considered merely an establishment victim in his parting with Surrey". On the field, Crawford was successful in 1913–14. He took 34 wickets, and against New South Wales he hit 91 in 89 minutes before taking 10 wickets; against Victoria he took eleven wickets. This latter match was his last for South Australia. Later in 1914, Crawford toured New Zealand with an Australian team—assembled without the approval of the Australian Board of Control—which contained many leading players, under the captaincy of the former Canterbury batsman Arthur Sims. In first-class games on this tour Crawford took 21 wickets and scored a century, but his most notable innings came in a minor match. Against the "XV of South Canterbury", he scored 354 in five-and-a-quarter hours, striking 14 sixes and 45 fours. He and Victor Trumper shared a partnership of 298 in 69 minutes, and Monty Noble helped him to score 50 runs in 9 minutes as the Australian team scored 922 for nine. +In March 1914, the Otago Cricket Association (OCA) offered Crawford a three-year contract worth £350 per year, and a share in the management of a sports store, to play for them. Briefly returning to Adelaide after the tour, he resigned from his South Australia contract before moving to Dunedin in June 1914. The South Australian Cricket Association were widely criticised for failing to retain Crawford; in their defence, the committee publicly stated that Crawford had been offered an increased deal and had promised to turn down any offers made to him in New Zealand, but went back on his word. Later historians—Nigel Hart in his biography of Crawford, and Chris Harte in his A History of Australian Cricket (1993)—condemned Crawford as arrogant and mercenary for his behaviour towards South Australia. Harte also suggests that Crawford left Australia with many debts. Burns, however, suggests that Crawford may have wanted to maximise his earnings to support his upcoming marriage, and notes that his South Australian teammates held no grudges and even organised a farewell presentation. Once in his new position, Crawford immediately organised a Colts team which produced several future Otago players. While qualifying to play for Otago, he played club cricket in Dunedin, scoring 559 runs and taking 88 wickets. During the 1914–15 season he appeared in four first-class games for Otago, in which he scored 337 runs and took 30 wickets. He briefly returned to Australia in 1915 to marry Anita Schmidt in Melbourne in April. Schmidt—from Adelaide and described in the society press as a "beauty"—and Crawford met in 1912 when the former was 18 years old and became well known as a couple at fashionable events in Adelaide. Burns suggests that the wedding took place in Melbourne rather than Adelaide because Crawford may have been avoiding his creditors. Two days after the wedding, the couple returned to New Zealand. +Crawford's wages caused the OCA some financial difficulty and were the cause of extended negotiations in 1915. Part of the settlement involved Crawford receiving a lower salary in return for freedom to offer his services to other clubs on a freelance basis. As a consequence, he represented different teams throughout the season in local cricket. He was also paid to coach at Otago Boys' High School. There were other difficulties; the association complained about the lateness of Crawford's report on the Otago team, and were unhappy that he also coached golf. Crawford in turn told the association that several Otago players were late for practice. After further complaints about Crawford's coaching in 1916, the OCA decided to terminate his contract. Crawford initially offered to continue for less money—£245. The OCA bargained, suggesting a payment of £200, whereupon Crawford insisted that he should be paid £300. The OCA then proposed to terminate his contract immediately for a payment of £150 but he refused. He was eventually paid £200 in June to leave Otago. That November, with the First World War into its third year, New Zealand brought in conscription. Crawford was called up in July 1917, and was posted to a training camp near Wellington in late 1917. During this time, he played twice for Wellington's cricket team in first-class matches during early 1918. When given weekend passes, he visited his wife who was left alone in Dunedin. He travelled to England prior to a posting to the Western Front, although he arrived too late to join the fighting. He was part of the New Zealand Rifle Brigade but it is unclear what his rank was. The Repton School War Register states that he was a rifleman but New Zealand press accounts named him as a quartermaster sergeant. According to a 1992 article in Wisden Cricket Monthly by Jim Sullivan, Crawford was demoted while he was in the army. The reasons are unknown, but Burns speculates that his "independent and obstinate nature" made it difficult for him to accept military discipline. While Crawford was in the army, his wife moved to Adelaide. According to Anita, the marriage became unhappy in 1916 and Crawford left her in November of that year. She claimed that he had not supported her financially from that point. When the war ended, she returned to live in Australia with her parents. Her subsequent career as a dress designer and fashion buyer necessitated a move to London, where Crawford was living, in 1921 but the couple never reconciled. They were divorced, with some publicity in Australia, in 1923; in court, she gave evidence that he had an affair with a "third party". She went on to remarry and to have a successful career as a designer and orchestral conductor. +After the First World War, Crawford returned to live in England, and was demobilised from the New Zealand Army in April 1919. Looking for work, he wrote to Surrey offering to play for them if they found him some employment in turn, but the committee, while writing that they would be pleased if he played for the county again, could not "see [their] way to find him employment". Crawford may have been contemplating playing professional cricket—unheard of for a former Public School cricketer—but instead found work at Repton, his old school. It is unclear what his position at Repton was at this time; he seems to have had no official cricketing role and it is possible he worked as a teacher simply to maintain his amateur status. In any case, Crawford remained at Repton only until the end of the academic year; his short stay may have been the result of poor results by the cricket team. However, Burns suggests: "Another indication that the school was not entirely happy to be associated with the Surrey rebel, whose baggage also included an indifferent war record, was that when the Old Boys cricket team, the Repton Pilgrims, was formed in 1921, Crawford was alone among the school's former distinguished players not to be made a member. He was not invited to join the club until 1952." +Having settled his disagreement with Surrey, Crawford resumed his English first-class cricket career in 1919. After appearing for the Gentlemen against the Players, Crawford returned to play for Surrey against the Australian Imperial Forces. He scored 144 not out, which was later described by Wisden as the innings of his life. Surrey were 26 for five in reply to the tourists' innings of 436 when Crawford came in to bat. Neville Cardus reported: "[Crawford] fell upon the advancing Australian attack, and by driving seldom equalled, threw it back." He more than doubled his score after the ninth wicket had fallen, hitting 73 out of the last 80 runs scored in 35 minutes to take Surrey past the target required to avoid the follow-on. Among his other successes, he scored 92 against Yorkshire. Playing against Kent, he hit 48 not out as Jack Hobbs and he scored 96 in 32 minutes; this partnership took Surrey to victory as they chased an apparently impossible chase in the short time remaining in the match. +In total, Crawford played in eight games in 1919, scoring 488 runs and taking 20 wickets. Wisden commented that his batting was as good as it had ever been, but his bowling lacked spin and accuracy. He played only four more times in first-class cricket. Part of the explanation was that he joined Dunlop Rubber as a manager at one of its mills in Rochdale. He played as an amateur for Rochdale Cricket Club in 1920, but by 1921 he had left the club—the Manchester Guardian speculated that he had returned to Surrey. He returned to live with his family in Merton Park, and played for the local cricket club. Of his first-class matches, one was for Surrey against the Australian touring team of 1921 and the others were for teams representing the Gentlemen. In his entire first-class career, Crawford scored 9,488 runs at an average of 32.60 and took 815 wickets at 20.66. +From the mid-1920s until his retirement in 1952, Crawford worked for the importers Elders and Fyffes. He played cricket and hockey for the firm until the Second World War, but kept a far lower profile than in his earlier cricketing life. He married his second wife, Hilda May Beman, in December 1925, but he never had children. He maintained a loose connection with cricket; he appeared at a birthday dinner for Pelham Warner, at a centenary celebration for the Free Foresters Cricket Club and in a radio broadcast for Jack Hobbs's 80th birthday. Crawford had a stroke in early 1962 and remained ill for the rest of the year; he died, aged 76, in a Surrey hospital on 2 May 1963. +Crawford's obituary in The Times described him as one of the best young players to play cricket in England and said: "Although he invariably played in glasses, he was a most attractive player to watch, an aggressive hitter of the ball and a dangerous medium-paced bowler". Wisden described him as a "hard-hitting batsman", and said he played mainly from the front foot. He had an orthodox batting technique, moved his feet well to get to the ball, and played very straight. Herbie Collins, who played with Crawford in Australia, described one of his innings as "a hurricane innings, full of classical shots charged with dynamite." +As a bowler, Crawford's technique was also orthodox, although he was unusual in using his second and third fingers to spin the ball—most bowlers used their first and second. His bowling pace varied from fast to medium, and he spun the ball so much that his fingers snapped audibly as he released it. An accurate bowler, it was difficult for batsmen to score runs against him. Crawford could swing the ball away from the bat, but his most effective delivery was his off break: Clem Hill stated that Crawford could make the ball turn several inches, despite the hard pitches prevalent in Australia when he played there. John Arlott described him as "the schoolboy genius who turned on Australian pitches where no one else deviated from straight". In 1937, Herbie Collins wrote, at a time when Wally Hammond was considered to be the world's leading all-rounder, that "people who have seen both men consider [Crawford] a better all-rounder". +Commenting on the interruption of Crawford's career by his dispute with Surrey, Neville Cardus wrote: "His break with Surrey must be regarded as a sad deprivation of fame and pleasure to himself, and a grievous loss to the annals of English cricket. It is as certain as anything in a man's life can be confidently postulated, that had he continued to play in English county cricket ... he would have taken his place amongst the select company of England's captains." The Times commented: "It was one of the great disappointments of English cricket in the first quarter of this century that his outstanding promise was never fully realized." Hart observes that Crawford's on-field successes were many up to 1909 but after that, his career effectively stalled during his absence from English cricket. Hart notes Crawford's "capacity to get on the scoreboard and up establishment noses. He could turn games around and agreements over, antagonise the powerful, endear himself to the young and those young enough at heart to care to characterise great personal performances as 'heroic'". He concludes that Crawford spent the last 40 years of his life "in comparative sporting obscurity". No-one on the Surrey committee ever expressed regret at what had happened, and Leveson Gower did not mention the affair in his autobiography. +Burns, Michael (2015). A Flick of the Fingers: The Chequered Life and Career of Jack Crawford. Worthing, Sussex: Pitch Publishing Ltd. ISBN 978-1-78531-063-8. +Green, Benny, ed. (1982). Wisden Anthology 1900–1940. London: Queen Anne Press. ISBN 0-7472-0706-2. +Green, Benny (1988). A History of Cricket. London: Guild Publishing. ISBN 978-0-71262-080-2. OCLC 671855644. +Hart, Nigel (2003). Whimpress, Bernard (ed.). J. N. Crawford: His Record Innings By Innings. Nottingham: Association of Cricket Statisticians and Historians. ISBN 1-902171-79-9. +McKinstry, Leo (2011). Jack Hobbs: England's Greatest Cricketer. London: Yellow Jersey Press. ISBN 978-0-224-08329-4. +Swanton, E. W., ed. (1980). Barclays World of Cricket (2nd ed.). Collins. ISBN 0-00-216349-7. +Wynne-Thomas, Peter (1989). The Complete History of Cricket Tours at Home and Abroad. London: Guild Publishing. ISBN 978-0-60055-782-1. +Booth, Keith; Booth, Jennifer (2016). Rebel With A Cause: The Life and Times of Jack Crawford. Sheffield: Chequered Flag Publishing. ISBN 978-0-9932152-5-4. +Lemmon, David (1989). The History of Surrey County Cricket Club. London: Christopher Helm. ISBN 0-7470-2010-8. +Jack Crawford at Cricinfo +Droxford railway station was an intermediate station on the Meon Valley Railway, built to a design by T. P. Figgis and opened in 1903. It served the villages of Droxford, Soberton and Hambledon in Hampshire, England. The railway served a relatively lightly populated area, but was built to main line specifications in anticipation of it becoming a major route to Gosport. Consequently, although the station was built in an area with only five houses, it was designed with the capacity to handle 10-carriage trains. It initially proved successful both for the transport of goods and passengers, but services were reduced during the First World War and the subsequent recession, and the route suffered owing to competition from road transport. +In 1944, amid World War II, Droxford station was used by the Prime Minister Winston Churchill as his base during preparations for the Normandy landings. Based in an armoured train parked in the sidings at Droxford, Churchill met with numerous ministers, military commanders and leaders of allied nations. On 4 June 1944, shortly before the landings were due to take place, Free French leader Charles de Gaulle visited Churchill at Droxford, and was informed of the invasion plans. When discussing the future governance of liberated France at this meeting, Churchill expressed his view that if forced to side with France or the United States he would always choose the United States, a remark which instilled in de Gaulle a suspicion of British intentions and caused long-term damage to the postwar relationship between France and Britain. +After the war, with Britain's railway network in decline, services on the Meon Valley Railway were cut drastically. A section of the line north of Droxford was closed, reducing Droxford to being the terminus of a short 9+3⁄4-mile (15.7 km) branch line. In early 1955 the station closed to passengers, and in 1962 it closed to goods traffic. Following its closure, Droxford station and a section of its railway track were used for demonstrating an experimental railbus until the mid-1970s. It then briefly served as a driving school for HGV drivers, before becoming a private residence and being restored to its original appearance. +The River Meon rises in East Meon, and flows 21 miles (34 km) through rural Hampshire before entering the Solent at Hill Head. With fertile farmland and access to the coast, the valley of the Meon has been inhabited since at least the Mesolithic era. The village of Droxford (from the Old English Drokeireford, "dry ford") is situated on sloping ground on the west bank of the Meon, overlooking a broad flood plain. The village has existed since at least the early 9th century; the vill of Drokeireford was granted by Ecgberht, King of Wessex to Herefrith, bishop of Winchester in 826 "for the sustenance of the monks of Winchester". At that time, the village already contained three mills. Droxford became the most important of the villages in the immediate area; by the late 19th century it housed the local post office and telegraph office, police station, workhouse and courthouse, and had its own brewery, manor house and flour mill. In the 1901 census, taken shortly before the opening of the railway, Droxford and neighbouring Soberton on the east bank of the Meon had a combined population of 1687. +Since 1851 there had been proposals to build a railway line through the Meon Valley, but all proposals were abandoned owing to high construction costs or objections from local landowners. Consequently, by the end of the 19th century the area still had no easy access to a railway line, unlike most other communities in the country. +To address this lack of service, in 1896 a group of wealthy locals proposed a railway running north–south from Basingstoke to Portsmouth. Although it would serve a lightly populated area and consequently be of little economic benefit, the proposal's supporters argued that it would be of strategic importance, connecting military facilities in Aldershot and Portsmouth with military training grounds in north Hampshire. The line was to have been operated by the Great Western Railway (GWR) and the South Eastern Railway. +The London and South Western Railway (LSWR) and London, Brighton and South Coast Railway, who between them controlled rail access to the major ports of Portsmouth and Southampton, were aghast at the prospect of the GWR controlling a route to the south coast, and lobbied against the proposal. The LSWR promised the House of Lords Committee considering the proposal that if the scheme were rejected, the LSWR would build a line from Basingstoke to Alton and from Alton south along the Meon Valley to Fareham between Southampton and Portsmouth. Although originally intending to build a light railway, when the LSWR made a formal proposal to Parliament in 1897 the proposal was for a railway built to main line specifications. The Long Depression had by this time been underway for over 20 years, severely lowering agricultural incomes and allowing the LSWR to buy land along the Meon Valley cheaply. +Droxford lies on the west of the River Meon, and at the time was a large village surrounded by farmland; the station was built on the opposite side of the river, in neighbouring Soberton. Soberton was closely associated with Droxford but had its own identity, and consisted of agricultural land interspersed with small settlements. Brockbridge, the part of Soberton chosen for the station, contained only five houses; it was chosen for its location at the meeting point of five roads close to a bridge over the Meon. It was accessible to Droxford, Soberton, and the important nearby village of Hambledon. Although sited in Soberton, the station was to be named for the more important Droxford. +The chosen site was glebe land, historically owned by the Diocese of Winchester for the benefit of the parish of Meonstoke, of which Soberton was a part. After centuries as a chapel of ease of the neighbouring parish of Meonstoke, in May 1897 Soberton became an independent parish in its own right. As such, the LSWR needed to persuade William Hammond Morley, the newly appointed rector of Soberton, of the benefits of the railway. On 22 November 1898 Morley provisionally agreed to sell the land for the station site for £425 (about £48,000 in 2025 terms). On 9 February 1899, following undertakings from the LSWR regarding fencing around the site and arrangements for access to remaining glebe land, the land was transferred to the LSWR. Over the following weeks compensation was paid to those people whose homes or land would be affected by the building of the station or the rerouting of roads, and the LSWR was ready to proceed with construction. +Construction of the line began in 1900, beginning at Alton and working south. The station site at Droxford was near high ground, and teams dug a deep cutting through the chalk to allow level access to the station. In mid-1900, the work team digging this cutting found human remains on the site. Construction work ceased while police investigated; it was eventually determined that the remains were Anglo-Saxon. Further bodies, along with metal jewellery and weapons, were discovered, and many of the human bodies had been covered with large pieces of flint. It became obvious that the site was an Anglo-Saxon burial ground. (Later excavations in 1974 identified the site as a pagan cemetery from the 5th and early 6th centuries. A total of 41 graves were identified.) Although the digging of the cutting was further delayed owing to labour shortages, work was largely completed by January 1902. By this time a hotel had opened near the site of the future station, named the Railway Hotel. +The exact date of construction is unrecorded, but it is likely that Droxford station was built in 1902. As with other stations on the line, the station building was designed by T. P. Figgis, an Irish architect who had designed many of the stations of the City and South London Railway (later to become part of London Underground's Northern line). All the new stations for the line were of a very similar Tudor Revival design, built largely of red brick with Portland stone stonework, stained glass windows and lavatories in the shape of pagodas. +In anticipation of the route potentially becoming a major main line the station had two very long 600-foot (180 m) platforms, capable of handling 10-carriage trains, connected by a wooden footbridge. In addition, the station had a corrugated iron goods shed with its own siding, and a signal box on the west (northbound) platform. The two lines converged to a single track immediately north of the station, but continued southwards as a double track for just over 500 yards (450m). One line (the one laid on the same side as the main station building) was the running line to and from Wickham while the other was used as a siding - a set of points connected the two lines just south of the platforms to allow northbound trains to access the west platform and goods trains to shunt the long siding. The entire route was single track outside stations and sidings, but all cuttings, tunnels and bridges were built wide enough to accommodate a second track in future. The station building also incorporated a house for the stationmaster and his family, situated immediately to the left of the main entrance to the platforms. In addition to the stationmaster's accommodation in the main building, four cottages for railway staff were built to the immediate west of the station, and a coal yard was built near the station. +Although the station building was complete, work was proceeding more slowly than anticipated on the construction of the railway line, and the proposed opening date of 25 March 1903 was missed. While the route between Alton and Droxford was built on stable chalk, south of Droxford it entered the Reading Formation of stony clay, a surface on which it was difficult to build; in addition, there were very few water sources for the steam-powered construction machinery and for mixing concrete, necessitating the drilling of multiple deep wells. As well as this, construction work was regularly interrupted by bad weather. The Meon Valley Railway was eventually completed and inspected for fitness by the Board of Trade on 6 April 1903, and the line opened to passengers on 1 June 1903. At the time of opening, the station was staffed by a stationmaster and three signalmen, as well as assorted porters and clerical staff. +The opening was relatively low-key, and no formal ceremony took place. The opening day was Whit Monday, a bank holiday; local residents were each allowed to take one journey free of charge to the next station in either direction. (The free fares were only valid for a single journey; those taking advantage of the offer were obliged either to pay for a return ticket, or to walk back.) Although express services between London and Gosport were hauled by powerful LSWR A12 class locomotives, the only services to stop at Droxford were local trains between Fareham and Alton, hauled by less powerful LSWR 415 class tank engines built in the early 1880s. Initially, Droxford was served by six services a day in each direction, with two trains in each direction on Sundays. +The railway proved successful, and the availability of convenient travel to London prompted the construction of large luxury homes on the undeveloped land to the south of the station. By 1915 there were 19 new homes on the site; 12 of the 19 homes had a woman as the householder, and the development became nicknamed the "Widow's Villas". Although important as a passenger facility, the main significance of Droxford station was its impact on the local agricultural economy. For the first time milk, livestock, vegetables and in particular strawberries from the numerous local strawberry farms could be shipped cheaply and quickly to market towns. (Strawberry farming became such a pivotal part of the local economy following the construction of the Meon Valley Railway that the railway became nicknamed the "Strawberry Line". During the picking season special trains known as "strawberry specials" were laid on, carrying entire trainloads of strawberries to market.) Likewise, bulk goods such as coal and beer could for the first time be shipped cheaply to Droxford and the surrounding villages. The station also proved popular with spectators visiting the Hambledon Hunt's racecourse, approximately 1 mile (1.6 km) from the station, and with people travelling to the market in Alton, a service which proved so popular that the railway offered special ticket prices on market days. +On the outbreak of the First World War in 1914, Britain's railway network was taken into government control. While initially there was little impact on services through Droxford, the need to prioritise military needs and the transfer of railway equipment to France caused a reduction in services on the line. As the war went on, wartime shortages meant there were few goods available in the shops while labour shortages led to a rise in household incomes. Consequently, holidays increased in popularity as there was little else on which people could spend their money, causing an increase in passenger numbers on the Meon Valley railway which connected London to picturesque country villages and busy coastal resorts. The increased popularity of holidays and a national shortage of rolling stock and skilled railway staff led the government to take drastic measures. Fares were increased by 50 per cent, special holiday trains were abolished, along with almost all concessionary tickets, and a new intentionally slow timetable was introduced. The measures were not as successful in curbing demand as the government had hoped, and passenger numbers fell by only 7 per cent. +In the lengthy recession following the First World War, services on the line were further reduced. A proposed doubling of the track was abandoned. Trains were reduced from four to six carriages to small sets of two carriages hauled by LSWR M7 class locomotives. The 1923 absorption of the LSWR into the Southern Railway (SR) led to further economies, and around 1926 the footbridge connecting the platforms of Droxford station was removed, passengers henceforward having to cross the line on foot from the northern ends of the platforms. By this time, cars and lorries were beginning to come into widespread use, providing direct competition with the railway for both goods and passenger traffic. Although the railway continued to operate six trains a day in each direction, they remained slower than they had been in 1914. With the expansion of Southampton—which already had a fast and direct service from London—as a major port, there was little need for a route from London to Gosport. In 1937 the line from London to Alton was electrified, making through trains to London no longer feasible and leaving the steam-powered Meon Valley Railway as a rural branch line. +On 3 September 1939 Britain declared war on Germany, and the Second World War began. Following the outbreak of war the Meon Valley railway remained in passenger and goods use during the day, and at night was used by troop trains carrying soldiers to Southampton to be shipped to France. Portsmouth and Gosport, both major military facilities within easy reach of German aircraft, were expected to come under severe bombardment, and many children from these towns were evacuated along the line to the villages of the Meon Valley. As a potential strategic route the Meon Valley Line came under occasional bombing from German aircraft attempting to disrupt traffic to and from the Channel ports. In 1940 the station building was lightly damaged and two of the four railway cottages were destroyed by a Junkers Ju 88 aircraft, the only significant damage to Droxford station during the war. +As the line was relatively lightly used, it was occasionally used for experimental purposes; in 1941 a special train visited Droxford hauled by an LSWR 700 class locomotive, carrying 35 Bren Gun Carriers and their associated troops, as an experiment into the feasibility of carrying troops and their equipment together. Having developed vacuum problems, the train remained in the siding at Droxford for two days, before eventually progressing to the coast. +Because railway managers had proven skills in administration and of managing logistics they were in demand from the government for strategic management, and many of the managers of Britain's four railway companies were seconded to government. Additionally, although railway work was a reserved occupation, large numbers of railway staff nonetheless volunteered for military service. Staff shortages and the lack of supplies caused services on the Meon Valley line to deteriorate; the maximum speeds were reduced to 10 miles per hour (16 km/h) for goods traffic and 30 miles per hour (48 km/h) for passenger trains. Despite the decline in the standard of services, Droxford station remained busy. Fuel rationing had made road transport between farms and the towns largely unviable; consequently, goods and passengers going to and from the surrounding villages largely ceased to be carried by road, and instead would be taken to Droxford station for onward shipment by train. +On the morning of 2 June 1944 orders were telephoned along the length of the Meon Valley Railway that it was to be kept free of trains to ensure a special train could use the route without interruption. The royal train of the London, Midland and Scottish Railway, a T9 class locomotive hauling armoured carriages, pulled into the sidings at Droxford station. Troops surrounded the station and its sidings, and the local post office was ordered to let no mail other than official business leave the village. Droxford's signalman Reg Gould, whose daughter had been born the previous month, was treated to a full breakfast off the ration book. +Although officially kept secret from local residents, it soon transpired that Winston Churchill had chosen the station as a secure base, to be near the coast and to the Allied command centre at Southwick House during the forthcoming Normandy landings. His reason for choosing Droxford is not recorded, other than its proximity to the Channel ports; there was local speculation that the site was chosen owing to its deep cutting into which the train could be repositioned should it come under attack. Droxford also had the advantage of being overshadowed by beech trees, which obscured the view of the train. +In the company of his secretary Marion Holmes, General "Pug" Ismay and South African prime minister Jan Smuts, Churchill was to remain in the carriage at Droxford for the next four days other than a few brief excursions. On those occasions in which the party needed to leave the train, it would be driven the short distance from the siding to the platform, and those on board would exit through the booking hall to staff cars waiting at the exit. For reasons unknown, Churchill refused to follow this procedure, and insisted on exiting through the goods yard. +On 3 June Anthony Eden (at that time the Foreign Secretary) and Pierson Dixon visited Churchill on the train. Eden was unimpressed, and was later to describe the train as a place where "it was almost impossible to conduct any business" owing to there only being one bath and one telephone, each of which was constantly in use by Churchill or Ismay. A string of visits from other close confidantes of Churchill followed, including Ernest Bevin, Geoffrey William Geoffrey-Lloyd, Duncan Sandys and Arthur Tedder, as well as fellow Commonwealth prime ministers Peter Fraser, Sir Godfrey Huggins and William Lyon Mackenzie King. In addition to the meetings held on board, the train took Churchill, Bevin and Smuts to Southampton to watch invasion forces embark. Churchill hosted a dinner aboard the train, at which Eden and Bevin—both considered potential successors to Churchill—discussed the possibility of working together to continue Britain's wartime coalition government in peacetime should Churchill retire. Later that night Dwight Eisenhower, from his nearby base at Southwick House, decided to postpone the invasion from 5 to 6 June owing to predicted bad weather. +Although the Allied invasion of France was only days away, Free French leader Charles de Gaulle had yet to be informed of the invasion plans. Not wishing to risk communicating with the French government in exile in Algeria about the forthcoming invasion plans, but wary of a potentially disastrous diplomatic incident should the invasion begin without French knowledge, the British cabinet had agreed to invite de Gaulle to visit England, at which time the invasion plans would be disclosed to him in person. In the morning of 4 June, de Gaulle landed at RAF Northolt, to be met with a telegram from Churchill: +TOP SECRET My dear General de Gaulle,Welcome to these shores! Very great military events are about to take place. I should be glad if you could come to see me down here in my train, which is close to General Eisenhower's Headquarters, bringing with you one or two of your party. General Eisenhower is looking forward to seeing you again and will explain to you the military position which is momentous and imminent. If you could be here by 1.30 p.m., I should be glad to give you dejeuner and we will then repair to General Eisenhower's Headquarters. Let me have a telephone message early to know whether this is agreeable to you or not. +De Gaulle was apprehensive as to why he had been invited to this unusual location. He arrived at Droxford at about 1.00 pm and was met by Anthony Eden and they walked up the track towards Churchill's siding, to be greeted with open arms by Churchill on the track. They entered the train to meet Jan Smuts and Ernest Bevin, where Churchill informed de Gaulle of the imminent invasion. De Gaulle asked Churchill how a liberated France was to be administered; Churchill told de Gaulle he would need to take it up with Franklin D. Roosevelt and that if ever he had to make a choice between France and the United States he would always side with the United States, a statement at which de Gaulle took great offence. +Anthony Eden was horrified at how Churchill had handled such a sensitive meeting. He spoke to de Gaulle privately to make amends, but the relationship between de Gaulle and Churchill had been badly damaged. From this point onwards de Gaulle was distrustful of the relationship between Britain and the United States, ultimately leading to a breakdown of relations between post-war Britain and France and to de Gaulle's vetoing of British membership of the European Economic Community. +At 6.58 pm on 5 June, Churchill's train pulled out of Droxford and returned to London. The time of his arrival at Downing Street is not recorded, but at 10.30 pm his duty typist was summoned. At 16 minutes past midnight the following morning, British glider troops attacked Pegasus Bridge and the American airborne landings in Normandy began shortly after. +On 26 July 1945 the Labour Party won a landslide victory on a manifesto that included bringing strategic industries into public control, and on 1 January 1948 the Southern Railway became the Southern Region of the publicly owned British Railways. Although still invaluable for the transport of bulk goods, the use of railways in the area fell sharply after the war. The abolition of petrol rationing in 1950 made the car far more practical than it had been, and the location of Droxford station between the two villages meant that it was inconvenient for both. Railway stations and rolling stock had suffered from poor maintenance since the outbreak of war; trains were slow and dirty, and there was an increasing number of high-profile railway accidents. To address the decline, services on the Meon Valley Railway were drastically cut. Goods charges were increased, Sunday services were scrapped, and weekday passenger service was reduced to four trains per day in each direction with the last train leaving Alton at 4.30 pm, making day return travel to London impossible. By this time Droxford no longer had its own stationmaster, and the stationmaster of nearby Wickham was in nominal charge of the station with Reg Daniels, Droxford's senior porter, responsible for its administration. +With the railway network struggling, the British Transport Commission (BTC) proposed drastic measures. On 3 May 1954 the BTC proposed the closure of the Meon Valley Railway to passengers and the total closure of the line between Droxford and Farringdon Halt, leaving a short 3+1⁄2-mile (5.6 km) stub between Alton and Farringdon and a longer 9+3⁄4-mile (15.7 km) route between Fareham and Droxford to carry any remaining goods traffic, by this time primarily coal for J. E. Smith's coal yard near Droxford station, and agricultural produce from the area's farms. For those passengers without access to cars, the BTC proposed improving bus services between Portsmouth and Meonstoke; they calculated that replacing four passenger trains per day with six buses would achieve annual savings of £38,000 (about £900,000 in 2025 terms). On 17 November 1954 the BTC announced the withdrawal of passenger services, and freight services between Droxford and Farringdon Halt, effective from 7 February 1955. The BTC guaranteed that one freight train per day would run on each of the two remaining sections of the line. +As 6 February 1955, officially the last day of service, was a Sunday on which no passenger trains were due to run, the last scheduled passenger services to Droxford were those of 5 February 1955. At 7.14 pm the last scheduled passenger train to stop at Droxford pulled out, arriving in Alton at 7.43 pm. On 6 February 1955 a special train named The Hampshireman, consisting of two locomotives hauling ten carriages, went on a circular journey leaving London Waterloo station at 9.45 am, passing through Pulborough and along the line from Pulborough to Petersfield via Midhurst which was also due to close the next day, north from Fareham to Alton along the Meon Valley Railway, and back to Waterloo, passing through Droxford at about 3.20 pm without stopping. With the station now a terminus with no need for signalling, the signal box was demolished later that year. +While the goods service from Droxford was kept busy during the sugar beet harvest, there was little demand for the rest of the year, and in 1956 it was reduced to three trains a week. Despite these economies, the line remained unprofitable. On 30 April 1961 the Solent Express, an excursion train chartered by the Locomotive Club of Great Britain, paid a brief visit to Droxford, the last passenger train to visit the station. The Hampshire Narrow Gauge Railway Trust approached Droxford council in May 1961 with a proposal to build a narrow-gauge line along part of the disused railway between Droxford and West Meon, arguing this would bring tourists to the area, but the proposal was abandoned in 1962. +On 1 June 1961 Dr Richard Beeching became chairman of the newly formed British Railways Board, with a brief of returning the network to financial stability. The branch line to Droxford, profitable only during the sugar beet harvest, was an obvious candidate for closure. On 30 April 1962 goods service was ended on the branch, and Droxford station was formally closed. Reg Gould, who by this time had been signalman for over 20 years and was the de facto manager of the station, was transferred to nearby Swanwick railway station. The tracks remained in place, and were used for the storage of redundant goods wagons awaiting scrapping. The short stub between Alton and Farringdon Halt remained operational by goods trains until 13 August 1968, the last part of the Meon Valley Railway to close. +Following its closure, Droxford station and the track south to Wickham were leased by the Sadler Rail Coach Company. The company's proprietor Charles Sadler Ashby had built the Pacerailer, a vehicle consisting of a bus-style body on rails, and used the line to demonstrate it to potential buyers; a section was rebuilt with a 1:10 incline to demonstrate its abilities on steep gradients. At the request of the Southern Locomotive Preservation Society, Ashby allowed the use of the sidings to store engines and carriages. A serious problem with vandalism arose at the site when the tracks were intentionally blocked and points jammed in an effort to derail vehicles, and on 4 May 1970 the Pacerailer prototype was burned out and badly damaged. Eventually British Rail closed the Knowle junction, ending the last connection between Droxford and the railway network, and the Southern Locomotive Preservation Society abandoned the site. Hampshire County Council considered building a road along the former railway line, and in September 1972 purchased the land and leased the station building and a 4+1⁄2 miles (7.2 km) section of track at Droxford to Ashby. This effectively freed British Rail from maintaining utilities it had not used for many years. +Ashby died in February 1976, by which time the tracks to Droxford had been removed. The dates are not recorded, but Ashby's test track between Droxford and Wickham was probably removed in 1975, as had been the track south of Wickham in 1974. The Sadler Rail Coach Company was dissolved in December 1976, having failed to sell the Pacerailer to any railway company. A similar concept, the Pacer, was successfully developed by British Rail in the 1980s. Droxford station and its associated land were taken over by a driving school; the surrounding land was tarmacked and used for the training of HGV drivers. In 1984 the building was sold to Colin and Elizabeth Olford, who removed the surrounding tarmac and restored the building to close to its original appearance. As of 2016 it remained a private residence. +List of closed railway stations in Britain +Beevor, Antony (2009). D-Day: The Battle for Normandy. New York; Toronto: Viking. ISBN 978-0-670-02119-2. +Buttrey, Pam (2012). A History of Droxford Station. Corhampton: Noodle Books. ISBN 978-1-906419-93-6. +Gilbert, Martin (1986). Winston S Churchill. Vol. VII: Road to Victory 1941–1945. London: Heinemann. ISBN 978-0-434130-17-7. +Stone, R. A. (1983). The Meon Valley Railway. Southampton: Kingfisher Railway Productions. ISBN 978-0-946184-04-0. +Struthers, Jane (2004). Royal Palaces of Britain. London: New Holland Publishers (UK) Ltd. ISBN 978-1-84330-733-4. +Sy-Wonyu, Aïssatou (2003). The "Special Relationship". Mont-Saint-Aignan, France: Presses universitaires de Rouen et du Havre. ISBN 978-2-877758-62-8. +Wolmar, Christian (2007). Fire & Steam. London: Atlantic Books. ISBN 978-1-84354-630-6. +The concerto delle donne (lit. 'consort of ladies') was an ensemble of professional female singers of late Renaissance music in Italy. The term usually refers to the first and most influential group in Ferrara, which existed between 1580 and 1597. Renowned for their technical and artistic virtuosity, the Ferrarese group's core members were the sopranos Laura Peverara, Livia d'Arco and Anna Guarini. +The Duke of Ferrara, Alfonso II d'Este, founded a group of mostly female singers for his chamber music series, musica secreta (lit. 'secret music'). These singers were exclusively noble women, such as Lucrezia and Isabella Bendidio. In 1580, Alfonso formally established the concerto delle donne for both his wife Margherita Gonzaga d'Este and reasons of prestige. The new group included professional singers of upper-class, but not noble, backgrounds, under the direction of the composers Luzzasco Luzzaschi and Ippolito Fiorini. Their signature style of florid, highly ornamented singing brought prestige to Ferrara and inspired composers of the time such as Lodovico Agostini, Carlo Gesualdo, and Claudio Monteverdi. +The concerto delle donne revolutionized the role of women in professional music, and continued the tradition of the Este court as a musical center. Word of the ladies' ensemble spread across Italy, inspiring imitations in the courts of the Medici and Orsini. The founding of the concerto delle donne was among the most important events in secular music in late 16th-century Italy. The musical innovations established in the court were important in the development of the madrigal, and eventually the seconda pratica. +Northern Italy was a leading center of Renaissance music, which broadly covered the 15th and 16th centuries of Europe. Regional courts, ruled by competing families—such as the Este, Gonzaga, and Medici—patronized secular music immensely, commissioning compositions and forming large ensembles. Although the frottola style held early popularity, it was quickly overtaken by the madrigal in the 1520s. The madrigal became the most important secular genre of 16th-century Italy, and possibly the entire Renaissance; according to J. Peter Burkholder, "through the madrigal, Italy became the leader in European music for the first time in history". Unlike the frottola, composed exclusively by native Italians, the first leading madrigal composers were foreign Franco-Flemish musicians, referred to as Oltremontani (those from lit. '"over the mountains"'), such as Philippe Verdelot and Jacques Arcadelt. +At the court in Ferrara, the Duke Alfonso II d'Este formed a group of mostly female singers by at least 1577. They performed madrigals within the context of the Duke's ongoing musica secreta (lit. 'secret music'), a regular series of chamber music concerts performed for a private audience. Although it is uncertain whether the group's members were amateur or professional musicians, they were noblewomen and would have attended court regardless. These singers included sisters Lucrezia and Isabella Bendidio, as well as Leonora Sanvitale, and Vittoria Bentivoglio. The professional bass singer Giulio Cesare Brancaccio also joined the ensemble. +The Duke formally established the concerto delle donne (lit. 'consort of ladies') in 1580. He did not announce the creation of a new professional, all-female ensemble; instead, the group infiltrated and gradually dominated the musica secreta concerts. This ensemble was created by the Duke in part to amuse his young new wife, Margherita Gonzaga d'Este who was musically inclined herself, and in part to help the Duke achieve his artistic goals for the court. Margherita's influence on the church through her brother-in-law, the cardinal Luigi d'Este, allowed the concerto to use church assets such as the San Vito convent in Ferrara. The first recorded performance by the professional ladies was on 20 November 1580; Brancaccio joined the new group the next month. By the 1581 carnival season, they were performing together regularly. +This new "consort of ladies" was viewed as an extraordinary and novel phenomenon; most witnesses did not connect the concerto delle donne with the earlier group of ladies from the 1570s. However, modern musicologists now view the earlier group as a crucial part of the creation and development of the social and vocal genre of the concerto delle donne. The culture at the Italian courts of that time had a political dimension, as families aimed to present their greatness by non-violent means. +The most prominent member of the new ensemble was Laura Peverara, whose musical abilities prompted the Duke to specifically ask the Duchess to bring her from Mantua as part of her retinue. She was particularly lauded for her skill in accompanied solo singing. Peverara was joined by Livia d'Arco and Anna Guarini, daughter of the prolific poet Giovanni Battista Guarini. The latter wrote poems for many of the madrigals which were performed by the ensemble, and wrote texts for the balletto delle donne dances. The well-known singer Tarquinia Molza was involved with the group, but modern scholars disagree on whether she sang with them or was solely as an advisor and instructor. Whether Molza ever performed with them or not, she was ousted from any role in the group after her affair with the composer Giaches de Wert came to light in 1589. After the dismissal of Brancaccio for insubordination in 1583, no more permanent male members of the musica secreta were hired; however, the ensemble occasionally sang with male singers. +The singers of the concerto delle donne were officially ladies-in-waiting of the Duchess Margherita, but were hired primarily as singers. Despite their upper-class background, the singers would not have been welcomed into the court's inner circle had they not been such skilled performers. D'Arco belonged to the nobility, but a minor family only. Peverara was the daughter of a wealthy merchant, and Molza came from a prominent family of artists. The musicologist Thomasin LaMay posits that the women of the concerti delle donne provided sexual favors for members of the court, but there is no evidence for this, and the circumstances of their marriages and dowries argues against this interpretation. The women were paid salaries and received other benefits, such as dowries and apartments in the ducal palace. Peverara received 300 scudi a year and lodging in the ducal palace for herself, her husband, and her mother – as well as a dowry of 10,000 scudi upon her marriage. +The new singers played instruments, including the lute, harp, and viol, but focused their energies on developing vocal virtuosity. This skill became highly prized in the mid-16th century, beginning with basses like Brancaccio, but by the end of the century virtuosic bass singing went out of style, and higher voices came into vogue. The composer Luzzasco Luzzaschi directed and wrote music to showcase the ensemble, and accompanied them on the harpsichord. The composer and lutenist Ippolito Fiorini was the maestro di cappella, in charge of the entire court's musical activities. In addition to his duties to the overall court, Fiorini accompanied the concerto on the lute. +Despite having married three times in the hopes of producing an heir, Alfonso II died in 1597 without issue, legitimate or otherwise. His cousin Cesare inherited the Duchy, but the city of Ferrara, which was legally a Papal fief, was annexed to the Papal States in 1598 through a combination of "firm diplomacy and unscrupulous pressure" by Pope Clement VIII. The Este court had to abandon Ferrara in disarray and its music establishment was disbanded. While the existence of the concerto delle donne was widely known, its detailed history was largely lost, dispersed between archival records, until the beginning of the 20th century when the Italian literature critic Angelo Solerti drew attention to Ferrara's 16th century court culture. +The concerto delle donne transformed the musica secreta series. In the past, performers and audience members would alternate roles, as the gatherings were "social music for the enjoyment of the singers themselves". During the ascendancy of the concerto delle donne the roles within the musica secreta became fixed, resulting in "concert music for the pleasure of an audience". The performances had a restricted audience; only selected dignitaries and few courtiers saw the concerto delle donne; one such dignitary may have been the Russian ambassador Istoma Shevrigin, in 1581. +The performers were thoroughly coached and rehearsed in their work, down to all hand gestures and facial movements. The women performed up to six hours a day, either singing their own florid repertoire from memory, sight-reading from partbooks, or participating in the balletti as singers and dancers. The ladies' musical duties included performing with the duchess's balletto delle donne, a group of female dancers who frequently crossdressed. +Aside from Brancaccio, all the singers in the concerto were female sopranos. There is no evidence that the ensemble used falsettists. This fact is surprising, considering that castrati were shortly to become the biggest stars of a new art form, opera. In 1607, Monteverdi's influential L'Orfeo featured four castrato roles out of a cast of nine, showing the new dominance of this vocal type. It also contrasts with the court of Margherita's father, where Guglielmo Gonzaga actively sought out eunuchs. +The elite, hand-selected audience members favored with admission to performances by the concerto delle donne demanded diversions and entertainment beyond the pleasures of beautiful music alone. During the concerts, members of the concerto's audience would sometimes play cards. The ambassador of the Grand Duke of Tuscany Orazio Urbani, having waited several years to see the concerto, complained that he was forced not only to play cards, distracting him from the performance, but also simultaneously admire and praise the women's music to their patron Alfonso. After at least one concert, to continue the entertainment, a dwarf couple danced. Alfonso was not as interested in these peripheral entertainments; in one instance he excused himself from the party to go sit under a tree to listen to the concerto, and follow along with the madrigal texts and musical scores, which were made available to listeners. +The greatest musical innovation of the concerto delle donne was its departure from one voice singing diminutions above an instrumental accompaniment to two or three highly ornamented voices singing varying diminutions at once. Such ornaments were meticulously notated by the composers, leaving a detailed record of the concerto delle donne's performance practice. Although traditionally such ornaments were improvised in performance, notation was used to coordinate and rehearse the multiple voices; the singers may have continued improvised diminutions in their solo repertoire. +Specific ornaments used by the concerto delle donne, mentioned in a source from 1581, were such popular 16th-century devices as passaggi (division of a long note into many shorter notes, usually stepwise), cadenze (decoration of the penultimate note, sometimes quite elaborate), and tirate (rapid scales). Accenti (connection of two longer notes, using dotted rhythms), a staple of early Baroque music, are absent from the list. In 1592 Giulio Caccini claimed that Alfonso asked him to teach his ladies the new accenti and passaggi styles. +Many Italian Renaissance composers wrote music either inspired by the concerto delle donne or specifically for them. Between 1581 and 1586 especially, Alfonso's court saw its most "vibrant and culturally productive period, during which its literary and musical talents were focused most keenly on providing repertoire for the ladies' performances, both in +private and as part of court spectacle". +The output of the ducal printer, Vittorio Baldini, consisted largely of music written for the concerto delle donne. Baldini's first publication for the Duke was Il lauro secco (1582), which was followed by Il lauro verde (1583), both containing music by the leading composers of Rome and Northern Italy. Music in honor of the concerto was printed as far away as Venice, with Paolo Virchi's First Book à 5, published by Giacomo Vincenti and Ricciardo Amadino containing the madrigal which begins SeGU'ARINAscer LAURA e prenda LARCO / Amor soave e dolce / Ch'ogni cor duro MOLCE. This capitalization is in the original, clearly spelling out the equivalent of the names Anna Guarini, Laura Peverara, Livia d'Arco, and Tarquinia Molza. +Musically, their repertoire was written to display the skill of the upper-voiced singers; oftentimes lower static voices accompanied them in contrast. Such works are characterized by a high tessitura, a virtuosic and florid style, and a wide vocal range. There were two separate styles of madrigals written for and inspired by the concerto delle donne. The first is the "luxuriant" style of the 1580s, which set the poetry of Ferrarese natives—such as Tasso and G.B. Guarini—which were generally short and witty with single sections. The second is music in the style of the seconda pratica, written in the 1590s, which treated harmony with more freedom than the preceding prima pratica style. +The chief composer for the concerto delle donne was their director, Luzzasco Luzzaschi, who wrote works in both the "luxuriant" and seconda pratica styles. Luzzaschi's book of madrigals for one, two, and three sopranos with keyboard accompaniment, published in 1601 as the well-known Madrigali per cantare e sonare, comprises works written throughout the 1580s. The musicologist Newcomb considers this publication the exemplar of the ladies' signature musical style. +Musically, Luzzaschi's works are highly sectionalized and based on melodic themes, rather than harmonic structures. Luzzaschi lessens the sectionalizing effect of his compositional techniques by weakening cadences. His music includes progressive and conservative elements: although his use of vocal imitation creates dense polyphonic textures, akin to earlier 16th-century compositions, his individualistic use of jarring melodic leaps and harmonic dissonance is at odds with older conventions. This freer use of dissonances was closely connected with the style of the concerto delle donne. +Other composers who wrote for the concerto include Lodovico Agostini, whose third book of madrigals is among the first publications fully dedicated to the new singing style. He dedicated songs to Guarini, Peverara, and Luzzaschi. Carlo Gesualdo also wrote music for the group while visiting Ferrara in 1594 to marry the Duke's niece Leonora d'Este; much of Gesualdo's music for the group does not survive. Other publications include De Wert's Seventh Book of Madrigals à 5 and Marenzio's First Book à 6, while Monteverdi's Canzonette a tre voci was probably influenced by the group. Some madrigals in the two-book Madrigaletti et napolitane by Giovanni de Macque were written with the Concerto delle donne in mind, due to their technically demanding content. +Works written for the concerto delle donne were not limited to music: The poets Torquato Tasso and Giovanni Battista Guarini wrote works dedicated to the ladies in the concerto, some of which were later set by composers. Tasso wrote over seventy-five poems to Peverara alone. +While they were neither the first nor only female musicians in Ferrara, the concerto delle donne was a revolutionary musical establishment that helped effect a shift in women's role in music; Newcomb remarks that its success took women from obscurity to "the apex of the profession". Women were openly brought to court to train as professional musicians, and by 1600, a woman could have a viable career as a musician, independent of her husband or father. New women's ensembles inspired by the concerto delle donne resulted in more positions for women as professional singers and more music for them to perform. The concerto delle donne contested the viewpoint of some contemporaries that women were unfit to achieve noteworthy deeds. +Despite Alfonso not publicizing the composed music and the dissolution of the court in 1597, the musical style which was inspired by the concerto delle donne spread throughout Europe, and remained prominent for almost fifty years. The concerto delle donne was so influential that other courts developed similar concerti and it became a cliché of northern Italian courts; having one was a sign of prestige. It heavily influenced the development of the madrigal and eventually the seconda practica. The group brought Alfonso and his court international prestige, as the ladies' reputation spread throughout Italy and southern Germany; in 1619, the German composer Michael Praetorius described it as "the latest new Italian style for achieving a good manner of singing". It functioned as a powerful tool of propaganda, projecting an image of strength and affluence. +Having seen the concerto delle donne in Ferrara, Caccini created a rival group made up of his family and a pupil. This ensemble was sponsored by the Medici, and travelled as far abroad as Paris to perform for Marie de' Medici. Francesca Caccini had much success composing and singing in the style of the concerto delle donne. Rival groups were planned in Florence by the Medici, Rome by the Orsini, and Mantua by the Gonzaga. There was even a rival group in Ferrara based in the Castello Estense, the very palace where the concerto delle donne performed. This group was formed by Alfonso's sister Lucrezia d'Este, Duchess of Urbino. She had lived at the Este court since 1576, and shortly after Margherita's marriage to Alfonso in 1579, Alfonso and his henchmen killed Lucrezia's lover. Lucrezia was unhappy about being replaced as the matron of the house by Margherita, and upset by the murder of her lover, leading to her desire to be separate from the rest of her family during her evening entertainments. +Barbara Strozzi was among the last composers and performers in this style, which by the mid-17th century was considered archaic. At least one instrument used by the concerto delle donne, the harp L'Arpa di Laura in the Galleria Estense art gallery, has become famous. +The blue-faced honeyeater (Entomyzon cyanotis), also colloquially known as the bananabird, is a passerine bird of the honeyeater family, Meliphagidae. It is the only member of its genus, and it is most closely related to honeyeaters of the genus Melithreptus. Three subspecies are recognised. At around 29.5 cm (11.6 in) in length, the blue-faced species is large for a honeyeater. Its plumage is distinctive, with olive upperparts, white underparts, and a black head and throat with white nape and cheeks. Males and females are similar in external appearance. Adults have a blue area of bare skin on each side of the face readily distinguishing them from juveniles, which have yellow or green patches of bare skin. +Found in open woodland, parks and gardens, the blue-faced honeyeater is common in northern and eastern Australia, and southern New Guinea. It appears to be sedentary in parts of its range, and locally nomadic in other parts; however, the species has been little studied. Its diet is mostly composed of invertebrates, supplemented with nectar and fruit. They often take over and renovate old babbler nests, in which the female lays and incubates two or rarely three eggs. +The blue-faced honeyeater was first described by ornithologist John Latham in his 1801 work, Supplementum Indicis Ornithologici, sive Systematis Ornithologiae. However, he described it as three separate species, seemingly not knowing it was the same bird in each case: the blue-eared grackle (Gracula cyanotis), the blue-cheeked bee-eater (Merops cyanops), and the blue-cheeked thrush (Turdus cyanous). It was as the blue-cheeked bee-eater that it was painted between 1788 and 1797 by Thomas Watling, one of a group known collectively as the Port Jackson Painter. +It was reclassified in the genus Entomyzon, which was erected by William Swainson in 1825. He observed that the "Blue-faced Grakle" was the only insectivorous member of the genus, and posited that it was a link between the smaller honeyeaters and the riflebirds of the genus Ptiloris. The generic name is derived from the Ancient Greek ento-/εντο- 'inside' and myzein/μυζειν 'to drink' or 'suck'. The specific epithet, cyanotis, means 'blue-eared', and combines cyano-/κυανο 'blue' with otis (a Latinised form of ωτος, the Greek genitive of ous/ους) 'ear'. Swainson spelt it Entomiza in an 1837 publication, and George Gray wrote Entomyza in 1840. +The blue-faced honeyeater is generally held to be the only member of the genus, although its plumage suggests an affinity with honeyeaters of the genus Melithreptus. It has been classified in that genus by Glen Storr, although others felt it more closely related to wattlebirds (Anthochaera) or miners (Manorina). A 2004 molecular study has resolved that it is closely related to Melithreptus after all. Molecular clock estimates indicate that the blue-faced honeyeater diverged from the Melithreptus honeyeaters somewhere between 12.8 and 6.4 million years ago, in the Miocene epoch. It differs from them in its much larger size, brighter plumage, more gregarious nature, and larger patch of bare facial skin. +Molecular analysis has shown honeyeaters to be related to the Pardalotidae (pardalotes), Acanthizidae (Australian warblers, scrubwrens, thornbills, etc.), and the Maluridae (Australian fairy-wrens) in the large superfamily Meliphagoidea. +"Blue-faced honeyeater" has been designated as the official common name for the species by the International Ornithologists' Union (IOC). Early naturalist George Shaw had called it the blue-faced honey-sucker in 1826. Other common names include white-quilled honeyeater and blue-eye. Its propensity for feeding on the flowers and fruit of bananas in north Queensland has given it the common name of banana-bird. A local name from Mackay in central Queensland is pandanus-bird, as it is always found around Pandanus palms there. It is called morning-bird from its dawn calls before other birds of the bush. Gympie is a Queensland bushman's term. Thomas Watling noted a local indigenous name was der-ro-gang. John Hunter recorded the term gugurruk (pron. "co-gurrock"), but the term was also applied to the black-shouldered kite (Elanus axillaris). It is called (minha) yeewi in Pakanh, where minha is a qualifier meaning 'meat' or 'animal', and (inh-)ewelmb in Uw Oykangand and Uw Olkola, where inh- is a qualifier meaning 'meat' or 'animal', in three aboriginal languages of central Cape York Peninsula +Three subspecies are recognised: +E. c. albipennis was described by John Gould in 1841 and is found in north Queensland, west through the Gulf of Carpentaria, in the Top End of the Northern Territory, and across into the Kimberley region of Western Australia. It has white on the wings and a discontinuous stripe on the nape. The wing-patch is pure white in the western part of its range, and is more cream towards the east. It has a longer bill and shorter tail than the nominate race. The blue-faced honeyeater also decreases in size with decreasing latitude, consistent with Bergmann's rule. Molecular work supports the current classification of this subspecies as distinct from the nominate subspecies cyanotis. +E. c. cyanotis, the nominate form, is found from Cape York Peninsula south through Queensland and New South Wales, into the Riverina region, Victoria, and southeastern South Australia. +E. c. griseigularis is found in southwestern New Guinea and Cape York, and was described in 1909 by Dutch naturalist Eduard van Oort. It is much smaller than the other subspecies. The original name for this subspecies was harteri, but the type specimen, collected in Cooktown, was found to be an intergrade form. The new type was collected from Merauke. This subspecies intergrades with cyanotis at the base of the Cape York Peninsula, and the zone of intermediate forms is narrow. The white wing-patch is larger than that of cyanotis and smaller than that of albipennis. Only one bird (from Cape York) of this subspecies was sampled in a molecular study, and it was shown to be genetically close to cyanotis. +A large honeyeater ranging from 26 to 32 cm (10 to 12.5 in) and averaging 29.5 cm (11.6 in) in length. The adult blue-faced honeyeater has a wingspan of 44 cm (17.5 in) and weighs around 105 g (3.7 oz). In general shape, it has broad wings with rounded tips and a medium squarish tail. The sturdy, slightly downcurved bill is shorter than the skull, and measures 3 to 3.5 cm (1.2 to 1.4 in) in length. It is easily recognised by the bare blue skin around its eyes. The head and throat are otherwise predominantly blackish with a white stripe around the nape and another from the cheek. The upperparts, including mantle, back and wings, are a golden-olive colour, and the margins of the primary and secondary coverts a darker olive-brown, while the underparts are white. Juveniles that have just fledged have grey head, chin, and central parts of their breasts, with brown upperparts, and otherwise white underparts. After their next moult, they more closely resemble adults and have similar plumage, but are distinguished by their facial patches. The bare facial skin of birds just fledged is yellow, sometimes with a small patch of blue in front of the eyes, while the skin of birds six months and older has usually become more greenish, and turns darker blue beneath the eye, before assuming the adult blue facial patch by around 16 months of age. The blue-faced honeyeater begins its moult in October or November, starting with its primary flight feathers, replacing them by February. It replaces its body feathers anywhere from December to June, and tail feathers between December and July. 422 blue-faced honeyeaters have been banded between 1953 and 1997 to monitor movements and longevity. Of these, 109 were eventually recovered, 107 of which were within 10 km (6.2 mi) of their point of banding. The record for longevity was a bird banded in May 1990 in Kingaroy in central Queensland, which was found dead on a road after 8 years and 3.5 months in September 1998, around 2 km (1.2 mi) away. +The blue-faced honeyeater produces a variety of calls, including a piping call around half an hour before dawn, variously described as ki-owt, woik, queet, peet, or weet. Through the day, it makes squeaking noises while flying, and harsh squawks when mobbing. Its calls have been likened to those of the yellow-throated miner (Manorina flavigula), but are deeper. Blue-faced honeyeaters make a soft chirping around nestlings and family members. +A distinctive bird, the blue-faced honeyeater differs in coloration from the duller-plumaged friarbirds, miners and wattlebirds, and it is much larger than the similarly coloured Melithreptus honeyeaters. Subspecies albipennis, with its white wing-patch, has been likened to a khaki-backed butcherbird in flight. +The blue-faced honeyeater is found from the Kimberleys in northwestern Australia eastwards across the Top End and into Queensland, where it is found from Cape York south across the eastern and central parts of the state, roughly east of a line connecting Karumba, Blackall, Cunnamulla and Currawinya National Park. It has a patchy distribution in New South Wales, occurring in the Northern Rivers and Northern Tablelands regions, and along the coast south to Nambucca Heads. To the south, it is generally absent from the Central and South Coast, and is instead found west of the Great Divide across the South West Slopes and Riverina to the Murray River. It is common in northern Victoria and reaches Bordertown in southeastern South Australia, its range continuing along the Murray. It is also found in the Grampians region, particularly in the vicinity of Stawell, Ararat and St Arnaud, with rare reports from southwestern Victoria. The species occasionally reaches Adelaide, and there is a single record from the Eyre Peninsula. The altitude ranges from sea level to around 850 m (2,790 ft), or rarely 1,000 m (3,300 ft). +In New Guinea, it is found from Merauke in the far southeast of Indonesia's Papua province and east across the Trans-Fly region of southwestern Papua New Guinea. It has also been recorded from the Aru Islands. +The blue-faced honeyeater appears to be generally sedentary within its range, especially in much of the Northern Territory, Queensland and New South Wales. However, in many places (generally south of the Tropic of Capricorn), populations may be present or absent at different times of the year, although this appears to result from nomadic, rather than seasonal, migratory movements. Around Wellington in central New South Wales, birds were recorded over winter months, and were more common in autumn around the Talbragar River. Birds were present all year round near Inverell in northern New South Wales, but noted to be flying eastwards from January to May, and westwards in June and July. In Jandowae in southeastern Queensland, birds were regularly recorded flying north and east from March to June, and returning south and west in July and August, and were absent from the area in spring and summer. +They live throughout rainforest, dry sclerophyll (Eucalyptus) forest, open woodland, Pandanus thickets, paperbarks, mangroves, watercourses, and wetter areas of semi-arid regions, as well as parks, gardens, and golf courses in urban areas. The understory in eucalypt-dominated woodland, where the blue-faced honeyeater is found, is most commonly composed of grasses, such as Triodia, but sometimes it is made up of shrubs or small trees, such as grevilleas, paperbarks, wattles, Cooktown ironwood (Erythrophleum chlorostachys) or billygoat plum (Terminalia ferdinandiana). One study in Kakadu National Park found that blue-faced honeyeaters inhabited mixed stands of eucalypts and Pandanus, but were missing from pure stands of either plant. +The social organisation of the blue-faced honeyeater has been little studied to date. Encountered in pairs, family groups or small flocks, blue-faced honeyeaters sometimes associate with groups of yellow-throated miners (Manorina flavigula). They mob potential threats, such as goshawks (Accipiter spp.), rufous owls (Ninox rufa), and Pacific koels (Eudynamys orientalis). There is some evidence of cooperative breeding, with some breeding pairs recorded with one or more helper birds. Parents will dive at and harass intruders to drive them away from nest sites, including dogs, owls, goannas, and even a nankeen night-heron (Nycticorax caledonicus). A study published in 2004 of remnant patches of forest in central Queensland, an area largely cleared for agriculture, showed a reduced avian species diversity in areas frequented by blue-faced honeyeaters or noisy miners. This effect was more marked in smaller patches. The study concluded that conserved patches of woodland containing the two aggressive species should be larger than 20 ha (44 acres) to preserve diversity. +Social birds, blue-faced honeyeaters can be noisy when they congregate. When feeding in groups, birds seem to keep in contact with each other by soft chirping calls. In Mackay, a bird would fly up 10 or 12 metres (33 or 39 ft) above the treetops calling excitedly to its flock, which would follow and fly around in what was likened to an aerial corroboree, seemingly at play. A single bird was recorded aping and playing with an immature Australian magpie (Gymnorhina tibicen) in Proserpine, Queensland. The blue-faced honeyeater has been reported to be fond of bathing; a flock of 15–20 birds was observed diving into pools one bird at a time, while others were perched in surrounding treetops preening. +The parasite Anoncotaenia globata (a worldwide species not otherwise recorded from Australia) was isolated from a blue-faced honeyeater collected in North Queensland in 1916. The habroneme nematode, Cyrnea (Procyrnea) spirali, has also been isolated from this among other honeyeater species. The nasal mite, Ptilonyssus philemoni, has been isolated from the noisy friarbird (Philemon corniculatus) and blue-faced honeyeater. +The blue-faced honeyeater probably breeds throughout its range. The breeding season is from June to January, with one or two broods raised during this time. The nest is an untidy, deep bowl of sticks and bits of bark in the fork of a tree, Staghorn or bird's nest ferns, or grasstree. Pandanus palms are a popular nest site in Mackay. They often renovate and use the old nests of other species, most commonly the grey-crowned babbler (Pomatostomus temporalis), but also the chestnut-crowned babbler (P. ruficeps), other honeyeaters, including noisy (Philemon corniculatus), little (P. citreogularis) and silver-crowned friarbirds (P. argenticeps), the noisy miner (Manorina melanocephala) and the red wattlebird (Anthochaera carunculata), and artamids, such as the Australian magpie and butcherbird species, and even the magpie-lark. In Coen, an old babbler nest in a paperbark (Melaleuca), which had been lined with messmate bark, had been occupied by blue-faced honeyeaters and re-lined with strips of paperbark. Two or, rarely, three eggs are laid, 22 mm × 32 mm (0.9 in × 1.3 in) and buff-pink splotched with red-brown or purplish colours. The female alone incubates the eggs over a period of 16 or 17 days. +Like those of all passerines, the chicks are altricial; they are born blind and covered only by sparse tufts of brown down on their backs, shoulders and parts of the wings. By four days they open their eyes, and pin feathers emerge from their wings on day six, and the rest of the body on days seven and eight. Both parents feed the young, and are sometimes assisted by helper birds. The Pacific koel (Eudynamys orientalis) and pallid cuckoo (Cuculus pallidus) have been recorded as brood parasites of the blue-faced honeyeater, and the laughing kookaburra recorded as preying on broods. +The blue-faced honeyeater generally forages in the branches and foliage of trees, in small groups of up to seven birds. Occasionally, larger flocks of up to 30 individuals have been reported, and the species has been encountered in a mixed-species foraging flock with the little friarbird (Philemon citreogularis). The bulk of their diet consists of insects, including cockroaches, termites, grasshoppers, bugs such as lerps, scale (Coccidae) and shield bugs (Pentatomidae), beetles such as bark beetles, chafers (subfamily Melolonthinae), click beetles (genus Demetrida), darkling beetles (genera Chalcopteroides and Homotrysis), leaf beetles (genus Paropsis), ladybirds of the genus Scymnus, weevils such as the pinhole borer (Platypus australis), and members of the genera Mandalotus, Polyphrades and Prypnus, as well as flies, moths, bees, ants, and spiders. Blue-faced honeyeaters have been reported preying on small lizards. Prey are caught mostly by sallying, although birds also probe and glean. In Kakadu National Park, birds prefer to hunt prey between the leaf bases of the screw palm (Pandanus spiralis). +The remainder of their diet is made up of plant material, such as pollen, berries, and nectar, from such species as grasstrees (Xanthorrhoea) and scarlet gum (Eucalyptus phoenicea), and from cultivated crops, such as bananas or particularly grapes. In general, birds prefer feeding at cup-shaped sources, such as flowers of the Darwin woollybutt (Eucalyptus miniata), Darwin stringybark (E. tetrodonta) and long-fruited bloodwood (Corymbia polycarpa), followed by brush-shaped inflorescences, such as banksias or melaleucas, gullet-shaped inflorescences such as grevilleas, with others less often selected. +Usually very inquisitive and friendly birds, they will often invade a campsite, searching for edible items, including fruit, insects, and remnants from containers of jam or honey, and milk is particularly favoured. Parent birds feed the young on insects, fruit and nectar, and have been recorded regurgitating milk to them as well. +Keeping blue-faced honeyeaters in an aviary in New South Wales requires a Class 2 Licence. Applicants must show they have appropriate housing, and at least two years' experience of keeping birds. Blue-faced honeyeaters are exhibited at Lincoln Park Zoo in Chicago, Philadelphia Zoo, Columbus Zoo and Aquarium (Ohio), Birmingham Zoo (Alabama), and Tracy Aviary (Utah), Woodland Park Zoo (Seattle) Children's Zoo at Celebration Square (Michigan) in the United States, Marwell Zoo in England, Edinburgh Zoo in Scotland and Taronga Zoo in Sydney, Australia. +Higgins, Peter J.; Peter, Jeffrey M.; Steele, W. K., eds. (2001). Handbook of Australian, New Zealand and Antarctic Birds. Volume 5: Tyrant-flycatchers to Chats. Melbourne: Oxford University Press. ISBN 0-19-553258-9. +Blue-faced honeyeater videos, photos and sounds on the Internet Bird Collection +Sound recording of blue-faced honeyeater on Cornell Laboratory of Ornithology's Macaulay Library website +Meliphagoidea – Highlighting relationships of Meliphagidae on Tree of Life Web Project +The Cock Lane ghost was a purported haunting that attracted mass public attention in 1762. The location was a lodging in Cock Lane, a short road adjacent to London's Smithfield market and a few minutes' walk from St Paul's Cathedral. The event centred on three people: William Kent, a usurer from Norfolk; Richard Parsons, a parish clerk; and Parsons' daughter Elizabeth. +Following the death during childbirth of Kent's wife, Elizabeth Lynes, he became romantically involved with her sister, Fanny. Canon law prevented marriage with a deceased wife's sister but they nevertheless moved to London and lodged at the property in Cock Lane, then owned by Parsons. Several accounts of strange knocking sounds and ghostly apparitions were reported, although for the most part they stopped after the couple moved out, but following Fanny's death from smallpox and Kent's successful legal action against Parsons over an outstanding debt, they resumed. Parsons claimed that Fanny's ghost haunted his property and later his daughter. Regular séances were held to determine "Scratching Fanny's" motives; Cock Lane was often made impassable by the throngs of interested bystanders. +The ghost appeared to claim that Fanny had been poisoned with arsenic and Kent was publicly suspected of being her murderer. But a commission whose members included Samuel Johnson concluded that the supposed haunting was a fraud. Further investigations proved the scam was perpetrated by Elizabeth Parsons, under duress from her father. Those responsible were prosecuted and found guilty; Richard Parsons was pilloried and sentenced to two years in prison. +The Cock Lane ghost became a focus of controversy between the Methodist and Anglican churches and is referenced frequently in contemporary literature. Charles Dickens is one of several Victorian authors whose work alluded to the story and the pictorial satirist William Hogarth referenced the ghost in two of his prints. +In about 1756–57 William Kent, a usurer from Norfolk, married Elizabeth Lynes, the daughter of a grocer from Lyneham. They moved to Stoke Ferry where Kent kept an inn and later, the local post office. They were apparently very much in love, but their marriage was short-lived as within a month of the move Elizabeth died during childbirth. Her sister Frances—commonly known as Fanny—had during Elizabeth's pregnancy moved in with the couple and she stayed to care for the infant and its father. The boy did not survive long and rather than leave, Fanny stayed on to take care of William and the house. The two soon began a relationship, but canon law appeared to rule out marriage; when Kent travelled to London to seek advice he was told that as Elizabeth had borne him a living son, a union with Fanny was impossible. In January 1759 therefore, he gave up the post office, left Fanny and moved to London, intending to "purchase a place in some public office" in the hope that "business would erase that passion he had unfortunately indulged". Fanny meanwhile stayed with one of her brothers at Lyneham. +Despite her family's disapproval of their relationship, Fanny began to write passionate letters to Kent, "filled with repeated entreaties to spend the rest of their lives together". He eventually allowed her to join him at lodgings in East Greenwich near London. The two decided to live together as man and wife, making wills in each other's favour and hoping to remain discreet. In this, however, they did not reckon on Fanny's relations. The couple moved to lodgings near the Mansion House, but their landlord there may have learnt of their relationship from Fanny's family, expressing his contempt by refusing to repay a sum of money Kent loaned him (about £20). In response, Kent had him arrested. +While attending early morning prayers at the church of St Sepulchre-without-Newgate, William Kent and Fanny met Richard Parsons, the officiating clerk. Although he was generally considered respectable, Parsons was known locally as a drunk and was struggling to provide for his family. He listened to the couple's plight and was sympathetic, offering them the use of lodgings in his home on Cock Lane, to the north of St Sepulchre's. Located along a narrow, winding thoroughfare similar to most of central London's streets, the three-storey house was in a respectable but declining area, and comprised a single room on each floor, connected by a winding staircase. Shortly after Mr and Mrs Kent (as they called themselves) moved in, Kent loaned Parsons 12 guineas, to be repaid at a rate of a guinea per month. +It was while Kent was away at a wedding in the country that the first reports of strange noises began. Parsons had a wife and two daughters; the elder, Elizabeth, was described as a "little artful girl about eleven years of age". Kent asked Elizabeth to stay with Fanny, who was then several months into a pregnancy, and to share her bed while he was away. The two reported hearing scratching and rapping noises. These were attributed by Mrs Parsons to a neighbouring cobbler, although when the noises recurred on a Sunday, Fanny asked if the cobbler was working that day; Mrs Parsons told her he was not. James Franzen, landlord of the nearby Wheat Sheaf public house, was another witness. After visiting the house he reported seeing a ghostly white figure ascend the stairs. Terrified, he returned home, where Parsons later visited him and claimed also to have seen a ghost. +As Fanny was only weeks away from giving birth Kent made arrangements to move to a property at Bartlet's Court in Clerkenwell, but by January 1760 it was not ready and so they moved instead to an "inconvenient" apartment nearby, intending only a temporary stay. However, on 25 January Fanny fell ill. The attending doctor diagnosed the early stages of an eruptive fever and agreed with Kent that their lodgings were inadequate for someone at so critical a stage of pregnancy. Fanny was therefore moved, by coach, to Bartlet's Court. The next day her doctor returned and met with her apothecary. Both agreed that Fanny's symptoms were indicative of smallpox. On hearing this, Fanny sent for an attorney, to ensure the will she had had made was in good order, and that Kent would inherit her estate. An acquaintance of Kent's, Stephen Aldrich, Rector of St John Clerkenwell, reassured her that she would be forgiven for her sins. She died on 2 February. +As sole executor of Fanny's will, Kent ordered a coffin, but fearful of being prosecuted should the nature of their relationship become known, asked that it remain nameless. On registering the burial he was, however, forced to give a name, and he gave her his own. Fanny's family was notified and her sister Ann Lynes, who lived nearby at Pall Mall, attended the funeral at St John's. When Ann learned of the terms of Fanny's will, which left her brothers and sisters half a crown each and Kent the rest, she tried but failed to block it in Doctors' Commons. The bulk of Kent's inheritance was Fanny's £150 share of her dead brother Thomas's estate. This also included some land owned by Thomas, sold by the executor of his estate, John Lynes, and Kent received Fanny's share of that too (almost £95). Her family resented this. Legal problems with Lynes's sale meant that each of Thomas's beneficiaries had to pay £45 in compensation to the purchaser, but Kent refused, claiming that he had already spent the money in settling Fanny's debts. In response to this, in October 1761 John Lynes began proceedings against Kent in the Court of Chancery. Meanwhile, Kent became a stockbroker and remarried in 1761. +Echoing the actions of Kent's previous landlord, Parsons had not repaid Kent's loan—of which about three guineas was outstanding—and Kent therefore instructed his solicitor to sue him. He managed to recover the debt by January 1762, just as the mysterious noises at Cock Lane began again. Catherine Friend had lodged there shortly after the couple left but moved out when she found the noises, which had returned intermittently and which were becoming more frequent, could not be stopped. They apparently emanated from Elizabeth Parsons, who also suffered fits, and the house was regularly disturbed by unexplained noises, likened at the time to the sound of a cat scratching a chair. Reportedly determined to discover their source, Richard Parsons had a carpenter remove the wainscotting around Elizabeth's bed. He approached John Moore, assistant preacher at St Sepulchre's since 1754 and rector of St Bartholomew-the-Great in West Smithfield since June 1761. The presence of one ghost, presumed to belong to Fanny's sister, Elizabeth, had already been noted while Fanny lay dying, and the two concluded that the spirit now haunting Parsons' house must be that of Fanny Lynes herself. The notion that a person's spirit might return from the dead to warn those still alive was a commonly held belief, and the presence of two apparently restless spirits was therefore an obvious sign to both men that each ghost had an important message to disclose. +Parsons and Moore devised a method of communication; one knock for yes, two knocks for no. Using this system, the ghost appeared to claim that Fanny had been murdered. It was conjectured that the mysterious figure in white which so terrified James Franzen, presumed to be the ghost of Elizabeth, had appeared there to warn her sister of her impending death. As the first ghost had seemingly vanished, this charge against Kent—that he murdered Elizabeth—was never acted on, but through repeated questioning of Fanny's ghost it was divined that she had died not from the effects of smallpox, but rather from arsenic poisoning. The deadly toxin had apparently been administered by Kent about two hours before Fanny died and now, it was supposed, her spirit wanted justice. Moore had heard from Parsons how Kent had pursued the debt he was owed, and he had also heard from Ann Lynes, who had complained that as Fanny's coffin lid was screwed down she had not been able to see her sister's corpse. Moore thought that Fanny's body might not show any visible signs of smallpox and that if she had been poisoned, the lack of scarring would have been something Kent would rather keep hidden. As a clergyman with inclinations toward Methodism he was inclined to trust the ghost, but for added support he enlisted the aid of Thomas Broughton, an early Methodist. Broughton visited Cock Lane on 5 January and left convinced the ghost was real. The story spread through London, The Public Ledger began to publish detailed accounts of the phenomenon, and Kent fell under public suspicion as a murderer. +After reading the veiled accusations made against him in the Public Ledger, Kent, determined to clear his name, and accompanied by a witness, went to see John Moore. The Methodist showed Kent the list of questions he and Parsons had drawn up for the ghost to answer. One concerned William and Fanny's marital status, prompting Kent to admit that they never married. Moore told him he did not think he was a murderer, rather, he believed the spirit's presence indicated that "there was something behind darker than all the rest, and that if he would go to Parson's house, he might be a witness to the same and convinced of its reality". On 12 January therefore, Kent enlisted the aid of the two physicians who attended Fanny in her last days, and with Broughton, went to Cock Lane. On the house's upper floor Elizabeth Parsons was publicly undressed, and with her younger sister was put to bed. The audience sat around the bed, positioned in the centre of the room. They were warned that the ghost was sensitive to disbelief and told that they should accord it due respect. When the séance began, a relative of Parsons, Mary Frazer, ran around the room shouting "Fanny, Fanny, why don't you come? Do come, pray Fanny, come; dear Fanny, come!" When nothing happened, Moore told the group the ghost would not come as they were making too much noise. He asked them to leave the room, telling them he would try to contact the ghost by stamping his foot. About ten minutes later they were told the ghost had returned and that they should re-enter the room. Moore then started to run through his and Parsons' list of questions: +"Are you the wife of Mr. Kent?" —Two knocks +"Did you die naturally?" —Two knocks +"By poison?" —One knock +"Did any person other than Mr. Kent administer it?" —Two knocks +After more questions, a member of the audience exclaimed "Kent, ask this Ghost if you shall be hanged". He did so, and the question was answered by a single knock. Kent exclaimed "Thou art a lying spirit, thou are not the ghost of my Fanny. She would never have said any such thing." +Public interest in the story grew when it was discovered that the ghost appeared to follow Elizabeth Parsons. She was removed to the house of a Mr Bray, where on 14 January, in the presence of two unidentified nobles, more knocking sounds were heard. A few days later she was returned to Cock Lane, where on 18 January another séance was held. In attendance were Kent, the apothecary, and local parish priest and incumbent of St John Clerkenwell, Stephen Aldrich. On that occasion, when a clergyman used a candle to look under the bed, the ghost "refused" to answer, Frazer claiming "she [the ghost] loving not light". After a few minutes of silence the questioning continued, but when Moore asked if the ghost would appear in court against Kent, Frazer refused to ask the question. +When they lived at Cock Lane William and Fanny had employed a maid, Esther "Carrots" Carlisle (Carrots on account of her red hair). She had since moved to a new job and knew nothing of the haunting, but seeking evidence of Fanny's poisoning, Moore went to question her. Carrots told him that Fanny had been unable to speak in the days before she died, so Moore invited her to a séance, held on 19 January. Once there, she was asked to confirm that Fanny had been poisoned, but Carrots remained adamant that Fanny had said nothing to her, telling the party that William and Fanny had been "very loving, and lived very happy together." Kent arrived later that night, this time with James Franzen and priests William Dodd and Thomas Broughton. Frazer began with her usual introduction before Moore sent her out, apparently irritated by her behaviour. He then asked the party of about 20 to leave the room, calling them back a few minutes later. This time, the séance centred on Carrots, who addressed the ghost directly: +"Are you my mistress?" —One knock, followed by scratches +"Are you angry with me, Madam?" —One knock +"Then I am sure, Madam, you may be ashamed of yourself for I never hurt you in my life." +At this, the séance was ended. Frazer and Franzen remained alone in the room, the latter reportedly too terrified to move. Frazer asked if he would like to pray and was angered when he apparently could not. The séance resumed and Franzen later returned to his home, where he and his wife were reportedly tormented by the ghost's knocking in their bedchamber. +On 20 January another séance was held, this time at the home of a Mr Bruin, on the corner of nearby Hosier Lane. Among those attending was a man "extremely desirous of detecting the fraud, and discovering the truth of this mysterious affair", who later sent his account of the night to the London Chronicle. He arrived with a small party which included James Penn of St Ann's in Aldersgate. Inside the house, a member of the group positioned himself against the bed, but was asked by one of the ghost's sympathisers to move. He refused, and following a brief argument the ghost's supporters left. The gentleman then asked if Parsons would allow his daughter to be moved to a room at his house, but was refused. For the remainder of the night the ghost made no sound, while Elizabeth Parsons, now extremely agitated, displayed signs of convulsions. When questioned she confirmed that she had seen the ghost, but that she was not frightened by it. At that point several of the party left, but at about 7 am the next morning the knocking once more recommenced. Following the usual questions about the cause of Fanny's death and who was responsible, the interrogation turned to her body, which lay in the vaults of St John's. +Parsons agreed to move his daughter to Aldrich's house for further testing on 22 January, but when that morning Penn and a man of "veracity and fortune" called on Parsons and asked for Elizabeth, the clerk told them she was not there and refused to reveal her whereabouts. Parsons had spoken with friends and was apparently worried that Kent had been busy with his own investigations. Instead, he allowed Elizabeth to be moved that night to St Bartholomew's Hospital, where another séance was held. Nothing was reported until about 6 am, when three scratches were heard, apparently while the girl was asleep. The approximately 20-strong audience complained that the affair was a deception. Once Elizabeth woke she began to cry, and once reassured that she was safe admitted that she was afraid for her father, "who must needs be ruined and undone, if their matter should be supposed to be an imposture." She also admitted that although she had appeared to be asleep, she was in fact fully aware of the conversation going on around her. +Initially only the Public Ledger reported on the case, but once it became known that noblemen had taken an interest and visited the ghost at Mr Bray's house on 14 January, the story began to appear in other newspapers. The St. James's Chronicle and the London Chronicle printed reports from 16 to 19 January (the latter the more sceptical of the two), and Lloyd's Evening Post from 18 to 20 January. The story spread across London and by the middle of January the crowds gathered outside the property were such that Cock Lane was rendered impassable. Parsons charged visitors an entrance fee to "talk" with the ghost, which, it was reported, did not disappoint. After receiving several requests to intercede, Samuel Fludyer, Lord Mayor of London, was on 23 January approached by Alderman Gosling, John Moore and Parsons. They told him of their experiences but Fludyer was reminded of the then recent case of fraudster Elizabeth Canning and refused to have Kent or Parsons arrested (on charges of murder and conspiracy respectively). Instead, against a backdrop of hysteria caused in part by the newspapers' relentless reporting of the case, he ordered that Elizabeth be tested at Aldrich's house. Meanwhile, Elizabeth was again the subject of study, in two séances held 23–24 January. Parsons accepted the Lord Mayor's decision, but asked that "some persons connected with the girl might be permitted to be there, to divert her in the day-time". This was refused, as were two similar requests, Aldrich and Penn insisting that they would accept only "any person or persons, of strict character and reputation, who are housekeepers". Aldrich and Penn's account of their negotiations with Parsons clearly perturbed the clerk, as he defended his actions in the Public Ledger. This prompted Aldrich and Penn to issue a pointed retort in Lloyd's Evening Post: "We are greatly puzzled to find Mr. Parsons asserting that he hath been always willing to deliver up the child, when he refused a gentleman on Wednesday evening the 20th inst. [...] What is to be understood, by requiring security"? +Elizabeth was taken on 26 January to the house of Jane Armstrong, sleeping there in a hammock. The continued noises strengthened the resolve of the ghost's supporters, while the press's ceaseless reporting of the case continued. Horace Walpole, 4th Earl of Orford, announced that with the Duke of York, Lady Northumberland, Lady Mary Coke and Lord Hertford, he was to visit Cock Lane on 30 January. After struggling through the throngs of interested visitors though, he was ultimately disappointed; the Public Advertiser observed that "the noise is now generally deferred till seven in the morning, it being necessary to vary the time, that the imposition may be the better carried on". +With Lord Dartmouth Aldrich began to draw together the people who would be involved in his investigation. They chose the matron of a local lying-in hospital as principal lady-in-waiting, the critic and controversialist Bishop John Douglas, and Dr George Macaulay. A Captain Wilkinson was also included on the committee; he had attended one séance armed with a pistol and stick; the former to shoot the source of the knocking, and the latter to make his escape (the ghost had remained silent on that occasion). James Penn and John Moore were also on the committee, but its most prominent member was Samuel Johnson, who documented the séance, held on 1 February 1762: +On the night of the 1st of February many gentlemen eminent for their rank and character were, by the invitation of the Reverend Mr. Aldrich, of Clerkenwell, assembled at his house, for the examination of the noises supposed to be made by a departed spirit, for the detection of some enormous crime. About ten at night the gentlemen met in the chamber in which the girl, supposed to be disturbed by a spirit, had, with proper caution, been put to bed by several ladies. They sat rather more than an hour, and hearing nothing, went down stairs, when they interrogated the father of the girl, who denied, in the strongest terms, any knowledge or belief of fraud. The supposed spirit had before publickly promised, by an affirmative knock, that it would attend one of the gentlemen into the vault under the Church of St. John, Clerkenwell, where the body is deposited, and give a token of her presence there, by a knock upon her coffin; it was therefore determined to make this trial of the existence or veracity of the supposed spirit. While they were enquiring and deliberating, they were summoned into the girl's chamber by some ladies who were near her bed, and who had heard knocks and scratches. When the gentlemen entered, the girl declared that she felt the spirit like a mouse upon her back, and was required to hold her hands out of bed. From that time, though the spirit was very solemnly required to manifest its existence by appearance, by impression on the hand or body of any present, by scratches, knocks, or any other agency, no evidence of any preter-natural power was exhibited. The spirit was then very seriously advertised that the person to whom the promise was made of striking the coffin, was then about to visit the vault, and that the performance of the promise was then claimed. The company at one o'clock went into the church, and the gentleman to whom the promise was made, went with another into the vault. The spirit was solemnly required to perform its promise, but nothing more than silence ensued: the person supposed to be accused by the spirit, then went down with several others, but no effect was perceived. Upon their return they examined the girl, but could draw no confession from her. Between two and three she desired and was permitted to go home with her father. It is, therefore, the opinion of the whole assembly, that the child has some art of making or counterfeiting a particular noise, and that there is no agency of any higher cause. +Disappointed that the ghost had failed to reveal itself, Moore now told Kent he believed it was an imposter, and that he would help reveal it. Kent asked him to admit the truth and write an affidavit of what he knew, so as to end the affair and restore Kent's reputation, but Moore refused, telling him that he still believed that the spirit's presence was a reminder of his sin. Moore's view of the couple's relationship was shared by many, including Mrs Parsons, who believed that the supposed ghost of Elizabeth Kent had disapproved of her sister's new relationship. +Another séance on 3 February saw the knocking continue unabated, but by then Parsons was in an extremely difficult—and serious—situation. Keen to prove the ghost was not an imposture he allowed his daughter to be examined at a house on The Strand from 7–10 February, and at another house in Covent Garden from 14 February. There she was tested in a variety of ways which included being swung up in a hammock, her hands and feet extended. As expected, the noises commenced, but stopped once Elizabeth was made to place her hands outside the bed. For two nights the ghost was silent. Elizabeth was told that if no more noises were heard by Sunday 21 February, she and her father would be committed to Newgate Prison. Her maids then saw her conceal on her person a small piece of wood about 6 by 4 inches (15 by 10 cm) and informed the investigators. More scratches were heard but the observers concluded that Elizabeth was responsible for the noises, and that she had been forced by her father to make them. Elizabeth was allowed home shortly after. +On or about 25 February, a pamphlet sympathetic to Kent's case was published, called The Mystery Revealed, and most likely written by Oliver Goldsmith. Meanwhile, Kent was still trying to clear his name, and on 25 February he went to the vault of St John's, accompanied by Aldrich, the undertaker, the clerk and the parish sexton. The group was there to prove beyond any doubt that a recent newspaper report, which claimed that the supposed removal of Fanny's body from the vault accounted for the ghost's failure to knock on her coffin, was false. The undertaker removed the lid to expose Fanny's corpse, "and a very awful shocking sight it was". For Moore this was too much and he published his retraction: +In justice to the person, whose reputation has been attacked in a most gross manner, by the pretended Ghost in Cock-lane; to check the credulity of the weak; to defeat the attempts of the malicious, and to prevent further imposition, on account of this absurd phenomenon, I do hereby certify, that though, from the several attendances on this occasion, I have not been able to point out, how, and in what manner, those knockings and scratchings, of the supposed Ghost, were contrived, performed, and continued; yet, that I am convinced, that those knockings and scratchings were the effects of some artful, wicked contrivance; and that I was, in a more especial manner, convinced of its being such, on the first of this month, when I attended with several persons of rank and character, who assembled at the Rev. Mr. Aldrich's, Clerkenwell, in order to examine into this iniquitous imposition upon the Public. Since which time I have not seen the child, nor heard the noises; and think myself in duty bound to add, that the injured person (when present to hear himself accused by the pretended Ghost) has not, by his behaviour, given the least ground of suspicion, but has preserved that becoming steadfastness, which nothing, I am persuaded, but innocence could inspire. +It was not enough to keep him from being charged by the authorities with conspiracy, along with Richard Parsons and his wife, Mary Frazer, and Richard James, a tradesman. +The trial of all five was held at the Guild Hall in London on 10 July 1762. Presiding over the case was Lord Chief Justice William Murray. Proceedings began at 10 am, "brought by William Kent against the above defendants for a conspiracy to take away his life by charging him with the murder of Frances Lynes by giving her poison whereof she died". The courtroom was crowded with spectators, who watched as Kent gave evidence against those in the dock. He told the court about his relationship with Fanny and of her resurrection as "Scratching Fanny" (so-called because of the scratching noises made by the "ghost"). James Franzen was next on the stand, his story corroborated by Fanny's servant, Esther "Carrots" Carlisle, who testified later that day. Dr Cooper, who had served Fanny as she lay dying, told the court that he had always believed the strange noises in Cock Lane to be a trick, and his account of Fanny's illness was supported by her apothecary, James Jones. Several other prosecution witnesses described how the hoax had been revealed, and Richard James was accused by the prosecution's last witness of being responsible for some of the more offensive material published in the Public Ledger. +The defence's witnesses included some of those who had cared for Elizabeth Parsons and who presumably still believed that the ghost was real. Other witnesses included the carpenter responsible for removing the wainscotting from Parsons' apartment and Catherine Friend, who to escape the knocking noises had left the property. One witness's testimony caused the court to burst into laughter, at which she replied "I assure you gentlemen, it is no laughing matter, whatever you may think of it." Thomas Broughton was also called, as was a priest surnamed Ross, one of those who had questioned the ghost. Judge Murray asked him "Whether he thought he had puzzled the Ghost, or the Ghost had puzzled him?" John Moore was offered support by several esteemed gentlemen and presented Murray with a letter from Thomas Secker, Archbishop of Canterbury, who sought to intercede on his behalf. Murray placed the letter in his pocket, unopened, and told the court "it was impossible it could relate to the cause in question." Richard James and Richard Parsons also received support from various witnesses, some of whom although acknowledging Parsons' drink problem, told the court they could not believe he was guilty. +The trial ended at about 9:30 pm. The judge spent about 90 minutes summing up the case, but it took the jury only 15 minutes to reach a verdict of guilty for all five defendants. The following Monday, two others responsible for defaming Kent were found guilty and later fined £50 each. The conspirators were brought back on 22 November but sentencing was delayed in the hope that they could agree on the level of damages payable to Kent. Having failed to do so they returned on 27 January 1763 and were committed to the King's Bench Prison until 11 February, by which time John Moore and Richard James had agreed to pay Kent £588; they were subsequently admonished by Justice Wilmot and released. The following day, the rest were sentenced: +The Court chusing that Mr. Kent, who had been so much injured on the occasion, should receive some reparation by punishment of the offenders, deferred giving judgment for seven or eight months, in hopes that the parties might make it up in the meantime. Accordingly, the clergyman, and tradesman agreed to pay Mr. Kent a round sum—some say between £500 and £600 to purchase their pardon, and were, therefore, dismissed with a severe reprimand. The father was ordered to be set in the pillory three times in one month—once at the end of Cock–Lane; Elizabeth his wife to be imprisoned one year; and Mary Frazer six months in Bridewell, with hard labour. The father appearing to be out of his mind at the time he was first to standing in the pillory, the execution of that part of his sentence was deferred to another day, when, as well as the other day of his standing there, the populace took so much compassion on him, that instead of using him ill, they made a handsome subscription for him. +Parsons, all the while protesting his innocence, was also sentenced to two years' imprisonment. He stood in the pillory on 16 March, 30 March and finally on 8 April. In contrast to other criminals the crowd treated him kindly, making collections of money for him. +The Cock Lane ghost was a focus for a contemporary religious controversy between the Methodists and orthodox Anglicans. Belief in a spiritual afterlife is a requirement for most religions, and in every instance where a spirit had supposedly manifested itself in the real world, the event was cherished as an affirmation of such beliefs. In his youth, John Wesley had been strongly influenced by a supposed haunting at his family home and these experiences were carried through to the religion he founded, which was regularly criticised for its position on witchcraft and magic. Methodism, although far from a united religion, became almost synonymous with a belief in the supernatural. Some of its followers therefore gave more credence to the reality of the Cock Lane ghost than did the Anglican establishment, which considered such things to be relics of the country's Catholic past. This was a view that was epitomised in the conflict between the Methodist John Moore and the Anglican Stephen Aldrich. In his 1845 memoirs, Horace Walpole, who had attended one of the séances, accused the Methodists of actively working to establish the existence of ghosts. He described the constant presence of Methodist clergymen near Elizabeth Parsons and implied that the church would recompense her father for his troubles. +Samuel Johnson was committed to his Christian faith and shared the views of author Joseph Glanvill, who, in his 1681 work Saducismus Triumphatus, wrote of his concern over the advances made against religion and a belief in witchcraft, by atheism and scepticism. For Johnson the idea that an afterlife might not exist was an appalling thought, but although he thought that spirits could protect and counsel those still living, he kept himself distant from the more credulous Methodists, and recognised that his religion required proof of an afterlife. Ever a sceptic, in his discussions with his biographer James Boswell, he said: +Sir, I make a distinction between what a man may experience by the mere strength of his imagination, and what imagination cannot possibly produce. Thus, suppose I should think I saw a form, and heard a voice cry, "Johnson, you are a very wicked fellow, and unless you repent you will certainly be punished;" my own unworthiness is so deeply impressed upon my mind, that I might imagine I thus saw and heard, and therefore I should not believe that an external communication had been made to me. But if a form should appear, and a voice tell me that a particular man had died at a particular place, and a particular hour, a fact which I had no apprehension of, nor any means of knowing, and this fact, with all its circumstances, should afterwards be unquestionably proved, I should, in that case, be persuaded that I had supernatural intelligence imparted to me. +Johnson's role in revealing the nature of the hoax was not enough to keep the satirist Charles Churchill from mocking his apparent credulity in his 1762 work The Ghost. He resented Johnson's lack of enthusiasm for his writing and with the character of 'Pomposo', written as one of the more credulous of the ghost's investigators, used the satire to highlight a "superstitious streak" in his subject. Johnson paid this scant attention, but was said to have been more upset when Churchill again mocked him for his delay in releasing Shakespeare. Publishers were at first wary of attacking those involved in the supposed haunting, but Churchill's satire was one of a number of publications which, following the exposure of Parsons' deception, heaped scorn on the affair. The newspapers searched for evidence of past impostures and referenced older publications such as Reginald Scot's Discoverie of Witchcraft (1584). The ghost was referenced in an anonymous work entitled Anti-Canidia: or, Superstition Detected and Exposed (1762), which sought to ridicule the credulity of those involved in the Cock Lane case. The author described his work as a "sally of indignation at the contemptible wonder in Cock-lane". Works such as The Orators (1762) by Samuel Foote, were soon available. Farcical poems such as Cock-lane Humbug were released, theatres staged plays such as The Drummer and The Haunted House. +Oliver Goldsmith, who had in February 1762 published The Mystery Revealed, may also have been responsible for the satirical illustration, English Credulity or the Invisible Ghost (1762). It shows a séance as envisioned by the artist, with the ghost hovering above the heads of the two children in the bed. To the right of the bed a woman deep in prayer exclaims "O! that they would lay it in the Red Sea!" Another cries "I shall never have any rest again". The English magistrate and social reformer John Fielding, who was blind, is pictured entering from the left saying "I should be glad to see this spirit", while his companion says "Your W——r's had better get your Warrant back'd by his L—rds—p", referring to a Middlesex magistrate's warrant which required an endorsement from the Lord Mayor, Samuel Fludyer. A man in tall boots, whip in hand, says: "Ay Tom I'll lay 6 to 1 it runs more nights than the Coronation" and his companion remarks "How they swallow the hum". A clergymen says "I saw the light on the Clock" while another asks "Now thou Infidel does thou not believe?", prompting his neighbour to reply "Yes if it had happen'd sooner 't would have serv'd me for a new Character in the Lyar the Story would tell better than the Cat & Kittens". Another clergyman exclaims "If a Gold Watch knock 3 times", and a Parson asks him "Brother don't disturb it". On the wall, an image of The Bottle Conjuror is alongside an image of Elizabeth Canning, whose fraud had so worried Samuel Fludyer that he had refused to arrest either Parsons or Kent. +Playwright David Garrick dedicated the enormously successful The Farmer's Return from London to the satirical artist William Hogarth. The story concerns a farmer who regales his family with an account of his talk with Miss Fanny, the comedy being derived from the reversal of traditional roles: the sceptical farmer poking fun at the credulous city-folk. Hogarth made his own observations of the Cock Lane ghost, with obvious references in Credulity, Superstition and Fanaticism (1762). This illustration makes a point of attacking Methodist ministers, one of whom is seen to slip a phallic "ghost" into a young woman's bodice. He again attacked the Methodists in The Times, Plate 2 (1762–1763), placing an image of Thomas Secker (who had tried to intervene on behalf of the Methodists) behind the Cock Lane ghost, and putting the ghost in the same pillory as the radical politician John Wilkes, which implied a connection between the demagoguery surrounding the Methodists and Pittites. The print enraged Bishop William Warburton, who although a vocal critic of Methodism, wrote: +I have seen Hogarth's print of the Ghost. It is a horrid composition of lewd Obscenity & blasphemous prophaneness for which I detest the artist & have lost all esteem for the man. The best is, that the worst parts of it have a good chance of not being understood by the people. +The 19th-century author Charles Dickens—whose childhood nursemaid Mary Weller may have affected him with a fascination for ghosts—made reference to the Cock Lane ghost in several of his books. One of Nicholas Nickleby's main characters and a source of much of the novel's comic relief, Mrs. Nickleby, claims that her great-grandfather "went to school with the Cock-lane Ghost" and that "I know the master of his school was a Dissenter, and that would in a great measure account for the Cock-lane Ghost's behaving in such an improper manner to the clergyman when he grew up." Dickens also very briefly mentions the Cock Lane ghost in A Tale of Two Cities and Dombey and Son. +Hammersmith Ghost murder case +Footnotes +Notes +Bibliography +Bakewell, Sarah (2001), "Scratching Fanny", Fortean Times, vol. 150, pp. 30–33 +Churchill, Charles (1762), The Ghost, London: Printed for the author and sold by William Flexney +Garcia Oliva, Javier, and Helen Hall. "The Trial of the Cock Lane Ghost: Criminal Law, Religious Orthodoxy and Social Values in Enlightenment London." Law and Justice: The Christian Law Review 190 (2023): 40+ +Gaston, Sean. "An Event Without An Object: The Cock Lane Ghost, London." The Literary London Journal 12#1–2 (Spring/Autumn 2015) online. +Lewis, Jane Elizabeth. "Sight Unseen: The Cock Lane Ghost among the Infidels", in Sustaining Literature: Essays on Literature, History, and Culture 1500-1800: Commemorating the Life and Work of Simon Varney (Lewisburg: Bucknell University Press, 2007) pp. 247–70 +Neil Brooks (born 27 July 1962) is an Australian former sprint freestyle swimmer best known for winning the 4 × 100 m medley relay at the 1980 Olympics in Moscow as part of the Quietly Confident Quartet. Brooks was as much known for his swimming achievements as he was for disciplinary incidents, and he often found himself in conflict with officialdom and threatened with sanctions. +Born in England, Brooks emigrated to Australia as a toddler and started swimming lessons after nearly drowning in a childhood accident. After initially being known for his lack of technique, Brooks quickly rose through the youth ranks. Brooks made his debut at the Australian Championships in 1976, but it was not until 1979 that he medalled at national level and made his debut for Australia at a FINA (Fédération Internationale de Natation) Swimming World Cup meet. In 1980, he gained prominence by breaking the Australian record in the 100 m freestyle and being invited to a national team camp. There he had his first clash with officialdom, walking out after accusing the officials of neglecting him. He then qualified for the Australian team for the 1980 Moscow Olympics, defying political pressure to boycott the Games in the wake of the Soviet invasion of Afghanistan. Arriving in Moscow, Brooks' experience in the 100 m freestyle was an unpleasant one, suffering an asthma attack and missing the final. The peak of his swimming career came in the 4 × 100 m medley relay, when he caught and passed the Soviet Union's Sergey Kopliakov during the anchor leg to seal a narrow victory for Australia. This victory was the first time that the United States did not win the event at Olympic level, and the last until China's victory in 2024. +Following the Olympics, Brooks was expelled from the Australian Institute of Sport by Don Talbot for disciplinary reasons. He accepted a swimming scholarship at the University of Arkansas, where he enjoyed the more liberal disciplinary standards. He returned to Australia for the 1982 Commonwealth Games in Brisbane and again raised the ire of officials during a preparatory training camp. After lobbying for improved accommodation conditions, Brooks was involved in a physical altercation with the team manager. As a result, he was given a suspension that was to take effect after the Commonwealth Games. However, his teammates protested and threatened to walk out, resulting in the ban being rescinded. Despite the turbulent preparation, Brooks had a successful meet, winning the 100 m freestyle and anchoring the 4 × 100 m freestyle and medley relays to gold medals. Brooks competed at his second Olympics in Los Angeles in 1984 Games, where he won silver in the 4 × 100 m freestyle relay and bronze for swimming the heats of the medley relay. +Brooks' international career ended at the 1986 Commonwealth Games in Edinburgh with silver in the 100 m freestyle and gold in the freestyle relay, after which he was suspended for drinking 46 cans of beer on the return flight to Australia. In retirement, he became a news presenter and sports commentator, but was sacked in 1999 after a string of incidents amidst an alcohol addiction. Later in life he faced a number of legal issues. +An only child, Brooks was born in Crewe, England, before migrating to Western Australia when he was four, along with his working-class parents Mick and Norah. His first aquatic adventure was almost his last. Aged seven, he was playing with a friend on the shore of the Swan River when they climbed into a boat that drifted deeper into water and overturned. Brooks was forced to cling to the boat as his friend swam ashore to seek help. His parents immediately enrolled him in swimming lessons at the Marylands Swim Club. Shortly after, he switched to the tutelage of Kevin Duff, who coached him for the next fifteen years. After just six weeks under Duff, Brooks came third in the 50 m breaststroke at the State Age Championships. Despite coaching the likes of Olympic medallists Kevin O'Halloran, Lyn McClements, David Dickson and Lynne Watson, Duff was virtually unknown outside Western Australia. Brooks was known for his rebellious nature, and had a glowing assessment of his coach, opining that "He's not pushy like many other Australian coaches and he's not in the politics of Australian swimming". +At the age of 13, Brooks suffered a loss of confidence. After being champion in all four strokes for the previous four years, he was now frequently losing. The other children had grown more at the start of their adolescence and he was struggling to match them. Within a year, Brooks' physical growth began to catch up and he started to regain the dominant position. He also switched from distance to sprint events. +Brooks attended Hale Primary School and trained at Beatty Park Pool, routinely dominating the State Age Championships. He won bronze in the 100 m and 200 m backstroke, and silver in the 200 m freestyle at the 1974 Australian Age Championships. Owing to a lack of style, he was known during his primary school years as "Basher Brooks", but by the time he entered Churchlands Senior High School, his stroke had become more technically refined. Nevertheless, he was always confident in his mental ability, stating "before I just swam on guts, now I had the stroke as well as the guts. I was always the toughest kid in the race." +In 1976, at the age of 13, Brooks competed at his first open Australian Championships, but did not gain any podium finishes and as such missed selection in the team for the 1976 Summer Olympics in Montreal. In the same year, the Australian team came to Perth for its pre-Olympic training camp and were billeted in the homes of members of the local swimming community. Brooks' family took his future relay team-mate Mark Tonelli, who had a reputation for indiscipline. Brooks, however, was inspired by Tonelli and cited him as a key motivating factor in him wanting to become an Olympian. +During the 1976–77 season, Brooks came to prominence as a possible Australian representative sprinter. He broke six records in winning two events at the 1977 Western Australian Championships. He swam the 100 m freestyle in 56.56 s, breaking the state records for 14-, 15-, and 16-year-olds, before repeating the achievement in the 200 m backstroke. Still aged 14, he competed at his second Australian Championships and came fourth in the 100 m freestyle. In 1978, he came third in the 100 m freestyle and missed selection for the 1978 Commonwealth Games in Edmonton, Canada by 0.03 s. He spent the rest of the year training and studying accounting at Leederville Technical College. During the year, Brooks set Australian age group records in the 200 m individual medley, the 200 m backstroke and the 100 m freestyle. At the inaugural Australian Short Course Championships in Launceston, Tasmania in 1979, he won silver in the 200 m individual medley and bronze in the 100 m freestyle. His performances earned him selection in the Australian team for the first time, competing in the FINA Swimming World Cup event in Tokyo in April 1979, aged 16 years and nine months. +Aged 17, Brooks swam 51.91 s in Perth in January 1980, breaking Tonelli's Australian record in the 100 m freestyle. As a result, he was invited to his first national training camp under Bill Sweetenham for prospective Olympic swimmers. He had the first of his many clashes with sporting administrators, first claiming that the officials did not want him to board with Tonelli's family and then stating that Sweetenham had only coached him two or three times, which he felt was insufficient. Brooks walked out of the camp and returned to Perth to train under Duff. At the Australian Championships, the 100 m was seen as a clash between Tonelli and Brooks. The latter came second in the 100 m freestyle, outsprinted in the dying stages by Tonelli, who reclaimed his national record in a time of 51.80 s. As a result, Brooks was selected in both the individual event and the 4 × 100 m medley relay. +However, another obstacle arose with the Soviet invasion of Afghanistan, which resulted in a boycott of the Games by a large part of the Western World, led by the United States. The Australian Prime Minister Malcolm Fraser was also the patron of the Australian Olympic Committee, and significant political pressure came to bear on the athletes to boycott the Games. Tonelli, however, realised that only the sportspeople would suffer from a boycott and that trade relations would continue unabated. He took a leadership role among the athletes to fight for their right to compete. +The 4 × 100 m medley relay was the focal point of Brooks' Moscow campaign and it came only two days before his 18th birthday. The event had always been won by the United States since its inception at Olympic level in 1960, and their boycott had opened up the field in the event. In the five times the event had been contested, Australia's best result was a silver in the inaugural race. A bronze in 1964 was the only other medal success and the 1976 edition of the medley relay had seen Australia eliminated in the heats. This time, Australia were regarded as a medal chance, but were not seen as the main threats; Sweden, Great Britain and the Soviet Union were the most heavily fancied teams. The hosts boasted the silver medallists in the 100 m backstroke and breaststroke, and their butterflyer had come fifth; their freestyler would place fourth a few days later. The British had Duncan Goodhew, the breaststroke gold medallist, while Sweden's butterflyer and backstroker had won their respective events and their freestyle swimmer would come second in the 100 m. On paper, Australia's team paled in comparison. Peter Evans was the only individual medallist over a 100 m race, claiming bronze in the breaststroke. Mark Kerry had been eliminated in the backstroke semifinals, while Tonelli was swimming as a makeshift butterflyer, despite having performed better than Kerry in the 100 m backstroke. Adding to the pressure was that Australia won no gold medals at the 1976 Olympics in any sport, and were yet to win in Moscow, so the public were still awaiting their first victory since Munich in 1972. Coming into the Olympics, Australia were ranked seventh out of the thirteen competing countries. Australia's prospects improved after the morning heats in which Sweden was disqualified. Tonelli, the eldest swimmer in the quartet at the age of 23, convened the team as its de facto leader. He asked his team-mates to commit to swimming their legs in a certain time; Kerry vowed to swim the backstroke in 57 s, Evans the breaststroke in 63 s flat, Tonelli the butterfly in 54 s and Brooks promised to anchor the team in 49.8 s, even though he had never gone faster than 51 s. Tonelli named the foursome as the Quietly Confident Quartet, and they exhibited a quiet confidence as they lined up for the race. +Kerry led off in a faster time than he had clocked in the individual event, but it was still two seconds slower than his personal best time of 57.87 s. This left Australia in fourth place at the end of the first leg. Evans then swam a personal best of 63.01 s, leaving the team almost level with the host nation at the halfway mark. Tonelli then swam his leg in 54.94 s, almost two seconds faster than his previous best. He did so with an uneven arm technique due to the disparity in the strength of his arms. He began to lose ground in the last 50 m and was a bodylength behind until a late surge brought him to within a metre of the lead by the end of his leg. Brooks then made a powerful, well-timed dive and surfaced almost even with his Soviet counterpart. At the halfway mark, he had drawn level and made a superior turn to take the lead. The Soviet freestyler Kopliakov pulled level at the 25 m mark before Brooks again sprinted away to seal an Australian victory by 0.22 s. He did not breathe in the last ten metres, and claimed to be laughing for the final five metres, confident that his opponent could not pass him. The Australian freestyler had finished his leg in 49.86 s as he had vowed to his team mates. The time of 3 m 45.70 s sealed Australia's first ever win in a medley relay at the Olympics, for men or women. Brooks dedicated the team's win to his mother, who had died from cancer the previous Christmas. Upon returning to Australia, he was greeted as a hero, but he considered retiring due to waning desire for success after his triumph in Moscow. In 2000, Brooks and the other members of the quartet were each awarded the Australian Sports Medal for their victory in Moscow. +In the individual event, which occurred after the relay, Brooks had come equal first in his heat with eventual bronze medallist Per Johansson from Sweden in a time of 52.11 s. This made him the seventh fastest qualifier for the semi-finals, but he suffered a severe asthma attack and had to be hospitalised. He swam the semi-final regardless, despite having a heart-rate before the start of race of 120, compared to his usual 72. He finished seventh in a time of 52.70 s, which saw him place 14th, missing the final by 0.83 s. +Brooks suffered from a lack of motivation after the Olympics. He was recruited to the Australian Institute of Sport by swimming coaches Bill Sweetenham and Dennis Pursley, but his stay was brief. The inaugural director Don Talbot, a former head coach of the Australian swimming team, expelled him for indiscipline. He then won the 1981 Australian title in the 100 m freestyle without training, but his time of 52.61 s was substantially slower than his personal best. Brooks then accepted a scholarship to go to the University of Arkansas to train under Sam Freas. He enjoyed the more liberal culture in the American collegiate system, particularly the relationship between swimmers and their coaches. He enjoyed the less paternalistic treatment that the coaches accorded to their swimmers, saying that "In Australia, after a race they want to lock you in your room. In America, you can share a beer after the meet with the coach." The shortcourse pools used in the United States played into the hands of Brooks, as his strong legs gave him an advantage in pushing away from the turns, which came twice as often in comparison to longcourse (50 m) pools. He had initially planned to stay in the United States for only a year, but decided to extend his stay by a year, having enjoyed the high frequency of racing and the recognition accorded to university athletes in the United States. In both years, he won the 50 m and 100 m freestyle double in the Southwest Conference, and was fourth in the 100 m freestyle at the 1981 National Collegiate Athletic Association (NCAA) Championships. He also came second in the 100 m freestyle at the AACC Championships to Rowdy Gaines. +Despite his successes in America, the Australian Swimming Union did not offer to fund a return home for Brooks for the 1982 Australian Championships, but the University of Arkansas paid his fare as a reward for his performances in collegiate competition. Despite failing to defend his 100 m title, he qualified for the 1982 Commonwealth Games in Brisbane. The team then went into a five-week training camp in the seaside Sydney suburb of South Coogee. The team resided at a migrant hostel, which at the time was primarily occupied by Vietnamese refugees from the Vietnam War. Brooks spoke out about the quality of the accommodation, leading officials to make improvements. +Brooks' protests against the accommodation was overshadowed by a much-publicised physical confrontation with a team official. He telephoned his coach in the United States to make return travel arrangements for the conclusion of the Commonwealth Games. Due to the difference in time zones, he made his call after the team curfew, prompting team manager Jeff Hare to attempt to disconnect the line. After Hare threatened to expel him from the team, Brooks lost his temper and pinned the manager against the wall. The swimmer was summoned to a meeting with ASU officials the following day, where he remained unrepentant. He received a one-month ban, effective after the conclusion of the Commonwealth Games. Brooks demanded that if the ASU were to suspend him, they should do so immediately for the Commonwealth Games, rather than letting him compete and win medals before punishing him. His complaints about the training conditions had gained the attention of his team-mates, and several senior swimmers threatened to leave if he was banned. In the end, the ASU rescinded the suspension. Brooks has remained strongly critical of the attitude of swimming bureaucracy, claiming that "too many officials expect unquestioning obedience", calling for the formation of a swimmers' committee. +Arriving in Brisbane for the Commonwealth Games, Brooks and some of his fellow sprinters shaved their heads, something that received much attention from the Australian public. He won his heat of the 100 m freestyle in a Commonwealth and Commonwealth Games record of 51.09 s. He swam slower in the final, but his time of 51.14 s was enough to secure the gold medal in a close contest. Just 0.43 s separated him and the bronze and silver medallists Greg Fasala and Michael Delany, both of Australia. +Brooks then won gold as part of the winning 4 × 100 m freestyle relay, anchoring the team of Fasala, Delany and Graeme Brewer to a victory by almost three seconds, putting in a split of 50.56 s. The shaven-headed quartet was dubbed the Mean Machine. He collected another gold in the medley relay, combining with David Orbell, Evans and Jon Sieben, this time completing his freestyle anchor leg in 50.44 s. Canada had finished the race far ahead of the Australians, but were disqualified for an early changeover. After the games, Brooks completed the American college season before returning to Australia. He was named as Western Australia's Sportsman of the Year. +Brooks came into the 1984 Australian Championships as the favourite, but this time he was on the receiving end of a close result. He placed third in the 100 m freestyle behind Mark Stockwell and Delany, missing individual selection by 0.05 s. The trio was separated by a total of only 0.15 s. Selected as a relay swimmer only, he went to Los Angeles hoping to win the 4 × 100 m freestyle relay, an event that the United States had always won. +During the heats, the Australian team of Brooks, Fasala, Delany and Stockwell showed their intention to deny the Americans the gold for the first time at Olympic level. Drawn alongside the Americans in the third and final heat, the Australians set a new Olympic record of 3 m 19.94 s. Brooks led off in the heat, setting a time of 50.36 s, before Stockwell made up a deficit of 0.41 s during the anchor leg to beat the hosts by 0.20 s. Australia and the United States had stamped their authority on the event, qualifying almost four seconds faster than third-placed Sweden. However, the Americans had more speed in reserve, having rested 200 m freestyle silver medalist Mike Heath and 100 m freestyle gold medallist Rowdy Gaines, whereas Australia used its full-strength team. +In the final, Australian coach Terry Buck switched the swimming order, putting Fasala as the leadoff leg, while the Americans brought in Heath and Gaines. Fasala's time of 51.00 s put the Australians second at the first change, behind the Americans, for whom Chris Cavanaugh had built a 0.17 s lead. After the first leg, the race was still close—0.90 s separated all the teams. Brooks completed his leg in 49.36 s, the fourth fastest in the race, 0.24 s faster than his American counterpart Heath, giving the Australians a slender 0.07 s lead at the halfway point. Australia and the United States had broken away from the field, which was now the best part of two seconds in arrears. However, the Australian lead was short-lived. Matt Biondi took 0.59 s from Delany and Gaines took another 0.13 s from Stockwell, as the United States won in a world record time of 3 m 19.03 s. Australia were 0.65 s behind with Sweden a further 2.99 s in arrears. Brooks remained adamant that either he or Stockwell should have led off, stating that "the gold was there for the taking". +Brooks collected a bronze in the medley relay after swimming the freestyle leg in the heats before being replaced by first-choice Stockwell in the final. Competing in the third and final heat, Australia and the United States were equal at the last change before Brooks posted the fastest freestyle leg in the heats, pulling out a 0.40 s margin over Tom Jager. Australia again qualified fastest, but the Americans again had more in reserve, having rested all of their first-choice quartet in the heats. Stockwell combined with Kerry, Evans and Glenn Buchanan to finish behind the United States and Canada in the final, missing silver by just 0.02 s. Australia improved on their qualifying time by only 0.68 s, while the Americans sped up by 5.03 s. +In 1985, Brooks dead-heated with Stockwell in the 100 m freestyle at the Australian Championships, clocking a time of 51.12 s. He then combined with Tom Stachewicz, Paul Lee and Barry Armstrong, as Western Australia won the 4 × 100 m freestyle relay for the first time. Brooks maintained his form and despite not defending his Australian 100 m title, was selected for the 1986 Commonwealth Games in Edinburgh, where he came second to Fasala in the 100 m freestyle in a time of 51.01 s. He claimed gold in the 4 × 100 m freestyle relay along with Fasala, Stockwell and Matthew Renshaw, anchoring the quartet in setting a Commonwealth Games record. +On the return flight from the games, Brooks consumed 46 cans of beer and was banned for six months after he talked about the incident during a television interview. In his memoirs, Talbot, a non-drinker, cited Brooks and his Mean Machine teammates as being one of the biggest proponents of a hard-drinking culture that had permeated the Australian swimming scene at the time. In particular, he singled out Brooks as being a focal point of disruptive activity during his time at the Australian Institute of Sport. Talbot sees alcohol as the main reason behind the collapse of Australia's standing in the swimming world in the 1970s and the subsequent prolonged period of international uncompetitiveness in the 1980s. +Brooks retired thereafter, moving to Nambour, Queensland and starting a rock band called The Union. He played the electric guitar, and also wrote his own music and songs. He also represented Western Australia in water polo and indoor cricket and played Australian rules football at district level. +Having graduated from the University of Arkansas with a degree in journalism in 1985, Brooks entered the media after his retirement from swimming. Before his graduation, he had been a cadet with the Seven Network in Perth for five years, and was again hired by Seven and read the sports segment on the weekday evening news. He served as a commentator for Seven's coverage of the 1988, 1992 and 1996 Summer Olympics. Aside from covering the swimming and water polo, he also commentated on volleyball and in 1998, he called various downhill skiing events at the Nagano Winter Olympics. Brooks also called Australian Football League matches, and for three years hosted Brooksy's Footy Show. He also hosted a Western Australian travel/holiday show called Wild West, and in the lead-up to the Sydney Olympics, co-hosted The Games with Tracey Holmes. +However, Brooks' media career began to unravel in the late-1990s. He became addicted to alcohol, leading to a series of on-screen incidents. He once read the sports news segment while inebriated, and was then involved in a drunken argument with the Nine Network's Australian rules football pundit Sam Newman. In early 1999, in an interview with a magazine that had not been authorised by Seven, Brooks was asked what event he was looking forward to most at the 2000 Summer Olympics in Sydney, and replied "The after-Olympics piss-up". Seven suspended Brooks from on-screen duties for six weeks after he made the comments, which they deemed to be "tasteless and offensive", and he was eventually sacked, denying him the opportunity to commentate on the swimming events and costing him an annual salary of A$700,000, equivalent to A$1,302,302 in 2022. +Having moved to South Australia in 2003, Brooks started Local, which he billed as the state's leading lifestyle magazine. The venture was run solely by him and his wife, and was dominated by advertising. Journalist Peter FitzSimons criticised the lack of grammar checking and copyediting, citing a quote from Kieren Perkins, which was rendered thus in the publication: "I was over the moon. Winning is something you strive to do but when I consider all the factors being married two children twenty seven years of age competing in my third Games and I broke fifteen minutes twice in two days it really was quite outstanding and whichever way you cut it Grant Hackett was just the next generation of swimmer [sic]." +Brooks stood 200 cm (79 in) and weighed 95 kg (209 lb) during his career, but in the early part of the 21st century fought a battle with obesity, after ballooning to 150 kg (330 lb). As of 2007, he had lost substantial weight and fought off his alcohol problems. +Brooks' first marriage was to Lynette Quinlivan in January 1985. Their son Luke is a member of the Australian water polo team, playing as a goalkeeper. In 2000, he married his third wife Linda. As of 2022 the Brooks reside in Bali, Indonesia. +In late 2000, Brooks was declared bankrupt by the Federal Court of Australia in Perth after failing to repay a $15,000 debt to Bankwest. In May 2001, the police raided Brooks' Perth home and found a metre-high cannabis plant. Brooks claimed that the plant belonged to a friend, and then announced that he would be leaving the state. +Brooks later became a partner in Nitro Energy Drink Company, which was involved in motorsport sponsorship. However, the firm suffered from financial trouble and he had a falling out with his business partner. In 2009, Brooks was removed from the board and the company was put into administration, and both he and his former partner started legal proceedings. +On 13 October 2017 Linda Brooks was arrested in Perth after returning there from Bali. An extradition request to Queensland was granted, and police questioned her over an alleged fraud in 2008 relating to a sports clothing company of which she was a director. Neil Brooks was later arrested upon entering the country from Hong Kong, and both were charged with fraud. It is alleged that both made dishonest representations about the success of their business in order to persuade two investors to purchase a stake in their company. Both were later released on bail and were allowed to return overseas. +In April 2022 Brooks and his wife were ordered to return to Australia in order to face court over the fraud allegations after being earlier unable to due to the COVID-19 pandemic. In October 2022 they returned to Australia and faced a two-day committal hearing that will determine whether the case will go to trial. +List of Commonwealth Games medallists in swimming (men) +List of Olympic medalists in swimming (men) +Neil Brooks at IMDb +Matthew Brettingham (1699 – 19 August 1769), sometimes called Matthew Brettingham the Elder, was an English architect who supervised the construction of Holkham Hall, and became one of the best-known architects of his generation, despite coming from modest origins. Much of his principal work has since been demolished, particularly his work in London, where he revolutionised the design of the grand townhouse. As a result, he is often overlooked today, remembered principally for his Palladian remodelling of numerous country houses, many of them situated in the East Anglia area of Britain. As Brettingham neared the pinnacle of his career, Palladianism began to fall out of fashion and neoclassicism was introduced, championed by the young Robert Adam. +Born in Norwich, into a family of craftsmen, Brettingham worked initially as a surveyor, gaining experience through jobs for the city's civic authorities. Work included restorations at Norwich Cathedral, at the castle, at the local prison and the shire hall. His professional ascent began in 1743 with his appointment to the post of Clerk of Works at Holkham. The succeeding decades saw many aristocratic commissions, predominantly in East Anglia, but including work at Kedleston Hall in Derbyshire. +In addition to designing their country houses, Brettingham developed a substantial practice in the construction of town houses for the aristocracy. Major commissions included Norfolk House and Cumberland House, both demolished. Drawing inspiration from Italian urban palazzi, and from Andrea Palladio's rural villas, he created a style and arrangements of rooms perfectly suited to the lavish entertaining undertaken by the mid-18th-century nobility. Here, as in the country, he was ultimately superseded, and his reputation eclipsed, by the rise and success of Robert Adam. Brettingham died in 1769. His son, Matthew Brettingham the Younger, also pursued an architectural career. +Brettingham was born in 1699, the second son of Launcelot Brettingham, a bricklayer or stonemason from Norwich, the county town of Norfolk, England. The younger Brettingham married Martha Bunn at St Augustine's Church, Norwich, on 17 May 1721 and they had nine children together. His early life is little documented, and one of the earliest recorded references to him is in 1719, when he and his elder brother Robert were admitted to the city of Norwich as freemen bricklayers. A critic of Brettingham's at this time claimed that his work was so poor that it was not worth the nine shillings a week (£88 in 2025) he was paid as a craftsman bricklayer. Whatever the quality of his bricklaying, he soon advanced himself and became a building contractor. +During the early eighteenth century, a building contractor had far more responsibilities than the title suggests today. A contractor often designed, built, and oversaw all the details of a building's construction to completion. Architects, often called surveyors, were employed only for the grandest and largest of buildings. By 1730, Brettingham is referred to as a surveyor, working on more important structures than cottages and agricultural buildings. In 1731, it is recorded that he was paid £112 (£22,500 in 2025) for his work on Norwich Gaol. From then, he appears to have worked regularly as the surveyor to the Justices (the contemporary local authority) on public buildings and bridges throughout the 1740s. Projects dating from this time include the remodelling of the Shirehouse in Norwich, the construction of Lenwade Bridge over the river Wensum in 1741, repairs to the façade of Norwich Cathedral in the 1740s, and to Norwich Castle between 1747 and 1749, as well as the rebuilding of much of St Margaret's Church, King's Lynn, which had been severely damaged by the collapse of its spire in 1742. His work on the Shirehouse, which was in the gothic style and showed a versatility of design rare for Brettingham, was to result in a protracted court case that was to rumble on through a large part of his life, with allegations of financial discrepancies. In 1755, the case was eventually closed. Brettingham was left several hundred pounds out of pocket—several tens of thousands, in present-day terms—and with a stain—if only a local one—on his character. Transcripts of the case suggest that it was Brettingham's brother Robert, to whom he had subcontracted and who was responsible for the flint stonework of the Shirehouse, who may have been the cause of the allegations. Brettingham's brief flirtation with the Gothic style, in the words of Robin Lucas, indicates "the approach of an engineer rather than an antiquary" and is "now seen as outlandish". The Shirehouse was demolished in 1822. +In 1734, Brettingham had his first great opportunity when two of the foremost Palladian architects of the day, William Kent and Lord Burlington, were collaboratively designing a grandiose Palladian country palace, Holkham Hall, at Holkham in Norfolk for Thomas Coke, 1st Earl of Leicester. Brettingham was appointed Clerk of Works (sometimes referred to as executive architect), at an annual salary of £50 (£9,800 per year in 2025). He retained the position until the Earl died in 1759. The illustrious architects were mostly absent; indeed, Burlington was more of an idealist than an architect, thus Brettingham and the patron Lord Leicester were left to work on the project together, with the practical Brettingham interpreting the architects' plans to Leicester's requirements. It was at Holkham that Brettingham first worked with the fashionable Palladian style, which was to be his trademark. Holkham was to be Brettingham's springboard to fame, as it was through his association with it that he came to the attention of other local patrons, and further work at Heydon Hall and Honingham Hall established Brettingham as a local country-house architect. +Brettingham was commissioned in 1742 to redesign Langley Hall, a mansion standing in its own parkland in South Norfolk. His design was very much in the Palladian style of Holkham, though much smaller: a large principal central block linked to two flanking secondary wings by short corridors. The corner towers, while similar to those later designed by Brettingham at Euston Hall, were the work of a later owner and architect. The neoclassical entrance lodges were a later addition, by Sir John Soane. In 1743, Brettingham began work on the construction of Hanworth Hall, Norfolk, also in the Palladian style, with a nine-bay façade of brick with the centre three bays projected with a pediment. +In 1745, Brettingham designed Gunton Hall in Norfolk for Sir William Harbord, three years after the former house on the site was gutted by fire. The new house of brick had a principal façade like that of Hanworth Hall, however, this larger house was seven bays deep, and had a large service wing on its western side. His commissions began to come from further afield: Goodwood in Sussex and Marble Hill, Twickenham. +In 1750, now well-known, the architect received an important commission to remodel Euston Hall in East Anglia, the Suffolk country seat of the influential 2nd Duke of Grafton. The original house, built circa 1666 in the French style, was built around a central court with large pavilions at each corner. While keeping the original layout, Brettingham formalised the fenestration and imposed a more classically severe order whereby the pavilions were transformed to towers in the Palladian fashion (similar to those of Inigo Jones's at Wilton House). The pavilions' domes were replaced by low pyramid roofs similar to those at Holkham. Brettingham also created the large service courtyard at Euston that now acts as the entrance court to the mansion, which today is only a fraction of its former size. +The Euston commission seems to have brought Brettingham firmly to the notice of other wealthy patrons. In 1751, he began work for Charles Wyndham, 2nd Earl of Egremont at Petworth House, Sussex. He continued work intermittently at Petworth for the next twelve years, including designing a new picture gallery from 1754, and a pair of lodges, originally called Gog and Magog but renamed the Gohanna lodges in honour of Lord Egremont's prize-winning racehorse, in 1756–1763. Over the same period his country-house work included alterations at Moor Park, Hertfordshire; Wortley Hall, Yorkshire; Wakefield Lodge, Northamptonshire; and Benacre House, Suffolk. +From 1747, Brettingham operated from London as well as Norwich. This period marks a turning point in his career, as he was now no longer designing country houses and farm buildings just for the local aristocrats and the Norfolk gentry, but for the greater aristocracy based in London. +One of Brettingham's greatest solo commissions came when he was asked to design a town house for Edward Howard, 9th Duke of Norfolk in St James's Square, London. Completed in 1756, the exterior of this mansion was similar to those of many of the great palazzi in Italian cities: bland and featureless, the piano nobile distinguishable only by its tall pedimented windows. This arrangement, devoid of pilasters and a pediment giving prominence to the central bays at roof height, was initially too severe for the English taste, even by the fashionable Palladian standards of the day. Early critics declared the design "insipid". There were, however, exceptions to this view; after attending the opening party in 1756, Horace Walpole described the house as "a scene of magnificence and taste". +But it was the design of the interior of Norfolk House that was particularly influential, coming to define the layout and ground plan of the London town house for the next century. The floor plan was based on an adaptation of one of the secondary wings he had built at Holkham Hall. A circuit of reception rooms centred on a grand staircase, with the staircase hall replacing the Italian traditional inner courtyard or two-storey hall. This arrangement of salons allowed the circulation of guests at large parties; having been received at the head of the staircase they were able to move through the reception rooms without having to double back on later-arriving guests. The second advantage was that while each room had access to the next, it also had access to the central stairs, thus allowing only one or two rooms to be used at a time for smaller functions. Previously, guests in London houses could only reach the principal salon by progressing through a long enfilade of minor reception rooms. In this square and compact way, Brettingham came close to recreating the layout of an original Palladian villa. He transformed what Andrea Palladio had conceived as a country retreat into a London mansion appropriate for the lifestyle of the British aristocracy, with its reversal of the usual Italian domestic pattern of a large palazzo in town, and a smaller villa in the country. As happened so often in Brettingham's career, Robert Adam later developed this design concept further, and was credited with its success. However, Brettingham's plan for Norfolk House was to serve as the prototype for many London mansions over the next few decades. +Brettingham's additional work in London included two more houses in St James's Square: No. 5 for the 2nd Earl of Strafford and No. 13 for Henry Liddell, 1st Baron Ravensworth. Lord Egremont, for whom Brettingham was working in the country at Petworth, gave Brettingham another opportunity to design a grandiose London mansion—the Egremont family's town house. Begun in 1759, this Palladian palace, known at the time as Egremont House, or more modestly as 94 Piccadilly, is one of the few great London town houses still standing. It later came to be known as Cambridge House and was the home of Lord Palmerston, and then of the Naval & Military Club. The club sold the house at the end of the 20th century, and planning permissions for conversion, firstly to a single private house and subsequently to a hotel and apartments, were approved in 2007, 2013 and 2017. As at 2021, the plans remained uncompleted. +In 1759 Sir Nathaniel Curzon, later 1st Baron Scarsdale, commissioned Brettingham to design a great country house on his Derbyshire estate. Thirty years before a prospective design for a new Kedleston Hall had been drawn up by James Gibbs, one of the leading architects of the day, but Curzon wanted his new house to match the style and taste of Holkham. Lord Leicester, Holkham's owner and Brettingham's employer, was a particular hero of Curzon's. Curzon was a Tory from a very old Derbyshire family, and he wished to create a showpiece to rival the nearby Chatsworth House owned by the Whig Duke of Devonshire, whose family were relative newcomers in the county, having arrived little more than two hundred years earlier. However, the Duke of Devonshire's influence, wealth, and title were far superior to Curzon's, and Curzon was unable either to complete his house or to outdo the Devonshires (William Cavendish, 4th Duke of Devonshire, had been prime minister in the 1750s). This commission might have been the ultimate accolade Brettingham was seeking, to recreate Holkham but this time with full credit. Kedleston Hall was designed by Brettingham on a plan by Palladio for the unbuilt Villa Mocenigo. The design by Brettingham, similar to that of Holkham Hall, was for a massive principal central block flanked by four secondary wings, each a miniature country house, themselves linked by quadrant corridors. From the outset of the project, Curzon seems to have presented Brettingham with rivals. In 1759, while Brettingham was still supervising the construction of the initial phase, the northeast family block, Curzon employed architect James Paine, the most notable architect of the day, to supervise the kitchen block and quadrants. Paine also went on to supervise the construction of Brettingham's great north front. However, this was a critical moment for architecture in England. Palladianism was being challenged by a new taste for neoclassical designs, one exponent of which was Robert Adam. Curzon had met Adam as early as 1758, and had been impressed by the young architect newly returned from Rome. He employed Adam to design some garden pavilions for the new Kedleston. So impressed was Curzon by Adam's work that by April 1760 he had put Adam in sole charge of the design of the new mansion, replacing both Brettingham and Paine. Adam completed the north façade of the mansion much as Brettingham had designed it, only altering Brettingham's intended portico. The basic layout of the house remained loyal to Brettingham's original plan, although only two of the proposed four secondary wings were executed. +Brettingham moved on to other projects. In the 1760s, he was approached by his most illustrious patron, Prince Edward, Duke of York and Albany (brother of King George III), to design one of the greatest mansions in Pall Mall, York House. The rectangular mansion that Brettingham designed was built in the Palladian style on two principal floors, with the state rooms as at Norfolk House, arranged in a circuit around the central staircase hall. The house was a mere pastiche of Norfolk House, but for Brettingham it had the kudos of a royal occupant. +Its royal occupant may very well have made York House the pinnacle of Brettingham's career. Built during the 1760s, it was one of his last grand houses. His last country-house commission was at Packington Hall, Warwickshire. In 1761, he published his plans of Holkham Hall, calling himself the architect, which led critics, including Horace Walpole, to decry him as a purloiner of Kent's designs. Brettingham died on 19 August 1769 at his house outside St Augustine's Gate, Norwich, and was buried in the aisle of the parish church. Throughout his long career, Brettingham did much to popularise the Palladian movement. His clients included a royal duke and at least twenty-one assorted peers and peeresses. He is not a household name today largely because his provincial work was heavily influenced by Kent and Burlington, and unlike his contemporary Giacomo Leoni he did not develop, or was not given the opportunity to develop, a strong personal stamp to his work on country houses. Ultimately, he and many of his contemporary architects were eclipsed by the designs of Robert Adam. Adam remodelled Brettingham's York House in 1780 and, having supplanted both Brettingham and James Paine at Kedleston, went on to replace Paine as architect at Nostell Priory, Alnwick Castle, and Syon House. +Brettingham's principal contribution to architectural change was perhaps the design of the grand London town house; often with unremarkable exteriors, Brettingham replaced the traditional long Baroque enfilades with a circulating plan of reception rooms suitable for lavish and large scale entertaining. Many of these anachronistic palaces are now long demolished; York House was torn down between 1908 and 1912, and Norfolk House in 1938; or have been transformed for other uses and are inaccessible for public viewing. Hence, what little remains in London of his work is generally unknown. Outside of London, of Brettingham's work, only the buildings he re-modelled have survived, and for this reason Brettingham tends to be thought of as an "improver" rather than an architect of country houses. His entry in the Art UK directory describes him as "never more than an orthodox and unenterprising Palladian." George Wardlaw Burnet's entry in the 1885 Dictionary of National Biography concluded, "Exhibiting no great novelty of conception, it must be admitted he displayed knowledge and skill equal to those of any architect of his time". +There is no evidence that Brettingham ever formally studied architecture or travelled abroad. Reports of him making two trips to Continental Europe, are the result of confusion with his son, Matthew Brettingham the Younger. That he enjoyed success in his own lifetime is beyond doubt—Robert Adam calculated that when Brettingham sent his son on the Grand Tour (1747), he went with a sum of money in his pocket of around £15,000 (£2.93 million in 2025), an enormous amount at the time. However, part of this sum was probably used to acquire the statuary in Italy (documented as supplied by Matthew Brettingham the Younger) for the nearly completed Holkham Hall. Matthew Brettingham the Younger wrote that his father "considered the building of Holkham as the great work of his life", and it is perhaps the building for which he is best remembered although the exact nature and extent of his contribution continues to be a matter of scholarly debate. +Burnet, G. W. (1886). "Brettingham, Matthew". In Stephen, Leslie (ed.). Dictionary of National Biography. Vol. 6. London: Smith, Elder & Co. pp. 287–288. +Colvin, Howard (1995). A Biographical Dictionary of British Architects 1600–1840 (3 ed.). New Haven, US and London: Yale University Press. ISBN 0-300-06091-2. +Girouard, Mark (1978). Life in the English Country House. New Haven, US and London: Yale University Press. ISBN 0-300-02273-5. +Harling, Robert (1969). Historic Houses. Conde Nast Publications. +Hewlings, Richard (1993). "Wakefield Lodge and Other Houses of the Second Duke of Grafton" (PDF). The Georgian Group. 3: 43–61. Retrieved 18 November 2023. +Howell, James D. E. (1964). "Matthew Brettingham and the County of Norfolk". Norfolk Archaeology. 33, Part III (3). doi:10.5284/1077966. +Jackson-Stops, Gervase (1990). The Country House in Perspective. Pavilion Books. ISBN 1-85145-383-0. +Nicolson, Nigel (1965). Great Houses of Britain. London: George Weidenfeld & Nicolson Ltd. ISBN 0-600-01651-X. +National Trust, ed. (1998). Kedleston Hall. Derbyshire Countryside. +Schmidt, Leo (2015). "Holkham Hall: An Architectural 'Whodunnit'". Architectural History. 58: 83–108. doi:10.1017/S0066622X00002598. S2CID 194901745. Retrieved 18 November 2023. (Subscription required) +Sheppard, F. H. W. (1960). St. James Westminster, Part 1. Vol. 29. Survey of London. +Wilson, Michael I. (1984). William Kent. London: Routledge & Kegan Paul. ISBN 0-7100-9983-5. +Holkham Hall +Kedleston Hall +Carrington Moss is a large area of peat bog near Carrington, Greater Manchester, England, south of the River Mersey, 10 miles (16 km) south-west of Manchester. It occupies an area of about 1,100 acres (450 ha). The depth of peat varies between 17 and 20 feet (5.2 and 6.1 m). +Originally an unused area of grouse moorland, the moss was reclaimed in the latter half of the 19th century for farming and the disposal of Manchester's waste. A system of tramways was built to connect it with the Manchester Ship Canal and a nearby railway line. In the Second World War, the land was used as a Starfish site and in the latter half of the 20th century a large industrial complex was built along its northern edge. More recently, several sporting facilities have been built on Carrington Moss. +The land is still used for farming and several nature reserves have been established within its bounds. Parts of Carrington Moss are accessible to the public over several rights of way. +A History of Flixton, Urmston, and Davyhulme (1898) claims that the name Carrington might be derived from the Goidelic Celtic root Cathair, a fortress, but a more recent theory is that it derives from an Anglicised form of a Scandinavian personal name. A Carrington Hall, seat of the Carrington family (descended from William de Caryngton) once existed to the north of Carrington Moss, at the junction formed by the modern-day A6144 and B5158 roads. The word moss, first used during the 15th century, forms part of the local name for a lowland peat bog, "mosslands". Today the term is also used to describe former bogs that have been converted to farmland. +Manchester's population increased by more than 150% between 1831 and 1851. This placed considerable pressure on the city's ability to dispose of refuse, exacerbated during the 1870s by a gradual switch from the older cesspit methods of sewage disposal to pail closets. These needed to be emptied regularly and by the 1880s, night soil accounted for about 75% of Manchester's 200,000 long tons (200,000 t; 220,000 short tons) of refuse. +Along with parts of Moss Side and Withington, in 1885 Bradford, Harpurhey and Rusholme became part of the City of Manchester. To cope with the extra demands placed on the area's refuse disposal systems, Manchester Corporation began to look for disposal sites. A number of locations were considered, including one on Deeside and another in Nottinghamshire, but Carrington Moss was chosen due to the nature of its land and its accessibility. Therefore, in 1886, the corporation bought the Carrington Moss Estate—an area of grouse moorland—from Harry Grey, 8th Earl of Stamford. +The purchase was part of the corporation's ultimately unsuccessful plan to retain the pail closet system (now superseded by the water closet), and followed a public scandal created by the daily dumping of 30–60 tons of human faeces into the Medlock and Irwell rivers, at Holt Town sewage works. It paid about £38,000 (£4 million as of 2025), for the site, but the bog's depth, between 17 and 20 feet or 5.2 and 6.1 metres deep pushed the total development cost to almost £94,000 (£11 million as of 2025). The 1,101-acre (446 ha) estate included 600 acres (2.4 km2) of wild mossland, 209 acres (0.85 km2) of partly cultivated mossland, 282 acres (1.14 km2) of mossland under cultivation and 10 acres (40,000 m2) of incomplete roads. A number of brick buildings were included, along with Asphodel Farm and Ash Farm, both with wooden farmhouses. The corporation rented 700 acres (2.8 km2) of land in small holdings to local farmers and kept 400 acres (1.6 km2) for itself. +The bog's virgin moss was cultivated and drainage channels cut through at regular intervals, the first step in the area's reclamation. This drainage caused the characteristically convex Moss to sag noticeably; some residents of Dunham Town commented that they could see parts of Carrington previously obscured by the moss. A network of tramways and roads was constructed using clinker and other materials brought from the city. Drains were laid and the land cleared of scrub. A water supply was also installed. Some of the more dangerous buildings were demolished, while others were either repaired or replaced. +Refuse was loaded from a number of locations and was first transported along the Mersey and Irwell Navigation, until that waterway was closed on 11 November 1888. For several years until the completion of its replacement, the Manchester Ship Canal, the corporation was reliant on Manchester's local railway network. Refuse was loaded at the corporation's Water Street Depot on to Cornbrook sidings and in waggons to Carrington on a junction from the Cheshire Lines Committee's (CLC) Glazebrook to Stockport Tiviot Dale line. The canal company installed a temporary dock on the new canal, although this was considered impractical and was rarely used. A more permanent arrangement was made several years later. New railway sidings were also built; once complete, refuse was loaded from near Oldham Road railway station and the corporation's Water Street Depot. It was then transported along the Ship Canal to a newly built wharf, and thereafter, by tramway across the moss. +Once delivered, refuse was normally placed in heaps and allowed to dry before being put into the ground. The naturally acidic water was a perfect receptacle for the contents of pail closets, rich in urea and nitrogen. Bacteria quickly broke the refuse down into ammonium compounds and free ammonia, which neutralised the soil's acidity and created ammonium nitrate—an essential fertiliser for arable land. +By the 1890s, over 70,000 long tons of excrement annually were being disposed of on the moss. The land was a useful source of income for Manchester; for the year ending 31 March 1900 the estate made a profit of £777 5s 2d (by comparison, the larger Chat Moss made £2,591 13s 4d). Its success helped persuade Manchester Corporation to purchase 2,583 acres of nearby Chat Moss in 1895. By 1897, 37,082 long tons of nightsoil, 587 long tons of sweepings and litter and 11,673 long tons of cinders were being sent to Carrington. Various crops were grown on the land, including wheat, oats, potatoes and carrots. A variety of ornamental shrubs, including rhododendrons, were grown in a nursery and used in the parks and gardens of Manchester. +By the 1930s, extensive use of the water closet meant that the amount of night soil being delivered to Carrington Moss had dropped significantly. During this period, the majority of refuse placed on the Moss came from ash bins, although some was from slaughterhouses and lairage facilities. In 1923, manure of only moderate value was being delivered, supplemented by sulphate of potash, sulphate of ammonia, and super-phosphates. Altrincham Sewage Farm (visible on the above map) was used to flood the surrounding fields with sewage water. To the west, a series of disused marl-pits formed Timperley Sewage Beds, a further source of manure. +Carrington Wharf had fallen out of use by 1934 and with the advent of the Second World War, five miles (8 km) of railway were lifted and all the waggons scrapped. At the Ministry of Supply's request, much of the infrastructure supporting both Carrington Moss and Chat Moss was sold. The sidings at Carrington continued to be used by the CLC for waggon storage, but Carrington Wharf was subsumed in 1946 by the construction of Carrington Power Station. During the war, the moss became one of four sites in Manchester used as a Starfish site—decoy targets for enemy aircraft. Operational control was the responsibility of RAF Balloon Command. The site contained an air raid shelter for the operational crew and several combustible devices used to simulate fires and lights. The site was activated in December 1940 but closed several years later, following a reduction in enemy aircraft attacks and lack of manpower. +In 1948, the estate was valued at £82,615. In the year ending March 1971, it produced an income of £20,268. By that time the entire Moss had been fully reclaimed; 872.785 acres (3.53204 km2) of cultivated land, 39.012 acres (157,880 m2) of roads and plantations, and 30.140 acres (121,970 m2) of 'industrial area' were available for use. The principal land uses were dairy, arable farming, and glasshouse culture. No refuse was delivered for the year ending March 1971, and the Moss had by that time taken a total of 1,305,822 tons of refuse. +Industrialisation of the moss took place from 1947–1952 when Petro-Carbon ltd began to build what would later become known as the Shell Site. The estate was leased on 1 October 1968 to Shell Chemicals, who in 1957 had purchased a propylene oxide plant along the moss's northern edge. Shell had built an ethylene oxide plant in 1958 and began to produce polyether polyols the following year. Council housing was built nearby, at Carrington and Partington, for workers and their families. By 1985 the Shell plant had a turnover of about £200M and employed 1,150 people, but a major restructuring of the business reduced the workforce to less than 500 by 1986. By 1994, four distinct plants operated on the 3,500-acre (14 km2) site, producing a range of chemicals, and materials including polystyrene, polyethylene and polypropylene. In 2005 it was reported that Shell would close their polyols and ethoxylates units, a decision which came into effect in 2007. The estate is currently managed by chartered surveyors Bell Ingram. LyondellBasell operate the last remaining chemical plant on site. +Manchester United opened their Trafford Training Centre training ground and Academy in 2000, on land formerly owned by Shell. Bury F.C.'s Carrington Training Centre, formerly occupied by Manchester City F.C., is located nearby. Because of the Moss's history as a dumping ground for waste, bottle diggers often frequent the area. Several rights of way exist on the land, and a horse-riding school operates in the area. +At 53°25′14″N 2°23′16″W (53.42056, 2.38778), 65.6 feet (20.0 m) above sea level, Carrington Moss lies along the southern edge of the Lancashire Plain, an area of Bunter sandstones overlaid with marls laid down during the Late Triassic period. These rocks are themselves overlaid by a layer of boulder clay deposited during the last ice age, about 10,000 years ago. The combination of the flat topography and the underlying clay resulted in extensive peat bogs developing along the Mersey Valley, and overflowing beyond the valley. Along with large parts of Chat Moss and Holcroft Moss, Carrington Moss began to form during the Flandrian period from 7100 to 5000 BP. +Carrington Moss is a lowland raised bog. The area drains slowly, which slows the decomposition of plant life and leads to the accumulation of peat. Over thousands of years this raises the level of peat and forms a gently sloping dome (hence, raised). Such areas support a wide range of flora and fauna; Sphagnum balticum, a medium-sized bog moss, was recorded on Carrington Moss in the 1880s, although locally it is now presumed to be extinct. Bog asphodel (Narthecium ossifragum), white beak-sedge (Rhynchospora alba), cranberries, bog-rosemary (Andromeda polifolia), and the cotton sedge have also been recorded. In 1923 species of trees recorded by E. Price Evans for the Journal of Ecology included English oak (Quercus robur), and common ash (Fraxinus excelsior). Undergrowth included common hazel (Corylus avellana), blackberry (Rubus fruticosus), and European holly (Ilex aquifolium). Several species of ground vegetation included creeping soft grass (Holcus mollis), common bluebell (Hyacinthoides non-scripta), common foxglove (Digitalis purpurea), dog's mercury (Mercurialis perennis), iris (Iris pseudacorus), mad-dog weed (Alisma plantago-aquatica), and cat-o'-nine-tails (Typha latifolia). +Birch Moss Covert is a small woodland containing birch, alder and willow trees, as well as various species of flora and fauna. The small mammal population includes the wood mouse, which attract both kestrel and sparrowhawk. Red foxes, stoats, weasels, and badgers, are often seen. The area is part of Shell's estate, covering about 15 acres (61,000 m2) of land managed by the Cheshire Wildlife Trust. The trust also manages a small nature reserve located within Manchester United's training ground. This provides a habitat for a number of species including the red admiral, meadow pipit, and grey partridge. Carrington Moss is home to the only recorded pairs of breeding grey partridge in Trafford. Six pairs of Eurasian bullfinch were recorded in 2003. The reduction in the population of these and similar birds is attributed to modern farming methods, the loss of broad hedgerows, and the lack of winter stubble. Action for Nature in Trafford has therefore included the site in its Biodiversity Action Plan. The group intends to develop Carrington Moss as a home for other species, such as reed bunting. Stigmella continuella (a species of moth occurring in southern and north-west England) has been observed in the area. +Notes +Bibliography +Carrington Moss ammunition dump at Manchester Local Image Collection Archived 18 August 2007 at the Wayback Machine +Prince Edward, Duke of Windsor, and Wallis, Duchess of Windsor, visited Nazi Germany in October 1937. Edward had abdicated the British throne in December 1936, and his brother George VI had become king. Edward had been given the title Duke of Windsor on abdication, and he married Wallis Simpson in June 1937. He appeared to have been sympathetic to Germany in this period and, that September, announced his intention to travel privately to Germany to tour factories. His interests, officially researching the social and economic conditions of the working classes, were against the backdrop of looming war in Europe. The Duke's supporters saw him as a potential peacemaker between Britain and Germany, but the British government refused to sanction such a role, opposed the tour and suspected that the Nazis would use the Duke's presence for propaganda. Prince Edward was keen for his wife, who had been rejected by the British establishment, to experience a state visit as his consort. He promised the government to keep a low profile, and the tour went ahead between 12 and 23 October 1937. +The Duke and the Duchess, who were officially invited to the country by the German Labour Front, were chaperoned for much of their visit by its leader, Robert Ley. The couple visited factories, many of which were producing materiel for the rearmament effort, and the Duke inspected German troops. The Windsors were greeted by the British national anthem and Nazi salutes. They dined with high-ranking Nazis such as Joseph Goebbels, Hermann Göring, Joachim von Ribbentrop, and Albert Speer, and had tea with Adolf Hitler in Berchtesgaden. The Duke had a long private conversation with Hitler, but it is uncertain what they discussed, as the minutes of their meeting were lost during the war. The Duchess took afternoon tea with Hitler's deputy, Rudolf Hess. Hitler was sympathetic to the Windsors and treated the Duchess like royalty. +The British government was unable to affect the course of events but forbade its diplomatic staff in Germany from having any high-level interaction with the Windsors. British popular opinion of the tour was muted, and most of the public viewed it as in poor taste and disrupting the first year of George's reign. The tour of Germany was intended to have been followed by one of the United States, but Nazi repression of working-class activists in Germany led to a wave of disapproval for the Windsors in the American labour movement, which led to the U.S. visit being cancelled. +Some modern historians tend to view the 1937 tour as a diplomatic misstep reflecting the Duke's disregard for advice. +Edward VIII became king after the death of his father, George V, in early 1936. Almost immediately, he announced his intention to marry Wallis Simpson, a twice-divorced American. On political and moral grounds, she was unacceptable as a royal consort to the British government and royal family. As king, Edward was the titular governor of the Church of England, which forbade the divorced from remarrying during the lifetime of their former spouses, and both of Simpson's previous husbands were still alive. The proposed marriage was believed by critics to breach Edward's coronation oath, and weakened his position as constitutional monarch. Edward knew that if he forced the issue, Stanley Baldwin's government would almost certainly resign en masse. +Edward realised that his family, the government, the Church, and the people would not support the marriage. Thus, in December 1936, he abdicated. His younger brother, the Duke of York, succeeded him as George VI, and Edward was given the new title of Duke of Windsor. Edward and Simpson married in France in June 1937, and having honeymooned in Vienna, they returned to Paris and established their headquarters there. Internationally, journalist Andrew Morton stated that the Duke was viewed as: +Modern, progressive, vigorous, and accessible. Even his mock Cockney accent with a touch of American seemed more down-to-earth and unaffected than the disdainful patrician tones of a man like Foreign Secretary Anthony Eden. He remained an intriguing international celebrity, his marital turmoil only enhancing the iconic mystery surrounding the man. +The European political background to the tour was tense. The Spanish Civil War, which had broken out the previous year, upset the balance of power and drew in the Soviet Union, Italy, and Germany. Also, Germany was becoming increasingly aggressive and had spent the previous few years rearming. In the United Kingdom, there was a sense of political unease towards the future and an expectation of war although foreign policy remained predicated on appeasement. Baldwin resigned as prime minister in May 1937 and was replaced by his deputy, Neville Chamberlain. +Historian Michael Bloch states that although the tour can be viewed as a poor decision with hindsight, it was not out of place for the time. He notes that "war was still two years away, curiosity about the Nazis was intense, and many respectable people accepted government invitations. It was fashionable to go to Germany and visit Hitler in the mid-thirties just as it was to go to China and visit Mao Tse-tung in the sixties". The former prime minister, David Lloyd George, had visited Germany two years before the Windsors. The leader of the Labour Party, pacifist George Lansbury, met with Hitler in April 1937. Also, Lord Halifax, later foreign secretary, visited to do so, at Göring's invitation, the following month. Halifax's trip was "ostensibly ... a social one", but it was also an opportunity for the British government to initiate talks with Hitler, according to modern historian Lois G. Schwoerer. Similarly, Hitler hosted many non-Germans, including Aga Khan III, papal nuncio Cesare Orsenigo, ambassadors, government ministers, and European royals, at his residence in Bavaria, the Berghof. +George VI is said to have been horrified by his brother's entry into European political affairs at such a delicate time. George wrote to Edward's political advisor, Walter Monckton, that the Duke's plan was "a bombshell, and a bad one". George took particular umbrage because on abdication, Edward had said that he intended to avoid public appearances. Royal biographer Sarah Bradford suggests that the visit indicated that Edward had no intention of retiring: rather, he intended to behave independently of the king's and the government's wishes. +Contemporaries were aware of the negative connotations of a trip to Germany at the time. The announcement took everyone by surprise, and those sympathetic to Edward, such as Winston Churchill and Lord Beaverbrook, attempted to dissuade him from going. The intervention of an old friend of the Duchess, Herman Rogers, against the trip also proved unsuccessful. The government already suspected that Edward had "strong views on his right to intervene in affairs of state", argues historian Keith Middlemas, but its "main fears ... were of indiscretion". The Foreign Office warned the Duke that the Nazis were propaganda experts; the Duke agreed but promised not to speak publicly while he was there. The government, argues historian Deborah Cadbury, was concerned that the Duke would gather a party around him and promote his own personal foreign policy, outside government control. +Edward stated that his intention for the visit was "without any political considerations and merely as an independent observer studying industrial and housing conditions". He said that one could not ignore what was happening in Germany "even though it may not have one's entire approval". The Duke was sympathetic to the cause of improving working conditions. +Historian Frances Donaldson suggests that his views "had caused offence in England because, according to opinion there, such matters were not the concern of the throne". Statements such as that one, scholar Adrian Philips emphasises, were intended to deflect from Edward's public relationship with Simpson. +Edward was an admirer of Germany and fluent in its language, which the Duke in his memoirs called "the Muttersprache [mother tongue] of many of our relations". He knew, too, that German blood "flowed strongly in him", and researcher Mark Hichens speculates that Edward's ancestry led him to favour German culture. As Prince of Wales, he had studied at Magdalen College, Oxford, under Hermann Georg Fiedler, and he had toured Germany twice before World War I broke out in 1914. One of his friends, Chips Channon, Conservative MP for Southend West, commented in 1936 that he "is going the dictator way, and is pro-German". Simpson was also believed to hold similar views on account of her rejection by the British ruling class, and many within the government suspected her to have spied for Hitler while she lived in Britain, though she denied that in her autobiography. The FBI also monitored her throughout the period, and concluded that she had Nazi sympathies. It had been rumoured that she and Joachim von Ribbentrop had a sexual relationship during his tenure as German ambassador in London during the mid-1930s. Albert von Mensdorff-Pouilly-Dietrichstein, a former Austrian ambassador to the UK who was George V's second cousin, believed that both Windsors favoured German fascism as a bulwark against communism in Europe. Edward also, according to the Count, favoured an alliance with Nazi Germany around the time. +Edward himself later contextualised his position in the 1930s as being a reaction to what he termed "the unending scenes of horror" of the First World War. He said that led him to support appeasement with Hitler. The latter is known to have seen the Duke as an ally, believing that as king, Edward would have strengthened Anglo-German relations. Albert Speer later said that Hitler was certain that "through him permanent friendly relations could have been achieved. If he had stayed, everything would have been different." The Duke, suggests biographer Anne Sebba, probably wanted to restore the countries' close ties, which had been broken by the First World War. He also wanted to make his new wife the centrepiece of a state visit. Historian Ted Powell suggests that the Duke would have visited any country that would accept his wife on his terms. Edward's equerry, Dudley Forwood, points out that the only possible state visit was to Germany and also suggested that the Duke wished to prove to his wife that he had lost nothing by abdicating. +A tour of Germany had been broached with the Duke before his wedding by French businessman Charles Bedaux, whom Bloch describes as an "enigmatic time and motion tycoon". Edward was agreeable and saw it as a way of raising his profile. By April 1937, Colonel Oscar Solbert had suggested that the Duke take a tour of Germany, which was soon intended to be the first of several planned international tours. Bedaux offered to organise the Duke's side of the arrangements. Solbert had been with Edward on his 1924 tour of the United States and had been impressed by his gravitas and professional demeanour. That led him to suggest to the Duke that he should "head up and consolidate the many and varied peace movements throughout the world". Swedish millionaire Axel Wenner-Gren acted as a go-between for the Duke in the early discussions. Bedaux wrote to Solbert to tell him: +The Duke of Windsor is very much interested in your proposal that he lead a movement so essentially international. We all know that as Prince of Wales and as King, he has always been keenly interested in the lot of the working man and he has not failed to show both his distress and his resolve to alter things whenever he has encountered injustice ... Yet he is not satisfied with the extent of his knowledge. He is determined to continue, with more time at his disposal, his systematic study of this subject and to devote his time to the betterment of the life of the masses ... He believes his is the surest way to peace. For himself he proposes to begin soon with a study of housing and working conditions in many countries ... +The tour of Germany was planned to be a brief visit of 12 days but was to be followed by a longer one of the U.S. The German side of things was organised by Hitler's adjutant, Captain Fritz Wiedemann, with final preparations discussed at the Paris Ritz in late September. The same month, the Duchess wrote to her aunt in Washington that they were planning a trip to observe European working conditions. The Duchess explained that "the Duke is thinking of taking up some sort of work in that direction. The trip is being arranged by Germany's No. 1 gentleman so should be interesting". However she noted that at that stage, it was still only a proposal. Writer Hugo Vickers suggests that Edward believed himself to be able to influence Hitler and avert war in Europe. If that was the case, says Vickers, Edward "severely overestimated his own importance". +Several contacts visited the Windsors at their Paris hotel, Le Meurice, but the nature of their discussions remains unknown, which has encouraged what Cadbury terms colourful theories. One such, for example, by Charles Higham, suggests that on one occasion the Duke received Hitler's deputy, Rudolf Hess, Hess's assistant Martin Bormann, and Hollywood actor Errol Flynn together. It is more likely, she claims, that the rooftop restaurant meetings involved men such as Wiedemann finalising the itinerary and other minutiae. +Powell suggests that Edward found the German government's response to be sufficiently sympathetic to proceed without delay. In late September, he received a personal invitation from Robert Ley, the head of the German Labour Front (DAF). Edward first indicated that he intended to accept in a letter to the British chargé d'affaires in Berlin, George Ogilvie-Forbes, on 20 September. A public announcement followed two weeks later. A telegram to the Foreign Office stated: +In accordance with the Duke of Windsor's message to the world press last June that he would release any information of interest regarding his plans or movements, His Royal Highness makes it known that he and the Duchess of Windsor are visiting Germany and the United States in the near future for the purpose of studying housing and working conditions in these two countries. +Historian Jonathan Petropoulos suggests that the British government knew that it could not prevent what was officially a visit by a private individual. In private, the news angered both Downing Street and Buckingham Palace. The Permanent Under-Secretary at the Foreign Office, Robert Vansittart, wrote to the King's Private Secretary, Alec Hardinge, to condemn the tour. Hardinge agreed and described it as a "private stunt for publicity purposes". He also reasoned that the premise of the tour was flawed: neither the Duke nor his visit, he said, could "obviously ... bring any benefit to the workers themselves". Ley proposed to hold Nazi rallies at each stop on the Windsors' tour, but the Duke had vetoed it on the grounds that it constituted anti-British propaganda. +Historian Andrew Roberts suggests that the German government believed that Edward had been forced to abdicate as a result of his pro-German views, which encouraged them to "lay out the red carpet" for him. On 10 October, the Duke's cousin, the Duke of Saxe-Coburg and Gotha, wrote to him, "dear David! I hear that you are coming to Germany ... I naturally would be delighted if you could take this opportunity to see me; perhaps I could introduce you to a couple of interesting personalities whom you otherwise wouldn't meet". Hitler and Ribbentrop planned that although the tour was a private rather than state visit, the Windsors, particularly the Duchess, would effectively undertake a royal progress. That was first demonstrated on their arrival, early Monday morning, at Berlin's Friedrichstraße station on 11 October. Scholar Susanna de Vries describes how the Duchess "covered in jewels ... did her best to look suitably royal", dressed in royal blue. They were greeted by Ley, who kissed her hand and called her "Your Highness". With Ley was a welcoming delegation including Ribbentrop and the deputy to the Gauleiter of Berlin, Artur Görlitzer. Also waiting was the British Embassy's Third Secretary, George Ogilvie-Forbes, to present a letter informing the Duke that the Embassy would not be available to assist him or his wife formally in the course of their visit. Ogilvie-Forbes later visited the Duke in his hotel to pay the personal respects that he had been unable to pay him in public. +A welcoming crowd of approximately 2,000 lined the streets outside the station. The German media had set great store by the Windsors' visit from the beginning. As the Windsors were leaving, the crowd surged forward, and a crush ensued. That, noted Cadbury, destroyed the "majestic air" of the reception that Ley had organised. With few of the crowd having seen them, the couple were driven away at high-speed in their Mercedes, to their hotel, the Kaiserhof. +Pathé caught the moment they emerged from the station into a large crowd that had gathered determined to see this unique couple: a king who had thrown away the greatest throne in the world for love, and the woman herself, who must possess some magical quality. Dr Ley, the head of the German delegation, wearing his brown Nazi uniform and for once not drunk, delighted them both by deferring to her as 'Her Royal Highness'. +The couple were treated like royalty by the German aristocracy, which "would bow and curtsy towards [the Duchess], and she was treated with all the dignity and status that the Duke always wanted". On their first night in Berlin, they joined Ribbentrop for dinner at Horcher. The night was attended by Speer with whom they discussed classical music, and Magda and Joseph Goebbels, who were Germany's de facto first lady and Reich Minister of Propaganda respectively. After their meeting, Goebbels wrote in his diary that "the duke is wonderful—a nice, sympathetic fellow who is open and clear and with a healthy understanding of people ... It's a shame he is no longer king. With him we would have entered into an alliance." The Duchess did not reciprocate, describing him as "a tiny, wispy gnome with an enormous skull", but Magda, she continued, was "the prettiest woman I saw in Germany". The Windsors dined with his cousin the Duke of Saxe-Coburg and Gotha on the 19th. The Duke later recalled that the dinner was attended by over 100 guests, many whom he had "hobnobbed" with at both his father's jubilee and then his funeral. +The Berlin correspondent of the British Observer newspaper, reporting the couple's arrival, wrote that they could look forward to a "heavy programme" of events. The couple and their entourage, which included the Duke's cousin Prince Philipp von Hessen, travelled around Germany on Hitler's personal train, the Führersonderzug, while their telephones were bugged by Prince Christoph of Hesse on the orders of Reichsstatthalter Hermann Göring to keep the Nazi leadership informed of the Windsors' private opinions. The German government was funding the visit, which modern historian John Vincent suggests allowed them to choreograph it. Hichens, too, notes that the Windsors "saw only what the Nazis wanted them to see, and the Duke saw what he wanted to see turning a blind eye on the horrors of Nazidom". For example, according to Morton, they visited a barracks of apparently-empty concrete buildings that they later realised had been a concentration camp. When the Duke enquired as to their purpose, Ley replied, wrote Forwood later, " 'it is where they store the cold meat.' In a horrible sense that was true." +Although the couple were in Germany at Ley's personal invitation, he was a poor host. Bloch describes him as coarse, "addicted to alcohol [and] high-speed driving", and risqué jokes. Hichens views Ley as "loud-mouthed", brutal and a "particularly odious Nazi thug." On one journey, he was drunk at the wheel of the Windsors' Mercedes while he was driving at speed and crashed them into the gates of the Munich factory that they were visiting. One of Ley's aides, Hans Sopple, later described events, saying Ley "drove the car through the locked gates and then raced up and down at full speed between the barracks, scaring hell out of the workers and nearly running over several. The next day Hitler told Göring to take over the Duke's visit before Ley killed him." That was not, comments Morton, "at all what the Duke had in mind when he described the nature of a royal tour to his wife". +Bloch describes the couple's itinerary as an "exhausting" series of visits to industrial and housing areas. A letter from the Duchess confirms that although the tour was interesting, it involved walking "miles a day through factories", including one that produced lightbulbs. Among other sights, they saw a winter relief centre, a Wagnerian opera in a workers' concert hall, and inspected a Pomeranian SS squadron with the Leibstandarte Adolf Hitler, Hitler's personal bodyguard. The Duchess did not accompany her husband everywhere; he visited the Daimler Motoren Gesellschaft at Untertürkheim alone, which was intended to showcase German precision engineering to the Duke, and while there he met British racing driver Richard Seaman, who had signed for the Mercedes-Benz team earlier in the year. +On 14 October, the Duke and the Duchess visited Göring at his jagdschloss in Carinhall, where they saw his miniature railway. There, Hitler's deputy gave them high tea, followed by a tour of his large art collection and gymnasium, where, although dressed in full uniform and decorations, he demonstrated his massage machine. The three conversed in Göring's study, where Edward noticed a new, official map of Nazi Germany on the wall. Reflecting the party's Anschluss policy, Austria was shown as annexed to Germany. Cadbury quotes Simpson: "Göring's face wrinkled with amusement ... The Austrians would want to be part of the Reich", he had said. Simpson noted that "the moment passed, the statement left unchallenged" by the Duke. +They visited an Academy for Youth Leadership, where they observed the training of Hitler Youth members. On an inspection of the Krupp factory in Essen, production of tanks and U-boats had already begun. On each visit, the couple were presented with enthusiastic workers keen to extol their working conditions to the Duke. He, in turn, was at his most charming, says Hichens. On one occasion, he joined a session of rowdy drinking songs in a staff beer garden, where he wore a false moustache and played skittles. The couple were regularly greeted with the Nazi salute, which they sometimes reciprocated (that was not unusual, and most visitors to Germany, including sports teams, then made the salute). The couple were welcomed at each venue by both the German and the British national anthems. The Nazis, researcher Peter Allen finds, knew the Duchess to have a keen interest in china, and as such, they included a trip to the Meissen porcelain works. Allen suggests that this demonstrated a policy of pleasing the Duke through his wife. On a visit to one of Ley's DAF meetings Edward made a speech, telling the assembly: +I have travelled the world and my upbringing has made me familiar with the great achievements of mankind, but that which I have seen in Germany, I had hitherto believed to be impossible. It cannot be grasped, and is a miracle; one can only begin to understand it when one realizes that behind it all is one man and one will. +Simpson, meanwhile, notes Morton, maintained the fiction in her letters to her friends and family that they were merely sightseeing. +The tour culminated on 22 October, when they met Hitler at the Berghof. It is possible that the meeting was a last-minute addition to their itinerary, as they were supposedly told of it only the previous day, although Allen suggests that this was unlikely, as Hitler had expressed a wish to meet the Duke. +The Duke and the Duchess had to wait before Hitler was ready to see them although, notes Vickers, he was in a genial mood when he did. The two men had an hour-long discussion, with Hitler doing most of the talking. The Duke is known to have encouraged Hitler in Germany's desired territorial expansion into Central and Eastern Europe. The minutes of the meeting appear to have been lost, presumably destroyed during the war. The Duchess did not join her husband but instead had tea with Hess. General Ernst Wilhelm Bohle acted as her interpreter. A friend of the Windsors, French millionaire Paul-Louis Weiller, later said that the Duchess had organised the meeting with Hitler and that being excluded from it had angered her. At the end of their visit, the three had tea together. Hitler's partner, Eva Braun, was not present: whenever he entertained guests of high rank, she had to stay in her bedroom until they had left. The Windsors made a good impression on Hitler, suggests Hichens; the Duchess later wrote how she was both "fascinated and repelled" by Hitler. Hitler, comments historian Philip Ziegler, "mildly irritated the Duke by insisting on using an interpreter rather than speaking directly to him in German". Interpreter Paul Schmidt later recalled the meeting: +Hitler was evidently making an effort to be as amicable as possible towards the Duke, whom he regarded as Germany's friend, having especially in mind a speech the Duke had made some years before, extending the hand of friendship to Germany's ex-servicemen's associations. In these conversations, there was, so far as I could see, nothing whatever to indicate whether the Duke of Windsor really sympathised with the ideology and practices of the Third Reich, as Hitler seemed to assume he did. Apart from some appreciative words for the measures taken in Germany in the field of social welfare, the Duke did not discuss political questions. +Forwood disagrees with Schmidt's recollection and says that the Duke raised criticisms of Nazi social policy. Forwood also says that at the same time, Forwood accused Schmidt of mistranslating for Hitler and that Forwood interjected "Falschübersetzt!" or "wrongly translated!" The Duke departed, he believed, under the impression that Hitler was a pacifist. An observer describes how they returned to their car and were escorted by their host: +The Duchess was visibly impressed with the Führer's personality, and he apparently indicated that they had become fast friends by giving her an affectionate farewell. [Hitler] took both their hands in his saying a long goodbye, after which he stiffened to a rigid Nazi salute that the Duke returned. +Historian Volker Ullrich argues that Hitler seems to have been flattered that the Windsors wanted to see him. Weidemann later said that he had rarely seen Hitler "so relaxed and animated as during that visit". The meeting concerned the British government since it appeared to be almost an informal summit. Three days earlier, Hitler had been telephoned by the future British Foreign Secretary, Lord Halifax, regarding Germany's expansionist policies. Halifax had pressed the benefits of a mutual understanding between their two countries. The Windsors' visit soon after, says Sebba, probably encouraged Hitler to see Edward as an ally. Edward later said that he had thought Hitler was "a somewhat ridiculous figure, with his theatrical posturings and his bombastic pretensions", and he denied to his wife that he and Hitler had discussed politics at all. The Duke's interpreter, Dudley Forwood, also put down his different recollection of what was said by writing how "my Master said to Hitler the Germans and the British races are one, they should always be one. They are of Hun origin." +The Duke and Duchess spent the last night of their tour back in Munich, where they stayed at the Vier Jahreszeiten Hotel; the Duke received some personal guests. One of them was a Kreisleiter of the Nazi Party, who had previously been Master of Ceremonies for Grand Duke Adolphus Frederick VI of Mecklenburg-Strelitz, a personal friend of the Duke's father. The main event was a dinner given by Rudolf and Ilse Hess and attended by high-ranking Nazi officials. Petropoulos comments that although there are no records of what may have been discussed at the dinner, "it is striking that the Duke and Hess, both future advocates of a negotiated peace, had the opportunity to spend the evening together and review the Windsors' tour". Ilse Hess later told how at one point, the Duke and her husband had been gone for over an hour. She found them in an upstairs games room. There, Hess had a large collection of model ships, and he and the Duke were "excitedly" re-enacting a World War I naval battle. +The British government vainly tried to control public relations during the visit. Cadbury notes how a former English king "turning up in ... Berlin was an unexpected bonus" for German diplomacy. German newspaper Deutsche Allgemeine Zeitung boasted about the number and the quality of the people who wanted to see the Nazis' social programme first hand, and it wrote that "the Duke of Windsor, too, has come to convince himself personally of the energy with which the new Germany has tackled her social problems". The German government took advantage as soon as the Duke and Duchess departed. Ogilvie-Forbes reported that Ley had already announced that Edward had praised Hitler's leadership. Hitler subsequently asserted that Simpson, in his opinion, would have been a good queen. Hitler believed Edward understood the Führerprinzip, and that he was a man with whom the Nazis could work. The tour may have given rise to later suspicions that in the event of a successful outcome to Operation Sea Lion, a German invasion of Britain, the Duke would have been appointed as a puppet king. In his diary, the Earl of Crawford summed up the British establishment's views on the Duke: +He had put himself hopelessly in the wrong by starting his visit with a preliminary tour in Germany where he was, of course, photographed fraternizing with the Nazi, the Anti-Trade Unionist and the Jewbaiter. Poor little man. He has no sense of his own and no friends with any sense to advise him. I hope this will give him a sharp and salutary lesson. +Similarly, diplomat and soldier Sir Robert Bruce Lockhart noted in his diary that he expected Edward to return sooner rather than later as "a social-equalising king, inaugurate an English form of fascism and an alliance with Germany". To the British ruling class, comments Morton, the Windsors' "farrago was greeted with undisguised glee". On the other side of the parliamentary divide, the Labour Party MP Herbert Morrison (leader of the London County Council) wrote that "if the Duke wants to study social problems he had far better quietly read books and get advice in private, rather than put his foot in it in this way". The Times reported how "His Royal Highness acknowledges with smiles and the National Socialist salute the greetings of the crowds gathered at his hotel and elsewhere during the day". The Daily Express, meanwhile, stated that Edward had received "the kind of reception that only the old kings of Bavaria could expect". The reaction in Germany, according to the British attaché in Leipzig, was that the tour had demonstrated the Duke's "strong pro-fascist sympathies". In the Soviet Union, the view was that the British royal family had "warm feelings" for Germany. +The most positive aspect of the visit, comments Powell, "was that it had been well-organised, albeit for the benefits of the hosts". Philips calls the tour "an embarrassment at best, and at worse, glaring proof of his complete lack of judgement". Piers Brendon describes it as "the worst blunder of his career". Roberts calls the tour "fantastically ill-judged", and Bloch notes that the Duke's political contemporaries were all in agreement that starting the tour in Nazi Germany at such a time was nothing short of "disastrous". Scholar Julia Boyd, comparing the meeting with Hitler with others that had taken place—the Aga Khan, for example—notes that while attracting a great deal of comment, they "could not compete with the Duke and Duchess of Windsor ... in terms of celebrity and sheer inappropriateness". +Sebba explains Edward's lack of judgement by the fact that while he had been able to call on a wide spectrum of counsel as Prince of Wales, he now had only his wife and acquaintances. Powell, similarly, believes that Edward's reputation "was at the mercy of unscrupulous strangers". Ziegler, conversely, suggests that while the trip may have been "ill-advised and ill-timed ... [it was] not a crime". Vickers, similarly, suggests that while the tour may have helped fuel the theory that the Duke was a Nazi, "he was no such thing. But he was naive, and having been brought up with people to advise him all his life until December 1936 he was hardly competent or equipped to deal with men like Hitler. Nor should he have undertaken this trip independently." +According to Sebba, Edward promised to refrain from making speeches so that his words could not be used against him by critics. Some scholars, such as Bradford, believe the visit to be directly the result of "pro-German and even more pro-Nazi" views. German people who witnessed the Duke on tour, suggests Morton, did not see him "either publicly or privately, as a collaborator, appeaser or traitor to his country. Far from it." Scholar Gerwin Strobl agrees and writes: +When the Nazis were dealing with a useful fool, they could never quite disguise an element of contempt in their language; when they met a rogue, their words betray a shared contempt for others. There is nothing of this in the descriptions of the Duke's conversations in Berlin or the later wartime recollections of his actions and opinions. Instead, there is something one comes across only very rarely in Nazi utterances: genuine respect; the respect felt for an equal. +The Windsors' German tour made little impact on the British public, and the main criticism seems to have been the failure to keep the low profile that he had promised. Churchill, for example, wrote to the Duke to imply that there had been little notice taken of the Nazi aspect and that he was "glad it all passed off with such distinction and success". The new prime minister, Neville Chamberlain, disagreed with the tour and privately worked against it, but, comments historian of Nazi Germany Karina Urbach, "as a convinced monarchist [he] did everything to keep the institution intact". +In 1966, the Duke described his memories of meeting Hitler, who had, Edward said, "made me realize that Red Russia [sic] was the only enemy and that Great Britain and all of Europe had an interest in encouraging Germany to march against the east and to crush communism once and for all ... I thought that we ourselves would be able to watch as the Nazis and the Reds would fight each other". His equerry, Forwood, said something similar in his memoirs: +Whereas the Duke, Duchess and I had no idea that the Germans were or would be committing mass murder on the Jews, we were none of us averse to Hitler politically. We felt that the Nazi regime was a more appropriate government than the Weimar Republic, which had been extremely socialist. +The Windsors returned to Paris on 24 October, with a fortnight to prepare for the tour of the United States. The week after the Windsors left Munich, the Nazis executed two KPD organisers and labour leaders: Adolf Rembte and Robert Stamm. They were widely admired in the American labour movement for their trade union and anti-Nazi activity; their deaths swung popular opinion against the Duke and the Duchess. Labour unions campaigned against the tour, particularly in the Duchess's hometown of Baltimore. Unions said that they would not support the Windsors' visit and called them either "emissaries of a dictatorship or uninformed sentimentalists". +Bedaux, who, Vincent suggests, intended to use the Duke to regain possession of his confiscated German business, was irreparably damaged by the fallout from the Windsors' tour. In 1938, his German businesses were confiscated by the Nazis permanently. His reputation also suffered in America, where his operations were forcibly taken over by a U.S.-based subordinate. The Duke's public connection to Bedaux, combined with the bad publicity, persuaded Edward to cancel the tour. The New York Times reported on 23 October that in its view, the German tour, "demonstrated adequately that the Abdication did rob Germany of a firm friend, if not indeed a devoted admirer, on the British throne. He has lent himself, perhaps unconsciously, but easily to National Socialist propaganda." Another correspondent wrote that "the poor fellow must have very little discretion and must be very badly advised. His going to Germany and hobnobbing with Hitler and Ley just before visiting America was enough to enrage every liberal organization in the country." +The American trip had been intended to demonstrate the Duke's leadership qualities, and its cancellation was sufficiently traumatic to induce him to retire temporarily from public life. U.S. President Franklin D. Roosevelt wrote a conciliatory letter to the Windsors that expressed hope that the tour would eventually go ahead. After the outbreak of the Second World War in 1939, says Bloch, the British government removed the Windsors from Europe for the war's duration. The Duke was appointed governor of the Bahamas. Churchill wrote to Roosevelt in July 1940: +The position of the Duke of Windsor in recent months has been causing His Majesty and His Majesty's Government some embarrassment as though his loyalties are unimpeachable there is always a backwash of Nazi intrigue which seeks to make trouble about him now that the greater part of the continent is in enemy hands. There are personal and family difficulties about his return to this country. In all the circumstances it was felt that an appointment abroad might appeal to him, and the Prime Minister has with His Majesty's cordial approval offered him the Governorship of the Bahamas. His Royal Highness has intimated that he will accept the appointment. +The Duchess called the Bahamas "the St Helena of the 1940s" for them. +Ivan Khristoforovich Bagramyan, born Hovhannes Baghramyan (2 December [O.S. 20 November] 1897 – 21 September 1982), was a Soviet military commander of Armenian origin who held the rank of Marshal of the Soviet Union. As commander of the 1st Baltic Front, he orchestrated the offensives which pushed German forces out of the Baltic countries on the Eastern Front of World War II. +Bagramyan was the second non-Slavic military officer, after Latvian Max Reyter, to become a commander of a front. He was among several high-ranking Armenian officers serving in the Soviet Army during the war. Bagramyan's experience in military planning as a chief of staff allowed him to distinguish himself as a capable commander in the early stages of the Soviet counter-offensives against Nazi Germany. He was given his first command of a unit in 1942, and in November 1943 received his most prestigious command as the commander of the 1st Baltic Front. +He did not immediately join the Communist Party after the consolidation of the October Revolution, becoming a member only in 1941, a move atypical for a Soviet military officer. After the war, he served as a deputy of the Supreme Soviets of the Latvian Soviet Socialist Republic and Armenian Soviet Socialist Republic and was a regular attendee of the Party Congresses. In 1952, he became a candidate for entry into the Central Committee and, in 1961, was inducted as a full member. For his contributions during the war, he was widely regarded as a national hero in the Soviet Union, and continues to hold such esteemed status among Armenians and Russians today. +Ivan Khristoforovich Bagramyan was born on 2 December [O.S. 20 November] 1897 in Yelizavetpol, a city then part of the Russian Empire and now part of Azerbaijan. His parents, Mariam and Khachatur Bagramyan, were ethnic Armenians originally from the village of Chardakhlu. Bagramyan's paternal grandfather came from the village of Koti. While Bagramyan's father went to work all day at the local railway station, his mother stayed at home to take care of her seven children. Bagramyan's parents decided to enroll him at a recently opened local two-year school because they could not afford to send him to the local gymnasium. +Graduating in 1912, Bagramyan, whom everyone affectionately called Vanya, followed his father and his brothers in a path in rail work, attending the three-year railway technical institute located in Tiflis. He graduated with honors and was slated to become a railway engineer within a few years when events in World War I changed his life. +Bagramyan was well aware of the military situation at the Caucasus front during the first months of the world war. In the winter of 1914–15, the Imperial Russian Army was able to withstand and repel the Ottoman offensive aimed at capturing Sarikamish. Bagramyan also began reading harrowing reports in the Russian press of what was taking place against his fellow kinsmen across the border: the Committee of Union and Progress-led government had embarked on a campaign to carry out a genocide of the Ottoman Armenians. He desperately sought to join the military effort but because he was only seventeen and a railway mechanic, he was not subject to conscription. This did not dissuade him from trying, as he later remarked, "My place is at the front." +His opportunity came on 16 September 1915, when he was accepted as a volunteer in the Russian army. He was assigned to the 116th Reserve Battalion and sent to Akhaltsikhe for basic training. With his training complete in December, he joined the 2nd Caucasian Border Regiment of the Russian Caucasus Army, which was sent to dislodge the Ottomans in Persia. Bagramyan participated in battles in Asadabad, Hamedan and Kermanshah, the Russian victories here sending Ottoman forces reeling toward Anatolia. +Learning about the exploits of the men in the outfit, the chief of staff of the regiment, General Pavel Melik-Shahnazarian, advised Bagramyan to return to Tiflis to enroll in the Praporshchik Military Academy. But in order to attend the school, Bagramyan needed to satisfy the academy's requirement of having completed school at a gymnasium. This did not deter him and, after preparing for the courses in Armavir, he passed his exams and began attending the academy on 13 February 1917. He graduated in June 1917 and was assigned to the 3rd Infantry Regiment, stationed near Lake Urmia. But with the overthrow of the Russian Provisional Government in the midst of the October Revolution of 1917, his unit was demobilized. +With the creation of the newly established First Republic of Armenia in 1918, Bagramyan enlisted in the newly formed 3rd Regiment. From 1 April 1918, that is, after the Ottoman Empire signed the Treaty of Brest-Litovsk (3 March 1918) with the Russian SFSR, he was in the 1st Cavalry Regiment, which took part in battles in Karaurgan, Sarikamish and Kars against units of the advancing Ottoman Third Army. Notably, he took part in the May 1918 battle at Sardarapat, where the Armenian military scored a crucial victory against Ottoman forces. He remained in the regiment until May 1920. +Three years after the toppling of the Provisional Government by the Bolsheviks in October 1917, the Red Army invaded the southern Caucasus republics of Azerbaijan, Georgia and Armenia. In May 1920, Bagramyan, upset with the country's social and political conditions, participated in the failed May Uprising against the Dashnak-led government of Armenia. Later that year, he joined the Red Army. He was jailed and sent to work in the fields for several months but was allowed to rejoin the military with the outbreak of the Turkish invasion of Armenia. By December 1920, Armenia was Sovietized and the national army was subsequently disbanded. Bagramyan, however, chose to join the 11th Soviet Army and was appointed a cavalry regiment commander. +As life in Armenia grew relatively more stable under Soviet rule, Bagramyan sought to locate a woman he had met several years earlier, Tamara Hamayakovna. Tamara, who was at this time living in Nakhichevan with her family, had been married to an Armenian officer who had been killed during the Turkish invasion, leaving her with their one-year-old son, Movses. Bagramyan visited her and the two decided to get married at the end of 1922. In addition to their son Movses, who went on to become a painter, they had a daughter, Margarit, who later became a doctor. Tamara remained at Bagramyan's side until her death in 1973. +In 1923, Bagramyan was appointed commander of the Alexandropol Cavalry Regiment, a position he held until 1931. Two years later, Bagramyan graduated from the Leningrad Cavalry School and, in 1934, from the Frunze Military Academy. In his memoirs, Pyotr Grigorenko, a Ukrainian commander who attended the Academy, recalled how Bagramyan was expelled from the academy by his superiors after they had learned that he had been a secret member of the banned Armenian nationalist party Dashnaktsutiun for more than a decade. Pending his arrest, Grigorenko described Bagramyan "deeply depressed, saying he only wished they'd arrest him soon so that he could get it over with." Grigorenko advised that he appeal the arrest warrant, which Bagramyan reluctantly did. The arrest warrant was ultimately dropped, with the help of Armenian politburo member Anastas Mikoyan. From 1934 to 1936, Bagramyan served as the chief of staff of the 5th Cavalry Division, and from 1938, he worked as a senior instructor and lecturer at the Military Academy of the Soviet General Staff even as Stalin purged the senior Soviet officer corps. While fellow students from the military academy, Andrei Yeremenko and Georgy Zhukov, had seen their careers rise, Bagramyan's had remained stagnant. +In 1940, when General Zhukov was promoted to commander of the Kiev Military District in the Ukrainian SSR, Bagramyan wrote a letter asking to serve under his command. Zhukov agreed, and in December asked for his help writing a paper to be presented to the commanders of the Soviet Military Districts. Bagramyan's paper, "Conducting a Contemporary Offensive Operation," apparently impressed Zhukov, as he promoted Bagramyan to become the head of operations for the Soviet 12th Army based in Ukraine. Within three months, however, Bagramyan, then a colonel, was appointed deputy chief of staff of the Southwestern Front, headquartered in Kiev. +In June 1941, Nazi Germany invaded the Soviet Union. Unlike many of the border troops who were caught off guard by the offensive, Bagramyan and his commander, General Mikhail Kirponos, believed an invasion by Germany was inevitable. However, Kirponos chose to ignore Bagramyan's belief that the German offensive would employ the Blitzkrieg-like tactics like those seen in the campaigns in Poland in 1939 and western Europe in 1940. Since the winter of 1939–40, Bagramyan had been busy devising a battle plan that would counter threats from the direction of western Ukraine, which was approved after numerous revisions on 10 May 1940. +On the morning of 22 June, he was tasked with overseeing a transfer of a military convoy to Ternopol. While his column was passing the Soviet airfields near the city of Brody, German air strikes hit the aircraft on the ground. Several hours later, they arrived in Ternopol, having been strafed twice by the planes. Three days after the invasion, the plans for the counter-offensive were implemented, but disorder engulfed the troops, and the counter-attack collapsed. Bagramyan took part in the great tank battles in western Ukraine and the defensive operation around Kiev, in which Kirponos was killed and the entire Front captured by the Germans. He was one of a handful of senior officers who escaped from the encircled Front. +Bagramyan was then appointed chief of staff to Marshal Semyon Timoshenko and along with future Soviet premier Nikita Khrushchev, then a political officer, coordinated the fighting around Rostov. In his memoirs, Khrushchev described Bagramyan as a "very precise person who reported on everything just as it was. How many troops we had, their positions, and the general situation." +Khrushchev went on to detail an account where Marshal Semyon Budyonny, sent by the chief of the operations department from Moscow as a representative of Stavka, arrived in Kiev to preside over Bagramyan's court-martial. Bagramyan protested vigorously and said that if his competence was in question, then he should instead be given a field unit to command. To Bagramyan's incredulity, Budyonny went on to attempt to convince him to agree to his execution. Khrushchev remarked that the argument was sparked arbitrarily and had taken place after an "abundant feast with cognac" and that "in those days we didn't take that kind of conversation seriously." According to him, at the time, however, the Soviet military was especially suspicious of the men in its ranks, itself judging that there were "enemies of the people...everywhere, especially the Red Army." +Bagramyan was instrumental in the planning of two Soviet counter-offensives against the Germans, including the major push made by Soviet forces in December during the Battle of Moscow and for this was promoted to the rank of lieutenant general. In the same month, he was made the chief of staff of a military operations group that would oversee three Army Groups: the Southern, the Southwestern and Bryansk Fronts. In March 1942, he accompanied Khrushchev and Timoshenko to Moscow to present the plans of a new counter-offensive in the direction of Kharkov to Stalin. Stalin, impressed with his plan, approved the operation and on 8 April, promoted Bagramyan as chief of staff of the Southwestern Front. On 12 May 1942, armies of the Southwestern Front attacked Kharkov but the launch of the offensive came at an inopportune moment since they were attacking from the Barvenkovo Salient, a region that German forces were near closing. +While Soviet forces were initially successful in recapturing Kharkov, they found themselves trapped by the German Army after the closing of Barvenkovo. On 18 May, Bagramyan asked Timoshenko to alter the plans but Timoshenko along with Stalin refused to approve his request. Soviet losses were heavy as the 6th, 9th and 57th armies (approximately 18–20 divisions) comprising a large portion of the Southwestern Front, were all destroyed and Bagramyan was removed from his post on 28 June by Stavka. According to Khrushchev, Bagramyan was so devastated by the immense loss of men that after the operation was called off: "he burst into tears. His nerves cracked...He was weeping for our army." Held responsible for the failure of the operation and "poor staff work", he was demoted to chief of staff of the Soviet 28th Army. Several days later, he wrote a letter to Stalin asking to "serve at the front at any capacity, however modest." British military historian John Erickson contends that Bagramyan was unfairly scapegoated by Stalin in his attempts to "hunt for [the] culprits" of the mismanagement of operations. +Though he had never led a fighting unit prior to the war, he was given his first command of an army in the Western Front as his superiors, and particularly Marshal Zhukov were impressed with his skills and capabilities as a staff officer. Zhukov, with the approval of Stavka, appointed him commander of the 16th Army (2nd formation), (July 1942 – April 1943) replacing its former commander, Konstantin Rokossovsky who had been sent to command the Bryansk Front. The 16th Army transferred its troops to the 5th Army, and its command and staff were moved to the second echelon of the Western Front where the Army took up command of part of the 10th Army's troops, and its defensive positions. On 11 August, however, German forces mounted a surprise offensive on the southern flank of Western Front, splitting the 61st Army from the 16th Army which was not taking part in the Rzhev-Sychevka Offensive operation. The German forces threatened Bagramyan's left flank as he quickly moved his forces to counter their movements and halted them from advancing further on 9 September. +With the rest of the Eastern Front battles almost entirely focused on Stalingrad and the Germans' attempts to advance into the Caucasus, the 16th Army was not called up to action until February 1943. By then, the German 6th Army besieged in Stalingrad had been encircled and surrendered. The 16th Army at the time was composed of four divisions and one infantry brigade and in light of the new offensive, Bagramyan's force was given two extra divisions, an infantry brigade, four tank brigades and several artillery regiments. +As the battle of Stalingrad marked the turning point of the war, German forces reorganized for a new offensive in the summer of 1943 to attack the Soviet held Kursk salient in Russia. The German High Command was to deploy veteran units to destroy the salient, including the Ninth Army and the II SS Panzer Corps. Stavka, already informed of the impending offensive, called for an advance toward the German defenses positioned near the town of Kozelsk, which would drive south with the help of the armies of the Central Front. The forces would then proceed to cut off a 75-mile (121 km) gap that would effectively surround the Germans and cut it off from reinforcements. This was similar to Operation Uranus, where the Soviet Army encircled and trapped the Sixth Army in Stalingrad. +Bagramyan's 11th Guards Army (the renamed 16th Army) was tasked to take part in the offensive and was given an additional three infantry divisions and two tank corps, a force composed of fifteen divisions. Bagramyan, however, argued to Stavka that its planning was too audacious in the hopes of repeating a successful encirclement like that in Uranus. He claimed that his forces would be overstretched and would have difficulty in attacking the entrenched German positions in Bolkhov. To avoid a repetition of the failure in Kharkov the previous year, he instead asked that the 61st Army from the Bryansk Front aid his 11th Guards in destroying the German forces in Bolkhov, thus eliminating the Ninth Army's protection from the north. He appealed to his front commander Vasily Sokolovsky as well as the Bryansk's M. A. Reyter, both of whom rejected his proposal. In April, Stavka summoned the main Front commanders to Moscow to brief them on the preparations for the battle. Against the protestations of Sokolovsky and Reyter, Bagramyan proposed his alternative plan to Stalin, who agreed that it would be the more appropriate course to follow. Bagramyan was given twenty days to prepare the 11th Guards Army, and on 24 May reported that his forces were ready. The 11th Guards now was composed of 135,000 men, 280 armored fighting vehicles, 2,700 artillery pieces and several hundred ground-support aircraft. Stalin, however, felt it necessary to further wear thin the fighting abilities of the German forces and delayed the offensive. +Ultimately, it was the Germans who took the initiative by launching Operation Citadel on 5 July in the area around Kursk. German losses were initially heavy due to Soviet defensive preparations. Taking advantage of this, on 12 July, Bagramyan's forces commenced their offensive, codenamed Operation Kutuzov, and quickly breached the German defenses, advancing a distance of 45 miles (72 km) by 18 July. By 28 July, the operation concluded successfully and he was promoted to the rank of Colonel-General. In the following month, his forces took part in the large-scale tank offensives which routed the German assaults and forced Germany to remain on the defensive for the remainder of the war. +With the end of operations in Kursk, the Soviets began a series of offensives on various fronts to push the Germans out of the occupied Soviet republics. In October 1943, Bagramyan's 11th Guards Army was transferred to the Second Baltic Front which was concentrated on the retaking of Belarus and namely, the Baltic republics. In November, Stalin offered Bagramyan the position of head commander of the First Baltic Front which had similar objectives of the Second but was making little headway in its attempts to advance northwards. +Stalin would allow him to retain the 11th Guards and suggested that Colonel-General N. E. Chibisov, an officer he had served under, assume his position. Bagramyan, however, commented that he had had a frictional relationship with Chibisov and instead nominated Lieutenant-General K. N. Galitsky. Stalin, belatedly realizing that Bagramyan was implying that the two would be unable to coordinate harmoniously due to a conflict of holding the same rank, agreed to Bagramyan's suggestion and promoted him to the rank of Army General. He also agreed to have the Second Baltic Front return a tank corps and an infantry division that was taken from the 11th Guards, thus bolstering the forces under Bagramyan to a total of four armies: 11th Guards, 39th, 43rd and the 4th Shock. +In the winter of 1943, his forces advanced forward towards the Belarusian city of Vitebsk. One of the key elements to Bagramyan's success was that many of the soldiers were part of veteran units that had been trained in the Arctic regions of Siberia, enabling them to easily push through entrenched defenses the Germans had spent months preparing. Among the key locations imperative to reach Vitebsk was the small town of Gorodok, which served as a heavily-fortified German communications hub. Despite the strong defenses, Bagramyan was able to utilize his heavy artillery and air support from the Red Air Force in late December to bombard the town and then launched a three-pronged attack, the Gorodok offensive from the ground. The German garrison was overwhelmed, and by 24 December, two infantry divisions and one tank division had surrendered. In Moscow, the news of the victory at Gorodok prompted a 124-cannon salute in honor of Bagramyan and the First Baltic Front. +On 2 April 1944 Stalin granted Bagramyan's request to relieve the troops of the Front from offensive duties. However, German forces took advantage of the lull to mount an operation against Soviet partisans in Belarus. Bagramyan's senior staff diverted air support and other crucial supplies to aid the partisans, allowing most of them to escape German encirclement. With the advance of Soviet forces in the Baltic and Ukraine, German Army Group Center had largely been isolated as Stavka prepared to eliminate the pocket (consisting of Third Panzer, Second, Fourth, and Ninth Armies). Stavka's plan, codenamed Operation Bagration was kept secret from all of the involved Front commanders. Bagramyan himself was only informed in May 1944 of his role in the offensive. +Bagration called for the First, Second and Third Belorussian and the First Baltic Fronts to engulf Army Group Center. Bagramyan was tasked with attacking the forces in the pocket, cross the Daugava River and, along with Third Belorussian, clear the surrounding areas of Vitebsk of German forces. Although he felt the plans for the Bagration were sound, he worried about the possibility of a German incursion by Army Group North against his forces from the north. He appealed to his superiors once more, Zhukov and Alexander Vasilevsky, to have the First Baltic Front move westward to help eliminate the Third Panzer Army, thus splitting Army Group North in two. Zhukov and Vasilevsky accepted his argument, introducing it to Stalin in a meeting on 23 May who formally approved it in a directive on 31 May. +Although Bagramyan found it acceptable to sustain heavy casualties (as did all the commanders of the Red Army), he was disturbed with the immense loss of life sustained by his forces. He attempted to reduce those levels primarily by maintaining the element of surprise in operations. In his preparations for Bagration, he planned for the 43rd Army to move through the more geographically difficult swamps and marshlands to Army Group North's right flank. This maneuver would thus take North by surprise since it expected the Soviet offensive to move through more suitable terrain. He proved correct, as in early June 1944, the 43rd achieved success in its attack. Commander of the 43rd Army, General Afanasy Beloborodov, wrote that during the offensive they apprehended a German general who stated that German forces had been blindsided by the attacking forces. +As Bagramyan pushed towards Vitebsk, his forces were aided by the same Belorussian guerrilla fighters who had escaped the German encirclement in April. They provided vital intelligence, including information on the location of bridges and troop movements, and launched attacks against German logistic lines. On 22 June 1944, Bagration began as Bagramyan proceeded in moving westwards as previously planned. However, a widening gap on the Front's northern flank grew as it advanced while the Second Baltic Front, tasked to help defend that area, took no action. Stalin agreed to send a tank corps to reinforce Bagramyan's forces but ordered him to capture Polotsk, which would sever Army Group North's communication lines and open up a route towards the central Baltic. By 3 July, his troops had accomplished the tasks set forth in the directive, destroyed the Third Panzer Army and captured Polotsk. For his achievements, on 7 July he was decorated with the title of Hero of the Soviet Union. +With the overall success of Soviet forces in Bagration, his Front was expanded by three armies (although he ceded the 4th Shock Army to Second Baltic), the 39th Army (previously under the command of Third Belorussian), the 51st and the 2nd Guards Armies. The First Baltic Front was ordered by Stavka to move westward in order to stop Army Group North's remaining forces from escaping to Germany. Despite this, Bagramyan inferred that since many of the general orders were being issued to the Wehrmacht by Hitler, rather than the General Staff, and surmised that while there was a possibility that they would confront them in the Lithuanian town of Kaunas, he felt the more likely location would be Riga. He spoke with Vasilevsky, who agreed to change the plans if his theory and intuition proved correct. +As the First Baltic began moving towards Lithuania and into eastern Latvia, it became clear that Army Group North would attempt to outflank Bagramyan's forces near Daugavpils, as he had previously predicted. Vasilevsky, keeping his promise, appealed to Stalin to allow Bagramyan to move to Daugavpils but he refused. Vasilevsky in turn, took it upon his own initiative and gave Bagramyan the go ahead. However, with the loss of 4th Shock Army, Bagramyan was left shortchanged since his promised 39th Army had not only not arrived but was composed of only seven divisions (in comparison to 4th Shock's ten). Feeling that time was being lost, he pressed on with the units he had. +By 9 July, his ground forces had made significant gains in cutting off a vital road that connected Kaunas to Daugavpils. Taking advantage of this, Bagramyan worked with other Front commanders to attack the rear guard of Army Group Center but poor coordination between the units led a stall in the advance. At this time, Bagramyan realized that German forces were most probably not going to easily retreat from the Baltics and so further advances towards Kaunas would be pointless. He proposed to Stavka to launch a full-scale offensive towards Riga but the former rejected his plans, stating that the armies of Second and Third Baltic Fronts would have already pushed Army Group Center to Prussia by the time of the offensive. He attempted to convince them otherwise, citing the numerically deficient forces in the two Fronts, but was rebuffed and ordered to drive towards a road connecting the Lithuanian city of Šiauliai to Riga, resulting in its capture in late July. +With its capture, he persuaded Vasilevsky to allow his forces to move towards Riga, receiving a formal go-ahead by Stavka in a directive on 29 July. On 30 July, his forces finally reached the seaside city of Tukums, near the Bay of Riga, thereby cutting off a total of 38 German infantry and armored divisions in Latvia. For his achievements in this battle, he was given the title of Hero of the Soviet Union. During the month of August, Soviet forces stalled in the Riga offensive, concentrating on halting German attacks. Finally on 14 September 1944, the First, Second and Third Baltic Fronts launched full-scale offensives with the objective of Riga, encountering fierce resistance by its defenders. On 24 September, with his forces only 12 miles (19 km) from Riga, Stavka ordered Bagramyan's forces to abandon it to the Second and Third Baltic Fronts, regroup, and instead advance against Memel. His forces attacked Memel on 5 October and on 10 October, reached the city, effectively preventing Army Group North from retreating to Prussia. +In early 1945, Bagramyan's army, under the overall command of Vasilievsky, took part in the East Prussian offensive. In Operation Samland, Bagramyan's First Baltic Front, now known as the Samland Group, captured Königsberg (now Kaliningrad), in April. On 9 May 1945, he accepted the surrender of the German forces penned up in Latvia, capturing a total of 158 aircraft, 18,000 vehicles, 500 tanks and assault guns among other weaponry. +After the war, Bagramyan remained in command of the Baltic Military District, commanding operations against partisans in Lithuania and Latvia. In 1954, he was appointed chief inspector of the Ministry of Defence. On March 11, 1955, he was appointed Deputy Minister of Defence with the rank of Marshal of the Soviet Union. He was also head of the Military Academy of General Staff. +On 2 June 1958, he was again appointed Deputy Minister of Defence of the USSR - Head of Rear Services of the USSR Ministry of Defence (later - Deputy Minister of Defence of the USSR - Head of Rear Services of the Armed Forces of the USSR). +He spent much of his time writing articles in military journals on Soviet strategic operations and most notably, co-authored the six-volume work on Soviet involvement during World War II, The Soviet Union's Great Patriotic War (1941–1945). In August 1967, Bagramyan accompanied General Secretary Leonid Brezhnev and Premier Alexei Kosygin to North Vietnam, where they met with Vietnamese leaders to arrange the transfer of arms and supplies in advance of the Tet Offensive. +Bagramyan retired on 25 April 1968, and was transferred to the post of General Inspector in the Group of Inspectors General of the Ministry of Defence of the USSR. In 1971, he completed the first volume of his memoirs in This is How the War Began, and in 1977 the second volume, Thus We Went to Victory, was published. Among the numerous points he noted in the second volume was an analysis of the Red Army's costly offensives in the early stages of the war: +There is no point in hiding that before the war we mostly learned to attack, and did not pay enough attention to such an important manoeuvre as retreat. Now we have paid for this. It turned out that the commanders and the staff were not sufficiently prepared to prepare and execute the retreat manoeuvre. Now, in the second week of war, we had in fact to learn from the beginning the most difficult art – the art of the execution of retreat. +In 1979, another book of Bagramyan titled My Memoirs was published and was based on the first and second volumes. A large portion of the book was dedicated to the Armenian issues including the territories of Western Armenia, massacres of Armenians by the Ottoman Empire and the Armenian Genocide, the Ottoman invasion of Armenia and the Battle of Sardarapat, as well as other topics. +Marshal Bagramyan was awarded numerous Soviet and foreign orders and medals for his service, including two Orders of the Hero of the Soviet Union, seven Orders of Lenin, the Order of the October Revolution, three Orders of the Red Banner, two Orders of Suvorov and the Order of Kutuzov. Among the other commendations he received were the Polish Polonia Restituta twice and the Medal "For the Victory Over Germany". +After the death of Marshal Vasily Chuikov on 18 March 1982, he was the last surviving Soviet Marshal who had held a high command in World War II. However, Bagramyan was ill and died a few months later, on 21 September 1982, at the age of 84. He was buried with full military honors at the Kremlin Wall Necropolis in Moscow. A town and district (raion) in Soviet Armenia, a military firing range, an Armenian Army training brigade, and a subway station and a major avenue in the capital Yerevan, have been named in his honor. On 11 May 1997, the government of Armenia established the commemorative 100th Anniversary of Marshal Bagramyan medal. It is awarded to service and civilian personnel who participated in the Second World War. +Foreign +During the 1987–88 English football season, Gillingham F.C. competed in the Football League Third Division. It was the 56th season in which the club competed in the Football League, and the 38th since the club was voted back into the league in 1950. In the previous season, Gillingham had qualified for the final of the play-offs for promotion to the Football League Second Division but had been defeated. The team began the new season strongly, including winning matches 8–1 and 10–0 on consecutive Saturdays, but their performances quickly declined and by December the team had slipped down the league table. After a heavy loss to Aldershot in the last game of 1987, manager Keith Peacock was dismissed from his job and replaced by his former assistant Paul Taylor, a decision which was extremely unpopular with the club's supporters. Although the team's performances improved in the second half of the season, briefly bringing them close to a potential play-off place, their form declined once again and they finished the season 13th in the 24-team division, the same position as when Peacock lost his job. +During the season, Gillingham also reached the third round of the FA Cup and the second round of the Football League Cup, but were eliminated from the Associate Members' Cup at the earliest stage. The team played 55 competitive matches, winning 17, drawing 17, and losing 21. Steve Lovell was the club's leading goalscorer, with 27 goals in all competitions, three times as many as any other player. He also made the most appearances, playing in all of the team's 55 matches; two other players each appeared over 50 times. The highest attendance recorded at the club's home ground, Priestfield Stadium, was 9,267 for an FA Cup match against Birmingham City. +The 1987–88 season was Gillingham's 56th season playing in the Football League and the 38th since the club was elected back into the League in 1950 after being voted out in 1938. It was the club's 14th consecutive season in the Football League Third Division, the third tier of the English football league system, since the team gained promotion from the Fourth Division as runners-up in 1974. In the 13 seasons since then, the team had achieved a best finish of fourth place, one position away from promotion to the Second Division, a feat achieved in both the 1978–79 and 1984–85 seasons. In the 1986–87 season, Gillingham had finished fifth, qualifying for the newly introduced play-offs for the third promotion place, but lost to Swindon Town in the final. The two teams drew 2–2 on aggregate, necessitating a replay at a neutral venue, which Swindon won 2–0 to ensure that Gillingham would again be playing in the Third Division the following season. +Keith Peacock was the club's manager for a seventh season, having been appointed in July 1981. Paul Taylor continued in the role of assistant manager, having been appointed at the same time as Peacock. Bill Collins, who had been with the club in a variety of roles since the early 1960s, held the post of first-team trainer, and John Gorman managed the youth team. Mark Weatherly was club captain, but for much of the early part of the season he was unable to play following an operation and subsequent injuries, so Colin Greenall served as captain on the field of play in his absence. +Following Gillingham's failure to win promotion, Tony Cascarino, the previous season's top goalscorer, joined Millwall of the Second Division for a transfer fee of £225,000. This was at the time the highest fee which Gillingham had ever received for a player. Despite having lost the highest-scoring player of the previous season, the club did not immediately sign any new forwards; the only two players to join the club ahead of the new season were George Shipley, a midfielder signed from Charlton Athletic for £40,000, and Gary West, a defender who arrived from Lincoln City for £50,000. The Gillingham team prepared for the new season with several friendly matches and the club also staged an open day, when supporters could attend autograph sessions with players and watch them training. The club's first-choice kit featured shirts in the club's traditional blue with a white zigzag band across the chest; the away shirts, to be worn in the event of a clash of colours with the home team, were the reverse of this. At the start of the season, Gillingham were anticipating playing at Wembley Stadium, England's national stadium, for the first time in the club's history, as all 92 League teams were originally set to play there in April 1988 in the Football League Centenary Tournament. The format of the tournament was later significantly changed, however, such that only 16 teams would compete. Gillingham failed to qualify for the revised tournament and did not make their first appearance at Wembley until 1999. +Gillingham's first match of the season was at home to Blackpool and resulted in a 0–0 draw. After a second consecutive league match in which the team failed to score a goal, a 2–0 defeat away to Grimsby Town, Gillingham defeated Southend United 8–1 on 29 August, the most goals scored by Gillingham in a single match in the Football League since a 9–4 victory over Exeter City in 1951. Forward Steve Lovell scored four times, the first time a Gillingham player had scored as many goals in a league game since 1958. Seven days later, Gillingham played Chesterfield, who were joint top of the league table going into the game and had the best defensive record in the division, having not conceded any goals in the first four matches of the season. Gillingham won the game 10–0, setting a new record for the club's biggest win in the Football League; four players each scored two goals but on this occasion Lovell did not score at all. The win was the largest by any team in the Football League since 1964; it was the 61st game in Football League history in which a team scored double figures, but the first such game in which no player scored more than twice. +Victory over York City on 19 September left Gillingham second in the table, but one draw and two defeats in the next three games meant that the team dropped to twelfth. New signing Shipley was seriously injured against York and did not play again for nearly six months. In early October, the club signed forward Mark Cooper from Tottenham Hotspur for a fee of £102,500, setting a new record for the highest fee Gillingham had paid to sign a player. He went straight into the starting line-up for the home game against Bristol Rovers on 10 October and scored his first goal two games later in a 1–1 draw with Preston North End. Gavin Peacock, son of the manager, also made his debut against Bristol Rovers, having joined the club on a one-month loan from Queens Park Rangers. An unbeaten run of five games pushed Gillingham back up among the teams contending for a play-off place; the team ended the month of October in seventh position, two places behind the lowest position which would qualify for the play-offs. +The team's unbeaten run ended with a 1–0 defeat at home to Brentford on 3 November and the rest of the month's matches resulted in two draws and another defeat, after which Gillingham had once again fallen to 12th place in the league table. During November, the club signed Bobby Hutchinson from Walsall on loan, but he suffered an injury less than 10 minutes into his first training session and the loan was curtailed. In early December, Gavin Peacock rejoined the club, this time on a permanent basis for a fee of £40,000. The team defeated Rotherham United 2–1 on 12 December, but then lost to Walsall. Gillingham's last match of 1987 was away to Aldershot and resulted in a 6–0 defeat, the most goals conceded by Gillingham in a match for more than three years. The team had now won only one of the last eight games and had slipped into the bottom half of the league table. The following day, manager Peacock was dismissed from his job by the club's board of directors. Assistant manager Taylor was appointed to the manager's role for the remainder of the season. +The team's first match under new manager Taylor was away to Southend United on 1 January 1988 and resulted in a 3–1 win for Gillingham; veteran goalkeeper Ron Hillyard, who was in his 14th season with the club but had been kept out of the team by Phil Kite before Christmas, was recalled to the team. The first home game since the change of manager took place the following day and ended in a 2–2 draw with Fulham. The home fans staged a protest against the dismissal of Peacock, who was in attendance at the game; after the match some supporters caught up with the former manager in the car park and carried him on their shoulders down the road outside the stadium. Afterwards Peacock told the press "There was no way I could have stopped the fans doing what they did. Everything was very orderly and non-violent and I considered it merely the supporters' way of saying thank you for what I have done at Gillingham." The team ended the month of January with consecutive defeats to Wigan Athletic and Sunderland. +Gillingham were unbeaten in five matches played in February, winning three and drawing two, which took the team back up to eighth in the table. During this run, Greenall was transferred to Oxford United for a new club-record fee of £285,000. Lovell scored a total of eight goals in the five matches, including his second hat-trick of the season in a 4–1 win over Chesterfield. Gillingham scored only a single goal in the next four games, which resulted in two draws and two defeats. Shipley made his return to the team in the defeat away to Chester City, having not played since the previous September. On 26 March, Gillingham ended their run of winless games by defeating league leaders Notts County away from home. Defender Alan Walker, a new signing from Millwall, made his debut in that match, after which Gillingham were ninth in the table. +Over the Easter weekend, Gillingham lost to both promotion-chasing Brighton & Hove Albion and Northampton Town. In the next game away to Brentford, Gillingham took the lead eight minutes from the end of the game but then conceded a late equaliser; the draw left them 11th in the table. The team next defeated Preston 4–0, their biggest victory since September, but the attendance of 2,721 was the lowest of the season for a league game at Priestfield. Gillingham then lost consecutive matches to Bury and Rotherham. The final match of the season was away to third-placed Walsall, and finished as a 0–0 draw; the team had only won two of the last twelve games of the season and finished in 13th place in the Third Division, the same position as when Keith Peacock was dismissed in December. +Key +Results +As a Third Division team, Gillingham entered the 1987–88 FA Cup in the first round and were drawn to play fellow Third Division team Fulham; Gillingham won 1–0 with a goal from Greenall. In the second round they played another Third Division team, Walsall, whom they defeated 2–1. The First and Second Division teams entered the competition in the third round and Gillingham were drawn to play Birmingham City of the Second Division. Greenall scored an own goal after three minutes and Birmingham added two more goals to win 3–0 and end Gillingham's participation in the FA Cup. Further protests against Peacock's dismissal occurred at the Birmingham game. +Key +Results +As a Third Division team, Gillingham entered the 1987–88 Football League Cup in the first round and were drawn to play fellow Third Division team Brighton. In the first match of the two-legged tie, Gillingham won 1–0 with a goal from Greenall. In the second leg, Chris Hutchings scored for Brighton midway through the first half to level the aggregate score. Shortly afterwards, Gillingham goalkeeper Kite was sent off for fouling an opposition player, reducing his team to ten men. As no substitute goalkeeper was available, defender Paul Haylock volunteered to take over in goal. With his team-mates adopting a defensive strategy, Haylock conceded no goals during the remainder of the game; with the scores still level, a penalty shoot-out was required. Both teams scored with their first four penalties; Gillingham scored their fifth and then Haylock saved Brighton's fifth kick to give Gillingham the victory. In the second round, Gillingham played Stoke City of the Second Division, who won the first leg 2–0 and the second 1–0 to eliminate Gillingham from the competition. +Key +Results +The 1987–88 Associate Members' Cup, a tournament exclusively for Third and Fourth Division teams, began with a preliminary round in which the teams were drawn into groups of three, contested on a round-robin basis. Gillingham's group also contained Aldershot of the Third Division and Leyton Orient of the Fourth. Gillingham lost 3–1 to Aldershot in their first match and drew 2–2 with Leyton Orient in the second and finished bottom of the group, which meant that they failed to qualify for the next round. +Key +Results +Lovell made the highest number of appearances during the season; he was in the starting line-up for every one of the club's 55 matches. Pritchard and West also made over 50 appearances, each playing 51 times. Both players played in every game in the FA Cup, League Cup, and Associate Members' Cup, but each missed four Third Division matches. Four other players made more than 40 appearances. Four players, all members of the club's youth team, played only one game each. Of these, Ian Docker, Ivan Haines and Lee Palmer would all go on to become regulars in the first team, but Neil Luff's single appearance as a substitute would prove to be the only game of his professional career. +Lovell was also the team's top goalscorer, with 25 goals in the Third Division and a total of 27 in all competitions, three times as many as any other player. It was the first of four consecutive seasons in which he would be Gillingham's top scorer. He was the season's overall top scorer in Third Division matches; Southend's David Crown finished the season with 26 league goals, but this included 9 scored in the Fourth Division for Cambridge United before he was transferred in November. +Taylor remained manager of Gillingham for the start of the 1988–89 season, but was dismissed in October 1988, after only ten months in the job, following a run of poor results. Gillingham finished 23rd in the Third Division and were relegated to the fourth tier of English football, where they would remain for seven seasons. Peacock returned to management in the summer of 1989 when he was appointed by Maidstone United after they were promoted into the Fourth Division to become Kent's second Football League club. He remained a popular figure with Gillingham supporters, referred to as "Sir Keith". Speaking in 2004, he attributed his dismissal in part to the inexperience of the club's directors, saying "A new board had taken over and they were a little bit green. They got carried away by the play-off final and 18 goals in two weekends, thinking we would steamroller the league. I'm proud of my time with the club, though." +Bateson, Bill; Sewell, Albert, eds. (1987). News Of The World Football Annual 1987–88. London: Invincible Press. ISBN 978-0-85543-100-6. +Bateson, Bill; Sewell, Albert, eds. (1988). News Of The World Football Annual 1988–89. London: Invincible Press. ISBN 978-0-85543-145-7. +Bateson, Bill; Sewell, Albert, eds. (1989). News Of The World Football Annual 1989–90. London: Invincible Press. ISBN 978-0-85543-172-3. +Brown, Tony (2003). The Definitive Gillingham F.C.: A Complete Record. Nottingham: Soccerdata. ISBN 978-1-89946-820-1. +Dunk, Peter, ed. (1987). Rothmans Football Yearbook 1987–88. London: Queen Anne Press. ISBN 978-0-35614-354-5. +Elligate, David (2009). Gillingham FC On This Day. Durrington: Pitch Publishing. ISBN 978-1-90541-145-0. +Rollin, Jack, ed. (1988). Rothmans Football Yearbook 1988–89. London: Queen Anne Press. ISBN 978-0-35615-880-8. +Rollin, Glenda; Rollin, Jack, eds. (1996). Rothmans Football Yearbook 1996–97. London: Headline Publishing Group. ISBN 978-0747277811. +Triggs, Roger (2001). The Men Who Made Gillingham Football Club. Stroud: Tempus Publishing Ltd. ISBN 978-0-75242-243-5. +Bride of Frankenstein is a 1935 American horror film as well as the first sequel to Universal Pictures' 1931 film Frankenstein. As with the first film, Bride was directed by James Whale, starring Boris Karloff as the Monster and Colin Clive as Dr. Frankenstein. Additionally, it features Elsa Lanchester in the dual role of Mary Shelley and the bride, and Ernest Thesiger as Doctor Septimus Pretorius with Valerie Hobson replacing Mae Clarke as Elizabeth Frankenstein. +Taking place immediately after the events of the earlier film, it is rooted in a subplot of the original Mary Shelley novel, Frankenstein; or, The Modern Prometheus (1818). Bride follows a chastened Henry Frankenstein as he attempts to abandon his plans to create life, only to be tempted and finally blackmailed by his old mentor Dr. Pretorius, along with threats from the Monster, into constructing a bride for the Monster. +Preparation to film a sequel to Frankenstein began shortly after its premiere, but script problems delayed the project. Principal photography on Bride of Frankenstein finally commenced in January 1935, with creative personnel from the original returning both in front of and behind the camera. The film was released to critical and popular acclaim, although it encountered difficulties with some state and national censorship boards. +Since its release Bride of Frankenstein's reputation has grown, and it is now frequently considered one of the greatest sequels ever made. Many critics consider it to be an improvement on the original, and have hailed it as Whale's masterpiece. In 1998, it was selected by the Library of Congress for preservation in the National Film Registry, having been deemed "culturally, historically or aesthetically significant". +In a castle on a stormy night, Percy Bysshe Shelley and Lord Byron praise Mary Shelley for her story of Frankenstein and his Monster. She reminds them that her intention for writing the novel was to impart a moral lesson, the consequences of a mortal man who tries to play God, then says she has more of the story to tell. +Following the events of Frankenstein (1931), villagers gather around the burning windmill to cheer the apparent death of the Monster. Hans, the father of the girl the Monster drowned, wants to see the Monster's bones. He falls into a flooded pit underneath the mill, where the Monster—having survived the fire—strangles him. Hauling himself from the pit, the Monster also casts Hans' wife to her death. He next encounters Frankenstein's servant Minnie, who flees in terror. +The body of Henry Frankenstein, who is thought to have died at the windmill, is returned to his fiancée Elizabeth at his castle home. Minnie arrives to sound the alarm about the Monster, but her warning goes unheeded. Elizabeth, seeing Henry move, realizes he is still alive. Nursed back to health by Elizabeth, Henry renounces his creation, but still believes he may be destined to unlock the secret of life and immortality. A hysterical Elizabeth cries that she foresees death. +Henry visits the lab of his former mentor Doctor Septimus Pretorius, where Pretorius shows Henry several homunculi he has created. Pretorius wishes to work with Henry to create a mate for the Monster, with the proposed venture involving Pretorius growing an artificial brain while Henry gathers parts for the mate. +Meanwhile, the Monster saves a young shepherdess from drowning. Her screams upon seeing the Monster alert two hunters, who shoot and injure him. They raise a mob that sets out in pursuit. Captured and trussed to a pole, the Monster is hauled to a dungeon and chained. Left alone, he breaks his chains, overpowers the guards, and escapes into the woods. That night, the Monster encounters an old blind hermit who thanks God for sending him a friend. He teaches the Monster words like "friend" and "good" and shares a meal with him. Two lost hunters stumble upon the cottage and recognize the Monster. He attacks them and accidentally burns down the cottage as the hunters lead the hermit away. +Taking refuge from another angry mob in a crypt, the Monster spies Pretorius and his cronies Karl and Ludwig breaking open a grave. The henchmen depart as Pretorius has supper. The Monster reveals himself, eats some of the food, and learns that Pretorius plans to create a mate for him. +Henry and Elizabeth, now married, are visited by Pretorius. Henry expresses his refusal to assist Pretorius, who calls the Monster. The Monster demands Henry's help, to no avail. Pretorius orders the Monster out, secretly signaling him to kidnap Elizabeth. Pretorius guarantees her safe return upon Henry's participation. Henry returns to his tower laboratory and, despite himself, grows excited over his work. After being assured of Elizabeth's safety, Henry completes the Bride's body. +A storm rages as final preparations are made to bring the Bride to life. Her bandage-wrapped body is raised through the roof, where electricity is harnessed from lightning to animate her. Henry and Pretorius lower her and, after realizing their success in bringing her to life, remove her bandages and help her to stand. +The Monster comes down the steps after killing Karl on the rooftop and sees the Bride. The Monster reaches out to her and asks, "Friend?" The Bride, screaming, rejects him. He observes, "She hate me! Like others." As Elizabeth races to Henry's side, the Monster rampages through the lab. Before destroying everything, the Monster pauses and tells Henry and Elizabeth: "Go! You live! Go!" To Pretorius and the Bride, he says: "You stay. We belong dead." While Henry and Elizabeth flee, the Bride hisses at the Monster. Shedding a tear, the Monster pulls a lever to trigger the laboratory and tower's destruction. +Universal considered making a sequel to Frankenstein as early as its 1931 preview screenings, following which the film's original ending was changed to allow for Henry Frankenstein's survival. James Whale initially refused to direct Bride, believing he had already "squeezed the idea dry". Kurt Neumann was originally scheduled to replace Whale but decided to film The Black Cat instead. Following the success of Whale's The Invisible Man, producer Carl Laemmle, Jr. realized that Whale was the only possible director for Bride; Whale took advantage of the situation by persuading Universal to let him make One More River. Whale believed the sequel would not top the original, so he decided instead to make it a memorable "hoot". According to a studio publicist, Whale and Universal's studio psychiatrist decided "the Monster would have the mental age of a ten-year-old boy and the emotional age of a lad of fifteen". +Screenwriter Robert Florey wrote a treatment entitled The New Adventures of Frankenstein: The Monster Lives!, but it was rejected without comment early in 1932. Universal staff writer Tom Reed wrote a treatment as The Return of Frankenstein, a title retained until filming began. Following its acceptance in 1933, Reed wrote a full script that was submitted to the Hays office for review. The script passed its review, but Whale, who by then had been contracted to direct, complained that "it stinks to heaven". L. G. Blochman and Philip MacDonald were the next writers assigned, but Whale also found their work unsatisfactory. In 1934, Whale set John L. Balderston to work on yet another version, and it was he who returned to an incident from the novel in which the Monster demands a mate. Frankenstein subsequently creates a mate, but destroys it without bringing it to life. Balderston also created the Mary Shelley prologue. After several months Whale was still not satisfied with Balderston's work and handed the project to playwright William James Hurlbut and Edmund Pearson. The final script, combining elements of a number of these versions, was submitted for Hays office review in November 1934. Kim Newman reports that Whale planned to make Elizabeth the heart donor for the bride, but film historian Scott MacQueen states that Whale never had such an intention. +Sources report that Bela Lugosi and Claude Rains were considered, with varying degrees of seriousness, for the role of Frankenstein's mentor, Pretorius; others report that the role was created specifically for Ernest Thesiger. Because of Mae Clarke's ill health, Valerie Hobson replaced her as Henry Frankenstein's love interest, Elizabeth. Early in production, Whale decided that the same actress cast to play the Bride should also play Mary Shelley in the film's prologue, to represent how the story — and horror in general — springs from the dark side of the imagination. He considered Brigitte Helm and Phyllis Brooks before deciding on Elsa Lanchester. Lanchester, who had accompanied husband Charles Laughton to Hollywood, had met with only moderate success while Laughton had made a strong impact with several films, even winning an Academy Award for Best Actor. Lanchester had returned alone to London when Whale contacted her to offer her the dual role. Lanchester modeled the Bride's hissing on that of swans. She gave herself a sore throat while filming the hissing sequence, which Whale shot from multiple angles. +Colin Clive and Boris Karloff reprised their roles from Frankenstein as creator and creation, respectively. Hobson recalled Clive's alcoholism had worsened since filming the original, but Whale did not recast the role because his "hysterical quality" was necessary for the film. Karloff strongly objected to the decision to allow the Monster to speak: "Speech! Stupid! My argument was that if the monster had any impact or charm, it was because he was inarticulate – this great, lumbering, inarticulate creature. The moment he spoke you might as well ... play it straight". This decision also meant that Karloff could not remove his dental plate, so now his cheeks did not have the sunken look of the original film. Whale and the studio psychiatrist selected 44 simple words for the Monster's vocabulary by looking at test papers of ten-year-olds working at the studio. Dwight Frye returned to play the doctor's assistant, Karl, having played the hunchback Fritz in the original. Frye also filmed a scene as an unnamed villager and the role of "Nephew Glutz", a man who murdered his uncle and blamed the death on the Monster. Boris Karloff is credited simply as KARLOFF, which was Universal's custom during the height of his career. Elsa Lanchester is credited for Mary Wollstonecraft Shelley, but in a nod to the earlier film, the Monster's bride is credited only as "?" just as Boris Karloff had been in the opening credits of Frankenstein. +Universal makeup artist Jack Pierce paid special attention to the Monster's appearance in this film. He altered his 1931 design to display the after-effects of the mill fire, adding scars and shortening the Monster's hair. Over the course of filming, Pierce modified the Monster's makeup to indicate that the Monster's injuries were healing as the film progressed. Pierce co-created the Bride's makeup with strong input from Whale, especially regarding the Bride's iconic hair style, based on Nefertiti. Lanchester's hair was given a Marcel wave over a wire frame to achieve the style. Lanchester disliked working with Pierce, who she said "really did feel that he made these people, like he was a god ... in the morning he'd be dressed in white as if he were in hospital to perform an operation". To play Mary Shelley, Lanchester wore a white net dress embroidered with sequins of butterflies, stars, and moons, which the actress had heard required 17 women 12 weeks to make. Lanchester said of her bride costume: "I drank as little liquid as possible. It was too much of an ordeal to go to the bathroom--all those bandages--and having to be accompanied by my dresser". +Kenneth Strickfaden created and maintained the laboratory equipment. Strickfaden recycled a number of the fancifully named machines he had created for the original Frankenstein for use in Bride, including the "Cosmic Ray Diffuser", and the "Nebularium". A lightning bolt generated by Strickfaden's equipment has become a stock scene, appearing in any number of films and television shows. The man behind the film's special photographic effects was John P. Fulton, head of the special effects department at Universal Studios at the time. Fulton and David S. Horsley created the homunculi over the course of two days by shooting the actors in full-size jars against black velvet and aligning them with the perspective of the on-set jars. The foreground film plate was rotoscoped and matted onto the rear plate. Diminutive actor Billy Barty is briefly visible from the back in the finished film as a homunculus infant in a high chair, but Whale cut the infant's reveal before the film's release. +Whale met Franz Waxman at a party and asked him to score the picture. Whale told him: "Nothing will be resolved in this picture except the end destruction scene. Would you write an unresolved score for it?" Waxman created three distinctive themes: one for the Monster; one for the Bride; and one for Pretorius. The score closes, at Whale's suggestion, with a powerful dissonant chord, intended to convey the idea that the on-screen explosion was so powerful that the theater where the film was being screened was affected by it. Constantin Bakaleinikoff conducted 22 musicians to record the score in a single nine-hour session. +Shooting began on January 2, 1935, with a projected budget of US$293,750 ($6.9 million as of 2025) – almost exactly the budget of the original – and an estimated 36-day shooting schedule. On the first day, Karloff waded in the water below the destroyed windmill wearing a rubber suit under his costume. Air got into the suit and expanded it like an "obscene water lily". Later that day, Karloff broke his hip, necessitating a stunt double. Clive had also broken his leg. Shooting was completed on March 7. The film was ten days over schedule because Whale shut down the picture for ten days until Heggie became available to play the Hermit. With a final cost of $397,023 ($11.7 million as of 2025), Bride was more than $100,000 ($2.35 million as of 2025) over budget. As originally filmed, Henry died fleeing the exploding castle. Whale re-shot the ending to allow for their survival, although Clive is still visible on-screen in the collapsing laboratory. Whale completed his final cut, shortening the running time from about 90 to 75 minutes and re-shooting and re-editing the ending, only days before the film's scheduled premiere date. +Bride of Frankenstein was subjected to censorship, both during production by the Hays office and following its release by local and national censorship boards. Joseph Breen, lead censor for the Hays office, objected to lines of dialogue in the originally submitted script in which Henry Frankenstein and his work were compared to that of God. He continued to object to such dialogue in revised scripts, and to a planned shot of the Monster rushing through a graveyard to a figure of a crucified Jesus and attempting to rescue the figure from the cross. Breen also objected to the number of murders, both seen and implied by the script and strongly advised Whale to reduce the number. The censors' office, upon reviewing the film in March 1935, required a number of cuts. Whale agreed to delete a sequence in which Dwight Frye's "Nephew Glutz" kills his uncle and blames the Monster, and shots of Elsa Lanchester as Mary Shelley in which Breen felt too much of her breasts were visible. Despite his earlier objection, Breen offered no objection to the cruciform imagery throughout the film – including a scene with the Monster lashed Christ-like to a pole – nor to the presentation of Pretorius as a coded homosexual. Bride of Frankenstein was approved by the Production Code office on April 15. +Following its release with the Code seal of approval, the film was challenged by the censorship board in the state of Ohio. Censors in England and China objected to the scene in which the Monster gazes longingly upon the body intended for reanimation as the Bride, citing concerns that it looked like necrophilia. Universal voluntarily withdrew the film from Sweden because of the extensive cuts demanded, and Bride was rejected outright by Trinidad, Palestine, and Hungary. Additionally, Japanese censors objected to the scene in which Pretorius chases his miniature Henry VIII with tweezers, asserting that it constituted "making a fool out of a king". +Bride of Frankenstein premiered on April 19, 1935 at the Orpheum Theater in San Francisco, and went into general release the next day. The film was profitable for Universal, with a 1943 report showing that it had earned approximately $2 million, a profit of about $950,000. +The film was critically praised upon its release, although some reviewers did qualify their opinions based on the film's being in the horror genre. The New York World-Telegram called the film "good entertainment of its kind". The New York Post described it as "a grotesque, gruesome tale which, of its kind, is swell". The Hollywood Reporter similarly called the film "a joy for those who can appreciate it". +Variety did not so qualify its review: "[It is] one of those rare instances where none can review it, or talk about it, without mentioning the cameraman, art director, and score composer in the same breath as the actors and director". Variety also praised the cast, writing that "Karloff manages to invest the character with some subtleties of emotion that are surprisingly real and touching ... Thesiger as Dr. Pretorious [is] a diabolic characterization if ever there was one ... Lanchester handles two assignments, being first in a preamble as author Mary Shelley and then the created woman. In latter assignment she impresses quite highly". +In another unqualified review, Time wrote that the film had "a vitality that makes their efforts fully the equal of the original picture ... Screenwriters Hurlbut & Balderston and Director James Whale have given it the macabre intensity proper to all good horror pieces, but have substituted a queer kind of mechanistic pathos for the sheer evil that was Frankenstein". The Oakland Tribune concurred it was "a fantasy produced on a rather magnificent scale, with excellent stagecraft and fine photographic effects". While the Winnipeg Free Press thought that the electrical equipment might have been better suited to Buck Rogers, nonetheless the reviewer praised the film as "exciting and sometimes morbidly gruesome", declaring that "all who enjoyed Frankenstein will welcome his Bride as a worthy successor". The New York Times called Karloff "so splendid in the role that all one can say is 'he is the Monster'". The Times praised the entire principal cast and Whale's direction in concluding that Bride is "a first-rate horror film", and presciently suggested that "the Monster should become an institution, like Charlie Chan". +The film's reputation has persisted and grown in the decades since its release. In 1998, Bride of Frankenstein was added to the United States National Film Registry, having been deemed "culturally, historically or aesthetically significant". Frequently identified as James Whale's masterpiece, the film is lauded as "the finest of all gothic horror movies". Time rated Bride of Frankenstein in its "All-Time 100 Movies", in which critics Richard Corliss and Richard Schickel overruled the magazine's original review to declare the film "one of those rare sequels that is infinitely superior to its source". On the review aggregator website Rotten Tomatoes, 97% of 60 critics' reviews of the film are positive, with an average rating of 9.2/10 and the consensus: "An eccentric, campy, technically impressive, and frightening picture, James Whale's Bride of Frankenstein has aged remarkably well." Metacritic, which uses a weighted average, assigned the film a score of 95 out of 100, based on 16 critics, indicating "universal acclaim". Roger Ebert of The Chicago Sun-Times added Bride of Frankenstein to his list of The Great Movies in 1999. He described it as "the best of the Frankenstein movies--a sly, subversive work that smuggled shocking material past the censors by disguising it in the trappings of horror. Some movies age; others ripen." Ebert also added how Lanchester's character provided "one of the immortal images of the cinema with lightning-like streaks of silver in her weirdly towering hair". +In 2008, Bride was selected by Empire magazine as one of The 500 Greatest Movies of All Time. Also that year, the Boston Herald named it the second greatest horror film after Nosferatu. In 2016, James Charisma of Playboy ranked the film #7 on a list of 15 Sequels That Are Way Better Than The Originals. Entertainment Weekly considers the film superior to Frankenstein. +Bride was nominated for one Academy Award, for Sound Recording (Gilbert Kurland). +Christian imagery appears throughout the film. In addition to the scenes of the Monster trussed in a cruciform pose and the crucified figure of Jesus in the graveyard, the hermit has a crucifix on the wall of his hut – which, to Whale's consternation, editor Ted Kent made glow during a fade-out – and the Monster consumes the Christian sacraments of bread and wine at his "last supper" with the hermit. Horror scholar David J. Skal suggests that Whale's intention was to make a "direct comparison of Frankenstein's monster to Christ". Film scholar Scott MacQueen, noting Whale's lack of any religious convictions, disputes the notion that the Monster is a Christ-figure. Rather, the Monster is a "mockery of the divine" since, having been created by Man rather than God, it "lacks the divine spark". In crucifying the Monster, he says, Whale "pushes the audience's buttons" by inverting the central Christian belief of the death of Christ followed by the resurrection. The Monster is raised from the dead first, then crucified. +In the decades since its release, modern film scholars have noted the possible gay interpretations of the film. Director James Whale was openly gay, and some of the actors in the cast, including Ernest Thesiger and, according to rumor, Colin Clive, were respectively gay or bisexual. Although James Curtis, Whale's biographer, rejects the notion that Whale would have identified with the Monster from a homosexual perspective, scholars have perceived a gay subtext suffused through the film, especially a camp sensibility, particularly embodied in the character of Pretorius and his relationship with Henry. +Gay film historian Vito Russo, in considering Pretorius, stops short of identifying the character as gay, instead referring to him as "sissified", "sissy" itself being Hollywood code for "homosexual". Pretorius serves as a "gay Mephistopheles", a figure of seduction and temptation, going so far as to pull Frankenstein away from his bride on their wedding night to engage in the unnatural act of creating non-procreative life. A novelization of the film published in the United Kingdom made the implication clear, having Pretorius say to Frankenstein: "Be fruitful and multiply. Let us obey the Biblical injunction: you of course, have the choice of natural means; but as for me, I am afraid that there is no course open to me but the scientific way". +The Monster, whose affections for the male hermit and the female Bride he discusses with identical language ("friend") has been interpreted as sexually "unsettled" and bisexual. Gender studies author Elizabeth Young writes: "He has no innate understanding that the male-female bond he is to forge with the bride is assumed to be the primary one or that it carries a different sexual valence from his relationships with [Pretorius and the hermit]: all affective relationships are as easily 'friendships' as 'marriages'". Indeed, his relationship with the hermit has been interpreted as a same-sex marriage that heterosexual society will not tolerate: "No mistake – this is a marriage, and a viable one ... But Whale reminds us quickly that society does not approve. The monster – the outsider – is driven from his scene of domestic pleasure by two gun-toting rubes who happen upon this startling alliance and quickly, instinctively, proceed to destroy it", writes cultural critic Gary Morris for Bright Lights Film Journal. The creation of the Bride scene, Morris continues, is "Whale's reminder to the audience – his Hollywood bosses, peers, and everyone watching – of the majesty and power of the homosexual creator". +Filmmaker Curtis Harrington, a friend and confidant of Whale's, dismissed this as "a younger critic's evaluation. All artists do work that comes out of the unconscious mind and later on you can analyze it and say the symbolism may mean something, but artists don't think that way and I would bet my life that James Whale would never have had such concepts in mind". Specifically in response to the "majesty and power" reading, Harrington stated: "My opinion is that's just pure bullshit. That's a critical interpretation that has nothing to do with the original inspiration". He concludes: "I think the closest you can come to a homosexual metaphor in his films is to identify that certain sort of camp humor". Whale's companion David Lewis stated flatly that Whale's sexual orientation was "not germane" to his filmmaking: "Jimmy was first and foremost an artist, and his films represent the work of an artist – not a gay artist, but an artist". +In 1985, MCA Home Video released Bride of Frankenstein on LaserDisc. In the 1990s, MCA/Universal Home Video released the film on VHS as part of the "Universal Monsters Classic Collection", a series of releases of Universal Classic Monsters films. +In 1999, Universal released Bride of Frankenstein on VHS and DVD as part of the "Classic Monster Collection". In April 2004, Universal released Frankenstein: The Legacy Collection on DVD as part of the "Universal Legacy Collection". This two-disc release includes Bride of Frankenstein, as well as the original Frankenstein, Son of Frankenstein, The Ghost of Frankenstein, and House of Frankenstein. +In 2012, Bride of Frankenstein was released on Blu-ray as part of the Universal Classic Monsters: The Essential Collection box set, which also includes a total of nine films from the Universal Classic Monsters series. In 2014, Universal released Frankenstein: Complete Legacy Collection on DVD. This set contains eight films: Frankenstein, Bride of Frankenstein, Son of Frankenstein, The Ghost of Frankenstein, Frankenstein Meets the Wolf Man, House of Frankenstein, House of Dracula, and Abbott and Costello Meet Frankenstein. In 2015, the six-film Universal Classic Monsters Collection was released on DVD. In 2016, Bride of Frankenstein received a Walmart-exclusive Blu-ray release featuring a glow-in-the-dark cover. That same year, the Complete Legacy Collection was released on Blu-ray. In September 2017, Bride of Frankenstein received a Best Buy-exclusive SteelBook Blu-ray release with cover artwork by Alex Ross. +Bride of Frankenstein was included in the Universal Classic Monsters: Complete 30-Film Collection Blu-ray box set in August 2018. This box set also received a DVD release. +Boris Karloff filmography +List of films featuring Frankenstein's monster +Frankenstein in popular culture +Gods and Monsters, a 1998 James Whale biopic that draws its title from a quote from Bride of Frankenstein +List of horror films of the 1930s +List of films featuring miniature people +List of cult films +The Bride! +Brunas, Michael, John Brunas & Tom Weaver (1990). Universal Horrors: The Studios Classic Films, 1931–46. Qefferson, NC, McFarland & Co. +Curtis, James (1998). James Whale: A New World of Gods and Monsters. Boston, Faber and Faber. ISBN 0-571-19285-8. +Gelder, Ken (2000). The Horror Reader. New York, Routledge. ISBN 0-415-21355-X. +Gifford, Denis (1973) Karloff: The Man, The Monster, The Movies. Film Fan Monthly. +Goldman, Harry (2005). Kenneth Strickfaden, Dr. Frankenstein's Electrician. McFarland. ISBN 0-7864-2064-2. +Johnson, Tom (1997). Censored Screams: The British Ban on Hollywood Horror in the Thirties. McFarland. ISBN 0-7864-0394-2. +Lennig, Arthur (1993). The Immortal Count: The Life and Films of Bela Lugosi. University Press of Kentucky. ISBN 0-8131-2273-2. +Mallory, Michael (2009) Universal Studios Monsters: A Legacy of Horror. Universe. ISBN 0-7893-1896-2. +Mank, Gregory W. (1994). Hollywood Cauldron: Thirteen Films from the Genre's Golden Age. McFarland. ISBN 0-7864-1112-0. +Picart, Carolyn Joan, Frank Smoot and Jayne Blodgett (2001). The Frankenstein Film Sourcebook. Greenwood Press. ISBN 0-313-31350-4. +Russo, Vito (1987). The Celluloid Closet: Homosexuality in the Movies (revised edition). New York, HarperCollins. ISBN 0-06-096132-5. +Skal, David J. (1993). The Monster Show: A Cultural History of Horror. Penguin Books. ISBN 0-14-024002-0. +Vieira, Mark A. (2003). Hollywood Horror: From Gothic to Cosmic. New York, Harry N. Abrams. ISBN 0-8109-4535-5. +Young, Elizabeth. "Here Comes The Bride". Collected in Gelder, Ken (ed.) (2000). The Horror Reader. Routledge. ISBN 0-415-21356-8. +Official website +Frankenstein and The Bride of Frankenstein essay by Richard T. Jameson on the National Film Registry site +"The Bride of Frankenstein" essay by Daniel Eagan in America's Film Legacy: The Authoritative Guide to the Landmark Movies in the National Film Registry, A&C Black, 2010 ISBN 0826429777, pages 234-235 +Bride of Frankenstein at IMDb +Bride of Frankenstein at the TCM Movie Database (archived) +Bride of Frankenstein at the AFI Catalog of Feature Films +The Derfflinger class was a class of three battlecruisers of the German Kaiserliche Marine (Imperial Navy). The ships were ordered for the 1912–1913 Naval Building Program of the German Imperial Navy as a reply to the Royal Navy's two new Lion-class battlecruisers that had been launched a few years earlier. The preceding Moltke class and the incrementally improved Seydlitz represented the end of the evolution of Germany's first generation of battlecruisers. The Derfflinger class had considerable improvements, including a larger primary armament, all of which was mounted on the centerline. The ships were also larger than the preceding classes. The Derfflinger class used a similar propulsion system, and as a result of the increased displacement were slightly slower. +The class comprised three ships: Derfflinger, Lützow, and Hindenburg. All three of the ships saw active service with the High Seas Fleet during World War I. Derfflinger was commissioned shortly after the outbreak of war, and was present at most of the naval actions in the North Sea, including the battles of Dogger Bank and Jutland. Lützow was commissioned in August 1915, and participated only in the raid on Yarmouth before being sunk at Jutland. Hindenburg was commissioned into the fleet in May 1917, and saw no major action. Derfflinger and Hindenburg were interned at Scapa Flow following the armistice in November 1918. Rear Admiral Ludwig von Reuter, who was in command of the interned High Seas Fleet, ordered the ships to be scuttled in an attempt to prevent their possible seizure by the Royal Navy. +Discussions began in April 1910 over the basic design requirements for the next German battlecruiser, which was to follow SMS Seydlitz. The Construction Department's design section requested general specifications in mid-April, but the General Department replied that it was too early. They cited the fact that the Nassau-class battleships had not yet completed firing trials and other design projects—such as a triple gun turret and a large diesel engine—were still in very early stages of development. At the same time, the General Department's Vizeadmiral (VAdm—Vice Admiral) Adolf Paschen noted that the British Royal Navy had increased the caliber of its battlecruisers' main battery from 30.5 cm (12 in) to 34 cm (13.5 in), and so the Germans would need to increase their own from 28 cm (11 in) to 30.5 cm. He also argued that the propulsion system should be changed from four to three screw propellers, with the center screw powered by a diesel engine for improved fuel economy. +Admiral Alfred von Tirpitz, the State Secretary of the Reichsmarineamt (Imperial Naval Office), convened the first of a series of meetings to discuss the next battlecruiser design at around the same time as the discussions between the Construction and General departments. The first meeting, held on 11 May, examined the results of recent firing tests of the 28 and 30.5 cm guns, along with that of an experimental 32 cm (13 in) gun. Konteradmiral (Rear Admiral) Gerhard Gerdes, a member of the Weapons Department, stated that expected battle ranges of 8,000 to 10,000 meters (8,750 to 10,940 yd), the 28 cm gun would not be able to penetrate the armor of the latest British battlecruisers, which was believed to be 250 mm (9.8 in) thick. Tirpitz argued that fighting would take place at much closer distances, ranges at which the 28 cm gun could still penetrate armor of that thickness. Paschen and VAdm Max Rollmann, the latter from the Construction Department, also voted for the 28 cm gun, which Tirpitz announced was at least temporarily the selection. +Tirpitz next pressed the question of the ship's propulsion system; he strongly favored the adoption of diesel engines, which he saw as a way to take a significant qualitative step ahead of foreign navies that still relied entirely on steam turbines. The naval engineer Hans Bürkner opposed the measure, given the untested nature of the new engine (and associated risks should the project fail), along with the excessive workload that would be created for the design staff, since they would need to redesign the underwater protection scheme and other aspects of the hull. The designers suggested that simply ordering the next cruiser, provisionally titled "K", as a sister ship to Seydlitz. The final decision on diesel engines, either as the sole engine type or as part of a mixed installation with turbines, was forced after the head of the Construction Department's propulsion section informed the naval leadership on 1 September that the diesel engine being developed by MAN AG would not be ready for "K". +Detailed design work began soon after the initial discussions in early 1910. The design staff started with abandoned proposals for Seydlitz, projects "IIIc" and "IVe"; the former was a four-turret-armed ship equipped with 30.5 cm guns and the latter carried ten 28 cm guns in five turrets, all of which were mounted on the centerline. In June 1910, a third alternative that combined the two variants—four 30.5 turrets on the centerline—was produced. By that time, Tirpitz was convinced that the larger gun was preferable. The decision was formally adopted in a meeting on 1 September, which was memorialized by Tirpitz the following day. In the memo, he expressed a preference for two superfiring pairs, one forward and one aft, but conceded that if this were impossible, an offset wing turret arrangement as used in the battlecruiser SMS Von der Tann would be acceptable. He also indicated that a fifth turret should be incorporated if weight and cost limits could be met, though such an increase proved to be prohibitive by both metrics. +On 22 September, Bürkner submitted his evaluation of three different proposals, designated 1, 2, and 3. The first used the Von der Tann arrangement for the main battery, while the latter two used the centerline layout. The primary differences between 2 and 3 were the spacing of the rear pair of turrets (2 being placed close together, while 3 increased the distance) and the placement of the secondary armament; 2 raised the secondary guns to the upper deck. All three proposals maintained the same level of armor protection as in Seydlitz, along with speed and coal provisioning. Bürkner recommended design 3, but with the raised secondary battery as in 2; this refined design was then designated 4, which Kaiser Wilhelm II approved on 26 September. Further detailed improvements produced design 4b. +On 18 March, the design staff presented design 5, which included a number of internal improvements over 4b. One of the internal decks was removed from the central portion of the ship, which allowed greater space for coal storage. The designers also adopted a new construction technique to save weight. Previous battlecruisers were built with a combination of transverse and longitudinal steel frames; the Derfflinger-class ships dispensed with the transverse frames and used only longitudinal ones. This enabled the ship to retain structural strength and a lower weight. Lastly, because of the internal reorganization, the boiler rooms could be divided by an internal bulkhead that reduced the risk of flooding in the event of underwater damage. Large capacity double-ended boilers that had been tested aboard Von der Tann would also be used, which should provide the ship with the same speed as Seydlitz or better. +Design 5 had one significant drawback; the ship's metacentric height would have been around 2.7 to 2.9 m (8 ft 10 in to 9 ft 6 in), which was too high, as it would have caused the ship to roll too quickly. The fast roll period would have degraded the gunners' ability to aim their weapons. The beam could be reduced by about 2 m (6 ft 7 in), but this would corresponding reduce the space between the outer hull and the torpedo bulkhead, which would have worsened the ship's resistance to underwater damage. Frahm anti-roll tanks were also considered, but these would have caused an increase in displacement and cost. Another conference was held on 30 March 1911 to evaluate the proposals, along with ideas to save weight to accommodate the anti-roll tanks. Among the options considered were a reduction in the thickness of the torpedo bulkhead or the bow armor, and the substitution of some oil-fired boilers for coal-burning boilers. +A finalized version, 5b, was completed on 15 June. The changes from design 5 included the reduction of the torpedo bulkhead and lowering the height of the bow armor, but not the adoption of partial oil firing for the boilers. In addition, the secondary battery was lowered by a level and brought to the edges of the hull, which had the effect of improving the ship's seakeeping. Displacement had increased from about 25,900 to 26,600 t (25,500 to 26,200 long tons; 28,500 to 29,300 short tons), and the hull had been slightly lengthened and widened to accommodate the additional weight. Wilhelm II approved the design soon thereafter. On 30 June, the naval leadership met again to discuss oil-burning boilers; those in favor argued that they were lighter and easier to operate, which would afford a significant reduction in crews. Those opposed cited the difficulty in obtaining fuel oil, especially in a crisis, and that coal storage bunkers provided protection that oil tanks could not. The conference participants concluded that eight of the twenty-two boilers should be oil fired. They also decided to install the Frahm anti-roll tanks, despite the fact that tests aboard Von der Tann revealed they only reduced rolling by about 33%. +The Derfflinger-class battlecruisers were ordered under the fourth and final Naval Law, which was passed in 1912. Tirpitz used public outcry over the British involvement in the Agadir Crisis of 1911 to pressure the Reichstag into appropriating additional funds to the Navy. The Fourth Naval Law secured funding for three new dreadnoughts, two light cruisers, and an extra 15,000 officers and men in the Navy for 1912. The three dreadnoughts secured in the bill became Derfflinger, Lützow, and Hindenburg. In the meantime, pressure from the British public and media had forced the British Parliament to step up ship building. Kaiser Wilhelm II requested that the build time for the new battlecruisers be reduced to two years each, as opposed to three years. This proved unfeasible, because neither the armor or armament firms could supply the necessary materials according to an expedited schedule. +Derfflinger and Lützow were 210 m (690 ft) long at the waterline and 210.4 m (690 ft 3 in) long overall. Hindenburg was slightly longer, at 212.5 m (697 ft 2 in) at the waterline and 212.8 m (698 ft 2 in) overall. All three ships had a beam of 29 m (95 ft 2 in), and a draft of between 9.2 m (30 ft 2 in) forward and 9.57 m (31 ft 5 in) aft. The first two ships were designed to displace 26,600 t (26,200 long tons; 29,300 short tons) with a standard load, and up to 31,200 t (30,700 long tons; 34,400 short tons) at combat weight. Hindenburg displaced slightly more, at 26,947 t (26,521 long tons; 29,704 short tons) standard and 31,500 t (31,000 long tons; 34,700 short tons) fully laden. The ships' hulls were constructed from longitudinal steel frames, over which the outer hull plates were riveted. Derfflinger's hull contained sixteen watertight compartments, though Lützow and Hindenburg had an additional seventeenth compartment. All three ships had a double bottom that ran for 65% of the length of the hull. This was a decrease from preceding German battlecruisers, which had a double bottom for at least 75% of the hull. +The ships were regarded as excellent sea boats by the German navy. The Derfflinger-class ships were described as having had gentle motion, though they were "wet" at the casemate deck. Steering was controlled by a pair of rudders placed on the centerline. The ships lost up to 65% speed with the twin rudders hard over, and heeled up to 11 degrees. This was greater than any of the preceding battlecruiser designs, and as a result, anti-roll tanks were fitted to Derfflinger. the tanks were tested at sea in December 1914 with inconclusive results. The three ships had a metacentric height of 2.6 m (8 ft 6 in). The standard crew for one of the vessels was 44 officers and 1,068 men; when serving as the flagship for the I Scouting Group, the ships carried an additional 14 officers and 62 men. The Derfflingers carried smaller craft, including one picket boat, three barges, two launches, two yawls, and two dinghies. +By the time construction work on Derfflinger began, it was determined that the diesel engine was not ready for use. Instead, the plan to use a three-shaft system was abandoned and the ships reverted to the standard four-shaft arrangement. Each of the three ships was equipped with two sets of marine-type turbines; each set drove a pair of 3-bladed screws that were 3.9 m (12 ft 10 in) in diameter on Derfflinger and Lützow and 4 m (13 ft 1 in) in diameter on Hindenburg. The starboard side screws turned to the right and the port side turned to the left to counteract any effect on the ships' steering. Each set consisted of a high- and low-pressure turbine—the high-pressure machines drove the outer shafts while the low-pressure turbines turned the inner pair. Steam was supplied to the turbines from 14 coal-fired marine-type double boilers and eight oil-fired marine-type double-ended boilers. Each ship was equipped with a pair of turbo-electric generators and a pair of diesel-electric generators that provided a total of 1,660 kilowatts at 220 volts. Each ship was equipped with two rudders. +The engines for first two ships were designed to provide 63,000 metric horsepower (62,000 shp), at 280 revolutions per minute. This would have given the two ships a top speed of 26.5 knots (49.1 km/h; 30.5 mph). During trials, Derfflinger's engines achieved 76,634 PS (75,586 shp), but a top speed of 25.5 knots (47.2 km/h; 29.3 mph). Due to the start of World War I, the ship's trials were conducted in shallower water, which reduced speed; it was expected that in deep water, her top speed would have been about 2 knots higher. Lützow's engines reached 80,988 PS (79,880 shp) and a top speed of 26.4 knots (48.9 km/h; 30.4 mph). Hindenburg's power plant was rated at 72,000 PS (71,000 shp) at 290 rpm, for a top speed of 27 knots (50 km/h; 31 mph). On trials she reached 95,777 PS (94,467 shp) and 26.6 knots (49.3 km/h; 30.6 mph). Derfflinger could carry 3,500 t (3,400 long tons) of coal and 1,000 t (980 long tons) of oil; at a cruising speed of 14 knots (26 km/h; 16 mph), she had a range of 5,600 nautical miles (10,400 km; 6,400 mi). Lützow carried 3,700 t (3,600 long tons) of coal and 1,000 tons of oil, though she had no advantage in range over her sister Derfflinger. Hindenburg also stored 3,700 tons of coal, as well as 1,200 t (1,200 long tons) of oil; her range at 14 knots was rated at 6,100 nautical miles (11,300 km; 7,000 mi). +The Derfflinger-class ships were armed with eight 30.5 cm (12 in) SK L/50 guns in four twin gun turrets, two forward of the main superstructure in a superfiring pair and two to the rear of the ship, in a similar arrangement. The guns were housed in C/1912 mounts on the first two ships, and in C/1913 mounts on Hindenburg. The turrets were trained with electric motors, while the guns were elevated hydraulically, up to 13.5 degrees. The guns fired 405.5-kilogram (894 lb) armor-piercing shells at a muzzle velocity of 855 meters per second (2,805 ft/s). At 13.5 degrees, the shells could hit targets out to 18,700 m (20,500 yd). After the Battle of Dogger Bank in January 1915, where British ships had been able to open fire first, outside the range of the German guns, the turrets were modified in 1916 to increase the elevation maximum to 16 degrees. This correspondingly increased the range to 20,500 m (22,400 yd). The ships carried 720 shells, or 90 per gun; each gun was supplied with 65 armor-piercing (AP) shells and 25 semi-AP shells for use against targets with less armor protection. The 30.5 cm gun had a rate of fire of between 2–3 shells per minute, and was expected to fire 200 shells before replacement was necessary. The guns were also capable of firing 405.9 kg (894.8 lb) high explosive shells. The shells were loaded with two RP C/12 propellant charges: a main charge in a brass cartridge that weighed 91 kg (201 lb) and a fore charge in a silk bag that weighed 34.5 kg (76 lb). The propellant magazines were located underneath the shell rooms for the two forward turrets as well as the rear superfiring turret; the arrangement was reversed for the rearmost turret. +The ships were designed to carry a secondary armament of fourteen 15 cm (5.9 in) SK L/45 guns, mounted in casemates along the superstructure. Because Derfflinger had to be fitted with anti-roll tanks, two of the casemated guns had to be removed, to allow enough room in the hull. Lützow and Hindenburg were equipped with the designed number of guns. Each gun was supplied with 160 rounds, and had a maximum range of 13,500 m (14,800 yd), though this was later extended to 16,800 m (18,400 yd). The guns had a sustained rate of fire of six to eight rounds per minute. The shells were 45.3 kg (99.8 lb). The guns fired at a muzzle velocity of 835 meters per second (2,740 ft/s). +The three ships carried a variety of 8.8 cm (3.5 in) SK L/45 guns in several configurations. The Derfflinger-class ships were initially equipped with eight of these weapons, all in single mounts; four were placed in the forward superstructure and four in the aft superstructure. The ships also carried four 8.8 cm Flak L/45 anti-aircraft guns, which were emplaced around the forward funnel, with the exception of Lützow, which carried the Flak guns around the rear funnel. After 1916, the four 8.8 cm guns in the forward superstructure were removed. The Flak guns were emplaced in C/13 pivot mountings, which allowed depression to −10 degrees and elevation to 70 degrees. These guns fired 10 kg (22 lb) shells. +The ships were also armed with submerged torpedo tubes in their hulls. Derfflinger was equipped with four 50 cm tubes; the later ships were armed with more powerful 60 cm weapons. The tubes were arranged with one in the bow, one in the stern, and two on the broadside. Derfflinger's 50 cm torpedoes were the G7 type, 7.02 m (276 in) long and armed with a 195 kg (430 lb) Hexanite warhead. The torpedo had a range of 4,000 m (4,370 yd) when set at a speed of 37 knots, and up to 9,300 m (10,170 yd) at 27 knots. The 60 cm torpedoes were the H8 type, which were 8 m long and carried a 210 kg (463 lb) Hexanite warhead. The torpedoes had a range of 6,000 m (6,550 yd) when set at a speed of 36 knots; at a reduced speed of 30 knots, the range increased significantly to 14,000 m (15,310 yd). +The Derfflinger-class ships were protected with Krupp cemented steel armor, as was the standard for German warships of the period. They had an armor belt that was 300 mm (11.8 in) thick in the central citadel of the ship, where the most important parts of the ship were. This included the ammunition magazines and the machinery spaces. The belt was reduced in less critical areas, to 120 mm (4.7 in) forward and 100 mm (3.9 in) aft. The belt tapered down to 30 mm (1.2 in) at the bow, though the stern was not protected by armor at all. A 45 mm (1.8 in) thick torpedo bulkhead ran the length of the hull, several meters behind the main belt. The main armored deck ranged in thickness from 30 mm thick in less important areas, to 80 mm (3.1 in) in the sections that covered the more critical areas of the ship. +The forward conning tower was protected with heavy armor: the sides were 300 mm thick and the roof was 130 mm (5.1 in) thick. The rear conning tower was less well armored; its sides were only 200 mm (7.9 in) thick and the roof was covered with 50 mm (2 in) of armor plate. The main battery gun turrets were also heavily armored: the turret sides were 270 mm (11 in) thick and the roofs were 110 mm (4.3 in) thick. On Hindenburg, the thickness of the turret roofs was increased to 150 mm (5.9 in). The 15 cm guns had 150 mm-worth of armor plating in the casemates; the guns themselves had 70 mm (2.8 in) thick shields to protect their crews from shell splinters. +Of the three ships in its class, only Derfflinger was ordered as an addition to the fleet, under the provisional name "K". The other two ships were to intended to replace obsolete vessels; Lützow was ordered as Ersatz Kaiserin Augusta for the elderly protected cruiser Kaiserin Augusta and the contract for Hindenburg was issued under the provisional name Ersatz Hertha, to replace the protected cruiser Hertha. +Derfflinger was constructed at Blohm & Voss in Hamburg under construction number 213. She was the least expensive of the three ships, at a cost of 56 million gold marks. The ship was ready to be launched on 14 June 1913, but during the ceremony, one of the wooden sledges upon which the hull rested became jammed. It took until 12 July for her to enter the water. She was commissioned into the High Seas Fleet on 1 September 1914, shortly after the outbreak of World War I. Lützow was built at the Schichau dockyard in Danzig under construction number 885, at the cost of 58 million gold marks. The ship was launched on 29 November 1913, and after lengthy trials, commissioned on 8 August 1915. Hindenburg, the final member of the class, was built at the Imperial Dockyard in Wilhelmshaven, under construction number 34. The ship was built at a cost of 59 million gold marks, the most expensive of the three vessels. She was launched on 1 August 1915 and commissioned on 10 May 1917. +Named after Georg von Derfflinger, a German field marshal during the Thirty Years' War, Derfflinger was commissioned on 1 September 1914. A dockyard crew transferred the ship from Hamburg to Kiel, via the Skagen. The ship was assigned to the I Scouting Group at the end of October. Damage to the ship's turbines sustained during trials prevented the ship from seeing active service until 16 November. On 15 December, the ship took part in the raid on Scarborough, Hartlepool and Whitby. She was also present during the battle of Dogger Bank on 24 January 1915. The ship was hit once by a 13.5-inch shell from one of the British battlecruisers; in response, she heavily damaged HMS Lion. Repair work was completed by 16 February, but Derfflinger's starboard turbine was accidentally damaged on 28 June, and the ship was again in the dockyard until August. On 24 April 1916, Derfflinger took part in the bombardment of Yarmouth. +On 31 May, Derfflinger was heavily engaged during the Battle of Jutland, as the second ship in the German battlecruiser line. She sustained 21 major hits during the battle, but dealt considerable damage to the British battlecruiser force as well. At 16:26, HMS Queen Mary sank after a magazine explosion that tore the ship apart; she had been targeted with a hail of heavy-caliber gunfire from Derfflinger and Seydlitz. Two hours later, at 18:30, HMS Invincible suffered a similar fate, though Derfflinger was assisted by her sister Lützow. During the engagement, Derfflinger had both of her rear turrets knocked out by British gunfire. Her crew suffered 157 men killed and 26 wounded, which was the highest casualty figure for any German ship not sunk. The resilience of the vessel earned her the nickname "Iron Dog" from her British adversaries. Repair work lasted until 15 October, during which the ship had her pole mast removed and replaced with a tripod mast. The ship conducted training operations until November, at which point she returned to active duty with the fleet. +Following the German capitulation in November 1918, Derfflinger was interned with a significant portion of the High Seas Fleet in Scapa Flow. On 21 June 1919, with the guard ships of the Royal Navy out on maneuvers, Rear Admiral Ludwig von Reuter ordered that the fleet be scuttled. The resulting scuttling of the German fleet saw some 66 vessels of various types sunk. Among those was Derfflinger, which sank at 14:45. The ship was raised in 1939 to be broken up for scrap metal, but the outbreak of World War II intervened. The ship, which remained capsized, was anchored off the island of Risa until 1946, at which point she was sent to Faslane Port, where she was broken up. The ship's bell was delivered to the German Federal Navy on 30 August 1965. +Lützow was named after Ludwig Adolf Wilhelm von Lützow, a Prussian lieutenant-general who fought during the Napoleonic Wars. The ship was commissioned on 8 August 1915, and then underwent trials. On 25 October, while still running sea trials, Lützow's port low pressure turbine was severely damaged. She was sent to Kiel for repairs, which lasted until late January 1916. The ship went on additional trials that lasted until 19 February. Lützow was by then fully operational, and assigned to I Scouting Group on 20 March 1916. She took part in two fleet advances, on 25 March and 21–22 April, without any major incidents. The following day, on 23 April, Lützow, along with her sister Derfflinger and the battlecruisers Seydlitz, Moltke, and Von der Tann, bombarded Yarmouth. While en route to the target, Vice Admiral Franz von Hipper's flagship Seydlitz was heavily damaged by mines. As a result, Lützow was transferred to the role of squadron flagship. During the operation, the German battlecruisers encountered British light forces, and a running battle ensued. Lützow engaged the light cruiser HMS Conquest and hit her several times. +At the Battle of Jutland, she was the first ship in the German line, and Hipper's flagship, and drew fire from the British battlecruisers which included hits below her waterline. Shortly after the start of the battlecruiser action, Lützow hit her opponent Lion several times; one hit knocked out Lion's "Q" turret, and the resulting magazine fire nearly destroyed the ship. Shortly after 19:00, the armored cruisers Defence and Warrior inadvertently ran into the German line; Lützow opened fire immediately, followed by several German dreadnoughts. In a hail of shells, Defence's ammunition magazines detonated and the ship was sunk. At around the same time, the fresh battlecruisers of the 3rd Battlecruiser Squadron engaged their German opposites. Between 19:26 and 19:34, Lützow sustained four 12-inch shell hits in her bow from the British battlecruisers; these eventually proved to be fatal. Despite this, at 19:30, the combined fire of Lützow and her sister Derfflinger destroyed the battlecruiser Invincible. By 20:15, Lützow had been hit five more times, including hits on her two forward turrets. +By 22:15, Lützow had shipped nearly 2,400 tons of water, and the ship was dangerously down by the bows. After midnight, attempts were made to steer the ship in reverse. This failed when the bow became submerged enough to bring the stern out of the water; by 02:20, the screws and both rudders were coming out of the water and the ship was no longer able to steer. The order to abandon ship was given, and at 02:47, Lützow was sunk by the torpedo boat G38. The ship was lost because the flooding in the bow could not be controlled; the forward pump system failed and the central system could not keep up with the rising water. The crew was picked up by four torpedo boats that had been escorting the crippled battlecruiser; during the battle the ship suffered 116 men killed. +Hindenburg was the last battlecruiser completed for the Imperial German Navy, and as such had a very short career. She was commissioned 10 May 1917, and was fully operational by 20 October 1917, too late to see any major action in World War I. On 17 November Hindenburg and Moltke, along with the light cruisers of II Scouting Group, were acting as distant support for German minesweepers off the German coast when they were attacked by British battlecruisers. The raid was brief; by the time Hindenburg and Moltke arrived on the scene, the British ships had broken off the attack and withdrawn. Six days later, Hindenburg replaced Seydlitz as flagship of I Scouting Group. On 23 April 1918, the ship took part in an abortive fleet advance into the North Sea that attempted to intercept an Allied convoy. Moltke sustained mechanical damage while en route, and as a result, Vice Admiral Hipper decided to cancel the operation. On 11 August, Hipper was promoted to Admiral and given command of the entire High Seas Fleet. Rear Admiral Ludwig von Reuter replaced Hipper as the commander of I Scouting Group; he raised his flag on Hindenburg the following day. +Hindenburg was interned at Scapa Flow, along with her sister Derfflinger and the rest of the German battlecruisers. She was scuttled on 21 June 1919, and sank at 17:00. Several unsuccessful attempts to raise her were made; on 23 July 1930 the ship was finally raised. From 1930 to 1932 she was scrapped at Rosyth. Her bell was presented to the German Federal Navy on 28 May 1959. +Bennett, Geoffrey (2005). Naval Battles of the First World War. Barnsley: Pen & Sword Military Classics. ISBN 978-1-84415-300-8. +Dodson, Aidan (2016). The Kaiser's Battlefleet: German Capital Ships 1871–1918. Barnsley: Seaforth Publishing. ISBN 978-1-84832-229-5. +Grießmer, Axel (1999). Die Linienschiffe der Kaiserlichen Marine: 1906–1918; Konstruktionen zwischen Rüstungskonkurrenz und Flottengesetz [The Battleships of the Imperial Navy: 1906–1918; Constructions between Arms Competition and Fleet Laws] (in German). Bonn: Bernard & Graefe Verlag. ISBN 978-3-7637-5985-9. +Gröner, Erich (1990). German Warships: 1815–1945. Vol. I: Major Surface Vessels. Annapolis: Naval Institute Press. ISBN 978-0-87021-790-6. +Herwig, Holger (1998) [1980]. "Luxury" Fleet: The Imperial German Navy 1888–1918. Amherst: Humanity Books. ISBN 978-1-57392-286-9. +Schmalenbach, Paul (1993). Die Geschichte der deutschen Schiffsartillerie [The History of German Naval Artillery] (in German) (3rd ed.). Herford: Koehler. ISBN 978-3-7822-0577-1. +Staff, Gary (2006). German Battlecruisers: 1914–1918. Oxford: Osprey Books. ISBN 978-1-84603-009-3. +Staff, Gary (2014). German Battlecruisers of World War One: Their Design, Construction and Operations. Barnsley: Seaforth Publishing. ISBN 978-1-84832-213-4. +Tarrant, V. E. (2001) [1995]. Jutland: The German Perspective. London: Cassell Military Paperbacks. ISBN 978-0-304-35848-9. +DiGiulian, Tony (21 April 2007). "German Torpedoes Pre-World War II". NavWeaps.com. Retrieved 24 July 2009. +Breyer, Siegfried (1997). Die Kaiserliche Marine und ihre Großen Kreuzer [The Imperial Navy and its Large Cruisers] (in German). Wölfersheim: Podzun-Pallas Verlag. ISBN 978-3-7909-0603-5. +Campbell, N. J. M. (1978). Battle Cruisers. Warship Special. Vol. 1. Greenwich: Conway Maritime Press. ISBN 978-0-85177-130-4. +Dodson, Aidan; Cant, Serena (2020). Spoils of War: The Fate of Enemy Fleets after the Two World Wars. Barnsley: Seaforth Publishing. ISBN 978-1-5267-4198-1. +Koop, Gerhard; Schmolke, Klaus-Peter (1998). Die Grossen Kreuzer Von der Tann bis Hindenburg [The Battlecruisers Von der Tann to Hindenburg] (in German). Bonn: Bernard & Graefe. ISBN 978-3-7637-5972-9. +Maya Angelou ( AN-jə-loh; born Marguerite Annie Johnson; April 4, 1928 – May 28, 2014) was an American memoirist, essayist, poet, and civil rights activist. She published seven autobiographies, three books of essays, several books of poetry, and is credited with a list of plays, movies, and television shows spanning more than 50 years. She received dozens of awards and more than 50 honorary degrees. Angelou's series of seven autobiographies focus on her childhood and early adult experiences. The first, I Know Why the Caged Bird Sings (1969), tells of her life up to the age of 17 and brought her international recognition and acclaim. +She became a poet and writer after a string of odd jobs during her young adulthood. In 1982, Angelou was named the first Reynolds Professor of American Studies at Wake Forest University in Winston-Salem, North Carolina. Angelou was active in the Civil Rights Movement and worked with Martin Luther King Jr. and Malcolm X. Beginning in the 1990s, she made approximately 80 appearances a year on the lecture circuit, something she continued into her eighties. In 1993, Angelou recited her poem "On the Pulse of Morning" (1993) at the first inauguration of Bill Clinton, making her the first poet to make an inaugural recitation since Robert Frost at the inauguration of John F. Kennedy in 1961. +With the publication of I Know Why the Caged Bird Sings, Angelou publicly discussed aspects of her personal life. She was respected as a spokesperson for African Americans and women, and her works have been considered a defense of African-American culture. Her works are widely used in schools and universities worldwide, although attempts have been made to ban her books from some U.S. libraries. Angelou's most celebrated works have been labeled as autobiographical fiction, but many critics consider them to be autobiographies. She made a deliberate attempt to challenge the common structure of the autobiography by critiquing, changing, and expanding the genre. Her books center on themes that include racism, identity, family, and travel. +Maya Angelou was born Marguerite Annie Johnson on April 4, 1928, in St. Louis, Missouri, the younger of two children of Bailey Johnson, a doorman and navy dietitian, and Vivian (née Baxter) Johnson, a nurse and card dealer. For the first three years of her life, her family lived in her maternal grandparents' home. Angelou's older brother, Bailey Jr., nicknamed Marguerite "Maya", derived from "My" or "Mya Sister". When Angelou was three and her brother four, their parents' "calamitous marriage" ended, and their father sent them to Stamps, Arkansas, alone by train, to live with their paternal grandmother, Annie Henderson. In "an astonishing exception" to the harsh economics of African Americans of the time, Angelou's grandmother prospered financially during the Great Depression and World War II, because the general store she owned sold basic and needed commodities and because "she made wise and honest investments". +Four years later, when Angelou was seven and her brother eight, the children's father "came to Stamps without warning" and returned them to their mother's care in St. Louis. At the age of eight, while living with her mother, Angelou was sexually abused and raped by her mother's boyfriend, a man named Freeman. She told her brother, who told the rest of their family. Freeman was found guilty but was jailed for only one day. Four days after his release, he was murdered, probably by Angelou's uncles. Angelou became mute for almost five years, believing she was to blame for his death; as she stated: "I thought, my voice killed him; I killed that man, because I told his name. And then I thought I would never speak again, because my voice would kill anyone." According to Marcia Ann Gillespie and her colleagues, who wrote a biography about Angelou, it was during this period of silence when Angelou developed her extraordinary memory, her love for books and literature, and her ability to listen and observe the world around her. +Shortly after Freeman's murder, when Angelou was eight and her brother nine, Angelou and her brother were sent back to their grandmother. She attended the Lafayette County Training School, in Stamps, a Rosenwald School. Angelou credits a teacher and friend of her family, Mrs. Beulah Flowers, with helping her speak again, challenging her by saying: "You do not love poetry, not until you speak it." Flowers introduced her to Charles Dickens, William Shakespeare, Edgar Allan Poe, Georgia Douglas Johnson, and James Weldon Johnson, authors who would affect Angelou's life and career, as well as Black female artists such as Frances Harper, Anne Spencer, and Jessie Fauset. +When Angelou was 14 and her brother 15, she and her brother moved in once again with their mother, who had since moved to Oakland, California. During World War II, Angelou attended the California Labor School. At the age of 16, she became the first Black female streetcar conductor in San Francisco. She wanted the job badly, admiring the uniforms of the operators, describing the women in uniform as having "their little money-changing belts and with bibs on their caps and well-fitted uniforms"—so much so that her mother referred to it as her "dream job". Her mother encouraged her to pursue the position, but warned her that she would need to arrive early and work harder than others. In 2014, Angelou received a lifetime achievement award from the Conference of Minority Transportation Officials as part of a session billed "Women Who Move the Nation". +Three weeks after completing school, at the age of 17, she gave birth to her son, Clyde (who later changed his name to Guy Johnson). +In 1951, Angelou married Tosh Angelos, a Greek electrician, former sailor, and aspiring musician, despite the condemnation of interracial relationships at the time and the disapproval of her mother. She took modern dance classes during this time and met dancers and choreographers Alvin Ailey and Ruth Beckford. Ailey and Angelou formed a dance team, calling themselves "Al and Rita", and performed modern dance at fraternal Black organizations throughout San Francisco but never became successful. Angelou, her new husband, and her son moved to New York City so she could study African dance with Trinidadian dancer Pearl Primus, but they returned to San Francisco a year later. +After Angelou's marriage ended in 1954, she danced professionally in clubs around San Francisco, including the nightclub The Purple Onion, where she sang and danced to calypso music. Up to that point, she went by the name of "Marguerite Johnson", or "Rita", but at the strong suggestion of her managers and supporters at The Purple Onion, she changed her professional name to "Maya Angelou" (her nickname and former married surname). It was a "distinctive name" that set her apart and captured the feel of her calypso dance performances. During 1954 and 1955, Angelou toured Europe with a production of the opera Porgy and Bess. She began her practice of learning the language of every country she visited, and in a few years she gained proficiency in several languages. In 1957, riding on the popularity of calypso, Angelou recorded her first album, Miss Calypso, which was reissued as a CD in 1996. She appeared in an off-Broadway review that inspired the 1957 film Calypso Heat Wave, in which Angelou sang and performed her own compositions. +In 1959, Angelou met novelist John Oliver Killens and, at his urging, moved to New York to concentrate on her writing career. She joined the Harlem Writers Guild, where she met several major African American authors, including John Henrik Clarke, Rosa Guy, Paule Marshall, and Julian Mayfield, and was published for the first time. In 1960, after meeting civil rights leader Martin Luther King Jr. and hearing him speak, she and Killens organized "the legendary" Cabaret for Freedom to benefit the Southern Christian Leadership Conference (SCLC), and she was named SCLC's Northern Coordinator. According to scholar Lyman B. Hagen, her contributions to civil rights as a fundraiser and SCLC organizer were successful and "eminently effective". Angelou also began her pro-Castro and anti-apartheid activism during this time, joining the Fair Play for Cuba Committee. She had joined the crowd cheering for Fidel Castro when he first entered the Hotel Theresa in Harlem, New York, during the United Nations 15th General Assembly on September 19, 1960. +In 1961, Angelou performed in Jean Genet's play The Blacks (playing the part of the Queen), along with Abbey Lincoln, Roscoe Lee Browne, James Earl Jones, Louis Gossett, Godfrey Cambridge, and Cicely Tyson. Also in 1961, she met South African freedom fighter Vusumzi Make; they never officially married. She and her son Guy moved with Make to Cairo, where Angelou worked as an associate editor at the weekly English-language newspaper The Arab Observer. In 1962, her relationship with Make ended, and she and Guy moved to Accra, Ghana, so he could attend college, but he was seriously injured in an automobile accident. Angelou remained in Accra for his recovery and ended up staying there until 1965. She became an administrator at the University of Ghana and was active in the African American expatriate community. She was a feature editor for The African Review, a freelance writer for the Ghanaian Times, wrote and broadcast for Radio Ghana, and worked and performed for Ghana's National Theatre. She performed in a revival of The Blacks in Geneva and Berlin.In Accra, she became close friends with Malcolm X during his visit in the early 1960s. Angelou returned to the U.S. in 1965 to help him build a new civil rights organization, the Organization of Afro-American Unity; he was assassinated shortly afterward. Devastated and adrift, she joined her brother in Hawaii, where she resumed her singing career. She moved back to Los Angeles to focus on her writing career. Working as a market researcher in Watts, Angelou witnessed the riots in the summer of 1965. She acted in and wrote plays and returned to New York in 1967. She met her lifelong friend Rosa Guy and renewed her friendship with James Baldwin, whom she had met in Paris in the 1950s and called "my brother", during this time. Her friend Jerry Purcell provided Angelou with a stipend to support her writing. +In 1968, Martin Luther King Jr. asked Angelou to organize a march. She agreed, but postponed once again, and in what Gillespie calls "a macabre twist of fate", he was assassinated on her 40th birthday (April 4). Devastated again, she was encouraged out of her depression by her friend James Baldwin. As Gillespie states, "If 1968 was a year of great pain, loss, and sadness, it was also the year when America first witnessed the breadth and depth of Maya Angelou's spirit and creative genius". Despite having almost no experience, she wrote, produced, and narrated Blacks, Blues, Black!, a ten-part series of documentaries about the connection between blues music and Black Americans' African heritage, and what Angelou called the "Africanisms still current in the U.S." for National Educational Television, the precursor of PBS. Also in 1968, inspired at a dinner party she attended with Baldwin, cartoonist Jules Feiffer, and his wife Judy, and challenged by Random House editor Robert Loomis, she wrote her first autobiography, I Know Why the Caged Bird Sings, published in 1969. This brought her international recognition and acclaim. +Released in 1972, Angelou's Georgia, Georgia, produced by a Swedish film company and filmed in Sweden, was the first produced screenplay by a Black woman. She also wrote the film's soundtrack, despite having very little additional input in the filming of the movie. Angelou married Paul du Feu, a Welsh carpenter and ex-husband of writer Germaine Greer, in San Francisco in 1973. Over the next ten years, as Gillespie has stated, "She [Angelou] had accomplished more than many artists hope to achieve in a lifetime." Angelou worked as a composer, writing for singer Roberta Flack, and composing movie scores. She wrote articles, short stories, TV scripts, documentaries, autobiographies, and poetry. She produced plays and was named a visiting professor at several colleges and universities. She was "a reluctant actor", and was nominated for a Tony Award in 1973 for her role in Jerome Kilty's play Look Away. As a theater director, in 1988 she undertook a revival of Errol John's play Moon on a Rainbow Shawl at the Almeida Theatre in London. +In 1977, Angelou appeared in a supporting role in the television mini-series Roots. She was given multiple awards during this period, including more than thirty honorary degrees from colleges and universities from all over the world. In the late 1970s, Angelou met Oprah Winfrey when Winfrey was a TV anchor in Baltimore, Maryland; Angelou would later become Winfrey's close friend and mentor. In 1981, Angelou and du Feu divorced. +She returned to the southern United States in 1981 because she felt she had to come to terms with her past there and, despite having no bachelor's degree, accepted the lifetime Reynolds Professorship of American Studies at Wake Forest University in Winston-Salem, North Carolina, where she was one of a few full-time African American professors. From that point on, she considered herself "a teacher who writes". Angelou taught a variety of subjects that reflected her interests, including philosophy, ethics, theology, science, theater, and writing. The Winston-Salem Journal reported that even though she made many friends on campus, "she never quite lived down all of the criticism from people who thought she was more of a celebrity than an intellect ... [and] an overpaid figurehead". The last course she taught at Wake Forest was in 2011, but she was planning to teach another course in late 2014. Her final speaking engagement at the university was in late 2013. Beginning in the 1990s, Angelou actively participated in the lecture circuit in a customized tour bus, something she continued into her eighties. She also taught at the University of California, Los Angeles, the University of Kansas, and the University of Ghana and was recognized as a Rockefeller Foundation Scholar and a Yale University Fellow. She wrote the poetry and played a role in the 1993 movie Poetic Justice starring Janet Jackson. +In 1993, Angelou recited her poem "On the Pulse of Morning" at the presidential inauguration of Bill Clinton, becoming the first poet to make an inaugural recitation since Robert Frost at John F. Kennedy's inauguration in 1961. Her recitation resulted in more fame and recognition for her previous works, and broadened her appeal "across racial, economic, and educational boundaries". The recording of the poem won a Grammy Award. In June 1995, she delivered what Richard Long called her "second 'public' poem", entitled "A Brave and Startling Truth", which commemorated the 50th anniversary of the United Nations. +Angelou achieved her goal of directing a feature film in 1996, Down in the Delta, which featured actors such as Alfre Woodard and Wesley Snipes. Also in 1996, she collaborated with R&B artists Ashford & Simpson on seven of the eleven tracks of their album Been Found. The album was responsible for three of Angelou's only Billboard chart appearances. In 2000, she created a successful collection of products for Hallmark, including greeting cards and decorative household items. She responded to critics who charged her with being too commercial by stating that "the enterprise was perfectly in keeping with her role as 'the people's poet'". More than thirty years after Angelou began writing her life story, she completed her sixth autobiography A Song Flung Up to Heaven, in 2002. +Angelou campaigned for the Democratic Party in the 2008 presidential primaries, giving her public support to Hillary Clinton. In the run-up to the January Democratic primary in South Carolina, the Clinton campaign ran ads featuring Angelou's endorsement. The ads were part of the campaign's efforts to rally support in the Black community; but Barack Obama won the South Carolina primary, finishing 29 points ahead of Clinton and taking 80% of the Black vote. When Clinton's campaign ended, Angelou put her support behind Obama, who went on to win the presidential election and become the first African American president of the United States. After Obama's inauguration, she stated, "We are growing up beyond the idiocies of racism and sexism." +In late 2010, Angelou donated her personal papers and career memorabilia to the Schomburg Center for Research in Black Culture in Harlem. They consisted of more than 340 boxes of documents that featured her handwritten notes on yellow legal pads for I Know Why the Caged Bird Sings, a 1982 telegram from Coretta Scott King, fan mail, and personal and professional correspondence from colleagues such as her editor Robert Loomis. In 2011, Angelou served as a consultant for the Martin Luther King, Jr. Memorial in Washington, D.C. She spoke out in opposition to a paraphrase of a quotation by King that appeared on the memorial, saying, "The quote makes Dr. Martin Luther King look like an arrogant twit", and demanded that it be changed. Eventually, the paraphrase was removed. +In 2013, at the age of 85, Angelou published the seventh volume of autobiography in her series, entitled Mom & Me & Mom, which focuses on her relationship with her mother. +Evidence suggests that Angelou was partially descended from the Mende people of West Africa through her maternal line. In 2008, a DNA test revealed that 45 percent of her genetic makeup was from Central African peoples from Congo-Angola region and 55 percent came from West Africans. A 2008 PBS documentary found that Angelou's maternal great-grandmother, Mary Lee, who had been emancipated after the Civil War, became pregnant by her white former owner, John Savin. Savin forced Lee to sign a false statement accusing another man of being the father of her child. After Savin was indicted for forcing Lee to commit perjury, and despite the discovery that Savin was the father, a jury found him not guilty. Lee was sent to the Clinton County poorhouse in Missouri with her daughter, Marguerite Baxter, who became Angelou's grandmother. Angelou described Lee as "that poor little black girl, physically and mentally bruised." Her maternal grandfather, Thomas Baxter, was from Trinidad and Tobago. He jumped ship in Florida at the turn of the 20th century, and later moved north and became a railway porter in St. Louis. +The details of Angelou's life described in her seven autobiographies and in numerous interviews, speeches, and articles tended to be inconsistent. Critic Mary Jane Lupton has explained that when Angelou spoke about her life, she did so eloquently, but informally, and "with no time chart in front of her." For example, she was married at least twice, but never clarified the number of times she had been married, "for fear of sounding frivolous." According to her autobiographies and to Gillespie, she married Tosh Angelos in 1951, and Paul du Feu in 1973 or 1974, and began her relationship with Vusumzi Make in 1961, but never formally married him. Angelou held many jobs, including some in the sex trade working as a prostitute and madam for lesbians, and describes so in her second autobiography, Gather Together in My Name. In a 1995 interview, Angelou said: +I wrote about my experiences because I thought too many people tell young folks, "I never did anything wrong. Who, Moi? – never I. I have no skeletons in my closet. In fact, I have no closet." They lie like that and then young people find themselves in situations and they think, "Damn I must be a pretty bad guy. My mom or dad never did anything wrong." They can't forgive themselves and go on with their lives. +Angelou had one son, Guy, whose birth she described in her first autobiography; one grandson, two great-grandchildren, and, according to Gillespie, a large group of friends and extended family. Angelou's mother Vivian Baxter died in 1991 and her brother Bailey Johnson Jr. died in 2000 after a series of strokes; both were important figures in her life and her books. In 1981, the mother of her grandson disappeared with him; finding him took four years. +Angelou did not earn a university degree, but according to Gillespie it was Angelou's preference to be called "Dr. Angelou" by people outside of her family and close friends. She owned two homes in Winston-Salem, North Carolina, and a "lordly brownstone" in Harlem, which was purchased in 2004 and was full of her "growing library" of books she collected throughout her life, artwork collected over the span of many decades, and well-stocked kitchens. The Guardian writer Gary Younge reported that in Angelou's Harlem home were several African wall hangings and her collection of paintings, including ones of several jazz trumpeters, a watercolor of Rosa Parks, and a Faith Ringgold work entitled "Maya's Quilt Of Life". +According to Gillespie, she hosted several celebrations per year at her main residence in Winston-Salem; "her skill in the kitchen is the stuff of legend—from haute cuisine to down-home comfort food". The Winston-Salem Journal stated: "Securing an invitation to one of Angelou's Thanksgiving dinners, Christmas tree decorating parties or birthday parties was among the most coveted invitations in town." The New York Times, describing Angelou's residence history in New York City, stated that she regularly hosted elaborate New Year's Day parties. She combined her cooking and writing skills in her 2004 book Hallelujah! The Welcome Table, which featured 73 recipes, many of which she learned from her grandmother and mother, accompanied by 28 vignettes. She followed up in 2010 with her second cookbook, Great Food, All Day Long: Cook Splendidly, Eat Smart, which focused on weight loss and portion control. +Beginning with I Know Why the Caged Bird Sings, Angelou used the same "writing ritual" for many years. She would wake early in the morning and check into a hotel room, where the staff was instructed to remove any pictures from the walls. She would write on legal pads while lying on the bed, with only a bottle of sherry, a deck of cards to play solitaire, Roget's Thesaurus, and the Bible, and would leave by the early afternoon. She would average 10–12 pages of written material a day, which she edited down to three or four pages in the evening. She went through this process to "enchant" herself, and as she said in a 1989 interview with the British Broadcasting Corporation, "relive the agony, the anguish, the Sturm und Drang". She placed herself back in the time she wrote about, even traumatic experiences such as her rape in Caged Bird, to "tell the human truth" about her life. She was quoted as saying: "The way I deal with any pain is to admit it – let it come." Angelou stated that she played cards to get to that place of enchantment and to access her memories more effectively. She said, "It may take an hour to get into it, but once I'm in it—ha! It's so delicious!" She did not find the process cathartic; rather, she found relief in "telling the truth". +In 2009, the gossip website TMZ erroneously reported that Angelou had been hospitalized in Los Angeles when she was alive and well in St. Louis, which resulted in rumors of her death and, according to Angelou, concern among her friends and family worldwide. +Angelou died on the morning of May 28, 2014, at age 86. Although Angelou had been in poor health and had canceled recent scheduled appearances, she was working on another book, an autobiography about her experiences with national and world leaders. During her memorial service at Wake Forest University, her son Guy Johnson stated that despite being in constant pain due to her dancing career and respiratory failure, she wrote four books during the last ten years of her life. He said, "She left this mortal plane with no loss of acuity and no loss in comprehension." +Tributes to Angelou and condolences were paid by artists, entertainers, and world leaders, including President Obama, whose sister was named after Angelou, and Bill Clinton. Harold Augenbraum, from the National Book Foundation, said that Angelou's "legacy is one that all writers and readers across the world can admire and aspire to." The week after Angelou's death, I Know Why the Caged Bird Sings rose to number 1 on Amazon.com's bestseller list. +On May 29, 2014, Mount Zion Baptist Church in Winston-Salem, of which Angelou was a member for 30 years, held a public memorial service to honor her. On June 7, a private memorial service was held at Wait Chapel on the campus of Wake Forest University in Winston-Salem. The memorial was shown live on local stations in the Winston-Salem/Triad area and streamed live on the university web site with speeches from her son, Oprah Winfrey, Michelle Obama, and Bill Clinton. On June 15, a memorial was held at Glide Memorial Church in San Francisco, where Angelou was a member for many years. Rev. Cecil Williams, Mayor Ed Lee, and former mayor Willie Brown spoke. +Angelou wrote a total of seven autobiographies. According to scholar Mary Jane Lupton, Angelou's third autobiography Singin' and Swingin' and Gettin' Merry Like Christmas marked the first time a well-known African American autobiographer had written a third volume about her life. Her books "stretch over time and place", from Arkansas to Africa and back to the U.S., and take place from the beginnings of World War II to the assassination of Martin Luther King Jr. In her fifth autobiography, All God's Children Need Traveling Shoes (1986), Angelou tells about her return to Ghana searching for the past of her tribe. She published her seventh autobiography Mom & Me & Mom in 2013, at the age of 85. Critics have tended to give Caged Bird the highest praise. Angelou wrote five collections of essays, which writer Hilton Als called her "wisdom books" and "homilies strung together with autobiographical texts". Angelou used the same editor throughout her writing career, Robert Loomis, an executive editor at Random House; he retired in 2011 and has been called "one of publishing's hall of fame editors". Angelou said regarding Loomis: "We have a relationship that's kind of famous among publishers." +Angelou's long and extensive career also included poetry, plays, screenplays for television and film, directing, acting, and public speaking. She was a prolific writer of poetry; her volume Just Give Me a Cool Drink of Water 'fore I Diiie (1971) was nominated for the Pulitzer Prize, and she was chosen by U.S. president Bill Clinton to recite her poem "On the Pulse of Morning" during his inauguration in 1993. +Angelou's successful acting career included roles in numerous plays, films, and television programs, among them her appearance in the television mini-series Roots in 1977. Her screenplay, Georgia, Georgia (1972), was the first original script by a Black woman to be produced, and she was the first African American woman to direct a major motion picture, Down in the Delta, in 1998. +I Know Why the Caged Bird Sings (1969): Up to 1944 (age 17) +Gather Together in My Name (1974): 1944–48 +Singin' and Swingin' and Gettin' Merry Like Christmas (1976): 1949–55 +The Heart of a Woman (1981): 1957–62 +All God's Children Need Traveling Shoes (1986): 1962–65 +A Song Flung Up to Heaven (2002): 1965–68 +Mom & Me & Mom (2013): overview +When I Know Why the Caged Bird Sings was published in 1969, Angelou was hailed as a new kind of memoirist, one of the first African American women who were able to publicly discuss their personal lives. According to scholar Hilton Als, up to that point, Black female writers were marginalized to the point that they were unable to present themselves as central characters in the literature they wrote. Linguist John McWhorter agreed, seeing Angelou's works, which he called "tracts", as "apologetic writing". He placed Angelou in the tradition of African American literature as a defense of Black culture, which he called "a literary manifestation of the imperative that reigned in the black scholarship of the period". Writer Julian Mayfield, who called Caged Bird "a work of art that eludes description", argued that Angelou's autobiographies set a precedent for not only other Black women writers, but also African American autobiography as a whole. Als said that Caged Bird marked one of the first times that a Black autobiographer could, as he put it, "write about blackness from the inside, without apology or defense". Through the writing of her autobiography, Angelou became recognized and highly respected as a spokesperson for Blacks and women. It made her "without a doubt, ... America's most visible black woman autobiographer", and "a major autobiographical voice of the time". As writer Gary Younge said, "Probably more than almost any other writer alive, Angelou's life literally is her work." +Als said that Caged Bird helped increase Black feminist writings in the 1970s, less through its originality than "its resonance in the prevailing Zeitgeist", or the time in which it was written, at the end of the American Civil Rights Movement. Als also claimed that Angelou's writings, more interested in self-revelation than in politics or feminism, have freed other female writers to "open themselves up without shame to the eyes of the world". Angelou critic Joanne M. Braxton stated that Caged Bird was "perhaps the most aesthetically pleasing" autobiography written by an African American woman in its era. Angelou's poetry has influenced the modern hip-hop music community, including artists such as Kanye West, Common, Tupac Shakur, and Nicki Minaj. +Reviewer Elsie B. Washington called Angelou "the black woman's poet laureate". Sales of the paperback version of her books and poetry rose by 300–600% the week after Angelou's recitation. Random House, which published the poem later that year, had to reprint 400,000 copies of all her books to keep up with the demand. They sold more of her books in January 1993 than they did in all of 1992, accounting for a 1,200% increase. Angelou famously said, in response to criticism regarding using the details of her life in her work, "I agree with Balzac and 19th-century writers, black and white, who say, 'I write for money'." Younge, speaking after the publication of Angelou's third book of essays, Letter to My Daughter (2008), has said, "For the last couple of decades she has merged her various talents into a kind of performance art—issuing a message of personal and social uplift by blending poetry, song and conversation." +Angelou's books, especially I Know Why the Caged Bird Sings, have been criticized by many parents, causing their removal from school curricula and library shelves. According to the National Coalition Against Censorship, some parents and some schools have objected to Caged Bird's depictions of lesbianism, premarital cohabitation, pornography, and violence. Some have been critical of the book's sexually explicit scenes, use of language, and irreverent depictions of religion. Caged Bird appeared third on the American Library Association (ALA) list of the 100 Most Frequently Challenged Books of 1990–2000 and sixth on the ALA's 2000–2009 list. +Angelou was honored by universities, literary organizations, government agencies, and special interest groups. Her honors included a Pulitzer Prize nomination for her book of poetry, Just Give Me a Cool Drink of Water 'fore I Diiie, a Tony Award nomination for her role in the 1973 play Look Away, and three Grammys for her spoken-word albums. She served on two presidential committees, and was awarded the Spingarn Medal in 1994, the National Medal of Arts in 2000, and the Presidential Medal of Freedom in 2011. Angelou was awarded more than fifty honorary degrees. In 2021, the United States Mint announced that Angelou would be among the first women depicted on the reverse of the quarter as a part of the American Women quarters series. The coins were released in January 2022. She is the first Black woman to be depicted on a quarter. +Angelou's autobiographies have been used in narrative and multicultural approaches in teacher education. Jocelyn A. Glazier, a professor at George Washington University, has trained teachers how to "talk about race" in their classrooms with I Know Why the Caged Bird Sings and Gather Together in My Name. According to Glazier, Angelou's use of understatement, self-mockery, humor, and irony have left readers of Angelou's autobiographies unsure of what she left out and how they should respond to the events she described. Angelou's depictions of her experiences of racism have forced white readers to either explore their feelings about race and their own "privileged status", or to avoid the discussion as a means of keeping their privilege. Glazier found that critics have focused on the way Angelou fits within the genre of African American autobiography and on her literary techniques, but readers have tended to react to her storytelling with "surprise, particularly when [they] enter the text with certain expectations about the genre of autobiography". +Educator Daniel Challener, in his 1997 book Stories of Resilience in Childhood, analyzed the events in Caged Bird to illustrate resiliency in children. He argued that Angelou's book has provided a "useful framework" for exploring the obstacles many children like Maya have faced and how their communities have helped them succeed. Psychologist Chris Boyatzis has reported using Caged Bird to supplement scientific theory and research in the instruction of child development topics such as the development of self-concept and self-esteem, ego resilience, industry versus inferiority, effects of abuse, parenting styles, sibling and friendship relations, gender issues, cognitive development, puberty, and identity formation in adolescence. He found Caged Bird a "highly effective" tool for providing real-life examples of these psychological concepts. +In 2025, Connie and Peter Roop published a children's book illustrated by Noa Denmon, entitled Maya Angelou Finds Her Voice, which Publishers Weekly described as a "sensitively rendered biography". +Angelou is best known for her seven autobiographies, but she was also a prolific and successful poet. She was called "the black woman's poet laureate", and her poems have been called the anthems of African Americans. Angelou studied and began writing poetry at a young age, and used poetry and other great literature to cope with her rape as a young girl, as described in Caged Bird. According to scholar Yasmin Y. DeGout, literature also affected Angelou's sensibilities as the poet and writer she became, especially the "liberating discourse that would evolve in her own poetic canon". +Many critics consider Angelou's autobiographies more important than her poetry. Although all her books have been bestsellers, her poetry has not been perceived to be as serious as her prose and has been understudied. Her poems were more interesting when she recited and performed them, and many critics emphasized the public aspect of her poetry. Angelou's lack of critical acclaim has been attributed to both the public nature of many of her poems and to Angelou's popular success, and to critics' preferences for poetry as a written form rather than a verbal, performed one. Zofia Burr has countered Angelou's critics by condemning them for not taking into account Angelou's larger purposes in her writing: "to be representative rather than individual, authoritative rather than confessional". +In the view of Harold Bloom, Professor of Literature (Yale University and New York University) and literary critic: +Her poetry has a large public, but very little critical esteem. It is, in every sense, "popular poetry," and makes no formal or cognitive demands upon the reader. Of Angelou's sincerity, good-will towards all, and personal vitality, there can be no doubt. She is professionally an inspirational writer, of the self-help variety, which perhaps places her beyond criticism. [...] Angelou seems best at ballads, the most traditional kind of popular poetry. The function of such work is necessarily social rather than aesthetic, particularly in an era totally dominated by visual media. One has to be grateful for the benignity, humor, and whole-heartedness of Angelou's project, even if her autobiographical prose necessarily centers her achievement. +Angelou's use of fiction-writing techniques such as dialogue, characterization, and development of theme, setting, plot, and language has often resulted in the placement of her books into the genre of autobiographical fiction. Angelou made a deliberate attempt in her books to challenge the common structure of the autobiography by critiquing, changing, and expanding the genre. Scholar Mary Jane Lupton argues that all of Angelou's autobiographies conform to the genre's standard structure: they are written by a single author, they are chronological, and they contain elements of character, technique, and theme. Angelou recognizes that there are fictional aspects to her books; Lupton agrees, stating that Angelou tended to "diverge from the conventional notion of autobiography as truth", which parallels the conventions of much of African American autobiography written during the abolitionist period of U.S. history, when as both Lupton and African American scholar Crispin Sartwell put it, the truth was censored out of the need for self-protection. Scholar Lyman B. Hagen places Angelou in the long tradition of African American autobiography but claims that Angelou created a unique interpretation of the autobiographical form. +According to African American literature scholar Pierre A. Walker, the challenge for much of the history of African American literature was that its authors have had to confirm its status as literature before they could accomplish their political goals, which was why Angelou's editor Robert Loomis was able to dare her into writing Caged Bird by challenging her to write an autobiography that could be considered "high art". Angelou acknowledged that she followed the slave narrative tradition of "speaking in the first-person singular talking about the first-person plural, always saying I meaning 'we'". Scholar John McWhorter calls Angelou's books "tracts" that defend African American culture and fight negative stereotypes. According to McWhorter, Angelou structured her books, which to him seem to be written more for children than for adults, to support her defense of Black culture. McWhorter sees Angelou as she depicts herself in her autobiographies "as a kind of stand-in figure for the Black American in Troubled Times". McWhorter views Angelou's works as dated but recognizes that "she has helped to pave the way for contemporary black writers who are able to enjoy the luxury of being merely individuals, no longer representatives of the race, only themselves". Scholar Lynn Z. Bloom compares Angelou's works to the writings of Frederick Douglass, stating that both fulfilled the same purpose: to describe Black culture and to interpret it for their wider, white audiences. +According to scholar Sondra O'Neale, Angelou's poetry can be placed within the African American oral tradition, and her prose "follows classic technique in nonpoetic Western forms". O'Neale states that Angelou avoided using a "monolithic Black language", and accomplished, through direct dialogue, what O'Neale calls a "more expected ghetto expressiveness". McWhorter finds both the language Angelou used in her autobiographies and the people she depicted unrealistic, resulting in a separation between her and her audience. As McWhorter states, "I have never read autobiographical writing where I had such a hard time summoning a sense of how the subject talks, or a sense of who the subject really is". McWhorter asserts, for example, that key figures in Angelou's books, like herself, her son Guy, and mother Vivian do not speak as one would expect, and that their speech is "cleaned up" for her readers. Guy, for example, represents the young Black male, while Vivian represents the idealized mother figure, and the stiff language they use, as well as the language in Angelou's text, is intended to prove that Blacks can use standard English competently. +McWhorter recognizes that much of the reason for Angelou's style was the "apologetic" nature of her writing. When Angelou wrote Caged Bird at the end of the 1960s, one of the necessary and accepted features of literature at the time was "organic unity", and one of her goals was to create a book that satisfied that criterion. The events in her books were episodic and crafted like a series of short stories, but their arrangements did not follow a strict chronology. Instead, they were placed to emphasize the themes of her books, which include racism, identity, family, and travel. English literature scholar Valerie Sayers has asserted that "Angelou's poetry and prose are similar". They both rely on her "direct voice", which alternates steady rhythms with syncopated patterns and uses similes and metaphors (e.g., the caged bird). According to Hagen, Angelou's works were influenced by both conventional literary and the oral traditions of the African American community. For example, she referenced more than 100 literary characters throughout her books and poetry. In addition, she used the elements of blues music, including the act of testimony when speaking of one's life and struggles, ironic understatement, and the use of natural metaphors, rhythms, and intonations. Angelou, instead of depending upon plot, used personal and historical events to shape her books. +Official website +Maya Angelou discography at Discogs +Maya Angelou at IMDb +Maya Angelou at the Internet Broadway Database +Maya Angelou at the Internet Off-Broadway Database (archived) +Maya Angelou papers at New York Public Library +Maya Angelou memorial service at Wake Forest University, June 7, 2014. +Spring, Kelly. "Maya Angelou". National Women's History Museum. 2017. +Gail Mitchell, "Maya Angelou's Posthumous Album, 'Caged Bird Songs,' Debuts", Billboard, November 4, 2014. +Appearances on C-SPAN +"'Maya Angelou: And Still I Rise': First Film on Writer and Activist Chronicles an Extraordinary Life", Democracy Now!, February 16, 2016. +Caroline Agnes Brady (also known as Caroline Agnes Von Egmont Brady; October 3, 1905 – November 5, 1980) was an American philologist who specialised in Old English and Old Norse works. Her works included the 1943 book The Legends of Ermanaric, based on her doctoral dissertation, and three influential papers on the Anglo-Saxon poem Beowulf. She taught at the University of California, Berkeley, the University of Pennsylvania, Johns Hopkins University, and Harvard University, among other places. +Brady was born an American citizen in Tientsin, China, and traveled frequently as a child, spending time in Los Angeles, California, British Columbia, and Austin, Texas. She studied in the University of California system, receiving her bachelor's and master's degrees, and her Ph.D. in 1935. She next became an English instructor at that university's College of Agriculture, and worked as an assistant professor of languages and literature at Berkeley from 1941 to 1946. The following three years were spent at the University of Pennsylvania, until, at the end of 1949, Brady moved to teach at Central Oregon Community College; her resignation due to "ill health" was announced a few months later. After being named the 1952–53 Marion Talbot Fellow of the American Association of University Women and writing two articles, Brady's scholarship ceased for a quarter of a century. In 1979, and posthumously in 1983, her final two articles were published. +Brady's monograph, The Legends of Ermanaric, argued that the Gothic king Ermanaric was subject to two competing traditions, and earned her a reputation as "a broad and discriminating investigator" with "a sovereign disregard of established opinion". Her papers on Beowulf, meanwhile, were identified by Katherine O'Brien O'Keeffe, a scholar of Anglo-Saxon literature, as "three fundamental studies" that were "philological in the traditional sense", shedding light on "the shades of meaning of the diction" used in the poem. Brady concluded that the Beowulf poet "is no artificer mechanically piling up synonyms and conventional metaphors, but an artist who knows how to use a variety of words and phrases". +Caroline Agnes Brady was born on October 3, 1905, in Tientsin, China. She was the daughter of United States Army Colonel David John Brady, an engineer who was then the general manager of the Chinese Engineering and Mining Company in Qinhuangdao, and his wife (Annie Lucy) Maude Short, daughter of John Short, the deputy prothonotary of Sherbrooke, Quebec. The two had married the year before, at the Holy Trinity Church in Shanghai. Caroline was one of two children, and the eldest by ten years of her sister, Frances Maud Brady. Her father, the son of British emigrants, had been raised in Austin, Texas, and traveled as the army took him. His two brothers, John W. and Will P. Brady—Caroline Brady's uncles—both became prominent Texas attorneys and jurists. Will P. Brady worked as the first district attorney of Reeves County, Texas, and later as a judge of the county court in El Paso. John W. Brady rose to prominence within Austin, and Texas generally, as an assistant attorney general and judge, before killing his mistress in 1929 and being sentenced to three years in prison. Caroline Brady's mother, meanwhile, traced back through four generations of her matriline to Anthony Van Egmond (Caroline Brady's great-great-great-grandfather), an early settler of Canada, whose true identity as the Dutch fugitive Antonij Jacobi Willem Gijben—not, as he claimed, as a direct descent of the counts of Egmond—was not uncovered until the second half of the twentieth century. +In May 1910, when Brady was four, her family arrived in Los Angeles, California, via Shanghai, aboard the steamer Bessie Dollar. The ship carried only two families and a woman traveling alone, in addition to a cargo of pig iron, and had what the Los Angeles Herald described as "a rough voyage across the Pacific", striking a whale. By the end of the year, the family was living in British Columbia. Though the Herald had described Brady's father as a Standard Oil engineer, by the end of World War I he was serving overseas as part of the Rainbow Division of the United States Army National Guard, in France and Germany as first a captain and then a major. During these years, until about September 1919, Caroline Brady and her family stayed with her uncle, John W. Brady, in his large Austin house. David Brady returned in August or September 1919; by the following year he was working with his brother Will at the Los Angeles-based Sunshine Oil Corporation. +In August 1924 Brady matriculated at the University of California at Los Angeles (UCLA), then known as the Southern Branch of the University of California. She entered Teachers College, for studies in kindergarten-primary education. Brady was active in a number of organizations at UCLA, including Beta Phi Alpha, the YWCA, and the Prytanean Society, of which she was president. She graduated with a Bachelor of Arts in English in 1928, and two years later, on May 15, 1930, received a Master of Arts from the school's Berkeley campus. That same year she began her Ph.D., also at Berkeley, and graduated in 1935, with the thesis The Legends of Ermanaric. Her dissertation committee was chaired by Arthur Gilchrist Brodeur, with whom Brady would coauthor an article five years later. +Brady is occasionally referred to as Caroline Agnes Von Egmont Brady. Though her published output universally refers to her as either "Caroline A. Brady" or "Caroline Brady", the program for her dissertation defense names her "Caroline Agnes Von Egmont Brady". Several library entries and membership lists of the Modern Language Association also use the longer name. +In 1935, the same year that she received her Ph.D., Brady became an English instructor at the College of Agriculture at the University of California. Brady was promoted on July 13, 1941, to assistant professor of languages and literature at the Berkeley campus. In 1943, her "completely rewritten" dissertation was published under the same title, The Legends of Ermanaric. Brady continued teaching at Berkeley until 1946. Thereafter, she taught for three years at the University of Pennsylvania as an assistant professor of English. +In 1949 Brady moved to Klamath Falls, Oregon, as one of the four inaugural instructors for the newly opened Central Oregon Community College. The college had campuses at both Bend and Klamath Falls; Brady taught at both, offering courses in English Composition and Survey of English Literature. After only a few months in the position, Brady resigned, citing "ill health". +By May 1952, Brady was working as the synonym editor for C. L. Barnhart, Inc., the publisher of Thorndike-Barnhart dictionaries, in Bronxville, New York. That year she was named the 1952–53 Marion Talbot Fellow of the American Association of University Women. The $2,200 fellowship was for "a study and reinterpretation of the substantial compounds and phrases in Old English poetry", looking at contextual word usage to "determine whether the various poets used them in exactly the same way". Brady's work was to take place at Johns Hopkins and Harvard, and in 1952 she published two related articles: "The Synonyms for 'Sea' in Beowulf", and "The Old English Nominal Compounds in -rád". +Brady's subsequent activities are unclear. In 1979, she was listed again with a California address. That year, she published the second piece in her Beowulf trilogy, "'Weapons' in Beowulf." The final work in the trilogy, "'Warriors' in Beowulf," was published posthumously, in 1983. +In her 1941 Who's Who in California entry, Brady was described as a Democrat and an Episcopalian. Her father died in late January 1953, and her mother in November 1959. Caroline Brady died on November 5, 1980, in Bellevue, Washington. The year before she was listed with an address in Corona del Mar by Anglo-Saxon England, the journal that published her final two works. Brady's sister, by then Frances Brady Ackley, died on December 14, 1993; her obituary mentioned only cousins among her survivors. +Brady's book The Legends of Ermanaric suggests the Gothic king Ermanaric, who ruled in the fourth century AD, was the subject of two competing traditions: one, in Ostrogothic lore, viewing him as a good king, and a second, promulgated by those subjugated by him, as evil. Brady's thesis gained less traction than her ability to investigate the intractable problems of Germanic myth, and the convoluted nature of the related scholarship. She was noted as "a broad and discriminating investigator", who had "a sovereign disregard of established opinion". Such disregard caused one reviewer to label Brady's work "more valuable in the sphere of criticism than construction," and another to note that her "conclusions are reached without reliance on the views of predecessors, and one may be sure that, in some quarters, the volume will be thoroughly combed for flaws to match those it has uncovered in the reasoning of others". Indeed, after Brady's "vigorous tilting with no less a scholar than Kemp Malone," he penned two separate reviews disparaging what he termed her scholastic immaturity, and suggesting "she overestimates the worth of debaters' points". Others shared concerns with Brady's thesis while being generally supportive, including the Old English scholar Philip W. Souers, who wrote that: +Her knowledge, from linguistics to archaeology, is great; her command of bibliography is sure; her acquaintance with languages shows the temper of a true scholar. ... It was worth doing, to try to establish a late Gothic legend that could be seen reflected in the Norse, to see where the results would lead. Others have always worked from the German sources. Though I cannot accept her hypothesis as proved, [the book] is without doubt one of the most important works in that difficult subject of heroic legend that has come from American scholarship in recent years. +Brady's 1979 and 1983 articles on the words used to describe weapons and warriors in the Anglo-Saxon poem Beowulf suggested that, unlike the interchangeability of words used for other subjects such as strong drink, the words used to describe weapons and warriors were precisely tailored to fit their specific contexts. Taken with her 1952 article "The Synonyms for 'Sea' in Beowulf", these are described by Katherine O'Brien O'Keeffe in A Beowulf Handbook as "three fundamental studies" that examine the context in which the Beowulf poet chose a word rather than simply the word itself. Brady concluded that "this poet is no artificer mechanically piling up synonyms and conventional metaphors, but an artist who knows how to use a variety of words and phrases". Her approach was considered "philological in the traditional sense" by O'Keeffe, and to have shed light on "the shades of meaning of the diction" used in the poem. +In addition to her book and the Beowulf articles, Brady published a number of other works during her career. She also presented several papers, including some which ultimately went unpublished, at academic conferences—notably at meetings of the Philological Association of the Pacific Coast and the Modern Language Association. +Brady, Caroline (September 1935). The Legends of Ermanaric (Ph.D. thesis). University of California Press, Berkeley. +Brady, Caroline (1943). The Legends of Ermanaric. Berkeley: University of California. OCLC 878278262. +Brady, Caroline (November 1933). "A Note on the Historical Prototype of Sigfried". Modern Philology. 31 (2). University of Chicago Press: 195–196. doi:10.1086/388087. JSTOR 433891. S2CID 162361312. +Brady, Caroline (1937). The Eormanric of the Wīdsīð. University of California Publications in English. Vol. III. Berkeley: University of California Press. +Brady, Caroline (April 1938). "Becca of the Banings". The Journal of English and Germanic Philology. XXXVII (2). University of Illinois Press: 169–188. JSTOR 27704379. +Brady, Caroline (April 1939). "The Date and Metre of the Hamðismál". The Journal of English and Germanic Philology. XXXVIII (2). University of Illinois Press: 201–216. JSTOR 27704484. +Brady, Caroline (October 1940). "Innweorud Earmanrices". Speculum. XV (4). University of Chicago Press: 454–459. doi:10.2307/2853463. JSTOR 2853463. S2CID 225090506. +Brodeur, Arthur Gilchrist & Brady, Caroline (November 1940). "Sundrmœðri–Sammœðra". Scandinavian Studies and Notes. XVI (4). Society for the Advancement of Scandinavian Study: 133–137. JSTOR 40908177. +Brady, Caroline (December 1940). "Óðinn and the Norse Jǫrmunrekkr-Legend". Publications of the Modern Language Association. LV (4). Modern Language Association: 910–930. doi:10.2307/458885. JSTOR 458885. S2CID 163229366. +Brady, Caroline (1952a). "The Synonyms for "Sea" in Beowulf" (PDF). Studies in Honor of Albert Morey Sturtevant. Lawrence, Kansas: University of Kansas Press. pp. 22–46. +Brady, Caroline (June 1952b). "The Old English Nominal Compounds in -rád". Publications of the Modern Language Association. LXVII (4). Modern Language Association: 538–571. doi:10.2307/459826. JSTOR 459826. +Brady, Caroline (1979). "'Weapons' in Beowulf: an analysis of the nominal compounds and an evaluation of the poet's use of them". Anglo-Saxon England. 8. Cambridge University Press: 79–141. doi:10.1017/S0263675100003045. +Brady, Caroline (1983). "'Warriors' in Beowulf: an analysis of the nominal compounds and an evaluation of the poet's use of them". Anglo-Saxon England. 11. Cambridge University Press: 199–246. doi:10.1017/S0263675100002611. +Brady, Caroline. "Kings Frotho I-V: A Study in Saxo's Historical Method". Unpublished. +Brady, Caroline (January–June 1941). "The Orkneyinga Saga by Alexander Burt Taylor". The Journal of American Folklore. 54 (211–212). American Folklore Society: 90–92. doi:10.2307/535815. JSTOR 535815. +Brady, Caroline (April 1951). "Walter of Aquitaine: Materials for the Study of His Legend by F. P. Magoun, Jr. & H. M. Smyser". Speculum. XXVI (2). University of Chicago Press: 397–401. doi:10.2307/2852428. JSTOR 2852428. +Brady, Caroline (November 1955). "The Digressions in Beowulf. by Adrien Bonjour". Modern Language Notes. LXX (7). Johns Hopkins University Press: 521–524. doi:10.2307/3039650. JSTOR 3039650. +Bonjour, Adrien (January 1955). "On Sea Images in Beowulf". The Journal of English and Germanic Philology. LIV (1). University of Illinois Press: 111–115. JSTOR 27706524. +"Contents". Anglo-Saxon England. 8. Cambridge University Press: v–vi. 1979. doi:10.1017/S0263675100002982. +"Contents". Anglo-Saxon England. 11. Cambridge University Press: v–vi. 1983. doi:10.1017/S0263675100002519. +"Fellowship Awards, 1952–53". Journal of the American Association of University Women. 45 (4). American Association of University Women: 225–230. May 1952. ISSN 0001-0278. +Frank, Roberta (Summer 1987). "Did Anglo-Saxon Audiences have a Skaldic Tooth?". Scandinavian Studies. 59 (3). Society for the Advancement of Scandinavian Study: 338–355. JSTOR 40918869. +Girvan, Ritchie (October 1944). "The Legends of Ermanaric by Caroline Brady". The Modern Language Review. XXXIX (4). Modern Humanities Research Association: 403–404. doi:10.2307/3716937. JSTOR 3716937. +"List of Members of the Modern Language Association of America". Publications of the Modern Language Association. LXIVIII (4 (part 2, Supplement)). Modern Language Association: 68–169. September 1953. JSTOR 2698988. +"List of Members of the Modern Language Association of America". Publications of the Modern Language Association. LVI (Supplement). Modern Language Association: 1417–1524. 1941. JSTOR 459009. +Malone, Kemp (March 1944a). "The Legends of Ermanaric by Caroline Brady". Modern Language Notes. LIX (3). Johns Hopkins University Press: 183–185. JSTOR 2910880. +Malone, Kemp (October 1944b). "The Legends of Ermanaric by Caroline Brady". The Journal of English and Germanic Philology. XLIII (4). University of Illinois Press: 449–453. JSTOR 27705152. +Morgan, Lynn A., ed. (1984). Loyalist Lineages of Canada, 1783–1983. Vol. I. Agincourt, Toronto: Generation Press. ISBN 0-920830-24-2. +O'Donoghue, Bernard (May 1986). "The Old English Elegies. New Essays in Criticism and Research by Martin Green". The Review of English Studies. XXXVII (146). Oxford University Press: 237–238. doi:10.1093/res/XXXVII.146.237. JSTOR 516977. +O'Keeffe, Katherine O'Brien (November 1997). "Diction, Variation, the Formula". In Bjork, Robert E. & Niles, John D. (eds.). A Beowulf Handbook. Lincoln: University of Nebraska Press. pp. 85–104. ISBN 0-8032-1237-2. +"Proceedings of the Sixty-Eighth Annual Meeting of the American Philological Association. Also of the Thirty-Eighth Annual Meeting of the Philological Association of the Pacific Coast". Transactions and Proceedings of the American Philological Association. LXVII. Johns Hopkins University Press: i–cxi. 1936. doi:10.2307/283247. JSTOR 283247. +"Proceedings of Modern Language Association of America". Publications of the Modern Language Association. LIV (Supplement). Modern Language Association: 1356–1381. 1939. JSTOR 458756. +Remein, Daniel (February 18, 2016). "Kinetic Beowulf". Harvard Medieval Colloquium. Working paper: 1–26. +Rypins, Stanley (June 1945). "The Legends of Ermanaric by Caroline Brady". Modern Language Quarterly. VI (2). Johns Hopkins University Press: 225–226. doi:10.1215/00267929-6-2-225. +Souers, Philip W. (October 1945). "The Legends of Ermanaric by Caroline Brady". Speculum. XX (4). The Mediaeval Academy of America: 502–507. doi:10.2307/2856754. JSTOR 2856754. +Wahlgren, Erik (July 1944). "The Legends of Ermanaric by Caroline Brady". California Folklore Quarterly. III (3). Western States Folklore Society: 248–250. doi:10.2307/1495888. JSTOR 1495888. +Wilson, Elane (1999). Skeats, Terry (ed.). Sherbrooke Daily Record: Vital Statistics. Vol. 2. North Hatley, Quebec: North Hatley Historical Society. ISBN 978-0-9682366-2-8. +Aldwych is a closed station on the London Underground, located in the City of Westminster in Central London. It was opened in 1907 with the name Strand, after the street on which it is located. It was the terminus of the short Piccadilly line branch from Holborn that was a relic of the merger of two railway schemes. The station building is close to the Strand's junction with Surrey Street, near Aldwych. During its lifetime, the branch was the subject of a number of unrealised extension proposals that would have seen the tunnels through the station extended southwards, usually to Waterloo. +Served mostly by a shuttle train and having low passenger numbers, the station and branch were considered for closure several times. Service was offered only during weekday peak hours from 1962 and discontinued in 1994, when the cost of replacing the lifts was considered too high for the income generated. +Disused parts of the station and the running tunnels were used during both world wars to shelter artworks from London's public galleries and museums from bombing. The station is the work of Leslie Green and has long been popular as a filming location, appearing as itself and as other London Underground stations in a number of films. In recognition of its historical significance, the station is a Grade II listed building and is an example of the Modern Style (British Art Nouveau style). +London Transport Museum runs guided tours of the station via its "Hidden London" programme throughout the year. The tour takes visitors through the ticket hall, the original lifts, disused platforms and tunnels; and shares findings from the museum's archives on the history of the site. +The Great Northern and Strand Railway (GN&SR) first proposed a station in the parish of St Clement Danes in a private bill presented to Parliament in November 1898. The station was to be the southern terminus of an underground railway line planned to run from Wood Green station (now Alexandra Palace) via Finsbury Park and King's Cross and was originally to be located at the corner of Stanhope Street and Holles Street, north of the Strand. When the two streets were scheduled for demolition as part of the London County Council's plans for the construction of Kingsway and Aldwych, the GN&SR moved the location to the junction of the two new roads. Royal assent to the bill was given and the Great Northern and Strand Railway Act 1899 was enacted on 1 August. +In September 1901, the GN&SR was taken over by the Brompton and Piccadilly Circus Railway (B&PCR), which planned to build an underground line from South Kensington to Piccadilly Circus via Knightsbridge. Both were under the control of Charles Yerkes through his Metropolitan District Electric Traction Company and, in June 1902, were transferred to Yerkes' new holding company, the Underground Electric Railways Company of London (UERL). Neither of the railways had carried out any construction, but the UERL obtained permission for new tunnels between Piccadilly Circus and Holborn to connect the two routes. The companies were formally merged as the Great Northern, Piccadilly and Brompton Railway (GNP&BR) following approval in the Great Northern and Strand Railway Act 1902 (2 Edw. 7. c. ccxxxv) November 1902. Prior to confirmation of the merger, the GN&SR had sought permission to extend its line southwards from the future junction of Kingsway and Aldwych, under Norfolk Street to a new interchange under the Metropolitan District Railway's station at Temple. The extension was rejected following objections from the Duke of Norfolk under whose land the last part of the proposed tunnels would have run. +In 1903, the GNP&BR sought permission for a branch from Piccadilly Circus to run under Leicester Square, Strand, and Fleet Street and into the City of London. The branch would have passed and interchanged with the already approved Strand station, allowing travel on the GNP&BR from Strand in three directions. The deliberations of the Royal Commission on London Traffic prevented parliamentary consideration of the proposal, which was withdrawn. +In 1905, with the royal commission's report about to be published, the GNP&BR returned to Parliament with two bills for consideration. The first bill revived the 1903 proposal for a branch from Piccadilly Circus to the City of London, passing and interchanging with Strand station. The second proposed an extension and relocation of Strand station to the junction of Strand and Surrey Street. From there the line was to continue as a single tunnel under the River Thames to Waterloo. The first bill was again delayed and withdrawn. Of the second, it became the Great Northern, Piccadilly and Brompton Railway (Various Powers) Act 1905 (5 Edw. 7. c. clxiii), but only the relocation of Strand station was permitted. +The linking of the GN&SR and B&PCR routes meant that the section of the GN&SR south of Holborn became a branch from the main route. The UERL began constructing the main route in July 1902. Progress was rapid, so that it was largely complete by the Autumn of 1906. Construction of the Holborn to Strand section was delayed while the London County Council constructed Kingsway and the tramway subway running beneath it and while the UERL decided how the junction between the main route and the branch would be arranged at Holborn. +Strand station was built on the site of the Royal Strand Theatre, which had closed on 13 May 1905 and been demolished. Construction of the station began on 21 October 1905, to a design by the UERL's architect Leslie Green in the UERL house style of a two-storey steel-framed building faced with red glazed terracotta blocks, with wide semi-circular windows on the upper floor. The station building is L-shaped, with two façades separated by the building on the corner of Strand and Surrey Street. The Strand façade is narrow with a single semi-circular window above the entrance. The façade in Surrey Street is wider with a separate entrance and exit and a shop unit. In anticipation of a revival of the extension to Waterloo and the City route, the station was built with three circular lift shafts able to accommodate six trapezium-shaped lifts. Only one of the shafts was fitted out, with two lifts. The other two shafts rose from the lower concourse to the basement of the station, but could have been extended upwards into the space of the shop unit when required. A fourth smaller-diameter shaft accommodated an emergency spiral stair. +The platforms are 92 feet 6 inches (28.2 m) below street level and are 250 feet (76 m) long; shorter than the GNP&BR's standard length of 350 feet (110 m). As with other UERL stations, the platform walls were tiled with distinctive patterns, in this case cream and dark green. Only parts of the platform walls were decorated because it was planned to operate the branch with short trains. Owing to the reduced lift provision, a second route between the platforms and lifts was never brought into use and was left in an unfinished condition without tiling. +The GNP&BR's main route opened on 15 December 1906, but the Strand branch was not opened until 30 November 1907. Initially, shuttle trains operated to Holborn from the eastern platform into the through platform at Holborn. At peak times, an additional train operated alternately in the branch's western tunnel into the bay platform at Holborn. During the first year of operation, a train for theatregoers operated late on Monday to Saturday evenings from Strand through Holborn and northbound to Finsbury Park; this was discontinued in October 1908. +In March 1908, the off-peak shuttle service began to use the western platform at Strand and the through platform at Holborn, crossing between the two branch tunnels south of Holborn. Low usage led to the withdrawal of the second peak-hour shuttle and the eastern tunnel was taken out of use in 1914. On 9 May 1915, three of the Underground stations in the area were renamed and Strand station became Aldwych. Sunday services ended in April 1917 and, in August of the same year, the eastern tunnel and platform at Aldwych and the bay platform at Holborn were formally closed. A German bombing campaign in September 1917 led to the disused platform being used as storage for around 300 paintings from the National Gallery from then until December 1918. +In October 1922, the ticket office was replaced by a facility in the lifts. Passenger numbers remained low: when the station was one of a number on the network considered for closure in 1929, its annual usage was 1,069,650 and takings were £4,500. The branch was again considered for closure in 1933, but remained open. +Wartime efficiency measures led to the branch being closed temporarily on 22 September 1940, shortly after the start of The Blitz, and it was partly fitted out by the City of Westminster as an air-raid shelter. The tunnels between Aldwych and Holborn were used to store items from the British Museum, including the Elgin Marbles. The branch reopened on 1 July 1946, but patronage did not increase. In 1958, the station was one of three that London Transport announced would be closed. Again it survived, but the service was reduced in June 1958 to run during Monday to Friday peak hours and Saturday morning and early afternoons. The Saturday service was withdrawn in June 1962. +After operating only during peak hours for more than 30 years, the closure announcement came on 4 January 1993. The original 1907 lifts required replacement at a cost of £3 million. This was not considered justifiable in relation to the passenger numbers using the station and it was losing London Regional Transport £150,000 per year. The Secretary of State for Transport granted permission on 1 September 1994 to close the station, and the branch closed on 30 September. Recognising the station's historical significance as a mostly unaltered station from the early 20th century, the station was given Grade II listed building status in 2011. Office floors above the station are used by the Classics department of King's College London, and part of the station is used by the university's Rifle and Pistol Club. +Although the Piccadilly Circus to City of London branch proposal of 1905 was never revisited after its withdrawal, the early plan to extend the branch south to Waterloo was revived a number of times during the station's life. The extension was considered in 1919 and 1948, but no progress towards constructing the link was made. +In the years after the Second World War, a series of preliminary plans for relieving congestion on the London Underground had considered various east–west routes through the Aldwych area, although other priorities meant that these were never proceeded with. In March 1965, a British Rail and London Transport joint planning committee published "A Railway Plan for London", which proposed a new tube railway, the Fleet line (later renamed the Jubilee line), to join the Bakerloo line at Baker Street then run via Bond Street, Green Park, Charing Cross, Aldwych and into the City of London via Ludgate Circus, Cannon Street and Fenchurch Street before heading into south-east London. An interchange was proposed at Aldwych and a second recommendation of the report was the revival of the link from Aldwych to Waterloo. London Transport had already sought legal powers from Parliament to construct tunnels from Aldwych to Waterloo in November 1964, and these were granted in the London Transport Act 1965 (c. xli) in August 1965. Detailed planning took place, although public spending cuts led to postponement of the scheme in 1967 before tenders were invited. +Planning of the Fleet line continued and approval was given in the London Transport Act 1969 (c. l) in July 1969 for the first phase of the line, from Baker Street to Charing Cross. Tunnelling began on the £35 million route in February 1972 and the Jubilee line opened north from Charing Cross in May 1979. The tunnels of the approved section continued east of Charing Cross under Strand almost as far as Aldwych station, but no work at Aldwych was undertaken and they were used only as sidings. Funding for the second phase of the work was delayed throughout the 1970s whilst the route beyond Charing Cross was reviewed to consider options for serving anticipated development in the London Docklands area. By 1979, the cost was estimated as £325 million, a six-fold increase from the £51 million estimated in 1970. A further review of alternatives for the Jubilee line was carried out in 1980, which led to a change of priorities and the postponement of any further effort on the line. When the extension was eventually constructed in the late 1990s it took a different route, south of the River Thames via Westminster, Waterloo and London Bridge to provide a rapid link to Canary Wharf, leaving the tunnels between Green Park and Aldwych redundant. +Following the station's closure, in 1996 the Civic Trust and other organisations including Transport 2000 proposed to create a heritage railway. The station would be restored to an inter-war appearance, and old Underground rolling stock would be used for journeys on the disused tracks. London Underground rejected the proposal on safety and cost grounds. +In July 2005, Ove Arup & Partners produced a report, DLR Horizon 2020 Study, for the Docklands Light Railway (DLR) examining schemes to expand and improve the DLR network between 2012 and 2020. One of the proposals was an extension of the DLR from Bank to Charing Cross via City Thameslink and Aldwych. The disused Jubilee line tunnels would be enlarged to accommodate the larger DLR trains and Aldwych station would form the basis for a new station on the line, although requiring considerable reconstruction to accommodate escalators. The estimated cost in 2005 was £232 million for the infrastructure works and the scheme was described as "strongly beneficial" as it was expected to attract passengers from the London Underground's existing east–west routes and from local buses and reduce overcrowding at Bank station. The business case assessment was that the proposal offered high value, although similar values were calculated for other extension proposals from Bank. Further detailed studies were proposed. +In 2015, a scheme was proposed by the design firm Gensler to convert disused London Underground tunnels into subterranean rail trails, enabling the disused branches of the Piccadilly line and Jubilee line to be used as cycle paths. The scheme, which would involve re-opening Aldwych station as an access point for cyclists, has not been officially approved. +Because it was a self-contained section of the London Underground that was closed outside weekday peak hours, Aldwych station and the branch line from Holborn were popular locations for filming scenes set on the Tube even before their closure. Since the branch's closure in 1994, its use in film productions has continued, with the station appearing as itself and, with appropriate signage, as other stations on the network. The track and infrastructure are maintained in operational condition, and until 2021 a train of ex-Northern line 1972 tube stock was stabled on the branch. This train could be driven up and down the branch for filming. The physical connection with the Piccadilly line northbound tracks remains, but requires manual operation. +Films and television productions that have been shot at Aldwych include: +The pre-war operation of the station features in a pivotal scene in Geoffrey Household's novel Rogue Male, when the pursuit of the protagonist by an enemy agent sees them repeatedly using the shuttle service on the branch line. A chase through Aldwych station ends with the agent's death by electrocution on the track. The station is the setting for the climax of the Doctor Who novel The Pescatons. A much modified and expanded version of the station appears as a level in the video game Tomb Raider III. The music video for The Prodigy's song "Firestarter" was filmed in the disused eastern tunnel and one of the unused lift shafts. The station was the subject of an episode of Most Haunted in 2002. The station was also the subject of an episode of The Magnus Archives in 2018. +List of former and unopened London Underground stations +"Aldwych Station". Underground History. Archived from the original on 10 December 2005. +"Aldwych". London's Abandoned Tube Stations. Archived from the original on 17 August 2008. +"Aldwych-Holborn Branch". Disused stations. Subterranea Britannica. Archived from the original on 27 January 2003. +London Transport Museum Photographic Archive +Strand station, mid-1907 +Surrey Street elevation, mid-1907 +Surrey Street elevation after vertical extension, 1958 +"An ENSA concert party entertains shelterers in Aldwych station". The Daily Telegraph. 3 May 2014. Retrieved 29 May 2014. +Oliver, Luke (12 June 2010). The Secret Station – Aldwych. YouTube. Archived from the original on 11 December 2021. Retrieved 29 June 2014. +In computer science, binary search, also known as half-interval search, logarithmic search, or binary chop, is a search algorithm that finds the position of a target value within a sorted array. Binary search compares the target value to the middle element of the array. If they are not equal, the half in which the target cannot lie is eliminated and the search continues on the remaining half, again taking the middle element to compare to the target value, and repeating this until the target value is found. If the search ends with the remaining half being empty, the target is not in the array. +Binary search runs in logarithmic time in the worst case, making O(log⁡n) comparisons, where n is the number of elements in the array. Binary search is faster than linear search except for small arrays. However, the array must be sorted first to be able to apply binary search. There are specialized data structures designed for fast searching, such as hash tables, that can be searched more efficiently than binary search. However, binary search can be used to solve a wider range of problems, such as finding the next-smallest or next-largest element in the array relative to the target even if it is absent from the array. +There are numerous variations of binary search. In particular, fractional cascading speeds up binary searches for the same value in multiple arrays. Fractional cascading efficiently solves a number of search problems in computational geometry and in numerous other fields. Exponential search extends binary search to unbounded lists. The binary search tree and B-tree data structures are based on binary search. +Binary search works on sorted arrays. Binary search begins by comparing an element in the middle of the array with the target value. If the target value matches the element, its position in the array is returned. If the target value is less than the element, the search continues in the lower half of the array. If the target value is greater than the element, the search continues in the upper half of the array. By doing this, the algorithm eliminates the half in which the target value cannot lie in each iteration. +Given an array A of n elements with values or records A0,A1,A2,…,An−1sorted such that A0≤A1≤A2≤⋯≤An−1, and target value T, the following subroutine uses binary search to find the index of T in A. +Set L to 0 and R to n−1. +If L>R, the search terminates as unsuccessful. +Set m (the position of the middle element) to L plus the floor of R−L2, which is the greatest integer less than or equal to R−L2. +If AmT, set R to m−1 and go to step 2. +Now Am=T, the search is done; return m. +This iterative procedure keeps track of the search boundaries with the two variables L and R. The procedure may be expressed in pseudocode as follows, where the variable names and types remain the same as above, floor is the floor function, and unsuccessful refers to a specific value that conveys the failure of the search. +function binary_search(A, n, T) is +L := 0 +R := n − 1 +while L ≤ R do +m := L + floor((R - L) / 2) +if A[m] < T then +L := m + 1 +else if A[m] > T then +R := m − 1 +else: +return m +return unsuccessful +Alternatively, the algorithm may take the ceiling of R−L2. This may change the result if the target value appears more than once in the array. +In the above procedure, the algorithm checks whether the middle element (m) is equal to the target (T) in every iteration. Some implementations leave out this check during each iteration. The algorithm would perform this check only when one element is left (when L=R). This results in a faster comparison loop, as one comparison is eliminated per iteration, while it requires only one more iteration on average. +Hermann Bottenbruch published the first implementation to leave out this check in 1962. +Set L to 0 and R to n−1. +While L≠R, +Set m (the position of the middle element) to L plus the ceiling of R−L2, which is the least integer greater than or equal to R−L2. +If Am>T, set R to m−1. +Else, Am≤T; set L to m. +Now L=R, the search is done. If AL=T, return L. Otherwise, the search terminates as unsuccessful. +Where ceil is the ceiling function, the pseudocode for this version is: +function binary_search_alternative(A, n, T) is +L := 0 +R := n − 1 +while L != R do +m := L + ceil((R - L) / 2) +if A[m] > T then +R := m − 1 +else: +L := m +if A[L] = T then +return L +return unsuccessful +The procedure may return any index whose element is equal to the target value, even if there are duplicate elements in the array. For example, if the array to be searched was [1,2,3,4,4,5,6,7] and the target was 4, then it would be correct for the algorithm to either return the 4th (index 3) or 5th (index 4) element. The regular procedure would return the 4th element (index 3) in this case. It does not always return the first duplicate (consider [1,2,4,4,4,5,6,7] which still returns the 4th element). However, it is sometimes necessary to find the leftmost element or the rightmost element for a target value that is duplicated in the array. In the above example, the 4th element is the leftmost element of the value 4, while the 5th element is the rightmost element of the value 4. The alternative procedure above will always return the index of the rightmost element if such an element exists. +To find the leftmost element, the following procedure can be used: +Set L to 0 and R to n. +While LT, set R to m. +Else, Am≤T; set L to m+1. +Return R−1. +If R>0 and AR−1=T, then AR−1 is the rightmost element that equals T. Even if T is not in the array, n−R is the number of elements in the array that are greater than T. +Where floor is the floor function, the pseudocode for this version is: +function binary_search_rightmost(A, n, T): +L := 0 +R := n +while L < R: +m := L + floor((R - L) / 2) +if A[m] > T: +R := m +else: +L := m + 1 +return R - 1 +The above procedure only performs exact matches, finding the position of a target value. However, it is trivial to extend binary search to perform approximate matches because binary search operates on sorted arrays. For example, binary search can be used to compute, for a given value, its rank (the number of smaller elements), predecessor (next-smallest element), successor (next-largest element), and nearest neighbor. Range queries seeking the number of elements between two values can be performed with two rank queries. +Rank queries can be performed with the procedure for finding the leftmost element. The number of elements less than the target value is returned by the procedure. +Predecessor queries can be performed with rank queries. If the rank of the target value is r, its predecessor is r−1. +For successor queries, the procedure for finding the rightmost element can be used. If the result of running the procedure for the target value is r, then the successor of the target value is r+1. +The nearest neighbor of the target value is either its predecessor or successor, whichever is closer. +Range queries are also straightforward. Once the ranks of the two values are known, the number of elements greater than or equal to the first value and less than the second is the difference of the two ranks. This count can be adjusted up or down by one according to whether the endpoints of the range should be considered to be part of the range and whether the array contains entries matching those endpoints. +In terms of the number of comparisons, the performance of binary search can be analyzed by viewing the run of the procedure on a binary tree. The root node of the tree is the middle element of the array. The middle element of the lower half is the left child node of the root, and the middle element of the upper half is the right child node of the root. The rest of the tree is built in a similar fashion. Starting from the root node, the left or right subtrees are traversed depending on whether the target value is less or more than the node under consideration. +In the worst case, binary search makes ⌊log2(n)+1⌋ iterations of the comparison loop, where the ⌊⋅⌋ notation denotes the floor function that yields the greatest integer less than or equal to the argument, and log2 is the binary logarithm. This is because the worst case is reached when the search reaches the deepest level of the tree, and there are always ⌊log2(n)+1⌋ levels in the tree for any binary search. +The worst case may also be reached when the target element is not in the array. If n is one less than a power of two, then this is always the case. Otherwise, the search may perform ⌊log2(n)+1⌋iterations if the search reaches the deepest level of the tree. However, it may make ⌊log2(n)⌋ iterations, which is one less than the worst case, if the search ends at the second-deepest level of the tree. +On average, assuming that each element is equally likely to be searched, binary search makes ⌊log2(n)⌋+1−(2⌊log2(n)⌋+1−⌊log2(n)⌋−2)/n iterations when the target element is in the array. This is approximately equal to log2(n)−1 iterations. When the target element is not in the array, binary search makes ⌊log2(n)⌋+2−2⌊log2(n)⌋+1/(n+1) iterations on average, assuming that the range between and outside elements is equally likely to be searched. +In the best case, where the target value is the middle element of the array, its position is returned after one iteration. +In terms of iterations, no search algorithm that works only by comparing elements can exhibit better average and worst-case performance than binary search. The comparison tree representing binary search has the fewest levels possible as every level above the lowest level of the tree is filled completely. Otherwise, the search algorithm can eliminate few elements in an iteration, increasing the number of iterations required in the average and worst case. This is the case for other search algorithms based on comparisons, as while they may work faster on some target values, the average performance over all elements is worse than binary search. By dividing the array in half, binary search ensures that the size of both subarrays are as similar as possible. +Binary search requires three pointers to elements, which may be array indices or pointers to memory locations, regardless of the size of the array. Therefore, the space complexity of binary search is O(1) in the word RAM model of computation. +The average number of iterations performed by binary search depends on the probability of each element being searched. The average case is different for successful searches and unsuccessful searches. It will be assumed that each element is equally likely to be searched for successful searches. For unsuccessful searches, it will be assumed that the intervals between and outside elements are equally likely to be searched. The average case for successful searches is the number of iterations required to search every element exactly once, divided by n, the number of elements. The average case for unsuccessful searches is the number of iterations required to search an element within every interval exactly once, divided by the n+1 intervals. +In the binary tree representation, a successful search can be represented by a path from the root to the target node, called an internal path. The length of a path is the number of edges (connections between nodes) that the path passes through. The number of iterations performed by a search, given that the corresponding path has length l, is l+1 counting the initial iteration. The internal path length is the sum of the lengths of all unique internal paths. Since there is only one path from the root to any single node, each internal path represents a search for a specific element. If there are n elements, which is a positive integer, and the internal path length is I(n), then the average number of iterations for a successful search T(n)=1+I(n)n, with the one iteration added to count the initial iteration. +Since binary search is the optimal algorithm for searching with comparisons, this problem is reduced to calculating the minimum internal path length of all binary trees with n nodes, which is equal to: +I(n)=∑k=1n⌊log2(k)⌋ +For example, in a 7-element array, the root requires one iteration, the two elements below the root require two iterations, and the four elements below require three iterations. In this case, the internal path length is: +∑k=17⌊log2(k)⌋=0+2(1)+4(2)=2+8=10 +The average number of iterations would be 1+107=237 based on the equation for the average case. The sum for I(n) can be simplified to: +I(n)=∑k=1n⌊log2(k)⌋=(n+1)⌊log2(n+1)⌋−2⌊log2(n+1)⌋+1+2 +Substituting the equation for I(n) into the equation for T(n): +T(n)=1+(n+1)⌊log2(n+1)⌋−2⌊log2(n+1)⌋+1+2n=⌊log2(n)⌋+1−(2⌊log2(n)⌋+1−⌊log2(n)⌋−2)/n +For integer n, this is equivalent to the equation for the average case on a successful search specified above. +Unsuccessful searches can be represented by augmenting the tree with external nodes, which forms an extended binary tree. If an internal node, or a node present in the tree, has fewer than two child nodes, then additional child nodes, called external nodes, are added so that each internal node has two children. By doing so, an unsuccessful search can be represented as a path to an external node, whose parent is the single element that remains during the last iteration. An external path is a path from the root to an external node. The external path length is the sum of the lengths of all unique external paths. If there are n elements, which is a positive integer, and the external path length is E(n), then the average number of iterations for an unsuccessful search is T′(n)=E(n)n+1, with the one iteration added to count the initial iteration. The external path length is divided by n+1 instead of n because there are n+1 external paths, representing the intervals between and outside the elements of the array. +This problem can similarly be reduced to determining the minimum external path length of all binary trees with n nodes. For all binary trees, the external path length is equal to the internal path length plus 2n. Substituting the equation for I(n): +E(n)=I(n)+2n=[(n+1)⌊log2(n+1)⌋−2⌊log2(n+1)⌋+1+2]+2n=(n+1)(⌊log2(n)⌋+2)−2⌊log2(n)⌋+1 +Substituting the equation for E(n) into the equation for T′(n), the average case for unsuccessful searches can be determined: +T′(n)=(n+1)(⌊log2(n)⌋+2)−2⌊log2(n)⌋+1(n+1)=⌊log2(n)⌋+2−2⌊log2(n)⌋+1/(n+1) +Each iteration of the binary search procedure defined above makes one or two comparisons, checking if the middle element is equal to the target in each iteration. Assuming that each element is equally likely to be searched, each iteration makes 1.5 comparisons on average. A variation of the algorithm checks whether the middle element is equal to the target at the end of the search. On average, this eliminates half a comparison from each iteration. This slightly cuts the time taken per iteration on most computers. However, it guarantees that the search takes the maximum number of iterations, on average adding one iteration to the search. Because the comparison loop is performed only ⌊log2(n)+1⌋ times in the worst case, the slight increase in efficiency per iteration does not compensate for the extra iteration for all but very large n. +In analyzing the performance of binary search, another consideration is the time required to compare two elements. For integers and strings, the time required increases linearly as the encoding length (usually the number of bits) of the elements increase. For example, comparing a pair of 64-bit unsigned integers would require comparing up to double the bits as comparing a pair of 32-bit unsigned integers. The worst case is achieved when the integers are equal. This can be significant when the encoding lengths of the elements are large, such as with large integer types or long strings, which makes comparing elements expensive. Furthermore, comparing floating-point values (the most common digital representation of real numbers) is often more expensive than comparing integers or short strings. +Fast floating point comparison is possible via comparing as an integer. However, this kind of comparison forms a total order, which makes every floating-point value compare differently from each other and the same as itself. This is different from the typical comparison where -0.0 should be the same as 0.0 and NaN should not compare the same as any other value including itself. +According to Steel Bank Common Lisp contributor Paul Khuong, binary search leads to very few branch mispredictions despite its data-dependent nature. This is in part because most of it can be expressed as conditional moves instead of branches. The same applies to most logarithmic divide-and-conquer search algorithms. +On most computer architectures, the processor has a hardware cache separate from RAM. Since they are located within the processor itself, caches are much faster to access but usually store much less data than RAM. Therefore, most processors store memory locations that have been accessed recently, along with memory locations close to it. For example, when an array element is accessed, the element itself may be stored along with the elements that are stored close to it in RAM, making it faster to sequentially access array elements that are close in index to each other (locality of reference). On a sorted array, binary search can jump to distant memory locations if the array is large, unlike algorithms (such as linear search and linear probing in hash tables) which access elements in sequence. This adds slightly to the running time of binary search for large arrays on most systems. +Paul Khuong has noted that binary search on large (≥ 512 KiB) arrays of exactly a power-of-two size tends to cause an additional problem with how CPU caches are implemented. Specifically, the translation lookaside buffer (TLB) is often implemented as a content-addressable memory (CAM), with the "key" usually being the lower bits of a requested address. When searching on an array of exactly a power-of-two size, memory address with the same lower bits tend to be accessed, causing collisions ("aliasing") with the "key" used to fetch the CAM. The typical TLB is 4-way associative, meaning it can handle at most four addresses hitting the same "key", after which TLB thrashing happens. (Although the other levels of CPU caches also use a similar setup, they manage smaller areas with a higher way count, usually 8 or 16, so they are less affected.) This can be prevented by offsetting the split point of the binary search so it divides at 31⁄64 instead of exactly the middle. +Sorted arrays with binary search are a very inefficient solution when insertion and deletion operations are interleaved with retrieval, taking O(n) time for each such operation. In addition, sorted arrays can complicate memory use especially when elements are often inserted into the array. There are other data structures that support much more efficient insertion and deletion. Binary search can be used to perform exact matching and set membership (determining whether a target value is in a collection of values). There are data structures that support faster exact matching and set membership. However, unlike many other searching schemes, binary search can be used for efficient approximate matching, usually performing such matches in O(log⁡n) time regardless of the type or structure of the values themselves. In addition, there are some operations, like finding the smallest and largest element, that can be performed efficiently on a sorted array. +Linear search is a simple search algorithm that checks every record until it finds the target value. Linear search can be done on a linked list, which allows for faster insertion and deletion than an array. Binary search is faster than linear search for sorted arrays except if the array is short, although the array needs to be sorted beforehand. All sorting algorithms based on comparing elements, such as quicksort and merge sort, require at least O(nlog⁡n) comparisons in the worst case. Unlike linear search, binary search can be used for efficient approximate matching. There are operations such as finding the smallest and largest element that can be done efficiently on a sorted array but not on an unsorted array. +A binary search tree is a binary tree data structure that works based on the principle of binary search. The records of the tree are arranged in sorted order, and each record in the tree can be searched using an algorithm similar to binary search, taking on average logarithmic time. Insertion and deletion also require on average logarithmic time in binary search trees. This can be faster than the linear time insertion and deletion of sorted arrays, and binary trees retain the ability to perform all the operations possible on a sorted array, including range and approximate queries. +However, binary search is usually more efficient for searching as binary search trees will most likely be imperfectly balanced, resulting in slightly worse performance than binary search. This even applies to balanced binary search trees, binary search trees that balance their own nodes, because they rarely produce the tree with the fewest possible levels. Except for balanced binary search trees, the tree may be severely imbalanced with few internal nodes with two children, resulting in the average and worst-case search time approaching n comparisons. Binary search trees take more space than sorted arrays. +Binary search trees lend themselves to fast searching in external memory stored in hard disks, as binary search trees can be efficiently structured in filesystems. The B-tree generalizes this method of tree organization. B-trees are frequently used to organize long-term storage such as databases and filesystems. +For implementing associative arrays, hash tables, a data structure that maps keys to records using a hash function, are generally faster than binary search on a sorted array of records. Most hash table implementations require only amortized constant time on average. However, hashing is not useful for approximate matches, such as computing the next-smallest, next-largest, and nearest key, as the only information given on a failed search is that the target is not present in any record. Binary search is ideal for such matches, performing them in logarithmic time. Binary search also supports approximate matches. Some operations, like finding the smallest and largest element, can be done efficiently on sorted arrays but not on hash tables. +A related problem to search is set membership. Any algorithm that does lookup, like binary search, can also be used for set membership. There are other algorithms that are more specifically suited for set membership. A bit array is the simplest, useful when the range of keys is limited. It compactly stores a collection of bits, with each bit representing a single key within the range of keys. Bit arrays are very fast, requiring only O(1) time. The Judy1 type of Judy array handles 64-bit keys efficiently. +For approximate results, Bloom filters, another probabilistic data structure based on hashing, store a set of keys by encoding the keys using a bit array and multiple hash functions. Bloom filters are much more space-efficient than bit arrays in most cases and not much slower: with k hash functions, membership queries require only O(k) time. However, Bloom filters suffer from false positives. +There exist data structures that may improve on binary search in some cases for both searching and other operations available for sorted arrays. For example, searches, approximate matches, and the operations available to sorted arrays can be performed more efficiently than binary search on specialized data structures such as van Emde Boas trees, fusion trees, tries, and bit arrays. These specialized data structures are usually only faster because they take advantage of the properties of keys with a certain attribute (usually keys that are small integers), and thus will be time or space consuming for keys that lack that attribute. As long as the keys can be ordered, these operations can always be done at least efficiently on a sorted array regardless of the keys. Some structures, such as Judy arrays, use a combination of approaches to mitigate this while retaining efficiency and the ability to perform approximate matching. +Uniform binary search stores, instead of the lower and upper bounds, the difference in the index of the middle element from the current iteration to the next iteration. A lookup table containing the differences is computed beforehand. For example, if the array to be searched is [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], the middle element (m) would be 6. In this case, the middle element of the left subarray ([1, 2, 3, 4, 5]) is 3 and the middle element of the right subarray ([7, 8, 9, 10, 11]) is 9. Uniform binary search would store the value of 3 as both indices differ from 6 by this same amount. To reduce the search space, the algorithm either adds or subtracts this change from the index of the middle element. Uniform binary search may be faster on systems where it is inefficient to calculate the midpoint, such as on decimal computers. +Exponential search extends binary search to unbounded lists. It starts by finding the first element with an index that is both a power of two and greater than the target value. Afterwards, it sets that index as the upper bound, and switches to binary search. A search takes ⌊log2x+1⌋ iterations before binary search is started and at most ⌊log2x⌋ iterations of the binary search, where x is the position of the target value. Exponential search works on bounded lists, but becomes an improvement over binary search only if the target value lies near the beginning of the array. +Instead of calculating the midpoint, interpolation search estimates the position of the target value, taking into account the lowest and highest elements in the array as well as length of the array. It works on the basis that the midpoint is not the best guess in many cases. For example, if the target value is close to the highest element in the array, it is likely to be located near the end of the array. +A common interpolation function is linear interpolation. If A is the array, L,R are the lower and upper bounds respectively, and T is the target, then the target is estimated to be about (T−AL)/(AR−AL) of the way between L and R. When linear interpolation is used, and the distribution of the array elements is uniform or near uniform, interpolation search makes O(log⁡log⁡n) comparisons. +In practice, interpolation search is slower than binary search for small arrays, as interpolation search requires extra computation. Its time complexity grows more slowly than binary search, but this only compensates for the extra computation for large arrays. +Fractional cascading is a technique that speeds up binary searches for the same element in multiple sorted arrays. Searching each array separately requires O(klog⁡n) time, where k is the number of arrays. Fractional cascading reduces this to O(k+log⁡n) by storing specific information in each array about each element and its position in the other arrays. +Fractional cascading was originally developed to efficiently solve various computational geometry problems. Fractional cascading has been applied elsewhere, such as in data mining and Internet Protocol routing. +Binary search has been generalized to work on certain types of graphs, where the target value is stored in a vertex instead of an array element. Binary search trees are one such generalization—when a vertex (node) in the tree is queried, the algorithm either learns that the vertex is the target, or otherwise which subtree the target would be located in. However, this can be further generalized as follows: given an undirected, positively weighted graph and a target vertex, the algorithm learns upon querying a vertex that it is equal to the target, or it is given an incident edge that is on the shortest path from the queried vertex to the target. The standard binary search algorithm is simply the case where the graph is a path. Similarly, binary search trees are the case where the edges to the left or right subtrees are given when the queried vertex is unequal to the target. For all undirected, positively weighted graphs, there is an algorithm that finds the target vertex in O(log⁡n) queries in the worst case. +Noisy binary search algorithms solve the case where the algorithm cannot reliably compare elements of the array. For each pair of elements, there is a certain probability that the algorithm makes the wrong comparison. Noisy binary search can find the correct position of the target with a given probability that controls the reliability of the yielded position. Every noisy binary search procedure must make at least (1−τ)log2(n)H(p)−10H(p) comparisons on average, where H(p)=−plog2(p)−(1−p)log2(1−p) is the binary entropy function and τ is the probability that the procedure yields the wrong position. The noisy binary search problem can be considered as a case of the Rényi-Ulam game, a variant of Twenty Questions where the answers may be wrong. +Classical computers are bounded to the worst case of exactly ⌊log2n+1⌋ iterations when performing binary search. Quantum algorithms for binary search are still bounded to a proportion of log2n queries (representing iterations of the classical procedure), but the constant factor is less than one, providing for a lower time complexity on quantum computers. Any exact quantum binary search procedure—that is, a procedure that always yields the correct result—requires at least 1π(ln⁡n−1)≈0.22log2n queries in the worst case, where ln is the natural logarithm. There is an exact quantum binary search procedure that runs in 4log605n≈0.433log2n queries in the worst case. In comparison, Grover's algorithm is the optimal quantum algorithm for searching an unordered list of elements, and it requires O(n) queries. +Classical binary search implicitly treats the target as equally likely to lie anywhere in the search space, and so splits each interval at its midpoint, requiring ⌊log2n+1⌋ iterations in the worst case. Bayesian binary search (BBS) instead learns a probability density over the search space and splits at the median of that density, bisecting probability mass rather than the interval itself. Viewed through Bayesian inference, classical binary search is the special case in which the prior is uniform and each comparison is a deterministic likelihood that assigns zero probability to half the remaining interval. BBS generalizes this by replacing the uniform prior with a learned prior Plearned(x)—estimated by supervised methods such as Gaussian process regression, Bayesian neural networks or quantile regression, or by unsupervised methods such as Gaussian mixture models, kernel density estimation and maximum likelihood estimation—and probing at the point pk satisfying ∫akpkP(x∣𝒟k)dx=12. The posterior update after each comparison is equivalent to bisection in probability density space. +The idea of sorting a list of items to allow for faster searching dates back to antiquity. The earliest known example was the Inakibit-Anu tablet from Babylon dating back to c. 200 BCE. The tablet contained about 500 sexagesimal numbers and their reciprocals sorted in lexicographical order, which made searching for a specific entry easier. In addition, several lists of names that were sorted by their first letter were discovered on the Aegean Islands. Catholicon, a Latin dictionary finished in 1286 CE, was the first work to describe rules for sorting words into alphabetical order, as opposed to just the first few letters. +In 1946, John Mauchly made the first mention of binary search as part of the Moore School Lectures, a seminal and foundational college course in computing. In 1957, William Wesley Peterson published the first method for interpolation search. Every published binary search algorithm worked only for arrays whose length is one less than a power of two until 1960, when Derrick Henry Lehmer published a binary search algorithm that worked on all arrays. In 1962, Hermann Bottenbruch presented an ALGOL 60 implementation of binary search that placed the comparison for equality at the end, increasing the average number of iterations by one, but reducing to one the number of comparisons per iteration. The uniform binary search was developed by A. K. Chandra of Stanford University in 1971. In 1986, Bernard Chazelle and Leonidas J. Guibas introduced fractional cascading as a method to solve numerous search problems in computational geometry. +Although the basic idea of binary search is comparatively straightforward, the details can be surprisingly tricky +When Jon Bentley assigned binary search as a problem in a course for professional programmers, he found that ninety percent failed to provide a correct solution after several hours of working on it, mainly because the incorrect implementations failed to run or returned a wrong answer in rare edge cases. A study published in 1988 shows that accurate code for it is only found in five out of twenty textbooks. Furthermore, Bentley's own implementation of binary search, published in his 1986 book Programming Pearls, contained an overflow error that remained undetected for over twenty years. The Java programming language library implementation of binary search had the same overflow bug for more than nine years. +In a practical implementation, the variables used to represent the indices will often be of fixed size (integers), and this can result in an arithmetic overflow for very large arrays. If the midpoint of the span is calculated as L+R2, then the value of L+R may exceed the range of integers of the data type used to store the midpoint, even if L and R are within the range. If L and R are nonnegative, this can be avoided by calculating the midpoint as L+R−L2. +An infinite loop may occur if the exit conditions for the loop are not defined correctly. Once L exceeds R, the search has failed and must convey the failure of the search. In addition, the loop must be exited when the target element is found, or in the case of an implementation where this check is moved to the end, checks for whether the search was successful or failed at the end must be in place. Bentley found that most of the programmers who incorrectly implemented binary search made an error in defining the exit conditions. +Many languages' standard libraries include binary search routines: +C provides the function bsearch() in its standard library, which is typically implemented via binary search, although the official standard does not require it to do so. +C++'s standard library provides the functions binary_search(), lower_bound(), upper_bound() and equal_range(). Using the C++20 std::ranges library, it can be applied over a range as std::ranges::binary_search(). +D's standard library Phobos, in std.range module provides a type SortedRange (returned by sort() and assumeSorted() functions) with methods contains(), equalRange(), lowerBound() and trisect(), that use binary search techniques by default for ranges that offer random access. +COBOL provides the SEARCH ALL verb for performing binary searches on COBOL ordered tables. +Go's sort standard library package contains the functions Search, SearchInts, SearchFloat64s, and SearchStrings, which implement general binary search, as well as specific implementations for searching slices of integers, floating-point numbers, and strings, respectively. +Java offers a set of overloaded binarySearch() static methods in the classes Arrays and Collections in the standard java.util package for performing binary searches on Java arrays and on Lists, respectively. +Microsoft's .NET Framework 2.0 offers static generic versions of the binary search algorithm in its collection base classes. An example would be System.Array's method BinarySearch(T[] array, T value). +For Objective-C, the Cocoa framework provides the NSArray -indexOfObject:inSortedRange:options:usingComparator: method in Mac OS X 10.6+. Apple's Core Foundation C framework also contains a CFArrayBSearchValues() function. +Python provides the bisect module that keeps a list in sorted order without having to sort the list after each insertion. +Ruby's Array class includes a bsearch method with built-in approximate matching. +Rust's slice primitive provides binary_search(), binary_search_by(), binary_search_by_key(), and partition_point(). +Bisection method – Algorithm for finding a zero of a function – the same idea used to solve equations in the real numbers +Multiplicative binary search – Binary search variation with simplified midpoint calculation +This article was submitted to WikiJournal of Science for external academic peer review in 2018 (reviewer reports). The updated content was reintegrated into the Wikipedia page under a CC-BY-SA-3.0 license (2019). The version of record as reviewed is: Lin, Anthony; et al. (2019). "Binary search algorithm" (PDF). WikiJournal of Science. 2 (1): 5. doi:10.15347/wjs/2019.005. Wikidata Q81434400. +NIST Dictionary of Algorithms and Data Structures: binary search +Comparisons and benchmarks of a variety of binary search implementations in C Archived 25 September 2019 at the Wayback Machine +Hermann Philipp Detzner (16 October 1882 – 1 December 1970) was a German engineer and surveyor, who served as an officer in the German colonial security force (Schutztruppe) in Kamerun (Cameroon) and German New Guinea. He gained fame for evading capture after Australian troops invaded German New Guinea at the start of World War I. +In early 1914, the German government sent Detzner to explore and chart central Kaiser-Wilhelmsland, the imperial protectorate on the island of New Guinea. When World War I broke out in Europe, he was far from civilisation and without radio contact. He refused to surrender to Australian troops when they occupied German New Guinea, concealing himself in the jungle with a band of approximately 20 soldiers. For four years, Detzner and his troops provocatively marched through the bush, singing "Watch on the Rhine" and flying the German Imperial flag. He led at least one expedition from the Huon Peninsula to the north coast, and a second by a mountain route, to attempt an escape to the neutral Dutch colony to the west. He explored areas of the New Guinea's hinterland formerly unseen by Europeans. +After finding out that the war had ended, Detzner surrendered in full dress uniform, flying the Imperial flag, to Australian forces in January 1919. He received a hero's welcome when he returned to Germany. He wrote a book about his adventures – Four Years Among the Cannibals in the Interior of German New Guinea under the Imperial Flag, from 1914 until the Armistice – that sold well in Great Britain and Germany, entered three printings, and was translated into French, English, Finnish and Swedish. He received a position in the Imperial Colonial Archives, and appeared frequently on the lecture circuit throughout the 1920s. In the late 1920s, scientific portions of his book were discredited. In 1932, he admitted that he had mixed fact and fiction and, after that time, eschewed public life. +Detzner was the son of a dentist, Johann Philipp Detzner and his wife, Wilhelmine Katharina Faber, in Speyer, in the Bavarian Palatinate, a cultural, economic, and historical city on the Rhine River. His father received his degree from Heidelberg University and was licensed to practise by the Kingdom of Bavaria in 1867; Detzner's father pioneered innovations in dental prosthetics. His large family included nine children. Hermann Detzner was trained as a topographer, surveyor, and an engineer, and received his promotion to Fahnrich in the 6 Infantry Regiment (Prussian), 2nd Pioneer Battalion, in February 1902. During World War I, military authorities transferred his commission to the 1st Bavarian Pioneer Battalion. +Hermann Detzner participated in a joint British-German scientific and surveying expedition to Kamerun in 1908 and 1909 and again in 1912–1913. He and one Captain Nugent, Royal Artillery, identified and marked the frontiers of Kamerun and explored the Niger valley. Detzner later published a paper on the marking of the boundary. +Navigators charted the coastline of the northern and eastern portions of New Guinea in the early 17th century and, later in the century, British Admiralty navigators named the visible mountain ranges. Most German surveying efforts had focused on coastal regions and river basins, where Germans had established plantations, leaving the hinterland unexplored. In late 1913, the Imperial Colonial office appointed Detzner to lead an expedition to survey the border between the British protectorate, called Papua, and the German territory, called Kaiser-Wilhelmsland, and to survey and map unexplored inland regions. +Detzner's mission was also to be the first serious attempt to explore the hinterland and to evaluate and describe its resources. The boundary between Papua and Kaiser Wilhelmsland had been broadly established by a joint British-German expedition in 1909, but the terrain had not been mapped and the German colonial administration maintained that the boundary was imprecise. From the German perspective, an accurately defined boundary was essential to monitor and control the activities of Papuan gold prospectors near the border with German territory. Detzner had had experience in joint operations in Kamerun in 1907–08 and could be expected to understand the challenges faced by the previous commission; he had a reputation as a methodical and precise engineer. Although small, he was tough and wiry, extremely focused and determined, and seemed like the right man for the job. +In January 1914, Detzner travelled to Rabaul on New Pomerania (now New Britain). In February, he began his expedition into Kaiser-Wilhelmsland. His survey immediately revealed inaccuracies in the 1909 joint survey; by March, Detzner had concluded that the border corridor was already showing a discrepancy of more than 650 meters (2,133 ft) from the 8°0'S parallel. The discrepancy increased the further west he travelled, revealing a widening wedge in the boundary as it was agreed upon, and as it was marked. The discrepancy favoured German interests. +He had progressed well inland when, on 4 August 1914, Britain declared war on Germany. As World War I spread to the Pacific, Australian troops invaded German New Guinea, taking the German barracks in Herbertshöhe (Kokopo) and forcing the defending German colonial troops to capitulate on 21 September after their defeat at Bita Paka. At the beginning of October, he was still unaware of the state of war that now existed between his country and the Commonwealth. +The several months following the outbreak of war found Detzner on border survey work with his sergeant, Konradt, 25 police and 45 carriers, two servants and an interpreter in the high country between Mt Chapman, the Ono River, and Mt Lawson, that is to say along the territorial boundary between Australian-governed Papua and German New Guinea. Eventually he sighted people he named Rockpapua or "skirted Papuans' – these were Kamea people perhaps in the area south of Tekadu in the southernmost part of Morobe Province or the northern part of Gulf Province. On 11 November 1914, one of the carriers, left with several others to rest at a temporary camp, arrived bearing a note from Frederick Chisholm, an Australian Patrol Officer, informing him of the state of war between Germany and Great Britain and asking him to surrender at Nepa on the Lakekamu River, five days walk away. +Rather than comply, Detzner led his party on a forced march north to the Markham Valley. His route is uncertain, but his description of a valley with steep grassy ravines entering from east and west may place the latter part of his journey in the Langimar Valley, through which flows a tributary of the Watut River. A clash with local people that he describes, has been identified as having occurred at Rangama among with Middle Watut people. Once on the Watut itself, the party built rafts and floated downstream to the Markham. +His final destination was Sattelberg on the Huon peninsula. His second in command, Sergeant Konradt, who suffered from frequent bouts of malaria, and a German officer, were captured by the Australians by spring 1915. Eventually, Detzner found his way to the vicinity of a Lutheran mission at the Sattelberg, at a foggy, cool area at 800 meters (2,625 ft), above Finschhafen. The Sattelberg mission was one of the Neuendettelsau Mission Society enterprises established by the Old Lutheran missionary, Johann Flierl, in 1885. This station, and additional mission stations in Heldbach, Simbang, Tami Islands, and Simbu, were an important evangelical presence in the Morobe Province. The missionaries had signed oaths of neutrality for the Australians, who allowed them to remain at their Stations and continue their work. +Once Detzner reached the vicinity of the Sattelberg Mission, nearby villagers in the Borrum valley housed him and his remaining men, which had dwindled to about 20 soldiers, plus four European officers and, on his behalf, the villagers sought assistance from the Sattelberg director, Christian Keyser, and another missionary, Otto Thiele. They reluctantly agreed to keep Detzner's presence a secret. Among the villagers, Detzner established a base camp from which he could depart at short notice. The valley was relatively secure for him, and inaccessible for the Australians, but if they ventured too close to his base, Detzner and his men would retreat into the mountainous Saruwaged, or, if necessary, further into the Finisterre mountains. These were rugged and remote locations, accessible to Detzner, who had the help of native guides, but which the Australians, who usually travelled in larger patrols, could not penetrate. +Detzner and his band stayed near the Sattelberg Mission for the remainder of the war. After the war, however, Detzner would claim he had roamed throughout the eastern jungles of the island, eluding Australian patrols and making little effort to hide. He said he flew the Imperial German flag (sewn from dyed loincloths) in villages throughout the bush, and marched his command through the jungle, loudly singing such patriotic German songs as "Watch on the Rhine" (Die Wacht am Rhein) and popular sentimental ballads like Der Lindenbaum. The Australian garrison was probably not aware of Detzner's activities. +Detzner could have made attempts to reach West New Guinea, which was then neutral Dutch New Guinea, but his claims to have been the first outsider to enter the Papua New Guinea highlands can be discounted. In 1915, and again in 1917, Detzner and some of his men tried to escape along the coast in two canoes. In 1917, they reached the vicinity of Friedrich-Wilhelmshafen, which today is Madang. There lay anchored the Australian ship, HMAS Una, which earlier had been the German imperial yacht, the SMS Komet, designated for use by the German governor of the colony. The ship blocked any further travel, and ended any notions they had of a water escape to Dutch New Guinea. On this escape attempt, Detzner also learned the Australians had orders to shoot him on sight. He made one further attempt to escape overland to Dutch New Guinea, but had to be carried back suffering from an internal hemorrhage. He spent the remainder of the time investigating the island's inhabitants and its flora and fauna, particularly in the Huon Peninsula and Huon Gulf. +On 11 November 1918, Detzner received the news of the official end of the war with the German defeat from a worker at the Sattelberg Mission Station, he wrote a letter to the Australian commander in Morobe in which he offered his capitulation. On 5 January 1919, he surrendered at the Finschhafen District headquarters, marching with his remaining German troops in a column, and wearing his carefully preserved full-dress uniform. He was brought to Rabaul, the Australian headquarters, and on 8 February 1919, was transferred to Sydney aboard the Melusia; after a brief internment in the prisoner of war camp at Holsworthy, he was repatriated to Germany. +On his arrival home, Detzner received a hero's welcome. The press likened him to the successful commander of German East Africa, Major General Paul von Lettow-Vorbeck, who tied down British forces in Africa for the duration of the war. Detzner had been promoted to the rank of captain during the war; upon his return, he was promoted to major. In that year, he wrote Kaiser-Wilhelmsland, nach dem Stande der Forschung im Jahre 1919, (Kaiser-Wilhelmsland, According to the State of Research in the Year 1919), which was widely read in scientific circles. +The Geographical Society of Berlin (Gesellschaft für Erdkunde zu Berlin) honored him 1921 with the Nachtigal Medal "in Eisen" (iron), named after the German explorer Gustav Nachtigal, not in Gold, as sometimes reported. The Geographic Society of Leipzig (Gesellschaft für Erdkunde zu Leipzig) awarded him their Eduard-Vogel-Medaille in gold (Eduard Vogel) in the same year. The University of Köln granted him an honorary degree (Dr. phil. h. c.) on 31 July 1921, and the military awarded him the Iron Cross (2nd and 1st Class) together on 31 August 1919. He received a position in the colonial administration (Reichskolonialamt) in Berlin and in the imperial archive (Reichsarchiv) in Potsdam. +To satisfy the public curiosity about his adventures, Detzner wrote Four years among cannibals : New Guinea. The book brought him fame in Germany and Britain, and he became a sought-after speaker on the lecture circuit. In the 1920s, in addition to several articles and two maps of New Guinea, Detzner published a memoir of his adventures in the Niger valley—In the land of the Dju-Dju: travel experiences in the eastern watershed of the Niger—in 1923, but it did not achieve the popularity of his previous work. +Detzner's book was wildly popular among the general population for its incredible tales of stubborn patriotism and its narratives describing the exotic locales of the lost imperial colonies. His descriptions touched a chord in the German imagination: one of their own had explored the colony, walked its paths, seen its mountains and valleys, and met its people. His vivid descriptions brought to life the images Germans had seen on postcards (such as the one to right), newspapers, and in school books. Furthermore, he had defended Germany's "place in the sun" when others had failed to do so. His book was translated into English, Finnish, Swedish and eventually French. (See below.) +In a speech at the Berlin Geographical Society in 1919, Detzner claimed that the natives of New Guinea had opposed Australian domination and resisted a military recruitment that amounted to slavery, that even the English plantation holders wished to remain independent of Australia, and that the natives were collecting money to build a war memorial for the Germans. The report on Detzner's speech, transmitted from a news agency in London, caused a small flutter in Australian government circles, but generally was dismissed; an earlier report by the Australian judiciary had absolved the Australian force of improper recruiting or treatment of the New Guineans. An angry letter to the editor from another Australian source, who claimed to have been in service in Morobe from 1914 to 1915, received little attention. +In the scientific world, several of his descriptive passages generated excitement and curiosity. In early 1914, he had been surveying a portion of the international boundary in the Upper Waria River between the German and British protectorates. By late September, he had passed into a different portion of the highlands, where the clay-slate mountains changed into limestone highlands. In this geologic transition, he said, he also found a change in the ethnographic character of the population, whom he described as a "new" people. They were stocky, powerfully built, and long–limbed; they wore their hair in knots on the centre of their heads, which were otherwise shaven, and painted yellow and black lines across their chests. They also wore grass skirts, so he called them the skirted ones. They used bows and arrows, slings and stone axes. As he pushed west to Mount Joseph, Detzner claimed, he had found the southern hills of the central watershed cut by numerous rivers flowing north to south. He had surmised that there were no insurmountable obstacles between him and the Sepik river. In 1917, he had travelled through the Ramu valley into the Bismarck range, northwest of the Kratke Mountains, and had continued on that route for 100 kilometres (62 mi). He also described the presence of an indigenous variation of German, called Unserdeutsch (our German), in several New Guinean dialects. +In 1919, after an account of Detzner's speech in Berlin to the Geographical Society was published in Australia, an angry Australian wrote anonymously to the editor of The Argus, a Melbourne newspaper, and described what he claimed were Detzner's lies. "There was no mystery about the disappearance of Captain Detzner and his party", he claimed. The writer attributed Detzner's success at staying ahead of the Australians to the perfidy of the German missionaries, who had agreed to remain neutral and in return for such agreement were allowed to continue their mission work. Detzner was a civilian [emphasis in the original] surveyor, the writer claimed, not a soldier and he survived on mission station rations supplied by public subscription from the German plantation owners. Furthermore, this writer asserted, Detzner's movements were so well known to the district officer at Morobe that he was prevented from escaping; they could have shot him several times, but did not. The writer dismissed Detzner's claims about Australian recruitment of the natives as "in keeping with his dozens of other lying statements in all cases endeavoring to belittle Englishmen or British officers, in every case pure fabrications and typical scurrilous Hun lies". +While such criticism of Detzner's adventures might have been dismissed as post-bellum bellicosity, in 1929, Detzner's assertions came under more serious assault. Two of the German missionaries in the Finschhafen District, Christian Keyser (also spelled Kayser or Keysser) and Otto Thiele, claimed Detzner had not spent the war roaming the jungle, one step ahead of the Australians, but had been under the Mission's protection the entire time. Keyser's additional accusations were particularly specific: Detzner had appropriated his own scientific observations. Keyser's claims carried some weight. He had published a dictionary of the Kâte language, and was a reliable expert on New Guinean dialects, and the German-based creole languages that had arisen in New Guinea; he was also a bona fide explorer and adventurer, having lived from 1899 to 1920 among the mountain peoples of the island. In 1913, Keyser climbed the 4,121-meter (13,520 ft) Saruwaged Massif; over the course of his 21 years in New Guinea, he had identified hundreds of new plant and animal species, and had maintained a regular correspondence with the German Geographical Society in Berlin. Ernst Mayr, a rising star in ornithology, had heard about Detzner from Australians on a research trip to New Guinea. In Germany, during a meeting with Keyser, they discussed Detzner's claims, and Mayr lost no time in broadcasting the discrepancy to his scientific contacts in Europe and the United States. +More problematically, Detzner had no documentation of his findings. As he explained in his many speeches, although he had kept notebooks with drawings of plants, animals, maps, and people, and journals recounting his day-to-day experiences, some of his notebooks and journals had been destroyed by the Australians as they over-ran his hiding places; others, which he had buried to keep them from being destroyed, had rotted beyond repair in the jungle humidity. He implied that what remained of his notes had been confiscated when he surrendered. Detzner's narrative also was rife with contradictions and omissions: Detzner named few villages or streams and stated that the valleys he discovered were thinly populated, whereas they actually contained large populations, at least by New Guinea standards. He also stated that the highest point in the range was 3,600 meters (11,800 ft), a 1,200-meter (3,900 ft) miscalculation, which, for a mapmaker and a surveyor, needed to be explained. +Detzner made attempts to explain away specific ambiguities, contradictions, and errors. In 1915, he said, he lost his surveying instruments while eluding an Australian patrol, which explained why many of his assertions were vague and inconclusive, and his calculations inaccurate. This was true, the Australians did find a box of Detzner's equipment in the location where the missionary Johann Flierl's oldest son, Wilhelm, had kept (or stored) his small canoe. Although Wilhelm denied helping Detzner and his men, the Australians arrested him. The event coincided with one of Detzner's narrow escapes from Australian patrols in 1915 and was inadvertently corroborated in 1919 by the angry letter to The Argus's editor. Some of Detzner's assertions could be sustained through observable physical evidence: he had reportedly wasted to a mere 40 kilograms (88 lb) while roaming in the bush, which should not have happened, some supporters claimed, if he had indeed been under the protection of Keyser and Thiele. On the other hand, this weight loss could also have been due to his debilitating illness in 1917. +Despite his explanations, the missionaries Thiele and Keyser, whose own autobiography appeared in 1929, and the widely respected Mayr, who by this time had become the leader of the Whitney South Seas Expeditions, continued to challenge the bulk of Detzner's scientific "discoveries". Detzner's position became increasingly untenable. In 1932, he admitted that he had mixed fact and fiction in his book, explaining that he had never intended it to be taken as science, but rather at its face-value, as the story of his adventurous years in the jungles of New Guinea. The following year, Detzner resigned from the prestigious Geographical Society of Berlin. +I wish to state that my book, Vier Jahre unter Kannibalen, contains a number of misrepresentations regarding my journeys in New Guinea. The book in question is a scientific report in part only; it is primarily a fictional account of my experiences in New Guinea and owes its origin to the unusual circumstances prevailing in Germany at the time of my return. Some of the journeys I had actually undertaken are not described at all; on the other hand it contains passages that do not correspond with the facts. +After this, he withdrew entirely from public life, although he retained his position in the colonial archive. He lived in Schmargendorf, Berlin, on Auguste-Viktoria-Straße, and later became the director of the Carl-Pfeffer Verlag, a publishing house in Heidelberg. He died there in 1970, at the age of 88. +The ambiguous wording of Detzner's resignation from the Geographical Society of Berlin—the use of such phrases as contains misrepresentations, scientific report in part only, primarily fictional, unusual circumstances in Germany, and so on—misled later scholars, many of whom remained unaware of the controversy surrounding his book. Consequently, his work continued to inform the geographical, linguistic, and anthropological investigations of New Guinean culture and geography well into the 1950s and 1960s, much to the dismay of Ernst Mayr, who had been instrumental in discrediting Detzner in the 1920s. +Since the mid-1970s, references to Four Years Among the Cannibals have continued to appear in studies on New Guinea. In the 1990s, Detzner's work received some rehabilitation from ethnographer Terence Hays, who placed Detzner's work in its contemporary context: Four Years Among the Cannibals, he wrote, "paved the way for me [to become an ethnographer] by creating romanticized images that served as a backdrop for more serious readings". Since then, geographer Robert Linke has raised some important questions: "Why did Detzner resort to lies to embellish his wonderful story? The unadorned truth would have been enough to establish him as one of the great figures in New Guinea history." Detzner had remained at large for four years, as a fugitive in enemy-held territory: surely, Linke concluded, this was an exceptional feat. No doubt the Australians could have made a more broadly organized attempt to capture him, and probably would have succeeded, but they did not make the effort; they preferred instead the more convenient "shoot-at-sight" method. "It is impossible", Linke wrote, "not to admire his sheer elan, his courage and tenacity." In 2008, Detzner's book was retranslated, reprinted, and marketed as a modern translation of a rare and valuable book about the exploration of the New Guinean interior during World War I. +(in German) "Kamerun-Boundary: Die nigerische Grenze von Kamerun zwischen Yola und dem Cross-Fluss". Mitteilungen aus den Deutschen Schutzgebieten. 26:13, pp. 317–338. +(in German) "Der Saruwaged und seine östlichen und südöstlichen Anschlussgebiete". Kolonial Rundschau. Number 25 (1919) (Booklets, 8, 9, 10) pp. 209–221. +(in German) "Kreuz- und Querzüge in Kaiser-Wilhelmsland während des Weltkrieges: Februar 1914 bis November 1918". Mitteilungen aus den Deutschen Schutzgebieten. Volume 32 (1919), pp. 4–19. +(in German) Kaiser-Wilhelmsland, nach dem Stande der Forschung im Jahre 1919, with Max Moisel, Map. Berlin [Mittler], 1919. +(in German) Vier Jahre unter Kannibalen. Von 1914 bis zum Waffenstillstand unter deutscher Flagge im unerforschten Innern von Neuguinea, Scherl, Berlin, 1920, 1921. +(in English) Four Years among the Cannibals, from 1914 to the Armistice under the German Flag in unexplored interior of New Guinea. Berlin, August Scherl, [1921]. +(in Finnish) Neljä vuotta ihmissyöjien parissa: Saksan lipun suojassa Uuden-Guinean tutkimattomissa sisäosissa 1914–1918. Porvoo, WSOY, 1925. +(in French) Mœurs et coutumes des Papous: quatre ans chez les cannibales de Nouvelle-Guinée (1914–1918) Avec une carte. Paris, Payot, 1923, 1935. +(in Swedish) Fyra år bland kannibaler. Ani Mari Nordman, trans. Helsingfors, Schildt, 1925. +(in English) Four Years Among the Cannibals ... Gisela Batt, (Trans.), Pacific Press, Gold Coast, Australia, 2008. (Retranslation and reprint). +(in German) "Medizinische und hygienische Streiflichter aus dem Innern von Neuguinea". Archiv für Schiffs- und Tropen-Hygiene, Pathologie und Therapie exotischer Krankheiten. 1921, 25(3): pp. 67–79. +(in German) Im Lande der Dju-Dju. Reiseerlebnisse im östlichen Stromgebiet des Niger, Scherl, Berlin 1923. +(in German) "Stammesgemeinschaften im Zentralgebiet von Deutsch-Neuguinea". Mitteilungen aus den deutschen Schutzgebieten, Volume 36 (1928), pp. 112–130. +(in German) "Unter Unbekannten Kannibalen", Die Woche, 24. 1. 1925, Nr. 4. +(in German) Die Kolonien unter Mandatsherrschaft. Berlin, Deutscher Wille, 1927. +(in German) Das "Zentralgebirge" Neuguineas im Gebiet der Wasserscheide zwischen Hüon- und Papua-Golf. Map. Mitteilungen aus den deutschen Schutzgebieten, Kt. 3 im 2. Heft, Bd. XXXVI, 1928. +Detzner's publications at the city library of Berlin. (in German) +Crush is a 2007 puzzle-platform video game developed by Zoë Mode and published by Sega for the PlayStation Portable. Its protagonist is Danny, a young man suffering from insomnia, who uses an experimental device to explore his mind and discover the cause of his sleeplessness. Each level of the game, representing events from Danny's life and inspired by artists such as Tim Burton and M.C. Escher, requires the player to control Danny as he collects his "lost marbles" and other thoughts. +Crush's primary gameplay feature involves manipulating each game level between 3D and 2D views, allowing the player to reach platforms and locations inaccessible from within a different view. This element was noted by critics to be similar to one in Super Paper Mario, also released in 2007, though the Zoë Mode team had envisioned the concept five years prior. Crush received positive reviews upon release, with critics praising its incorporation of this dimension-shifting component alongside other aspects of the game presentation. Though Crush won several gaming awards, including PSP game of the month, it failed to meet the developer's sales expectations. A port of the game for the Nintendo 3DS called CRUSH3D was announced in January 2011 and was made available in January 2012 in Europe; in February 2012 in Australia; and in March 2012 in North America. +While Crush and its Nintendo 3DS port CRUSH3D retain the same gameplay mechanics and premise, the two versions feature different plots. +The protagonist of the game, a young man named Danny, suffers from chronic insomnia caused by worry, stress, and repressed memories. He is admitted to a mental institution for it, where he consults a mad scientist, Dr. Reubens, who treats Danny with his Cognitive Regression Utilizing pSychiatric Heuristics (C.R.U.S.H.) device, which has a sentient female persona. The device's helmet places Danny under hypnosis, during which he can regain control of his sanity by collecting his lost marbles, and facing his primal fears in the form of monsters (i.e. cockroaches, blockwalkers, slugs). Danny starts recounting key moments in his life that may have contributed to his insomnia: Moving into his first apartment and his suffering job as a chef in the city, his love life some few years ago at the beach with a girl named Tina (which unfortunately ended in him being jilted), and the time he went to a local funfair alone as a young boy, where he ended up assaulted by a group of thugs. When Danny returns to the lab a fourth time after none of these prove to be the case, Dr. Reubens snaps at Danny for wearing out his machine further, and regresses Danny all the way to his childhood, when he was only six years old, against Danny's own will. In the end, Danny reveals that he was left home alone during his parents' date night, with disturbing shadows of the night leaving Danny traumatized. Danny discovers that his childhood fears at that time, principally his nyctophobia, are the real cause of his insomnia. Reubens is disgusted that his treatment was a complete waste and that Danny ruined his machine due to such a petty issue, but tells him he's cured. Right when Reubens begins to snap Danny out of hypnosis, C.R.U.S.H. seems to attack Danny's mind, with Reubens panicking and Danny's life on the line while the screen fades to black. +Doctor 'Doc' Doccerson (instead of Reubens in the PSP version), is infuriated that all of his inventions failed, but today he 'shall surpass them all' with his latest invention, C.R.U.S.H. He uses a tape recorder for his scientific narrative on the experiment. With the help of his best friend and protégé, Danny (who recurrently corrects Doc throughout the game), he enters his mind as his subject to C.R.U.S.H., but after a few tests, Danny is trapped inside C.R.U.S.H., where he can collect his lost marbles (which will unlock new dressing gowns/robe designs) and facing his fears in the form of monsters (i.e. cockroaches, slugs). At one point, Doc tells Danny that C.R.U.S.H. will not let Danny out due to "unresolved feelings." The two question that reasoning as they progress. Towards the end, C.R.U.S.H. interrupts contact between Danny and Doc, overly infatuated with Danny. Danny slowly admits to C.R.U.S.H. that she is simply a machine and compares it to "flirting with a parking meter." C.R.U.S.H., infuriated, sends Danny back to his childhood as a result. When Doc comes back into contact with Danny, Danny lashes out at Doc for building a machine "with the heart of a teenage girl" who "LOLd" at him. Doc, calming Danny down, explains how this "subconscious" is not really his, but old data, and suddenly Danny becomes shocked when he realizes he took the place of someone who underwent treatment with C.R.U.S.H. (being the PSP Danny), but "she knows [he's] not the same." C.R.U.S.H. really wanted Danny to clear her head. In the ending, Danny finally escapes and is free out of C.R.U.S.H's mind. He and Doc discuss C.R.U.S.H's future, while Doc says that he'll fix her. In a sudden plot twist, it turns out that they were not in the real world at all, but all the levels throughout the game are outside Doc's lab. C.R.U.S.H. smiles and laughs into the camera, she winks and the screen cuts to black. +Crush contains ten levels in each of the four locations, all based on an event in Danny's past. The levels represent Danny's mind: a dark city landscape with many tall buildings and the occasional street lamp, a hotel resting aside a seaside location, a dark and mysterious funfair, and a haunted childhood bedroom. Levels are mostly composed of platforms formed by blocks. The player's goal in each level is to collect marbles, which give the player points based on their color. The exit from the level is opened once a predetermined number of points have been collected. Danny can crawl into narrow areas and jump a small height. +The main gameplay feature of Crush is the ability to transpose the layout of a level between 2D and 3D representations to reach seemingly inaccessible areas and solve the game's puzzles. The player can switch the third-person camera between four directional side views and a top-down view at any time while in 3D. When in these views, the player can have Danny "crush" the level, collapsing all 3D elements into 2D; crushing from side views results in a 2D platformer-like view, while crushing from the top-down view provides a 2D top-down perspective. Crushing can connect and merge platforms on the same visual plane in the 3D view but separated by a large distance, creating pathways across the level in the 2D view. The player can also "uncrush" the level at any time. Certain blocks, when crushed, become either obstacles through which Danny cannot pass, or ledges the player can use to reach other parts of the level. Attempts to crush a level in any manner that would harm Danny are thwarted. However, uncrushing can leave Danny in a helpless state, such as hanging in mid-air. +Enemy monsters inhabit the levels, but the player can crush them by flattening impassable blocks against them. The player may also encounter timers in the form of alarm clocks that will begin to elapse when crushed for the first time and can only be stopped when Danny jumps on them. Danny wakes from his mental explorations if he falls off the level or too high of a surface, is touched by a monster, or fails to stop a timer, but the C.R.U.S.H. device reinserts him at the start of the level or the last checkpoint Danny passed. +Throughout the levels are scattered large spheres and cylinders, which the player can roll when crushed appropriately. These can then be used as platforms or to depress switches. Jigsaw pieces can be collected to reveal concept art and extras in the game's menus. Some of Danny's "thoughts", represented by glowing neon icons on the walls of the level, are only activated when the level is crushed in a manner that does not obscure them. Some thoughts allow Danny to jump higher or stop time. Once the player completes each level, they are graded by the duration of Danny's stay on the level, how many times Danny "woke up", and a bonus for collecting all marbles, the jigsaw piece, and a hidden thought trophy. The thought trophy, once completed by crushing, unlocks a level's special challenge mode for later play which requires time-limited completion of the level with an allotted number of crushes. +In an interview, Zoë Mode executive producer Paul Mottram claimed that the game concept was envisioned in 2002, but work did not actually begin until 2006. The initial concept was built on the crushing mechanism between 2D and 3D, and they only had to create appropriate obstacles to prevent players from simply "crushing" across the level. Mottram noted that during the development of Crush, the gameplay of Super Paper Mario had not yet been revealed, and thus the team was surprised to learn that it shared a similar feature. +Mottram stated that the crushing mechanism had been developed and refined for six months before developing the story and characters; the development team wanted to have "a normal person in an impossible situation". The art and level design were inspired by Tim Burton, Mike Mignola, and M.C. Escher. The plot was originally more morbid than in the final product, with Danny dying and the rest of the game told as flashbacks. +The game levels were developed on a level editor on the PlayStation Portable, but Zoë Mode were not able to refine the editor in time for shipping. Mottram said "It would be great to see user generated content and this is something we have been seriously thinking about for the future" and that downloadable content "would work perfectly with the Crush level structure and I am sure that fans of the game would be eager to see more levels." Mottram has stated they would like to market a sequel based on the highly positive feedback they had received; however, with other Zoë Mode projects such as SingStar and Play taking precedence, he does not know when this could be. +Crush received "favorable" reviews, while CRUSH3D received "average" reviews, according to the review aggregation website Metacritic. The PSP version was highly praised for its innovative approach to gameplay. Ryan Davis of GameSpot appreciated Crush for owing "very little of its novel concept to games that preceded it". Nick Suttner of 1UP called the game a "cognitively rewarding, expertly designed puzzle experience that truly plays like nothing else". Reviews were mixed on the game's learning curve. IGN's Jeremy Dunham praised the ordering of the puzzle elements, that new gaming elements are introduced at "an ideal pace", and that most puzzles have solutions where the player must "think 'outside the box'". Eurogamer's Dan Whitehead commented that the game introduces these elements too quickly and "doesn't give you much time to put the basics into practice". Some critics found that elements of the game detracted from the game's uniqueness. X-Play's Greg Orlando, while stating that this was "one of the most novel games ever made", noted that it was "simply not very fun", as it lacked many player incentives beyond manipulating the world and collecting objects on each level. Reviewers noted that some puzzles were awkward due to the selection of the PlayStation Portable's controls. Reviewers found the game's story poor, but this was overcome by the gameplay elements; Charles Herold of The New York Times said "the minimal story is as forgettable as its puzzles are ingenious." Gus Mastrapa of The A.V. Club said that the game "satisfies the innate gamer need to express the world in easy-to-manipulate numbers and objects." GamesRadar+ included Crush in their list of the 100 most overlooked games of its generation. Editor Jason Fanelli stated that the ability to switch from 2D to 3D was "one of the most unique (at the time) features we've ever seen." +The Digital Fix's Ryan Poxon gave CRUSH3D a score of seven out of ten and called it a "puzzle game that genuinely surprised me at how brain teasing the game actually is." 411Mania's Adam Larck gave it a score of 6.6 out of 10 and said, "For a puzzle game, Crush 3D is a good choice for 3DS fans. While the mechanic may not be as creative as it was a few years ago, it's still a decent game to check out. If you never saw the PSP version or haven't played games like Fez or Echochrome, I'd recommend checking this out to see what crushing is all about." However, Digital Spy's Liam Martin gave it three stars out of five and stated: "Despite the five-year gap between releases, Crush 3D takes a few steps backwards in terms of visual style and presentation. Other than a rather limited and lacklustre selection of new levels and StreetPass features, it also offers very little in the way of brand new bonus content, making it difficult to recommend to anybody familiar with its PSP counterpart. Despite its shortcomings, however, Crush 3D gets it right where it counts." In Japan, where CRUSH3D was ported for release under the name Nightmare Puzzle: Crush 3D (ナイトメア パズル: クラッシュ 3D, Naitomea Pazuru: Kurasshu Surī Dī)) Famitsu gave it a score of three sevens and one six for a total of 27 out of 40. +IGN awarded Crush PSP Game of the Month for May 2007. GameSpy called Crush the third best PlayStation Portable game and the "PSP Puzzle Game of the Year" for its Game of the Year 2007 awards. Similarly, IGN awarded Crush the "Best PSP Puzzle Game", "Most Innovative PSP Game", and "Best PSP Game No One Played" awards in their Game of the Year 2007 selections. Crush won the 2007 Develop Conference Industry Award for "Best New Handheld IP". According to Paul Mottram, positive reception of the game did not translate into high sales, but the game "will hopefully stick around for a while and continue to shift units". GameTrailers made Crush a nominee for "Best PSP Game" in its Game of the Year Awards 2007. +Echochrome +Fez +Super Paper Mario +Zoë Mode website +Crush at MobyGames +Barthélemy Boganda (c. 1910 – 29 March 1959) was a Central African politician and independence activist. Boganda was active prior to his country's independence, during the period when the area, part of French Equatorial Africa, was administered by France under the name of Oubangui-Chari. He served as the first Premier of the Central African Republic as an autonomous territory. +Boganda was born into a family of farmers, and was adopted and educated by Roman Catholic missionaries after the death of his parents. In 1938, he was ordained as a Roman Catholic priest. During World War II, Boganda served in a number of missions and afterwards was persuaded by the Bishop of Bangui to enter politics. In 1946, he became the first Oubanguian elected to the National Assembly of France, where he spoke out against racism and the abuses of the colonial regime. He then returned to Oubangui-Chari to form a political organisation, culminating in the 1949 foundation of the Movement for the Social Evolution of Black Africa (MESAN), which became popular among villagers and the peasantry. Boganda was laicized from the priesthood after developing a relationship with and eventually marrying Michelle Jourdain, a parliamentary secretary. Nonetheless, he continued to advocate for equal treatment and fundamental rights for blacks in the territory well into the 1950s. As France conceded measures of representation to its colonies, MESAN won local elections and he gained influence in Oubangui-Chari's government, though his reputation suffered when he backed an unsuccessful economic scheme. +In 1958 French Prime Minister Charles de Gaulle proposed the creation of a French Community through which France's colonies could associate with the metropole. After being assured that Oubangui-Chari's membership in the community would not preclude it from securing independence at a later time, Boganda supported joining it. He sought to do so as part of a federation with other territories in French Equatorial Africa as a "Central African Republic", which he believed would bolster the financial situation of the member states. He hoped this would serve as the basis for a United States of Latin Africa, a conglomeration including other countries in central Africa. This never came to fruition, and on 1 December, Boganda declared the establishment of the Central African Republic for only Oubangui-Chari. Boganda became the autonomous territory's first premier as the President of the Council of Government, and began drawing up administrative reforms and preparing for the next election. He was killed in a plane crash on 29 March 1959, while en route to Bangui. Experts found a trace of explosives in the plane's wreckage, but a full report on the incident was never published, and the possibility of an assassination remains unresolved. The Central African Republic attained formal independence from France in 1960. Boganda's death is annually commemorated in the country, and his presence in the national collective memory remains politically potent. +Little is known about Boganda's early life. He was born around the year 1910 to a family of farmers in Bobangui, a large M'Baka village in the Lobaye basin located at the edge of the equatorial forest some 80 kilometres (50 mi) southwest of Bangui. His father, Swalakpé, was a village head and the wealthy owner of several palm plantations who had taken numerous wives. Boganda's mother, Siribé, was Swalakpé's third wife. French commercial exploitation of Central Africa had reached an apogee around the time of Boganda's birth, and although interrupted by World War I, activity resumed in the 1920s. The French consortia used what was essentially a form of slavery—the corvée—and one of the most notorious was the Compagnie Forestière de la Haute Sangha-Oubangui (CFSO), involved in rubber gathering in the Lobaye district. Coercive labour practices, violence, and disease had severely disrupted traditional society by the time Boganda was born. Bobangui was particularly affected by these elements. His uncle, whose son, Jean-Bédel Bokassa, would later crown himself as the Emperor of the Central African Empire, was beaten to death at a colonial police station as a result of his alleged resistance to forced labor. +Both of Boganda's parents died when he was young; his father was reportedly killed in a punitive campaign conducted by colonial forces shortly after his birth. His mother died before 1915, probably having been murdered by a CFSO militiaman for not having met a rubber collection quota. Boganda was then given to the care of a guardian who joined the French Army during World War I and was killed at the Battle of Verdun. He was subsequently placed in the tutelage of other relatives, and in 1920 he contracted smallpox. In June his brother was instructed to take him to an uncle, and along the way they encountered a French patrol led by a Lieutenant Mayer. His brother fled out of fear. Left alone, Boganda said, "Gboganda". This was probably a Ngbaka phrase meaning "I am [from] elsewhere", and he was probably hoping to explain that he was lost. The soldiers believed this was his name, rendering it "Boganda", and the name was used for the rest of his life. Mayer took him to the orphanage in the nearby town of Mbaïki. Once there, a Spiritan missionary touring the area decided to take him to the mission station of Saint Jean Baptiste in Bétou, a town further south on the Ubangi River which was home to a school. +At Bétou, Boganda was instructed in reading and writing in Lingala. Most accounts concur that he was an excellent student. In December 1921 he was taken to the main Spiritan mission of Saint Paul des Rapides in Bangui, the capital of Oubangui-Chari. He was baptised there under the name Barthélemy in late 1922. He later wrote, "To be a Christian meant for me to free myself from ancestral customs, to become a brother of humanity". At Saint Paul he learned French, the catechism, and agricultural labour. By mid-1924 Boganda had completed his primary education and had indicated his desire to become a priest. In November he was sent to the Jesuit petit séminaire in Lemfu, Belgian Congo. The school's curriculum included Latin, French, mathematics, history, and philosophy and was scheduled to take six years to complete, though Boganda had left by 1928/1929. After failing to enroll in a school in France due to lack of money, he entered the Spiritian petit séminaire in Brazzaville. He spent his final year of studies in Bangui, where he was tutored by Monsignor Marcel Grandin, the head of the Catholic Church in Oubangui-Chari. Once this was completed, Grandin enrolled Boganda in the Saint Laurent grand séminaire in Mvolyé, Yaoundé, French Cameroon, in 1931. The first African student at the school, he learned history, Latin, philosophy, theology, and other subjects. +On 17 March 1938 Boganda was ordained. He was subsequently posted to the new Saint Marcel petit séminaire in Bangui as a teacher. In 1939 his bishop denied his request to join the French Army, deeming it necessary for him to remain, as many persons involved with the church had been recalled to the metropole to fight in World War II. He was posted in the Grimari region from 1941 to 1946 to evangelize the resident Banda population. Boganda was enthusiastic about his work and was angered by local resistance to some of his teachings and practices. This especially included his efforts which contravened local cultural norms, such as his encouragement to abandon polygamy and fetishism and work to enroll girls in school. His response to such hesitation sometimes included violence, and he was accused of beating people and once suggesting a recalcitrant chief be shot with a cartridge of salt. Despite this, he showed a measure of appreciation for some indigenous culture, and was particularly proud that some locals fearfully dubbed him mourou, or leopard, a traditional symbol of power and violence. Boganda believed that the church was providing him with insufficient support and, by the mid-1940s, was in conflict with the local government administrator and felt he was facing racial discrimination from colonial officials, settlers, and some missionaries. Further strain on his position was incurred when he fathered a child in the Bakala region, an action his missionary colleagues saw as scandalous. As a result of these tensions, in 1946 Boganda was transferred to a mission at Bangassou. +After World War II, Boganda was urged by Grandin to involve himself in politics. In particular, Grandin hoped he would seek election to the National Assembly of France. Some Oubanguians had already indicated that they would support Boganda if he contested a seat in the assembly, and Grandin hoped that Boganda could preserve the interests of the Catholic Church from the local growth of Protestantism and increasing leftist and anti-colonial thought in both France and its colonies. Boganda decided to compete, and on 10 November 1946, he was elected deputy for Oubangui-Chari, becoming the first native Oubanguian to join the assembly after winning 10,846 votes—almost half of the total votes cast—and defeating three other candidates, including the incumbent, François Joseph Reste, who had formerly served as the Governor-General of French Equatorial Africa. Formally, Boganda identified with the Popular Republican Movement (MRP). His election was confirmed on 20 December and he was made a member of the Assembly's Overseas Territories Commission and the Supply Commission. He arrived in Paris attired in his clerical garb and introduced himself to his fellow legislators as the son of a polygamous cannibal, probably in a deliberate attempt to project a personal aura of omnipotence. Grandin sought to maintain influence over his former pupil and had requested that the Spiritans in France welcome him, provide lodging, and introduce him to Catholic politicians and keep him away from left-leaning groups. They did not do this, and Boganda expressed disappointment at the lack of a reception upon his arrival and the weak support from French deputies for his proposals to help his constituents in Oubangui-Chari. +Boganda left the MRP in 1950 and served as an independent thereafter. He was reelected to the National Assembly in 1951 and 1956. He made only two parliamentary interventions during his tenure, in August 1947 on the abuses of colonialism and in June 1950 on the lack of social justice in French Equatorial Africa. After 1956, he largely stopped attending the Paris parliament, though he remained a deputy until 1958. Frustrated by the faults of colonial rule in Oubangui-Chari and an unwillingness on the part of local officials to accept reforms, he quickly resorted to vocal criticism of French administration in the colony. He paid particular attention to racism and highlighted incidents of settler violence against black Africans to boost his own political following. Among his complaints were instances of arbitrary arrest, low wages, compulsory cotton cultivation, and the barring of blacks from restaurants and cinemas. In April 1947, Grandin complained to the governor of Oubangui-Chari that Boganda had "escaped from his cage" and was "flying like an idiot". While his rhetoric was anti-colonial, Boganda nominally supported French political ideals and did not oppose continuing connections between France and Oubangui-Chari. He also identified as a staunch anti-communist and accused colonial administrators of being "anti-French" and "worthy sons of Stalin", making it difficult for them to criticise him. He proposed several measures aimed at reforming communal land ownership and ensuring the prohibition of forced labour, but his serious attacks on French colonial policy upset the other deputies and as a result his ideas were never incorporated into the parliamentary agenda. +Feeling as though his actions in the National Assembly were engendering little substantive change in Oubangui-Chari, Boganda grew disenchanted with parliamentary politics and decided to seek direct political action within the territory. In an attempt to improve Oubanguian farmers' incomes, spur colonial reform, and form a political organisation for himself, in 1948 he launched a co-operative project, the Société Coopérative Oubangui, Lobaye, Lesse (SOCOULOLE), which aimed to provide food, clothing, lodging, medical care, and education. Boganda hastily established the organization without regard for the competence of its staff or the standards of trading goods, though he made sure to include communal healers in its management. Initially operating with a French subsidy, over time the co-operative became embroiled with allegations of financial improprieties and ran low on money. Boganda attempted to fund it with his parliamentary salary, but its deficits rapidly increased. +On 28 September 1949, at Bangui, Boganda established the Movement for the Social Evolution of Black Africa (MESAN), a mass political party. He wrote its founding code, which stipulated that the organisation sought "to develop and liberate the black race by progressive and pacific evolution, achieved by the combined efforts of all negroes throughout the world". Politically, the party supported liberty and equality for Africans, while economically it endorsed the use of co-operative ventures. Framing his political programme as a matter of fundamental rights, Boganda coined the Sango phrase zo kwe zo, which translated to "every human being is a person". He praised "peasant virtues" and the rustic life before colonialism, and these messages resonated with rural farmers. Furthermore, Oubanguians appreciated his willingness to angrily confront colonial officials. Boganda deliberately strived to instill his rhetoric with a sense of religiosity and mysticism, and he frequently used Latin in his speeches. Rumours began to circulate of his supposed invulnerability and supernatural powers, and at one point later in his career a large crowd waited on the shore of the Ubangi River to see him walk over the water (he did not appear). +MESAN's activities angered the French administration and the companies trading in cotton, coffee, diamonds, and other commodities. The Bangui Chamber of Commerce was controlled by these companies, and its members resented the end of forced labour and the resultant rise of black nationalism. They despised Boganda, viewing him as a dangerous revolutionary demagogue and a threat to their "free enterprise", and they resolved to get rid of him. French colonists and administrators established local Gaullist Rally of the French People (RPF) branches to counter MESAN. Drawing on the support of government workers, clerks, and Free French veterans of World War II, the party sought to take credit for colonial reforms, but failed to generate popular traction. The presence of the African Democratic Rally (RDA) in the other three territories of French Equatorial Africa posed some threat to MESAN, but they were eventually reduced to minor groups. +Boganda was not particularly concerned with his religious mission once he entered politics, but he used the enormous popular respect for the Catholic Church to his advantage, manipulating religious symbols for political purposes. Boganda's attachment to the clergy weakened when he met and fell in love with a young Frenchwoman, Michelle Jourdain, who was employed as a parliamentary secretary. By 1949 they were cohabitating and Boganda wrote a letter to his Catholic superiors, pointing out that clerical celibacy was a rule created by the Catholic Church and had no biblical basis. On 25 November he was expelled from the priesthood. Despite this, Boganda remained a devout Catholic and sympathetic to missionary interests. The couple was married on 13 June 1950, and would later have two daughters and a son. +On 10 January 1951, SOCOULOLE agents in the village of Bokanga became involved in a heated dispute with local Portuguese merchants, with the former objecting to the latter's practice of forming a coalition amongst themselves. The co-operative agents insisted that the village market be closed until Boganda could arrive to represent their case. By the time he arrived, SOCOULOLE members had blocked the roads leading out of the locale with trees to prevent the merchants' trucks from leaving. Fearing a loss of law and order, the head of the district of Mbaïki arrested Boganda (along with his accompanying wife) and held him in detention for two days. He was charged with "endangering the peace" and on 29 March the local court sentenced him to two months in prison (his wife was condemned to two weeks incarceration for aiding him). Since he was arrested in flagrante delicto, his parliamentary immunity offered him no protection. Boganda terminated SOCOULOLE later that year due to financial difficulties. Boganda's arrest occurred five months before the next round of French National Assembly elections, and he framed it as a campaign issue. He ultimately won re-election, defeating challengers from the RPF and RDA, and the colonial administration acceded to his return to office. +In 1952, the French government appointed more reform-oriented officials in French Equatorial Africa, allowing for a relaxation in tensions between Boganda and the local administration. In March 1953 RPF leader Charles de Gaulle visited Bangui. Boganda refused to see him due to his leadership of the party, but de Gaulle refrained from taking a public stance on the politics in Oubangui-Chari, a move which was interpreted as an expression of disapproval of the local RPF's tactics. +On 30 April 1954, disorder erupted in Berbérati when news emerged that two Africans who worked for a European—who was known for his abusive treatment of the locals—had died. The families of the deceased demanded the European's arrest, and a crowd gathered in the town and began rioting and assaulting public officials. Once word reached Brazzaville, the colonial authorities began mobilising troops to march on Berbérati, and Oubangui-Chari Governor Louis Sanmarco pleaded with Boganda to accompany him to the locale and intervene. The following day Boganda appeared before the crowd and told them that "the same justice would be administered to white as to black". Thus assured, the crowd dispersed and order was restored. The riot deeply worried the administration, which acknowledged in its own reports that Europeans' racism towards Africans was pervasive in the territory. +No substantial violence succeeded the Berbérati riot, and in the following months the RPF increasingly struggled as MESAN continued to grow. Boganda praised the educational and health work done by the colonial administration before the French National Assembly, saying "doctors, administrators, and colonists are our friends [...] We are not as ungrateful as we are black. We know how much has been done for our country". Addressing the Grand Council of French Equatorial Africa, he complimented Sanmarco and declared, "Oubangui-Chari has embarked on a positive undertaking at last, after years of negative grievances and sterile struggles, and a better future is ahead". With the assistance of the colonial administration, Boganda established his own coffee plantation and encouraged rural residents to follow suit if possible. Meanwhile, district councils were established, and he attended their meetings in Boda and Mbaïki, urging the MESAN members to collaborate with the European district heads. +In June 1956 the French National Assembly passed the Loi-cadre Defferre, an act which conceded a measure of internal autonomy to French colonies. The Europeans in Oubangui-Chari politically organised themselves to seize control of the new local institutions, particularly the office of Mayor of Bangui. René Naud—the European president of the Bangui Chamber of Commerce—and other white merchants offered themselves as candidates in the November election, but Boganda entered the race and quickly became the favourite to win. On 18 November he won the election and became the first Mayor of Bangui. +On 4 February 1957, in accordance with the loi-cadre, the French government formally decreed the semi-autonomous status of each of the four territories of French Equatorial Africa, including Oubangui-Chari. Collectively, the four territories constituted a federation led by a General High Commissioner with the assistance of a Grand Council. Within Oubangui-Chari, the former French governor became the High Commissioner, who was to preside over a Council of Government, with its members to be chosen by the newly created Territorial Assembly. On 31 March, MESAN won all seats in the Assembly and, at Boganda's request, Hector Rivierez was elected its presiding officer. He also arranged for Abel Goumba, the territory's only African doctor and a former catechism student of his, to become Vice President of the Council of Government. The council was installed on 17 May without Boganda as a member, since he did not want to participate in a government presided over by a French commissioner, and was also increasingly concerned with political organising at a federal level. Instead, he became President of the Grand Council. +The only European minister in the Council of Government was Roger Guérillot, who had previously secured the confidence of Boganda and worked as MESAN's treasurer. He was given charge of the portfolio for economic and administrative affairs. Guérillot sought to increase Africanisation of the administration, since there was a lack of trained Oubanguians and such a change would weaken the government and strengthen the position of the French colonists. He encouraged Boganda to go on a rhetorical offensive against French officials, whom he dubbed "the saboteurs of the loi-cadre". In a speech before the Territorial Assembly, Boganda suggested that the French administrators should leave and Oubanguians could "curse their shameful memory for ever", but also noted that it would take several years to train African personnel to replace them. He softened his stance a few days later while addressing the Grand Council, suggesting the colonies needed "a new form of administration" and proposing the transformation of districts into "rural communities" with trusted officials from the existing bureaucracy serving as directors of each. +In the mid-1950s a Bangui study group had proposed the construction of a rail line from Bangui to Chad. This was both to secure Oubangui-Chari's commercial relations with southern Chad—which was facing strain due to competition from Cameroon—and to satisfy private firms that sought a large state contract to make up for the decline in foreign investment driven by uncertainties in the territory's political future. Boganda believed that it would only be reasonable to embark on the railway project if Oubanguian economic output was greatly increased, so he requested that Guérillot draw up a programme for improving production and raising the standard of living. +Guérillot proposed a large scheme totaling four billion Central African CFA francs in expenditures to greatly increase the cultivation of coffee trees, cotton, and ground-nuts. As a part of this, he conceived a Committee of Economic Safety, which would consist of more regional bodies of European merchants and MESAN officials who would oversee peasants' production efforts. On 30 December, Boganda praised the committee as "the union of capital and Oubanguian labour" and convinced the Territorial Assembly to allow Guérillot to proceed. French colonists found the project risky and did not invest in it, and were followed in their abstention by banks and French economic aid organizations. Goumba also thought the proposal demanded too much of peasants and began formulating his own economic platform. Meanwhile, Guérillot recruited unemployed whites in Bangui as "inspectors" with African auxiliaries to directly manage cultivation. Peasants regarded the scheme as a return to the concessionary system and began publicly protesting it. Facing skepticism in the press and a measure of isolation for earlier excluding Antoine Darlan—Oubangui-Chari's representative to the French Union—from MESAN, Boganda traveled throughout the territory to try to allay peasants' fears and exhort them to work. He stressed that increased agricultural production was the only way for Oubangui-Chari to become economically viable without French aid. Angered by public criticism, he proposed banning all political activity. The economic scheme ultimately failed to achieve its goals and damaged Boganda's reputation, as well as tarnishing the view of the Council of Government both domestically and in the French and Belgian governments. +Following the failure of the Algiers putsch in May 1958, de Gaulle reassumed power in France as Prime Minister and prepared to draft a new French constitution, stressing the importance of re-examining a federal relationship between France and its colonies. Boganda was not included in the new constitutional commission, to his dismay. De Gaulle hosted him in Paris in July, and upon his return to Oubangui-Chari he expressed to the Territorial Assembly that the loi-cadre was insufficient for the territory and the rest of Sub-Saharan Africa. He told the body he wanted assurance of "the people's right to self-determination and a voluntary freely-consented independence. The ways of introducing it are to be examined". The Assembly passed a motion repeating Boganda's demands. +De Gaulle proposed the creation of a new federal French Community that would encompass the African colonies. Boganda opposed Oubangui-Chari joining the community, fearing it would forestall independence. In August a meeting was held in Brazzaville between de Gaulle and political leaders in French Equatorial Africa. Boganda presented a petition signed by the leaders which requested that the new French constitution recognise the right of its colonies to declare independence. De Gaulle assured him that Oubangui-Chari's membership in the community would not preclude it from securing independence at a later time. A referendum was to be held in each colony in order to determine its support for the new constitution and joining the community; de Gaulle warned that while a negative vote would grant a territory immediate independence, it would also lead to the termination of all French aid. On 30 August Boganda told MESAN leaders he supported an affirmative vote in favour of the constitution, and he subsequently traveled around Oubangui-Chari to tell the people that the French would remain slightly longer "to set right the ravages of colonisation". The referendum was held on 28 September and 98% of voters chose to support the new constitution. +While the French constitution had placed political responsibility upon each territorial assembly in Africa and expected them to ratify the results of their referendums, it left open the possibility of federations. Boganda had been worried about balkanisation in Africa for some time, and believed that independence of Oubangui-Chari as a single state would be disastrous. He used his position as President of the Grand Council to encourage the formation of a united state in Central Africa. He wrote a tract which stated, "A united state with a united government and united parliament would reduce our expenses considerably. We could restrict the administrative budget and devote more of our resources to developing the welfare of our countries, so that all citizens would benefit, not just one privileged category. It is obvious that such an arrangement would encourage investment". +Boganda articulated a new framework for the states of French Equatorial Africa whereby there would be a central government and legislature. There would be an annually rotating presidency in which each former territory would take turns supplying the officeholder. The territories would become departments under the supervision of ministers of state and be divided into urban sections and rural communes. Due to the geographic span of such a federation—which would include Oubangui-Chari, Congo, Gabon, and Chad—he proposed that the state would be known as the "Central African Republic". He stressed the urgency of accomplishing this as quickly as possible, saying, "The Central African Republic must be built today, for tomorrow it will be too late [...] Chad and Oubangui-Chari will surely be solicited by other voices and other means". In a speech, Boganda revealed he envisioned the Central African Republic as a step in creating a larger United States of Latin Africa: +Next we have to examine the question of the right bank of the Congo [river]. Since the official historical frontier is the Congo and not the Oubangui [river], we must regard that area from now on as belonging to the Central African Republic. Thirdly, we must work towards re-uniting the two Congos. The fourth stage will be to create the United States of Latin Africa, including the Central African Republic, the so-called Belgian Congo, Ruanda-Urundi, Angola, and Cameroon. +Boganda dispatched Rivierez and David Dacko to Gabon, Chad, and the Congo to explore their interest in a united state. The two were unable to secure a meeting with authorities in Gabon, while Chadian leaders rejected the idea. Jacques Opangault, the leader of the government in the Congo, was enthusiastic about the proposal, but his majority in the Congolese legislature was slim and his position weak. Gabon was the richest of the states, and its refusal to engage with the proposal made the French hesitant to sanction the federation. In late November the French High Commissioner convened a meeting of the equatorial leaders in Brazzaville and told them that each territorial assembly was to independently ratify its referendums and finalise its decision to adhere to the new constitution. By 28 November, all the other territories had decided to join the French Community as separate entities. Disheartened, Boganda resigned himself to proclaiming only Oubangui-Chari as the Central African Republic on 1 December. +The Central African Republic adopted a design drawn up by Boganda for its flag, including a star, the French tricolour, and colors of other African flags. He also penned the lyrics for the national anthem, "La Renaissance". On 6 December, the CAR's first government was established with Boganda as President of the Council of Government (premier), though a French High Commissioner was retained. Frustrated with Guérillot's economic failures and political maneuverings to be elected to the French Senate, he sent Guérillot to France as a diplomat and replaced him at the Ministry of Interior and Economic Affairs with Dacko. He made several other changes to the composition of the original council, but retained Goumba as Minister of State and de facto vice president of the council. +The new government's first action was to adopt a law banning nudity and vagrancy. Its main objective, however, was to draw up a constitution. Democratic in nature, the document provided for a unicameral parliament with a five-year term and a prime minister for the same period. The text was largely borrowed from the French constitution, though Boganda had some influence over the wording of the preamble and pushed for the inclusion of a provision that allowed the country to cede its sovereignty to a wider union. The draft was approved by the assembly on 16 February 1959. Boganda then set about creating extensive administrative reforms, including the establishment of rural and urban municipalities, the creation of district councils with broad authority, and the institution of mutual development societies. The government also drafted new electoral constituencies and scheduled elections for the Legislative Assembly on 5 April. Boganda personally oversaw the selection of MESAN's candidates and agreed to include five Frenchmen on its lists. The government also created a new electoral law which stipulated that civil servants could not run for office unless they had been on leave of absence for at least six months prior to the polling date. Furthermore, the law required all parties to nominate their candidates in lists instead of individually, and if one candidate was disqualified, the whole list would be dismissed. As a result, all opposition lists were thrown out by the courts, leaving MESAN unopposed. Opposition politicians were infuriated, and when asked about the impending lack of a parliamentary opposition, Boganda told the press, "We will create our own opposition within our party". +On 29 March 1959 Boganda boarded a Nord Noratlas plane at Berbérati, where he had been campaigning, for a flight to Bangui. The plane was owned by Union Aéromaritime de Transport, which transported the mail between the two cities. The plane went missing, and its wreckage was discovered the following day in the district of Boda. All four crew and five passengers, including the government's information chief and a member of the Assembly, were found dead. Boganda's body was recovered from the pilot's cabin. +The French General Secretariat of Civil Aviation ordered an inquiry and sent a team to investigate the crash site. A report was never published, but shortly afterwards the Paris weekly L'Express revealed that investigators had identified traces of explosive in the wreckage. The French High Commissioner ordered all copies of the reporting edition suppressed in the Central African Republic. No cause for the crash has ever been definitively determined. Many Central Africans believed that the crash was an assassination; in particular, many suspected that expatriate businessmen from the Bangui Chamber of Commerce, possibly aided by the French secret service, played a role. Michelle Jourdain was also suspected of being involved; by 1959, relations between Boganda and his wife had deteriorated, and he thought of leaving her and returning to the priesthood. She had a large insurance policy on his life, taken out just days before the accident. Historian Gérard Prunier wrote that "the probability of foul play was very high", noting, "The whites who worked for what was left of the Grandes Compagnies Concessionaires hated Boganda, who had been instrumental in finally getting compulsory labor outlawed in 1946. They also hated his intelligence, which was unsettling to their view of black inferiority". +Aside from some minor disorder in Mbaïki, the country received Boganda's death in relative calm. Some of his followers suggested that he had not died and would return to the public in the future. His funeral was held on 3 April outside of the Notre-Dame Cathedral in Bangui and attended by thousands, with Robert Lecourt representing the French government and Raymond Janot representing the French Community. Charles Féraille, a priest who had been personally acquainted with Boganda, declared that he had been "chosen by God" to lead the country. Goumba replaced him as interim President of the Council of Government, while Étienne Ngounio took over the office of Mayor of Bangui and the MESAN party presidency. Boganda's parliamentary constituency was declared vacant in the elections held on 5 April, which were handily won by MESAN, though with a sharp drop in voter turnout. With its founder gone, MESAN substantively ceased to exist aside as a label. +Dacko, with the backing of the French High Commissioner, the Bangui Chamber of Commerce, and Jourdain, offered himself as a candidate to lead the Council of Government. Goumba was hesitant to divide the populace, and after a month in power conceded the presidency to Dacko. Dacko became consumed with administrative work and, though he had initially retained Goumba as Minister of State, dismissed him after several months. In 1960 Goumba founded a new political party, the Democratic Evolution Movement of Central Africa (MEDAC), and claimed it carried the ideals of Boganda and MESAN. Frightened by its rapid growth, Dacko declared his intent to revive MESAN. Under his government, political focus moved away from the peasantry and was drawn to the creation of a new moneyed elite, mostly favouring officials who received large salaries. The Central African Republic received its full independence from France on 13 August 1960. Dacko pushed several measures through the Assembly which invested him as President of the Republic and head of state, and gave the government wide authority to suppress political opposition. By 1962 he had arrested Goumba and declared MESAN the sole party of the state. +The Legislative Assembly declared Boganda the "Father of the Nation" in June 1959. He was posthumously awarded Knight of the Legion of Honour, Grand Cross of the Order of Central African Merit, and Commander of the Order of Agricultural Merit. The Boganda National Museum, named for the former premier and hosted in his former Bangui residence, was opened in 1966. A secondary school and an avenue were also named in his honour, while a statue of him was erected at an independence memorial in the capital. Jean-Bédel Bokassa, leader of the Central African Republic between 1966 and 1979, promoted a cult of personality for Boganda as the founder of MESAN and the republic. Boganda Day is observed annually on 29 March to commemorate his death. +Mythical perceptions of Boganda's invulnerability persisted after his death, and his presence in Central African collective memory remains politically potent, serving as a unifying element among both the country's elite and the general populace. His phrase, zo kwe zo, was incorporated into the state's coat of arms. The preamble of the republic's 2004 constitution read, in part: "Animated by the wish of assuring to man his dignity with respect to the principle of 'ZO KWE ZO' enunciated by the Founder of the Central African Republic Barthélemy BOGANDA". Despite this, his political ideas have generally not been studied by successive Central African leaders. Historian Klaas van Walraven wrote, "his contemporary significance may lie precisely in the memory of his comportment and the widespread ignorance of his ideas". +Historian Georges Chaffard described Boganda as "the most prestigious and the most capable of Equatorial political men", while Prunier called him "probably the most gifted and most inventive of French Africa's decolonization generation of politicians". Historian Brian Titley suggested that Boganda's death "robbed the country of a charismatic leader" able to maintain legitimacy and in the long term facilitated General Bokassa's overthrow of Dacko and subsequent military takeover in 1966. +Boganda's life has some presence in French language histography, but much of what has been written about his biographical details, especially by Central African authors, is hagiographic in nature. His ideas and speeches have been more thoroughly incorporated into general analyses of political philosophies. Boganda is rarely mentioned in English historiography, and where he is included it is generally within the context of his Pan-African project of the United States of Latin Africa. +Bigo, Didier (1988). Pouvoir et obéissance en Centrafrique (in French). Karthala Editions. ISBN 978-2865372133. +Bradshaw, Richard; Fandos-Rius, Juan (2016). Historical Dictionary of the Central African Republic (4th, reprint ed.). Lanham: Rowman & Littlefield. ISBN 978-0810879928. +Chirot, Daniel (1996). Modern Tyrants: The Power and Prevalence of Evil in Our Age. Princeton: Princeton University Press. ISBN 0691027773. +Falola, Toyin; Jean-Jacques, Daniel, eds. (2015). Africa: An Encyclopedia of Culture and Society. ABC-CLIO. ISBN 978-1598846669. +Goumba, Abel (2007). Les Mémoires et les Réflexions (in French). Paris: Ccinia Communication. ISBN 978-2915568073. +Heyns, Christoph, ed. (1999). Human Rights Law in Africa. The Hague: Martinus Nijhoff Publishers. ISBN 9041102876. +Kalck, Pierre (1971). Central African Republic: A Failure in De-Colonisation. translated by Barbara Thomson. London: Pall Mall Press. ISBN 0269028013. +Kalck, Pierre (2005). Historical Dictionary of the Central African Republic. translated by Xavier-Samuel Kalck (3rd ed.). Lanham: The Scarecrow Press. ISBN 0810849135. +Kinata, Côme (2008). "Barthélémy Boganda et l'Église catholique en Oubangui-Chari" [Barthélémy Boganda and the Catholic Church in Oubangui-Chari]. Cahiers d'Études Africaines (in French). 48 (191): 549–565. doi:10.4000/etudesafricaines.12292. JSTOR 40379941. +Le Vine, Victor T. (2004). Politics in Francophone Africa: The States of West and Equatorial Africa. Boulder: Lynne Rienner Publishers. ISBN 1588262499. +O'Toole, Thomas E. (2019). The Central African Republic: The Continent's Hidden Heart. London: Routledge. ISBN 978-1000315134. +Prunier, Gérard (2009). Africa's World War: Congo, the Rwandan Genocide, and the Making of a Continental Catastrophe. Oxford: Oxford University Press. ISBN 978-0195374209. +Titley, Brian (1997). Dark Age: The Political Odyssey of Emperor Bokassa. Montreal: McGill-Queen's University Press. ISBN 0773516026. +van Walraven, Klaas (2017). "The Diaries of Barthélémy Boganda, Priest and Politician in French Equatorial Africa (1910–1959)". History in Africa. 44: 237–264. doi:10.1017/hia.2016.14. ISSN 1558-2744. S2CID 201721582. +van Walraven, Klaas (2019). "The historical long-term in the politics of the Central African Republic: Insights from the biography of Barthélémy Boganda (1910–1959)". ASC Working Paper (146). African Studies Centre Leiden. +Villalón, Leonardo A.; VonDoepp, Peter (2005). The Fate of Africa's Democratic Experiments: Elites and Institutions. Bloomington: Indiana University Press. ISBN 978-0253003119. +Yansané, Aguibou Y. (1984). Decolonization in West African States, with French Colonial Legacy: Comparison and Contrast : Development in Guinea, the Ivory Coast, and Senegal, 1945–1980. Schenkman Publishing Company. ISBN 978-0870733291. +Kalck, Pierre (1995). Barthélemy Boganda, 1910-1959: élu de Dieu et des Centrafricains (in French). Editions Sépia. ISBN 978-2907888585. +van Walraven, Klaas (2020). "Barthélémy Boganda between Charisma and Cosmology: Interpretive Perspectives on Biography in Equatorial African History". The Individual in African History: The Importance of Biography in African Historical Studies. Brill. ISBN 978-9004407824. +Rupert Leo Scott Bruce-Mitford (14 June 1914 – 10 March 1994) was a British archaeologist and scholar. He spent the majority of his career at the British Museum, primarily as the keeper of the Department of British and Medieval Antiquities, and was particularly known for his work on the Sutton Hoo ship-burial. Described as the guiding spirit of such research, he oversaw the production of the monumental three-volume work The Sutton Hoo Ship-Burial, termed by the president of the Society of Antiquaries as "one of the great books of the century". +Though Bruce-Mitford was born in London, his preceding two generations had lived largely abroad: his maternal grandparents as early settlers of British Columbia, his paternal grandparents as missionaries in India, and his parents as schoolteachers in Japan. When Bruce-Mitford was five, his father died. His mother was left to raise four sons, of whom Bruce-Mitford was the youngest, on a tiny salary; the stresses were substantial, and he was fostered for a time after his mother had a breakdown. He attended preparatory school with the support of a relative, enrolled at the charity school Christ's Hospital five years later, and, in 1933, earned a Baring Scholarship in History to attend Hertford College, Oxford. Recommending him for a museum curatorship in 1936, the University Appointments Board noted that he "has an exceptional gift for research, a sphere in which he could do work of outstanding merit". +After spending a year as an assistant keeper at the Ashmolean Museum, during which he produced the first standardised chronology of medieval pottery, in December 1937, Bruce-Mitford joined the British Museum's Department of British and Medieval Antiquities. The ship-burial was excavated in 1939, weeks before the outbreak of the Second World War; Bruce-Mitford spent 1940 to 1946 in the Royal Corps of Signals, and returned with responsibility over Sutton Hoo. Bruce-Mitford spent much of the next four decades focused on the subject, publishing dozens of works, studying contemporary graves in Scandinavia (excavating a boat-grave in Sweden and learning Swedish and Danish along the way), and leading a second round of excavations at Sutton Hoo from 1965 to 1970. +In his other duties, Bruce-Mitford excavated at the Mawgan Porth Dark Age Village, published significant works on the Lindisfarne Gospels, the Codex Amiatinus, and (posthumously) Celtic hanging bowls, translated P. V. Glob's book The Bog People into English, and oversaw acquisitions including Courtenay Adrian Ilbert's collection of thousands of clocks and watches, considered the greatest such collection in the world. He also founded the Society for Medieval Archaeology, and served as secretary, and later vice-president, of the Society of Antiquaries. After his retirement from the British Museum in 1977, he served as Slade Professor of Fine Art at the University of Cambridge, a visiting fellow at All Souls College, Oxford, and a faculty visitor in the Department of English at the Australian National University. +Rupert Leo Scott Bruce-Mitford was born on 14 June 1914 at 1 Deerhurst Road, Streatham, London. Following Terence, Vidal and Alaric (Alec), he was the fourth of four sons born to Charles Eustace and Beatrice Jean Bruce-Mitford; a daughter did not survive. Family tradition has it that Rupert's brothers were responsible for his given names, selecting them from their reading: Rupert from Anthony Hope's Rupert of Hentzau, Leo from Rider Haggard's She, and Scott from either Robert Falcon Scott's diary, or his "Message to England". +Bruce-Mitford's paternal great-grandparents, George and Elizabeth Beer, sailed to the Godavari River Delta in India to work as Baptist missionaries in 1836; poor and unordained when they left, they went on, according to Anthony Norris Groves's biographer, to "stand amongst the most tenacious Christian workers of all time". Their two sons, John William and Charles Henry, continued the calling, while their two daughters married schoolteachers in the area. In 1866 John Beer married Margaret Anne Midford, the daughter of an English family living in Machilipatnam. They had five children, including in 1871 Herbert Leonard and in 1875 Eustace, Rupert Bruce-Mitford's father. The family returned to Devon in 1884, when John Beer fell ill. He died shortly after arrival; his wife returned to India, but died there four years later. Eustace Beer, Rupert Bruce-Mitford would later write, was thereby "himself twice orphaned while still a small boy". By 1891 he was in England, having either returned or never left following his father's death. After studying in Exeter he taught English and Classics at Blackburn Grammar School, but then sailed from Genoa in 1901 to teach at the Weihaiwei School in China, an institution for European boys founded by his brother Herbert. He left less than nine months later for Japan, intending to establish his own school there with a curriculum and ethos reflecting his own ideas. +Shortly before his 1902 departure to China, Eustace Beer adopted the surname Bruce-Mitford—perhaps indicative of his desire to separate himself from his family's missionary past. "Mitford" was a take on "Midford", his mother's maiden name, and, perhaps not unintentionally, that of the unrelated Bertram Freeman-Mitford, 1st Baron Redesdale, whose name carried respect in the British expatriate community in Japan. "Bruce" may have been taken from Major Clarence Dalrymple Bruce, an acquaintance who commanded the Weihaiwei Regiment. In Japan Eustace founded the Yokohama Modern School, which targeted the sons of English, and English-speaking, businessmen and missionaries. In 1903, and likely on the basis of his book and articles on Weihaiwei, he was elected a fellow of the Royal Geographical Society; he subsequently became interested in geography and vulcanology, writing additional works on Japan. +Eustace Bruce-Mitford had met Beatrice Allison on his ship to Yokohama, and soon after founding his school recruited her as an assistant teacher; they married on 27 July 1904, at Christ Church, Yokohama. She was the eldest daughter of early settlers of British Columbia, Susan Louisa (née Moir) and John Fall Allison, an explorer, gold prospector, and cattle rancher. In 1908, however, by which time the family had three sons, William Awdry, the Bishop of South Tokyo, announced from the pulpit of Christ Church that "certain marriages of British subjects celebrated in Japan" might not be legally valid, and if so "the couples ... will find that they have been and are living together ... in concubinage and that their children are 'illegitimate'". Though a legal technicality, and one which was remedied by an Act of Parliament in 1912, the announcement disgraced the Bruce-Mitfords, and Eustace lost his leadership of the Yokohama Modern School. He was taken on as an assistant editor by Francis Brinkley, owner and editor of The Japan Mail, though by 1911 had returned to England as a freelance journalist. Rupert Bruce-Mitford was born three years after his family returned from Japan. Three years later, his father left for India to work as an assistant editor at The Madras Mail. Eustace died following a short fever in 1919, when he was forty-four and Rupert five. +Following the death of his father, Bruce-Mitford later wrote, "the family was stranded in London and fell on very hard times". His mother then earned roughly £170 a year (equivalent to £8,900 in 2025), of which she lent £120 (equivalent to £6,300 in 2025) to Terence and Vidal, to be repaid after their studies, and spent approximately £42 18s yearly (equivalent to £2,250 in 2025) for part of a house, with rent paid weekly. Bruce-Mitford was frequently sick as a child, coming down with scarlet fever and diphtheria when aged two, and influenza when around six. The stresses on the family were substantial, and at one point Beatrice had a breakdown, causing Rupert to be fostered for a time. +Fatherless and poor, Rupert Bruce-Mitford was educated with the financial support of his mother's cousin. The support hinged, Bruce-Mitford later wrote, "on one condition – that my father's novel, depicting life in Yokohama at the turn of the century, should be burnt; she thought it immoral and scurrilous". Around 1920, Bruce-Mitford was thereby sent to Brightlands preparatory school in Dulwich, London, which his brothers Terence and Alec also attended before receiving scholarships to Dulwich College. Bruce-Mitford was baptised around the same time, perhaps to improve his later chances of admittance to the charity school Christ's Hospital. Five years later the Brightlands headmaster nominated Bruce-Mitford to take an examination for Christ's Hospital. Following success in the examination—covering the three compulsory subjects of English, arithmetic and practical mensuration, and history and geography, as well as all three optional subjects of Latin, French, and mathematics—and his mother's application, he was admitted on 17 September 1925. +Bruce-Mitford was successful, and happy, at Christ's Hospital. He was also introduced to archaeology; in 1930 he participated in a dig with S. E. Winbolt at the Jacobean ironworks in Dedisham, Sussex. Winbolt wrote in the school magazine that "unhappily the 'dig' produced no useful results", but added that "possibly, however, the C.H. diggers learnt something", and named Bruce-Mitford "among willing helpers, mentioned honoris causa". Meanwhile, Bruce-Mitford was active in school events, including playing rugby and cricket, acting in (and directing the orchestra for) John Galsworthy's The Little Man, debating at the Horsham Workers' Educational Association, and writing his first article, on a ten-day signals camp held over the 1931 summer holiday. +By the time Bruce-Mitford was 16 or 17, he had switched his studies from classics to history—the result, in no small part, of his shaky grasp of Greek and Latin, despite devoted tutoring by his brother Terence. Around the same time, he came across Samuel Gardner's English Gothic Foliage Sculpture in the school's library, and upon reading it discovered his love of the concrete and visual. In 1933, he was awarded a £100 (equivalent to £6,200 in 2025) Baring Scholarship in History to attend Hertford College, Oxford. This came as surprise, he wrote, "for I never had a head for dates and treaties". But as he recalled four decades later, at Oxford "I fell in love with the atmosphere and smell of the oldest part of the Library where, under the flat-arched 15th century ceiling, cases displaying illuminated manuscripts were set out". One, the twelfth-century Ashmole Bestiary, open to a folio of a red eagle on a background of gold, so captured his attention that "after some weeks I could stand my ignorance and quell my curiosity no longer", and, "[s]crewing up my courage", asked for permission to see it; he remained absorbed in the work through lunch and until evicted at the end of the day. +During school vacations, Bruce-Mitford often took the tram to the British Museum, where he spent time in the Reading Room. He also walked around the building, listening to guest lecturers speak on the objects, and particularly enjoying hearing about the Chinese paintings and the Royal Gold Cup. In 1936, he obtained his bachelor's degree with a Second Class in Modern History, and in Michaelmas term began a postgraduate Bachelor of Letters on "The Development of English Narrative Art in the Fourteenth Century". The research included an investigation of the pigments (particularly Egyptian blue) used in early illuminated manuscripts. Bruce-Mitford's supervisor was Robin Flower, deputy keeper of manuscripts at the British Museum. The same year, the University Appointments Board recommended Bruce-Mitford for the curatorship of the York Castle Museum, writing that he "would do well in a trading or administrative post, but has an exceptional gift for research, a sphere in which he could do work of outstanding merit". He never finished the Bachelor of Letters, although he was promoted to the rank of Master of Arts in 1961 and awarded a Doctor of Letters in 1987, both from Hertford College. +By 1937, Bruce-Mitford had taken a one-year position as an assistant keeper at the Ashmolean Museum. Initial work included rearranging and displaying the museum's collection of medieval pilgrims' badges. Soon, however, he was introduced to what would be later termed rescue archaeology, when a group of seventeenth-century houses was demolished in favour of a large extension to the Bodleian Library. Before the demolition, the head of the library invited the Oxfordshire Architectural and Historical Society to investigate the houses for their architectural or antiquarian interest, and observe the forthcoming demolition and excavations of foundations. The Society, in turn, created a subcommittee consisting of Edward Tudor Long, Edward Thurlow Leeds, and William Abel Pantin to direct an investigation of the houses and watch for finds during their demolition. +Demolition lasted from December 1936 to March 1937, after which began, according to the geologist William Joscelyn Arkell, "the removal of the greatest quantity of subsoil ... ever taken out of one hole within the City of Oxford". Bruce-Mitford was tasked with watching the site during the excavation. Much of his work involved waiting for the well in front of each house to be dug out, revealing two or three feet (around half a metre to a metre) of mud at the bottom, filled with broken medieval pottery and other artefacts. He waited "impotently", he later recalled, for the jaws of the mechanical diggers (which would not wait for the archaeologists) to pick up the mud and transfer it to a lorry; he then jumped aboard, and picked out the artefacts as the lorry made its way "to some gravel hungry site at Cumnor". When back at the Ashmolean, he would wash the sherds and stick them together. Bruce-Mitford's energy and eye resulted in a collection of pottery that was exceptionally useful. Because the wells quickly silted up during their use and had to be replaced by new ones every 50 or so years, Bruce-Mitford found it possible to accurately date pottery within particularly short time-frames. In 1939, he published an article on the finds, to be read in conjunction with an article by Pantin, in which he described five distinct groups of pottery in their probable chronological order; his brother Alaric provided the illustrations. This was "the first serious study of medieval pottery", wrote the archaeologist Maureen Mellor in 1997, and "has never had to be challenged, although refined and extended". Bruce-Mitford's work also influenced him, decades later, to create a national reference collection of medieval pottery at the British Museum. +On 3 December 1937, Bruce-Mitford was named assistant keeper (second class) of the Department of British and Medieval Antiquities at the British Museum. He was possibly helped in this position by his professor from two years previously, Robin Flower, also the deputy keeper of manuscripts. The following year Bruce-Mitford was reacquainted with archaeological work, spending three weeks with Gerhard Bersu at the Iron Age site of Little Woodbury in Wiltshire. "I learned a lot", he later wrote, "and loved being out on the chalk, in the fresh air." There, Bruce-Mitford met Charles Phillips, the secretary of the Prehistoric Society (for which Bersu was digging). +In 1939 Bruce-Mitford was tasked with leading an excavation, this time at the medieval village of Seacourt in Oxfordshire. Though Seacourt was a difficult site, Bruce-Mitford thought it would be possible to determine a complete ground plan of domestic buildings and of the church. It was also a village that was known to have been deserted and archaeologically sealed by 1439, within a century of the Black Death. This provided a terminus ante quem for any artefacts found there, offering the chance to obtain important chronological evidence as to pottery and small objects such as brooches, ornaments, bucks, fittings, shears, horseshoes, and nails, the dating of which was particularly imprecise. Excavations wrapped up 15 July 1939, seven weeks before Britain's entry into the Second World War. +From 1940 to 1946, Bruce-Mitford served in the Royal Corps of Signals. Joining as a lance corporal and initially assigned to a territorial unit in Essex, he transmitted morse code during the day, after which he watched for fires from the dome of St Paul's Cathedral. He was in Catterick Camp in North Yorkshire by autumn, when The Yorkshire Archæological Journal reported that he and his friends cleared out a Roman hypocaust at Middleham. The site had previously been excavated around 1881 and 1906. Bruce-Mitford's notes were published in The Journal of Roman Studies, with photographs taken by Eric Lomax. +Bruce-Mitford was commissioned as a second lieutenant on 1 February 1941, and promoted to war substantive lieutenant on 1 August 1942, acting captain on 20 November 1942, and temporary captain on 26 February 1943. By 1943 he was working on the publications staff of the Royal School of Signals at Catterick, where he wrote a booklet on wireless communication, attempted to reorganise the Northern Command's signals system, and travelled around Yorkshire by motorcycle, laying cable. From 1943 to 1945, he led parties from the School of Signals to archaeological and other sites across Northern England, including Richmond Castle, Jervaulx Abbey, Easby Parish Church, Stanwick St John, Middleham Castle, and the Georgian Theatre Royal, recording notes and commentaries when there. +Bruce-Mitford spent the war awaiting his return to the Department of British and Medieval Antiquities. As early as 1940, T. D. Kendrick—then keeper of the department, later director of the museum—wrote to Bruce-Mitford at his army camp, telling him he would be responsible for the collection of Anglo-Saxon antiquities, the Germanic collections of Europe, and the Late Celtic collections of the British Isles. The letter closed with a warning: "You will also be responsible for Sutton Hoo. Brace yourself for this task." Bruce-Mitford's responsibility for the Anglo-Saxon Sutton Hoo ship-burial, wrote the Oxford scholar Martin Biddle, would become "the defining moment of Rupert's life, his greatest challenge, the source of almost insuperable difficulties, and his greatest achievement". Discharged from the army as an honorary captain in early 1946, Bruce-Mitford immediately returned to the museum; the first lecture he attended after discharge was by Phillips on Sutton Hoo. +Bruce-Mitford returned to a museum that had suffered during the war. Understaffed and with inadequate facilities, the museum had much of its collection still in storage. The Sutton Hoo finds, excavated in 1939 and promptly taken to the safety of the tunnel connecting the Aldwych and Holborn tube stations, had been returned to the museum only a year or two before. Herbert Maryon, a technical attaché recruited for the task, set to work restoring what Bruce-Mitford later termed "the real headaches – notably the crushed shield, helmet and drinking horns". "When I began work", he continued, "I sat with Maryon while he took me through the material and with infectious enthusiasm, demonstrated what he was doing". "There followed great days for Sutton Hoo when new, often dramatic, discoveries were being made in the workshops all the time. Built from fragments, astonishing artefacts – helmet, shield, drinking horns, and so on – were recreated." +Early in 1946, Kendrick and Bruce-Mitford placed restored artefacts from Sutton Hoo on display in the museum's King Edward VII Gallery. In January 1947, Bruce-Mitford was elected a fellow of the Society of Antiquaries of London, and the museum published The Sutton Hoo Ship-Burial: A Provisional Guide, which Bruce-Mitford had written and produced during evenings at his kitchen table. The work quickly proved to be one of the museum's most successful publications, going through ten impressions even before the second edition was issued. Also in 1947, in what he later termed "one of the most rewarding experiences of my life", he visited Sweden for six weeks at the invitation of the archaeologist Sune Lindqvist. Bruce-Mitford studied the similar finds from Vendel and Valsgärde and helped Lindqvist excavate the boat-grave from Valsgärde 11, learning Swedish along the way. +Although Bruce-Mitford continued to write prolifically on Sutton Hoo—he quickly became, in the words of Francis Peabody Magoun, the "spiritus rector of present-day Sutton Hoo research"—a definitive publication remained elusive. Writing a memorandum to Kendrick in May 1949, Bruce-Mitford outlined a plan for "[t]hree large volumes" and possibly a fourth, buttressed by "a formidable array of technical reports obtained at my request from outside scientists", and accompanied by the "hope that the publication will set a new standard in archaeological publication". But he admitted that he could not see "any real prospect of getting the catalogue out the way things are at present". The museum had other needs, and Bruce-Mitford other responsibilities; the country plunged into the Korean War, and resources were diverted elsewhere. In a 1957 addendum to his 1949 memorandum, Bruce-Mitford wrote "[t]here was no reply to this report". +Throughout the 1950s, Bruce-Mitford's attention was directed away from Sutton Hoo. In these "fallow years for Sutton Hoo", as Biddle termed them, germinated many of the other contributions of Bruce-Mitford's career. He began two projects that would result in posthumous publications: excavations at the Mawgan Porth Dark Age Village in Cornwall from 1949 to 1952 (continued in 1954 and 1974), and the compilation of information on Celtic hanging bowls. He undertook another effort in 1955, unsuccessfully searching Lincoln Cathedral with Sir Wilfred Le Gros Clark and Harold Plenderleith for the burial place of Saint Hugh of Lincoln. +Also starting in the 1950s, Bruce-Mitford began to shoulder increasing roles and responsibilities, including deputy keepership of the Department of British and Medieval Antiquities in February 1954, full keepership upon the retirement of A. B. Tonnochy that August, appointment to the Ancient Monuments Board for England that same year (serving until at least 1975), membership in the Museums Association by 1955, and founder and president of the Society for Medieval Archaeology in 1957. In the latter half of the decade Bruce-Mitford became a semi-regular participant in the show Animal, Vegetable, Mineral?, appearing in 1955, 1956, 1958, and 1959. In 1960, he opened an exhibition, "Archaeology from the Air", at the Victoria Galleries in Kingston upon Hull. +This time also saw Bruce-Mitford's primary work on early medieval manuscripts. In 1956 and 1960, he published a two-volume work—facsimile and commentary—on the Lindisfarne Gospels. The work was his first major publication; the museum gave him four months' leave to focus on the work, including time at the Royal Library in Copenhagen and the Laurentian Library in Florence. The result, according to the journal Antiquity, was "magistral". While at the Laurentian, Bruce-Mitford also studied the c. 700 AD bible known as the Codex Amiatinus, eventually resulting in a Jarrow Lecture on the subject in 1967. Around the same time, he translated P. V. Glob's 1965 book The Bog People from Danish to English, with the result published in 1969. +Though Bruce-Mitford helped secure acquisitions throughout his 21 years as keeper, two of his most significant efforts came in 1958. That year, the museum purchased the Lycurgus Cup from Victor Rothschild, 3rd Baron Rothschild, for £20,000 (equivalent to £542,000 in 2024). The museum also purchased Courtenay Adrian Ilbert's collection of clocks and watches. This, wrote Biddle, was "the greatest collection of horology in the world", and Bruce-Mitford's "greatest coup". After Ilbert died in 1956, his collection—some 210 clocks and 2,300 watches and watch movements—was set for auction at Christie's. Although the treasury declined a request for funds, Bruce-Mitford approached the Worshipful Company of Clockmakers, which was able to secure a donor to purchase the clocks for the museum. The company then again approached the treasury, the head of which this time agreed to petition parliament for funds to purchase the remainder of the collection. The collection was purchased for the museum, and the company made Bruce-Mitford a liveryman. +Shortly thereafter, in 1960, Bruce-Mitford embarked on an ultimately unsuccessful two-year attempt to acquire what would become known as the Cloisters Cross. The ivory cross, which a panel of experts at the museum declared "one of the finest and most impressive objects of the 12th century they [had] ever seen", appeared at the museum on 5 December 1960, prior to which it was virtually unknown; Bruce-Mitford was alerted by an urgent note that a man had arrived with an ivory cross, and that it was his last day in London. Bruce-Mitford studied the cross over the ensuing two years—including four days spent in a bank vault in Zurich—assembled a file an inch and a half thick, and successfully persuaded the treasury to allocate £195,000 (equivalent to £5,290,000 in 2024) for its purchase. But as unsavoury rumours swirled around the owner, Ante Topić Mimara, and doubts persisted that the unprovenanced cross might be Nazi loot, Mimara steadfastly refused to disclose how he had obtained possession. With the museum unwilling to pay without this information and Mimara unwilling to disclose it, the museum's option expired at midnight on 31 January 1963. The Metropolitan Museum of Art had been waiting in the wings; its curator Thomas Hoving, untroubled by the issues with the cross and owner, stayed up over dinner with Mimara and purchased it immediately. +Frank Francis succeeded Kendrick as director in 1959, and the following year made two floors of a Montague Street house available for Bruce-Mitford to devote to the Sutton Hoo operation. A research assistant was added in 1962 and, eventually, thirteen people were involved. By then, criticism over the delays in publication had begun to mount; a 1964 article by Christopher Hawkes lamented the fact that "[a] quarter of a century has passed ... and Sutton Hoo is still not published", and concluded that the museum "really must go to it". Much of the criticism landed on the shoulders of Bruce-Mitford, leading the archaeologist Robert T. Farrell to observe that "it has become something of a trend to slate Bruce-Mitford for delay". After the volumes were ultimately published, Hawkes and his wife Sonia went as far as to translate, privately publish, and distribute amongst colleagues a biting German review by Joachim Werner. Sir David M. Wilson, who became director of the museum in 1977, disparaged Bruce-Mitford in his 2002 book The British Museum: A History, and wrote that work on the Sutton Hoo volumes "presents a precautionary, if unique, tale of procrastination and obfuscation". To others, including Biddle and Rosemary Cramp, while Bruce-Mitford's perfectionism and penchant for detail did not speed matters, the criticism failed to account for the many other demands on his time, and the museum's delay in making resources available for the publication. Nor, wrote Biddle, did it appreciate the scope of the undertaking, which involved modernising multiple fields of research—such as the proper chronology of Merovingian coins, used to date the burial; the study of textiles; and the study of soil, used to determine whether the burial was a grave or a cenotaph—and the re-excavation of the ship-burial. +From 1965 to 1970, Bruce-Mitford led another round of excavations at Sutton Hoo in order to acquire more information about the mound, the ship, and the burial. Especially given that the original two-week excavation had been an exercise in rescue archaeology, it was hoped that an excavation without the looming spectre of war might discover items or features that had been missed. In 1968, Bruce-Mitford updated the Provisional Guide with a new edition, now entitled The Sutton Hoo Ship-Burial: A Handbook; second and third editions of the Handbook followed in 1972 and 1979. In 1973, he delivered the second G. N. Garmonsway Memorial Lecture at the University of York, speaking on Sutton Hoo, and in 1974, he published Aspects of Anglo-Saxon Archaeology. The book contained twelve updated and rewritten chapters that had appeared elsewhere, along with four new chapters; originally conceived as a reference work which the forthcoming Sutton Hoo publication would cite throughout, it also provided a more high-level overview of certain portions of the burial. +The first volume of The Sutton Hoo Ship-Burial finally appeared in 1975. It focused on the larger issues of the burial—such as the site, the excavations, the ship, the date of the burial, and the questions of whom it honoured, and whether or not it was a cenotaph. The book was widely welcomed; A. J. Taylor, then president of the Society of Antiquaries, hailed the publication as "one of the great books of the century". The book was "splendid", wrote Birgit Arrhenius, and to Biddle, "the firm foundation on which all subsequent work on the site and on the contents of the mound will safely rest". Cramp wrote of "the consistently high standard of argument and presentation" in the work, and even beyond the specific site of Sutton Hoo, the art critic Terence Mullaly suggested the book "sets standards for archaeologists everywhere". The authors and the museum, James Graham-Campbell wrote in Antiquity, "merit every congratulation on the appearance of this first volume". +Also in 1975, Bruce-Mitford relinquished his role as keeper of the Department of British and Medieval Antiquities to spend two years as research keeper. This period was the only time during his career which he was able to devote exclusively to Sutton Hoo. Amidst some internal conflict, some of the museum's trustees, led by Eric Fletcher, Baron Fletcher, sought to terminate the keepership six months early; the effort was thwarted by the advocacy of Sir Grahame Clark and A. J. Taylor, who had seen the first volume of The Sutton Hoo Ship-Burial shortly after printing, and were able to report being "impressed ... by the copiousness and quality of the volume". During this period, in 1976, Bruce-Mitford was also elected a fellow of the British Academy. +The second volume of The Sutton Hoo Ship-Burial, covering arms, armour, and regalia, followed in 1978. The work was again widely praised, with Graham-Campbell calling it a "tour de force", Farrell a "very great accomplishment", and Vera Evison a "mighty effort" with "no sign of limitation on the length of the text or the number and scale of the illustrations". This resulted in what J. N. L. Myres termed an "open-ended approach" that emphasised exploring judgement calls in reconstructions and interpretations rather than presenting conclusions as unimpeachable. The third volume, published in two parts and covering many items including the silver objects, hanging bowls, drinking vessels, textiles, and lyre, came in 1983. Writing of the series as a whole, Farrell termed it "a series which in scope and format is one of the most monumental to have been undertaken in the field of medieval archaeology", Catherine Hills "[a] monument of publication [which] enshrines the original monument", and Myres "a truly heroic achievement in archaeological publishing". Martin Carver, who led research and excavation at Sutton Hoo after Bruce-Mitford, called the publication "the most compendious ever produced for a British archaeological site". +A fourth volume had been planned to offer thoughts on the relationship between Sutton Hoo and Beowulf, the connection between Sutton Hoo and Sweden, studies of comparable materials, and any needed revisions, although its fate was uncertain even after volume one was in press. The plan was ultimately shelved in the face of financial considerations and a desire to focus on facts rather than interpretation. Writing in the preface to the final volume, Bruce-Mitford stated that "the Trustees felt that with the completion of the third volume their essential responsibility to archaeology had been discharged, the factual record being completed with Volume 3". +Bruce-Mitford retired from the British Museum after his research keepership ended in 1977. From 1978 to 1979, he served as Slade Professor of Fine Art at the University of Cambridge, and in the same year served as a visiting fellow at All Souls College, Oxford. In 1981, he took his last position at the Australian National University, where he was a faculty visitor in the Department of English. Two years later, he was elected an honorary fellow of Hertford College, his alma mater. +In his years after leaving the British Museum, Bruce-Mitford worked to bring old works to publication. He finished his work on the excavations at Mawgan Porth; the book was published posthumously in 1997. He also resumed work on A Corpus of Late Celtic Hanging-Bowls, on which he had begun work in the 1940s, and which was finished by Shiela Raven and published in 2005. The result, a 500-page tome with 800 illustrations, was reviewed as "a fitting memorial to Dr Bruce-Mitford whose contribution to early medieval archaeology—and to metalwork studies in particular—was immense". +Bruce-Mitford was married three times and had three children. In November 1941 he married Kathleen Dent, with whom he fathered Myrtle (b. 1943), Michael (b. 1946), and Miranda (b. 1951). A professional cellist, Myrtle Bruce-Mitford herself contributed to the Sutton Hoo finds, being employed by the British Museum to work on the remnants of the lyre and co-authoring a paper with her father. She was also the longtime partner of Nigel Williams, who from 1970 to 1971 reconstructed the Sutton Hoo helmet. +Bruce-Mitford's relationship with Dent was long troubled, and he left home in the later 1950s (with the marriage officially dissolved in 1972) and formed a series of relationships. In March 1975 he and his former research assistant Marilyn Roberta Luscombe announced their engagement, marrying on 11 July. The two had met eight years prior, when Bruce-Mitford was interviewing her for the position; knowing Bruce-Mitford's work but believing him to be dead, Luscomb said she spoke at length about one of his papers before realising she was interviewing with its author. The marriage was dissolved in 1984, at which point Bruce-Mitford found it necessary to sell his library, which went to Okinawa Christian Junior College in Japan. In 1986 he married for a third time, to Margaret Edna Adams, a child psychiatrist and published poet, whom he had met at Oxford fifty years before. +In 1987, Bruce-Mitford visited British Columbia, the homeland of his mother's family. He met cousins living on a reservation there and, Biddle wrote, was "deeply moved" by The Lake, an opera about early life in the Okanagan Valley. Biddle noted that Bruce-Mitford "was fascinated by his family's background" in both British Columbia and Japan; Bruce-Mitford wrote two essays to feature in the 1989 auction catalogue for his library, one of which was titled "My Japanese Background". +After years of inherited heart disease, on 10 March 1994 Bruce-Mitford died of a heart attack at the John Radcliffe Hospital; he had driven himself there two days before. He was buried eight days later by St Mary's Church in Bampton, Oxfordshire. The Guardian recalled him as amongst "that tiny band of scholars whose names are linked with great archaeological discoveries". A service was held at St George's, Bloomsbury on 14 June, with the address given by Biddle. Additional obituaries were published in The Daily Telegraph, The New York Times, The Independent, Old English Newsletter, and Medieval Archaeology. Biddle's memorial address was later published in The Hertford College Magazine, and in 2015 expanded into a bibliographic memoir published by the British Academy. Bruce-Mitford's widow, Margaret Edna Adams, died in 2002. +Bruce-Mitford was a fan of boxing, rugby, and cricket. He was a member of Marylebone Cricket Club, and frequented the Athenaeum and the Garrick. An avid reader of the books of Dick Francis, he was fond of recalling a two-day transcontinental train journey in Australia during which he shared a compartment with the author. +Many of the works below are listed in the 1989 catalogue of Bruce-Mitford's library produced in preparation for its sale. The first 156 items in the catalogue are works by or about Bruce-Mitford; Bruce-Mitford's personal copy is held by Columbia University's Avery Architectural and Fine Arts Library, and contains 14 additional works added by hand. +Bruce-Mitford, Rupert (1947a). The Sutton Hoo Ship-Burial: A Provisional Guide. London: Trustees of the British Museum. OCLC 869758063. +——— (2 November 1951). The Society of Antiquaries of London: Notes on its History and Possessions. London: Society of Antiquaries of London. OCLC 1030053915. +Kendrick, Thomas Downing; Brown, Thomas Julian; Bruce-Mitford, Rupert; Roosen-Runge, Heinz; Ross, Alan Strode Campbell; Stanley, Eric Gerald & Werner, Alfred Emil Anthony, eds. (1956). Codex Lindisfarnensis. Vol. I. Olten: Urs Graf. OCLC 1440544. +Kendrick, Thomas Downing; Brown, Thomas Julian; Bruce-Mitford, Rupert; Roosen-Runge, Heinz; Ross, Alan Strode Campbell; Stanley, Eric Gerald & Werner, Alfred Emil Anthony, eds. (1960). Codex Lindisfarnensis. Vol. II. Olten: Urs Graf. OCLC 1440544. +Bruce-Mitford, Rupert (1968). The Sutton Hoo Ship-Burial: A Handbook. London: Trustees of the British Museum. SBN 7141-1320-4. +——— (1972). The Sutton Hoo Ship-Burial: A Handbook (2nd ed.). London: Trustees of the British Museum. SBN 7141-1330-1. +——— (1979). The Sutton Hoo Ship-Burial: A Handbook (3rd ed.). London: British Museum Publications Ltd. ISBN 0-7141-1343-3. +Glob, P. V. (1969). The Bog People: Iron Age Man Preserved. Translated by Bruce-Mitford, Rupert. London: Faber and Faber. OCLC 1015925879. +Published in United States as Glob, P. V. (1969). The Bog People: Iron Age Man Preserved. Translated by Bruce-Mitford, Rupert. Ithaca: Cornell University Press. OCLC 13404. +Bruce-Mitford, Rupert (1974a). Aspects of Anglo-Saxon Archaeology: Sutton Hoo and Other Discoveries. London: Victor Gollancz Limited. ISBN 0-575-01704-X. +Published in the United States as ——— (1974a). Aspects of Anglo-Saxon Archaeology: Sutton Hoo and Other Discoveries. New York: Harper's Magazine Press. LCCN 73-13220. SBN 06-120480-3. +———, ed. (1975). Recent Archaeological Excavations in Europe. London: Routledge & Kegan Paul. ISBN 0-7100-7963-X. +——— (1975). The Sutton Hoo Ship-Burial, Volume 1: Excavations, Background, the Ship, Dating and Inventory. London: British Museum Publications Limited. ISBN 0-7141-1334-4. +——— (1978a). The Sutton Hoo Ship-Burial, Volume 2: Arms, Armour and Regalia. London: British Museum Publications. ISBN 0-7141-1335-2. +——— (1983a). The Sutton Hoo Ship-Burial, Volume 3: Late Roman and Byzantine Silver, Hanging-Bowls, Drinking Vessels, Cauldrons and Other Containers, Textiles, the Lyre, Pottery Bottle and Other Items. Vol. I. London: British Museum Publications. ISBN 0-7141-0529-5. +——— (1983b). The Sutton Hoo Ship-Burial, Volume 3: Late Roman and Byzantine Silver, Hanging-Bowls, Drinking Vessels, Cauldrons and Other Containers, Textiles, the Lyre, Pottery Bottle and Other Items. Vol. II. London: British Museum Publications Limited. ISBN 0-7141-0530-9. +——— (1979). The Sutton Hoo Ship-Burial: Reflections After Thirty Years. University of York Medieval Monograph Series. Vol. 2. York: William Sessions. ISBN 0-900657-46-4. +——— (1997). Taylor, Robin J. (ed.). Mawgan Porth: A Settlement of the Late Saxon Period on the North Cornish Coast: Excavations 1949–52, 1954 and 1974. London: English Heritage. doi:10.5284/1028203. ISBN 978-1-848-02186-0. +——— & Raven, Sheila (2005). A Corpus of Late Celtic Hanging-Bowls with an Account of the Bowls Found in Scandinavia. Oxford: Oxford University Press. ISBN 978-0-19-813410-7. +Bruce-Mitford, Rupert (July 1938). "A Hoard of Neolithic Axes from Peaslake, Surrey". The Antiquaries Journal. XIII (3). London: Society of Antiquaries of London: 279–284. doi:10.1017/S0003581500094737. S2CID 164198113. +Correction issued in "The Peaslake Hoard". The Antiquaries Journal. XIX (1). London: Society of Antiquaries of London: 94. January 1939. doi:10.1017/S0003581500094907. S2CID 163195581. +——— (June 1939). "Two Medieval Pottery Vessels". The British Museum Quarterly. XIII (2). London: The Trustees of the British Museum: 35–38. doi:10.2307/4422140. JSTOR 4422140. +——— (1939). "The Archaeology of the Site of the Bodleian Extension in Broad Street, Oxford" (PDF). Oxoniensia. IV. Oxford: Oxfordshire Architectural and Historical Society: 89–146. ISSN 0308-5562. +——— (July 1939). "Anglo-Saxon Brooch and Pot from Brixworth, Northants". The Antiquaries Journal. XIX (3). London: Society of Antiquaries of London: 325–326. doi:10.1017/S0003581500095081. S2CID 164096144. +Skinner, Frederick George & Bruce-Mitford, Rupert (January 1940). "A Celtic Balance-beam of the Christian Period". The Antiquaries Journal. XX (1). London: Society of Antiquaries of London: 87–102. doi:10.1017/S0003581500045595. S2CID 162218406. +Bruce-Mitford, Rupert (January 1940a). "Medieval Tripod Pitchers". The Antiquaries Journal. XX (1). London: Society of Antiquaries of London: 103–112. doi:10.1017/S0003581500045613. S2CID 162835240. +——— (1940b). "The Excavations at Seacourt, Berks, 1939: An Interim Report" (PDF). Oxoniensia. V. Oxford: Oxfordshire Architectural and Historical Society: 31–41. ISSN 0308-5562. +——— & Jope, E. M. (1940). "Eleventh and Twelfth Century Pottery from the Oxford Region" (PDF). Oxoniensia. V. Oxford: Oxfordshire Architectural and Historical Society: 42–49. ISSN 0308-5562. +——— (September 1946). "Sutton Hoo Ship-Burial". East Anglian Magazine. 6 (1): 2–9, 43. ISSN 0012-8392. +Myres, J. N. L.; Hawkes, C. F. C.; Bruce-Mitford, Rupert & Radford, C. A. Ralegh (1946). "The Archaeology of Lincolnshire and Lincoln: Anglian and Anglo-Danish Lincolnshire" (PDF). The Archaeological Journal. CIII. London: The Royal Archaeological Institute of Great Britain and Ireland: 85–101. doi:10.1080/00665983.1946.10853806. +Bruce-Mitford, Rupert (January 1947b). "Swedish Museums and the Public". The Museums Journal. 46 (10). London: The Museums Association: 188–194. ISSN 0027-416X. +——— (June 1947). "Anglo-American Museums Officers Visit to Finland (September–October, 1946)". The Museums Journal. 47 (3). London: The Museums Association: 41–51. ISSN 0027-416X. +——— (1948). "Saxon Rendlesham: Some Preliminary Considerations" (PDF). Proceedings of the Suffolk Institute of Archaeology. XXIV (3). Ipswich: 228–251. Archived (PDF) from the original on 19 February 2016. Retrieved 12 February 2026. +Edited and republished in Bruce-Mitford 1974a, ch. 2 +——— (1948). "Sutton Hoo och Sverige". Ord och Bild. 57. Stockholm: Sven Rinman: 97–104. +——— (May 1948). "Sutton Hoo and Sweden". The Archaeological News Letter. 1 (2). Linden Publicity: 5–7. ISSN 0003-8040. +Lindqvist, Sune (September 1948). "Sutton Hoo and Beowulf". Antiquity. XXII (87). Translated by Bruce-Mitford, Rupert. Gloucester: John Bellows Ltd.: 131–140. doi:10.1017/S0003598X00019669. S2CID 164075295. +Bruce-Mitford, Rupert (October 1948). "Medieval Archaeology". The Archaeological News Letter. 1 (6). Linden Publicity: 1–4. ISSN 0003-8040. +——— (November–December 1948). "The Sutton Hoo Ship-Burial". The Norseman. VI (6): 394–400. OCLC 1760471. +——— (1949). "The Sutton Hoo Ship-Burial: Recent Theories and Some Comments on General Interpretation" (PDF). Proceedings of the Suffolk Institute of Archaeology. XXV (1). Ipswich: 1–78. +Summarised, with Bruce-Mitford's input, in Crawford, O. G. S. (March 1952). "Sutton Hoo? A Summary". Antiquity. XXVI (101): 4–8. doi:10.1017/S0003598X00021232. S2CID 246041741. +Edited and republished in Bruce-Mitford 1974a, ch. 1 +Bruce-Mitford, Rupert (1950). "The Sutton Hoo Ship-Burial". Proceedings of the Royal Institution of Great Britain. XXXIV (III). London: The Royal Institution of Great Britain: 440–449. ISSN 0035-8959. +——— & Hill, George Francis (January–April 1950). "A Note by Sir George Hill on the Sutton Hoo Treasure Trove Inquest". The Antiquaries Journal. XXX (1–2). London: Society of Antiquaries of London: 67–68. doi:10.1017/S0003581500088193. S2CID 163275205. +——— (March 1950). "The Problem of the Sutton Hoo Cenotaph". The Archaeological News Letter. 2 (10). Linden Publicity: 166–169. ISSN 0003-8040. +——— (4 March 1950). "The Sutton Hoo Ship-Burial". Nature. CLXV (4192). Nature Publishing Group: 339–341. Bibcode:1950Natur.165..339B. doi:10.1038/165339a0. +——— (April 1950). "The Sutton Hoo Ship-Burial: A New Chapter in Anglo-Swedish Relations". The Anglo-Swedish Review. London: The Swedish Chamber of Commerce for the United Kingdom: 69–72. OCLC 51228631. +——— (October–November 1950). "3rd International Congress of Prehistoric and Protohistoric Sciences, Zurich, August 14th to 19th, 1950". The Archaeological News Letter. 3 (5). Linden Publicity: 78–79. ISSN 0003-8040. +——— (1951). "Anglo-Saxon Suffolk" (PDF). The Archaeological Journal. CVIII. Royal Archaeological Institute: 132–133. +——— (1951). "A Late-Medieval Chalk-Mine at Thetford" (PDF). Norfolk Archaeology. XXX (III). Norwich: Norfolk and Norwich Archæological Society: 220–222. +——— (January–April 1951). "A Saxon Jewelled Circular Brooch from Long Bennington, Lincs". The Antiquaries Journal. XXXI (1–2). London: Society of Antiquaries of London: 67–68. doi:10.1017/S0003581500058005. S2CID 163358427. +——— (March 1951). "The Sutton Hoo Ship-Burial". The American-Scandinavian Review. XXXIX (1). Princeton, New Jersey: The American-Scandinavian Foundation: 27–32. ISSN 0003-0910. +——— (April 1951). "The Sutton Hoo Ship-Burial". Scientific American. 184 (4): 24–30. Bibcode:1951SciAm.184d..24B. doi:10.1038/scientificamerican0451-24. JSTOR 24945139. S2CID 45498190. +——— (1952). "An Anglo-Saxon Gold Pendant from High Wycombe, Bucks". The British Museum Quarterly. XV. London: The Trustees of the British Museum: 72. doi:10.2307/4422266. JSTOR 4422266. +——— (1952). "The Castle Eden Vase". The British Museum Quarterly. XV. London: The Trustees of the British Museum: 73. doi:10.2307/4422267. JSTOR 4422267. +——— (1952). "A Late Saxon Disk-Brooch and Sword Pommel". The British Museum Quarterly. XV. London: The Trustees of the British Museum: 74–75. doi:10.2307/4422268. JSTOR 4422268. +——— (1952). "Other Dark-Age Acquisitions". The British Museum Quarterly. XV. London: The Trustees of the British Museum: 75–76. doi:10.2307/4422395. JSTOR 4422269. +——— (1952). "A Medieval Polychrome Pottery Aquamanile from Stonar, Kent". The British Museum Quarterly. XV. London: The Trustees of the British Museum: 80–81. doi:10.2307/4422275. JSTOR 4422275. +——— & King, William (1952). "Medieval Pottery, Tiles, and Glass". The British Museum Quarterly. XV. London: The Trustees of the British Museum: 81–82. doi:10.2307/4422276. JSTOR 4422276. +——— (June 1952). "A Late-Saxon Silver Disk-Brooch from the Isle of Ely". The British Museum Quarterly. XVII (1). London: The Trustees of the British Museum: 15–16. doi:10.2307/4422367. JSTOR 4422367. +——— & Allan, John (June 1952). "Sutton Hoo—A Rejoinder: With a Note on the Coins". Antiquity. XXVI (102): 76–82. doi:10.1017/S0003598X00023619. S2CID 163980192. +——— (December 1952). "The Fuller Brooch". The British Museum Quarterly. XVII (4). London: The Trustees of the British Museum: 75–76. doi:10.2307/4422395. JSTOR 4422395. +——— (1952). "The Snape Boat-Grave" (PDF). Proceedings of the Suffolk Institute of Archaeology. XXVI (1). Ipswich: 1–26. +Edited and republished in Bruce-Mitford 1974a, ch. 3 +——— (1952–1953). "A Bronze Strap-End of c. A. D. 900 from Souldern, Oxon" (PDF). Oxoniensia. XVII–XVIII. Oxford: Oxfordshire Architectural and Historical Society: 236. ISSN 0308-5562. +——— (January 1954). "A Late- or Sub-Roman Buckle-Plate from College Wood, near Winchester". The Antiquaries Journal. XXXIV (1–2). London: Society of Antiquaries of London: 75–76. doi:10.1017/S0003581500073224. S2CID 163239398. +Tonnochy, A. B.; Brailsford, J. W.; Bruce-Mitford, Rupert & King, William (June 1954). "British and Medieval Antiquities, 1753–1953". The British Museum Quarterly. XIX (1). London: The Trustees of the British Museum: 18–27. doi:10.2307/4422462. JSTOR 4422462. +Bruce-Mitford, Rupert (1955). "Gold and Silver Cloisonné Buckle from Wynaldum, Friesland". Verslag van het Fries Genootschap van Geschied-, Oudheid- en Taalkunde. 126. Leeuwarden: Friesch Genootschap van Geschied-, Oudheid- en Taalkunde: 16–17. OCLC 73136678. +Edited and republished in Bruce-Mitford 1974a, pp. 270–273 +——— (1955). "Recent Studies of Late Saxon Metalwork". Transactions of the Lancashire and Cheshire Antiquarian Society. LXV. Manchester: Lancashire and Cheshire Antiquarian Society: 134–135. ISSN 0950-4699. +——— (October 1955). "Edward Thurlow Leeds". Obituary. The Museums Journal. 55 (7). London: The Museums Association: 189–190. ISSN 0027-416X. +——— (1956). "The Benty Grange Helmet". Annual Report for the Year Ending 31st March 1956. Sheffield: Sheffield City Museum. pp. 13–15. OCLC 694999446. +——— (January 1956). "National Museums and Local Material". The Museums Journal. 55 (10). London: The Museums Association: 251–258. ISSN 0027-416X. +——— (July 1957). "A Medieval Polychrome Jug from the City of London". The British Museum Quarterly. XXI (2). London: The Trustees of the British Museum: 54–55. doi:10.2307/4422573. JSTOR 4422573. +Andrew O'Dell, Andrew; Bruce-Mitford, Rupert; Stevenson, Robert B. K.; Plenderleith, Harold James & Brown, Thomas Julian (December 1959). "The St Ninian's Isle Silver Hoard". Antiquity. XXXIII (132): 257–268. doi:10.1017/S0003598X00027654. S2CID 163046639. +Bruce-Mitford, Rupert (November 1960). "The Treasure of St. Ninian's". Scientific American. 203 (5): 154–166. Bibcode:1960SciAm.203e.154B. doi:10.1038/scientificamerican1160-154. JSTOR 24940701. +——— (June 1961). "The Book of Kells Exhibition". Museums Journal. 61 (1). London: The Museums Association: 7–20. ISSN 0027-416X. +——— (1963). "The Early Gospel-Book in the British Isles, with Special Reference to the Gospels of St. Chad". Annual Report. 30. Llandaff, Wales: Friends of Llandaff Cathedral: 16–19. OCLC 896111779. +——— (1964). "Excavations at Sutton Hoo in 1938" (PDF). Proceedings of the Suffolk Institute of Archaeology. XXX (1). Ipswich: 1–43. +——— (1964a). "A National Reference Collection of Medieval Pottery" (PDF). Medieval Archaeology. 8. London: The Society for Medieval Archaeology: 229–230. doi:10.5284/1000320. +——— (September 1964). "A Hiberno-Saxon Bronze Mounting from Markyate, Hertfordshire". Antiquity. XXXVIII (151). Cambridge: Antiquity Publications Ltd.: 219–220. doi:10.1017/S0003598X00105095. +——— (December 1964c). "National Reference Collection of Medieval Pottery". Antiquity. XXXVIII (152). Cambridge: Antiquity Publications Ltd.: 303–304. doi:10.1017/S0003598X00108270. S2CID 246047709. +——— (February 1965). "Sutton Hoo and Sweden". The Anglo-Swedish Review (1). London: The Swedish Chamber of Commerce for the United Kingdom: 20–22. OCLC 51228631. +——— (Summer 1965). "The Lindisfarne Gospels in the Middle Ages and Later". The British Museum Quarterly. XXIX (3–4). London: The Trustees of the British Museum: 98–100. doi:10.2307/4422900. JSTOR 4422900. +——— (September 1966). "Archaeology: A National Museum Viewpoint". The Relationship of National and Provincial Museums. Museums Journal. 66 (2). London: The Museums Association: 99–106. ISSN 0027-416X. +——— (1967). "A New Wooden Ship's Figure-Head Found in the Scheldt, at Moerzeke-Mariekerke". Miscellanea. Acta Archaeologica. XXXVIII: 199–209. ISSN 0065-101X. +Edited and republished in Bruce-Mitford 1974a, ch. 6 +——— (4 November 1967). "Sutton Hoo Revisited". Archaeological Section No 2277. The Illustrated London News. Vol. 251, no. 6692. London. pp. 26–27. ISSN 0019-2422. +——— (March 1968). "Sutton Hoo". Gypsum Journal (47). London: The Gypsum Plasterboard Development Association: 11–14. OCLC 21731710. +——— (March 1968). "Sutton Hoo Excavations, 1965–7". Antiquity. XLII (165). Cambridge: Antiquity Publications Ltd.: 36–39. doi:10.1017/S0003598X00033810. S2CID 163655331. +Edited and republished in Bruce-Mitford 1974a, ch. 5 +——— (December 1968). "Fresh Observations on the Torslunda Plates". Frühmittelalterliche Studien. 2. Münster: 233–236. doi:10.1515/9783110242027.233. S2CID 185602132. +Edited and republished in Bruce-Mitford 1974a, ch. 10 +——— (1969). "The Art of the Codex Amiatinus: Jarrow Lecture 1967". Journal of the British Archaeological Association. XXXII: 1–25. doi:10.1080/00681288.1969.11894883. +Also published as a hardcover offprint. See OCLC 926900345, 7214133300 +——— (1969). "The Ruined Church of Stone-by-Faversham: Appendix III, The Amber Pendant". The Antiquaries Journal. XLIX (2). London: Society of Antiquaries of London: 292–293. doi:10.1017/S0003581500053798. S2CID 161808688. +——— & Bruce-Mitford, Myrtle (March 1970). "The Sutton Hoo Lyre, Beowulf, and the Origins of the Frame Harp". Antiquity. XLIV (173). Cambridge: Antiquity Publications Ltd.: 7–13. doi:10.1017/S0003598X00040916. +Edited and republished in Bruce-Mitford 1974a, ch. 7 +——— (June 1970). "Ships' Figure-heads in the Migration Period and Early Middle Ages". Antiquity. XLIV (174). Cambridge: Antiquity Publications Ltd.: 146–148. doi:10.1017/S0003598X00104764. S2CID 163282907. +Edited and republished in Bruce-Mitford 1974a, ch. 6 +——— (Spring 1971). "Envoi". The British Museum Quarterly. XXXV (1–4). London: The Trustees of the British Museum: 8–16. doi:10.2307/4423066. JSTOR 4423066. +——— (Autumn 1972). "The Sutton Hoo Helmet: A New Reconstruction". The British Museum Quarterly. XXXVI (3–4). London: The Trustees of the British Museum: 120–130. doi:10.2307/4423116. JSTOR 4423116. +Edited and republished in Bruce-Mitford 1974a, ch. 8 +——— (February 1973). "Sutton Hoo Drinking Horns". The British Museum Society Bulletin (12): 20. ISSN 0525-5260. +——— (1974b). "Exhibits at Ballots: 5. A Replica of the Sutton Hoo Helmet Made in the Tower Armouries, 1973". The Antiquaries Journal. LIV (2). London: Society of Antiquaries of London: 285–286. doi:10.1017/S0003581500042529. S2CID 246055329. +——— (1974c). "Exhibits at Ballots: 6. Anglo-Saxon Gold Sword Mount from Maidstone, Kent". The Antiquaries Journal. LIV (2). London: Society of Antiquaries of London: 286. doi:10.1017/S0003581500042529. S2CID 246055329. +——— (February 1974d). "The Sutton Hoo Helmet". The British Museum Society Bulletin (15): 6–7. ISSN 0525-5260. +Edited and republished in Bruce-Mitford 1974b +——— (March 1975). "New Galleries for Medieval and Later Antiquities". The British Museum Society Bulletin (18): 6–7. ISSN 0525-5260. +——— (1977). "Obituary: Basil Brown" (PDF). Proceedings of the Suffolk Institute of Archaeology. XXXIV (1). Ipswich: 71. +——— (January 1978b). "The Archaeologist". Antique Collector. 49 (1). London: National Magazine Company LTD.: 68–69. +——— (1980). "Obituary: Leslie Dow, F.S.A" (PDF). Proceedings of the Suffolk Institute of Archaeology. XXXIV (4). Ipswich: 287–288. +——— (December 1982). "The Sutton Hoo Helmet-Reconstruction and the Design of the Royal Harness and Sword-Belt: A Reply to Hofrat Dr. Ortwin Gamber with Some Additional Comments on the Sutton Hoo Arms and Armour". The Journal of the Arms & Armour Society. X (6). London: 217–274. ISSN 0004-2439. +Response to: Gamber, Ortwin (December 1982). "Some Notes on the Sutton Hoo Military Equipment". The Journal of the Arms & Armour Society. X (6). London: 208–216. ISSN 0004-2439. +Bruce-Mitford, Rupert (1989a). "Early Thoughts on Sutton Hoo". Saxon (10). Woodbridge, Suffolk: The Sutton Hoo Society. +——— (1989b). Anglo-Saxon and Mediaeval Archaeology, History and Art, with Special Reference to Sutton Hoo: The Highly Important Working Library and Archive of More than 6,000 Titles Formed by Dr. Rupert L.S. Bruce-Mitford FBA, D.Litt., FSA. Wickmere: Merrion Book Co. OCLC 858531182. +Includes prefatory essays My Japanese Background and Forty Years with Sutton Hoo by Bruce-Mitford. +Bruce-Mitford, Rupert (1952). "The Sutton Hoo Ship-Burial". In Hodgkin, Robert Howard (ed.). A History of the Anglo-Saxons. Vol. II (3rd ed.). London: Oxford University Press. pp. 696–734, 750–756. +Footnote 21 edited and republished in Bruce-Mitford 1974a, ch. 9 +——— (1953). "Some Recent Results of the Application of Laboratory Technique to Antiquities of the Anglo-Saxon Period in Britain". In Vogt, Emil (ed.). Actes de la IIIe session: Zurich, 1950. Zurich: Congrès International des Sciences Préhistoriques et Protohistoriques. pp. 321–323. OCLC 797756211. +——— (1954). "Preface". In Lasko, Peter (ed.). Catalogue of an Exhibition of Ivory Carvings Lent by the City of Liverpool Public Museums, Mostly from the Mayer-Fejervary Collection. London: Trustees of the British Museum. p. 3. OCLC 30175308. +———, ed. (1956). Recent Archaeological Excavations in Britain: Selected Excavations 1939–1955 with a Chapter on Recent Air-Reconnaissance. New York: Macmillan. +Contains ——— (1956). "A Dark-Age Settlement at Mawgan Porth, Cornwall". In Bruce-Mitford, Rupert (ed.). Recent Archaeological Excavations in Britain: Selected Excavations 1939–1955 with a Chapter on Recent Air-Reconnaissance. New York: Macmillan. pp. 167–196. +Briefly summarised in Griffiths, W. E. (May 1955). "The Second Viking Congress". The Archaeological News Letter. 5 (1). Linden Publicity: 17–19. ISSN 0003-8040. +Contains ——— (1956). "Treasure Trove: A Note on Law and Practice". In Bruce-Mitford, Rupert (ed.). Recent Archaeological Excavations in Britain: Selected Excavations 1939–1955 with a Chapter on Recent Air-Reconnaissance. New York: Macmillan. pp. 297–301. +——— & Harden, Donald Benjamin (1956). "Edward Thurlow Leeds: 1877–1955". In Harden, Donald Benjamin (ed.). Dark-Age Britain: Studies Presented to E. T. Leeds with a Bibliography of His Works. London: Methuen & Co. Ltd. pp. ix–xvi. OCLC 30193045. +——— (1956). "Late Saxon Disc-Brooches". In Harden, Donald Benjamin (ed.). Dark-Age Britain: Studies Presented to E. T. Leeds with a Bibliography of His Works. London: Methuen & Co. Ltd. pp. 171–201. OCLC 30193045. +Republished in part in Bruce-Mitford 1964b, app'x B +Edited and republished in Bruce-Mitford 1974a, ch. 15 +——— (1956). "The Pectoral Cross". In Battiscombe, Christopher Francis (ed.). The Relics of Saint Cuthbert. Oxford: Oxford University Press. pp. 308–325, 542–544. OCLC 4071903. +——— (1964). "Preface & Appendix B: Iconography of the Fuller Brooch". In Wilson, David M. (ed.). Anglo-Saxon Ornamental Metalwork 700–1100. Catalogue of Antiquities of the Later Saxon Period. Vol. I. London: The Trustees of the British Museum. pp. v, 91–98. OCLC 886634250. +——— (1967). "The Reception by the Anglo-Saxons of Mediterranean Art Following their Conversion from Ireland to Rome". La conversione al Cristianesimo nell'Europa dell'Alto Medioevo: 14–19 aprile 1966. Settimane di studio del Centro italiano di studi sull'alto Medioevo. Vol. XIV. Spoleto: Centro Italiano di Studi sull'Alto Medioevo. pp. 797–825. OCLC 1115912. +——— (1967). "The Lindisfarne Gospels". Great Books of Ireland: Thomas Davis Lectures. Dublin: Clonmore & Reynolds Ltd. pp. 26–37. OCLC 20194. +——— (1970). "Preface". In Grohskopf, Bernice (ed.). The Treasure of Sutton Hoo. New York: Atheneum. pp. vii–x. LCCN 74-86555. +——— (1971). "Medieval and Later Antiquities". In Francis, Frank (ed.). Treasures of the British Museum. London: Thames & Hudson. pp. 179–209. ISBN 0-500-18125-X. +——— (1971). "Sutton Hoo and the Background to the Poem". In Girvan, Ritchie (ed.). Beowulf and the Seventh Century: Language and Content (2nd ed.). London: Methuen & Co. Ltd. pp. 85–98. +Edited and republished in Bruce-Mitford 1974a, ch. 12 +——— (1972). "The Dating of the Sutton Hoo Coins: Some Comments". In Hall, E. T. & Metcalf, David Michael (eds.). Methods of Chemical and Metallurgical Investigation of Ancient Coinage: A Symposium Held by the Royal Numismatic Society at Burlington House, London on 9-11 December 1970. Royal Numismatic Society Special Publication. Vol. 8. London: Royal Numismatic Society. pp. 108–109. OCLC 962994865. +——— (1976). "The Chapter House Vestibule Graves at Lincoln and the Body of St. Hugh of Avalon". In Emmison, Frederick & Stephens, Roy (eds.). Tribute to an Antiquary: Essays Presented to Marc Fitch by Some of His Friends. London: Leopard's Head Press. pp. 127–140. ISBN 0-904920-00-3. +——— (1976). "Excavation at the Sutton Hoo Site, England, 1969". In Oehser, Paul Henry (ed.). Abstracts and Reviews of Research and Exploration Authorized Under Grants from the National Geographic Society During the Year 1968. National Geographic Society Research Reports. Vol. 9. Washington, D.C.: National Geographic Society. pp. 49–52. LCCN 68-26794. SBN 87044-136-1. +——— (1978). "A Comparison Between the Sutton Hoo Burial Deposit and Childeric's Treasure". Actes du Colloque International d'Archéologie, Rouen 3-4-5 Juillet 1975: La Période Mérovingienne. Vol. III. Rouen: Musée départemental des antiquités de Seine-Maritime. pp. 365–372. OCLC 633625288. +——— (1978). "La Matériel archéologique de la sépulture royale de Sutton Hoo (Grand-Bretagne, Suffolk): dernier bilan des recherches" [The archaeological material of the royal burial of Sutton Hoo (Great Britain, Suffolk): latest research report]. In Fleury, Michel & Périn, Patrick (eds.). Problèmes de chronologie relative et absolue concernant les cimetières mérovingiens d'entre Loire et Rhin: Actes du IIe colloque archéologique de la IVe Section de l'Ecole pratique des Hautes Etudes (Paris, 1973) [Problems of relative and absolute chronology concerning the Merovingian cemeteries between Loire and Rhine: Proceedings of the 2nd archaeological colloquium of the 4th Section of the École pratique des hautes études (Paris, 1973)]. Bibliothèque de l'Ecole des hautes études: IVe Section — Sciences historiques et philologiques (in French). Vol. 326. Paris: Librairie Honoré Champion. pp. 13–17. OCLC 932310155. +——— & Wilson, David M. (1979). "Die Angelsachsen" [The Anglo-Saxons]. In Roth, Helmut (ed.). Kunst der Völkerwanderungszeit [Art of the Migration Period]. Propyläen Kunstgeschichte: Supplementbänd (in German). Vol. IV. Berlin: Propyläen Verlag. pp. 206–222. OCLC 1078702695. +——— (1986). "The Sutton Hoo Ship-Burial: Some Foreign Connections". Angli e Sassoni al di qua e al di là del mare: 26 aprile-lo maggio 1984, tomo primo. Settimane di studio del Centro italiano di studi sull'alto Medioevo. Vol. XXXII. Spoleto: Centro italiano di studi sull'alto Medioevo. pp. 171–210. +——— (1987). "Ireland and the Hanging Bowls—A Review". In Ryan, Michael (ed.). Ireland and Insular Art, A.D. 500–1200: Proceedings of a Conference at University College Cork, 31 October-3 November 1985. Dublin: Royal Irish Academy. pp. 30–39. ISBN 0-901714-54-2. +——— (1989). "The Durham-Echternach Calligrapher". In Bonner, Gerald; Rollason, David & Stancliffe, Clare (eds.). St Cuthbert, His Cult and His Community to AD 1200. Woodbridge, Suffolk: The Boydell Press. pp. 175–188. ISBN 0-85115-510-3. LCCN 88-16805. +——— (1991). "Thomas Downing Kendrick: 1895–1979" (PDF). Lectures and Memoirs. Proceedings of the British Academy. Vol. 76. Oxford: Oxford University Press. pp. 445–471. ISBN 0-19-726107-8. ISSN 0068-1202. +Bruce-Mitford, Rupert (1945). "Review of London Museum Catalogues, No. 7: Medieval Catalogue" (PDF). The Archaeological Journal. CII. London: The Royal Archaeological Institute of Great Britain and Ireland: 173–174. doi:10.5284/1018054. +——— (November 1948). "Review of Medieval English Pottery, by Bernard Rackham". The Archaeological News Letter. 1 (7). Linden Publicity: 7. ISSN 0003-8040. +——— (June 1949). "Review of Essai sur les Origines de la Miniature Dite Irlandaise, by François Masai". New Books Reviewed. The Connoisseur. CXXIII (512): 126. ISSN 0010-6275. +——— (August 1949). "Review of Late Saxon and Viking Art, by T. D. Kendrick". The Archaeological News Letter. 2 (4). Linden Publicity: 61–62. ISSN 0003-8040. +——— (January–April 1950). "Review of Nubian Treasure, by Walter B. Emery". The Antiquaries Journal. XXX (1–2). London: Society of Antiquaries of London: 76–80. doi:10.1017/S0003581500088259. S2CID 163791671. +——— (September 1950). "Review of Merowingerzeit, by Gustav Behrens". Antiquity. XXIV (95). Gloucester: John Bellows Ltd.: 163–164. doi:10.1017/S0003598X00023243. S2CID 162972699. +Republished with edits in ——— (June 1951). "Review of Merowingerzeit, by Gustav Behrens". Antiquity. XXV (98). Gloucester: John Bellows Ltd.: 107–108. doi:10.1017/S0003598X00021001. +——— (December 1950). "Review of Culture in Early Anglo-Saxon England, by D. Elizabeth Martin-Clarke". Antiquity. XXIV (96). Gloucester: John Bellows Ltd.: 211–212. doi:10.1017/S0003598X00023462. S2CID 163141521. +——— (August 1951). "Review of The Coffin of Saint Cuthbert, by Donald McIntyre". The Burlington Magazine. XCIII (581): 274. JSTOR 870757. +——— (29 May 1952). "Review of Pre-Conquest England and Byzantium, by David Talbot Rice". Summer Books. The Listener. Vol. XLVII, no. 1213. London: The British Broadcasting Corporation. p. 879. +——— (14 May 1954). "Boats of the North". History. The Times Literary Supplement. No. 2728. London. p. 314. Archived from the original on 9 May 2023. Retrieved 9 May 2023. +——— (July–October 1954). "Review of Der Tassilokelch, by Gunther Haseloff". The Antiquaries Journal. XXXIV (3–4). London: Society of Antiquaries of London: 245–246. doi:10.1017/S0003581500060042. S2CID 162310675. +——— (July–October 1954). "Review of Die langobardischen Fibeln aus Italien, by Siegfried Fuchs and Joachim Werner". The Antiquaries Journal. XXXIV (3–4). London: Society of Antiquaries of London: 246–247. doi:10.1017/S0003581500060054. S2CID 163720738. +——— (July–October 1954). "Review of Das Alamannische Gräberfeld von Bülach, by Joachim Werner". The Antiquaries Journal. XXXIV (3–4). London: Society of Antiquaries of London: 247. doi:10.1017/S0003581500060066. S2CID 162670894. +——— (December 1954). "Review of The English Windmill, by Rex Wailes". The Archaeological News Letter. 5 (7). Linden Publicity: 128. ISSN 0003-8040. +——— (January–April 1956). "Review of Oreficerie Langobarde a Parma, by Giorgio Monaco". The Antiquaries Journal. XXXVI (1–2). London: Society of Antiquaries of London: 99–100. doi:10.1017/S0003581500060595. S2CID 162256332. +——— (January–April 1957). "Review of Fynske Jernaldergrave II: Ældre romersk jernalder, by Erling Albrechtsen". The Antiquaries Journal. XXXVII (1–2). London: Society of Antiquaries of London: 79–80. doi:10.1017/S0003581500077441. S2CID 162911783. +——— (1958). "Review of Early Christian Ireland, by Máire and Liam de Paor" (PDF). Medieval Archaeology. 2. London: The Society for Medieval Archaeology: 214–217. doi:10.5284/1000320. +——— (September 1959). "Review of The Beginnings of Christian Art, by David Talbot Rice". Antiquity. XXXIII (131): 226–227. doi:10.1017/S0003598X00027551. S2CID 163981765. +——— (1962). "Review of The Sword in Anglo-Saxon England: Its Archaeology and Literature, by Hilda Ellis Davidson" (PDF). The Archaeological Journal. CXIX. London: The Royal Archaeological Institute: 367. doi:10.5284/1018054. +——— (1962). "Review of Excavations at Helgö, I: Report for 1954–1956, edited by Wilhelm Holmqvist in collaboration with Birgit Arrhenius and Per Lundström". The Antiquaries Journal. XLII (1). London: Society of Antiquaries of London: 99–100. doi:10.1017/S000358150002415X. +——— (June 1962). "Review of Archäologische Denkmäler der Gepiden im Mitteldonaubecken (454–568 u. Z.), by Dezsö Csallány". Museums Journal. 62 (1). London: The Museums Association: 354–355. ISSN 0027-416X. +——— (1962–1963). "Review of Ladby-Skibet, by Knud Thorvildsen" (PDF). Medieval Archaeology. 6–7. Society for Medieval Archaeology: 353–356. doi:10.5284/1000320. +——— (22 February 1963). "The Third Great Codex: The Book of Durrow's Place in the Celtic World". The Times Literary Supplement. No. 3182. London. p. 143. Archived from the original on 9 October 2023. Retrieved 30 July 2021. +——— (1965). "Review of The Fifth Century Invasions South of the Thames, by Vera Evison". The Archaeological Journal. CXXII. London: The Royal Archaeological Institute: 247–249. doi:10.1080/00665983.1965.11077377. +——— (1967). "Review of Three Viking Graves in the Isle of Man, by Gerhard Bersu and David M. Wilson" (PDF). Medieval Archaeology. 11. Society for Medieval Archaeology: 332–334. doi:10.5284/1000320. +——— (28 May 1971). "Topographical Past". History. The Times Literary Supplement. No. 3613. London. p. 612. Archived from the original on 9 October 2023. Retrieved 9 May 2023. +——— (August 1971). "Review of Early Celtic Art, by Paul Jacobstahl". The Burlington Magazine. CXIII (821): 478. JSTOR 876721. +——— (1975). "Review of Britain before the Norman Conquest". The Antiquaries Journal. LV (1). London: Society of Antiquaries of London: 155. doi:10.1017/S0003581500035733. S2CID 162394640. +——— (1985). "Review of Vendel Period Studies: Transactions of the Boat-grave Symposium, Stockholm, 1981, edited by Jan Peder Lamm and Hans-Åke Nordström" (PDF). Medieval Archaeology. 29. Society for Medieval Archaeology: 231–233. doi:10.1080/00766097.1985.11735471. +——— (4 January 1985). "The Age of the Cross". Art History. The Times Literary Supplement. No. 4266. London. p. 21. Archived from the original on 4 April 2023. Retrieved 30 July 2021. +——— (1986). "Review of Corpus of Anglo-Saxon Sculpture in England, Vol. I: County Durham and Northumberland, by Rosemary Cramp". The Antiquaries Journal. LXVI (1). London: Society of Antiquaries of London: 179–181. doi:10.1017/S0003581500085000. +——— (1988). "Review of The Sutton Hoo Ship Burial, by Angela Care Evans". The Antiquaries Journal. LXVIII (2). London: Society of Antiquaries of London: 356–357. doi:10.1017/S0003581500069882. S2CID 163211667. +——— (1988). "Review of From Durrow to Kells: The Insular Gospel-Books 650–800, by George Henderson". The Antiquaries Journal. LXVIII (2). London: Society of Antiquaries of London: 359–360. doi:10.1017/S0003581500069912. S2CID 161482786. +——— (11 November 1988). "The Sign of the Cross". Archaeology. The Times Literary Supplement. No. 4467. London. p. 1256. +——— (1993). "Review of The Origins of Beowulf and the Pre-Viking Kingdom of East Anglia, by Sam Newton". The Antiquaries Journal. LXXIII. London: Society of Antiquaries of London: 210–211. doi:10.1017/S0003581500071985. S2CID 164034442. +Sheppard, Thomas (February 1940). "Saxon Remains at Barton, Part II". The Naturalist (997). London: A. Brown & Sons, Ltd.: 37–47. +Contains (at pp. 40–42) letter by Bruce-Mitford on weights and scales excavated at Barton-upon-Humber. Part I of the article published at Sheppard, Thomas (October 1939). "Saxon Relics from Barton, Lincs". The Naturalist (993). London: A. Brown & Sons, Ltd.: 257–262. +"The Sutton Hoo Musical Instrument". Reports of Meetings. The Archaeological News Letter. 1 (1). London: Linden Publicity: 11–13. April 1948. ISSN 0003-8040. +Summary of lecture given by Bruce-Mitford to the Society of Antiquaries of London on 26 February 1948. +Bruce-Mitford, Rupert (May 1949). "The Alfred Jewel". Letters to the Editor. The Museums Journal. 49 (2). London: The Museums Association: 47–48. ISSN 0027-416X. +——— (1950). "Gold and Silver Work: Early Christian". Chambers's Encyclopædia. Vol. VI. London: George Newnes Limited. pp. 418–419. OCLC 265899. +——— (1950). "Jewellery: Dark Ages". Chambers's Encyclopædia. Vol. VIII. London: George Newnes Limited. pp. 87–89. +——— (October 1951). "Archaeology in Cornwall". Letters. The Archaeological News Letter. 4 (3). London: Linden Publicity: 43. ISSN 0003-8040. +——— (1955). "Sutton Hoo Ship Burial". The Encyclopedia Americana. Vol. 26. New York: Americana Corporation. p. 104. LCCN 55-5910. +Republished in subsequent editions, such as ——— (1956). "Sutton Hoo Ship Burial". The Encyclopedia Americana. Vol. XXVI. New York: Americana Corporation. pp. 104–104b. LCCN 56-5839, and ——— (1970). "Sutton Hoo Ship Burial". The Encyclopedia Americana. Vol. 26. New York: Americana Corporation. pp. 103–104b. LCCN 72-97500. SBN 7172-0101-5. +——— (September 1955). "Points from Correspondence". The South African Archaeological Bulletin. X (39). Claremont, Cape Town: The South African Archaeological Society: 104. doi:10.2307/3887567. JSTOR 3887567. +——— (1956). "Preface to Second Edition". In Watson, William (ed.). Flint Implements: An Account of Stone Age Techniques and Cultures. London: The Trustees of the British Museum. p. v. OCLC 4847841. +——— (1968). "Preface to Third Edition". In Watson, William (ed.). Flint Implements: An Account of Stone Age Techniques and Cultures. London: The Trustees of the British Museum. p. 6. ISBN 0-7141-1306-9. +——— (June 1956). "The Woman in Grey". Under the Dome. 15 (6). London: British Museum: 2–4. OCLC 561848219. +Short ghost story. +——— (1958). "Anglo-Sassoni e Irlandesi Centri e Tradizioni". Enciclopedia Universale dell'Arte (in Italian). Vol. I. Venice: Istituto per la Collaborazione Culturale. col. 408–425, pl. 248–265. OCLC 6109600. +Translated and republished as ——— (1959). "Anglo-Saxon and Irish Art". Encyclopedia of World Art. Vol. I. New York: McGraw-Hill Book Company, Inc. col. 446–463, pl. 274–291. +——— (1961). "Revival of Roman Influences in the 7th and 8th Centuries in Anglo-Saxon Archaeology". In Bersu, Gerhard & Dehn, Wolfgang (eds.). Bericht über den V. Internationalen Kongress für Vor- und Frühgeschichte Hamburg vom 24. bis 30. August, 1958. Berlin: Verlag Gebr. Mann. p. 158. OCLC 1046238019. +——— (19 July 1963). "Mr. A. B. Tonnochy". Obituary. The Times. No. 55757. London. p. 14. Archived from the original on 4 April 2023. Retrieved 10 August 2021. +——— (23 July 1965). "Mr. Herbert Maryon". Obituary. The Times. No. 56381. London. p. 14. Archived from the original on 5 November 2023. Retrieved 9 May 2023. +——— & Jenkins, Gilbert Kenneth (29 November 1966). "Treasure Trove". Letters to the Editor. The Times. No. 56801. London. p. 11. Archived from the original on 9 May 2023. Retrieved 9 May 2023. +——— (1967). "Anglo-Saxon Ship-Burials". The Twentieth Aldeburgh Festival of Music and the Arts. p. 33. OCLC 28358875. +——— (1967). "Celtic Art". New Catholic Encyclopedia. Vol. III. New York: McGraw-Hill Book Company. pp. 382–383. LCCN 66-22292. +LCCN incorrectly printed as LCCN 80--84291. +——— (1967). "Durrow, Book of". New Catholic Encyclopedia. Vol. IV. New York: McGraw-Hill Book Company. pp. 1123–1124. LCCN 66-22292. +——— (2003). "Durrow, Book of". New Catholic Encyclopedia: Com-Dyn. New Catholic Encyclopedia. Vol. 4 (2nd ed.). Detroit: Thomson Gale. pp. 953–954. ISBN 0-7876-4008-5. +——— (1967). "Irish Art". New Catholic Encyclopedia. Vol. VII. New York: McGraw-Hill Book Company. pp. 633–666. LCCN 66-22292. +——— (1967). "Irish Crosses". New Catholic Encyclopedia. Vol. VII. New York: McGraw-Hill Book Company. pp. 644–645. LCCN 66-22292. +——— (2003). "Irish Crosses". New Catholic Encyclopedia: Hol-Jub. New Catholic Encyclopedia. Vol. 7 (2nd ed.). Detroit: Thomson Gale. pp. 583–584. ISBN 0-7876-4011-5. +——— (1967). "Kells, Book of". New Catholic Encyclopedia. Vol. VIII. New York: McGraw-Hill Book Company. pp. 145–146. LCCN 66-22292. +——— (2003). "Kells, Book of". New Catholic Encyclopedia: Jud-Lyo. New Catholic Encyclopedia. Vol. 8 (2nd ed.). Detroit: Thomson Gale. pp. 138–139. ISBN 0-7876-4012-3. +——— (1967). "Lindisfarne Gospels". New Catholic Encyclopedia. Vol. VIII. New York: McGraw-Hill Book Company. p. 771. LCCN 66-22292. +——— (2003). "Lindisfarne Gospels". New Catholic Encyclopedia: Jud-Lyo. New Catholic Encyclopedia. Vol. 8 (2nd ed.). Detroit: Thomson Gale. pp. 593–594. ISBN 0-7876-4012-3. +——— (1969). "Preface". In Zürn, Hartwig & Schiek, Siegwalt (eds.). Die Sammlung Edelmann im Britischen Museum zu London [The Edelmann Collection in the British Museum in London]. Urkunden zur Vor- und Frühgeschichte aus Südwürttemberg-Hohenzollern (in German). Vol. 3. Stuttgart: Verlag Müller & Gräff. p. 5. OCLC 17063707. +——— (2 February 1971). "Sutton Hoo Stag". Letters to the Editor. The Times. No. 58088. London. p. 15. Archived from the original on 9 May 2023. Retrieved 9 May 2023. +——— (27 July 1971). "Treasure Trove". Letters to the Editor. The Times. No. 58235. London. p. 20. Archived from the original on 27 September 2022. Retrieved 30 July 2021. +——— (23 March 1977). "Mr Basil Brown: Discovery of Sutton Hoo Ship". Obituary. The Times. No. 59959. London. p. 18. Archived from the original on 9 May 2023. Retrieved 9 May 2023. +——— (May 1976). "The Paglesham Brooch". Letters. Current Archaeology. Vol. V, no. 56. p. 287. ISSN 0011-3212. +Letter regarding article in previous issue: Clarke, David Tyrwhitt-Drake (January 1976). "Saved for Britain!". Current Archaeology. Vol. V, no. 54. pp. 214–215. ISSN 0011-3212. +Bruce-Mitford, Rupert (12 December 1979). "Mr Leslie Dow". Obituary. The Times. No. 60498. London. p. 16. Archived from the original on 4 April 2023. Retrieved 3 August 2021. +Not attributed to Bruce-Mitford in The Times, but listed in Bruce-Mitford 1989b, p. 28 (#55). +——— (11 March 1980). "A Long Way Back in Tipperary". Letters to the Editor. The Times. No. 60572. London. p. 13. Archived from the original on 4 April 2023. Retrieved 30 July 2021. +——— (28 August 1989). "Ship or Tomb at Sutton Hoo?". Letters to the Editor. The Times. No. 63484. London. p. 11. Archived from the original on 30 July 2021. +Oddy, Andrew & Bruce-Mitford, Rupert (1 May 1992). "Nigel Reuben Rook Williams, 1944–1992: Two Tributes". Unpublished. +Alexander, John (December 1969). "Review of The Bog People, by P. V. Glob". Antiquity. XLIII (172): 320–321. doi:10.1017/S0003598X00040758. S2CID 163281870. +Allison, Susan (1976). Ormsby, Margaret (ed.). A Pioneer Gentlewoman in British Columbia: The Recollections of Susan Allison. Vancouver: University of British Columbia Press. ISBN 0-7748-0039-9. +Arkell, William Joscelyn (1938). "The Geology of the Site of the Bodleian Extension in Broad Street" (PDF). Oxoniensia. III. Oxford: Oxfordshire Architectural and Historical Society: 1–5. ISSN 0308-5562. Archived (PDF) from the original on 19 May 2016. +Arrhenius, Birgit (1978). "Review of The Sutton Hoo Ship-Burial, volume 1, by Rupert Bruce-Mitford". Medieval Archaeology. 22. Society for Medieval Archaeology: 189–195. doi:10.1080/00766097.1978.11735412. +Biddle, Martin; Binns, Alan; Cameron, J. M.; Metcalf, David Michael; Page, Raymond Ian; Sparrow, Charles & Warren, F. L. (1977). "Sutton Hoo Published: A Review". Anglo-Saxon England. 6. Cambridge: Cambridge University Press: 249–265. doi:10.1017/s0263675100001058. S2CID 161813648. +Biddle, Martin (1997–1998). "Dr Rupert Bruce-Mitford F.B.A., F.S.A." (PDF). The Hertford College Magazine (83): 115–121. Archived (PDF) from the original on 15 January 2022. +Biddle, Martin (3 December 2015). "Rupert Leo Scott Bruce-Mitford: 1914–1994" (PDF). Biographical Memoirs of Fellows of the British Academy. XIV. British Academy: 58–86. Archived (PDF) from the original on 28 September 2020. +Blair, Peter Hunter (January 1976). "Review of Aspects of Anglo-Saxon Archaeology, by Rupert Bruce-Mitford". The English Historical Review. XCI (358). London: Longman Group Limited: 172–173. JSTOR 565221. +"Boat Graves in Sweden". The Archaeological News Letter. 1 (5). Linden Publicity: 4, 7. August–September 1948. ISSN 0003-8040. +Bromley, Eustace Blake (1937). They Were Men Sent From God (PDF). Bangalore: The Scripture Literature Press. Archived (PDF) from the original on 30 July 2021. +Bruce-Mitford, Charles Eustace (1902). The Territory of Wei-hai-wei: A Descriptive Guide and Handbook with Maps and Illustrations a Historical Note and Appendix. Shanghai: Kelly & Walsh, Limited. +Bruce-Mitford, Charles Eustace (June 1903a). "Wei-Hai-Wei: Some Physical Characteristics of Our New Dependency in the Far East". The Geographical Teacher. II (2): 66–70. JSTOR 40554245. +Bruce-Mitford, Charles Eustace (July 1903b). "Mountain and Cliff at Weihaiwei". The East of Asia Magazine. II (2): 169–173. +Bruce-Mitford, Charles Eustace (1905). A New Geography of Japan for the Upper Forms of Schools and Colleges. Yokohama: Japan Gazette Press. +Bruce-Mitford, Charles Eustace (February 1908). "Notes on the Physiography of Certain Volcanoes in Northern Japan". The Geographical Journal. XXXI (2). London: The Royal Geographical Society: 187–198. Bibcode:1908GeogJ..31..187M. doi:10.2307/1776879. JSTOR 1776879. +Bruce-Mitford, Charles Eustace (1909). "The Active Volcanoes of Japan". Transactions of the Asiatic Society of Japan. XXXVII (I). Yokohama: Asiatic Society of Japan: 89–112. +Bruce-Mitford, Charles Eustace (19 July 1910). "The Volcanoes of Japan". The Times. No. 39, 328. London. p. 70. Archived from the original on 24 December 2025. Retrieved 24 December 2025 – via Newspapers.com. +Bruce-Mitford, Charles Eustace (1914). Japan's Inheritance: The Country, Its People, and Their Destiny. New York: Dodd, Mead & Co. +The Celebrated Collection of English and Continental Clocks Formed by the Late Courtenay A. Ilbert, Esq., F.B.H.I . London: Christie, Manson & Woods, Inc. 1958. OCLC 4020841. +Cherry, John (1995). "Rupert Bruce-Mitford 1914–1994" (PDF). Medieval Archaeology. 39. Society for Medieval Archaeology: 178–179. doi:10.5284/1000320. Archived (PDF) from the original on 2 September 2017. +Clark, Mary Kitson (1941). "Roman Yorkshire, 1940". The Yorkshire Archæological Journal. XXXV (138). Yorkshire Archæological Society: 222–227. +Cramp, Rosemary (1978). "Review of The Sutton Hoo Ship-Burial, volume 1, by Rupert Bruce-Mitford". The Antiquaries Journal. LVIII (2). London: The Oxford University Press: 426–428. doi:10.1017/S0003581500056158. +Cramp, Rosemary (23 March 1994). "Rupert Bruce-Mitford". Obituaries. The Independent. No. 2, 314. London. p. 14. Archived from the original on 24 October 2023 – via Newspapers.com. +Also published online +Cunliffe, B. W. (26 July 1969). "Bog Burials". Nature. 223 (5204). Macmillan (Journals) Ltd.: 423–424. Bibcode:1969Natur.223..423C. doi:10.1038/223423b0. +Dann, Robert Bernard (2004). Father of Faith Missions: The Life and Times of Anthony Norris Groves. Waynesboro, Georgia: Authentic Media. OCLC 58027950. +Dickinson, Tania M. (September 1975). "Review of Aspects of Anglo-Saxon Archaeology, by Rupert Bruce-Mitford". Antiquity. XLIX (195): 227–228. doi:10.1017/S0003598X00070101. S2CID 163138211. +Evison, Vera (1980). "Review of The Sutton Hoo Ship-Burial, volume 2, by Rupert Bruce-Mitford". The Antiquaries Journal. LX (1). London: The Oxford University Press: 124–127. doi:10.1017/S0003581500036179. +Farrell, Robert T. (October 1977). "Review of The Sutton Hoo Ship-Burial, volume 1, by Rupert Bruce-Mitford". Speculum. LII (4). Cambridge, Massachusetts: The Mediaeval Academy of America: 932–935. doi:10.2307/2855397. JSTOR 2855397. +Farrell, Robert T. (July 1981). "Review of The Sutton Hoo Ship-Burial, volume 2, by Rupert Bruce-Mitford, and of The Sutton Hoo Ship-Burial: Reflections after Thirty Years, by Rupert Bruce-Mitford". Speculum. 56 (3). Cambridge, Massachusetts: The Mediaeval Academy of America: 590–593. doi:10.2307/2847749. JSTOR 2847749. +Farrell, Robert T. (January 1985). "Review of The Sutton Hoo Ship-Burial, volume 3, by Rupert Bruce-Mitford". Speculum. 60 (1). Cambridge, Massachusetts: The Mediaeval Academy of America: 126–130. doi:10.2307/2852142. JSTOR 2852142. +Gardner, Samuel (1927). English Gothic Foliage Sculpture. Cambridge: Cambridge University Press. +Graham-Campbell, James (March 1977). "Review of The Sutton Hoo Ship-Burial, volume 1, by Rupert Bruce-Mitford". Antiquity. LI (201): 79–80. doi:10.1017/S0003598X00100754. S2CID 163133488. +Graham-Campbell, James (July 1979). "Review of The Sutton Hoo Ship-Burial, volume 2, by Rupert Bruce-Mitford". Antiquity. LIII (208): 159–160. doi:10.1017/S0003598X00042435. S2CID 163087755. +Hawkes, Christopher (December 1964). "Sutton Hoo: Twenty-Five Years After". Antiquity. XXXVIII (152): 252–257. doi:10.1017/S0003598X00031136. S2CID 164039505. +Hawkes, Christopher (1976). "Review of Aspects of Anglo-Saxon Archaeology, by Rupert Bruce-Mitford". The Antiquaries Journal. LVI (1). London: The Oxford University Press: 104–105. doi:10.1017/S0003581500019892. S2CID 163031781. +Henry, Françoise (June 1963). "The Lindisfarne Gospels". Antiquity. XXXVII (146): 100–110. doi:10.1017/S0003598X00037200. S2CID 163055695. +Hills, C. M. (March 1985). "Review of The Sutton Hoo Ship-Burial, volume 3, by Rupert Bruce-Mitford". Antiquity. LIX (225): 60–61. doi:10.1017/S0003598X00056611. S2CID 163185472. +Hinton, David A. (1977). ""Rudely Made Earthen Vessels" of the Twelfth to Fifteenth Centuries A.D.". In Peacock, David Philip Spencer (ed.). Pottery and Early Commerce: Characterization and Trade Roman and Later Ceramics. London: Academic Press. pp. 221–238. ISBN 0-12-547850-X. LCCN 77-77360. +Hoving, Thomas (1981). King of the Confessors. New York: Simon & Schuster. ISBN 0-671-43388-1. LCCN 81-9050. +Jope, E. M. (1989). "Preface". In Henderson, Julian (ed.). Scientific Analysis in Archaeology and Its Interpretation. Monograph (Oxford University Committee for Archaeology). Vol. 19. Oxford: Oxford University Committee for Archaeology Institute of Archaeology. pp. xi–xv. ISBN 0-947816-19-4. LCCN 89-26932. +Kirby, David Peter (January 1971). "Review of The Sutton Hoo Ship-Burial: A Handbook, by Rupert Bruce-Mitford". The English Historical Review. LXXXVI (338). London: Longman Group Limited: 153–154. doi:10.1093/ehr/LXXXVI.CCCXXXVIII.153. +Lomax, Eric (1995). The Railway Man. London: Jonathan Cape. ISBN 0-224-04187-8. +Magoun, Francis P. Jr. (January 1954). "The Sutton Hoo Ship-Burial: A Chronological Bibliography". Speculum. XXIX (1). Cambridge, Massachusetts: The Mediaeval Academy of America: 116–124. doi:10.2307/2853871. JSTOR 2853871. S2CID 162599371. +Mann, James (July–October 1954). "Anniversary Address". The Antiquaries Journal. XXXIV (3–4). London: Society of Antiquaries of London: 149–154. doi:10.1017/S0003581500059825. S2CID 246042776. +Martin-Clarke, D. Elizabeth (1947). Culture in Early Anglo-Saxon England (PDF). Baltimore: Johns Hopkins Press. Archived (PDF) from the original on 3 January 2019. +Mellor, Maureen (1997). Pots and People that Have Shaped the Heritage of Medieval and Later England. Oxford: Ashmolean Museum. ISBN 1-85444-080-2. +Myres, J. N. L. (1974). "Anniversary Address". The Antiquaries Journal. LIV (1). London: Society of Antiquaries of London: 1–7. doi:10.1017/S0003581500030055. S2CID 246045948. +Myres, J. N. L. (October 1977). "Review of The Sutton Hoo Ship-Burial, volume 1, by Rupert Bruce-Mitford". The English Historical Review. XCII (365). Longman Group Limited: 847–851. doi:10.1093/ehr/XCII.CCCLXV.847. JSTOR 567658. +Myres, J. N. L. (July 1980). "Review of The Sutton Hoo Ship-Burial, volume 2, by Rupert Bruce-Mitford". The English Historical Review. XCV (376). Longman Group Limited: 607–611. doi:10.1093/ehr/XCV.CCCLXXVI.607. JSTOR 568060. +Myres, J. N. L. (January 1985). "Review of The Sutton Hoo Ship-Burial, volume 3, by Rupert Bruce-Mitford". The English Historical Review. C (394). Longman Group Limited: 120–121. doi:10.1093/ehr/C.CCCXCIV.120. JSTOR 569930. +Nield, Robert (2015). China's Foreign Places: The Foreign Presence in China in the Treaty Port Era, 1840–1943. Hong Kong: Hong Kong University Press. ISBN 978-988-8139-28-6. +Pantin, William Abel (1937). "The Recently Demolished Houses in Broad Street, Oxford" (PDF). Oxoniensia. II. Oxford: Oxfordshire Architectural and Historical Society: 171–200. ISSN 0308-5562. Archived (PDF) from the original on 13 June 2012. +"Proceedings of the Society of Antiquaries". The Antiquaries Journal. XXX (3–4). London: Society of Antiquaries of London: 233–240. July–October 1950. doi:10.1017/S0003581500088077. S2CID 246042921. +"Proceedings". The Antiquaries Journal. LII (2). London: Society of Antiquaries of London: 438–460. 1972. doi:10.1017/S000358150002148X. +"Roman Britain in 1940". The Journal of Roman Studies. XXXVI. London: The Society for the Promotion of Roman Studies: 128–148. 1941. JSTOR 297109. +Ross, Edward Hunter (October 1975). "Review of Aspects of Anglo-Saxon Archaeology, by Rupert Bruce-Mitford". Archaeology. 28 (4). New York: Archaeological Institute of America: 279–280. JSTOR 41710181. +Rowlett, Ralph M. (December 1970). "Review of The Bog People, by P. V. Glob". American Anthropologist. 72 (6). Washington, D.C.: American Anthropological Association: 1568–1569. doi:10.1525/aa.1970.72.6.02a01080. JSTOR 672952. +Ryan, Michael (June 2006). "Review of A Corpus of Late Celtic Hanging Bowls, by Rupert Bruce-Mitford & Sheila Raven". Antiquity. 80 (308): 487–488. doi:10.1017/S0003598X00093923. S2CID 163688085. +Stunt, William Thornburgh; Pulleng, Alexander; Pickering, Arnold; Simmons, George Philip; Boak, Donald Kenneth & Warren, Stanley Francis (1972). Turning the World Upside Down: A Century of Missionary Endeavour (2nd ed.). Bath: Echoes of Service. +Tatford, Frederick Albert (1983). "The Godavari Delta". The Challenge of India. That the World May Know. Vol. 3. Bath: Echoes of Service. pp. 91–128. ISBN 0-946-214-02-6. +Tatford, Frederick Albert (1986). The Islands of the Sea. That the World May Know. Vol. 10. Bath: Echoes of Service. ISBN 0-946214-11-5. +Taylor, A. J. (1976). "Anniversary Address". The Antiquaries Journal. LVI (I). London: The Oxford University Press: 1–10. doi:10.1017/S0003581500019582. S2CID 246044560. +Werner, Joachim (1982). "Das Schiffsgrab von Sutton Hoo: Forschungsgeschichte und Informationsstand zwischen 1939 und 1980" [The Sutton Hoo Ship-Burial: Research and Publication Between 1939 and 1980]. Germania (in German). 60 (1): 193–209. doi:10.11588/ger.1982.94642. +Translated as Werner, Joachim (1985). The Sutton Hoo Ship-Burial: Research and Publication Between 1939 and 1980. Translated by Hawkes, Christopher & Hawkes, Sonia. Oxford: Privately printed. OCLC 848873715. +Williams, Harold S. (1958). Tales of the Foreign Settlements in Japan. Tokyo: Charles E. Tuttle Company. LCCN 58-11101. +Williams, Nigel (1992). "The Sutton Hoo Helmet". In Oddy, William Andrew (ed.). The Art of the Conservator. London: British Museum Press. pp. 73–88. ISBN 0-7141-2056-1. +Wilson, David M. (1992). "Sutton Hoo – Pros and Cons". In Farrell, Robert & Neuman de Vegvar, Carol (eds.). Sutton Hoo: Fifty Years After. American Early Medieval Studies. Vol. 2. Oxford, Ohio: Miami University. pp. 5–12. ISBN 1-879836-01-7. +ISBN incorrectly printed as ISBN 1-879836-02-7 {{isbn}}: ignored ISBN errors (link) +Wilson, David M. (2002). The British Museum: A History. London: British Museum Press. ISBN 0-7141-2764-7. diff --git a/corpus/fetch.py b/corpus/fetch.py new file mode 100755 index 0000000..0c86ecd --- /dev/null +++ b/corpus/fetch.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Fetches the prose corpus: plain text of featured Wikipedia articles. + + ./corpus/fetch.py writes corpus/sv.txt and corpus/en.txt + +- A fixed-seed sample of the featured articles of sv and en Wikipedia, so a + rerun fetches the same articles (unless the categories change). +- The text is the TextExtracts plain-text extract: no markup. Section + headings ("== ... ==") and empty lines are dropped. +- Stops at about TARGET characters per language. +""" +import json, random, sys, time, urllib.error, urllib.parse, urllib.request +from pathlib import Path + +TARGET = 1_500_000 +CATEGORIES = { + 'sv': 'Kategori:Wikipedia:Utmärkta artiklar', + 'en': 'Category:Featured articles', +} +HERE = Path(__file__).resolve().parent + + +def api(lang, **params): + params.update(format='json', formatversion='2') + url = f'https://{lang}.wikipedia.org/w/api.php?' + urllib.parse.urlencode(params) + req = urllib.request.Request(url, headers={'User-Agent': 'keyscore-corpus/0.1'}) + for attempt in range(8): + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + except urllib.error.HTTPError as e: + if e.code != 429: raise + wait = int(e.headers.get('Retry-After') or 0) or 5 * 2 ** attempt + print(f' rate-limited, waiting {wait} s', file=sys.stderr) + time.sleep(wait) + raise RuntimeError(f'still rate-limited: {url}') + + +def titles(lang, limit_pages=4): + out, cont = [], {} + for _ in range(limit_pages): + r = api(lang, action='query', list='categorymembers', cmtitle=CATEGORIES[lang], + cmnamespace=0, cmlimit=500, **cont) + out += [m['title'] for m in r['query']['categorymembers']] + if 'continue' not in r: break + cont = {'cmcontinue': r['continue']['cmcontinue']} + return out + + +def extract(lang, title): + r = api(lang, action='query', prop='extracts', explaintext=1, titles=title) + text = r['query']['pages'][0].get('extract', '') + lines = [l.strip() for l in text.split('\n')] + return '\n'.join(l for l in lines if l and not l.startswith('==')) + + +def main(): + for lang in CATEGORIES: + names = titles(lang) + random.Random(1).shuffle(names) + parts, size = [], 0 + for name in names: + if size >= TARGET: break + text = extract(lang, name) + parts.append(text); size += len(text) + print(f'{lang}: {name} ({len(text)} chars, {size} in all)', file=sys.stderr) + time.sleep(1.5) + (HERE / f'{lang}.txt').write_text('\n'.join(parts) + '\n', encoding='utf-8') + + +if __name__ == '__main__': + main() diff --git a/corpus/sv.txt b/corpus/sv.txt new file mode 100644 index 0000000..8b96b4e --- /dev/null +++ b/corpus/sv.txt @@ -0,0 +1,3160 @@ +Jorge Mario Pedro Vargas Llosa, förste markis av Vargas Llosa, född 28 mars 1936 i Arequipa, död 13 april 2025 i Lima, var en peruansk författare, journalist och politiker. Vargas Llosa räknas som en av Latinamerikas ledande romanförfattare och essäister, och som en av de främsta författarna i sin generation. Han anses vara en av de inflytelserikaste av de latinamerikanska författare som slog igenom under den latinamerikanska boomen under 1960- och 1970-talet. 2010 tilldelades han nobelpriset i litteratur för sin "kartläggning av maktens strukturer och knivskarpa bilder av individens motstånd, revolt och nederlag". +Vargas Llosa fick sitt genombrott på 1960-talet med romanerna Staden och hundarna (1963) och Det gröna huset (1965). Han var aktiv inom en rad litterära genrer och verkade som litteraturkritiker och journalist. Bland hans verk finns komedier, deckare, historiska romaner och politiska thrillers. I många av sina verk gav han sin syn på det peruanska samhället, men han blev med tiden mer internationellt inriktad, och hans senare verk berörde också andra delar av världen. I karriären gick Vargas Llosa från en litterär modernism till en lekfullare postmodernistisk stil. +Som många andra latinamerikanska författare var Vargas Llosa politiskt aktiv genom hela sin författarkarriär. Hans politiska åsikter gick gradvis från vänster till höger. Den kubanska revolutionen under Fidel Castros ledning hade till en början Vargas Llosas stöd, något han senare ångrade. 1990 ställde han upp som presidentkandidat i det peruanska valet för center-högerkoalitionen Frente Democrático (FREDEMO) och förespråkade nyliberala reformer. Betecknande för Vargas Llosas politiska hållning är hans grundläggande liberala värderingar, såsom vikten av individens frihet och moraliska ansvar, och hans försvar av demokratin. Senare gav han sin stöd åt mer traditionellt konservativa kandidater. År 1990 påstod han i ett uttalande på mexikansk teve att "Mexiko är den perfekta diktaturen", ett uttalande som levde kvar länge i Mexiko och internationellt. +Mario Vargas Llosa föddes i en medelklassfamilj den 28 mars 1936 i staden Arequipa i Peru. Han var det enda barnet till Ernesto Vargas Maldonado och Dora Llosa Ureta. Fadern var busschaufför och modern var dotter till en kreolfamilj. Makarna separerade några månader före Vargas Llosas födelse. Fadern hade en kärleksrelation med en tysk kvinna och från den relationen har Mario två yngre halvbröder, Enrique och Ernesto Vargas. +Släkten Vargas kom till Peru på 1500-talet. Namnet var taget från en adelsman i Trujillo, Juan de Vargas, på vars mark man hade tjänat som bönder eller vasaller. Det fanns en Vargas i den handfull conquistadorer som tog tillfånga Atahualpa. Släkten Vargas blev en av de mest spridda och Mario kommer ur en av dess grenar. Släkten Llosa kom till Peru på 1600-talet, namnet kommer från en by med samma namn i Katalonien. En gren kom med tiden till Santillana del Mar och därifrån utvandrade Juan de la Llosa y Llaguno till Arequipa och hans efterkommande höll sig sedan kvar i trakten, "de flesta med yrken såsom advokater, präster, nunnor, domare, lärare, funktionärer, poeter, tokstollar och en och annan militär." +Vargas Llosa levde hos sin mors familj i Arequipa till ett år efter föräldrarnas skilsmässa, då hans morfar Pedro Llosa blev honorärkonsul för Peru i Bolivia. Med sin mor och hennes familj flyttade Vargas Llosa därefter till Cochabamba i Bolivia, där han tillbringade sina tidiga barnaår. Familjen Llosa hölls ihop av morfadern, som drev en bomullsfarm. Modern och hennes familj ville inte förklara att Vargas Llosas föräldrar hade separerat och han växte upp i tron att hans far var död. När José Bustamante y Rivero var president i Peru fick hans morfar en befattning som prefekt i den peruanska kuststaden Piura och hela familjen flyttade tillbaka till Peru. Medan han bodde i Piura gick Vargas Llosa i grundskola vid den religiösa akademin Colegio Salesiano. 1946, vid tio års ålder, flyttade han till Lima där han för första gången träffade sin far. Föräldrarna återupprättade sin relation och bosatte sig under Vargas Llosas tonår i Magdalena del Mar, ett medelklassdistrikt i Lima. Han studerade då vid Colegio La Salle, en kristen salesiansk grundskola, från 1947 till 1949. +Fadern som hade upptäckt att sonen skrev poesi var orolig för hans framtid och försörjning. Han ansåg att det var lämpligt att skicka Vargas Llosa till en militärskola där han skulle få en "manlig" uppfostran (hombria). Vid fjorton års ålder skickades han sålunda till militärskolan Colegio Militar Leoncio Prado i La Perla, Callao. Det var ett internat där han gick tredje och fjärde året i educación secundaria åren 1950-51. Där fick han uppleva en järnhård militär disciplin och det var den epok då han läste och skrev "som han aldrig hade gjort förut", och formade så sin förmåga som författare. Han tyckte mest om att läsa romaner av franska författare som Alexander Dumas och Victor Hugo. Bland hans lärare figurerade den surrealistiske poeten César Moro, som under en tid undervisade Vargas Llosa i franska. +När sommarlovet började 1952 började Vargas Llosa arbeta som assistent på tidningen La Crónica. Kort därefter övergick han till att vara journalistlärling, och fick arbeta med reportage och intervjuer om lokala händelser. Ett år innan han tog sin examen började Vargas Llosa arbeta som amatörjournalist för lokala tidningar. Han slutade studera vid militärskolan och avslutade sina studier i Piura, där han arbetade för den lokala tidningen La Industria och bevittnade framförandet av sitt första dramatiska verk, Inkans flykt (La huida del Inca). +År 1953, under Manuel Odrías regering, skrev Vargas Llosa in sig vid det anrika universitetet San Marcos i Lima, för att studera juridik och litteraturvetenskap. Han gifte sig år 1955 vid 19 års ålder med sin morbrors svägerska, den 10 år äldre nyskilda Julia Urquidi Illanes. Om detta har Vargas Llosa skrivit i den självbiografiska "Tant Julia och författaren" (där han dock ändrade hustruns ålder till 13 år äldre). Familjen Llosa uppfattade äktenskapet som en riktig skandal. Vargas Llosa började sin litterära karriär på allvar 1957 när hans första noveller publicerades, Los jefes (Ledarna) och El abuelo (Farfadern/morfadern), medan han arbetade för två peruanska tidningar: El Mercurio Peruano och El Comercio. När han tog examen vid San Marcos 1958 fick han ett stipendium, Javier Prado, som gjorde det möjligt för honom att åka till Madrid, där han studerade vid Universidad Complutense, 1960, efter att stipendiet hade gått ut, flyttade han till Frankrike i förvissningen om att han skulle få ett stipendium för att studera där. Men när han anlände till Paris fick han veta att hans ansökan om stipendium hade avslagits. Trots Marios och Julias osäkra ekonomiska läge bestämde sig paret för att stanna i Paris där han började skriva produktivt. . +Vargas Llosas kusin Patricia Llosa Urquidi flyttade som 15-åring in hos paret under sina studier och Vargas Llosa förälskade sig så småningom i henne och skilde sig från Julia Urquidi Illanes 1964. Året därpå gifte han om sig med den då 19-åriga Patricia Llosa Urquidi som han sedermera fick tre barn med: Álvaro (född 1966), författare och redaktör, Gonzalo (född 1967), arbetar för FN-organet UNHCR i Genève, och Morgana (född 1974), fotograf/journalist för tidningen El País i Madrid. 1971 lade han också fram sin doktorsavhandling i Madrid med titeln Gabriel García Márquez: historia de un deicidio som publiceras samma år. Under äktenskapet skötte hustrun Patricia allt det praktiska och ekonomiska runt Mario Vargas Llosa. Då han mottog Nobelpriset prisade han Patricia Llosa som "bra på allt" och citerade henne ""Mario, para lo único que tú sirves es para escribir" ("Mario, det enda du duger till är att skriva."), +Efter 50 års äktenskap lämnade Mario Vargas Llosa sin andra hustru Patricia Llosa Urquidi för Julio Iglesias före detta hustru Isabel Preysler, spansk modell och programledare med filippinskt ursprung, mor till bland annat Enrique Iglesias. Paret levde tillsammans från 2015 till december 2022 då de meddelade att de hade avslutat sin relation. +Vargas Llosas första roman Staden och hundarna (La ciudad y los perros) publicerades 1963. Handlingen utspelar sig kring en grupp kadetter på Limas militärskola, och bygger på författarens egna erfarenheter under hans två år vid Colegio Militar Leoncio Prado. Detta tidiga verk fick stor uppmärksamhet och var en omedelbar framgång. Kritiker imponerades av romanens vitalitet och författarens skickliga användning av sofistikerade litterära tekniker. Romanen vann också kritikerpriset Premio de la Crítica Española. Bokens skarpa kritik av det peruanska militäretablissemanget ledde dock till kontrovers. Tusen exemplar av boken brändes ceremoniellt vid skolan och flera peruanska generaler gick till hårt angrepp. En av generalerna sa att boken hade skrivits av ett ”degenererat sinne” och en annan att romanen var betald av Ecuador för att underminera den peruanska arméns anseende. Boken hade framgång, men hos Vargas Llosa kvarstod för alltid ett tvivel på om detta berodde på bokens kvaliteter eller på skandalen runt den. +År 1965 publicerades Det gröna huset (La casa verde), en roman om en bordell med samma namn och om hur dess halvmytiska närvaro påverkar bokens gestalter. Huvudhandlingen kretsar kring Bonifacia, en flicka som har för avsikt att avlägga klosterlöfte, men som förvandlas till la Selvatica, bordellens kändaste prostituerade. Boken hyllades av kritikerna och befäste Vargas Llosa som en central figur inom den latinamerikanska litteraturen. Det gröna huset tilldelades det första Rómulo Gallegos-priset 1967 i tävlan med verk av den uruguayanske veteranförfattaren Juan Carlos Onetti och av Gabriel García Márquez. Vargas Llosa fick tillräckligt många priser enbart genom Det gröna huset för att bli en av de centrala författarna inom den latinamerikanska boomen. Många kritiker anser fortfarande att boken är författarens bästa och viktigaste verk. Gerald Martin, en brittisk kritiker av latinamerikansk litteratur, har framhållit den som en av de största latinamerikanska romanerna någonsin. +Vargas Llosa's tredje roman Samtal i katedralen publicerades 1969 när han var 33 år gammal. Den är en berättelse om ministersonen Santiago Zavala och hans chaufför Ambrosio. När de två av en slump träffas på ett hem för herrelösa hundar uppstår en intensiv konversation som fortsätter på Katedralen, en näraliggande bar. Under samtalet, som belyser den härskande diktaturen, söker Zavala efter sanningen om sin fars roll i mordet på en ökänd peruansk motståndsman. Zavalas sökande leder honom dock in i en återvändsgränd utan svar eller tecken på en bättre framtid. Romanen angriper Odrías hårdföra regim genom att visa hur en diktatur kontrollerar och förstör människors liv. Hopplösheten är ett genomgående tema i Samtal i katedralen, som är en av Vargas Llosas bittraste romaner. +År 1971 publicerade Mario Vargas Llosa sin doktorsavhandling vid Complutenseuniversitet i Madrid. García Márquez: historia de un deicidio ('García Márquez: berättelsen om ett gudamord') är en omfattande studie av hans vän den colombianske författaren och nobelpristagaren i litteratur, Gabriel García Márquez som han senare blev ovän med. År 1976 slog han Márquez i ansiktet i Mexico City vid Palacio de Bellas Artes; detta blev slutet på deras vänskap. Ingen av författarna har gett någon offentlig förklaring till osämjan, och de båda har en överenskommelse om att inte prata om varandra. Ett fotografi av Márquez med en blåtira publicerades 2007, vilket åter väckte allmänhetens intresse för fejden. Trots flera decenniers tystnad godkände Vargas Llosa år 2007 att delar av hans bok skulle användas som inledning till en specialutgåva till 40-årsjubileet av Márquez bok Hundra år av ensamhet (1967). Historia de un Deicidio gavs också ut igen samma år, som en del av Vargas Llosas samlade verk. +Efter Samtal i katedralen gick Vargas Llosas från allvarligare teman som politik och samhällsproblem till mer satiriska skildringar. Den latinamerikanska litteraturforskaren Raymond L. Williams beskriver denna fas i Vargas Llosas författarkarriär som "upptäckten av humor". Hans första satiriska roman var Pantaleón och soldatbordellen, som publicerades 1973. Denna korta, komiska roman består av vinjetter av samtal och dokument om de peruanska arméstyrkorna och en kår med prostituerade med uppdrag att besöka militära utposter i avlägsna djungelområden. Handlingen liknar den i hans tidigare roman Det gröna huset men i annan form. Pantaleón och soldatbordellen är sålunda en sorts parodi på både Det gröna huset och den litterära utgångspunkt som romanen representerar. Idén till romanen fick han genom att själv bevittna hur prostituerade anlitades av den peruanska armén för att betjäna soldater i djungeln. +Från 1974 till 1987 fokuserade Vargas Llosa på sitt skrivande, men tog sig även tid till andra initiativ. 1975 var han med och regisserade filmatiseringen av Pantaleón och soldatbordellen. Efter denna föga framgångsrika produktion valdes han till ordförande i den internationella författarklubben PEN. Vid den tiden reste Vargas Llosa ofta för att föreläsa på konferenser som organiserades av internationellt kända institutioner, såsom Hebreiska universitetet i Jerusalem och Universitetet i Cambridge. +År 1977 publicerade Vargas Llosa Tant Julia och författaren (La tía Julia y el escribidor), som delvis bygger på hans äktenskap med sin första fru, Julia Urquidi, som han även tillägnade romanen. Hon skrev senare sina memoarer, Lo que Varguitas no dijo (Vad lille Vargas inte sa), där hon gav sin egen syn på deras relation. Hon menar att Vargas Llosa på flera ställen överdrivit många negativa delar av deras relation, samtidigt som han bagatelliserat hennes stöd för sin författarkarriär. Tant Julia och författaren anses vara ett av de mest slående exemplen på hur bild- och talspråket i populärkulturen kan användas i litteraturen. Romanen filmatiserades 1990 i Hollywood under namnet Tune in Tomorrow. +Mario Vargas Llosas fjärde längre skönlitterära verk Kriget vid världens ände gavs ut 1981 och var hans första historiska roman. Boken blev början på en radikal utveckling av hans litterära stil mot frågor som messianism och irrationellt mänskligt beteende. Romanen utspelar sig under Canudoskriget i Brasilien i mitten av 1890-talet, en väpnad konflikt där en milleniaristisk sekt höll stånd mot den brasilianska armén i flera år. Precis som Vargas Llosas tidigaste verk behandlar den ett allvarligt motiv på ett sakligt sätt, och är skriven i en mörk ton. Kritiker uppskattade hur Vargas Llosas utforskade mänsklighetens fallenhet för att idealisera våld och hans skildringar av en katastrof orsakad av fanatism. Trots att romanen hyllades även i Brasilien fick den till en början ett dåligt mottagande, som en utlännings skildring av ett brasilianskt tema. Den kritiserades också för att vara revolutionär och antisocialistisk. Vargas Llosa säger själv att det är hans egen favoritroman och den som varit svårast att skriva. +Efter att ha avslutat Kriget vid världens ände började Vargas Llosa skriva romaner som var betydligt kortare än många av hans tidigare böcker. År 1983 färdigställde han Den sanna berättelsen om kamrat Mayta (Historia de Mayta), en roman med fokus på ett vänsteruppror som ägde rum den 29 maj 1962 i staden Jauja i Anderna. Senare samma år, under Sendero Luminosos uppror, inbjöds Vargas Llosa av den peruanska presidenten Fernando Belaúnde Terry att ingå i en kommission som skulle granska massakern på åtta journalister av byinvånarna i Uchuraccay. Kommissionens huvudsyfte var att undersöka morden för att kunna informera allmänheten om händelsen. Efter sitt engagemang i kommissionen publicerade Vargas Llosa en serie artiklar där han försvarade sin hållning i affären. År 1986 blev han klar med sin nästa roman, Vem dödade Palomino Molero? (¿Quién mató a Palomino Molero?), en morddeckare som han började skriva strax efter undersökningen av vad som hände i Uchuraccay. Handlingen i romanen ligger nära händelserna i Uchuraccay, men litteraturkritikern Roy Boland har framhållit att den inte är så mycket ett försök att rekonstruera morden som en form av "litterär exorcism" av Vargas Llosas egna erfarenheter som medlem av kommissionen. Arbetet med kommissionen inspirerade även en av Vargas Llosas senare romaner, Döden i Anderna (Lituma en los Andes), som först gavs ut 1993 i Spanien. +Det tog nästan 20 år innan Vargas Llosa skrev nästa större verk. Bockfesten (La fiesta del chivo), en politisk thriller som publicerades år 2000. Enligt Raymond L. Williams är Bockfesten Vargas Llosas närmast kompletta och ambitiösaste roman sedan Kriget vid världens ände. Romanen är baserad på diktatorn Rafael Trujillo, som styrde Dominikanska republiken från 1930 till dess han mördades 1961. Verket har tre huvudmotiv. Det första rör Urania Cabral, dotter till en före detta politiker, samt lojal Trujilloanhängare som för första gången återvänder till Dominikanska republiken efter mordet på Trujillo trettio år tidigare. Det andra motivet är själva mordet; konspiratörerna som utförde det och dess konsekvenser. Det tredje motivet handlar om Trujillo själv under slutet av hans regim. Boken fick snabbt positiva recensioner i Spanien och Latinamerika. Den anses vara ett av Vargas Llosas bästa verk. +År 2006 skrev Vargas Llosa Den stygga flickans rackartyg (Travesuras de la niña mala) som journalisten Kathryn Harrison anser vara en omskrivning (snarare än en bearbetning) av Gustave Flauberts Madame Bovary från 1856. I Vargas Llosas version handlar boken om en peruansk utvandrares besatthet av en flicka som han har älskat i årtionden, sedan tonåren. År 2013 utkom Den blygsamme hjälten, hans första roman efter nobelpriset. Den har beskrivits som en roman där Vargas Llosa i pusseldeckarens form skildrar det samtida Peru. +Som många andra latinamerikanska intellektuella stödde Mario Vargas Llosa i början Fidel Castros kubanska revolutionära regering. Han studerade marxism på djupet som universitetsstudent och övertygades senare av kommunistiska ideal efter den kubanska revolutionens framgångar. Gradvis började dock Vargas Llosa anse att kubansk socialism var oförenlig med hans syn på mänskliga fri- och rättigheter. Den officiella brytningen mellan författaren och den politik som den kubanska regeringen förde kom med den så kallade Padilla-affären, när Castro fängslade poeten Heberto Padilla. Vargas Llosa skrev tillsammans med andra intellektuella en protest mot det kubanska politiska systemet och dess fängslande av Padilla som de skickade till Castro. Vargas Llosa har sedan dess identifierat sig med liberalismen snarare än vänsterns ideologier. Efter att han gav upp sin tidigare vänsterpolitik har han motsatt sig både såväl höger- som vänsterorienterade auktoritära regimer. +När Vargas Llosa fick en befattning i undersökningskommissionen 1983 ledde det till vad litteraturkritikern Jean Franco har kallat "den obekvämaste händelsen i hans politiska karriär". Hans medverkan i kommissionen gav upphov till omedelbara negativa reaktioner och förtal från den peruanska pressen. Många menade att massakern var en konspiration för att hindra journalisterna att rapportera om regeringens paramilitära aktiviteter i Uchuraccay. Kommissionens slutsats var att byborna själva låg bakom morden. Vargas Llosa såg händelsen som ett tecken på "hur sårbar demokratin är i Latinamerika och hur lätt den kan utplånas av både höger- och vänsterdiktaturer. Dessa slutsatser och Vargas Llosa själv kritiserades hårt. Antropologen Enrique Mayer anklagade honom för paternalism, medan Meyers kollega Carlos Iván Degregori kritiserade honom för hans okunnighet om samhället i Anderna. Vargas Llosa anklagades också för att aktivt hjälpa regeringen att mörklägga arméns roll i massakern. Den amerikanska litteraturvetaren Misha Kokotovic har sammanfattat kritiken med att Vargas Llosa anklagades för att betrakta "inhemska kulturer som ett 'primitivt' hinder mot förverkligandet av hans västerländska modernitetsideal". Vargas Llosa chockades av massakern i sig och sedan av reaktionen mot hans slutsatser i kommissionen. Hans reaktion mot kritiken var att de brydde sig mer om hans rapport än de hundratals bönder som senare mördades av Sendero Luminosos gerilla. +Under 1980-talet blev Vargas Llosa känd för sin principfasta nyliberala hållning. Han stödde den konservativa regeringen under Fernando Belaúnde Terry men avvisade 1984 en inbjudan att bli premiärminister. År 1987 hjälpte han till att bilda och senare leda Movimiento Libertad. Året därpå gick hans parti in i en koalition med Perus två största konservativa politiska partier vid den tiden, Acción Popular under före detta presidenten Fernando Belaúnde Terry och Partido Popular Cristiano, lett av Luis Bedoya Reyes, för att skapa den tredelade center-högerkoalitionen Frente Democrático (FREDEMO). Han ställde upp i Perus presidentval 1990 som kandidat för FREDEMO. Han föreslog ett drastisk ekonomiskt sparprogram som skrämde fattiga väljare. Programmet betonade behovet av privatisering, marknadsekonomi, frihandel och framför allt spridningen av privat egendom. Han vann den första valomgången med 34% av alla röster, men besegrades av den då okända jordbruksingenjören Alberto Fujimori i den andra valomgången. Vargas Llosa redogjorde för valet i sin memoar El pez en el agua, ('En fisk i vattnet') 1993. Efter valet 1990 har han främst fokuserat på att skriva, dock med ett sporadiskt engagemang inom politiken. +En månad efter att han förlorat valet bjöds Vargas Llosa in till en konferens för intellektuella i Mexiko av det landets mest framstående författare, Octavio Paz. (En månad efter konferensen tilldelades Paz Nobelpriset i litteratur.) Konferensen, som gick under namnet "El siglo XX: la experiencia de la libertad" ('1900-talet: frihetens erfarenhet') ägnades åt att diskutera kommunismens sammanbrott i Öst- och Centraleuropa. Den sändes i mexikansk tv mellan den 27 augusti och den 2 september. I ett uttalande den 30 september 1990 fördömde Vargas Llosa det mexikanska maktsystemet, som baserades på partiet Partido Revolucionario Institucionals (PRI, som suttit vid makten i 61 år) permanenta styre, i ett uttalande: "Mexiko är den perfekta diktaturen. Den perfekta diktaturen är inte kommunism, inte Sovjetunionen, inte Fidel Castro; den perfekta diktaturen är Mexiko. För det är en kamouflerad diktatur." Med hänvisning till PRI lade han till: "Jag tror inte att det någon annanstans i Sydamerika har funnits en diktatur som så effektivt har rekryterat intelligentsian och så omärkligt mutat den." Uttalandet "Mexiko är den perfekta diktaturen" blev en kliché i Mexiko och internationellt, ända tills PRI förlorade makten i Mexiko år 2000. +Vargas Llosa bodde mestadels i London från 1990-talet, men tillbringade ungefär tre månader om året i Peru. Vargas Llosa fick spanskt medborgarskap 1993. Han besökte ofta Spanien för olika konferenser och semestrade ofta där. 1994 valdes han till medlem av Real Academia Española (Spanska kungliga akademien) där han från januari 1996 innehade stol ”L” och han var aktiv på den spanska politiska arenan. I februari 2008 slutade han att stödja Partido Popular till förmån för det nyskapade partiet Enhet, framsteg och demokrati. Vargas Llosa påstod att vissa av Partido Populars konservativa ståndpunkter gick emot hans klassiskt liberala åsikter. Hans politiska ideologier framträder i boken Política razonable ('Rimlig politik'), skriven av politikerna och kulturpersonligheterna Fernando Savater, Rosa Díez, Álvaro Pombo, Albert Boadella och Carlos Martínez Gorriarán. I nätverket Fundación Internacional para la Libertad knöt Vargas Llos samman ett stort antal tankesmedjor och forskningsinstitut för att skapa en samlingspunkt för den spansktalande världen, med syfte att utveckla och sprida liberala värderingar. Han verkade även som gästprofessor vid flera framstående universitet. Vid Georgetown University, Washington DC, fick han en tjänst hösten 2009, vid institutet Ibero-American Literature and Culture som hade i flera år. Den 3 februari 2011 adlades han av Spaniens kung Juan Carlos I med den ärftliga titeln markis av Vargas Llosa (spanska: Marqués de Vargas Llosa). Hans äldste son Álvaro är arvtagare till titeln. +År 2010 kom Mario Vargas Llosa med ett uppseendeväckande uttalande där han menade att kriget mot droger var ett enda stort misslyckande. Llosa berömde stater som Mexiko, som agerat med stort mod för att bekämpa drogkarteller men menade samtidigt att det inte gick att vinna detta krig. Samhället ”måste legalisera narkotikan” istället för att lägga enorma summor på att bekämpa den. Att satsa på ”rehabilitering, prevention och utbildning”. Llosa drog paralleller med dagens syn på tobak och den informationskampanj som fått stor framgång. Detta skulle vara det enda sättet att få slut på det våld knarkindustrin för med sig. +I juni 2011 vände sig Llosa tillsammans med flera andra framstående personer i samhället, däribland FN:s tidigare generalsekreterare Kofi Annan och entreprenören Richard Branson, till FN där de bland annat krävde: +Att det blir ett slut på kriminaliseringen och stigmatiseringen av personer som nyttjar droger men som inte skadar andra +Att regeringar uppmuntras att testa legalisering av droger, i första hand marijuana. +Att kampanjer som riktar sig till unga med budskapen "säg nej till droger" eller "nolltolerans mot droger" slopas, i stället bör andra utbildningsmetoder användas. +Mario Vargas Llosas stil bygger på såväl historiska händelser som personliga erfarenheter. Hans erfarenheter vid militärskolan Leoncio Prada var grunden för beskrivningen av den korrumperade samhällsinstitutionen i hans första roman Staden och hundarna. Skolans korruption är också en återspegling av korruptionen i det peruanska samhället vid tiden för romanens tillkomst. Berättartekniken är inspirerad av den nya romanen. Vargas Llosa använder ofta sina böcker för att utmana det otillräckliga i samhället, såsom moralens förfall och förtrycket från dem som har politisk makt gentemot dem som utmanar denna makt. Ett av de teman som han har utforskat är individens strävan efter frihet i en verklighet fylld av förtryck. Till exempel baseras hans roman Samtal i katedralen på Perus president Manuel Odrías diktatur. Bokens huvudperson Santiago gör uppror mot den kvävande diktaturen genom att delta i vänsterpolitiska gruppers subversiva aktiviteter. Vargas Llosas andra roman, Det gröna huset, utforskar förutom teman som korruption och förtryck också "fördömandet av Perus grundläggande institutioner", och tar upp korrupta officerares misshandel och utnyttjande av de anställda vid en bordell. +Många av Vargas Llosas tidiga romaner utspelar sig i Peru, medan de senare verken utspelar sig i andra delar av Sydamerika, såsom Brasilien och Dominikanska republiken. Kriget vid världens ände var den första av hans stora romaner som utspelade sig utanför hans hemland. Trots att handlingen berör Canudoskriget och den brasilianska regeringen bygger romanen inte direkt på historiska fakta. Dess huvudsakliga inspirationskälla är snarare den icke-fiktiva redogörelsen för dessa händelser som publicerades av den brasilianske författaren Euclides da Cunha 1902. Bockfesten, som baserats på Rafael Trujillos diktatur, utspelar sig i Dominikanska republiken. För att förbereda boken studerade Vargas Llosa grundligt dominikansk historia. Romanen är realistisk till sin karaktär, och Vargas Llosa understryker att han "respekterade den grundläggande informationen, [...] jag har inte överdrivit", men samtidigt påpekar han att "det är en roman, ingen historiebok, så jag tog mig många, många friheter". +En av Vargas Llosas senare romaner, Paradiset finns om hörnet, utspelar sig till stor del i Frankrike och på Tahiti. Den är baserad på före detta samhällsutvecklaren Flora Tristan, och visar hur Flora och Paul Gaugin inte fann paradiset, men ändå lyckades inspirera efterföljare att arbeta för en socialistisk utopi. Vissa kritiker, såsom Barbara Mujica, menar dock att Paradiset finns om hörnet saknar den "djärvhet, energi, politiska vision och berättartekniska geni" som fanns i hans tidigare verk. +Vargas Llosas verk betraktas som både moderna och postmoderna. Trots att skillnaden mellan modern och postmodern litteratur är flitigt omdebatterad hävdar litteraturvetaren M. Keith Booker att problematiken och den tekniska komplexiteten i Vargas Llosas tidiga verk, som Det gröna huset och Samtal i katedralen, är inslag som är typiska för den moderna romanen. De tidiga romanerna innehåller också ett visst mått av allvar, även det en viktig del av definitionen av modern konstutövning. Vargas Llosas senare romaner, till exempel Pantaleón och soldatbordellen, Tant Julia och författaren, Den sanna berättelsen om kamrat Mayta och El hablador ('Berättaren'), bär tydliga spår av postmodernt skrivande. Dessa har en mycket lättare, farsartad och komisk ton, något som är typisk för postmodernismen. Booker har jämfört två av Vargas Llosas romaner, Det gröna huset och Pantaleón och soldatbordellen, och diskuterar kontrasten mellan modernism och postmodernism som återfinns i författarens verk. Booker påpekar att samtidigt som båda romanerna utforskar ämnen som prostitution och den peruanska militären är den förra gravallvarlig medan den senare är löjligt komisk. +Litteraturvetaren M. Keith Booker menar att Vargas Llosa fulländar tekniken med sammanvävda dialoger i sin roman Det gröna huset. Genom att väva samman två samtal som utspelar sig vid olika tillfällen skapar han illusionen av en återblick. Vargas Llosa använder också denna teknik ibland för att byta miljö genom att väva ihop två samtal som tar plats samtidigt men på olika ställen. Denna teknik, som han började använda mot slutet av sin första roman, Staden och hundarna, är kännetecknande för hans litterära repertoar. Han använder sig dock inte av sammanvävda dialoger på samma sätt i alla romaner. Till exempel används tekniken i Det gröna huset på ett seriöst sätt, för att uppnå en nykter ton och för att fokusera på släktskapet mellan händelser som äger rum vid olika tillfällen och på olika platser, medan samma teknik skapar komiska effekter i Pantaleón och soldatbordellen, där enklare scenbyten används. Denna teknik liknar både Virginia Woolfs sammanflätande av olika karaktärers monologer och Gustave Flauberts kontrapunktiska teknik där han flätar samman samtal med andra händelser, såsom föreläsningar. +Mario Vargas Llosas första litterära förebilder var relativt okända peruanska författare som Martín Adán, Carlos Oquendo de Amat och César Moro. Som ung författare inspirerades han av dessa revolutionära författare för att skapa nya berättarstrukturer och tekniker för att ge en mer samtida, mångfacetterad bild av det urbana Peru. Han sökte en stil som skiljde sig från de traditionella skildringar av landet och landsbygdslivet som blivit berömda genom José María Arguedas, Perus främsta romanförfattare vid den tiden. Vargas Llosa beskrev Arguedas som "ett exempel på en gammalmodig regionalism som redan hade uttömt sina visionära möjligheter". Trots att han inte delade Arguedas intresse för ursprungsbefolkningens situation, beundrade och respekterade Vargas Llosa honom för hans bidrag till den peruanska litteraturen. År 1996 publicerade han också La utopía arcaica ('Den ålderdomliga utopin'), en studie över Arguedas verk. +Vargas Llosa begränsade sig inte till peruansk litteratur, han letade också inspiration utomlands. Två fransmän, existentialisten Jean-Paul Sartre och romanförfattaren Gustave Flaubert, influerade både hans teknik och hans stil. Sartres påverkan är som påtagligast i Vargas Llosas flitiga användning av dialog. Flauberts litterära självständighet (hans romaner går i många fall emot såväl verklighet som moral) är något som väckt Vargas Llosas beundran, och som även kom till uttryck i La orgía perpetua. Flaubert y "Madame Bovary" ('Den eviga orgien. Flaubert och Madame Bovary'), Vargas Llosas studie av Flauberts estetik. I sin analys av den franske författaren ifrågasätter Vargas Llosa litteraturens revolutionära kraft i ett politiskt sammanhang, något som skiljer sig tydligt från hans tidigare syn på "litteratur som en form av uppror". +En av Vargas Llosas favoritförfattare, och den som troligen haft störst inflytande på hans skrivande, är amerikanen William Faulkner. Vargas Llosa anser att Faulker är "författaren som fulländade den moderna romanens teknik". Båda använder komplicerade förändringar i tid och berättande i sin litterära stil. Till exempel har handlingen och utvecklingen av huvudpersonen i Staden och hundarna till viss del influerats av Vargas Llosas favoritroman av Faulkner, Ljus i augusti. +Förutom studierna av Arguedas och Flaubert har Vargas Llosa skrivit litterär kritik av andra författare som han beundrat, såsom Gabriel García Marquez, Albert Camus, Ernest Hemingway och Jean-Paul Sartre. Målet med hans icke-fiktiva verk har främst varit att visa på hur dessa författare influerat hans skrivande och visa på kopplingen mellan honom själv och de andra författarna. Kritikern Sara Castro–Klarén menar att han inte erbjuder mycket till systematisk analys av dessa författares litterära tekniker. Till exempel fokuserar Vargas Llosa i La orgía perpetua: Flaubert y "Madame Bovary" på förhållandet mellan sin estetik och Flauberts, snarare än att endast diskutera Flauberts egen estetik. +Mario Vargas Llosa anses vara en av de stora latinamerikanska författarna, tillsammans med författare som Octavio Paz, Julio Cortázar, Jorge Luis Borges, Gabriel García Márquez och Carlos Fuentes. Fuentes själv analyserade redan 1971 i boken La Nueva Novela ('Den nya romanen') de positiva influenser som Varga Llosas verk har haft på den latinamerikanska litteraturen. Kritikern Gerald Martin skrev 1987 att Vargas Llosa var "kanske den framgångsrikaste [...och] tveklöst den kontroversiellaste latinamerikanska författaren under de senaste tjugofem åren". +Vargas Llosa är känd för sitt journalistiska arbete. Han publicerade i maj 1960 sin första tidningsartikel i nummer 198 av tidskriften Caretas (Lima). Den 25 juli 1977 var det premiär för hans kolumn Piedra de toque, en författares betraktelse över mänskliga angelägenheter. Sedan 1997 publiceras Varga Llosas kolumner varannan vecka i Caretas, liksom i mer än 20 tidningar och tidskrifter i hela västvärlden. Han var också programledare i peruansk teve åren 1980–1982, för kulturmagasinet La torre de Babel ('Babels torn'). +De flesta av Vargas Llosas verk har översatts till flera språk och nått betydande internationell framgång. Även hans journalistiska arbete har blivit publicerat internationellt, något som relativt få andra latinamerikanska författare lyckats med. Under sin karriär har han skrivit mer än ett dussin romaner och ett antal andra böcker och berättelser, och har i flera decennier varit en tongivande röst inom latinamerikansk litteratur. Han har vunnit flera priser för sitt författarskap, från Premio Leopoldo Alas 1959 och Premio Biblioteca Breve 1962 till Premio Planeta 1993 för Döden i Anderna och Jerusalem Prize 1995. Litteraturkritikern Harold Bloom inkluderade Kriget vid världens ände i sin lista över litterära verk som bör ingå i den västerländska litterära kanonen. Vargas Llosa tilldelades även Miguel de Cervantes-priset 1994, ett av de mest prestigefyllda priserna inom den spanskspråkiga litteraturen, som delas ut till författare "vars arbete har bidragit till att påtagligt berika det spanskspråkiga litterära arvet". Vargas Llosa fick även Irving Kristol Award 2005 från American Enterprise Institute och mottog Harold and Ethel L. Stellfox Visiting Scholar and Writers Award vid Dickinson College 2008. År 2010 tilldelades han Nobelpriset i litteratur för sin "kartläggning av maktens strukturer och knivskarpa bilder av individens motstånd, revolt och nederlag". +Flera av Vargas Llosas verk har filmatiserats, bland andra Staden och hundarna och Pantaleón och soldatbordellen (båda i regi av landsmannen Francisco Lombardi), samt Bockfesten som regisserades av Vargas Llosas kusin Luis Llosa. Tant Julia och författaren blev filmatiserad på engelska som Tune in Tomorrow. Bockfesten har även dramatiserats för teatern av Jorge Alí Triana, en colombiansk pjäsförfattare och regissör. +Svenska översättningar inom parentes med utgivningsår om det skiljer sig från det spanska originalet. +1959 – Los jefes +1963 – La ciudad y los perros (Staden och hundarna, översättning Jan Sjögren 1966 och Peter Landelius 2013) +1966 – La casa verde (Det gröna huset, översättning Jens Nordenhök 1979) +1967 – Los cachorros (Valparna, översättning Annika Ernstson 1978, inkluderar även Los jefes) +1969 – Conversación en la catedral (Samtal i katedralen, översättning Annika Ernstson 1994) +1973 – Pantaleón y las visitadoras (Pantaleón och soldatbordellen, översättning Jan Sjögren 1975) +1977 – La tía Julia y el escribidor (Tant Julia och författaren, översättning Jens Nordenhök 1981) +1981 – La guerra del fin del mundo (Kriget vid världens ände, översättning Jens Nordenhök 1984) +1984 – Historia de Mayta (Den sanna berättelsen om kamrat Mayta, översättning Jens Nordenhök 1987) +1986 – ¿Quién mató a Palomino Molero? (Vem dödade Palomino Molero?, översättning Jens Nordenhök 1989) +1987 – El hablador (Berättaren, översättning Annika Ernstson 1991) +1988 – Elogio de la madrastra (Till styvmoderns lov, översättning Annika Ernstson 1992) +1993 – Lituma en los Andes (Döden i Anderna, översättning Annika Ernstson 1998) +1997 – Los cuadernos de don Rigoberto (Don Rigobertos anteckningsböcker, översättning Elisabeth Helms 2000) +2000 – La fiesta del chivo (Bockfesten, översättning Peter Landelius 2002) +2003 – El paraíso en la otra esquina (Paradiset finns om hörnet, översättning Peter Landelius 2003) +2006 – Travesuras de la niña mala (Den stygga flickans rackartyg, översättning Peter Landelius 2007) +2010 – El sueño del celta (Keltens dröm, översättning Peter Landelius 2011) +2013 – El héroe discreto (Den blygsamme hjälten, översättning Peter Landelius 2014) +2016 – Cinco esquinas (En peruansk affär, översättning Peter Landelius 2017) +1971 – García Márquez: historia de un deicidio +1975 – La orgía perpetua: Flaubert y "Madame Bovary" +1990 – La verdad de las mentiras: ensayos sobre la novela moderna +1993 – El pez en el agua. Memorias +1996 – La utopía arcaica: José María Arguedas y las ficciones del indigenismo +1997 – Cartas a un joven novelista +2001 – El lenguaje de la pasión +2004 – La tentación de lo imposible +2006 – Diccionario del Amante de América Latina (har biografiska inslag) +2007 – El Pregón de Sevilla +2009 – El Viaje a la Ficcion +2018 – La llamada de la tribu (Lockrop, översättning Elisabeth Helms 2019) +2022 – El fuego de la imaginación (Libros, escenarios, pantallas y museos. Obra periodística I) +1952 – La huida del inca (Inkans flykt) +1981 – La señorita de Tacna +1983 – Kathie y el hipopótamo +Mario Vargas Llosas essäer och journalistik har samlats under titeln Contra viento y marea och givits ut i tre volymer 1983, 1986 och 1990. +Den här artikeln är helt eller delvis baserad på material från engelskspråkiga Wikipedia, Mario Vargas Llosa, den 15 oktober 2010. +Alemany, Luis (28 januari 2011), ”En el infierno de Mario” (på spanska), El Mundo, http://www.elmundo.es/elmundo/2011/01/28/cultura/1296206852.html, läst 28 januari 2011 +Armas Marcelo, J. J. (2002) (på spanska), Vargas Llosa, el vicio de escribir, Madrid: Alfaguara, ISBN 84-204-4286-0 . +Boland, Roy; Harvey, Sally (1988) (på engelska), Mario Vargas Llosa: From Pantaleón y las visitadoras to Elogio de la madrastra, Auckland: Antipodas, the Journal of Hispanic Studies of the University of Auckland / VOX/AHS, ISBN 0-9597858-1-7 . +Booker, M. Keith (1994) (på engelska), Vargas Llosa Among the Postmodernists, Gainsville, FL: University Press of Florida, ISBN 0-8130-1248-1 . +Campos, Jorge; Jose Miguel Oviedo (1981) (på spanska), Vargas Llosa y su Guerra del fin del mundo, Madrid: Taurus Ediciones, ISBN 84-306-2131-8 +Castro-Klarén, Sara (1990) (på engelska), Understanding Mario Vargas Llosa, Columbia, SC: University of South Carolina Press, ISBN 0-87249-668-6 . +Cevallos, Francisco Javier (1991), ”García Márquez, Vargas Llosa, and Literary Criticism: Looking Back Prematurely” (på engelska), Latin American Research Review (Latin American Research Review, Vol. 26, No. 1) 26 (1): 266–275, http://www.jstor.org/stable/2503775, läst 6 april 2008 . (Prenumeration krävs för tillgång online.) +Coca, César (27 augusti 2006), ”30 años después de la ruptura con García Márquez, Vargas Llosa desvela las claves literarias y personales” (på spanska), Hoy, http://www.hoy.es/prensa/20060827/sociedad/anos-despues-ruptura-garcia_20060827.html, läst 16 april 2008 . +Cohen, Noam (29 mars 2007), ”García Márquez’s Shiner Ends Its 31 Years of Quietude” (på engelska), The New York Times, http://www.nytimes.com/2007/03/29/books/29marq.html?_r=1&em&ex=11&oref=slogin, läst 31 mars 2008 . +Gran Enciclopedia Universal, "Mario Vargas Llosa", "18", Madrid: El Mundo/Espasa Calpe, 2004, s. 11782–11785, ISBN 84-670-1345-1 (spanska) +Dammann, Guy (10 januari 2007), ”Let's hear it for literary feuds” (på engelska), The Guardian, http://www.guardian.co.uk/books/booksblog/2007/jan/10/letshearitforliteraryfeud, läst 7 oktober 2010 . +Fernández, Casto Manuel (1977) (på spanska), Aproximación formal a la novelística de Vargas Llosa, Madrid: Editora Nacional, ISBN 84-276-0383-5 . +Franco, Jean (2002) (på engelska), The Decline and Fall of the Lettered City: Latin America in the Cold War, Cambridge, MA: Harvard University Press, ISBN 0674008421 . +Gussow, Mel (28 mars 2002), ”Lacing his Fiction with History: Vargas Llosa Keeps a Latin American Literary Boom Booming” (på engelska), The New York Times 151 (52071), http://query.nytimes.com/gst/fullpage.html?res=9D03E4DD113BF93BA15750C0A9649C8B63, läst 27 mars 2008 . +Harrison, Kathryn (14 oktober 2007), ”Dangerous Obsession” (på engelska), The New York Times, http://www.nytimes.com/2007/10/14/books/review/Harrison.html, läst 14 april 2008 . +Heawood, Jonathan (24 november 2003), ”Past Master: Review of Mario Vargas Llosa, The Way to Paradise” (på engelska), New Statesman 132 (4665): 55, http://search.ebscohost.com/login.aspx?direct=true&db=a9h&AN=11449072&site=ehost-live, läst 7 april 2008 . (Prenumeration krävs för tillgång online.) +Igartua, Francisco (1998) (på spanska), Huellas de un destierro, Lima: Aguilar, ISBN 9972-00-266-7 . +Kirk, Robin (1997) (på engelska), The Monkey's Paw: New Chronicles from Peru, Amherst, MA: University of Massachusetts Press, ISBN 1558491090 . +Kokotovic, Micha (2007) (på engelska), The Colonial Divide in Peruvian Narrative: Social Conflict and Transculturation, Brighton: Sussex Academic Press, ISBN 1845191846 . +Kristal, Efraín (1998) (på engelska), Temptation of the Word: The Novels of Mario Vargas Llosa, Nashville, TN: Vanderbilt University Press, ISBN 0-8265-1301-8 . +Larsen, Neil (2000), ”Mario Vargas Llosa: The Realist as Neo-liberal” (på engelska), Journal of Latin American Cultural Studies 9 (2): 155–179, doi:10.1080/713679233, ISSN 1356-9325, http://search.ebscohost.com/login.aspx?direct=true&db=aph&AN=3954729&site=ehost-live . (Prenumeration krävs för tillgång online.) +Lamb, Ruth S (1971), ”El mundo mítico en la nueva novela latino americana” (på spanska) (PDF), Centro Virtual Cervantes (Claremont, CA: AIH Actas IV): 101–108, http://cvc.cervantes.es/obref/aih/pdf/04/aih_04_2_010.pdf, läst 7 april 2008 . +Martin, Gerald (1987), ”Mario Vargas Llosa: Errant Knight of the Liberal Imagination”, i John King (på engelska), Modern Latin American Fiction: A Survey, London: Faber and Faber, s. 205–233 . +Morote, Herbert (1998) (på spanska), Vargas Llosa, tal cual, Lima: Jaime Campodónico . +Mujica, Barbara (Mars/april 2004), ”Review of Mario Vargas Llosa, The Way to Paradise”, Américas 56 (2): 45, http://search.ebscohost.com/login.aspx?direct=true&db=a9h&AN=12400192&site=ehost-live, läst 8 april 2008 . +Muñoz, Braulio (2000) (på engelska), A Storyteller: Mario Vargas Llosa between Civilization and Barbarism, Lanham, ML: Rowman & Littlefield, ISBN 0-8476-9750-9 . +Navarro, Mireya (23 februari 2003), ”Spring Theater: Political Theater; At the Intersection Of Ruler and Ruled” (på engelska), The New York Times, http://query.nytimes.com/gst/fullpage.html?res=9801E1DC163DF930A15751C0A9659C8B63, läst 19 mars 2008 . +Parker, Emily (23 juni 2007), ”Storyteller: The Famous Novelist on Politics, and How Writing Can Change the Course of History” (på engelska), Wall Street Journal, http://opinionjournal.com/editorial/feature.html?issn=110010248, läst 6 mars 2008 . +Perales Contreras, Jaime (2007), ”Review on Historia de un Deicidio, en obras completas, vol. VI, Tomo I., in Americas Magazine, Organization of American States, July-August, 2007”, Americas, http://findarticles.com/p/articles/mi_go2043/is_4_59/ai_n29360449/ . +Reding, Andrew (1991) (på engelska), Mexico: The Crumbling of the "Perfect Dictatorship", World Policy Journal, våren 1991, 8(2):255-284, http://www.jstor.org/stable/40209208 +Setti, Ricardo A. (1981) (på spanska), Dialogos y Conferencias de Vargas Llosa, Madrid, Spain: Editora Inter Mundo., ISBN 84-86663-01-6 +Shaw, D.L.; Llosa, Mario Vargas (1973), ”Review of Vargas Llosa, García Márquez: historia de un deicidio” (på engelska), The Modern Language Review (Glasgow: The Modern Language Review, Vol. 68, No. 2) 68 (2): 430–431, doi:10.2307/3725901, http://www.jstor.org/stable/3725901, läst 6 april 2008 . (Prenumeration krävs för tillgång online.) +Vargas Llosa, Mario (1993) (på spanska), El pez en el agua: memorias (1. ed.), Barcelona: Seix Barral, ISBN 84-322-0679-2 +Vargas Llosa, Mario (2004), Paradiset finns om hörnet (Ny utg.), Stockholm: Pan, ISBN 91-7263-526-6 +Vargas Llosa, Mario (2006) (på spanska), Diccionario del amante de América Latina, Barcelona: Paidós, ISBN 84-493-1950-1 +Van Delden, Maarten; Grenier, Yvon (2009) (på engelska), Gunshots at the fiesta, Literature and Politics in Latin America (see chap on Vargas Llosa), Tennessee: Vanderbilt University Press / VOX/AHS, ISBN 9780826516336 . +Vincent, Elizabeth (27 augusti 2007), ”Interview with Mario Vargas Llosa” (på engelska), Macleans, arkiverad från ursprungsadressen den 6 februari 2008, https://web.archive.org/web/20080206001401/http://www.macleans.ca/culture/media/article.jsp?content=20070827_108283_108283, läst 22 mars 2008 . +Williams, Raymond L. (2001) (på spanska), Vargas Llosa: otra historia de un deicidio, Mexico: Taurus, ISBN 968-19-0814-7 . +The Guardian profile: Mario Vargas Llosa (läst 2010-10-30 +Zetterholm, Tore (1981) (på svenska), Levande litteratur: från Gilgamesj till Bob Dylan, Bra böcker . +Knut Ahnlund, Spansk öppning, Atlantis 2002 +Kjell A. Johansson, Latinamerikansk prosa under 1900-talet, Btj 1970 +Artur Lundkvist, Utflykter med utländska författare, Aldus 1969 +Jöran Mjöberg, Latinamerikanska författare, Norstedts 1988 +Bernt Olsson/Ingemar Algulin, Litteraturens historia i världen, Norstedts 1995 +Mauricio Rojas, Frihetens flämtande flamma: Mario Vargas Llosas politiska tänkande, Timbro 2010 +René Vázquez Díaz, Anteckningar i marginalen, Kentaur 2000 +Wikimedia Commons har media som rör Mario Vargas Llosa. +es +Mario Vargas Llosa hos Nobelstiftelsen (på engelska) +Mario Vargas Llosa i Libris +Länk till beståndet hos Svenska Akademiens Nobelbibliotek +En man som heter Ove är en svensk-norsk dramakomedifilm, i regi och skriven av Hannes Holm och som hade biopremiär i Sverige den 25 december 2015. Filmen bygger på Fredrik Backmans roman med samma namn från 2012. Rolf Lassgård spelar titelrollen som Ove, en vresig gammal man som upprätthåller strikt ordning i sitt radhusområde. När filmen börjar har Oves hustru Sonja (spelad av Ida Engvoll) dött av cancer och efter ett halvår planerar Ove att följa med henne i döden. Hans självmordsplaner blir dock saboterade när den höggravida Parvaneh (Bahar Pars) med sin familj blir Oves nya grannar. Deras inträde i området förändrar Oves liv för evigt. +Filmprojektet tillkännagavs den 17 maj 2014. Redan vid ett tidigt stadium bestämde sig Hannes Holm för att tilldela Rolf Lassgård rollen som Ove, trots att producenterna hade efterfrågat en mer "komisk" skådespelare. Vid manusförfattandet tog Holm inspiration dels från Carin Mannheimers berättelser, dels från sin egen barndom i ett radhusområde på Lidingö. Filmen inspelades mellan mars och maj 2015 i främst Trollhättan med omnejd liksom i andra delar av Västra Götaland och på Mallorca. Vid inspelningen använde sig Lassgård av en särskild mask för att få ett mer åldrat utseende. Filmen producerades av Tre Vänner i samproduktion med Fantefilm AS, Film i Väst, Nordisk Film, Sveriges Television och Nordsvensk Filmunderhållning. +En man som heter Ove blev en framgång både kommersiellt och hos kritikerna. Med sina 1,7 miljoner biobesökare är filmen den tredje mest sedda svenska filmen i Sverige genom tiderna. Filmen hyllades för sin tragikomiska berättelse och sitt skådespel, i synnerhet Rolf Lassgårds rollprestation. Vid Oscarsgalan 2017 var filmen nominerad till bästa icke-engelskspråkiga film och bästa smink, men vann ingen av kategorierna. Filmen tilldelades en European Film Award för bästa komedi och vid Guldbaggegalan 2016 fick den publikpriset och två guldbaggar för bästa mask och smink och bästa manliga huvudroll till Lassgård. +En amerikansk nyinspelning hade svensk premiär den 13 januari 2023 under titeln A Man Called Otto, med Marc Forster som regissör och med Tom Hanks i huvudrollen. +Ove Lindahl är en femtionio år gammal enstörig man som bor i ett svenskt radhusområde, där han upprätthåller strikt ordning till grannarnas stora förtret. Samtidigt lider han av depression och besöker flera gånger sin frus gravsten. Han hamnar i en självmordskris när han efter fyrtiotre års arbete på samma tågbolag tvingas gå i pension. När han försöker hänga sig själv i sitt hus blir han avbruten när en familj, bestående av den höggravida svenskiraniern Parvaneh, hennes svenske make Patrick och deras döttrar Sepideh och Nasanin, flyttar in i huset tvärs över gatan. Ove blir förbannad och försöker ignorera dem och andra grannar för att göra ett nytt självmordsförsök. +Under sina olika självmordsförsök blickar Ove tillbaka till sitt förflutna. Hans mamma dog när han var sju år gammal. Hans pappa arbetar som mekaniker på ett tågbolag och har ett stort intresse för bilmotorer, vilket han delar med sig till Ove. När Ove blir ung vuxen dör pappan i en olycka på bolagets bangård, varpå Ove övertar pappans jobb i bolaget. Kort senare får Ove besök av två kommunanställda, vilka Ove kallar för "Vitskjortorna", som förklarar för honom att hans hus kommer att rivas. En natt råkar hans grannars hus fatta eld, från vilken Ove räddar två personer. Men elden sprider sig till Oves hus och "Vitskjortorna" förhindrar brandmännen från att släcka den. Utan någonstans att ta vägen övernattar Ove i en tågkupé. När Ove vaknar upp träffar han i samma kupé den unga lärarstudenten Sonja. Efter tre veckor börjar de dejta och några år senare friar Ove till Sonja, som senare blir gravid. De flyttar in i radhusområdet och Ove blir vän med en granne vid namn Rune, med vilken han samarbetar för att skapa ordning och reda i grannskapet. Genom åren växer de ifrån varandra och kontakten bryts när Rune sparkar Ove från bostadsrättsföreningens styrelse. Senare blir Rune helt förlamad i sviterna av en hjärninfarkt och tas om hand av sin fru Anita. +Tillbaka i nutiden fortsätter Ove med sina självmordsförsök. En dag försöker Ove ta livet av sig genom kolmonoxidförgiftning, men han blir avbruten av Parvaneh. Hon frågar honom om hon kan få skjuts till sjukhuset och om han kan vara barnvakt till hennes båda döttrar, efter att Patrick råkat ramla från en stege. När Ove motvilligt fullgjort detta går han till en järnvägsstation och planerar att hoppa framför ett tåg. När en man på perrongen plötsligt svimmar och faller ner på spåret, hoppar Ove ner och räddar honom. På vägen hem tar Ove in en herrelös katt som han tidigare skrämt bort. En dag frågar Parvaneh om Ove kan lära henne att köra bil. Efter en tids reflektion hjälper Ove henne att ta sitt körkort. Ove reparerar en cykel som han först konfiskerat och senare lämnar tillbaka till tonåringen Adrian, en gammal elev till Sonja. Ove försöker senare begå självmord med ett hagelgevär, men han avbryts av att Adrian och hans kompis Mirsad ringer på hans dörr. De frågar om Mirsad kan få bo hos Ove eftersom Mirsad hade blivit utslängd från sitt hem efter att ha kommit ut som homosexuell inför sin familj. Då Adrian påpekar att Sonja alltid hjälpt andra, bjuder Ove in Mirsad i sitt hus. +Anita och Ove hamnar i en konflikt med en "Vitskjorta" som vill tvinga Rune att flytta in på ett äldreboende. Ove beslutar sig för att hjälpa Rune, men de lokala myndigheterna vägrar att lyssna på hans protester och Parvaneh klagar på hans arga temperament. Ove lugnar ner sig och berättar för henne om när han och den gravida Sonja åkte på en romantisk semester till Spanien. På hemresan råkar de ut för en allvarlig bussolycka. Båda överlever, men Sonja förlorar barnet och blir rullstolsburen. Efter att Sonja läst klart sin lärarutbildning nekas hon att få jobb eftersom skolorna inte är handikappanpassade. När de lokala myndigheterna ignorerar Oves förslag om att bygga en rullstolsramp till en skola tillbringar Ove en hel natt med att bygga den själv. Oves gärning gör att Sonja äntligen får arbeta som lärare och i skolundervisningen kämpade hon för barnen hon aldrig fick. Ove tröstas av Parvaneh när han avslöjar att Sonja dog i cancer för ett halvår sedan och att han lovade att följa henne i döden. +Med hjälp av en journalist lyckas Ove stoppa "Vitskjortan" från att omplacera Rune. På vägen hem kollapsar plötsligt Ove och på sjukhuset anger han Parvaneh som sin nära anhörig. Parvaneh får veta att hennes "pappa" lider av ett förstorat hjärta, men att han kommer återhämta sig. Skrattande säger hon till Ove att han är "sjukt dålig på att dö", innan hon själv går in i sin förlossning och föder en pojke. Ove ger gåvor till Parvaneh och hennes döttrar, vilka kallar honom för "morfar". Flera månader senare vaknar Parvaneh upp på morgonen av ett snöfall och ser att Ove inte har skottat utanför sitt hus. Parvaneh och Patrick springer till huset och upptäcker att Ove har dött i sömnen. I en lista över strikta instruktioner önskade Ove om att få en enkel ceremoni med bara de närmaste och med de som "tycker att jag har gjort rätt för mig". På begravningen uppvaktas Ove av alla sina grannar. Ove vaknar sedan upp i ett tåg och får äntligen återförenas med Sonja. +Följande skådespelare medverkar i filmen: +Krönikören Fredrik Backman gjorde sin författardebut med romanen En man som heter Ove (2012). Romanen blev en internationell bästsäljare och hade 2017 översatts till 44 språk. Den hamnade på första plats på "Best Seller List" hos den amerikanska tidningen The New York Times och låg kvar på listan i 82 veckor i följd. År 2015 hade romanen sålts i 800 000 exemplar och 2017 uppgick antalet sålda exemplar till över sju miljoner. En man som heter Ove sattes senare upp som teaterpjäs på flera scener. Den första pjäsen hade premiär på Rival i Stockholm den 30 januari 2015, i regi av Emma Bucht, skriven av Marie Persson Hedenius och med Johan Rheborg i huvudrollen. Pjäsen blev mycket framgångsrik och har även satts upp i Norge och Finland. +Den 19 februari 2013 förvärvade produktionsbolaget Tre Vänner filmrättigheterna till En man som heter Ove. Filmprojektet sattes igång när Tre Vänners producent Annica Bellander Rune tog kontakt med regissören Hannes Holm och frågade honom ifall han kunde tänka sig att regissera filmen. Holm avvisade först projektet med förklaringen att Backmans roman enligt honom lät som en komedibok och att han inte var intresserad av att göra fler komedifilmer. Holm ändrade uppfattning när han hade läst igenom romanen, efter att ha fått ett exemplar av Bellander. Han blev djupt berörd av romanens berättelse och beskrev att den "innehåller såväl komiska som sorgliga inslag". Holm hade efteråt ett möte med Fredrik Backman. Då Backman var oerfaren inom filmbranschen kom båda parterna överens om att Holm skulle skriva filmens manus. +Vid en presskonferens under filmfestivalen i Cannes den 17 maj 2014 tillkännagav produktionsbolaget Film i Väst att Hannes Holm skulle bli regissör och manusförfattare till filmadaptionen av Fredrik Backmans roman En man som heter Ove. Filmen planerades att spelas in i Trollhättan och Spanien under hösten 2014 och våren 2015 och filmens premiär sattes till juldagen 2015. Den kom att bli Holms första långfilm baserad på en roman. Enligt producenten Bellander uppgick filmens budget till 40 miljoner svenska kronor. Enligt Holm uppgick filmens produktionsbudget till 350 000 amerikanska dollar (motsvarande 2 775 500 kronor). Filmen producerades av Tre Vänner i samproduktion med Fantefilm AS, Film i Väst, Nordisk Film, Sveriges Television och Nordsvensk Filmunderhållning. Filmen tilldelades 6 miljoner svenska kronor i automatstöd av Svenska Filminstitutet, 2 miljoner norska kronor i produktionsstöd av Norska filminstitutet och 2 miljoner norska kronor i produktionsstöd av Nordisk Film- & TV Fond. Den norska andelen av filmens budget uppgick till 11,5 procent, motsvarande 4 034 376 norska kronor. De norska produktionsbolagen försåg filmteamet med ljus- och kamerautrustning och ansvarade under produktionen för laboratoriearbetet, ljudinspelningen, ljudredigeringen, ljudmixningen och musikkompositionen. +Holm färdigställde filmens manus efter två månader och medgav att manusets första version "var mer eller mindre den film vi spelade in." Tidigt under manusförfattandet förstod Holm att han ville se Rolf Lassgård i huvudrollen som Ove. Så fort Holm hade färdigställt manuset presenterade han filmprojektet för Lassgård i samband med ett lunchmöte. Lassgård hade själv inte läst Backmans bok och var inledningsvis tveksam till att spela rollen som en komisk karaktär. Han ändrade uppfattning så småningom efter att ha läst igenom filmens manus och då flera personer i hans närhet, inklusive hans egen fru och dotter, som läst Backmans roman, uppmanade honom att acceptera rollen. +Bortsett från bokens innehåll tog Holm mycket inspiration från Carin Mannheimer, vars pjäser och tv-serie Svenska hjärtan (1987–1998) skildrar äldre människor som bor i idylliska radhusområden. Han tog även inspiration från sin egen barndomstid på Lidingö under 1960-talet, när han med sina föräldrar bodde först i en lägenhet i Näset och senare i ett radhus i Rudboda. När han var liten brukade han titta på sina föräldrars gamla fotoalbum om de romantiska utflykter de gjorde innan han var född. Han beskrev även sin pappa som väldigt lik karaktären Ove, främst genom att vara "väldigt ordningsam. Han är trubbig och hård på utsidan men kärleksfull och varm på insidan. Och han har keps och kör en Saab...". +Under produktionen hamnade Hannes Holm i en konflikt med styrelsen hos filmproduktionsbolaget SF Studios gällande filmens budget och innehåll. Bolaget är sedan 2013 ägare till Tre Vänner. Under produktionens gång fick Holm veta att SF Studios hade tagit en stor del av filmens budget, omkring 30 procent, för sig själva innan starten av filminspelningen, medan det normala för ett filmbolag, enligt Holm, är att ta ut mellan 10 eller 20 procent. Även Film i Västs verkställande direktör Tomas Eskilsson kritiserade bolaget för att det "tar de högsta marginalerna i branschen". SF:s verkställande direktör Jonas Fors, tidigare VD för Tre Vänner, förnekade att SF Studios tagit 30 procent av filmens budget och angav att bolagets marginal "som regel [låg] kring 10 procent". Holm avfärdade Fors svar som "trams" och påstod att han själv hade "sett dokument där de [SF Studios] tycker att 19,7 procent är alldeles för liten marginal och behöver åtgärdas." +Vidare kritiserade Holm bolaget för att det "drivs av ett pengaintresse och att kreativitet inte alls är lika viktigt. Dessutom präglas bolaget av en sexistisk grabbkultur." Produktionsbolaget ville exempelvis av budgetskäl ta bort de scener som utspelades i Spanien och en katt som hade en viktig roll i Backmans roman. Holm lämnade filmprojektet i protest, men kunde återvända efter att han fått sina krav tillgodosedda av bolaget. Efter produktionens slut skrev Holm ett brev till SF:s styrelse, där han beklagade att han "aldrig varit med om maken till ett så dåligt skött bolag". Holms kritik mot SF Studios hyllades av olika profiler inom den svenska filmbranschen, såsom Peter Birro, Ellen Tejle, Jonas Holmberg och Jan Gradvall. I samband med Sveriges Televisions program Kulturnyheternas granskning av SF Studios fick Jonas Fors sparken av Bonniers den 10 juni 2016. +Den 6 november 2014 tillkännagavs att Rolf Lassgård skulle spela huvudrollen som Ove Lindahl. Producenterna var inledningsvis tveksamma till att tilldela Lassgård rollen som Ove, då Lassgård huvudsakligen hade medverkat i dramaproduktioner och inte var en komediskådespelare. Holm argumenterade dock att han rollsatte honom för hans skådespelartalang, och att han inte sökte efter komiska skådespelare. Lassgård beskrev rollen som en utmaning för honom och att Ove "är ju lika gammal som jag är i boken men han känns som en äldre man från en annan tid. Jag har aldrig spelat morfar förut, det ska bli jävligt spännande att hitta den här gubben." Det svåraste med att spela rollen som Ove var att "hitta växlingarna mellan komiken och det dramatiska, förutom att rent fysiskt gestalta den. Det har varit en drömroll att jobba med." +När Lassgård efter genomläsningen av filmmanuset även läst igenom Backmans bok hade han ett lunchmöte med författaren, som han efteråt erkände "låste upp en nyckel till karaktären (Ove)." Lassgård fick dels veta av Backman om hur Oves "stora mörker" blev till, dels att Ove "kunde lära sig några nya onyttigheter, men han ville inte lära sig och säger nej." Vidare var Lassgård fascinerad av Oves tragiska levnadshistoria "som gör att han agerar som han gör, den här otroliga saknaden efter den konkreta kärleken till sin hustru och det som hänt henne, plus det som han upplever av samhällets framåtskridande och hur han försöker stå upp mot det med sin envishet." Han erkände att han personligen är ganska olik Ove: "vi är jämngamla, men i praktiken är han nog tio år äldre än jag. Han är sprungen ur det gamla folkhemmet men missade poprevolutionen och ego-boomen. Han är nog mer som 59-åringarna var i mina föräldrars generation". Lassgård erinrade att Oves utseende och beteende påminde en del om hans egen pappa när han var vid samma ålder som karaktären. Han beskrev samtidigt Ove som en "kinesisk ask. Man tror att saker och ting är på ett visst sätt, men så lyfter man på locket och upptäcker att det inte alls var så. Då ligger det en ny ask där inne och lyfter man där får man en ny aha-upplevelse. Det är drömrollen för en skådespelare." +Både Holm och Lassgård inspirerades till Oves gestaltning av Alexander Paynes film About Schmidt (2002), vars huvudroll Warren Schmidt spelad av Jack Nicholson var en "nypensionerad känslostum försäkringstjänsteman" med en avliden fru. Lassgård influerades även av filmen Still Life (2013) av Uberto Pasolini, vars huvudroll John May spelad av Eddie Marsan är en "solitär kommunal byråkrat" som ordnar begravningar till personer som saknar anhöriga. Lassgård förklarade att "de här ensamma männen gav tankar för Ove, även om vår film drar mer mot komedin. Samtidigt har vi dödat många älsklings-skratt för historiens skull. Vi har velat skapa en liten tragikomisk berg- och dalbana". +Den 4 februari 2015 tillkännagavs att Filip Berg tilldelades rollen som den yngre Ove. Innan inspelningen hade Berg en gemensam diskussion med Lassgård och Holm gällande karaktärens tankebanor och sätt att agera i olika situationer. Både Lassgård och Berg skulle göra sina egna tolkningar av karaktären. För att förbereda sin gestaltning hade Berg observerat Lassgårds inspelningar och studerat hans rörelsemönster och sätt att prata. Samtidigt läste Berg hälften av Backmans bok inför sina egna scener. Ida Engvoll tilldelades rollen som Oves fru Sonja. Hon hade läst boken inför sin provspelning och "kände redan då mycket för Sonja, hennes historia är ju ännu mer utmejslad i boken." +Bahar Pars reflekterade att hon inledningsvis var tveksam att acceptera rollen som Parvaneh, då hon i filmen skulle "representera en hel folkgrupp och typ alla invandrarkvinnor. För mig blev det inte bara att bära en roll, och det kan ju falla så otroligt platt. Det kan bli väldigt stereotypt och klyschigt om man gör det fel". Efter att hon läst igenom Backmans bok ville hon gestalta Parvaneh mindre stereotypt än i romanen. Hon beskrev bokens Parvaneh som en "väldigt simpel" sidokaraktär till Ove och "en färgklick som ­bjuder grannen Ove på exotisk mat." Innan hon tackade ja till rollen hade hon flera samtal med Holm angående hans planer för Parvanehs karaktär, varpå Holm gav Pars spelrum att porträttera "en hel människa och inte bara en schablon". Under inspelningen hade Pars flera diskussioner med filmteamet angående Parvanehs hus och karaktärens tal, kroppsspråk och utseende, där allt som Pars ansåg var "för kliché" skulle ändras eller tas bort. I efterhand var Pars glad att hon antagit rollen och berättade att hon efter filmens premiär fått flera beundrarbrev, "alla tycker att jag är världens finaste, Sverigedemokrater och allt skriver till mig." Tobias Almborg tilldelades rollen som Parvanehs make Patrick i juni 2014, vilken blev hans långfilmsdebut. +Maskörparet Love Larson och Eva von Bahr tillbringade flera månader med att skapa masker åt Rolf Lassgård. Enligt Holm, Larson och Bahr var Lassgård inte "tillräckligt mycket gubbe" och hade "för mycket hår" för att kunna gestalta Ove. Larson och Bahr inledde arbetet med att printa ut bilder på män som de tyckte såg ut som "Ovar" och de upptäckte att majoriteten av dessa hade håravfall. De skapade sedan en skiss på Lassgårds huvud i Photoshop där de tog bort hans huvudhår. Resultatet blev en mask i form av en silikonhjässa med ögonbryn, leverfläckar, blemmor och en lös huvudtopp med löshår. Masken var av samma typ som Johnny Depp använt när han gestaltade karaktären James "Whitey" Bulger i filmen Black Mass (2015). Masken tog två och en halv dag att tillverka, och då huvudtoppen bara höll för en inspelningsdag fick Love och Bahr skapa 27 kopior, och sedan slänga de gamla efter varje inspelningsdag. Varje mask tog två timmar att sätta på och en timme att ta av. Lassgård tilldelades även bruna kontaktlinser och specialtillverkade födelsemärken för att efterlikna Filip Berg. Berg hade först testat att använda blåa kontaktlinser för att hans ögon skulle efterlikna Lassgårds. Den lösningen valdes dock bort eftersom linserna fick Berg att se ut som en "vampyr". +Filmteamet letade runtom i landet efter ett lämpligt radhusområde som inspelningsplats. Enligt Holm hade teamet under sin spaning "nästan överallt" råkat påträffa "en Ove som stoppskylt och skrek "här får ni fan inte köra!"". Valet föll slutligen på ett radhusområde vid Bergkullevägen i stadsdelen Sylte i Trollhättan, där filmteamet hyrde tre radhus som skulle gestalta Oves, Parvanehs och Runes respektive hem. Stora delar av filmen spelades in i och utanför kedjehuset på Bergkullevägen 160, som användes som Oves hus. Filmteamet fick tillåtelse av husägaren att bygga om delar av husets interiör. Detta underlättade kameraarbetet och möjliggjorde för filmteamet att möblera om huset till vissa scener som utspelades under dåtid. Köket till huset byggdes om, en vägg revs, resten av väggarna målades om och samtliga möbler byttes ut. Radhusområdets gatstump döptes efter inspelningen om till "Oves gata". +Under produktionen bidrog Saab Car Museum i Trollhättan genom att låna ut några av sina Saabmodeller till filminspelningen. +Filminspelningen skedde under 39 dagar mellan 20 mars och 22 maj 2015. Filmfotograf var Göran Hallberg, som har fotograferat flera av Holms tidigare filmer. Filmen spelades in med digitalkameran ARRI Alexa SXT Plus, med Cooke Anamorphic/i som objektiv med en fast brännvidd och Angenieux Optimo 56-152 som zoomobjektiv. Stora delar av filmen inspelades med Anamorphic-objektiv, i synnerhet med brännvidden 40- och 60 mm, i syfte för Hallberg att komma nära skådespelarna. Till vissa scener använde Hallberg två kameror med Anamorphic- respektive Optimo-objektiv. Åkningen gjordes genom Steadicam med Joel Olsson som operatör. +För filmens ljussättning samarbetade Hallberg med filmelektrikern Kent Högberg. Filmens färgsättning syftades till att förstärka Oves karaktär och känslospel. Både interiör- och exteriörscener inspelades för att återspegla berättelsens sinnesstämning. Stora delar av filmen inspelades med mestadels molnigt väder och "blågrå" ljussättning, medan en av slutscenerna inspelades under "vår-sommar". Hallberg beskrev att när Ove går in i andra hus än hans eget "blir allting varmare och mer öppet". Under själva kameraarbetet skulle inledningen av filmen innehålla direktbilder och statiska bilder, medan när Ove senare interagerar med Parvaneh "börjar vi förflytta kameran" med jämna rörelser. Som belysningsutrustning använde Högberg PAR-lampor, som enligt Hallberg gav "fina reflektioner", Lite Panel-strålkastare, Kino Flo Celeb LED-strålkastare samt 4K och 6K HMI-strålkastare. +Inspelningen skedde under två sejourer. Lassgårds scener inspelades först medan flashback-scenerna med Filip Berg och Viktor Baagøe inspelades senare. Huvuddelen av filmen inspelades i Trollhättan och dess omnejd från den 20 mars fram till den 13 maj 2015. Lassgårds mask gjorde honom oigenkännlig såväl utanför inspelningen som bland vissa medarbetare i filmteamet under den första inspelningsdagen. Lassgård berättade att hans förändrade utseende gjorde att filmteamet behandlade honom intuitivt mer omhändertagande under inspelningens gång. För scenerna där Ove skulle hänga sig själv i sitt hem hängde Lassgård med hjälp av linor. Vid närbilder ställdes en liten låda som Lassgård kunde stå på, men vid helbilder fick han hänga utan skydd. För den herrelösa katten som följer med Ove använde sig filmteamet av två identiska katter, där den ena beskrevs som "mer aggresiv" medan den andra var mer lugn och som "inte har något emot att bli buren". +Oves och hans pappas hus inspelades i Höljebacka brandmuseum, som ligger 2 mil söder om Trollhättan. Bergslagernas Järnvägssällskap i Göteborg lånade ut sin bangård och tågsamling till filmteamet. Dess styrelseordförande Lars-Erik Karlsson var delaktig i inspelningen genom att med ett tåg köra över en docka som gestaltade Oves pappa. Övriga inspelningar skedde i Vänersborg, Uddevalla och Stockholm. +Scenerna i Spanien inspelades på Mallorca mellan 20 och 22 maj 2015. Inspelningsplatser var hotel Araxa i Palma de Mallorca, Cap de Formentor utanför Port de Pollença och Sa Calobra i Escorca kommun. För scenen med bussolyckan dekorerade filmteamet två identiska bussar, där den ena delades i tre delar som sedan svetsades ihop för att skapa illusionen av en autentisk olycka. Då Filip Bergs Ove sitter i busstoaletten under olyckans gång byggde filmteamet en separat toalett. För att simulera olyckan lyftes toaletten upp och roterades av en lastbil. +Filmens musik komponerades av den norske jazzmusikern och kompositören Gaute Storaas. +Följande licenserade låtar finns med i filmen. +En man som heter Ove distribuerades i Sverige, Norge och Danmark av Nordisk Film, som satte filmens världspremiär på juldagen 2015 i 265 biografer i Sverige och 125 i Norge. Enligt Pia Grünler på Nordisk Film var förväntningarna inför premiären höga, dels på grund av att den var baserad på en bästsäljande roman och dels därför att den populäre skådespelaren Rolf Lassgård spelade huvudrollen. Galapremiären skedde i biografen Rigoletto i Stockholm den 14 december 2015. Filmen sändes senare på SVT1 den 26 december 2017, 4 januari 2018 och 1–2 september 2018, liksom på SVT24 den 6 september och 6 december 2018. Den 26 december 2016 visades filmen på Internationella komedifilmfestivalen i Filmhuset i Stockholm för afghanska ensamkommande flyktingbarn. Filmen var textad på dari och visningen följdes av samtal med Bahar Pars som särskild gäst. +Redan den 3 november 2014 förvärvade Concorde Filmverleih och September Film Distribution filmens distributionsrättigheter i Tyskland respektive Benelux. Filmen hade biopremiär i Finland den 8 januari 2016 och i Danmark den 26 maj samma år. Den 1 mars 2016 förvärvade Music Box Films filmens distributionsrättigheter i USA och Kanada och satte nordamerikansk biopremiär den 30 september samma år i 40 amerikanska städer, inklusive under Minneapolis-Saint Paul International Film Festival. Filmen visades senare på filmfestivaler i Seattle, Scottsdale, Traverse City, Stonybrook, Madison, Wisconsin, Saint Louis, Houston och Orlando, Florida. Under filmfestivalen i Berlin såldes filmen till Japan, Frankrike, Spanien, Grekland, Taiwan och Litauen. Efteråt har filmen sålts till uppemot 50 länder. +En man som heter Ove blev en stor svensk kassasuccé. Under premiärdagen hade filmen lockat 76 000 svenska biobesökare och puttade ner filmen Star Wars: The Force Awakens (2015) från sin förstaplats på biotoppen under julhelgen. I jämförelse hade Star Wars: The Force Awakens 44 000 svenska biobesökare under sin premiärdag den 18 december. Under de tre första dagarna på svenska biografer hade En man som heter Ove lockat 241 726 biobesökare, en siffra som överträffade den tidigare rekordhållaren Hundraåringen som klev ut genom fönstret och försvann (2013). Filmen blev Nordisk Films mest framgångsrika filmlansering någonsin. +Enligt statistikrapporter från Svenska Filminstitutet lockade filmen mellan 2015 och 2016 sammanlagt 1 714 994 biobesökare i Sverige och intjänat totalt 175 008 799 svenska kronor. Den 13 mars 2016 blev En man som heter Ove med 1 656 215 biobesökare den tredje mest sedda svenska filmen i Sverige genom tiderna, efter Sällskapsresan (1980) och Att angöra en brygga (1965). Filmen lockade totalt 423 731 biobesökare under sin premiärmånad i december 2015 och drog in 43 358 387 kronor. Den blev 2015 års näst mest sedda svenska film, efter En underbar jävla jul (2015), och den tionde mest sedda filmen i Sverige samma år. Under 2016 lockade filmen 1 285 966 biobesökare och drog in 131 650 412 kronor. Den blev 2016 års mest sedda film i Sverige, följd av filmen Zootropolis (2016) som lockade 608 225 biobesökare samma år, liksom 2016 års mest köpta svenska film. Den blev både 2017 och 2018 års näst mest sedda svenska film på TV med 1 914 000 respektive 1 223 000 tittare. +Mellan 2015 och 2016 lockade filmen enligt Svenska Filminstitutet totalt 1 349 657 biobesökare från 13 länder utanför Sverige. Enligt databasen Lumiere som hålls av European Audiovisual Observatory, Europarådets organ för den audiovisuella sektorn i rådets länder, hade filmen mellan 2015 och 2021 lockat sammanlagt 1 248 753 biobesökare från 25 europeiska länder exklusive Sverige. Dessa inkluderar 504 743 biobesökare från Tyskland, 208 829 från Danmark, 138 962 från Norge, 74 305 från Nederländerna, 50 808 från Finland, 45 477 från Italien, 34 358 från Spanien, 27 439 från Storbritannien, 26 966 från Polen, 25 772 från Österrike, 19 914 från Ungern, 19 021 från Schweiz, 13 672 från Ryssland, 13 054 från Frankrike, 9 000 från Tjeckien, 8 496 från Island, 7 108 från Grekland, 4 791 från Slovakien, 4 680 från Belgien, 3 306 från Portugal, 1 948 från Bulgarien, 1 727 från Estland, 1 722 från Kroatien, 1 410 från Litauen och 1 245 från Luxemburg. +Enligt siffror från webbplatsen Box Office Mojo hade filmen mellan 2015 och 2017 dragit in sammanlagt 30 692 889 dollar från 24 länder, som inbegriper 15 länder i Europa, 5 i Amerika, 2 i Asien och 2 i Oceanien. 20 224 613 dollar av denna summa intjänades i Sverige, varav 2 924 071 dollar var under öppningshelgen från 309 biografer. Totalt 10 468 276 dollar intjänades utanför Sverige, varav 9 657 229 dollar (82 183 019 kronor) indrogs under 2016 och 809 854 dollar (7 410 164 kronor) indrogs under 2017. I USA och Kanada indrog filmen 3 479 315 dollar, med 368 405 dollar som sin högsta summa från 167 biografer mellan 4 och 6 november 2016. Fyra veckor efter filmens amerikanska premiär hade den intjänat omkring 800 000 dollar. Mellan perioden 28 oktober och 24 november 2016 insamlades 1 976 319 dollar från som flest 194 biografer, och fram till 10 december 2016 hade filmen spelat in motsvarande 30 miljoner svenska kronor. Bland 200 av de högst inkomstbringande filmerna lanserade i USA och Kanada år 2016 hamnade En man som heter Ove på 175:e plats. Den hamnar på 26:e plats bland 200 filmer som haft premiär i färre än 600 biografer. +I Tyskland indrog filmen 2 383 722 dollar från som flest 171 biografer. I Norge indrog filmen 1 604 790 dollar, varav 354 818 dollar från 128 biografer under öppningshelgen. Från övriga territorier i Europa intjänade filmen 518 971 dollar från Nederländerna, 464 636 dollar från Finland, 192 261 dollar från Spanien, 177 224 dollar från Schweiz, 115 741 dollar från Österrike, 80 834 dollar från Island, 48 612 dollar från Ryssland, 44 795 dollar från Storbritannien, 10 899 dollar från Portugal, 10 241 dollar från Slovakien, 3 027 dollar från Litauen och 1 193 dollar från Bulgarien. Från Asien och Oceanien intjänade filmen 648 662 dollar från Sydkorea, 378 378 dollar från Australien, 134 722 dollar från Nya Zeeland och 7 318 dollar från Förenade Arabemiraten. Från Latinamerika intjänade filmen 77 598 dollar från Mexiko, 63 130 dollar från Colombia och 22 207 dollar från Argentina. +Recensionerna för filmen var mestadels positiva. På webbsidan Rotten Tomatoes har filmen betyget 91 procent baserat på 119 recensioner (på deras så kallade "Tomatometer") med ett genomsnittligt betyg på 7.8 av 10. Webbplatsens kritiska konsensus lyder följande: "En man som heter Oves älskvärda uppriktighet – liksom Rolf Lassgårds flagranta och stenhårda prestation i titelrollen – förhindrar den från att ge efter för överdriven sentimentalitet." På Metacritic har filmen ett genomsnittsbetyg på 70 av 100, baserat på 21 recensioner. På Kritiker.se har filmen genomsnittsbetyget 3,5 av 5, baserat på 19 recensioner. +Aftonbladet, Expressen och Göteborgs-Posten gav alla filmen betyget 4 av 5. Jens Peterson från Aftonbladet hyllade filmens skådespel och dess berättelse som "rörande och känslo­stark". Maria Brander från Expressen hyllade Rolf Lassgårds och Bahar Pars rollprestationer; om den senare ansåg Brandar att Parvanehs skildring i filmen "fungerar [...] snäppet bättre än i boken". Mats Johnson från Göteborgs-Posten hyllade dess tragikomiska berättelse, den "träffsäkra dialogen" och Lassgårds skådespel, som han kallade för en "perfekt balans mellan komik och tragik". Samtidigt kritiserar Johnson filmen för att vara 30 minuter för lång och att Holm hade implementerat för många element från Backmans roman till filmen som han "ändå inte hinner utveckla". Nils Karlén från Moviezine gav filmen betyget 3 av 5. Karlén hyllade Lassgårds skådespel, men kritiserade filmens ojämna humor och att den var för lik filmen St. Vincent (2014). Fredrik Sahlin från Kulturnyheterna gav filmen betyget 3 av 5. Sahlin skrev att filmen "lutar sig lite för hårt mot lätt förutsägbara vändningar för att övertyga stort, men så är Holm inte heller ute efter att överraska. Målet är sannolikt en lättsmält men varm berättelse om saknad, alienation och människans behov av sammanhang och mening. Och det är precis det som Hannes Holm och hans nissar levererar." Clemens Poellinger från Svenska Dagbladet gav filmen betyget 3 av 6. Han hyllade dels dess "kompetent[a] manus- och filmarbete", dels dess flashback-scener som skapar "en skimrande bild av det gamla goda folkhemssverige, ett minnenas rike." +Krister Uggeldahl från finländska Hufvudstadsbladet gav filmen betyget 4 av 5. Uggeldahl hyllade Lassgårds rollprestation som hans "bästa arbete" och han beskrev att filmens komiska delar "skakar tass med det djupt tragiska, detta utan att regissören och manusförfattaren Hannes Holm [...] tappar fotfästet." Odie Henderson från RogerEbert.com gav filmen betyget 3,5 av 4. Han beskriver filmen som en "en sjukligt rolig och gripande framgång" och Rolf Lassgårds "rika och komplexa rollprestation som [...] både rolig och rörande". David Hughes från brittiska Empire gav filmen betyget 4 av 5. Han kritiserade filmen för att den åsidosatte karaktären Sonja i flashback-scenerna, men att filmen räddas tack vare karaktären Parvaneh som "ger Ove samvete och sällskap som han har saknat efter sin hustrus död." Shery Linden från The Hollywood Reporter hyllade filmens skådespel, men hon kritiserade filmens "melodrama[tiska] bakgrundshistoria", dess "tvådimensionella romantik" och att den innehåller en "subplot som involverar en karikatyrartad skurk, ytterligare en "Vitskjorta", som är för överdriven för att kunna uppfylla sitt avsedda syfte." I en mindre entusiastisk recension av Glenn Kenny från The New York Times skrev Kenny att filmen innehåller "förvisso godhjärtade saker, men som främst skulle intressera beundrare av filmisk tröstmat." +Dokumentärfilmaren Bengt Löfgren kritiserade filmen för att den var för lik Clint Eastwoods film Gran Torino (2008). Han utvecklade att En man som heter Ove "kunde visat lite mer fantasi och satt en egen tydlig signatur [...] Gran Torino har inte parallellhistorien där man tittar tillbaka till barndomen, men annars kom jag på mig själv med att kunna räkna ut vad nästa scen skulle handla om för att det var så likt Gran Torino. Hannes Holm besvarade Löfgrens kritik med att En man som heter Ove till visso har likheter och vars berättelse återfinns i flera andra filmer och böcker, men att till skillnad från Gran Torino innehåller den ett flertal flashback-scener som är "en av den filmens huvudbeståndsdelar". Aftonbladets filmkritiker Jan-Olov Andersson och Jens Peterson var båda skeptiska till Löfgrens kritik. Peterson påpekade att "huvudpersonerna är äldre män vars fruar nyss dött. Men skillnaderna är större. Den svenska är en kärlekshistoria som visar vad som formade Ove. Den amerikanska handlar om kriminella gäng och en äldre man som försöker dela med sig av erfarenheter till en tonåring. En stor skillnad är att många känner igen sig själva eller andra i Ove, både i Sverige och internationellt. Det har jag aldrig hört någon säga om Walt Kowalski som Clint Eastwood spelar." +I en gästkrönika på Dagens Samhälle skrev författaren Åsa Moberg att "Vitskjortan" som karaktärerna Ove och Anita har en konflikt med i filmen var ett "nidporträtt" som "representera[r] ondskan av en privat vårdföretagare." Moberg var ifrågasättande till att "en tjänsteman från ett riskkapitalägt vårdföretag" skulle kidnappa karaktären Rune. Hon argumenterade att det vanligtvis är mycket svårt för pensionärer att få plats på ett äldreboende och att "ingen kommun utreder äldre för att mot deras egen och närståendes vilja röva bort dem hemifrån". Hon sammanfattade sin krönika med att skriva: "Är det viktigt om en osannolik skurk är vinstdriven eller paragrafdriven i en roman och en film? Ja, när miljoner människor har tagit historien till sina hjärtan. I synnerhet inför ett valår där vänstern strävar efter att göra detta till en huvudfråga." +En man som heter Ove tilldelades ett flertal utmärkelser och nomineringar från filmfestivaler och filmgalor världen över. Vid Guldbaggegalan den 18 januari 2016 tilldelades filmen pris för bästa manliga huvudroll till Rolf Lassgård, bästa mask och smink till Eva von Bahr och Love Larson samt publikpriset. Den var även nominerad i kategorierna bästa film, bästa kvinnliga biroll till Bahar Pars och bästa visuella effekter till Torbjörn Olsson. Vid European Film Awards i Wrocław i Polen den 10 december 2016 tilldelades filmen pris för bästa europeiska komedi och var nominerad inom kategorierna bästa manliga huvudroll till Lassgård och publikpriset. +Den 1 september 2016 utvaldes filmen av den svenska Oscarskommittén som Sveriges officiella bidrag till Oscarsgalan den 26 februari 2017. Den 16 december hamnade filmen på den sista gallringen ihop med 8 andra filmer. När nomineringarna presenterades den 24 januari 2017 tillkännagavs att filmen blivit nominerad inom kategorierna bästa icke-engelskspråkiga film och bästa smink till Bahr och Larson. Vid galan tilldelades filmen inget pris. Både Rolf Lassgård och Bahar Pars riskerade att missa galan, dels för att Lassgård inte tilldelades någon galabiljett, dels då Pars på grund av sitt iranska medborgarskap nekades att flyga till USA som en konsekvens av president Donald Trumps inreseförbud. Bådas problem löstes genom att Lassgård tilldelades en extrabiljett av Bahr och Larson och att inresereglerna ändrades så att alla med svenskt pass, även de med dubbla medborgarskap, fick komma in i USA. +Den 18 januari 2022 tillkännagavs att det skulle produceras en engelskspråkig nyinspelning av En man som heter Ove, med Marc Forster som regissör och med Tom Hanks i huvudrollen. I kölvattnet av filmens Oscarsnomineringar inleddes förhandlingar kring en nyinspelning mellan producenten Fredrik Wikström Nicastro och olika filmstudior i Hollywood. Nyinspelningens förproduktion inleddes i september 2017, med SF Studios som produktionsbolag och Nicastro som producent i samproduktion med Tom Hanks eget produktionsbolag Playtone med Hanks, Gary Goetzman och Rita Wilson som producenter. I februari 2022 köpte Sony Pictures upp filmens globala distributionsrättigheter för 60 miljoner dollar. Filmen fick titeln A Man Called Otto och inspelningarna inleddes i Pittsburgh under februari 2022. Den 25 december 2022 hade filmen biopremiär. +Officiell trailer på Youtube +En man som heter Ove på Allmovie (engelska) +En man som heter Ove på Internet Movie Database (engelska) +En man som heter Ove på Svensk Filmdatabas +Psilanderaffären var en incident som inträffade i Skálafjørður (danska: Skaalefjord) nära Tórshavn på Färöarna den 20 juni 1940 då de fyra svenska jagarna HMS Psilander, Puke, Romulus och Remus togs i beslag av brittiska flottan. Tillslaget ska ha skett av oro för att tyska flottan annars skulle beslagta jagarna och ta dem i tysk tjänst när de lämnat Nordsjön. +Att svenska jagare befann sig vid Färöarna, vid en för Storbritannien kritisk tidpunkt i juni 1940, berodde på det akuta behov av krigsmateriel som uppstod på svensk sida vid andra världskrigets utbrott i september 1939. Italien var ett av få länder som kunde sälja lämplig materiel och ett snabbt inköp av fyra jagare genomfördes. Fartygen testades i La Spezia under mars och april 1940. Efter en lång och dramatisk hemresa, fördröjd bland annat av hastigt uppkomna reparationsbehov och svårigheten att skaffa drivmedel, hamnade jagarna tillsammans med det medföljande passagerarfartyget Patricia och tankern Castor vid Färöarna, sedan striderna i Västeuropa tvingat fartygen att undvika området runt Engelska kanalen. +En dryg vecka senare återfick Sverige de beslagtagna jagarna efter intensiv diplomatisk aktivitet, och fartygen kunde slutligen segla till Göteborg. Eftersom jagarstyrkans befälhavare kommendörkapten Torsten Hagman hade överlämnat rustade svenska örlogsfartyg till främmande makt utan strid, ifrågasattes hans handlande och förberedande förhör hölls vid fältkrigsrätten vid Västkustens marinkommando. Då ärendet inte gick vidare till rättegång blev Hagman heller aldrig offentligt friad i domstol för sitt beslut. +När andra världskriget bröt ut uppstod ett akut behov av att förstärka den svenska krigsmakten med ny krigsmateriel. I december 1939 utsåg Kungl Maj:t en kommission med uppdrag att i Italien köpa, till en början torpeder och motorer till torpedbåtar, och om så erbjöds färdiga fartyg i första hand jagare och motortorpedbåtar samt eventuellt även kryssare. Kommissionen bestod av mariningenjören av 1. graden i flottans reserv Holger Graffman, kaptenerna vid flottan Johan Gabriel Oxenstierna och Henning Hammargren, den senare marinattaché, samt mariningenjören Samzelius. +Det fanns tveksamhet från svensk sida om köp av två jagare ur italienska Sella-klassen. Dessa var sjösatta 1926 och ansågs ha passerat sin åldersgräns innan de tagits i bruk i Sverige. Men det italienska budet var två Sella- och två Spica-3-klass jagare. Den sistnämnda klassen var sjösatt 1934, och betydligt modernare. Den 21 januari 1940 var samtliga kontrakt klara för inköp av fyra jagare, fyra motortorpedbåtar med mera. De fyra motortorpedbåtarna av MAS-klassen, vilka i Sverige blev T11-T14, transporterades hem på M/S Boreland från La Spezia till Göteborg i mars 1940. Sverige fick också köpa annan krigsmateriel för sammanlagt 250 miljoner kronor. Totalt mellan 700 och 800 järnvägsvagnar krigsmateriel rullade från Italien till Sverige, minor, torpeder, ammunition, reservdelar, bombsikten och riktmedel till kanoner. Senare under våren 1940 köptes 216 flygplan, dels jaktplanen Fiat CR.42 (J 11) och Reggiane Re.2000 (J 20), och dels bomb- och spaningsplanet Caproni Ca.313 (B 16). Dessutom köptes ett stort antal flygplansmotorer och kulspruteammunition. +Statsminister Per-Albin Hansson stod personligen bakom utnämningen av kommendörkapten 2. graden Torsten Hagman, till avdelningschef (AC) för flottstyrkan. Hagman hade varit adjutant till Per-Albin Hansson, när den senare var försvarsminister på 1920-talet. +Svenska Lloyds passagerarfartyg Patricia hyrdes och omvandlades snabbt till "hjälpkryssare" i flottan. Fartyget utrustades med en 12 cm kanon och två 8 mm kulsprutor. Det örlogsflaggade fartyget var efter återkomsten avsett att bli moderfartyg för det stigande antalet ubåtar. De fyra jagarbesättningarna, sammanlagt 450 man, embarkerade i Göteborg och tillsammans med Patricias besättning på 36 man påbörjades resan mot La Spezia den 4 mars 1940. Jagarbesättningarna kom till stor del från de i Sverige infrusna jagarna HMS Sigurd och HMS Ragnar, samt HMS Vidar och HMS Wale. Då Göteborg blockerades av en halv meter tjock is, fick Patricia isbrytarhjälp ut i Skagerack. Utan allvarligare störningar eller tillbud anlände Patricia till La Spezia den 19 mars. +Den 27 mars halades de italienska flaggorna på jagarna Bettino Ricasoli, Giovanni Nicotera, Spica och Astore och svenska örlogsflaggor hissades på Puke, Psilander, Romulus och Remus, som jagarna döpts till. Från början var de två sistnämnda tänkta att heta Mode och Magne, men den italienska marinledningen lät förstå att ett par av fartygen borde få ett namn som påminde om deras ursprungsland. Namnändringen underlättade avsevärt det fortsatta samarbetet. Under provturerna gick fartygen på "mätt mil" utanför La Spezia, artilleri och torpeder provsköts, dimbildare provades, med mera. +För besättningarna fanns uppenbara risker vid provkörning och trimning av maskiner och materiel. En dödsolycka inträffade när 1. radiomannen, radiokorpralen Erik Larsson, arbetade med felsökning av sändaren i radiohytten på Romulus den 4 april tillsammans med 2. radiomannen Tage Sundgren. När Larsson skulle undersöka en kontakt på sändarens baksida kom han i kontakt med högspänning. Brännskadad i vänstra handen kom Larsson fram från sändarens baksida och sjönk ihop i radiohytten. Upplivningsförsök var förgäves, och på militärsjukhus dödförklarades Larsson. En minnesceremoni hölls två dagar senare med både svenskt och italienskt deltagande, och stoftet togs med på Patricia för begravning i Sverige. +Under arbetet med provturer och intrimning fick besättningsmedlemmarna olika inblickar i den italienska kulturen. +Utrustningsarbeten och provturer var avslutade den 13 april. Det italienska marinministeriet hade tidigare gett sitt tillstånd till en "shakedown cruise" till Tripolis och åter, men det tyska anfallet på Danmark och Norge den 9 april omöjliggjorde detta. Det var därefter angeläget att jagarna snarast lämnade Italien. På första delen av resan fanns också två italienska mariningenjörer och sju maskinunderofficerare med ombord på jagarna för att avhjälpa maskinhaverier. +I Rom försökte den svenska marinattachén att från engelska myndigheter utverka fri passage för flottstyrkan genom Gibraltar sund, och fram till Lissabon. Utverkande av fri lejd från Tyskland sköttes samtidigt av legationen i Berlin. +Jagarna avseglade från La Spezia den 14 april och anlände till Neapel dagen därpå, den 15 april klockan 12.00. Besöket i Neapel gjordes som ett anmält örlogsbesök, för att understryka att jagarna hade skiftat till svensk flagg. För besättningarna betydde det att man efter avslutade arbeten ombord kunde gå iland som vid ett normalt örlogsbesök. På depåfartyget Patricia lastades 30 inköpta torpeder samt en riklig mängd reservdelar till jagarna och de redan levererade motortorpedbåtarna. Även en del ammunition lastades ombord på Patricia. Den 18 april lämnade flottstyrkan Neapel med Lissabon som planerad bunkringsplats. Patricia gick först i kolonnen som farthållare, med 12 knops fart. Därefter följde i samma ekonomifart HMS Puke, med kommendörkapten Torsten Hagman som fartygschef (FC) och avdelningschef (AC). Efter Puke kom HMS Psilander med kapten Agne Löfgren som FC, HMS Romulus med kapten Rolf Ström som FC, och sist i kolonnen HMS Remus med kapten Edward Hamilton som FC. +Den 20 april meddelade den brittiske marinattachén i Rom att "Admiralty skulle garantera fri lejd för eskadern på samma sätt som hade skett med Boreland och Patricia. Bäst för Er att ta vägen genom Engelska kanalen, varvid tyskarna lämpligen kan överta ansvaret i höjd med Rotterdam." Sjöförsvarets kommandoexpedition meddelade trots det, till den tyska beskickningen i Stockholm, att jagarstyrkan planerade att färdas via Irland och Färöarna till norska kusten och därefter kustnära till Göteborg. +Vid middagstid samma dag, den 20 april, fick Puke maskinhaveri och gick ur formeringen utan att meddela sin avsikt till efterföljande Psilander, som inte hann att gira undan utan i måttlig fart träffade Pukes akter med sin förstäv. Puke fick ett propellerskydd bortslitet, och i hålet efter detta rann vatten in. På Psilander hade nitar lossnat i de tillbucklade plåtarna i stäven, vilket också orsakade en mindre läcka. Då de vattentäta skotten höll, kunde båda jagarna efter ett tag fortsätta med 12 knop, och flottstyrkan gick mot Cartagena i Spanien för reparation, dit man kom den 21 april. +I Cartagena arbetade de spanska arbetarna snabbt och effektivt med reparationerna, trots att förstörelsen på örlogsvarvet var stor efter inbördeskriget. Psilander dockades för reparation, medan Puke reparerades utan dockning. På grund av den oplanerade ankomsten till Cartagena kom en stor del av sekretessen kring jagarna och hemfärden, att brytas. Efter förrådskomplettering avgick jagarna den 26 april till Lissabon, dit Patricia hade sänts i förväg för att ta emot beställd olja. +Under tiden i Cartagena, den 23 april, hade den italienske marinchefen amiral Cavagnari, per telefon meddelat den svenske marinattachén att fartygen snarast borde komma iväg hem. "Det bör gå på mindre än tre veckor, men pass opp så att inte tyskarna gör fartygen illa i Kielkanalen." Detta uttalande har senare tolkats som att amiralen var införstådd med Hitlers planer att gå till anfall i väster i början av maj. +Flottstyrkan anlände till Lissabon 28 april och ankrade på floden Tejo, där ursprungsplanen var att snabbt bunkra olja och färskvatten, för att därefter fortsätta till Cobh på Irland. Väntan i Lissabon visade sig bli månadslång eftersom ingen olja gick att få tillgång till, enbart färskvatten. Pukes skador från kollisionen i Medelhavet åtgärdades slutligen med reparation i docka. +Jagarnas färd följdes nu noga i London. I ett PM till sjöförsvarets kommandoexpedition meddelade svenska UD: +Detta meddelande nådde däremot inte kommendörkapten Hagman i Lissabon. Den 10 maj gick Tyskland till anfall mot Nederländerna, Belgien och Frankrike på västfronten, vilket stängde jagarnas färdväg genom Engelska kanalen. Under tiden arbetade man i Stockholm på att få olja till jagarna från annat håll och chefen för marinen (CM) meddelade jagarstyrkan den 15 maj att ett svenskt tankfartyg var att vänta inom fyra till fem dagar. Den svenska tankern M/T Castor hade rekvirerats och beordrades att ansluta till jagarstyrkan. Den 21 maj kom Castor fram till Lissabon med 14 495 ton olja, varav hälften dieselolja. Dessutom hade fartyget mer än 1 000 ton färskvatten i sina tankar vilket förväntades göra jagarna självförsörjande. +En serie telegram sändes från chefen för marinen: +Tankfartyget blev nu formellt örlogsfartyget HM Transportfartyg Castor och befälhavaren John Juhlin blev alltså samtidigt löjtnant i flottans reserv. +Den 26 maj lämnade flottstyrkan Lissabon, och den 27 maj ankom man till Vigo i Spanien, där Castor och Patricia bunkrade fullt, samt den italienska maskinpersonalen på jagarna gick iland hemkallade av sin regering. Den 10 juni gick Italien in i kriget. +Trots att chefen för marinen tidigare hade gett instruktioner om att Castor skulle följa med jagarstyrkan, så innebar fartygets låga fart, cirka åtta knop, att olja och vatten troligen inte skulle räcka på jagarna för förflyttningen till Cobh på Irland, enligt Hagmans bedömning. Han detacherade därför Castor och lät jagarna och Patricia gå från Vigo i sin mest ekonomiska fart, tolv knop, i en vid båge ut i Atlanten mot Cobh den 29 maj. +Castor eskorterades ut genom det ubåtsfarliga området närmast spanska kusten, och fick order att på egen hand fortsätta till Cobh. Vid middagstid den 30 maj prejades Castor av ett franskt bevakningsfartyg. Då tankfartyget enligt dokument tillhörde svenska marinen och förde örlogsflagga, hade det ändå inga kanoner. Ett örlogsfartyg måste ha minst en kanon, menade den franska fartygschefen. Kapten Juhlin var visserligen löjtnant i flottan, men hade ingen uniform. Allt detta gjorde fransmännen misstänksamma, och Castor fick order att gå mot Casablanca, där oklarheter skulle redas ut. Efter ingripande av svenska regeringen släpptes tankern efter två dagar, och fick order att återförena sig med jagarstyrkan i Cobh. +I slutet av maj och början av juni genomfördes den dramatiska evakueringen av cirka 300 000 engelska och franska soldater från stränderna vid franska Dunkerque tvärs över Engelska kanalen. En ny rutt ut i Atlanten mot Färöarna anvisades av det brittiska amiralitetet, och även Castor skulle ansluta där. I praktiken var danska Färöarna besatta av brittisk militär sedan i april 1940, efter Danmarks kapitulation. Men britterna erkände formellt inte detta, eftersom det skulle inneburit ansvar för Färöarnas styrelse och försörjning. +Utanför Irlands sydkust den 1 juni upptäckte Patricia ett periskop om styrbord, och undanmanöver verkställdes av samtliga fartyg. Jagarstyrkan ankrade i Cobh (Queenstown) den 2 juni. Återigen gick det inte att få tag i någon olja i land. Genom att bunkra från Patricia kunde jagarna fylla på tillräckligt med olja för att nå Färöarna. Dessutom kom besked att 117 civila passagerare, varav 40 kvinnor och 8 barn, skulle följa med på Patricia som passagerare. De flesta var tidigare anställda i engelska företag, som nu blivit utan jobb sedan företagen ställt om till krigsproduktion. Bland passagerarna fanns åtskilliga unga kvinnor som jobbat som au pair, eller i serviceyrken, och nu ville hem på grund av kriget. Även sjömän från krigsförlista svenska fartyg, till exempel S/S Foxen, fanns bland passagerarna. Dessa väntades anlända i mitten av juni, och väntetiden utnyttjades av besättningarna bland annat till fysisk träning. Den 15 juni tog Patricia ombord sina civila passagerare, och på natten till den 16 juni lämnade man Cobh med destination Färöarna. +Den 14 juni marscherade tyska trupper in i Paris, och den 22 juni kapitulerade Frankrike. Det innebar att Storbritannien stod ensamt mot de dittills ständigt segrande tyska trupperna, med ett överhängande hot om tysk invasion. Den tyska flottan hade tidigare mist tio av sina modernaste jagare i slaget om Narvik den 10 och 13 april 1940, och britterna var angelägna om att Tyskland inte skulle få snabb tillgång till nya jagare. +Vid brittiska krigskabinettets sammanträde i London meddelade ministern för Dominions, att fyra svenska jagare hade lämnat Eire för att gå till Färöarna. Han föreslog krigskabinettet att överväga möjligheten att kapa jagarna vid deras ankomst till Färöarna. Utrikesministern förklarade sig vilja undersöka frågan närmare. +Vid krigskabinettets sammanträde behandlades förslaget från föregående dag om kapning av jagarna. Ministern för Dominions ansåg, att detta var ett fall där man kunde använda angarierätt. Detta är en i internationell rätt fastställd rättighet för en krigförande nation att beslagta en neutral stats egendom, inom den krigförande nationens territorium eller områden som ockuperats av den. Premiärminister Churchill ansåg det vara av största vikt att sätta sig i besittning av jagarna. Omedelbara åtgärder skulle vidtas och man skulle senare undersöka vilka förklaringar som ansågs lämpliga. Krigskabinettet gav Admiralty i uppdrag att snarast möjligt ge order om kapning av jagarna. +I tät dimma närmade sig Patricia och jagarna Färöarna på nordlig kurs under tidiga morgonen, med minimal fart växlande mellan "Sakta" och "Halv". Klockan 05.45 slogs "Stopp" och "Full back" på Psilander då klipporna var så nära att ekot från ångvisslan återkastades. Fartygen vände ut igen och avvaktade att dimman skulle lätta, för möjlighet till ny angöring. Castor gav sitt läge via radio klockan 09.00 cirka 120 distansminuter från Färöarna, och fartyget skulle vara framme morgonen därpå. +Samtidigt var den svenske konsuln på Färöarna inte i förväg informerad om jagarnas ankomst av svenska myndigheter, utan fick besked om att ett stort belopp ställts till hans förfogande hos Føroya Banki, att användas för jagarstyrkans behov. Inte heller den brittiske representanten "Naval Officer in Charge of Faroe Islands (NOIC)" Captain Crowther hade fått information om jagarna, innan han meddelades av den svenske konsuln. +Efter att dimman lättat under förmiddagen styrde Patricia med jagarna återigen in mot Färöarna, och tog ombord lots som förde fartygen till den långsmala Skaalefjorden nära Torshamn. Där blev Patricia anvisad ankringsplats längst in i fjorden, och jagarna fick ankra i taktisk nummerordning - i linje, vid 14-tiden på eftermiddagen. I samma fjord ankrade också ett antal bestyckade engelska trålare (naval drifters) på var sida om jagarna. I land besatte brittisk militär de små telefonstationerna utmed fjorden, och beslagtog kikare och kameror. +Kommendörkapten Hagman gav order om att släcka ångpannorna på jagarna för att spara olja. Det innebar att det skulle ta cirka fyra timmar att få jagarna klara för avgång igen. Dessutom beordrades vaktgöring på kvarter, det vill säga ingen stridsberedskap. +De tre brittiska Tribal-klass jagarna HMS Maori, HMS Tartar och HMS Mashona lämnade den brittiska flottbasen Scapa Flow på Orkneyöarna klockan 19.30 med kurs mot Färöarna, och med avsikt att överta de svenska jagarna. +Under tidiga morgonen kom Castor fram till Färöarna och fick order att ankra sydöst om Torshamn cirka tio distansminuter, en och en halv timmas gångtid, från jagarna i Skaalefjorden. Senare under morgonen kom också de brittiska jagarna fram. +Vakten på Remus, närmast fjordmynningen, märkte att några stora engelska jagare ankrade upp och några bestyckade trålare närmade sig den svenska styrkan. Han väckte vakthavande officeren (VO) och rapporterade. +I en motorbåt anlände den brittiske eskaderchefen Captain C. Caslon, samt en Lieutenant-Commander (paymaster) som sekreterare och en civilperson till svenska förbandschefsfartyget HMS Puke. Caslon framförde muntligt i ultimativ form att han direkt från brittiska regeringen hade fått order att ta jagarna i beslag. Som motiv för sitt handlande angavs att man från engelsk sida förutsatte att de svenska jagarna skulle hamna i tyska händer då fartygen närmade sig Sverige. Jagarna skulle enligt de brittiska kraven utrymmas inom två och en halv timma, och besättningarna skulle därefter föras över till England för vidare transport till Sverige. Patricia och Castor kunde fortsätta resan, men motstånd från jagarna eller försök att sänka fartygen, skulle mötas med "lämpliga motåtgärder". Alla radiosändare på de svenska fartygen skulle plomberas omedelbart, och försök att upprätta förbindelse med Sverige förhindras med våld. +Kommendörkapten Hagmans spontana svar var att avvisa kraven, och att han skulle slåss alternativt sänka fartygen. Captain Caslon beklagade detta och påpekade att det bara var fråga om att ta fartygen i förvar, de skulle förbli svensk egendom. +Hagman krävde då att få möjlighet att sätta sig i förbindelse med svenska regeringen, men Caslon förklarade att användning av radio skulle förhindras med våld. Hagman begärde att få den brittiska regeringens krav och motiv i skriftlig form, varefter han skulle svara skriftligt. Caslon lovade att tillmötesgå detta, och lämnade därefter HMS Puke. +Meddelande skickades från AC på Puke till övriga svenska fartyg inom synhåll: "Samtliga FC kallas inom en halvtimma". Enligt då gällande reglemente för marinen (RM II §56), kunde förbandschef vid viktigare tillfällen sammankalla underställda chefer för rådslag, där protokoll förs men beslutet fattas av chefen själv. +Strax efteråt kom ett nytt meddelande: "Förbered utrymning av jagarna". +Kommendörkapten Hagman var medveten om att jagaren HMS Psilander fått sitt namn av amiralen Gustav von Psilander, som i slaget vid Orford Ness år 1704 vägrade att vika sig för en kraftig engelsk övermakt. I väntan på övriga fartygschefer formulerade han sitt eget ställningstagande. +De brittiska kraven överlämnades skriftligt. Där anges som bilaga också "Right of Angary", där krigförande länder äger rätt att på eget eller ockuperat territorium tillfälligt beslagta neutral makts egendom. Prejudikat från tysk-franska kriget 1870-1871 och första världskriget anfördes, men ingen med tillämpning på örlogsfartyg. Enligt de skriftliga kraven skulle eventuella skador på de beslagtagna jagarna ersättas. +Efter att ha granskat de skriftliga kraven höll fartygscheferna rådslag ombord på Puke. Styrkeförhållandena var kraftigt i brittisk favör. De brittiska jagarna var under gång och stridsberedda i klart skepp (action stations). Vardera av Tribal-jagarna hade åtta stycken 12 cm kanoner, fyra stycken luftvärnsautomatkanoner och fyra stycken torpedtuber. Dessutom fanns bestyckade engelska "drifters" mellan de svenska jagarna, samt posteringar i land. +På svensk sida hade fartygen mycket låg nivå av olja och vatten och ett ofördelaktigt läge för strid i den smala fjorden, avsläckta och ankrade på linje. Efter att under några timmar ha fått upp ångtrycket kunde bara Remus två aktra 10 cm kanoner riktas med fritt skottfält mot de brittiska jagarna. Om de svenska jagarna hade brutit sig ut, skulle de snart ha slut på bränsle och bli enkla mål för Home Fleet. Dessutom fanns Patricia i skottlinjen, lastad med ammunition och med 117 civila personer ombord. +Att själv skada eller sänka de egna jagarna skulle dels hjälpa britterna i deras påstådda strävan att hindra jagarna att falla i tyska händer, dels äventyra svenska regeringens ersättningskrav. Dessutom skulle det medföra allvarliga konsekvenser för Patricia och Castor med passagerare och dyrbara laster av krigsmateriel och olja. +Flera av Tribal-klass jagarna styrde in i Skaalefjorden, vände och styrde ut igen. +I det pågående fartygschefsmötet, så skulle enligt reglerna den yngste av fartygscheferna, Edward Hamilton på Remus, yttra sig först för att inte påverkas av de äldre. Samtliga fartygschefer hade ändå samma åsikt, att motstånd var utsiktslöst och att situationen borde lösas med förhandlingar mellan respektive länders regeringar. Kommendörkapten Hagman, som befann sig i den för en svensk officer märkliga situationen att ensam behöva fatta beslut som kunde få storpolitiska konsekvenser, skrev i sitt svar till britterna att han under omständigheterna inte hade något annat val än att gå med på kraven. Men han protesterade också i svaret allvarligt mot hela proceduren, särskilt att han förvägrats möjligheten till kommunikation med egna regeringen. +Innan skrivelsen överlämnades hade en förfrågan gjorts, om besättningarna kunde föras över till Patricia och Castor i stället för att, enligt det ursprungliga ultimatumet, föras till England. Begäran godkändes av engelsmännen. +Efter mötet återvände fartygscheferna till sina fartyg och meddelade besättningarna. Endast personliga tillhörigheter skulle få packas ner i sjösäckarna enligt brittiska direktiv. På AC order medtogs dessutom flaggor och befälstecken utom de blåsande, kojer, filtar, gasmasker med mera. En del tillhörigheter kastades överbord, eftersom besättningarna inte ville överlämna det till engelsmännen. Marketenteriinnehållet delades ut så rättvist som brådskan medgav. +Trots order från AC att inte försämra läget genom sabotage eller motstånd, jagarna var fortfarande svensk egendom, förekom detta på enskilda besättningsmäns initiativ. På Remus, som låg närmast de brittiska jagarna, fick befälet avstyra begäran från besättningsmedlemmar som trots övermakten ville gå i strid. I maskinrummet på Remus hade vissa kikar, det vill säga ventiler som växlas mellan öppet och stängt med ett kvartsvarvs vridning, och kranar stängts. Dessutom hade några blindflänsar, det vill säga plattor som stoppar flödet i rörledningen, monterats. Slagstiften till kanonerna hade tagits bort, och gjort artilleriet obrukbart. På Romulus fick radiomännen tillstånd att göra sändaren obrukbar. De slog då sönder slutrör och reservrör, samt lämnade en skylt med texten "Pericolo di Morte" (Varning för döden) i radiohytten. +Tre drifters användes till att föra över besättningarna från Puke, Psilander och Romulus till Patricia samt från Remus till Castor. Utrymningen leddes av NOIC, Captain Crowther. Utrymningen var känsloladdad för besättningarna från jagarna som fick påspädning av en, i vissa fall, arrogant attityd från drifters besättningar. +Efter att svenska besättningarna lämnat kom brittiska besättningar ombord på jagarna, med första åtgärd att hala ned de svenska flaggorna och befälstecknen. +Klockan 09.00 kom NOIC, Captain Crowther, till Patricia och meddelade att fartyget kunde avgå. Vid 14-tiden på eftermiddagen lättades ankar och man gick ut genom Skaalefjorden med kurs mot norska kusten. Ute på internationellt vatten kunde radiosändare åter användas till kontakt med svenska myndigheter. +I en serie telegram informerade kommendörkapten Hagman, chefen för marinen (CM): +Patricia och Castor girade på kontrakurs tillbaka till Färöarna. Under tiden började en diplomatisk telegramväxling mellan Stockholm och London. Kabinettssekreterare Erik Boheman, som var på middag hos den brittiske ministern i Stockholm, Victor Mallet, blev mycket upprörd och krävde att Mallet skulle telegrafera till London om att jagarna omedelbart skulle friges. +De två svenska fartygen återvände till Torshamn, där de ankrade på yttre redden klockan 08.00. Den svenske vicekonsuln på Färöarna meddelade att jagarna hade förflyttats till en för honom obekant plats, vilket han protesterat emot hos den brittiske konsuln. +De diplomatiska kontakterna fortsatte. Bland annat samtalade den svenske ministern i London, Björn Prytz, med den brittiske utrikesministern Halifax. Prytz meddelade att, om britterna behöll jagarna, skulle tyskarna anklaga den svenska regeringen för att tillåta kränkning av den svenska neutraliteten, och som konsekvens invadera Sverige. Prytz meddelade också att jagarnas färdväg fullständigt hade diskuterats med den brittiska legationen i Stockholm, vilket Halifax inte ansåg sig känna till. +Senare granskning av källmaterial har gett en förklaring till att det brittiska agerandet eventuellt byggde på ett missförstånd. Den svenska formella begäran om fartygspassage hade blivit liggande hos amiralitetet i London på grund av semestertiderna. Därför blev den brittiska regeringen överraskad av den svenska sjöstyrkans förflyttning, som det uppfattades utan förvarning och utan att följa normal rutin och "proper channel". Misstanken väcktes därför att svenskarna var bulvaner i ett försök att förstärka den tyska flottan med italienska fartyg. +Under de följande dagarna insåg man på brittisk sida att jagarna var ganska små och inte särskilt effektiva. Man försökte också koppla ett eventuellt frisläppande av jagarna som argument i förhandlingar om andra frågor, till exempel möjlighet att skjuta upp svenska flygplansköp i USA, samt avbrytande av den planerade tyska transiteringstrafiken. Vid brittiska krigskabinettets möte den 24 juni fattades beslut om att släppa jagarna. +På kvällen den 25 juni meddelade NOIC att de svenska jagarna skulle återlämnas till sina besättningar. Två av jagarna befann sig i Kirkwall, och två var på väg dit. För att inte Castor skulle behöva färdas genom de farliga vattnen runt Orkneyöarna fördes Remus besättning också över till Patricia. Enligt direktiv från CM skulle de civila passagerarna inkvarteras i Torshamn, men detta tilläts inte av britterna. Med över 600 personer ombord på ett fartyg som var avsett för 250, avgick Patricia den 30 juni 01.00 till Kirkwall, och var framme cirka klockan 23. +På förmiddagen den 1 juli kunde fartygscheferna och vissa besättningsmän inspektera jagarna. Under färden mellan Färöarna och Kirkwall hade de fått åtskilliga skador. Dels på grund av de engelska besättningarnas ovana vid jagarnas maskineri, dels på grund av svenska sabotage. Detta hade lett till upprepade maskinstopp och bogseringar, allt i mycket hårt väder. På Psilander och Romulus hade britterna torrkokat en panna, som därigenom blivit helt förstörd. På Romulus hade olja dessutom av misstag pumpats upp i gångar och mäss. Puke hade skador efter kollision i samband med bogsering, och hade tappat ett ankare. Den engelska besättningen anspelade också på det kraftigt rullande fartygets namn, som på engelska betyder må illa, kräkas. +Fartygen hade till stor del plundrats på lösa inventarier och verktyg. På Remus lista över förlorade inventarier fanns 22 borddukar, 54 handdukar, 53 tallrikar, 77 kaffekoppar och så vidare i en lång lista. Alla förluster, samt lönekostnader för Patricia och Castors besättningar med mera värderades senare till 879 644 kronor, vilket betalades av britterna. +Jagarna återlämnades 2 juli till sina svenska besättningar, som snabbt utförde rengöring och provisorisk reparation av fartygen. Den 5 juli 02.00 lämnade Patricia och jagarna Kirkwall för att åter färdas mot Färöarna. Castor möttes vid en punkt 40 distansminuter öster om öarna på kvällen den 5 juli, och styrkan gick därefter i kolonn mot Norge, med åtta knops fart och ordningen Puke, Psilander, Castor, Patricia, Romulus och Remus. +Efter att ha siktat flygplan både 6 och 7 juli på lång distans kom, utefter norska kusten, vid 6-tiden på morgonen den 8 juli ett bombanfall av flygplan från brittiska kustkommandot, där fyra bomber fälldes mot Castor. Bomberna sprängdes på 50-75 m avstånd från tankern, som efter noggrann kontroll rapporterade att ingen läcka kunde märkas. Anfallet berodde på ett misstag, och britterna har senare bett om ursäkt. +Efter att både Puke och Remus sänkt drivande hornminor, kom styrkan fram till Kristiansand på morgonen den 9 juli. Då myndigheterna behövde få några timmar för att varsko den egna kustbevakningen om den svenska styrkans förflyttning, blev Hagman under tiden väl emottagen hos de tyska myndigheterna. +På eftermiddagen fortsatte resan, och på morgonen den 10 juli passerade styrkan in på svenskt territorialvatten. Vid 17-tiden var jagarna förtöjda vid Nya Varvet i Göteborg där de togs emot av chefen för marinen, amiral Fabian Tamm. Han hälsade fartyg och besättningar välkomna hem, men med en kylig underton. Ingen ifrån regeringen fanns representerad i Göteborg, vilket dåvarande statsminister Per-Albin Hansson senare ångrade. +Efter ankomsten till Göteborg hölls på CM:s initiativ förberedande förhör med Hagman vid fältkrigsrätten vid västkustens marindistrikt. På en fråga från förhörsledaren om Hagman ansåg att han uppehållit "svenska flottans anseende och svenska flaggans ära" svarade han, att han ansåg sig i det aktuella läget ha handlat riktigt, oberoende av reglementen och instruktioner. +Sedan fältkrigsrätten återigen sammanträtt den 6 augusti 1940, konstaterades att Hagmans handlande inte skulle leda till ansvar eller ersättningsskyldighet och att ärendet inte skulle gå vidare till rättegång. +Vid sammanträdet frågade åklagaren, om det kanske varit lämpligt att låta engelsmännen skjuta ett eller annat skott, och först därefter bestämma om jagarna skulle överlämnas. På det svarade Hagman, att han ofrånkomligt varit tvungen att besvara elden, därmed skulle också alla ersättningsanspråk äventyrats. Det var enligt Hagman utomordentligt viktigt att inte ett enda skott föll. +Trots att statsminister Per Albin Hansson personligen meddelande Hagman att han hade hans stöd för sitt handlande fick Hagman aldrig någon formell upprättelse, eftersom han inte blev offentligt friad i domstol. Han betraktades av många som en förrädare som överlämnat sina fartyg utan strid. +Kritiken från högsta marinledningen gällde också att Hagman detacherat Castor utanför Vigo, mot direktiven från CM att låta tankfartyget ingå i jagarstyrkan. Dagen efter det att fältkrigsrätten avslutat sin förberedande utredning, den 22 augusti 1940 utfärdade chefen för kustflottan, konteramiral Gösta Ehrensvärd, en hemlig order: +En pressdebatt bröt ut i samband med krigsrättens dom, där de flesta tidningar stödde Hagmans handlande. En kommentar var att "krigsmannaplikten får inte överflödiggöra bruket av hjärnan". Tidningarna på extrema vänsterkanten, och den för sin tyskvänlighet kända Östgöta Correspondenten var kritiska till Hagmans beslut. +På brittisk sida har flera företrädare efteråt uttryckt att man ångrade åtgärden att beslagta jagarna, som kallats "a dirty trick" igångsatt av Secret Service. En hög civil tjänsteman i Admiralty framförde till dåvarande konteramiral Stig H:son Ericson, att den brittiska åtgärden uteslutande kunde förklaras med det nervösa tillstånd som britterna befann sig i, samt att Admiralty gjort en mycket beklaglig blunder. Churchill erkände också senare för svenske ministern Prytz i London, att Hagman skött "den svåra och pinsamma affären" på ett beundransvärt sätt, och att "han besparade båda länderna mycket besvär". Han nämnde också "Om den svenske chefen varit en brittisk officer i motsvarande läge skulle vi ha givit honom en lämplig utmärkelse för att uttrycka vår beundran och nationens tacksamhet". +Incidenten gjorde att den redan tyskvänliga svenska sjöofficerskåren blev rent fientligt inställd mot Storbritannien i allmänhet och mot brittiska flottan i synnerhet. Detta kom att vålla britterna problem i samband med senare händelser kring de i Sverige instängda norska kvarstadsbåtarna, Operation Rubble och Operation Performance. +I förordet till jubileumsskriften för 50-årsjubileet av Italienexpeditionen år 1990 lovordade dåvarande chefen för Marinen, viceamiral Bengt Schuback, kommendörkapten Hagmans handlande och gav honom en postum upprättelse. +Flera av källorna nämner en fjärde Tribal-klass jagare vid Färöarna, förutom de tre namngivna. Däremot nämner inte någon av källorna vilken jagare det skulle varit. Enligt ett rykte på Romulus fanns också slagskeppet HMS Rodney utanför mynningen av Skaalefjorden. +Artéus, Gunnar (1996). Svenska officersprofiler under 1900-talet. Stockholm: Militärhögskolan. Libris 7762137. ISBN 91-87072-19-X +Barker, Ralph (1976). Blockadbrytarna. Lund: Rabén & Sjögren. Libris 7233733. ISBN 91-29-47167-2 +Fontander, Björn (2006). De onda åren: Sverige 1940-1945. Stockholm: Carlsson. Libris 10166527. ISBN 91-7203-784-9 (inb.) +Nilsson, Bengt (december 2000). ”Sven Hedin och Torsten Hagman : Psilander som förebild under 1900-talet”. http://members.tripod.com/Bengt_Nilsson/Marinhistoria/svenhedin.htm. Läst 7 januari 2011. +Lundberg, Lennart (2012). Svenska öden under andra världskriget. Stockholm: Instant Book. Libris 12339211. ISBN 978-91-86939-20-5 +Rumenius, John (1976). "Klart skepp för minfällning" : en reservfänrik berättar från beredskapsåren i flottan 1939-1940. Stockholm: Lindfors. Libris 7630300. ISBN 91-7268-044-X (inb.) +Smith, Gordon (18 oktober 2011). ”Naval events, June 1940, Saturday 15th - Friday 21st”. http://www.naval-history.net/xDKWW2-4006-19JUN03.htm. Läst 7 januari 2011. +Strömbäck, Stig; Jahr, Erik; Lilja, Nils-Hugo (1990). Jagarköp under krig : dramatisk hemfärd med Italien-jagarna. Stockholm: Jubileumskomm. för 1940-års Italienexpedition. Libris 936771. ISBN 91-87072-07-6 +Zetterberg, Kent; von Hofsten, Gustaf (2020). "Jagarincidenten vid Färöarna 1940" : Brittiskt försök att beslagta svenska örlogsfartyg. Stockholm: Svenskt Militärhistoriskt Biblioteks Förlag. ISBN 978-91-88885-22-7 (inb.) +Hagman, Torsten (1941). De i Italien inköpta jagarnas utrustning och hemfärd. Karlskrona. Libris 2062626 +Hagman, Torsten (1959). Tillägg 1959 till berättelsen "De i Italien inköpta jagarnas utrustning och hemfärd". Stockholm. Libris 3228778 +Hagwall, Åke (1993). Ett jagarköp och ett chefsbeslut 1940. Torekov. Libris 3228783 +Hammargren, Henning (1981). Vapenköp i krig : svenska krigsmaterielinköp i Italien under andra världskriget. Stockholm: Marinlitteraturfören. Libris 7753508. ISBN 91-85944-02-5 (inb.) +Hardy, Alfred Cecil (1950). Kampen på haven : sjökrigshändelserna under andra världskriget i populär framställning. Malmö. Libris 11634583 +Hagberg, Gunnar: Torsten Hagman är Norrköpings glömda hjälte (artikel i Norrköpings Tidningar den 12 mars 2005). +Rosenqvist, Staffan: Beredskapsmärken: Flottan 1939 - 1945 (se särskilt 1940-års ROMULUSMÄN). +Älvdalska eller älvdalsmål (övdalsk, övkallmą̊l, dalska) är en nordisk språklig varietet som talas av cirka 3 000 talare i Älvdalens socken i norra Dalarna i Sverige. +Älvdalskan är nära besläktad med närliggande ovansiljanmål såsom våmhusmål och orsamål, som tillsammans förgrenade sig från fornnordiskan tidigast på 800-talet och senast 1300-talet. Älvdalskan utmärker sig genom en rad arkaismer och novationer. Bland annat har älvdalskan bevarat ett äldre kasussystem, fornnordiskans stavelsekvantitetssystem med korta, långa och överlånga stavelser. Den har också bevarat ljud som [w] och [ð], och har även som ensam nordisk varietet bevarat fornnordiskans nasala vokaler. Älvdalskan har också genomgått stora ljudförändringar och förändringar i satsläran. Bland förändringarna har bland annat äldre långa vokaler diftongerats, och de primära nordiska diftongerna har monoftongerats eller utvecklats till andra diftonger. +Älvdalskan och de övriga ovansiljanmålen är inte ömsesidigt begripliga med svenska, norska eller danska, men är i stort sett förståeliga sinsemellan. Traditionellt sett har ovansiljanbor använt sitt modersmål inom Ovansiljan, medan man har gått över till svenska vid besök i till exempel Rättvik eller Malung utanför Ovansiljan. Talare av älvdalska och övriga ovansiljanmål är således tvåspråkiga och kodväxlar mellan ”dalska” och svenska. +Under 1900-talet har älvdalskan genomgått en utjämning i riksspråklig riktning, och för varje generation närmar den sig standardsvenskan mer och mer, vilket har lett till ett mer rikssvensktnära uttal, en förenkling av morfologin (bland annat färre kasus), med mera. Flera forskare skiljer därför mellan klassisk älvdalska, traditionell älvdalska och modern älvdalska. +Älvdalska har ingen status som minoritetsspråk i Sverige enligt språklagen (2009:600). Hösten 2015 lämnade tio lingvister in en ansökan till SIL International om att ge älvdalskan en ISO 639-3-kod på Ethnologue. En kritisk kommentar till denna ansökan lämnades in av Institutet för språk och folkminnen, den myndighet i Sverige som bland annat handhar frågor om dialekter, men 27 maj 2016 beviljades ansökan (efter överklagande). +Vid Oðer råðstemną um övdalskų (Andra konferensen om älvdalska) 2008 presenterade Gösta Larsson, tillsammans med Bengt och Ulla Welin, en uppskattning av antalet talare av älvdalska baserat på frågeformulär insamlade av olika ”byombud” runtom i Älvdalens socken. Dessa visade att cirka 1 700 boende i socknen kunde tala älvdalska, vilket är cirka 34 procent av befolkningen. Därtill uppskattar de att cirka 1 300 i socknen kan förstå, men inte tala älvdalska. I socknen kan omkring 2 000 personer varken förstå eller tala älvdalska. De uppskattar även antalet talare av älvdalska utanför socknen till cirka 700. De räknar därför med att cirka 2 500 personer år 2008 talade älvdalska. I åldrarna över 50 år var andelen älvdalsktalande 60 procent, i åldrarna mellan 15 och 50 var antalet 20 procent. Hos barn och ungdomar under 15 kunde bara 5 procent tala älvdalska, vilket motsvarar 45 personer. +Samma undersökning visar även att antalet älvdalsktalande generellt är högst i de perifera byarna, och lägst i kyrkbyn. I kyrkbyn talade endast 14 procent älvdalska, vilket motsvarar 186 personer, jämfört med mer perifera byar som Brunnsberg eller Åsen, med 64 procent (eller 145 personer) respektive 57 procent (eller 182 personer) älvdalsktalande. +Tidigare undersökningar av antalet älvdalsktalande gjordes av bland andra Helgander år 1991 som visade totalt 1 652 talare, varav 179 under 20 år. Denna undersökning inkluderade dock inte lika många byar som i Larsson och Welins uppskattning, och kompletterades år 2004 av Gunnar Nyström med ytterligare 668 talare i de byar som Helgander inte undersökte. Totalt minskade antalet älvdalsktalande med 37 procent i byarna Helgander undersökte mellan 1991 och 2008, från 1652 till 1034 talare. Mellan Nyströms kompletterande undersökning 2004 och Larsson och Welins 2008 minskade antalet i alla byar med undantag av två, Östermyckeläng och Mjågen, där antalet ökade från 85 till 147 respektive 10 till 13. I övriga byar minskade antalet med totalt 14 procent, från 573 till 495 talare. Den mest drastiska minskningen har skett hos unga talare, där Helgander år 1991 noterar 179 talare under 20 år, jämfört med 45 talare under 15 år 2008, vilket är en minskning på 75 procent. +Även inom det älvdalska språkområdet förekommer viss språklig variation och älvdalskan talas på olika sätt i olika byar. Skillnaderna är särskilt tydliga i uttalet, men det finns även viss grammatisk variation och viss lexikal variation, det vill säga dialektala ord. Lars Levander delar in älvdalskan i tolv ”typbymål”: Blyberg, Brunnsberg, Evertsberg (med Dysberg), Garberg, Gåsvarv (med Mjågen), Loka, Karlsarvet (med Månsta), Klitten, Näset (med Rot, Holen, Kittan, Liden), Väsa, Västermyckeläng (med Kåtilla) och Åsen. Han nämner dock att överensstämmelsen inom grupperingarna inte är fullständiga, och att ett ”vant öra med lätthet [kan] urskilja, om en person är t.ex. från Evetsbärg eller Dysbärg”. +Där svenskan i många fall har å eller a framför ng-ljud i ord som lång, trång, blank har älvdalskan au, a eller o. Av dessa vokaler är au det äldre uttalet och förekommer i de västra byarna, jämte Finnmarken, i Åsen och hos äldre i de södra byarna. Uttalet med a är något yngre och uppkom som en förenkling av au-diftongen och förekommer i de centrala östra byarna (inklusive Klitten). På 1920-talet noterade Levander inget uttal med a, men det fanns bevisligen redan då, och Adolf Noreen noterar 1882 ett a-uttal hos en talare från Rot född 1861. Brunnsberg har som ensam by ett o-uttal: longg, trongg, blonk. +Hos de nasala vokalerna har det skett vissa vokalsammanfall. Nästan alla bymål skiljer mellan de onasalerade vokalerna o och å som i roð ’röd’ och råð ’råd’, men de nasala motsvarigheterna sammanfaller alltid. Resultatet är dock olikt i olika byar. I de västra byarna samt Finnmarken och Åsen är resultatet ą̊ som i ną̊t ’natt’ och gą̊s ’gås’. Byarna på östsidan Dalälven, förutom Åsen, har i stället ǫ: nǫt och gǫs. I de sydligaste byarna som Blyberg och Garberg saknas ibland nasaliteten helt och vokalen är då ett onasalerat o. +På samma sätt finns inget bymål som skiljer mellan de nasala motsvarigheterna till vokalerna e och ä. I de allra flesta bymålen har resultatet blivit ett nasalerat ę som i męla ’mäta’ och nęta ’nöta’, men i Finnmarken och i byarna på västsidan om älven (förutom Väsa) är uttalet i stället ą̈: mą̈la och ną̈ta. +En diftong med mycket varierande uttal är den som med Råðdjärums ortografi skrivs uo, som också ingår i triftongen iuo. I byarna på östsidan om älven är dess första element en u-haltig vokal och västsidan en mera o-haltig vokal. Diftongens andra element är mera obetonad och förekommer i många varianter. På många östliga mål skrivs diftongen ofta uä eller ue, medan diftongen i flera västliga mål snarare liknar oå eller ou. +I vissa byar ersätts ljudet [ð] ibland med /r/, såsom i den bestämda artikeln i neutrum -eð, vilket gör att till exempel auseð ’huset’ i stället heter auser. Denna process började redan på 1600-talet. +Som nämns ovan är mindre än hälften av Älvdalens sockens befolkning älvdalsktalande, och andelen talare blir mindre för varje generation. Älvdalskan har dessutom påverkats starkt av svenskan under de senaste generationerna, och alla älvdalsktalande talar svenska. Detta resulterar i en mer splittrad variation, i vilken skillnaderna mellan generationerna, och ibland mellan individer, har blivit större än skillnaderna mellan de traditionella bymålen. +Älvdalska har ingen status som minoritetsspråk i Sverige enligt Språklagen (2009:600). Europarådets expertkommitté som kontrollerar hur den europeiska minoritetsspråksstadgan efterlevs har fem gånger uppmanat Sveriges regering att ompröva älvdalskans ställning. Sedan 2006 har ett antal motioner lämnats in till riksdagen om att ge minoritetsspråksstatus till älvdalskan, men det har till dags dato (januari 2021) inte givits någon ytterligare status i Sverige. +I december 2020 uppmanade Europarådets expertkommitté svenska myndigheter att ta in älvdalskan i Europarådets minoritetspråkskonvention. Detta efter att Europarådets expertråd undersökt frågan och fastslagit att älvdalskan uppfyller Europarådets kriterier. Expertkommittén uppmanar således svenska myndigheter att inbegripa älvdalskan sin nästa rapport till Europarådet. +I en artikel i Språktidningen 2008 diskuterade Östen Dahl älvdalskans status som dialekt gentemot språk. Han jämför bland annat älvdalskans ordförråd med svenskans, norskans och danskan och konstaterar att älvdalskans skiljer sig mer från svenskans än danskans och norskans. Han skriver också att älvdalskans grammatik och fonologi har ”en rad fenomen som saknas i både svenska och norska, som dativkasus, böjning av verb efter person och numerus, nasalvokaler och så vidare.” Å andra sidan skriver Dahl att många sockenmål i Sverige skiljer sig avsevärt från svenskan – även närliggande sockenmål såsom våmhusmål, moramål och orsamål. Det produceras också skriven text på älvdalska, till exempel en översättning av Antoine de Saint-Exupérys Lille prinsen. Garbacz tar även upp att älvdalingarna är mycket medvetna om sitt mål och att den normala kommunikationsstrategin i Älvdalen är kodväxling mellan älvdalska och svenska. +Hösten 2015 lämnade tio lingvister in en ansökan (Övdalian, ansökan 2015-046) till SIL International om att ge älvdalskan en ISO 639-3-kod på Ethnologue, bland annat på grunderna av att älvdalskan ”systematiskt skiljer sig från de andra nordiska språken (och dialekterna) på alla språkliga nivåer: fonetiskt, fonologiskt, morfologiskt, syntaktiskt och till ordförrådet”. De skriver vidare att dalmålen är kända för sin unika ställning bland svenska dialekter, och att älvdalskan är oförståelig för talare av svenska, norska och danska, och att till och med andra talare av dalmål har stora problem när de försöker kommunicera med talare av älvdalska på sina respektive mål på grund av dessa systematiska skillnader. +Ett antal kommentarer till denna ansökan lämnades in, bland annat en kritisk kommentar av Institutet för språk och folkminnen, den myndighet i Sverige som bland annat handhar frågor om dialekter, som menade att många andra svenska dialekter också i hög grad skiljer sig från standardsvenska, och om älvdalskan hade talats i Norge, hade den betraktats som en norsk dialekt, och inte som ett särskilt språk. De menade att vissa lingvister som specialiserat sig på älvdalska har ett snävt perspektiv, och tar inte hänsyn till att många av älvdalskans grammatiska drag kan återfinnas såväl i närliggande mål som i mera avlägsna mål, t.ex. i Finland. De menade också att den älvdalska som ofta framhävs är en ålderdomlig form av älvdalska, och att många av älvdalskans karakteristiska drag är på tillbakagång, eller i dag har försvunnit hos yngre talare och talare i medelåldern. De kommenterar att lingvisterna som lämnat in ansökan väljer att ignorera det faktum att många talare av traditionella mål upplever sina mål som skilda språk från svenskan, och att idén att älvdalskan skulle vara viktigare än andra mål är djupt problematisk. +Henrik Rosenkvist, en av lingvisterna som stod bakom ansökan, lämnade in ett svar på Institutet för språk och folkminnens kommentar där han skrev att det faktum att älvdalskan och andra traditionella svenska mål håller på att förlora många av sina karakteristiska drag är ett argument för beskydd och erkännande av dessa mål, och att ingen har påstått att älvdalskan skulle vara viktigare än andra svenska mål. Han efterfrågade i stället en oberoende undersökning baserad på lingvistiska kriterier snarare än traditionella dialektuppdelningar av vilka dialekter som skulle kunna tänkas få språkstatus. +John Helgander, som inte fanns bland de tio lingvister som lämnade in ansökan, kom även med en kommentar där han menade att det finns goda språkvetenskapliga grunder för att erkänna älvdalskan som ett särskilt språk, men att detta synsätt är alltför förenklat. Han menar att man i stället måste inkludera både de andra dalmålen och övriga nordiska varieteter i Sverige, där flera mål skiljer sig så pass mycket från standardspråket att många talare av sådana varieteter anser att även deras mål bör betraktas som egna språk på samma grunder, om denna status tilldelas älvdalskan. Han framhäver även att de stora förenklingar och utjämningar som älvdalskan genomgått har gjort att den i dag inte är lika oförståelig för utomstående som den var förr, och att dessa utjämningar är snarlika dem som även andra dalmål och svenska dialekter har genomgått. Han menar att älvdalskan i dag befinner sig på ett liknande språkligt stadium som många av dessa varieteter, vilka aldrig åtnjutit samma uppmärksamhet från språkvetare, eller pekats ut som egna språk. +Övriga kommentarer lämnades in av Ulum dalska, Språkförsvaret, Älvdalens kommun samt Råðdjärum (det älvdalska språkrådet). +Ansökan avslogs först, men beviljades efter överklagan 27 maj 2016 med tillägg av det engelska namnet Elfdalian. Språkets ISO 639-3-kod är ovd. +2008 sattes tvåspråkiga vägskyltar upp av Älvdalens kommun på svenska och älvdalska. +På 700- och 800-talen genomgick urnordiskan ett antal större ljudförändringar som lade grunden för de moderna nordiska språken. Levander skriver att det troligen fanns vissa dialektala skillnader inom urnordiskan, men eftersom övre Dalarna hade en ytterst liten befolkning före vikingatiden, är denna synpunkt av ringa betydelse. +Området där dalmålen talades var troligen större än det av idag, i synnerhet mot sydost, men krympte senare, mestadels på grund av gruvindustrin i Kopparberg från och med 1200-talet, vilken orsakade stor invandring från resten av Sverige och utlandet, bland annat från Tyskland. Detta påverkade kulturen och språket i detta område, och dialekterna i Bergslagen spreds till Dalarna, varigenom de egentliga dalmålen blev begränsade till Dalarnas nordvästra del. +Levander räknar med att ovansiljanmålen förgrenade sig från fornnordiskan på 1300-talet, då vokalerna började diftongeras. Han ger exempel på kristen terminologi med diftongerade vokaler, såsom praim ’runstav’ (av latinets prīmus) och kraungen ’krona’ (av latinets corōna), och eftersom övre Dalarna inte kristnades förrän i slutet av 1100-talet, måste diftongeringsprocessen ha varit produktiv vid den tiden. Å andra sidan hade de lågtyska lånord som lånades in under 1300-talet inte genomgått processen, såsom skrin och duk. Därför drar han slutsatsen att diftongeringen inte längre var produktiv under 1300-talet, och att dalmålen då stabiliserats. +Levander utgår också ifrån att ovansiljanmålen vid början av 1600-talet existerade som självständiga dialekter och hade i grunden samma utformning som nu, och att dialektförhållandena i övre Dalarna under 1500-talet var väsentligen desamma som i slutet av 1800-talet. +Älvdalskan (och de övriga dalmålen) placeras traditionellt in bland de östnordiska språken, tillsammans med svenska och danska, närmare bestämt bland sveamålen, baserat på ett antal drag som älvdalskan har gemensamt med dessa. Levander tar upp följande: +inget i-omljud i presens i starka verb: älvdalska ig ar ’jag har’, men isländska ég hef. +ovanligt med u-omljud på /a/: älvdalska saks ’sax’ men värmländska (Finnskoga) såks. +växling i ändelsevokalen /i/~/e/ ⟨i ~ e⟩ och /u/~/o/ ⟨o ~ å⟩ beror på vokalbalans, inte vokalharmoni: älvdalska eri ’hare’, men bokke ’backe’ +vokalbrytningen är av östnordisk typ +inskott av /b/ efter långt /mː/ och framför /l/ eller /r/: älvdalska sumbrer ’somrar’ +inga specifikt västnordiska pronomen som västnorska me ’vi’, de ’ni’ och døkk ’er’. +Dock påvisar älvdalskan även vissa västnordiska drag. Dessa västnordiska drag är enligt Levander ålderdomligheter som en gång i tiden varit gemensamma för många dialekter, men som bevarats i de mest konservativa målen öster och väster om Kölen. +/ʉ/ ⟨u⟩ i stället för /u/ ⟨o⟩ i ord som älvdalska buð, men svenska bod, jf. norska bu och isländska búð +övergången /e/ > /i/ > /j/ framför vokal: älvdalska ljå ’lie’ (av fornsvenska. līe, lē < urgermanska. *lewan-), jämför norska ljå och isl. ljár +i-omljud i konjunktiv preteritum: älvdalska wäre men svenska vore. +obruten form för 1:a person singular: älvdalska ig, men svenska jag, jämför norska eg. +formen för pronomenet för 2:a person. plural: älvdalska ið, ir, fornsvenska i(r). +utvecklingen av den urgermanska diftongen /eu/ > /jo/ > /iuo/: älvdalska briuota ’bryta’ är parallell med utvecklingen i östnorska dialekter. +Kunskapen om älvdalskan före 1900-talet är tämligen begränsad. Under 1600- och 1700-talen började det dock komma folkliga dikter med burleskt komiskt innehåll som exempelvis framfördes på bröllop i Stockholm, och några av dessa finns författade på älvdalska. 1621 författade även Uppsalaprofessorn Andreas Johannis Prytz ett studentskådespel om Gustav Vasa till minne av Gustav Vasas befrielsekrig med titeln En Lustigh Commoedia om […] Konung Gustaf Then Första. I detta skådespel har Prytz möjligen tagit hjälp av sin student, som enligt Björklund är identisk med Daniel Buskovius, till att författa fem sidor (870 ord) dialog på älvdalska. +Dessa är de tidigaste daterbara vittnesbörden om älvdalska och vittnar om en äldre älvdalska till form och ordförråd. Exempel på dessa är oapokoperade ord som bådhe ’både’ och iäne ’här’ för nutida älvdalska boð och jän, eller adjektivändelsen -e för maskulinum predikativ såsom unge, fyödde jämfört med feminina och nu allenarådande ung(g), fyödd. Den bestämda formen i dativ plural och dativ femininum anges som -omma respektive -ene, -une i stället för det moderna -um och -in, -un. En dativändelse med slutvokal levde kvar i till exempel åsenmål från tidigt 1900-tal som -ume, respektive -ine, -une och som -umma i äldre evertsbergsmål. Denna slutvokal lever ännu kvar i orsamålet, och går tillbaka på en ändelse motsvarande fornsvenska -umen. Diftongen /ai/ hade ännu inte på 1600-talet utvecklats fullt så långt, utan återges som ⟨ei⟩ eller ⟨äi⟩ ännu i en dikt från 1715, trots att övergången till /ai/ skedde runt sekelskiftet 1700. I samma dikt skrivs även bestämd form dativ som -um, det vill säga som i modern älvdalska. Björklund tar också upp ord som i dagens älvdalska är ålderdomliga, såsom frillij ’särdeles’ (jämför fornsvenska friligha med samma betydelse), frekli ’snällt’, gluspa ’supa’, konjunktionerna dätt ’men’ (som försvann på 1700-talet) och ädh ’att’ (som försvann på 1600-talet). Man finner även åtminstone ett exempel på ett oblikt subjekt, megh ticker ’mig tycker’, som inte finns i yngre texter. +Ovansiljanmålen har därför en unik position bland sveamålen eftersom de inte är ömsesidigt begripliga med svenska, norska eller danska, även om de i stort sett är förståeliga sinsemellan. Levander skriver 1925 att ovansiljanbor brukar sitt modersmål inom Ovansiljan, men går vid besök i till exempel Rättvik eller Malung över till svenska, ett språk som var tvunget att läras in som ett främmande språk. Talare av ovansiljanmål är således tvåspråkiga och kodväxlar aktivt emellan ”dalska” och svenska, men så sent som under 1800-talets andra hälft kunde man ännu träffa på älvdalingar, i synnerhet äldre kvinnor, som hade svårigheter att förstå svenska och knappt kunde yttra en mening på något annat språk än älvdalska. +Under 1900-talet har älvdalskan genomgått en ”utjämning i riksspråklig riktning”, det vill säga att älvdalskan för varje generation närmar sig standardsvenskan mer och mer, vilket har lett till ett mer rikssvensktnära uttal, en förenkling av morfologin (bland annat färre kasus), med mera. Den älvdalskan som beskrevs av till exempel Levander 1909 reflekterar inte den älvdalska som talas av älvdalingar i dag. +Sedan slutet av 1800-talet har älvdalskan genomgått stora förändringar i övergången från ett bondesamhälle till ett industrisamhälle. Piotr Garbacz citerar John Helgander, som menar att detta beror på förändringar i sociala nätverk. I det gamla bondesamhället var det sociala nätverket mycket lokalt, man gifte sig med folk inom socknen och ofta inom samma by. Dock var inte rörligheten låg i Älvdalen, utan det var vanligt att tillfälligt flytta av ekonomiska skäl. Helgander hävdar att rörligheten inte var en faktor som bidrog till språkförändring, vilket också hävdas av Levander. +Efter sekelskiftet började folk från utanför socknen bosätta sig i Dalarna på grund av expansionen av skogsindustrin. Därmed skedde en social förändring och giftermål skedde i större omfattning med inflyttade vilket ledde till ett behov av tvåspråkighet. De flesta inflyttade kom från Värmland, och älvdalingarna började tala svenska när de kommunicerade med folk som inte hade kunskaper i det lokala språket. Helgander föreslår även att svenskans inflytande ökade vid den här tidpunkten, vilket ledde språkliga influenser från svenskan i ordförrådet, men även i andra språkliga domäner. Levander nämner också familjer där älvdalskspråkiga föräldrar talade svenska med sina barn för att öka deras förutsättningar i livet, vilket tyder på att dessa familjer hade genomgått en identitetsförändring där det lokala samhället inte längre var den enda identiteten. +Efter andra världskriget blev Sverige ett högutvecklat land vilket också påverkade norra Dalarna. Många sökte arbete utanför Älvdalen, fler än tidigare flyttade dit och älvdalingar återvände med familjemedlemmar uppvuxna utanför Älvdalen. Denna identitetsförändring ledde till en språkförändring mellan den äldre och den yngre generationen, vilket har påverkat dalmålen i stor mån. +Baserad på Helganders beskrivning om norra Dalarnas samhälls- och kulturförändring gör Garbacz en uppdelning av älvdalskan i tre stadier: +klassisk älvdalska (talad av personer födda före ca 1920) +traditionell älvdalska (talad av personer födda mellan ca 1920 till slutet av 40-talet) +modern älvdalska (talad av personer födda ca 1950 och senare) +Klassisk älvdalska är bland annat den älvdalska (åsenmål) som beskrivs av Levander 1909 och som han kallar någorlunda stabil med avseende på fonologi, morfologi och syntax. Garbacz räknar med att de talare som representerar ett stabilt språk hos Levander föddes under första halvan av 1800-talet, och att inga talare av klassisk älvdalska är födda senare än i början av 1900-talet. Efter denna period av stabilitet kommer perioden som Helgander kallar ”brytningsperioden” och Garbacz antar att talare som representerar denna period föddes under de första två årtiondena av 1900-talet. Under denna period sker fler förändringar i målet än tidigare, bland annat som ett resultat av tvåspråkighet. Dessa förändringar synes vara mer signifikanta eftersom de påverkar formläran i större mån än tidigare. +Levander ger exempel på förändringar i morfologin i åsenmålet hos den yngre generationen, t.ex. en förändring där nominativen undanträngt ackusativen i långstaviga femininer, och där ackusativen undanträngt nominativen i kortstaviga former. Levander ger även andra exempel, där den yngsta generationen ersatt särskilda feminina former av adjektiv med den maskulina formen, och säger litn och druttjin i femininum i stället för det äldre litę och druttję, nominativändelsen -r i vissa adjektiv såsom nyr har försvunnit och blivit ny, och svaga verb med ändelselös presensform såsom sel ’säljer’ och stjil ’skiljer’ ersätts av seler och stjiler. Levander ger också exempel på betydelseglidningar i ordförrådet, och skriver att orden fjäll och krytyr som hos den äldsta generationen betyder ’kulle’ och ’djur’, hos den yngre generationen betyder ’fjäll’ respektive ’kreatur’, och ger exempel på ord från det äldre bondesamhället som fallit bort. +Garbacz refererar även den studie Bengt Åkerberg publicerade 1957, där Åkerbergs första informant född 1867 brukar tre kasus, de två informanterna födda 1898 brukar två (nominativ och dativ) och den yngsta född 1934 endast brukar nominativen. Garbacz drar slutsatsen att åtminstone formläran hos substantiv och adjektiv har genomgått stora förändringar hos den generation som föddes på 1930-talet, jämfört med dem födda runt sekelskiftet. Han antar att denna övergångsperiod börjar omkring år 1920 och slutar efter andra världskriget. Således kallar han målet hos generationen född under denna period för traditionell älvdalska. +Helganders sista period kallar han för ”revolutionen” och representeras av talare födda efter 1950. Deras mål karakteriseras av stora förändringar i älvdalskan, och kallas av Garbacz för modern älvdalska. Garbacz och Johannessen fann att verbkongruensen i modern älvdalska är intakt, medan bara spillror återstår av kasussystemet i substantiv och att kasussystemet bland pronomenen är på väg att upplösas likaså. Däremot finns många syntaktiska drag fortfarande kvar i modern älvdalska. +Älvdalska har existerat i skriven form i flera hundra år. I förstone användes så kallade dalrunor för att rista i trä, t.ex. i skålar och bord. Den sista människa som i rakt nedstigande led lärt sig skriva med runor dog 1939. Det finns emellertid få ristningar som är på älvdalsmål, utan många är skrivna på svenska med ett eller annat älvdalskt ord och med älvdalska influenser. I början av 1900-talet började man skriva mer med papper och penna, och det latinska alfabetet blev vanligare. Dock fanns inga stavningsregler, och var och en skrev efter eget tycke. Man använde sällan, eller aldrig, särskilda bokstäver eller tecken för att symbolisera de ljud som var speciella för älvdalskan, vilket ledde till att de endast blev ungefärligt återgivna. +Eftersom älvdalskan varierar mellan byarna har en enhetlig stavning för alla områden varit svår att skapa. Man har inte heller velat använda dialekten i en viss by som norm för hela skriftspråket, eftersom inget bymål kan anses vara bättre lämpat än något annat. +1986 gav Lars Steensland ut den första älvdalska ordboken för praktiskt bruk, där han använde en ortografi som han själv utvecklat några år tidigare. Han baserade ortografin på språket i byarna runt kyrkbyn, eftersom många bodde där. Han gjorde emellertid vissa avsteg och skapade en någorlunda byneutral älvdalska, med en viss lutning åt språket som talades öster om älven. Denna ortografi använde han sedan också i sin översättning av Johannesevangeliet 1989. +2000 gavs boken Rattjin (”Hunden”) ut. Denna bok använde en ny ortografi som skapats av Bengt Åkerberg och föreningen Ulum dalska. Det var första gången som nya tecken introducerades för att symbolisera älvdalska ljud, med undantag för att teckna nasalljud med en svans som även hade använts på 1980-talet. Denna ortografi, jämte andra ortografier som använde tecken främmande för svenskan, har mötts med kritik på grund av bland annat de tekniska svårigheterna som medföljer främmande bokstäver. +2004 gav Ulum dalska i uppgift till det nybildade älvdalska språkrådet Råðdjärum, att utarbeta en anpassad stavning för älvdalskan. Samtliga medlemmar av rådet hade varsin ingång till lingvistik och till älvdalska. Efter sju månader lades stavningsförslaget fram, som godkändes av Ulum dalska. Sedan dess har Råðdjärums ortografi varit vanligt förekommande i flera älvdalska skrifter, t.ex. i Lisslprinsn, översättningen av Antoine de Saint-Exupérys Lille prinsen. Dock blev inte Råðdjärums ortografi allenarådande, utan de två olika ortografier Steenslands och Åkerbergs fick sällskap av en tredje. +2010 gav Lars Steensland ut en utökad ordbok, och en grammatika av Bengt Åkerberg planerades vid samma tid. Ordbokens ortografi är baserad på Råðdjärums ortografi, med små ändringar som närmade sig Åkerbergs ortografi. På samma sätt baserade Åkerberg grammatikan på sin egen ortografi, med små ändringar som närmade sig Råðdjärums ortografi. Detta var ett medvetet samarbete mellan författarna för att jämka sina ortografier i riktning mot en mer enhetlig stavning. +2021 utkom andra upplagan av Lars Steenslands älvdalska ordbok. I den närmar sig ortografin Råðdjärum ännu mer, så att digraferna som i första upplagan stavades ⟨tş⟩ och ⟨dş⟩ nu stavas ⟨tj⟩ och ⟨dj⟩ i enlighet med Råðdjärums ortografi. Dock finns fortfarande vissa skillnader, bland annat stavningen ⟨ǫ⟩ mot Råðdjärums ⟨ą̊⟩. +Standardiseringen av skriftspråket har senare mötts med viss kritik. Till en början fick Råðdjärums antagna modell för ett enhetligt skriftspråk ett positivt gensvar, men många svängde i frågan. Rut ”Puck” Olsson som skrivit böckerna Mumunes masse och Mier um Masse på älvdalska uttalade sig i en artikel i Språktidningen 2011: ”De, forskarna, säger att vi ska lära barnen skrivreglerna när vi lär barnen läsa och skriva, men älvdalskan har aldrig haft något skriftspråk så det är ju konstruerat.” Hon ger exempel med ordet för golvet som med Råðdjärums ortografi stavas guoveð men vilket på många håll uttalas guäver: ”Golvet, g-u-ä-v-e-r. Men då har forskarna skrivit ett d med ett kryss över svansen, ð, i stället för r, och så ska forskarna vara i skolorna med det där och lära ut till de stackars småungarna, som har fullt upp med att lära sig de vanliga bokstäverna.” Mats Elfqvist, tidigare ordförande i Ulum dalska, säger även han: ”Låt oss skriva som vi vill! Som det är nu vågar ingen skriva på älvdalska i vår egen tidning Dalskum sedan forskarna har bestämt hur det ska vara.” +Helgander belyser problemen med att skapa ett enhetligt skriftspråk för älvdalskan och tar bland annat upp älvdalskans heterogenitet mellan byar och i synnerhet mellan generationer, och han inbegriper här även variationer i grammatiken mellan de olika generationerna, där den äldsta älvdalskan (”klassisk älvdalska”) ofta lyfts fram som normen. Han skriver vidare att förutsättningarna för ett spontant och funktionellt skrivande saknas och att älvdalskan fortfarande i fortsättningen främst kommer att vara ett talat språk och att det påminner om en diglossisituation, där man har en språkvarietet som brukas av folk i allmänhet (lågspråk), och en varietet, som är inskränkt till mycket specifika tillfällen, ett avvikande skriftspråk (högspråk). +Råðdjärum poängterade redan 2005 att deras ortografi inte kan återspegla alla älvdalingars uttal, utan att den är en ”kompromiss mellan olika möjligheter” – precis som att svensktalande uttalar svenska väldigt olika men ändå kan kommunicera i skrift. De framhåller också att ”skapandet av en enhetlig stavning” är ”[väsentligt för] älvdalskans överlevnad”. +Sapir, som bland annat ledde Råðdjärum, har också senare betonat att Råðdjärums stavning inte är ett försök att styra folks uttal, eller att stavningen eftersträvar att radera de skillnader som finns mellan byarna. Han anser snarare att en standardiserad stavning underlättar skriftlig kommunikation och dessutom ger älvdalskan ökad prestige hos talarna.” +Råðdjärums alfabet består av följande 36 bokstäver: +Utöver de bokstäver som förekommer i det svenska alfabetet används även bokstäverna ⟨ą⟩, ⟨ę⟩, ⟨į⟩, ⟨ų⟩, ⟨y̨⟩ och ⟨ą̊⟩ för att beteckna de nasala varianterna av respektive vokaler. Därtill används även bokstaven ⟨ð⟩ för ljudet tonande dental frikativa (som i engelska father). Bokstäverna ⟨c⟩, ⟨q⟩, ⟨x⟩, ⟨z⟩ förekommer enbart i namn och främmande ord. Alla tecken, t.ex. ⟨ǫ⟩ och ⟨ş⟩ från Steensland och Åkerberg, stöds av Unicode, men ⟨Ą̊⟩ och ⟨ą̊⟩ får skapas med Ą eller ą och tecknet "Combining Ring Above" (U+030A) som lägger en ring ovanför valfritt tecken. +Vissa likheter finns mellan alla tre ortografier (Råðdjärums, Åkerbergs och Steenslands). Bland annat använder sig samtliga ortografier av tecknet ⟨ð⟩ för att beteckna en tonande dental frikativa, och av en svans för att beteckna nasalitet. Däremot skiljer sig ortografierna från varandra i hur man ska beteckna ljud som [ʦ̺] och [ʣ̺] samt på andra plan såsom vilket vokalljud som representeras av vilket tecken, med flera. Några skillnader och likheter mellan de tre ortografierna presenteras i tabellen nedan: +Älvdalskans ljudlära utmärker sig genom en rad arkaismer, d.v.s. fornspråkligt bevarade drag, och novationer, d.v.s. nyutvecklade drag. Bland annat har älvdalskan bevarat fornnordiskans stavelsekvantitetssystem med korta, långa och överlånga stavelser. Den har också bevarat ljuden [w] och [ð], så att fornnordiskans vindr ’vind’ (med [w]-uttal), och rauðr ’röd’ motsvaras av wind och roð på älvdalska. Älvdalskan har också som ensam nordisk varietet bevarat fornnordiskans nasala vokaler. +Älvdalskan har också genomgått stora ljudförändringar. Bland förändringarna har bland annat äldre långa vokaler diftongerats, och de primära nordiska diftongerna har monoftongerats eller utvecklats till andra diftonger. Vissa konsonantkluster med /r/ har också assimilerats och /l/ har fallit bort framför flera konsonanter, så att svenskans ’björn’ och ’kalv’ motsvaras av älvdalskans byönn och kåv. Älvdalskan har också genomgått [h]-bortfall, i likhet med t.ex. roslagsmålet i Uppland, så att svenska ’hus’ och ’himmel’ motsvaras av aus respektive imil. +Följande tabell visar några av älvdalskans vokalförändringar sedan fornnordiska och motsvarande ord på svenska: +Älvdalskans konsonantsystem skiljer i några större avseenden från svenskans. Älvdalskan har bevarat de äldre ljuden, halvvokalen [w] respektive frikativan [ð], som står i komplementär distribution med frikativan [v] och klusilen [d], där [w] förekommer framför vokaler inom samma morfem, t.ex. i warg, och [v] förekommer efter vokaler inom samma morfem, t.ex. i kåv. Ibland kan [v] övergå till ett [b] framför [d], som i Övdaln ’Älvdalen’ och lovdag ’lördag’ som då uttalas Öbbdaln och lobbdag. I äldre tider kunde [v] uttalas som en tonande bilabial frikativa, [β] mellan vokaler och stavelsefinalt. Ljudet [ð] förekommer efter vokaler och inte framför konsonant, i övriga fall används [d] och dessa ljud utgör tillsammans fonemet /d/. +Älvdalskan har även ett frikativt uttal av klusilen /g/, nämligen en tonande velar frikativa, [ɣ], som altererar på samma sätt som [d] och [ð], men detta ljud är på väg ur älvdalskan. +Älvdalskan har två affrikator, som i Råðdjärums ortografi skrivs ⟨tj⟩ och ⟨dj⟩, men i Steenslands och Åkerbergs ortografier skrivs ⟨tş⟩ och ⟨dş⟩. Av Steensland betecknas dessa som [tsʲ] och [dzʲ] medan Yair Sapir betecknar dem som dentala apikala affrikator, [ʦ̺] och [ʣ̺]. Av Åkerberg beskrivs de som ”tj-ljud med t‑förslag” respektive ”tonande tj-ljud med d-förslag”. Dessa har uppkommit genom palatalisering av äldre /k/ och /g/ och har till skillnad från svenskan inte uppkommit ur sekvenserna /tj/ och /dj/, vilka har bevarats i språket i ord som tiokk ’tjock’ och diuop ’djup’. +Steensland uppger att kvaliteten hos /s/ skiljer sig från svenskan och artikuleras längre bak och mera apikalt än dess svenska motsvarighet, och uttalet av /r/ är vibrantiskt och apikoalveolart. Älvdalskans lateral /l/ altererar komplementärt mellan tjockt kakuminalt [ɽ] och tunt [l], där det tjocka ljudet används då ljudet är kort och det tunna då ljudet är långt. Detta bruk är dock numera vacklande, och tjockt /l/ ersätts ofta av tunt. Älvdalskans tjocka /l/ kan till skillnad från andra dialekter även förekomma i början av ord, vilket är mycket ovanligt bland dialekter i Sverige. Enligt Gunnar Nyström altererade tjockt /l/ med tunt inom paradigm i vissa bymål väster om älven, där maskulina ord på finalt -l hade tunt, men feminina och neutrala ord hade tjockt. I adjektiv hade maskulinen tunt, men femininen tjockt. Detta har sitt ursprung i en fornspråklig längdskillnad. Denna distinktion kan alltjämt påträffas hos vissa äldre talare i Västermyckeläng och Evertsberg. +Konsonantsystemet har också genomgått en rad skiften. Som nämnt ovan har [h] fallit bort, [r] assimilerats i kluster med /l n s/ så att svenska ’karl’, ’stjärna’ och ’björn’ motsvaras av kall, stienna och byönn, samt att [l] försvunnit framför konsonanterna [g k m p s v], jämför t.ex. svenska ’talg’, ’folk’, ’holme’, ’hjälp’, ’hals’ och ’älv’ med älvdalska tåg, fuok, uome, jåp, ås och öve. Dessa ljudbortfall återfinns redan i Dalalagen. +Konsonantkombinationen /wr/ i början av ord har genomgått metates, så att t.ex. svenskans ’vrida’ och ’vrång’ motsvaras av älvdalska rwaiða och rwaungg. Konsonanterna /g/ och /k/ har genomgått palatalisering (förmjukning) likt i svenskan framför höga främre vokaler, så att fornnordiska gefa ’ge’ och kýr ’ko’ motsvaras av älvdalskans djävå och tjyr. Till skillnad från standardsvenskan har detta även skett inuti ord, så kallad norrländsk förmjukning, vilket gör att palatalisering kan uppträda i böjningar av ord, så att svenska ’ägg, ägget’ och ’dricka, drack, druckit’ i älvdalskan motsvaras av egg, eddjeð och drikka, drokk, druttjeð. Till skillnad från svenskan har detta inte skett framför /e/, så att [k] och [g] kvarstår i ord som kenna och gelda. +Älvdalskan har också bevarat de äldre klusterna ld, nd, mb, rg, gd och ng (med hörbart [g]), som i kweld ’kväll’ och lamb ’lamm’ (av fornnordiska kveld och lamb). Dock har den första konsonanten i dessa kluster blivit förlängd, så att t.ex. kweld uttalas med ett långt [l]. Konsonantkombinationen ⟨sl⟩ har utvecklats till ett tonlöst /l/-ljud, och realiseras som [l̥], oftast följd av ett tonande [l], som i ord som slaik ’sådan’ och kessler 'kittlar, grytor’. +Konsonanterna [r ð] beskrivs av Åkerberg som ”flyktiga”, vilket innebär att de i ordslut elideras, d.v.s. inte uttalas före en efterföljande konsonant, såväl som i fulla satser som i sammansättningar. Om de i stället följs av en vokal eller avslutar ett yttrande uttalas de, t.ex. an i’ frek (< an ir frek) ’han är snäll’ och e’ sir aut bra (< eð sir aut bra) ’det ser bra ut’. Samma förhållande råder för /g/, men endast i orden ig, mig, dig, sig, nog och og, samt i ändelser som -ig, -lig och -ug. +Vissa enstaviga pronomen och adverb på /d/, såsom du och dig samt dą̊ och dar, som går tillbaka på ett äldre fornnordiskt ⟨þ⟩ (þú, þig, þá, þar) uttalas frikativt som [ð] när föregående ord slutar på vokal. I kombination med elisionsreglerna ovan gör det att en mening som ig war dar dą̊ ’jag var där då’, uttalas i’ wa’ ða’ ðą̊. +Älvdalskan har nio orala vokalfonem (monoftonger), som kan förekomma både korta och långa, liksom i svenskan; dessa är ⟨a o u å e i y ä ö⟩. Därtill tillkommer sex diftonger och en triftong; dessa är ⟨ie yö uo ai åy au⟩ samt ⟨iuo⟩. De stigande diftongerna ⟨ie yö uo⟩ samt triftongen ⟨iuo⟩ kan förekomma både långa och korta, medan de tre fallande diftongerna ⟨ai åy au⟩ i regel alltid är långa, eftersom de utvecklats ur långa vokaler. Utöver dessa orala vokaler finns även fem nasala vokaler (monoftonger), fyra nasala diftonger och en nasal triftong; dessa är ⟨ą ǫ~ą̊, ųˌ ę į⟩, samt ⟨įe y̨ö ųo ąi⟩ och ⟨įuo⟩ (se vidare Vokalnasalitet nedan). +I skriften uttalas vokal framför dubbeltecknad konsonant eller framför konsonantgrupp generellt kort (som i kall, buosta), och vokal som står framför enkel konsonant i långstaviga ord uttalas generellt långt (som i kal, buota). I kortstaviga ord uttalas vokalen däremot kort, jämför ripa ’repa’ (lång vokal) och ripå ’rapa’ (kort vokal). Många ord som har lång vokal självständigt, har kort vokal som förled i sammansättningar, t.ex. glasrut ’glasruta’ eller snįuopluog ’snöplog’. +Den exakta realiseringen av vokalerna skiljer sig ofta mellan byarna (se avsnittet Variation ovan). +Älvdalskan har nasala vokaler, vilket innebär att jämte de vanliga orala vokalerna, förekommer även vokaler som uttalas med mjuka gommen (velum) sänkt, så att luft går igenom både näsan och munnen, såsom i franskans bon /bõ/ ’god, bra’ som kontrasterar med beau /bo/ ’vacker’. Den älvdalska nasaliteten uppträder företrädesvis på två sätt i älvdalskan: (1) hos vokaler som följs eller föregås av en nasal konsonant (2) hos vokaler som hade en efterföljande nasal konsonant i ett tidigare språkskede (t.ex. fornnordiska eller urgermanska). Utöver dessa finns ett antal ord där etymologin inte kan härledas till någon av ovanstående, d.v.s. (3) hos vokaler där nasaliteten är mera etymologiskt dunkel. Vissa byar i Älvdalens socken saknar vokalnasalitet, bland annat de sydligaste byarna Garberg och Blyberg. I såväl Råðdjärums som Åkerbergs ortografi markeras vokalnasaliteten med en svans på vokalen: ⟨ą ǫ ų ą̊ ę į y̨⟩. Grafemet ⟨ą̊⟩, ett nasalerat å-ljud, förekommer endast i älvdalskan. Grafemet ⟨y̨⟩ (nasalerat ⟨y⟩) förekommer endast i diftongen y̨ö. +Älvdalskan har nio orala vokaler men endast fem nasala vokaler. Detta beror på bl.a. ett vokalsammanfall av de nasala vokalernas motsvarighet till e och ä samt av o och å, vilket är ett vanligt fenomen bland nasala vokaler i språk. I älvdalskans fall skiljer det sig mellan bymålen vilken vokal som sammanfallet har resulterat i. På östsidan älven utom Åsen har sammanfallet mellan de nasala motsvarigheterna till o och å resulterat i ett nasalt o-ljud, ǫ, medan på västsidan älven (utom Evertsberg i ett äldre skede), Finnmarken och Åsen i stället har ett nasalerat å-ljud, ą̊. I de byar där nasaliteten saknas uttalas vokalen som ett onasalerat o. Detsamma gäller för de nasala motsvarigheterna till vokalerna e och ä, där sammanfallet i de allra flesta byar har resulterat i ett nasalerat e-ljud, ę, men byarna på västsidan älven (utom Väsa) samt Finnmarken har ett nasalerat ä-ljud, ą̈. I Råðdjärums ortografi används ą̊ och ę, medan Åkerberg och Steensland använder ǫ och ę. Dessa vokalsammanfall beskrivs ingående av Nyström. +De nasala motsvarigheterna till monoftongerna ⟨y⟩ och ⟨ö⟩ uppträder endast allofoniskt. +Älvdalskans nedärvda nasala vokaler är av två typer, dels primära nasala vokaler och sekundära nasala vokaler. De primära nasala vokalerna var nasala redan på fornnordiska och uppkom i kontakt med en nasal konsonant på urgermanska eller urnordiska, som föll bort i övergången till fornnordiska. Exempel på dessa är gą̊s (< fn. gás < urger. *gans, jf. ty. Gans) och lą̊s (< fn. láss < urger. *lamsaz, jf. is. löm ’gångjärn’) De sekundära nasala vokalerna har i älvdalskan fallit bort i ett senare skede av språket där svenskan har bevarat den nasala konsonanten, men den har gått förlorad i älvdalskan. Exempel på dessa är wįster ’vänster’, ųosdag ’onsdag’, įesum ’ensam’. Vissa ord har med tiden förlorat sin nasalitet, t.ex. i ’i’ (< urger. *in) och uoss ’oss’ (< urger. *uns), vilket Noreen menar förklaras av att de varit obetonade och genomgått vokalförkortning. Precis som på fornnordiska förekommer minimala par mellan ord med oral respektive nasal vokal, t.ex. ą̊s ’ås’ och ås ’hals’. +Vokalnasalitet har även funnits i andra dialekter i Norden, bl.a. i synnerhet i närliggande dialekter i Orsa och Venjan. I Skattungbyn i Orsa har både primär och sekundär nasalitet förekommit, medan övriga orsamål endast haft nasalt /a/ som på grund av sin nasalitet ofta fick klangen av ett å-ljud. Redan 1918 uppger Boëthius att vokalnasaliteten hos /a/ hade ersatts med ett oralt å-ljud hos yngre talare utom i Skattungbyn. I Orsamålsordboken från 2011 nämns ingenting om nasala vokaler, däremot finns övergången från a till å framför nasal konsonant kvar, vilket markeras i skriften i ordboken. Även venjansmålet hade på Noreens tid nasala vokaler, men dessa var enligt Noreen begränsade till långa vokaler framför nasal konsonant. Även i Norge i Selbu fanns åtminstone tidigare vokalnasalitet av sekundär typ i tre vokaler, men är i dag på utdöende. +Noreen och Steensland nämner att vokalnasalitet framför /s/ är vanligt på älvdalska. Detta kan bero på att älvdalskan ljudlagsenligt förlorat nasal konsonant före just /s/, som i wįster ’vänster’ och ųosdag ’onsdag’. Jämför även detta med t.ex. gą̊s och lą̊s, en process som överlag är mycket vanlig i språk. +Vokaler på älvdalska som följs av nasal konsonant nasaleras automatiskt (med vissa undantag). Både korta och långa vokaler kan vara nasala. I de flesta älvdalska ortografier som markerar nasalitet markeras oftast inte nasalitet framför en nasal konsonant eftersom den då är förutsägbar, såsom i gåmål ’gammal’ eller wind ’vind’. Däremot brukar nasalitet efter en nasal konsonant markeras eftersom den är mera sporadisk och oförutsägbar, såsom i mįer ’mer’, mą̊l ’mål’, mangnįet ’magnet’ eller femą̊ga ’förmåga’. En vokal före en nasal konsonant nasaleras generellt inte: +vid synkront bortfall av konsonant, t.ex. bu’n (< buð) ’boden’ och we’n (< weg) ’vägen’. Vid äldre (medeltida) bortfall (t.ex. assimilering) har vokalen nasalerats: kwęnn ’kvarn’ (< fn. kvern), ą̊m (< fn. hálmr) ’halm’, +vid ”större” morfemgränser, t.ex. sju-mąnn ’sjöman’, stugu-nytjyl ’stugnyckel’, slå-ningg ’slåning’, +om vokalen står i slutet av en obetonad förstavelse: kamįn, tomat, minut, termomįeter, astronomi. +Det finns även ett antal ord med nasal vokal vars nasalitet är oetymologisk. Steensland kallar detta för parasitisk nasalering och föreslår olika förklaringar till denna uppkomst, bl.a. analogi, assimilation och onomatopoetiska ord. +Steensland tar både upp morfologisk analogi, och fonologisk analogi. Morfologisk analogi skulle vara det som Noreen föreslår, att nasaliteten i wįð, wįr ’vi’ beror på analogi med den possessiva formen ųor ’vår’. Lars Levander föreslår också att den dialektala formen įð, įr ’ni’ skulle bero på inverkan från wįð, wįr, en teori som Steensland bedömer trolig. En annan analogibildning är fonologisk morfologi, där nasaliteten beror på påverkan av fonetiskt liknande ord med nasal vokal. Hit härleder Noreen t.ex. tųosdag ’torsdag’ som skulle bero på inverkan från ųosdag ’onsdag’. Steensland avfärdar denna hypotes, men konstaterar att įð och wįð inte bara tillhör samma grammatiska kategori, utan också rimmar på varandra vilket kan orsaka analogibildningen. Steensland tar också upp andra potentiella fall av fonologisk analogibildning. Många av fallen har en sidoform som ej är nasalerad, t.ex. rįesa ’resa’ (jämte riesa) som möjligen skulle kunna ha fått sin nasalering per analogi med rįesa ’rensa’. Noreen nämner också lånord som ką̊l (< latinets caulis) och kęse (< latinets cāseus) som kan blivit nasalerade, och anslutit sig till t.ex. mą̊l och stą̊l, samt många av de andra ord med strukturen /Ṽs/, såsom gręsas ’skratta ilsket’. +Steensland tar också upp att vissa nasaleringar kan bero på nasalassimilation med ett närliggande nasalt ljud. Precis som att den initiala konsonanten i pronomenet ni samt pronomen som me i norskan och my i slaviska språk påverkats av verbändelsen för andra respektive första person plural, kan enligt Steensland nasaliteten i wįð och įð uppkommit på samma sätt, som en form av nasal sandhi. Wįð skulle således kunna ha fått sin nasalering från ändelsen -um, och įð från -in. +Ett flertal onomatopoetiska älvdalska ord verkar också vara nasala, såsom fręsa ’fräsa’, ųosta ’hosta’, pįster ’pipa’, tįeta ’tita’, tiųota ’tjuta’, kwęka ’kväka’, sįster ’syrsa’, węsa ’andas’, pęsa ’flämta’, m.fl. +Till skillnad från svenskan har älvdalskan bevarat fornnordiskans tre stavelsetyper, vilket betyder att det finns kort stavelse, det vill säga kort vokal + kort konsonant (VK); lång stavelse, kort vokal + lång konsonant (VKː); lång vokal + kort konsonant (VːK), eller kort vokal + konsonantkluster. Slutligen finns det även överlång stavelse som är lång vokal + lång konsonant (VːKː). I Råðdjärums och Steenslands ortografi markeras överlång stavelse med en apostrof, ⟨’⟩, före dubbeltecknad konsonant, medan Åkerberg markerar det med ett streck under konsonanten, ⟨tt⟩. Följande stavelsetyper finns således i älvdalskan (efter Sapir): +I älvdalskan är tendensen starkare att första stavelsen i ett ord får huvudbetoning, däremot kan såväl vokaler som konsonanter i bibetonade stavelser fortfarande vara långa, i ord som pagas /ˈpagɑːs/ ’patrask, pack’ (< bagage) och putell /ˈpʉtɛlː/ ’flaska, butelj’. Detta är i synnerhet vanligt efter en kort huvudbetonad stavelse. +Sammansättningsbitrycket skiljer sig också från svenskan. Detta innebär att andra ledet i en sammansättning får bitryck på första stavelsen, vilket är det vanliga i svenskan, endast om denna är lång och inte följs av en lång stavelse, såsom i juolaftun. Annars förskjuts betoningen till nästa stavelse, som i såmårstugu ’sommarstuga’, med bitryck på sista stavelsen eftersom stugu har kort rotstavelse, och grasainklingg ’gräsänkling’, med bitryck på suffixet -lingg, eftersom det har en lång konsonant. +Älvdalskans har akut och grav tonaccent (accent 1 och 2) precis som i svenskan. Däremot finns sammansättningar där första ledet har akut accent men andra ledet tydligt bitryck som i gråbryms ’gråbroms’ och ljåuorv ’lieorv’. +I älvdalskan är det också möjligt för enstaviga ord att ha grav accent (accent 2). Detta sker genom att slutvokalen i ett tvåstavigt ord med grav accent apokoperas, men den enstaviga formen som återstår behåller sin grava accent. Dock kan den grava accentens andra tontopp realiseras på efterföljande ord. T.ex. i frasen an yöpte ’han ropade’ har yöpte grav accent, och i frasen an yöpt it apokoperas slutvokalen, men den grava accenten behålls, och den andra tontoppen realiseras på ordet it i frasen. +I kortstaviga ord realiseras grav accent som en jämviktsaccent, i ord som båkå, som kan ge intryck av en jämn accent över stavelserna, eftersom tonkurvan inte gör någon större rörelse över stavelserna. +Älvdalska uppvisar vokalbalans, vilket enligt Tomas Riad intill nyligen har varit någorlunda produktiv. Vokalbalans innebär att kvaliteten i slutstavelsen bestäms av kvantiteten i rotstavelsen, d.v.s. att vokalen i slutstavelsen varierar beroende på huruvida rotstavelsen är ”tung” eller ”lätt”. Tunga stavelser är de med lång vokal eller vokal + konsonant. Lätta stavelser är öppna stavelser med kort vokal. Stavelsernas vikt har betydelse för ett antal processer i språket, t.ex. verbböjning och apokope. +I älvdalskans fall innebär detta att vissa slutvokaler altererar beroende på rotvokalens vikt, t.ex. altererar -i med -e, -e med -ä och -a med -å. Av förtydligande skäl representerar kolonet (ː) i tabellen nedan en lång vokal. +Enligt Riad beror dessa vokalförändringar på två separata processer, dels vokalreduktion, d.v.s. att ändelsevokalerna till de tunga stavelserna har stått i prosodiskt svaga positioner, vilket har gjort vokalerna mera reducerade och centraliserade. Den andra processen kallar han vokalförstärkning, vilket är att prosodiskt starka stavelser utanför rotstavelsen blir förstärkta, vilket i det här fallet gjort a till å. Detta sker i ord med så kallad jämviktsaccent, vilket är accent 2 såsom den realiseras i ord med lätt rotstavelse. +I de ord med jämviktsaccent förekommer också tilljämning, en sorts vokalharmoni eller vokalassimilation, som innebär att en stavelses kvalitet (i regel rotstavelsen) påverkats av vokalkvaliteten i en intilliggande stavelse, vilket ger ord som spårå (< *sparå) ’spara’, spärär (< *sparär) ’sparar’, nytjyl (< *nykil) ’nyckel’, skuomäkär (< *skuomakär) ’skomakare’, men kasta, kaster ’kastar’, dymbel ’dymmelspik’ och skradder ’skräddare’. +I älvdalskan förekommer olika former av apokope, det vill säga bortfall av slutvokaler. Apokope drabbar endast långstaviga ord som slutar på ⟨a⟩, ⟨e⟩ eller ⟨u⟩, med undantag för vissa kortstaviga infinitiver, och den drabbar aldrig nasala vokaler. Apokope i älvdalska delas av Åkerberg in i två olika kategorier: permanent apokope och tillfällig apokope. Levander och Steensland kallar tillfällig apokope för villkorlig apokope. +Permanent apokope är lexikaliserad och drabbar ett ord i samtliga fall. Detta sker först och främst i sammansättningar, till exempel i skaulkull ’skolflicka’ (av kulla), smą̊kåyt ’småspringa’ (av kåyta), smyörbytt ’smörbytta’. När dessa ord får bestämd artikel behandlas de dock som om de slutade på vokal, t.ex. skaulkullą ’skolflickan’. Permanent apokope drabbar även ord med betoning på första stavelsen som ursprungligen hade tre eller fler stavelser. Detta gäller ord av olika betoningstyper, till exempel suoter ’sotare’ (med lång första stavelse), fundir ’fundera’ och studir ’studera’ (med lång respektive kort första stavelse och sammansättningsbetoning) och även kortstaviga ord som spiläð ’spelade’ eller kuolär ’kolare’. Flerstaviga ord som inte har betoning på första stavelsen följer dock inte reglerna för permanent apokope, t.ex. dividira ’dividera’. +Tillfällig apokope beror på ordets ställning i meningen. Den inträffar när ett flerstavigt långstavigt ord, som slutar på vokalerna ovan, omedelbart följs av ett annat ord. Om ordet följs av paus apokoperas inte ordet. Åkerberg ger som exempel: +An yöpte. (’Han ropade.’) +An yöpt að mig. (’Han ropade på mig.’) +Eð kam įe kulla. (’Det kom en flicka.’) +Eð kam įe kull daitað uoss. (’Det kom en flicka till oss.’) +Dessa regler gäller även för ord med tre eller fler stavelser som betonas på näst sista stavelsen (penultiman), såsom dividira ovan. När orden apokoperas behåller de sin tonaccent, vilket gör att ett enstavigt ord kan ha både akut och grav accent (accent 2) om det senare ordet är apokoperat. Därför har yöpt och kull i exemplen ovan fortfarande grav accent, och en talare av älvdalska är således medveten om när en vokal har uteslutits. +Ovanstående regler för tillfälliga apokope gäller även vanliga äldre förnamn och vissa släktskapsord såsom muna ’mor’ och tytta ’faster’. Förnamnen apokoperas därför i: +Eð war Ann og Lass so kamu. (’Det var Anna och Lasse som kom.’) +Detta gäller oftast dock inte mer moderna förnamn som Lisa eller Elvira. +Det är dessutom vanligt att apokopera namnen vid tilltal; apokoperingen blir då en sorts vokativ. Detta sker även med namn som annars inte apokoperas och namnen får då akut accent. Detta kan även ske i slutet av meningen. +Kom jųot, Lass, og kuogä! (’Kom hit, Lasse, och titta!’) +Är ą̊ Elvir! (’Hör på, Elvira!’) +Förnamn apokoperas alltid när de föregås av ett gårdsnamn, såsom Felt-Lass och Draguns-Ann; ett undantag är Maja som inte apokoperas: Bjärg-Maja. Namn på platser i Älvdalen med omnejd följer även ofta reglerna för apokopering, till exempel Luok och Mųor för Loka och Mora, medan platser längre bort såsom Skåne sällan apokoperas. +Älvdalskans satslära (syntax) och i synnerhet formlära (morfologi) har genomgått stora förändringar under 1900-talet. Många språkforskare skiljer därför mellan tre huvudsakliga varieteter av älvdalska. Dels klassisk älvdalska, talad av älvdalingar födda före 1920, vilket är den varietet som beskrivs i Lars Levanders avhandling, en av de mest omfattande beskrivningarna av älvdalskans ordböjning och satslära, dels traditionell älvdalska som talas av älvdalingar födda mellan 1920 och 1950, samt modern älvdalska som talas av älvdalingar födda efter 1950. +Skillnaden i formlära skiljer sig mycket mellan dessa varieteter och de flesta beskrivningar av älvdalskans formlära, såväl Levanders avhandling som Åkerbergs grammatika, behandlar klassisk älvdalska, medan älvdalingar i dag talar traditionell eller modern älvdalska. +Vissa studier av t.ex. Peter Svenonius och Piotr Garbacz behandlar främst traditionell älvdalska och dess skillnader gentemot klassisk älvdalska, medan t.ex. Garbacz & Johannessen i en studie jämför äldre formlära och syntax med modern älvdalska. +Älvdalska skiljer likt fornsvenskan mellan tre genus, maskulinum, femininum och neutrum, och varje substantiv har ett av dessa genus. Till exempel är substantiv som warg ’varg’, kall ’man’ och est ’häst’ maskulina, medan suol ’sol’, buok ’bok’ och bru ’bro’ är feminina och substantiv som aus ’hus’ och buord ’bord’ är neutrala. +I början av 1900-talet hade älvdalskan ett trekasussystem, där man skilde mellan nominativ, ackusativ och dativ. Levander tar i sin avhandling även med genitiv i sina uppställningar, men endast för substantiv i bestämd form, andra forskare hävdar dock att klassisk älvdalska hade ett trekasussystem, och räknar därmed inte med genitiven. +Levander noterar att skillnaden mellan nominativ och ackusativ hos substantiven hade börjat luckras upp i de större samhällena i Älvdalen redan på 1920-talet, och redan var försvunnen i närliggande områden. Han visar också på en förlorad distinktion hos unga i den mindre byn Åsen och ger som exempel en talare född 1909 och dennes bekanta som inte gör någon skillnad, men vars tio äldre syskon upprätthåller skillnaden. Svenonius skriver att en nominativ/ackusativ-distinktion kan i dag i bästa fall anses ålderdomlig, även hos de mest konservativa talarna, och Steenslands ordbok listar dativformer, men inga separata ackusativformer. Vissa kunniga äldre talare känner till ackusativformerna och anser dem ibland vara korrekt i ackusativa sammanhang, men Svenonius skriver att han inte kunnat elicitera en distinktion i spontant tal. +Klassisk älvdalska hade även en distinkt dativform i obestämd form som möjligen har gått förlorad. I klassisk älvdalska markerades denna hos starka maskuliner med ett e-suffix, så att dativformen av kall ’man’ är kalle. Dock kan vokalen utelämnas på grund av älvdalskans apokopering vilket kan ha bidragit till dess försvinnande. Det ska dock nämnas att dativformen får accent 2 (grav accent) som bibehålls även vid apokopering, trots att ordet är enstavigt, och Svenonius noterar att det är möjligt att den inte är helt försvunnen från traditionell älvdalska utan kan finnas bevarad i ton, vilket inte har utforskats. +Den äldre distinktionen mellan warger ’vargar’ och wargär ’vargarna’ har också mestadels gått förlorad förutom hos vissa talare av traditionell älvdalska i vissa byar, som i Brunnsberg. +Skillnaderna mellan klassisk och traditionell älvdalska kan således sammanfattas i tabellerna till höger där skillnaderna i övergången till traditionell älvdalska har markerats med en grå ruta i tabellen. +Garbacz och Johannessen undersökte substantivens formlära i modern älvdalska och fann att endast spillror finns kvar av kasussystemet hos substantiven, och dativen förekommer endast sporadiskt. Helgander undersökte även användandet av kasus mellan tre olika talare från tre olika generationer och fann att talarna födda 1937 och 1984 inte använde dativformer, men däremot fanns dativen kvar hos den äldsta talaren född 1914. +Adjektiv böjs likt substantiv efter genus, numerus och kasus, till exempel ien duktin kall ’en duktig karl’ men įe duktig kelingg ’en duktig kvinna’ och iet duktit aus ’ett duktigt hus’. En fullständig böjning för adjektivet stur ’stor’ i klassisk älvdalska kan ses i tabellen till höger, som beskriven i Åkerbergs grammatik, i stort sett identisk med beskrivningen i Levanders avhandling. Älvdalska adjektiv bildar dock ofta sammansättningar med det substantiv det beskriver, vilket gör att de böjda adjektiven förekommer relativt sällan. Av denna anledning har inte Svenonius gett en uppställning av adjektivens böjning i traditionell älvdalska, eftersom de sällan förekom osammansatta i materialet. I stället förekommer uttryck som finkullur ’fina flickor’ och kroppknaiv ’böjd kniv’. De former som förekom i materialet visade på böjning efter genus och numerus och ibland en dativ plural, men de andra ändelserna var ovanliga, och ändelsen för maskulinum ackusativ användes för andra former än den ursprungliga ackusativen. +Formerna inom parentes i tabellen används när adjektivet står självständigt eller i predikativ ställning, och de andra när adjektivet står attributivt, t.ex. flier stur kaller ’flera stora karlar’ men kallär war sturer ’karlarna var stora’. Ändelsen -ų för neutrum plural ersätts oftast av -er i predikativ ställning men inte i attributiv, och man säger därför bjärrę irå blåer ’bergen är blåa’ men blåų bjärr ’blåa berg’. +I bestämd form bildar adjektiven en sammansättning med efterföljande substantiv, likt många andra nordiska dialekter, men olikt andra nordiska standardspråk: guolbuotję ’den gula boken’, sturkartaundję ’den stora kartongen’. Om adjektiven används substantiverade, d.v.s. självständiga i bestämd form, fogar man substantivändelser till adjektivet, t.ex. roðn ’den röde’, gambeln ’den gamle’, Liuotn ’den onde, djävulen’, Lonn’n ’den ludne, björnen’. +Adjektiven kompareras i komparativ och superlativ med ändelserna -er(a) och -est, t.ex. stark, starkera, starkest ’stark’. Komparativformen kan även apokoperas, men den blir då identisk med den självständiga pluralformen starker. Ett antal adjektiv har en oregelbunden komperation, t.ex. stur, styörrera, styöst ’stor’ eller bra, wildera/bettera, wildest/best ’bra’. +Ett antal adjektiv kan styra dativ, till exempel laik ’lik’, iwari ’varse’ eller uoni ’van’ som i dugum it werd uoni plågum ’vi kan inte bli vana vid plågorna’. +Det älvdalska verbsystemet liknar i stort det svenska, men skiljer sig i att verben fortfarande personböjs efter första, andra och tredje person plural. Likt svenska finns det både svaga och starka verb, och de starka verben karakteriseras av en avsaknad av presensändelse (t.ex. skain ’skiner’ eller foll ’faller’). Både starka och svaga verb får lite olika ändelser beroende på stavelsens längd, t.ex. böjs dalska ’tala älvdalska’, ett långstavigt svagt verb, som dalsker i presens och dalskað i supinum, medan båkå ’baka’, ett kortstavigt svagt verb, böjs som bäkär i presens, med en vokalförändring i stammen, och som båkåð i supinum. Liknande skillnader finns i starka verb; kortstaviga starka verb får en supinumändelse på -ið, t.ex. bitið ’bitit’ och långstaviga tar ett -eð, t.ex. bundeð ’bundit’. I första och andra person plural utelämnas pronomenet, och verbets unika ändelse visar vilken person det rör sig om, således betyder dalskum och bakið ’vi talar dalmål’ respektive ’ni bakar’. När pronomenet följer verbet utelämnas pronomenet i andra person, men inte i första person, t.ex. kamum wįð? ’kom vi?’, men kamið? ’kom ni?’. +Konjunktiv brukas inte i älvdalskan, men två konjunktivformer finns bevarade: edde ’hade’ (i konditionalsatser) och wäre ’vore’. Verbet go ’gå’ har en särskild imperativform gokk, jämför äldre svenska ’gack’. +Likt andra germanska språk med kasus kan vissa verb styra antingen ackusativ eller dativ. När ett verb kräver två objekt, ett indirekt och ett direkt objekt, står det indirekta objektet i dativ och det direkta i ackusativ: Ulåv djäv kullun (dativ) byökär (ackusativ) ’Olov ger flickan böckerna’. När ett verb enbart kräver ett objekt står det objektet för det mesta i ackusativ, men det finns några verb som styr dativ, t.ex. jåpa ’hjälpa’. Gunnar Nyström uppskattar antalet till cirka 300. +Presensparticip kan utöver att uttrycka något samtidigt pågående även användas för att uttrycka något som är möjligt, t.ex. jätend ’ätbar’ (möjlig att äta) eller warend ’möjlig att vara’. +Till skillnad från kasussystemet är verbböjningssystemet mer stabilt i älvdalskan. Helgander jämförde språkdrag hos tre informanter från tre generationer, och i hans studie använde sig talarna från mellangenerationen (född 1937) och den yngsta generationen (född 1984) inte av kasus, men däremot av personböjda verb. Garbacz och Johannessen drar också slutsatsen att verbsystemet i älvdalskan är robust, och att modern älvdalska skiljer mellan person och numerus i verben. +I älvdalskan finns prepositioner som likt tyskan och isländskan styr antingen ackusativ eller dativ, och prepositioner som kan styra både ackusativ och dativ beroende på sammanhang. Exempel på prepositioner som styr ackusativ är brieðwið ’bredvid’, gainum ’genom’, ringgum ’kring’, um ’om’, wið ’vid’, m.fl. Exempel på prepositioner som styr dativ är að ’åt’, frą̊ ’från’, millą̊ ’mellan’, mųota ’mot’, nest ’hos’, yr ’ur’, åv ’av’, m.fl. Exempel på prepositioner som kan styra båda är etter ’efter’, firi ’före, framför’, för ’för’, i ’i’, min ’med’, ą̊ ’på’, under ’under’, yvyr ’över’. +Prepositionen et styrde förr genitiv och är nu endast halvproduktiv i genitivkonstruktioner som i et skuogs ’till skogs’, et sainggs ’till sängs’, et Brunnsbjärrs ’till Brunnsberg’. +Älvdalskan föredrar ofta sammansatta prepositioner före enkla prepositioner, vilket är en sammansättning mellan ett adverb och en preposition. Dessa sammansättningar bildar totalt runt 200 prepositioner och valet mellan enkel och sammansatt preposition styrs av komplicerade regler. Exempel på dessa sammansatta prepositioner är auti ’ut(e) i’, uppi ’upp(e) i’, daiti ’bort(a) i’, nordi ’nord i’, m.fl. +I konkreta sammanhang vid riktning och befintlighet är det obligatoriskt med en sammansatt preposition och satserna kan inte formuleras utan nyanseringen i prepositionens förled som i mass ligg uppi saindjin ’katten ligger (uppe) i sängen’, där det är ogrammatiskt med endast den enkla prepositionen i. Sammansättningsbenägenheten är olika stark hos olika prepositioner. Några står nästan alltid tillsammans med ett förled, och andra mer sällan, och det skiljer sig också från sammanhang till sammanhang. +I klassisk älvdalska, liksom isländska och delvis färöiska, fanns det fornnordiska systemet där räkneorden 1–4 böjdes efter kasus och genus. I modern älvdalska finns inte detta system kvar. Vid uppräkning, vid till exempel ett telefonnummer eller klockslag, används oftast twå, tri och fyra i stället för twer, trair och fiuorer, former som också används av många i alla sammanhang och som därmed ersatt det klassiska systemet. +I tabellen till höger där parentesformer existerar, används de längre formerna självständigt, det vill säga fiuor biler kamu (fyra bilar kom), men fiuorer kamu (fyra kom). +Nyström, citerad av Helgander, noterade att äldre, och vissa yngre, vid den tiden gjorde skillnad mellan de tre genusformerna för två, medan de flesta unga generaliserat till maskulinformen för alla tre, och att ackusativformerna och dativformerna verkade bli ”alltmer inskränkt[a]”. Så länge användes twå mest i uttryck som twå tusn och klukką twå (”två tusen” respektive ”klockan två”). Helgander jämför även tre olika talare från tre olika generationer, födda 1914, 1937 och 1984, där den äldsta använder formen twer tillsammans med ett neutralt substantiv, medan den född 1937 övergått till twå och den yngsta övergått från [w] till [v] och använder två som i riksspråket. +Traditionellt användes former som ien-og-tiugu för ”tjugoen”, likt tyska och danska, men i dag används formerna tiuguien eller tjuguien. +Älvdalsk satslära beskrevs redan av Levanders avhandling, men har också beskrivits i ett antal artiklar, och varit föremål för en doktorsavhandling som behandlar ordföljden i traditionell älvdalska jämfört med klassisk älvdalska. Garbacz och Johannessen jämför syntaxen i modern älvdalska med den i traditionell och klassisk älvdalska och finner att vissa äldre syntaktiska drag i klassisk älvdalska har gått förlorade, medan vissa älvdalska syntaktiska innovationer fortfarande finns kvar, såsom referentiella nollsubjekt, subjektsdubblering, negationsdubblering och avsaknaden av objektsskifte (object shift). +Possessiva pronomen följer vanligtvis sitt huvudord, till skillnad från svenskan där de oftast föregår dem, som i buotję mąi ’min bok’. Om pronomenet betonas föregår det i stället huvudordet, som i mąi buok ’min bok’ (betonat), likt norska och isländska. +Ägandekonstruktioner kan konstrueras på tre sätt, dels kan ägaren med ett ägandesuffix föregå det ägda, som i Lasses buord ’Lasses bord’, dels kan det uttryckas perifrastiskt med prepositionen að: Buordeð að Lasse, bokstavligen ’bordet åt Lasse’, eller kan ägaren placeras efter huvudordet: buordeð Lasse. Om ägaren är efterställd står den i dativ: peninggpundjin kallem ’mannens penningpung’. +Bestämd form används ofta i älvdalskan i andra sammanhang än i svenskan, ett fenomen som älvdalskan delar med många andra nordnordiska dialekter. Obestämd form används endast i några fall, t.ex. om ordet föregås av en obestämd artikel, vid yrkes- och släktskapsord, i vissa prepositionsfraser, i vissa fasta uttryck m.fl., men generellt sett används i stället bestämd form: djär upp jäldn! ’gör upp eld!’, ir eð bar abuorrn dar? ’finns det bara abborre där?’, eð ir grannweðreð i dag ’det är fint väder i dag’. +Garbacz grupperar den traditionella älvdalskans syntaktiska drag i fyra grupper: (1) drag älvdalskan delar med alla nordiska språk, (2) drag älvdalskan delar med andra fastlandsskandinaviska språk, såsom svenska och norska, (3) drag älvdalskan delar med önordiska språk, d.v.s. isländskan och färöiskan, samt (4) unika drag för traditionell älvdalska. +Den första gruppen innefattar drag som V2-ordföljd, verb–objekt-ordföljd i verbfrasen, possessiva reflexiver (sin, sitt sina), m.fl. Den andra gruppen innefattar drag som obligatoriskt formellt subjekt i väderuttryck, såsom nų far eð raingen ’nu börjar det regna’, där det formella subjektet det kan utelämnas i önordiska språk. Älvdalskan saknar även oblika subjekt som mig drömmer, som är standard på isländska. +Vissa andra drag delar älvdalskan med önordiska språk, men inte med fastlandsskandinaviska språk. Nedan följer två exempel på detta. Ett av dem är V-till-I-flyttning, d.v.s. att det finita verbet kan föregå adverbet, se exempel (1) nedan. Detta existerade i fastlandsskandinaviska språk under medeltiden, men finns inte i dag, däremot finns det på isländska. I klassisk älvdalska var denna flytt troligen obligatorisk, men är valfri i traditionell älvdalska. +Ett annat drag som älvdalskan delar med de önordiska språken är att den inte tillåter flytt av hela verbfraser till början av en sats, vilket generellt accepteras i svenskan. Exempel (2) nedan är ogrammatiskt på älvdalska, vilket indikeras av en asterisk. +Som nämnt ovan finns ett antal syntaktiska innovationer i älvdalskan, d.v.s. drag som inte finns i några andra nordiska språk, och vissa av dessa är ovanliga i germanska språk överlag. Fyra exempel på detta är referentiella nollsubjekt, subjektsdubblering, negationsdubblering och avsaknaden av objektsskifte (object shift). +Referentiella nollsubjekt syftar på att älvdalskan, som beskrivet i avsnittet om formläran, utelämnar subjektet i första och andra person plural, så att blott kamum betyder ’vi kom’, och kamið betyder ’ni kom’. +Traditionell älvdalska kan även dubblera subjektet i en sats, vilket generellt inte återfinns i andra önordiska språk, åtminstone inte med samma funktion. I älvdalskans fall kommer alltid det första subjektet i början av satsen, och det andra på vedertagen subjektsposition men föregås alltid av ett adverb som speglar talarens attityd. Data från Levanders avhandling antyder att även trippla subjekt var möjliga i klassisk älvdalska. +Älvdalskan har även negationsdubblering, vilket syftar på att negationen inte följs av andra negerade uttryck, utan att dessa blir affirmativa uttryck, såsom i svenskan. +Till sist saknar även älvdalskan objektsskifte i pronomen, d.v.s. möjligheten för ett pronomen att flytta till positionen före adverbet i satsen. Detta finns i alla andra nordiska standardspråk och även i fornnordiskan, och är därför troligen en innovation som älvdalskan delar med dialekter av finlandssvenska och några danska dialekter. Svenskan tillåter objektsskifte och man kan säga såväl han såg inte mig som han såg mig inte. På älvdalska är däremot endast den första möjlig och satsen an såg mig it är ogrammatisk. +I utvecklingen från klassisk älvdalska har många äldre syntaktiska drag försvunnit, även om många också bevarats. De som har försvunnit är generellt drag som även funnits i äldre stadier av svenskan, och speglar utvecklingen från fornsvenska till nusvenska, medan de drag som är älvdalska innovationer generellt bevarats. Exempel på de syntaktiska drag som fanns i klassisk älvdalska som den beskrivs av Levander men som inte bevarats i traditionell älvdalska är exempelvis objekt–verb-ordföljd och kilkonstruktionen. +Objekt–verb-ordföljd finns belagt i klassisk älvdalska med pronomenobjekt och innebär att objektet föregår verbet i en sats. +Kilkonstruktionen innebär för klassisk älvdalska att verbet ofta står sist i korta relativsatser, men detta har inte bevarats i traditionell älvdalska. +De äldsta beläggen på skriven älvdalska är skrivna med dalrunor, som ristades på föremål av trä, såsom skålar, dryckeskärl eller i stockar till eldhus och lador, ibland järn av någon runkunnig smed. Dessa inskrifter kunde ofta ha ett tillfälligt ändamål där ristarna har berättat om arbetet de utförde medan de snidade, eller fått lust att visa prov på sin runkunskap och snidat strofer ur en psalm. Dessa ristningar var ofta på svenska, eller svenska blandat med mål. Annars hade de ett praktiskt syfte och man ristade in äganderätten till skiften i byn på revstickor eller fäste barnens namn och födelseår på kasspinnar. De äldsta bevarade ristningarna på älvdalsmål är från början av 1600-talet och dalrunorna brukades fram till 1800-talet. +Litteratur skriven på älvdalska har ökat i modern tid, men börjar redan på 1600-talet med Anders Johannis Prytz’ En Lustigh Comoedia om […] Konung Gustaf Then Första från 1622, med fem sidors dialog på älvdalska, troligen författade av Daniel Buskovius. Efter det följde olika gratulations- och bröllopsdikter, t.ex. en bröllopsdikt på hexameter av Samuel Petri Elfving (1668), brorson till Buskovius; några rader ur en bröllopsdikt (1678); lycköskningsdikten Geäsle-Kall up i daalen (1683), troligen skriven av Johannes Laurentii Elvius, brorson till Elfving, från avhandlingen Pastor in Parnasso av Jacob Svedelius; ytterligare en bröllopsdikt av Elvius (1685), en bröllopsdikt av Ands Andsunä (1688). Från slutet av 1600-talet finns även Johan Eenbergs försök att översätta Lukasevangeliet till älvdalska, orsamål och moramål. +Från 1700-talet återfinns Jacob Svedelius’ bröllopsdikt Gilliare-Snack (1715) och en gratulationsdikt (1716); dikten Ad Prost- Ulof i Lieksand (1739) av Reinhold Näsman till en avhandling av Olof Siljeström. Med utgång från sina översättningsstudier författade Johan Eenberg senare Kort Berättelse om Dahlska Språkets Egenskaper, vilket är vår första dialektavhandling, som dock aldrig trycktes. Trettio år senare kom den första tryckta avhandlingen Historiola Lingvæ Dalekarlicæ (1733) (En liten dalsk språkhistoria) med jämförelser mellan ovansiljanmål inbördes och språk som isländska och gotiska, vilket enligt Adolf Noreen är ”högeligen bristfälligt” och ”föga upplysande”. +Bland större verk från 1800-talet återfinns avhandlingarna Conspectus lexici lingvæ dalekarlicæ (1813) av Eric Michael Fant, vilken enligt Noreen innehåller ”en mängd betänkliga fel som gör att den icke kan utan den största varsamhet ock sträng kontroll begagnas” och han säger detsamma om Conspectus grammatices linguæ dalekarlicæ (1818) av Erik Gustaf Geijer, men nämner att den är av intresse tack vare sina syntaktiska anmärkningar. Från 1800-talet finns även vissa lexikon och skrifter om visor, folksägner, folklynne m.m. som innehåller vissa dalord. +Den moderna forskningen om älvdalska börjar med Adolf Noreen och Carl Säve, följd av Lars Levander. +I slutet av 1900-talet började ny litteratur på älvdalska skrivas, och den första boken skriven på älvdalska är Hjalmar Larssons Kunundsin kumb ’Kungen kommer’ från 1985. 1987 kom de två barnböckerna Mumunes masse (Mormors katt) och Mier um Masse (Mer om Masse) och Lars Steensland har översatt Johannesevangeliet (Juanneswaundsjilą, 1989), Markusevangeliet (Markuswaundşilą, 2013), Lukasevangeliet (Lukaswaundşilą, 2015) och Matteusevangeliet (Matiaswaundşilą, 2017). År 2000 översattes Kerstin Ekmans Hunden till älvdalska (Rattşin) av Bengt Åkerberg. Björn Rehnström har även skrivit barnböckerna Rodh, ien brindkåv fro Övdalim (1999), och Byönn-Byöker (2000), tio pixiböcker på älvdalska, samt Trair byönner fro Övdalim (2006). År 2007 översattes Lille prinsen till älvdalska (Lisslprinsn) av Bo Westling. +2012 hittades en dagbok på Näset skriven av porfyrsliparen Frost Anders Andersson med flera hundra sidor skrivna på älvdalska från mitten av 1940-talet. Att skriva på älvdalska var mycket ovanligt på den tiden; många tyckte det var omöjligt att skriva på älvdalska och det råder brist på autentiska texter. Språket är ålderdomligt med nutida ögon men har enligt Bengt Åkerberg ”ett vardagsnära, enkelt och begripligt innehåll [som] återges med ett välutvecklat, grammatiskt korrekt språk. Framställningen har dessutom stilistiska kvaliteter, som tyder på ett medvetet uppsåt att ibland skriva drastiskt och uttrycksfullt.” Utöver dagboksanteckningar och skildringar av Frost Anders’ upplevelser, innehåller texterna även hans egna verser, dikter, karikatyrer och teckningar. Dagboken publicerades 5 augusti 2017. +Runinskrift på Lillhärdalstolen (ca 1600) +Ett av de äldsta skriftliga beläggen av älvdalska. +Prytz’ En Lustigh Comoedia om Konung Gustaf Then Första (1622) +En av de första replikerna på älvdalska i Prytz’ pjäs. +Björnen vid slåtterboden (1917) +Utdrag ur Lars Levanders uppteckning av Erkols Anna Olsdotter från Åsen (född 1859) på fonograf från 1917. Textversion och översättning av Bengt Åkerberg, dock med Råðdjärums ortografi av tekniska skäl. +Frost Anders dagbok (1946) +Utdrag ur Frost Anders dagbok 16 oktober 1946, med översättning ur Åkerbergs utgåva 2017. +Elfdalens hembygdsförening +Ulum dalska (Föreningen för älvdalskans bevarande) +Björklund, Stig (1956), Älvdalsmålet i Andreas Johannis Prytz’ Comoedia om konung Gustaf then Första 1622, Uppsala, http://www.sprakochfolkminnen.se/download/18.5850f85e15732ead0b3569/1474056509815/Svenska+landsm%C3%A5l+och+Svenskt+folkliv_1956_bilaga.pdf +Björklund, Stig (1958), ”Gammalt och nytt i älvdalsmålet”, Älvdalens sockens historia D. 5: 149–193 +Boëthius, Johannes (1918), Orsamålet. 1, Ljudlära., Uppsala +Committee of Experts of the European Charter for Regional or Minority Languages (2020), Seventh evaluation report on Sweden, Europarådet, https://rm.coe.int/swedenecrml7-en3c-pdf/16809fbaab +Dahl, Östen; Koptjevskaja-Tamm, Maria (2006), ”The resilient dative and other remarkable cases in Scandinavian vernaculars”, Sprachtypologie und Universalienforschung (STUF), "59(1)", s. 56–75, http://www.intercult.su.se/publications/Dahl_&_Koptjevskaja-Tamm_2006.pdf +Garbacz, Piotr (2008), Älvdalska – ett mindre känt nordiskt språk, Oslo universitet, http://www.hf.uio.no/iln/om/organisasjon/tekstlab/aktuelt/arrangementer/arkiv/alvdalska.pdf +Garbacz, Piotr (2010) (på engelska), Word order in Övdalian: A study in variation and change, Lund: Språk- och litteraturcentrum, Lunds universitet, ISBN 978-91-628-8068-2, http://lup.lub.lu.se/luur/download?func=downloadFile&recordOId=1567392&fileOId=1567413 +Garbacz, Piotr; Johannessen, Janne Bondi (2015), ”Övdalian from 1909 to 2009”, i Bentzen, Kristine; Rosenkvist, Henrik; Johannessen, Janne Bondi, Studies in Övdalian morphology and syntax: New research on a lesser known Scandinavian language, Linguistik aktuell 221, Amsterdam: John Benjamins Publishing Co, s. 11–46, ISBN 9789027257048, https://benjamins.com/#catalog/books/la.221/fulltext +Helgander, John (1996), Mobilitet och språkförändring: exemplet Övre Dalarna och det vidare perspektivet., Falun: Högsk. Dalarna +Helgander, John (2005), ”Älvdalsmål i förändring – några reflektioner kring en fallstudie”, Fuost konferensn um övdalskų, http://uu.diva-portal.org/smash/get/diva2:431173/FULLTEXT03.pdf +Helgander, John (2013), ”»Det avvikande skriftspråket» – dödar eller stöttar det älvdalskan?”, Svenska landsmål och svenskt folkliv, arkiverad från ursprungsadressen den 2017-01-01, https://web.archive.org/web/20170101000946/http://www.kgaa.nu/upload/tidskrifter/3_2013.pdf#page=29, läst 31 december 2016 +Larsson, Gösta; Welin, Bengt & Welin, Ulla (2008), ”Hur många talar älvdalska idag?”, Föredrag vid Oðer råðstemną um övdalskų +Levander, Lars (1909a), Älvdalsmålet i Dalarna: ordböjning ock syntax, Uppsala universitet, http://www.sprakochfolkminnen.se/download/18.5850f85e15732ead0b3334/1474043788133/Svenska+landsm%C3%A5l+och+Svenskt+folkliv_1909_h2.pdf +Levander, Lars (1909b), ”I vad mån kan ett bymål kallas enhetligt? (Undersökning rörande Åsenmålet i Älvdalen).”, i Hesselman, Bengt, Språk och stil. Tidskrift för nysvensk språkforskning., "12", Uppsala +Levander, Lars (1925), Dalmålet. Beskrivning och historia., "1", Uppsala +Levander, Lars (1928), Dalmålet. Beskrivning och historia., "2", Uppsala +Noreen, Adolf (1881), ”Dalmålet I. Inledning till Dalmålet.”, Svenska landsmål och svenskt folkliv IV: 1–23, http://www.sprakochfolkminnen.se/download/18.5850f85e15732ead0b32ac/1474036182797/Svenska+landsm%C3%A5l+och+Svenskt+folkliv_1881_d.pdf +Noreen, Adolf (1883), ”En lustigh Comoedia om Konung Gustaf then Första af Andreas Johannis Prytz, tredje gången upplagd. Med ett tillägg om de folkliga beståndsdelarne i det svenska skoldramat. Öfversättning af dalmålet.”, Svenska landsmål och svenskt folkliv Bihang 1 I: 69–76, http://www.sprakochfolkminnen.se/download/18.5850f85e15732ead0b32b1/1474036187530/Svenska+landsm%C3%A5l+och+Svenskt+folkliv_1882_c.pdf#page=71 +Noreen, Adolf (1886), ”De nordiska språkens nasalerade vokaler”, Arkiv för nordisk filologi. III: 1–41, https://runeberg.org/anf/1886/0005.html +Nyström, Gunnar (1964), Ett prov på Karlsarvsmålet i Älvdalen. Utskrift och översättning av bandinspelning, jämte kommentar. (Opublicerad uppsats för nordiska proseminariet), Uppsala universitet +Nyström, Gunnar (1982), ”Om maskulina substantiv på -l i Älvdalsmålet.”, Svenska landsmål och svenskt folkliv 105: 52–77, http://www.sprakochfolkminnen.se/download/18.5850f85e15732ead0b358c/1474126375341/Svenska+landsmål+och+Svenskt+folkliv_1982.pdf#page=54 +Nyström, Gunnar (1995), ”Två fonemsammanfall i dalmål - och två ordstudier”, Svenska landsmål och svenskt folkliv 118: 227–237 +Nyström, Gunnar (2015), ”Nya rön om älvdalskans adjektivmorfologi”, Föredrag vid Trið råðstemną um övdalskų, http://video.ku.dk/elvdalsk-nya-ron-om-alvdalskans +Olander, Eva (2011), ”Orsamålet, älvdalskans grannspråk”, Oðer råðstemną um övdalskų, http://uu.diva-portal.org/smash/get/diva2:431171/FULLTEXT02 +Rehnström, Björn (2008), ”Älvdalskans Higgs-partikel”, Oðer råðstemną um övdalskų, arkiverad från ursprungsadressen den 2016-03-09, https://web.archive.org/web/20160309191946/http://www2.valentin.uu.se/information/Elfdalian_radstemna_program.pdf, läst 30 december 2016 +Riad, Tomas (2005), ”Balans och harmoni i älvdalsmål”, Fuost konferensn um övdalskų, http://uu.diva-portal.org/smash/get/diva2:431173/FULLTEXT03.pdf +Rosenkvist, Henrik (2007), ”Subject doubling in Övdalian”, Working Papers in Scandinavian Syntax 80: 77–102 +Rosenkvist, Henrik (2010), ”Null referential subjects in Övdalian”, Nordic Journal of Linguistics 33: 231–267, doi:10.1017/S033258651000020X, ISSN 0332-5865 +Rosenkvist, Henrik (2015), ”The syntax and meaning of subject doubling in Övdalian”, i Bentzen, Kristine; Rosenkvist, Henrik; Johannessen, Janne Bondi, Studies in Övdalian morphology and syntax: New research on a lesser known Scandinavian language, Linguistik aktuell 221, Amsterdam: John Benjamins Publishing Co, s. 107–136, doi:10.1075/la.221.05ros, ISBN 9789027257048, https://benjamins.com/#catalog/books/la.221/fulltext +Råðdjärum (2005), Förslag till en enhetlig stavning för älvdalska, arkiverad från ursprungsadressen den 2017-01-13, https://web.archive.org/web/20170113233252/http://alvdalen.se/upload/Dokument/kultur_fritid/%c3%84lvdalskan/alvdalsk_ortografi.pdf, läst 30 december 2016 +Røset, Ingulv (2011), Selbumålet, Novus +Sapir, Yair (2006), ”Elfdalian, the vernacular of Övdaln”, i Nyström, Gunnar, Rapport från första konferensen om älvdalska, http://hkr.diva-portal.org/smash/get/diva2:773379/FULLTEXT02.pdf +Sapir, Yair (2008), ”Råðdjärums älvdalska ortografi”, Oðer råðstemną um övdalskų, arkiverad från ursprungsadressen den 2016-03-09, https://web.archive.org/web/20160309191946/http://www2.valentin.uu.se/information/Elfdalian_radstemna_program.pdf, läst 30 december 2016 +Sapir, Yair (2017), ”The revitalization of Elfdalian”, i Ostler, Nicholas; Ferreira, Vera; Moseley, Chris (pdf), FEL XXI: Communities in Control: Learning Tools and Strategies for Multilingual Endangered Language Communities: Proceedings of the 21st FEL Conference, 19-21 October 2017, Portugal, s. 50–57, ISBN 9780956021090, http://hkr.diva-portal.org/smash/get/diva2:1211694/FULLTEXT01.pdf, läst 27 november 2018 +Steensland, Lars (2003), ”Älvdalska”, i Vamling, Karina; Svantesson, Jan-Olof, Världens språk – en typologisk och geografisk översikt, Lund: Lunds universitet, s. 361–375 +Steensland, Lars (2010), Material till en älvdalsk ordbok: svenska-älvdalska, älvdalska-svenska, Älvdalen: Ulum dalska, ISBN 978-91-978943-0-2 +Steensland, Lars (2011b), ”Parasitisk (oetymologisk) nasalisering i älvdalskan”, Oðer råðstemną um övdalskų, s. 116–126, http://uu.diva-portal.org/smash/get/diva2:431171/FULLTEXT02 +Steensland, Lars (2015a), Älvdalskans stavning – historia, nuläge och framtidsperspektiv, arkiverad från ursprungsadressen den 2017-03-20, https://web.archive.org/web/20170320215411/http://larssteensland.se/Stavning.pdf, läst 27 november 2018 +Steensland, Lars (2015b), ”Is there a vocative case in the Övdalian language? Some observations on forms of address in Övdalian.”, i Bentzen, Kristine; Rosenkvist, Henrik; Johannessen, Janne Bondi, Studies in Övdalian morphology and syntax: New research on a lesser known Scandinavian language, Linguistik aktuell 221, Amsterdam: John Benjamins Publishing Co, s. 171–176, ISBN 9789027257048, https://benjamins.com/#catalog/books/la.221/fulltext +Steensland, Lars (2021), Älvdalsk ordbok: svenska-älvdalska, älvdalska-svenska (2:a uppl.), Älvdalen: Ulum dalska, ISBN 978-91-984940-2-0 +Svenonius, Peter (2015), ”The morphological expression of case in Övdalian”, i Bentzen, Kristine; Rosenkvist, Henrik; Johannessen, Janne Bondi, Studies in Övdalian morphology and syntax: New research on a lesser known Scandinavian language, Linguistik aktuell 221, Amsterdam: John Benjamins Publishing Co, s. 177–230, ISBN 9789027257048, https://benjamins.com/#catalog/books/la.221/fulltext +Åkerberg, Bengt (1957), Om böjningen av feminina långstaviga svaga substantiv i singularis i Älvdalsmålet (Lokamålet) (Opublicerad uppsats), Institutionen för nordiska språk, Uppsala universitet +Åkerberg, Bengt (2012), Älvdalsk grammatik, Mora: Ulum dalska, ISBN 978-91-633-9251-1 +Åkerberg, Bengt; Elfquist, Mats; Westerling, Mats (2017), Frost-Anders dagbok, Spånga: Bengt Åkerberg, ISBN 9789163917721 +Sapir, Y.; Lundgren, O. (2024). A Grammar of Elfdalian. London: UCL Press (PDF) +Helgander, John (1994), ”Dalmålen i upplösning – bakgrund och förklaringsmodeller.”, i Kotsinas, Ulla-Britt Kotsinas; Helgander, John, Dialektkontakt, språkkontakt och språkförändring i Norden (MINS 40), Stockholm, s. 63–80 +Melerska, Dorota (2010), ”Vem är ”en riktig älvdaling”? – Identitetsmarkörer i dagens Älvdalen”, Folia Scandinavica Posnaniensia (Poznań) 11: 123–133, https://repozytorium.amu.edu.pl/bitstream/10593/825/3/123-133.pdf +Melerska, Dorota (2011), Älvdalskan – mellan språkdöd och revitalisering, Poznań, https://repozytorium.amu.edu.pl/bitstream/10593/1054/2/Dorota_Melerska_rozprawa%20doktorska.pdf +Steensland, Lars (1994), Älvdalska växtnamn: förr och nu, Uppsala: Dialekt- och folkminnesarkivet, ISBN 91-85540-66-8 +Steensland, Lars (2000), Fågelnamn och fågelliv i Älvdalen med omnejd, Uppsala: Centrum för biologisk mångfald, ISBN 91-89232-01-1 +Steensland, Lars (2006), War åvå dier ferid?: älvdalska namn på orter utanför Älvdalen med kulturhistoriska kommentarer, Uppsala: Uppsala universitet, ISBN 91-506-1883-0 +Älvdalsk ordbok – Lars Steenslands ordbok online +Swedia 2000 – Inspelningar av älvdalska +Institutet för språk- och folkminnens inspelning av älvdalska +Ulum dalskas webbsida +Brev skrivna under en kort vistelse i Sverige, Norge och Danmark (engelska: Letters Written During a Short Residence in Sweden, Norway, and Denmark) är en personlig reseskildring skriven av den brittiska författaren och tidiga feministen Mary Wollstonecraft. De 25 breven täcker ett brett spektrum av ämnen, från sociologiska reflektioner över Skandinavien och dess invånare till filosofiska frågor som den om personlig identitet. Boken utgavs av Joseph Johnson 1796 och var den sista som Wollstonecraft fick utgiven under sin levnadstid. +Wollstonecraft genomförde sin resa till Sverige, Norge och Danmark för att återta ett skepp som hade blivit stulet från hennes älskare, Gilbert Imlay. I hopp om att resan skulle bidra till att deras förhållande skulle återupptas, påbörjade hon entusiastiskt resan. Under sina tre månader i Skandinavien förstod hon dock att Imlay inte hade något intresse av att återuppta förhållandet. Breven som boken består av skildrar hennes ilska och melankoli över Imlays upprepade svek, och boken blir på så vis både en reseskildring och en självbiografi. I Brev skrivna under en kort vistelse i Sverige, Norge och Danmark utforskar Wollstonecraft förhållandet mellan den enskilda individen och samhället. Hon värdesätter subjektiva erfarenheter, främjandet av jämställdhet och utbildning för kvinnor. Hon kritiserar även vad hon ser som marknadens olyckliga påverkan på samhället. +Brev skrivna under en kort vistelse i Sverige, Norge och Danmark var Wollstonecrafts populäraste bok på 1790-talet. Boken sålde bra, blev översatt till flera språk och togs emot väl av de flesta kritiker. Filosofen och sedermera Wollstonecrafts make William Godwin sa om boken att "om det någon gång var en boks syfte att få en man att bli förälskad i författaren, så anser jag det vara denna bok". Romantiska diktare som William Wordsworth och Samuel Taylor Coleridge inspirerades av boken. Brev skrivna under en kort vistelse i Sverige, Norge och Danmark inspirerade läsare att resa till Skandinavien, men intresset för boken minskade dramatiskt efter att William Godwin utgav sin biografi om Mary Wollstonecraft 1798. Biografin orsakade en skandal i och med att den avslöjade Wollstonecrafts självständiga och okonventionella livsstil. +År 1790 gjorde Wollstonecraft, 31 år gammal, en dramatisk entré i offentligheten med pamfletten A Vindication of the Rights of Men ('Ett försvar för människors rättigheter'). Pamfletten var ett inlägg i den brittiska debatten om den revolution som just då pågick i Frankrike. Två år senare gav hon ut det som har kommit att bli hennes mest kända verk, A Vindication of the Rights of Woman (Till försvar för kvinnans rättigheter). Ivrig att få uppleva den franska revolutionen på nära håll flyttade hon till Frankrike och bodde där i två år. Hon återvände till Storbritannien 1795 efter att situationen i Frankrike hade förvärrats, övergiven av sin amerikanske älskare Gilbert Imlay och med dottern Fanny Imlay född utom äktenskapet. Kort efter återkomsten till Storbritannien försökte Wollstonecraft att begå självmord, men Imlay lyckades rädda henne. +I juni 1795, en månad efter självmordsförsöket, gick Wollstonecraft med på att företa den långa och svåra resan till Skandinavien för att lösa Imlays affärsproblem. Resan var inte bara farlig (hon var en kvinna som reste ensam i krigstid), utan präglades även av sorg och nedstämdhet. Med sig hade hon även sitt lilla barn och en barnflicka. Wollstonecraft trodde först att resan skulle bidra till att förhållandet till Imlay återupptogs, men insåg gradvis att det var meningslöst, speciellt efter att Imlay inte mötte henne i Hamburg. +Vid återkomsten till Storbritannien i september gjorde Wollstonecraft ännu ett försök att begå självmord genom att dränka sig i Themsen, men hon räddades av förbipasserande. Brev skrivna under en kort vistelse i Sverige, Norge och Danmark, som bygger på materialet från resan och breven hon skrev till Imlay under den tre månader långa resan, gavs ut i januari 1796. Förläggare var Wollstonecrafts nära vän och utgivare, Joseph Johnson. Boken präglades av självmordsförsöket och beskriver Wollstonecrafts mentala tillstånd under hennes tid i Skandinavien. Boken har förklarats som ett långt självmordsbrev till Imlay, även om hans namn aldrig nämns i texten. Den var det sista verk som Wollstonecraft fick utgivet under sin livstid och hon dog i barnsäng året efter. +I Brev skrivna under en kort vistelse i Sverige, Norge och Danmark framstår Wollstonecraft som en turist, medan hon i verkligheten genomförde komplicerade affärsförhandlingar å Imlays vägnar. I nästan 200 år var det oklart varför Wollstonecraft hade rest till Skandinavien, men på 1980-talet upptäckte historikern Per Nyström, tack vare dokument i svenska och norska arkiv, avsikten med resan. Nyström fann att Wollstonecraft eftersökte ett skepp och en last som hade stulits från Imlay. Wollstonecraft hade fullmakt att agera i Imlays ställe – Imlay refererade till henne i juridiska dokument som "fru Mary Imlay, min bästa vän och hustru" trots att de inte var gifta. +Nyström kunde avslöja de invecklade detaljerna i Imlays affärer. Den 18 juni 1794 köpte norrmannen Peder Ellefsen skeppet Liberty av Imlays agent i Le Havre, Frankrike. Det skulle senare visa sig att Ellefsen aldrig ägde skeppet, utan fungerade som bulvan i en pro forma-försäljning på uppdrag av Imlay. Ellefsen döpte om skeppet till Maria and Margaretha, antagligen efter Mary Wollstonecraft och hennes tjänarinna Marguerite, och fick det dansk-norska konsulatet i Le Havre att godkänna transaktionen så att fartyget kunde passera den brittiska blockaden av Frankrike. På detta sätt kunde Imlay agera som blockadbrytare. Med en värdefull last av silver och guld seglade skeppet från Frankrike under dansk-norsk flagg och ankom Köpenhamn den 20 augusti 1794. Trots att det sägs att Ellefsen beordrade skeppet vidare till Göteborg kom det aldrig dit. Imlay försökte vid flera tillfällen att återfinna skeppet och dess värdefulla last innan han skickade Wollstonecraft att förhandla fram ett avtal med Ellefsen, som hade arresterats för att ha stulit skeppet och lasten. Resultatet av Wollstonecrafts förhandlingar är okänt, detsamma gäller även skeppet och lastens öden. +För dessa förhandlingar reste Wollstonecraft, med dottern och barnflickan, till Göteborg där de stannade i två veckor. Därifrån reste hon ensam vidare till Strömstad, där hon gjorde en kort avstickare för att besöka Fredrikstens fästning. Därefter fortsatte hon till Larvik och till Tønsberg, där hon stannade i tre veckor. Hon besökte även Helgeroa, Risør och Christiania (Oslo) innan hon via Strömstad vände tillbaka till Göteborg, där hon återförenades med dottern och barnflickan. De tre återvände till England via Köpenhamn och Hamburg med ankomst Dover september 1795, efter tre månaders resa. +Brev skrivna under en kort vistelse i Sverige, Norge och Danmark består av 25 brev som berör en mängd olika områden, bland annat omstridda politiska frågor som reformering av fängelsestraffen, äganderätt och skilsmässolagstiftning, såväl som mindre kontroversiella ämnen som trädgårdsskötsel, saltutvinning och det sublima. I Wollstonecrafts politiska kommentarer utvecklar hon idéerna som hon först presenterade i An Historical and Moral View of the French Revolution. Diskussionen om en fängelsereform är till exempel präglad av hennes upplevelser i revolutionens Frankrike, där många av hennes vänner fängslades. +Brev skrivna under en kort vistelse i Sverige, Norge och Danmark uppfattas lätt som en reseskildring, men den tillhör mer en blandad genre och litteraturvetare är oense om vad som präglar boken mest. Någon betonar Wollstonecrafts kombination av reseskildring med självbiografin eller memoaren, där det senare var ett begrepp hon själv använde vid marknadsföringen av boken. Andra ser boken som en reseskildring i form av en brevroman. +Texten visar Wollstonecrafts tankeprocess och rör sig fritt från biografisk information till funderingar kring naturen och vidare till politiska teorier. Det finns emellertid två tydliga linjer i texten: den ena är Wollstonecrafts argumentation om samhällets struktur och utveckling, den andra är hennes tilltagande melankoli. Även om Wollstonecrafts avsikt är att skriva som en filosof är det bilden av den lidande kvinnan som dominerar boken. +Den ena hälften av Brev skrivna under en kort vistelse i Sverige, Norge och Danmark är en reseskildring i form av en brevroman. Wollstonecrafts uppfattning av denna genre formades av samtida reseskildringar som Oliver Goldsmiths The Traveller, or a Prospect of Society (1764), Laurence Sternes A Sentimental Journey Through France and Italy (1768), Samuel Johnsons A Journey to the Western Islands of Scotland (1775), James Boswells Journal of a Tour to the Hebrides (1785) och Arthur Youngs reseböcker. +Efter att ha recenserat 24 reseböcker för Joseph Johnsons tidskrift The Analytical Review var Wollstonecraft välbekant med genren. Den omfattande läsningen befäste hennes uppfattning om hur en god resebok skulle utformas. I en recension framhåller hon att författare av reseböcker skulle ha "ett bestämt mål, ett större projekt som kunde hjälpa till att koncentrera tanken och knyta samman de olika reflektionerna". Reseböcker skulle inte vara "isolerade observationer, utan någon röd tråd eller författarens förhärskade idéer". Hennes recensioner rosar detaljerade och engagerade beskrivningar av folk och städer, funderingar kring historia och den omättliga nyfikenheten. +Wollstonecraft skriver att "konsten att resa är bara en del av konsten att tänka". Hennes resa och själva reseskildringen är därför inte bara sentimental utan även filosofisk. Hon använder de båda "formerna" för att utveckla civilisationskritiken och kritiken av dåtidens kvinnoroll, som hon redan beskrivit i A Vindication of the Rights of Men (1790), Till försvar för kvinnans rättigheter (1792) och An Historical and Moral View of the French Revolution. Efter att ha brutit mot konventionen för hur man skriver om historiska och politiska fenomen introducerar Wollstonecraft, vad experten Gary Kelly kallar "revolutionär feminism", vilket är ytterligare ett område som tidigare hade ansetts vara ett territorium för manliga skribenter. Enligt Kelly förnyar Wollstonecraft reseskildringens "blandning av objektiv fakta och individuella intryck ... till en grund för självbiografiskt utlämnande". Boken är, som en av utgivarna för Brev skrivna under en kort vistelse i Sverige, Norge och Danmark skriver, "inget annat än en revolution i den litterära genren", dess högstämdhet, framställt genom partier med intensiva känslor utgör "en ny vildhet och rikedom i känslouttryck" eftersträvansvärd hos den resande. +En forskare i ämnet har utnämnt Wollstonecraft till den "perfekta lidelsefulla resenären". Hennes strävan att fördjupa sig och fullt ut uppleva varje ögonblick är inspirerad av Jean-Jacques Rousseaus verk, särskilt hans Les Rêveries du promeneur solitaire (1782). Flera av Rousseaus teman återfinns i Brev skrivna under en kort vistelse i Sverige, Norge och Danmark, som "sökandet efter källan till mänsklig lycka, en stoisk avvisning av det materiella, den extatiska omfamningen av naturen och den centrala rollen som känslor har i uppfattningen av saker och ting". Men medan Rousseau till sist avvisar samhället prisar Wollstonecraft framstegen både i hemmet och i näringslivet. +I en av de mer inflytelserika tolkningarna av Brev skrivna under en kort vistelse i Sverige, Norge och Danmark argumenterar Mary Favret att Wollstonecrafts brev inte endast bör ses som personliga brev utan även som affärskorrespondens, en genre som ideologiskt hade varit tvetydig för Wollstonecraft. Enligt Favret försöker Wollstonecraft reformera den opersonliga genre som affärskorrespondens representerar och genomsyra den med egna erfarenheter. En teknik som hon använder för detta är att med hjälp av omfattande användning av fantasirika berättelser tvinga läsaren att bli en deltagare i händelserna som beskrivs. +Favret pekar på att Brev skrivna under en kort vistelse i Sverige, Norge och Danmark klart skiljer sig från de förtvivlade och klagande kärleksbrev som Wollstonecraft skickade till Imlay. Brev skrivna under en kort vistelse i Sverige, Norge och Danmark ligger mycket närmare resedagboken hon förde där hon registrerade tankar om människor hon mötte och städer hon besökte. Medan breven till Imlay har långa passager där hon nästan uteslutande skriver om sig själv har Brev skrivna under en kort vistelse i Sverige, Norge och Danmark kommentarer om sociala förhållanden och sympatier med offren för katastrofer och orättvisor. Inför Imlay framställer sig Wollstonecraft som nedstämd på grund av tvivel, medan hon inför resten av världen skildrar sig som den som övervinner alla dessa farhågor. Hon grubblar över farhågorna och omformar dem till grundmaterial för ett brev som motsvarar de öppna politiska brev som var populära mot slutet av 1700-talet, och hon låter sina personliga erfarenheter utgöra utgångspunkten för diskussionen kring en nationell politisk reformation. +Starkt influerad av Rousseaus ärliga och avslöjande Les Confessions (1782) blottlägger Wollstonecraft sitt innersta i Brev skrivna under en kort vistelse i Sverige, Norge och Danmark och beskriver i detalj både sin fysiska och sin psykiska resa. Precis som med många andra självbiografier skrivna av kvinnor har hennes personliga avslöjanden ofta framställts som oavsiktliga och till synes indirekta. Men enligt Mitzi Myers, som forskat i ämnet, använder sig Wollstonecraft av denna stil för att formulera ett stabilt och begripligt jag för läsaren. Med ett allt större självförtroende som författare kontrollerar hon framställningen och dess effekt på läsaren på en nivå hon inte når upp till i sina andra verk. Hon omvandlar de personliga sorger som både den faktiska och den inre resan innebär till gripande litteratur. +I Brev skrivna under en kort vistelse i Sverige, Norge och Danmark använder sig Wollstonecraft i stor omfattning av diskursen kring det sublima. Hon utvecklar och omdefinierar Edmund Burkes centrala begrepp i A Philosophical Enquiry into the Origin of Our Ideas of the Sublime and Beautiful (1757). Burke framhåller det sublima (som han associerar med maskulinitet, terror, vördnad och styrka) över det vackra (som han förbinder med femininitet, passivitet, skörhet och svaghet) medan Wollstonecraft knyter det sublima till sterilitet och det vackra till fruktbarhet. För henne är det vackra knutet till det kvinnliga och det estetiska skiftet är uppenbart i de många passager som fokuserar på det nära förhållandet mellan Wollstonecraft och hennes dotter Fanny. Hon förknippar därmed den feminina kategorin av "det vackra" med den mest ärbara och dugliga gruppen av kvinnor: mödrarna. +Wollstonecraft förnyar även de konventionella negativa associationerna mellan det sublima och döden. Tankar om döden, frammanade av ett vattenfall, får henne istället att tänka på återfödelse och odödlighet: +Liksom hennes övriga förskjutningar av språket kring det sublima är även denna passage starkt påverkad av genus. En forskare skriver: "eftersom Wollstonecraft är kvinna, och som sådan begränsas av juridiska och sociala restriktioner för hennes kön på 1700-talet, kan hon bara se en självständig tillvaro efter döden". +Wollstonecraft karaktäriseras ofta som en rationell filosof, men i Brev skrivna under en kort vistelse i Sverige, Norge och Danmark visar hon sitt engagemang och värdesättande av känslor. Hon argumenterar för att subjektiva upplevelser, som de känslor av transcendens som framkallas av det sublima och det vackra, har ett värde motsvarande de objektiva sanningar som kan avslöjas genom förnuft. I Wollstonecrafts tidigare arbeten var förnuftet överordnat eftersom de gav tillgång till universella sanningar. I Brev skrivna under en kort vistelse i Sverige, Norge och Danmark är förnuftet istället ett redskap för reflektion, en övergång från den sinnliga upplevelsen av den jordiska tillvaron till en abstrakt uppfattning av sanningen – inte nödvändigtvis en universell sådan. Att mogna är här inte bara att förvärva ett förnuft, vilket var ett resonemang som Wollstonecraft utvecklat i Original Stories from Real Life (1788), utan även att utveckla en förståelse för hur och när man bör lita på sina känslor. +Wollstonecrafts teorier om förnuft, känslor och fantasi är nära knutna till varandra. Vissa forskare i ämnet hävdar att Wollstonecraft använder fantasin för att frigöra jaget, och då särskilt det kvinnliga jaget, eftersom detta gav henne möjligheten att föreställa sig kvinnoroller utanför de traditionella gränser som rådde under 1700-talet, och gjorde det möjligt för henne att formulera dessa nya idéer. Andra forskare menar att hennes fokus på fantasins styrka har en menlig inverkan på hennes resonemang eftersom detta fjättrar henne till ett "individualiserat, borgerligt begär", vilket aldrig helt kan omfatta samhällstillvändhet. +Favret argumenterar för att Wollstonecraft använder fantasin som ett medel för att förena "mannens begreppsvärden" med "kvinnans sensibilitet". Läsaren måste låta fantasin arbeta, och dessa ansträngningar kommer att rädda läsaren från sentimentalitet men även från att hänfallas åt överdrivna spekulationer. Än viktigare är att läsaren involveras i berättarens historia. Wollstonecrafts språk kräver att läsarna engagerar sig i handlingen: +Genom att ge fantasin styrkan att omforma samhället (en kraft som antyds genom flera referenser till Shakespeares Stormen) avslöjar Wollstonecraft att hon blivit en romantiker. +I Brev skrivna under en kort vistelse i Sverige, Norge och Danmark utforskar Wollstonecraft genomgående förhållandet mellan samhället och individen. Medan hennes tidigare verk fokuserar på samhällets brister och dess ansvar fokuserar hon i detta verk istället inåt, och argumenterar uttryckligen för värdet av personliga erfarenheter. I reklamen för boken, som även ingår som förord till texten, förklarar hon sin roll som berättelsens "hjälte": +Genom hela boken knyter Wollstonecraft sin andliga resa och utveckling till civilisationens framsteg. Nationer, liksom individer, framhåller forskaren Mary Poovey, har "en kollektiv förståelse som utvecklas organiskt och mognar gradvis". Wollstonecraft ser samtidigt civilisationens umbäranden som mer värda omtanke än de individuella eller fiktiva umbärandena, vilket påvisar att för henne är sympati kärnan i alla sociala relationer: +En stor del av Brev skrivna under en kort vistelse i Sverige, Norge och Danmark innehåller beskrivningar av naturen och Wollstonecrafts känslomässiga reaktioner på dessa naturupplevelser. Ett av hennes mer effektiva grepp är att knyta tankar och känslor till naturfenomen, som i passagen om vattenfallet ovan. Naturen, antar Wollstonecraft, utgör "en gemensam referenspunkt" som delas av henne och läsaren, och därför borde breven etablera ett tillstånd av gemensam sympati dem emellan. Många av breven innehåller "små romantiserade utvikningar" som visualiserar Wollstonecrafts idéer om förbindelsen mellan naturen, Gud och människan. Den materiella världen utgör "den fundamentala utgångspunkten för resonemang och ett av de viktigaste områdena för upplevelser och erfarenheter". +Alla Wollstonecrafts skrifter, inklusive Brev skrivna under en kort vistelse i Sverige, Norge och Danmark, tar upp kvinnornas, och då främst medelklasskvinnornas, belägenhet i slutet av 1700-talet. Liksom i tidigare texter diskuterar hon även här konkreta frågor som barnuppfostran och förhållandet till tjänstefolket, men till skillnad från hennes mer polemiska skrifter som Thoughts on the Education of Daughters (1787) och Till försvar för kvinnans rättigheter (1792) lägger denna text vikten vid hennes känslomässiga reaktioner på naturupplevelser och moderskapet. Samtidigt fjärmar hon sig inte från sitt engagemang i att förespråka utbildning och rättigheter för kvinnor. I det nittonde brevet, där hennes feministiska tankar tydligast kommer fram, förutser Wollstonecraft läsarens kritik: "fortfarande tjatar hon om samma ämne, kommer du att utropa – men hur skulle jag kunna låta bli, när flertalet av de strider som förekommit i mitt händelserika liv, har orsakats av förtrycket av mitt kön: vi begrundar djupt det vi känner starkast för." Wollstonecraft inser att hon alltid tvingats möta världen utifrån det faktum att hon är kvinna – och detta är ett av de mest utmärkande dragen för hennes självbild. +Genom hela Brev skrivna under en kort vistelse i Sverige, Norge och Danmark kommenterar Wollstonecraft den otrygga position kvinnor hade i samhället. Bland annat försvarar hon, och sympatiserar med, Drottning Caroline Mathilde av Danmark och Norge, som anklagats för "lösaktighet" efter en utomäktenskaplig romans med livsläkaren Struensee, medan hon var gift med den sinnessjuke Kristian VII. Wollstonecraft beskriver drottningen, som var en progressiv social reformator, som en modig kvinna som försökte revolutionera sitt land innan det var redo. Sådana exempel förstärkte Wollstonecrafts ökande desperation och melankoli. Vid ett tillfälle förtvivlar hon sig över sin dotters öde: +Wollstonecrafts ilska och förtvivlan över kvinnans underordnande roll tvingade henne att definiera sig själv i motsats till dåtidens konventionella föreställningar av kvinnlighet. I sitt första brev berättar hon stolt att: "Vid kvällsvarden sade min värd mig uppriktigt att jag var en kvinna med iakttagelseförmåga, för de frågor jag ställde till honom var mansfrågor" (Wollstonecrafts kursivering). +Wollstonecraft ställer den kvinnliga fantasin som den konstruktiva motsatsen till den destruktiva manliga handelsverksamheten, ett konststycke hon i huvudsak genomför genom användningen av genren brevroman. Medan hon i Till försvar för kvinnans rättigheter argumenterar för att kvinnor borde vara nyttiga och produktiva, och därmed överförde marknadens språk till hemmet, applicerar Brev skrivna under en kort vistelse i Sverige, Norge och Danmark hemmets värderingar på den sociala och politiska världen. +Även om Wollstonecraft i Brev skrivna under en kort vistelse i Sverige, Norge och Danmark ägnar stort utrymme åt funderingar kring naturen och dess kopplingar till jaget avhandlar en stor del av texten handelns negativa effekter på kulturen. Hon hävdar till exempel att skadorna på Hamburg och Frankrike, förorsakade av legosoldater och en allt mer kommersialiserad kultur, är mer omfattande än den förstörelse som den franska revolutionen orsakade. Hon skriver att "svärdet har varit barmhärtigt, i jämförelse med de skövlingar som mänskligheten utsatts för av entreprenörer, och av den svärm av frossande gräshoppor som sprider sig som en farsot till andra länder". Wollstonecraft menar att handelsverksamheten brutaliserade känslolivet och fostrade en egoistisk hållning hos utövaren. Enligt henne borde handel "reguleras efter idéer om rättfärdighet och likabehandling och inrättas mot att stötta oberoende och givmildhet". +Imlay hade gjort Wollstonecraft desillusionerad, inte bara på grund av hans avvisande inställning till henne, utan även på grund av hans girighet. Genom Brev skrivna under en kort vistelse i Sverige, Norge och Danmark knyter hon kritiken av marknaden till den anonyme älskaren som svikit henne: +Genom hela texten sätter hon konstruktiv och kreativ fantasi i motsats till destruktiv handel. Genom att associera kommersialism med den anonyma älskaren i texten kritiserar Wollstonecraft Imlay, som hon ansåg var mer upptagen av sina företag än av henne och deras gemensamma barn. +I flera avsnitt av Brev skrivna under en kort vistelse i Sverige, Norge och Danmark spekulerar Wollstonecraft kring möjligheterna till en social och politisk revolution och skisserar en utveckling för civilisationen. I hennes jämförelse av Norge med England och Frankrike argumenterar hon för att norrmännen var mer progressiva eftersom de hade pressfrihet, praktiserade religiös tolerans, hade en rättvis fördelning av landområden och en politiskt aktiv befolkning. Hennes beskrivning av en norsk guldålder blir emellertid mindre översvallande när hon upptäcker att landet inte har något universitet eller några vetenskapsmän. +På många sätt förverkligade dåtidens norska samhälle den brittiska radikala tanken om ett "samfund av småproducenter, där intäkterna är jämnt fördelade för att säkra en rimlig jämlikhet", motsvarande det som Wollstonecraft hade skissat i A Vindication of the Rights of Men (1790). Efter noggranna överväganden kring hur man bäst kunde förbättra de sociala och politiska problemen i länderna hon besökte kommer Wollstonecraft fram till slutsatsen att social utveckling måste ske i en kontrollerad och "naturlig" takt. Hon argumenterar för att varje land måste finna sin egen väg för att utvecklas och att en demokratisk revolution inte kan tvingas på ett folk. Hon anser att de lägre klasserna och småbrukarna utgör de mest lovande "potentiella källorna till revolutionär social förändring". I denna bedömning finns det dock indirekt en borgerlig nedlåtande hållning, då hon ser arbetarklassen som en grupp långtifrån henne själv, och vid ett tillfälle beskriver hon deras beteende som pittoreskt. +Wollstonecrafts främsta skäl för att ge ut Brev skrivna under en kort vistelse i Sverige, Norge och Danmark var hennes stora skulder. Den framgångsrika försäljningen av boken, som blev hennes mest populära på 1790-talet, skedde vid en gynnsam tidpunkt. Efter goda recensioner översattes boken till tyska, nederländska, svenska och portugisiska, den publicerades i USA och trycktes i en andra upplaga 1802. +Amelia Alderson prisade boken, men särskiljde kvinnan och filosofen: "Med en gång när jag hade läst dina brev från Norge försvann den kalla fruktan som filosofen hade framkallat, till förmån för en varm sympati framkallad av kvinnan." William Godwin, Wollstonecrafts blivande make, skrev i Memoirs of the Author of A Vindication of the Rights of Woman att boken fick honom att förälska sig i henne: +Godwin kopplar samman boken med Wollstonecrafts första utgivning, Mary: A Fiction (1788), och framhäver textens känslosamhet och hur den "erotiserar den kvinnliga sorgen". Godwin såg boken som en romantisk brevroman och inte som ett verk präglat av politiska kommentarer. Efter Wollstonecrafts död 1797 publicerade Godwin hennes originalbrev till Imlay och i samband med detta förstördes också originalen. Han tog bort alla hänvisningar till aktuella politiska händelser och Wollstonecrafts affärsmöten, och underströk istället det romantiska sambandet mellan de två brevsamlingarna. Favret menar att Godwin ville att allmänheten skulle betrakta Wollstonecrafts kärleksförhållande som en sentimental romans motsvarande den mellan Charlotte och Werther i Goethes Den unge Werthers lidanden (1774). +Att en kvinna med ett barn på ett år och en hushållerska reste till Skandinavien utan beskydd av en man saknade motstycke vid slutet av 1700-talet. Boken tedde sig mycket uppseendeväckande. Att Wollstonecraft reste till en så avlägsen plats (en av bokens redaktörer beskrev det som "en nordlig vildmark") fängslade och chockerade flera läsare. Bokens oortodoxa teologi avskräckte också läsare. Monthly Magazine and American Review skrev: +Brev skrivna under en kort vistelse i Sverige, Norge och Danmark tog avstånd från Wollstonecrafts tidigare syn på en dömande Gud, till Gud som skapare. Detta chockade konservativa läsare som inte var mottagliga för något som påminde om deism. Den samtida feministen och författaren Mary Hays kritiserade istället boken för att vara överdrivet sentimental och känslosam. Thomas Brown, professor i moralfilosofi, publicerade ett poetiskt svar till boken i The Wanderer in Norway (1816). I stället för att glädja sig över friheten som är konsekvensen av Wollstonecrafts argument om förhållandet mellan natur och känslor, presenterade Brown boken som ett fiasko och Wollstonecraft som en tragisk figur. Brown läste Brev skrivna under en kort vistelse i Sverige, Norge och Danmark som en förmanande moralisk berättelse, medan Wollstonecrafts avsikt var att presentera sociala och personliga utvecklingsmöjligheter. Favret visar i sin forskning att nästan alla dåtidens uttolkare ser författaren (Mary) som den tragiska hjältinnan, trots att texten i sig, med sin blandning av känslor och politik, utmanar en sådan tolkning på många sätt. +Publiceringen av Godwins memoarer, som avslöjade och stödde Wollstonecrafts romanser och barnafödande utanför äktenskapet, resulterade i att en majoritet av allmänheten kom att förakta hennes böcker. Men trots detta "väckte boken en lust att resa bland de kultiverade människorna i Europa". Dristiga kvinnliga brittiska resenärer på 1800-talet som Isabella Bird och Mary Kingsley inspirerades av Wollstonecrafts pionjärinsats. Brev skrivna under en kort vistelse i Sverige, Norge och Danmark trycktes i en ny upplaga mot slutet av 1800-talet. Robert Louis Stevenson, författaren av Skattkammarön tog med ett exemplar på sin resa till Samoa år 1890. +Romantikens poeter påverkades kraftigt av Brev skrivna under en kort vistelse i Sverige, Norge och Danmark. Poeten Robert Southey skrev till exempel till sin förläggare: "Har du stött på Mary Wollstonecrafts [resebok]? Hon har fått mig att älska ett kallt klimat, och frost och snö, med ett nordiskt månljus." Bokens progressiva sociala syn, i kombination med dess försvar av de subjektiva erfarenheterna, inspirerade författare som William Wordsworth och Samuel Taylor Coleridge. Wollstonecrafts "inkarnationsteori om kreativ fantasi" banade väg för Wordsworths genomgång av fantasin och dess förhållande till jaget i den femte boken av The Prelude (1805; 1850) Hennes bok hade också ett starkt inflytande på Coleridges The Rime of the Ancient Mariner (1797–1799) och Percy Shelleys Alastor (1815). Den mest slående hyllningen till Wollstonecrafts bok finns emellertid i Coleridges berömda dikt Kubla Khan (1797; 1816) I stora delar är stilen i dikten hämtad från Brev skrivna under en kort vistelse i Sverige, Norge och Danmark, och i en passage anspelar han direkt på Wollstonecraft när han beskriver en kall ödemark: +Favret, Mary. Romantic Correspondence: Women, politics and the fiction of letters. Cambridge: Cambridge University Press, 1993. ISBN 0-521-41096-7. +Furniss, Tom. «Mary Wollstonecraft's French Revolution». The Cambridge Companion to Mary Wollstonecraft Ed. Claudia L. Johnson. Cambridge: Cambridge University Press, 2002. ISBN 0-521-78952-4. +Godwin, William; Erik Carlquist (översättning) (2009). Mary Wollstonecraft: författaren av Till försvar för kvinnans rättigheter : en minnesteckning. Umeå: h:ström Text & kultur. ISBN 9789173271134 . Efterord av Elisabeth Mansén. +Holmes, Richard. «Introduction». A Short Residence in Sweden, Norway and Denmark and Memoirs of the Author of A Vindication of the Rights of Woman. New York: Penguin Books, 1987. ISBN 0-14-043269-8. +Jacobus, Mary. «In Love With a Cold Climate: Traveling with Wollstonecraft». First Things: Reading the Maternal Imaginary. New York and London: Routledge, 1995. ISBN 0-415-90383-1. +Jones, Chris. «Mary Wollstonecraft's Vindications and their political tradition». The Cambridge Companion to Mary Wollstonecraft Ed. Claudia L. Johnson. Cambridge: Cambridge University Press, 2002. ISBN 0-521-78952-4. +Kaplan, Cora. «Mary Wollstonecraft's reception and legacies». The Cambridge Companion to Mary Wollstonecraft Ed. Claudia L. Johnson. Cambridge: Cambridge University Press, 2002. ISBN 0-521-78952-4. +Kelly, Gary. Revolutionary Feminism: The Mind and Career of Mary Wollstonecraft. New York: St. Martin's, 1992. ISBN 0-312-12904-1. +Moore, Jane. «Plagiarism with a Difference: Subjectivity in 'Kubla Khan' and Letters Written during a Short Residence in Sweden, Norway and Denmark». Beyond Romanticism. Ed. Stephen Copley and John Whale. London: Routledge, 1992. ISBN 0-415-05201-7. +Moskal, Jeanne. «The Picturesque and the Affectionate in Wollstonecraft's Letters from Norway». Modern Language Quarterly 52 (1991): 263–294. +Myers, Mitzi. «Wollstonecraft's Letters Written . . . in Sweden: Towards Romantic Autobiography». Studies in Eighteenth-Century Culture 8 (1979): 165-185. +Nyström, Per. «Mary Wollstonecraft's Scandinavian Journey». Acts of the Royal Society of Arts and Letters of Gothenburg, Humaniora 17 (1980). +Nyström, Per; Forser, Tomas (1994). Tre kvinnor mot tiden: drottning Kristina, Mary Wollstonecraft och Alma Åkermark. Stockholm: Tiden. sid. 93–164. ISBN 9155041213 Nyströms artikel från 1980 bearbetad och översatt till svenska, inklusive tio av de privata breven till Imlay. +Parks, George B. «The Turn to the Romantic in the Travel Literature of the Eighteenth Century». Modern Language Quarterly 25 (1964): 22–33. +Poovey, Mary. The Proper Lady and the Woman Writer: Ideology as Style in the Works of Mary Wollstonecraft, Mary Shelley and Jane Austen. Chicago: University of Chicago Press, 1984. ISBN 0-226-67528-9. +Sapiro, Virginia. A Vindication of Political Virtue: The Political Theory of Mary Wollstonecraft. Chicago: University of Chicago Press, 1992. ISBN 0-226-73491-9. +Swaab, Peter. «Romantic Self-Representation: The Example of Mary Wollstonecraft's Letters in Sweden». Mortal pages, Literary Lives: Studies in Nineteenth-Century Autobiography. Ed. Vincent Newey and Philip Shaw. Aldershot: Scholar Press, 1996. ISBN 1-85928-206-7. +Taylor, Barbara. Mary Wollstonecraft and the Feminist Imagination. Cambridge: Cambridge University Press, 2003. ISBN 0-521-66144-7. +Todd, Janet (2000) (på engelska). Mary Wollstonecraft: a revolutionary life. London: Weidenfeld & Nicolson. ISBN 0297842994 . +Wollstonecraft, Mary. Brev skrivna under en kort vistelse i Sverige, Norge och Danmark. Översättning Jane Lundblad. Stockholm: Tiden, 1978. ISBN 91-550-2265-0. +Wollstonecraft, Mary (2022). Brev från en resa i Sverige, Norge och Danmark. Ellerströms förlag. ISBN 9789172476813 . Översättning, inledning, noter och kommentarer Elisabeth Mansén. +Wollstonecraft, Mary (2024). Brev till en svekfull älskare. Ellerströms förlag. ISBN 9789172477469 Översättning Elisabeth Mansén. +Letters from Sweden i fulltext, inte formaterat, från Project Gutenberg (engelska) +Mary Wollstonecraft: A «Speculative and Dissenting Spirit» av Janet Todd fra www.bbc.co.uk (engelska) +The treasure seeker Artikel i The Guardian (engelska) +Liverpool FC (The Reds) är en engelsk professionell fotbollsklubb i Liverpool som spelar i Premier League. Klubben grundades den 3 juni 1892 och har Anfield som sin hemmaarena, vilken tillhörde den 14 år äldre rivalen Everton FC innan Liverpool FC grundades. +Klubben har stått som engelska mästare totalt 20 gånger, senaste Premier League-titel 2024/25, och har därtill vunnit sex Europacup-/Champions League-titlar (senast 2019) vilket är tre fler än rivalen Manchester United. De båda har lika många ligatitlar (20 stycken) i kampen om att titulera sig Englands mest framgångsrika klubb någonsin. Liverpool FC hade sin mest framgångsrika tid under 1970- och 1980-talen, och vann år 1984 den så kallade trippeln: engelska ligan, Europacupen för mästarlag (nuvarande Champions League) och Ligacupen. Under 2019 blev Liverpool först i England att vinna en internationell trippel i form av Champions League, Uefa Super Cup och världsmästerskapet i fotboll för klubblag. Framgångsrika tränare under klubbens drygt 130-åriga historia har varit David Ashworth, Bill Shankly, Bob Paisley, Kenny Dalglish och Jürgen Klopp. Bland egenfostrade världsspelare finns Ian Callaghan, Jamie Carragher, Michael Owen, Robbie Fowler och Steven Gerrard. +Klubben har en global skara av supportrar och vid tifo till hemmamatcherna spelas låten "You'll Never Walk Alone" (som också står textat på klubbmärket) sedan 1960-talet. Läktarkaos på bortaplan ledde dock till två omtalade incidenter med dödlig utgång på 1980-talet (däribland Hillsborougholyckan 1989). +Klubben ägs sedan 2010 av Fenway Sports Group och i maj 2024 rankades klubben av den ansedda tidskriften Forbes som världens fjärde rikaste fotbollsklubb, detta till ett värde av 5,37 miljarder dollar. +År 1989 bildades damsektionen Liverpool WFC, som blivit engelsk mästare två gånger (2013 och 2014) och bidragit till ett genombrott för professionell damfotboll i England. +Historien om Liverpool FC börjar i grannklubben Everton, som bildades 1878. Everton hade spelat på Anfield sedan 1884, men under sju år höjde arenaägaren John Houlding hyran från 100 pund till 250 pund. Everton bestämde sig därför för att flytta till Goodison Park och Houlding hade nu en arena men inget lag. I januari 1892 försökte Houlding bilda en ny klubb kallad "Everton Football Club and Athletic Grounds Company plc", men namnet godkändes inte av fotbollsförbundet eftersom det redan fanns en klubb med namnet Everton. Ett fåtal medlemmar från Everton stod på Houldings sida och den 15 mars 1892 bildades Liverpool Football Club. Med hjälp av ett lån från Houlding på 500 pund kunde lagledaren John McKenna värva ett dussintal spelare från Skottland, vilket gjorde att laget blev känt som "Team of all the Macs", efter spelarnas efternamn. +Liverpool ansökte om medlemskap i The Football League, men fick avslag och gick istället med i Lancashire League – en av både Englands och världens tidigaste fotbollsligor, som bedrevs i blott 14 år (1889–1903). Den 1 september 1892 spelade Liverpool sin första match; en vänskapsmatch mot Rotherham Town, som Liverpool vann med 7–1, men bevittnades bara av några få personer. Samma dag hade Everton spelat inför 10 000 åskådare på Goodison Park. Två dagar senare spelade Liverpool sin första tävlingsmatch, som vanns mot Higher Walton med 8–0 i Lancashire League. Säsongen slutade med att laget vann både ligatiteln och Liverpool Senior Cup. I cupfinalen besegrades Everton med 1–0 i det första Merseysidederbyt. 1893 valdes klubben in i The Football League, där klubben så småningom vann division två utan att förlora en enda match. Efter att ha besegrat Newton Heath (sedermera Manchester United) med 2–0, i en så kallad testmatch, tog Liverpool steget upp i division ett, men väl där kom laget sist och flyttades ner till division två efter förlust mot Bury i en testmatch. +Ett år senare, 1896, var Liverpool tillbaka i division ett efter att ha besegrat Small Heath (sedermera Birmingham City) och West Bromwich Albion i en serie testmatcher. Tom Watson anställdes som lagledare, varefter Liverpool blev ett etablerat lag i division ett, där Watsons främsta spelarvärvning var den skotske centerhalven Alex Raisbeck från Stoke City. Säsongen 1900/01 blev Liverpool ligamästare för första gången, men sjönk till elfte plats året efter. Säsongen 1902/03 kom Liverpool på femte plats, fyra poäng efter mästarlaget The Wednesday, men kom näst sist i tabellen 1903/04 och flyttades ner till division två. Liverpool tog sig dock snabbt tillbaka och blev redan efter återkomsten ligamästare igen. På denna tid framträdde Arthur Goddard och Joe Hewitt som klubbens stjärnspelare. +Efter ligasegern fick Anfield två nya läktare: en huvudläktare på långsidan och den så kallade Spion Kop på ena kortsidan. Anfield hade då plats för 50 000 åskådare, men under de närmast följande åren nådde laget inga framgångar bortsett från en andraplats i tabellen 1909/10. Under åren fram till första världskrigets utbrott höll Liverpool till i nedre halvan av ligatabellen. Viss framgång nåddes dock 1914, då laget gick till final i FA-cupen som förlorades mot Burnley med 0–1. Denna final var den sista som spelades på National Sports Centre i Crystal Palace (London) och den första som bevittnades av en monark – på läktaren satt kung Georg V. Ett år senare var klubben inblandad i en skandal: Liverpool låg i mitten av tabellen och mötte i sista omgången Manchester United, som behövde vinna matchen för att inte ramla ur serien. Efter en överenskommelse lagen emellan fick Manchester United vinna med 2–0 och när det avslöjades att matchen var uppgjord blev fyra spelare i vardera lag avstängda på livstid. Trots detta fick både Liverpool och Manchester United spela kvar i högsta divisionen. Avstängningen av spelarna hävdes också och den officiella förklaringen var att båda lagen behövde all hjälp de kunde få efter att ha drabbats av stora förluster under första världskriget. +David Ashworth anställdes som ny tränare och värvade den irländske landslagsmålvakten Elisha Scott samt backarna Ephraim Longworth och Don MacKinlay. Efter två raka fjärdeplatser tog Liverpool sin tredje ligatitel 1922 – och förväntades bli den dominerande klubben under 1920-talet, då klubben således också vann sin andra raka ligatitel 1923. Detta trots att Ashworth lämnade klubben mitt under säsongen då istället Matt McQueen, som tidigare suttit i klubbens styrelse, tog över som tränare. Efter fem år i ett mindre framgångsrikt Liverpool FC avgick McQueen i februari 1928, på grund av hälsoskäl. Innan han lämnade klubben värvade han den sydafrikanske anfallaren Gordon Hodgson, som under de följande åren kom att göra 241 mål på 377 matcher för Liverpool. +George Patterson ersatte McQueen som tränare efter att tidigare ha varit assistent till Tom Watson och även haft hand om laget under första världskriget. Under hans ledning var Liverpool ett mittenlag i ligan och laget gjorde heller inget resultat i FA-cupen. Säsongen 1935/36 kom laget på nittonde plats i tabellen och var nära att åka ur. Detta i kombination med en sjukdom gjorde att Patterson avgick efter säsongen, men han hade under sin tid i klubben fostrat unga blivande stjärnspelare som Jack Balmer och Phil Taylor. Han värvade även Matt Busby, som senare skulle fira stora framgångar som tränare i Manchester United. Den nye tränaren George Kay kunde inte heller han göra Liverpool till en toppklubb innan andra världskriget satte stopp för ligaspelet 1939. +Efter kriget var Liverpool med i FA-cupen säsongen 1945/46, men blev utslaget av Bolton Wanderers i fjärde omgången. Ligaspelet återupptogs följande säsong och Liverpool blev ligamästare, en poäng före Wolverhampton Wanderers och Manchester United. I laget fanns bland andra Billy Liddell, Bob Paisley, Jack Balmer och Albert Stubbins. Liddells inflytande var så stort att laget fick smeknamnet "Liddellpool". Ligasegern 1947 följdes av att laget sjönk till mitten av tabellen under en längre tid, men nådde dock finalen i FA-cupen 1950 där laget föll mot Arsenal med 0–2. Tränaren George Kay fick problem med sin hälsa, vilket ledde till att han avgick i januari 1951, och ersattes av Don Welsh som hade gästspelat i Liverpool under andra världskriget. +Under Welshs ledning blev klubben alltmer ett bottenlag och kom att degraderas till division två, efter att ha kommit på sista plats i tabellen säsongen 1953/54, och väl där slutade laget på elfte plats, vilket var klubbens lägsta placering någonsin. I slutet av säsongen 1955/56 fick Welsh sparken och Phil Taylor, som var med i truppen vid ligasegern 1947, tog över som huvudtränare. Han ledde laget till en tredjeplats och två fjärdeplatser i division två innan han lämnade klubben i november 1959 och blev därmed den förste tränaren i Liverpool som aldrig ledde laget i högstadivisionen. Han värvade Roger Hunt, som kom att bli en av klubbens främsta målskyttar genom tiderna. +Liverpools ordförande T.V. Williams anställde Bill Shankly som ny tränare i december 1959 och han krävde att Liverpools styrelse inte skulle lägga sig i laguttagningar och värvningar av spelare. Shankly värvade ett flertal spelare som Ron Yeats och Ian St. John, och plockade även in unga spelare som Ian Callaghan och Chris Lawler, för att göra Liverpool till ett storlag igen. På ledarsidan anställdes Bob Paisley och Joe Fagan som assistenter till Shankly och efter två tredjeplaceringar i division två vann laget serien 1962 för ett nytt avancemang uppåt. Återkomsten till högsta divisionen slutade med en åttondeplats och ett år senare, 1964, var Liverpool ligamästare för sjätte gången. +Säsongen 1964/65 slutade laget på sjunde plats i ligan, men tog sig till final i FA-cupen på Wembley. Matchen vanns mot Leeds United med 1–0 efter ett förlängningsavgörande, där Gerry Byrne fick spela en stor del av matchen med brutet nyckelben, då avbytare inte var tillåtna. Därmed vann Liverpool sin första FA-cuptitel. Laget hade även gått till semifinal i Europacupen, men förlorade detta dubbelmöte mot Inter efter avgörande på målskillnad. Under säsongen etablerade sig Tommy Smith i laget och Liverpool kunde vinna sin sjunde ligatitel säsongen 1965/66. Laget deltog också i Cupvinnarcupen, där finalplats nåddes. +Efter framgångarna i mitten av 1960-talet skulle det dröja några år till klubbens nästa stora titel. Shankly inledde en generationsväxling och värvade flera unga spelare som Steve Heighway, Emlyn Hughes, John Toshack och målvakten Ray Clemence. Kvar i laget fanns Ian Callaghan, Chris Lawler och Tommy Smith, som tillsammans med de nya spelarna tog laget till final i FA-cupen 1971, där det dock blev förlust mot Arsenal med 1–2 efter förlängning. Efter säsongen värvades Kevin Keegan och två år senare var Liverpool återigen ligamästare. Laget vann också Uefacupfinalen 1973 med totalt 3–2 i dubbelmötet mot Borussia Mönchengladbach. Säsongen 1973/74 kom Liverpool på andra plats i ligan, men vann FA-cupen för andra gången efter finalvinst mot Newcastle United med 3–0. Efter säsongen valde Bill Shankly att kliva åt sidan och det sista han gjorde var att värva Ray Kennedy. +Bill Shankly efterträddes av den tidigare assisterande tränaren Bob Paisley, som hade varit i klubben som spelare och ledare sedan 1939. Paisley fortsatte det arbete som Shankly påbörjat; i detta ingick värvningarna av Phil Neal och Terry McDermott. Under Paisleys första säsong som huvudtränare kom laget på tredje plats i ligan och blev också utslaget av Ferencvaros i andra omgången av Cupvinnarcupen. I första omgången besegrades norska Strömsgodset med 11–0 på Anfield, vilket är Liverpools största seger genom tiderna. 1976 blev Liverpool ligamästare för nionde gången efter att ha slutat en poäng före skrällaget Queens Park Rangers. I Uefacup-semifinalen besegrades Barcelona och i finalen Club Brugge KV, vilket blev Liverpools andra titel. +Liverpool försvarade ligatiteln säsongen 1976/77, då laget var obesegrat på Anfield. I slutet av andra kvartsfinalmötet i Europacupen mot Saint-Étienne avgjorde inhopparen David Fairclough, som var specialist på sådana inhopp och därför kallades "Superavbytaren". Laget nådde ytterligare en cuptitel, efter seger mot Borussia Mönchengladbach på Olympiastadion i Rom med 3–1. Detta var Kevin Keegans sista match i Liverpool innan han såldes till Hamburger SV för 500 000 pund. Liverpool hade även gått till final i FA-cupen, men förlorade mot Manchester United med 1–2. +Inför säsongen 1977/78 värvades Kenny Dalglish från Celtic för rekordsumman 440 000 pund för att ersätta Kevin Keegan. Även Alan Hansen värvades och han kom att spela över 600 matcher för Liverpool. Laget kom på andra plats i ligan och gick till final i Ligacupen för första gången, men förlorade mot Nottingham Forest. I Europacupen blev det återigen final mot Club Brugge, som Liverpool vann. Nästa säsong tog Liverpool rekordhöga 68 poäng efter 30 segrar, åtta oavgjorda och fyra förluster. I början av ligasäsongen stod laget bland annat för en 7–0-seger mot Tottenham hemma på Anfield. +Ännu en ligatitel bärgades 1979/80, men laget blev utslaget i semifinalen av både FA-cupen och Ligacupen. 1981 vann Liverpool Ligacupen för första gången, efter finalvinst mot West Ham United. Det blev även seger i Europacupen för tredje gången på fem år. +Under säsongen 1981/82 etablerade sig den walesiske anfallaren Ian Rush i laget och han skulle komma att göra totalt 30 mål. Målvakten Bruce Grobbelaar ersatte Ray Clemence. Från Brighton värvades Mark Lawrenson för 900 000 pund, vilket gjorde honom till Liverpools dyraste spelarköp och Storbritanniens dyraste försvarsspelare genom tiderna. Liverpool vann ligan fyra poäng före Ipswich Town; ett lag som också besegrades i semifinalen av Ligacupen; för att sedan slå Tottenham med 3–1 i finalen efter förlängning. Liverpool blev ligamästare också 1982/83 och då avgjordes förstaplatsen tidigt. Det blev även seger i Ligacupen efter finalseger med 2–1 över Manchester United efter förlängning. Efter säsongen avgick Bob Paisley som Liverpools mest meriterade tränare genom tiderna. Under de nio säsongerna förde han klubben till sex ligatitlar, tre ligacupvinster, tre europacupvinster och en UEFA-cupvinst. +Den tidigare assisterande tränaren Joe Fagan tog över efter Paisley och under sin första säsong som huvudansvarig ledde han laget till tre stora titlar; ligan för tredje året i rad, Ligacupen för fjärde året i rad samt Europacupen. +Inför säsongen 1984/85 värvades danske Jan Mölby och John Wark, och laget tog sedermera andraplatsen i ligan. I FA-cupen blev laget utslaget i semifinalen av Manchester United, och i Europacupen gick Liverpool åter till final men förlorade mot Juventus. Matchen kom dock att överskuggades av händelserna på Heyselstadions läktare där anhängare till Liverpool bröt sig igenom ett stängsel och gick till attack mot Juventussupportrarna som då försökte fly. Trycket blev för stort på en vägg som kollapsade och krossade de flyende människorna. 39 människor miste livet, de flesta italienare, och Liverpool blev avstängda från europeiskt cupspel i sex år. Övriga engelska klubbar blev avstängda i fem år. +Joe Fagan valde att inte förlänga kontraktet efter Heyselkatastrofen och Kenny Dalglish fick rollen som spelande tränare. Säsongen 1985/86 lyckades Liverpool som tredje klubb under 1900-talet att vinna Dubbeln, det vill säga både ligan och FA-cupen. Följande år kom laget på andra plats i ligan, och gick till final i Ligacupen som förlorades mot Arsenal med 2–1. Sommaren 1987 såldes Rush och klubben värvade John Aldridge, John Barnes (som under sin första säsong i klubben blev vald till årets spelare i England) och Peter Beardsley (för den brittiska rekordsumman 1,9 miljoner pund). Värvningarna betalade sig då Liverpool återtog ligatiteln, och gick på nytt till final i FA-cupen men förlorade mot Wimbledon som hade stått för en stor skräll. +Inför säsongen 1988/89 köptes Ian Rush tillbaka från Juventus. Liverpool kom på andra plats i ligan, på samma poäng som Arsenal men med färre gjorda mål. I FA-cupfinalen vann Liverpool mot Everton med 3–2. Inför semifinalen mot Nottingham Forest på Hillsborough Stadium i Sheffield den 15 april 1989 inträffade en av de största läktarkatastroferna som drabbat England. Det stora antalet supportrar som ville ta sig in på arenan gjorde att polisen öppnade ett antal ingångar som saknade vändkors. Detta ledde till att publiken vällde in på arenan och folk som redan hade kommit in krossades mot stängslen som skiljde läktarna från planen. Totalt omkom 96 personer och händelserna ledde till den så kallade Taylor-rapporten, som uppmanade till att avskaffa stängsel och ståplatsläktare på de engelska fotbollsarenorna. +Säsongen 1989/90 tog Liverpool sin artonde ligatitel efter att bland annat ha besegrat Crystal Palace med 9–0. I FA-cupens semifinal förlorade Liverpool mot samma lag med 4–3. Dagen efter att Liverpool spelat 4–4 mot Everton i omspelet av femte omgången i 1991 års FA-cup avgick tränaren Kenny Dalglish. Ronnie Moran tog tillfälligt över, men laget förlorade det andra omspelet mot Everton och lyckades inte heller försvara ligatiteln utan kom på andra plats. Vid det laget hade klubben anställt en permanent efterträdare till Dalglish, skotten Graeme Souness, en tidigare mittfältare i klubben. Andraplatsen i ligan säsongen 1990/91 innebar att Liverpool hade varit bland de två bästa tio år i rad. +Under Graeme Souness första säsong kom laget sexa i ligan, men vann istället FA-cupen efter att ha besegrat Sunderland med 2–0 på Wembley. Under säsongen gjorde Liverpool återkomst i det europeiska cupspelet efter den sex år långa avstängningen, men blev utslaget av italienska Genoa i fjärde omgången av UEFA-cupen. Efter segern i FA-cupen 1992 lyckades inte Souness föra laget till några nya framgångar, och efter förlust mot Bristol City i femte omgången av FA-cupen i januari 1994 avgick han då istället Roy Evans tog över. +Evans tog över ett lag med unga stjärnspelare som Robbie Fowler och Steve McManaman, och säsongen 1994/95 vann Liverpool Ligacupen efter seger med 2–1 över Bolton Wanderers i finalen. Sommaren 1996 satte Liverpool nytt brittiskt övergångsrekord genom att värva anfallaren Stan Collymore från Nottingham Forest för 8,5 miljoner pund. Laget gick till final i FA-cupen 1996, men förlorade mot Manchester United med 1–0. Efter säsongen lämnade Rush Liverpool; detta efter 346 mål på 660 matcher för klubben. Följande säsong ledde Liverpool ligan vid flera tillfällen, men slutade till sist på fjärde plats. Laget tog sig även till semifinal i Cupvinnarcupen, men förlorade mot franska Paris Saint-Germain. Säsongen 1997/98 kom Liverpool på tredje plats i Premier League och under året etablerade sig unge Michael Owen i laget. Owen blev delad skytteligavinnare med 18 mål och blev samtidigt också det engelska landslagets dittills yngste debutant genom tiderna. +Inför säsongen 1998/99 anställdes den franske tränaren Gérard Houllier för att arbeta tillsammans med Roy Evans, men till följd av lagets placering lämnade Evans Liverpool i november, då laget skulle sluta på plats sju i ligan. Nya spelare som finländaren Sami Hyypiä och unga Steven Gerrard bidrog i hög grad till att laget säsongen 2000/01 kunde vinna tre cuptitlar. Uefa-cupfinalen mot spanska Deportivo Alavés på Westfalenstadion i Dortmund blev den målrikaste UEFA-cupfinalen någonsin då ställningen vid full tid var 4–4 och Liverpool kunde vinna med ett mål i förlängningen. +Säsongen 2001/02 slutade Liverpool på andra plats i Premier League; klubbens högsta ligaplacering på elva år. Året efter togs ytterligare en cuptitel då laget besegrade Manchester United med 2–0 i Ligacupfinalen på Millennium Stadium i Cardiff. Efter säsongen 2003/04 lämnade Houllier klubben och ersattes av spanjoren Rafael Benitez, som åren innan hade tagit Valencia till seger både i Spanska ligan och UEFA-cupen. Även Michael Owen lämnade efter säsongen klubben för spel i Real Madrid. +Rafael Benitez första säsong slutade med en femteplats i Premier League, men kunde ta sig till final i Ligacupen (förlust mot Chelsea med 3–2 efter förlängning). Säsongens stora händelse för klubben blev Champions League-finalen på Atatürk-stadion i Istanbul, där Liverpool mötte Milan och kunde vinna efter straffsparkläggning då laget hade hämtat in ett 0–3-underläge i andra halvlek. Eftersom Liverpool slutade femma i ligan skulle laget inte fått möjlighet att försvara sin Champions League-titel, men som regerande mästare i turneringen gavs chansen för Liverpool att kvala till Champions League 2005/06. Laget blev dock utslaget av Benfica i åttondelsfinalen, men Liverpool vann istället FA-cupfinalen mot West Ham United i en match som slutade 3–3 där Liverpool vann straffsparksläggningen och tog sin sjunde inteckning i cupen. Säsongen 2006/2007 tog sig laget åter till final i Champions League, och på nytt stod Milan för motståndet som fick revansch genom att vinna matchen med 2–1. +Den 6 november 2007 skrev Liverpool Champions League-historia genom att vinna med 8–0 mot turkiska Besiktas på hemmaplan, vilket dittills var den största segermarginalen någonsin i turneringen. +Säsongen 2008/2009 nådde laget en kvartsfinalplats i Champions League och slutade på en andra plats i ligan bakom Manchester United. Säsongen 2009-2010 misslyckades klubben med att ta sig vidare från gruppspelet i Champions League för första gången på fem år. Istället fick laget spela i Europa League och tog sig till semifinal som förlorades mot Atletico Madrid. Liverpool slutade under våren sedan sjua i Premier League och i juni 2010 meddelade klubben att ha kommit överens med Benítez om att säga upp hans kontrakt som manager för klubben. +Den 1 juli 2010 hade Liverpool skrivit ett kontrakt med Roy Hodgson som innebar att denne skulle vara klubbens tränare under de följande tre åren. Efter bara 7 månader, i januari 2011, fick dock Hodgson lämna Liverpool efter lagets resultat i ligan, bland annat med endast en bortavinst, mot Bolton på Reebok Stadium. +När Hodgson lämnat Liverpool och Kenny Dalglish tagit över som interimtränare för resten av säsongen blev han den förste tränaren i klubbens historia som anställts för en andra period. Under vinterns transfer köpte klubben först uruguayanen Luis Suárez från Ajax för 26,5 miljoner euro och senare Andy Carroll från Newcastle United för 35 miljoner pund (den dittills högsta transfersumma en brittisk klubb betalat). Rekordet slogs dock endast någon timme senare då Liverpool i sin tur sålde Fernando Torres till Chelsea för 50 miljoner pund. Den 12 maj samma år förlängde Dalglish sitt kontrakt som manager för klubben med ytterligare tre år. +Under sommaren införskaffade sedan klubben ett antal nya spelare: Jordan Henderson, Charlie Adam, Stewart Downing, Alexander Doni, Sebastian Coates och nygamle Craig Bellamy. Säsongen 2011/2012 vann klubben ännu en gång ligacupen på Wembley den 26 februari mot Cardiff City, efter straffsparksavgörande. Cuptiteln till trots blev inte Dalglish långvarig i sin andra tränarsejour på Anfield. Efter klubbens sämsta resultat i ligan på 18 år fick han efter säsongen lämna sin post. +Efter Dalglishs sorti skrev Brendan Rodgers den 1 juni 2012 på ett kontrakt som ny Liverpoolmanager, efter att först ha tackat nej. Trots sin ålder hade Rodgers varit med i lagledarroller i nästan 20 år efter att hans karriär som fotbollsspelare tagit slut i 20-årsåldern på grund av ett genetiskt fel i knät. Rodgers första nyförvärv blev Fabio Borini och Rodgers gjorde omgående små ändringar på klubbens arena. Han tog bland annat tillbaks de röda nätmaskorna från Liverpools storhetstid och bytte ut den nya This Is Anfield-skylten från 1990-talet till den gamla modellen som Bill Shankly en gång satt upp. Rodgers inledde sin första säsong med att komma på en sjundeplats i ligan, men skulle under säsongen därpå leda laget till en andraplats. Säsongen 2014/15 var klubblegendaren Steven Gerrards sista säsong i klubben. Liverpool hade storsatsat med värvningarna av Adam Lallana och Mario Balotelli, men avslutade ändå säsongen utan titlar. +Den 8 oktober 2015 anställdes Jürgen Klopp som ny manager efter en skakig säsongsinledning. Under sin första säsong vid rodret tog Klopp med sitt höga presspel klubben till final i både ligacupen den 28 februari 2016, där man förlorade på straffar mot Manchester City, och i Europa League den 18 maj 2016, där Sevilla blev för starka. Inför säsongen 2016/2017 värvades spelare som Sadio Mané, Georginio Wijnaldum och Joël Matip in vilket bidrog till säkra den fjärde plats i Premier League som krävs för kvalificering till Champions League kommande år. Med inköp av Mohamed Salah, Andy Robertson och Alex Oxlade-Chamberlain under sommaren 2017, och ett nytt transferrekord för en mittback vintern 2018 i och med köpet av Virgil van Dijk, slutade klubben återigen på en fjärde plats i tabellen samt gick till final i Champions League mot Real Madrid FC. Finalen slutade dock i förlust med 3–1 efter en tidig skada på Mohamed Salah (som under säsongen vunnit Premier Leagues skytteliga med 32 mål), samt två omtalade misstag av målvakten Loris Karius. Inför säsongen 2018/2019 värvades Alisson Becker som ny förstemålvakt och mittfältet stärktes med köpen av Fabinho, Naby Keita och Xherdan Shaqiri. Med den målfarliga trion Salah, Mané och Firmino pressade Liverpool de regerande mästarna Manchester City om segern in i Premier Leagues sista omgång för att sluta tvåa på 97 poäng. Det var första gången ett lag tog mer än 90 poäng utan att vinna titeln. Säsongen slutade dock i dur då Tottenham Hotspur besegrades i Champions League-finalen den 1 juni på Estadio Metropolitano, Madrid, med 2–0. +Efter seger med 2–0 mot ärkerivalen Manchester United den 19 januari 2020 inledde Liverpool FC 2019/2020 års Premier League-säsong utan att förlora och radade upp 21 vinster på 22 matcher. År 2019 blev Liverpool den första engelska klubben att vinna den internationella trippeln; Champions League, Uefa Super Cup och världsmästerskapet i fotboll för klubblag. Premier League-säsongen 2019/20 blev avbruten mellan den 13 mars och den 17 juni, på grund av Covid-19-pandemin. Efter seger mot Chelsea den 25 juni 2020 stod Liverpool som engelska mästare för första gången sedan 1990, och sin första Premier League-titel någonsin, med sju omgångar kvar (vilket var PL-rekord). Lagkaptenen Jordan Henderson utsågs till Årets fotbollsspelare i England (FWA). +Säsongen 2020/21 var komplicerad i idrottsvärlden, på grund av Covid-19-pandemin, då matcherna i bland annat Premier league spelades inför tomma läktare. Trots att tre av klubbens äldre mittbackar drabbades av långtidsskador kunde lagets svit i antal förlustfria hemmamatcher slå ett nytt rekord för klubben vid 68 (tidigare 63) sådana, där Burnley FC satte stopp (0–1) den 21 januari 2021. En formsvacka med sex raka förluster till en åttondeplats följdes av åtta raka segrar, däribland ett avgörande mål på stopptid av målvakten Alisson Becker mot West Bromwich (det första någonsin av en målvakt i klubbens då 129-åriga historia), och Liverpool slutade trea. Säsongen därpå var Liverpool nära att vinna en unik kvadrupel genom att vinna FA-cupen och en rekordstor EFL Cup, mot Chelsea i final i bägge turneringar, och tog sig till final i CL 2021/22 samt kom tvåa i Premier League med endast en poäng bakom Manchester City. +Den 30 juli 2022 inledde Liverpool säsongen 2022/23 med att vinna FA Community Shield, 3–1 mot manchester City. I Premier League stod Liverpool för flera storsegrar; bland annat 9–0 hemma mot AFC Bournemouth vilket tangerade den största segern i Premier Leagues historia (och Bournemouths största förlust någonsin) och klubbens tidigare 9–0-seger hemma mot Crystal Palace FC 1989/90. Roberto Firmino noterades i denna match för sitt 100:e mål med Liverpool. Även 7–0 hemma, den största segern inbördes, mot ärkerivalen Manchester United FC. I 7–1-segern borta mot Rangers FC i Champions League slog Mohamed Salah nytt rekord i det snabbaste CL-hattricket någonsin. Liverpool kom att sluta femma i Premier League, och missade därmed Champions League. Inför öppningen av januarifönstret kontrakterades Cody Gakpo för övergångssumman omkring 40 miljoner pund. Inför säsongen 2023/24 skedde en större spelaromsättning i Liverpool där elva spelare lämnade klubben; däribland Jordan Henserson (för vilken Virgil van Dijk tog över lagkaptensrollen), Roberto Firmino och Fabinho som flyttade till Saudiarabien. In kom bland andra nederländarna Ryan Gravenberch och Sepp van den Berg. +Den 26 januari 2024 meddelade Jürgen Klopp att han kommer avgå som tränare för Liverpool när säsongen 2023/24 är slut. Hans sista tävlingsmatch med klubben slutade 2–0 på Anfield, mot Wolverhampton Wanderers FC, och en tredjeplats i Premier League. +Liverpools traditionella klubbfärger är rött och vitt, men under de första åren spelade laget i blåvita tröjor. I början av 1900-talet började laget spela i röda tröjor och vita byxor (färgerna på strumporna alternerade under åren mellan svart, rött och vitt). Under Bill Shanklys tid introducerades det berömda helröda matchstället. Inför en Europacupmatch mot Anderlecht 1964 kom Shankly in i omklädningsrummet och bad Ron Yeats ta på sig ett par röda shorts som skulle matcha de röda tröjorna. Tanken var att den röda färgen skulle ha en psykologisk effekt – spelarna såg större och farligare ut och motståndarna skulle bli ängsliga. Shankly gillade vad han såg, och när Ian St. John föreslog att laget även skulle ha röda strumpor föddes den numera klassiska Liverpooldräkten. +Liverpools reservställ har ofta bestått av vita tröjor, svarta byxor och vita strumpor. I slutet av 1970-talet introducerades ett tredje matchställ i helgult som användes i bortamatcher mot lag som spelade i rött och vitt, som till exempel Sunderland. Under några år i slutet av 1980-talet gick reservstället i helgrått och på 1990-talet förekom reservställ bestående av gröna tröjor och vita byxor. Från och med 2020 är Nike officiell leverantör av utrustning. +Liverpools klubbmärke utgörs av en sköld med texten Liverpool Football Club. Inuti skölden finns en avbildning av en så kallad liverbird, en fantasifågel som är en korsning mellan en örn och en skarv. Liverbirden är också symbol för staden Liverpool. Ovanför skölden återfinns texten You'll Never Walk Alone tillsammans med det järnkonstverk som sitter ovanpå Shankly Gates, en av ingångarna till Anfield. På varsin sida om skölden finns två lågor som symboliserar Hillsborough memorial som är en minnesplats utanför Anfield vid The Shankly Gates uppförd för att hedra de 96 människor som omkom vid Hillsborougholyckan 1989. Liverpool återgick till en äldre variant av klubbmärket på tröjorna säsong 2012/2013, med en gul/guld-färgad Liver bird med texten "L.F.C." istället för den vanliga skölden. +Som ägare av Anfield och grundare av Liverpool FC var John Houlding klubbens förste ordförande, en post han innehade från 1892 till 1904. John McKenna tog över som ordförande, vilken senare blev president för Football League. ordförandeskapet bytte ägare många gånger innan John Smith, vars far var akrieägare i klubben, tog över rollen 1973. Han ledde den mest framgångsrika perioden i Loverpools historia innan han avgick 1990. Hans efterträdare blev ordföranden Noel White 1990. I augusti 1991 blev David Moores ordförande, vars familj hade ägt klubben i mer än 50 år. Hans farbror John Moores var också aktieägare i Liverpool och ordförande i Everton FC från 1961 till 1973. Moores ägde 51 procent av klubben och 2004 uttryckte han sin vilja att överväga ett bud på sina aktier i klubben. +Moores sålde så småningom klubben till de amerikanska affärsmännen George Gillett och Tom Hicks den 6 februari 2007. Affären värderade klubben och dess utestående skulder till 218,9 miljoner pund. Duon betalade 5 000 pund per aktie, eller 174,1 miljoner pund för det totala aktieinnehavet och 44,8 miljoner pund för att täcka klubbens skulder. Meningsskiljaktigheter mellan duon, och fansens ovilja att stötta dem, resulterade i att de två ville sälja klubben. Martin Broughton utsågs till klubbens ordförande den 16 april 2010 för att övervaka försäljningen. I maj 2010 offentliggjordes räkenskaper som visade att klubbens holdingbolag hade 350 miljoner pund i skulder (på grund av ett övertagande av lånefinansieringen) med förluster på 55 miljoner pund, vilket fick revisorn KPMG att göra ett reservationsuttalande. Gruppens fordringsägare, inklusive Royal Bank of Scotland, tog Gillett och Hicks till domstol för att tvinga dem att tillåta styrelsen att gå vidare med försäljningen av klubben, holdingbolagets största tillgång. En domare i brittiska High Court dömde till förmån för fordringsägarna och banade väg för försäljningen av klubben till amerikanska Fenway Sports Group (då benämnt New England Sports Ventures, NESV), den 15 oktober 2010 för 300 miljoner pund, även om Gillett och Hicks fortfarande hade möjlighet att överklaga. +Liverpool har beskrivits som ett globalt varumärke; en rapport från 2010 värderade klubbens varumärken och tillhörande immateriella rättigheter till 141 miljoner pund, en ökning med 5 miljoner pund jämfört med året innan, och Liverpool fick betyget "AA (Very Strong)". I april 2010 rankade den amerikanska affärstidningen Forbes Liverpool som världens sjätte mest värdefulla fotbollsklubb, till ett värde av 822 miljoner dollar (532 miljoner pund) exklusive skulder. Revisorn Deloitte rankade Liverpool som åtta i Deloitte Football Money League, vilken rankar världens fotbollsklubbar när det gäller intäkter. Liverpools intäkter säsongen 2009/10 var 225,3 miljoner euro. Enligt en rapport från Deloitte 2018 hade klubben en årsintäkt på 424,2 miljoner euro för 2017, och Forbes värderade klubbens värde till 1,944 miljarder dollar. År 2018 ökade intäkterna till 513,7 miljoner euro, och Forbes värderade klubben till 2,183 miljarder dollar. Under 2019 passerade intäktsiffrorna gränsen för en halv miljard pund som mätte 533 miljoner, vilket motsvarade 604 miljoner euro enligt Deloitte. +I april 2020 fick klubbens ägare kritik från supportrar och media för att ha beslutat att permittera all icke-spelande personal under Covid-19-pandemin. Som svar på detta gjorde klubben en tvärvändning i det beslutet och bad om ursäkt. I april 2021 värderade Forbes klubben till 4,1 miljarder dollar, en tvåårig ökning med 88 procent, vilket gjorde den till världens femte rikaste. Sedan maj 2023 rankas Liverpool som världens fjärde rikaste av Forbes. +Liverpools supportrar kallas ofta för "Kopites", ett namn taget från sektionen The Kop på hemmaarenan Anfield. På 1960-talet växte låten "You'll Never Walk Alone", ursprungligen från musikalen Carousel men senare inspelad av Liverpool-gruppen Gerry & the Pacemakers, fram som klubbens officiella hymn. Den spelas inför varje hemmamatch då spelarna tågar in på planen och sjungs ofta även under matcherna av publiken. Textraden finns med på klubbmärket och på "Shankly Gates", en av ingångarna till arenan. +Under säsongen 2009/2010 hade Liverpool det fjärde högsta publiksnittet av alla engelska klubbar, 42 863, vilket var 94,4 % av Anfields totala kapacitet. +Klubben har varit inblandad i två av de största olyckorna i fotbollshistorien: Katastrofen på Heyselstadion under Europacupfinalen (nuvarande UEFA Champions League) 1985 mot Juventus, då 39 supportrar (mestadels italienska Juventussupportrar) dog, och Olyckan på Hillsborough under semifinalen av FA-cupen 1989 då 96 Liverpoolsupportrar miste sina liv. +Huvudartiklar: Merseysidederbyt och Rivaliteten mellan Liverpool FC och Manchester United +Liverpool FC:s äldsta rivalitet är Merseysidederbyt där de möter Liverpool-laget Everton FC. Denna rivalitet härstammar från Liverpool FC:s bildande och dispyten mellan Evertons tjänstemän och de dåvarande ägarna av Anfield. Merseysidederbyt är ett av få derbyn som inte upprätthåller segregation för supportrar, och har därför blivit känt som "vänskapsderbyt". Sedan mitten av 1980-talet har rivaliteten dock intensifierats både på och utanför planen och sedan starten av Premier League 1992 har Merseysidederbyt fått fler spelare utvisade än något annat Premier League-möte. Det har kallats "de mest odisciplinerade och explosiva matcherna i Premier League". När det gäller supporterstödet i staden Liverpool är Liverpool FC:s fans dubbelt så många som Evertons, enligt en undersökning 2020. +Liverpools rivalitet med Manchester United härstammar från städernas konkurrens i den industriella revolutionen på 1800-talet. Liverpool och Manchester, som är förbundna med världens första regionala järnväg, är belägna på 48 kilometers avstånd längs East Lancs Road. Liverpool FC och Manchester United FC rankas som de två största klubbarna i England av tidningen France Football, och är de mest framgångsrika engelska lagen i både inhemska och internationella tävlingar, och båda klubbarna har en global skara av supportrar. Deras möten ses som engelsk fotbolls mest kända, och en av världsfotbollens största rivaliteter. De två klubbarna alternerade som mästare mellan 1964 och 1967, och Manchester United blev den första engelska klubben att vinna Europacupen 1968 – följt av fyra Europacuptitlar av Liverpool. Trots deras sammanlagt 39 ligatitlar och nio Europacup-/Champions League-titlar däremellan har de två rivalerna sällan varit framgångsrika samtidigt; Liverpools storhetstid på 1970- och '80-talen sammanföll med Manchester Uniteds 26-åriga titeltorka, och Manchester Uniteds framgångar i Premier League-eran sammanföll med Liverpools 30-åriga titeltorka. De två klubbarna har slutat (samtida) etta och tvåa i ligan fem gånger. Rivaliteten mellan klubbarna är så stor att de sällan gör transferaffärer med varandra; den senaste spelarövergången mellan dem var Phil Chisnall som flyttade till Liverpool från Manchester United 1964. +Anfield byggdes 1884 och var Evertons hemmaarena fram till 1892. Efter en konflikt gällande hyran flyttade Everton till Goodison Park och Anfields ägare John Houlding bildade Liverpool FC som sedan dess har haft Anfield som hemmaarena. +1906 döptes en av läktarna på ena kortsidan till Spion Kop (eller bara The Kop) efter en ås i Natalprovinsen i Sydafrika där ett slag utkämpades under andra boerkriget med flera omkomna soldater från Liverpool. The Kop kunde ta uppemot 30 000 åskådare, men efter Hillsborougholyckan bestämdes att ståplatsläktare skulle avskaffas. I mitten av 1990-talet revs The Kop och ersattes av en sittplatsläktare med plats för 12 409 åskådare. +På motsatt sida av The Kop finns Anfield Road Stand där bortalagets supportrar håller till. Läktaren byggdes om 1998 och fick två våningar med plats för sammanlagt 9 274 åskådare. Huvudläktaren, Main Stand, uppfördes 1895 och var då arenans första stora utbyggnad. Main Stand, vars kapacitet är 12 277 åskådare, byggdes om 1973 och fick då ett nytt tak. Sittplatserna på Main Stand är handmålade. Läktaren på den andra långsidan hette ursprungligen Kemlyn Road Stand, men i samband med klubbens 100-årsjubileum 1992 döptes den om till Centenary Stand (kapacitet: 11 762). 2014 påbörjades utbyggnaden av Main Stand, som stod klar September 2016, med en kapacitet på 54 074 åskådare. Oktober 2017 ändrade också namnet på Centenary Stand till Sir Kenny Dalglish Stand. Det finns planer på att bygga ut Anfield Road Stand och därmed utöka Anfield till att rymma 61 000 åskådare. +Anfields publikrekord är 61 905 åskådare och sattes i en match mellan Liverpool och Wolverhampton Wanderers 1952. +Liverpool var med i den första upplagan av BBC:s Match of the Day, som visade höjdpunkter från deras match mot Arsenal FC på Anfield den 22 augusti 1964. Den första fotbollsmatchen som sändes i färg var mellan Liverpool och West Ham United, som direktsändes i mars 1967. Liverpool-supportrar medverkade i Pink Floyds låt "Fearless", där de sjöng utdrag ur "You'll Never Walk Alone". För att markera klubbens framträdande i FA-cupfinalen 1988 släppte Liverpool "Anfield Rap", en låt med John Barnes och andra medlemmar i truppen. +En dramadokumentärsfilm om Hillsborougholyckan visades 1996, skriven av Jimmy McGovern. I denna medverkade Christopher Eccleston som Trevor Hicks, vilken förlorade två tonårsdöttrar i denna katastrof och kom att föra kampanjer för säkrare arenor och hjälpa till att bilda Hillsborough Families Support Group. Liverpool var med i filmen The 51st State (2001), där ex-torpeden Felix DeSouza (Robert Carlyle) är en ivrig supporter av laget och den sista scenen utspelar sig vid en match mellan Liverpool och Manchester United FC. Klubben var också med i den brittiska TV-serien Scully (1984), där en ung pojke försöker få ett provspel med Liverpool. I Doctor Who-avsnittet "The halloween Apocalypse", som sändes i oktober 2021, visar doktorn (spelad av Jodie Whittaker) när hon lämnar tardis utanför Anfield och utbrister: "Liverpool? Anfield! Klopp-eran, klassisk!" +Liverpool Football Club Museum är ett fotbollsmuseum som tillhör Liverpool FC. Museet samorganiseras med de guidade turerna på Anfield Road där museet också är beläget. Museet grundades 1998 och innehåller föremål, foton, berättelser dylikt från klubbens många framgångar. Från klubbens första år 1892 finns namnändringsbeviset, då interna motsättningar i Everton FC ledde till ett misslyckat kapningsförsök av klubbnamnet, vilket resulterade i grundandet av Liverpool FC. Museet har cirka 100 000 besökare årligen. +Fem replikabucklor från vinsterna i Europacupen för mästarlag +Vandringsbucklan för Champions League, som man fick behålla efter finalvinsten 2005 +Jamie Carraghers samlade medaljsamling +Roger Hunts guldmedalj från Fotbolls-VM 1966 +Michael Owens Ballon d'Or från 2001 +Billy Lidells ligaguldmedalj från 1947 +Ray Houghtons matchtröja från ligaguldåret 1990 +Fem fotbollar som Kenny Dalglish gjort hat-trick med +Ian St Johns matchtröja som han hade på sig när han avgjorde FA-Cupfinalen 1965 +Under åren har flera framstående spelare uppträtt i Liverpools tröja. Exempel på spelare som var aktiva i Liverpool före andra världskriget är backen Ephraim Longworth som blev klubbens förste engelske landslagskapten och målvakten Elisha Scott som tillbringade 22 år i Liverpool. Klubbens främste målskytt under 1920- och 30-talen var Gordon Hodgson som är den spelare i Liverpool som gjort flest hattrick. Under perioden efter andra världskriget gjorde anfallaren Billy Liddell så stort intryck att klubben fick smeknamnet "Liddellpool". +När Liverpool etablerade sig som en storklubb under Bill Shanklys ledning var mittbacken Ron Yeats, som beskrevs som en "koloss", en av nyckelspelarna. En annan viktig spelare var anfallaren Roger Hunt som gjorde 245 ligamål och även var med och blev världsmästare med engelska landslaget 1966. I början av 1970-talet värvades Kevin Keegan som senare blev utsedd till årets spelare i Europa (vid det laget hade han dock sålts till Hamburg). +Under Bob Paisleys ledning värvades två spelare från Skottland 1977: Alan Hansen, som var med om att vinna Europacupen tre gånger, och Kenny Dalglish, som firade stora framgångar både som spelare och tränare i klubben. År 1980 värvades Ian Rush som blev klubbens bäste målskytt genom tiderna. +Under 1990-talet kom flera unga spelare fram från Liverpools ungdomsakademi, däribland Steve McManaman, Robbie Fowler, Jamie Carragher och Michael Owen. Den sistnämnde blev utsedd till årets spelare i Europa 2001. Från akademin kom även Steven Gerrard, som tog över lagkaptensbindeln efter Sami Hyypiä 2003. +När klubben i slutet på 1990- och på 00-talet anställde utländska managers präglades spelartrupperna i viss del av deras nationstillhörighet. Fransmannen Gérard Houllier värvade flera franskspråkiga landslagsspelare, till exempel Djimi Traore, Nicolas Anelka, Bruno Cheyrou, El Hadji Diouf och Djibril Cissé. När spanjoren Rafael Benitez tog över efter Houllier 2004 köptes flera spanska spelare in. Under hans tid i klubben representerade Liverpool-spelarna Xabi Alonso, Luis García, Fernando Morientes, Jose Reina, Álvaro Arbeloa, Fernando Torres och Albert Riera det spanska landslaget. +Uppdaterad 14 september 2026. +Not: Spelare i kursiv stil är inlånade. +Endast tävlingsmatcher, inhopp i matcher inom parentes. Uppdaterat 26 maj 2024. +Endast tävlingsmatcher, antal matcher (inklusive inhopp) inom parentes. Uppdaterat 19 okt 2023. +Listkriterier: +Listan inkluderar alla icke-brittiska spelare (som ej kommer ifrån England, Nordirland, Skottland, Wales eller Irland) som representerat klubben. +Listan inkluderar enbart spelare som representerat klubben i tävlingsmatcher. +Siffrorna står för antal tävlingsmatcher, vinster, förluster och oavgjorda matcher. (Uppdaterad 26 maj 2024) +Engelska mästare: 20 gånger (1901, 1906, 1922, 1923, 1947, 1964, 1966, 1973, 1976, 1977, 1979, 1980, 1982, 1983, 1984, 1986, 1988, 1990, 2020, 2025) +FA-cupen: 8 gånger (1965, 1974, 1986, 1989, 1992, 2001, 2006, 2022) +Engelska ligacupen: 10 gånger (1981, 1982, 1983, 1984, 1995, 2001, 2003, 2012, 2022, 2024) +FA Community Shield: 16 gånger (1964*, 1965*, 1966, 1974, 1976, 1977*, 1979, 1980, 1982, 1986*, 1988, 1989, 1990*, 2001, 2006, 2022) [*=delad] +UEFA Champions League: 6 gånger (1977, 1978, 1981, 1984, 2005, 2019) +Uefa Europa League: 3 gånger (1973, 1976, 2001) +Uefa Super Cup: 4 gånger (1977, 2001, 2005, 2019) +Klubblags-VM: 1 gång (2019) +I statistiken för flest matcher och mål ingår förutom matcher från ligan, FA-cupen, Ligacupen, Europacupen/Champions League, UEFA-cupen och Cupvinnarcupen även matcher i följande mindre turneringar: UEFA Super Cup, Interkontinentalcupen/VM för klubblag, Screen Sport Super Cup och Charity Shield. Notera även att snittet för Ian Rush beror på att han säsongen 1987/88 spelade i Juventus och att han därför spelade sammanlagt femton säsonger i Liverpool. +Rekord kopplade till Liverpool FC: +Publikrekord: 61 905 stycken mot Wolverhampton Wanderers (1951/1952) +Största ligasegern: 10-1 mot Rotherham Town (1896), 9-0 mot Bournemouth (2022) +Största ligaförlusten: 1-9 mot Birmingham City (1954) +Flest ligamatcher: Ian Callaghan, 640 stycken +Flest ligamål: Roger Hunt, 245 mål +Meste spelare genom tiderna: Ian Callaghan, 857 matcher +Meste målgörare genom tiderna: Ian Rush, 346 mål +Flest ligamål på en säsong: Roger Hunt, 41 mål (1961/1962) +Flest landskamper under sin tid i klubben: Steven Gerrard, 114 landskamper +Liverpool Women Football Club bildades 1989 under namnet Newton Ladies, som deltog i seriespel för första gången 1991 under namnet Knowsley Women's Football Club och slutade då på fjärde plats. Sedan 1995 har klubben verkat under dess nuvarande namn, Liverpool WFC (fram till 2018 Liverpool LFC). Liverpools damlag var en av grundarna till FA Women's Super League 2011. Året därpå blev Liverpool den första engelska klubben att erbjuda alla spelare professionella heltidskontrakt, ett beslut som banade vägen för professionaliseringen av damfotbollen i England. Klubben blev engelska mästare säsongerna 2013 och 2014. +Liverpool FC:s säsonger +Goldstein, Dan (2000). The Rough Guide to English Football, 2nd Edition: A Fans' Handbook. ISBN 1-85828-557-7 +Heatley, Michael (2005). The Little Book of Liverpool - A Liverpool A to Z. Kina: Green Umbrella Publishing. ISBN 1-905009-26-7 +Thylin, Stefan (1998). Första Ligan : 1888 : Engelska Ligan 110 år. ISBN 91-88540-97-9 +”Klubbens historia”. liverpoolfc.com. http://www.liverpoolfc.com/history. Läst 27 oktober 2006. +”Matcher och målskyttar säsong för säsong”. LFChistory.net. http://www.lfchistory.net/SeasonArchive/. Läst 27 oktober 2006. +Wikimedia Commons har media som rör Liverpool FC. +Officiell webbplats +Museets webbplats +Praktblåsmyg (Malurus splendens) är en liten långstjärtad tätting i familjen blåsmygar (Maluridae). Den förekommer över stora delar av den australiska kontinenten från centrala västra New South Wales och sydvästra Queensland till Western Australias kust och bebor främst torra eller halvtorra regioner. Den uppvisar en hög grad av sexuell dimorfism där hanen i häckningsdräkt är klarblå med svarta detaljer medan icke-häckande hanar, honor och juveniler till största delen är gråbruna. Detta gav upphov till uppfattningen att hanen var polygam då alla gråbruna individer bedömdes som honor. Idag omfattar arten ett antal taxa som tidigare kategoriserades som egna arter. +Som andra blåsmygar uppvisar praktblåsmygen ett antal ekologiska särdrag. Den är socialt monogam och sexuellt promiskuös, vilket innebär att trots att den bildar par med en hane och en hona så parar sig båda parter med andra individer och hjälper även till att föda upp ungar som kan bli resultatet ifrån sådana möten. Hanarna plockar rosa eller violetta blomblad och visar upp dem för honan som en del av parningsleken. +Dess biotop varierar från skog till torr buskmark, ofta med riklig vegetation för skydd. Till skillnad från den östliga vitbukad blåsmyg (Malurus cyaneus), har den inte anpassat sig väl till mänsklig påverkan av landskapet och har försvunnit från urbaniserade områden. Praktblåsmygen lever främst av insekter som den kompletterar med frön. +Praktblåsmygen är en av tolv arter inom släktet Malurus som alla återfinns i Australien och på Nya Guineas lågland. Dess närmsta släkting är vitbukad blåsmyg och dessa båda helblå arters närmsta släkting utgörs av purpurkronad blåsmyg (Malurus coronatus) som förekommer i nordvästra Australien. +De första specimen av arten insamlades vid King George-sundet, och praktblåsmygen beskrevs sedan 1830 som Saxicola splendens av den franske naturalisten Jean René Constant Quoy och Joseph Paul Gaimard, tre år före John Gould gav den det vetenskapliga namnet Malurus pectoralis och trivialnamnet Banded Superb-warbler. Trots att han placerade arten i det korrekta släktet Malurus, så hade artepitetet som Quoy och Gaimard givit arten förtur. Artepitetet härstammar ifrån latinets splendens, som betyder "skinande". Blåsmygarna är inte nära besläktade med gärdsmygarna. Först placerades släktet i flugsnapparfamiljen Muscicapidae av Richard Bowdler Sharpe, men omplacerades senare till sångarfamiljen Sylviidae av samme auktor, innan den 1975 placerades i den sentida erkända familjen Maluridae. DNA-analyser har visat att familjen är besläktad med Meliphagidae (Honungsfåglar), och Pardalotidae som tillsammans utgör den större överfamiljen Meliphagoidea. +Den nuvarande taxonomin erkänner fyra underarter: +M. s. splendens – förekommer i Western Australia +M. s. musgravei (G. Mathews, 1922) – förekommer i centrala Australien (synonym: M. s. callainus) +M. s. melanotus (Gould, 1841) – förekommer i östra Australiens inland. +M. s. emmottorum (Schodde & Mason, 1999) – förekommer i sydvästra Queensland. +Initialt kategoriserades tre av dessa underarter; melanotus, splendens och musgravei som egna arter då de beskrevs långt ifrån varandra. I och med att Australiens inland blev utforskat blev det uppenbart att taxonen hybridiserar i områden där utbredningsområdena överlappar. På grund av detta omkategoriserades dessa tre taxon 1975 som underarter av Malurus splendens. +Utifrån specimen insamlade av Samuel White beskrevs taxonet callainus 1867 av John Gould som den egna arten M. callainus. Numera är det känt att detta taxon är en hybrid mellan musgravei och melanotus. +Praktblåsmygen är en liten, långstjärtad fågel och arten mäter cirka 14 cm. Den uppvisar en hög grad av sexuell dimorfism. Adult hane i häckningsdräkt har klarblå panna och örontäckare, violett strupe och djupare blå vingovansida, bröst och stjärt. Den har ett svart ögonstreck, som sträcker sig från den svarta näbben till ett svart parti i nacken. Utöver detta har den även ett svart bröstband. Icke-häckande hane är brun med blå inslag på vingarna och en blåaktig stjärt. Honan liknar icke-häckande hanar men har ett rödbrunt område runt ögat och rödbrun näbb. Juvenila hanar ruggar till häckningsdräkt redan under första häckningssäsongen efter att de har kläckts men den kan vara partiell med kvarvarande bruna partier och kan ta ett eller två år till innan den är komplett. Båda könen ruggar under hösten efter häckningen, då hanarna anlägger en eklipsdräkt och de ruggar igen under vinter och vår för att anlägga häckningsdräkt. Hanarnas blå häckningsdräkt, speciellt örontäckarna, är mycket iriserande Den blå fjäderdräkten reflekterar även kraftigt ultraviolett ljus. +M. s. melanotus skiljer sig från nominatformen genom sin svarta övergump, avsaknad av lila toner och ljusare nedre buk. M. s. musgravei har, i förhållande till splendens, ljusare blå, eller turkos ovansida och svart övergump. +Sången har beskrivits som ett kassettband som snabbspolas. Lätet är hårdare och ljudligare än hos andra blåsmygar och varierar från individ till individ. Ett mjukt enstaka trrt fungerar som kontaktläte inom en födosökande grupp, medan varningslätet är ett tsit. Honan yttrar ett purrande läte under ruvningen. +Arten är vida spridd i torra och halvtorra områden i Australien. Dess habitat är vanligtvis torra och buskiga områden med acacia och dvärgeukalyptus men i sydväst även i skogsområden med undervegetation. Den västliga underarten splendens och den östliga melanotus är till största delen stannfåglar medan musgravei delvis är nomadisk. Till skillnad från den östliga vitbukiga blåsmygen (Malurus cyaneus), har praktblåsmygen inte anpassat sig väl till mänsklig påverkan av landskapet och arten har försvunnit från vissa urbaniserade områden. +Praktblåsmygen är främst insektsätare och dess diet består av en stor variation av leddjur som myror, gräshoppor, syrsor, spindlar och skalbaggar. Utöver detta kompletteras dieten med mindre mängder frön, blommor och frukt. Likt de andra blåsmygarna är arten en rastlös och aktiv födosökare, ofta i grupp och främst på barmark i närheten av skydd eller i buskage. Ovanligt för blåsmygarna så födosöker den högre upp i de lägre nivåerna i bladverken av blommande Eucalyptus. Under vår och sommar, då insekterna är talrika och lättfångade, är de aktiva i perioder under dagen och vilar sig mellan födosöken. De söker ofta skydd i grupp under den värsta hettan under dagen. Under vintern är födan mindre lättillgänglig och sökandet efter föda tar större delen av dagen. +Den förflyttar sig i serier av kvicka hopp, och balanserar med hjälp av den långa stjärten, vilken den oftast håller upprätt och i rörelse. De korta, runda vingarna ger god initial lyftförmåga och duger väl för kortare flykt. Utöver detta är praktblåsmygen även en starkare flygare än många andra blåsmygar. +Praktblåsmygen försvarar sina revir i grupper av två till åtta individer året runt. Revirets storlek är i genomsnitt 4,4 hektar i skogs- och hedlandskap och storleken minskar med ökad växtlighet och ökar i förhållande till hur många hanar gruppen omfattar. Gruppen består av socialt monogama par med en eller flera icke-häckande understödjande individer, som kläckts i reviret men som inte nödvändigtvis är avkomman till de par som de understödjer. Praktblåsmygen är sexuellt promiskuös, där partnerna parar sig med andra individer och till och med hjälper till att föda upp ungar som kan bli resultatet ifrån sådana möten. Över en tredjedel av avkomman härstammar ifrån "utomäktenskaplig" parning. De icke-häckande individerna i gruppen hjälper till att försvara reviret och föda upp ungar. Fåglarna i gruppen vilar sida vid sida och ägnar sig även åt gemensam putsning. +De viktigaste bopredatorerna utgörs av flöjtkråka (Gymnorhina tibicen), törnkråkor (Cracticus), skrattkokaburra (Dacelo novaeguineae), kurrawonger (Strepera), kråkor och korpar (Corvus), sottörntrast (Colluricincla umbrina) men även vissa introducerade däggdjur som rödräv (Vulpes vulpes), katt (Felis catus) och svartråtta (Rattus rattus). Likt andra blåsmygar kan den genomföra en form av distraktionsuppvisning för att komma undan hot, eller skydda ungar och ägg från predatorer. De sänker då huvudet och stjärten samtidigt som den burrar upp sig och sträcker ut vingarna. På detta sätt springer den sedan snabbt iväg samtidigt som den ger ifrån sig ett ihållande varningsläte. +Det finns flera olika dokumenterade uppvaktningsbeteenden hos praktblåsmygen. Bland annat ägnar sig hanen åt så kallad "sjöhästsflygning" , uppkallad efter sjöhästen på grund av dess liknande rörelsemönster. Under denna uppvisning flyger hanen med halsen sträckt och huvudfjädrarna uppspärrade, samtidigt som han lutar kroppen från horisontellt läge till vertikalt och med mycket snabb vingföring sänker och höjer sig överdrivet långsamt från marken. Hanen kan också resa de blå fjädrarna på sidan av huvudet så att de ser ut som örontofsar, att vifta med dem lockar till sig honor och fungerar som en hotfull signal mot andra hanar. +Ett annat särdrag som blåsmygarna uppvisar är att hanarna under häckningssäsongen plockar blomblad, och där praktblåsmygshanen främst väljer rosa och lila blad som han visar för den uppvaktade honan. Utanför häckningstid så förekommer det att hanar visar upp blomblad för honor i ett annat revir för att visa upp sig. Arten är socialt monogam och sexuellt promiskuös, vilket innebär att trots att den bildar livslånga par så parar sig båda parter regelbundet med andra individer och hjälper även att föda upp ungar som kan bli resultatet ifrån sådana möten. En del av ungarna härstammar från hanar utanför revirgruppen. Juveniler tas ofta om hand av inte enbart föräldrarna, utan även av andra hanar som också parat sig med honan. +Häckningen pågår från slutet av augusti till januari, men kraftigt regn i augusti kan försena häckningen. Boet byggs av honan och består av en rund eller klotformad struktur av löst sammanvävt gräs och spindelnät med en väl dold öppning nära marken ofta i tjock och taggig vegetation som Acacia pulchella eller arter inom släktet Hakea. En till två kullar läggs per häckningssäsong. Varje kull består av två till fyra smutsvita ägg med rödbruna prickar som mäter 12 × 16 mm. Honan ruvar äggen i ungefär två veckor. Ungarna är flygga efter 10–13 dagar. Juvenilerna stannar i familjegruppen under ett år eller mer och flyttar sedan till en ny revirgrupp, ofta en angränsande revirgrupp, eller också stannar den kvar och intar en dominant position i den ursprungliga revirgruppen. +Praktblåsmygen är ofta värd för häckningsparasiten mulgaglansgök (Chrysococcyx basalis), och det finns även observationer av guldglansgök (Chrysococcyx lucidus). +Arten har ett stort utbredningsområde och beståndet anses vara stabilt. Internationella naturvårdsunionen IUCN listar den därför som livskraftig (LC). +Texten bygger till stora delar på en översättning av engelskspråkiga wikipedias artikel Splendid Fairywren, läst 2010-01-18 +Rowley, Ian & Russell, Eleanor (1997), Bird Families of the World:Fairy-wrens and Grasswrens, Oxford: Oxford University Press, ISBN 0198546904 +Wikimedia Commons har media som rör praktblåsmyg. +Wikispecies har information om praktblåsmyg. +www.birdpedia.com - information om arten, bland annat med ljudfiler av dess läten. +Tree Of Life Web Project – släktskapen inom familjen Maluridae. +Mary Shelley, ursprungligen Mary Wollstonecraft Godwin, född 30 augusti 1797 i London, död 1 februari 1851 i samma stad, var en brittisk (engelsk) roman- och novellförfattare som också publicerade dramatik, essäer, biografier och reseskildringar. Hon är mest känd för sin roman Frankenstein: eller den moderne Prometeus (Frankenstein: or The Modern Prometheus) från 1818. Hon var dotter till den politiske filosofen William Godwin och filosofen och feministen Mary Wollstonecraft samt styvsyster till poeten Claire Clairmont. Mary Shelleys far såg till att hon fick en för den tiden mycket god, om än informell, utbildning som präglades av liberala idéer. Hon gifte sig med en av romantikens poeter, Percy Bysshe Shelley. +1816 tillbringade paret en omtalad sommar vid Genèvesjön i Schweiz tillsammans med Lord Byron, John William Polidori och Claire Clairmont. Där fick Mary idén till romanen Frankenstein. 1818 lämnade paret Shelley Storbritannien och for till Italien, där deras andra och tredje barn dog innan Mary Shelley födde sitt sista och överlevande barn, Percy Florence. 1822 drunknade maken Percy Shelley när hans segelbåt sjönk i en storm i La Spezia-bukten. Ett år senare återvände Mary till England och kom återstoden av sin levnad att ägna sig åt sin sons uppfostran och sin författarkarriär. Hennes sista tio år kom att präglas av sjukdom, troligen orsakad av en hjärntumör som också kom att orsaka hennes död vid 53 års ålder. +Fram till 1970-talet var romanen Frankenstein, samt hennes ansträngningar för att bevara och publicera sin avlidne makes verk, praktiskt taget det enda Mary Shelley uppmärksammades för. Senare års forskning har dock bidragit till en mer omfattande och utvecklad syn på Mary Shelleys livsverk. Det akademiska intresset för hennes övriga romaner har tilltagit; bland annat har man studerat hennes historiska romaner Valperga (utgiven år 1823) och The Fortunes of Perkin Warbeck (utgiven år 1830), den apokalyptiska romanen The Last Man (utgiven år 1826), och hennes sista två romaner Lodore (utgiven år 1835) och Falkner (utgiven år 1837). Studier av hennes mindre välbekanta verk såsom reseberättelsen Rambles in Germany and Italy (utgiven år 1844) och de biografiska artiklar hon skrev för Dionysius Lardners Lives of the Most Eminent Literary and Scientific Men, Lardner's Cabinet Cyclopaedia (mellan åren 1829–1846) har givit stöd åt åsikten att Mary Shelley förblev politiskt radikal hela sitt liv. Hon återkom ofta till grundtesen att samarbete och sympati, särskilt i de former kvinnor applicerar på sitt arbete med hem och familj, borde vara de metoder man skulle använda för att förändra och förbättra det civila samhället. Denna ståndpunkt var en direkt utmaning mot den individualistiska livssyn som romantiken stod för och som upphöjdes av Percy Shelley, och av upplysningens politiska teorier som framförts av hennes far, William Godwin. +Mary Shelley föddes i Somers Town i London, år 1797 och hette då Mary Wollstonecraft Godwin. Hon var den politiske filosofen, författaren och journalisten William Godwins första och den feministiska filosofen Mary Wollstonecrafts andra barn. Modern dog i barnsängsfeber tio dagar efter Marys födelse. Godwin lämnades därmed ensam att uppfostra Mary och hennes äldre halvsyster Fanny Imlay, Wollstonecrafts barn med den amerikanske prospektören Gilbert Imlay. Ett år efter Wollstonecrafts död gav Godwin ut verket Memoirs of the Author of A Vindication of the Rights of Woman (1798), som var avsedd att vara en uppriktig och känslosam hyllning till hans avlidna hustru. Istället väckte boken skandal, då han i den avslöjar att Wollstonecraft haft en kärleksaffär utan att vara gift och dessutom fött ett utomäktenskapligt barn. Mary fick dock läsa boken och lärde sig under sin barndom att vårda minnet efter sin mor. +Marys tidiga barndom var lycklig att döma av de brev William Godwins hushållerska och barnjungfru Louisa Jones skrev. Godwin var dock ofta hårt skuldsatt, och då han upplevde att det var svårt att uppfostra två flickor på egen hand började han se sig om efter en ny hustru. I december 1801 gifte han om sig med Mary Jane Clairmont, en välutbildad kvinna som hade två egna barn, Charles och Claire. De flesta av Godwins vänner ogillade hans nya hustru och har beskrivit henne som grälsjuk, omogen, ful och ilsken. Godwin var dock sin hustru mycket tillgiven och äktenskapet var lyckligt. Mary, å andra sidan, kom att avsky sin styvmor. C. Kegan Paul som skrev en biografi över William Godwin på 1800-talet antyder att styvmodern på klassiskt manér favoriserat sina egna barn på Marys bekostnad. +Tillsammans bestämde sig paret Godwin för att starta förlaget M. J. Godwin, som även sålde barnböcker, kartor, skrivmateriel och spel. Emellertid blev företaget ett misslyckande och Godwin tvingades ta stora lån för att kunna hålla företaget flytande. Han tog också nya lån för att kunna betala tillbaka gamla skulder, vilket bidrog till familjens växande ekonomiska problem. 1809 var Godwins företag nära konkurs, och själv stod han "på förtvivlans rand". Godwin slapp hamna på gäldstuga tack vare bidrag från filosofiska supportrar och meningsfränder såsom Francis Place, som hjälpte honom genom att låna ut mer pengar. +Mary Godwin fick bara en försumbar formell utbildning, men hennes far undervisade henne i en mängd olika ämnen. Han tog med sig barnen på små utflykter i studiesyfte, och de hade tillgång till hans mycket omfattande bibliotek. De tilläts även lyssna på, och samtala med, de många intellektuella besökare fadern mottog i hemmet, till exempel den romantiske poeten Samuel Taylor Coleridge och USA:s tidigare vice-president Aaron Burr. Godwin medgav att han inte följde Mary Wollstonecrafts anvisningar för fostran och utbildning, så som hon beskrivit dem i A Vindication of the Rights of Woman (1792) men Mary Godwin fick trots detta en mycket avancerad och ovanlig utbildning med tidens mått mätt och med tanke på hennes kön. Hon hade en guvernant, en informator, och läste flera av sin fars böcker om romersk och grekisk historia innan de gick i tryck. År 1811 tillbringade hon även sex månader på en internatskola i Ramsgate. När hon var 15 år gammal beskrev hennes far henne som "anmärkningsvärt framåt, något påstridig, och väldigt intelligent. Hennes kunskapstörst är stor, och hennes uthållighet i allt hon tar sig för är i det närmaste oövervinnerlig." +I juni 1812 sände Marys far henne för att bo hos den radikale filosofen William Baxter och hans familj i närheten av Dundee i Skottland. Baxter hade gjort sig känd för sina avvikande åsikter kring religion. Godwin skrev till Baxter: "Jag är angelägen om att hon ska uppfostras...som en filosof, ja till och med som en cyniker". Forskare har föreslagit att det även kan ha funnits hälsomässiga skäl till att Mary skickades på denna resa, kanske ville hennes föräldrar att hon skulle komma bort från familjens ekonomiska problem. Syftet kan även ha varit att introducera henne för radikala, politiska, idéer. Mary Godwin njöt av de vida omgivningarna kring Baxters hus och av umgänget med hans fyra döttrar. År 1813 återvände hon och tillbringade ytterligare tio månader hos familjen. I förordet till Frankenstein år 1831 skriver hon: "Jag skrev då – men i en mycket alldaglig stil. Det var under träden på markerna kring huset, eller på de trädlösa bergens kala sluttningar i närheten, som min verkliga stil, min fantasis luftiga flykt, föddes och utvecklades". +Mary Godwin träffade antagligen den radikale poeten och filosofen Percy Bysshe Shelley för första gången under perioden mellan hennes två vistelser i Skottland. När hon kom hem för andra gången hade Percy Shelley separerat från sin hustru Harriet Shelley och blivit en regelbunden besökare hos Godwin. Han lovade hjälpa Godwin med hans skulder. Percy Shelleys politiska radikalism, i synnerhet hans åsikter om ekonomi som han hade hämtat från Godwins Political Justice (utgiven 1793), hade lett till att han kommit på kant med sin förmögna, aristokratiska familj. De ville att han skulle forma sitt liv efter de normer som gällde för medlemmar av den besuttna adeln, medan han själv ville använda sin förmögenhet till att hjälpa nödställda. Percy Shelley hade därför problem med att komma åt pengar, till dess han kunde tillträda sitt arv, då hans familj ville förhindra att han slösade bort medlen på att försöka komma till rätta med "politiska orättvisor". Efter att under flera månader ha lovat att hjälpa Godwin tillkännagav Shelley till slut att han inte kunde, eller inte ville, betala samtliga Godwins skulder. Godwin upplevde detta som ett svek och blev mycket arg. +Mary och Percy började träffas i hemlighet vid Mary Wollstonecrafts grav på kyrkogården vid St Pancras Old Church och de blev förälskade; hon var nästan 17 år, han var nästan 22. Till Marys stora besvikelse var hennes far emot förhållandet, som han gjorde allt för att försöka avbryta och därmed rädda dotterns "ofläckade rykte". Detta var vid samma tid som Godwin slutligen fick veta att Shelley inte ämnade betala hans skulder. Mary, som senare skrev och berättade om sitt "överdrivna och romantiska förhållande till min far", förvirrades av hans uppträdande. Hon betraktade Percy Shelley som ett förkroppsligande av sina föräldrars liberala, reformivrande, idéer, i synnerhet Godwins åsikt att äktenskapet var ett hindrande monopol, en åsikt han hade förfäktat i Political Justice, men sedan tagit avstånd ifrån. Den 28 juli begav sig paret i hemlighet till Frankrike, och de tog med sig Marys styvsyster Claire. Percys gravida hustru lämnades dock kvar. +Efter att ha övertygat sin styvmor som förföljt dem till Calais att de inte ämnade återvända reste systrarna och Shelley till Paris, och sedan vidare genom det av krig härjade Frankrike till Schweiz. De fick använda sig av de transportmedel som stod till buds, allt från vagn, till åsnor och mulor. "Det var som att befinna sig i en roman, en förkroppsligad romantisk berättelse", erinrade sig Mary Shelley resan år 1826." Medan de reste läste Mary och Percy verk av Mary Wollstonecraft och andra, de skrev en gemensam dagbok och fortsatte även med sina egna författargärningar. I Luzern tvingades de av brist på pengar att vända tillbaka. De färdades på Rhen och fortsatte sedan landvägen till den holländska hamnstaden Maassluis och anlände till Gravesend i Kent den 13 september 1814. +Den situation som väntade Mary Godwin när hon återvände till England var full av komplikationer, även sådana hon inte kunnat förutse. Antingen före eller under resan hade hon blivit gravid. Hon och Percy stod nu utan medel att kunna försörja sig själva, och till Marys förvåning vägrade hennes far att hjälpa dem eller ens ha något med dem att göra. Paret flyttade in i ett par anspråkslösa hyresrum i Somers Town, och senare flyttade de till liknande rum vid Nelson Square. De fortsatte att läsa och skriva lika mycket som tidigare, och de besöktes ofta av Shelleys vänner, till exempel Thomas Jefferson Hogg och författaren Thomas Love Peacock. Ibland tvingades Percy Shelley att lämna hemmet för att gömma sig undan från sina långivare. Parets förtvivlade brev till varandra vid dessa tillfällen visar hur smärtsamt det var för dem att vara åtskilda. +Gravid och vid dålig hälsa tvingades Mary att uthärda Percys lycka då hans hustru Harriet födde honom en son sent år 1814 samt det faktum att han ofta åkte ensam på utflykter med Claire när Mary var för svag för att följa med. Hon tröstades en smula av att få besök av Hogg, som hon till en början tyckt illa om men som sedan blev en nära vän. Det förefaller som om Percy Shelley ville att Mary Godwin och Hogg skulle ha en relation. Mary avvisade inte denna idé, eftersom hon principiellt var för den fria kärleken. I praktiken var det dock så att hon enbart älskade Percy Shelley och hon gick aldrig längre än till en oskyldig flört med Hogg. Den 22 februari, två månader för tidigt, födde Mary en dotter som man inte trodde skulle överleva. Den 6 mars skrev hon till Hogg: +"Min käre Hogg, min baby är död – kan du komma och hälsa på mig så snart som möjligt. Jag längtar efter att få träffa dig – den mådde alldeles bra när jag gick och lade mig – jag vaknade på natten för att amma den, den verkade "sova" så stilla att jag inte ville väcka den. Den var död redan då, men "det" upptäckte vi inte förrän på morgonen – det ser ut som om den dog av kramper att döma av utseendet – Kan du komma – du är en så lugn varelse & Shelley är rädd att jag ska få feber på grund av bröstmjölken – för jag är inte längre en mor nu". (My dearest Hogg my baby is dead—will you come to see me as soon as you can. I wish to see you—It was perfectly well when I went to bed—I awoke in the night to give it suck it appeared to be sleeping so quietly that I would not awake it. It was dead then, but we did not find that out till morning—from its appearance it evidently died of convulsions—Will you come—you are so calm a creature & Shelley is afraid of a fever from the milk—for I am no longer a mother now.) +Som en följd av förlusten av barnet drabbades Mary Godwin av en akut depression. Till sommaren hade hon dock åter blivit gravid och hon repade sig då. Percy Shelleys ekonomi förbättrades dramatiskt då hans farfar, Sir Bysshe Shelley, dog och efterlämnade ett arv åt Percy. Paret åkte till Torquay på semester, och hyrde sig sedan ett tvåvåningshus i Bishopsgate, vid kanten av Windsor Great Park. Man vet ganska lite om denna period av Marys liv, då hennes dagbok från maj 1815 till juli 1816 gått förlorad. Det var i Bishopsgate som Percy skrev sin dikt Alastor, or The Spirit of Solitude, och den 24 januari 1816 födde Mary honom en andra son, William, som uppkallades efter hennes far men snart fick smeknamnet Willmouse. I sin roman The Last Man beskriver hon senare Windsor som en "Edens lustgård". +I maj 1816 reste Mary Godwin, Percy Shelley och deras son till Genève tillsammans med Claire. De planerade att tillbringa sommaren hos poeten Lord Byron, som nyligen hade haft en affär med Claire, vilket lett till att hon var gravid. Följet anlände till Genève den 14 maj 1816. Mary kallade sig under resan "Mrs Shelley". Byron anslöt sig till sällskapet den 25 maj tillsammans med sin unge läkare John Polidori och hyrde sig ett hus, Villa Diodati, i närheten av Genèvesjön. Percy Shelley ett mindre hus, Maison Chapuis, vid stranden i närheten. De ägnade sig åt att skriva, ro på sjön och samtala till långt ut på nätterna. +"Det visade sig bli en blöt och otrevlig sommar", erinrade sig Mary Shelley år 1831, "och ihållande regn tvingade oss ofta att stanna inomhus flera dagar i sträck".("It proved a wet, ungenial summer", Mary Shelley remembered in 1831, "and incessant rain often confined us for days to the house") Sällskapet ägnade sig sådana dagar åt att samtala, bland annat diskuterade man naturfilosofen Erasmus Darwin experiment på 1700-talet som sades ha lett till att han kunnat ge "liv" åt död materia. Man talade även om huruvida det kunde bli möjligt att återge livet till någon som avlidit. medan de satt kring brasan i Byrons villa roade sig sällskapet även med att läsa tyska spökhistorier, vilket föranledde Byron att föreslå att de var och en skulle försöka sig på att skriva en egen historia på ett övernaturligt tema. Strax därefter, i en sorts vakendröm, fick Mary Godwin uppslaget till Frankenstein. +"Jag såg den bleke man som studerat de oheliga konsterna böja knä bredvid den tingest han skapat. Jag såg den ohyggliga skepnaden av en man som låg utsträckt, och sedan, med hjälp av någon mäktig maskin, började skepnaden vakna till liv och röra sig med en obekväm, halvt levande, rörelse. Skrämmande var det, för överlägset skrämmande skulle resultatet bli om någon människa skulle företa sig att håna Skaparens fantastiska plan." ("I saw the pale student of unhallowed arts kneeling beside the thing he had put together. I saw the hideous phantasm of a man stretched out, and then, on the working of some powerful engine, show signs of life, and stir with an uneasy, half vital motion. Frightful must it be; for supremely frightful would be the effect of any human endeavour to mock the stupendous mechanism of the Creator of the world.") +Hon började skriva på vad hon inledningsvis trodde skulle bli en novell. Tack vare uppmuntran från Percy Shelley valde hon sedan att utarbeta texten till sin första roman, Frankenstein: eller den moderne Prometeus, utgiven år 1818. Hon beskrev senare den där sommaren i Schweiz som det tillfälle i livet då "jag lämnade barndomen och steg ut i livet". +När de återvände till England i september flyttade Mary och Percy, tillsammans med Claire som skaffade sig rum i närheten, till Bath där de hoppades att det skulle vara möjligt att hemlighålla Claires graviditet. I Schweiz hade Mary fått två brev från sin halvsyster Fanny Imlay, som skrev om hur olycklig hon var. Den 9 oktober skrev Fanny ett oroväckande brev från Bristol, vilket fick Percy Shelley att ge sig iväg och leta efter henne, utan framgång. På morgonen den 10 oktober hittades Fanny Imlay död i ett hotellrum i Swansea med ett självmordsbrev och en tom flaska laudanum. Den 10 december hittades Percys hustru Harriet drunknad i en sjö i Hyde Park. Båda dessa självmord tystades ner. Harriets familj försökte förhindra att Percy, uppmuntrad av Mary Godwin, skulle få vårdnaden av hans två barn med Harriet. Hans advokater rådde honom att gifta sig, då detta skulle förstärka hans ställning i vårdnadstvisten, så Percy och Mary, som var gravid igen, gifte sig den 30 december 1816 i St Mildred's Church, London. Mr och Mrs Godwin var där vid vigseln, vilket satte punkt för familjekonflikten. +Den 13 januari födde Claire en flicka som först fick namnet Alba, men som senare döptes till Allegra. I mars samma år kom domen att Percy Shelley ansågs vara moraliskt inkompetent att ha hand om sina två barn, och de placerades i stället i fosterhem hos en prästfamilj. I mars flyttade också paret Shelley tillsammans med Claire och Alba till Albion House i Marlow, Buckinghamshire, ett stort, fuktigt hus vid Themsen. Här födde Mary sitt tredje barn, Clara, den 2 september. I Marlow umgicks de med nya vänner, skrev och diskuterade politik. +I början av sommaren 1817 skrev Mary Shelley färdigt Frankenstein, som publicerades anonymt i januari 1818 och av vissa kritiker betraktas som den första science fiction-romanen. Kritikerna och allmänheten utgick från att det var Percy Shelley som var författaren då han skrivit förordet och boken var dedicerad till hans politiske hjälte William Godwin. I Marlow ägnade sig Mary även åt att bearbeta dagböckerna från parets resa till kontinenten år 1814 och hon lade även till material som skrivits i Schweiz år 1816 samt Percys dikt Mont Blanc. Arbetet resulterade i History of a Six Weeks' Tour, som publicerades 1817. Den hösten tvingades Percy Shelley bo borta från hemmet långa perioder, då han hotades att hamna på gäldstuga på grund av sina många och stora skulder. Som en följd av detta, och att paret båda led av dålig hälsa, samt att man riskerade att förlora vårdnaden om de gemensamma barnen beslutade Mary och Percy sig för att lämna England och flytta till Italien den 12 mars 1818. De tog med sig Claire och Alba, och hade aldrig för avsikt att återvända. +Bland det första sällskapet gjorde när de anlänt till Italien var att överlämna Alba till hennes far, Byron, som lovat att ta hand om henne mot ett löfte från Claire om att fortsättningsvis hålla sig borta från henne. Percy, Mary och Claire reste sedan omkring planlöst utan att slå sig ner någonstans. Nyfunna vänner anslöt emellanåt till trion. Man besökte turistmålen samt ägnade sig åt +skrivande och läsande och social samvaro. Italienäventyret omintetgjordes emellertid för Mary Shelley av hennes båda barns död. I september 1818 i Venedig dog Clara i dysenteri, och ett halvår senare i Rom dog William i malaria, tre år fyllda. Tragedierna förde Mary in i en svår depression, som skapade ett avstånd mellan henne och Percy. Percy skrev i sin anteckningsbok: +My dearest Mary, wherefore hast thou gone, +And left me in this dreary world alone? +Thy form is here indeed—a lovely one— +But thou art fled, gone down a dreary road +That leads to Sorrow’s most obscure abode. +For thine own sake I cannot follow thee +Do thou return for mine. +(Min älskade Mary, varför har du lämnat mig alldeles ensam i denna trista värld? Din skepnad är här, och den är vacker, men du har flytt, gett dig av längs en trist väg som leder till Sorgens mest undangömda boning. För din egen skull kan jag inte följa dig, vill du återvända för min.) +För en tid fann Mary tröst i sitt skrivande. Hennes fjärde barn Percy Florence Shelleys födelse var det som slutligen ryckte upp henne ur depressionen. Hon vårdade dock minnet av sina döda barn resten av sitt liv. +Italien försåg paret Shelley, Byron och andra kulturpersonligheter i exil med en politisk frihet som inte var tillgänglig hemma i England. Trots att hon kom att associera Italien med personliga förluster, så betraktade Mary Shelley Italien som "ett land som minnet återgav som ett paradis". Deras år i Italien var en tid av intensiv intellektuell och kreativ verksamhet för både Mary och Percy. Medan Percy skrev en serie av sina mest framstående dikter, ägnade sig Mary åt att skriva den självbiografiska romanen Matilda, den historiska romanen Valperga samt pjäserna Proserpine och Midas. Mary skrev Valperga för att kunna hjälpa sin far ekonomiskt, då han åter hade svåra finansiella problem, och Percy fortfarande vägrade hjälpa honom Hon led ofta av sjukdomar och depressioner. Hon var också tvungen att stå ut med Percys intresse för andra kvinnor. Då Mary Shelley delade hans åsikt att äktenskapet inte borde innebära ett monopol, så ägnade hon sig åt att bygga upp relationer till männen och kvinnorna i deras bekantskapskrets. Hon blev särskilt förtjust i den grekiske revolutionären prins Alexandros Mavrokordatos. +I december 1818 färdades paret Shelley söderut till Neapel, där de stannade i tre månader tillsammans med Claire. Under denna tid tog de bara emot en besökare, en läkare. År 1820 utsattes de för anklagelser och hotelser från två tidigare anställda, Paolo och Elise Foggi, som Percy hade avskedat i Neapel strax efter att paret Foggi hade gift sig. Paret Foggi avslöjade att Percy Shelley den 27 februari 1819 i Neapel hade registrerat att Mary hade fött honom ett barn, en flicka vid namn Elena Adelaide Shelley. Paret Foggi hävdade dock att det var Claire som var barnets mor. I biografier har det framkommit olika teorier kring detta: att Percy Shelley beslutat sig för att adoptera ett barn, att barnet var hans och Claires eller hans och Elises eller någon annan kvinnas eller till och med att barnet skulle ha varit Elises och Lord Byrons. Mary Shelley hävdade bestämt att hon hade vetat om Claire varit gravid, men det är osäkert hur mycket hon faktiskt visste. Händelserna i Neapel en stad Mary senare kallade ett paradis bebott av djävlar,The events in Naples, förblir höljda i dunkel. Det enda som är säkert är att Mary inte var barnets mor. Elena Adelaide Shelley dog i Neapel den 9 juni 1820. +Sommaren 1822 flyttade en gravid Mary tillsammans med Percy, Claire och vännerna Edward och Jane Williams till det ensligt belägna huset Villa Magni, nära San Terenzo. När de slagit sig ner där fick Percy till Claire frambringa den sorgliga nyheten att hennes dotter Allegra hade avlidit i tyfus i ett kloster i Bagnacavallo. Mary Shelley vantrivdes i den trånga och isolerade Villa Magni, som hon kom att betrakta som en fängelsehåla. Den 16 juni drabbades hon av ett missfall, vilket orsakade henne sådana blodförluster att hon nästan avled. Hellre än att invänta läkarhjälp placerade Percy henne i ett badkar fullt med is för att hämma blodflödet. Denna handling fick han senare veta av en läkare räddade hennes liv. Förhållandet mellan makarna var inte bra denna sommar, och Percy tillbringade vanligtvis mycket mer tid med Jane Williams än med sin deprimerade och sjukliga hustru. De flesta av de dikter han skrev i San Terenzo är riktade till Jane, inte till Mary. +Att de bodde nära kusten tillät Percy Shelley och Edward Williams möjligheten att njuta av den "perfekta sommarleksaken" – en ny segelbåt. Båten hade byggts av Daniel Roberts and Edward Trelawny, som tillhörde Byrons beundrare och hade anslutit sig till sällskapet i januari 1822. Den 1 juli 1822 gav sig Percy Shelley, Edward Williams, och Daniel Roberts ut på en segeltur söderut till kusten utanför Livorno. Där diskuterade Percy Shelley, Byron och Leigh Hunt möjligheten att ge ut en radikal tidskrift som de tänkte kalla The Liberal. Den 8 juli gav sig Shelley och Edward Williams iväg för att segla hem igen, tillsammans med en 18-årig skeppshand vid namn Charles Vivian. De kom aldrig fram. Ett brev anlände till Villa Magni från Hunt till Percy Shelley, daterat den 8 juli, där det stod, "vänligen skriv till oss och berätta hur hemresan gick, för det sägs att vädret var dåligt så vi är oroliga". "papperet föll ur mina händer" berättade Mary senare för en vän, "Jag började skaka i hela kroppen". Hon och Jane Williams skyndade sig desperat till Livorno och vidare till Pisa i det svaga hoppet att deras makar fortfarande skulle vara vid liv. Tio dagar efter stormen spolades tre kroppar upp vid kusten utanför Viareggio, halvvägs mellan Livorno och Lerici. Trelawny, Byron och Hunt kremerade sedan Percy Shelleys lik på stranden i Viareggio. +"Frankenstein is the most wonderful work to have been written at twenty years of age that I ever heard of. You are now five and twenty. And, most fortunately, you have pursued a course of reading, and cultivated your mind in a manner the most admirably adapted to make you a great and successful author. If you cannot be independent, who should be?"– William Godwin till Mary Shelley +Efter sin makes död bodde Mary Shelley i ett år hos Leigh Hunt och hans familj i Genua. Här träffade hon ofta Byron och hon hjälpte till att skriva rent hans dikter. Hon beslutade sig för att försöka försörja sig själv och sin son med sitt skrivande, men hon hade det mycket svårt ekonomiskt. Den 23 juli 1823 lämnade hon Genua och återvände till England där hon flyttade in hos sin far och styvmor i deras hus vid the Strand, tills hon fick lite pengar av sin svärfar som tillät henne att hyra ett eget hem i närheten. Sir Timothy Shelley hade först gått med på att ge ett underhåll till sin sonson Percy Florence enbart på villkor att denne lämnades över till en utsedd vårdnadshavare. Detta hade Mary Shelley vägrat att gå med på. Hon lyckades istället få Sir Timothy att betala ut ett mindre, årligt, underhåll, som skulle betalas tillbaka när pojken blev myndig och fick tillträda sitt arv. Sir Timothy vägrade hela sitt liv att sammanträffa med Mary och samtalade bara med henne genom advokater. Mary Shelley sysselsatte sig med att sammanställa sin makes poesi för utgivning, men oro för sonen begränsade hennes handlingsutrymme. Sir Timothy hotade med att upphöra med utbetalningarna om hon lät publicera något biografiskt material om sonen. År 1826 blev Percy Florence juridisk arvtagare till familjen Shelleys egendomar då Charles Shelley, faderns son med Harriet Shelley, avled. Sir Timothy höjde då underhållet till Mary från £00 pund om året till 250 pund, men han fortsatte att göra livet svårt för henne. Mary Shelley njöt av det intellektuella umgänge som William Godwin höll sig med, men fattigdom hindrade henne från att umgås i den utsträckning hon hade önskat. Delar av samhället uteslöt henne också fortfarande ur gemenskapen då de inte kunnat förlika sig med hennes förhållande med Percy Shelley. +Sommaren 1824 flyttade Mary Shelley till Kentish Town i norra London för att kunna vara nära vännen Jane Williams. Det är möjligt att hon var, som Muriel Spark skriver i sin biografi över Mary "lite kär" i Jane. Jane gjorde henne senare mycket besviken genom att sprida ryktet att Percy föredragit henne framför Mary, då Mary varit en undermålig hustru åt honom. Det var ungefär vid denna tid som Mary skrev sin roman The Last Man (1826), och hon ägnade också tid åt att hjälpa vänner som skrev biografiskt material om Byron och Shelley; hon hade alltså redan inlett sin strävan att bidra till sin avlidne makes odödlighet. Hon träffade också en amerikansk skådespelare vid namn John Howard Payne och hon fascinerades av den amerikanske författaren Washington Irving. Payne förälskade sig i henne och 1826 friade han till henne. Hon svarade nej med motiveringen att efter att ha varit gift med ett geni kunde hon bara tänka sig att gifta om sig med ett annat. Payne accepterade avslaget och försökte sedan utan framgång förmå sin vän Irving att också fria till henne. Mary Shelley kände till detta, men det är osäkert om hon tog planen på allvar. +År 1827 var Mary Shelley med och utarbetade en plan som skulle tillåta hennes vänner Isabel Robinson och Isabels älskarinna Mary Diana Dods, som var författare under pseudonymen David Lyndsay, att rymma och bygga upp ett liv tillsammans i Frankrike som man och hustru. Med Paynes hjälp kunde Mary Shelley skaffa fram falska pass till de båda älskande. År 1828 smittades Mary av smittkoppor när hon besökte dem i Paris. Flera veckor senare var hon återställd, men sjukdomen hade berövat henne hennes skönhet, trots att hon inte hade fått de fruktansvärda ärr i ansiktet som var vanligt. +Under perioden 1827–40 var Mary Shelley upptagen med sitt arbete som författare och utgivare av sin mans skrifter. Hon skrev romanerna Perkin Warbeck (1830), Lodore (1835), och Falkner (1837). Hon skrev också artiklar i tidskrifter för damer. Hon hjälpte fortfarande sin far ekonomiskt, och de hjälpte varande med att finna förläggare. År 1830 sålde hon rättigheterna till en ny utgåva av Frankenstein för 60 pund till Henry Colburn och Richard Bentley. När hennes far dog vid 80 års ålder 1836 började hon arbetet med att sammanställa hans memoarer och brev för utgivning, vilket han hade begärt i sitt testamente. Två år senare övergav hon dock detta projekt. Hon lade också ner mycket tid på att propagera för sin makes poesi genom att utarbeta reklamkampanjer och referera till hans verk i sina egna texter. År 1837 var Percy Shelleys verk redan välkända och väckte allt större beundran. Sommaren 1838 föreslog förläggaren Edward Moxon som bland annat gav ut Alfred Lord Tennysons poesi och som var svärson till författaren Charles Lamb, att man skulle ge ut en utgåva med Percy Shelleys samlade verk. Mary fick 500 pund för att utarbeta denna sammanställning som gavs ut under titeln Poetical Works (1838). Sir Timothy insisterade på att utgåvan inte fick innehålla någon biografi, ett problem Mary lyckades lösa genom att inkludera biografiskt material i kommentarerna till dikterna. +Mary Shelley var även fortsättningsvis försiktig med att bilda nya relationer. År 1828 mötte hon och flörtade med, den franske författaren Prosper Mérimée, men det enda brev som är bevarat från Mary till honom verkar vara ett avvisande. Hon blev förtjust när hennes gamle vän från Italien, Edward Trelawny, återvände till England och i brev skämtade de om att gifta sig. De blev dock osams över ett förslag till biografi över Percy som han hade, och som Mary inte ville stödja. Vaga kommentarer i hennes dagbok under 1830-talet tyder på att Mary hyste känslor för den radikale politikern Aubrey Beauclerk, som kanske gjorde henne besviken då han två gånger gifte sig med andra kvinnor. +Mary Shelleys största uppgift under dessa år var dock enligt hennes egen mening att ta hand om sonen, Percy Florence. Hon respekterade sin avlidne makes önskan att hans son skulle gå på en internatskola, och med sir Timothys motvilliga hjälp lyckades hon få honom antagen till Harrow School. För att slippa betala inackorderingsavgifter flyttade hon till Harrow on the Hill så att Percy Florence skulle kunna få sin utbildning som dagelev. Trots att Percy sedan studerade vid Trinity College, försökte sig på både politik och juridik visade han aldrig några drag av den enorma begåvning båda hans föräldrar haft. Han var sin mor varmt tillgiven, och när han lämnade universitetet flyttade han tillbaka hem till henne. +Åren 1840 och 1842 reste mor och son tillsammans ner på kontinenten. Dess resor beskriver Mary Shelley i Rambles in Germany and Italy in 1840, 1842 and 1843 (utgiven år 1844). År 1844 dog slutligen Sir Timothy Shelley vid 90 års ålder, "han föll från stjälken som en vissen blomma", skrev Mary. För första gången var hon och hennes son därmed ekonomiskt oberoende, även om det visade sig att arvet var mindre än de hade hoppats. +Vid mitten av 1840-talet utsattes Mary Shelley för tre separata utpressningsförsök. År 1845 hotade en italiensk emigrant med att publicera brev hon skrivit till honom. En av hennes sons vänner mutade då en polis till att beslagta italienarens alla papper, inklusive dessa brev som därefter brändes. Kort därefter köpte Mary brev hon själv och Percy skrivit av en man som utgav sig för att vara Byrons illegitime son. År 1845 sökte också Percys kusin, Thomas Medwin, upp henne och sade sig ha skrivit en biografi över Percy som skulle dra skam över hans minne. Han erbjöd sig att låta bli att publicera, om Mary gav honom 250 pund, men Mary Shelley vägrade. +År 1848 gifte sig Percy Florence med Jane Gibson St John. Äktenskapet blev lyckligt, och Mary Shelley och Jane tyckte om varandra. Mary bodde hos sin son och sonhustru på Field Place, Sussex, vilket var familjen Shelleys fädernehem. Hon följde även med dem på resor utomlands. +Mary Shelleys sista år försvårades av en serie sjukdomar. Från år 1839 led hon av huvudvärk och tillfälliga anfall av förlamning i olika delar av kroppen, vilka ibland hindrade henne från att läsa och skriva. Den 1 februari 1851 dog hon i London vid 53 års ålder till följd av vad hennes läkare trodde var en hjärntumör. Enligt Jane Shelley hade Mary Shelley bett om att få bli begravd bredvid sin mor och far, men Percy och Jane som tyckte illa om deras gravplats valde istället att låta gräva upp deras kvarlevor och begrava dem bredvid Mary i Bournemouth där de nu bodde. När precis ett år gått efter Marys död öppnade Percy och Jane hennes sekretär. Där fann de några lockar av hennes döda barns hår, en anteckningsbok hon delat med sin make, en kopia av hans dikt Adonis samt ett litet paket innehållande lite av askan efter Percy och en del av hans hjärta. +Mary Shelley levde ett litterärt liv. Hennes far uppmuntrade hennes skrivartalang, bland annat genom att lära henne att skriva vackra brev, och hennes favoritsysselsättning som barn var att skriva berättelser. Alla hennes barndomsalster försvann när hon rymde med Percy Shelley 1814, och inget av hennes återstående manuskript kan dateras före detta år. Hennes första publicerade verk antas vara Mounseer Nongtongpaw vilket är en samling komiska verser skrivna för Godwins Juvenile Library när hon var tio och ett halvt år. Hennes dikt står dock under en annan författares namn i den senaste samlingsutgåvan av hennes verk. Percy Shelley uppmuntrade entusiastiskt hennes skrivande: ”Redan från första början var min man väldigt angelägen att jag skulle visa mig värdig min härkomst, och skriva in mig i berömmelsens bok. Han försökte alltid sporra mig att uppnå litterär ryktbarhet.” +Delar av Mary Shelleys romaner tolkas ofta som maskerade omskrivningar av hennes liv. I synnerhet har kritiker pekat på det återkommande far-dotter-motivet som bevis på detta självbiografiska drag. Exempel på detta är den vanligt förekommande tolkningen av Matilda (1820) där de tre karaktärerna identifieras som Mary Shelley, William Godwin och Percy Shelley. Mary Shelley erkände att hon baserat sina huvudpersoner i The Last Man på sin italienska cirkel. Lord Raymond, som lämnar Europa för att slåss för Grekland och som dör i Konstantinopel, är baserad på Lord Byron; utopisten Adrian, Earl of Windsor, som leder sina anhängare i sökandet efter det förlorade paradiset och som dör när hans båt går under i en storm, är ett skönlitterärt porträtt av Percy Bysse Shelley. I sin recension av Godwins roman Cloudesley (1830), skrev hon dock att hon inte trodde på tesen att författare ”endast kopierade från sina hjärtan”. William Godwin ansåg att hans dotters karaktärer var typer snarare än porträtt från det levande livet. Några moderna kritiker såsom Patricia Clemit och Jane Blumberg, har intagit samma ståndpunkt och tar avstånd från en självbiografisk läsning av Mary Shelleys verk. +Mary Shelley använde sig av tekniker hämtade från många romangenrer, i synnerhet från Godwins romaner, Walter Scotts nya historiska roman och den gotiska romanen. Den Godwinska romanen, som blev populär på 1790-talet genom Godwins Caleb Williams (1794), ”använde sig av den rousseauianska bekännelsens form för att kontrastera relationen mellan jaget och samhället”. Frankenstein utforskar flera av de teman och stilistiska konstgrepp som återfinns i Godwins roman, men Mary Shelley kritiserar de upplysningsideal som Godwin upphöjer i sina verk. I The Last Man använder Shelley sig av den filosofiska formen från Godwins romaner för att påvisa den ultimata meningslösheten i världen. Medan Godwins romaner hade visat hur rationella individer sakteliga kunde förändra samhället, visade The Last Man och Frankenstein på individens bristande kontroll över historien. +Mary Shelley använder sig av den historiska romanens form för att kommentera könens relationer. Romanen Valperga är till exempel en feministisk version av Walter Scotts traditionellt manliga genre. Genom att föra in kvinnor i berättelsen som inte är historiskt belagda kan Shelley ifrågasätta teologiska och politiska institutioner. Mary Shelleys manliga huvudpersons ohämmade ärelystnad ställs i direkt kontrast till det kvinnliga alternativet: förstånd och sensibilitet. I Shelleys andra historiska roman, Perkin Warbeck, symboliserar Lady Gordon värden som vänskap, familjeliv och jämställdhet. Genom henne erbjuder Shelley ett feministiskt alternativ till den manliga maktpolitik som förstör den manlige huvudpersonen. Romanen utmanar den historiska berättelsens såtillvida att Shelleys roman är en historisk berättelse som inkluderar, snarare än skildrar, händelser som rör männens värld. +I och med att det under 1970-talet blev vanligt att bedriva litteraturkritik ur ett feministiskt perspektiv började Mary Shelleys verk, i synnerhet Frankenstein, dra till sig mer uppmärksamhet bland akademikerna. Forskare med en feministisk eller psykoanalytisk synvinkel blev i hög grad ansvariga för Shelleys återupptäckt som författare. Ellen Moers var en av de första att hävda att Shelleys förlust av sitt barn var en avgörande inspiration till att skriva Frankenstein. Hon menar att romanen är en "födelsemyt", där Shelley försöker förlika sig med att både ha orsakat sin egen mors död och att ha förlorat sitt barn. Enligt Moers handlar berättelsen om "vad som händer när en man försöker få ett barn utan en kvinna ... Frankenstein handlar i grund och botten om motsättningarna mellan naturligt och onaturlig reproduktion". Victor Frankensteins misslyckande som förälder i romanen har tolkats som ett uttryck för den ångest som är förknippad med graviditet, förlossning och moderskap. +Sandra Gilbert och Susan Gubar argumenterar i sin bok The Madwoman in the Attic (1979) för att i Frankenstein i synnerhet försöker Shelley följa upp, och svara på, den manliga litterära tradition som tidigare representerats av till exempel John Miltons Det förlorade paradiset. Deras tolkning är att Shelley bekräftar den maskulina traditionen, inklusive det könsförtryck som är en del av den, samtidigt som hon " på ett fördolt sätt introducerar fantasier om jämställdhet som då och då får utbrott i monstruösa bilder av vrede". Mary Poovey ser den första utgåvan av Frankenstein som en del av ett större mönster i Shelleys författarskap, som inleds med försök att bekräfta sig själv genom sin litteratur och som avslutas i mer traditionell femininitet. Poovey menar att Frankensteins många parallella berättelseteman möjliggör för Shelley att leva ut sin konstnärliga personlighet på flera plan; hon kan därmed "uttrycka och gestalta sig själv samtidigt". Shelleys rädsla för att hävda sig själv avspeglas i Frankensteins öde, då han straffas för sin egoism genom att förlora sina kopplingar till hemmet. +Feministiska litteraturvetare fokuserar ofta på hur själva författarskapet, särskilt detta med ett kvinnligt författarskap, representeras i, och av, Shelleys romaner. Som den framstående experten på Shelleys författarskap Anne K. Mellor påpekar, använder sig Shelley av ett nygotiskt uttryck inte bara för att utforska den undertryckta kvinnliga sexualiteten utan även för att "censurera sina egna uttalanden" i Frankenstein. Enligt Poovey och Mellor ville Shelley inte framhäva sin egen författarpersonlighet utan kände sig djupt otillräcklig som författare, vilket "ledde till att hon skapade påhittade abnorma, perverterade och destruktiva skepnader". +Det fokus på familjens roll i samhället, och kvinnans roll i familjen, som Shelley har i sina verk är en hyllning till "feminina känslor och medkänsla", och hon antyder att det civila samhället skulle vara förlorat utan dem. Shelley var "absolut hängiven samarbetets etik, ömsesidigt beroende och självuppoffring". I Lodore, till exempel så beskriver huvudtemat inte huvudkaraktärens öde, utan hans hustrus och döttrars, efter att han dödats i en duell i slutet av den första delen av romanen. Han lämnar därmed sin familj i juridiska, sociala och ekonomiska problem som de kvinnliga karaktärerna får lösa. Romanen behandlar politiska och ideologiska frågor, särskilt intresserar den sig för kvinnors utbildning och sociala status. Den skärskådar den patriarkala kulturen som höll könen åtskilda och gjorde kvinnor beroende av män. En annan expert på Shelley, Betty T. Bennett menar att "Romanen förespråkar en paradigm av jämställd utbildning för kvinnor och män, som skulle föra med sig både social rättvisa och en andlig och intellektuell beredskap att handskas med livets oundvikliga problem". Dock är Falkner den enda av Shelleys romaner där hjältinnan triumferar. Romanens läsart är att när de kvinnliga värderingarna får besegra den våldsamma och destruktiva maskuliniteten kan männen bli fria att uttrycka "sina bättre jags medlidande, sympati och generositet". +Frankenstein blandar liksom många andra gotiska romaner från tidsperioden avståndstagande i objekten med spekulativa och provocerande teman. Hellre än att fokusera på alla vändningar i historien understryker man protagonistens mentala och moraliska kamp. Shelley fyller texten med sin egen form av politiserad romantik, en som ställer sig kritisk till den traditionella romantikens individualism och egoism. Victor Frankenstein för tankarna till Satan i Det förlorade paradiset, han gör uppror mot traditionen, han skapar liv och han formar sitt eget öde. Dessa drag porträtteras inte som något positivt, utan som Blumberg påpekar: "hans omedgörliga ambition är ett självbedrägeri, förklädd till en jakt på sanningen". Han måste överge sin familj för att fullfölja sitt öde. +Mary Shelley trodde på Upplysningen och dess idéer om att människor kan förbättra samhället genom att utöva politiskt ansvar., men hon fruktade att politik som utövades utan ansvar skulle leda till kaos. I praktiken kritiserar hon i sina romaner 1700-talets tänkare som liksom hennes föräldrar trodde att man verkligen skulle kunna åstadkomma en sådan förändring. Frankensteins monster till exempel läser böcker om radikala idéer men den kunskap han får från den är i slutänden meningslös. Shelleys verk avslöjar att hon i grund och botten var mindre optimistisk än Godwin och Wollstonecraft, hon saknar övertygelsen att människan någonsin skulle kunna bli perfekt. +Som litteraturvetaren Kari Lokke skriver: "The Last Man, ännu mer än Frankenstein, bildar genom sitt ifrågasättande av mänskligheten som universums mittpunkt, av vårt privilegierade förhållande till resten av naturen, en grundläggande och profetisk utmaning av västvärldens humanism". I synnerhet Shelleys paralleller till vad världen betraktade som en misslyckad fransk revolution, och de radikala filosofernas syn på densamma, utmanar upplysningens övertygelse om att gemensamma ansträngningar oundvikligen leder till utveckling. Precis som i Frankenstein åstadkommer Shelley i denna roman en fullständigt desillusionerad kommentar till revolutionseran, som leder till ett slutgiltigt avståndstagande från hennes generations progressiva ideal. Inte enbart förkastar hon upplysningens politiska ideal, hon förnekar också romantikens tes att den poetiska eller litterära fantasin kan erbjuda ett alternativ. +Litteraturvetare har tills helt nyligen brukat hänvisa till Lodore och Falkner som bevis på att Shelley successivt blev alltmer politiskt konservativ. År 1984 fick Mary Poovey stort genomslag med sin tes att Shelleys reformivrande idéer alltmer kom att fokuseras på hennes privata sfär, hennes familjeliv. Poovey hävdar att Mary Shelley skrev Falkner för att analysera sin egen tudelade inställning till faderns kombination av liberal radikalism och strikta krav på att man skulle följa samhällets regler och normer. Mellor instämmer i stort sett i detta och menar att "Mary Shelley baserade sin alternativa politiska ideologi på visionen av den fridfulla, kärleksfulla, borgerliga familjen. Underförstått stödde hon därmed den konservativa drömmen om gradvis, evolutionär utveckling." Denna vision tillät kvinnor att befinna sig i den offentliga sfären, men den bar också med sig det inneboende könsförtryck som fanns i den borgerliga familjen. +Senare har dock dessa tidigare slutsatser blivit ifrågasatta. Till exempel hävdar Bennett att Shelleys verk visar på en aldrig upphörande tro på romantikens idealism och hopp om politiska reformer. och Jane Blumbergs studie kring Shelleys tidiga romaner argumenterar för att det inte så enkelt går att dela in hennes karriär i en radikal och en konservativ fas. Hon menar att "Shelley var aldrig en övertygad politisk radikal som hennes make, och hennes livsstil senare i livet var varken plötsligt påkommen eller ett svek. Hon utmanade de facto sin egen cirkels politiska och litterära inflytande redan i sitt allra första verk." Utifrån detta argument kan man se Shelleys tidiga verk som hennes sätt att utmana Godwins och Percy Shelleys politiska radikalism. Victor Frankensteins "tanklösa förnekande av familjen" till exempel ses som ett bevis på Shelleys ständiga fokus på den privata sfären. +På 1820- och 30-talen hände det ofta att Mary Shelley författade berättelser för presentböcker och tidningsmagasin. Hon skrev till exempel sexton noveller åt tidningen The Keepsake, som var inbunden i siden och hade guldkantade blad, och vars målgrupp var medelklassens kvinnor. Mary Shelleys arbeten i den här genren har beskrivits som "litterär hötorgskonst" Litteraturvetaren Charlotte Sussman pekar å andra sidan på att andra framstående författare från samma tidsperiod, till exempel poeterna William Wordsworth och Samuel Taylor Coleridge också skrev för denna lukrativa marknad. Hon förklarar att tidningsmagasinen "var en stor litterär genre under 1820- och 1830-talen", och bland dessa tidningar var The Keepsake den mest framgångsrika. +Många av Shelleys berättelser utspelar sig i tider och på platser som var avlägsna från det tidiga 1800-talets England, såsom Grekland och Henrik IV av Frankrikes regeringstid. Shelley intresserade sig speciellt för "den individuella identitetens bräcklighet" och försökte ofta återge hur "en persons uppgift i världen kan förändras i grunden till följd av känslomässiga stormar, eller på grund av övernaturliga händelser som speglar ett inre tumult". I hennes berättelser kopplas kvinnans värde till hennes kortvariga attraktionskraft på äktenskapsmarknaden, medan mannens identitet kan bevaras och utvecklas genom pengarnas betydelse. Även om Mary Shelley skrev tjugoen noveller åt tidningar mellan åren 1823 och 1839 betraktade hon alltid sig själv som romanförfattare. Hon skrev till vännen Leigh Hunt att "Jag skriver dåliga artiklar som gör att jag känner mig olycklig – men jag tänker ge mig i kast med att skriva en roman och hoppas att dess rena vatten kan hjälpa mig bli av med tidningarnas gyttja". +När Mary Godwin och Percy Shelley rymde till Frankrike sommaren 1814 började de föra en gemensam dagbok som de publicerade 1817 med titeln History of a Six Weeks' Tour. I denna utgåva hade de även med fyra brev, två av dem var, från deras vistelse i Genève 1816, och även Percy Shelleys dikt Mont Blanc. Verket hyllar ungdomlig kärlek och politisk idealism, och följer i spåren av Mary Wollstonecraft och andra som tidigare använt sig av reseskildringen som form för att föra fram filosofiska idéer. Bokens perspektiv är filosofiskt och reformvänligt, snarare än det konventionella för reseskildringar. Särskilt skildras politikens och krigens samhälleliga effekter i Frankrike. Breven paret skrev på sin andra resa skildrar och analyserar de "stora och extraordinära händelser" som äger rum i samband med att Napoleon I besegras vid Waterloo efter sina "hundra dagars återkomst" år 1815. De utforskar även Genèvesjöns och Mont Blancs filosofiska sublimitet på ett sätt som för tankarna till Petrarca och arvet efter Jean-Jacques Rousseau. +Den sista boken av längre format som Mary Shelley skrev, i form av en samling brev, publicerades 1844 med titeln Rambles in Germany and Italy in 1840, 1842 and 1843. Häri beskrevs de resor hon gjort tillsammans med sonen Percy Florence och hans vänner från universitetet. I detta verk följer Mary i sin mors fotspår, då formatet mycket liknar det som Mary Wollstonecraft använde i Letters Written in Sweden, Norway, and Denmark och som Mary Shelley tidigare använt sig av i A History of a Six Weeks' Tour Hon använder här sina egna definitioner av begreppen känsla och sympati för att beskriva både ett faktiskt och ett politiskt landskap. För Shelley är det enda sättet att bygga ett bättre samhälle att skapa sympatiska kopplingar mellan folk och individer: "kunskap, upplysning, att befria sinnet från destruktiva fördomar – en vidare cirkel av sympati för våra medmänniskor – däri ligger nyttan med att resa". Mellan kommentarer om landskap och utsikter, kultur och "människorna, särskilt ur politisk synvinkel", använder hon sig av reseskildringens form för att analysera sin egen roll som änka och mor, och för att göra reflexioner kring den revolutionära nationalismen i Italien. Hon berättar även om sina "pilgrimsfärder" till platser hon associerar till Percy Shelley. Enligt litteraturvetaren Clarissa Orr tjänar Mary Shelleys anammande av en roll som förkroppsligar det filosofiska moderskapet till att ge Rambles en unik form av prosadikt, med "döden och minnet" som centrala teman. Samtidigt tar Shelley ställning för ett jämlikt samhälle, mot monarkin, klasskillnader, slaveri och krig. +Mellan åren 1832 och 1839 skrev Mary Shelley en mängd biografier över kända italienska, spanska, portugisiska och franska män och även några kvinnor åt Dionysius Lardners Lives of the Most Eminent Literary and Scientific Men. Detta verk hade tillkommit för att svara på medelklassens växande efterfråga på bildande litteratur. Fram tills att dessa essäer återutgavs 2002 hade ingen större uppmärksamhet ägnats denna del av hennes produktion. Det ska dock sägas att det är svårt att helt säkert säga vem som skrivit vad i detta samlingsverk. Shelley använde sig i dessa biografier av en narrativ teknik som blivit populär inom genren under 1700-talet och som inspirerats av Samuel Johnson. Tekniken innebar att man kombinerar andrahandskällor och memoarer med anekdoter och kommentarer från författaren. +Shelley använde sig av biografiskrivandet för att "skaffa mig en utbildning i hur man studerar historiefilosofi". och även för att "ge lektioner". Det är viktigt att lägga märke till att dessa "lektioner" ofta innehöll kritik mot patriarkala institutioner. Shelley betonar företeelser som familj, hem, romantik, sympati och medlidande när hon beskriver människor. Hennes övertygelse att kraften från dessa är vad som kan åstadkomma varaktiga förbättringar av samhället förbinder hennes biografiska text med den som användes av tidiga feministiska historiker såsom Mary Hays och Anna Jameson. Dessa essäer kom i samlingsverket att ges ut i över 4 000 exemplar, vilket kan jämföras med hennes romaner som oftast trycktes i hundratal per utgåva. Detta innebar att hennes biografiska essäers föresats att befordra den kvinnliga historieskrivningens sociala agenda" blev ett av Shelleys mest inflytelserika politiska inlägg. +En kort tid efter Percy Shelleys död bestämde sig Mary Shelley för att skriva en biografi över honom. I ett brev daterat den 17 november 1822 skriver hon: "Jag ska skriva boken om hans liv... och på så sätt sysselsätta mig själv med det enda som kan skänka mig tröst". Hon förbjöds dock av sin svärfar, sir Timothy Shelley, att göra detta. Mary började arbeta med att befordra Percys verk och förstärka hans rykte som poet genom att publicera samlingsverket Posthumous Poems 1824. År 1839, samtidigt som hon skrev biografiska artiklar, började hon förbereda en nyutgåva av hans samlade dikter. Denna utgåva kom att bli vad litteraturvetaren Susan J. Wolfson kallar "den händelse som helgonförklarade Percy Shelley". Påföljande år redigerade Mary Shelley en sammanställning av sin makes essäer, brev, översättningar och textfragment, och hon fortsatte sedan under hela 1830-talet att väcka en bredare publik för hans poesi genom att låta trycka utvalda dikter i tidskriften The Keepsake. +Ett sätt för Mary Shelley att kringgå sin svärfars förbud mot en biografi var att i dessa samlingar och nyutgåvor bifoga kommentarer och anekdoter som tillät henne att smyga in biografiskt material. "Jag ska rättfärdiga hans gärningar" tillkännagav hon 1824, "Jag ska göra honom älskad för all framtid." Det var detta mål, menar Blumberg, som föranledde henne att presentera Percys verk för allmänheten på det mest "populära sättet hon kunde komma på". För att skräddarsy hans verk för den viktorianska smaken lät hon framställa Percy Shelley som en romantisk poet, snarare än en radikal, politisk poet. Som Mary Favret skriver: "Den förandligade Percy blev en bild av poesins själva själ". Mary lyckades utmåla Percys politiska radikalism som en form av sentimentalitet, i det hon hävdade att hans republikanska sympatier härstammade från medlidande med nödlidande människor. Hon såg till att föra fram romantiska anekdoter som beskrev hans välvilja, hans kärlek till familjen och till naturen. Hon beskrev också hur hon själv, i rollen som hans "praktiska musa" hade föreslagit förbättringar i texterna medan han skrev. +Trots att hon var så känslomässigt engagerad, visade sig Mary Shelley vara en kunnig och professionell redaktör. Hon försökte få fram en korrekt kronologi för alla hans texter genom att studera hans slarviga, ibland otydbara, anteckningsböcker, och hon inkluderade dikter såsom Epipsychidion som han skrivit till Emilia Viviani, trots att hon egentligen själv hade velat utelämna den. Hon tvingades dock acceptera vissa kompromisser, och som Blumberg påpekar har hon av senare litteraturvetare anklagats för att ha manipulerat Percys verk genom misstolkningar, urval och felskrivningar för att framställa poeten som något han inte var. I princip ville Mary Shelley att varje ord hennes make skrivit skulle publiceras. men hon fann att detta var omöjligt, både med tanke på förläggarnas krav och med tanke på läsekretsens moraliska normer. Till exempel tog hon bort de ateistiska antydningarna i dikten Queen Mab i den första utgåvan. När hon sedan lät dem vara med i den andra blev förläggaren åtalad och fälld för hädelse, även om inget straff utdömdes. Mary Shelleys censur föranledde kritik från Percy Shelleys tidigare umgängeskrets, och kritiker anklagade henne bland annat för okunnig selektivitet. Hennes anteckningar har ändå förblivit en viktig källa för dem som studerar Percy Shelleys författarskap. Som Bennet uttrycker det: "biografiskrivare och litteraturvetare är överens om att Mary Shelleys arbete med att se till att Percys verk fick den uppmärksamhet hon ansåg att de förtjänade var den största enskilda kraft som bidrog till att hans rykte etablerades under en tid då hans livsverk annars hade riskerat att falla i glömska". +Mary Shelley bemöttes med respekt som författare under sin egen livstid, även om kritiker ofta missade hennes texters politiska undertoner. Efter sin död blev hon dock främst ihågkommen som Percy Shelleys hustru och för ett enda verk – Frankenstein. När hennes brev publicerades 1945 skriver till och med redaktören Frederick Jones i förordet att "en samling som den som här föreligger kunde knappast rättfärdigas av brevens allmänna kvalitet eller av Mary Shelleys betydelse som författare. Det är som hustru till Percy Bysshe Shelley som hon väcker vårt intresse". Denna inställning hade inte försvunnit 1980 då Betty T. Bennett publicerade den första kompletta utgåvan av Mary Shelleys brev. Hon förklarar att "Faktum är att Mary Shelley ända till nyligen betraktats som en produkt. Först en produkt av sina föräldrar, William Godwin och Mary Wollstonecraft, och sedan av sin man, Percy Shelley" Inte förrän Emily Sunsteins Mary Shelley: Romance and Reality kom ut 1989 hade det funnits någon fullödig, akademisk avhandling om Mary Shelleys liv och författarskap. +Mary Shelleys sons försök att "viktorianisera" hennes minne genom att censurera det material hon lämnade efter sig bidrog till att eftervärlden bildade sig uppfattningen att Mary Shelley varit en konventionell författare i högre grad än vad hennes texter faktiskt låter oss förstå. Hennes egna metoder att popularisera Percy Shelleys verk, och hennes tendens att undvika offentliga debatter under senare år bidrog till denna uppfattning. Kommentarer från medlemmar av Percy Shelleys umgängeskrets gick också ofta ut på att nedvärdera Mary Shelleys radikalism. Trelawnys Records of Shelley, Byron, and the Author (1878) till exempel hyllade Percy Shelley på bekostnad av Marys insatser, då han ifrågasatte hennes intelligens och till och med att hon verkligen var upphovsman till Frankenstein. Lady Shelley, Percy Florence hustru, svarade på dessa anklagelser genom att 1882 låta publicera en privat utgåva av en brevsamling hon ärvt med titeln Shelley and Mary. +Ända sedan man först skrev om Frankenstein för scenen har det blivit vanligt att ny publik först möter Mary Shelleys verk genom iscensättningar. Den första filmatiseringen kom 1910, sedan gjordes den klassiska versionen av James Whale 1931, uppföljd av till exempel Mel Brooks Young Frankenstein 1974 och Kenneth Branaghs Mary Shelley's Frankenstein 1994. Under 1900-talet kom Mary Shelley att betraktas som en "en-romans författare" snarare än som den yrkesskribent hon var. Flera av hennes verk har inte kommit i nyutgåvor förrän under de allra senaste åren, vilket har hindrat publiken att lära känna hela hennes produktion. De senaste decennierna har dock nästan samtliga hennes texter kommit ut i nya utgåvor, vilket bidragit till uppvärderingen av hennes författarskap. Hennes studievanor och kunskaper, som de framstår av hennes anteckningar och brev, kan nu bättre uppskattas. Hennes egen uppfattning om sig själv som författare har också uppmärksammats: "Jag tror att jag kan försörja mig genom mitt skrivande, och jag inspireras av den tanken". Litteraturvetare och forskare anser idag att Mary Shelley är en av de största romantiska författarna och att hon är betydelsefull på grund av sina litterära verk samt sin offentliga röst som liberal kvinna. +History of a Six Weeks' Tour (1817) +Frankenstein; or, The Modern Prometheus (1818) +Frankenstein (översättning av Monica Stolpe, Christofer, 1959) +Frankenstein (anonym översättning?, Delta, 1974) +Frankenstein (översättning: Anna Pyk, Niloe, 1982) +Frankenstein eller den moderne Prometeus (översättning Måns Winberg, Bakhåll, 2008) +Matilda (1819) +Matilda (översättning av Daniel Martinez, Sjösala, 2018) +Valperga (1823) +Posthumous Poems of Percy Bysshe Shelley (1824) +The Last Man (1826) +Den sista människan (översättning Erik Carlquist, Hström, 2005) +The Fortunes of Perkin Warbeck (1830) +Lodore (1835) +Falkner (1837) +The Poetical Works of Percy Bysshe Shelley (1839) +Rambles in Germany and Italy in 1840, 1842, and 1843 (1844) +Mörk romantik +Skräcklitteratur +Den här artikeln är helt eller delvis baserad på material från engelskspråkiga Wikipedia, Mary Shelley, läst 3 okt 2009. +Shelley, Mary Wollstonecraft; Robinson Charles E. (1976) (på engelska). Collected tales and stories: with original engravings. Baltimore: Johns Hopkins U.P. Libris 5675153. ISBN 0-8018-1706-4. https://archive.org/details/collectedtalesst00mary +Shelley, Mary. Frankenstein; or, The Modern Prometheus. Ed. Susan J. Wolfson. New York: Pearson Longman, 2007. ISBN 0321399536. +Shelley, Mary. The Journals of Mary Shelley, 1814–44. Ed. Paula R. Feldman and Diana Scott-Kilvert. Baltimore: Johns Hopkins University Press, 1995. ISBN 0801850886. +Shelley, Mary. The Last Man. Ed. Morton D. Paley. Oxford: Oxford Paperbacks, 1998. ISBN 0192838652. +Shelley, Mary. Lodore. Ed. Lisa Vargo. Ontario: Broadview Press, 1997. ISBN 1551110776. +Shelley, Mary. Mary Shelley's Literary Lives and Other Writings. 4 vols. Ed. Tilar J. Mazzeo. London: Pickering & Chatto, 2002. ISBN 1851967168. +Shelley, Mary. Mathilda. Ed. Elizabeth Nitchie. Chapel Hill: University of North Carolina Press, 1959. Project Gutenberg. Hämtad den 16 february 2008. +Wollstonecraft, Mary; Todd Janet, Shelley Mary Wollstonecraft (1992) (på engelska). Mary: Maria. Penguin classics. London: Penguin. Libris 4579877. ISBN 0-14-043371-6 +Shelley, Mary, ed. The Poetical Works of Percy Bysshe Shelley. London: Edward Moxon, 1840. Google Books. Hämtad den 6 april 2008. +Shelley, Mary. Selected Letters of Mary Wollstonecraft Shelley. Ed. Betty T. Bennett. Baltimore: Johns Hopkins University Press, 1995. ISBN 0801848865. +Shelley, Mary. Valperga. Ed. Michael Rossington. Oxford: Oxford Paperbacks, 2000. ISBN 0192832891. +Shelley, Percy Bysshe. Shelley's Poetry and Prose. Eds. Donald H. Reiman and Neil Fraistat. 2nd ed. New York: W. W. Norton and Co., 2002. ISBN 0393977528. +Bennett, Betty T. (1992). Brinkley Robert, Hanley Keith. red (på engelska). Romantic Revisions. Cambridge: Cambridge University Press. Libris 5017346. ISBN 0-521-38074-X. https://archive.org/details/romanticrevision0000unse +Bennett, Betty T., ed. Mary Shelley in her Times. Baltimore: Johns Hopkins University Press, 2003. ISBN 0801877334. +Bennett, Betty T. (1998) (på engelska). Mary Wollstonecraft Shelley: an introduction. Baltimore, Md.: Johns Hopkins University Press. Libris 5677610. ISBN 0-8018-5975-1 (inb.). https://archive.org/details/marywollstonecra0000benn_e5t2 +Bennett, Betty T. (1978). Reiman Donald H.. red (på engelska). The evidence of the imagination: studies of interactions between life and art in English romantic literature. The Gotham library of the New York university press. New York: New York U.P. Libris 5749214. ISBN 0-8147-7372-9 (inb.). https://archive.org/details/evidenceofimagi00atbr +Bieri, James (2005) (på engelska). Percy Bysshe Shelley: a biography : exile of unfulfilled reknown, 1816-1822. Newark: University of Delaware Press. Libris 9526854. ISBN 0-87413-893-0 (alk. paper). https://archive.org/details/percybyssheshell0000bier +Blumberg, Jane. Mary Shelley's Early Novels: "This Child of Imagination and Misery". Iowa City: University of Iowa Press, 1993. ISBN 0877453977. +Brewer, William D. "William Godwin, Chivalry, and Mary Shelley's The Fortunes of Perkin Warbeck". Papers on Language and Literature 35.2 (Spring 1999): 187–205. Rpt. on bnet.com. Hämtat den 20 february 2008. +Bunnell, Charlene E. "All the World's a Stage": Dramatic Sensibility in Mary Shelley's Novels. New York: Routledge, 2002. ISBN 0415938635. +Carlson, Julie Ann (2007) (på engelska). England's first family of writers: Mary Wollstonecraft, William Godwin, Mary Shelley. Baltimore, Md.: Johns Hopkins University Press. Libris 10532139. ISBN 978-0-8018-8618-8 (hbk.). https://archive.org/details/englandsfirstfam0000carl +Clemit, Pamela. "From The Fields of Fancy to Matilda." Mary Shelley in her Times. Ed. Betty T. Bennett. Baltimore: Johns Hopkins University Press, 2003. ISBN 0801877334. +Clemit, Pamela (1993) (på engelska). The Godwinian novel: the rational fictions of Godwin, Brockden Brown, Mary Shelley. Oxford English monographs, 99-0112816-1. Oxford: Clarendon. Libris 4620918. ISBN 0-19-811220-3. https://archive.org/details/godwiniannovelra0000clem +Conger, Syndy M., Frederick S. Frank, and Gregory O'Dea, eds. Iconoclastic Departures: Mary Shelley after "Frankenstein". Essays in Honor of the Bicentenary of Mary Shelley's Birth. Madison, NJ: Farleigh Dickinson University Press, 1997. ISBN 0838636845. +Eberle-Sinatra Michael, red (2000) (på engelska). Mary Shelley's fictions: from Frankenstein to Falkner. Basingstoke: Macmillan. Libris 4837556. ISBN 0-333-77106-0 (St. Martin). https://archive.org/details/maryshelleysfict0000unse +Schor Esther H., Mellor Anne K., Fisch Audrey A., red (1993) (på engelska). The other Mary Shelley: beyond Frankenstein. New York: Oxford University Press. Libris 4611470. ISBN 0-19-507740-7. https://archive.org/details/othermaryshelley0000unse +Frank, Frederick S. "Mary Shelley's Other Fictions: A Bibliographic Consensus". Iconoclastic Departures: Mary Shelley after "Frankenstein". Essays in Honor of the Bicentenary of Mary Shelley's Birth. Ed. Syndy M. Conger, Frederick S. Frank, and Gregory O'Dea. Madison, NJ: Farleigh Dickinson University Press, 1997. ISBN 0838636845. +Gilbert, Sandra M.; Gubar Susan (1984) (på engelska). The madwoman in the attic: the woman writer and the nineteenth-century literary imagination ([New ed.]). New Haven: Yale Univ. Press. Libris 4768245. ISBN 0-300-02596-3 +Gittings, Robert; Manton Jo (1992) (på engelska). Claire Clairmont and the Shelleys: 1798-1879. Oxford: Oxford University Press. Libris 4623172. ISBN 0-19-818594-4. https://archive.org/details/claireclairmonts0000gitt +Holmes, Richard. Shelley: The Pursuit. 1974. London: Harper Perennial, 2003. ISBN 0007204582. +Jump, Harriet Devine, Pamela Clemit, and Betty T. Bennett, eds. Lives of the Great Romantics III: Godwin, Wollstonecraft & Mary Shelley by Their Contemporaries. London: Pickering & Chatto, 1999. ISBN 1851965122. +Levine George, Knoepflmacher Ulrich Camillus, red (1979) (på engelska). The endurance of Frankenstein: essays on Mary Shelley's novel. Berkeley. Libris 5004150. ISBN 0-520-03612-3 +Mellor, Anne Kostelanetz (1989[1988]) (på engelska). Mary Shelley: her life, her fiction, her monsters. New York: Routledge. Libris 4923097. ISBN 0-415-90147-2 (pbk). https://archive.org/details/maryshelleyherli0000mell_i5g1 +Myers, Mitzi. "Mary Wollstonecraft Godwin Shelley: The Female Author between Public and Private Spheres." Mary Shelley in her Times. Ed. Betty T. Bennett. Baltimore: Johns Hopkins University Press, 2003. ISBN 0801877334. +Orr, Clarissa Campbell. "Mary Shelley's Rambles in Germany and Italy, the Celebrity Author, and the Undiscovered Country of the Human Heart". Romanticism On the Net 11 (August 1998). Hämtad den 22 february 2008. +Poovey, Mary. The Proper Lady and the Woman Writer: Ideology as Style in the Works of Mary Wollstonecraft, Mary Shelley and Jane Austen. Chicago: University of Chicago Press, 1985. ISBN 0226675289. +Schor Esther H., red (2003) (på engelska). The Cambridge companion to Mary Shelley. Cambridge companions to literature. Cambridge, UK: Cambridge University Press. Libris 9027896. ISBN 0-521-80984-3 (inb.). https://archive.org/details/isbn_9780521007702 +Seymour, Miranda (2000) (på engelska). Mary Shelley. London: John Murray. Libris 8302529. ISBN 0-7195-5711-9. https://archive.org/details/maryshelley00seym +Sites, Melissa. "Re/membering Home: Utopian Domesticity in Mary Shelley's Lodore". A Brighter Morn: The Shelley Circle's Utopian Project. Ed. Darby Lewes. Lanham, MD: Lexington Books, 2003. ISBN 0739104721. +Shelley, Mary Wollstonecraft; Smith Johanna M. (2000) (på engelska). Frankenstein: complete, authoritative text with biographical, historical, and cultural contexts, critical history, and essays from contemporary critical perspectives. Case studies in contemporary criticism, 99-1950712-1 (2. ed.). Boston, Mass.: Bedford/St. Martin's. Libris 4795962. ISBN 0-312-19126-X (hft.). https://archive.org/details/frankensteincomp00shel +Spark, Muriel (1988) (på engelska). Mary Shelley (Repr.). London: Constable. Libris 4525878. ISBN 0-09-468460-X +James Edward, Mendlesohn Farah, red (2003) (på engelska). The Cambridge companion to science fiction. Cambridge companions to literature. Cambridge: Cambridge University Press. Libris 9003183. ISBN 0-521-01657-6 (pbk.). https://archive.org/details/isbn_9780521016575 +St. Clair, William (1989) (på engelska). The Godwins and the Shelleys: the biography of a family. London: Faber. Libris 5054213. ISBN 0-571-15422-0. https://archive.org/details/godwinsshelleysb0000stcl_y7n2 +Sterrenburg, Lee. "The Last Man: Anatomy of Failed Revolutions". Nineteenth Century Fiction 33 (1978): 324–347. +Sunstein, Emily W. (1991) (på engelska). Mary Shelley: romance and reality. Baltimore, Md.: Johns Hopkins Univ. Press. Libris 5676417. ISBN 0-8018-4218-2. https://archive.org/details/maryshelleyroman00suns +Wake, Ann M Frank. "Women in the Active Voice: Recovering Female History in Mary Shelley's Valperga and Perkin Warbeck". Iconoclastic Departures: Mary Shelley after "Frankenstein". Essays in Honor of the Bicentenary of Mary Shelley's Birth. Ed. Syndy M. Conger, Frederick S. Frank, and Gregory O'Dea. Madison, NJ: Farleigh Dickinson University Press, 1997. ISBN 0838636845. +White, Daniel E. "'The god undeified': Mary Shelley's Valperga, Italy, and the Aesthetic of Desire". Romanticism on the Net 6 (May 1997). Hämtad den 22 february 2008. +Wikimedia Commons har media som rör Mary Shelley. +Originalverk av Mary Shelley på engelskspråkiga Wikisource. +Mary Shelley på Project Gutenberg +Frankenstein: A New Reality! +Mary Shelley chronology and bibliography – ingår i Romantic Circles +Nuncio, född 17 april 2011 på Hanover Shoe Farms i Hanover, Pennsylvania, är en hingst av rasen amerikansk travare, mest känd för sin seger i Elitloppet 2016. +Nuncio tränades under större delen av sin tävlingskarriär av Stefan Melander på gården Tillinge-Åby utanför Enköping, och sköttes av sambon Catarina Lundström. Hästen köptes av Melander för 7 000 dollar den 6 november 2012 på en hästauktion i Harrisburg, Pennsylvania men sedan december 2017 är även Menhammar stuteri delägare. Som två- och treåring tävlade Nuncio i Nordamerika där han först stod i träning hos Jim Oscarsson och sedan hos Jimmy Takter. I februari 2015 överfördes han till Sverige och placerades i Melanders egen träning där han sedan stannade karriären ut. +Nuncio tävlade åren 2013–2017 och tillhörde under denna period travets världselit. Han gjorde totalt 63 starter vilket resulterade i 42 segrar, 14 andraplatser och 4 tredjeplatser. Han sprang in 28,8 miljoner kronor – siffror som gör honom till Andover Halls vinstrikaste avkomma och Melanders genom tiderna vinstrikaste häst. Fram till och med karriärens 54:e start var han aldrig sämre än trea i ett lopp (39 vinster, 13 andraplatser, 2 tredjeplatser). Han tog 15 raka segrar från november 2015 till mars 2017. Nuncio var obesegrad under hela 2016 då han också blev årets vinstrikaste häst i Sverige där han följdriktigt också utsågs till "Årets Häst" samma år. År 2020 hyllades han genom att få ett eget travlopp på Solvalla uppkallat efter sig, Nuncios Lopp. +Nuncio tog karriärens största seger i Elitloppet 2016. Bland andra stora segrar räknas Kentucky Futurity (2014), Yonkers Trot (2014), Sprintermästaren (2015), Breeders' Crown (2015), Solvalla Grand Prix (2015), Oslo Grand Prix (2016), Årjängs Stora Sprinterlopp (2016), Jubileumspokalen (2016), Sundsvall Open Trot (2016) och UET Trotting Masters (2016). Han kom även på andraplats i Peter Haughton Memorial (2013) och Hambletonian Stakes (2014) samt på tredjeplats i Elitloppet (2015). +Nuncio är en hingst med mankhöjden 161 cm. Han har lång smal hals, lång markerad manke, smal bringa, liggande bogar, djup i bålen, ett smalt och något sluttande toppigt kors. Långa kotor, korta skenor fram och långa skenor bak, bred has och något tåvid bak. +Nuncio är svartbrun med ett par signalement. På huvudet har han en stjärn, vilket är en vit fläck mellan ögonen. På höger ben fram har han en oregelbunden vit krona med svarta fläckar. På vänster ben fram en oregelbunden vit kota med svarta fläckar. På vänster ben bak en vit halvstrumpa med svarta fläckar. +Nuncio föddes på Hanover Shoe Farms i Hanover i Pennsylvania, vilket är ett av världens största stuterier. Hanover Shoe Farms säljer varje år i början av november sina åringar på hästauktionen i Harrisburg, vilken varar i sex dagar och är Nordamerikas största auktion. Stefan Melander brukade besöka den i travkretsar välkända auktionen och hade tidigare ropat in stjärnhästar som Scarlet Knight och Super Photo Kosmos där. Han besökte auktionen återigen 2012, och ropade då in nummer 544 Nuncio för 7 000 dollar (motsvarande cirka 50 000 kronor med dåvarande växelkurs) under auktionens andra dag den 6 november. Melander har förklarat att han inte förstod det låga priset för Nuncio och att han hade varit beredd att gå högre i budgivningen. Han ska vid köpet ha sagt till den person som visade upp Nuncio att "den här hästen kommer att vinna Hambletonian". +För att sätta priset i perspektiv kan konstateras att det under den andra dagen av auktionen såldes totalt över 300 hästar och Nuncio tillhörde de 50 billigaste. Totalt under dag två såldes 260 hästar för en högre summa än Nuncio och 46 såldes för en lägre summa (43) eller samma summa (3) som Nuncio. Den dyraste hästen under dag två köptes för 275 000 dollar. Den dyraste hästen under hela 2012 års hästauktion på Harrisburg köptes för 380 000 dollar. Totalt köpte Melander sex hästar under auktionen i Harrisburg 2012 – Nuncio (dag 2), Simple Solution (dag 2), Bella Beluga (dag 3), Viking Eagle (dag 3), Stirling Cheer (dag 4) och Party Boat Hanover (dag 4). Nuncio var billigast av dessa medan Party Boat Hanover, som köptes för 32 000 dollar, var dyrast. +Efter att Melander köpt Nuncio vid hästauktionen i Harrisburg var planen att ta med honom hem till Sverige för träning i egen regi. Det visade sig dock att Nuncio inte var importberättigad till Sverige eftersom hans mor Nicole Isabelle inte hade ett tillräckligt snabbt löpningsrekord i tävlingssammanhang för gångarten trav (1.13,4 som slagen). Detta gjorde att Melander blev tvungen att hitta en tränare i Nordamerika till Nuncio, vilket blev den svenske tränaren Jim Oscarsson som under denna tid var verksam vid träningsanläggningen Palema Trotting Center i Vero Beach i Florida. +Nuncio gick ett första kvallopp på Meadowlands Racetrack i East Rutherford i New Jersey den 1 juni 2012, där han kördes av Oscarssons son Kevin. Nuncio gjorde därefter sin första start och tog sin första seger den 12 juli 2013 i ett tvååringslopp på Meadowlands Racetrack tillsammans med kusken John Campbell. Två veckor senare följdes segern i debutloppet upp med en start i ett försökslopp till världens största tvååringslopp Peter Haughton Memorial. Han slutade på andraplats bakom Father Patrick i försöket, och kvalificerade sig därmed för final den 3 augusti under 2013 års Hambletoniandag på Meadowlands. I finalen kom han återigen på andraplats bakom Father Patrick. Efter andraplatsen i Peter Haughton Memorial följde två raka segrar, varav den ena i Reynolds Memorial Stakes den 5 september på Vernon Downs i Vernon i New York där han segrade på nytt banrekord för tvååringar. Nästa start gjordes i semifinalen av amerikanska Breeders Crown för tvååriga hingstar och valacker den 11 oktober på Pocono Downs i Wilkes-Barre i Pennsylvania. Där kom han på andraplats efter Father Patrick. Andraplatsen innebar dock att Nuncio kvalificerade sig för finalen, som gick av stapeln den 19 oktober. I finalen placerade han sig återigen på andraplats slagen av Father Patrick. +Efter Breeders Crown tvingades ägaren Stefan Melander i november att flytta Nuncio från Jim Oscarssons till Scott Andrews träning. Detta eftersom Oscarsson valde att flytta sin verksamhet tillbaka till Sverige. I de två första starterna hos Andrews vann Nuncio både försöks- och finalloppen av Matron Stakes för tvååringar på Dover Downs Racetrack i Dover i Delaware. I finalloppet den 17 november 2013 satte Nuncio och kusken Campbell nytt banrekord för tvååringar på Dover Downs. I nästa start avslutade Nuncio säsongen med en andraplats i Valley Victory Trot den 30 november 2013 på Meadowlands. Totalt sprang Nuncio in över 400 000 dollar på tio starter under debutsäsongen, vilket gjorde honom till säsongens näst vinstrikaste tvååriga travhäst i Nordamerika. +Inför säsongen 2014 flyttade ägare Melander i december 2013 Nuncio till Jimmy Takters träning. Den 2 augusti 2014 på Meadowlands Racetrack kom Nuncio på andraplats i Hambletonian Stakes, slagen av stallkamraten Trixton med en halv längd. I samband med detta satte han även sitt personliga löpningsrekord 1:50.3 över sprinterdistansen 1609 meter. Drygt två månader efter andraplatsen i Hambletonian vann han i oktober Kentucky Futurity på The Red Mile i Lexington. Denna seger följdes upp med en seger i även Yonkers Trot den 25 oktober 2014 på Yonkers Raceway i Yonkers. Efter två raka storloppssegrar tog Nuncio den tredje raka storloppssegern när han den 6 november 2014 vann Matron Stakes för treåringar på Dover Downs. Efter loppet påpekade kusken John Campbell hur Nuncio "aldrig gör ett dåligt lopp" utan är en av de jämnaste hästar han kört. Nuncio gjorde säsongens sista start den 22 november 2014 på Meadowlands i amerikanska Breeders Crown för treåriga hingstar och valacker, där han slutade på tredjeplats. +Nuncio var den näst vinstrikaste travhästen i kullen i Nordamerika under säsongen 2014. Endast Father Patrick var vinstrikare. Totalt sprang Nuncio in 1,5 miljoner amerikanska dollar på 17 starter under året. Vid en sammanfattning av säsongen bör även nämnas att Nuncio var den som dittills varit närmast att ta en Triple Crown inom amerikansk travsport sedan Glidemaster 2006. Detta då Nuncio vann de två Triple Crownloppen Kentucky Futurity och Yonkers Trot men däremot inte Hambletonian Stakes, utan var där som tvåa slagen med en halv längd. Utöver Glidemaster har endast sju andra travare sedan starten 1955 lyckats med bedriften att ta en Triple Crown. +Efter två framgångsrika säsonger som unghäst i Nordamerika beslutade ägare Melander att importera Nuncio till Sverige. Den 11 februari 2015 genomfördes importen och Melander placerade Nuncio i sin egen träning på gården Tillinge-Åby utanför Enköping. I The Onions Lopp på den nya hemmabanan Solvalla den 22 april 2015 debuterade Nuncio på svensk mark. Han startade från spår 9 med Melander i sulkyn och segrade med två längder från positionen utvändigt om ledaren. Segern i debutloppet följdes upp med ytterligare en seger, tagen den 6 maj i Dartster F:s Lopp på Solvalla. Efter den andra raka segern på svensk mark blev fyraårige Nuncio inbjuden till 2015 års upplaga av Elitloppet. Elitloppet gick av stapeln den 31 maj på Solvalla. Nuncio startade i det första av de två försöken och slutade i detta på andraplats bakom vinnande Royal Fighter, vilket innebar att han kvalificerade sig för finalloppet som kördes senare under eftermiddagen. I finalen slutade han på tredjeplats, slagen av vinnaren Magic Tonight och tvåan Mosaique Face med en halv längd. Nuncio travade i tredjespår under hela finalloppet, och han ansågs därför av sin ägare, tränare och kusk Melander vara "den moraliska vinnaren" av loppet. Nuncio blev med sitt deltagande i Elitloppet 2015 den 48:e fyraåringen att delta i loppet. Med sin tredjeplats i finalen tillhör han även skaran av de mest framgångsrika fyraåringarna i loppets historia – enbart Mack Lobell som segrade 1988 och Peace Corps som slutade på andraplats 1990 har som fyraåringar lyckats bättre. +Tredjeplatsen i Elitloppet följdes upp med segrar både i försöks- och finallopp av Sprintermästaren den 2 juli 2015 på Halmstadtravet. Segern var hans första seger i ett Grupp 1-lopp i Europa. Därefter startade han den 5 augusti i ett fyraåringslopp på Solvalla, där han segrade på nytt svenskt rekord för utländska fyraåringar efter en lång upploppsspurt förbi loppets ledare Daley Lovin. Drygt två veckor senare, den 19 augusti, segrade han i ett fyraåringslopp inom V75 på Solvalla. Den 10 september startade han i Fyraåringsstjärnan på Gävletravet. I loppet kördes han barfota runt om för första gången i karriären och segrade på löpningsrekordet 1.09,6 över sprinterdistansen 1640 meter med autostart. Han tog sin sjätte raka seger sedan tredjeplatsen i Elitloppet den 7 oktober i Cancerfondens Lopp på Solvalla. Med sex raka segrar i ryggen startade han i semifinalen av Breeders' Crown för fyraåriga hingstar och valacker den 25 oktober på Solvalla. Han travade i ledningen men besegrades med en hals av storskrällen Owen Face som kördes av Adrian Kolgjini. Med andraplatsen kvalificerade han sig dock för final. Finalen gick av stapeln den 8 november på Sundbyholms travbana utanför Eskilstuna. Han vann finalen och tog därmed också sin andra raka seger i ett europeiskt Grupp 1-lopp. Han avslutade sedan säsongen med en seger i Solvalla Grand Prix på Solvalla den 14 november. Med denna seger, värd 400 000 kronor, gick Nuncio om Scarlet Knight som Melanders genom tiderna näst vinstrikaste häst (endast efter Gigant Neo). Totalt sprang Nuncio in 3,6 miljoner kronor på 13 starter varav 10 segrar under sin första säsong i Sverige, vilket gjorde honom till den svenska travsäsongens vinst- och segerrikaste fyraåring och femte vinstrikaste travhäst. På topplistan över säsongens 30 vinstrikaste hästar var han och Readly Express de enda som aldrig var sämre än trea i något lopp under året. +Nuncio inledde säsongen 2016 med årsdebut i Kentucky Fibbers lopp den 30 mars på Solvalla. Han vann årsdebuten och segermarginalen beskrevs som "många längder". Årsdebuten följdes upp med segrar i The Onions Lopp den 13 april och Dartster F:s Lopp den 27 april på Solvalla, vilket innebar att Nuncio försvarade sina segrar i dessa lopp från föregående år. Segern i Dartster F:s Lopp togs på tiden 1.09,9 över sprinterdistansen 1640 meter, vilket var den dittills snabbaste tiden i Europa under 2016. +Säsongens fjärde start gjordes den 11 maj i Meadow Roads Lopp på Solvalla. Efter segern i loppet fick Nuncio en inbjudan till att delta i Elitloppet den 29 maj 2016 på Solvalla. Den 22 maj 2016 blev det officiellt att tränare Melander satte upp Örjan Kihlström som kusk att köra Nuncio i Elitloppet istället för sig själv. Kihlström valde att köra Nuncio framför sin fjolårsvinnare av Elitloppet, Magic Tonight. Elitloppet 2016 gick av stapeln den 29 maj. I försöksloppet skar Nuncio mållinjen på andraplats, och kvalificerade sig därmed för final. I efterhand framkom det dock att försöksvinnaren Un Mec d'Héripré (tränad av Fabrice Souloy) varit dopad i loppet, varför Nuncio i efterhand tilldelades segern i loppet. I finalloppet vann Nuncio från utvändigt ledaren på tiden 1.09,2 över sprinterdistansen 1609 meter, vilket var den dittills snabbaste segertiden någonsin i en Elitloppsfinal. Det tidigare rekordet låg på 1.09,5, vilket Timoko travade från ledningen i 2014 års upplaga. I och med segern, som var värd tre miljoner kronor, gick Nuncio även om Gigant Neo som tränare Melanders vinstrikaste häst genom tiderna samt Donato Hanover som hingsten Andover Halls vinstrikaste avkomma. +Den 4 juni blev det klart att Nuncio skulle följa upp sin Elitloppsseger med start i Oslo Grand Prix den 12 juni på Bjerke Travbane i Oslo. Nuncio skar mållinjen på andraplats, slagen med en hals av Your Highness, efter att ha travat utvändigt om loppets ledare Mosaique Face. Nuncio satte nytt personligt löpningsrekord över medeldistans med autostart med tiden 1.10,5. Liksom i Elitloppet kördes han av Kihlström, som vid denna tid kunde betraktas som hans förstekusk. I efterhand framkom det dock att vinnaren Your Highness (tränad av Fabrice Souloy) varit dopad, varför Nuncio tillskrevs segern i Oslo Grand Prix. Hans segertid 1.10,5 blev därmed också den snabbaste segertiden i loppets historia. +Nästa start gjordes den 16 juli i Årjängs Stora Sprinterlopp på Årjängstravet, där Nuncio vann loppet från utvändigt ledaren. Segern innebar en inbjudan till Hugo Åbergs Memorial, men tränare Melander tackade nej till förmån för Jubileumspokalen. Den 3 augusti skulle Nuncio ha startat i ett kvallopp till Jubileumspokalen på Solvalla. Under förmiddagen den 3 augusti ströks dock Nuncio från loppet, då han inte kunde delta på grund av feber. Trots det missade kvalloppet kom Nuncio med i finalen av Jubileumspokalen, tack vare sina höga startpoäng. Loppet gick av stapeln den 17 augusti på Solvalla. Nuncio vann loppet från andra utvändigt. Vinnartiden skrevs till 1.10,7 över 2140 meter med autostart, vilket innebar att Nuncio slog Varennes tidigare rekord i Jubileumspokalen. Vinnartiden innebar även nytt svenskt rekord och nytt personligt rekord över distansen. Segern i Jubileumspokalen följdes upp med en start i Sundsvall Open Trot den 27 augusti på Bergsåker travbana. Nuncio vann loppet från ledningen. Efter segern påtalade kusken Kihlström hur Nuncio har "the total package" – med startsnabbhet, styrka och löphuvud. +Den 18 september gjorde Nuncio säsongens sista start då han tillsammans med kusken Kihlström deltog i finalen av 2016 års UET Trotting Masters på Bjerke. Nuncio segrade från ledningen, i vad som kom att bli hans sista seger i ett grupplopp. Han var i loppet spelad till vinnaroddset 1.18, vilket är det lägsta vinnaroddset och största favoritskapet någonsin i en final av UET Trotting Masters. Det tidigare lägsta vinnaroddset var 1.70, vilket Ready Cash var spelad till i 2013 års upplaga. Vid en summering av säsongen 2016 kan konstateras att Nuncio var obesegrad under hela säsongen med segrar i samtliga elva starter. Han var dessutom fortsatt obesegrad i karriären med kusken Kihlström, då ekipaget dittills hade vunnit sju av sju starter tillsammans. Totalt sprang Nuncio in 9,4 miljoner kronor på 11 starter och lika många segrar under säsongen 2016, vilket gjorde honom till den svenska travsäsongens såväl vinstrikaste som segerrikaste travhäst. +Säsongen 2017 årsdebuterade Nuncio den 18 februari i ett Gulddivisionslopp inom V75 på Axevalla travbana. Nuncio vann loppet med fem längder från ledningen tillsammans med kusken Örjan Kihlström. Årsdebuten följdes upp med ytterligare en seger den 22 mars i Kentucky Fibbers lopp på Solvalla, vilket innebar att han nu hade 15 raka segrar sedan oktober 2015. +Den 1 april 2017 startade han i Gulddivisionens final (som också var ett av uttagningsloppen till Olympiatravet) på Solvalla och besegrades där av Kadett C.D. Nuncios segersvit med 15 raka segrar blev därmed bruten. Efter att ha tilldelats ett wild card, startade Nuncio den 29 april i finalen av Olympiatravet på Åbytravet i vad som var hans 55:e start i karriären. Han galopperade i startrusningen och slutade oplacerad. Detta var första gången som han galopperade i ett lopp och även första gången som han slutade oplacerad. Fram till och med karriärens 54:e start hade Nuncio nämligen aldrig kommit sämre än trea i ett lopp. Förlusten i Olympiatravet var även första gången som Nuncio, under sin tid i Sverige, förlorade ett lopp utanför hemmabanan Solvalla. Säsongens femte start gjordes den 10 maj i Meadow Roads Lopp på Solvalla. Nuncio lyckades inte försvara sin seger i loppet från föregående år, utan slutade på tredjeplats. Tredjeplatsen innebar tre raka lopp utan seger, vilket endast hänt en gång tidigare i Nuncios karriär – vid skiftet mellan säsongerna 2013 och 2014, då han kom på andraplats i säsongens sista start 2013, och andra- respektive tredjeplats i de två första starterna 2014. +Nuncio deltog i Elitloppet för tredje gången i karriären den 28 maj 2017. Han vann sitt försökslopp före Propulsion och Timoko, och kvalificerade sig därmed för sin tredje raka Elitloppsfinal. Det andra försöksloppet vanns av Bold Eagle, en häst som Nuncio aldrig tidigare mött och det talades om en "drömduell" dem emellan inför finalen. Finalen vanns dock av Timoko, som med segertiden 1.09,0 aak (amerikansk autostart) från ledningen, också slog Nuncios rekord från 2016 års upplaga med den snabbaste segertiden någonsin i en Elitloppsfinal. Nuncio skar mållinjen som femma efter att ha travat utvändigt om ledaren Timoko. I efterhand fråntogs Propulsion sin andraplats (liksom alla sina andra resultat på svensk mark) då det 2020 framkom att han varit nervsnittad i sina hovar och inte varit startberättigad. Nuncios femteplats blev därför en fjärdeplats. +Elitloppet följdes upp med start i Midsommarloppet på Rättviks travbana den 24 juni 2017. Han slutade på tredjeplats bakom vinnaren On Track Piraten. Den 15 juli skulle han enligt sin tränare Stefan Melanders planering ha startat i Årjängs Stora Sprinterlopp, för att försvara sin seger i loppet från föregående år. Inför anmälan till loppet, drygt en vecka innan det skulle gå av stapeln, meddelade dock Nuncios dåvarande ordinariekusk Örjan Kihlström att han valde att köra Propulsion för tränare Daniel Redén framför Nuncio i loppet. Detta resulterade i att tränare Melander inte anmälde Nuncio till loppet. Det fick även till följd att samarbetet mellan Melander och Kihlström blev svagare. Från att ha varit förstavalet som kusk för Melander fick inte Kihlström någon förfrågan att köra ett enda travlopp med någon häst från Melanders Stall TZ mellan den 8 juli 2017 och den 30 maj 2018. Melander anlitade istället främst Johnny Takter och Ulf Ohlsson eller körde själv. Den 30 maj 2018 körde Kihlström återigen lopp för Stall TZ då han var tillbaka som kusk till Digital Ink i Gösta Bergengrens Minneslopp. Han körde även Melanders Zack's Zoomer till seger i Ina Scots Ära den 2 juni 2018. +Efter tre månaders tävlingsuppehåll gjorde Nuncio comeback den 5 oktober 2017 i John Deeres Pokal på Bollnästravet. I loppet kördes han, för första gången sedan maj 2016, av tränare Melander själv och de segrade med fyra längder från ledningen. I segerintervjun efter loppet bekräftade Melander att Nuncio matchas mot Frankrike och det franska vintermeetinget med målet att starta i världens största travlopp Prix d'Amérique i januari 2018. Detta var också anledningen till att Nuncio i loppet tävlade i en europeisk sulky istället för en amerikansk sulky för första gången i karriären. De så kallade "jänkarvagnarna" är nämligen förbjudna i Frankrike. Nästa start gjordes den 21 oktober i Express Gaxes Lopp på Örebrotravet. Loppet kördes som ett Gulddivisionslopp inom V75. Nuncio kom femma efter att ha travat utvändigt om loppets ledare Day or Night In. Starten i Örebro följdes upp med ytterligare en start i Gulddivisionen den 4 november i Hilda Zonetts Lopp på Bergsåker. Han segrade från ledningen på tiden 1.12,0 över distansen 2140 meter. Segern kom att bli den sista i karriären. Med denna seger passerade Nuncio även en miljon kronor insprunget under 2017. +Den 13 november 2017 blev det offentligt att Nuncio preliminäranmälts till Prix de Bretagne, ett lopp som skulle köras den 19 november på Vincennesbanan i Paris som det första av de fyra så kallade "B-loppen" inför Prix d'Amérique. Fram till den 15 november hade dock tränare möjlighet att dra tillbaka anmälningar, vilket Melander valde att göra på grund av bättre väderförhållanden hemma i Enköping än väntat. Melanders plan för Nuncio var istället att starta i Prix du Bourbonnais den 10 december. Nuncio anmäldes senare inte till Prix du Bourbonnais utan Melander ändrade sig och valde istället att anmäla honom till en start i Gösta Nordins Lopp på hemmabanan Solvalla den 13 december. Han slutade på andraplats, slagen med ett huvuds marginal av Food Money (efter målfoto). Nästa start skulle ha blivit Prix de Bourgogne den 31 december, men under julhelgen visade det sig att han drabbats av en infektion varför han inte anmäldes dit och den planerade transporten ner till Frankrike den 27 december fick ställas in. Totalt sprang Nuncio in 1,2 miljoner kronor på tolv starter varav fem segrar, två andraplatser och två tredjeplatser under 2017. Han tjänade minst femsiffriga belopp i samtliga elva felfria starter. Han hamnade emellertid endast på plats 62 över den svenska travsäsongens vinstrikaste hästar, vilket var hans sämsta placering i karriären efter att 2016 ha varit den vinstrikaste hästen och 2015 den femte vinstrikaste. +Den 6 januari 2018 meddelade Melander i det direktsända tv-programmet V75 Direkt att Nuncio avslutar tävlingskarriären och blir avelshingst. Andraplatsen på Solvalla den 13 december 2017 blev därmed karriärens sista start. Totalt gjorde Nuncio 63 starter under sin fyra år långa karriär (2013–2017) och tog 42 segrar, 14 andraplatser och 4 tredjeplatser. Fram till och med karriärens 54:e start var han aldrig sämre än trea i ett lopp (39 segrar, 13 andraplatser, 2 tredjeplatser). Han sprang in totalt 28,8 miljoner kronor – siffror som gör honom till hingsten Andover Halls vinstrikaste avkomma (före Donato Hanover) och tränare Melanders genom tiderna vinstrikaste häst (före Scarlet Knight). +Efter att Nuncio importerats till Sverige i februari 2015 gjordes den 15 mars 2015 inför avelsvärderingsnämnden i Eskilstuna en första avelsvärdering för att värdera hans avelsvärde. Nuncio fick i avelsvärderingen 82 av 100 möjliga poäng, vilket innebär "högt skattat avelsvärde" (71-85 poäng). I värderingen gjordes bedömningar av hans härstamning, tävlingsprestationer, temperament och exteriör. Vidare gjordes en veterinärundersökning med röntgentagning. +Under vintern 2017 fick ägare Stefan Melander flera erbjudanden från amerikanska Hanover Shoe Farms, som ville ha Nuncio som avelshingst från och med avelssäsongen 2018. Den 30 december 2017 meddelade dock Melander i det direktsända tv-programmet V75 Direkt att han valt att tacka nej till förmån för svenska Menhammar stuteri i Ekerö. Melander lät stuteriets ägare Margareta Wallenius-Kleberg köpa in sig som delägare i Nuncio. Efter nyheten kom information om kostnaden för Nuncio i aveln upp på stuteriets hemsida. Intresset för Nuncio visade sig vara mycket stort, han blev snabbt fullbokad för året och stuteriet beskrev i nyhetsmedia att de blivit nedringda om Nuncios tjänster efter beskedet att han ska verka i aveln där. Det stora intresset ledde till att Melander den 6 januari 2018 meddelade beslutet att Nuncio inte kommer att tävla mer, utan från och med januari 2018 stallas upp vid stuteriet för att ägna sig åt avelsverksamhet på heltid. +Nuncios debutkull från 2019 innehåller 101 svenskfödda hästar. Han fick sin första avkomma natten den 6 februari 2019, då Ursula Knick (efter Ready Cash) hos Lena Martell i Stånga på Gotland nedkom med ett hingstföl efter honom. Fölet fick senare namnet King Kennedy, döpt efter fotbollsspelaren Kennedy Bakircioglü. Sin första avkomma i USA fick Nuncio den 11 mars 2019, då Tuonela (efter Cantab Hall) hos hans gamla uppfödare Russell C. Williams på Hanover Shoe Farms i Pennsylvania nedkom med ett hingstföl efter honom. Den första avkomman till Nuncio som registrerades hos Svensk Travsport var Ytowns Skylark, ett sto (fux) som föddes den 2 april 2019, undan Unter Uns (efter Viking Kronos). Hon dog senare under året. +Nuncios avkommor från debutkullen var ute på hästauktioner under 2020. Den dyraste avkomman blev hingsten Roadtohana (undan Coco Truffles) som klubbades för 900 000 kronor på Solvalla Yearling Sales den 27 augusti 2020. Han var den femte dyraste hästen på hela auktionen. På Kriterieauktionen den 28 september 2020 blev avkomman Perkins, en hingst undan Lightning Kronos, hela auktionens dyraste häst. Han köptes för 775 000 kronor av Stefan Hultman. Även utanför Sverige har det funnits ett stort intresse för Nuncios avkommor. På den årliga finska hästauktionen Varsahuutokauppa den 11 september 2020 såldes hingsten Ignacio (undan Locksweeper) för 36 000 euro, vilket gjorde honom till den näst dyraste ettåringen genom tiderna i Finland. Nuncios dyraste avkomma vid hästauktionen i Lexington i Kentucky i oktober 2020 blev stoet Penthouse (undan Ruby Trap) som köptes för 60 000 dollar av Stefan Melander genom Stall TZ. +Nuncios första startande avkomma blev Joygo, som debuterade den 29 maj 2021 på Lebanon Raceway i Kentucky i USA. Han vann debuten i överlägsen stil från ledningen. Nuncios hittills vinstrikaste avkomma är hingsten Following som sprungit in 2,9 miljoner kronor. +Nuncios andra kull från 2020 innehåller 97 svenskfödda hästar. Kullen var ute på hästauktioner under 2021. På Elitauktionen under Elitloppshelgen 2021 blev stoet Twigs Khaleesi (undan Twigs Likepine) den dyraste Nuncio-avkomman och hela auktionens fjortonde dyraste häst. Hon köptes för 400 000 kronor av Daniel Redén. +Nuncios tredje kull från 2021 innehåller 65 svenskfödda hästar. +Nuncios fjärde kull från 2022 innehåller 95 svenskfödda hästar. +Vid den svenska Hästgalan 2015 var Nuncio en av de fyra nominerade hästarna i kategorin "Årets Häst". Nuncio förlorade utmärkelsen till Delicious U.S. som fick flest antal tittarröster medan Nuncio fick näst flest röster. Nuncio var återigen en av fyra nominerade i kategorin "Årets Häst" vid Hästgalan 2016, och denna gång vann han utmärkelsen. Nuncios kusk Örjan Kihlström vann dessutom utmärkelsen "Årets kusk" efter att ha segrat i flera stora lopp med Nuncio, bland annat Elitloppet. +Under Elitloppshelgen 2018 hyllades Nuncio den 27 maj 2018 på Solvalla för sin framgångsrika tävlingskarriär. Vid hyllningen närvarade Nuncio själv, tränare och ägare Stefan Melander, skötare Catarina Lundström och Nuncios två kuskar John Campbell och Örjan Kihlström. +Nuncio har hedrats med att få ett eget travlopp på hemmabanan Solvalla uppkallat efter sig, Nuncios Lopp. Premiärvinnaren den 11 november 2020 blev Nuncios tidigare stallkamrat Disco Volante. +Hoffman, Dean A. (2003) (på engelska). Quest for Excellence: Hanover Shoe Farms, The First 75 Years. Neenah, WI, USA: Russell Meerdink Company. ISBN 0-9293-4674-2. https://archive.org/details/questforexcellen0000hoff +Wikimedia Commons har media som rör Nuncio (travhäst). +Stall TZ:s officiella webbplats +Nuncio hos Svensk Travsport +Nuncios hingstprofil hos Menhammar stuteri +Det danska fälttåget i Skåne och Blekinge 1709–1710 var ett danskt fälttåg mot Sverige under stora nordiska kriget. Fälttåget ägde rum mellan 2 november 1709 och 5 mars 1710 enligt den svenska kalendern; 1 november 1709 till 4 mars 1710 enligt den julianska kalendern; och 12 november 1709 till 15 mars 1710 enligt den gregorianska kalendern. Det slutliga målet var att återta Skånelandskapen (Skåne, Halland och Blekinge), som förlorats vid freden i Roskilde 1658. +Den danske kungen Fredrik IV hade länge planerat ett anfall mot Sverige, trots att freden i Traventhal år 1700 hade tvingat Danmark ur stora nordiska kriget. Problemet var att för att kunna försörja en stor stående armé under fredstid hade man tvingats hyra ut en stor del av denna till konflikter i Europa, främst spanska tronföljdskriget. De svenska styrkorna hade däremot länge varit upptagna i Ryssland och det förkrossande svenska nederlaget i Poltava den 28 juni 1709 gjorde dessutom läget allt mer fördelaktigt för en invasion. Efter förhandlingar om att föra tillbaka de uthyrda trupperna från kontinenten, samt förhandlingar med de allierade Ryssland och Sachsen-Polen för att säkra deras stöd, kunde planerna till slut sättas i verket och den danska krigsförklaringen nådde det svenska riksrådet den 18 oktober 1709. +Den 2 november samma år landsteg den 16 000 man starka danska invasionsarmén under befäl av general Christian Ditlev Reventlow vid Råå söder om Helsingborg. Då de svenska styrkorna i Skåne, ledda av guvernör Magnus Stenbock, vid tillfället inte var starka nog att motstå den danska armén tvingades de slå till reträtt och danskarna kunde snabbt överta stora delar av landskapet. Helsingborg och Lund erövrades utan motstånd och de viktiga fästningsstäderna Landskrona och Malmö belägrades. Svenskarna trycktes tillbaka till Blekinge, men samtidigt pågick det från den svenska defensionskommissionens sida en intensiv verksamhet för att rekrytera och träna nya trupper för att ersätta de som hade utplånats vid Poltava. I januari 1710 kunde Stenbock tåga in i Skåne med sin armé och Reventlow ansåg sig tvungen att dra sig tillbaka med den efter hand allt mer försvagade danska invasionsarmén. +Under det danska återtåget insjuknade Reventlow och ersattes av generallöjtnant Jørgen Rantzau som drog tillbaka trupperna till Helsingborg. De två arméerna möttes slutligen vid Ringstorp utanför Helsingborg den 28 februari 1710, vilket slutade med en avgörande svensk seger. De danska trupperna flydde tillbaka innanför Helsingborgs befästningar och kunde därifrån evakueras till Danmark. De sista trupperna lämnade Skåne den 5 mars 1710. Fälttåget blev Danmarks sista större försök att återta Skånelandskapen. +Danmark hade tvingats ur det stora nordiska kriget vid freden i Traventhal år 1700 men den danske kungen Fredrik IV hade länge planerat att åter ge sig in i kriget med målet att återerövra de förlorade landskapen Skåne, Halland och Blekinge. Dock hade Danmark sedan 1701 varit inblandat i spanska tronföljdskriget genom att man hade lånat ut totalt omkring 18 000 man till de allierade mot Frankrike och Ludvig XIV. Ungefär 12 000 man var utlånade till Storbritannien och Nederländerna och stred i Nederländerna och Frankrike, medan närmare 6 000 man var utlånade till den tysk-romerske kejsaren Josef I och stred i Ungern. Den danska statskassan var inte heller redo för ett krig. Flottan saknade sjömän, krut och proviant, rytteriet saknade hästar, infanteriet saknade skor och kläder och artilleriet saknade hästar och krut. Finansminister Johan Georg Holstein och överkrigssekreterare Christian von Lente argumenterade emot en invasion med ekonomin och arméns bristande beredskap som argument. De menade att innan man kunde förklara krig skulle man förvissa sig om tillräcklig hjälp från sina allierade i Ryssland och Sachsen-Polen. Större delen av kungens rådgivare var däremot för en invasion, däribland general Christian Ditlev Reventlow och generalmajor Valentin von Eickstedt, som ansåg att Sverige borde vara helt uttömt på stridsdugligt folk och att en erövring av Skånelandskapen skulle bli en enkel sak. I slutändan valde kungen att lyssna på de rådgivare som förordade krig. +Även om krigsförberedelserna redan hade varit påtänkta i början av 1709, hade ännu ingen konkret upprustning företagits till sommaren det året. Fredrik IV hade julen 1708 åkt på en resa till Italien och var inte åter i Danmark förrän i juli 1709. Under hans bortavaro hade alla förberedelser stått stilla eftersom alla sådana beslut behövde godkännas av kungen under det danska enväldet. Ryssland hade året innan gett Danmark ett erbjudande om ett ekonomiskt stöd på en miljon daler, 10 000 man fotfolk avlönade av ryska kronan, samt beck, tjära och hampa till flottan, i utbyte mot att de åter skulle gå med i kriget mot Sverige. Men den danska regeringen avstod då man ville invänta utgången av Karl XII:s infall i Ryssland. Medan Fredrik IV vistades i Italien förnyade tsar Peter den stores sändebud i Venedig erbjudandet och meddelades att danske kungen skulle underteckna förbundet när han på sin hemresa besökte Dresden. Den 18 juni i Dresden ingick Fredrik IV i ett försvars- och angreppsförbund med August den starke av Sachsen där han förband sig att invadera Sverige så fort Danmark nått ett fördelaktigt avtal med Ryssland. Anledningen till det sista förbehållet var att det ryska sändebudet saknade fullmakt från tsaren för sitt tidigare erbjudande, vilket gjorde den sachsiske kurfursten något mer försiktig. Ett avtal ingicks den 5 juli även med Preussen, där de ålade sig att neka svenska trupper genomfart genom landet, men gick med på att förklara Sverige krig enbart om övertalningen av tsaren att dela upp Polen mellan Preussen, Sachsen och Ryssland lyckades. +Vid sin återkomst till Danmark den 16 juli tryckte kungen på för ett fördrag med Ryssland, men på grund av det förkrossande svenska nederlaget i Poltava den 28 juni 1709 drog Peter den store tvärt tillbaka erbjudandet om ekonomiskt och militärt understöd till en dansk invasion. Han ansåg sig efter denna seger inte längre vara i behov av Danmarks stöd i kriget. Återigen anmodade storkansler Holstein och överkrigssekreterare von Lente kungen om att vänta med sin krigsförklaring, då man med alla utlånade trupper inte ansåg sig redo för krig utan stöd utifrån. Förhandlingar hade redan påbörjats under 1708 om att föra tillbaka trupperna i Ungern till Danmark med målsättningen att använda dem i invasionen av Skånelandskapen. Den tysk-romerske kejsaren hade inte längre råd att betala dem. +Sommaren 1709 påbörjade trupperna sin marsch hemåt och efter den långa tiden i Centraleuropa bestod förbandet mestadels av tyskar och ungrare. Dessa trupper förväntades däremot inte nå dansk mark förrän sent in på hösten. Den 8 augusti förklarade August av Sachsen krig mot Sverige och ryckte den 14 augusti in i Polen där han snart därefter hade fördrivit den svenska lydkungen. Nya förhandlingar inleddes senare mellan de allierade, och Ryssland och Danmark nådde slutligen ett avtal den 12 oktober i Köpenhamn. Båda länderna förband sig till att inte sluta någon enskild fred med Sverige, att garantera varandras av Sverige erövrade besittningar, och bekräftade August av Sachsens anspråk på Polens tron. Ryssland lovade även att sända 6 000 hjälptrupper till invasionen av Skånelandskapen och för att invadera Finland. +I slutet av juli 1709 hade den danska regeringen börjat värva sjömän utomlands och tusentals tyskar och nederländare lockades till att ta värvning. Trupperna som marscherade från Ungern var utslitna och decimerade och förbanden erhöll därför rikliga rekryteringspengar för att återfå sin styrka. Soldater värvades även från Tyskland och Nederländerna och hästar köptes in för att komplettera rytteriet. En invasionsflotta började ansamlas och alla länsherrar ålades att sända alla båtar och pråmar som kunde användas till att skeppa trupper. I Köpenhamn låg till hösten 128 olika typer av trupptransportfartyg i väntan på klartecken. Den danska flottan saknade inte krigsdugliga skepp att skydda dem med, utan hade 40 linjeskepp, 10 fregatter och runt 50 mindre skepp. Den 19 augusti skickade Fredrik IV order till överbefälhavaren för de norska trupperna, generallöjtnant Hans Ernst von Tritzschler, att förbereda sig för att dra in i Sverige från Norge. I slutet av oktober hade 6 000 man samlats vid Svinesund och 1 400 man vid Kongsvinger, även om trupperna ännu inte var i stridbart skick. +De danska avsikterna gick inte den svenska ambassadören i Köpenhamn, Anders Leijoncloo, förbi. Han varnade vid flera tillfällen i början av 1709 det svenska riksrådet om de danska krigsförberedelserna. De svenska spioner som verkade i Köpenhamn kunde också bekräfta uppgifterna. Även Helsingborgs borgmästare Gabriel Löfgren, hade tillsammans med kommendanten på Helsingborg slott, regementskvartermästaren Peter Bäfverfeldt, på eget bevåg spionerat på danskarnas verksamhet. De skickade en inspektor Magnus Aulin till Helsingör varifrån denne kunde rapportera om hur danskarna börjat utrusta sina krigsskepp och hur man i Köpenhamn värvade soldater. Det avtal som Preussen tecknat med Danmark hade dessutom hamnat i svenska händer genom att den svenske generalen Mauritz Vellingk erhållit en avskrift från greve Ditlev Reventlow, kusin till danske generalen Reventlow. Redan på våren hade guvernören i Skåne, general Magnus Stenbock, påbörjat upprustningar av befästningarna i Landskrona och Malmö. Helsingborgs och Kristianstads befästningar, som till största delen nedrivits till följd Skånska kriget var däremot i så illa skick att inte mycket kunde göras åt dem. +Efter nederlaget i slaget vid Poltava var hotet om krig uppenbart från flera håll: Ryssland, Danmark, Sachsen och Preussen smidde alla krigsplaner. Den 22 augusti skickade Stenbock en underrättelse till defensionskommissionen om att det finns en överhängande fara för en dansk invasion. Stora delar av den svenska krigsmakten hade dock gått under vid Poltava; när det kom till de skånska förbanden hade till exempel både Norra skånska kavalleriregementet och Södra skånska kavalleriregementet upphört att existera. Det enda regementet i full styrka var Göran Gyllenstiernas Skånska tre- och femmänningsregemente som av försörjningsskäl vid tillfället var utspritt och soldaterna inkvarterade hos sina rusthållare. Regementet led dessutom av en långvarig brist på utrustning, då resurserna istället hade gått till de två ordinarie skånska regementena som stred i fältarmén i Ryssland. Bristen på trupper var akut och man saknade både pengar och hästar till kavalleriet. För att förstärka försvaret skickades Västgöta tre- och femmänningsregemente under överste Gustaf Fredrik Lewenhaupt och Västgöta kavalleriregemente under överste Lars Hierta till Skåne. Vid tillfället hade man fått ihop totalt 1 300 man. +Arbetet med att ersätta de regementen som förgåtts vid Poltava påbörjades också så fort som möjligt och i motsats till vad de danska rådsherrarna antog fanns det tillräckligt med dugligt folk i lämplig ålder kvar i Sverige att fylla upp de återskapade regementena med. Stenbock själv blev överraskad över skåningarnas vilja att låta sig värvas, men vågade trots detta inte samla regementena förrän de uppsvenska regementena anlänt. Stenbock betvivlade generellt de skånska regementenas lojalitet och behandlade dem ständigt med försiktighet. Han var även orolig för att en snapphanerörelse likt den som uppstått under Skånska kriget skulle bildas igen. Han lät därför den 11 september utfärda en proklamation som varnade om att all form av snapphaneri, landsflykt, samt flykt upp i skogarna i norra Skåne skulle ses som landsförräderi och vara straffbart med döden. Det svåra när det kom till att återskapa nya regementen kom istället att bli tillgången till officerare, då tillräckligt erfarna sådana saknades efter alla år av krig. För att lösa detta tog man in flera yngre och otränade ungdomar i universitetsålder och från adelssläkter. +Det uppstod också svårigheter att skaffa utrustning och mundering åt de nya krigarna. När inga andra medel fanns att tillgå i det utarmade landet tvingades man ta de ännu ej utbetalta avlöningsmedlen för de soldater som var fångna i Ryssland, eller som stupat eller frusit ihjäl. På detta sätt tillgrep man alltså medel som egentligen skulle tillkomma soldaternas hustrur och barn. Det var, som ett av riksråden sade, att "kläda den nye soldaten med den gamles svett och blod". Nästan alla regementena kunde också beklädas med den vanliga blå karolinska uniformen, fast en del av dessa kläder inte hann fram till regementena förrän mot krigets slut, vilket ledde till att vissa av Stenbocks soldater fick slåss i sina grå vadmalskläder. Inom några av de äldre regementena fick åtskilliga ryttare också rida i getskinnspälsar. På fötterna hade somliga av dessa ryttare inte annat än träskor som de band fast vid foten med bast. Danskarna talade också föraktfullt skämtande om "Bocken och hans getapågar". På Kalmar regemente räckte beklädnadsmedlen inte till varken för kappor eller ränslar; och såväl strumpor och skor som skjortor fick soldaten själv skaffa sig så gott han förmådde. Detta regemente fick under fälttåget också en mycket hög sjuk- och dödlighetssiffra. På sommaren 1710 uppgavs det att av 1 100 man hade "på marschen och i landet" 341 dött, och 434 låg sjuka. Till tross åt de nya regementena räckte tillgångarna så gott som inte alls. Varken proviant- eller ammunitionsvagnar, och inte heller tält kunde man förse dem med. Bristen på utrustning var så mycket kännbarare med tanke på att det var ett vinterfälttåg som förestod. De nyuppsatta regementenas hästar var i allmänhet små och oansenliga, och ett ögonvittne påstod att de "nästan kunde krypa under buken på de stora danska hästarna". Stenbock skrev till regeringen: "I 26 år, som jag tjänat, har jag aldrig sett en armé i slätare tillstånd; dock låter jag icke modet falla." Han ansåg sig nämligen veta att det i soldaternas bröst bodde "ett frimodigt hjärta, fastän kroppen var betäckt med utslitna vallmars- och allehanda brokota kläder". De unga rekryterna visade också "en stor lust att gå till Skåne" och "stark åtrå att slåss med fienden", enligt Stenbock. +I oktober hade både Norra skånska kavalleriregementet och Södra skånska kavalleriregementet återuppsatts och den skånska Adelsfanan hade åter nått upp till full styrka. I och med detta hade det skånska försvaret förstärkts till att omfatta 3 550 man. En upprustning av norska trupper hade också noterats längs gränsen till Bohuslän och generallöjtnant Karl Nieroth sändes dit för att möta upp ett eventuellt angrepp. +Planen enligt fördraget i Dresden var att det danska anfallet skulle ske mot Sveriges skandinaviska landområde och inte mot de svenska besittningarna i Tyskland. Anledningen var att Storbritannien och Nederländerna fruktade att det stora nordiska kriget annars skulle sprida sig till de tyska staterna, vilket riskerade att betyda att de tyska stater som deltagit i spanska tronföljdskriget skulle tvingas att återvända hem. Det var illa nog att Danmark och Sachsen, som försåg dessa länder med trupper i kriget, skulle låsas i stridigheter på annan plats. Den 18 oktober 1709 nådde krigsförklaringen det svenska riksrådet och som grund hade danskarna angivit att Sverige hade fuskat med Öresundstullen, samt att befolkningen i de forna danska landskapen Skåne, Halland, Blekinge och Bohuslän hade behandlats illa. +Fredrik IV godkände i september 1709 invasionshärens sammansättning. Denna bestod av 15 000 man indelade i sex kavalleriregementen, fyra dragonregementen, åtta infanteriregementen, sex artillerikompanier, samt träng. Den endast 39 år gamla greve Christian Ditlev Reventlow, son till den avlidne tidigare storkanslern, utsågs som chef för invasionshären. Reventlow hade visat sin duglighet i fält under spanska tronföljdskriget under både Eugen av Savojen och Vilhelm av Oranien och var en kunglig favorit. Trots sin ringa ålder åtnjöt Reventlow uppenbarligen de äldre generalernas respekt, då inga av dem ska ha protesterat mot utnämningen. Reventlow hade dock inte mycket inflytande över hur fälttåget skulle utföras då den delen styrdes hårt från Fredrik IV. Trots detta hade Reventlow inte fått någon tydlig fälttågsplan från kungen. Instruktionerna sträckte sig bara till att så snart som möjligt inta Landskrona. Tidpunkten för invasionen var inte den bästa då man i största mån ville undvika ett vinterfälttåg, men eftersom man ville ta vara på den uppfattade svenska svagheten så mycket som möjligt och inte ge dess armé någon tid att återhämta sig ansåg man sig tvungna att ta tillfället i akt. +Ansamlingen av danska fartyg hade inte undgått de svenska trupperna på den skånska sidan av Öresund. Stridsledningen hade identifierat tre tänkbara landstigningsplatser: Barsebäck, Råå och Helsingborg. Stenbock hade placerat ut sina tre kavalleriregementen längs skånska västkusten och gett order att olika antal signalskott skulle avfyras från Landskronas eller Malmös befästningar så fort det stod klart var landstigningen ska ske rum och att kavalleriregementena genast skulle bege sig till angiven plats. De första danska trupperna började embarkera trupptransportfartygen den 24 oktober och slutligen den 31 oktober avseglade en flotta bestående av mer än 250 civila skepp skyddade av 12 linjeskepp och 5 fregatter, ledd av kungens halvbror, generalamiral Ulrik Christian Gyldenløve på flaggskeppet Dannebroge, från Köpenhamn norrut genom Öresund. +När de svenska trupperna såg flottan passera Landskrona citadell avfyrade fästningens kanoner sex skott: signalen för att landstigningen kommer att ske vid Råå, på samma plats som de danska styrkorna landstigit år 1676 i början av Skånska kriget. Vid ljudet av signalskotten begav sig kavalleriregementena mot platsen och placerade ut sig vid Raus kyrka, Rya och Katslösa. Stenbock själv hade sitt huvudkvarter i Rya. +Vid tretiden på eftermiddagen den 1 november ankrade den danska flottan på redden utanför fiskeläget och krigsråd hölls på Dannebroge mellan Reventlow och hans närmaste befälhavare, där dessa erhöll sina order. Från skeppen kunde danskarna se hur vårdkasar tändes längs de skånska stränderna och kyrkklockorna började ringa för att meddela om krigets ankomst. Tidigt på morgonen den 2 november rodde Reventlow, tillsammans med generalmajor Valentin von Eickstedt ur Grenadjärkåren, generaladjutant Lothar Adolf Henrik Klepping samt en mindre eskort, i land söder om Råå. Väl där såg de inte några större hinder. De skymtade endast en grupp svenska ryttare på en höjd längre inåt land, men dessa vände om och red i riktning mot Landskrona. +Klockan åtta på morgonen anslöt Fredrik IV till landstigningsflottan och alla dess bestyckade fartyg avgav salut på 27 skott vardera. Med kungen väl på plats kunde landstigningen påbörjas. De första att landsättas blev bataljoner ur Grenadjärkåren, Gardet och Drottningens livregemente, som fördes i land i kanonslupar, specialbyggda flatbottnade båtar med en kanon i fören som skydd. Den totalt 2 000 man stora styrkan anfördes av Reventlows nästkommenderande, generallöjtnant Frederik Gersdorff. Landstigningen täcktes av beskjutning från de danska linjeskeppen som låg med bredsidorna vända mot stranden. De landstigna trupperna intog ställningar för att försvara brofästet, grenadjärkåren längs Råån inåt land, Gardet mot norr och Drottningens livregemente mot söder. Därefter påbörjades överskeppningen av resterande delar av det danska infanteriet och på kvällen hade man hunnit föra över totalt 5 000 man. +Klockan 14 den 2 november landsteg Fredrik IV och inspekterade de redan landstigna truppernas försvarspositioner, innan han inkvarterade sig på Råå värdshus. Där övernattade han för att dagen efter bege sig ut för att rekognoscera tillsammans med sina generaler. Samma dag slog vädret om till storm och regn, vilket gjorde att landstigningen av de resterande trupperna gick trögt. På förmiddagen den 3 november hade nästan allt det danska fotfolket skeppats i land, men man hade stora svårigheter att få i land kavalleriet och dess hästar i den höga sjön. +Den 4 november kunde till slut de sista delarna av kavalleriet stiga i land. Därefter började man föra över tross och annan utrustning. De svenska trupperna var inte tillräckligt många för att bjuda den danska invasionshären motstånd. Stenbock skickade ryttmästare Henrik Hammarberg till Stockholm för att underrätta defensionskommissionen om invasionen och att han inte hade tillräckligt med trupper för att möta den. Han gav order till kavalleriregementena att dra sig tillbaka ner förbi Glumslöv och Landskrona ner till Kävlinge för att där nå andra sidan av Lödde å. På vägen rev man broarna vid Kävlinge och Getinge. +Det första målet för invasionshären var att säkra Helsingborg, och den 3 november anlände en delegation från Helsingborg som erbjöd att överlämna staden åt danskarna i utbyte mot deras beskydd, då borgarna hört rykten om att de svenska trupperna skulle bränna ner staden för att förneka de danska trupperna möjligheten att använda den för inkvartering. Samtidigt meddelade de att den skrala svenska garnisonen på Helsingborgs slott, som bestod av endast 36 man ur William Sinclairs Västgöta femmänningsinfanteriregemente, hade övergett staden. Garnisonens befälhavare, regementskvartermästare Bäfverfeldt, hade från Kärnans topp kunnat följa den danska landstigningen och hade blivit så avskräckt att han beordrade sina män att nagla igen de åtta kanonerna på Kärnans tak för att sedan lämna staden med sina trupper. Man hade även rivit upp alla brädor på stadens skeppsbro. Natten till den 5 november ryckte överste Albrecht Philip von Eynden med Fynska regementet in i staden och mötte där inget motstånd. Senare under dagen anlände Prinsen av Hessens regemente för att bilda garnison i staden och man hissade en enorm Dannebrog på Kärnans topp. Chefen för fortifikationen, överstelöjtnant Marcus Heinsohn, sändes den 6 november till staden för att besiktiga och förbättra dess befästningar. +Den danske kungen reste på kvällen den 5 november till Helsingborg, eskorterad av 50 man ur Livgardet till häst. På väg dit hälsades han av dansk lösen (tre skott) från kanonerna på Kärnans topp, kanoner som hade borrats upp av de danska trupperna under dagen. Han bosatte sig tillsammans med sitt följe på 200 personer i den danskvänliga rådmannen Herman Schlyters gård, numera Henckelska gården på Norra Storgatan. I och med att Helsingborg säkrats hade danskarna tillgång till stadens skeppsbro, över vilken man efter att den reparerats, enklare kunde föra över proviant och manskap från Danmark. När överskeppningen av utrustningen från landstigningsflottan var klar skickades de större transportfartygen tillbaka till Köpenhamn, medan de mindre transportskeppen skickades till Helsingör för att därifrån kunna föra över resterande materiel direkt till Helsingborg. Det handlade i första hand om det viktiga fältartilleriet. Örlogsskeppen låg kvar utanför kusten fram till den 15 november innan de till slut återvände till Köpenhamn. Helsingborgs svenskfödda borgmästare, Gabriel Löfgren, hade vid invasionen lämnat staden och Schlyter utsågs till tillförordnad borgmästare. Han lät inventera alla borgares vapeninnehav, och dessa vapen kom senare att beslagtas av de danska trupperna. Genom ett plakat som upplästes från predikstolen i Mariakyrkan lät Fredrik IV proklamera att han tagit Skåne i "skydd och besittning". Stadens 170 borgare svor den danske kungen trohet, vilket även borgarna i Ängelholm, samt allmogen i Luggude, Rönneberga och Onsjö härader gjorde. Helsingborgs kyrkoherde, Olof Troilius, vägrade dock läsa upp den danska kungens plakat och bad istället den vanliga förbönen för Karl XII, varefter även han flydde staden. Troilius uppsökte därefter Stenbocks armé, där han utsågs till dess generalstabspräst. +Koordineringen av den danska överfarten hade inte varit helt smärtfri. De 25 kanoner som togs i land kunde inte användas då man försummat att föra över draghästar för dessa. Den ursprungligen planerade artilleriparken på 42 tunga och 56 lätta kanoner, 23 mörsare och 6 haubitsar saknade bemanning då manskapet fortfarande var kvar i Köpenhamn. Proviantmängden räckte också bara till 14 dagar och började snabbt sina och för att få in förnödenheter lockade man den skånska allmogen med betalning i "klingande mynt" för den proviant de kunde förse den danska armén med. De danska finanserna var dock i så dåligt skick att man fann att man inte kunde hålla detta löfte och när allmogen ankom med proviant fick de istället betalt i kvittenser och meddelades att dessa skulle avräknas mot den planerade krigsskatt man ämnade samla in. Reventlow hade varnat den danska kungen att det skulle skada det danska anseendet hos Skånes befolkning om de inte höll sitt löfte om betalning, men detta ignorerades. Ett annat problem var den danska byråkratin, som krävde att tull skulle tillföras på alla varor som fördes över Öresund, även för utrustning och proviant åt den danska armén. Administrationen av detta gjorde att viktig försörjning från Danmark ständigt försenades. Reventlow anmodade om att få undslippa avgifterna, men fick i den frågan inget gehör från de danska myndigheterna. +Den danska hären bröt den 16 och 17 november upp från landstigningsplatsen vid Råå för att inkvarteras i byarna omkring Helsingborg och Landskrona. Reventlow förlade sitt högkvarter till godset Bälteberga strax norr om Rååns dalgång. Som gränslinje i söder valde man Lödde å, medan den nordöstra gränslinjen utgjordes av Vege å. Vädret var dock eländigt och ständigt regn gjorde vägarna leriga och oanvändbara för några större truppförflyttningar. Mestadels var det besvärligt för tross och artilleri. +I slutet av november satte till slut frosten in och det blev lättare för den danska armén att manövrera i landskapet. Den 25 november påbörjades den danska belägringen av Landskrona genom att överste von Eyndens Fynska regemente förflyttades till staden. Eftersom inneslutningen begränsade sig till att bara bevaka infartsvägarna var denna inte särskilt effektiv. Någon örlogsblockad från sjösidan utfördes inte heller, vilket gjorde att garnisonen fritt kunde kommunicera med Malmö genom sjövägen. Då belägringen inte startats tidigare hade kommendanten på Landskrona Citadell, överste Sinclair, dessutom haft tre veckor på sig efter den danska landstigningen att samla proviant för att klara just en sådan. +Den 28 november höll Fredrik IV ett sista krigsråd innan han återvände till Själland. Kungen fortsatte därefter att styra fälttåget genom korrespondens. I ett brev gav han order om att Reventlow skulle röra sig mot Kristianstad och sedan till Karlskrona för att förstöra den svenska flottan. Reventlow föredrog hellre att inta Landskrona och Malmö för att på så sätt ha ryggen fri och säkra sina försörjningslinjer innan offensiven. Något bifall fick Reventlow inte, utan kungen följde istället ett förslag från Rantzau om att innesluta städerna fram till våren. I början av december gjordes en dansk truppförflyttning mot Rönne å. Vid Röstånga stod tre svenska kavalleriregementen under generalmajor Göran Gyllenstierna med order om att bevaka övergångarna vid Rönne å och Lödde å. Gyllenstierna valde vid beskedet om danskarnas ankomst att retirera till Kristianstad. Den 8 december korsade generalmajor Ditlev Brockdorff med Första jylländska kavalleriregementet Lödde å och marscherade med en trupp in Lund med målet att besätta det bördiga Söderslätt. Väl där plundrade de danska trupperna stadens kornförråd och Brockdorff installerade sig i professor Johan Hörlings gård med ett följe på 18 personer. Någon aktion mot Malmö hade ännu inte företagits. Vädret hade slagit om till regn igen och vägarna var åter ofarbara. +Magnus Stenbock hade den 5 november anlänt till Malmö där det fanns 20 välbehövliga kanoner och där även hans gravida hustru och parets barn bebodde en våning av Malmöhus slott. Han fann att stadens försvarsförmåga var god och den 9 november utsågs Stenbock till överbefälhavare för de trupper som skulle verka i Skåne. Den 1 december bröt Stenbock upp från Malmö och förflyttade sig till Kristianstad, där han övertog befälet från Gyllenstierna. De kanoner som fanns i Malmö lyckades han dock inte få med sig då han saknade tross att transportera dem med. I början av december kontrollerade danskarna praktiskt taget hela mellersta Skåne utom Landskrona och Malmö. Runt nyår 1710 återvände frosten och i början av januari 1710 fick generalmajor Brockdorff order om att marschera mot Malmö med omkring 2 000 man, bestående av två skvadroner ur Själländska lantdragonregementet och två bataljoner ut Drottningens livregemente. Dessa posterades ut ungefär en mil från Malmös befästningar, vid landsvägarna som ledde in till staden. Därmed var det fortfarande möjligt att ta sig in och ut ur staden genom mindre stigar. Inte heller denna stad blockerades från sjösidan och blockaden var därför lika verkanslös som den av Landskrona. De danska trupperna saknade dessutom både belägringsartilleri och en duglig belägringsingenjör. +För att rycka fram mot Kristianstad ansåg sig Reventlow behöva ytterligare förstärkningar. Många av de danska trupperna var upplåtna för belägringarna av Malmö och Landskrona och en stor del av trupperna behövde stanna i Helsingborg för att försvara mot svenska trupper som skulle kunna anfalla från Halland. Båda sidorna hade goda underrättelsenätverk och fick snabbt bud om läget hos sina motståndare. Reventlow hade fått tydliga uppgifter om styrkan i den svenska armé som höll på att byggas upp och enligt hans bedömning hade han med sina ungefär 6 700 tillgängliga soldater inte tillräckligt för att möta den svenska armén i en avgörande drabbning. Han ansåg att han på sin höjd kunde använda sina trupper till att blockera ytterligare svenska förstärkningar från Småland. +Reventlow meddelande den danske kungen om sina farhågor och anmodade om att de danska regementen som för tillfället stod sysslolösa i Holstein skulle skickas till Skåne. Han ville även att kungen skulle utöva påtryckningar på Ryssland om att de skulle efterleva sin del av fördraget genom den utlovade hjälpkåren och invasionen av Finland. Slutligen yrkade Reventlow återigen om att man skulle inta Malmö och Landskrona och ville att städerna skulle bombarderas av flottan. Inga av dessa förfrågningar kom att hörsammas. Det var bal- och maskeradsäsong i Köpenhamn och Fredrik IV sysselsatte sig hellre med dessa nöjen tillsammans med sin unga älskarinna Charlotte Helene von Schindel än med invasionen av Skåne. Många av Reventlows förfrågningar förblev till och med obesvarade. Den danska statsledningen såg fortfarande invasionen som en enkel sak, men inte heller statsledningen kunde bevilja några förstärkningar utan kungens godkännande. +Reventlow ansåg sig även ha brist på dugliga officerare. Två av hans mer kapabla befälhavare hade på kungliga order förflyttats från den danska armén till Skåne: hans nästkommenderande, generallöjtnant Frederik Gersdorff, som utsetts till befälhavare för de danska hjälptrupperna i Flandern, samt generalmajor Frederik Leegaard som sänts till Holstein. Lyckligtvis hade Gersdorff ersatts av den erfarne Frantz Joachim von Dewitz. Om von Dewitz skrev Reventlow i ett brev till Fredrik IV att han var "den enda ledaren som givits mig i armén" och att han "säkert vet hur man beställer och reglerar gemene man såväl som [är] en man vilken man borde fortsätta att beakta". I övrigt var Reventlow mycket kritisk till de officerare han hade tilldelats. Han hade inga högre tankar om generalmajor Valentin von Eickstedts militära egenskaper och var av åsikten att generalmajor Christian Rodsteen (Leegaards ersättare) helt borde avlägsnas från armén då denne hade ett alltför ängsligt lynne för att effektivt kunna anföra manskapet. Utöver detta hade Reventlow drabbats av en envis förkylning. +Den ursprungliga invasionsplanen angav att de trupper som var baserade i Norge snart skulle dra in i Sverige för att öppna en andra front, och Reventlow ansåg därför att hotet norrifrån kunde bortses från. Han lät ändå ge order om att förstärka försvaret längs Rönne å, något som försvagade hans armé ytterligare. Efter att vädret blivit mer gynnsamt under de första dagarna av 1710 bröt Reventlow den 3 januari upp från sitt högkvarter i Bälteberga och beordrade de trupper han ansåg sig kunna undvara att marschera mot Kristianstad. Expeditionskåren bestod av tre infanteriregementen: Livgarden till fots, Prins Christians regemente och Marinregementet; fyra kavalleriregementen: Första och Tredje själländska kavalleriregementet, Andra fynska kavalleriregementet, och Första jylländska kavalleriregementet; samt 200 man från artilleriet som medtog sex fyrpunds fältkanoner, två haubitsar, pontonmaterial och 50 trossvagnar. Armén marscherade ner mot Lödde å, där den sedan kunde nyttja landsvägen som gick från Malmö upp till Kristianstad (motsvarande dagens Europaväg 22). +I Kristianstad fanns ungefär 3 500 svenska trupper. Dessa bestod mestadels av kavalleri genom Västgöta kavalleriregemente, Västgöta tre- och femmänningsregemente till häst, samt Skånska tre- och femmänningsregementet till häst. Det enda infanteri på plats var en bataljon ur Sachsiska infanteriregementet bestående av 494 sachsiska krigsfångar som valt att gå i svensk tjänst. Dessa hade anlänt till Kristianstad den 17 december som några av de första förstärkningarna norrifrån. Stenbock ansåg inte att Kristianstad, vars vallar hade rivits efter Skånska kriget, hade något större strategiskt värde och planerade istället att mota den danska armen vid passen vid Norje och Mörrumsån. Kristianstads befästningar förstärktes så gott det gick, men huvuddelen av arméförråden förflyttades till Karlshamn, dit Stenbock begett sig den 10 januari efter att ha överlämnat befälet av staden till Gyllenstierna. Det viktiga krutbruket i Torsebro norr om Kristianstad hade för säkerhets skull tömts på inventarier för att inte falla i danskarnas händer. Stenbock gav också order om att alla broar skulle förstöras för att sakta ner fiendens framfart. +Den 12 januari anlände Reventlow till Vä sydväst om Kristianstad, men valde att inte anfalla staden. Eftersom armén skulle bli tvungen att korsa den 200 meter långa och smala Långebro för att ta sig till stadens portar skulle den vara alldeles för blottad för kanoneld från stadens palissader, och ett anfall på Kristianstad skulle enligt Reventlow bli alltför blodigt. I stället skickade han överstelöjtnant Scheel med 50 dragoner till Torsebro för att se om det var möjligt att korsa Helge å vid denna plats. På så sätt skulle han kunna inringa staden och skära av dess garnison från Blekinge. Bron vid Torsebro bevakades av 60 man ur Sachsiska infanteriregementet under kapten Johan von Rohr, samt 150 ryttare ur Västgöta tre- och femmänningsregemente till häst under ryttmästare Sven Hierta. Med sig hade trupperna två kanoner som på grund av avsaknad av kanonkulor var laddade med blyskrot. +Framme vid bron beordrade Scheel sina trupper att sitta av och trupperna började utbyta musköteld från vardera sida av ån. Efter ungefär en timmes strid hade danskarna mist fem soldater, medan svenskarna förlorat tre, och Scheel tvingades dra sig tillbaka. Reventlow skickade generaladjutant Hartvig Huitfeldt till platsen och han ankom mitt under eldstriden. Scheel underrättade honom om läget och då det fanns en god chans att erövra Torsebro red Huitfeldt i full galopp tillbaka till Reventlow och avgav rapport. Reventlow gav order till huvudstyrkan i Härlöv om att bege sig till platsen. Han tog själv de 700 ryttare han hade med sig och red mot Torsebro, men då han kom fram hade mörkret redan börjat lägga sig. Från den svenska sidan hade Hierta sänt en kurir till Kristianstad för att underrätta Gyllenstierna om läget och samtidigt satte man eld på bron, som dock inte förstördes helt. Vid beskedet gav Gyllenstierna order om att förstärka vakten under natten. +På morgonen den 13 januari hade huvuddelen av den danska styrkan hunnit fram och tagit ställning vid Helge ås västra strand. Marinregementet stod uppställt med 300 man och så fort det ljusnade öppnade man eld med fältartilleriet. Den inledande salvan dödade tre svenska ryttare och resterande styrkor tvingades att söka skydd. Samtidigt vadade trupper från Livregementets dragoner över ån vid vadställen norr och söder om bron och kunde beskjuta de svenska styrkorna från sidorna. Under täckande eld från resterande delar av regementet stormade sedan 80 soldater ur Marinregementet tillsammans med generaladjutanterna Bendix Meyer och Lothar Adolf Henrik Klepping resterna av bron vid Torsebro, som fortfarande var överfarbar. När de danska styrkorna nått över bron flydde de svenska ryttarna och omringade tvingades de sachsiska trupperna att överlämna sig. Klockan var halv nio på förmiddagen och de danska ingenjörstrupperna satte genast igång med att reparera bron. Ungefär två timmar senare kunde den danska huvudstyrkan korsa Helge å. +Gyllenstierna hade vid gryningen låtit sina trupper ta ställning vid Nosaby medan han själv stannat kvar i Kristianstad för att överse evakueringen av staden samt förstörelsen av Långebro. Efter erövringen av Torsebro sände Reventlow iväg Livregementets dragoner under Christian Rodsteen för att jaga efter de flyende svenska ryttarna. Runt lunchtid nådde ryttmästare Hierta fram till Gyllenstierna och kunde berätta att danskarna hade reparerat bron och att huvuddelen av armén var på väg över. Då Gyllenstierna insåg att hans styrka snart skulle vara underlägsen den danska gav han order om reträtt och lämnade Kristianstad. En förtrupp ur Livregementets dragoner om 150 man nådde ifatt Gyllenstiernas trupper strax innan Fjälkinge. För att sakta ner danskarnas framfart gav han order till Hierta att ta ställning strax öster om Fjälkinge med Västgöta kavalleriregemente och bataljonen ur Sachsiska infanteriregementet för att förhindra att bli kringränd av de danska trupperna. Förtruppens befälhavare, överstelöjtnant Koppelau, rapporterade tillbaka om svenskarnas ställning och Rodsteen beordrade sina trupper att göra halt medan han skickade generaladjutant Huitfeldt till Reventlow för att fråga om han skulle gå till anfall. Reventlow skickade genast tillbaka order om att Rodsteen omedelbart skulle gå till anfall. +Anfallet utfördes vid skymningstid och vid åsynen av de anfallande danskarna tog de svenska ryttarna till flykten och lämnade de sachsiska fotsoldaterna ensamma kvar. Rodsteens livdragoner fortsatte att jaga efter de svenska ryttarna medan nästa danska trupp, ledda av generallöjtnant von Dewitz, nådde sachsarna och erbjöd dem att överlämna sig, vilket de gjorde. Öster om Fjälkinge träffade Rodsteens trupper på Västgöta tre- och femmänningsregemente till häst, som överrumplades och flydde hals över huvud, vilket skapade oordning i de övriga skvadronerna. Snart anlände dessutom von Dewitz med sina trupper. Gyllenstierna försökte organisera ett motanfall och fick samlat ihop 300 ryttare, men vid det tillfället hade det redan hunnit bli mörkt och istället fick han ge upp och fortsätta sin reträtt. Mörkret gjorde även att Reventlow avblåste de danska kavalleritruppernas jakt på svenskarna. I och med den svenska reträtten hade danskarna säkrat övergången vid Helge å. +Stenbock stod vid Sissebäck vid Valje på gränsen mellan Skåne och Blekinge och lyckades där fånga upp största delen av de flyende svenska trupperna, även om vissa av dem inte stannade förrän de nått Karlshamn. Ovetande om att Reventlow dragit tillbaka sina trupper lät han tända vakteldar för att ge intryck av att hans styrka var större än den egentligen var och drog sig tillbaka till Sölvesborg. Där möttes han av 500 man från Skånska tre- och femmänningsregementet till häst som varit stationerat i Åhus och senare ankom även de mindre vaktstyrkor som varit utplacerade längs Helge å. +Från Sölvesborg drog sig Stenbock vidare mot Norje, där landsvägen var inklämd mellan sjön Vesan och havet vilket var en utmärkt försvarsposition för trupper i numerärt underläge. Dock hade vinterkylan gjort att isarna frusit på sjön och träsken däromkring, vilket fick Stenbock att inse att det inte längre var möjligt att försvara dessa ställningar. Han gav den 14 januari order om att ställningen vid Norje skulle överges och drog sig tillbaka till Mörrumsån. Även här hade vattnet frusit till is till den grad att man kunde ta sig över ån med häst och släde. Dessutom gjorde kylan att man inte kunde bygga några befästningar i den frusna jorden. Stenbock kallade till krigsråd med alla sina regementschefer där även amiralgeneralen Hans Wachtmeister som anlänt från Karlskrona deltog. Bedömningen gjordes att man inte kunde hålla ställningen och dessutom var trupperna utmattade och moralen låg efter nederlagen vid Torsebro och Fjälkinge. Man insåg att man inte kunde hindra danskarna att ta sig in i Blekinge och beslutet togs att överge Mörrumsställningen och återsamla armén i Ronneby. Alla broar på vägen till Karlskrona skulle brännas för att ge tid åt stadens försvar att göra sig redo. +Den danska kåren slog läger öster om Fjälkinge och Reventlow tog högkvarter i Ljungbygård (nuvarande Trolle Ljungby slott) och trupperna fick vila ut sig några dagar. Där uppsöktes han av det danska sändebudet till Stockholm, Johan Diderik Grüner, som hade utvisats från Sverige till följd av den danska krigsförklaringen. Innan Grüner hade tvingats lämna den svenska huvudstaden hade han nåtts av säkra underrättelser om svenskarnas återuppsättande av nya regementen och hur dessa snart skulle kunna vara på väg. Efter beskedet skickade Reventlow förnyade uppmaningar till Fredrik IV om förstärkningar. Morgonen den 19 januari bröt Reventlow upp från Ljungbygård och marscherade mot Norje. Där fann de bron sönderbränd, men inga svenska trupper. Reventlow antog därför att Stenbock hade valt att hålla stånd vid Mörrumspasset, men väl där mötte han inte heller då något motstånd. +Karlshamn hade i och med detta lämnats helt öppet för danskarna och den 19 januari kunde Reventlow tåga in i staden oemotstridd. Man undvek däremot att göra ett försök att inta Karlshamns kastell som låg på en ö utanför staden. Kastellet bemannades av 70 svenska soldater förstärkta av 145 trossdrängar. Den danska överbefälhavaren installerade sig i köpmannen Peter Mestertons gård och till en början var ockupationen närmast vänskaplig. Soldaterna uppförde sig väl och officerarna roade sig på stadens gästgiveri. Danskarna hade dock fått reda på att svenska förråd skulle ha förvarats i staden, men kunde inte påträffa några sådana trots att man förhört rådmannen Gislou Pijhlgard om saken. Därför beordrades en total genomsökning av staden och all proviant som påträffades togs i beslag vare sig det var i privat ägo eller svenska statens. +Den danske kungens önskan om att inta Karlskrona ansåg Reventlow skulle bli svårt att uppfylla. Stadens befästningar hade dessutom förstärkts och örlogsflottans skepp med uppemot 1 800 kanoner samt de omkring 4 000 sjömän som var stationerade på fartygen kunde användas till stadens försvar. På grund av oron för att bli avskuren av den nyuppsatta svenska armén norrifrån hade han varit tvungen att lämna kvar flera eftertrupper för att bevaka de olika passen vid Norje och Mörrum och hade därför endast kvar två infanteribataljoner och ett fåtal artilleripjäser. Dessa skulle inte räcka för att inta staden och trots att han utöver infanteriet hade ett stort kavalleri lämpade detta inte sig för att storma en befäst stad. Dessutom hade de danska trupperna töjt sina försörjningslinjer och började få slut på proviant. Bedömningen gjordes att den svenska armé som började ansamlas i Småland skulle användas för att återta Helsingborg och bryta belägringarna av Malmö och Landskrona och därför valde Reventlow att vända tillbaka till Kristianstad. Dessutom fick Reventlow fortfarande inga förstärkningar från dansk sida, vilket innebar att hans styrka sakta vittrade bort, medan den svenska armén byggde sig allt starkare. +För att säkra resurstillgången sammankallades Karlshamns borgare den 23 januari och presenterades med ett ultimatum att betala 20 000 riksdaler i brandskatt för att undvika att danskarna brände ner staden. En så stor summa bedyrade stadens borgare att de inte kunde frambringa och Reventlow gick med på att sänka summan till 12 000 riksdaler. Av detta förmådde Karlshamnsborna samma dag få fram 6 000. För att säkra betalning av resterande del av summan togs borgarna Peter Masterton, Jörgen Hegardt, Jacob Wildschiött, Peter Jacobsson och Carl Aschenborg som gisslan och fördes via Helsingborg hela vägen till Korsør. De släpptes dock slutligen fria i juni samma år. Den 24 januari påbörjades marschen tillbaka till Skåne och på vägen rev man de nylagade broarna vid Mörrum, Norje och Nymölla. +Den 27 januari anlände Reventlow åter till Ljungbygård och den danska armén inkvarterades i och kring Kristianstad. Där fick han den 28 besked om hur svenska trupper ur den armé som var placerad i Dalsland och Bohuslän för att skydda mot ett anfall från Norge var på väg söderut för att antingen ansluta sig till Stenbocks armé eller skära av försörjningslinjen från Helsingborg. Det blev uppenbart för Reventlow att han inte kunde lita på någon andra front från generallöjtnant von Tritzschler. Denne hade visat sig vara synnerligen handlingsförlamad när det kom till att bryta in i Bohuslän. När hans trupper stod redo att korsa Svinesund upptäckte han att armén saknade bröd och beordrade halt. En månad senare upptäcktes vid en inspektion att den skärgårdsflotta som samlats för att skydda överfarten saknade bestyckning. Förtvivlat skrev Reventlow åter till Fredrik IV: +Något svar från Fredrik IV på sin anmodan fick Reventlow inte. Under dessa dagar arrangerade kungen en karusellridning i Köpenhamn inför sin älskarinna och verkade inte villig att skänka en tanke på fälttåget i Sverige. I bittert sarkastiska brev till ledande ämbetsmän bad Reventlow ironiskt vederbörande att tala med kungen om krigshärens angelägna behov "efter karusellen, ty för närvarande är det ju ej tid därtill" och tänka på saken "efter karnevalen". +Den 8 november 1709, sex dagar efter den danska landstigningen vid Råå, hade ryttmästare Henrik Hammarberg ankommit till Stockholm och avgett sin rapport till defensionskommissionen. Denna gav genast order om att de helt nyuppsatta Kronobergs, Jönköpings, Kalmar, Östgöta och Södermanlands infanteriregementen, samt Smålands och Östgöta kavalleriregementen så fort som möjligt skulle göras redo att marschera ner mot Skåne och den avtalade mötesplatsen vid Loshult. Besked om ytterligare trupper i form av Upplands fyr- och femmänningsregemente till häst, Västmanlands infanteriregemente och Älvsborgs regemente nådde Stenbock den 6 december. Stenbock lämnade Ronneby och anlände den 21 januari i Växjö där de svenska trupperna hade börjat ansamlas. Dagligen lät man de oerfarna trupperna öva på isen på en frusen sjö utanför staden och snart fick soldaterna även uniformer hämtade ur militärförråden. Hästar och proviant hade också börjat samlas in från omgivningarna, även om kavalleriet kom att få klara sig med vanliga bondhästar. +Defensionskommissionen kunde i januari skicka även Upplands regemente, Adelsfanan och Livregementet till häst till Stenbocks förfogande. Den myt som senare uppkom om "Måns Bocks getapojkar", där man föreställde sig hur den snabbt ihopskrapade armén bestod av dåligt utrustade bonddrängar, som slogs i trätofflor för att försvara landet mot den lede dansken var långt från sanningen. Rekryterna var friska och starka män mellan 20 och 30 år som tilldelats moderna skjutvapen. Dock fanns det en brist på värjor och endast pikenarna utrustades med sådana medan övriga tilldelades bajonetter att sätta på musköterna. Den snabbhet med vilken den nya armén sattes upp visade på indelningsverkets effektivitet. +När det kom till att hitta officerare till den nya armén hade flera pensionerade officerare tidigt anmält sig för tjänstgöring, bland annat överste Jacob Grundel, som utsågs till chef för Södermanlands infanteriregemente, och överste Lamoral van der Noth, som utsågs till chef för Upplands fyr- och femmänningsregemente. Dessutom dök flera veteraner från Karl XII:s armé i Poltava upp månaderna efter slaget då de antingen utväxlats eller flytt från krigsfångenskapen i Ryssland. Bland dessa kan man hitta generalmajor Johan August Meijerfeldt och artillerimajoren Carl Cronstedt, som båda kom att spela avgörande roller för utkomsten av kriget i Skåne. Meijerfeldt hade erbjudits platsen som generallöjtnant under Stenbock, men avstod på grund av ett inte helt läkt benbrott och tjänade istället som generalmajor i Stenbocks generalstab. Andra var överste Gustaf Adam Taube som befordrades till generalmajor och blev del av Stenbocks generalstab, samt Vilhelm Bennet som utsågs till chef för Upplands fyr- och femmänningsregemente till häst. I den svenska arméns generalstab ingick, förutom Meijerfeldt och Taube, generallöjtnant Jacob Burensköld, landshövding i Östergötland vilken Stenbock själv begärt från defensionskommissionen då de tjänat tillsammans i början av stora nordiska kriget i Kurland, och Magnus Palmqvist, som senast varit kommendant i Wismar. +Innan alla regementen hade hunnit sluta upp i Växjö bröt Stenbock den 31 januari upp och marscherade med en stor del av armén mot Osby. Anledningen var att trakten kring Växjö nästan var helt tömd på proviant och inte längre kunde försörja trupperna. Man behövde ta sig ner till det mer bördiga Skåne för att säkra tillgången på proviant. De förband som inte hunnit fram fick order om att följa efter. Den 3 februari anlände de första trupperna till Osby och Stenbock skickade genast ut trupper för att säkra broarna över Helgå å vid Hönjarum och Hasslaröd. Den 8 februari anlände generalmajor Christian Ludvig von Ascheberg med ytterligare trupper från Bohuslän och den 11 februari anlände Gyllenstierna med de tre kavalleriregementen som hade kvarstannat i Blekinge. Till slut hade Stenbock här samlat en styrka på 19 regementen och över 16 000 man. +Försörjningssituationen var kritisk för både den danska och den svenska armén. Den 3 februari nåddes Reventlow av beskedet att Stenbocks armé var i rörelse och dagen efter siktade en spaningspatrull de svenska vakttrupperna vid Hasslaröds bro. Målsättningen blev nu att förhindra att den svenska armén kunde rycka ner i det proviantrika Skåne och hålla emot till dess att förstärkningar skulle kunna komma från Norge eller Danmark. Reventlow placerade ut sina disponibla trupper i en linje som sträckte sig från Kristianstad i öster till Hässleholm och vidare till Herrevads kloster längst i väster. Den östra flanken skyddades på så sätt av Helge å medan den huvudsakliga västra flanken skyddades av Almaån och Finjasjön. I ryggen hade man Nävlingeåsen. +Denna gång var det danskarnas tur att få sina väl lagda försvarsplaner kullkastade av kylan, då frosten fick de flesta vattendragen i trakten att frysa till is. Därför koncentrerade Reventlow istället sina trupper till en kortare linje söder om Almaån och de två arméerna stod nu omkring 23 km från varandra. Han gjorde även bedömningen att det skulle bli väldigt svårt för Stenbock att ta sin armé öster om Helge å då detta skulle bli en väldigt lång marsch där flera sjöar utgjorde naturliga hinder och flyttade därför den 6 februari sitt huvudkvarter till Sinclairsholms slott. Därifrån skickade han flera rekognosceringspatruller för att underrätta sig om de svenska truppförflyttningarna. När det stod klart att de svenska trupperna från Bohuslän inte var något hot mot Helsingborg, skickades Prinsen av Hessens regemente och Fynska infanteriregementet från sin position vid Ängelholm österut för att ansluta den danska huvudarmén. Med dessa förstärkningar disponerade Reventlow över runt 9 000 man. Några besked om ytterligare förstärkningar från Danmark eller Norge uteblev och den 8 februari skickade Reventlow ett skarpt formulerat brev till överkrigssekreterare von Lente där han skrev: +Reventlows anmodan förmedlades något nedtonad till Fredrik IV av von Lente. Helt oväntat hade kungen vid detta tillfälle vaknat till liv och engagerade sig plötsligt väldigt mycket i den skånska kampanjen. Kungen meddelade att flottan hade fått order om att påbörja en blockad och bombardemang av Malmö från sjösidan. Förstärkningar skulle också ordnas fram, dels genom rytteriregementen från Själland, men kungen planerade även att så fort vädret tillät det låta flottan skeppa ner styrkorna i Norge och hämta trupper från Holstein. Det enda av detta som kom att bli av var att delar av Östsjälländska och Västsjälländska lantmiliserna skeppades över till Helsingborg i början av februari för att avlösa trupperna där. +Stenbock lade fram en fälttågsplan vars mål var att manövrera bort den danska armén från Skånes inre och tvinga dem tillbaka till Helsingborg. För att göra detta var han tvungen att röra sig söderut ner mot Kävlingeån och vidare till Malmö för att på så sätt säkra upp de sydskånska slätterna med deras rika tillgång på proviant. I Malmö fanns dessutom 20 kanoner som Stenbock desperat var i behov av. Om planen lyckades skulle den svenska armén erhålla ett stort förplägningsområde och dessutom stöd från fästningarna i Malmö och Landskrona. Stenbock påbörjade sin framryckning från Osby den 12 februari med armén uppdelad i två kolonner. Den östra rörde sig ner mot Glimminge, medan den västra marscherade mot Hästveda. Trupprörelserna var en skenmanöver från Stenbocks sida och en del i planen att få Reventlow att röra sig västerut mot Helsingborg för att öppna upp de södra slätterna. +För att ta reda på vad den svenska armén hade för sig skickade Reventlow på morgonen den 12 februari en trupp på 200 dragoner under generallöjtnant Rantzau norrut från Sinclairsholm. Med sig hade han generalmajorerna Rodsteen och von Dewitz. Vid Hästveda påträffade rekognosceringstruppen en förtrupp till den svenska västra kolonnen om 30 man. Den lilla truppen tog till flykten och danskarna följde efter, men snart kunde de se hur den svenska huvudstyrkan närmade sig. En trupp på 50 man lämnades kvar för att sakta ner svenskarna. Vägen kantades med jämna intervall av vinkelräta stengärdesgårdar som vardera var försedda med grindar som skärmade av vägen. De avsuttna dragonerna barrikaderade den första grinden vid Bäckaröd så gott det gick och tog ställning bakom stengärdet på vardera sida grinden. +Det första svenska regementet att nå fram var Östgöta kavalleriregemente följt av en skvadron på 300 ryttare från Västgöta kavalleriregemente. En eldstrid utbröt och de danska dragonerna drog sig tillbaka med de svenska styrkorna efter sig för att sedan ta skydd bakom nästa stengärde där de avlossade en salva och drog sig tillbaka till stengärdet där bakom. Detta fortsatte stengärde efter stengärde hela vägen till Rävninge. Striderna saktade ner de svenska styrkorna avsevärt, vilket gav de danska generalerna möjlighet att sätta sig i säkerhet då de annars var i verklig fara för att bli tillfångatagna. Under striderna stupade 13 danska dragoner medan 16 togs till fånga och därutöver tillfångatogs även truppens chef, överstelöjtnant Hans Henrik Bibow som fallit från sin häst under striderna. +När Reventlow underrättades om striden och den svenska arméns förflyttning tolkade han det som att Stenbock planerade att marschera mot Kristianstad för att sedan anfalla den danska armén i ryggen. Den fortsatta frosten gjorde också att den danska armén inte kunde lita på vattendragen som skydd. För att inte blotta sig för ett anfall drog han därför tillbaka sina trupper till Vä och sedan till Västra Vram, där ett tillfälligt högkvarter etablerades. I stället för att dra sig mot Helsingborg som Stenbock hoppades på drog Reventlow sig alltså åt söder. Livgardet, som förlagt i Kristianstad, fick den 13 februari order om att lämna staden och återsluta sig till armén. Innan man lämnade staden meddelades borgarna att de skulle vara tvungna att betala en brandskatt på 9 000 riksdaler. Denna summa kunde staden inte frambringa, men danskarna gav staden en kredit och tog åtta av dess mest framstående borgare som gisslan: Anton Cortmejer, Paul Friis, Ludolph Plattfuss, Johan Wegman, Jöns Dunckler, Jöns Munck, Anders Munck och Jöns Kockum. Dessa kom att föras till Korsør på samma sätt som de borgare som tagits som gisslan från Karlshamn. +I Västra Vram höll Reventlow den 12 februari krigsråd med sina generaler. Reventlow förordade att skapa en ny försörjningslinje till Själland över Barsebäck där han hoppades kunna få förstärkningar och, om det värsta skulle inträffa, varifrån den danska armén skulle kunna skeppas hem igen. Reventlow ville också skapa en försvarslinje vid Lödde å och genom denna skära av Stenbock från Malmö och de sydskånska slätterna. På så sätt skulle han kunna tvinga Stenbock till drabbning i centrala Skåne, något som den svenska generalen helst ville undvika. Rantzau och von Dewitz ansåg att man skulle möta den svenska armén i ett avgörande slag så snart hela armén var samlad och hade fått tillräckliga förstärkningar och ville att man skulle gå mot Ystad, där det fanns gott om proviant. Rodsteen förordade att man skulle bryta belägringarna av Landskrona och Malmö och ställa sig i en linje längs Rönne å. Reventlow beslutade att följa sin egen intuition och armén beordrades att fortsätta över Linderödsåsen mot Skarhult söder om Eslöv. +Armén var framme den 16 februari och inkvarterades så att den var skyddad av Rönne å, vars kraftiga vattenflöde gjort att den inte frusit i kylan, och Ringsjön på vänsterflanken och Vombsjön och Lödde å på högerflanken. Den envisa förkylning som Reventlow lidit av under hela kampanjen blev under den nattliga förflyttningen ner till Skarhult ännu värre och natten till den 17 februari drabbades han av svåra ryggsmärtor och feber. På morgonen kunde han knappt röra sig och lät därför åderlåta sig, men då han inte blev bättre insåg han att han inte längre kunde utföra sin plikt som befälhavare för armén och sände samma dag ett brev till kungen där han meddelade sin avgång. Som sin ställföreträdare hade han utsett generallöjtnant Jørgen Rantzau. +De ytterligare förstärkningar kungen utlovat hade fortfarande inte setts till. De hade uppenbarligen blivit försenade på grund av sådana bagatellartade detaljer som att regementenas igenkänningsfärger på foder och uppslag inte hade rätt nyans. Reventlow skickade återigen ett syrligt brev till kungen där han försäkrade om att det i stridens hetta inte har någon betydelse om vapenrocken har en mörkare blå eller ljusare blå nyans. Reventlow inkvarterades efter sjukdomsutbrottet i Lund för att där få vård. +Efter träffningen vid Hästveda rörde Stenbock sig den 14 februari mot Vankiva vid dagens Hässleholm med avsikten att säkra det strategiskt viktiga Möllerödspasset. Målsättningen var nu att röra sig ner till mitten av Skåne för att skära av den danska arméns förbindelse med Helsingborg. Vid Vankiva slog armén läger och armén fick ett drygt dygns vila medan Stenbock skickade iväg Gyllenstierna med 1 000 man västerut för att säkra så många broar som möjligt på väg till Rönne å. Man lyckades säkra bron vid Riseberga och en spaningsstyrka skickades till Hasslebro vid Stockamöllan där huvudvägen mellan Kristianstad och Helsingborg korsade Rönne å. Då danskarna hade sparat bron för att behålla sina kommunikationer österut fann man den oskadd, men efter rykten om att den danska armén var på väg beordrade befälet för spaningsstyrkan att bron skulle rivas. +Den 16 februari bröt armén upp och rörde sig söderut. De svenska trupperna var i akut behov av proviant. Enligt Stenbock hade manskapet inte fått bröd på fyra dagar och både soldater och hästar började kuva under av hunger. Vid beskedet om att Hasslebro hade förstörts gjordes halt vid Norra Rörum den 17 februari. Senare inkom rapporter från spaningspatruller att de siktat danska trupper på andra sidan av Rönne å. Trupperna de sett kom från Bülows ungerska dragonregementet, vilka tillsammans med Tredje själländska kavalleriregementet och Själlands lantdragonregemente låg utplacerade längst åt norr, ohållbart långt från den danska huvudstyrkan. Stenbock såg en möjlighet att överrumpla dessa och utarbetade en plan som innebar att Jacob Burensköld skulle säkra Forestads bro och korsa Rönne å med en mindre styrka, antingen över bron eller vid det kända vadstället, för att angripa danskarna i ryggen medan Stenbock själv anföll vid Hasslebro. +Stenbock skickade bud till Burensköld om sin plan, men på grund av kommunikationsproblem missförstod denne Stenbocks order och när anfallet skulle utföras stod han kvar på östra sidan om Rönne å. När Stenbock på morgonen den 18 februari trots den förstörda bron vid Hasslebro lyckades korsa ån med Upplands fyr- och femmänningsregemente till häst fann han därför inga trupper från Burensköld. Istället träffade han på Bülows ungerska dragoner under befäl av överstelöjtnant Magnus Ernst von Prehn. Den danska generalstaben, medvetna om att Stenbock närmade sig och de tre nordligt placerade ryttarregementenas utsatthet, hade natten innan gett order om att dessa skulle röra sig längre söderut. När Stenbock väl utförde sitt anfall hade de ungerska dragonerna ännu inte hunnit bege sig, men de hann se hur svenskarna korsade Rönne å och begav sig då snabbt söderut. En skvadron på 300 svenska ryttare satte efter dem ner mot Bosarp. +Vid Bosarp stod Tredje själländska kavalleriregementet och gjorde sig klart för avmarsch och deras befälhavare, överste Hans Frederik Leegel, kunde se de rödklädda ungerska dragonerna komma ridande efterföljda av blåklädda ryttare. Först antog han att dessa var Själlands lantdragonregemente, då de också bar blå uniformer, men snart insåg han att det var svenskar och beordrade sina trupper att anfalla för att skydda de danska ryttarna. På håll kunde Stenbock uppfatta faran och tog personligen befälet över de resterande delarna av Upplands fyr- och femmänningsregemente till häst för att anfalla Leegels regemente i ryggen. När Leegel såg Stenbocks trupper avbröt han sitt anfall och drog sig tillbaka österut, men hanns upp av svenskarna och lyckades endast efter svåra strider och stora förluster ta sig tillbaka till det danska högkvarteret, mycket tack vare mörkrets inbrott som fick svenskarna att avbryta sin jakt. De ungerska dragonerna jagades till Gullarps kyrka, där överstelöjtnant Prehn beordrade en del av sitt regemente att sitta av och ta täckning bakom kyrkogårdsmuren, medan resterande del tog ställning i skydd av kyrkan. När de svenska ryttarna anlände avlossade de avsuttna dragonerna en muskötsalva varefter de beridna trupperna gick till anfall. Sex svenska ryttare dödades och sex sårades och togs till fånga, medan resten av de jagande trupperna drevs tillbaka. +Under tiden som ryttarstriderna pågick hade en provisorisk bro över ån uppförts vid Hasslebro och det svenska infanteriet hade under dagen börjat korsa över. Senare anslöt även Burensköld till huvudarmén då han slutligen lyckats korsa ån vid Forestad med sina trupper. Armén återsamlades vid Eriksholm (nuvarande Trolleholm), där de uttröttade ryttarna och deras hästar fick vila ut efter en lång dag. Rantzau hade meddelats om att svenskarna korsat Rönne å och skickade en kurir till Köpenhamn för att rådfråga kungen om vad han skulle göra. Under tiden fortsatte han enligt Reventlows ursprungliga plan att söka drabbning så fort som möjligt i centrala Skåne och han letade en lämplig position att möta svenskarna på söder om Eslöv. Han lät också säkra ställningen vid Getinge bro över Kävlingeån och generaladjutant Huitfeldt skickades för att beordra generalmajor Brockdorff att bryta belägringen och ansluta sina trupper till huvudarmén. Brockdorff hade samtidigt fått order från kungen att förstärka belägringen av Malmö, som gett besked om att en bataljon ur Viborg-lolländska lantmilisregementet hade skickats som förstärkning. Brockdorff valde att följa Rantzaus order och drog sig mot Skarhult. Under natten skickades den sängliggande Reventlow för säkerhets skull från Lund till Helsingborg. +På morgonen den 19 februari ställde sig den danska armén i slagordning i öst-västlig riktning vid den utvalda platsen mellan byarna Roslöv och Skreglinge söder om Skarhult, men några svenska trupper dök inte upp. Vid middagstid kallade Rantzau sina befälhavare till krigsråd. I och med den svenska positionen hade oron att Stenbock planerade att skära av de danska försörjningslinjerna från Helsingborg ökat. Rantzau, som tillsammans med von Dewitz tidigare hade förordat ett avgörande slag så fort som möjligt, sällade sig nu till den majoritet av staben som ansåg att man skulle dra sig tillbaka till Helsingborg. Den danska armén bröt upp och förflyttade sig österut med målet att ta sig till Annelöv. Trupperna som belägrade Landskrona fick också order om att avbryta sin inneslutning och förena sig med huvudarmén. Den danska armén hann under dagen endast till Stora Harrie norr om Kävlinge där man inför natten åter ställde upp sig i slagordning. +På kvällen återkom kuriren från Köpenhamn med en skrivelse från Fredrik IV undertecknad den 18 februari med svar på Rantzaus förfrågan. I denna godkände kungen Rantzaus utnämning till ny befälhavare och gav sitt tillstånd för armén att möta svenskarna i ett avgörande slag. Han varnade också för att armén inte skulle dra sig för långt från Helsingborg, då Stenbock riskerar kunna skära av denna från stadens proviantförråd och de nya förstärkningarna. I samma skrivelse uppmanade kungen också befälhavarna att välja terrängen för slaget med stor noggrannhet då det svenska kavalleriet var starkare än det danska. Beskedet från kungen sågs av den danska arméledningen som en bekräftelse av beslutet att dra sig mot Helsingborg. +Stenbock nåddes på kvällen den 19 februari av beskedet om den danska arméns läge vid Stora Harrie, knappt en mil söder om Eriksholm. Då Stenbock var osäker på om den danska armén var på väg mot Helsingborg eller planerade anfalla hans position gav han order till armén att ställa sig i slagordning. Ett krigsråd sammankallades klockan tre på morgonen där det beslutades att det var dags att möta den danska armén i ett avgörande slag så fort som möjligt och armén sattes i rörelse. På morgonen den 20 inkom underrättelser om att den danska armén redan avmarscherat och rörde sig mot Landskrona. Den synbara danska tillbakaryckningen till Helsingborg mottogs med glädje och förvåning av Stenbock eftersom danskarna hade gett honom fri lejd till södra Skåne. I en försändelse till Karl XII hade Stenbock skrivit att han inte vågat skära av den danska arméns från Helsingborg på grund av "om på den händelse fienden ej hade satt sig bakom Kjeflinge bro och således sökt evitera all combat och att blifva mästare af slätten, hvarigenom hela mitt propos hade studsat". Danskarna hade alltså genom att dra sig tillbaka till Helsingborg spelat Stenbock i händerna. +På morgonen den 20 februari bröt den danska armén upp från Stora Harrie och marscherade norrut i vinterkylan. Omkring klockan 11 passerade de Annelöv med målet att ta sig över Braån vid bron vid Norra Möinge (mellan Asmundtorp och Billeberga). Stenbock såg en möjlighet att trakassera den danska armén då den skulle vara sårbar medan den korsade bron, kanske till och med få den att helt stanna upp. Han skickade tidigt på morgonen ut överste Vilhelm Bennet med de mest utvilade kavalleriförbanden för att bita den danska armén i hälarna medan huvudarmén sattes i marsch efter. +Den danska armén hade på eftermiddagen nått fram till bron vid Norra Möinge och påbörjat överfarten av sina trupper. Generalmajor Rodsteen var chef för det danska arriärgardet (eftertruppen) och ansvarade som sådan för försvaret av bron. Längst bak gick Livregementets dragoner under befäl av generalmajor Brockdorff. Denna påträffades av Bennets trupper längs vägen och strider utbröt. Brockdorffs dragoner satt av och tog skydd bakom de pilevallar som genombröt landskapet. Generaladjutant Huitfeldt, som ridit med Brockdorffs trupp, skickades till Rantzau med besked om striderna. Där fick Huitfledt instruktioner om att rida till Rodsteen och beordra honom att ställa upp sina trupper till försvar på norra sidan av Braån. Rodsteen körde fram fältkanonerna och beordrade sin styrka på 400 man från Själländska lantdragonregementet att förskansa sig medan han även skickade ut spanare för att underrätta sig om var den svenska huvudarmén var. +Efter att ha sakta stridit sig fram från pilevall till pilevall medan de danska dragonerna taktiskt drog sig tillbaka nådde Bennets trupper vid solnedgången fram till bron. Vid denna tidpunkt hade nästan hela den danska armén hunnit passera över bron, så när som på delar av trossen och en liten trupp dragoner, och de svenska trupperna lyckades erövra flera bagage- och proviantvagnar och tillfångata de danska trupperna. Danskarna på andra sidan ån öppnade eld med fältartilleriet och de svenska ryttarna tog skydd på södra sidan i väntan på att resten av armén skulle anlända. När mörkret fallit på utan att den svenska huvudstyrkan dykt upp valde Bennet att dra tillbaka sina trupper. Rantzau beordrade vid 18-tiden generalmajor von Eickstedt att ta infanteriet och artilleriet och marschera till Råå. Vid klockan ett på natten drog sedan även Rantzau sig med kavalleriet mot Råå, men lämnade kvar en eftertrupp på 200 man som fortsatte beskjuta de svenska posteringarna under natten. Inte förrän klockan tre drog även denna sig tillbaka. +Natten till den 21 februari anlände Rantzau med sin armé till Råå och därigenom var den danska armén efter fyra månader tillbaka vid landstigningsplatsen. Armén var helt utmattad då detta var tredje natten i rad utan vila och till råga på allt hade trossen skickats i förväg till Helsingborg vilket innebar att man saknade tält när man slog läger. Fälttåget hade slitit hårt på armén och över tusen av de danska soldaterna fick skeppas över till Helsingör för vård så fort som möjligt. På förmiddagen ställdes armén upp i slagordning i en linje från bron över Råån till Glumslöv för att vara redo för ett eventuellt svenskt anfall. Rapporter från rekognosceringspatruller berättade dock snart att den svenska armén inte hade förföljt danska armén mot Helsingborg. +Den utökade tidsfristen utnyttjade Rantzau genom att hålla krigsråd där man noga valde ut arméns positioner med hänsyn till terrängen. Att armén skulle placeras i nära anknytning till Helsingborg stod klart, men var en sådan ställning i relation till staden var bäst kunde diskuteras. Generalmajor Ditlev Brockdorff förespråkade att armén skulle ställa sig med ena flanken vid resterna av de gamla befästningarna (vid nuvarande lasarettet), för att sedan sträcka sig över Bollbrolyckan ner till stranden. Efter en inspektion av terrängen drog Rantzau slutsatsen att detta läge skulle bli alltför trångt och svårmanövrerat och förslog vid ett nytt krigsråd att armén skulle ställas i linje på den öppna slätten bakom mossen mellan Kropp och Välluv för att därigenom ha större manöverutrymme. De danska befälhavarna fruktade att en ställning så långt från Helsingborg skulle försvåra försörjningen av armén och föreslog att man istället placera armén närmare staden, på höjderna utanför befästningsverken. Rantzau godtog förslaget och den 23 februari slog den danska hären läger nordöst om staden, tvärs över landsvägen mot Ängelholm, mellan Ringstorpshöjden och Fredriksdal. +Samma dag anlände oväntat Fredrik IV till Helsingborg för att bese arméns förberedelser. Där höll han en timmes möte bakom stängda dörrar med den sjuke Reventlow och red därefter ut för att inspektera de danska ställningarna. Nöjd med vad han såg återvände han följande dag till Köpenhamn, men innan han reste såg han till att förse Rantzau med ytterligare förstärkningar. Mellan den 25 och 27 februari anlände Första fynska kavalleriregementet och Gardet till häst i omgångar från Danmark. Eventuella hopp om att Reventlow skulle hinna friskna till innan avgörandet tynade allt eftersom bort, och generalen skeppades över till Helsingör den 26 februari. +Efter den danska arméns tillbakadragande valde Stenbock att låta sina utmattade trupper vila ut och armén inkvarterades i Västra Karaby, Norrvidinge och Skrävlinge där de äntligen kunde återhämta sig och få en mindre mängd mat. När det stod klart att danskarna hade övergett belägringen av Malmö red Stenbock genast dit för att hämta de 20 kanoner som stått där och väntat tillsammans ett stort förråd av proviant. Där återförenades han även med sin familj. Efter några dagars vila bröt den svenska armén den 26 februari upp från sina lägerplatser och marscherade norrut och korsade Råån vid Tågarp varefter man fortsatte till Gedsholm och Hässlunda. Här stannade man för natten och Stenbock inkvarterade sig på Gedsholms herrgård. För att underrätta sig om svenskarnas position skickade Rantzau ut spaningspatruller i flera olika riktningar: Ramlösa, Hässlunda, Filborna, Kropps socken och Pålsjö. Truppen från Hässlunda kunde rapportera att de där siktat lägereldar, vilket innebar att svenskarna korsat Råån. +På morgonen den 27 februari var Stenbocks plan att vända västerut för att anfalla den danska armén från öster, mellan Välluv och Kropp, på grund av terrängens försvarsmöjligheter. När besked kom att det inte stod någon dansk armé vid denna plats valde han på grund av terrängens svårframkomlighet att istället marschera ytterligare norrut mot Kropp och Fleninge nordost om Helsingborg. Stenbock hade därefter föredragit att fortsätta mot Helsingborg och gå till anfall direkt, men avstod eftersom det var söndag. Istället slog armén läger för natten och Stenbock etablerade sitt högkvarter i Hjälmshults kungsgård några kilometer västerut. Då trossen hade hamnat på efterkälken tvingades de svenska trupperna att sova utan tält genom den regniga och vindpinade natten. +Efter kvällens vila under bar himmel var den svenska armén klockan två natten till den 28 februari marschfärdig och formerades i fem kolonner med artilleriet i mitten omgivet på båda sidor av infanteriet och med kavalleriet på flankerna. Stenbock utgick från att den danska armén skulle stå i slagordning tvärs över den nordostliga landsvägen mellan Brohuset och Väla. Han valde därför att, istället för att marschera längs landsvägen in mot Helsingborg, låta armén röra sig åt väster mot Allerum, för att sedan svänga söderut mot Pilshult. Den landsväg armén färdades på var så smal att endast artilleriet kunde utnyttja den vilket innebar att infanteriet och kavalleriet tvingades marschera genom den täta skog som gick ända fram till vägkanten. +När solen började gå upp på morgonen den 28 februari föll en tät dimma över hela området. Rantzau red klockan sex på morgonen ut med sina generaler för att rekognoscera och konstaterade att dimman var så tät att han inte kunde se tre hästlängder framför sig. Tillsammans med generaliteten nådde han slutsatsen att de svenska styrkorna omöjligen kunde förflytta sig i så dålig sikt och att ett anfall troligen inte skulle ske förrän om flera timmar. Klockan tio på förmiddagen började dimman lätta och de danska spejarna kunde då se de första svenska trupperna marschera fram genom skogen vid Pilshult. Tidigare underrättelser hade gjort givande att den svenska armén låg nordost om staden och den danska stridsledningen hade antagit det som mest troligt att den svenska armén skulle komma längs landsvägen från nordost. Att den svenska armén kom marscherande rakt norrifrån kom därför som en stor överraskning. Den danska armén ryckte ut och klockan elva stod den uppställd och klar för strid. Även Stenbock överraskades när han såg att den danska positionen var mycket mer fördelaktig än han tänkt då den stod närmre Helsingborg med vänstra flanken väl skyddad på Ringstorpshöjden och centern skyddad av svagt frusna kärr. +Då den svenska armén kom från en mer nordlig riktning än Rantzau väntat sig och dess högra flank sträckte sig ner till Pålsjö, antog han att det svenska huvudangreppet skulle ske mot hans vänstra flank. Rantzau gav därför order om att två infanteribataljoner skulle förflyttas från andra linjen i centern till vänstra flygeln för att förstärka artilleriställningen på Ringstorpshöjden. Medan den svenska armén fortfarande formerade sig påbörjade de danska kanonerna på Ringstorpshöjden vid klockan halv tolv sin beskjutning. Stenbock såg att den danska positionen på Ringstorpshöjden var alltför stark för att gå till direkt angrepp och beordrade därför hela armén att göra en rörelse österut. Han började även vrida armén något söderut med målet att så småningom kunna stå parallellt med den danska linjen och möjliggöra en utflankering av den danska högra flygeln. Stenbock gav order till Burenskiöld att bege sig med sina kavalleriregementen fram mot Brohuset och Filborna för att säkra vänsterflanken. Den svenska centern utvecklade sig samtidigt över våtmarkerna norr om landsvägen och började sakta rycka fram under ständig kanoneld med cheferna till häst framför linjerna. +Rantzau satt till häst framför den danska centern och iakttog de svenska trupprörelserna. När han såg hur armén rörde sig österut insåg han att det svenska huvudanfallet inte skulle ske vid Ringstorpshöjden utan att Stenbock sökte drabbning över hela slaglinjen. Det stod också klart att den svenska vänsterflygelns framryckning mot Brohuset hotade att utflankera den danska högerflygeln. Rantzau var väl medveten om att hans flyglar var svagare än sina svenska motsvarigheter på grund av det danska kavalleriets numerära underläge. Han beordrade genast Rodsteen matt han skulle förflytta sina trupper på högerflygeln mot Filborna för att möta den svenska kavalleristyrkan. +I denna stund gjorde sig Rantzaus ovana som härförare påmind, då han förbisåg att sända bud till resten av sina generaler om att denna förflyttning endast gällde högra flygeln. När Rodsteen påbörjade sin framryckning mot Filborna skapade detta en stor lucka mellan centern och högerflygeln, vilken snart var en kilometer stor. Luckan i linjen gjorde chefen för den danska centern, von Eickstedt, orolig och han gav därför på eget initiativ order om en generell framryckning och rörelse åt höger av den danska centern. Då den danska vänsterflygeln på Ringstorpshöjden däremot höll sina ställningar började den danska armén att töjas ut och vred sig till att stå längs med landsvägen åt nordost. +Strax efter klockan tolv hade Burenskiöld tagit fram sig till Brohuset och ställde upp sina trupper i formation vid en lång stengärdesgård som avskilde Brohuset från Filborna. Rodsteen hade vid ungefär samma tid nått Filborna och ryckte därefter fram mot Brohuset, där de svenska trupperna höll på att formera sig. Rodsteen avancerade och började ta ställning vid en 150 meter bred lucka mellan en stengärdesgård och ett moras, nu mycket nära de svenska trupperna. Burenskiöld kunde inte tillåta det danska kavalleriet att stänga öppningen mot Filborna och så fort han såg det danska kavalleriet rusa fram från höjden beordrade han sina tillgängliga trupper att gå till motanfall. +När Rantzau nådde fram till Filborna övertog han befälet över de danska kavalleristyrkorna under anfallet. Hans långa vana av att leda kavallerianfall tog i denna stund överhanden, då han som överbefälhavare istället borde ha stannat kvar vid centern för att hålla uppsikt över hela sin här. Det svenska kavalleriet drabbade snart samman med den danska fronten, ledd av Livgardet till häst. De numerärt överlägsna danska trupperna tryckte dock snart tillbaka svenskarna mot stengärdesgården. Under stridens togs Burenskiöld till fånga och fördes till Helsingborg. De svenska trupperna skingrades och retirerade, men många fastnade vid ett grindhål i gärdesgården och stacks i trängseln ner av de förföljande själländska lantdragonerna. Total förlorade svenskarna under denna del av slaget 147 man i stupade och 401 man i sårade. +Den svenska vänsterflygelns resterande trupper började efter hand anlända till platsen och generalmajor Göran Gyllenstierna lyckades samla ihop de flyende resterna av Västgöta och Östgöta kavalleriregementen och begav sig sedan genast in i striden med sina trupper. Under tiden hade Stenbock stått vid centern och sett hur de svenska trupperna tryckts tillbaka. För att stärka upp vänsterflanken beordrade han befälhavaren för andra linjen på högerflygeln, överste Carl Gustaf Dücker, skulle ta 1 200 trupper och rida bort till Brohuset. När Rantzau såg att allt fler svenska regementen började ansluta sig återsamlade han sina trupper och beordrade anfall. Den första danska anfallsvågen lyckades åter trycka tillbaka de svenska trupperna, men efter hand som att allt fler svenska regementen började ansluta sig till striderna stannades framryckningen av och striderna blev allt mer desperata. Bud skickades till von Dewitz på den danska vänsterflygeln med anhållan om förstärkningar och han skickade över 600 ryttare till högerflygeln. +I de hårda striderna skadades danska Livgardets chef svårt och både Livregementets dragoners och Själländska lantdragonregementets chefer stupade under de hårda stridigheterna. Generaladjutant Bildstein lyckades samla ihop en trupp flyende svenska ryttare, vända dem till anfall och överraskade Livregementets dragoner i flanken. Ytterligare svenska trupper anslöt och regementet började ta till flykten. Vid den tidpunkten ankom ännu fler svenska trupper och anföll resten av det danska kavalleriet i flanken som även det började retirera. Rantzau lyckades stanna upp en del av de flyende trupperna och samlade dem till ett motanfall, men blev under handgemänget skjuten genom skuldran och bröstet av en svensk fänrik. Den danska högerflygeln var dock redan bruten och flykten mot Helsingborg gick inte att hindra. +Ungefär samtidigt som den andra ryttarstriden närmade sig sin början vid Brohuset började de två arméernas centrar närma sig varandra. Den danska positionen stod högre än den svenska och var väl skyddad bakom spanska ryttare som medtagits under framryckningen. Den svenska centern rörde sig så fort det över huvud taget gick över de frusna våtmarkerna men terrängen gjorde dock att avancemanget gick långsamt och under tiden fortsatte beskjutningen från det danska artilleriet. De svenska regementskanonerna hade stora svårigheter att ta sig fram genom morasen och flera gånger tvingades den svenska frammarschen stannas av i väntan på att artilleriet skulle kunna medföras. +Den första infanterisammandrabbningen skedde vid östra delen, mellan Smålands och Upplands femmänningsregementen, samt Sachsiska regementet på svenska sidan, och Livgardet till fots och Drottningens livregemente på den danska. När de svenska regementena kom inom 10 till 15 meter från den danska linjen avfyrades massiva salvor av druvhagel- och karteschladdningar från regementskanonerna samtidigt som infanteriet unisont avfyrade sina vapen. Strax efter föll det svenska infanteriet in mot danskarna med pikar och blanka vapen. Drottningens regemente drabbades särskilt hårt av den svenska artillerielden och regementschefen träffades av artillerield och föll ner död. Samtidigt såg några av soldaterna i det bakre ledet hur danska kavallerister från högerflanken flydde mot Helsingborg jagade av svenska ryttare och fruktan att stängas av från Helsingborg fick soldaterna att ta till flykten. Det närmaste regementena till vänster, Fynska regementet och Jylländska regementet, drogs med i flykten. +Kvar i första linjen stod endast Livgardet till fots vid högerflygeln och Grenadjärkåren vid vänsterflygeln, som bjöd envist motstånd mot de svenska trupperna. I stridens hetta träffades Livgardets regementschef av kanoneld som slog av hans ben i knähöjd. Under en kort tid fortsatte han ändock att kommendera sina trupper, men stacks till slut ner av en svensk pikenär. Snart anslöt sig Södermanlands regemente till de anfallande svenska trupperna och Livgardets situation blev ohållbar. Regementets förluster var oerhörda. Med 500 döda eller sårade och 490 tillfångatagna blev hela förbandet i praktiken upplöst. +I den västra delen av slagfältet stod även Grenadjärkåren isolerat efter mittenregementenas flykt. Detta regemente stod emot det svenska trupperna lika obstinat som Livgardet och häftiga strider utspelade sig mellan dem och Älvsborgs regemente. Efter hand som fler svenska infanteritrupper ryckte fram mot Grenadjärkåren blev deras ställning blev allt mer trängd. När försvagade danska kavalleriet på vänsterflygeln tvingades till flykten av det numerärt överlägsna svenskt kavalleri insåg generalmajor von Eickstedt att vidare motstånd var lönlöst och beordrade dansk reträtt. +Runt klockan tre på eftermiddagen var huvuddelen av slaget över. Enstaka mindre strider utspelade sig utanför Helsingborgs vallar. De sista striderna bedarrade slutligen vid halv fyra-tiden. De danska förlusterna var förödande. Närmare 7 500 man hade stupat, sårats eller tillfångatagits och dessutom var artilleriet i svenska händer. Mellan 1 500 och 2 000 hade stupat och mellan 3 000 och 3 500 sårades under slaget. Antalet tillfångatagna uppgick till 2 677. Förlusterna för den svenska armén uppgick till 810 döda och 2 098 sårade, totalt 2 908. +Resterna av den danska armén sökte sin tillflykt innanför de hastigt uppförda vallarna kring Helsingborg. Under generalmajor von Eickstedts befäl fördelades delar av infanteriet till att besätta olika delar av Helsingborgs förskansningar för att skydda mot ett möjligt svensk anfall mot staden. Något svenskt anfall kom dock inte då Stenbock trots sitt övertag valde att inte genast anfalla Helsingborg de det skulle kräva alltför stor manspillan, vilket man inte hade råd med. En kurir skickades till Köpenhamn med besked om nederlaget och senare under dagen ankom en eskader av örlogsskepp för att beskydda Helsingborg från kustsidan. +Den 1 mars förde svenskarna ner ett batteri på 10 kanoner, 2 haubitsar och 2 mörsare till stranden nedanför Pålsjö för att därifrån beskjuta Helsingborg och den danska flotteskadern. Någon större effekt hade beskjutningen dock inte och elden besvarades av de danska örlogsskeppen. Den 3 mars ryckte svenskt kavalleri och infanteri fram mot Helsingborg, troligen vid Ängelholmsporten, men då danskarna öppnade eld från kanonerna på Kärnans topp drog de svenska trupperna sig tillbaka. Samtidigt fortsatte eldgivningen från det svenska batteriet på udden norr om Kullaporten, dels mot staden och dels mot skeppsbryggan och de danska skeppen. En av de svenska artilleripjäserna lyckades träffa det danska huvudmagasinet för krut och ammunition som var inhyst i ett hus i närheten av Mariakyrkan, vilket resulterade i en våldsam explosion som utplånade ett helt kvarter och en hög eldkvast reste sig över staden. Granaterna fortsatte att brisera i serier och smällarna skapade stor förvirring och oro då många av de danska trupperna trodde att svenskarna hade brutit sig in i staden. +Evakueringen av trupperna till Helsingör påbörjades den 4 mars. Det stod klart att det inte skulle vara möjligt att inom rimlig tid föra med sig det omfattande proviantförråd som samlats i Helsingborg, trossen och dess vagnar, samt större delen av arméns hästar. Då man inte ville att detta skulle falla i svenskarnas händer började man redan samma kväll bränna upp trossens rustvagnar och ammunitionsvagnar. De första beväpnade trupperna att föras över var kavalleriet. Bedömningen gjorde att man inte hade möjlighet att skeppa över alla kavalleriets hästar eller de hästar som följt med trossen. Då man inte ville att dessa skulle falla i fiendens händer valde man att sticka eller skjuta ihjäl dessa, totalt runt 5 000. Endast officerarnas hästar skonades och fördes över till Danmark. För att vålla så stort besvär som möjligt för svenskarna hade hästarna till och med släpats ner i källare och släppts ner i brunnar. +Den 5 mars överskeppades dragonerna och infanteriet. Under evakueringen tog danskarna all proviant som samlats i stadens förråd, runt 10 000 tunnor råg, korn och havre, och dumpade ut det på stadens gator. Där blandades det med gyttja, hästblod och hästavföring och blev fullständigt oanvändbart. Resterande 22 kanoner förnaglades och allt kvarvarande krut ströddes ut på marken. Klockan nio på kvällen steg von Dewitz tillsammans med generaladjutant Huitfeldt och trettio man ur Tredje själländska kavalleriregementet som stannat kvar vid foten av skeppsbron ner i den sista slupen och därmed lämnade de sista danska trupperna Helsingborg och Skåne. +De flesta historiker pekar på den danske kungens passivitet som en stor anledning till att fälttåget blev ett sådant misslyckande. Både den i synen på fälttåget banbrytande historikern Arthur Stille och den mer sentida historikern Anders Blidberg lyfter i sina respektive böcker fram flera exempel där Reventlow vädjar till Fredrik IV om förstärkningar och fler resurser men att dessa till största delen går obesvarade från Fredrik IV. Blidberg spekulerar att pressen från alla beslut som krävde den danske kungens godkännande blev honom övermäktig och att han då istället valde att distrahera sig med sin älskarinna eller maskeradbaler. +Hur det danska fälttåget skötts pekas också ut att ha haft stor betydelse. Den danske historikern Karsten Skjold Petersen pekar i sin bok om fälttåget på det faktum att Reventlow inte fick gehör för sina krav att Malmö och Landskrona skulle intas, vilket hade kunnat frigöra många trupper för fälttåget mot Karlskrona. Eftersom dessa städer var befästa hade de också varit säkrare brohuvuden för överföring av förstärkningar och materiel. Utöver det hade man förnekat Stenbock tillgång till de 20 kanoner som fanns på Malmöhus slott. Även Stille påpekar att danske kungen borde ha följt Reventlows råd och intagit städerna. Blidberg, Stille och Claes-Göran Isacson menar alla att Reventlows sjukdom och det faktum att han ersattes av Rantzau hade ödesdigra konsekvenser för fälttåget. Reträtten till Helsingborg, som till största delen var Rantzaus beslut, understödd av Fredrik IV, har också kritiserats historiskt sett. Den långa reträtten tillbaka till Helsingborg, föranledd av just bristen på trupper av att många var uppbundna vid belägringarna, anser både Stille och Skjold Petersen bör ha brutit ner den danska arméns moral. Artur Stille, som var den första historikern i Sverige att egentligen se till hela fälttågets betydelse och inte bara slaget vid Helsingborg, framhäver också hur framgångsrik Stenbocks fälttågsplan var och hur han lyckades manövrera motståndararmén dit han ville ha den innan det avgörande slaget. Rantzaus brist på fältherreerfarenhet och det faktum att han ska ha saknat det lugn och känsla för överblick som Reventlow besatte är några punkter som förs fram för betydelsen för utkomsten av slaget vid Helsingborg. Detta kom sedan i uttryck av två mycket konkreta misstag från Rantzaus sida under slaget. +En annan faktor som framhålls är den svenska statsapparatens effektivitet när det kom till att mobilisera resurser och sätta upp nya förband. Även om Sverige hade en enväldig monark i Karl XII, hade denne delegerat ansvaret för landets försvar till defensionskommissionen, ledd av Arvid Horn, och Stenbock fick gott om stöd från denna. Blidberg framhäver även betydelsen av Stenbocks egen organisationsförmåga. Lars Ericson Wolke visar i sin artikel i Militärhistorisk tidskrift 2009–2010 att om tidsfaktorn krävde det kunde landshövdingar och guvernörer på eget initiativ besluta om egna krigsbördor på sin befolkning, något som Stenbock utnyttjade till hög grad. Handlingskraften inom de svenska myndigheterna sätts av Blidberg i kontrast till den danska beslutsgången, som fortfarande innebar att kungen skulle tillfrågas inom alla ärenden. Då kungen som nämnts tidigare hellre tillbringade sin tid på andra aktiviteter kom detta att länge lamslå det danska fälttåget. +Efter Stenbocks seger vid Helsingborg började en hjältekult växa kring honom i det svenska riket. Han fick personliga gratulationer från änkedrottningen Hedvig Eleonora, prinsessan Ulrika Eleonora, hertig Karl Fredrik av Holstein-Gottorp, kung Stanisław Leszczyński och hertigen av Marlborough. Över hela riket firades Stenbocks seger med hyllningstal, skrifter och konstnärliga verk, och i en allmän tacksägelseceremoni den 18 mars lovprisades Stenbocks namn. I april reste Stenbock till Stockholm för att informera riksrådet om det nya strategiska läget. Vid ankomsten dit hyllades han av stadens invånare och han höll flera segertal inför präst- och bondestånden. Rikets ständer skänkte honom Bååtska palatset och den 21 maj utnämnde riksrådet honom till fältmarskalk, och skickade fullmakten till Karl XII:s högkvarter i Bender för kungens underskrift. Kungen underlät däremot att besvara fullmakten och utnämnde i stället Stenbock till riksråd, vilket betydde att han motvilligt fick avsäga sig sin tjänst som guvernör över Skåne. Kungen ändrade sig dock och stadfäste fältmarskalksutnämningen två år senare. +Från danska sidan konstaterades att fälttåget hade varit en katastrof och Fredrik IV begärde in redogörelser från alla i den danska generalstaben för att reda ut vad som gått fel. Särskilt ville han veta vilka order som getts och huruvida dessa efterföljts. Klart var att det var tid att utpeka syndabockar. Rantzau kritiserades hårt för sin insats under slaget, särskilt av överkrigssekreteraren von Lente, men bedömningen gjordes att han trots allt visat på initiativ och personligt mod, och han undkom större konsekvenser. Han fick 1711 ansvaret för inneslutningen av Wismar. Skulden föll i stället på Reventlow, som avskedades som överbefälhavare den 12 april. Reventlow togs dock i nåder igen och våren 1712 gick han i kungens tjänst som diplomat och var Danmarks sändebud till Peter den store och August den starke inför slaget vid Gadebusch. Efter Stenbocks seger var han avgörande då han lyckades övertala en uppgiven Peter den store att inte avbryta sitt fälttåg och återvända till Ryssland. Det visade sig att Reventlow hade drabbats av gikt och generalen blev sängliggande i sju månader efter fälttåget. Samtidigt med Reventlow avskedades chefen för det danska förplägnadsväsendet, justitieråd Christoffer Joachim Giese. Han fick något mer rättmätigt skulden för de senfärdiga materiel och proviantleveranserna. Den 17 maj avskedades överkrigssekreteraren von Lente och ersattes på tjänsten av Valentin von Eickstedt. Befälhavaren i Norge, von Tritzschler, avskedades för sin oförmåga att öppna en andra front i norr och utreddes utöver detta för landsförräderi, men utredningen nådde aldrig till rättegång. +Stora nordiska kriget rullade dock på och den danska armén var trots förlusten i Helsingborg fortfarande i stridbart skick vilket gjorde att hotet om ytterligare invasioner fortfarande var påtagligt. Danskarna fortsatte göra flotträder längs Öresundskusten för att sabotera för svenskarna. Till exempel anfölls Båstad den 25 juli och brandskattades och den 25 augusti anfölls Råå och plundrades. Då hotet inte var helt över inkvarterades den svenska armén under våren runt om i Skåne. Meningen var att de skånska bönderna skulle förse de svenska trupperna med den mat de behövde, men efter att ha försörjt två arméer på totalt över 30 000 under två vintermånader var proviantbristen stor. Till råga på allt drabbades Skåne sommaren 1710 av missväxt och de svenska soldaterna hemföll åt stöld för att överleva. Uppgivet beklagade sig Stenbock till vännen Hans Wachtmeister: +I september 1710 lyckades den svenska flottan under Hans Wachtmeister att överraska den danska flottan vid sjöslaget vid Köge bukt och därefter genskjuta en transportflotta menad för att transportera 6 000 ryska soldater från Gdańsk till Själland. Under slaget fattade linjeskeppet Dannebroge, som varit flaggskepp för invasionsflottan vid Råå 1709, eld och exploderade. Slaget gav ytterligare andrum för Sverige och minskade under en period hotet för invasion, vilken innebar att en del av trupperna i Skåne kunde lämna landskapet. För att långsiktigt förhindra fler landstigningar vid Råå beordrade Karl XII att en lång befästning i form av ett retranchement skulle uppföras längs kuststräckan norr om fiskeläget. Retranchementet, som numera benämns Råå vallar, stod klart 1713, men kom aldrig till användning. Olyckorna fortsatte dock för den hårt prövade skånska befolkningen och i september 1710 nådde den pestepidemi som härjat runt Östersjön sedan 1709 landskapet. +Ännu år 1716 planerade Danmark att återta Skåne med hjälp av ryska och brittiska trupper. Detta företag blåstes dock av av Peter av Ryssland i september med hänvisning till försörjningsläget och den förestående vintern. De bakomliggande orsakerna till den inställda invasionen torde dock varit de allierades samstämmiga intressen och osäkerhet om vilken riktning de svenska motanfallen kunde tänkas rikta sig. De allierade hade inte helt samstämmiga intressen och läget var otydligt: den dansk-norska flottan bands upp till den norska kusten och det gick rykten i Europa om att Sverige planerade landsätta trupper i Skottland för att stödja huset Stuarts försök att återta den skotska tronen vilket band upp brittiska trupper. I oktober lämnade de ryska trupperna som hade avsetts delta i invasionstrupperna Själland. Denna inställda invasion var det sista konkreta försöket av Danmark att återta sina forna östra provinser. +Blidberg, Anders (2018), Sista striden om Skåne, Stockholm: Bokförlaget Langenskiöld, ISBN 978-91-88439-26-0 +Ericson Wolke, Lars (2011), ”Mellan Poltava och Tönningen : Svenska operativa förutsättningar och resursmobiliseringen 1709–1713”, i Nilsson, Mikael, Militärhistorisk tidskrift 2009–2010, Militärhistorisk tidskrift, Stockholm: Försvarshögskolan, s. 45–71, ISSN 0283-8400, http://fhs.diva-portal.org/smash/get/diva2:1234145/FULLTEXT01.pdf +Hårdstedt, Martin; Iko, Per; Ericson Wolke, Lars; Sjöblom, Ingvar; Åselius, Gunnar (2003), ”Helsingborg 1710: Danskarna lämnar Skåne”, Svenska slagfält, Stockholm: Medströms Bokförlag, ISBN 91-46-21087-3 +Isacson, Claes-Göran (2000), Vinterbrand: Fälttåget i Skåne 1709–1710 och slaget vid Helsingborg, Helsingborg: Helsingborgs museiförening, ISBN 91-87274-22-1 +Johannesson, Gösta, red. (1969), Hälsingborgs historia, del IV:1 : Försvenskningen, Stockholm: Almqvist & Wiksell +Liljegren, Bengt (2000), Karl XII: en biografi, Lund: Historiska media, ISBN 91-85377-14-7 +Skjold Petersen, Karsten (2017), Det ulykkelige slag : Helsingborg 1710, Köpenhamn: Gads forlag, ISBN 978-87-12-05413-9 +Stille, Arthur (1903), Kriget i Skåne 1709–1710, Stockholm: Militärlitteraturföreningens förlag, https://www.tacitus.nu/karoliner/kallor/Stille/index.htm +Vaupell, Otto Frederik (1872), Den dansk-norske hærs historie indtil 1814, Köpenhamn: Gyldendal +Larsson, Olle (2009). Stormaktens sista krig : Sverige och stora nordiska kriget 1700-1721. Lund: Historiska Media. ISBN 9789185873593 +Persson, Bodil E. B. (1997). ”1710 års fältsjuka i Skåne.”. Förhandlingar från XVI Nordiska medicinhistoriska kongressen (Stockholm : Fören. för utgivande av Svensk medicinhistorisk tidskrift, 1997): sid. 195-209 : diagr.. Libris 2797653 +Tuxen, A. P.; With, C. L.; Hansen, A. L. (1903). Felttoget i Skaane 1709–1710. Bidrag til Den Store Nordiske Krigs Historie. "2. Bind". Köpenhamn: Det Nordiske Forlag. http://www.marinehist.dk/orlogsbib/SNK/SNK-bind2.pdf +Gunnar, Wetterberg (2009). Skånes historia, del 2 : 1376–1720. Stockholm: Albert Bonniers Förlag. ISBN 9789100142070 +Åberg, Alf; Göransson Göte (1976). Karoliner. Höganäs: Bra Böcker. Libris 128073 +Wikimedia Commons har media som rör Stora nordiska kriget. +Wikisource har originalverk som rör Danska fälttåget i Skåne och Blekinge 1709–1710. +Helsingborgs Museum. Arkeologiska fynd från Slaget vid Helsingborg +Kriget i Skåne 1709-10. Korrespondens mellan svenska tjänstemän inför och under fälttåget. +Li Si (李斯; Lǐ Sī), född cirka 280 f.Kr., död 208 f.Kr., var en framstående kinesisk politiker, filosof och kalligraf. +Li Si verkade som Kinas försteminister från 221 f.Kr. till 209 f.Kr. och hade en betydande roll under Kinas enande under Qins föreningskrig. Han var en tung maktfaktor under stora delar av Qindynastin (221 f.Kr.–206 f.Kr.) och var förespråkare för legalismen. Li Si låg bakom den omtalade bokbränningen och begravningen av akademiker och hade även en central roll i tronföljdskonspirationen efter att kejsar Qin Shi Huangdi avlidit. Li Si avrättades 208 f.Kr. efter att ha protesterat mot maktkoncentrationen runt kejsar Qin Er Shi. +Li Si föddes ungefär 280 f.Kr. under tiden för De stridande staterna. Han kom från Shangcai i dagens Henan som tillhörde staten Chu. Li Si lärde sig statsvetenskap av den kända konfucianska filosofen Xun Zi där han studerade tillsammans med Han Feizi. Som färdigutbildad bedömde han att det bara var i staten Qin som det fanns någon framtid eftersom Qin såg ut att vara den starkaste av de sju staterna som stred om makten över Kina. Li Si begav sig därför mot Qin med ambitioner att bli rådgivare åt dess kung, och han anlände till Qin 247 f.Kr. i samband med att Qins kung Zhuangxiang just avlidit. Li Si sökte jobb hos Lü Buwei som då var Försteminister (丞相). Efter att kungen avlidit tillsattes 246 f.Kr. den trettonåriga Ying Zheng som kung av Qin. Lü Buwei fortsatte som försteminister och Li Si utsågs till Ministerns tjänare (舍人). +I samband med kungens myndighetsdag 238 f.Kr. uppdagades att markis Lao Ai planerade ett uppror, vilket slogs ner efter ett blodigt motangrepp. År 237 f.Kr. blev Lü Buwei avsatt som försteminister på grund av sitt samröre med Lao Ai, och Li Si fick en starkare position och kom närmare kungen. Han steg snabbt i graderna och utsågs till Översekreterare (长史) och därefter till Gästminister (客卿). +År 237 f.Kr. i samband med gästingenjören Zheng Guo från Hans kostsamma uppförande av Zhengguokanalen blev kungafamiljen och flera ministrar misstänksamma mot alla gästarbetare och beskyllde dem för att vara spioner och krävde deras utvisning. Även Li Si nämndes som tänkbar spion, och han skrev då en lång framställan till kungen där han förklarade hur stora historiska fördelar som Qin haft av gästarbetare såsom till exempel filosofen Shang Yang. Kungen tog till sig Li Sis argument och inga gästarbetare utvisades. +Li Si övertygade kungen om vikten av att erövra de övriga konkurrerande staterna, vilket genomfördes under Qins föreningskrig 230 f.Kr. till 221 f.Kr. Li Si befordrades också till Justitieminister (廷尉). +När Qin 221 f.Kr. besegrat och erövrat alla konkurrerande staterna utropade sig kungen till kejsare Qin Shi Huangdi, och Li Si blev upphöjd till försteminister. +Det var många i hovet som förespråkade att kejsaren skulle återinföra ett feodalsystem likt den tidigare Zhoudynastin (1046 f.Kr.–256 f.Kr.) där kungens familjemedlemmar styrde olika regioner i landet, men Li Si övertygade kejsaren om att inte göra om Zhoudynastins misstag. I stället delades det nya riket upp i provinser vilka var för sig styrdes av en guvernör, en militär befälhavare och en inspektör. +År 213 f.Kr drev Li Si tesen att Qindynastin hade trätt in i en ny tid där allt hade fått en fast form. Han menade att tidigare dygder och akademiker som hänvisar till antiken bara skulle förvilla folket. Lärda hade tidigare i historien använt sin kunskap för att revoltera mot överheten. Folk som ägnar sig åt självstudier kommer att ifrågasätta lagar. Li Si drev igenom förslaget att bränna alla historiska verk som inte var skrivna av Qin-historiker. Böcker såsom Sångernas bok, Dokumentens bok och andra filosofiska verk från de "hundra skolorna" skulle brännas, och att diskutera Sångernas bok skulle leda till avrättning. Året efter bokbränningen utfördes stora mängder förhör med akademikerna i Qindynastins huvudstad Xianyang om hur påbuden om förbjuden litteratur hade respekterats. Många akademiker angav varandra och kejsaren valde ut 460 personer som bedömdes ha brutit mot de nya lagarna. Dessa avrättades, och information om händelsen spreds som ett varnande exempel. +Under hemresan på kejsar Qin Shi Huangdis femte inspektionsresa 210 f.Kr. blev kejsaren allvarligt sjuk. Kejsare Qin Shi Huangdi insåg att han var döende och skrev ett brev till kronprinsen Fu Su att denne skulle ta över makten och överlämna armén till general Meng Tian. Brevet överlämnades till eunucken Zhao Gao som var ansvarig för postgången. För att undvika att något oförutsett skulle ske med prinsarna eller landet beslöt Li Si att dödsfallet skulle hållas hemligt. +Eunucken Zhao Gao skickade dock aldrig iväg brevet och han hade även tillgång till kejsarens sigill. Med på resan var även kejsarens yngre son Ying Huhai. Eunucken Zhao Gao och kejsarens yngre son Ying Huhai beslutade att förfalska tronföljdsbrevet, men insåg att de även behövde få med Li Si på konspirationen. Zhao Gao övertygade Li Si om att ifall kronprinsen Fu Su skulle komma till makten skulle den högt meriterade Meng Tian bli försteminister, och att avskedade försteministrar alltid blir avrättade. Med tårar i ögonen accepterade Li Si konspirationen. När de återkom till huvudstaden tillträdde Ying Huhai som kejsare under namnet Qin Er Shi +Li Si är den första dokumenterade kinesiska kalligrafen, och det var Li Si som skapade stilen Liten sigillskrift efter att Kina enats 221 f.Kr. Ett känt exempel på Li Sis kalligrafi är en stentavla i ett tempel vid foten av det heliga berget Taishan. Stentavlan restes ursprungligen på toppen av berget. +Li Si behöll sin position som försteminister under kejsar Qin Er Shi, men Zhao Gao hade blivit en betydande maktfaktor i hovet. Under kejsarens andra regentår 209 f.Kr. styrdes landet enbart av kejsaren och Zhao Gao. Li Si lämnade tillsammans med minister Quji och general Feng Jie in protester mot parets beslut. Protesterna ledde till att Quji och Feng Ji begick självmord för att de inte accepterade förödmjukelsen att förbises i maktordningen. Li Si fängslades och torterades med "de fem straffen" (五刑). Zhao Gao utsågs till försteminister, och Li Si avrättades 208 f.Kr. genom att offentligt huggas i två delar, och tre generationer av hans familj avrättades (三族). +Sima, Qian (2016). Kinas förste kejsare. övers. Pettersson, Bengt. Natur & Kultur. ISBN 9789127143029. http://www.adlibris.com/se/bok/kinas-forste-kejsare-9789127143029 +张, 英聘 (2003) (på engelska). THE HISTORY AND CIVILIZATION OF CHINA. 中央文献出版社. ISBN 7-5073-1360-3. http://www.amazon.com/History-Civilization-China-Zhang-Yingpin/dp/7507313603 +Loewe, Michael (1999) (på engelska). The Cambridge History of Ancient China: From the Origins of Civilization to 221 BC. Cambridge University Press. ISBN 978-0521470308. https://www.amazon.co.uk/Cambridge-History-Ancient-China-Civilization/dp/0521470307/ref=sr_1_1?s=books&ie=UTF8&qid=1479241132&sr=1-1&keywords=9780521470308 +Chen, Tingyou (2011) (på engelska). Chinese Calligraphy. Cambridge University Press. ISBN 978-0521186452. https://www.amazon.co.uk/Chinese-Calligraphy-Introductions-Culture/dp/0521186455/ref=sr_1_1?s=books&ie=UTF8&qid=1483429400&sr=1-1&keywords=9780521186452 +Hakarps kyrka är en kyrkobyggnad i Hakarps socken i Växjö stift. Den är församlingskyrka i Hakarps församling i Huskvarna pastorat. Kyrkan ligger i Hakarps kyrkby och är en rosa korskyrka och sevärdhetskyrka som byggdes 1694. Det har före stenkyrkan funnits två kyrkor: en stavkyrka, som byggts under 1100-talets mitt, samt en kyrka byggd av liggande timmerstockar, vilken uppförts omkring 1283. Timmerkyrkan från 1200-talet revs samtidigt som den nuvarande stenkyrkan byggdes. I den nya kyrkan finns bland annat ett krucifix som tros ha tillhört kyrkan sedan den första byggdes. +När stenkyrkan skulle byggas 1694 hade församlingen bevljats en kollekt, som togs upp under påsken. Dock räckte inte pengarna från kollekten till nybyggnationen, och generaltullinspektör Nils Arvidsson Hägerflycht bekostade då den nya kyrkan. Hägerflycht anställde Johan Olsson Ståhle som arbetsledare och en möjlig arkitekt var Erik Dahlbergh. +Hägerflycht bekostade den nya kyrkan med 6 000 daler kopparmynt samt 1 700 daler kopparmynt för orgeln och andra inventarier. +I enlighet med den trend som rådde i övriga Sverige under 1800-talet målades många av kyrkans inventarier och möbler över med vit färg, något som emellertid togs bort vid kyrkans 200-årsjubileum 1894. +Från och med 2014 ingår Hakarps församling i Huskvarna pastorat. +Det exakta byggnadsåret för Hakarps första kyrka är inte känt, inte heller varför kyrkan placerades där den gjorde. Man tror dock att kyrkan är samtida med kristnandet av området och grundandet av Linköpings stift, dit kyrkan först hörde. Linköpings stift grundades i mitten av 1100-talet. Under den första tiden efter kristnandet i området förrättades förmodligen gudstjänsterna utomhus med ett enkelt träkors som symbol, då Hakarp var för glesbebott för att ha råd med en egen kyrka. Ville man gå till en kyrka kunde man gå till Sanda kyrka som låg nere vid Vätterstranden vilket lär ha varit en av Södra vätterbygdens tidigaste kyrkor. Invånarna i Hakarp ville emellertid ha en kyrka och man timrade upp en träkyrka mitt i Hakarp, eller Hakatorp som det då hette. Det var både dyrt och tungt att bygga en egen kyrka och man tror att en församling därför bildades i samband med bygget. Man hade inte råd med en egen kyrkoherde utan man var en annexförsamling till Skärstads församling till 1566. Klockaren förrättade de gudstjänster som firades i kyrkan, vilket var fallet till in på 1500-talet. Den äldsta befintliga uppgiften om Hakarps kyrka är från Gustav Vasas klockuppbörd 1530. +Man vet ändock att det omkring 1100-talets mitt, efter 1141, uppfördes en stavkyrka. 2012 kom det fram uppgifter om att den nuvarande kyrkan kan ha haft två föregångare. När man skulle ta bort hussvamp 2011-2012 fann man kraftiga ekplankor. Undersökningar visade att några plankorna avverkades 1141. Detta tyder på att kyrkan har haft två föregångare, och att virke från båda återanvänts i den nuvarande kyrkan. Den första kyrkan var förmodligen en stavkyrka. Det finns ett trettiotal liknande fynd av rivna stavkyrkor i Sverige, och ekplankorna från Hakarps första kyrka motsvarar andra småländska kyrkor i Aringsås, Sjösås och Nöttja. +Från omkring samma tid är triumfkrucifixet som än finns kvar i kyrkan, samt en sandstensdopfunt. Det är osäkert om dessa ursprungligen kommer från Hakarps kyrka eller Sanda kyrka, som lades ned efter reformationen - varpå inventarierna delades upp mellan de närliggande kyrkorna. 1283 plockas stavkyrkan ner och ersätts av en timmerkyrka. Delar av materialet från stavkyrkan användes också för timmerkyrkan - förmodligen i golvet. +1283, enligt de dendrokronologiska dateringar man kan göra, plockades stavkyrkan ner, antingen för att den var uttjänt eller för trång. Den ersattes av en timmerkyrka. Delar av materialet från stavkyrkan återanvänds, förmodligen i golvet. Timmerkyrkan består av furutimmer som huggits i fyrkanter, samt ektimmer till golv och syllar. Fasaderna hade klätts med spån. +När man byggde den nya stenkyrkan stod den gamla träkyrkan kvar innanför de murade väggarna och man kan därför på ett ungefär uppskatta den gamla kyrkans storlek. Man använde mer än 120 sträckmeter virke från timmerkyrkan till stenkyrkans tak. Kyrkan hade en romansk plan. Långhuset var 9,45x6,6m, och det mindre koret 5,75x5,15m. Landytan uppskattas ha varit runt 60 km² och med 20-30 gårdar i församlingen vid den tiden bör alla ha fått plats i kyrkan. Kyrkan låg med koret åt öster och andra sidan åt väster. Den var uppförd av liggande timmer med synliga knutskallar. Uppskattningsvis hade kyrkan fem fönster och två dörrar. Huvudingången låg i den södra långväggens västra del och trappstenen utanför dörren bar inskriptionen MARIA. +En hel del av virket från timmerkyrkan ligger på stenkyrkans vind och är dekorerade med motiv från den bibliska historien. Konterfejarens namn är inte känt, men man ser likheter i bilderna på virket i Grännas, Säbys, Askeryds, Marbäcks och Ölmstads kyrkor. Målningarna tillkom vid en om- eller tillbyggnad år 1508. Konstnären kan ha varit lärjunge till Amund men var inte lika skicklig som denne, vilket framgår om man jämför foton från bilder gjorda av Amund i Gränna kyrka från innan kyrkan brann ner i slutet av 1800-talet. Vissa detaljer i bilderna från Hakarps kyrka liknar målningarna i Södra Råda kyrka. Efter detta stod förmodligen kyrkan relativt oförändrad fram till dess att den rivs 1694. +1556 ändrades kyrkans förutsättningar när Sanna socken upplöstes och ett antal av socknens gårdar överfördes till Hakarps församling som därigenom ökade i folkmängd. Församlingen fick då råd med en egen kyrkoherde. Anledningen till att Sanna socken upplöstes var att landhöjningen som pågått sedan inlandsisens avsmältning, med följden att Vättern kantrar åt söder, medförde att kyrkan riskerade att översvämmas. Socknens invånare ansåg sig inte ha råd att bygga en ny kyrka. Inkorporeringen gjorde att Hakarp kunde bilda ett eget pastorat. Den förste kyrkoherden blev Haqvinius Laurenti (Håkan Larsson). +De tidigare kyrkorna hade en dopfunt som dateras till runt 1150. På dopfunten, som är gjord av sandsten, är en biskop avbildad med en kräkla. Det fanns även ett krucifix som är bevarat från den gamla kyrkan och som idag står i den nuvarande kyrkan. Krucifixet anses härstamma från slutet av 1100-talet. Krucifixet kan ha kommit till Hakarp först runt 1500-talets mitt, från Sanda kyrka. +Under golvet i den gamla kyrkan fanns ett antal gravar övertäckta med släta hällar av kalksten med inskriptioner om vilka som ligger begravda. Kvar från den gamla kyrkan finns även ett altarskåp med figurer som glänste i guld och silver. På dörrarna till altarskåpet är Sankt Erik och Sankt Olof avbildade. En nattvardskalk, brudkrona och ljuskrona anses också vara kvar sedan den gamla kyrkan. +I slutet av 1600-talet började en reparation av kyrkan bli nödvändig. År 1675 skrev prästen Nicolaus Ingemari en vädjan om bidrag. Församlingen var liten, resurserna små och tjuvar hade stulit undansatta pengar. I en annan vädjan från 1690 stod det "att kyrkotaket är alldeles förruttet och odugligt". Ytterligare en vädjan från sockenbönder till landshövdingen löd: +Den 29 maj 1691 beviljades en kollekt för byggandet av en stenkyrka i Hakarps socken. I Huskvarna anlades emellertid ett bruk vilket ledde till folkökning och en kyrka planerades att byggas. Konungen godkände förslaget om bruket och kyrkan och Hakarpsborna blev oroade att de inte skulle kunna underhålla sin gamla kyrka, och inte heller ha råd med en kyrkoherde. Oron visade sig dock vara obefogad. Kollekten upptogs i kyrkorna i Växjö stift på annandag påsk 1694 och den nya kyrkan kunde börja byggas. +När kyrkan skulle byggas hade precis två år av missväxt passerat och församlingen hade inte råd att reparera sin kyrka eller bygga en ny. Problemet löstes när ägaren till Hakarps säteri, generaltullinspektör Nils Arvidsson Hägerflycht, åtog sig att bekosta den nya kyrkans byggnation. Hägerflycht var vid den tiden ägare till nästan halva Hakarps socken och en mycket rik man. Hägerflycht blev adlad 1675. Han tog sitt efternamn när han tillsammans med Karl XI med en lodbössa sköt ner en häger. Som de flesta andra adelsmän försökte man få sin familj begraven i en familjegrav inne i kyrkan. Då Hägerflycht började bli gammal är det troligt att löften om gravplats i kyrkan bidrog till att han bekostade kyrkbygget. +Man vet inte säkert hur lång tid kyrkan tog att bygga men det gick väldigt fort. Den 13 maj 1693 hade man inte börjat bygga kyrkan och 11 månader senare var väggarna klara och taket lagt. Att taket var lagt till påsk 1694 vet man av ett brev som den dåvarande kyrkoherden, Lars Lilliander, skrev. +Den nya kyrkan var inte helt klar och pengarna som kom in via den påbjudna stiftskollekten användes till de sista detaljerna. Den 2 december 1694 invigdes Hakarps nuvarande kyrka av biskopen. +Medan man byggde den nya kyrkan stod den gamla ännu kvar innanför murarna på den nya. Under byggnationstiden användes den gamla kyrkan som gudstjänstlokal. När den nya var klar slogs väggarna på den gamla in och bars ut genom porten. Kyrkan uppfördes enligt en grekisk korsplan, vilket var vanligt på barockkyrkor. Kyrkan är försedd med en sluten lanternin som spänner över det spåntäckta takets korsmitt. Från medeltidskyrkan har ett romanskt triumfkrucifix och en dopfunt bevarats. Edvard Orm målade 1729 de plana brädtaken, samt bibliska gestalter på sidoläktarna. På bänkarna är den ursprungliga bemålningen framtagen. Altaruppsatsen och predikstolen kommer från Eksjö kyrka. +Man vet inte vem som var arkitekt för kyrkan men en möjlighet är att det var Erik Dahlbergh. Arbetsledare för byggnaden var Johan Olsson Ståhle. Hägerflycht bekostade bygget med 6 000 daler kopparmynt samt 1 700 daler för orgel och andra inventarier. Han skänkte ett område till bostad för klockaren och bidrog med 114 daler till förflyttningen av prästgården. Hägerflycht och hans familj begravdes under altaret i kyrkan och till vänster om altaret sattes en minnestavla upp som än är kvar. Hans son, Arvid Hägerflycht, fick patronatsrätt vilket innebar att han fick tillsätta präst i pastoratet, något som följde med äganderätten till Hakarps säteri fram till mitten av 1800-talet. Därmed blev kyrkan vad som kallas en patronatskyrka. Altartavlan och predikstolen kom till kyrkan först fem år efter invigningen, 1699. +Ett antal saker såsom takmålningar vars färger flagnade, sprickor i stöttepelare, från muren fallande murbruk och så vidare gjorde att en renovering och restaurering under slutet av 1900-talet blev nödvändig. På grund av kyrkans kulturhistoriska värde behövde vissa försiktighetsåtgärder tas. Man anställde arkitekten Per Rudenstam och konservatorn Sven Wahlgren. Konservatorns arbete omfattade takmålningarna framförallt samt en allmän översyn på övriga inventarier. Ett förslag till renovering gavs av arkitekten till kyrkorådet den 24 juni 1969, vilket godkändes av alla berörda parter med enbart vissa små ändringar av Landsantikvarien och domkapitlet. Ett slutgiltigt förslag drevs igenom den 5 maj 1970. Arbetet kunde emellertid inte sättas igång förrän 1 september samma år på grund av att noggranna förberedelser krävdes. Arbetet beräknades vara klart vid midsommar 1971. Vid restaureringen av murbruket uppdagades att kyrkan tidigare varit tonad i rosa, vilket landsantikvarier med flera önskade. Kyrkorådet ville behålla den vita färgen och en kompromiss gjordes att färgen är vit med väldigt lätt rosa toning. +Då värmesystemet var en del av anledningen till att kyrkan var tvungen att restaureras valde man med omsorg ut det nya värmesystemet. Till slut valde man ett värmesystem konstruerat av dansken Horwath som användes i en del kyrkor. Det är konstruerat så att värmen inte stannar under taket och torkar sönder målningarna. Den 5 september 1971 öppnades kyrkan igen. Biskop Olof Sundby återinvigde kyrkan samt höll ett installationstal. Redan 1991 kunde man emellertid se att även detta danska värmesystem orsakade skador på kyrkan. Därför infördes elvärme i form av bänkvärmare av modell EVECO. Takmålningarna konserverades och ett vapenhus uppfördes. 2008 upptäckte man återigen skador, nu på takmåleri och under korgolv. Därför infördes ett nytt styrsystem, JEFF CC Kyrka El, för värme 2011-2012, tillsammans med ett omfattande konserveringsarbete. +När golvet vid restaureringen under 1900-talets andra hälft bröts upp för att läggas nytt upptäckte man plankor som tillhörde den gamla kyrkan. Plankorna kunde inte användas till golvet men togs tillvara. Man blottlade även en gravkammare som innehöll två kistor och som liknade gravkammaren där Hägerflycht begravdes. På grund av de upptäckta föremålen krävdes en oförutsedd arkeologisk utgrävning vilket försenade arbetet till att istället för midsommar vara klart i början av september. Av den arkeologiska utgrävningen framkom delar av något man tror var en kista, sju gravvalv innehållande gravstenar varav en del idag kan beskådas utanför kyrkan. Ett antal skelettdelar upptäcktes. Järn, spikar, mynt, mässingskedjor, glasögon, böcker, psalmböcker med mera upptäcktes också under utgrävningen. +Påträffade mynt: +Vid restaurationerna 2011 gjordes flera nya byggnadsarkeologiska rön. Altaret hade restaurerats tidigare 1953. Man fann dock att under putsen var det mesta av konstruktionen ursprunglig. Originalkonstruktionen bestod av huvudsakligen otuktade fältstenar i kalkbruk, med visst inslag av sandsten. Krönet hade sandstenshällar som man hittade rester av kalkputs på, avfärgade med vit och grå kalkfärg. Det var uppmurat på två grova ekplankor, som i sin tur vilade på tre ekbjälkar. En ekbjälke låg inmurad i originalkonstruktionen. Denna visades bära en ornering. Man fann även, när altarringen hade avlägsnats, spår av muralmåleri där altaret mötte väggen i norr. Måleriet var förmodligen en drapering som bröstning i ockra och järnoxid. Under altaret påträffade man, när man tog bort det gamla bjälklaget, ett kranium, ett lårben, en knapp, ett öre kopparmynt (Ulrika Eleonora) från 1719 och ett fragment av dekorationsmålad puts. +Det främsta fyndet utgjorde emellertid det äldre ekmaterialet. Tidigare hade man trott att det funnits en medeltidskyrka. Genom dendroanalys kunde man emellertid finna att det funnits två tidigare kyrkor. Det finns dessutom på en av plankorna svårtolkade uthuggningar, och på båda spår av brännmärken. Ytorna har bearbetats noggrant och väl, och det finns spår av sågmärken och hyvling. En av plankorna kunde man fastställa har fällts någon gång efter 1141. Förmodligen var plankorna väggplankor i en stavkyrka. Det finns likheter mellan dem och plankor som funnits i andra kyrkor. Väggplankorna har förmodligen stått i en syll och hållits samman av ett hammarband i ovankant. Genom nåtar som det har passats in lösa fjädrar i hölls plankorna ihop. Den bäst bevarade av plankorna exponeras i kyrkorummet, och den skadade togs till länsmuseets samlingar. +Motivet på den ristade bjälken utgörs av tre arkader. På dessa är människor framställda. Tre av dessa har tydlig gloria. Arkaden i mitten är tre decimeter bred. De omgivande har inga avslutande pelare. I den vänstra arkaden står två helgon vända mot höger. Dessa bär var sitt ljus. I den mittersta arkaden står ett helgon med toga. Helgonet gör en gest, vänd åt vänster. I den högra arkaden avbildas fyra gestalter, av vilka de till vänster vänds mot varandra. En av dem har förmodligen ett barn i famnen. De till höger är vända åt höger. En av dem håller ett föremål i höger hand. Den andra går inte att urskilja, eftersom bjälken har bearbetats här med yxa. Det fanns bunden tjära i virket. Det tyder på att bjälken var placerad på timmerkyrkans – den andra kyrkans – utsida. Den utgjorde, såsom var vanligt på medeltida stavkyrkor, därför förmodligen en del av ett hammarband. Det finns emellertid inga sådana med framställningar av människor, vilket gör dessa ristningar i så fall unika. Det finns växtornament och drakar på Kumlaby och Väversunda kyrka. Dessa är gjorda på 1100-talet. Det annorlunda motivet kan tyda på att dessa ristningar gjorts i gotisk tid, efter 1200-talets mitt. Den spräckning som förekommer kan ha orsakats vid den förmodade ombyggnaden 1508. +Från timmerkyrkan kom förmodligen de fyra bjälkar av ek man fann under altaret. Två av dem är förmodligen nedhuggna omkring vintern 1282-1283, enligt dendrokronologiska analyser. Den längre bjälken har en tydlig sprättäljning, om var en teknik som användes under tidig medeltid. En av ekbjälarna daterades till vintern 1507-1508, och en till 1509±7. Dessa tyder på att en ombyggnation ägt rum omkring 1508. Av de ekbjälkar man fann 1971 fanns fem bevarade på Hakarps säteri (de övriga svarvades till ekljusstakar saluförda som 'tusenårig ek från Hakarps kyrka'). De liknar syllbjälkarna, med ett undantag som har urtag - förmodligen för bindbjälkar. +Slutsatsen som länsarkeolog Robin Gullbrandsson drar från fynden är att det förmodligen restes en stavkyrka i Hakarp omkring 1100-talets mitt. Denna kyrkas väggar var gjorda av kluvna ekstockar, som bearbetats fint och satts i en syll. De var dessutom förmodligen sammanfogade med lösa fjädrar i långsidornas spår. Ett hammarband höll samman dem upptill. Detta liknar stavkyrkorna i Nöttja, Aringsås och Sjösås. Nöttjas kyrka uppfördes omkring 1146 och Sjösås omkring 1229, vilket innebär att man använt liknande teknik under nästan ett sekel i Småland. Denna stavkyrkas planform eller mått finns det ingen kunskap om. +Hakarps kyrka är en typisk korskyrka byggt med grekisk korsplan i ljusrosa och med väggar av sten. Stilmässigt är den en blandning av barock och rokoko. Korsmitten bär upp ett torn som, i likhet med takkonstruktionen i övrigt, är av trävirke. Kyrkan har två fönster i varje korsarm. Koret är i den östra delen som i många andra kyrkor. Kyrkans längd är 21 meter och bredden 20 meter. Murarna är runt sex meter höga. Tornet har en förgylld kyrktupp på toppen av en spira. +Läktaren är uppdelad i tre delar. Den norra var avsedd för fabriksarbetare och den södra för drängar och ungdomar. Den mittersta läktaren är avsedd för orgeln. Vid mitten av 1700-talet fanns även en läktare över den södra utgången och en ovanför orgelläktaren. Båda dessa togs dock bort 1858. +I början hängde kyrkklockorna i kyrktornet men det var för tungt och taket påfrestades när man ringde i klockorna. År 1747 byggdes därför en fristående klockstapel där de båda medeltida klockorna och en ny klocka hängdes upp. År 1991 infördes elektrisk ringning. +I början av 1800-talet upptäckte man röta i bjälkarna som bär upp kyrktornet, och man var tvungen att sätta upp stöttepelare. För att få stabilt underlag för pelarna var man tvungen att fylla de gravar som fanns i den norra korsarmens östra del. Stoftet efter de avlidna begravdes i kyrkogårdens nordöstra hörn. +År 1858 målades altartavlan, predikstolen, taklister, tavelramar och kyrkbänkar över med vit färg, såsom trenden var i Sverige. Vissa detaljer på predikstolen och altartavlan blev förgyllda med guld. Till kyrkans 200-årsjubileum 1894 rentvättades dock kyrkan från det vita. +År 1729 bekostade församlingen en konstnärlig utsmyckning av kyrkan. Gränna kyrka hade försetts med takmålningar tio år tidigare av Edvard Orm (f. 1670) och man bestämde sig för att anlita samma konterfejare. Hans målningar har inte bara ett dekorativt syfte utan skulle även för en icke läskunnig illustrera bibelns berättelser. +Paradiset +Norra korsarmens takmålning föreställer paradiset med Adam och Eva sittande under kunskapens träd. Längst till höger ser man hur de blir utdrivna ur lustgården. +Försoningen +En takmålning föreställande försoningen är placerad rakt ovanför altaret där Fadern, Sonen och den Helige Ande är omgivna av änglar som bär redskap som användes vid korsfästelsen och nedtagningen från korset. Texten på ett papper som en mörk figur ger till Jesus lyder Wärldens synder. +Uppståndelsen +Ovanför orgelläktaren är två kyrkor avbildade: den ena Hakarps nuvarande kyrka och den andra förmodligen den gamla kyrkan. Det är dessutom tre kors på en kulle avbildade varifrån kropparna är borttagna. Flera gestalter reser sig upp ur gravarna på Hakarps kyrkogård, för att symbolisera uppståndelsen. +Pinorummet +I södra delen av kyrkan har Orm avbildat helvetet. En kyrklig konstkännare ska ha sagt att det är den förskräckligaste bild av helvetet i hela Sverige. Det påstås även att konstnären har avbildat ett par smeder ifrån Huskvarna som han var osams med. +Den yttersta domen +I mitten av kyrkan i spegelvalvet har Orm avbildat Kristus i en 2,5 meter i diameter stor cirkel. Målningen föreställer den yttersta domen. Taklisterna och annat i trä runtomkring är prytt med bibelspråk och liknande. +Övriga målningar +På läktaren har Orm avbildat ett antal för kristendomen viktiga personer såsom de 12 lärjungarna, evangelisterna, Johannes Döparen och så vidare. Flera av dem är avbildade tillsammans med symboler för de sätt på vilka de dog som martyrer. +För sitt målningsarbete i Hakarps kyrka erhöll Edvard Orm 400 daler silvermynt, men vid en proteststämma behjärtade församlingen en förhöjning med ytterligare 60 daler silvermynt – på grund av Orms ”usla och släta tillstånd”. +Altartavlan var en gåva från Nils Hägerflycht. Den inköptes för 360 daler silvermynt från Eksjö stads kyrka. +Längst ner står texten "Kommer Ty Allting Är Nu Redo" och ovanför det avbildas de fyra evangelisterna. De omges och åtskiljs av byster som antagligen är porträtt av personer som bidrog till tavlans tillkomst under 1600-talet. Ovanför det syns reliefbilder med motiv från den bibliska historien; Syndafallet, Isaks offrande, herdarnas tillbedjan, Jesu dop, frestelse och hans kamp i Getsemane, påskalammsmåltiden och nattvardens instiftande. Ovanför det finns tre bilder omgivna av fyra pelare som avbildar korsfästelsen, nedtagningen från korset och gravläggningen. Längst upp på tavlan ser man Jesu uppståndelse. +Även predikstolen köptes in från Eksjö och bekostades av Hägerflycht. Både predikstolen och altartavlan är från 1636. De fyra evangelisterna som sitter bakom pulpeter och skriver är avbildade på predikstolens sidor. Nära sakristian hänger en målning i olja. Tavlan fanns redan när Anders Ståhl blev kyrkoherde 1712. Ursprunget är okänt men den är antagligen målad av en holländare eller fransman. Vid renoveringen av kyrkan 1894 försågs tavlan med en text. +Kyrkans dopängel hänger drygt två meter över golvnivån. Vid dop sänker man den. I ena handen håller ängeln en dopskål gjord av silver. Dopskålen är en gåva från KJ Gustafssons dödsbo. Dopängeln skänktes till kyrkan 1759 av P. Bremer. Den hamnade därefter hos Norra Smålands Fornminnesförenings museum innan den gavs tillbaka till Hakarps kyrka i samband med 250-årsjubileet 1944. +Det finns ett antal gamla ljuskronor. +Man vet inte om den gamla träkyrkan hade ett orgelverk eller inte. Inspektor Nils Arvidsson Hägerflycht skänkte till den nuvarande kyrkan en femstämmig orgel 1691/1694 till en kostnad av 500 daler silvermynt. +År 1775 diskuterade man huruvida man skulle bygga om orgelverket. Andra åtgärder på kyrkan hade emellertid kostat så mycket pengar att man av ekonomiska skäl sköt upp ombyggnationen. År 1783 togs frågan upp igen. Beslutet blev att man skulle renovera det gamla positivet. Man tillkallade den kände orgelbyggaren Per Schiörlin från Linköping för att undersöka orgeln. Hans uttalande var att orgelns pipor, bälg, klaver och regeringar var så slitna, förfallna och förmultnade att det inte gick att reparera och således tecknades med Schiörlin ett kontrakt på ett bygge av ett nytt orgelverk våren 1783. Orgeln blir färdig 1784 och har då 6 stämmor. Orgeln renoverades omfattande igen år 1864 av Erik Nordström. +År 1905 byggdes en ny, niostämmig orgel av Åkerman och Lund till en kostnad av 4 800 kr. Delar av den gamla orgeln såldes till Båraryds kyrka i Gislaveds församling, varpå den flyttades runt innan den revs våren 1982. +År 1948 renoverades och utökades orgeln från Åkerman och Lund med sju stämmor av Olof Hammarberg till en kostnad av 12.300 kr. Då orgeln på grund av pneumatisk traktur hade kort livslängd var man tvungen att redan 1975 bygga en ny orgel. Detta var dock inte lätt då Riksantikvarieämbetet hade synpunkter på utformningen och disponeringen av orgelverket. Till slut fick församlingen dock börja bygga och orgeln stod klar till pingsten 1982. Orgelläktaren är dock den ursprungliga från 1784 och likaså piporna som sitter i fasaden. Orgelns traktur är idag mekanisk och har därför mycket lång hållbarhet. Orgeln har 19 stämmor. Orgeln är mekanisk och har en cymbelstjärna. +Disposition: +1965 byggde J. Künkels Orgelverkstad en mekanisk kororgel. +Hakarps kyrkas äldsta skrud, en rödbrun sidenbrokad sådan med invävda blommor, är från 1741 alternativt 1734 och återbördades till kyrkan 1952 efter att ha varit på Norra Smålands Fornminnesförening. Den svarta skruden av sammet med silverbroderier, härstammar från 1800-talet. Alla andra är gjorda på 1900-talet. +A Dopfunten finns idag på Länsmuseet i Jönköping. +B Sanda kyrkas romanska dopfunt finns också bevarad på Länsmuseet. +C Gravstenarna står idag utanför kyrkan. +D Från en viss vinkel ser man att taket buktar inåt på grund av den tunga kyrkklockan. +E Anordning för clusterspel finns och cymbelstjärna med variabel hastighet. +Listorna över komministrar och kyrkoherdar innehåller enbart födelse- och dödsdatum om sådant finns tillgängligt. Annars är det ersatt med -. +Andersson, Per-Åke; Jonzon, Per-Axel; Karlsmo, Ragnar m.fl. (1994). Till minnet av Hakarps kyrka 300 år. Libris 2004711 +Möllerström, Harald; Redaktionen (1971). Hakarps krönika 1970-1971. Libris 3617433 +Bergenblad, Harry; Redaktionen (1972). Hakarps krönika 1971. Libris 10558833 +Gullbrandsson, Robin (2012). Hakarps kyrka: sanering och renovering av korgolv : Hakarps socken i Jönköpings kommun, Jönköpings län, Växjö stift : antikvarisk medverkan. Byggnadsvårdsrapport (Jönköpings läns museum) ; 2012:18. Jönköping: Jönköpings läns museum. Libris 13619736 +Hakarps krönika. 1952-2005 +Johansson, Hans (1967, andra upplagan utgiven 1996). ”En kristen tro; En kyrka byggs”. Hakarps socken genom tiderna +Tore Johansson, red (1988). Inventarium över svenska orglar: 1988:II, Växjö stift. Tostared: Förlag Svenska orglar. Libris 4108784 +Wikimedia Commons har media som rör Hakarps kyrka. +In Utero är det amerikanska grungebandet Nirvanas tredje och sista studioalbum, släppt den 1 september 1993 via skivbolaget Geffen Records. Efter framgångarna med bandets tidigare album Nevermind ville Kurt Cobain göra ett mer extremt album med ett råare sound. Nirvana kontaktade därför Steve Albini, som hade sina rötter i noiserockbandet Big Black, för att se om han var intresserad av att producera albumet, vilket han var. Cobain, som skrev samtliga låtar på albumet, var imponerad av två av Albinis tidigare produktioner, Surfer Rosa av Pixies och Pod av The Breeders, och ville utnyttja Albinis teknik att fånga upp den naturliga atmosfären i inspelningsstudion med hjälp av flera olika mikrofoner utplacerade runt om i studion. +In Utero, som är latin för "i livmodern", gick från början under namnet I Hate Myself and I Want to Die och senare Verse Chorus Verse innan det slutligen bestämdes att albumet skulle heta just In Utero. Inspelningen av albumet skedde i februari 1993 i Pachyderm Studio i Cannon Falls, Minnesota och Nirvana och Albini satte själva en deadline på två veckor för inspelningen av In Utero. Albini ville undvika all sorts inblandning av skivbolaget DGC Records vid producerandet av albumet och föreslog därför att Nirvana själva skulle betala för inspelningssessionerna, vilket de gick med på. Efter att inspelningen och mixningen av albumet var färdigställda kände Cobain att han inte var helt nöjd med slutresultatet. Nirvana kontaktade därför producenten Scott Litt och mixade tillsammans med honom om låtarna "Heart-Shaped Box" och "All Apologies" i Bad Animals Studio i Seattle, Washington. Albini var kritisk till den nya mixningen av albumet och ansåg att slutproduktionen inte alls lät som den han och Nirvana hade spelat in tidigare. +När In Utero lanserades fick det främst positiva reaktioner även om vissa recensenter var kluvna till albumet. Recensionerna handlade främst om hur olika In Utero och Nevermind var och att det verkade beröra de problem Cobain hade med att hantera berömmelsen han fick i och med Nirvanas genombrott två år tidigare. Albumet har i efterhand benämnts både som Cobains självmordsbrev och hans svanesång eftersom han avled i april 1994, drygt ett halvår efter lanseringen av In Utero. Albumet nådde som bäst plats 1 i USA, Sverige och Storbritannien samt nådde upp på topp 10 i länder såsom Kanada, Australien, Finland och Norge. In Utero har certifierats för 6× platina i Kanada, 5× platina i USA och platina i Frankrike. I Sverige har albumet certifierats för guld. In Utero kom på plats 435 på Rolling Stones lista "The 500 Greatest Albums of All Time" och nominerades i kategorin Best Alternative Music Album vid Grammy Awards 1994. 2013, tjugo år efter albumets ursprungliga lansering, släpptes In Utero i en 20th Anniversary Edition, en 20th Anniversary Deluxe Edition och en 20th Anniversary Super Deluxe Edition innehållande flera tidigare outgivna låtar av Nirvana. +Nirvana (bestående av Kurt Cobain, Krist Novoselic och Dave Grohl) fick sitt genombrott med albumet Nevermind, som gavs ut 1991. Från början hade Geffen Records hoppats att Nevermind skulle sälja runt 250 000 exemplar, ungefär samma försäljningssiffror som skivbolaget haft med Sonic Youths album Goo. Geffen Records största förhoppningar var att om alla inblandade arbetade hårt kunde Nevermind möjligtvis nå upp till att bli certifierad guld i USA till september 1992. Dock sålde albumet över allas förväntningar och certifierades för både guld och platina i USA i november 1991 av Recording Industry Association of America och för diamant i mars 1999. Nevermind har sagts vara det album som gjorde alternativ rock till en mainstreamgenre i början av 1990-talet, men trots alla framgångar var Nirvana inte helt nöjda med albumet. De klagade särskilt över mixningen av Nevermind, vilken utfördes av Andy Wallace. Även om Nirvana hade tyckt om mixningen från början uttryckte de missnöje med den vid senare tillfällen då de ansåg den var alldeles för polerad. Cobain har sagt att när han såg tillbaka på Nevermind skämdes han över hur albumet lät och han tyckte att det lät mer som ett album Mötley Crüe hade kunnat släppa än som ett punkrockalbum. +Cobain sade i en intervju med Rolling Stone under våren 1992 att Nirvanas nästa album skulle bli mer "extremt" och att albumet skulle innehålla ett råare sound, men även låtar som skulle gränsa till att klassas som tuggummipop. Cobain nämnde även att albumet inte skulle vara lika endimensionellt som Nevermind och under en intervju senare samma år sade Cobain att han förväntade sig att Nirvana skulle förlora en hel del fans när de släppte sitt nya album. Cobain ville börja arbeta med det nya albumet redan under sommaren 1992, men då bandmedlemmarna bodde i olika städer vid tillfället hade de svårt att få ihop detta. Dessutom var Cobains fru Courtney Love gravid med deras dotter Frances Bean Cobain, vilket påverkade planeringen av albumet. Nirvanas skivbolag DGC Records hade hoppats att Nirvana skulle ha det nya albumet färdigt att lanseras till julhelgen 1992, men när de märkte att detta inte skulle vara möjligt bestämde de sig istället för att släppa samlingsalbumet Incesticide i december detta år. +I en intervju i juli 1992 sade Cobain att han var intresserad av att spela in det nya albumet tillsammans med Jack Endino (som tidigare hade arbetat med Nirvana på deras debutalbum Bleach) och Steve Albini (tidigare sångare i noiserockbandet Big Black). Cobain sade även att han efter inspelningarna skulle välja ut det bästa låtmaterialet och sätta ihop detta till Nirvanas nästa album. I oktober 1992 spelade bandet in några demolåtar under en inspelningssession med Endino i Seattle, Washington, där flera av dessa låtar senare skulle spelas in igen för det tredje studioalbumet. Alla dessa demolåtar var vid tillfället helt instrumentella utom "Rape Me" under vilken Cobain sjöng, sittande med sin nyfödda dotter Frances Bean i knäet. Nirvana valde inte Endino för att producera In Utero. Endino kom ihåg att bandet hade många diskussioner om de verkligen ville ha Albini som producent. Nirvana spelade in ytterligare demolåtar när de var på en turné i Brasilien i januari 1993. En av låtarna från denna inspelningssession var "Gallons of Rubbing Alcohol Flow Through the Strip" som släpptes som sista låten på albumet, men enbart på de utgåvor som trycktes utanför USA. +Till slut bestämde sig Nirvana för att ta in Albini som producent för In Utero eftersom han hade ett gott rykte inom indiegenren. Cobain har även sagt att de valde att arbeta med Albini eftersom han hade producerat två av Cobains favoritalbum: Surfer Rosa av Pixies och Pod av The Breeders. Inspirerad av dessa album ville Cobain utnyttja Albinis teknik att fånga upp den naturliga atmosfären i inspelningsstudion, vilket gjordes med hjälp av flera olika mikrofoner utplacerade runt om i studion. Detta var något Nirvanas föregående producenter hade varit motståndare till att försöka sig på. Flera månader innan Nirvana ens hade tagit kontakt med Albini cirkulerade det rykten om att han var påtänkt att producera In Utero. Albini tog då kontakt med den brittiska pressen och förklarade för dem att han inte hade någon som helst inblandning i produktionen av albumet, vilket skedde enbart några få dagar innan han officiellt blev tillfrågad att producera In Utero. Albini ansåg vid tillfället att Nirvana lät snarlikt R.E.M., fast med ett mer förvrängt ljud, och han tyckte även att bandet var ett praktexempel på det sound som kom från Seattle. Albini sade att han accepterade att arbeta med Nirvana eftersom han tyckte synd om dem och han förväntade sig att de skulle vara som vilka andra artister han arbetat med tidigare. Innan själva huvudinspelningen av In Utero påbörjades skickade Nirvana en demokassett med de låtar de tidigare spelat in i Brasilien till Albini och i gengäld skickade Albini ett exemplar av PJ Harveys album Rid of Me till Cobain, för att på så sätt ge honom en känsla för vad det var för sorts studio de skulle spela in albumet i. +Nirvana och Albini satte själva en deadline på två veckor för inspelningen av In Utero. Albini ville undvika inblandning av DGC Records och föreslog att Nirvana skulle betala för inspelningssessionerna, vilket bandet gick med på. Inspelningskostnaderna uppgick till $24 000 medan Albini hade en fast avgift på $100 000 för sina tjänster. Albini vägrade ta emot royalties för albumet, trots att han blev rekommenderad att göra så då dessa uppgick till runt $500 000, eftersom han kände att detta hade varit omoraliskt och en förolämpning mot Nirvana. +I februari 1993 tog sig Nirvana till Pachyderm Studio i Cannon Falls, Minnesota för att påbörja inspelningen av In Utero. För att undvika opåkallad uppmärksamhet bokades studion i bandnamnet The Simon Ritchie Bluegrass Ensemble istället för Nirvana; det påhittade bandnamnet var en anspelning på Sid Vicious födelsenamn John Simon Ritchie. Albini hade inte träffat bandmedlemmarna tidigare, även om de hade pratat tillsammans om vilket sorts album de ville producera. Albini kände av att Nirvana ville spela in precis den sortens album han själv var bekväm med att producera. Bandmedlemmarna bodde i ett hus i närheten av studion under dagarna de spelade in In Utero. Novoselic jämförde deras levnadsförhållanden med det sovjetiska systemet av koncentrationsläger kallat "gulag" och sade att det enda de gjorde under tiden de var där var att arbeta. Under större delen av inspelningen var det bara Nirvana, Albini och teknikern Bob Weston som var närvarande i studion. Nirvana gjorde det klart för DGC Records att de inte ville att skivbolaget skulle lägga sig i produktionen av In Utero och de gick så långt att de vägrade spela upp något låtmaterial för DGC Records A&R-representanter. Albini införde en policy att ignorera alla som inte var medlemmar av Nirvana, för att på så sätt få bort skivbolagets inblandning. Albini förklarade att alla som hade med Nirvana att göra, men inte var medlemmar av bandet var "de största skitstövlarna jag någonsin träffat" och han har även benämnt dem som "parasiter". +Början av inspelningen gick långsamt för Nirvana och Albini. Bandet anlände till Pachyderm Studio utan instrument och de första tre dagarna gick åt till att vänta på att de skulle anlända med posten. När instrumenten väl hade kommit fortskred inspelningen i ett relativt högt tempo. Inspelningsdagarna följde ungefär samma rutin där Nirvana började spela in mitt på dagen, gjorde ett uppehåll för lunch och middag och sedan spelade in igen fram till midnatt. Cobain, Novoselic och Grohl spelade in sina instrumentslingor tillsammans för nästan alla låtar. Undantagen var låtarna "Very Ape" och "tourette's", där Grohls trumslinga spelades in i ett kök i närheten av studion på grund av dess naturliga reverb. Albini använde sig av ungefär trettio stycken mikrofoner som han ställde runtomkring Grohls trumset när han spelade in trumslingorna. Cobain arbetade med att först spela in gitarrslingorna, sedan sina gitarrsolon och till sist sin sång. Nirvana gjorde sig inte av med något av det material de spelade in utan använde i slutändan nästan allt som spelats in i studion. Albini har sagt att han kände sig mer som en tekniker än en producent och trots sina egna åsikter lät han oftast Nirvana välja ut vilka tagningar de ville använda sig av i slutproduktionen. Albini sade även att Cobain visste vad han tyckte var acceptabelt och att Cobain visste hur han skulle förbättra de tagningar som var oacceptabla. Cobain spelade in all sin sång på runt sex timmar. Albumet var färdiginspelat den 18 februari, sex dagar efter inspelningsstart. Cobain hade från början trott att inspelningen skulle bli komplicerad (eftersom han hade hört rykten om att Albini skulle vara ett "sexistiskt svin"), men sade efteråt att inspelningen av In Utero var den lättaste inspelningen bandet någonsin hade haft. Det enda störande momentet skedde efter cirka en vecka när Courtney Love kom förbi studion, då hon ska ha saknat Cobain. Varken Nirvana, Love eller Albini har velat gå in på detaljer precis vad som hände när Love dök upp, men Westons flickvän (som skötte om matlagningen i studion) sade att Love skapade en negativ stämning i studion eftersom hon kritiserade Cobains arbete och konfronterade de andra bandmedlemmarna och Albini i tid och otid. +Mixningen tog fem dagar att genomföra och var klar den 26 februari. Enligt Nirvanas standard var detta en ganska snabb mixningsprocess, men för Albini var det tvärtom eftersom han brukade mixa hela album på en eller två dagar. När Nirvana och Albini körde fast i mixningsprocessen bestämde de sig för att ta ledigt resten av dagen och under dessa tillfällen tittade de på naturfilmer, tände eld på saker eller busringde. +Efter att inspelningen av In Utero var färdigställd skickades de omastrade versionerna av låtarna till flera personer, bland annat Geffen Records dåvarande VD Ed Rosenblatt. Cobain har sagt att den feedback Nirvana fick på dessa versioner främst var negativ. Cobain fick klagomål på att hans förmåga att skriva låttexter inte var tillräckligt bra, att In Utero inte gick att lyssna på och att det inte var säkert att albumet skulle bli lika uppskattat som Nevermind. Från början hade få personer från Geffen Records tyckt att Nirvana överhuvudtaget skulle spela in något tillsammans med Albini och Cobain kände det som att detta var ett outtalat meddelande till honom att slänga bort allt material de spelat in med Albini och börja om från början igen. Cobain blev upprörd över detta, men kände att han hade skapat ett album som han själv kunde tänka sig att lyssna på när han var hemma. Några av bandmedlemmarnas vänner gav dock låtmaterialet tummen upp och i april 1993 kände Nirvana att de ville släppa In Utero som det var. Cobain sade att det fanns många personer som ville att albumet skulle ha låtit som Nevermind, men att han inte hade några som helst planer på att spela in ytterligare ett sådant album. +Nirvana började ändå vid denna tidpunkt att tvivla på albumets sound. Cobain sade att den första gången han lyssnade igenom albumet kände han direkt att något inte stod rätt till. Han sade att under den första veckan lyssnade han knappt på In Utero eftersom han inte var intresserad av albumet. Cobain sade även att detta inte brukade hända och att han inte kände någon känslomässig anknytning till In Utero. Bandet kände att det främst var fel på basljudet och att det knappt gick att höra vad Cobain sjöng. De kontaktade Albini för att fråga honom om han kunde hjälpa dem att göra en ny mixning av In Utero. Albini vägrade göra detta eftersom han var rädd för att om han mixade om albumet skulle detta bara leda till mer oro från Nirvana i framtiden, vilket i sin tur skulle leda till att albumet aldrig blev helt färdigt. Nirvana kontaktade då istället Bob Ludwig som hjälpte dem att göra en ny mixning av albumet. Novoselic var nöjd med Ludwigs mixning, men Cobain hade fortfarande sina tvivel. +I en intervju med Chicago Tribune i april 1993 sade Albini att han var osäker på om Geffen Records skulle lansera albumet överhuvudtaget. Albini kommenterade flera år senare att hans uttalande var förhastat och att det var grundat på andrahandsinformation. Albinis uttalande i Chicago Tribune väckte inte några direkta reaktioner från varken Geffen Records eller Nirvana, men ett liknande uttalande i Newsweek en kort tid därefter fick större uppmärksamhet. Nirvana dementerade ryktena om att Geffen Records tvingade dem att ändra albumets sound och de sade att artikelns författare hade misstolkat vad som sagts. Bandet valde även att trycka upp sitt svar på en helsida i tidskriften Billboard. Rosenblatt gick ut med ett pressmeddelande där det klart och tydligt stod att Geffen Records skulle lansera Nirvanas nästa album och skivbolagets grundare, David Geffen, gjorde något så ovanligt som att personligen ringa upp Newsweek för att klaga på deras artikel. +Nirvana var fortfarande inte helt nöjda med albumet och de ville fortsätta arbeta med det. De funderade på att ta in producenten Scott Litt och för att mixa låtarna ville de ta hjälp av Andy Wallace, som tidigare hade arbetat med bandet på Nevermind. Albini blev upprörd över detta och sade att Nirvana inte fick lov att ändra på några låtar utan att han var inblandad i processen. Albini vägrade till en början att ge ifrån sig masterversionerna av låtarna, men gick med på det efter att Novoselic hade ringt upp honom. Nirvana valde att inte arbeta med Wallace utan istället att bara ta hjälp av Litt. I maj 1993 träffades Nirvana och Litt för att mixa om och förbättra låtarna "Heart-Shaped Box" och "All Apologies" i Bad Animals Studio i Seattle, Washington. En låt, "I Hate Myself and Want to Die", valdes bort från albumet eftersom Cobain kände att det redan fanns för många låtar på In Utero som till större delen bestod av oljud. I övrigt lämnades albumet som det tidigare var förutom att basljudet förstärkdes och att Cobains sång höjdes med cirka tre decibel. Albini var kritisk till den nya mixningen av albumet och ansåg att slutproduktionen inte alls lät som det han och Nirvana hade spelat in. +Albini ville producera ett album som lät helt olikt Nevermind. Han tyckte att Neverminds sound var alldeles för kommersiellt och att det lät som vilket album som helst, något han inte tyckte var smickrande för Nirvana. Albinis avsikt var istället att spela in ett album med ett mer naturligt och känslomässigt sound. Albini vägrade använda sig av dubblering, vilket var något Nirvana hade använt sig av mycket på Nevermind. Albini fick istället Cobain att spela in sin sång ensam i ett resonantrikt rum. Albini märkte hur intensiv och kraftfull Cobains röst var och han nämnde att Cobain i slutet av låtarna "Milk It" och "Rape Me" ville skrika ut sin sång för att på så sätt överrösta resten av bandet. Albini fick fram det glesa trumljudet genom att använda ungefär trettio stycken mikrofoner, som han ställde runtomkring Grohls trumset när han spelade in trumslingorna. Albini lyckades då även fånga upp rummets naturliga reverb och han kommenterade med att säga: "Om du tar en bra trumslagare och sätter honom framför ett trumset som låter bra akustiskt och sedan bara spelar in det, då har du gjort ditt jobb." +Nirvanas levnadstecknare Michael Azerrad har skrivit att In Utero reflekterade de omvälvande känslor Cobain uppvisade innan albumet var färdigställt. Azerrad tog särskilt fasta på hur olika låtarna på albumet var, där han kallade "Dumb" för en typisk The Beatles-inspirerad låt och "Milk It" för ett exempel på en punkrocklåt. Azerrad uppmärksammade även hur den lugna "All Apologies" var helt skild från den apoplektiska "Scentless Apprentice" och skrev att det verkade som att Cobain hade gett upp den poppunk-blandning som han hade använt sig av på Nevermind. Cobain själv ansåg dock att In Utero varken var musikaliskt tyngre eller mer känslosamt än de album Nirvana tidigare hade lanserat. Novoselic höll med om Azerrads argument att In Utero var mer aggressivt och konstnärligt än något av bandets tidigare album. Han förklarade det med följande ord: "Det har alltid funnits låtar [av Nirvana] som 'About a Girl' och det har alltid funnits låtar som 'Paper Cuts' [...] Nevermind lät mer som 'About a Girl' medan detta album låter mer som 'Paper Cuts'." Cobain använde "Milk It" som exempel på en låt som var mer aggressiv och experimentell och att det var åt det hållet som Nirvanas musik lutade månaderna innan de började spela in In Utero. Novoselic ansåg att "All Apologies" och "Heart-Shaped Box" var inkörsportar som gav de andra låtarna på albumet ett mer strävt ljud. Novoselic har sagt i en intervju att när man lyssnade på In Utero skulle man känna av hur aggressivt albumet egentligen var. +Flera av låtarna på In Utero är skrivna flera år innan de spelades in och vissa av dem kan dateras till 1990. Cobain arbetade med låttexterna även när Nirvana spelade in i Pachyderm Studio. Han har dock sagt att han fokuserade mer på låttexterna under arbetet med In Utero än han tidigare hade gjort med både Bleach och Nevermind. Azerrad hävdade att låttexterna på In Utero var mer rakt på sak och inte lika impressionistiska som Nirvanas tidigare låttexter. Azerrad lade även märke till att mer eller mindre varenda låt berörde sjukdomar på ett eller annat sätt. Några av låtarna var inspirerade av böcker som Cobain hade läst: "Frances Farmer Will Have Her Revenge on Seattle" hämtade inspiration från William Arnolds bok Shadowland från 1978, vilken är en biografi om skådespelerskan Frances Farmer. Cobain hade varit fascinerad av boken ända sedan han läste den i sin ungdom och med titeln "Frances Farmer Will Have Her Revenge on Seattle" ville Cobain ha ett långt låtnamn för att gå emot den dåvarande trenden att använda sig av låtnamn som innehöll enbart ett ord. "Scentless Apprentice" var baserad på Patrick Süskinds bok Parfymen – berättelsen om en mördare. Denna bok handlar om parfymören Jean-Baptiste Grenouille, som är isolerad på grund av den känsla av obehag han väcker eftersom han saknar kroppslukt. Dock utvecklar Grenouille ett särdeles luktsinne och blir besatt av dofter och för att skapa parfymer mördar han unga flickor för att extrahera deras kroppsdofter. +Cobain sade i en intervju att det mesta av låtmaterialet från In Utero var väldigt opersonligt. Han sade även att fastän han precis själv hade blivit far var alla bilder på spädbarn och födslar på albumet där av en ren tillfällighet. Dock ansåg Azerrad att In Utero visst hade vissa personliga kopplingar och han fick medhåll av Grohl om detta. Grohl hade sagt att mycket av låtmaterialet på albumet handlade om sådant som Cobain själv hade upplevt och han sade även att det nu inte rörde sig om "tonårsångest" längre utan "rockstjärneångest". Cobain sade att trots att han hade blivit en världsstjärna ansåg han inte att han hade haft ett särskilt spännande liv. Han berättade för Azerrad att han inte kände för att skriva någon låt som handlade om hans avsky för massmedias rapportering om hans privatliv, men Azerrad påpekade att låten "Rape Me" verkade handla om just detta. Även om Cobain säger att låten skrevs innan det kom fram att han hade drogproblem visade han förståelse för att låten kunde tolkas på detta sätt. Cobain har dock sagt att refrängen i "Heart-Shaped Box" ("Hey, wait, I've got a new complaint") var ett exempel på hur han uppfattades av just massmedia. "All Apologies" var tillägnad Cobains fru Courtney Love och deras dotter Frances Bean Cobain. Cobain har sagt att även om låten i sig inte handlar om varken hans fru eller dotter, så menade han att stämningen i låten (som han beskrev som "lugn, lycklig och bekväm") var menad för dem. "Serve the Servants" innehåller kopplingar både till Cobains uppväxt och till hans liv som vuxen. Låtens öppningsrader "Teenage angst has paid off well, now I'm bored and old" visade vad Cobain kände över det faktum att Nirvana hade blivit populära. Cobain refererade även till sina föräldrars skilsmässa med både raderna "That legendary divorce is such a bore" och "I tried hard to have a father, but instead I had a dad, I just want you to know that I don't hate you anymore, there is nothing I could say that I haven't thought before". Cobain kommenterade detta genom att säga att han ville att hans far skulle veta att han inte hatade honom, men att han ändå inte hade någon lust att prata med honom. +Cobain ville från början att namnet på albumet skulle vara I Hate Myself and I Want to Die, vilket var en mening som började dyka upp i hans dagbok runt mitten av 1992. Vid denna tidpunkt hade Cobain börjat använda meningen som svar när någon frågade honom hur han mådde. Cobain har dock sagt att detta albumnamn skulle ses som ett skämt eftersom han var trött på att alla, även han själv, tog Nirvana på så stort allvar. Novoselic övertygade Cobain att ändra albumnamnet då han var rädd att det annars skulle kunna resultera i en stämning av Nirvana. Namnet I Hate Myself and I Want to Die låg istället till grund för låten "I Hate Myself and Want to Die" som spelades in för albumet, men som togs bort från slutproduktionen; låten var från början placerad mellan "tourette's" och "All Apologies". Bandet funderade senare på att använda namnet Verse Chorus Verse, vilket var ett namn på en av Nirvanas låtar och även ett tidigare arbetsnamn för låten "Sappy", innan de slutligen bestämde sig för In Utero. Detta namn kom från en dikt skriven av Love. +Robert Fisher, albumets reklamformgivare, hade arbetat med Nirvana under hela deras tid på DGC Records. Han sade att Cobain brukade komma på idéerna för vad han ville ha för illustrationer och fotografier på albumen och singlarna, men att han sedan sade åt Fisher att det var upp till honom att göra något med hans idéer. Till singeln "All Apologies/Rape Me" sade Cobain till exempel bara till Fisher att han ville ha "något med sjöhästar" på skivomslaget. På skivomslaget av In Utero syns en transparent, anatomiskt korrekt docka med änglavingar fastsatta på ryggen. På baksidan syns ett kollage skapat av Cobain som består av konstgjorda foster och kroppsdelar som ligger utspridda på en bädd av orkidéer och liljor. Cobain själv beskrev albumets baksida med orden: "Sex och kvinna och In Utero och slidor och födslar och död." Cobain satte ihop kollaget hemma hos sig och det fotograferades av Charles Peterson som hävdade att han blev ditkallad efter ett oväntat telefonsamtal från Cobain. Albumets låtlista och symboler positionerades runt kanten av baksidan, med inspiration hämtad från Barbara G. Walkers bok The Woman's Dictionary of Symbols and Sacred Objects från 1988. +In Utero släpptes den 1 september 1993 i Storbritannien, den 13 september 1993 i Sverige och den 21 september 1993 i USA. För att albumet inte skulle få för stor hype runt sig valde DGC Records att inte marknadsföra albumet på något storslaget sätt. I en intervju med Billboard förklarade skivbolagets marknadsföringschef att de främst tänkte stå bredvid och låta albumet sälja sig självt, som de tidigare hade gjort med Nevermind. Albumets marknad ansågs främst vara fans av alternativ rock samt pressen och därför trycktes albumet upp i LP-format för att möta efterfrågan därifrån. Till skillnad från de flesta av singlarna från Nevermind valde DGC Records att inte ha någon officiell lansering av någon av singlarna från In Utero i USA. Istället skickades promosingeln "Heart-Shaped Box" ut till flera olika radiostationer i början av september 1993, men DGC Records hade inte som mål att denna skulle ta sig upp på topp fyrtio på någon av topplistorna. Nirvana var säkra på att In Utero inte skulle sälja lika bra som Nevermind. Cobain sade att han var övertygad om att In Utero inte skulle uppnå en fjärdel av Neverminds försäljningssiffror, men han tog detta med ro eftersom han verkligen tyckte om albumet. Bandmedlemmarna marknadsförde albumet genom att spela in en trettio sekunder lång TV-reklam tillsammans med komikern Bobcat Goldthwait. Goldthwait var även öppningsakt åt Nirvana under deras In Utero-turné. +In Utero gick in på plats 1 på topplistan Billboard 200 och albumet sålde 180 000 exemplar i USA under den första veckan efter lanseringen. Dock vägrade butikskedjor som Walmart och Kmart att sälja albumet. Enligt The New York Times köpte inte Walmart in albumet på grund av för liten efterfrågan medan anställda på Kmart ansåg att albumet inte passade in i deras utbud. Sanningen var dock att båda butikskedjorna var rädda för att deras kunder skulle ta illa upp på grund av albumets baksida. DGC Records tog därför fram en annorlunda version av In Utero som började säljas i mars 1994. På dessa utgåvor är de konstgjorda fostren och kroppsdelarna borttagna och låten "Rape Me" är omdöpt till "Waif Me"; Cobain försökte dock från en början få låten omdöpt till "Sexually Assault Me". En talesman för Nirvana berättade att när Cobain och Novoselic var barn kunde de bara få tag på sin musik via de större butikskedjorna och det var därför de gick med på att censurera albumets baksida för att ge fler barn möjligheten att köpa deras album. +I oktober 1993 gav sig Nirvana ut på sin första USA-turné på två år. Albumets andra singel, "All Apologies/Rape Me", släpptes sedan i december samma år. I februari 1994 begav sig Nirvana ut på en tänkt sexveckors Europaturné, men delar av turnén ställdes in efter att Cobain tagit en drogöverdos i Rom, Italien den 6 mars. Cobain gick med på att gå igenom drogrehabilitering på Exodus Recovery Center i Los Angeles, Kalifornien, men kort efteråt rymde han därifrån. Han hittades död i sitt hem i Seattle, Washington den 8 april 1994 och dödsorsaken var ett hagelgevärsskott mot huvudet. Lanseringen av den tredje singeln från In Utero, "Pennyroyal Tea", drogs tillbaka efter att Cobain hade avlidit. Ett begränsat antal kopior av singeln hann dock säljas i Tyskland. I och med Cobains död klättrade In Utero upp på topplistan Billboard 200 igen, från plats 72 till plats 27. +2013, tjugo år efter albumets ursprungliga lansering, släppte Geffen Records tre olika utgåvor av albumet: en 20th Anniversary Edition innehållande tre LP, en 20th Anniversary Deluxe Edition innehållande två CD och en 20th Anniversary Super Deluxe Edition innehållande tre CD och en DVD. Under åtta timmar den 24 september 2013 erbjöd tre skivbutiker i USA sina kunder att få en gratis tatuering av Nirvanas smileylogotyp i samband med återlanseringen av In Utero. 2023, trettio år efter albumets ursprungliga lansering, släppte Geffen Records en 30th Anniversary Edition av In Utero. +När In Utero lanserades fick albumet främst positiva reaktioner, även om vissa recensenter var kluvna till albumet. Stephen Thomas Erlewine från Allmusic skrev att albumet kändes som "Cobains självmordsbrev" och han ansåg att Albinis sound passade perfekt samman med Cobains nihilistiska texter. Erlewine skrev även att han tyckte att det var synd att låtar som "Verse Chorus Verse" och "I Hate Myself and Want to Die" förpassades till samlingsalbum istället för att vara med på In Utero men att han i slutändan ändå ansåg att detta album var uppsåtligen svårhanterligt, vilket han menade på ett positivt sätt. Erlewine gav albumet betyget 5 av 5. Eamonn Stack från BBC Music ansåg att med detta album kunde Nirvana vara sig själva igen och han beskrev dem som antihjältar. Stack skrev i sin positiva recension av albumet att med In Utero får man följa med in under Cobains hud till alla de demoner han slogs mot dagligen och han beskrev albumet som en dagbok fylld med alla de konsekvenser som uppstår i och med att man blir världskänd. Raziq Rauf från Drowned in Sound beskrev In Utero som Nirvanas mest kommersiella album även om han skrev att Cobain med en viss desperation inte ville göra en direkt uppföljare till Nevermind. Rauf fortsatte sin positiva recension med att skriva att albumet visar att Nirvana har mognat jämfört med tidigare album och att In Utero har en viss mångfald som varken Bleach eller Nevermind hade. David Browne från Entertainment Weekly skrev att även om albumet saknar en låt i samma klass som "Smells Like Teen Spirit" ansåg han att låtarna "Frances Farmer Will Have Her Revenge on Seattle" och "Pennyroyal Tea" var några av bandets mest kraftfulla låtar någonsin. Browne ansåg även att vissa delar av "Heart-Shaped Box" uppvisade en ny och mer ömsint sida av Nirvana. Browne gav albumet betyget B+. +John Mulvey från New Musical Express beskrev In Utero som ett väldigt förvirrat album. Han skrev att det varken var självdestruktivt, nihilistiskt eller en direkt uppföljare till Nevermind. Mulvey ansåg att In Utero var ett kärleksbrev som skriks ut av den man som är rädd för den belåtenhet han hittat i livet. Mulvey gav albumet 8 av 10 i betyg, men han skrev samtidigt att albumet inte var lika revolutionerande som Nevermind. David Fricke från Rolling Stone kallade albumet för lysande, rasande och tankeväckande men att det mer än något annat visar på viljans triumf. Fricke skriver att In Utero är fullt med påhopp mot både fans och media och han kallar Cobain för "Generation X:s John Lennon". Fricke gav albumet 4,5 av 5 i betyg. Ben Thompson från The Independent skrev i sin positiva recension att In Utero mycket oftare var ett vackert album snarare än ett fult, rent musikaliskt sett. Han var glad över att Nirvana valde att negligera den typiska punkrocken för något som han tyckte var mer passande. Robert Christgau från The Village Voice kallade In Utero för ett svårt album, men gav det ändå betyget A-; ett betyg som senare ändrades till ett A. Christopher John Farley från Time anser att Nirvana med detta album blev pånyttfödda. Farley skrev att Nirvana med Nevermind byggde upp grungekulturen och med In Utero rev ned den. Farley avslutade sin positiva recension med att säga att "Nirvana kanske inte betyder himmelriket, men trions senaste album är väldigt nära att vara gudomligt." +In Utero hamnade överst på Rolling Stones lista över de bästa albumen från 1993 och på plats två på The Village Voices motsvarande lista Pazz & Jop. The New York Times tog med albumet på sin lista över de tio bästa albumen från 1993 och albumet nominerades i kategorin Best Alternative Music Album vid Grammy Awards 1994. 2003, tio år efter albumets ursprungliga lansering, uppmärksammades albumet i en artikel i tidskriften Guitar World. I artikeln kallade Nirvanas levnadstecknare Charles R. Cross In Utero för ett bättre album än Nevermind och skrev att det än idag fortsätter att influera andra band. Samma år utsåg Pitchfork In Utero till det trettonde bästa albumet från 1990-talet och året efter hamnade albumet på plats 94 på Blenders lista "100 Greatest American Albums of All Time". Q placerade In Utero på plats 20 på deras lista "The Best 50 CD's of the Last 15 Years" och samma tidskrift har även listat albumet på listorna "Q Readers All Time Top 100 Albums" (plats 63), "Q Readers 100 Greatest Albums Ever" (plats 11), "Q Readers Best Albums Ever" (plats 22) "Album of the Year" 1993 samt att In Utero placerade sig som ett av albumen på listan "In Our Lifetime: Q's 100 Best Albums" och på listan "50 Heaviest Albums of All Time". 2005 satte Spin albumet på plats 51 på sin lista "100 Greatest Albums 1985–2005" och tio år senare placerade tidskriften albumet på plats 28 på listan "The 300 Best Albums of the Past 30 Years (1985–2014)". Rolling Stone hade med In Utero på plats 435 när de gjorde sin lista "The 500 Greatest Albums of All Time". In Utero har certifierats för 6× platina i Kanada, 5× platina i USA och platina i Frankrike. I Sverige har In Utero certifierats för guld. +Alla låtar är skrivna och komponerade av Kurt Cobain, om inte annat anges. +Kurt Cobain – sång, sologitarr, fotografi och design +Krist Novoselic – elbas +Dave Grohl – trummor +Den här artikeln är helt eller delvis baserad på material från engelskspråkiga Wikipedia, In Utero (album), 31 juli 2012. +Azerrad, Michael (1994) (på engelska). Come as You Are: The Story of Nirvana. New York: Doubleday. ISBN 0385471998 +Berkenstadt, Jim; Cross, Charles (1998) (på engelska). Classic Rock Albums: Nirvana - Nevermind. Schirmer. ISBN 0028647750 +Cross, Charles R. (2001) (på engelska). Heavier Than Heaven: A Biography of Kurt Cobain. Hyperion. ISBN 0786884029 +DeRogatis, Jim (2003) (på engelska). Milk It!: Collected Musings on the Alternative Music Explosion of the 90's. Cambridge: Da Capo. ISBN 0306812711. https://archive.org/details/milkitcollectedm00dero +Gaar, Gillian G. (2006) (på engelska). In Utero 33⅓. Continuum International Publishing Group. ISBN 0826417760. https://archive.org/details/inutero0000gaar +Kent, David (1993; 2006) (på engelska). Australian Chart Book 1970-1992; Australian Chart Book 1993-2005. St Ives; Turramurra: Australian Chart Book Pty Ltd. ISBN 0646119176 +Oricon (2006) (på japanska). Album Chart-Book Complete Edition 1970~2005. Orikonmāketingupuromōshon (オリコン・マーケティング・プロモーション,). ISBN 4871310779 +Pennanen, Timo (2003) (på finska). Sisältää hitin: levyt ja esittäjät Suomen musiikkilistoilla vuodesta 1972. Otava Publishing Company Ltd. ISBN 951121053X +Roberts, David (2006) (på engelska). British Hit Singles & Albums (19). HIT Entertainment. ISBN 1904994105 +Scapolo, Dean (2001) (på engelska). New Zealand Music Charts, 1975-2000: Albums. ISBN 0908876343 +Skånska kriget (danska/norska: Skånske Krig; tyska: Schonischer Krieg) var ett krig som huvudsakligen utkämpades mellan 1675 och 1679. Det var nära förbundet med det fransk-nederländska kriget (1672–1678) och ingick i den större europeiska maktkampen. Sverige stod på Frankrikes sida, medan Danmark-Norge anslöt sig till Frankrikes motståndare. För Danmark var ett viktigt mål att återta de tidigare danska landskap i Skåneland som hade förlorats till Sverige genom frederna i Roskilde och Köpenhamn. Sverige försökte samtidigt försvara sina nya områden och sin stormaktsställning kring Östersjön. +Trots namnet var kriget inte begränsat till Skåne. Det hade sin upptakt i Tyskland, där svenska trupper gick in i Brandenburg i slutet av 1674. Efter det svenska nederlaget i slaget vid Fehrbellin i juni 1675 anslöt sig flera av Frankrikes fiender i kriget mot Sverige. Därmed utvecklades konflikten till ett flerfrontskrig. Sveriges besittningar i Pommern, Bremen-Verden och Wismar angreps, samtidigt som Sverige förlorade herraväldet till sjöss efter nederlagen mot den dansk-nederländska flottan vid Öland 1676 och Köge bukt 1677. Det danska sjöherraväldet gjorde det möjligt att erövra Gotland och att landsätta en armé vid Råå i Skåne den 29 juni 1676. +Den danska armén ryckte därefter snabbt fram genom Skåne och Blekinge och erövrade flera viktiga städer. Den svenska armén under Karl XI tvingades först retirera, men kriget vände efter segrarna vid slaget vid Halmstad och det blodiga slaget vid Lund 1676. Efter slaget vid Landskrona 1677 behöll Sverige det militära initiativet i stora delar av Skåne. Samtidigt angreps Sverige från Norge under Ulrik Frederik Gyldenløve, främst i Bohuslän och Jämtland. I delar av Skåne och Blekinge fördes dessutom ett lokalt gerillakrig av snapphanar och danska friskyttar, vilket möttes av hårda svenska motåtgärder. +Medan Sverige återtog initiativet i Skåne fortsatte motgångarna i Tyskland. Brandenburg och dess allierade erövrade stora delar av de svenska besittningarna, och 1678 hade Sverige förlorat huvuddelen av Svenska Pommern. Under vintern 1678–1679 slog Brandenburg även tillbaka ett svenskt angrepp mot Ostpreussen. Kriget avslutades 1679 framför allt genom fransk diplomati. Genom frederna i Saint-Germain, Fontainebleau och Lund återfick Sverige i stort sett de områden som hade gått förlorade, medan Danmark inte fick några territoriella vinster. +Kriget orsakade omfattande förstörelse, särskilt i Skåneland och de svenska besittningarna i Tyskland. Det visade också på svagheter i Sveriges armé, flotta, finanser och förvaltning. Under Karl XI:s fortsatta regering genomfördes därför omfattande reformer, bland annat reduktionen, utvecklingen av indelningsverket, uppbyggnaden av flottbasen i Karlskrona och en starkare integrering av Skåne i det svenska riket. Samtidigt stärktes kungamakten genom det karolinska enväldet. Erfarenheterna från kriget bidrog därmed till omformningen av Sverige till en starkare men också mer militärt organiserad stormakt. +Bakgrunden till skånska kriget låg dels i den långvariga maktkampen mellan Danmark-Norge och Sverige om herraväldet i Norden och Östersjöområdet, dels i den större europeiska konflikten mellan stormakten Frankrike och sjömakten Nederländerna. Från 1500-talets senare del hade Sverige stegvis stärkt sin ställning genom erövringar i Baltikum, Ingermanland och Livland. Genom trettioåriga kriget och Westfaliska freden 1648 fick Sverige även besittningar i Nordtyskland, vilket gav riket strategiska positioner runt Östersjön och befäste dess ställning som europeisk stormakt. Den svenska expansionen hotade samtidigt Danmarks ställning, eftersom Östersjöhandeln och kontrollen över Öresund med dess värdefulla tull var av central ekonomisk och strategisk betydelse för det danska riket. +Vid 1600-talets mitt hade Danmark successivt förlorat ett flertal provinser till Sverige. Genom frederna i Brömsebro (1645), Roskilde (1658) och Köpenhamn (1660) hade Danmark avträtt Jämtland, Härjedalen, Gotland, Ösel, Bohuslän, Ven, Skåne, Blekinge och Halland. Trondheims län och Bornholm återlämnades dock till Danmark 1660. För Danmark-Norge blev revanschfrågan därför central, särskilt när Skåne, Halland, Blekinge och Bohuslän förblev under svensk kontroll efter 1660. Relationerna mellan länderna präglades därefter av ett spänt status quo, där Sverige ville bevara sina erövringar medan Danmark sökte möjligheter att bryta den svenska dominansen. Motsättningarna förstärktes av frågan om Holstein-Gottorp, som styrdes av hertigen Kristian Albrekt, vars nära band till Sverige uppfattades som ett hot mot Danmarks säkerhetspolitiska ställning. Samtidigt var Sverige beroende av den europeiska stormaktspolitiken. Riket hade fortfarande betydande militär prestige, men var i praktiken försvagat. De svenska besittningarna var geografiskt utspridda, flera av de erövrade områdena var ännu svagt integrerade och statsfinanserna var ansträngda efter de nordiska krigen under Karl X Gustavs regeringstid (1655–1660). Under den förmyndarregering som styrde under Karl XI:s minderårighet var dessutom armén, flottan och befästningsväsendet eftersatta. +Sverige slöt den 4 april 1672 ett förbund med Frankrike. Sverige åtog sig att ingripa mot tyska furstar som stödde Nederländerna mot Frankrike och fick i gengäld franska subsidier samt en garanti om att Frankrike inte skulle stödja ett danskt angrepp. För Sverige var subsidierna av stor betydelse, eftersom de bidrog till att upprätthålla rikets militära beredskap och stormaktsställning. Det svenska agerandet följde därför i hög grad av beroendet av franskt stöd och av de förpliktelser som alliansen medförde. När Frankrike och England inledde krig mot Nederländerna 1672 försökte Sverige först undvika att bli direkt indraget. Den svenska regeringen ville behålla de franska subsidierna utan att gå in i ett öppet krig, särskilt eftersom Sverige saknade resurser för en större konflikt. Läget förändrades när flera tyska furstar och Brandenburg-Preussen närmade sig den franskfientliga sidan. Det franska sändebudet Isaac de Feuquières pressade då den svenska ledningen att sända trupper till Tyskland för att binda Frankrikes fiender. Rikskanslern Magnus Gabriel De la Gardie försökte vinna tid, men efter misslyckade fredsförsök och ett försämrat franskt läge ökade trycket på Sverige. +Under 1674 övertalades den svenska regeringen att överföra trupper till de tyska besittningarna. Beslutet mötte invändningar i riksrådet, eftersom Sverige saknade resurser för ett större krig och eftersom ett danskt ingripande kunde hota förbindelserna över Östersjön. Trots detta drevs politiken vidare av Frankrikes krav, subsidierna och Karl XI:s växande stöd för ett mer aktivt ingripande. Svenska trupper samlades därför i Svenska Pommern inför ett fälttåg i norra Tyskland. Danmark, under ledning av kung Kristian V, kunde samtidigt inte stå på samma sida som Sverige och närmade sig därför den antifranska koalitionen, där bland annat Nederländerna, Brandenburg-Preussen, Tysk-romerska riket och flera nordtyska furstar ingick. Särskilt viktigt blev stödet från den nederländska flottan, som senare fick stor betydelse för krigföringen i Östersjön. Svenska förhandlingar i Köpenhamn ledde visserligen till att Danmark tills vidare försäkrade sin neutralitet, men den danska ledningen var splittrad i frågan om ett nytt krig mot Sverige. +Eftersom relationerna med Danmark var osäkra övervägde den svenska krigsledningen ett förebyggande angrepp mot Holstein. Planen stöddes av riksmarsken Carl Gustaf Wrangel, men stoppades eftersom Frankrike, som under sommaren 1674 stred mot kejserliga och brandenburgska trupper i Rhenlandet, prioriterade ett svenskt angrepp mot Brandenburg. I september 1674 förband sig de svenska förhandlarna att låta trupperna rycka över Brandenburgs gräns mot Frankrikes fiender, mot ytterligare franska penningförskott. I december samma år förberedde sig Wrangel för att gå in i Brandenburg med en svensk här. Därmed blev ett större krig i praktiken oundvikligt, och utvecklingen ledde fram till det krig som i svensk historieskrivning blev känt som ”skånska kriget”. Karl XI hade nyligen blivit myndig när kriget inleddes. Den unge kungen hade inledningsvis begränsad erfarenhet av krig och statsledning, men konflikten kom att få stor betydelse för hans utveckling som regent och för hans senare maktkoncentration inom det svenska riket. +Styrkeförhållandena under skånska kriget skiftade mellan olika fronter och år, eftersom kriget fördes samtidigt på flera fronter. På svensk sida uppgick landsstridskrafterna 1674 till omkring 40 000 man. Av dessa fanns cirka 15 500 man i Sverige och Finland, 6 000 i Svenska Livland och 18 500 i de tyska besittningarna. När garnisoner hade lagts in i de tyska fästningarna återstod en fältarmé på omkring 11 000–13 000 man. Den svenska försvarsplanen hösten 1675 byggde på att minska garnisonerna och samla större rörliga styrkor. Enligt planen skulle garnisonsstyrkorna reduceras till omkring 14 000 man, medan de rörliga trupperna skulle koncentreras till minst 22 000 man i Tyskland, omkring 12 000 man vid västgränsen mot Norge och omkring 4 000 man i Livland. Totalt räknade planen med 13 000 kavallerister, 4 500 dragoner och 37 000 fotsoldater, sammanlagt 54 500 man. Planen var dock mer ambitiös än vad Sverige i praktiken kunde genomföra, eftersom bristen på manskap, utrustning och samordning var betydande. Till sjöss var Sverige beroende av att flottan kunde hålla förbindelserna över Östersjön öppna. Under 1676 rustades den svenska örlogsflottan till 25 linjeskepp och 35 andra örlogsfartyg, med omkring 2 000 kanoner och cirka 10 000 man, vilket var en av de största flottor som någonsin utrustats i Sverige. +Danmark-Norge hade inför kriget påbörjat omfattande rustningar. Under våren 1675 planerades en dansk fältstyrka på 16 000 man för operationer i Holstein. Den danska stående armén uppgick enligt en samtida sammanställning till omkring 29 200 man, fördelade på cirka 20 000 fotsoldater, 8 000 ryttare och 1 200 dragoner. Alla dessa kunde dock inte användas som rörlig fältarmé, eftersom en del av manskapet behövdes till fästningsbesättningar, kustförsvar och andra bevakningsuppgifter. Den norska armén organiserades samtidigt för operationer mot Sverige från väster. Styrkan angavs först till drygt 11 000 man, men ökade efter organisatoriska förändringar till omkring 12 000 man. +I Nordtyskland samlades drygt 20 000 brandenburgska och kejserliga soldater, en styrka som kunde växa till som mest mellan 30 000 och 45 000 man. I den allierade krigsplaneringen våren 1675 räknade man dessutom med att omkring 20 000 brandenburgare, 5 000–6 000 münsterska soldater, 10 000 kejserliga trupper och 14 000 lüneburgska trupper skulle sättas in mot Sverige i Nordtyskland. Dessa planer ändrades under förhandlingarnas gång, och siffrorna bör därför ses som planerade eller mobiliserade kontingenter snarare än som en exakt totalsumma över de styrkor som faktiskt var i strid samtidigt. +Till sjöss understöddes Danmark-Norge av Nederländerna. I den ursprungliga planeringen våren 1675 räknade man med att 16 danska och 9 nederländska örlogsfartyg skulle operera gemensamt i Östersjön, även om den första förenade flottstyrkan endast omfattade 7 danska och 7 nederländska örlogsfartyg. Under 1676 och 1677 fortsatte samarbetet med nederländska eskadrar under bland andra Cornelis Tromp, Philip van Almonde och Willem Bastiaensz Schepers. Den allierade flottan bestod då av 26 danska örlogsfartyg under amiral Niels Juels ledning, medan den nederländska hjälpflottan omfattade omkring 10 örlogsfartyg. Under 1678 och 1679 redovisades däremot främst dansk-norska flottstyrkor, med 80 fartyg 1678 och 107 örlogsfartyg 1679. +Enligt Sveriges avtal med Frankrike skulle Karl XI ingripa mot stater inom det Tysk-romerska riket. Efter franska påtryckningar gick svenska trupper den 25 december 1674 från Svenska Pommern in i Uckermark i Brandenburg utan krigsförklaring. Inmarschen motiverades främst av Sveriges förpliktelser gentemot Frankrike och syftade till att binda Brandenburgs styrkor för att avlasta Frankrike vid Rhenfronten. Fälttåget leddes av Carl Gustaf Wrangel och dennes halvbror Wolmar Wrangel, och den svenska armén kunde till en början rycka fram utan större motstånd eftersom Brandenburgs huvudarmé befann sig på annat håll. Den svenska framryckningen blev dock försiktig och präglades av bristande disciplin, låg moral och försörjningsproblem. Armén bestod delvis av värvade soldater från Pommern och andra europeiska områden, och obetalda löner samt svårigheter att skaffa proviant försvagade truppernas sammanhållning. Den svenska närvaron drabbade också civilbefolkningen hårt genom inkvarteringar, plundringar och övergrepp, vilket bidrog till lokalt motstånd mot svenska patruller och mindre avdelningar. +Under våren 1675 planerade svenskarna att gå över Elbe och förena sig med en allierad styrka från Braunschweig-Lüneburg för att sedan ockupera Havelland. Kurfursten Fredrik Vilhelm I beslutade då att först driva ut svenskarna ur Brandenburg innan han återvände till kriget mot Frankrike. Nederländerna och Spanien förklarade krig mot Sverige, men Brandenburg fick inget omedelbart militärt stöd från vare sig Danmark eller kejsaren Leopold I. I juni 1675 ryckte den brandenburgska armén fram under ledning av Fredrik Vilhelm och fältmarskalk Georg von Derfflinger. Genom snabba anfall utnyttjade brandenburgarna att den svenska armén var splittrad. Efter ett överraskande anfall vid Rathenow och en skärmytsling vid Nauen tvingades svenskarna till reträtt mot Fehrbellin. I slaget vid Fehrbellin den 18 juni 1675 led den svenska armén ett kännbart nederlag. Slaget var inte militärt förintande, men fick stor politisk och psykologisk betydelse. Svenskarna tvingades retirera mot Svenska Pommern, och den svenska arméns rykte som närmast oövervinnerlig skadades. Segern stärkte Brandenburg-Preussens militära anseende och uppmuntrade flera av Sveriges motståndare att ansluta sig till kriget. +Slaget vid Fehrbellin fick allvarliga följder för Sverige. Det visade att rikets krigsplanering var osäker och att Sveriges ställning som offensiv stormakt hade försvagats. Karl XI svarade med ett omfattande upprustningsprogram, där armén och särskilt flottan prioriterades. Efter Fehrbellin stärktes den svenskfientliga koalitionen. Leopold I förklarade Sverige som fiende till det Tysk-romerska riket, och flera tyska furstar vände sig mot Sverige eller intog en neutral hållning. Även Danmark anslöt sig till Sveriges fiender och såg en möjlighet att återta områden som tidigare förlorats till Sverige. Därmed vidgades konflikten från ett svenskt ingripande i Brandenburg till ett större krig på flera fronter: i norra Tyskland, i Östersjön, i Skåneland och vid gränsen mot Norge. +Efter slaget vid Fehrbellin riktades uppmärksamheten mot Sveriges nordtyska besittningar. Danmark ville begränsa Sveriges inflytande i norra Tyskland, särskilt i Holstein-Gottorp. Genom Rendsburgrecessen den 10 juli 1675 tvingades hertig Kristian Albrekt avstå från sin suveränitet och sina utländska allianser, varefter den gottorpska delen av Schleswig ockuperades av Danmark fram till krigsslutet. Därefter kunde Danmark rikta större militära ansträngningar mot de svenska stödjepunkterna i norra Tyskland. När Danmark den 2 september 1675 öppet anslöt sig till Sveriges fiender stod Sverige inför ett flerfrontskrig. Danmark fick stöd av bland andra Brandenburg-Preussen, Münster och Braunschweig-Lüneburg, medan även Nederländerna och kejsaren ingick i den bredare svenskfientliga koalitionen. Samarbetet mellan Sveriges fiender präglades av skilda intressen, men tvingade ändå Sverige att försvara flera utspridda fronter samtidigt. Sveriges strategiska svaghet var att riket och dess besittningar var geografiskt splittrade. Försvaret byggde på att flottan kunde hålla förbindelserna öppna och flytta trupper mellan olika fronter. När Danmark och dess allierade angrep från flera håll blev denna utspridda stormaktsställning en allvarlig belastning för Sverige. +Wismar och Svenska Pommern var Sveriges viktigaste besittningar i Tyskland, där den senare omfattade de befästa städerna Stralsund, Greifswald, Stettin, Wolgast, Anklam och Demmin. När danska, brandenburgska och kejserliga styrkor gick till angrepp hösten 1675 drog sig de svenska styrkorna tillbaka mot fästningarna. Försvaret försvagades av splittrad ledning, bristande samordning och flottans misslyckade försök att ingripa. De svenska styrkorna tvingades successivt tillbaka. Stora delar av de svenska områdena i Pommern gick förlorade, däribland Tribsees, Usedom, Wolin och Wolgast, som förstördes i samband med kapitulationen den 10 november 1675. Vid årets slut hade Sverige endast kvar några större fästen i Svenska Pommern, bland annat Stralsund, Stettin, Greifswald och Rügen. Även Wismar belägrades av danska styrkor hösten 1675. Staden var en viktig svensk bas vid södra Östersjön, men blev isolerad när svensk undsättning inte längre kunde väntas. Efter en längre belägring och hård beskjutning kapitulerade Wismar den 24 december 1675. Fallet blev ett allvarligt bakslag för Sverige och försvagade sambandet mellan de svenska områdena i Tyskland. +Vid slutet av 1675 var Sveriges militära läge allvarligt. Wismar hade fallit, Pommern var utsatt och flottans svaghet gjorde det svårt att föra över förstärkningar. Samtidigt växte hoten mot andra svenska områden, däribland Bremen-Verden, Skåne och gränsen mot Norge. Kriget hade därmed utvecklats från ett svenskt ingripande i Brandenburg till ett flerfrontskrig där även rikets eget försvar stod på spel. +Ett mindre men viktigt delavsnitt av kriget utkämpades i den svenska besittningen Bremen-Verden. Området hade begränsat ekonomiskt värde men stor strategisk betydelse genom sitt läge nära Nordsjön och som ett svenskt brohuvud i västra Tyskland. När kriget utvecklades blev det dock svårt att försvara. Den svenske fältmarskalken Henrik Horn hade begränsade styrkor, medan fienden kunde angripa från flera håll. +År 1675 angreps Bremen-Verden av styrkor från Münster, Brandenburg-Preussen, Danmark och Braunschweig-Lüneburg under gemensam ledning av furstbiskopen Christoph Bernhard von Galen och hertigen Georg Wilhelm av Sachsen-Lauenburg. De svenska styrkorna pressades gradvis tillbaka, bland annat genom deserteringar bland tyska soldater i svensk tjänst, och tvingades överge landsbygden och dra sig tillbaka mot Stade. Under oktober föll Ottersberg, Buxtehude och Bremervörde till de allierade styrkorna. Vid årets slut återstod i praktiken endast några starkare svenska fästen, främst Stade och Carlsburg. +Under 1676 försämrades läget ytterligare. Stade led brist på förnödenheter, och den svenska flottan kunde inte bryta blockaden eller undsätta området. Efter belägring kapitulerade Horn den 13 augusti 1676, varefter de återstående svenska fästena föll i de allierades händer. Sverige förlorade därmed i praktiken kontrollen över Bremen-Verden. Förlusten minskade Sveriges handlingsutrymme i norra Tyskland. Danmark behövde inte längre oroa sig för ett svenskt hot från söder, och de allierade kunde rikta större uppmärksamhet mot Sveriges återstående besittningar i Pommern. Samtidigt gjorde dansk kontroll över sjöförbindelserna det svårt för Sverige att sända förstärkningar, vilket bidrog till att de svenska styrkorna i Nordtyskland blev alltmer isolerade. +Det dansk-nederländska sjöherraväldet i Östersjön gjorde det svårt för Sverige att försörja och undsätta sina trupper i Svenska Pommern. Kriget i de tyska provinserna fick därför främst karaktären av ett belägrings- och fästningskrig. De svenska styrkorna var utspridda och beroende av sjöförbindelser som fienden i stor utsträckning kunde hota eller skära av. Under 1676 och 1677 fortsatte de svenska positionerna i Pommern att falla. Brandenburg erövrade flera viktiga platser, däribland Peenemünde, Anklam och Demmin. Till slut återstod främst Stralsund och Stettin som större svenska fästen i området. +Stettin blev därefter ett av Brandenburgs huvudmål. Belägringen återupptogs i juli 1677 med större styrkor och tungt artilleri. Stadens försvar leddes av Jacob Johann von Wulffen, men efter flera månaders strider, bombardemang och omfattande förstörelse kapitulerade den svenska garnisonen den 26 december 1677. Förlusten av Stettin blev ett av Sveriges allvarligaste nederlag i den tyska delen av kriget. Samtidigt fortsatte striderna om Rügen. Danska trupper landsteg på Rügen den 17 september 1677, men svenska styrkor under Otto Wilhelm von Königsmarck återtog ön den 8 januari 1678 efter slaget vid Warksow. Framgången blev dock kortvarig. Under september 1678 genomförde Danmark och Brandenburg en ny landstigning på Rügen, och ön föll snabbt åter i de allierades händer. Därmed blev Stralsunds ställning alltmer utsatt. +Efter Rügens fall riktades huvudanfallet mot Stralsund. Staden belägrades i september 1678 och utsattes för kraftig artilleribeskjutning. Königsmarck försvarade staden, men de allierades överlägsenhet i manskap och artilleri gjorde läget ohållbart. Den svenska garnisonen kapitulerade den 11 oktober och fick lämna staden med militär heder. Kort därefter återstod endast Greifswald som Sveriges sista större fäste i Pommern. Staden kapitulerade den 7 november 1678. Därmed hade Sverige förlorat sina sista större fästen i Svenska Pommern, och det svenska försvaret av området var i praktiken avslutat. +Under natten mellan den 4 och 5 december 1678 inträffade sjökatastrofen vid Bornholm, när en svensk transportflotta var på väg från Pommern till Sverige. I hårt vinterväder misstolkade flera fartyg ljuset från Sose på Bornholms västkust, drev in mot land och strandade. Katastrofen ledde till stora förluster i människor, fartyg och krigsmateriel och blev ytterligare ett svårt slag mot Sverige efter motgångarna i Nordtyskland. +När Sveriges besittningar i Nordtyskland höll på att gå förlorade försökte Sverige öppna en ny front längre österut. Syftet var att avlasta de svenska trupperna i Pommern genom ett fälttåg från Livland in i Ostpreussen och därmed tvinga Brandenburg-Preussen att sprida sina styrkor. Sverige hoppades även på stöd från Polen-Litauen, men kung Johan III Sobieski avstod från att gå in i kriget på svensk sida. Befälet gavs till Henrik Horn, som i oktober 1678 marscherade från Riga med omkring 12 000 man. Den svenska styrkan gick in i Preussen den 15 november och intog bland annat Tilsit. När Stralsund föll kunde Brandenburg dock frigöra styrkor från Pommern, vilket gjorde att fälttåget förlorade sitt ursprungliga syfte. Horn beordrades därför att avbryta framryckningen mot Königsberg och dra sig tillbaka eller inta vinterkvarter. +Fälttåget utvecklades därefter till ett nytt svenskt nederlag. Den svenska armén led av kyla, sjukdomar och brist på förnödenheter, medan kurfurst Fredrik Vilhelm I samlade sina styrkor för ett motangrepp. Genom ett snabbt vinterfälttåg, där brandenburgska trupper bland annat rörde sig över den frusna Kuriska sjön, kunde de nå fram till svenskarna. Horn beslöt då att retirera mot Livland. Svenskarna undvek ett större fältslag men led svåra förluster under förföljelsen och i skärmytslingarna vid Tilsit, Splitter och Telschi. När Horn återvände till Riga i februari 1679 återstod endast omkring 2 000 soldater av den ursprungliga styrkan. Fälttåget hade därmed kostat Sverige stora förluster utan att uppnå något strategiskt resultat. För Brandenburg blev vinteroperationen däremot en framgång som ytterligare stärkte Fredrik Vilhelms militära anseende. +Under skånska kriget fick kontrollen över sjövägarna stor strategisk betydelse. Sveriges tyska besittningar låg längs Östersjöns södra kust och var beroende av sjöförbindelser med Sverige för trupper, förnödenheter och förstärkningar. För Sverige gällde det därför att hålla förbindelserna öppna, skydda Skåne från invasion och hindra dansk-nederländsk samverkan. Danmark försökte i stället bryta de svenska transportlederna, isolera de tyska besittningarna och skapa förutsättningar för en invasion av Skåne. Utgångsläget var dock ojämnt. Danmark hade större sjömilitär erfarenhet, ett mer gynnsamt basläge i Köpenhamn och kunde dessutom få stöd av Nederländerna, en av Europas främsta sjömakter. Sverige hade visserligen byggt upp en stor örlogsflotta, men den led av brist på erfarna sjömän, svag organisation, otillräckligt underhåll och ledningsproblem. Den svenska huvudbasen vid Stockholm låg dessutom långt från krigsskådeplatsen och påverkades av långa transportvägar genom skärgården och av isförhållanden. När Danmark gick in i kriget 1675 försökte den danska flottan utnyttja dessa svagheter. Den svenska flottan låg vid Dalarö och var på papperet stark, men hade svårt att snabbt bli stridsklar. En svensk expedition försenades, drabbades av stormar och återvände utan att ha nått sitt mål. Därmed isolerades de svenska besittningarna i Nordtyskland i ett tidigt skede och fick inte det stöd sjövägen som behövdes. +Inför 1676 års fälttåg byggde den svenska strategin på att flottan först skulle föra förstärkningar till Tyskland och därefter blockera de danska farvattnen. Karl XI planerade samtidigt en offensiv mot de danska öarna, med en tänkt landstigning på Själland. På dansk sida blev erövringen av Gotland ett viktigt mål, eftersom ön kunde stärka Danmarks ställning inför en större invasion av Skåne. +Den svenska flottans ledning var samtidigt försvagad. Riksamiralen Gustaf Otto Stenbock hade misslyckats med att få flottan till sjöss 1675 och ersattes av riksrådet Lorentz Creutz, som utsågs till amiralgeneral. Creutz hade administrativ erfarenhet men saknade större sjömilitär bakgrund, vilket blev en svaghet när flottan skulle användas i ett avgörande skede av kriget. Våren 1676 gick den danska flottan under amiral Niels Juel till sjöss och förenade sig med en nederländsk eskader under Cornelis Tromp och Philips van Almonde. Ett svenskt försök att hävda kontrollen över Östersjön ledde först till ett oavgjort resultat i slaget mellan Bornholm och Rügen den 25–26 maj, men utan att förändra styrkeförhållandena till svensk fördel. +Den 1 juni 1676 möttes de svenska och dansk-nederländska flottorna i slaget vid Ölands södra udde, som blev ett svårt svenskt nederlag. Den svenska flottan splittrades av svag samordning, hårt väder och bristande sjövana. Amiralgeneral Creutz stupade när regalskeppet Kronan kantrade och exploderade efter en misslyckad manöver, och kort därefter förlorades även amiral Claes Ugglas fartyg Svärdet. Flera andra svenska fartyg förstördes, erövrades eller tvingades dra sig undan, och flottan drog sig tillbaka mot Stockholms skärgård. Nederlaget vid Öland fick stora följder. Sverige förlorade inte hela flottan, men dess handlingsfrihet i Östersjön minskade kraftigt. Danmark och Nederländerna fick övertaget i Öresund och södra Östersjön, vilket försvårade Sveriges förbindelser med de tyska besittningarna. Därmed fanns det inte längre något större hinder för en dansk landstigning i Skåne. Kriget försköts därmed från att främst ha berört Sveriges områden i norra Tyskland till att direkt hota svenskt territorium i södra Sverige. Under resten av 1676 behöll den danska flottan initiativet och kunde operera vid bland annat Rügen och Blekingekusten. Den danska sjödominansen bidrog till att hota de svenska förbindelserna i södra Östersjön, men kunde inte ensam avgöra kriget. +År 1677 var Danmark beroende av säkra förbindelser över Öresund för att kunna försörja och förstärka sina trupper i Skåne. Sverige försökte därför återta initiativet till sjöss och bryta de danska försörjningslinjerna. Den svenska flottan hade dock fortfarande problem med manskap, utrustning och sjöduglighet. En svensk eskader under amiral Erik Carlsson Sjöblad seglade från Göteborg för att förena sig med huvudflottan, men mötte i stället en starkare dansk eskader under Niels Juel vid Møn. I slaget vid Møn den 1 juni 1677 besegrades den svenska eskadern. Flera svenska fartyg erövrades eller gick förlorade, och många sjömän och soldater togs till fånga. Nederlaget försvagade Sveriges möjligheter att störa de danska sjöförbindelserna och stärkte Danmarks sjöherravälde. Den 1 juli 1677 möttes de svenska och danska huvudflottorna i slaget vid Köge bukt. Den svenska flottan under Henrik Horn hade till uppgift att bryta Danmarks förbindelser över Öresund, men slaget slutade med ett avgörande svenskt nederlag. Den svenska flottan hamnade i oordning, flera fartyg gick på grund eller blev avskurna, och danskarna utnyttjade skickligt sitt övertag. Efter segern vid Köge bukt befästes Danmarks sjödominans. Sverige kunde inte längre på allvar utmana Danmark till havs under krigets fortsättning och fick svårt att undsätta Pommern sjövägen. De danska sjösegrarna gjorde det möjligt att fortsätta stödja armén i Skåne över Öresund, medan Sverige fick svårare att isolera de danska styrkorna i landskapet. Samtidigt försvårades kontakterna med de svenska besittningarna i Tyskland, och svensk handel utsattes för danska kapare. +Under 1678 och 1679 fortsatte Danmark att dominera Östersjön, men krigets politiska utveckling begränsade flottans betydelse. Den svenska flottan undvek större avgöranden till sjöss och hölls i huvudsak i skyddade positioner, medan den danska flottan försökte bevaka Kalmarsund, hindra svenska transporter och angripa svenska fartyg. Sammantaget gjorde sjökriget invasionen av Skåne möjlig och försvårade Sveriges försvar av sina utspridda besittningar, men sjöherraväldet räckte inte för att Danmark skulle kunna framtvinga ett avgörande nederlag för Sverige. +Efter de svenska motgångarna till sjöss 1676 kunde Danmark föra över trupper till svenska områden. Den svenska ledningen hade räknat med att Skåne kunde angripas, men försvaret var otillräckligt. De svenska styrkorna var få och utspridda, flera fästningar var svagt bemannade och armén saknade resurser för att möta en större dansk landstigning. Ett av Danmarks första mål blev Gotland, som hade strategisk betydelse genom sitt läge i Östersjön. Det svenska försvaret på ön var svagt, och när en dansk styrka under Niels Juel landsteg vid Klintehamn i slutet av april 1676 kunde inget effektivt motstånd organiseras. Visborgs slott kapitulerade den 1 maj, och Gotland föll snabbt i danska händer. Ön förblev därefter under dansk kontroll under större delen av kriget, även om den inte präglades av samma våldsamma krigföring som Skåne. +Efter Gotlands fall blev Skåne huvudfront. Det dansk-nederländska sjöherraväldet öppnade vägen för den planerade invasionen. Den danska huvudstyrkan samlades på Själland och fördes över under skydd av flottan. En mindre landstigning vid Ystad genomfördes den 27 juni 1676, men huvudlandstigningen skedde den 29 juni vid Råå söder om Helsingborg. Den danska armén uppgick till omkring 14 000–15 000 man och stod under befäl av kung Kristian V och hertig Johan Adolf av Holstein-Plön. Landstigningen mötte begränsat motstånd. De svenska styrkorna i Skåne var för svaga för att hindra övergången, och Karl XI:s fältstyrka på 5 000 man drog sig tillbaka österut och norrut för att undvika att bli instängd i västra Skåne. Efter landstigningen försökte Kristian V vinna stöd i landskapet genom kungörelser, krav på trohetsed och manifest där invasionen framställdes som en befrielse från svenskt styre. +Den danska framryckningen gick snabbt. Ängelholm intogs i samband med landstigningen, och Helsingborg föll den 30 juni utan större strid. Även stadens slott kapitulerade den 4 juli, vilket väckte stark kritik från svensk sida. Under sommaren föll flera andra viktiga fästen. Landskrona intogs den 3 augusti 1676 och blev en viktig dansk bas i västra Skåne. I Blekinge kapitulerade Karlshamn den 7 augusti och Kristianopel den 25 september, vilket stärkte den danska ställningen där och försvagade Sveriges försvar i de södra landskapen. Därefter riktades offensiven mot Kristianstad, en viktig befäst stad i nordöstra Skåne. Efter hårda strider natten mellan den 14 och 15 augusti föll staden, och en dansk garnison lades in där. Därmed hade Danmark på kort tid erövrat flera centrala orter i Skåne och Blekinge. Den svenska huvudarmén drog sig tillbaka mot Småland, samtidigt som Karl XI även behövde hantera hotet från norsk-danska styrkor i Västsverige. Efter den danska offensiven återstod i praktiken Malmö som den viktigaste större svenska stödjepunkten i Skåne. +Efter den danska landstigningen i Skåne föll den svenska kontrollen över landskapet snabbt samman. Kristian V framställde sig som landskapets befriare, och större delen av Skåne hamnade under dansk kontroll, även om Malmö fortsatte att hålla stånd. Sveriges läge var samtidigt allvarligt: Danmark och Nederländerna hade övertaget till sjöss, de svenska besittningarna i Nordtyskland var hotade och dansk-norska angrepp väntades från väster. +Ett första viktigt svenskt motangrepp kom vid slaget vid Halmstad, även kallat slaget vid Fyllebro, den 17 augusti 1676. En dansk styrka under general Jakob Duncan hade sänts mot Halmstad för att verka i Halland och därefter förena sig med dansk-norska styrkor västerifrån. Karl XI och fältmarskalk Simon Grundel-Helmfelt hann dock fram med en svensk styrka och tvingade fram strid innan danskarna kunde fullfölja planen. Slaget slutade med svensk seger, och Duncan togs till fånga. Segern fick stor moralisk betydelse och försvårade de danska planerna på att samordna offensiverna i Halland, Bohuslän och Västergötland. Trots segern var Sveriges läge fortsatt pressat. Den danska huvudarmén stod kvar i Skåne, medan de svenska trupperna led av brist på proviant och förstärkningar. Samtidigt utnyttjade Danmark inte sitt övertag fullt ut. Kristian V valde att avvakta, bland annat sedan Frankrike förklarat krig mot Danmark, vilket gav Karl XI tid att samla nya styrkor. Under hösten 1676 gick den svenska armén söderut för att undsätta Malmö och möta den danska huvudarmén. Fälttåget försvårades av regn, dåliga vägar och försörjningsproblem, men svenskarna lyckades ändå föra kriget tillbaka in i Skåne. +Vid slutet av 1676 befann sig båda arméerna i ett svårt läge med sjukdom, hunger och bristande försörjning. Karl XI övervägde reträtt, men beslöt i stället att försöka nå Malmö eller tvinga fram ett avgörande. När vintern kom frös vattendragen, vilket gjorde det möjligt för svenskarna att korsa Kävlingeån och överraska danskarnas ställning vid Lund. Den 4 december 1676 möttes de svenska och danska huvudarméerna i slaget vid Lund. Den svenska armén gick över Kävlingeån under natten och anföll den större danska armén i gryningen. Slaget blev ett av de blodigaste i Nordens historia och utvecklades till en lång och kaotisk strid där initiativet växlade flera gånger. Den svenska armén hamnade periodvis i ett mycket svårt läge, men efter hårda strider bröt den danska arméns sammanhållning samman, och danskarna tvingades retirera till Landskrona. Den danske överbefälhavaren Carl von Arenstorff stupade efter slaget. +Segern vid Lund fick stor betydelse för krigets fortsättning. Den danska offensiven förlorade sin kraft, Malmö var inte längre lika isolerat och Sverige kunde återta initiativet i Skåne och Blekinge. För Karl XI stärkte segern kungens auktoritet och gav den svenska armén nytt handlingsutrymme. Kriget var dock inte avgjort. Danmark behöll viktiga fästen, bland annat Landskrona och Kristianstad, och striderna fortsatte på flera fronter. Efter segern vid Lund riktade Karl XI åter uppmärksamheten mot Kristianstad och Helsingborg, där danskt rytteri hade dragit sig tillbaka. Helsingborg föll till svenskarna den 30 december 1676, vilket avslutade året med ytterligare en svensk framgång i Skåne. +Skånska kriget drabbade Skånelands städer och landsbygd hårt. I de tidigare danska landskapen uppstod tidigt lokalt motstånd mot de svenska styrkorna, särskilt i norra Skåne och Blekinge, men även i delar av Bohuslän. Motståndet berodde delvis på fortsatt starka band till Danmark, men förstärktes också av inkvarteringar, utskrivningar, proviantkrav och den svenska statens försök att knyta de erövrade landskapen närmare riket. Danmark försökte utnyttja missnöjet genom att framställa Kristian V som landskapens befriare och uppmana befolkningen att stödja den danska krigföringen. +I svensk terminologi kallades de väpnade grupperna ofta ”snapphanar”, medan danskarna hellre använde beteckningen ”friskyttar” för grupper som ansågs strida på dansk sida. Begreppet snapphanar var dock oklart och kunde syfta på deltagare i bondeuppbåd, lokala partisaner, danska friskyttar, fredlösa, förrymda soldater eller stråtrövare. I svenska källor framställdes de ofta som landsförrädare och rövare, medan dansk och senare lokal tradition ibland har beskrivit dem som patrioter eller frihetskämpar. Rörelsen var inte enhetlig. Under krigets första fas fanns inslag av lokalt bondeuppror och självförsvar, medan verksamheten senare delvis organiserades som friskyttekompanier i dansk tjänst. Deltagarna kom ofta från småbrukar-, tjänstefolks- och obesuttna grupper, vilket visar att motståndet även hade sociala och lokala orsaker. Många i gränsbygderna agerade dessutom pragmatiskt och försökte framför allt överleva i ett krigsläge där de pressades av både svenska och danska krav. +Ett av de mest uppmärksammade exemplen på det lokala motståndet var Loshultskuppen den 24 juli 1676, då lokala göingebönder, småländska bönder och andra beväpnade grupper överföll svenska trosstransporter och lade beslag på krigskassan. Händelsen fick stor symbolisk betydelse och visade att den svenska reträtten och den danska framryckningen hade skapat ett maktvakuum i delar av gränsbygderna. Den visade också att kriget inte enbart utkämpades mellan reguljära arméer, utan även genom lokala uppror, bakhåll och plundringar. Snapphaneverksamheten blev särskilt betydelsefull i Skåne och Blekinge, där lokala friskyttar angrep svenska transporter, patruller, ämbetsmän, soldater och försörjningslinjer. Enligt bevarade danska mönsterrullor fanns 17 friskyttekompanier med sammanlagt 949 man, medan svenska och danska uppskattningar angav mellan 1 200 och 1 500 man; så många var dock sannolikt inte aktiva samtidigt. Danskarna försökte organisera sådana grupper och använda dem som stöd för sina militära operationer. Friskyttarna uppträdde ofta i små grupper och kunde fungera som spejare, vägvisare och störande förband i skogsbygderna. Deras militära värde var dock begränsat på längre sikt, eftersom många deltagare främst försvarade sina hemtrakter eller deltog i tillfälliga aktioner. Gerillakrigföringen försvårade ändå den svenska kontrollen över landsbygden och bidrog till att konflikten i Skåne fick en särskilt hård karaktär. +Efter den svenska segern vid Lund skärptes politiken mot snapphanarna. Karl XI och den svenska ledningen bedömde att rörelsen måste isoleras från civilbefolkningen. Den svenska motstrategin växlade mellan amnesti och hårda repressalier: plakat erbjöd förlåtelse åt dem som underkastade sig men hotade samtidigt med stränga straff mot dem som stödde snapphanar eller danska styrkor. Johan Göransson Gyllenstierna, Karl XI:s gunstling vid denna tid, fick en central roll genom det så kallade ”edkrävartåget”, då befolkningen i delar av Blekinge och Skåne tvingades avlägga trohetseder till den svenska kronan och lova att inte bistå väpnade grupper. Den svenska politiken fick efter hand effekt, särskilt när Sveriges militära ställning i Skåne förbättrades och det öppna stödet till snapphanarna minskade. Samtidigt drabbades civilbefolkningen hårt. Misstänkta snapphanar, friskyttar och medhjälpare kunde dömas till stränga straff, och avrättningar användes som avskräckning. Det hårdaste exemplet var åtgärderna mot Örkeneds socken under april 1678, där gårdar brändes, boskap och livsmedel fördes bort och flera invånare dödades eller avrättades. Syftet var att slå mot snapphanarnas stöd och visa att kronan var beredd att använda hårda medel mot väpnat motstånd. +Mot slutet av kriget minskade snapphanerörelsen i omfattning. När det stod klart att Skåne skulle förbli svenskt försvagades stödet från den bofasta befolkningen, och de kvarvarande grupperna kom i högre grad att bestå av stråtrövare, fredlösa och förrymda soldater. En del följde med de danska trupperna tillbaka, medan andra försökte återvända hem eller höll sig gömda. Snapphanekriget avgjorde inte skånska kriget militärt, men bidrog till att den svenska staten efter kriget skärpte kontrollen över de erövrade landskapen och drev på deras inordning i det svenska riket. +Under vintern och våren 1677 förberedde båda sidor nya operationer i Skåne och Blekinge. Sverige försökte stärka sina förband, återta förlorade positioner och bekämpa snapphanarna, medan Danmark samlade en ny armé i Landskrona. Under vintern hade svenskarna återtagit bland annat Helsingborg, Kristianopel och Karlshamn, men hade fortsatt brist på pengar, utrustning, förnödenheter och sjöförbindelser. +Våren 1677 återtog Danmark initiativet i Skåne. Svenska styrkor tvingades avbryta belägringen av Kristianstad, och stora delar av landskapet kom åter under dansk kontroll. I maj stod Karl XI vid Rönneberga med en mindre svensk styrka på 5 000 man mot Kristian V:s överlägsna armé på 12 000 man, men trots sitt övertag lyckades danskarna inte utnyttja situationen fullt ut. Svenskarna kunde dra sig undan, samtidigt som snapphaneverksamheten fortsatte på landsbygden. Därefter riktades den danska offensiven mot Malmö. Malmö var ett av Sveriges viktigaste kvarvarande fästen i Skåne. Under fältmarskalk Fabian von Fersen förstärktes försvaret, men staden var utsatt eftersom danskarna kontrollerade stora delar av landskapet. Kristian V inledde belägringen av Malmö den 11 juni 1677 med omkring 14 000 man. Ett större danskt stormningsförsök natten mellan den 25 och 26 juni slogs tillbaka med svåra förluster. Misslyckandet blev ett betydande bakslag för Danmark, och den 5 juli drog sig danskarna tillbaka mot Lund och Landskrona. Malmö förblev därmed i svenska händer. +Efter försvaret av Malmö marscherade Karl XI norrut med 13 000 man för att möta Kristian V:s huvudarmé, som hade dragit sig tillbaka mot Landskrona. Den 14 juli 1677 möttes arméerna i slaget vid Landskrona, även kallat slaget vid Tirup eller Ylleshed. Slaget blev krigets sista stora fältslag och slutade med svensk seger. Den danska armén drog sig tillbaka innanför Landskronas murar och senare mot Kristianstad. Segern vid Landskrona hindrade Danmark från att snabbt återta initiativet efter nederlaget vid Malmö, men den avgjorde inte kriget. De svenska förlusterna var stora, och Karl XI saknade resurser för att driva ut danskarna ur Skåne eller belägra Landskrona under längre tid. Efter slaget mattades de större fältrörelserna av, medan striderna kring Kristianstad och snapphanekriget fortsatte. Kriget i Skåne övergick därmed till ett utdraget kraftprov där båda sidor försökte behålla eller återta kontrollen över landskapet. +Efter de svenska framgångarna på land fortsatte Danmark och Nederländerna att utnyttja sitt sjöherravälde. Den svenske riksamiralen Hans Wachtmeister förlade därför en del av flottan till Kalmarsund, där farvattnet och Kalmars fästningsartilleri gav visst skydd. I augusti 1677 genomförde den allierade flottan under Niels Juel och Cornelis Tromp ett angrepp mot området. Ett försök att hota Kalmar misslyckades, men danska och nederländska styrkor kunde därefter genomföra landstigningar och plundringar på Öland och längs Smålandskusten. Mellan den 10 och 18 augusti 1677 utsattes Öland för omfattande härjningar. Gårdar, hus och kvarnar brändes, boskap och livsmedel fördes bort och flera ölänningar tvångsrekryterades till de allierade fartygen. Borgholms fästning hotades men kunde inte intas, sedan Wachtmeister den 18 augusti gått över Kalmarsund med en mindre styrka och överraskat danskarna. Efter nederlagen vid Borgholm och på norra Öland fortsatte danskarna med strandhugg längs Smålandskusten, där bland annat Västervik drabbades. Räderna fick svåra följder för Öland, som drabbades av hungersnöd vintern 1677–1678 och därefter inte längre kunde bidra ekonomiskt till den svenska krigsledningen på samma sätt. De danska operationerna mot Kalmar, Öland och Smålandskusten visade att Danmarks styrka främst låg i rörlig sjömakt och förmågan att hota utsatta svenska kustbygder. Samtidigt kunde danskarna inte inta Kalmar eller Borgholm, och räderna ledde därför inte till något avgörande militärt resultat i Skåne. +Samtidigt som skånska kriget fördes i Skåneland, Nordtyskland och Östersjön pågick även strider i Västsverige och längs gränsen mot Norge. Denna del av konflikten har i historieskrivningen kallats ”Gyldenløvefejden”, efter den dansk-norske ståthållaren Ulrik Frederik Gyldenløve. Den dansk-norska krigsledningen ville utnyttja Sveriges utsatta läge genom att angripa från väster, binda svenska styrkor och hota Bohuslän, Västergötland och den svenska västkusten. Inför kriget hade Gyldenløve stärkt försvaret i Norge, organiserat armén och förbättrat fästningarna. Den svenska sidan var däremot svagt förberedd i gränsområdena. Krigföringen inleddes med räder och mindre anfall, men utvecklades snart till större operationer. I oktober 1675 gick Gyldenløve till anfall mot Bohuslän, men han stoppades vid Kvistrum av general Rutger von Ascheberg och tvingades retirera. +År 1676 gick Gyldenløve till nytt anfall mot Bohuslän och Västergötland. Under juni och juli det året intog dansk-norska styrkor bland annat Uddevalla och Vänersborg och besegrade svenska bondeuppbåd vid slaget vid Vänersborg och slaget vid Hunneberg. Den fortsatta framryckningen söderut hindrades dock av Bohus fästning, som spärrade vägen mot Göteborg och Göta älv. Gyldenløve belägrade fästningen i augusti, men garnisonen under Harald Stake höll stånd. När det svenska motståndet hårdnade avbröts belägringen i september, och de dansk-norska styrkorna drog sig tillbaka mot Norge. +Sommaren 1677 fortsatte de dansk-norska operationerna mot Bohuslän. Det viktigaste målet blev Marstrand och den strategiskt viktiga Carlstens fästning, som hade stor betydelse för kontrollen över västkusten och hotet mot Göteborg. Efter hårda strider kapitulerade fästningen den 23 juli 1677. Marstrands fall blev en betydande dansk-norsk framgång och gav Gyldenløve både militär och symbolisk prestige. Efter Marstrands fall försökte den svenska ledningen hejda den dansk-norska framryckningen. Förstärkningar skickades till Bohuslän under Magnus Gabriel De la Gardie, men det svenska försvaret försvagades av bristande samordning och försörjningsproblem. I slaget vid Uddevalla den 28 augusti 1677 led svenskarna ett nytt nederlag, vilket blottade svagheterna i försvaret av västra Sverige. Trots framgångarna kunde Gyldenløve inte utnyttja läget fullt ut och drog sig senare tillbaka mot Norge. +Kriget berörde även de nordligare gränsområdena. I augusti 1677 ryckte norska styrkor in i Jämtland, där det svenska motståndet var svagt. Vid Oviken vägrade jämtska soldater att strida för Sverige och gick i stället över till den norsk-danska sidan. Svenskarna drevs snabbt ut ur landskapet, men den dansk-norska kontrollen blev kortvarig. När de norska styrkorna drog sig tillbaka under hösten kunde Sverige återta Jämtland. +År 1678 riktades den dansk-norska offensiven åter mot Bohuslän. Det viktigaste målet blev på nytt Bohus fästning. Belägringen inleddes den 4 juni och blev hård, men den svenska garnisonen under Friedrich Börstell lyckades hålla stånd trots omfattande skador. När svenska förstärkningar under Gustaf Otto Stenbock nådde fram blev belägrarnas läge allt svårare, och den 21 juli drog sig Gyldenløve tillbaka. Fästningen förblev därmed i svensk hand. Under krigets sista år fortsatte dansk-norska operationer i västra Sverige, bland annat i Dalsland, Värmland och Bohuslän. Stora delar av landsbygden drabbades av plundring, bränder och förstörelse, medan svenska försök att återta initiativet förblev begränsade. Även längre norrut förekom svenska motangrepp mot Norge, bland annat mot Røros, men dessa ledde inte till någon avgörande framgång. Gyldenløvefejden fick därmed karaktären av ett utdraget gränskrig som band svenska resurser och slog hårt mot civilbefolkningen, men utan att leda till något avgörande genombrott för Danmark-Norge. +Efter de första årens danska framgångar hade kriget utvecklats till ett utdraget utnötningskrig. Danmark hade fortfarande övertaget till sjöss och höll viktiga stödjepunkter i Skåne, särskilt Landskrona och Kristianstad. De svenska segrarna vid Halmstad, Lund och Landskrona hade dock hindrat Danmark från att avgöra kriget på land. Konflikten blev därför långvarig och hårt belastande för civilbefolkningen. I början av 1678 hölls riksdag i Halmstad, där Karl XI begärde nya utskrivningar och större krigsansträngningar. Trots motgångar i Pommern och till sjöss ville kungen fortsätta kriget och hålla fast vid förbundet med Frankrike. Riksdagen gick med på nya uppoffringar, men önskade samtidigt att kungen skulle verka för fred så länge kriget mot Danmark fortsatte. +Under 1678 försökte Danmark återta initiativet i Skåne, medan svenskarna pressade Kristianstad och bevakade Landskrona. Helsingborg blev en viktig stödjepunkt och utsattes för dansk belägring den 9 juni. Genom list fick danskarna den svenska garnisonen att kapitulera den 27 juni, vilket tillfälligt tvingade Karl XI att avbryta rörelsen mot Kristianstad. Området mellan Helsingborg och Landskrona fortsatte därefter att vara ett utsatt krigsområde, med härjningar och mindre strider som slaget vid Örja utanför Landskrona den 2 september. Samtidigt fortsatte den danska flottan att genomföra räder längs de skånska kusterna och försvåra svensk försörjning. +Krigets viktigaste händelse i Skåne 1678 blev kampen om Kristianstad. Karl XI försökte nu pressa fram ett avgörande genom att göra ett nytt försök att belägra staden och samtidigt tvinga Kristian V till strid. Den danska armén var dock försvagad och undvek ett större fältslag, medan svenska styrkor under bland andra Rutger von Ascheberg gradvis inneslöt fästningen. Den danska garnisonen i Kristianstad höll länge stånd, men försvagades av strider, sjukdomar och bristande försörjning. Danska försök att undsätta staden fördröjdes av långsamhet och oenighet i krigsledningen, och den 4 augusti 1678 kapitulerade Kristianstad till svenskarna. Därmed förlorade Danmark en central bas i Skåne, vilket stärkte Sveriges militära läge inför krigets slutskede. Efter Kristianstads fall drog sig danskarna under generallöjtnant Friedrich von Arenstorffs ledning tillbaka mot Helsingborg och senare över till Själland. Arenstorffs reträtt följdes av omfattande förstörelse, där byar, gårdar, slott och städer såsom Lund, Ystad och Ängelholm brändes eller plundrades. Denna brända jord-politik syftade till att försvåra svensk kontroll och försörjning, men förvärrade också civilbefolkningens lidande. Årets fälttåg i Skåne hade därmed i huvudsak slutat till svensk fördel. +I början av 1679 hade Karl XI omkring 11 000 man i Skåne, och omkring 7 000 man hölls mot Gyldenløves styrkor vid västgränsen. Samtidigt planerade Karl XI ett svenskt anfall mot Själland för att tvinga Danmark till fred. Planen förutsatte dock stöd av en stark flotta, och eftersom den väntade franska flottan aldrig kom till Östersjön övergavs landstigningen. Under samma år påverkades kriget alltmer av diplomatin, då Frankrike pressade fram stillestånd och fredsavtal med flera av Sveriges fiender. I Skåne höll danskarna till slut endast Helsingborg och Landskrona, medan det större europeiska kriget närmade sig en uppgörelse. +Fredsförhandlingarna påverkades starkt av att skånska kriget var en del av den större europeiska konflikten mellan Frankrike, Nederländerna, Brandenburg och flera andra makter. Sedan 1676 hade Frankrike och Nederländerna förhandlat i Nijmegen, och när fred slöts mellan dem den 10 augusti 1678 stärktes Frankrikes ställning. Detta gynnade Sverige, eftersom Ludvig XIV inte ville att den svenska bundsförvanten skulle förlora områden efter ett krig som i hög grad hade förts för franska intressen. +Flera separata fredsavtal banade väg för slutuppgörelsen, inklusive med det Tysk-romerska riket den 26 januari 1679, med furstbiskopen av Münster den 19 mars 1679 och med Nederländerna den 2 oktober 1679. Frankrike drev linjen att de territoriella bestämmelserna från Westfaliska freden skulle ligga fast, vilket gynnade Sverige i de tyska besittningarna. I freden i Celle den 26 januari 1679 tvingades Sverige dock avträda Dörverden och Thedinghausen till furstarna av Lüneburg. Sveriges krig med Spanien avslutades inte genom ett särskilt fredsfördrag, utan genom att en gemensam svensk-spansk fredsförklaring ratificerades den 6 augusti 1679. Brandenburg-Preussen hade haft stora framgångar mot Sverige i Svenska Pommern, men blev under fredsprocessen isolerat. När Fredrik Vilhelm I vägrade återlämna sina erövringar satte Frankrike militärt tryck på Brandenburgs västliga besittningar vid Rhen. Genom freden i Saint-Germain den 29 juni 1679 tvingades Brandenburg återlämna större delen av de områden som hade erövrats från Sverige i Svenska Pommern. Sverige behöll därmed huvuddelen av sina tyska besittningar, även om vissa mindre gränsjusteringar gjordes till Brandenburgs fördel. Genom fredsavtalet ergavs Brandenburg en landremsa i Hinterpommern öster om Oder med orterna Kammin, Greifenhagen och Bahn, med undantag för orterna Damm och Gollnow. +Därmed försvagades Danmarks läge, eftersom landet stod kvar nästan ensamt mot Sverige. Karl XI försökte få till stånd direkta fredsförhandlingar med Danmark i Lund, men förhandlingarna präglades av misstro och hårda krav från båda sidor. Danmark ville behålla strategiska positioner och begränsa Sveriges militära möjligheter i Skåne, medan Sverige krävde full restitution för både sig självt och hertig Kristian Albrekt av Holstein-Gottorp. Frankrike kom att spela en avgörande roll även i fredsslutet med Danmark. Genom militärt och diplomatiskt tryck mot Danmark kunde Ludvig XIV tvinga fram en uppgörelse på villkor som i huvudsak gynnade Sverige. Den 2 september 1679 slöts freden i Fontainebleau, som innebar att Danmark skulle återlämna de områden som hade erövrats under kriget. Bestämmelserna från Westfaliska freden, freden i Roskilde och freden i Köpenhamn skulle fortsatt gälla, och även Sverige skulle återlämna sina erövringar. +Förhandlingarna fortsatte därefter i Lund, där villkoren bekräftades och kompletterades med försök att förbättra relationerna mellan Sverige och Danmark. Freden i Lund den 26 september 1679 blev därför i hög grad en bekräftelse på de villkor som redan hade fastställts genom franskt inflytande, snarare än ett resultat av helt fria dansk-svenska förhandlingar. Fredsslutet innebar att kriget avslutades i status quo ante bellum, utan större territoriella förändringar mellan Sverige och Danmark. +Fredsslutet innebar att Danmark inte lyckades återvinna sina förlorade provinser i Skåneland, trots de stora framgångarna i krigets inledning. Sverige behöll även huvuddelen av sina tyska besittningar, men kriget hade skadat bilden av landet som en militärt oövervinnerlig stormakt. Danmark hade under kriget erövrat flera viktiga områden och fästen, däribland Gotland, Landskrona, Helsingborg, Marstrand, Rügen och Wismar, men fick inte behålla någon av dessa erövringar. Krigets utgång och fredens villkor blev därmed en stor besvikelse för Danmark; en präst i Köpenhamn kallade det för ”en skiden og skammelig fred”. +Kriget lämnade efter sig omfattande mänskliga och materiella skador, särskilt i Skånelandskapen, Bohuslän och Öland. I de drabbade områdena återstod brända gårdar, skadade städer, förstörd sjöfart och en befolkning som under flera år hade pressats av truppförflyttningar, försörjningskrav, inkvarteringar och återkommande våld. Många som hade flytt eller samarbetat med den danska sidan hamnade efter freden i en osäker situation, medan den svenska staten skärpte kontrollen över tidigare oroliga områden och motverkade fortsatt snapphaneverksamhet och danskvänliga kontakter. +Som ett led i fredsprocessen fullföljdes äktenskapet mellan Karl XI och Kristian V:s syster, prinsessan Ulrika Eleonora. Bröllopet genomfördes på Skottorps slott den 6 maj 1680 och fick politisk betydelse som en symbolisk bekräftelse på att kriget var avslutat. Samtidigt slöts ett svensk-danskt försvarsförbund som syftade till att minska motsättningarna mellan rikena. Misstron mellan Sverige och Danmark bestod dock, särskilt kring Holstein-Gottorp och maktställningen i Östersjöområdet. +Efter kriget genomförde Karl XI omfattande inrikespolitiska reformer. Erfarenheterna från skånska kriget bidrog till att stärka kungamakten på bekostnad av riksrådet och högadeln. Vid riksdagen 1680 lades grunden för det karolinska enväldet, samtidigt som reduktionen ökade kronans inkomster och bröt stora delar av högadelns ekonomiska och politiska makt. Kriget blottlade också svagheter i Sveriges militära organisation. Den äldre ordningen med utskrivningar och värvade trupper hade visat sig otillräcklig när riket tvingades försvara både det svenska kärnlandet och utspridda besittningar. En central del av reformerna blev därför det yngre indelningsverket, där bönder i regel befriades från utskrivning mot att de genom rotar eller rusthåll svarade för soldaters och ryttares underhåll. Systemet skulle ge Sverige en mer permanent och snabbare mobiliserbar krigsmakt. Även flottan reformerades. Karl XI lät bygga upp flottan på nytt och anlägga Karlskrona som ny flottbas 1680, närmare de viktigaste operationsområdena. Samtidigt utvecklades båtsmanshållet för att ge flottan en fastare bemanning. +I de tidigare danska landskapen bidrog krigserfarenheterna till en starkare svensk integrering, i synnerhet i Skåne. Processen inriktades särskilt på kyrkan, språket, undervisningen och den lokala förvaltningen, samtidigt som myndigheterna övervakade tidigare snapphaneområden och försökte hindra olovliga kontakter med Danmark. Svensk lag, kyrklig ordning och förvaltning infördes successivt, och Skånelandskapen knöts därigenom närmare den svenska staten. Reformerna gjorde den svenska krigsmakten mer stabil och bättre förberedd än före skånska kriget, men de byggde fortfarande på lokala resurser och hade begränsad flexibilitet vid större kriser. Trots detta kom indelningsverket, den nya flottbasen och den karolinska taktiken att prägla Sveriges militära organisation under Karl XI:s senare regeringstid och vid inledningen av stora nordiska kriget (1700–1721). För Karl XI blev skånska kriget en avgörande erfarenhet, och Danmark förblev i hans ögon Sveriges huvudfiende. +Carlson, Fredrik Ferdinand (1856) (på engelska). Sveriges historia under konungarne af Pfalziska huset − Andra delen − Carl XI. Stockholm: Norstedt. Libris 392240 +Essen, Michael Fredholm von (2019) (på engelska). Charles XI's War: The Scanian War Between Sweden and Denmark, 1675-1679. Warwick: Helion & Company. Libris p0w3sbgdmz429xw5. ISBN 978-1-911628-00-2 +Florén, Anders; Dahlgren, Stellan; Lindegren, Jan (1992). Kungar och krigare, tre essäer om Karl X Gustav, Karl XI och Karl XII. Stockholm: Atlantis. Libris 8374783. ISBN 91-7486-992-2 +Lindbergh, Katarina Harrison (11 mars 2026). ”Den brutala grannfejden”. Populär Historia. Arkiverad från originalet den 12 april 2026. https://web.archive.org/web/20260412132916/https://popularhistoria.se/krig/svenska-krig/den-brutala-grannfejden. Läst 12 april 2026. +Lindqvist, Herman (1995). Historien om Sverige – Storhet och fall. Stockholm: Norstedt. Libris 1502998. ISBN 91-1-951282-1 +Lunde, Henrik O. (2014) (på engelska). A Warrior Dynasty: The Rise and Decline of Sweden as a Military Superpower. Stockholm: Norstedt. ISBN 978-1514632680 +Starbäck, Carl Georg; Bäckström, Per Olof (1886). Berättelser ur svenska historien: Sjette bandet – Carl X Gustaf. Carl XI. Stockholm: F. & G. Beijers förlag. Libris 8225055. https://runeberg.org/sverhist/6/ +Wahlöö, Claes; Larsson, Göran (1998). Slaget vid Lund: ett mord och icke ett fältslag. Illustrerad historia, 99-2734324-8. Lund: Historiska media. Libris 7776573. ISBN 91-88930-38-6 +Wimarson, Nils (1912). Sveriges krig i Tyskland 1675-1679. Lund: Gleerups förlag. Libris 1234768 +Wikimedia Commons har media som rör Skånska kriget. +Sarah Margaret Fuller Ossoli, känd som Margaret Fuller, född 23 maj 1810 i Cambridge i Massachusetts, död 19 juli 1850 i ett skeppsbrott utanför Fire Island i Suffolk County i New York, var en amerikansk författare, lärare, journalist, kritiker och feminist med anknytning till den amerikanska transcendentala rörelsen. Hon var den första kvinnliga bokrecensenten på heltid inom journalistiken. Hennes bok Woman in the Nineteenth Century betraktas som det första större feministiska verket i USA. +Hon föddes i Cambridge, Massachusetts och hon gavs tidigt en omfattande undervisning av sin far Timothy Fuller. Senare fick hon en mer formell utbildning och blev lärare, innan hon 1839 började leda vad hon kallade "konversationer", det vill säga diskussioner bland kvinnor som syftade till att kompensera för deras bristande tillgång på högre utbildning. Hon blev 1840 den första redaktören för den transcendentala publikationen The Dial innan hon ingick i redaktionen för New York Tribune under Horace Greeley 1844. Vid den tid då hon var i 30-årsåldern hade Fuller fått rykte om sig som den mest belästa personen i New England och blev den första kvinnan som tilläts använda biblioteket i Harvard College. Hennes betydelsefulla verk Woman in the Nineteenth Century publicerades 1845. Ett år senare skickades hon till Europa för Tribune som dess första kvinnliga korrespondent. Hon blev snart inblandad i revolutionen i Italien och allierade sig med Giuseppe Mazzini. Hon mötte också Giovanni Ossoli, som hon fick ett barn med. Alla tre dog i ett skeppsbrott utanför Fire Island i New York när de reste tillbaka till USA 1850; Fullers lik återfanns aldrig. Postumt utgavs hennes Memoirs (två band, 1851) och Julia Ward Howe utgav 1903 hennes Love Letters. +Fuller var en förespråkare för kvinnors rättigheter och i synnerhet kvinnors utbildning och rätten till anställning. Hon uppmuntrade också många andra reformer i samhället, däribland fängelsereformer och frigörandet av slavar i USA. Många andra förespråkare för kvinnors rättigheter och feminism, däribland Susan B. Anthony, nämner Fuller som inspirationskälla. Få av hennes samtida gav dock samma stöd, däribland hennes tidigare vän Harriet Martineau som sade att Fuller var en talare snarare än en aktivist. Kort efter Fullers död bleknade hennes betydelse. De redaktörer som beredde hennes brev för publicering trodde att hennes berömmelse skulle bli kortlivad och brydde sig inte om korrekthet och censurerade eller ändrade stora delar av hennes texter före publicering. +Sarah Margaret Fuller föddes den 23 maj 1810, i Cambridgeport i Massachusetts, som första barn till Timothy Fuller och Margaret Crane Fuller. Hon fick sitt namn efter sin farmor och mor, dock slutade hon använda sig av namnet "Sarah" vid nio års ålder och krävde att kallas "Margaret". Huset där hon föddes finns kvar än idag. Hennes far lärde Fuller att läsa och skriva vid tre och ett halvt års ålder, kort efter att parets andra dotter, Julia Adelaide, hade dött vid fjorton månaders ålder. Han erbjöd henne en utbildning som var lika rigorös som någon samtida pojkes och förbjöd henne att läsa den typiska kvinnliga repertoaren såsom etikettsböcker och sentimentala romaner. Han tog också med latin i undervisningen kort efter att parets son, Eugene, föddes i maj 1815 och snart översatte hon enkla stycken från Vergilius. Under dagarna tillbringade den unga Margaret tid med sin mor, som lärde henne hushållssysslor och sömnad. 1817 föddes hennes bror William Henry Fuller och hennes far valdes in i representanthuset i USA:s kongress. Under de kommande åtta åren skulle han tillbringa fyra till sex månader om året i Washington, D.C. Vid 10 års ålder skrev Fuller en kryptisk lapp som hennes far bevarade: "På den 23 maj 1810 föddes en som var dömd till sorg och smärta och liksom andra att utstå olyckor." +1824 skickades Fuller till skolan för unga damer i Groton på inrådan av släktingar, fastän hon gjorde motstånd mot idén till en början. Medan hon var i Groton ställde Timothy Fuller inte upp för omval utan valde istället att hjälpa John Quincy Adams med hans presidentvalskampanj 1824. Han hoppades att Adams skulle gengälda tjänsten med någon sorts statlig utnämning. Den 17 juni 1825 närvarade Fuller vid ceremonin då hjälten från det amerikanska frihetskriget, Gilbert du Motier, markis av Lafayette, lade hörnstenen till Bunker Hill-monumentet 50 år efter slaget. Fuller lämnade skolan i Groton och återvände hem vid 16 års ålder efter att ha studerat där i två år. Hemma studerade hon de klassiska ämnena och övade sig på flera moderna språk och olika exempel på världslitteraturen. Vid denna tid insåg Fuller att hon inte passade in med andra unga kvinnor i hennes ålder. Hon skrev: "Jag har känt att jag inte föddes till det vanliga kvinnliga ödet." Eliza Farrar, hustru till Harvardprofessorn John Farrar och författare till The Young Lady's Friend (1836), försökte öva henne i kvinnlig etikett tills hon var 20 år, men hon nådde aldrig fullständig framgång. +Fuller var en flitig läsare och när hon var i trettioårsåldern hade hon fått rykte om sig som den mest belästa personen i New England. Hon använde sin kunskap till att ge privatlektioner baserade på Elizabeth Palmer Peabodys undervisningsstil. Fuller hoppades kunna försörja sig på journalistik och översättande. Hennes första publicerade verk, ett svar till historikern George Bancroft, trycktes i november 1834 i North American Review. När hon var 23 gick hennes fars advokatpraktik omkull och han flyttade med familjen till en gård i Groton. Den 20 februari 1835 bad både Frederick Henry Hedge och James Freeman Clarke om skriftliga bidrag från henne för att publicera i sina respektive tidskrifter. Clarke hjälpte henne att publicera sin första litteraturrecension i Western Messenger i juni: kritik av nyligen utgivna biografier om George Crabbe och Hannah More. På hösten det året drabbades hon av en förfärlig migrän med feber som varade i nio dagar. Fuller skulle ofta plågas av huvudvärk under hela sitt liv. Medan hon fortfarande återhämtade sig dog hennes far av kolera den 2 oktober 1835. Hon påverkades djupt av hans död: "Bilden av min far följer mig ständigt" skrev hon senare. Hon lovade att träda in som familjens överhuvud och ta hand om sin mor och sina yngre syskon. Hennes far hade inte efterlämnat något testamente. Två av hennes farbröder fick kontroll över hans egendom och finanser (som senare värderades till $18 098,15) och tvingade familjen att förlita sig på dem för sin försörjning. Förödmjukad av det sätt som hennes farbröder behandlade familjen skrev Fuller att hon beklagade att hon tillhörde "det mjukare könet och aldrig mer än nu". +Omkring denna tid hade Fuller hoppats att färdigställa en biografi om Johann Wolfgang von Goethe, men kände att hon bara kunde arbeta på den om hon reste till Europa. Hennes fars död och hennes plötsliga ansvarsställning för sin familj fick henne att överge denna idé. 1836 blev Fuller anställd som lärare vid Bronson Alcott's Temple School i Boston, där hon stannade i ett år. Sedan tackade hon ja till en inbjudan om att undervisa vid Greene Street School i Providence i Rhode Island i april 1837 för en lön på $1 000 om året. Hennes familj sålde gården i Groton och Fuller flyttade med dem till Jamaica Plain i Massachusetts. Den 6 november 1839 höll Fuller den första av sina "konversationer", det vill säga diskussioner bland lokala kvinnor som träffades i Peabodys hem i Boston. Fuller tänkte sig att dessa möten skulle kompensera för kvinnors bristande tillgång till utbildning, med diskussioner och debatter som inriktades på ämnen som de sköna konsterna, historia, mytologi, litteratur och natur. Fuller fungerade som "konversationens kärna" och ämnade också besvara de "stora frågor" som kvinnor ställdes inför: "Vad är vi födda för att göra? Hur ska vid göra det? som så få någonsin frågar sig innan deras bästa år har förflutit". Ett antal viktiga personer i rörelsen för kvinnors rättigheter deltog i dessa "konversationer", däribland Sophia Dana Ripley, Caroline Sturgis och Maria White Lowell. +I oktober 1839 sökte Ralph Waldo Emerson en redaktör till sin transcendentala tidskrift The Dial och, efter att flera hade avböjt erbjöd han posten till Fuller, som han beskrev som "min livliga vän". Emerson hade mött Fuller första gången i Cambridge 1835. Om det mötet medgav han att "hon fick mig att skratta mer än jag ville". Följande sommar tillbringade Fuller två veckor i Emersons hem i Concord där deras vänskap växte. Fuller tackade ja till Emersons erbjudande om att redigera The Dial den 20 oktober 1839 men började inte arbeta förrän första veckan 1840, samma år som tidskriften började publiceras. Hon redigerade tidskriften under de första två åren av dess existens från 1840 till 1842, men hennes utlovade årslön på $200 utbetalades aldrig. På grund av sin roll erkändes hon snart som en av de viktigaste personerna i den transcendentala rörelsen och bjöds in till George Ripleys kollektivexperiment Brook Farm. Hon blev aldrig officiellt medlem av kollektivet, men besökte det ofta och tillbringade ofta nyårsafton där. På sommaren 1843 reste hon till Chicago, Milwaukee, Niagarafallen och Buffalo. Medan hon var där träffade hon flera indianer, däribland medlemmar av ottawastammen och ojibwastammen. Hon beskrev sina erfarenheter i en bok med titeln Summer on the Lakes, som hon skrev färdigt på sin 34-årsdag 1844. Kritikern Evert Augustus Duyckinck kallade den "den enda genuina bok jag kan tänka mig denna säsong". Hon hade använt biblioteket i Harvard University för ytterligare bakgrundsinformation om Stora sjöområdet och blev därmed den första kvinnan som tilläts använda Harvards bibliotek. +Ett av Fullers viktigaste verk, "The Great Lawsuit", skrevs i följetongsform för The Dial. Hon avsåg först att kalla verket "The Great Lawsuit: Man versus Men, Woman versus Women". När det utökades och publicerades oberoende 1845 kallades det istället Woman in the Nineteenth Century. Efter att hon färdigställt verket skrev hon till en vän: "Jag hade lagt en stor del av mitt sanna jag i det, som om, jag antar jag försvann nu, mitt fotavtryck skulle finnas kvar på jorden." Verket diskuterade den roll som kvinnor spelade i den amerikansk demokratin och Fullers åsikter om möjligheter till förbättring. Det har senare blivit ett av de främsta dokumenten i amerikansk feminism och betraktas som den första av sitt slag i USA. +Fuller lämnade The Dial 1844 delvis på grund av dålig hälsa, men också på grund av sin besvikelse över tidskriftens minskande antal prenumeranter. Hon flyttade till New York den hösten och gick med i Horace Greeleys New York Tribune som litteraturkritiker och blev därmed den första bokrecensenten på heltid inom journalistiken och var vid 1846 tidskriftens första kvinnliga redaktör. Hennes första artikel, en recension av en samling essäer av Emerson, trycktes i numret från 1 december 1844. Vid denna tid hade Tribune omkring 50 000 prenumeranter och Fuller tjänade $500 om året på sitt arbete. Förutom amerikanska böcker recenserade hon utländsk litteratur, konserter, föreläsningar och konstutställningar. Under sina första fyra år vid tidskriften publicerade hon mer än 250 kolumner, de flesta undertecknade med en asterisk (*) som signatur. I dessa kolumner diskuterade Fuller ämnen som sträckte sig från konst och litteratur till politiska och sociala frågor såsom slavarnas belägenhet och kvinnors rättigheter. Hon publicerade också poesi, dock har hennes dikter, utformade efter Emersons verk, inte samma intellektuella kraftfullhet som hennes kritik. +Omkring denna tid var hon också inblandad i ett skandal som rörde litteraturkritikerkollegan Edgar Allan Poe, som hade utfört en offentlig flört med den gifta poeten Frances Sargent Osgood. Samtidigt blev en annan poet, Elizabeth F. Ellet, förälskad i Poe och svartsjuk på Osgood och framförde att relationen mellan Poe och Osgood var mer än bara en oskyldig flört. Osgood skickade då Fuller och Anne Lynch Botta till Poes stuga å hennes vägnar för att be honom återlämna de personliga brev hon hade skickat honom. Poe blev ilsken över deras inblandning och kallade dem "busy-bodies" (det vill säga personer som lägger sig i andras privata affärer). En offentlig skandal utbröt och fortsatte tills Osgoodes frånseparerade make Samuel Stillman Osgood grep in och hotade med att stämma Ellet. +New York Tribune skickade Fuller till Europa år 1846, främst till England och Italien, som dess första kvinnliga utrikeskorrespondent. Hon reste från Boston till Liverpool i England i augusti på Cambria, ett fartyg som använde både segel och ånga till att korsa havet på tio dagar och sexton timmar. Under de följande fyra åren lämnade hon trettiosju rapporter från Europa. Hon intervjuade många framträdande författare, däribland George Sand och Thomas Carlyle, som hon upplevde som en besvikelse på grund av hans reaktionära politiska åsikter, bland andra saker. George Sand hade tidigare varit en av hennes idoler, men Fuller blev besviken när Sand beslöt att inte kandidera till Frankrikes nationalförsamling, med motiveringen att kvinnor inte var redo att rösta eller inneha politiska ämbeten. Fuller fick också ett introduktionsbrev från Cornelius Mathews för Elizabeth Barrett, men de två kvinnorna möttes inte då Barrett just hade rymt med Robert Browning. På våren 1846 i England mötte hon Giuseppe Mazzini, som hade varit i exil från Italien sedan 1837. Fuller mötte också den italienske revolutionären Giovanni Ossoli, en markis som hade gjorts arvslös av sin familj på grund av sitt stöd för Mazzini. Fuller och Ossoli flyttade ihop i Florens, troligen innan de var gifta, om de någonsin blev det. Fuller var först emot att gifta sig med honom, delvis på grund av deras olika religioner; hon var protestant och han var katolik. Emerson spekulerade att paret "gifte sig kanske i oktober, november eller december" 1847, men förklarade inte hur han kom fram till detta. Biografer har spekulerat om att paret gifte sig 4 april 1848, för att fira årsdagen av sitt första möte. Vid den tid då paret flyttade till Florens beskrevs de som man och hustru, men det är oklart om någon formell ceremoni ägde rum. Det tycks säkert att de inte var gifta när deras barn föddes. Fuller misstänkte att hon var gravid senast på nyårsdagen 1848, men höll det hemligt för Ossoli i flera veckor. Deras barn, Angelo Eugene Philip Ossoli, föddes i början av september 1848. De gav honom smeknamnet Angelino. Paret var mycket hemlighetsfulla om sitt förhållande, men efter att Angelino drabbats av en obestämd sjukdom kom de närmare varandra. Fuller berättade slutligen för sin mor om Ossoli och Angelino i augusti 1849. Brevet förklarade att hon hade tigit för att inte göra henne upprörd "men det har blivit nödvändigt, på grund av barnet, för oss att leva tillsammans offentligt och beständigt." Hennes mors svar gjorde det klart att hon kände till att inget juridiskt äktenskap hade ingåtts. Trots detta var hon lycklig för sin dotter och skrev: "Jag skickar min första kyss med min innerliga välsignelse till min dotterson." Moderna biografer är fortfarande osäkra på om Fuller och Ossoli någonsin gifte sig. +Paret stödde Giuseppe Mazzinis revolution för upprättandet av en romersk republik 1849; Ossoli deltog i kampen medan Fuller var volontär på ett stödjande sjukhus. Efter generationer då landet styrts av olika härskare, av vilka få var italienska, hade Italien lämnats utan en officiell central regering och olika bitar av landet styrdes av olika svaga regeringar, däribland en under påvedömets kontroll. När Pius IX utsågs till påve 1846 tog han små steg mot att upprätta en central italiensk demokratisk regering, men revolutionärer som Mazzini litade inte på påvens insatser. Den politiska oron var så stor att påven förklädde sig och flydde den 24 november 1848. En romersk republik med en representativ regering upprättades i februari 1849, bara för att bli krossad av en invasion från Frankrike några månader senare. Eftersom Fuller och Ossoli var på revolutionens sida blev de tvungna att fly från Italien när påven Pius IX återvände till Rom 1850 och de beslöt sig för att flytta till USA. Fuller hade för avsikt att använda sina erfarenheter till att skriva en bok om den romerska republikens historia, ett verk som hon kan ha påbörjat så tidigt som 1847, och hoppades hitta en amerikansk utgivare efter att en brittisk förläggare refuserat den. Hon ansåg att verket skulle bli hennes viktigaste och beskrev det i ett brev till sin bror Richard i mars 1849 som "något bra som kan överleva min besvärliga tillvaro." +I början av 1850 skrev Fuller till en vän: "Det har länge tyckts att i året 1850 skulle jag stå på någon viktig platå i livets uppförsbacke [...] Jag känner dock ingen markant eller viktig förändring än så länge." Samma år skrev Fuller: "Jag är absurt orolig och olika omen har samverkat till att ge mig en mörk känsla [...] Det tycks mig att min framtid på jorden snart kommer att avslutas [...] Jag har en vag förväntan om någon kris, men jag vet inte vad." Några dagar efter att hon skrev detta påbörjade Fuller, Ossoli och deras barn en fem veckors resa till USA på skeppet Elizabeth. Skeppet var ett amerikanskt handelsfartyg som var lastat med mestadels marmor från Carrara samt med en staty av John C. Calhoun skulpterad av Hiram Powers. Efter en kort fördröjning på grund av regn satte Elizabeth segel den 17 maj. På havet dog skeppets kapten, Seth Hasty, av smittkoppor. Fullers barn, Angelino, fick också sjukdomen men återhämtade sig. +Möjligen på grund av den oerfarna styrmannen, som nu tjänstgjorde som kapten, slog skeppet in i ett sandrev mindre än 100 meter från Fire Island i New York den 19 juli 1850, omkring 03.30 på morgonen. Många av de andra passagerarna och besättningsmedlemmarna lämnade skeppet. Styrmannen, herr Bangs, uppmanade Fuller och Ossoli att försöka rädda sig själva och sitt barn medan han själv hoppade överbord och hävdade senare att han trodde att Fuller hade velat lämnas att dö. På stranden anlände människor med vagnar i hopp om att dra fördel om någon last spolades iland. Ingen gjorde någon ansträngning att rädda besättningen eller passagerarna på Elizabeth, fastän de var mindre än 50 meter från stranden. Ossoli och Fuller och deras barn var några av de sista på skeppet. De flesta andra hade försökt simma iland. Slutligen kastades Ossoli överbord av en stor våg och efter att vågen passerat sade en besättningsman som bevittnade händelsen att Fuller inte var synlig. +Henry David Thoreau reste till New York, på Emersons uppmaning, för att genomsöka stranden men varken Fullers eller hennes makes lik återfanns någonsin. Endast Angelino hade spolats iland. Få av deras ägodelar hittades förutom några av barnets kläder och några brev. Fullers manuskript om romerska republikens historia gick också förlorat. Ett minnesmärke över Fuller restes på stranden på Fire Island 1901 genom Julia Ward Howes ansträngningar. En kenotaf över Fuller och Ossoli, under vilken Angelino är begravd, finns på Mount Auburn Cemetery i Cambridge i Massachusetts. Inskriften lyder till en del: :By birth a child of New England :By adoption a citizen of Rome :By genius belonging to the world +Inom en vecka efter hennes död föreslog Horace Greeley för Emerson att en biografi om Fuller, som skulle heta Margaret and Her Friends, skulle genomföras snabbt "innan intresset som har väckts av hennes tragiska bortgång har passerat". Många av hennes skrifter samlades snart ihop av hennes bror Arthur, som At Home and Abroad (1856) och Life Without and Life Within (1858). Han redigerade också en ny version av Woman in the Nineteenth Century 1855. I februari 1852 publicerades The Memoirs of Margaret Fuller Ossoli, redigerad av Emerson, James Freeman Clarke och William Henry Channing, men stora delar av verket hade censurerats eller omformulerats. Det utelämnade särskilt detaljer om hennes kärleksaffär med Ossoli och en tidigare relation med en man vid namn James Nathan. De tre redaktörerna, som trodde att allmänhetens intresse för Fuller skulle bli kortlivat och att hon inte skulle överleva som historisk person, brydde sig inte om dess korrekthet. Ändå var den en tid årtiondets mest sålda biografi och kom ut i tretton utgåvor före seklets slut. Boken fokuserade på hennes personlighet snarare än hennes verk och som en följd av detta ignorerade bokens belackare hennes status som kritiker och kritiserade istället hennes privatliv och hennes "okvinnliga" arrogans. +Fuller var en tidig företrädare för feminism och trodde särskilt på att ge utbildning åt kvinnor. När väl lika rättigheter till utbildning hade getts till kvinnorna, trodde hon, kunde kvinnorna kräva lika politiska rättigheter också. Hon förespråkade att kvinnor skulle söka vilket arbete de önskade, snarare än att sörja för tidens stereotypa "feminina" roller, såsom undervisning. Hon sade en gång: "Om du frågar mig vilken uppgift kvinnor skulle fylla, svarar jag: vilken som helst [...] låt dem vara sjökaptener om du vill. Jag tvivlar inte på att det finns kvinnor som är väl lämpade för en sådan uppgift." Hon hade stor tillit till alla kvinnor, men tvivlade på att en kvinna skulle åstadkomma ett bestående konstnärligt eller litterärt verk under hennes livstid och ogillade sin tids populära kvinnliga poeter. Fuller ville också att kvinnor skulle vara försiktiga med att gifta sig och se till att inte bli beroende av sina makar. Som hon skrev: "Jag önskar att kvinnan ska leva först för Guds skull. Då kommer hon inte att göra en ofullkomlig man till sin gud och därmed sänka sig till avgudadyrkan. Då kommer hon inte att ta det som inte förtjänar henne på grund av en känsla av svaghet och fattigdom". Senast 1832 hade hon gjort en personlig utfästelse att förbli ensamstående. Fuller ifrågasatte också att det fanns en definitiv gräns mellan manligt och kvinnligt: "Det finns ingen fullständigt maskulin man [...] ingen rent feminin" och menade att båda var närvarande i varje individ. Hon föreslog också att det inom en kvinna fanns två delar: den intellektuella sidan (som hon kallade Minerva) och den "lyriska" sidan eller "kvinnlighetssidan" (musan). Hon beundrade Emanuel Swedenborg, som ansåg att män och kvinnor delade "en änglalik ämbetstid" som hon skrev, liksom Charles Fourier, som satte "Kvinnan i fullständig jämlikhet med Mannen". Till skillnad från flera samtida kvinnliga författare, däribland "Mrs. Sigourney" och "Mrs. Stowe", omtalades hon förtroligt på ett mindre formellt sätt som "Margaret". +Fuller förespråkade även reformer på alla nivåer i samhället, inklusive fängelser. I oktober 1844 besökte hon Sing Sing och intervjuade de kvinnliga fångarna. Hon övernattade till och med där. Sing Sing höll på att utveckla ett mer humant system för sina kvinnliga interner, av vilka många var prostituerade. Fuller engagerade sig också för de hemlösa och de som levde i svår fattigdom särskilt i New York. Hon erkände också att, fastän hon hade uppfostrats till att tro "att indianen envist vägrade bli civiliserad", hennes resor i västra USA hade fått henne att inse att den vita mannen behandlade ursprungsbefolkningen orättvist. Hon såg indianerna som en viktig del av Amerikas arv. Hon stödde också afroamerikanernas rättigheter, talade om "denna slaveriets cancer" och föreslog att de som intresserade sig för abolitionism skulle följa samma tänkesätt när det gällde kvinnors rättigheter: "Såsom Negerns vän antar att en människa inte med rätt kan hålla en annan i slaveri, så borde Kvinnans Vän anta att Mannen inte med rätt kan lägga ens välmenande inskränkningar på Kvinnan." Hon framförde att de som talade mot slavarnas frigörelse var lika dem som inte stödde Italiens frigörelse. +Fuller instämde i transcendentalisternas omsorg om individens psykologiska välfärd, fastän hon aldrig kände sig hemma med att etiketteras som transcendentalist. Ändå skrev hon att om att kallas transcendentalist betyder "att jag har ett aktivt sinne som ofta ägnar sig åt stora ämnen hoppas jag att det är så." Hon kritiserade dock personer som Emerson för att inrikta sig för mycket på individuell bättring och inte tillräckligt på samhällsreform. Liksom andra medlemmar av den så kallade Transcendental Club gjorde hon uppror mot det förgångna och trodde på möjligheten till förändring. Till skillnad från andra i rörelsen var hennes uppror dock inte grundat på religion. +Fuller var särskilt känd under sin livstid för sin personlighet och i synnerhet för att vara överdrivet självsäker och ha ett dåligt humör. Denna personlighet var inspirationen till romanfiguren Hester Prynne i Nathaniel Hawthornes roman The Scarlet Letter, då särskilt hennes radikala tänkande om "hela kvinnosläktet". Hon kan också ha varit utgångspunkten för figuren Zenobia i ett annat av Hawthornes verk, The Blithedale Romance. Hawthorne och hans dåvarande fästmö Sophia Hawthorne hade mött Fuller för första gången i oktober 1839. +Hon var även en inspirationskälla för poeten Walt Whitman, som trodde på hennes uppmaning till skapandet av en ny nationell identitet och en sant amerikansk litteratur. Elizabeth Barrett Browning var också en stor beundrare, men trodde att Fullers okonventionella synsätt inte var uppskattade i USA och att hon därför hade det bättre som död. Hon sade också att Fullers historia om den romerska republiken skulle ha blivit hennes största verk: "Det verk som hon höll på att färdigställa om Italien skulle troligen mer ha motsvarat hennes förmåga än någonting som tidigare framställts av hennes penna (då hennes tidigare skrifter var märkligt underlägsna de intryck hon gav i samtal)." En essäsamling från 1860, Historical Pictures Retouched, av Caroline Healey Dall, kallade Fullers Woman in the Nineteenth Century "utan tvivel den mest briljanta, kompletta och lärda redogörelse som någonsin gjorts om ämnet". Trots sina personliga problem med Fuller skrev den typiskt stränga litteraturkritikern Edgar Allan Poe om boken som "en bok som få kvinnor i landet kunde ha skrivit, och ingen kvinna i landet skulle ha publicerat, med undantag för Miss Fuller", och noterade dess "oberoende" och "outspädda radikalism". Thoreau hade också höga tankar om boken och föreslog att dess styrka delvis kom från Fullers konversationsförmåga. Han kallade den "rikt oförberett skrivande, talande med pennan i handen" +En annan beundrare av Fuller var Susan B. Anthony, en pionjär för kvinnors rättigheter, som skrev att Fuller "ägde större inflytande på amerikanska kvinnors tänkande än någon kvinna före hennes tid". Fullers verk kan delvis ha inspirerat kvinnorättskonferensen i Seneca Falls 1848. Anthony skrev tillsammans med Elizabeth Cady Stanton och Matilda Joslyn Gage i sin History of Woman Suffrage att Fuller "var föregångaren till agitationen för kvinnors rättigheter". Moderna forskare har framkastat att Woman in the Nineteenth Century var det första större verket om kvinnors rättigheter sedan Mary Wollstonecrafts Till försvar för kvinnans rättigheter (1792), men en tidigare jämförelse mellan de två kvinnorna kom från George Eliot 1855. Det är oklart om Fuller var bekant med Wollstonecrafts verk. Under hennes barndom hindrade hennes far henne från att läsa dem. 1995 installerades Fuller i National Women's Hall of Fame. +Fuller saknade dock inte kritiker. En tidigare vän, den engelska författaren Harriet Martineau, var en av hennes hårdaste belackare efter Fullers död. Martineau sade att Fuller var en talare snarare än en aktivist, att hon hade "ytlig högfärd" och ofta "såg ned på personer som handlade istället för att tala fint [...] och föraktade dem som, liksom jag själv, inte kunde anta hennes värderingsskala" Den inflytelserike redaktören Rufus Wilmot Griswold, som tyckte att hon gick emot hans föreställning om kvinnlig blygsamhet, talade om Woman in the Nineteenth Century som "ett vältaligt uttryck för hennes missnöje med att ha skapats som kvinna". New York-författaren Charles Frederick Briggs sade att hon "slösade med sina läsares tid", särskilt eftersom hon var en ogift kvinna och därför inte kunde "verkligt företräda den kvinnliga karaktären." Den engelske författaren och kritikern Matthew Arnold hånade Fullers konversationer också och sade "Min G-d, vilket strunt pratade hon och de andra kvinnliga hundarna i Boston om grekisk mytologi!" Nathaniel Hawthorne, som tidigare hade varit anhängare till Fuller, var kritisk mot henne efter att Woman of the Nineteenth Century hade utgivits: +Fuller hade förargat poet- och kritikerkollegan James Russell Lowell när hon recenserade hans verk. Hon kallade honom "absolut i avsaknad av poesins sanna anda och ton [...] hans vers är stereotyp, hans tänkande sonderar inga djup, och eftervärlden kommer inte att minnas honom." Som svar hämnades Lowell i sin satiriska A Fable for Critics, första gången publicerad i oktober 1848. Till en början övervägde han att utesluta henne helt och hållet, men slutligen gav han henne vad som kallades den "mest fullständigt negativa karakteriseringen" i verket. Lowell, som gav henne namnet Miranda, skrev att hon stal gamla idéer och presenterade dem som sina egna och bara var bara genuin i sin illvilja. +Kort efter Fullers död avtynade hennes betydelse. Som biografen Abby Slater skrev: "Margaret hade flyttats ned från position som betydelsefull i sig själv till en där hennes enda betydelse var genom sin umgängeskrets". Åratal senare skrev Hawthornes son Julian Hawthorne: "Flertalet läsare kommer, tror jag, inte vara otröstliga över att stackars Margaret Fuller slutligen har intagit sin plats bland de otaliga andra förskräckliga bluffar som fyller den mänskliga pretentionens och misslyckandets limbo." På 1980-talet skrev den amerikanska författaren Elizabeth Hardwick essän "The Genius of Margaret Fuller" (1986). Hon jämförde sin egen flytt från Boston till New York med Fullers och sade att Boston inte var ett bra ställe för intellektuella, trots antagandet att det var det skulle vara detta. +Summer on the Lakes (1844) +Woman in the Nineteenth Century (1845) +Papers on Literature and Art (1846) +Postumt utgivna +Memoirs of Margaret Fuller Ossoli (1852) +At Home and Abroad (1856) +Life Without and Life Within (1858) +Den här artikeln är helt eller delvis baserad på material från engelskspråkiga Wikipedia, Margaret Fuller, 5 januari 2009. +Arvin, Newton (1963) (på engelska). Longfellow: His Life and Work. Boston: Little, Brown and Company. ISBN 0837195055. https://archive.org/details/longfellowhislif00arvi +Bayless, Joy (1943) (på engelska). Rufus Wilmot Griswold: Poe's Literary Executor. Nashville: Vanderbilt University Press +Blanchard, Paula (1987) (på engelska). Margaret Fuller: From Transcendentalism to Revolution. Reading, Massachusetts: Addison-Wesley Publishing Company. ISBN 020110458X. https://archive.org/details/margaretfullerfr0000blan +Brooks, Van Wyck (1952) (på engelska). The Flowering of New England. New York: E.P. Dutton and Company, Inc. ISBN 0395305225. https://archive.org/details/floweringofnewen0000broo_v7c2 +Callow, Philip (1992) (på engelska). From Noon to Starry Night: A Life of Walt Whitman. Chicago: Ivan R. Dee. ISBN 0929587952. https://archive.org/details/bwb_P7-CZE-669 +Cheever, Susan (2006) (på engelska). American Bloomsbury: Louisa May Alcott, Ralph Waldo Emerson, Margaret Fuller, Nathaniel Hawthorne, and Henry David Thoreau; Their Lives, Their Loves, Their Work. Detroit: Thorndike Press. ISBN 078629521X +Deiss, Joseph Jay (1969) (på engelska). The Roman Years of Margaret Fuller. New York: Thomas Y. Crowell Company. ISBN 0690010176. https://archive.org/details/romanyearsofmarg00deis +Dickenson, Donna (1993) (på engelska). Margaret Fuller: Writing a Woman's Life. New York: St. Martin's Press. ISBN 0312091451. https://archive.org/details/margaretfullerwr0000dick +Douglas, Ann (1977) (på engelska). The Feminization of American Culture. New York: Alfred A. Knopf. ISBN 0394405323. https://archive.org/details/feminizationofam0000doug_o5v8 +Duberman, Martin (1966) (på engelska). James Russell Lowell. Boston: Houghton Mifflin Company. https://archive.org/details/jamesrusselllowe00dube +Ehrlich, Eugene; Carruth, Gorton (1982) (på engelska). The Oxford Illustrated Literary Guide to the United States. New York: Oxford University Press. ISBN 0195031865. https://archive.org/details/oxfordillustrate00euge +Gura, Philip F. (2007) (på engelska). American Transcendentalism: A History. New York: Hill & Wang. ISBN 0809034778. https://archive.org/details/americantranscen00gura +James, Laurie (1988) (på engelska). Why Margaret Fuller Ossoli Is Forgotten. New York: Golden Heritage Press. ISBN 0944382010. https://archive.org/details/whymargaretfulle0000jame +Kane, Paul (1995) (på engelska). Poetry of the American Renaissance. New York: George Braziller. ISBN 0807613983. https://archive.org/details/poetryofamerican00paul +Marshall, Megan (2005) (på engelska). The Peabody Sisters: Three Women Who Ignited American Romanticism. Boston: Houghton Mifflin Company. ISBN 9780618711697. https://archive.org/details/peabodysisters00mega +McFarland, Philip (2004) (på engelska). Hawthorne in Concord. New York: Grove Press. ISBN 0802117767. https://archive.org/details/hawthorneinconco00mcfa +Meyers, Jeffrey (1992) (på engelska). Edgar Allan Poe: His Life and Legacy. New York: Cooper Square Press. ISBN 0815410387. https://archive.org/details/edgarallanpoehis0000meye_u7k6 +Miller, Edwin Haviland (1991) (på engelska). Salem Is My Dwelling Place: A Life of Nathaniel Hawthorne. Iowa City: University of Iowa Press. ISBN 0877453322. https://archive.org/details/salemismydwellin00mill +Moss, Sidney P. (1969) (på engelska). Poe's Literary Battles: The Critic in the Context of His Literary Milieu. Southern Illinois University Press. ISBN 0809303515 +Nelson, Randy F. (1981) (på engelska). The Almanac of American Letters. Los Altos, Kalifornien: William Kaufmann, Inc. ISBN 086576008X. https://archive.org/details/almanacofameric00nels +Rose, Anne C. (1981) (på engelska). Transcendentalism as a Social Movement, 1830–1850. New Haven, Connecticut: Yale University Press. ISBN 0300025874. https://archive.org/details/transcendentalis0000rose_p0o0 +Silverman, Kenneth (1991) (på engelska). Edgar A. Poe: Mournful and Never-ending Remembrance. New York: Harper Perennial. ISBN 0060923318. https://archive.org/details/edgarpoe00kenn +Slater, Abby (1978) (på engelska). In Search of Margaret Fuller. New York: Delacorte Press. ISBN 0440039444. https://archive.org/details/insearchofmargar0000slat +Stanton, Elizabeth Cady; Anthony, Susan B.; Gage, Matilda Joslyn (1881) (på engelska). History of Woman Suffrage. "1". New York: Fowler and Wells. ISBN 1274999456 +von Mehren, Joan (1994) (på engelska). Minerva and the Muse: A Life of Margaret Fuller. Amherst: University of Massachusetts Press. ISBN 1558490159. https://archive.org/details/minervamuse00joan +Watts, Emily Stipes (1977) (på engelska). The poetry of American women from 1632 to 1945. Austin: University of Texas-Pan American. ISBN 0292764359. https://archive.org/details/poetryofamerican00emil +Wilson, Susan (2000) (på engelska). Literary Trail of Greater Boston. Boston: Houghton Mifflin Company. ISBN 0618050132. https://archive.org/details/literarytrailofg00wils +Wineapple, Brenda (2001). Larry J. Reynolds. red (på engelska). "Nathaniel Hawthorne 1804-1864: A Brief Biography" i A Historical Guide to Nathaniel Hawthorne. New York: Oxford University Press. ISBN 0195124146 +Margaret Fuller Society +Margaret Fuller Neighborhood House +Margaret Fuller: Bicentennial 2010 +Margaret Fuller Family Papers at Houghton Library, Harvard University +George Brinton McClellan, född 3 december 1826 i Philadelphia, död 29 oktober 1885 i Orange, var en amerikansk militär och politiker. McClellan var generalmajor i unionsarmén (Nordstaterna) under amerikanska inbördeskriget, Demokraternas presidentkandidat i presidentvalet 1864 mot Abraham Lincoln och guvernör i New Jersey 1878–1881. +Under det amerikanska inbördeskriget organiserade McClellan Potomac-armén och tjänstgjorde från november 1861 till mars 1862 som högste militäre befälhavare för unionsarmén. McClellan var en skicklig organisatör och spelade en viktig roll under krigets inledning genom att sätta upp en vältränad och välorganiserad armé för unionen, den så kallade Potomacarmén. På slagfältet var han dock passiv och avvaktande. Han övervärderade ofta motståndarnas styrkor och tvekade att använda sina reserver i lägen som kunde givit avgörande segrar. Som överbefälhavare var han ofta hånfull, arrogant och ohörsam mot presidenten. +General McClellan förlorade med tiden president Abraham Lincolns förtroende. Sedan han fråntagits sitt befäl blev han Demokraternas kandidat i presidentvalet 1864 och förlorade mot Lincoln. Hans parti hade ett antikrigsprogram med löfte om att avsluta kriget och förhandla med Konfederationen. +De flesta specialister på perioden ser i dag McClellan som en dålig general på slagfältet. Det finns dock även en minoritetsuppfattning att han egentligen var en duglig befälhavare, men att hans rykte solkades ned av anhängare till Lincoln, vilka behövde en syndabock för unionens misslyckanden. +Under amerikanska inbördeskriget organiserade McClellan Potomac-armén och tjänstgjorde en kort tid (november 1861 till mars 1862) som högste militäre befälhavare för unionsarmén. I krigets inledning spelade han en viktig roll för att sätta upp en vältränad och organiserad arme för unionen. McClellan var noggrann när han planerade och förberedde sig, men dessa personlighetsdrag kan ha begränsat hans förmåga att utmana aggressiva motståndare på ett högrörligt slagfält. Han övervärderade oftast motståndarnas styrkor och tvekade att använda större arméenheter, han lämnade ofta stora delar av sin armé sysslolös vid avgörande tillfällen. +McClellans insats i unionens militära offensiv mot Virginia Peninsula 1862, kallad Peninsula campaign, slutade i fiasko. I inledningen mötte han den lika försiktiga Joseph E. Johnston, som sårades och ersattes av general Robert E. Lee. Trots att denne hade en mindre armé tvingades McClellan till reträtt och hans mål att inta konfederationens huvudstad Richmond misslyckades. Hans insatser vid det blodiga slaget vid Antietam stoppade Lees invasion i Maryland, men tillät Lees numerärt underlägsna styrkor att undkomma. Resultatet blev att McClellans ledarskapsförmåga i strid blev ifrågasatt av president Abraham Lincoln, som efter en tid fråntog McClellan befälet. Först från uppdraget som högste befälhavare, sedan från Potomac-armén. Lincolns citat blev välkänt: ”Om general McClellan inte vill använda armén, skulle jag vilja låna den ett tag.” Trots detta var McClellan den mest populäre befälhavaren bland manskapet, som kände att han hade deras välgång som sin högsta prioritet. +General McClellan misslyckades också med att behålla Lincolns förtroende, och han var hånfull och ohörsam mot presidenten som överbefälhavare. Sedan han fråntagits sitt befäl, blev han Demokraternas kandidat i presidentvalet 1864 och förlorade mot Lincoln. Hans parti hade ett antikrigsprogram, med löfte om att avsluta kriget och förhandla med Konfederationen. +Han var guvernör i New Jersey från 1878 till 1881. Han blev senare författare och försvarade det han gjorde under inbördeskriget. +Trots att den stora majoriteten av nutida auktoriteter ser McClellan som en dålig general på slagfältet, finns det en liten skara historiker som menar att han egentligen var en mycket duglig befälhavare, men att hans rykte solkades ned av anhängare till Lincoln, som behövde en syndabock för unionens inledande misslyckanden. Det är därför inte lätt att värdera hans eftermäle. Efter kriget ombads Ulysses S. Grant att värdera McClellan som general. Han svarade: "McClellan är, för mig, ett av krigets mysterier." +McClellan föddes i Philadelphia. Han var son till en framstående oftalmolog, läkaren George McClellan, som grundade Jefferson Medical College. Hans mor var Elizabeth Steinmetz Brinton McClellan tillhörde en framstående familj i Pennsylvania. Paret fick fem barn, Frederica, John, George, Arthur och Mary. Deras farfar var generalen från amerikanska revolutionen Samuel McClellan från Woodstock, Connecticut. George B. McClellan började vid University of Pennsylvania 1840 vid 13 års ålder. Från början hade han tänkt sig att läsa juridik, men ändrade sig efter två år och ville bli militär. Med hjälp av ett brev från sin far till president John Tyler kom han in på militärakademin 1842, med ett undantag från den normala åldersgränsen på 16 år. +På United States Military Academy i West Point var han en energisk och ambitiös kadett, som var mycket intresserad av den undervisning som Dennis Hart Mahan gav och av Antoine-Henri Jominis strategiska principer. Hans närmaste vänner var aristokratiska sydstatare som James Stuart, Dabney Maury, Cadmus M. Wilcox och A.P. Hill. Dessa vänskapsband gav McClellan det som han såg som en insikt i sydstaternas sinnelag, en förståelse för de politiska och militära betydelserna av de skillnader inom USA som ledde till inbördeskriget. Han tog examen 1846, näst bäst i sin årskull av 59 kadetter, och förlorade förstaplatsen (till Charles Seaforth Stewart) bara för att han var dålig på att rita. Han fick tjänst i amerikanska arméns ingenjörtrupper. +McClellans första tjänst var vid ett ingenjörskompani vid West Point, men han fick snabbt order om att segla till Mexikanska kriget. Han anlände nära Rio Grandes mynning i oktober 1846, väl förberedd för strid med en dubbelpipig hagelbössa, två pistoler, en sabel, ett litet svärd och en Bowiekniv. Han klagade på att han hade kommit fram för sent för att delta i den amerikanska segern vid Monterrey i september. Under en tillfällig vapenvila där general Zachary Taylors styrkor väntade på att få delta i strid, fick McClellan dysenteri och malaria. Han blev därför kvar i sjukhus i närmare en månad. Malarian skulle komma tillbaka senare, han kallade den sin "mexikanska sjukdom" Han tjänstgjorde modigt som ingenjörsofficer under kriget, under fientlig eld vid flera tillfällen, och befordrades till premiärlöjtnant för slagen vid Contreras och Churubusco och till kapten för slaget vid Chapultepec, Han utförde rekognosering för generallöjtnant Winfield Scott, en nära vän till McClellans far. +McClellans erfarenheter under kriget gjorde att han utvecklade en del inställningar som påverkade hans senare militära och politiska karriärer. Han lärde sig att föredra att angripa flankerna framför frontalangrepp (vilket Scott använde vid slaget vid Cerro Gordo) och värdet av belägringar (Veracruz). Han bevittnade hur Scott framgångsrikt balanserade politiska och militära ting, och hans goda relationer med civilbefolkningen där han invaderade, med strikt disciplin hos sina soldater för att minimera skadorna på civil egendom. Han utvecklade också avsmak för frivilliga soldater och officerare, särskilt för politiker som inte brydde sig det minsta om disciplin eller träning. +McClellan återvände till West Point för att ta befäl över sitt ingenjörkompani, vilket var anslutet till militärhögskolan i syfte att träna kadetter i ingenjörsarbeten. Han tyckte inte om den tråkiga, fredstida tjänsten i garnison, trots att han fann stort nöje i det sociala livet. I juni 1851 fick han förflyttning till Fort Delaware, ett tegelbygge under konstruktion på en ö i Delawarefloden, 64 km nedströms Philadelphia. I mars 1852 fick han order att inställa sig hos kapten Randolph B. Marcy vid Fort Smith, Arkansas, för att vara andrebefäl vid en expedition för att upptäcka källorna till Red River. I juni nådde expeditionen källan till den norra förgreningen och Marcy gav en liten biflod namnet McClellan's Creek. Vid deras återkomst till civilisationen den 28 juli, blev de förbluffade över att upptäcka att de hade blivit uppgivna såsom döda. En sensationell historia hade nått pressen om att expeditionen hade hamnat i bakhåll av 2 000 comancher och dödad till siste man. +På hösten 1852 publicerade McClellan en manual om bajonettaktik, som han hade översatt från sitt franska original. Han fick också i uppdrag att undersöka floder och hamnar i Texas. År 1853 deltog han i undersökningar inför byggandet av Pacific Railroad, på befallning av USA:s krigsminister Jefferson Davis, för att välja en sträckning för den första transkontinentala järnvägen. McClellan undersökte en nordlig sträckning längs 47:e och 49:e breddgraderna från St. Paul, Minnesota, till Puget Sound. Under uppdragets gång visade han en tendens att vara ohörsam mot höga politiker. Isaac Stevens, guvernör i Washingtonterritoriet, blev missnöjd med McClellans resultat när han undersökte bergspass i Kaskadbergen. (McClellan valde Yakima Pass utan en noggrann rekognoscering och vägrade följa guvernörens order att ta en grupp genom passet i vinterväder, då han litade på felaktiga uppgifter om snödjupet i området. Han missade också tre mycket bättre bergspass i den omedelbara närheten, vilka skulle komma att bli de som användes för järnvägar och landsvägar.) Guvernören beordrade McClellan att lämna över expeditionens loggböcker, men McClellan vägrade styvnackat, förmodligen för att han skämdes över personliga kommentarer som han hade gjort genom hela loggböckerna. +När McClellan återvände österut, började han uppvakta Ellen Mary Marcy (1836–1915), dottern till hans tidigare befälhavare. Ellen, eller Nelly, tackade nej till McClellans första frieri, ett av nio som hon fick från olika män, inklusive McClellans vän från West Point, A.P. Hill. Ellen tackade ja till Hills frieri 1856, men hennes familj godkände honom inte och han drog tillbaka frieriet. +I juni 1854 skickades McClellan på ett hemligt rekognosceringsuppdrag till Santo Domingo på order från Jefferson Davis. McClellan bedömde den lokala försvarskapaciteten åt krigsministern. (Denna information användes inte förrän 1870, när president Ulysses S. Grant gjorde ett misslyckat försök att annektera Dominikanska republiken.) Davis började behandla McClellan nästan som en skyddsling, och hans nästa uppdrag var att bedöma den logistiska kapaciteten hos olika järnvägar i USA, än en gång med syftet att planera för en transkontinental järnväg. I mars 1855 befordrades McClellan till kapten och fick tjänst vid 1:a amerikanska kavalleriregementet. +På grund av sina politiska kontakter och sina kunskaper i franska fick McClellan uppdraget att vara officiell observatör av de europeiska arméerna i Krimkriget 1855. Han reste långa vägar och träffade de högsta militära befälen och kungliga familjer. Han observerade belägringen av Sevastopol (1854-1855). När han kom tillbaka till USA 1856 bad han om att få en tjänst i Philadelphia för att förbereda sin rapport, vilken omfattade en kritisk analys av belägringen och en lång beskrivning av hur de europeiska arméerna var organiserade. Han skrev också en handbok om kavalleritaktik, som baserade sig på ryska kavallerireglementen. Ett fel som krigets observatörer begick, även McClellan, var att de inte förklarade betydelsen av utvecklingen av vapen som kombinerade gevärens räfflade pipor, som gav stor träffsäkerhet på håll, med musköternas möjlighet att laddas om snabbare, och hur detta skulle kräva grundläggande förändringar i taktiken i det kommande inbördeskriget. +Armén antog McClellans kavallerihandbok och även hans konstruktion av en sadel, "McClellansadeln", som han sade att han hade sett användas av husarer i Preussen och Ungern. Den blev standardutrustning för kavalleriet så länge hästar användes och används fortfarande för ceremoniellt bruk. +McClellan tog avsked från sin officerstjänst den 16 januari 1857 och blev, tack vare sin erfarenhet från rekognoscering för järnvägar, chefsingenjör och vice verkställande direktör för järnvägen Illinois Central Railroad och dessutom verkställande direktör för Ohio and Mississippi Railroad 1860. Han gjorde väl ifrån sig på båda arbeten, och byggde ut Illinois Central Railroad till New Orleans och hjälpte Ohio and Mississippi Railroad att komma tillbaka efter den ekonomiska kraschen i USA 1857. Trots hans framgångar och goda lön ($10 000 per år), trivdes han inte med att vara civilt anställd och fortsatte studera klassisk militärstrategi. Under striderna mot mormonerna funderade han på att ta värvning i armén igen. Han hade också tankar på att stödja Benito Juárez i Mexiko. +Före amerikanska inbördeskrigets utbrott blev McClellan aktiv politiker och stödde Demokraternas presidentkandidat Stephen A. Douglas i presidentvalet 1860. Han hävdade att han hade stoppat ett försök till valfusk av Republikanerna, genom att beordra försening av ett tåg som förde med sig män som skulle rösta olagligt i ett annat county. Det gjorde att Douglas vann detta county. +I oktober 1859 kunde McClellan återuppta sin uppvaktning av Ellen Marcy. De gifte sig i Calvary Church i staden New York den 22 maj 1860. +Vid början av inbördeskriget gjorde McClellans kunskaper i vad som kallades "det storskaliga krigets vetenskap" och hans erfarenheter av järnvägar att han förmodades kunna spela stor roll i militär logistik. Det gjorde att han blev efterfrågad när unionsarmén mobiliserades. Guvernörerna i Ohio, Pennsylvania och New York, de tre största delstaterna i USA, bad honom att föra befäl över deras delstaters miliser. Ohios guvernör William Dennison var den mest ihärdige, så McClellan fick tjänst som generalmajor för frivilligstyrkorna och tog befälet över Ohiomilisen den 23 april 1861. Till skillnad från vissa av officerarna i unionsarmén, vilka kom från familjer av abolitionister, var han motståndare till federal inblandning i frågan om slaveri. Några av hans kolleger från Södern frågade honom informellt om han ville ta parti för Södern, men han kunde inte acceptera secession. +Den 3 maj återinträdde McClellan i federal tjänst genom att bli befälhavare för Ohio, Indiana, Illinois och, senare, västra Pennsylvania, västra Virginia och Missouri. Den 14 maj fick han rangen av generalmajor i den reguljära armén. Vid 34 års ålder hade han då en högre rang än alla andra i armén än generallöjtnant Winfield Scott, högste militäre befälhavaren. McClellans snabba befordran berodde delvis på hans bekantskap med Salmon P. Chase, USA:s finansminister och före detta guvernör i Ohio och senator. +Medan McClellan kämpade med att ta hand om alla de tusentals män som anmälde sig frivilligt för militärtjänst och med att sätta upp träningsläger, funderade han också på strategiska planer. Fyra dagar efter att han tagit befälet i Ohio skrev han brev till generallöjtnant Scott den 27 april, med det första förslaget för en samlad strategi för kriget. Det innehöll två alternativ, båda med honom själv som befälhavare. Det första innebar att 80 000 man skulle invadera Virginia genom dalgången längs Kanawhafloden mot Richmond. Det andra innebar att samma styrka skulle gå söderut i stället, in i Kentucky och Tennessee. Scott avfärdade bägge planer på grund av logistiska problem. Även om han berömde McClellan, svarade han i ett brev att de 80 000 man som McClellan skrev om skulle komma till bättre användning på en flodexpedition för att ta kontroll över Mississippifloden och splittra Konfederationen, tillsammans med en stark blockad av Söderns hamnar. Denna plan, som skulle ha krävt mycket tålamod hos nordstaternas befolkning, kallades nedsättande för Anacondaplanen i pressen, men kom till sist att bli grunden för den strategi som vann kriget. Relationen mellan McClellan och Scott blev mer och mer ansträngd under sommaren och hösten. +McClellans första militära insats var att ockupera den del av västra Virginia där en majoritet ville stanna kvar i unionen, och som skulle komma att bli delstaten West Virginia. Han hade den 26 maj fått underrättelserapporter om att de betydelsefulla broarna som tillhörde järnvägen Baltimore and Ohio Railroad i den delen av Virginia brändes. Medan han snabbt tog fram planer för att invadera regionen, satte han igång sin första allvarliga politiska kontrovers genom att uttala för dem som bodde där att hans styrkor inte hade någon önskan att blanda sig i frågor om enskild egendom — inklusive slavar. Han insåg snabbt att han hade gått för långt och bad om ursäkt i ett brev till president Lincoln. Problemet var inte att hans uttalande var diametralt motsatt regeringens politik vid den tiden, utan att han hade gått långt utöver sin strikt militära roll. +Hans styrkor förflyttade sig snabbt in i området genom Grafton och vann i den lilla strid som fick namnet slaget vid Philippi, troligen den första striden till lands i kriget. Hans första personliga befäl i strid var vid Rich Mountain, som han också vann, men inte förrän han hade visat stor försiktighet och en ovilja att använda reservstyrkor, vilket skulle bli hans kännetecken under resten av hans karriär. Hans underlydande, William S. Rosecrans, klagade bittert på att hans anfall inte följdes upp av förstärkningar, så som McClellan hade sagt. Dessa två mindre segrar gjorde ändå McClellan till nationalhjälte. New York Herald rubricerade en artikel om honom: "General McClellan, Napoleon i detta krig." +Efter unionsstyrkornas förlust vid Bull Run den 21 juli 1861, kallade Lincoln McClellan från västra Virginia, där McClellan hade givit nordstaterna det första som man kunde likna vid militära segrar. Han for med ett specialtåg från Wheeling via Pittsburgh, Philadelphia och Baltimore till Washington, D.C., och blev överväldigad av de entusiastiska folkmassor som slöt upp vid hans tåg längs vägen. +Carl Sandburg skrev: "McClellan var dagens man, utpekad av händelserna, och utvald av en överväldigande tyngd av allmän och enskildas opinion." Den 26 juli, samma dag som han kom fram till huvudstaden, blev McClellan utnämnd till befälhavare av det militära området för Potomac, den huvudsakliga unionsstyrkan som var ansvarig för att försvara Washington. Den 20 augusti lades flera militära enheter i Virginia under hans befäl, och han bildade Potomac-armén med sig själv som dess förste befälhavare. Han uppskattade den makt och berömmelse som han hade fått. +Under sommaren och hösten organiserade McClellan den nya armén och förbättrade dess stridsmoral avsevärt genom ständiga resor för att se över och uppmuntra styrkorna. Det var en enastående bedrift, som fick honom att personifiera Potomac-armén och smickrade hans män. Han skapade för Washington försvarsverk som var nästan ogenomträngliga, med 48 fort och styrkekoncentrationer samt 480 kanoner bemannade av 7 200 artillerister. Potomac-armén växte från 50 000 i juli till 168 000 i november och betraktades som den mest kolossala militära styrka som världen hade sett i modern tid. Detta var dock även en tid då han hade ansträngda relationer till det högsta befälet och han fortsatte att ofta gräla med regeringen och den högste militäre befälhavaren, generallöjtnant Scott, i frågor om strategi. McClellan avvisade Scotts Anacondaplan, han föredrog i stället ett stort fältslag av samma stil som under Napoleonkrigen. Han föreslog att hans armé skulle få växa till 273 000 man och 600 kanoner och "krossa rebellerna i ett enda fälttåg" Han föredrog ett krig som skulle ha liten påverkan på civilbefolkningen och inte skulle behöva medföra frigivandet av slavar. +McClellans motstånd mot frigivandet av slavar ökade pressen mot honom, och han fick bitter kritik från radikala republikaner i regeringen. Han såg slaveri som en institution erkänd av USA:s konstitution och som därför skulle få federalt skydd varhelst det fanns. Hans sätt att uttrycka sig efter kriget var typiskt för många från nordstaterna: "Jag erkänner att jag har fördomar till förmån för min egen ras, & kan inte lära mig att tycka om odören från varken getter eller niggrer." Han skrev dock i november 1861 till sin fru: "Jag kommer att, om jag lyckas, kasta mitt svärd på vågen för att tvinga fram förbättringar för de stackars svarta." Han skrev senare att om han hade fått förhandla om fredsvillkoren, skulle han ha insisterat på ett stegvis frigivande, för att bevaka både slavarnas och slavägarnas rättigheter, som del av uppgörelsen. Han var dock öppen med sitt motstånd mot de radikala republikanerna. Han sade till sin fru Ellen: "Jag tänker inte kämpa för abolitionisterna." Detta gav honom ett uppenbart handikapp, eftersom många politiker i ledande position ansåg att han försökte införa oppositionspartiets politik. +Det största problemet med McClellans krigsstrategi var att han var övertygad om att de konfedererade styrkorna stod redo att anfalla honom med överväldigande numerär. Den 8 augusti trodde han att de konfedererade hade över 100 000 man som stod mot hans trupper (trots att inte hade använt fler än 35 000 vid Bull Run några veckor tidigare), och han utfärdade höjd beredskap i huvudstaden. Den 19 augusti bedömde han att det stod 150 000 fiender framför hans front. McClellans följande operationer skulle komma att påverkas i hög grad av de uppförstorade bedömningarna om fiendens styrka som han fick av sin underrättelsechef, detektiven Allan Pinkerton. Uppskattningarna i augusti 1861 var dock helt och hållet McClellans egna. Resultatet blev en väldig försiktighet som tog initiativet från McClellans armé och orsakade hård kritik från regeringen. Historikern och levnadstecknaren Stephen W. Sears har kallat McClellans handlande "i grunden sunt" om han hade varit så numerärt underlägsen som han trodde, men McClellan hade faktiskt sällan mindre än ett numerärt överläge på två mot en under 1861 och 1862. Exempelvis hade Konfederationens styrkor den hösten en numerär på mellan 35 000 och 60 000, medan Potomac-armén hade 122 000 man i september och 170 000 tidigt i december, vid årets slut 192 000. +Oenigheten med Scott skulle komma att bli mycket personlig. Scott (tillsammans med många på krigsdepartementet) var ursinniga över att McClellan vägrade avslöja några detaljer om sina strategiska planer, eller ens enkla detaljer som truppstyrkors numerär eller utplaceringar. (För egen del hävdade McClellan att han inte litade på att någon i den högre ledningen skulle hålla hans planer hemliga för pressen, och därmed för fienden.) Medan de var oeniga om försvarsstyrkorna längs Potomacfloden, skrev McClellan till sin fru den 10 augusti, på ett sätt som var karakteristiskt för en del av hans mer privata korrespondens: "General Scott är det stora hindret - han kan inte inse faran & är antingen en förrädare eller inkompetent. Jag måste kämpa mig förbi honom." Scott blev så besviken på sitt förhållande till den unge generalmajoren att han lämnade in sin avskedsansökan till president Lincoln, som initialt vägrade att acceptera den. Det gick rykten i huvudstaden om att McClellan skulle kunna avgå, eller göra en militärkupp, om Scott inte avsattes. Lincolns regering sammanträdde den 18 oktober och gick med på att acceptera Scotts avsked av "hälsoskäl". +Den 1 november 1861 avgick Winfield Scott och McClellan blev högste militäre befälhavare för alla unionsarméer. Presidenten uttryckte sin oro över det arbete som den dubbla rollen av att vara befälhavare över en armé och högste militäre befälhavare innebar, men McClellan svarade: "jag kan göra allt detta". +Lincoln och många andra ledare i nordstaterna blev alltmer otåliga på grund av McClellans senfärdighet i att angripa de konfedererade styrkor som fortfarande stod koncentrerade nära Washington. Unionsstyrkornas förlust vid det mindre Slaget vid Ball's Bluff nära Leesburg i Virginia i oktober ökade på missnöjet och skadade indirekt McClellan. I december bildade kongressen en kommitté för förandet av kriget, vilket genom hela kriget kom att bli ett horn i sidan på många generaler. Kommittén anklagade dem för inkompetens och, i några fall, förräderi. McClellan kallades som första vittne den 23 december, men han fick tyfoidfeber och kunde inte komma. Hans närmaste underordnade officer vittnade i stället. Deras uttryckliga erkännanden att de saknade kännedom om strategier för att avancera mot de konfedererade gav upphov till många krav på McClellans avsked. +McClellan skadade sitt rykte ännu mer genom att inte lyda sin överbefälhavare. Han refererade privat till Lincoln, som han hade känt innan kriget när Lincoln var jurist för Illinois Central, som "inget mer än en välmenande babian", en "gorilla" och "ständigt ovärdig...sin höga position". Den 13 november förolämpade han presidenten, som besökte McClellans hem, genom att låta honom vänta i 30 minuter, bara för att sedan få besked om att generalen hade gått till sängs och inte kunde ta emot honom. +Den 12 januari 1862 kallades McClellan till Vita huset, där regeringen krävde att få kännedom om hans planer. För första gången avslöjade han sina planer att transportera Potomac-armén med fartyg till Urbanna, Virginia, längs Rappahannockfloden, för att kringgå de konfedererade styrkorna nära Washington och fortsätta 80 km till lands för att inta Richmond. Han vägrade att lämna några specifika detaljer om den föreslagna operationen, till och med till sin vän, den nyligen tillsatte krigsministern Edwin M. Stanton. +Den 27 januari utfärdade Lincoln en order som krävde att alla hans arméer skulle inleda offensiver den 22 februari, Washingtons födelsedag. Den 31 januari utfärdade han en kompletterande order om att Potomac-armén skulle förflytta sig till lands för att angripa de konfedererade vid Manassas Junction och Centreville. McClellan svarade omedelbart med ett 22 sidor långt brev, där han med många detaljer argumenterade emot presidentens order och förespråkade sin egen Urbannaplan i stället. Detta var den första skriftliga presentationen av planens detaljer för presidenten. Även om Lincoln ansåg att hans egen plan var överlägsen, var han lättad över att McClellan äntligen gick med på att börja förflytta sig, och gick motvilligt med på McClellans plan. Den 8 mars, då han tvekade över McClellans beslutsamhet, blandade sig Lincoln åter i arméns befälhavares beslutsrätt. Han kallade till ett krigsråd i Vita huset där McClellans underlydande blev tillfrågade om sin tilltro till Urbannaplanen. De uttryckte tilltro i varierande grad. Efter mötet utfärdade Lincoln en ny order, där han namngav enskilda officerare som kårbefälhavare, vilka skulle rapportera till McClellan (som tidigare hade varit tveksam till att göra på detta sätt för att utröna sina befälhavares effektivitet i strid, trots att detta skulle ha givit honom direkt översyn i de tolv divisionerna i fält). +Ytterligare två kriser skulle drabba McClellan innan han kunde sätta sina planer i verket. De konfedererade styrkorna under general Joseph E. Johnston drog sig tillbaka från sina positioner vid Washington och intog nya positioner söder om Rappahannockfloden, vilket omöjliggjorde Urbannaplanen. McClellan gjorde om sin plan så att hans trupper skulle avsegla från Fort Monroe, Virginia, och avancera längs Virginiahalvön till Richmond, en operation som skulle bli känd som halvökampanjen. McClellan fick dock utstå extrem kritik från pressen och kongressen när det kom fram att Johnstons styrkor inte bara hade retirerat obemärkt, utan att de i månader hade lurat unionsarmén genom att använda svartmålade stockar för att efterlikna kanoner, så kallade kväkarkanoner (Quaker Guns). Kongressens kommitté för krigföringen besökte de övergivna konfedererade linjerna och radikala republikaner lade förslag i kongressen om att avskeda McClellan, men detta förslag blev genom parlamentariska manövrer nedröstat med en liten majoritet. Den andra krisen var när konfederationens bepansrade fartyg CSS Virginia visade sig i strid, vilket ledde till panik i Washington och gjorde att örlogsflottans stödoperationer på Jamesfloden blev problematiska. +Den 11 mars 1862 avskedade Lincoln McClellan som högste militäre befälhavare. Han fick bara behålla befälet över Potomac-armén, formellt för att McClellan skulle bli fri att ägna all sin uppmärksamhet åt framryckningen mot Richmond. Lincolns order gjorde inte klart om McClellan skulle kunna återinsättas om operationen blev framgångsrik. Han blev inte ersatt av någon annan officer. Lincoln, Stanton och en grupp officerare som kallades "krigsstyrelsen" bestämde över strategin den våren. Även om McClellan fick uppmuntrande kommentarer från Lincoln, skulle han med tiden se förändringen i befälsordningen annorlunda. Han beskrev den som en del av ett intrigerande "för att säkerställa att den förestående operationen skulle misslyckas." +McClellans armé började överföras från Alexandria i Virginia den 17 mars. Det skedde på en armada som var större än alla tidigare amerikanska expeditioner, med transport av 121 500 man, 44 artilleribatterier, 1 150 vagnar, över 15 000 hästar och tonvis med materiel. En engelsk observatör anmärkte att det var "en jättes kliv" Arméns framryckning från Fort Monroe upp längs Virginiahalvön gick långsamt. McClellans plan att snabbt inta Yorktown misslyckades, när det kom fram att de konfedererade styrkorna hade befäst en sträcka tvärs över halvön, vilket fick honom att besluta att belägra staden, något som krävde mycket förberedelser. +McClellan fortsatte att tro på underrättelseinformation som sade att de konfedererade styrkorna hade två till tre gånger fler man än vad de verkligen hade. Tidigt under operationen försvarade den konfedererade generalen John B. "Prince John" Magruder med en mycket mindre styrka halvön mot McClellans offensiv. Han skapade en illusion av att ha många trupper bakom linjerna och ännu fler som var på väg. Han åstadkom detta genom att låta små grupper av soldater marschera flera gånger förbi samma platser, där de kunde observeras på långt håll eller var precis utom synhåll, tillsammans med mycket ljud och fanfarer. Under denna tid kunde general Johnston sända förstärkningar till Magruder, men även då hade Magruder mycket färre soldater än vad McClellan trodde. +Efter en månads förberedelser, precis innan han skulle gå till anfall mot de konfedererade försvarsverken vid Yorktown, fick McClellan veta att Johnston hade dragit sig tillbaka upp längs halvön mot Williamsburg. McClellan blev då tvungen att följa efter, utan att ha hjälp av det tunga artilleri som hade ställts upp framför Yorktown. Slaget vid Williamsburg den 5 maj betraktas som en seger för unionen - McClellans första - men den konfedererade armén var inte tillintetgjord och huvuddelen av dess styrkor kunde förflyttas förbi Williamsburg till Richmonds yttre försvarsverk medan slaget pågick och under de påföljande dagarna. +McClellan hade också hoppats på en samtida framryckning från flottan mot Richmond via Jamesfloden. Denna framryckning kom inte till stånd, på grund av unionsflottans nederlag den 15 maj i slaget vid Drewry's Bluff, omkring 11 km nedströms den konfedererade huvudstaden. Genom att placera artilleri på en strategisk höjd högt över en krök i floden, och genom att sänka båtar för att skapa en serie hinder i själva floden, hade de konfedererade effektivt blockerat denna möjliga framryckningsväg mot Richmond. +McClellans armé framskred under de följande tre veckorna försiktigt mot Richmond och nådde fram till ett avstånd av omkring 6 km från staden. McClellan satte upp en bas vid Pamunkeyfloden (en farbar biflod till Yorkfloden) vid White House Landing, där järnvägen Richmond and York River Railroad mot Richmond korsade floden, tog kontroll över järnvägen, och transporterade ånglok och vagnar dit via pråmar. +Den 31 maj, medan McClellan planerade en offensiv, blev hans armé överraskad av ett konfedererat angrepp. Johnston insåg att unionsarmén efter kraftiga regn var delad i två halvor av Chickahominyfloden och hoppades att kunna besegra delarna var och en för sig vid Seven Pines och Fair Oaks. McClellan kunde inte själv föra befäl, på grund av att hans malaria gjorde sig påmind igen, men hans underordnade lyckades slå tillbaka angreppen. McClellan fick ändå kritik från Washington för att inte gå till motangrepp, vilket några ansåg att han skulle ha kunnat göra att de hade intagit Richmond. Johnston skadades i slaget och general Robert E. Lee tog över befälet över Norra Virginias armé. McClellan ägnade de följande tre veckorna åt att omdisponera sina styrkor och vänta på utlovade förstärkningar. Han förlorade då värdefull tid medan Lee fortsatte att förstärka försvaret av Richmond. +I slutet av juni inledde Lee en serie angrepp som blev kända som sjudagarsslagen. Vid det första riktiga slaget, vid Mechanicsville, led de konfedererade av dåligt samarbete mellan Lee och hans underlydande, vilket ledde till stora förluster mot små taktiska framgångar. Slaget hade dock stor påverkan på McClellan. När generalmajor Stonewall Jacksons trupper överraskande dök upp i slaget (när de senast hade rapporterats vara flera kilometer bort i Shenandoah Valley) övertygades McClellan om att han var ännu mer numerärt underlägsen än vad han redan hade förmodat. (Han rapporterade till Washington att han stod mot 200 000 konfedererade soldater, men det fanns egentligen bara 85 000.) +Medan Lee fortsatte sin offensiv under Gaines' Mill i öster, spelade McClellan en passiv roll. Han tog inga initiativ utan väntade på hur händelserna skulle utvecklas. Han höll två tredjedelar av sin armé utanför striden, lurad än en gång av Magruders taktik att lura honom var det fanns konfedererade styrkor. Samma natt bestämde han sig för att dra tillbaka sin armé till en säkrare plats, långt nedanför Richmond, vid en del av Jamesfloden som kontrollerades av unionsflottan. När han gjorde så, kan han oförhappandes ha räddat sin armé. Lee hade förutsatt att unionsarmén skulle dra sig tillbaka österut mot sin existerande bas där, och McClellans förflyttning söderut fördröjde Lees svar med åtminstone 24 timmar. McClellan erkände dock samtidigt att han inte längre hade möjlighet att inta Richmond, vilket hade varit syftet med operationen. Det tunga artilleriet som behövdes till belägringen skulle vara nästan omöjligt att förflytta utan de järnvägsförbindelser som fanns från hans första bas vid Yorkfloden. I ett telegram till krigsministern Edwin Stanton, där han rapporterade om händelserna, skyllde McClellan sin reträtt på Lincolns regering. "Om jag skall kunna rädda denna armé nu, säger jag rent ut att jag inte står i tacksamhetsskuld till dig eller någon annan person i Washington. Ni har gjort ert bästa för att offra denna armé." Som tur var för McClellans omedelbara framtid, såg Lincoln aldrig detta nedsättande uttalande (i vart fall inte då), eftersom det var censurerat av krigsdepartementets telegrafist. +McClellan hade också tur i att även om operationen misslyckandes kvarstod hans armé ändå till stora delar intakt, för han var för det mesta frånvarande från striderna och försummade att utse en andreman som kunde kontrollera reträtten. Militärhistorikern Stephen W. Sears skrev: "När han övergav sin armé vid Glendale och Malvern Hill under sjudagarsslagen, försummade han sina plikter. Om Potomac-armén hade slagits i grunden vid någon av dessa platser (vid Glendale hade detta varit en reell risk), hade detta troligen vänts mot honom enligt krigslagarna." Vid Glendale var McClellan 8 km därifrån, bortom Malvern Hill, utan telegrafsamband och för långt bort för att föra befäl över armén. Vid Malvern Hill var han på en kanonbåt som vid ett tillfälle var 16 km bort längs Jamesfloden. Under båda slagen fördes det faktiska befälet av hans vän, brigadgeneralen Fitz John Porter. När allmänheten hörde talas om att han hade varit på kanonbåten, var det ännu en stor pinsamhet, jämförbar med kväkarkanonerna vid Manassas. Skämtteckningar under presidentvalet 1864 skulle komma att driva med McClellan för att han föredrog säkerheten på ett fartyg medan ett slag fördes på avstånd.) +McClellan återanslöt till sin armé vid Harrison's Landing vid Jamesfloden. Diskussioner fördes om huruvida armén skulle evakueras eller försöka återuppta offensiven mot Richmond. McClellan fortsatte att distansera sig från Abraham Lincoln genom att fortsätta be om förstärkningar och skriva ett långt brev där han föreslog en strategisk och politisk linje för kriget och fortsatte motsätta sig frigivandet eller omhändertagandet av slavar som taktik. Han avslutade brevet med att föreslå att han skulle återinsättas som högsta militära befälhavare, men Lincoln svarade med att utnämna generalmajor Henry W. Halleck tíll den positionen utan att rådfråga, eller ens informera, McClellan. Lincoln och Stanton erbjöd också befälet över Potomacarmén till generalmajor Ambrose Burnside, som tackade nej. +I Washington ledde en omorganisation av de militära enheterna till att Virginiaarmén skapades och ställdes under befäl av generalmajor John Pope, som fick order att rycka fram mot Richmond från nordost. McClellan motsatte sig uppmaningar att förstärka Popes armé och fördröjde återresan för Potomacarmén från halvön tillräckligt länge för att förstärkningarna skulle komma fram när offensiven redan hade satt igång. Han skrev till sin fru innan striden: "Pope kommer att bli tillintetgjord...& besegras [av Lee]. ...En sådan skurk som han är borde medföra förlust för var och en som tar hjälp av honom. Lee hade bedömt McClellans offensiva kvaliteter och tog risken att dra bort stora delar av styrkorna från Virginiahalvön för att angripa Pope, som blev slagen i grunden vid andra slaget vid Bull Run i augusti. +Efter Popes nederlag vid andra slaget vid Bull Run återkom president Lincoln motvilligt till den man som hade lagat en skadad armé förut. Han insåg att McClellan var en god organisatör och skicklig tränare av soldater, med möjlighet att införliva trupperna från Popes armé med Potomacarmén snabbare än någon annan. Den 2 september 1862 utnämnde Lincoln McClellan till att föra befälet över fortifikationerna kring Washington och de trupper som skulle försvara huvudstaden. Utnämningen var kontroversiell i regeringen, en majoritet av regeringens ledamöter skrev under en deklaration där de menade att "vår uttryckliga mening, vid denna tid, är att det inte är säkert att betro generalmajor McClellan med befälet över någon av USA:s arméer." Presidenten medgav att det inte var en god lösning, men sade till sin sekreterare John Hay: "Vi måste använda de verktyg vi har. Det finns ingen man i armén som kan bemanna dessa fortifikationer och få dessa trupper i stridbart skick hälften så bra som han. Om han inte kan slåss själv, är han exceptionellt bra på att göra andra redo för att slåss." +Den rädsla som fanns i nordstaterna över en fortsatt offensiv av Robert E. Lee visade sig besannas när han satte sin Marylandkampanj i verket den 4 september, med förhoppningen att få sympatier för sydstaterna i slavstaten Maryland. McClellan inledde sitt försvar den 5 september. Han marscherade mot Maryland med sex av hans omorganiserade armékårer, omkring 84 000 man, medan han lämnade två kårer för att försvara Washington. McClellan fick ett mycket varmt mottagande i Frederick, Maryland, då han marscherade mot Lees armé +Lee delade sina styrkor på flera kolumner, vida spridda, medan han förflyttade sig in i Maryland. Han planerade också att försöka inta unionsstyrkornas arsenal vid Harpers Ferry. Detta var en riskabel manöver för en mindre armé, men Lee räknade med sin kunskap om McClellans temperament. Han sade till en av sina generaler: "Han är en kapabel general, men en väldigt försiktig sådan. Hans armé är i ett väldigt demoraliserat och kaotiskt skick och kommer inte att vara beredd på offensiv - eller så kommer han inte att tro att den är det - på tre till fyra veckor. Innan så sker hoppas jag vara vid Susquehanna." Det här var inte en riktigt korrekt bedömning, men McClellans armé rörde sig sakta och kom bara framåt med i genomsnitt 6 km om dagen. +McClellan hade emellertid snart mirakulös tur. Unionssoldater råkade av en händelse hitta en kopia av Lees order om att dela upp hans armé och levererade dessa till McClellans högkvarter i Frederick den 13 september. När han insåg värdet i denna information, slog McClellan upp sina armar och utbrast: "Nu vet jag vad jag skall göra!" Han viftade med ordern mot sin gamle vän, brigadgeneralen John Gibbon, och sade: "Här är ett papper som, om jag inte kan slå Lee med det, är jag villig att resa hem." Han telegraferade till president Lincoln: "Jag har hela rebellstyrkan framför mig, men jag har tillförsikt, och ingen tid skall gå förlorad. Jag tror att Lee har begått ett stort misstag, och att han kommer att bli hårt slagen för det. Jag har rebellernas alla planer, och kommer att fånga dem i deras egen fälla om mina män klarar av det...Kommer att sända dig troféer." +Trots sina utspel i högkvarteret fortsatte McClellan sin försiktiga strategi. Sedan han hade telegraferat till presidenten mitt på dagen den 13 september beordrade han inte sina styrkor att ge sig av mot South Mountain direkt, utan att avmarsch skulle ske morgonen därpå. De 18 timmarnas fördröjning gav Lee tid att reagera, för han fick underrättelser från en sydstatssympatisör om att McClellan kände till hans planer. Förseningen dömde också unionsgarnisonen vid Harpers Ferry, eftersom undsättningsstyrkan som McClellan skickat dit inte nådde fram innan garnisonen kapitulerade för Stonewall Jackson. Under slaget vid South Mountain lyckades McClellans armé slå sig igenom de försvarade pass som skilde dem från Lee, men gav också Lee tillräckligt med tid för att koncentrera många av sina män vid Sharpsburg, Maryland. +Slaget vid South Mountain gav McClellan möjligheten till ett av de storslagna tillfällena i sin karriär. +Unionsarmén nådde fram till Antietam Creek, öster om Sharpsburg, på kvällen den 15 september. Ett anfall planerat till den 16 september sköts upp på grund av dimma tidigt på morgonen, vilket lät Lee förbereda sitt försvar med en armé mindre än hälften så stor som McClellans. +Slaget vid Antietam den 17 september 1862 blev den enskilt blodigaste dagen i amerikansk militär historia. De numerärt underlägsna konfedererade styrkorna slogs desperat och väl. Trots betydande numerärt överläge lyckades inte McClellan koncentrera sina styrkor på ett effektivt sätt, vilket betydde att Lee kunde förflytta sina försvarsstyrkor för att stå emot vart och ett av unionsstyrkornas angrepp, vilka sattes igång separat och vid olika tid mot de konfedererade styrkornas vänstra flank, center och till sist högra flank. McClellan var dessutom ovillig att använda sina stora reservstyrkor för att dra ytterligare fördelar av lokala framgångar. Historikern James M. McPherson har påpekat att de två kårer som McClellan höll i reserv var större än Lees hela styrka. Orsaken till McClellans tvekan var att han, precis som vid tidigare slag, var övertygad om att han hade ett numerärt underläge. +Taktiskt gav slaget inget avgörande, även om Lee tekniskt sett var slagen, eftersom han drog sig tillbaka först och retirerade till Virginia. McClellan telegraferade till Washington: "Vår seger var komplett. Fienden drevs tillbaka till Virginia." Det uppstod dock en tydlig besvikelse över att McClellan inte hade krossat Lee, som slogs med en mindre armé med ryggen mot Potomacfloden. Även om McClellans underlydande hade del i ansvaret för förseningarna (såsom Ambrose Burnsides olycksaliga händelser vid Burnside Bridge) och misstagen (Edwin V. Sumners anfall utan rekognoscering), var detta lokala problem som armén som helhet hade kunnat klara av. Precis som vid de avgörande slagen vid sjundedagsslagen, var McClellans högkvarter för långt bort för att låta hans personal ta kontroll över striden. Han använde inte sitt kavalleri för rekognoscering. Han delade inte med sig av sin stridsplan till befälhavarna för de olika kårerna, vilket hindrade dem från att ta initiativ utanför sina sektorer. Han var också alltför villig att ta hänsyn till råd om att vara försiktig och spara sina reserver, såsom när ett stort genombrott i de konfedererades centrala linjer kunde ha utnyttjats, men Fitz John Porter sägs ha sagt till McClellan: "Kom ihåg, general, att jag för befäl över den sista reserven i den sista armén i republiken." +Trots att slaget taktiskt slutade oavgjort, anses Antietam som en vändpunkt i kriget och en seger för unionen, eftersom den avbröt Lees strategiska operation (hans första invasion av nordstaterna) och det tillät president Lincoln att besluta om sin Emancipation Proclamation den 22 september, som trädde i kraft den 1 januari 1863. Även om Lincoln hade tänkt fatta beslutet tidigare, hade hans regering givit honom rådet att vänta till en unionsseger, för att undvika uppfattningen att det var ett beslut fattat i desperation. Unionssegern och Lincolns proklamation spelade stor roll för att avhålla regeringarna i Frankrike och Storbritannien från att erkänna Konfederationen; det fanns annars de som misstänkte att dessa länder planerade ett sådant erkännande efter ytterligare en unionsförlust. McClellan hade ingen kännedom innan slaget om att planen för emancipationsproklamationen vilade på hans framgångar på slagfältet. +Då McClellan inte förföljde Lee efter Antietam, beordrade Lincoln att han skulle fråntas befälet den 5 november. Generalmajor Ambrose Burnside tog befälet över Potomacarmén den 7 november. McClellan skrev till sin fru: "De vars bedömningar jag litar på, berättar för mig att jag genomförde slaget på ett utmärkt sätt och att det var ett mästerverk i krigskonsten. ...Jag känner att jag har gjort allt som kan begäras av mig genom att två gånger rädda landet. ...Jag känner en viss stolthet i att jag, med en slagen & demoraliserad armé, slog Lee på ett sådant definitivt sätt. ...Nå ja, endera dagen kommer historien göra mig rättvisa." +Krigsminister Stanton beordrade McClellan att infinna sig i Trenton, New Jersey, för att invänta vidare order, men inga kom. Medan kriget fortgick förekom det flera önskemål om att återinsätta McClellan i en framstående position, efter unionens förluster i Fredericksburg och Chancellorsville, när Robert E. Lee avancerade norrut i början av Gettysburgkampanjen, och när Jubal Early hotade Washington 1864. När Ulysses S. Grant blev högste militäre befälhavare, diskuterade han möjligheten att ge McClellan en ny, ospecificerad position. Alla dessa möjligheter var dock omöjliga, på grund av motståndet hos regeringen och vetskapen om att McClellan innebar ett möjligt politiskt hot. McClellan arbetade i flera månader på en lång rapport som beskrev hans två huvudsakliga kampanjer och hans framgångar vid organiserandet av armén, där han besvarade sina kritiker och rättfärdigade sina handlingar genom att beskylla regeringen för att ha sålt ut honom och nekat honom nödvändiga förstärkningar. Krigsdepartementet var tveksamt till att publicera rapporten därför att, strax efter att han hade gjort den klar i oktober 1863, McClellan gick ut öppet och sade att han hade givit sig in i politiken som demokrat. +McClellan nominerades av 1864 års Demokratiska nationella konvent för att kandidera mot Abraham Lincoln i presidentvalet 1864. Han följde Winfield Scotts exempel och kandiderade som armégeneral som fortfarande var i aktiv tjänst; han sade inte upp sig från sin tjänst i det militära förrän på valdagen, den 8 november 1864. Han stödde en fortsättning av kriget och ett återupprättande av unionen. Partiets valplattform, som var skriven av krigsmotståndaren Clement Vallandigham från Ohio, gick dock emot denna position. Plattformen agiterade för ett omedelbart slut på fientligheterna och en framförhandlad överenskommelse med Konfederationen. McClellan var tvungen att förkasta plattformen, vilket gjorde hans kampanj inkonsekvent och svår. Han fick inte heller hjälp av partiets val av vicepresidentkandidat, George H. Pendleton, en fredsförespråkande kandidat från Ohio. +Den djupa splittringen inom partiet, enigheten inom Republikanerna (som gick till val under namnet +"Nationella enighetspartiet"), och de militära framgångarna för unionens styrkor hösten 1864 dömde McClellans kandidatur. Lincoln vann valet enkelt, med 212 elektorsröster mot 21 och 403 000 röster totalt, eller 55 procent. Medan McClellan hade varit mycket populär hos sina soldater när han var befälhavare, röstade de för Lincoln med marginaler på 3-1 eller mer. Lincolns andel av rösterna från Potomacarmén var 70 procent. +Efter kriget reste McClellan och hans familj på en lång resa till Europa (från 1865 till 1868). Under denna tid engagerade han sig inte i politik. När han kom tillbaka, uttalade Demokratiska partiet ett visst intresse av att nominera honom till president igen, men när det blev klart att Ulysses S. Grant skulle bli Republikanernas kandidat försvann detta intresse. McClellan arbetade på ingenjörsprojekt i New York City och erbjöds tjänsten som chef för det nyligen instiftade University of California. +McClellan utnämndes till chefsingenjör för New York City Department of Docks 1870. Denna position krävde uppenbarligen inte hans fulla tid, för från 1872 tjänstgjorde han också som verkställande direktör för Atlantic and Great Western Railroad. Han återvände till Europa med sin familj från 1873 till 1875. +I mars 1877 nominerades McClellan av New Yorks guvernör Lucius Robinson till att bli den förste chefen för New York State Department of Public Works, men han godkändes inte av New Yorks senat, då han ansågs vara "inkompetent för denna position". +År 1877 nominerades McClellan av Demokraterna till att bli guvernör i New Jersey. Detta överraskade honom, eftersom han inte hade uttryckt något intresse av en sådan position. Han valdes och tjänstgjorde en treårig mandatperiod från 1878 till 1881. Hans mandatperiod utmärkte sig för försiktigt, konservativt verkställande arbete och minimal politisk oenighet. +Den avslutande delen av hans politiska karriär var ett starkt stöd för valet av Grover Cleveland i presidentvalet 1884. Han hoppades bli utsedd till krigsminister i Clevelands regering, en position som han skulle ha varit välmeriterad för, men politiska rivaler från New Jersey lyckades blockera nomineringen av honom. +McClellans sista år ägnade han åt att resa och skriva. Han rättfärdigade sin militära karriär i McClellan’s Own Story, utgiven postumt 1887. Han avled oväntat vid 58 års ålder i Orange, New Jersey, sedan han hade haft smärtor i bröstet i några veckor. Hans sista ord, klockan 3 på morgonen den 29 oktober 1885, var: "Jag känner mig lättad nu. Tack." Han är begravd på Riverview Cemetery i Trenton, New Jersey. +McClellans son, George B. McClellan Jr. (1865-1940), föddes i Dresden i Tyskland under familjens första resa i Europa. Han kallades för Max inom familjen och var också politiker. Han var ledamot av USA:s representanthus för delstaten New York och som borgmästare i New York City från 1904 till 1909. McClellans dotter, Mary ("May") (1861-1945), gifte sig med en fransk diplomat och tillbringade stora delar av sitt liv utomlands. Hans fru Ellen avled 1915 i Nice i Frankrike, när hon besökte May i "Villa Antietam". Varken Max eller May gav paret McClellan några barnbarn. +The New York Post kommenterade i McClellans dödsruna: "Troligen har ingen soldat som slogs så litet någonsin blivit så omdiskuterad, och vi kan lägga till, med sådan skärpa, för sina kvaliteter som befälhavare." Denna skarpa diskussion har fortsatt i över ett sekel. McClellan rankas oftast bland de sämsta generalerna i amerikanska inbördeskriget. Debatten om McClellans förmåga och talang fortsätter dock att vara ett ämne för många kontroverser bland historiker inriktade på inbördeskriget och militärhistoria. Han har berömts av alla för sin organisatoriska förmåga och sina mycket goda relationer till soldaterna. Hans soldater gav honom smeknamnet "Little Mac"; andra kallade honom den "unge Napoleon". Det har föreslagits att hans ovilja att gå i strid till en del berodde på en innerlig önskan att undvika att förlora sina mannar. Ironiskt nog ledde detta till att han misslyckades med att ta initiativet gentemot fienden och sålunda missade tillfällen till avgörande segrar, som kunde ha avslutat kriget tidigt och således kunde ha besparat tusentals soldaters liv som dog i senare bataljer. Generaler som var mer framgångsrika i kriget, som Lee och Grant, hade en tendens att vara mer aggressiva och mer villiga att ta risken för ett stort slag även när inte alla förberedelser var perfekt utförda. +McClellan ansåg själv att det rätta sättet att leda militära operationer var att inte göra förflyttningar förrän förberedelserna är så kompletta som omständigheterna tillåter och att aldrig utkämpa ett slag utan något definitivt skäl som är värt den sannolika förlusten. +McClellans försiktighet när det kom till att pressa fienden var troligen inte en fråga om personligt mod, vilket han visade tydligt nog i fientlig eld i Mexikanska kriget. Stephen Sears skrev att hans förmåga att leda försvann under press: "Vid Glendale och Malvern Hill [...] flydde han fältet för att slippa ta ansvar. Vid Antietam, där det inte fanns något ställe att fly till, blev han paralyserad av obeslutsamhet. Sett i ett längre perspektiv kunde general McClellan vara både bekväm med och framgångsrik som verkställande befälhavare och också, om än inte fullt så framgångsrik, som strateg. Som befälhavare på slagfältet var han dock i fel position." +Ett av skälen till att McClellans rykte har solkats är hans egna memoarer. Historikern Allan Nevins skrev: "Historiestudenter måste alltid vara tacksamma för att McClellan så rättframt visade sina egna svagheter i sin postuma bok." Doris Kearns Goodwin hävdar att en genomgång av hans personliga korrespondens under kriget avslöjar en tendens till självförhärligande. Hans ursprungliga manus var klart 1881, men det enda exemplaret brann upp. Han började skriva ett nytt manus som kom att ges ut postumt 1887, som McClellan's Own Story. Han avled dock innan den var halvfärdig och William C. Prime, som hade uppdraget att göra det klart och var redaktör på Journal of Commerce i New York som var anhängare av McClellan, tog med utdrag ur omkring 250 brev som McClellan skrev till sin fru under kriget, där han hade som vana att avslöja sina innersta känslor och åsikter. +Medan McClellans rykte har blivit sämre över tid, särskilt de senaste 75 åren, är det en liten skara inbördeskrigshistoriker som anser att han har blivit orättfärdigt bedömd på åtminstone fyra punkter. För det första säger McClellans anhängare att eftersom McClellan var en konservativ demokrat med stor personlig karisma, fanns det radikala republikaner som var rädda för hans politiska potential och därför medveten underminerade hans operationer i fält. För det andra, genom att de radikala republikanerna var vinnarna i inbördeskriget, kunde de skriva dess historia och sätta deras värste politiske fiende vid den tiden, McClellan, i värsta möjliga dager. För det tredje är historiker angelägna om att stödja Lincoln, när denne som USA:s största politiska helgon arbetade på att slå ut en annan politiker genom att skylla tidiga militära motgångar på McClellan snarare än Lincoln och krigsministern Edwin M. Stanton. För det fjärde att Lincoln och Stanton medvetet undergrävde McClellan på grund av hans mer medgörliga inställning till sydstaterna, vilket hade kunnat leda till ett mindre destruktivt slut på kriget om Richmond hade fallit som resultat av halvökampanjen. Förespråkare för denna historiesyn hävdar att McClellan kritiseras mer för sin helt klart stötande personlighet än för sina faktiska förmågor på fältet. +Flera geografiska platser har fått namn efter George B. McClellan. Bland dessa finns Fort McClellan i Alabama, McClellan Butte i Mount Baker-Snoqualmie National Forest, där han reste medan han genomförde undersökningen för Pacific Railroad 1853, McClellan Street i North Bend, Washington, McClellan Street i södra Philadelphia, McClellan Road i Cupertino, Kalifornien, McClellan Elementary School i Chicago, och en ryttarstaty i brons i Washington, D.C. En annan ryttarstaty av honom finns framför Philadelphia City Hall. +Demokratiska nationella konventet 1864: +George B. McClellan - 174 (77,33 procent) +Thomas H. Seymour - 38 (16,89 procent) +Horatio Seymour - 12 (5,33 procent) +Charles O'Conor - 1 (0,44 procent) +Franklin Pierce - 0 (0,00 procent) +USA:s presidentval 1864 +Abraham Lincoln/Andrew Johnson (National Union Party) - 2 218 388 (55,0 procent) och 212 elektorer +George B. McClellan/George H. Pendleton (Demokraterna) - 1 812 807 (45,0 procent) och 21 elektorer (vann tre stater) +New Jerseys guvernörsval 1877: +George B. McClellan (Demokraterna) - 97 837 (51,65 procent) +William Augustus Newell (Republikanerna) - 85 094 (44,92 procent) +The Mexican War Diary of George B. McClellan (William Starr Myers, redaktör), utgiven postumt, 1917. +Bayonet Exercise, or School of the Infantry Soldier, in the Use of the Musket in Hand-To-Hand Conflicts (översatt från franska, Gomard), 1852. Återutgiven som Manual of Bayonet Exercise: Prepared for the Use of the Army of United States, 1862. +The Report of Captain George B. McClellan, One of the Officers Sent to the Seat of War in Europe, in 1855 and 1856, 1857. Återutgiven som The Armies of Europe, 1861. +European Cavalry, Including Details of the Organization of the Cavalry Service Among the Principal Nations of Europe, 1861. +Regulations and Instructions for the Field Service of the United States Cavalry in Time of War, 1861. Återutgiven som Regulations for the Field Service of Cavalry in Time of War, 1862. +McClellan's Own Story (William C. Prime, redaktör), 1887. +Lista över guvernörer i New Jersey +Den här artikeln är helt eller delvis baserad på material från engelskspråkiga Wikipedia, George B. McClellan, 22 oktober 2009. +Bailey, Ronald H., and the Editors of Time-Life Books, Forward to Richmond: McClellan's Peninsular Campaign, Time-Life Books, 1983, ISBN 0-8094-4720-7. +Bailey, Ronald H., and the Editors of Time-Life Books, The Bloodiest Day: The Battle of Antietam, Time-Life Books, 1984, ISBN 0-8094-4740-1. +Beagle, Jonathan M., "George Brinton McClellan", Encyclopedia of the American Civil War: A Political, Social, and Military History, Heidler, David S., and Heidler, Jeanne T., eds., W. W. Norton & Company, 2000, ISBN 0-393-04758-X. +Beatie, Russel H., Army of the Potomac: Birth of Command, November 1860 – September 1861, Da Capo Press, 2002, ISBN 0-306-81141-3. +Eckenrode, H. J., and Col. Bryan Conrad, George B. McClellan: The Man Who Saved the Union, University of North Carolina Press, 1941, ISBN 978-0-548-14788-7. +Eicher, John H., and Eicher, David J., Civil War High Commands, Stanford University Press, 2001, ISBN 0-8047-3641-3. +Goodwin, Doris Kearns, Team of Rivals, Simon & Schuster, 2005, ISBN 978-0-684-82490-1. +McPherson, James M., Battle Cry of Freedom: The Civil War Era (Oxford History of the United States), Oxford University Press, 1988, ISBN 0-19-503863-0. +McPherson, James M., Crossroads of Freedom: Antietam, The Battle That Changed the Course of the Civil War, Oxford University Press, 2002, ISBN 0-19-513521-0. +Nevins, Allan, The War for the Union, Vol. I: The Improvised War 1861 – 1862, Charles Scribner's Sons, 1959, ISBN 0-684-10426-1. +Rafuse, Ethan S., McClellan's War: The Failure of Moderation in the Struggle for the Union, Indiana University Press, 2005, ISBN 0-253-34532-4. +Rowland, Thomas J., "George Brinton McClellan", Leaders of the American Civil War: A Biographical and Historiographical Dictionary, Ritter, Charles F., and Wakelyn, Jon L., eds., Greenwood Press, 1998, ISBN 0-313-29560-3. +Rowland, Thomas J., George B. McClellan and Civil War History: In the Shadow of Grant and Sherman, Kent State University Press, 1998, ISBN 978-0-87338-989-1. +Sandburg, Carl, Storm Over the Land: A Profile of the Civil War, Harcourt Brace and Company, 1942, ISBN 978-0-8317-1433-8. +Sears, Stephen W., Controversies and Commanders: Dispatches from the Army of the Potomac, Houghton Mifflin Company, 1999, ISBN 0-395-86760-6. +Sears, Stephen W., George B. McClellan: The Young Napoleon, Da Capo Press, 1988, ISBN 0-306-80913-3. +Sears, Stephen W., Landscape Turned Red: The Battle of Antietam, Houghton Mifflin, 1983, ISBN 0-89919-172-X. +Sears, Stephen W., To the Gates of Richmond: The Peninsula Campaign, Ticknor and Fields, 1992, ISBN 0-89919-790-6. +Beatie, Russel H., Army of the Potomac: McClellan Takes Command, September 1861-February 1862, Da Capo Press, 2004, ISBN 0-306-81252-5. +Beatie, Russel H., Army of the Potomac: McClellan's First Campaign, March-May 1862, Savas Beatie, 2007, ISBN 978-1932714-25-8. +Burton, Brian K., Extraordinary Circumstances: The Seven Days Battles, Indiana University Press, 2001, ISBN 0-253-33963-4. +Cutrer, Thomas W., The Mexican War Diary and Correspondence of George B. McClellan, Louisiana State University Press, 2009, ISBN 978-0-8071-3451-1. +Ridgway, James M., Jr., Little Mac: Demise of an American Hero, Xlibris, 2000, ISBN 0-7388-0579-3. +McClellan Society +Georgia's Blue and Gray Trail McClellan timeline +Lincoln and Lee at Antietam +Mr. Lincoln and New York: George B. McClellan +National Park Service biography +Harper's Weekly political cartoon, October 27, 1877, "All Quiet on the Hudson", McClellan caricature in the campaign for governor of New Jersey +Marcy, Randolph B, assisted by McClellan, George B., Exploration of the Red River of Louisiana, in the year 1852 hosted by the Portal to Texas History +Biography of George B. McClellan (PDF), New Jersey State Library +New Jersey Governor George Brinton McClellan, National Governors Association +"L'il Mac" George McClellan Sångparodi +American Heritage on George McClellan's appointment +The Mexican War diary of George B. McClellan på www.archive.org +George Brinton McClellan på www.findagrave.com +Abraham Lincoln and George B. McClellan +The Comfort Zone är en låt framförd av den amerikanska sångaren Vanessa Williams, från hennes andra studioalbum med samma namn (1991). Låten skrevs av Williams tidigare samarbetspartner Kipper Jones i ett samarbete med Reggie Stewart och producerades av Jones och Gerry Brown. Wing Records gav ut "The Comfort Zone" som albumets andra musiksingel den 29 oktober 1991 genom att göra den tillgänglig kommersiellt på CD, 12" vinyl och kassettband. Den skickades till R&B-radio (urban contemporary) den 25 oktober medan popradio (contemporary hit radio) fick låten den 15 november. +"The Comfort Zone" är en R&B-komposition som innefattar influenser av jazz. Den utgår från ett medelsnabbt tempo och är sammansatt med flera instrument, däribland keyboardar, elbas, gitarr och slagverk. Den innehåller ett flöjtsolo av Hubert Laws och röstsamplingar från ett liveuppträdande av Run-DMC. Inspirationen bakom låttexten kom från samtal med Williams där hon beskrev för Jones hur hon kände sig glad och befann sig i en positiv fas i livet. Texten avhandlar sex utan att gå in på detaljer och hur hon bjuder in sin manliga partner till avslappning och trygghet i hans bekvämlighetszon. +"The Comfort Zone" mottog positiv respons från musikjournalister efter utgivningen, vilka lyfte fram Williams passionerade sånginsats och musiken, vilket sammantaget skapat en romantisk och sexig komposition. Låten blev en hit på den amerikanska R&B-marknaden. Billboard rankade låten på andraplatsen på singellistan Hot R&B Singles medan den intog förstaplatsen på Top Urban Contemporary sammansatt av Gavin Report. "The Comfort Zone" gav Williams hennes sjunde Grammy Award-nominering när den nominerades i kategorin Best Female R&B Vocal Performance år 1992. Musikvideon till låten regisserades av Ralph Ziman som tidigare jobbat med Williams på videon till singeln "Running Back to You". +Under år 1989 påbörjade Williams arbetet på en uppföljare till det guldsäljande debutalbumet The Right Stuff (1988) som genererade fyra musiksinglar som nådde topp-tio på den amerikanska R&B-topplistan. Skivbolagschefen Ed Eckstine, som hade befordrats till verkställande direktör för Mercury Records, fortsatte som chefsproducent och tillsammans med Williams valde de ut material till hennes andra studioalbum The Comfort Zone med större noggrannhet än tidigare. Eckstine kontaktade musikproducenten och låtskrivaren Kipper Jones som skapat Williams debutsingel "The Right Stuff" (1988) för att samarbeta med henne igen på ny musik. Jones berättade i en intervju: "[...] eftersom 'The Right Stuff' var en sådan stor framgång och hennes debutalbum gjorde riktigt bra ifrån sig så ringde han upp mig och sa 'Hej, vi borde jobba ihop på en andra låt. Vi bokar in ett möte med Vanessa och hör hur hon känner." Jones besökte Williams under inspelningen av den amerikanska långfilmen Another You (1991) som hon skådespelade i tillsammans med Richard Pryor och Gene Wilder. Jones samtalade med Williams och frågade henne om livet och hur hon kände sig varpå Williams sade: "Jag mår väldigt bra [...] allt i mitt liv är väldigt bra just nu. Jag är på en riktigt bekväm plats." Konversationen med Williams blev Jones inspiration inför skapandet av låttexten till "The Comfort Zone". I en intervju sade han: +"Det har blivit kännetecknet för mitt låtskrivande. Jag vill prata med dig först om jag skriver texter till dig. Jag vill sätta mig ner och utforska ditt sinne och lära mig hur du uttrycker dig. Jag vill veta vad som pågår för dig just nu, hur du mår. Vad känner du passion för just nu? Allt sådant är viktigt för jag vill kunna skriva med utgångspunkten att du som ska framföra låten känner dig bekväm och kan leverera rätt budskap." +Efter mötet med Williams begav sig Jones tillbaka till inspelningsstudion där han jobbade tillsammans med den yngre kollegan Reggie Stewart. Duon skrev texten till "The Comfort Zone" samma kväll och jobbade samtidigt på musiken tillsammans med Gerry Brown. Inspirationen i studion var vad Jones beskrev som "euro-soul" och musik utgiven av den brittiska sångaren Omar Lye-Fook, en artist som Williams tipsat honom om. Brown kom på iden med att inkludera ett flöjtsolo i låten varpå han ringde upp Hubert Laws som han var ytligt bekant med. Laws uttryckte att han gärna ville bidra till låten och besökte inspelningsstudion. Under skapandet bidrog Thomas Boje med elbas, Wah Wah Watson med gitarr, Eric Kupper spelade keyboardar och slagverk tillsammans med Randy Stern medan Terry Burrus spelade piano och stränginstrument. Låtens mastering sköttes av Herb Powers Jr. medan John Poppo arbetade som ljudtekniker. Efter att ha färdigställt en demo och spelat den för Williams gick hon med på att spela in låten. +Originalversionen av "The Comfort Zone" har en speltid på tre minuter och femtionio sekunder (3:59). Den har ett medelsnabbt tempo och utgår från 105 taktslag per minut. Den är skriven i tonarten e-moll. "The Comfort Zone" rör sig inom ramarna för R&B vars komposition även innefattar jazzinfluenser. Jones beskrev själv att låtens beats inspirerades av musikduon Soul II Soul då de var "riktigt heta" vid tidpunkten. "The Comfort Zone" inleds med ljudet av någon som söker rätt radiostation genom att vrida på en kanalväljare. På en station hörs en kort sekvens från Marvin Gayes version av "I Heard It Through the Grapevine" (1968). På en annan hörs en kvinna prata franska och säga "Bienvenue dans la zone de confort". En röstsampling från en man bryter in som säger "Hey, check this out". Den franska kvinnan var en språklärare och vän till Jones. Under ett telefonsamtal bad han henne säga "Välkommen till trygghetszonen" på franska och inkluderade inspelningen av hennes röst i låten. Den manliga röstsamplingen som upprepar "check this out" och "oh yeah" under låtens gång var från en livespelning av musikgruppen Run-DMC när dom framförde "Here We Go" (1994). +Michele Johnson från Classic Rock History noterade att låten har ett 1990-tals groove där keyboardar och flöjt är i framkant samtidigt som trumslagen bidrar till rytmiken. Johnson beskrev musiken som "sexig och romantisk" medan en annan musikjournalist ansåg att den förmedlar "feelgoodkänslor". Den brittiska webbplatsen On This Day in Pop ansåg att kompositionen badade i "eldig passion, finpolerade pianohouse-ackord och en förförisk refräng tillräckligt het för att smälta en glass." +Texten i "The Comfort Zone" avhandlar sex men går inte in detaljerat utan använder sig istället av antydningar. Williams bjuder in sin manliga partner till avslappning och trygghet i hans bekvämlighetszon. Den inleds med textversen: "Where do you go when there's a need to be loved like you need to be loved? I'll let you know just what to do and where to go". Musikjournalisten Carl Wiser ansåg att textens innebörd var intressant då det vanligtvis talas det om att "ta sig ur" sin bekvämlighetszon istället för att stanna i den. Om detta kommenterade Jones vid en tillbakablick från år 2022: +"Även om jag skriver åt en annan artist så är det ändå mycket av mig själv i texten. Vid den här tidpunkten i livet längtade jag efter något lugnande och familjärt. Något tryggt. Det var min impuls. Min avsikt var att gå mot något säkert. Nu är vi i ett nytt millennium och vi har lärt oss att om vi stannar kvar i vår bekvämlighetszon kommer vi aldrig få ut det vi vill av livet. Men detta var en annan tid och det var okej att vara i en bekväm och familjär plats. Det är en plats som jag tror många av oss fortfarande strävar att hitta. Man måste inte alltid tvinga sig ur sin bekvämlighetszon. Ibland behöver man hitta tillbaka till den och känna tröst." +Johnson noterade att "The Comfort Zone" passade Williams sångröst som mezzosopran och att hon tillämpade ett "passionerat, varmt och änglalikt" sånguttryck. Johnson konstaterade: "Hennes ton matchar den mjuka musiken perfekt. Hon behöver inte skrika för att höras. Hon sjunger passionerat. Hon sjunger en låt om sex, men hon låter inte vulgär. Hon förefaller ha klass och är samtidigt sexig." Mark Chappelle från Albumism skrev att Williams och Jones tillsammans skapat en "tabubelagd fantasi" som är väldigt ingående fram tills Williams slutar vara detaljrik vid versen: "No one ever has to know what turns you on". +Vid färdigställandet av The Comfort Zone ville Williams ge ut titelspåret som albumets huvudsingel, något som hennes skivbolag motsatte sig då de ville återintroducera Williams på musikmarknaden med en upptempolåt med "sommarkänsla" som skulle vara mer "popradiovänlig". Valet föll istället på danspopkompositionen "Running Back to You". I en intervju med Billboard sade Williams: "Den andra singeln kommer att bli '[The] Comfort Zone', den som jag egentligen hade sett som första singeln. Men 'Running Back [to You]' är väldigt popradiovänlig medan 'Comfort Zone' är mer R&B så dom [Wing] tyckte att vi skulle satsa på pop först som grund för att utnyttja min mångsidighet". "Running Back to You" blev en hit för Williams som nådde topp-tjugo på poplistan Billboard Hot 100 och förstaplatsen på R&B-listan. Albumet The Comfort Zone släpptes i augusti i USA och mottog positiv respons av musikjournalister och Chicago Tribune ansåg att Williams kunde "sjunga cirklar runt sina konkurrenter". +"The Comfort Zone" skickades till amerikansk R&B-radio (urban contemporary) den 25 oktober 1991 och till popradio (contemporary hit radio) den 15 november. Fysiska exemplar av singeln släpptes till skivaffärer den 29 oktober och gjordes tillgängliga på formaten CD-Maxi, "12 vinyl samt kassett. Två remixversioner skapades av låten som inkluderades på de fysiska utgåvorna. Phase 5 skapade "Vanessa's Vibe Mix" och "Comfortable Percapella It's A Late Nite Thing!" där den förstnämnda remixen gästades av den amerikanska rapparen J-Sun. Frankie Knuckles skapade "Frankie's Comfortable Mix". Musikvideon till låten regisserades av Ralph Ziman som även skapade videon för "Running Back to You". Den hade premiär den 7 december. +Alex Henderson från Allmusic var positiv till låten i sin recension av albumet The Comfort Zone och skrev: "Williams är som bäst på det sexiga, fängslande titelspåret". Han lyfte fram den som en av höjdpunkterna på albumet tillsammans med "Work to Do" och "Running Back to You". John Martinucci från Gavin Report var positiv till låten som han beskrev som en "sensuell midtempokomposition". Han fortsatte: "Denna flöjtfyllda groove är lugn och saktar ner tempot efter Ms. Williams förra utgivning. Vanessas inbjudande sång och tillbakalutade musikarrangemang kommer att tillfredsställa de som söker skydd i hennes 'trygghetszon'. Försök inte ens att justera kanalväljaren – låt henne göra det!"Vid en tillbakablick år 2016 beskrev den brittiska webbplatsen On This Day in Pop låten som en föregångare till Janet Jacksons "That's the Way Love Goes" (1993) och ansåg att Williams låt borde blivit en minst lika stor hit på amerikansk popradio. År 2021 rankade Classic Rock History låten på åttondeplatsen över Williams tio bästa låtar i karriären. Samma år recenserade Mark Chappelle från Albumism albumet och beskrev Laws flöjtsolo som "förstklassigt" och att det bidrog till låtens sensualism. +Vid den 35:e Grammy Awards-ceremonin år 1993 var Williams nominerad i kategorin Best Female R&B Vocal Performance tillsammans med Chaka Khan, Shanice, Whitney Houston och Oleta Adams. Williams förlorade utmärkelsen till Khan och hennes "The Woman I Am" (1991). +På amerikanska poplistan Billboard Hot 100 noterades "The Comfort Zone" för första gången den 30 november 1991 på plats 86. Tre veckor senare hade låten klättrat till plats 70. Den nådde som bäst plats 62 på listan den 18 januari 1992. Låten hade större framgång på R&B-marknaden. På singellistan Hot R&B Singles noterades den första gången den 2 november 1991. Den nådde andraplatsen på listan den 4 januari 1992 och blev hennes femte topp-tio-singel i karriären. Den toppade även "Sales"-kolumnen och nådde tredjeplatsen i "Airplay" kolumnen i Hot R&B Singles Sales & Airplay den 4 januari 1992. "The Comfort Zone" hade även framgång på danslistorna i USA. Den nådde som bäst plats 25 på Hot Dance Club Play. Låten gick in på plats 41 på topplistan 12-Inch Singles Sales den 7 december och tilldelades veckans "Hot Shot Debut"-titel. Följande vecka klättrade låten från plats 41 till plats 23 och fick då veckans "Power Pick"-titel. Den toppade listan den 25 januari 1992. +Enligt amerikanska musikpublikationer som enbart mätte radiospelningar var låten en framgång i kategorin R&B/Urban. I Gavin Report, i kategorin "Urban" den 25 oktober, var Williams den mest tillagda ("Most Added") nya artisten tillsammans med Shanice och hennes "I Love Your Smile" (1991). "The Comfort Zone" gick in på plats 39 på topplistan Urban Contemporary följande vecka och nådde topp-trettio den 8 november. Den nådde topp-tio den 29 november och klättrade till tredjeplatsen den 6 december. Låten toppade slutligen listan den 13 november. Även i R&R Magazine blev "The Comfort Zone" en av veckans mest populära nya singlar den 25 oktober. Följande vecka var Williams den fjärde mest tillagda artisten efter Lisa Stansfield, Prince och Luther Vandross. Den 1 november gick den in på plats 33 på radiotopplistan Urban Contemporary. Den nådde topp-tjugo den 22 november och topp-tio den 29 november. "The Comfort Zone" nådde andraplatsen på listan den 20 december vilket blev dess topposition. Williams blockerades från förstaplatsen av Shanice. +Enligt Discogs finns det 10 utgivningar av "The Comfort Zone". Nedan listas ett urval som ej är identiska. +Information hämtad från Discogs. +Malmö Fotbollförening, ofta förkortat Malmö FF eller MFF (även kallat Di Blåe), är en svensk fotbollsklubb från Malmö. De tävlar i Allsvenskan, den högsta divisionen i svensk fotboll, och spelar sina hemmamatcher på Eleda Stadion. Klubben har vunnit allsvenskan 27 gånger, blivit svenska mästare 24 gånger (flest SM-titlar av alla lag), senast 2024, och leder den allsvenska maratontabellen. De har även vunnit Svenska cupen 16 gånger. +Malmö FF grundades den 24 februari 1910 och gjorde sin första säsong i allsvenskan 1931/1932. De vann allsvenskan första gången 1944. Från och med mitten av 1940-talet och början av 1950-talet genomgick föreningen sin första storhetstid genom tre raka allsvenska guld (1949, 1950 och 1951). +Under 1970-talet vann klubben fem svenska mästerskap och fyra Svenska cupen-titlar. I Europacupen 1979 gick MFF till final och mötte där den engelska klubben Nottingham Forest, en match som slutade i förlust med 0–1, för denna prestation tilldelades föreningen Bragdguldet 1979. Malmö FF är den enda nordiska klubb som har varit representant vid Interkontinentala cupen (efterföljande av FIFA Club World Cup) där de tävlade om titeln 1979. +Sedan 2010-talet har klubben inlett vad som kan kallas sin tredje storhetstid då klubben tagit nio SM-titlar (2010, 2013, 2014, 2016, 2017, 2020, 2021, 2023 och 2024); vunnit svenska cupen vid två tillfällen (2022 och 2024); kvalat in till gruppspelet i Champions League säsongerna 2014/2015, 2015/2016 och 2021/2022; samt gått till slutspel i Europa League säsongerna 2018/2019 och 2019/2020. +Malmö FF:s främsta rivaler är Helsingborgs IF och IFK Göteborg. Från och med början av 1910-talet fram till 1960-talet, i cirka femtio år, hade klubben en rivalitet med IFK Malmö. Klubben är medlem i Skånes Fotbollförbund. Den officiella supporterföreningen är MFF Support. +Klubben uppstod som ett kommunalt initiativ 1905 för att uppmuntra ungdomar i Malmö att spela organiserad fotboll. Ett av ungdomslagen, Bollklubben Idrott, även känt som BK Idrott, var en föregångare till Malmö FF. BK Idrott gick ihop med IFK Malmö i deras nyskapade fotbollsavdelning 1909, men lämnade rätt snabbt på grund av problem mellan de två klubbarna. Den 24 februari 1910 grundade de 19 medlemmarna i BK Idrott, Malmö FF. Den första ordförande i klubben var Werner Mårtensson. +Klubben tillbringade sina första tio år i lokala och regionala divisioner eftersom det inte fanns någon officiell nationell liga. De spelade de flesta av sina matcher i stadsdivisionen som kallades Malmömästerskapen. De tävlade även i regionala tävlingar i Skåne och spelade matcher mot danska klubbar. År 1916 nådde Malmö FF finalen i distriktsmästerskapen för första gången, där de förlorade med 3–4 mot rivalen Hälsingborgs IF. Klubben besegrade den lokala rivalen IFK Malmö tre gånger under säsongen, vilket gav dem den inofficiella men mycket åtråvärda titeln, Malmös bästa fotbollsklubb. År 1917 deltog Malmö FF i Svenska mästerskapet för första gången, en turnering i cupform där vinnaren blev utnämnd till svenska mästare. Malmö FF förlorade dock sin match i den andra kvalomgången med 1–4 mot IFK Malmö. Klubben fortsatte att spela i cupen fram tills 1922 och nådde kvartsfinalen 1920 då de slogs ut av Landskrona BoIS. Cupen avbröts slutligen och titeln svenska mästare gavs senare till vinnarna av Allsvenskan som först skapades inför säsongen 1924/1925. +År 1920 bjöd Svenska Fotbollförbundet in svenska fotbollsklubbar till att tävla i officiella nationella tävlingar. Malmö FF fick då en plats i Division 2 Sydsvenska Serien. De vann den divisionen och blev uppflyttade till Svenska Serien Västra, den högsta divisionen i fotboll i Sverige på den tiden. De blev dock nedflyttade efter en enda säsong och fick spela i Sydsvenska Serien i ett årtionde innan de ännu en gång blev uppflyttade i Allsvenskan 1931. +Klubben nådde mittenplaceringar i ligatabellen i två säsonger innan de 1934 blev nedflyttade som ett straff för att ha brutet mot amatörreglerna. Klubben hade betalat sina spelare en mindre summa pengar för varje match de spelade. Även om det var mot reglerna så var det vanligt på den tiden. Malmö FF var dock den enda klubben som visade upp det i sin bokföring. Utöver nedflyttningen till division 2, så blev hela styrelsen och tjugosex spelare avstängda. Den version av händelserna som berättades av Malmö FF och den lokala pressen var att det var den lokala rivalen IFK Malmö som anmält brottet till Svenska Fotbollförbundet. Detta tros ha bidragit till den långvariga rivaliteten mellan klubbarna. +Klubben tog sig tillbaka till Allsvenskan 1937 efter två säsonger i division 2. Samma år valdes Eric Persson till ordförande efter att varit sekreterare sedan 1929, och höll den positionen fram till 1974. Persson anses av klubbens ledare och fans som den viktigaste personen i klubbens historia, efter att han bland annat gjorde klubben till professionell på 1970-talet. Klubben gick även under hans ledning från att var titellösa 1937 till att ha vunnit svenska mästerskapet tio gånger fram till slutet av säsongen 1974. År 1939 nådde klubben sin dittills högsta position, en tredjeplats i Allsvenskan, nio poäng bakom mästarna IF Elfsborg. Första gången Malmö FF blev Svenska mästare var 1944 när klubben vann säsongens näst sista, och för guldet avgörande, match mot AIK inför 36 000 åskådare på Råsunda. Den sista matchen för säsongen vann klubben med 7–0 mot Halmstad BK. +Alla de nästkommande nio säsongerna slutade Malmö FF i topp tre i ligan. Klubben vann det svenska mästerskapet 1949, 1950, 1951 och 1953 och kom tvåa 1946, 1948 och 1952. Klubben vann även Svenska cupen 1944, 1946, 1947, 1951 och 1953 och slutade tvåa 1945. Mellan den 6 maj 1949 och den 1 juni 1951 var laget obesegrat i 49 matcher i följd, varav 23 vinster i rad. +Klubben slutade på andraplats i Allsvenskan ytterligare två gånger, 1956 och 1957. Följande år lämnade klubben Malmö IP för Malmö stadion, som hade byggts inför Världsmästerskapet i fotboll 1958 och blev hemmaarena för klubben de kommande 50 åren. År 1964 värvade Malmö FF den spanska tränaren Antonio Durán, vilket var det första i en rad förändringar som ledde till den mest framgångsrika eran i klubbens historia. Unga talanger som Lars Granström och Bo Larsson dök upp under tidigt 1960-tal och skulle visa sig vara viktiga spelare på 1970-talet. Klubben slutade tvåa 1964 men fortsatte med att vinna sitt sjätte svenska mästerskap 1965, när Bo Larsson gjorde 28 mål och slutade som ligans skyttekung. Malmö FF vann återigen Allsvenskan 1967, efter ett mindre lyckat år 1966. Klubbens unga spelare, samt talanger som köps in från närliggande klubbar i Skåne 1967, blev ett lag som konsekvent slutade bland de tre bästa i Allsvenskan. +Efter att ha slutat på andraplats i Allsvenskan de två sista åren på 1960-talet inledde Malmö FF sitt mest framgångsrika årtionde i historien med att vinna svenska mästerskapet 1970. Klubben vann Allsvenskan 1970, 1971, 1974, 1975 och 1977 samt Svenska cupen 1976 och 1978. Vinsten i Allsvenskan 1977 kvalificerade klubben för Europacupen 1978/1979 och efter segrar mot AS Monaco, Dynamo Kiev, Wisła Kraków och Austria Wien nådde de, som enda svenska fotbollsklubb någonsin, finalen i tävlingen. Finalen spelades på Olympiastadion i München mot Nottingham Forest. Inför finalen drabbades man dock av ett svårt bakslag då nyckelspelaren Bosse Larsson inte kunde spela på grund av en knäskada. Utan Larsson på planen saknade Malmö FF sin naturliga målskytt och Nottingham Forest vann med 1–0 efter att Trevor Francis gjort finalens enda mål. Trots förlusten är detta fortfarande den största framgången i Malmö FF:s historia. Laget fick Svenska Dagbladets guldmedalj samma år med motiveringen: ”För den serie av bragder som förde MFF som första svenska lag fram till finalen i europeiska mästarcupen i fotboll.” +Mycket av framgångarna under 1970-talet berodde på en ny taktik och nya träningsmetoder som kom till klubben genom engelsmannen Bob Houghton, som tränade klubben mellan 1974 och 1980. Eric Persson efterträddes som ordförande 1974 av Hans Cavalli-Björkman. Efter att laget inte presterat tillräckligt bra under tränarna Keith Blunt och Tord Grip i början av 1980-talet tog Roy Hodgson över 1985. Han ledde klubben till två svenska mästerskap, 1986 och 1988 och klubben vann Allsvenskan fem år i rad mellan 1985 och 1989. Vid den tiden beslutades mästaren genom playoff mellan de bästa lagen efter att den ordinarie säsongen spelats klart. Playoff användes mellan 1982 och 1992 och klubben nådde finalen fyra gånger mellan 1986 och 1989 och vann två gånger. Utöver detta vann klubben Svenska cupen 1984, 1986 och 1989. +Under 1990-talet följde en lång nedgång för klubben. Förutom en andraplats i Allsvenskan 1996 lyckades laget inte utmärka sig och man vann varken Allsvenskan eller Svenska cupen på hela årtiondet, som blev klubbens resultatmässigt sämsta sedan 1930-talet. Årtiondet avslutades dessutom 1999 med att MFF degraderades ner till Superettan efter 62 säsonger i Allsvenskan. Degraderingen ledde till omfattande förändringar i klubbens styrelse, Hans Cavalli-Björkman efterträddes som ordförande av Bengt Madsen 1999 och före detta spelaren Hasse Borg anlitades som ny sportchef. Dessa förändringar i verksamheten, liksom det faktum att man lyckades behålla nästan hela sin spelartrupp samt framväxten av den unga talangen Zlatan Ibrahimović, ledde klubben tillbaka till Allsvenskan 2001. Ibrahimović blev berömd och var en viktig spelare i klubbens försök att återvända till högsta ligan. Han såldes senare till Ajax 2001 och har sedan dess spelat för Juventus, Inter, Barcelona, AC Milan, Paris Saint Germain och Manchester United. +Även om 2001 inte gav laget några högre placeringar lyckades man slå AIK i den allsvenska premiären efter ett år i Superettan, med två mål gjorda av Zlatan Ibrahimović. +Återkomsten i Allsvenskan var starten på en framgångsrik inledning av 2000-talet, under ledning av Tom Prahl, där klubben slutade bland de tre bästa tre gånger i rad. År 2004 vann man Allsvenskan vilket var klubbens femtonde vinst i Svenska mästerskapet. Året efter nådde klubben den sista kvalomgången till Champions League men besegrades av Thun. Framgångsrikt sponsorarbete och försäljning av spelare gjorde Malmö FF till den rikaste klubben i Sverige; en position de fortfarande innehar 2023. Klubben flyttade från Malmö Stadion till Swedbank Stadion under 2009, en stadion helt byggd för fotboll som ligger intill den gamla. +År 2009 meddelade Madsen att han skulle avgå som ordförande och ersattes av Håkan Jeppsson i början av följande år. Under 2010 firade klubben sitt 100-årsjubileum med flera festliga evenemang i början av säsongen. På dagen av klubbens 100-årsjubileum förklarade den svenska fotbollstidningen Offside att Malmö FF var den största fotbollsklubben i svensk historia efter ha analyserat tillgänglig historisk statistik. Säsongen blev en stor succé för klubben som vann Allsvenskan för nittonde gången och blev svenska mästare för sextonde gången. Till skillnad från 2004 skedde inte några större övergångar innan säsongen, utan dessa framgångar kom med en trupp som bestod mestadels av yngre spelare. Klubben har fortsatt att vara en av de mest dominerande fotbollsklubbarna i Sverige. Under säsongen 2011 var klubben på förstaplats i den allsvenska maratontabellen. Malmö FF är även rekordhållare för antalet vinster i Allsvenskan och Svenska cupen. +År 2011 slog Malmö ut det skotska laget Rangers i Champions League-kvalet, med resultaten 0–1 borta och 1–1 hemma. De åkte sedan ut i playoff med sammanlagda resultatet 3–4 mot Dinamo Zagreb, efter ha förlorat med 1–4 på bortaplan och vunnit med 2–0 på hemmaplan. Säsongen 2012 slutade Malmö FF på en tredjeplats i Allsvenskan efter att ha varit inblandade i en rafflande guldstrid som till slut vanns av IF Elfsborg. Säsongen har dock i efterhand främst blivit känd på grund av de ekonomiska problem som förföljde klubben, samt att man återtog förstaplats i maratontabellen från IFK Göteborg. +År 2013 säkrade Malmö FF sin 17:e SM-titel när de besegrade IF Elfsborg borta i näst sista omgången. Stor guldhjälte var Guillermo Molins som återvände till klubben under sommaren. Han gjorde 8 mål på 11 matcher och båda målen mot IF Elfsborg i näst sista omgången. +Inför säsongen 2014 presenterades Daniel Andersson som ny sportchef för Malmö FF och Markus Rosenberg återvände till klubben från West Bromwich Albion. MFF ledde serien från omgång 1 ända till sista omgången och säkrade därmed sin 18:e SM-titel. I kvalspelet till Champions League vann MFF den andra rundan mot lettiska Ventspils med totalt 1–0 (0–0 hemma) och den tredje rundan mot tjeckiska Sparta Prag på bortamål (2–4, 2–0). I den fjärde och sista rundan (playoff) mot österrikiska Red Bull Salzburg vann MFF med 4–2 (1–2 borta och 3–0 hemma) och tog sig därmed till gruppspel. I gruppspelet vann klubben matchen hemma mot grekiska Olympiakos med 2–0 efter två mål av Markus Rosenberg, men förlorade övriga fem matcher och slutade sist i gruppen. +Inför och under säsongen 2015 såldes bland andra Isaac Kiese Thelin, Emil Forsberg, Magnus Eriksson, Markus Halsti, Erik Johansson, Filip Helander, Ricardinho och Robin Olsen. Dessa ersattes av norrmännen Jo Inge Berget, Magnus Wolff Eikrem och Andreas Vindheim, serberna Nikola Djurdjic och Vladimir Rodić, sydamerikanerna Yoshimar Yotun och Felipe Carvalho, landslagsmeriterade backen Rasmus Bengtsson samt U21-landslagsmannen Oscar Lewicki, den isländske landslagsbacken Kári Árnason och den rutinerade målvakten Johan Wiland. Trots denna spelaromsättning lyckades Malmö FF för andra året i rad vinna samtliga 3 omgångar i Champions League-kvalet och ännu en gång kvalificera sig för gruppspel. I kvalspelet besegrades Zalgiris Vilnius, Red Bull Salzburg och Celtic. I gruppspelet blev det ännu en gång 3 poäng och en sistaplats i gruppen efter hemmaseger och bortaförlust mot Sjachtar Donetsk samt dubbla förluster mot Real Madrid och Paris Saint-Germain. I Allsvenskan slutade MFF femma efter att ha förlorat hemma i sista omgången mot IFK Norrköping, som tog SM-titeln. I samband med säsongsavslutningen lämnade också tränaren Åge Hareide Malmö FF efter två mycket framgångsrika säsonger och ersattes av Allan Kuhn. +Inför säsongen 2016 fanns isländske anfallaren Vidar Örn Kjartansson och den danske mittfältaren Anders Christiansen med bland nyförvärven. Senare anslöt också den rutinerade backen Behrang Safari. Årets enda chans att få kvala in till Europaspel för Malmö FF:s del gick via Svenska cupen 2015/2016, där man gick till sin första final sedan 1996. Mot BK Häcken hemma på Swedbank Stadion förlorade laget till slut efter straffavgörande (2–2 efter ordinarie tid och förlängning) och därmed inget Europacupspel för Malmö. I Allsvenskan 2016 fick bortamötet mot IFK Göteborg avbrytas under kaosartade former, sedan en IFK Göteborgssupporter kastat in en banger (kraftig smällare) på MFF-spelaren Tobias Sana, tidigare i IFK Göteborg. Även en av linjedomarna som stod alldeles i närheten drabbades. Därtill sprang en person in på planen, som senare visade sig vara ishockeyspelaren Mats Rosseli Olsen (Frölunda HC). Matchen kom aldrig att återupptas, utan Malmö FF tilldömdes i efterhand segern med 3–0. Malmö kom att säkra sin 19:e SM-titel den 26 oktober 2016, då man bortabesegrade Falkenbergs FF med 3–0 och därmed gick Malmö FF definitivt förbi IFK Göteborg i kampen om Sveriges "Mesta Mästare". I november meddelades det att Malmö FF sade upp Allan Kuhns anställning som tränare i Malmö FF, vilken ersattes av Magnus Pehrsson. +Inför säsongen 2017 lämnade bland andra Enoch Kofi Adu och Kari Arnason, in kom Lasse Nielsen närmast från belgiska Gent. I juli påbörjade MFF kvalspelet till Champions League, där man i den andra kvalomgången mot makedonska Vardar till slut förlorade med totalt 2–4 (1–1 hemma och 1–3 borta). I Allsvenskan 2017 hade laget haft både upp- och nedgång under säsongen, innan den 20:e SM-titeln var säkrad den 16 oktober 2017 genom att bortabesegra IFK Norrköping med 3–1. +Efter att ha inlett Allsvenskan 2018 med fyra förluster och två oavgjorda matcher på nio matcher, samt förlorat med 0–3 mot Djurgårdens IF i finalen av Svenska cupen, valde MFF att sparka huvudtränaren Magnus Pehrsson och låta sportchefen Daniel Andersson gå in som tillfällig tränare. I juni presenterades sedan Uwe Rösler som ny huvudtränare i Malmö FF. Vidare följde en period under sommarmånaderna där MFF inte förlorade en enda av totalt 16 tävlingsmatcher. I kvalet till Champions League besegrade MFF Drita från Kosovo och rumänska Cluj innan man åkte ut i den tredje kvalomgången mot ungerska Vidi. Mot danska FC Midtjylland i playoff till Europa League följdes 2–2 hemma upp med en bortaseger (0–2), varför MFF var klara för gruppspel. Med turkiska Beşiktaş, belgiska Genk och norska Sarpsborg 08 i gruppen kom Malmö att nå sextondelsfinal. I Allsvenskan slutade MFF på tredjeplats, vilket innebar en ny kvalplats till Europa League. Fredagen den 7 december meddelades det att Malmö FF:s ordförande sedan år 2010, Håkan Jeppsson, hastigt hade avlidit i en ålder av 57 år. Anders Pålsson trädde in som tillförordnad ordförande och blev ordinarie i mars 2019. +Malmö FF inledde tävlingsåret 2019 med ett dubbelmöte mot Premier League-storklubben Chelsea i sextondelsfinal i Europa League; 1–2 hemma och 0–3 borta (Chelsea kom att vinna hela turneringen). I Svenska cupen med Degerfors IF, Falkenbergs FF och Östers IF i sin grupp missades chansen att gå vidare till slutspel. I Allsvenskan 2019 gick MFF till sommaruppehåll som Allsvenska serieledare. Efter sommaruppehållet kom laget in i en viss formsvacka och tappade förstaplaceringen. I kvalspelet till Europa League gick MFF vidare till gruppspel efter segrar i kvalet mot Ballymena United (Nordirland), Domžale (Slovenien), Zrinjski Mostar (Bosnien-Hercegovina) och slutligen Bnei Yehuda Tel Aviv (Israel). Med Dynamo Kiev (Ukraina), FC Köpenhamn (Danmark) och Lugano (Schweiz) som konkurrenter vann Malmö slutligen gruppen. I den sista allsvenska omgången var MFF i guldläge en bit in i andra halvlek genom att besegra Örebro SK borta (5–0) – men Djurgårdens IF lyckades dock ta den bortapoäng mot IFK Norrköping som krävdes och vann SM-titeln, en poäng före MFF. Hemmamötet med Dynamo Kiev i Europa League (som slutade 4–3 till Malmö efter avgörande i den sjätte tilläggsminuten) var Markus Rosenbergs näst sista match i karriären och den sista på hemmaplan i Malmö och hyllades som en av de viktigaste spelarna i klubbens historia. +Tävlingssäsongen 2020 inleddes precis som fjolåret med sextondelsfinal i Europa League, denna gång mot tyska Wolfsburg, med förlust 1–5 totalt. I Svenska cupen 2019/2020 gick MFF till sin tredje final på fem säsonger genom att i gruppspel och slutspel besegra Syrianska FC, FK Karlskrona, AFC Eskilstuna, AIK och Mjällby AIF. I finalen, som spelades på Gamla Ullevi i Göteborg, stod dock IFK Göteborg som slutsegrare. Allsvenskan kom igång först i juni månad på grund av coronaviruspandemin. I kvalspelet till Europa League 2020/2021 besegrade MFF i tur och ordning Cracovia, Budapest Honvéd och Lokomotiva Zagreb, men i playoff-mötet mot spanska Granada åkte man ut och missade därmed chansen till gruppspel i Europa denna säsong. I Allsvenskan säkrade MFF till slut sin 21:a SM-titel den 8 november genom att besegra IK Sirius med 4–0. +I inledningen av 2021 missade MFF slutspelet i Svenska cupen 2020/2021 efter att ha blivit blott grupptrea. Man inledde dock Allsvenskan med sex segrar, två oavgjorda matcher och en förlust fram till sommaruppehållet. I kvalspelet till Champions League 2021/2022 besegrades i tur och ordning Riga FC, HJK, Rangers och Ludogorets Razgrad varpå MFF var klara för gruppspel i Champions League 2021/2022. I gruppspelet hamnade MFF sist i sin grupp efter oavgjort hemma och bortaförlust mot Zenit Sankt Petersburg samt dubbla förluster mot Chelsea och Juventus. MFF stod dock till slut som slutsegrare i Allsvenskan 2021 och försvarade därmed SM-titeln från 2020 och tog sin 22:a mästartitel. I slutet av året lämnade Jon Dahl Tomasson sin tjänst som huvudtränare i Malmö FF efter två säsonger. +Säsongen 2022 inleddes med att Malmö FF gick till final i Svenska cupen, som man sedan vann mot Hammarby IF efter ett avgörande på straffar. I Allsvenskan gick laget obesegrat de första sju omgångarna, men tappade sedan successivt under säsongen och slutade till sist på en sjundeplats, 18 poäng efter segrande BK Häcken. I kvalet till Champions League slogs Malmö ut redan i den andra omgången av Žalgiris, men tog sig via seger i playoff mot Sivasspor till gruppspel i Europa League. Väl där blev man dock sist i sin grupp på 0 poäng efter dubbla förluster mot Union Saint-Gilloise, Union Berlin och Braga. Under säsongens gång hann MFF både anställa och avskeda Milos Milojevic som huvudtränare, samt ha såväl Andreas Georgson som Åge Hareide som tillförordnade tränare. +Inför säsongen 2023 tog Henrik Rydström över som ny huvudtränare i Malmö FF. Malmö FF inledde den allsvenska säsongen med åtta raka vinster. I Svenska cupen nådde man kvartsfinal, som förlorades mot Djurgården efter straffläggning. Malmö FF bärgade sin 23:e SM-titel den 12 november efter en 1–0-seger i en dramatisk match mot Elfsborg. +I slutet av januari fick Anders Christiansen sin comeback i säsongens första träningsmatch mot danska Silkeborg IF efter fjolårets hjärtstillestånd. Samtidigt presenterades säsongens första nyförvärv: Jens Stryger och Erik Botheim. +Den 30 mars inleddes fotbollsallsvenskan för Malmö FF:s del med en 5–1 vinst mot IFK Norrköping. MFF vann de sex inledande omgångarna. Årets hemmapremiär mot Hammarby IF (2–0) präglades av stök; både inför och efter matchen inträffade flera incidenter mellan supportrar och polis. Anklagelser om oprovocerat våld riktades mot ordningsmakten. +Under årets första månader vinner MFF sin grupp i svenska cupen följt av vinst i både kvartsfinal och semifinal mot IFK Norrköping respektive Halmstads BK. Finalen spelades på Eleda Stadion den 1 maj mot Djurgårdens IF där Malmö FF vinner på straffar. Matchhjälte blev Johan Dahlin som räddade två straffar. Detta blev klubbens sextonde cuptitel. +I slutet av maj drabbades MFF av säsongens största skandal: lagkaptenen Pontus Jansson förekom i en förundersökning där ett flertal huliganer misstänks för brott. Detta ledde till en internutredning där Jansson blev av med kaptensbindeln och stängdes av från spel i en match. Jansson, som i media anklagades för bl.a antisemitism, JO-anmälde händelsen då det visade sig att det offentliggjorda materialet inte var komplett. +Under sommaren såldes Derek Cornelius till Marseille vilket blev allsvenskans åttonde dåvarande dyraste försäljning genom tiderna. Dessutom säljs Sebastian Nanasi till Strasbourg för allsvensk rekordsumma. In kom Colin Rösler och Sead Hakšabanović, klubbens dyraste värvning genom tiderna. +Under sommaren inleddes säsongens europaspel, med vinst i dubbelmötet mot färöiska KÍ Klaksvik. Därefter spelades 2–2 hemma mot PAOK. I returmatchen i Thessaloniki kvitterade Nils Zätterström i slutminuterna, och assisterade Anders Christiansen till ledningsmålet i förlängningen. I sista kvalrundan för Champions League förlorade man mot tjeckiska Sparta Prag med 0–4 totalt. Detta innebar dock spel i Europa League. Där förlorade man sina två första hemmamatcher, mot Rangers (0–2) och Olympiakos (0–1). Ligans första vinst tog man borta efter en vändning mot Qarabağ (2–1). +Den 28 oktober säkrades klubbens 24:e SM-guld inför ett fullsatt Eleda Stadion. Efter underläge mot IFK Göteborg vände och vann Malmö FF efter mål av Taha Ali och Hugo Bolin. +Malmö FF är en förening med årsmötet som högsta beslutande organ. Mötet godkänner räkenskaper och revision samt utser styrelse och ordförande. År 2018 utsågs Anders Pålsson till tillförordnad ordförande efter Håkan Jeppssons oväntade bortgång, och blev ordinarie ordförande i mars 2019. Föreningens dagliga verksamhet handhas av klubbens verkställande direktör i samråd med ordföranden. Malmö FF hade i slutet av år 2022 ett eget kapital på drygt 700 miljoner kronor och var sett till detta Sveriges rikaste fotbollsklubb. +Malmö FF hade tidigare flera avdelningar, men endast herr- och damfotbollsavdelningen är kvar. +Malmö FF Hockey – ishockeyavdelningen, grundad 1944, blev Malmö IF och skildes från klubben 1972 +Malmö FF Tennis – tennisavdelningen, grundad 1944, blev Bellevue TK och skildes från klubben 1977 +Malmö FF Handboll – handbollsavdelningen, grundad 1944, gick samman med Dalhems IF och skildes från klubben 1982 för att bli Dalhem/MFF +Malmö FF Bridge – bridgeavdelningen, grundad 1948 och nedlagd 1964 +Malmö FF Badminton – badmintonavdelningen, grundad 1948, blev Malmö BMK och skildes från klubben 1980 +Malmö FF Bowling – bowlingavdelningen, grundad 1948 och nedlagd 2004 +Malmö FF Dam – damfotbollsavdelningen, grundad 1970, blev 2007 LdB FC Malmö och skildes från klubben 2007 innan namnet 2013 ändrades till FC Rosengård. Återstartad 2019. +Malmö FF Bordtennis – bordtennisavdelningen, grundad 1975 och nedlagd 2006 +Klubben är känd under smeknamnen Di blåe och Himmelsblått på grund av klubbfärgerna himmelsblått och vitt. Spelarna bär himmelsblåa tröjor, vita shorts och himmelsblåa strumpor. Bortafärgerna är röd- och vitrandiga tröjor, svarta shorts och röda strumpor. Olika alternativa kit har använts för internationellt spel, bland annat helvita som infördes på 1950-talet och helsvarta med himmelsblå och gyllene kanter och text som användes för europeiskt spel 2005. +Klubbfärgerna har inte alltid varit himmelsblåa. Klubbens föregångare BK Idrott bar blå- och vitrandiga tröjor och vita shorts, och detta användes fortfarande det första halvåret 1910 när Malmö FF bildats. Det ändrades senare till röd- och vitrandiga tröjor och svarta shorts för att symbolisera att Malmö FF var en ny klubb och ett liknande kit används nu som bortadräkt av dessa historiska skäl. Det nuvarande himmelsblåa kitet infördes 1920. Sedan 2010 finns en liten skånsk flagga med på baksidan av tröjan strax under nacken. +Klubbmärket består av en sköld med två vertikala himmelsblå fält på sidorna och ett vertikalt vitt fält i mitten. Nedanför skölden står det skrivet "Malmö FF" i himmelsblåa bokstäver med en himmelsblå stjärna under texten. I övre delen av skölden är ett vitt horisontellt fält över de tre vertikala fälten. Förkortningen "MFF" av klubbens namn skrivs ut med himmelsblåa bokstäver i detta fält. På toppen av skölden finns fem tornlika förlängningar av det vita fältet. Det nuvarande klubbmärket gjorde sin debut på tröjan på 1940-talet. Det fanns andra klubbmärken innan detta men de fanns aldrig med på tröjorna. Medan det första klubbmärket var svart och vitt så var det andra klubbmärket rött och vitt i enlighet med klubbens huvudfärger mellan 1910 och 1920. I det ursprungliga klubbmärket fanns inte klubbens namn och den himmelsblåa stjärnan under skölden med. Dessa lades till i efterhand då klubbens ordförande Eric Persson noterade att åskådare utomlands hade svårt att identifiera från vilken stad klubben kom enbart genom att titta på klubbmärket. Till klubbens 100-årsdag 2010 var årtalen 1910 och 2010 skrivna på varsin sida av skölden på ett himmelsblått band bakom skölden. +I modern tid har två guldstjärnor, mästerskapsstjärnor, lagts till över skölden, men endast på spelartröjorna. En stjärna symboliserar att klubben vunnit fler än tio inhemska titlar och är en tradition som tagits från italienska ligan. Malmö FF placerade två stjärnor över klubbmärket efter att de vunnit allsvenskan 2013. De hade då segrat i den svenska högsta serien 20 gånger. På 1980-talet blev inte allsvenskans seriesegrare svenska mästare, eftersom det avgjordes i ett slutspel bland de bästa i serien. De båda stjärnorna väckte en del kritik hos motståndarnas supporterklubbar, bland annat anmäldes Malmö FF till Reklamombudsmannen för "falsk marknadsföring" och IFK Göteborg skickade en matematikbok till klubben. +Efter guldet 2017 lades den andra stjärnan till igen, för att symbolisera den 20:e mästerskapsinteckningen +Under 2011 introducerade klubben ett nytt helt vitt bortakit för användning i europeiska cuper och Allsvenskan. +Malmö FF är välkända för sin stora publik. Klubben har flera supporterklubbar, varav den största är den officiella supporterklubben MFF Support, grundad 1992. MFF Support beskriver sig själv som "en ideell och opolitisk förening som arbetar mot våld och rasism". I september 2011 blev Magnus Ericsson vald till ordförande för MFF Support. +Det finns även flera mindre fristående supportergrupper. Den mest framträdande av dessa är Supras Malmö, som grundades 2003 av en samling mindre ultrasgrupper och hängivna fans. Namnet "Supras" kommer från orden supporter och ultras, där det senare indikerar att gruppen är inspirerad av en fankultur med rötter i länder kring Medelhavet. Supras Malmö är den synligaste gruppen på stadions läktare med sina banderoller och flaggor. En annan grupp med liknande mål är Rex Scania. MFF Tifosi 96 (MT96) är ett nätverk av supportrar som skapar tifos för speciella tillfällen och viktiga matcher. +Malmösystrar är Europas största kvinnliga supportergrupp med över 200 medlemmar. +Supportrar till Malmö FF har vänskap med bland annat supportrar till Hertha BSC. +På grund av geografisk närhet finns en viss rivalitet med Trelleborgs FF och Landskrona BoIS, som båda är belägna i Skåne. Klubbens främsta rivaler är Helsingborgs IF, IFK Göteborg och IFK Malmö. Rivaliteten mellan Helsingborgs IF och Malmö FF har funnits sedan Malmö FF gick upp till Allsvenskan på 1930-talet, och är främst av geografiska skäl då båda lagen är från Skåne. Rivaliteten mellan IFK Göteborg och Malmö FF relaterar i högre grad till slag om titlar de två lagen emellan, då de är de två mest framgångsrika klubbarna i svensk fotbollshistoria och de enda två som har spelat europeiska cupfinaler: IFK Göteborg i Uefacupen 1982 och 1987 samt Malmö FF i Europacupen 1979. +Under 2000-talet har en rivalitet mellan Malmö FF och FC Köpenhamn växt sig starkare. Rivaliteten handlar framför allt om geografisk närhet och att lagen mer frekvent har spelat mot varandra. Under en match i Royal League år 2005 på Parken i Köpenhamn fick mötet att få en större betydelse än tidigare, detta efter att dansk polis under oklara grunder delade ut slag med batonger mot Malmösupportrar på läktaren. Vilket resulterade i ett antal skadade supportrar där flera fick vårdas för sina skador på sjukhus med bland annat en spräckt mjälte och ett flertal fick sys. +Rivaliteten mellan IFK Malmö och Malmö FF är både geografisk och historisk. Klubbarna kommer från samma stad och spelade på samma stadion i början av 1900-talet. Troligtvis bidrogs det till stora spänningar när styrelseledamöter i IFK Malmö 1933 avslöjade Malmö FF:s brott mot amatörfotbollsreglerna till Svenska Fotbollförbundet. IFK Malmö har inte spelat i Allsvenskan sedan 1962, vilket gör matcher mellan de två klubbarna sällsynta. +Malmö FF:s första arena var Malmö IP som de delade med ärkerivalerna IFK Malmö. Laget spelade där från grundandet av klubben år 1910, fram tills 1958. Arenan existerar fortfarande idag, dock med lägre kapacitet, och används av damlaget FC Rosengård, som tidigare var Malmö FF:s damfotbollssektion. Kapaciteten 2012 är 7 600, men den var vanligen mycket högre när Malmö FF spelade där. Under den sista säsongen år 1957 var den genomsnittliga publiksiffran 15 500. Klubbens publikrekord på Malmö IP är 22 436 mot Helsingborgs IF den 1 juni 1956. Arenan anses fortfarande vara en viktig del av klubbens historia, då det var där som klubben grundades, spelade sina första 47 säsonger och vann fem svenska mästerskap. MFF, och IFK Malmö, spelade på Malmö IP även under säsongerna 1999 och 2000. +En ny arena började att byggas i Malmö efter att Sverige blivit värdland för Världsmästerskapet i fotboll 1958, namnet för arenan blev Malmö stadion. Malmö FF spelade sin första säsong på stadion 1958/1959. Första gången klubben vann svenska mästerskapet på arenan var säsongen 1965. En övre läktare lades till på arenan på 1980-talet och stod klar 1992. Klubben hade sin mest framgångsrika era på denna stadion, då de vann tio av sexton svenska mästerskap. Arenan hade ursprungligen en kapacitet på 30 000, men detta sänktes till 26 500 på grund av förändringar i säkerhetsbestämmelserna. Klubbens publikrekord på stadion var 29 328 mot Helsingborgs IF den 24 september 1967. Arenan delades med IFK Malmö. +Efter segern i Allsvenskan 2004 kom det planer på att bygga en ny arena. I juli 2005 meddelade Malmö FF att arbetet på Stadion hade börjat. Arenan var avsedd att ha 18 000 sittplatser och 6 000 ståplatser. Arenan kan även göras om så att den rymmer 21 000 åskådare, dock endast sittplatser, detta för europeiskt spel där ståplatser inte är tillåtet. Byggandet började 2007 och var klart 2009. Den nya arenan ligger bredvid Malmö Stadion. Även om byggandet fortfarande pågick runt om i arenan vid tidpunkten så invigdes arenan den 13 april 2009 i klubbens första hemmamatch för säsongen 2009 mot Örgryte IS; där Malmö FF:s Labinot Harbuzi gjorde det första målet i den 61:a minuten. Klubben vann Allsvenskan under sin andra säsong på arenan 2010, när klubben slog Mjällby AIF den 7 november i sista matchen för säsongen med 2–0. I samma match sattes publikrekordet på arenan till 24 148 personer. Stadion är en kategori 4 arena (högsta kategori) enl. UEFA:s ranking. +Malmö FF gjorde övergången från att vara en amatörklubb till att bli fullt professionella i slutet av 1970-talet under ledning av klubbens ordförande Eric Persson. Malmö FF är en öppen förening med årsstämman som högsta beslutande organ, där varje medlem har en röst. Mötet godkänner räkenskaper och revision, utser styrelse och ordförande samt beslutar om inkomsterna. År 2010 valdes Håkan Jeppsson till ny ordförande efter Bengt Madsen. Föreningens dagliga verksamhet handhas av klubbens verkställande direktör i samråd med ordföranden. +Med ett eget kapital på 92,2 miljoner kronor år 2013 var MFF den rikaste fotbollsklubben i Sverige. Omsättningen för 2012, exklusive spelarövergångar var 163,5 miljoner kronor. Åren 2014 och 2015 bidrog intäkter från spel i Uefa Champions League till ett sammanlagt resultat på 375 miljoner. +Den högsta övergångssumman som Malmö FF fått för en spelare var +omkring 135 miljoner kronor för Hugo Larsson, som såldes till Eintracht Frankfurt 2023. Det är den högsta övergångssumman som någonsin betalats till en svensk fotbollsklubb. +MFF hade 2015 Swedbank som namnpartner till Stadion, Puma som leverantör av matchställ och Rörläggaren som tröjsponsor. Officiella partners var AAK, Elitfönster, Eon, Gigantprint, Intersport, Länsförsäkringar Skåne, Svenska Spel, Sydsvenskan och Volkswagen Malmö. +Malmö FF har varit huvudämnet i flera filmer. +Klubben har i media uppmärksammats, bland annat för fotbollsdokumentärerna Blådårar, Blådårar 2 och Framåt Malmö, som visar klubben från både supportrarna och spelarnas perspektiv under säsongen 1997 och 2000. Blådårar 1 utspelar sig 1997, då klubben slutade trea i Allsvenskan. Filmen fokuserades på supportern Lasse, spelaren Anders Andersson, tidigare ordföranden Hans Cavalli-Björkman och andra personer. +Blådårar 2 utspelar sig 2000, året efter att klubben blivit degraderade till Superettan, och följer laget när de kämpar för sin återkomst till Allsvenskan. Den andra filmen fortsätter att följa Lasse, men har även ett stort fokus på Zlatan Ibrahimović, hans framsteg och hur han till slut såldes till AFC Ajax under säsongen 2001. +Klubben har även varit med i Mitt hjärtas Malmö, en serie dokumentärer som handlar om Malmös historia. Klipp som användes var matchbilder från 1940-talet (volym 7), och matchbilder från finalen i Europacupen i fotboll 1978/1979 som spelades i München från en supporters perspektiv (volym 8). Volym 9 av serien ägnades helt åt klubbens 100-årsjubileum år 2010. +I Om Sara, en svenska dramafilm från 2005, spelar Alexander Skarsgård den fiktiva fotbollsstjärnan Kalle Öberg, som spelar för Malmö FF. +En återkommande sketch i den andra säsongen av komediserien Hipp Hipp! involverade en grupp med Malmö FF-fans som sjunger samtidigt som de utför dagliga uppgifter. +Listkriterier: +spelaren har spelat mer än 500 matcher för klubben, eller +spelaren har vunnit Guldbollen, en officiell UEFA-utmärkelse eller +spelaren har varit med som en av de 11 bästa spelarna i det officiella Sydsvenskan-laget. +Listkriterier: +Listan inkluderar alla utländska spelare som representerat klubben. +Svenska spelare med dubbelt medborgarskap är inkluderade ifall de valt att representera sitt andra land. +Spelare i fet stil representerar klubben. +Styrelsen under 2026. +Uppdaterad 30 juni 2026 +Detta är en lista över tränare som har vunnit en eller flera titlar med MFF +Malmö FF har spelat 88 säsonger (2023) i Allsvenskan. De enda klubbarna som har spelat fler säsonger är AIK med 95 stycken och IFK Göteborg med 91 stycken (2023). Klubben ligger trots detta på en förstaplats i Allsvenskans maratontabell. MFF är den enda svenska och enda nordiska klubben som har spelat i finalen av Europacupen, då de nådde denna 1979. Malmö FF är också den enda nordiska klubben som har varit representant vid Interkontinentala cupen (efterföljande av VM för klubblag) där de tävlade om 1979 års titel. +Malmö FF har vunnit inhemska, nationella och internationella titlar. Klubben innehar rekordet för flest titlar i Allsvenskan och Svenska cupen. Majoriteten av Malmö FF:s titlar är från 1970-talet. Senaste titeln klubben vann var 2024 när de vann Allsvenskan. Första gången MFF spelade i en europeisk turnering var under säsongen 1964/1965 i Europacupen och säsongen 2021/2022 var den senaste säsongen de spelade i Europa. Malmö FF kvalade för första gången in till Champions League säsongen 2014/2015. Klubben har även spelat i andra numera nedlagda europeiska tävlingar så som Cupvinnarcupen och Intertotocupen. +Svenska mästare +Vinnare (24): 1943/1944, 1948/1949, 1949/1950, 1950/1951, 1952/1953, 1965, 1967, 1970, 1971, 1974, 1975, 1977, 1986, 1988, 2004, 2010, 2013, 2014, 2016, 2017, 2020, 2021, 2023, 2024 +Allsvenskan +Vinnare (27): 1943/1944, 1948/1949, 1949/1950, 1950/1951, 1952/1953, 1965, 1967, 1970, 1971, 1974, 1975, 1977, 1985, 1986, 1987, 1988, 1989, 2004, 2010, 2013, 2014, 2016, 2017, 2020, 2021, 2023, 2024 +Andraplats (15): 1945/1946, 1947/1948, 1951/1952, 1955/1956, 1956/1957, 1964, 1968, 1969, 1976, 1978, 1980, 1983, 1996, 2002, 2019 +Division 2 Sydsvenska Serien +Vinnare (1): 1920/1921 +Andraplats (1): 1923/1924 +Division 2 Södra +Vinnare (3): 1930/1931, 1934/1935, 1935/1936 +Andraplats (1): 1929/1930 +Superettan +Andraplats (1): 2000 +Svenska cupen +Vinnare (16): 1944, 1946, 1947, 1951, 1953, 1967, 1972/1973, 1973/1974, 1974/1975, 1977/1978, 1979/1980, 1983/1984, 1985/1986, 1988/1989, 2021/2022, 2023/2024 +Andraplats (6): 1945, 1970/1971, 1995/1996, 2016, 2018, 2020 +Slutspel i Allsvenskan +Vinnare (2): 1986, 1988 +Andraplats (2): 1987, 1989 +Supercupen +Vinnare (2): 2013, 2014 +Andraplats (1): 2011 +Allsvenskan och Svenska cupen +Vinnare (9): 1943/1944, 1950/1951, 1952/1953, 1967, 1974, 1975, 1986, 1989, 2024 +Johan Dahlin (2010, 2013, 2017, 2020, 2021, 2023, 2024). +Krister Kristensson (1965, 1967, 1970, 1971, 1974, 1975, 1977). (Han tilldelades ingen av de få medaljerna som delades ut 1965). +Anders Christiansen (2016, 2017, 2020, 2021, 2023, 2024). +Oscar Lewicki (2016, 2017, 2020, 2021, 2023, 2024). +Erdal Rakip (2013, 2014, 2016, 2017, 2020, 2021). +Bosse Larsson (1965, 1970, 1971, 1974, 1975, 1977). +Pontus Jansson (2010, 2013, 2014, 2023, 2024). +Anton Tinnerholm (2014, 2016, 2017, 2023, 2024). +Roy Andersson (1970, 1971, 1974, 1975, 1977). +Tommy Andersson (1970, 1971, 1974, 1975, 1977). +Jan Möller (1970, 1971, 1974, 1975, 1977). +Staffan Tapper (1970, 1971, 1974, 1975, 1977). +Europacupen +Andraplats (1): 1978/1979 +Interkontinentala cupen +Andraplats (1): 1979 +Damlaget grundades den 7 september 1970, damerna vann fem mästerskapstitlar och två cuptitlar som Malmö FF innan damsektionen bröt sig ur föreningen 2007 och bildade LdB FC Malmö, som till säsongen 2014 blev FC Rosengårds damlag. +Den 26 november 2019 grundades ett damlag på nytt efter att Malmö FF haft ett extra insatt årsmöte, där medlemmarna röstade ja till att föreningen ska ha ett damlag igen, efter 12 års avsaknad. Detta efter att medlemmarna röstat nej till en sammanslagning med Limhamn Bunkeflos damlag, som då huserade i Damallsvenskan. Laget debuterade i Division 4 år 2020 och gick under säsongen obesegrade i serien med en målskillnad på 139–0. 2021 vann damlaget division 3 Sydvästra Skåne efter 18 raka vinster. 2022 vann de division 2 Södra Götaland, med 18 raka vinster och målskillnad på 143-5. 2023 spelade laget sig upp igenom division 1, södra till Elitettan 2024. Den 12 oktober 2024 säkrades uppflyttningen från elitettan till damallsvenskan. +Den portugisiske stjärnan Eusébio spelade en vänskapsmatch för Malmö FF:s A-lag den 12 juli 1971 mot Moskvas stadslag på Malmö stadion inför 12 000 åskådare. Eusébio gjorde själv ett mål, efter att först ha assisterat till Curt Olsbergs mål, och matchen slutade 2–2. +Smitt, Rikard (2009). Ända sedan gamla dagar-: [MFF:s jubileumsbok – 100 år]. Malmö: Malmö FF. Libris 11844810. ISBN 978-91-633-5767-1 +Törner, Ole (2005). Malmö FF: en supporters handbok. Stockholm: Bokförlaget DN. Libris 9712913. ISBN 91-7588-683-9 +Den här artikeln är helt eller delvis baserad på material från engelskspråkiga Wikipedia, Malmö FF, 6 april 2012. +Wikimedia Commons har media som rör Malmö FF. +Officiell webbplats +Malmö FF Instagram +System of a Down är ett amerikanskt alternativt metalband, grundat 1994 i Hollywood, Kalifornien. Bandet består av sångaren Serj Tankian, gitarristen Daron Malakian, basisten Shavo Odadjian och trumslagaren John Dolmayan. System of a Down släppte sitt självbetitlade debutalbum 1998, men fick sitt kommersiella genombrott 2001 med Toxicity. Tre av bandets album (Toxicity, Mezmerize och Hypnotize) har debuterat på plats 1 på Billboard 200. System of a Downs senaste album Hypnotize lanserades i november 2005. +Bandmedlemmarna har armeniskt, ursprung. Samtliga bandmedlemmar har släktingar som mördades under det armeniska folkmordet och de tillhör den armeniska diasporan. Även om bandmedlemmar inte vill beskriva System of a Down som ett politiskt band har bandmedlemmar engagerat sig för att sprida kunskap om det armeniska folkmordet och propagerat mot amerikansk militarism och totalitarism. +System of a Down gjorde ett uppehåll med start den 13 augusti 2006 och under några år framöver arbetade bandmedlemmarna med olika sidoprojekt. Den 29 november 2010 meddelades det att System of a Down hade återförenats och den 10 maj 2011 hölls den första konserten på närmare fem år. Ett sjätte studioalbum har diskuterats inom bandet. Den 6 november 2020 släppte System of a Down två låtar, vilket var det första nya materialet på 15 år. Sedan sin debut har System of a Down sålt över 40 miljoner kopior världen över. +Rötterna till System of a Down ligger i det Los Angeles-baserade bandet Soil som Serj Tankian, Daron Malakian och Shavo Odadjian var med i. De hade alla gått på Rose and Alex Pilibos Armenian School, men på grund av åldersskillnaden dem emellan träffades de inte förrän 1993 när de arbetade på separata projekt i repetitions- och inspelningsstudion Nightingale Studios i Burbank, Kalifornien. Innan Soil grundades spelade Tankian keyboard i bandet Forever Young, Malakian var med i bandet Snowblind och Odadjian hade spelat i banden Roswell och Van Buren. Det var under sin tid i Forever Young som Tankian först träffade Malakian, där deras första samarbete var under låten "Waco Jesus", som Tankian hade skrivit rörande David Koresh. +Tankian och Malakian grundade Soil och de tog in två andra medlemmar: David "Dave" Hakopyan som basist och Domingo "Dingo" Laranio som trumslagare. Odadjian anslöt sig först till Soil som deras manager; han blev dock mot slutet av Soils verksamma karriär bandets kompgitarrist. Under 1994 lämnade Hakopyan och Laranio bandet eftersom de kände att det inte ledde någonvart. Soil hade då endast uppträtt under en konsert, vilket var på Café Club Fais Do-Do i Los Angeles. Soil spelade under 1995 in åtminstone en demokassett och bandets totala verksamma karriär var åtta månader. +Malakian har i efterhand sagt att han inte ansåg att Soil var det band som kändes rätt för honom utan att det först var när System of a Down grundades som han kände sig hemma. Tankian har beskrivit Soils låtar som väldigt komplexa och han hävdade att "man skulle kunna sätta ihop 7 eller 8 System-låtar till en enda Soil-låt." Malakian har sagt att Soils låtar var 6–7 minuter långa och att de var mer som riff snarare än färdiga låtar. Han har beskrivit Soils sound som om System of a Down skulle blandas med Rush och Frank Zappa. Det var när Malakian började komprimera Soils låtar och ge dem verser, refränger och melodislingor som System of a Downs musikstil skapades. +Soil skulle senare komma att bli namnet på en låt som inkluderades på System of a Downs debutalbum från 1998 och det är även namnet på en dikt i Tankians poesibok Cool Gardens från 2001. +Efter att Soil hade splittrats 1995 grundade Tankian, Malakian och Odadjian bandet System of a Down, vars namn är baserat på Malakians dikt Victims of a Down. Tankian och Odadjian ansåg dock att ordet "Victim" syftade på svaghet och Malakian föreslog då istället användningen av ordet "System", vilket de andra bandmedlemmarna godtog. Ursprungligen var Odadjian bandets manager, men han antog kort därefter rollen som bandets basist. Ontronik "Andy" Khachaturian fick rollen som trumslagare efter att System of a Down hade sökt en sådan i tre månader. Khachaturian kände Malakian sedan tidigare eftersom de hade spelat i Snowblind tillsammans. Dessa händelser ägde rum under 1994 fram till februari 1995, när bandet till sist hade en färdig uppsättning medlemmar. +System of a Down hade problem att få ett första gig eftersom de ännu inte hade släppt någon demoutgivning. Odadjian förklarade att "vi [System of a Down] inte hade några pengar, men vi ville uppträda. Jag jobbade på banken First Interstate och mellan kreditöverföringarna brukade jag ringa till Roxy [Theatre] och säga 'Kan du inte ge oss ett gig? Eddie, ge oss ett gig.' Han [Eddie] svarade då att 'Jag behöver en demokassett. Jag kan inte bara ge bort gig.' Fem minuter senare ringde jag igen. En dag fick Roxy nog och sade 'Okej då! Gå och sälj 75 biljetter.' Vi sålde 150 biljetter." Deras första uppträdande någonsin som System of a Down var på Roxy Theatre den 28 maj 1995. +System of a Down släppte fyra obetitlade demokassetter mellan 1995 och 1997. 1997 ådrog sig Khachaturian en handfraktur på fjärde och/eller femte metakarpalbenet efter att ha slagit hål i en vägg, vilket ledde till att han inte kunde spela trummor på minst ett år. Khachaturian valde då att lämna System of a Down för att istället grunda The Apex Theory tillsammans med bland annat den förre Soil-basisten David Hakopyan. System of a Down rekryterade en ny trumslagare vid namn John Dolmayan, som gick med i bandet tidigt under 1997. Enligt Odadjian var Dolmayan från början bara tänkt att vara en tillfällig medlem, men bandmedlemmarna blev imponerade av hans trumspelande och valde därför att behålla Dolmayan som en permanent medlem i System of a Down. +Under 1997 släpptes även bandets första albumlåt, i detta fall "P.L.U.C.K.", vilken lanserades på samlingsalbumet April 24. I juli detta år fick musikproducenten Rick Rubin upp ögonen för System of a Down efter deras konsert på The Viper Room i Hollywood, Kalifornien; bandmedlemmarna beskrev själva sina konsert under denna tid som "mörkröda upplevelser." Bandet började samarbeta med Rubin i september 1997 och samma år spelade de in en femte obetitlad demokassett, som låg till grund för deras debutalbum. System of a Down började även samarbeta med skivbolaget Velvet Hammer Music and Management Group och manager för bandet blev David "Beno" Benveniste, som hade påbörjat sitt samarbete med System of a Down under 1996. +I juni 1998 släppte bandet sitt självbetitlade debutalbum System of a Down, som bäst nådde plats 124 på den amerikanska Billboard 200-listan den 8 april 2000. Runt denna tid började System of a Down också att turnera och de var förband åt både Slayer och Metallica innan de själva fick möjligheten att uppträda på Ozzfest. I november 1998 medverkade System of a Down på Chef Aid: The South Park Album med låten "Will They Die 4 You" tillsammans med Mase, Puff Daddy och Lil' Kim. I maj 1999 lanserade bandet Sugar E.P., vilket var deras första EP. +Under en konsert på nattklubben The Troubadour i Los Angeles den 4 december 1999 fick Khachaturian, den tidigare trumslagaren, ersätta Tankian som sångare, då denne hade blivit sjuk och inte kunde uppträda. Khachaturian, som från början bara kom till konserten som en åskådare, förklarade att han fick lyssna in sig backstage på System of a Downs debutalbum och lära sig sjunga deras låtar på cirka 45 minuter innan han gick ut och uppträdde på scen. Han beskrev upplevelsen som "passande för dem [System of a Down] och rolig för mig." Andra gästsångare under denna konsert var Jared Gomes, John Garcia, Logan Mader och Jack Osbourne. +Under 2000 släppte bandet "Snowblind", vilket var en coverversion av låten med samma namn av Black Sabbath, på hyllningsalbumet Nativity in Black II: A Tribute to Black Sabbath. System of a Downs andra EP, Limited Edition Tour CD, lanserades även detta år och bandet gjorde sitt första TV-framträdande den 16 mars 2000, med låten "Spiders", på Late Night with Conan O'Brien. Under 2000 turnerade System of a Down på SnoCore Tour tillsammans med banden Incubus, Puya och Mr. Bungle. Detta år medverkade Odadjian och Dolmayan i musikvideon för "Angel's Son" och Tankian medverkade på "Starlit Eyes", där båda låtarna kom med på Snots andra studioalbum Strait Up. +Genombrottet för System of a Down kom 2001, när bandet den 4 september släppte sitt andra album Toxicity. Albumet debuterade på plats 1 på Billboard 200 och den 13 september 2001 rapporterades en försäljning av 222 000 kopior i USA. Albumet låg i topp på Billboard 200 samma vecka som 11 september-attackerna ägde rum och i kölvattnet av dessa attacker tog Clear Channel Communications, den dåvarande ägaren av flest radiostationer i USA, fram en kontroversiell lista under namnet "Clear Channel memorandum". Listan tog upp låtar med "tveksamma låttexter" och rekommenderade att dessa låtar inte skulle spelas på radio. "Chop Suey!" från Toxicity var en av över 150 låtar som nämndes på "Clear Channel memorandum". Trots detta blev "Chop Suey!" en hit och låten nominerades 2002 till en Grammy Award i kategorin Best Metal Performance, men förlorade mot Tools "Schism". +Den 3 september 2001, dagen innan Toxicity släpptes, tänkte System of a Down hålla en gratiskonsert för att tacka alla sina fans för deras stöd och denna konsert skulle äga rum på en parkeringsplats i Hollywood. Antalet personer som arrangörerna räknade med skulle få plats på parkeringsplatsen var 3 500 personer, men uppskattningsvis mellan 7 000 och 10 000 personer fanns på plats när konserten skulle börja. På grund av säkerhetsrisken som detta medförde bestämde sig arrangörerna för att ställa in konserten precis innan bandet skulle gå på scen. Dock meddelades inte fansen att konserten var inställd och bandmedlemmarna nekades av arrangörerna att gå upp på scen för att berätta att det inte skulle bli något uppträdande. Fansen väntade i över en timme efter utsatt tid på parkeringsplatsen i hopp om att konserten skulle börja och efter att spänningen hade stigit började fansen att storma scenen. Ett upplopp bröt ut som varade i två timmar och resulterade i att sex personer greps av polisen. Kostnaderna för skadorna efter upploppet uppskattades till cirka $30 000. För att undvika liknande upplopp valde West Hollywood att ställa in bandets konsert dagen efter, som skulle ha hållits i skivbutiken Tower Records på Sunset Boulevard. Odadjian har i en intervju sagt att denna gratiskonsert som aldrig ägde rum fick stor uppmärksamhet i de lokala medierna, vilket han ansåg kan ha varit en av orsakerna till albumets höga försäljningssiffror veckan efter. +Under 2001 var System of a Down på turné i USA och Mexiko tillsammans med Slipknot. Under konserten den 20 oktober 2001 på Van Andel Arena i Grand Rapids, Michigan blev Odadjian trakasserad, etniskt förolämpad och fysiskt misshandlad av några väktare bakom scenen. Han fick efter attacken medicinsk hjälp av arenapersonalen och av polisen på plats. I mars 2003 stämde Odadjian DuHadway Kendall Security, vilket var företaget väktarna ifråga arbetade för. Trots denna incident blev turnén i sin helhet en succé och System of a Down uppträdde på Pledge of Allegiance Tour tillsammans med band såsom Slipknot, Rammstein och Mudvayne. +Inför inspelningen av Toxicity hade 40 låtar skrivits och 33 av dem spelats in, där det slutgiltiga albumet innehöll 14 av dessa låtar. I maj 2002 läckte några av de övriga inspelade låtarna ut på internet under titeln Toxicity II. Denna läcka gjorde bandmedlemmarna mycket upprörda. Malakian beskrev det som att "vi hade förlorat kontrollen över våra egna låtar" och han jämförde det med att "sälja en konstnärs tavla som inte var färdigmålad." Malakian har även sagt att det inte var potentiell förlorad inkomst för bandmedlemmarna som gjorde dem upprörda utan att de låtar som hade läckt ut inte var de bästa versionerna som System of a Down kunde spela in. Bandet bestämde sig för att ta kontrollen över låtarna igen och de begav sig in i studion för att spela in nya versioner av flera av de läckta låtarna. Dessa släpptes sedan i november 2002 på albumet Steal This Album!, som bäst nådde plats 15 på Billboard 200-listan den 14 december 2002. Titeln på albumet är förmodligen inspirerad av Abbie Hoffmans bok Steal This Book, men kan även ses som en sorts respons till de personer som läckte Toxicity II. System of a Down valde att inte marknadsföra Steal This Album! i någon större utsträckning. Endast ett uppträdande skedde i samband med att albumet lanserades och det var en albumsignering på Tower Records i Glendale, Kalifornien. Utöver detta släpptes enbart en singel, "Innervision", från Steal This Album! och en musikvideo spelades in, vilket var till "Boom!". +Under denna tid arbetade även några av bandmedlemmarna med andra musikaliska projekt. 2002 grundade Tankian den ideella organisationen Axis of Justice tillsammans med Tom Morello. Enligt Morello uppstod idén om Axis of Justice efter att han och Tankian hade sett några personer i publiken på Ozzfest 2002 som förespråkade rasistiska bilder och symboler. Organisationen påbörjade ett samarbete med Anti-Racist Action kort därefter för att visa på att Axis of Justice främst stod för antirasism och antifascism. Livealbumet Axis of Justice: Concert Series Volume 1 släpptes sedan i november 2004. Tankian samarbetade även med Arto Tunçboyacıyan, vilket resulterade i lanseringen av albumet Serart i maj 2003 och detta år medverkade Tankian på albumet Bird Up: The Charlie Parker Remix Project med låten "Bird of Paradise". Malakian hade 2002 grundat bandet Scars on Broadway tillsammans med Casey Chaos och Zach Hill. I december detta år spelade de in några demolåtar på det som senare skulle bli känt som Ghetto Blaster Rehearsals. En av dessa låtar var en tidig version av "B.Y.O.B.", som System of a Down senare spelade in. +I mars 2003 sade Malakian i en intervju att han "inte hade för avsikt att låta de nästa System-albumen låta exakt som de tre tidigare." Under 2004 pågick inspelningen av vad som skulle bli två stycken album: Mezmerize och Hypnotize. Det första av dessa, Mezmerize, släpptes i maj 2005. På albumet finns bland annat den Grammy Award-vinnande låten "B.Y.O.B.", vilket är en protestlåt mot Irakkriget. Det andra albumet, Hypnotize, släpptes ett halvår senare. Båda albumen nådde plats 1 på den amerikanska Billboard 200-listan. Detta gjorde System of a Down till det femte bandet/artisten någonsin som har haft två album under samma år som debuterat på plats 1 på Billboard 200-listan; de andra fyra är The Beatles, Guns N' Roses, 2Pac och DMX. +Den 7 maj 2005 uppträdde System of a Down på Saturday Night Live. De framförde "B.Y.O.B." som innehåller en del svordomar, bland annat raden "Where the fuck are you?". Svordomarna censurerades av programmets producenter, men mot slutet av låten skrek Malakian ut "Fuck yeah!", vilket inte censurerades. Detta väckte en viss uppståndelse i USA och svordomen har censurerats i senare sändningar av programmet. System of a Down har sedan dess inte uppträtt på Saturday Night Live och det har ryktats om att de är bannlysta från programmet på obestämd tid. +Under sommaren 2006 gav sig System of a Down ut på en avslutande turné för Mezmerize och Hypnotize. Den 13 augusti 2006 spelade bandet sin sista konsert tillsammans innan uppehållet, som skulle vara i nästan fem år. Denna konsert ägde rum i West Palm Beach, Florida, där bandmedlemmarna förklarade att även om de gjorde ett uppehåll från varandra skulle de återförenas i framtiden. Malakian förklarade det som att "vi [System of a Down] gör inte ett uppbrott. [...] Vi kommer att ta ett långt uppehåll ifrån varandra efter Ozzfest [2006] och göra våra egna saker. Vi har varit System i över 10 år och jag tror att det är hälsosamt för oss att vara ifrån det ett tag och sedan komma tillbaka till det senare." I juli 2018 förklarade Tankian att han var anledningen till att System of a Down gjorde ett uppehåll, vilket till stor del berodde på att han inte var nöjd med den riktning bandet var på väg. Tankian uppgav att det som låg till grund för hans missnöje gällde både den konstnärliga processen och jämlikheten inom bandet, främst berörande ägarskapet av andelarna till System of a Downs låtkatalog. +Några månader efter bandets uppehåll hade dokumentärfilmen Screamers premiär. Dokumentären, som syftar till att väcka uppmärksamhet runt det armeniska folkmordet som ägde rum 1915–1923, innehåller intervjuer och konsertmaterial med System of a Down, men Tankian ville göra det klart att det inte var en dokumentärfilm om själva bandet utan att filmens fokus ligger på det armeniska folkmordet. Screamers innehåller även en intervju med en av Tankians släktingar som överlevde folkmordet och det var tack vare denna intervju som Tankian själv valde att medverka i dokumentären. System of a Downs medverkan i dokumentären beskrevs av Ken Hachikian, dåvarande ordförande för Armenian National Committee of America (ANCA), som "väldigt väsentlig i kampen om att för evigt kunna få slut på cykeln av folkmord." +2007 nominerades låten "Lonely Day" till en Grammy Award i kategorin Best Hard Rock Performance, men förlorade mot Wolfmothers låt "Woman". +Under 2006 valde Malakian att starta upp Scars on Broadway igen dock utan Chaos och Hill, som hade varit med i bandet under 2002–2003. Dolmayan anslöt sig till Scars on Broadway kort därefter och de båda började söka efter andra medlemmar till bandet. Franky Perez, Danny Shamoun och Dominic Cifarelli gick med i Scars on Broadway och tillsammans uppträdde de för första gången den 11 april 2008 på Whisky a Go Go. Scars on Broadway hade några veckor tidigare släppt sin första singel, "They Say", efter en nedräkning på deras hemsida. Scars on Broadway släppte sitt självbetitlade debutalbum i slutet av juli 2008, men i oktober samma år tog Malakian en paus från bandet eftersom han kände att "han inte var sugen på att turnera." Kort därefter hävdade både Cifarelli och Dolmayan att Scars on Broadway hade upplösts; dock svarade Dolmayan i maj 2009 på ett mer tvetydigt sätt när han fick frågan om Scars on Broadway någonsin kommer att spela in något mer material, då han sade att han inte var säker på om detta kommer att hända men att han saknar det. I augusti 2009 åkte Dolmayan, Perez, Shamoun och Cifarelli till Irak för att uppträda på en USO-konsert. Perez skrev sedan följande på sin Twitter-sida: "Repetitionen gick bra. Jag laddar upp låtlistan innan vi sticker härifrån. Scars-låtarna låter fantastiska, men det är inte detsamma utan D...". Den 16 mars 2010 skrev Greg Watermann, bandets fotograf, på sin Myspace-sida att Scars on Broadway skulle spela ihop igen snart, men att han inte vill bekräfta något mer. Bandmedlemmarna själva meddelade den 13 april 2010 att de skulle uppträda live på The Troubadour i Hollywood den 2 maj samma år. På konserten var Odadjian närvarande under två av låtarna. Den 29 juli 2010 släppte bandet singeln "Fucking". +Utöver Scars on Broadway medverkade Malakian på låten "Trouble Seeker" på Cypress Hills album Rise Up från 2010. Dolmayan medverkade 2007 på några låtar på Tankians debutalbum som soloartist, Elect the Dead. I april 2009 rapporterades det att Dolmayan hade provspelat för The Smashing Pumpkins, efter att Jimmy Chamberlin hade lämnat bandet. I juni samma år var Dolmayan med om att starta upp bandet Indicator tillsammans med vännerna Tom Capossela, Ryan Huber och Ryan Murphy. De spelade sin första konsert tillsammans den 22 juli 2009 på The Bitter End i New York. Dolmayan grundade även Torpedo Comics, en återförsäljare av serietidningar, under 2007. Bolaget hade dock vissa ekonomiska svårigheter och under 2010 auktionerades Torpedo Comics lagervaror ut på internet för att kunna betala av sina skulder. +2005, året innan uppehållet, medverkade Tankian på tre av låtarna på Bucketheads album Enter the Chicken, som lanserades via Tankians skivbolag Serjical Strike Records. Det var via detta skivbolag, tillsammans med Reprise Records, som Tankian i oktober 2007 släppte sitt debutalbum som soloartist, Elect the Dead. På albumet spelade Tankian de flesta instrument själv, men han tog hjälp av bland annat Dolmayan för trumspelandet och under liveuppträdanden medverkade kompbandet The F.C.C. Tankian var en av presentatörerna under MTV Europe Music Awards den 1 november 2007. Under de följande åren turnerade Tankian med låtarna från Elect the Dead fast i mars 2010 lanserades livealbumet Elect the Dead Symphony, som hade spelats in den 16 mars 2009 tillsammans med Auckland Philharmonia Orchestra. I september 2010 släppte Tankian sitt andra soloalbum, Imperfect Harmonies. +Odadjian valde att grunda det experimentella hiphopbandet Achozen i maj 2006 tillsammans med RZA, Kinetic-9 och Reverend William Burke. Deras första konsert hölls den 1 december 2006 på Key Club i Los Angeles. Deras första singel "Deuces" lanserades den 22 april 2008 och låten spelas i början av filmen Babylon A.D. från 2008. Den andra låten som Achozen släppte var "Salute/Sacrifice" den 13 november 2009. Ett debutalbum planerades av Achozen under denna tid, men det dröjde till juli 2015 innan några nya låtar släpptes. +Förutom arbetet med Achozen startade Odadjian även en konst- och musikportal vid namn urSESSION i början av 2008. Han uppträdde även som discjockey, då under någon av pseudonymerna DJ Tactic eller DJ DecompozeR. Den 27 februari 2009 mottog Odadjian, tillsammans med filmskaparen Carla Garapedian, utmärkelsen GenNext's Community Hero Award av Armenian General Benevolent Union (AGBU) för sitt filantropiska arbete som en förebild för unga armenier i USA. +I september 2006 hävdade Casey Chaos, en av de ursprungliga medlemmarna i Scars on Broadway, att han var en av upphovsmännen till "B.Y.O.B.", som hade lanserats som en singel av System of a Down året innan. Genom Maxwood Music Ltd. krävde han 50 procent royalties istället för de två procent han tidigare hade fått i gåva av Malakian. Maxwood Music Ltd. hade den 2 juni 2006 köpt upp Chaos upphovsrättigheter till "B.Y.O.B.", för $50 000 plus eventuella royalties, och det var nu upp till en domstol att avgöra om Chaos verkligen hade någon upphovsrätt till låten. Upphovsrättsansökan lämnades in till domstol av Maxwood Music Ltd. den 21 februari 2008 och den huvudsakliga rättegången varade mellan 30 november till 8 december 2009. Slutpläderingen i rättegången ägde rum den 4 februari 2010 och den 17 maj samma år kom domslutet, nämligen att Chaos hade förlorat tvistemålet och kvar som de rättmätiga upphovsmännen stod Malakian och Tankian. +I augusti 2008 började rykten spridas om en möjlig återförening av System of a Down inför Eurovision Song Contest 2009, som skulle hållas i Moskva, Ryssland. Tankian förnekade inte dessa rykten utan sade istället att han kunde tänka sig att återförenas med resten av bandmedlemmarna fast på ett villkor, nämligen att bandet skulle få sjunga om det armeniska folkmordet. Tankian kommenterade att "Eurovision skulle bli ett perfekt sätt att få fram detta budskap. Vi måste verkligen tänka igenom detta." Något uppträdande eller någon återförening skedde dock inte vid Eurovision Song Contest 2009. Tankian har vid senare tillfällen sagt att Eurovision förmodligen inte längre är aktuellt för bandet men att det ur ett politiskt perspektiv hade varit en bra idé att vara med där, även om det är förbjudet att ha politiska budskap i låtarna vid Eurovision Song Contest. +Den 31 oktober 2009 uppträdde Malakian, Dolmayan, Odadjian och Perez på Odadjians Halloweenfest på Roxy Theatre och de spelade bland annat System of a Down-låten "Suite-Pee" och Scars on Broadway-låten "They Say". Den 20 november samma år uppträdde Malakian, Odadjian och Dolmayan under Chi Cheng Benefit Show i Hollywood. Anledningen till detta var att få ihop finansiellt stöd åt Chi Cheng, tidigare basist i Deftones, som hade legat i koma sedan en bilolycka i början av november 2008. Tankian medverkade inte under konserten utan bidrog istället genom att auktionera ut signerade utgåvor av sin diktbok Cool Gardens, sitt debutalbum Elect the Dead samt en Elect the Dead-affisch. System of a Down-låtar som spelades under konserten var "Aerials" och "Toxicity". +Den 29 november 2010 meddelade System of a Down att de hade återförenats. Dolmayan beskrev den första tiden efter återföreningen som "konstig" och att han själv kände sig obekväm. Enligt Dolmayan dröjde det månader innan bandmedlemmarna kände att det var roligt att uppträda tillsammans igen. Deras första konsert tillsammans på nästan fem år ägde rum i Edmonton, Kanada den 10 maj 2011. Malakian, Odadjian och Dolmayan uppträdde på hyllningskonserten Linkin Park and Friends - Celebrate Life In Honor of Chester Bennington den 27 oktober 2017 på Hollywood Bowl i Los Angeles, Kalifornien till minne av Chester Bennington, som hade avlidit den 20 juli samma år. +Under 10 års tid var det oklart om System of a Down skulle spela in nytt material tillsammans. Odadjian skrev den 20 maj 2013 på bandets Facebook-sida att han, Malakian och Dolmayan gärna ville släppa ett nytt album. Odadjian skrev även att det var Tankian som hindrade dem från att spela in nya låtar tillsammans och han ställde frågan om de resterande bandmedlemmarna skulle söka efter en ny sångare. Detta inlägg raderades kort därefter och ett officiellt uttalande lades istället ut, i vilket det stod att om System of a Down skulle spela in nytt material tillsammans skulle det göras med de medlemmar de hade i dagsläget. Malakian, Odadjian och Dolmayan tog in en potentiell ersättare till Tankian för en provspelning, men i slutändan valde de att forsätta med samma bandmedlemmar som tidigare. +I juni 2013 sade Tankian att ett nytt album krävde att bandet hängivet arbetade med det under tre års tid, något han inte såg att han hade tid med vid tillfället. I juli 2014 uttryckte Dolmayan sitt missnöje över att System of a Down inte hade varit i en inspelningsstudio på över nio år och han sade att han själv gärna ville spela in nytt material med resten av bandet. Dolmayan förklarade att en av anledningarna till att System of a Down inte hade spelat in några nya låtar tillsammans var på grund av personliga anledningar. I november 2016 sade Dolmayan att System of a Down arbetade med ett nytt album och att de hade gjort så under de senaste sex månaderna. Vid tillfället fanns det 15 låtar som potentiellt kunde komma med på det nya albumet, men Dolmayan sade att dessa skulle låta annorlunda jämfört med tidigare låtar släppta av bandet. Den 2 juli 2017 sade Odadjian att ett nytt album med System of a Down inte fanns med i planerna inom en snar framtid, vilket även bekräftades av Malakian i maj 2018 och Tankian i augusti samma år. I december 2019 visade Tankian intresse för att lansera tidigare inspelat material med System of a Down, men enligt honom var de andra bandmedlemmarna inte övertygade om denna idé. I maj 2020 berättade Tankian att bandmedlemmarna hade skrivit låtar ett par år tidigare för ett potentiellt nytt album, men att de inte kunde komma överens om vilken musikalisk riktning de skulle ta. +10 år efter återföreningen och 15 år efter att bandet lanserade sitt senaste studioalbum, Hypnotize, släppte System of a Down den 6 november 2020 en singel som innehöll låtarna "Protect the Land" och "Genocidal Humanoidz". Låtarna syftar till att väcka uppmärksamhet och ekonomiskt bistå Armenien och republiken Artsach/Nagorno-Karabach under kriget som rådde mellan republiken och Azerbajdzjan. +I januari 2023 lyfte Dolmayan återigen de oenigheter som bandmedlemmarna hade gällande ett potentiellt sjätte studioalbum. Enligt Dolmayan hade låtar skrivits för cirka fem år sedan som var redo att spelas in, men han menade på att Tankian inte visade något intresse för att genomföra studioinspelningarna. +System of a Down är ett band som gärna experimenterar med olika musikstilar. De brukar främst klassas som ett alternativt metalband, men de har även kategoriserats inom genrerna hårdrock, heavy metal, experimentell rock, nu metal, progressiv metal, progressiv rock, thrash metal, konstrock och avant-garde metal. Malakian sade i en intervju från 2005 att han inte tyckte att System of a Down enbart tillhörde en genre och i en intervju från 2003 sade han följande: "Vi är inte ett armeniskt band, vi är inte ett heavy metalband och vi är inte ett nu metalband. Vi är System of a Down. Folk verkar alltid vilja placera oss inom en viss kategori, men vi passar inte in i någon kategori. Vi är vad rockmusiken har utvecklats till." Malakian har även sagt att han inte tycker om vissa aspekter med gitarrspelandet inom nu metalgenren och att han var glad över att System of a Down inte hade fallit in under denna genre. Tankian har sagt att han tyckte att bandets musikaliska stil kunde jämföras med popmusiken vad det gällde arrangemanget och han ansåg inte att System of a Down var ett progressivt band. Tankian nämnde även att bandet lägger ned mycket tid på att få sin musik att låta som något som ingen har hört förut. Odadjian har sagt att han inte bryr sig om vilka System of a Down jämförs med och att de etiketter som vissa människor sätter på bandet inte har någon som helst effekt på deras musik. +System of a Downs låttexter har beskrivits som indirekta, skeva och dadaistiska. Tankian har sagt att han först skriver musiken till en låt för att sedan hitta ett tema eller en känsla denna musik vill förmedla. Därefter söker han främst bland de dikter han har skrivit tidigare för att få ihop låttexterna. System of a Down inkorporerar en varierande mängd instrument i sin musik såsom keyboard, elektrisk mandolin, elgitarr, akustisk gitarr, barytongitarr, 12-strängad gitarr, sitar och oud. +Malakian har sagt att han personligen har influerats av band såsom Wham!, The Beatles, Slayer, The Dead Boys, The Damned, Fleetwood Mac och Yes medan Odadjian och Tankian nämner Dead Kennedys, Bad Brains, Ozzy Osbourne, Black Sabbath, Metallica och Korn som inspirationskällor. Tankian har även influerats av jazzmusikerna Thelonious Monk, Chet Baker och Miles Davis. Dolmayan har blivit influerad av band och artister såsom Madonna, Lionel Richie, Kiss, Keith Moon, John Bonham, Stewart Copeland, Neil Peart och sin egen far, som var en saxofonspelare. Khachaturian har sagt sig blivit inspirerad av armenisk, grekisk och libanesisk musik samt Wolfgang Amadeus Mozart, Michael Jackson, Stevie Wonder, The Beatles, Pink Floyd, Björk, Metallica, The Prodigy, The Crystal Method, The Chemical Brothers och Underworld. Utöver dessa tidigare nämnda band och artister kan det även spåras influenser av Primus, Suicidal Tendencies och Frank Zappa i System of a Downs musikstil. +System of a Down agerar mycket i antipropagandasyfte inte minst vad gäller det kontroversiella armeniska folkmordet, då samtliga bandmedlemmar har haft släktingar som blev mördade i detta folkmord. Bandet har gjort ett aktivt val att aldrig hålla en konsert i Turkiet eftersom Turkiets regering hävdar att det armeniska folkmordet aldrig ägde rum. Tankian har sagt att om System of a Down skulle ha haft en konsert i Turkiet hade de varit tvungna att tala om det armeniska folkmordet, vilket hade fått dem fängslade. Tankian sade även att ifall bandmedlemmarna hade avstått från att tala om det armeniska folkmordet vid en möjlig konsert i Turkiet hade det varit hyckleri. I en intervju från 2015 nämnde Tankian att han hade ansökt om tillstånd från den turkiska regeringen för att System of a Down skulle få hålla en konsert i landet, men han fick aldrig något svar. Bandet medverkade 2006 i Screamers, vilket är en dokumentärfilm som fokuserar på det armeniska folkmordet, samt att bandmedlemmarna, med fokus på Tankian, medverkade i dokumentärfilmen Truth to Power (2021), vilken även den berör folkmordet. Tankian uppmanade USA:s dåvarande president Barack Obama att erkänna det armeniska folkmordet, vilket han var motvillig till att göra offentligt. När president Joe Biden den 24 april 2021 offentligt erkände det armeniska folkmordet tackade bandmedlemmarna i System of a Down honom kort därefter och benämnde erkännandet som en "oerhört viktig milstolpe mot rättvisa". System of a Down uppträdde 2015 på Wake Up the Souls-turnén som uppmärksammade att det var hundra år sedan det armeniska folkmordet ägde rum och den 23 april detta år höll bandet sin första konsert i Armenien någonsin. +Tankian har sagt att budskapet i System of a Downs låtar handlar om att bryta de sociala normer som finns i samhället och att förmedla känslomässiga upplevelser. System of a Down har öppet protesterat mot USA:s politiska inställning till krig. Låtarna "Temper", "A.D.D. (American Dream Denial)" och "Tentative" är exempel på antikrigslåtar medan "Boom!" och "B.Y.O.B." riktar stark kritik mot USA:s agerande under Irakkriget. +Bandmedlemmarna har ett flertal gånger dementerat ryktet om att System of a Down skulle vara ett politiskt band. Odadjian sade i en intervju från 1997 att System of a Down inte var ett politiskt band och Tankian kommenterade att han personligen var emot totalitarism, men att det inte gjorde dem till ett politiskt band. Odadjian har sedan dess upprepade gånger stått fast vid detta uttalande och sagt att bandet skriver om sådant som händer i dagens samhälle och att politik spelar en stor roll i människors liv, men att System of a Down även tar upp andra ämnen såsom sex, droger, livet och döden. Malakian sade i en intervju från 2003 att alla som hävdade att System of a Down var ett politiskt band inte hade lyssnat tillräckligt noga på bandets låtar och i en intervju från 2005 sade han att System of a Downs låtar mer förmedlar ett socialt budskap snarare än ett politiskt budskap. Tankian kommenterade i en intervju från 2000 att System of a Down inte var ett sociopolitiskt band och att de inte hade som avsikt att vara detta heller. Dolmayan hävdade i en intervju från 2020 att cirka 12 procent av System of a Downs låtar grundade sig i politik. Trots detta har System of a Down ansetts radikalisera unga lyssnare och bandmedlemmarna själva har sagt att de har "avvikande [politiska] åsikter [inom bandet]". +System of a Down tar upp ämnen såsom religion och mystik i bland annat låtarna "Science", "Tentative" och "Forest", där den senare har tolkats handla om Bibeln. Tankian har sagt att han "har samma religion som ett träd", vilket han förklarade bestod av den som USA:s ursprungsbefolkning har blandat med buddhism och transcendentala idéer. Malakian sade i en intervju att han ansåg att religion kan vara något underbart för en individ, men att han tyckte det blev problematiskt så fort religionen blev organiserad. Dolmayan har sagt att alla religioner är lika trovärdiga och att lärdomen om att religioner kräver att en människa måste mörda andra människor för sin egen överlevnad eller för att skydda landgränser enbart är baserad på rädsla. +Berelian, Essi (2005) (på engelska). The Rough Guide to Heavy Metal. Rough Guides. ISBN 1843534150. https://archive.org/details/roughguidetoheav0000bere_t6b9 +Bertoncelli, Riccardo; Thellung, Cris (2006) (på italienska). Ventiquattromila dischi: Guida a tutti i dischi degli artisti e gruppi più importanti. Baldini Castoldi Dalai. ISBN 9788860181510 +Della Cioppa, Gianni (2010) (på italienska). Heavy Metal: I Contemporanei. Giunti Editore. ISBN 9788809749627 +Hovanissian, Richard (2007) (på engelska). The Armenian genocide: Cultural and ethical legacies. New Brunswick, New Jersey: Transaction Publishers. ISBN 9781412806190. https://archive.org/details/armeniangenocide0000unse +Myers, Ben (2007) (på engelska). System of a Down: Right Here in Hollywood. The Disinformation Company. ISBN 1932857885 +Perry, Megan (2004) (på engelska). "Daron's Guitar Tunings" i Wired: Musicians' Home Studios: Tools & Techniques of the Musical Mavericks. Hal Leonard Corporation. ISBN 0879307943. https://archive.org/details/wiredmusiciansho0000perr +Tankian, Serj (2001) (på engelska). Cool Gardens. MTV Books. ISBN 9780743457415 +Wikimedia Commons har media som rör System of a Down. +System of a Down på Facebook +System of a Down på X +Slaget vid Poltava var ett fältslag, som utkämpades vid staden Poltava i dagens Ukraina den 28 juni 1709 (enligt den svenska kalendern; 27 juni enligt den julianska och 8 juli enligt den gregorianska). Det var den största och mest avgörande drabbningen under stora nordiska kriget och stod mellan en svensk armé under befäl av fältmarskalken greve Carl Gustaf Rehnskiöld och en rysk armé under befäl av tsar Peter I. +Under loppet av sex år i krigets inledande skede hade kung Karl XII och stormaktsriket Sverige besegrat nästan samtliga deltagare i anfallsförbundet mot riket, som inledningsvis bestod av Sachsen-Polen, Danmark-Norge och Ryssland. Det senare under tsar Peter I:s styre var den enda deltagaren som fortfarande var obesegrad. Karl XII valde därför att under hösten 1707 invadera Ryssland och marschera mot Moskva med en stor svensk armé. Fälttåget komplicerades dock av hårda väderförhållanden och av att ryssarna tillämpade den brända jordens taktik och gjorde överraskningsanfall, vilket tvingade Karl XII att avbryta sin marsch mot Moskva och i stället marschera söderut för att med hjälp av zaporizjakosackernas ledare Ivan Mazepa etablera goda vinterkvarter i Lillryssland. +Fram mot våren 1709 inledde Karl XII en mycket kostsam belägring av den befästa staden Poltava på den västra sidan av floden Vorskla. För att undsätta staden korsade Peter I, med en stor rysk armé, floden till den svenska sidan i mitten av juni. Den 17 juni insjuknade Karl XII till följd av en skottskada i foten och överlämnade befälet över den karolinska armén till Carl Gustaf Rehnskiöld. Trots sin skada ville Karl XII inte försumma ett tillfälle att ta upp striden med tsaren och beordrade att armén skulle gå till anfall den 28 juni. På grund av dålig rekognoscering och oordning i befälsföringen under anfallet blev konsekvenserna att omkring en tredjedel av den svenska huvudstyrkan förlorades före den avgörande drabbningen mot den ryska huvudstyrkan. Den svenska huvudstyrkan förintades och Karl XII retirerade med resten av sin armé söderut till floden Dnepr som kungen, Mazepa samt ett par tusen svenskar och kosacker korsade för att undkomma ryssarna och etablera sig i Osmanska riket. Resten av den karolinska armén tvingades att kapitulera inför den ryska övermakten vid byn Perevolotjna den 1 juli 1709. +Slaget vid Poltava, liksom den påföljande kapitulationen, slutade i en avgörande seger för Ryssland och blev den största militära katastrofen i Sveriges historia. Det markerade en vändpunkt i krigets fortsättning till förmån för anfallsförbundet mot Sverige, vilket till följd av slaget återuppväcktes och med förnyad kraft anföll det försvagade svenska riket på flera fronter. Slaget kom därmed att symbolisera början på Sveriges slutgiltiga nederlag i kriget och slutet för Sveriges tid som stormakt i norra Europa, en position som efter kriget övertogs av kejsardömet Ryssland. Slaget är därför av avgörande betydelse i såväl Sveriges som Rysslands och Ukrainas historia. +Den 12 februari 1700 inleddes stora nordiska kriget i och med att Polens kung och Sachsens kurfurste August II och hans sachsiska trupper korsade floden Düna och belägrade staden Riga i den svenska provinsen Livland. Samtidigt med August II:s angrepp beordrade kung Fredrik IV av Danmark sina danska trupper att invadera hertigdömet Holstein-Gottorp, Sveriges bundsförvant sedan flera år tillbaka. Ryssland bröt freden först i augusti samma år. I september invaderade ryska trupper under tsar Peter I den svenska provinsen Ingermanland och inledde en belägring av Narva i Estland. Dessa tre länder hade i hemlighet enats om en gemensam pakt som gick ut på att anfalla Sverige från tre fronter, och vart och ett hade som syfte att vinna tillbaka de landområden som förlorats till Sverige under tidigare krig. +Den svenska armén under kung Karl XII:s befäl avvärjde först det danska hotet genom en framgångsrik landstigning vid Humlebæk på Själland den 25 juli 1700, vilket tvingade Fredrik IV att dra sig ur kriget den 8 augusti samma år under freden i Traventhal. Därefter tvingades ryssarna utrymma Ingermanland efter deras förkrossande nederlag mot Karl XII:s huvudarmé i slaget vid Narva den 20 november. Året efter den 9 juli slog Karl XII en allierad sachsisk-polsk-rysk armé i slaget vid Düna, varefter han beslutade att invadera Polen för att avsätta kung August II som polsk kung. Efter flera sachsiska nederlag i slagen vid Kliszów, Pułtusk och Thorn blev August II slutligen avsatt av den storpolska konfederationen, och Karl XII lät välja adelsmannen Stanisław Leszczyński som sin marionett på den polska tronen. Under tiden som Karl XII stred mot Sachsen-Polen tog Peter I tillfället att omgruppera och omorganisera sina trupper, och genomförde ett förnyat fälttåg mot både Ingermanland och Livland. Efter ryssarnas segrar vid Errastfer, Hummelshof och Nöteborg fick de tillgång till Ingermanland 1703, där Peter I erövrade den svenska fästningen Nyenskans som bevakade floden Neva. Där kunde han grunda staden Sankt Petersburg, döpt efter hans skyddshelgon aposteln Petrus. Ryssarna erövrade efteråt Dorpat och Narva men deras fortsatta fälttåg mot Sveriges baltiska provinser förhindrades av general Adam Ludwig Lewenhaupt, befälhavaren för den "kurländska armén", och dennes segrar vid Saladen, Jakobstadt och Gemauerthof. +Mellan 1704 och 1706 utspelades en avgörande maktkamp mellan Sverige och Ryssland i Polen, där Peter I grep in med full kraft för att sätta August II tillbaka på sin tron. Efter svenskarnas segrar vid Warszawa, Grodno, Fraustadt och Kletsk lyckades Karl XII besegra och jaga bort sina fiender från polsk mark. Den svenska armén marscherade sedan in i Sachsen där August II tvingades underteckna freden i Altranstädt den 14 september 1706 och erkänna Stanisław Leszczyński som ny kung. Karl XII och hans armé förlades under ett år i Sachsen, där man rustade upp trupperna inför en slutlig konfrontation med Peter I i ryskt territorium. +I augusti 1707 lämnade Karl XII Sachsen med en till stor del nyvärvad och välutrustad armé på omkring 33 000 soldater, där Carl Gustaf Rehnskiöld i egenskap av fältmarskalk stod närmast kungen som dess högsta befäl. Kvar i Sverige och i övriga svenska provinser stationerades omkring 65 000 man. Rehnskiöld och den svenska arméledningen var ovetande om Karl XII:s planer för det ryska fälttåget, som denne höll för sig själv, men de kom överens om en preliminär marschriktning mot Rysslands huvudstad Moskva, där Peter I hade samlat majoriteten av sina försvarskrafter. Kvar i Polen detacherade kungen 8 000 svenska trupper under generalmajor Ernst Detlof von Krassow för att bistå kung Stanisław samt för att kunna skickas som förstärkningar tillsammans med polska styrkor följande år. Tsaren förfogade över en rysk armé på totalt omkring 100 000 reguljära och irreguljära trupper stationerade i Polen, Ingermanland, Livland och Lillryssland. Karl XII gav order till general Lewenhaupt att bryta upp och ansluta sig till huvudarmén i marschen mot Moskva. Lewenhaupt fick även uppdraget att skaffa en stor mängd proviant och trossvagnar som den kurländska armén skulle transportera till huvudarmén. I avvaktan på Lewenhaupts anslutning framryckte den svenska huvudarmén långsamt mot den ryska gränsen. Senare under året, vid övergången av floden Vabitj vid staden Holowczyn i juli 1708, stötte svenska förtrupper på en rysk armé som stod i försvarsposition på motsatta stranden. I slaget vid Holowczyn den 4 juli drevs de ryska försvarstrupperna under fältmarskalk Boris Sjeremetev och furst Aleksandr Mensjikov bort efter en åtta timmar lång strid, men slaget fick inga strategiska följder då den ryska armén lyckades undkomma mestadels intakt. +Efter denna seger dröjde den svenske kungen under nio veckor i Mohilew och trakterna närmast öster därom, mellan Dnepr och dess biflod Sozj, och väntade på Lewenhaupts försenade ankomst. Lewenhaupts uppgift var mycket komplicerad, och under halva vägen till huvudarmén blev hans armé på 12 000 man upphunnen av en rysk armé på 30 000 man under Peter I:s personliga ledning. Det resulterande slaget vid Lesna den 29 september slutade oavgjort, där båda parterna led stora förluster. För att förhindra att trossen skulle falla i ryska händer beslutade Lewenhaupt att bränna upp trossvagnarna och huvuddelen av provianten och avmarschera med sina resterande trupper. Den 23 oktober kunde Lewenhaupt till slut ansluta sig till huvudarmén, men bara med 6 500 man och utan den nödvändiga provianten. Armén led brist på livsmedel och tvingades förflytta sig till trakter där de kunde livnära sig; Peter I hade systematiskt låtit ryska strövkårer förhärja sitt eget land för att försvåra den svenska huvudarméns marsch mot Moskva. Under en överläggning mellan Karl XII, Rehnskiöld, kanslichefen Carl Piper och generalkvartermästaren Axel Gyllenkrok togs beslutet att armén skulle gå söderut till Severien, i riktning mot Lillryssland. Där skulle svenskarna kunna få goda vinterkvarter, proviant och truppförstärkningar genom en allians mellan Karl XII och zaporizjakosackernas hetman Ivan Mazepa. +Peter I fick höra om Mazepas allians med Karl XII, och skickade en straffexpedition under furst Mensjikov till att intaga och bränna ner Mazepas huvudstad Baturin och avrätta de kosacker som stod på Mazepas sida. De kosacker som avvisade Mazepa valde Ivan Skoropadskyj som ny hetman den 11 november 1708. Därmed hade Mazepa förlorat stora delar av sitt stöd hos zaporogerna och kunde förena sig med svenskarna med bara några tusen kosacker, utan den stora armé och de rikliga livsmedelstillgångarna han tidigare utlovade. I december 1708 kom dessutom en hård vinter som drabbade hela Europa, där tusentals svenska och ryska soldater dukade under för den bitande kylan. Svenska armén gick i bivack i början av december i trakterna kring Gadjatj. I närheten av staden låg fästningen Vepryk som spärrade vägen av den svenska framryckningen. Karl XII beslutade att storma fästningen den 7 januari 1709, som slutade med att den ryska garnisonen kapitulerade efter hårda strider där de svenska förlusterna var mycket omfattande. Den svenska armén stannade i trakterna runt Veprik innan den kunde röra sig i slutet av februari. Den marscherade söderut till en stark position mellan Dneprs bifloder Vorskla och Psiol för att där kunna ta hjälp av zaporogernas områden till att upprätta förbindelser med krimtatarerna, osmanerna och polackerna. I början av mars gick 4 000 zaporoger under deras hövding Kost Gordejenko över till Karl XII:s armé, och svenskarna byggde broar över nedre Vorskla för att kunna inleda militär samverkan med zaporogerna. När Peter I fick underrättelser om detta detacherade han ryska dragoner för att utföra straffexpeditioner. Den 19 april brändes byn Perevolotjna vid Vorsklas inlopp till Dnepr, liksom andra zaporogbyar uppströms Perevolotjna och ett flertal båtar. Den 15 maj stormades zaporogernas befästa läger Setj, beskyddad av omkring 2 000 zaporoger. Lägret brändes ner, 300 zaporoger togs tillfånga medan de övriga dödades eller lyckades fly. +Under mitten av våren gick trupperna över till Vorskla och anvisades kvarter längs floden mot staden Poltava och dess angränsande byar. Karl XII planerade att innesluta Poltavas fästning i ett försök att locka ut Peter I till en öppen strid. Poltava beskyddades av en ryskdominerad garnison under ledning av överste Aleksej Stepanovitj Kelin, som tillträdde kommendanttjänsten den 18 januari 1709 sedan Peter I efter Mazepas förräderi inte längre litade på fästningens tidigare kosackgarnison. Vid ankomsten till Poltava placerade Karl XII en vakt på 200 man som inneslöt hela staden och Gyllenkrok fick order att göra en ritning av staden och skapa en belägringsplan. De svenska regementena förflyttades till byarna runtom Poltava och med kungens tillstånd öppnade Gyllenkrok löpgravarna den 1 maj 1709. Arbetet gick långsamt eftersom svenskarna hade brist på skyfflar och eftersom infanteriet var svagt överläts arbetet till Mazepas kosacker. Då dessa var förskräckta av den ryska garnisonens beskjutning och flydde från löpgravarna var Gyllenkrok tvungen att hota dem med dödsstraff ifall de flydde på nytt. Från svenskt håll saknade man belägringsartilleri och ammunition för att kunna skapa en bräsch i fästningsvallen. Under belägringens gång utkämpade det svenska och ryska kavalleriet, tillsammans med sina allierade kosacker, flera skärmytslingar om kontrollen av de omkringliggande byarna och deras förnödenheter. Belägringen drog ut över sex veckor och för var dag blev flera svenska soldater dödsskjutna av ryska skarpskyttar eller av fästningens kanoner. Gyllenkrok övervägde olika metoder för att nå ett snabbt avgörande, men som slutade med misslyckade minsprängningar nära fästningsvallen och en planerad generalstormning som avblåstes på grund av brist på artilleriammunition. +På andra sidan Vorskla grupperade en större rysk styrka som steg för steg grävde löpgravar och byggde framskjutna fältbefästningar i form av skansar och redutter på den östra stranden för att skapa förbindelser med Poltavas garnison. För att motverka detta lät Gyllenkrok gräva löpgravar och bygga fältbefästningar längs floden söder, öster och väster om Poltava. När ryssarna genomförde olika aktioner för att skicka förnödenheter och truppförstärkningar till staden utvecklades situationen på de båda sidorna av floden till ett skyttegravskrig längs en nio kilometer lång frontlinje. Svenska insatsstyrkor låg stridsberedda uppe på högplatåerna nära floden för att snabbt kunna genomföra motanfall mot ryska trupprörelser. Dalgången var dock vattensjuk och under början av juni 1709 kom kraftiga regnfall som översvämmade båda stränderna. Den 5 juni anlände Peter I med ryska rekryter till de ryska fältbefästningarna på den östra stranden. Från detta håll kunde han inte uträtta något. Den ryska ledningen påbörjade ett flertal rekognosceringsturer längs floden för att leta efter en lämplig plats att föra över hela armén till den svenska sidan av Vorskla. Den 15 juni fattade ledningen beslutet om att genomföra en stor anfallsrörelse på flera ställen samtidigt mot de svenska posteringarna längs Vorskla. På kvällen innan det planerade anfallet proklamerade tsaren följande order: +På morgonen den 17 juni anföll Peter I:s armé de svenska och zaporogiska posteringarna vid Nizjnje Mlyny, Poltava och Petrovka. Vid Poltava anföll ryssarna vid det gamla broläget på en under natten byggd kavelbro, men som slogs tillbaka strax efter middagstid. Ryssarna återupptog striden samma kväll innan de utrymde sina ställningar på sin sida av Vorskla-floden. Vid byn Nizjnje Mlyny drygt tre kilometer söder om Poltavas fästning höll ett svenskt detachement ur Livdragonregementet ett par redutter. Dessa angreps av general Ludwig Nicolaus von Hallarts trupper, som under natten korsat Vorsklas huvudfåra och upprättat ett litet brohuvud. Anfallet slogs tillbaka när Karl XII kom dit med Dalregementets livbataljon. När Karl XII därefter var på väg tillbaka till Poltava träffades han i vänster fotblad av en muskötkula. Vid fästningen fortsatte han att övervaka verksamheten tills han matt av blodförlust måste låta fältskären Melchior Neumann behandla hans skottskada. Kungen insjuknade snabbt och under några dagar svävade hans liv i fara till följd av sårfeber och kraftig blodförgiftning. Då Karl XII inte som brukligt var kunde föra befälet övertogs det direkta befälet över den svenska armén till Rehnskiöld, som förhöll trupperna passiva vid Poltavas närhet. Under dagarna led den svenska armén av livsmedelsbrist och deserteringar. Den 21 juni hade den ryska huvudstyrkan hunnit över Vorskla och slagit läger vid byn Semjonovka. Den 22 juni fick Karl XII underrättelser om att situationen för kung Stanisław och generalmajor Krassow i Polen var mycket brydsam och att någon hjälp från dem inte var att vänta. Natten till den 26 juni flyttade Peter I sin armé söderut och slog läger vid byn Jakovtsi, fem kilometer norr om Poltava. Framför lägrets sydvästra hörn och vid Jakovtsi beordrade tsaren sina trupper att bygga ett antal redutter. +Det svenska högkvarterets vetskap om att inga förstärkningar var på väg, att ett ryskt storanfall skulle igångsättas inom kort och att underhållssituationen raskt förvärrades lämnades anfall som det enda tänkbara handlingsalternativet. Någon möjlighet att vänta ytterligare i hopp om att kungen skulle bli så återställd att han själv kunde leda sin armé på slagfältet fanns inte. Inte heller fanns möjlighet att pressa eller locka ryssarna till att anfalla först. På söndagen den 27 juni efter fältgudstjänsten tog Karl XII, som sakta började återhämta sig från sin feber, beslutet att gå till anfall följande morgon. Närvarande i beslutet var fältmarskalk Rehnskiöld, fältkanslichefen Piper och tjänstgörande "stabschefen" överste Gustaf Henrik von Siegroth, chef för Dalregementet. Kungen beordrade senare Gyllenkrok att indela infanteriet i fyra kolonner och ansvara för dess gruppering inför den kommande drabbningen. Rehnskiöld skulle senare sköta Gyllenkroks uppgift att rekognoscera slagfältet, och i slaget skulle Gyllenkrok fungera som observatör och sambandsofficer, fri från ansvar. Den svenska stridsplanen utformades av Rehnskiöld i samråd med kungen. Planen gick ut på att det svenska infanteriet skulle delas upp i fyra kolonner som innan gryningen skulle genomföra en överraskande genomrusning av det ryska reduttsystemet i form av ett kombinerat infanteri- och kavallerianfall. Det svenska kavalleriet skulle följa infanteriet bakom, uppdelade i sex kolonner, och driva bort det ryska kavalleriet stationerade bakom redutterna, för att därefter skära av de ryska reträttvägarna norr om det ryska lägret. Efter att infanterikolonnerna passerat redutterna skulle de anfalla det ryska fältlägrets västra front, samtidigt som de svenska ryttarna opererade mot lägrets norra front. Infanteriet skulle utdela huvudstöten i detta avslutande anfall mot lägret, varefter den ryska förskansade armén skulle stå instängd med ryggen mot floden och hotas av total undergång. +Enligt det svenska generalstabsverkets beräkningar kommenderade Karl XII vid Poltava och dess omnejd över 24 700 svenska reguljära trupper, varav 22 450 stridsdugliga, och omkring 6 000 irreguljära kosacktrupper och zaporoger. Det reguljära svenska infanteriet bestod av totalt 11 270 man, varav 1 070 underofficerare, med en effektiv stridande styrka på 9 300 man. Det reguljära svenska kavalleriet bestod av 6 840 ryttare och 6 240 dragoner, totalt 13 080 man, med en effektiv stridande styrka på 11 900 man. Antalet svenska officerare uppskattades till mellan 1 850 och 2 000. Mazepas och Gordejenkos trupper beordrades att stanna kvar i sitt härläger vid Pusjkarjovka. Inför huvuddrabbningen skulle den anfallande svenska huvudstyrkan bestå av 8 170 fotsoldater fördelade mellan 18 bataljoner och 7 800 ryttare och dragoner fördelade mellan 109 skvadroner. De i slaget deltagande officerarna uppskattades till 290 kavalleriofficerare, 240−290 dragonofficerare och omkring 680 infanteriofficerare eller totalt 1 210−1 260 officerare. Endast fyra 3-pundiga regementskanoner och fyra ammunitionsvagnar, betjänade av omkring 30 artillerister under kapten Hans Clerckberg, skulle medföras inför drabbningen. Innan drabbningen skulle inledas genomförde omkring 1 000 irreguljära valakryttare under överste Sandul Koltza ett avledande anfall mot de två ryska redutterna vid Jakovtsi för att dra ryssarnas uppmärksamhet från den svenska huvudstyrkan. De svenska trupperna var samlade med högkvarteret, Livgardet och infanteriet vid och väster om "klosterhöjden" nordöst om Poltava. Kavalleriet fanns i öppna läger mellan byarna Ribitsi och Pusjkarjovka, ett par kilometer väster om infanteriet. +I huvudstyrkan skulle det svenska infanteriet marschera på fyra kolonner under befäl av, från vänster till höger, av generalmajorerna Axel Sparre, Berndt Otto Stackelberg, Carl Gustaf Roos och Anders Lagercrona. Kavalleriet skulle ställas upp bakom infanteriet och indelas i sex kolonner. Den vänstra kavalleriflygelns tre kolonner skulle ledas av generalmajor Hugo Johan Hamilton och den högra av generalmajor Carl Gustaf Creutz. Rehnskiöld förde det högsta befälet, Lewenhaupt skulle leda infanteriet och kavalleriet skulle främst ansvaras av generalmajor Wolmar Anton von Schlippenbach. Karl XII ville iaktta striden personligen och lät sig bäras vid infanteriet på en bår som spänts mellan två hästar, tillverkad av Mora kompani i Dalregementet. En avdelning bestående av 24 särskilt utvalda soldater ur Livgardet och 15 ur Drabantkåren under befäl av drabantlöjtnant Johan Giertta skulle verka som kungens egen livvakt. I kungens uppvaktning som medföljde på slagfältet ingick hovstaten under överseende av hovintendent Gustaf von Düben, fältkansliet under Piper och statssekreteraren Olof Hermelin, utländska sändebud och militärer såsom Stanisław Poniatowski och James Jefferyes, hovpredikanten Jöran Nordberg samt flera adjutanter, läkare, taffeltäckare, pager och lakejer. +Den första infanterikolonnen längst till vänster under generalmajor Sparre bestod av Västmanlands regementes två bataljoner på 1 100 man under Erik Sparre, Närke-Värmlands regementes två bataljoner på 1 200 man, samt en bataljon på 300 man ur Jönköpings regemente under Georg von Buschwaldt. Den andra kolonnen under generalmajor Stackelberg bestod av Västerbottens regementes två bataljoner på 600 man under Gideon Fock, en bataljon på 380 man ur Östgöta infanteriregemente under Anders Appelgren samt Upplands regementes två bataljoner på 690 man under Gustav Stiernhöök. Den tredje kolonnen under generalmajor Roos bestod av Dalregementets två bataljoner på 1 100 man under Gustaf Henrik von Siegroth samt två bataljoner ur Livgardet. Den fjärde kolonnen längst till höger under generalmajor Lagercrona bestod av en bataljon på 500 man ur Kalmars regemente under Gustaf Ranck, en bataljon på 500 man ur Skaraborgs regemente under Carl Gustav Ulfsparre, samt två ytterligare bataljoner ur Livgardet. Livgardet bestod av sammanlagt 1 800 man under Carl Magnus Posse. +Den vänstra kavalleriflygelns tre kolonner under generalmajor Hamilton bestod av Skånska ståndsdragonregementets 8 eller 10 skvadroner under prins Maximilian Emanuel av Württemberg-Winnental, Östgöta kavalleriregementes 8 skvadroner under Hamilton, Nylands kavalleriregementes 10 skvadroner under Anders Torstenson, Åbo och Björneborgs läns kavalleriregementes 8 skvadroner under Gustaf Eneskjöld, Upplands ståndsdragonregementes 8 skvadroner under Anders Wennerstedt, 10 skvadroner ur Nils Gyllenstiernas dragonregemente samt 10 skvadroner ur Carl Gustaf Dückers dragonregemente. Den högra kavalleriflygelns tre kolonner under generalmajor Creutz bestod av 1 skvadron ur Drabantkåren under Carl Gustaf Hårdh, Livdragonregementets 8 skvadroner under Philip Örnestedt, 12 skvadroner ur Livregementet till häst under Creutz, Smålands kavalleriregementes 8 skvadroner under Johan Valentin von Daldorff, 8 skvadroner var ur Norra respektive Södra skånska kavalleriregementen under Gustaf Horn respektive Carl Gustaf Örnestedt, 8 eller 10 skvadroner ur Nils Hielms dragonregemente samt 8 skvadroner ur Gustaf Adam Taubes dragonregemente. +Både Karl XII och Rehnskiöld ansåg att de resterande 8 000 svenska trupperna inte skulle behövas inför striden och beordrade dessa att bevaka trossen, belägringsverken vid Poltava och förbindelserna längs Vorskla. Vid Poltava stod en belägringsstyrka på 1 300 man, bestående av en bataljon vardera ur Södermanlands respektive Kronobergs infanteriregementen under Gabriel von Weidenhaijn respektive Johan Cronman, 4 skvadroner ur Livdragonregementet och belägringsartilleriet, liksom en fältvakt på 30 ryttare vid Poltavas förstad. Det fanns en postering vid Nizjnje Mlyny bestående av en bataljon på 200 man ur Livgardet under Gabriel Johan Lagerhjelm, medan en postering vid Bulanovka tolv kilometer söder om staden, bestående av 30 dragoner under sekundlöjtnant Carl von Roland, drogs in den 27 juni. Vid Novyje Sanzjary stod generalmajor Johan August Meijerfeldts dragonregemente på 1 000 man och sedan 26 juni artilleriets bakre trossdelar. Längs Vorskla nedströms fästningen fanns svenska posteringar vid Bilyki, Kobeljaki och Sokolki, liksom svenskvänliga kosacker och zaporoger längs Psiols nedre lopp. Vid Bilyki stod 300 ryttare huvudsakligen från Dückers dragoner under överstelöjtnant Thomas Funck, och vid Kobeljaki och Stokolkow stod 500 ryttare huvudsakligen från Södra skånska kavalleriregementet under överstelöjtnant Göran Silfverhielm. +Kvar i det svenska högkvarteret vid Pusjkarjovka fanns 2 200 svenska soldater, mestadels sjuka och sårade, 28 varierande pjäser ur fältartilleriet och Mazepas och Gordejenkos kosacker och zaporoger. Trossen bestod av Adelsfanans fyra skvadroner under Anders Ramsvärd, en skvadron ur Livländska adelsfanan, fyra skvadroner var ur Karelska kavalleriregementet respektive Upplands ståndsdragonregemente under Carl Kruse, fyra skvadroner ur Schlippenbachs dragonregemente under överstelöjtnant Arvid Johan von Kaulbars, fyra skvadroner ur Gustaf Carl von Schreiterfelts dragonregemente samt 4 skvadroner ur Henrik Otto von Albedyls dragonregemente. Artilleriet under befäl av överste Rudolf von Bünow bestod av sexton 3-pundskanoner med 150 skott, fem 6-pundskanoner med 110 skott per eldrör, två haubitser på 16 pund med 45 skott per eldrör och fem mörsare på 6 pund med 15 granater per eldrör. +Peter I kommenderade över 51 000 reguljära ryska trupper och 23 000 irreguljära kosacker och kalmucker. Det ryska reguljära infanteriet räknade högst 24 500 man i slaglinjen inför huvuddrabbningen. Det ryska reguljära kavalleriet bestod inför huvuddrabbningen av omkring 14 600 dragoner. Av de ryska trupperna befann sig omkring 60 bataljoner, omkring 27 000 man, och artilleriet inom det befästa lägret. Där fanns även Peter I:s stab och uppvaktning. Två infanteriregementen stod som säkerhetsbesättning i det gamla lägret vid Semjonovka och redutterna vid Petrovka. De sjuka och sårade ryska soldaterna var bortförda från lägret, medan antalet hästar och civila är oklart. Som skyddstrupp vid den ryska tross som förts norrut till Rubljovka fanns minst ett regemente. Det ryska reguljära kavalleriet, som bestod av omkring 30 dragon- och dragongrenadjärregementen med 20–23 haubitser och kanoner, totalt omkring 20 000 dragoner var förlagda längs Maloje Budisjtjeskogens bryn, strax norr om reduttlinjen och väster om Jakovtsilägrets västra front. Kavalleriet leddes av Aleksandr Mensjikov i samverkan med generallöjtnant Carl Ewald von Rönne. Under generalmajoren furst Grigorij Semjonovitj Volkonskij avdelades sex dragonregementen eller omkring 4 000 dragoner för samverkan med Skoropadskyjs irreguljära kavalleri. +Under senare delen av slaget detacherades från lägret en särskild stridsgrupp som under Mensjikovs befäl skulle anfalla eventuella svenska reservtrupper. Gruppen bestod av 6 299 man fördelade mellan 3 813 dragoner under generalmajor Johan Kristian Heinske och 2 486 fotsoldater under generallöjtnant Samuel Rentzel. Kavalleriet utgjordes av Troitska regementets sju skvadroner på 771 man, Smolenska regementets tio skvadroner på 992 man, Jamburgska regementets sex skvadroner på 617 man, Sankt Petersburgska regementets sju skvadroner på 723 man samt Kargopolska regementets sju skvadroner på 710 man. Infanteriet utgjordes av Tobolska regementets två bataljoner på 1 006 man, Koporska regementets bataljon på 517 man, Andrej Jurivitj Inglis regementes bataljon på 654 man samt Fichtenheimska regementets bataljon på 309 man. +Enligt den rysk-ukrainske historikern Valerij Moltusovs studier ska de tolv redutterna utanför det ryska lägret ha bemannats av 4 730 man från sex infanteriregementen under brigadör Savva Vasiljevitj Ajgustov. Styrkan bestod av Belgorodska regementets två bataljoner på 1 067 man under Vasilij Jakovlevitj Levasjov, Netjajevs regementes två bataljoner på 634 man under Ivan Konstantinovitj Netjajev, Nelidovs regemente 2 bataljoner på 749 man under Ivan Fjodorovitj Nelidov, Nekljudovs regementes två bataljoner på 705 man under Matvej Nekljudov, Jamburgska regementets två bataljoner på 682 man under Jakov Jurievitj Vestov samt von Deldins regementes två bataljoner på 893 man under Jeremej Vilimovitj von Deldin. De två redutterna belägna vid Jakovtsi bemannades med 1 700 soldater, medan de tio övriga redutterna ska bemannades av i genomsnitt 300 soldater vardera. De sex första anlagda redutterna i baslinjen hade en besättning på 200 soldater vardera. Varje långsida skulle ha bemannats av 50 man, men i verkligheten skulle endast 25 aktiva skyttar ha stått i beredskap medan den halva styrkan skulle ladda om skyttarnas vapen. Vidare bestyckades redutterna av mellan 15 och 20 medeltunga kanoner och bakom den tvärgående reduttlinjen placerades 13 stycken 3-pundiga mörsare. Bakom tvärlinjen grupperades omkring 10 000 ryska dragoner eller 16 dragonregementen under Mensjikovs befäl, formerade mellan 74 skvadroner på tre linjer bakom varandra. Kavalleriet till vänster kommenderades av generallöjtnant Rönne, i mitten av generalmajor Heinske och till höger av generallöjtnant Rudolf Felix Bauer. Ett beridet grenadjärregemente ställdes på båda sidorna av kavalleriet som flankskydd. +I Poltavas fästning fanns omkring 3 000 soldater under kommendant Kelins befäl, chef för Tverska regementet. På fästningens vallar fanns 28 mindre kanoner med omkring 100 artillerister under fortifikationschef Jean de Terson. I belägringens inledning bestod garnisonen av fyra bataljoner ur Kelins regemente och Ustiuzjska regementet under överste Ivan von Mengden. Natten mellan den 15 och 16 maj förstärktes garnisonen med en avdelning om 900 man under befäl av brigadören Aleksej Aleksejevitj Golovin. I avdelningen ingick tre mindre bataljoner ur Fichtenheimska regementet under överste Ivan Samuilovitj Fichtenheim, Nevska eller Apraksins regemente under överstelöjtnant Vasilij Pestrikov och Permska regementet under överste Timofej Ivanovitj Treiden. Av Poltavska kosackregementet under överste Ivan Prokofievitj Levenet ingick under belägringen några kompanier i Poltavas garnison, medan de övriga var disponerade på andra håll. +Valplatsen låg på den västra sidan av floden Vorskla, som rann i sydlig riktning mot Dnepr. Floden var uppdelad i flera biflöden, åar och bäckar som slöt upp längs en sank och sumpig flodsänka som bildade en mindre dal, mellan en och två kilometer vid. Väster om Vorsklafloden reste sig en vidsträckt slätt och som bildade ett stort, flackt platålandskap som genomkorsades av flera skogbevuxna raviner. Söder om valplatsen, på platån alldeles vid de höga sluttningarna ned mot den västra stranden av Vorskla, låg staden Poltava. Staden hade sitt läge där den gamla leden mellan Kiev och Kharkov klev över Vorskla, och var omgiven av raviner och byar. Hela stadsområdet inklusive förstaden omfattade något över en kvadratkilometers yta. Själva fästningsområdet och den del av stadsbebyggelsen som det inneslöt var cirka 1 000 × 600 meter. Befästningsområdet klövs i två delar av en ravin som genomkorsade landskapet. Av befästningsområdets två delar var den norra större och rymde den egentliga staden, medan den södra utgjordes av den mindre förstaden Masurovka. Sluttningarna upp mot Masurovka kläddes av körsbärslundar. Strax nordväst om fästningen låg på slätten en under belägringens gång nedbränd förstad som omgavs av en lång bågformad jordvall. +Strax öster om fästningen ledde branta skogbeväxta backar ner till floden och ett drygt en kvadratkilometer stort fält, som översvämmades varje år. Söder om Poltava löpte en lång ravin där belägringstrupperna hade sin bivack. Norr om staden skar en lång och bred ravin in i höglandet ovan floden som löpte i nord-sydlig riktning mot den vidsträckta Jakovtsiskogen, med ett ravinsystem fyllda av bäckar och små tjärnar. Skogen sträckte sig från Poltavas utkanter i söder till byn Jakovtsi med det ryska huvudlägret i norr, byarna Pavlenki och Maloje Pavlenki i väst och sluttningarna till floden Vorskla i öst. Drygt en kilometer nordväst om Poltava reste sig en långsträckt höjd som slöt an till Jakovtsiskogen, och dess sluttningar var täckta av vindruvsodlingar och stora körsbärsträdgårdar. På denna höjds södra spets låg ett kloster där den svenska ledningen hade sitt högkvarter. Vid klosterhöjden hade också det svenska infanteriet sitt bivack dagarna innan slaget. Drygt fyra kilometer väster om infanteriets läger låg det svenska kavalleriets vidsträckta läger på den vågiga slätten mellan byarna Ribitsi och Pusjkarjovka. Söder om Pusjkarjovka stod större delen av arméns tross med dess tusentals vagnar uppställda delvis i skydd av en stor ravin. +Slätten som omgav det ryska lägret i väst och norr var relativt platt och svagt lutande, och täcktes av torr lössjord. Ungefär en kilometer väst om lägret bildade böljorna i terrängen en stor sänka i närheten av byn Pobivanka. Precis intill den låg det vidsträckta Maloje Budisjtjeskogen. Denna sträckte sig norrut och i en vid båge åt nordväst, där den följde Ivantjinskijbäcken som kröp fram i en svag sänka kallad "Lilla moraset", vars botten var fylld av gyttjiga moras och en del små dammar. Utspridda i trakterna fanns det också en stor mängd jordkulor, i vilka befolkningen förvarade sina livsmedel, och som utgjorde farliga hinder för förbipasserande ryttare. Vid bäckens flöde stod byarna Maloje Budisjtje och Ivantjinskij med stugor gjorda av lera och halmtak, omgivna av gärdesgårdar och körsbärsträd. Mellan Maloje Budisjtje- och Jakovtsiskogarna fanns en mellan 1 200 och 1 500 meter bred korridor, beväxt med en del buskar och en och annan trädgrupp. I dess längdriktning löpte en svag ås som skilde Maloje Budisjtjeskogen från den så kallade "Stora ouvragen", ett skogbeklätt ravinsystem med byarna Tachtaulov, Osmatjiki och Zjuki utanför dess norra skogsbryn. Korridoren mellan Maloje Budisjtje- och Jakovtsiskogarna var den enda raka vägen som svenska trupper kunde använda för att angripa det ryska lägret. Då den ryska ledningen förstod korridorens betydelse beslutade Peter I att anlägga befästningar för att blockera svenskarnas marschväg. Först anlades sex redutter tvärs över öppningen mellan de två skogarna. Sedan gavs order om att bygga fyra ytterligare redutter i rät vinkel ut från baslinjen i riktning mot svenskarna. +Klockan tjugotre den 27 juni gav Rehnskiöld order om uppbrott. Innan marschen höll armén korum med bön, varefter soldaterna sjöng tyst vers 6 ur psalm 96 från 1695 års psalmbok. Strax efter klockan ett på natten begav sig de svenska kolonnerna iväg under tystnad mot de ryska redutterna, samtidigt som Koltza med sitt utgångsläge från klosterhöjden genomförde sitt avledningsanfall utmed Vorskla mot byn Jakovtsi. Redan från början blev marschen problemfylld då en del enheter på grund av nattmörkret hamnade på fel plats, varefter Rehnskiöld hade ett häftigt meningsutbyte med Lewenhaupt. Längst ut på den vänstra flanken red generalmajor Hamilton fel med sina kavalleriavdelningar. Infanteriet som kom först på plats blev tvungna att stanna kvar i sina formationer. Vid denna tidpunkt gav sig generalmajor Schlippenbach ut på en rekognosceringstur för att bilda sig en uppfattning om terrängen och kavalleriets anfallsvägar. Han försvann dock bland de ryska linjerna och kvar på platsen var generalmajor Creutz, som nu i praktiken fick leda det svenska kavalleriet under resten av dagen. Vid gryningen gick det svenska infanteriet över till linjeformation. +På natten till den 28 juni red Rönne med en rysk rekognosceringspatrull mot det svenska lägret, där han upptäckte en tyst, men febril aktivitet. Han var övertygad om att svenskarna gjorde sig redo för utmarsch och återvände med sin patrull till reduttlinjen. Han beordrade sina dragoner att göra sig redo för strid och runtomkring honom pågick arbeten med att bygga nya redutter. +Under natten begav sig Gyllenkrok med några fortifikationsofficerare på en rekognosceringstur när han fick höra ett huggande och bultande ljud i horisonten. När han red ut på fältet för att undersöka ljudet upptäckte han att ryssarna under natten hade byggt fyra redutter som låg i en rät vinkel framför svenskarnas marschrutt. På vägen tillbaka stötte han på Rehnskiöld ensam på fältet och rapporterade för honom om hans upptäckt, som blev fullständigt överraskad och tvingades lägga om sin stridsplan. Rehnskiöld red tillbaka till infanteriet och beordrade dem att återigen ställa upp i marschordning i fyra kolonner. Gyllenkrok fortsatte sin rekognosceringstur utanför de ryska redutterna och avbröts plötsligt av pistolskott, följt av trumvirvlar. De ryska besättningarna som arbetade på de fyra nybyggda redutterna kastade ifrån sig sina verktyg och intog sina eldställningar. Rönnes dragoner rullade sina kanoner i ställning mellan redutterna och avfyrade sina första skott, som dödade två svenska ryttare. Ungefär samtidigt slog ryssarna larm vid de två redutterna nära Jakovtsi, där ryssarna fällde muskötsalvor mot Koltzas valakregemente. Anfallet slogs tillbaka och Koltza blev tillfångatagen, men aktionen gjorde att den ryska ledningen blev osäkra om var den svenska armén skulle kraftsamla sitt anfall. +Den svenska huvudstyrkan hade därmed förlorat överraskningsmomentet och den svenska ledningen blev osäkra över nästa drag. Armén saknade material och tungt eldunderstöd för att kunna storma de ryska fältbefästningarna, men samtidigt var arméns försörjningsläge i fortsatt nödläge. Efter ett rådslag med Karl XII, Piper och Lewenhaupt fattade Rehnskiöld beslutet att fullfölja anfallsrörelsen, som skedde klockan fyra på morgonen den 28 juni. På grund av tidsbristen för ordergivningen till de inblandade kolonnbefälhavarna och till regements- och bataljonscheferna blev resultatet att dessa fick tvetydiga tolkningar av sina uppgifter, där en del uppfattade att man skulle anfalla och genom en öppning slå sig förbi reduttlinjen, medan andra menade att man skulle neutralisera hotet från redutterna. En rusning genom den tvärgående reduttlinjen försvårades av de fyra nybyggda redutterna, som stod längs med den svenska anfallsriktningen. De svenska kolonnerna skulle därför marschera i behörigt avstånd från varandra och mellan redutterna, vilket gjorde att dessa riskerade artillerield från en längslinje som sträckte sig i en och en halv kilometer, och med en marschtakt om 75 meter per minut skulle kolonnerna riskera tjugo minuters eld i flanken. +Ryska posteringar hade omedelbart och i god tid meddelat svenskarnas rörelser mot reduttsystemet till Mensjikov, som vidarebefordrade nyheterna till Peter I och Sjeremetev. Mensjikov beordrade sedan det ryska kavalleriet att sitta upp och göra sig stridsberedda. I lägret beordrade Peter I de förskansade bataljonerna att göra sig färdiga till batalj. Dessa aktioner verkställdes redan innan de första svenska bataljonerna nådde fram till det ryska reduttsystemet. +Klockan fyra på morgonen avfyrade de fyra svenska regementskanonerna dubbel svensk lösen, och det svenska infanteriet med kavalleriet bakom sig började marschera mot de ryska redutterna. Vid anfallets inledning rådde förvirring i de svenska leden, då det rådde oklarhet om förbandens formering, vilket resulterade i att flera bataljoner inte kunnat slutföra sina formeringsrörelser under stridens inledning, och genom kavalleriets felridningar hade hela armén råkat i akut tidsnöd. När fälten skådade dagens ljus befann sig det svenska infanteriet inom räckvidd för de främsta redutternas kanoner, som påbörjade sin bombardemang på längsta skottvidden och dödade enstaka svenska fotsoldater, inklusive två gardister invid kungens hästbår. +De två mittersta kolonnernas anfall var oredigt från början och bataljonerna kom snabbt att skiljas från varandra, men trots detta gick de främsta förbanden raskt framåt. Den främsta redutten angreps från vänster av generalmajor Stackelberg med Västerbottens regementes två bataljoner och från höger av överste Siegroth med Dalregementets två bataljoner. Dessa fick även understöd från Livdragonregementets fyra skvadroner, som anföll en mindre rysk styrka som stod i mellanrummet till nästa redutt. Redutten hade inte byggts färdig och intogs snabbt, och de flesta i besättningen dödades medan de överlevande flydde in i nästa redutt. Den andra redutten var bättre förstärkt men den intogs på kort tid av Dalregementet och hela dess besättning dödades. Samtidigt som detta skedde framryckte kolonnernas bakre bataljoner, två gardesbataljoner ur Roos kolonn och en östgötsk och två uppländska bataljoner ur Stackelbergs kolonn, och strömmade förbi till vänster. De ryska redutterna sköt ihållande musköt- och kanoneld mot de anryckande svenska kolonnerna, som led svåra förluster, vilket skapade stora moln av fet krutrök som täckte hela slagfältet och försvårande de svenska officerarnas överseende. +Karl XII ihop med sitt följe och generalstaben följde med den fjärde kolonnen längst till höger, som under sin framryckning hade dragit sig längre högerut, in i Jakovtsiskogen, för att snabbare kunna ta sig förbi redutterna. Vid det här laget anmärkte generalmajor Roos att han förlorat kontakten med de två gardesbataljonerna som ingick i hans kolonn. Han fick därefter veta att bataljonerna, av någon outgrundlig anledning, fick order om att ansluta sig till generalmajor Lagercronas fjärde kolonn av Livgardets chef, överste Posse. Samtidigt fortsatte Siegroth och Roos framåt med Dalregementet för att understödja två bataljoner från Jönköpings och Närke-Värmlands regementen som påbörjat anfallet av en tredje och starkare befäst redutt. Tillsammans med Västerbottens regementes två bataljoner från Stackelbergs kolonn genomförde Roos tre anfall mot den starka redutten och samtliga slogs tillbaka av dess besättning. I samtliga sex svenska bataljoner som deltog i reduttstriden blev manfallet stort. I rask takt stupade Jönköpingsbataljonens regementschef Buschwaldt, liksom överstelöjtnant Natt och Dag och major Oxe, vilket lämnade kapten Mörner som kommenderande officer. I närheten blev även överste Siegroth dödligt sårad. Då denne var en av ytterst få officerare som varit med när kungen meddelade sin stridsplan, blev Roos och hans bataljoner kvarstående bland redutterna och saknade nya anvisningar. +Under tiden som de övriga tre kolonnerna efter anfallets inledning var på väg att passera reduttlinjen, var Roos trupper fortfarande kvar i den invecklade striden mot den tredje redutten. Med omkring 40 procent eller 1 100 i antalet stupade och sårade bland trupperna, blev löjtnant Olof Pommerijn från Dalregementet tvungen att uppsöka Roos och anhålla om att trupperna borde avbryta anfallet och dra sig tillbaka. Roos gav med sig och drog sedan slutsatsen att besättningen i den tredje redutten var för stark. Vid sextiden beordrade Roos att de överlevande förbanden skulle återsamlas österut i brynet av Jakovtsiskogen nära Maloje Pavlenki, där han sett sårade svenskar krypa till. Då flera officerare hade stupat blev Roos tvungen att själv omorganisera hans återstående 1 500 soldater. Dalregementets och Västerbottens regementes respektive två bataljoner slogs ihop till en enda, vilket lämnade Roos med en bataljon vardera från Dal-, Västerbottens, Närke-Värmlands och Jönköpings regementen. Kort därefter anlände på fältet generalmajor Schlippenbach och hans spaningsavdelning, åtföljd av kapten Nils Bonde, som brutit sig igenom och kom från Rehnskiöld för att visa vägen till armén. Roos bad Bonde att avvakta tills han organiserat sina trupper till strid. +Under hela den svenska framryckningen låg svenskarna under eldgivning från Rönnes dragonartilleri. Även hans chef Mensjikov hade kommit på plats för att rida till motangrepp med dragonerna. Dessa framryckte genom den bakre reduttlinjen och angav karakollartade karbinsalvor mot de anryckande svenska kolonnerna. Den svenska högra kavalleriflygeln under Creutz var i oordning och pressades ut till höger av artillerielden. Utrymmet var mycket trångt mellan det svenska infanteriet, redutterna och skogen. Creutz beordrade dock ett omedelbart motanfall och de svenska skvadronerna travade igenom trängseln för att möta de ryska dragonerna i strid. Dessa anföll svenskarna i omgångar och hade några lokala framgångar genom att kasta tillbaka de första svenska skvadronerna, vilket fick Mensjikov att anhålla för Peter I om att han kunde hejda det svenska anfallet ifall han fick stöd från det ryska infanteriet. Tsaren ville dock inte riskera sitt infanteri och ville först avvakta stridens utveckling. Först när svenska skarpskyttar från infanterikolonnerna började tillstöta de ryska dragonerna började dessa pressas bakåt mellan redutterna under mycket hårda strider. +Den svenska vänstra kavalleriflygeln under Hamilton anföll Bauers ryska dragoner på en bredare front än den högra. Den sträckte sig från Maloje Budisjtje i väst och mellan de närmast intill belägna redutterna nära skogsbrynet i öst. Samtidigt hittade Hamilton där en väg för att kunna kringgå tvärlinjens yttersta redutt. Generalmajor Sparre medföljde Hamilton med Västmanlands regementes båda bataljoner och en bataljon av Närke-Värmlands regemente, medan resten av hans kolonn var indragen i striden mot den tredje redutten. Även här var framkomligheten svår för den vänstra kavalleriflygeln, och både fotsoldaterna och ryttarna tvingades ta sig igenom skog med nedhuggna träd, moras, dolda jordkulor och Maloje Budisjtje. När ryttarna kom förbi reduttlinjen red dessa till anfall mot det ryska kavalleriet på denna flank. Efter en kort strid tvingades ryssarna retirera och pressades långsamt bakåt av de svenska ryttarna. +Hamiltons framgångsrika anfall sprängde flera ryska förband och skapade panik bland ryssarnas irreguljära förband. Vid ett tillfälle gick en parlamentär för en stor grupp kosacker fram till Skånska dragonregementets chef, prins Maximilian, och erbjöd att med kungens medgivande gå över till den svenska sidan med 2 000 kosacker. Parlamentären fick dock vända tillbaka efter att prinsen själv inte kunde bevilja dennes anbud, och besvarade: "emedan vår nådige konung under aktionen blivit från oss separerad och nu intet tillstädes var, att man hans nådige vilja häröver kunde inhämta." +Kavalleristriden på den svenska högra flygeln böljade fram och tillbaka och kunde avgöras till svensk fördel först efter ett antal kavallerichocker. Peter I hörde med stigande oro ljudet av de hårda striderna från reduttlinjen och beordrade Mensjikov att retirera. Mensjikov ville dock fortsätta striden och skickade 14 erövrade svenska fälttecken till tsaren ihop med en önskan om förstärkning. Men Peter I lät sig inte övertygas och gav en skarp tillsägelse till Mensjikov om att dra sig tillbaka. Efter en timslång strid vek det ryska kavalleriet bakåt och Creutz började förfölja dem tvärs över det öppna fältet mellan det ryska lägret och Maloje Budisjtjeskogen. Mensjikov lät skicka större delen av kavalleriet under Bauer till huvudlägrets norra flank och drog sig själv med en mindre avdelning tillbaka till lägrets västra och södra flank. De ryska kanonerna på lägervallarna avfyrade bakom de retirerande ryska ryttarna för att förhindra Creutz ryttare från mer energiskt förföljande. Jakten mot Bauers ryttare gick dock vidare förbi det ryska lägret mot den stora ouvragen. När omkring en kilometer återstod till ouvragen fick de svenska ryttarna order av Rehnskiöld att avbryta sin förföljelse och återvända till slagfältet. Genom denna befallning räddades det ryska kavalleriet ur en svår knipa och kunde dra sig undan till och vid lägret för att omgruppera sig. Kavalleriets förluster under slagets första skede räknade omkring 1 000 i antalet stupade och sårade. Bland de senare var generallöjtnant Rönne, som överlämnade sitt befäl till Bauer. +Medan kavalleristriderna pågick kunde det svenska infanteriet med ryttarnas understöd ordna sina led och fortsätta sin framryckning, trots den mördande elden från redutterna. Parallellt med Creutz förföljelse av det ryska kavalleriet stormades och erövrades en av de mellersta redutterna av tvärlinjen av Östgöta och Upplands infanteriregementen från den andra kolonnen, med understöd från en bataljon av Närke-Värmlands regemente ur Sparres kolonn, som därefter marscherade vidare åt nordväst. Parallellt med Hamiltons kavallerianfall till vänster gick Sparre och Västmanlands regemente rakt på och erövrade redutten längst till vänster av tvärlinjen och fortsatte vidare genom Maloje Budisjtjeskogen. Lewenhaupt som marscherade längst till höger med Lagercronas kolonn och Livgardets samlade styrka upptäckte att man gick fram för snabbt och att de övriga kolonnerna till vänster fortfarande satt fast bland redutterna, vilket gjorde att ett glapp uppstod i formationen. Lewenhaupt anhöll för Rehnskiöld om att man borde bromsa samtliga kolonner för att kunna invänta de övriga bataljonerna och åter ordna stridslinjen; ett förslag som delades av Gyllenkrok. Men Rehnskiöld som var i närheten beordrade full fart framåt för att inte ge ryssarna betänketid, en uppfattning som delades av Stackelberg. Därmed kom de bataljoner som stred mot den tredje redutten att bli efter och att till slut tappa kontakten med huvudstyrkan. +Under rådande omständigheter beslöt Lewenhaupt att med sina samlade sex bataljoner gå mot det ryska huvudlägrets södra flank, där han upptäckte en svag punkt i fältbefästningen. De passerade en ravin framför lägret och på omkring en kilometers avstånd förberedde man sig för att storma det ryska lägrets södra fältverk. De ryska artilleripjäserna som upptäckte vad som föregick nära lägret började beskjuta Lewenhaupts bataljoner, liksom det framstormande svenska kavalleriet som jagade fram över slätten. Samtidigt påbörjade åtskilliga ryska infanteriförband på Peter I:s order en utrymning av det ryska lägret över till den andra sidan av floden Vorskla. +Rehnskiöld var orolig över den utvecklade situationen. Han upptäckte att den vänstra infanteriflygeln hade hunnit en bit bort från redutterna, men att Roos bataljoner hade sackat efter och som fortfarande var invecklad i striden mot den tredje redutten och förlorat flera soldater bland sina led. I stället för Lewenhaupts påbörjade inbrytning av det ryska lägret hamnade stridens fokus på Roos onödiga kamp mot den tredje redutten, vilket gjorde att den svenska ledningen förlorade tyglarna över striden till förmån för ryssarna. Rehnskiöld ville nu hålla armén så tätt samlad som möjligt och skickade adjutanter till både Hamilton, Creutz och Lewenhaupt med strikta kontraorder om att avbryta sina anfall. Han beordrade dessa att marschera västerut och återsamla sig till Pobivankasänkan väst om det ryska lägret, i närheten av Maloje Budisjtjeskogen. +Under framryckningen förbi redutterna hade Karl XII och hans följe marscherat en bit bakom Lewenhaupts bataljoner. Följet blev ansatta av kosacker på alla sidor, men dessa utgjorde inget större hot och blev sedan bortdrivna av en livdragonskvadron. När följet sedan marscherade samma väg som de svenska ryttarna råkade man hamna i korseld från reduttlinjen. Hästbårens högra bärstång blev avskjuten vilket gjorde att båren behövde hastigt repareras mitt på slagfältet. Dessa reparationsarbeten skedde under kraftig rysk beskjutning som fällde 8 drabanter och 10 gardessoldater, medan kungen själv klarade sig med minsta möjliga marginal. När båren var reparerad följde Karl XII med Östgöta infanteriregemente i den andra kolonnen. Vid sextiden började Creutz kavalleri och Lewenhaupts bataljoner lämna sina utsatta positioner och armén samlades i Pobivankasänkan. Sparre som tagit en omväg genom skogen till vänster var den förste som kom till rätta med Västmanlands regemente. Det saknades dock infanteri från tre av de fyra kolonnerna. Rehnskiöld beordrade Sparre att marschera tillbaka mot reduttsystemet med sitt regemente och med Hielms dragonregemente för att leta efter Roos trupper och försöka lotsa dem tillbaka till huvudstyrkan, som nu började samlas vid sänkans norra ände i närheten av lilla moraset. Den samlade skaran soldater passade på att vila på sina strama formeringar, samtidigt som Karl XII slöt upp och gratulerades av sin uppvaktning till de vunna framgångarna. +Enligt Moltusovs studier hade ryssarna fått underrättelseunderlag som pekade på att en andra svensk anfallsväg var på väg med reservtrupper och fältartilleriet. Mensjikov fick order av Peter I att möta det förmodade hotet med en stridsgrupp, bestående av Samuel Rentzels fem infanteribataljoner och Heinskes fem dragonregementen. Rentzels bataljoner gick genom skogen samtidigt som Heinskes dragoner tog vägen via redutterna. Ryska bataljoner från lägret återbesatte de redutter som tidigare erövrats under morgonens strider. Detta tycks ha hänt samtidigt som Rehnskiöld försökte samla sina splittrade enheter. Någon ankommande svensk artilleristyrka kom inte, men av en händelse träffade Mensjikov i stället på Roos bataljoner i skogsudden. Han beslöt att omedelbart gå till anfall, i tron om att dessa var de svenska reservtrupperna. +Roos upptäckte de hotande ryska dragonerna på fältet och de ryska bataljonerna som framstack sig genom skogen och längs fältkanten. Han beordrade sina kvarvarande fyra bataljoner att formera sig i en öppen fyrkant; Jönköpingsbataljonen ställdes upp i rät vinkel mot Närke-Värmlands bataljon som ställdes upp med front mot de ryska bataljonerna. Invid dessa rangerades västerbottningarnas bataljon, medan dalkarlabataljonen ställdes upp på den vänstra kanten, som med sin vänstra flygel begravd i en liten hasseldunge fick uppdraget att hindra de ryska dragonerna från att anfalla svenskarna i ryggen. Några skvadroner under Heinskes befäl red an mot svenskarna, svepte in och vände om i en upprepad procedur utan skottlossning, i syfte att rekognoscera de svenska dispositionerna. Dessutom nåddes svenskarna fortfarande av artillerield från redutterna. Vid detta tillfälle bestämde sig Schlippenbach att lämna oordningen på platsen för att leta efter Karl XII och resten av armén. Hans avdelning red dock strax in i några ryska förband som höll på att anfalla Roos trupper. Efter en kort strid kastade ryssarna de svenska ryttarna över ända och tillfångatog Schlippenbach. Gardeskaptenen Carl Palmfelt lyckades dock komma undan och fortsätta sin vilda ritt mot resten av armén. Samtidigt anfölls Roos från två håll. +På grund av ryssarnas överlägsenhet i både manskap och material, liksom att nästan samtliga officerare hade stupat, tvingades Roos att med sin illa tilltygade skara retirera från skogsbrynet och gå längre in i Jakovtsiskogen mot det svenska infanteriets tidigare förläggning i söder, i hopp om att där kunna finna undsättning. Mensjikov beslöt därefter att överlämna befälet till Rentzel och återvända till tsaren, då han ansåg sig ha löst sin uppgift. Rentzels trupper förföljde svenskarna obönhörligt från flera sidor genom skogen, och emellanåt gjorde Roos halt för att sätta sig till motvärn mot förföljarna. Efter ojämna skottsalvor mot alla sidor satte svenskarna snabbt i rörelse igen. I det svenska infanteriets tomma förläggningsområde hade Heinskes dragoner hunnit före Roos trupper, varpå Roos drog sig vidare mot belägringstrupperna vid Poltava. De kom till klosterhöjden och hittade en liten grupp svenskar som barrikaderat sig inne i klostret. Roos uppmanade gruppen att ansluta sig till återtåget, men dessa vägrade att följa med. När Roos och hans trupper kom nedför backen mot belägringsverken blev dessa blockerade av fyra ryska bataljoner som marscherat ut från staden och ryckte an mot Vorskla, Roos till mötes. Svenskarna tvingades därför dra sig ned till någon av de övergivna befästningsverken vid Vorsklas strand. Den som låg närmast dem var den 120 × 140 meter stora gardesskansen, som låg ungefär 500–600 meter nordöst om klostret. Roos lät sina resterande trupper besätta skansens vallar och utsåg en reservstyrka, men de flesta var sårade och det rådde brist på vapen och ammunition. +Skansen omringades snart av 2 000 ryska fotsoldater och Rentzel lät skicka överstelöjtnant Alexej Kirov med en trumslagare som parlamentär till Roos för att erbjuda honom en hedersam kapitulation. Roos tvekade och begärde att få två timmars betänketid, vilket beviljades av Rentzel. Roos kallade då samman sina få överlevande officerare till ett krigsråd. Då man befann sig utan ammunition och andra medel för att stå emot ett ryskt stormanfall, beslutade Roos vid halv tio på förmiddagen att kapitulera, på villkoren att han ville få ackordet i skriftlig form, att samtliga sårade svenskar skulle få vård och att de skulle få behålla och bära sina värjor. Rentzel accepterade Roos villkor och lovade att hålla dem "som om de vore skrivna på över 100 ark papper". Kapitulationen blev nu ett faktum, och villkoren skulle av ryssarna efteråt följas till punkt och pricka. Av 2 600 man eller sex bataljoner som Roos kommenderade i striden mot den tredje redutten återstod nu endast 400 man, som nu skulle hamna i rysk fångenskap. Resten hade stupat, tillfångatagits eller lämnats dödligt sårade på slagfältet eller på vägen till gardesskansen. En tredjedel av den svenska huvudstyrkans infanteri var nu förlorad. +Ungefär samtidigt som Mensjikov fick sin anfallsorder mot Roos trupper utskickades från det ryska lägret ett mindre detachement under överste Ivan Michajlovitj Golovin, som med tre bataljoner skulle erövra klosterhöjden för att därifrån kunna upprätta kommunikation med Poltava. Uppdraget fullföljdes utan större svårighet eftersom det svenska artilleriet som var placerat där hade dragits tillbaka redan några dagar tidigare. Den grupp svenskar som barrikaderat sig själva i klostret kunde inte bjuda något nämnvärt motstånd. Golovin lät Rostovska regementet under överste Aleksej Mjakinin befästa sig i klosterruinerna och gruppera sitt regementes nya trepundiga kanoner. Genom Mensjikovs, Rentzels och Golovins aktioner öppnades en fri väg från det ryska lägret till Poltavas fästning redan innan slaget var över. +I det belägrade Poltava hade kommendant Kelin på morgonen nåtts av beskedet att undsättning var på väg och gjorde sin garnison redo för strid. Både Södermanlands och Kronobergs regementen var under tiden förskansade i belägringsverken sydväst om staden, och små avdelningar av livdragoner och zaporoger posterades runt staden för att uppehålla skenet om att staden var inringat. Sedan Kelins fyra bataljoner tvingat Roos att fly till gardesskansen och Golovins och Rentzels trupper etablerat kontakt med Kelins garnison blev situationen för de svenska belägringstrupperna allt svårare. Vid underrättelsen om Roos kapitulation genomförde Kelin ett större utfall med sina fyra bataljoner och tvingade en svensk stjärnskans nära Vorskla med en besättning på 40 man från Södermanlands regemente under sukundkapten Jesper Hård af Segerstad till kapitulation. Kelin utförde därefter mindre utfall mot de övriga belägringstrupperna. 140 man från Kronobergs regemente under sekundkapten Carl von Rango som var posterad på Masurovkahöjden tvingades ge vika av de ryska bataljonerna. Sedan fick Kelin understöd av allierade kosacker och Heinske som med fyra dragonregementen rensade upp runt Poltavas fästning. Regementschefen Cronman beordrade sina trupper att dra ihop sig och inrätta sig i löpgravarna utanför stadens murar och avvärja ett dubbelt omfattande ryskt anfall. När ryssarna tog sig till löpgravarna utvecklades striderna till ett slags skyttegravskrig. Ryssarna lyckades erövra ett svenskt batteri på två kanoner på en erövrad del av fältvallen, och regementschefen Weidenhaijn dödades av en kanonkula avlossad från ett av blockhusen nära fästningsvallen. Efter ett tag ebbade striderna ut; bortsett från att man förlorade kanonbatteriet på fältvallen lyckades de svenska soldaterna försvara sina ställningar. Vid ett tillfälle ville kapten Christoffer Adolf Wenden vid Södermanlands regemente förhandla om kapitulation med en rysk parlamentär, men han blev nedskjuten av egen trupp då han utan regementschefens tillstånd hade inlåtit sig i underhandlingar med ryssarna. Samma ryska parlamentär förhandlade sedan med överste Cronman, men han vägrade att kapitulera, och en strid bröt ut så fort parlamentären var tillbaka hos ryssarna. +Kampen var växlingsrik eftersom ryssarna vid flera tillfällen bytte anfallsförband. Flera anfall slogs ut mot svenskarna och stundtals gjorde ryssarna ett mindre avancemang, bara för att senare bli drivna tillbaka efter en svensk motstöt. I omkring en halvtimme böljade striden fram och tillbaka innan den ryska anfallskraften var slut. Framemot eftermiddagen avbröt svenskarna striden och på Cronmans order drog sig belägringstrupperna från löpgravarna, ned i den bakomliggande ravinen där de hade sitt läger. Isolerade svenska förband tvingades kämpa sig genom de överväldigande ryska trupperna i löpgravarna, innan de efter svåra förluster var i säkerhet nere i ravinen. Där fick soldaterna träffa dragonkaptenen Carl Gustaf von Trautwetter, som med kungliga order bad dem att förena sig med Livgardets postering vid Nizjnje Mlyny, för att sedan tillsammans marschera vidare till trossen vid Pusjkarjovka. När Cronmans soldater marscherade iväg förföljde ryssarna inte. Södermanlands regemente hade lidit de största förlusterna vid fästningen, men även Kronobergs regemente förlorade en stor del av sitt manskap och befäl. Totalt låg omkring 160 stupade svenskar kvar i löpgravarna, i körsbärslundarna och nedanför Poltavas palissader. +Efter klockan sex på morgonen blev slagets förste skede till ända och det inträdde en slags stridspaus bland de båda arméerna, som vid denna tid förhöll sig passiva. Kungens trupper befann sig i Pobivankasänkan nära Maloje Budisjtjeskogen medan tsarens trupper höll sig vid och i sitt läger samt i redutterna. För den ryska ledningen blev man lättad över att det svenska överraskningsanfallet hade misslyckats, att den ryska armén var intakt och att både lägret och stora delar av reduttsystemet fortfarande var i rysk hand. Dessutom hade de ryska dragonernas självförtroende ökat efter morgonens strider och de ryska förband som börjat sätta sig i säkerhet på andra sidan floden kallades tillbaka. Lägrets försvar hade säkrats genom att 10 bataljoner på sydsidan och 13 bataljoner på nordsidan ryckte ut från lägrets två sidor och rangerades i två linjer, där man dessutom etablerade kontakt med dragonförbanden på flyglarna. Irreguljärt kavalleri under Skoropadskyj, med understöd av Volkonskijs dragonregementen, förflyttade sig från Maloje Budisjtje till Tachtaulov i ett utmärkt utgångsläge för flank- eller rygganfall. Trots dessa framgångar var den ryska ledningen övertygad om att den svenska armén fortfarande var obesegrad, och man var osäker på var samtliga svenska trupper och dess artilleri befann sig. +Under stridspausen samlade den ryska ledningen underrättelserapporter från läget i Poltava, Semjonovka, Petrovka och längs Pobivankasänkan, och genomförde olika stridsförberedelser för att stå emot ett förnyat svenskt anfall. När läget hade klarnat sig kallade Peter I sina generaler till ett krigsråd omkring klockan sju på morgonen. Där diskuterade man olika strategier för att besvara svenskarnas numera passiva situation, såväl dess risker som konsekvenser. Uppdelade mellan två falanger yrkade de ryska generalerna på att den ryska armén antingen skulle stanna kvar bakom de skyddande lägervallarna eller att ta initiativet genom att marschera ut ur lägret och formera en regelrätt slagordning på det öppna fältet. General Hallart skildrade krigsrådet enligt följande: "Hans Tsariska Höghet höll krigsråd sittande till häst och beslöt att sedan vi nu slagit tillbaka det fientliga anfallet måste vi utnyttja detta för att uppnå seger." +Strax efter klockan sju tog Peter I beslutet att låta sina trupper marschera ut från lägret mot den plana öppna marken framför det ryska lägrets västra sida. Ungefär två timmar efter att beslutet fattades tillbringade ryssarna med att vidarebefordra tsarens order och genomföra en ordnad utmarsch. Från två håll rörde sig det ryska infanteriet från lägret mot mitten för att mötas framför lägrets västra front. Tsaren själv skulle kommendera trupperna med stöd av Sjeremetev, som hade det direkta ansvaret för infanteriet. Under uppställningen placerades den ryska gardesdivisionen (12–13 bataljoner) under befäl av generallöjtnant Michail Golitsyn platsen till höger, general Anikita Repnins division (14 bataljoner) placerades i mitten och general Hallarts division (15 bataljoner) på platsen till vänster. Peter I beordrade att trupperna skulle grupperas i två linjer. Den första ryska linjen utgjordes av 10 000 man fördelade i 24 bataljoner och i linjen 300 steg bakom stod 8 000 man fördelade i 18 bataljoner. Bataljonerna i varje linje skulle stå på fyra led och med tio meters luckor mellan bataljonerna, vilket gjorde den första linjen 200 meter bredare än den svenska linjen. 14 bataljoner beordrades tills vidare att stanna kvar i lägret som taktisk reserv, varur Rentzels och Golovins bataljoner detacherades mot Roos trupper. +Generallöjtnant Golitsyns division bestod av Preobrazjenskijregementets fyra bataljoner, Semjonovskijregementets tre bataljoner, Ingermanländska regementets fyra bataljoner samt Astrachanska regementets två bataljoner. General Repnins division bestod av två bataljoner ur Grenadjärregementet Taylor/Lacy, Kievska regementets tre bataljoner, Narvska regementets tre bataljoner, Schlüsselburgska regementets två bataljoner, Novgorodska regementets två bataljoner samt Batyrska regementets två bataljoner. General Hallarts division bestod av Moskovska regementets tre bataljoner, Pskovska regementets två bataljoner, Vologodska regementets två bataljoner, Kazanska regementets två bataljoner, Nizjnij Novgorodska regementets två bataljoner, Sibiriska regementets två bataljoner samt två bataljoner ur Grenadjärregementet Busch. +Mellan bataljonerna i den första linjen grupperades 25 lätta trepundiga regementsstycken som åtföljde infanteriet över slagfältet. De större kanonerna blev kvar i lägret under befäl av överste Günther. Flera av dem var av typen haubitser och mörsare vilka med sina höga kastbanor riskfritt kunde skjuta över den egna infanterilinjen. Andra direktskjutande pjäser stod upphöjda på plattformar av uppkastad jord. Placeringen tillät artilleristerna att i relativ flack bana kunna skjuta över huvudet på infanteriet och understödja dem ända fram till en eventuell närstrid. Inalles förfogade man över 70 artilleripjäser inför huvuddrabbningen. +På infanteriets båda flanker grupperades de ryska dragonerna, som räknade mellan 9 000 och 9 500 man fördelade på 73 skvadroner efter förlusterna i slagets första skede. 43 skvadroner utgjorde den högra kavalleriflygeln under ledning av generallöjtnant Bauer, medan den vänstra kavalleriflygeln bestod av 30 skvadroner under Mensjikovs direkta ledning. Den högra kavalleriflygeln grupperades i två linjer söder om Pobivanka by och ravin. Den första linjen bestod av Generalskvadronens fyra skvadroner, Sjeremetevs livskvadron, 3:e dragongrenadjärregementets tre skvadroner, Moskovska dragonregementets fem skvadroner, Novotroitska dragonregementets fem skvadroner samt Archangelska dragonregementets fem skvadroner. Den andra linjen bestod av fem skvadroner var ur Vladimirska, Vologodska, Nevska respektive Nizjnij Novgorodska dragonregementen. Den vänstra kavalleriflygeln grupperades på en enda linje och bestod av fem skvadroner var ur Livregementet samt Jaroslavska, Kievska, Ingermanländska, Vjatska respektive Novgorodska dragonregementen. +Under två timmar väntade den samlade huvudstyrkan i Pobivankasänkan, tills man vid åttatiden på långt håll fick syn på några bataljoner som den svenska ledningen trodde tillhörde Roos. Med denna övertygelse ville Rehnskiöld fullborda uppmarschen inför anfallet mot det ryska lägret, genom att från en position norr om lilla moraset slå ut ett anfall med både infanteri och kavalleri för att kunna skära av de ryska reträttvägarna mot norr. Från en ställning på andra sidan lilla moraset skulle huvudstyrkan dessutom kunna skilja huvuddelen av det ryska kavalleriet i norr från de ryska bataljonerna i lägret, för att därefter slå ut dessa grupper en efter en. För att genomföra denna anfallsmanöver beslutade den svenska ledningen att kalla på svenska förstärkningar och fältartilleriet från trossen, ett uppdrag som anförtroddes generaladjutant Anders Gideon Gyldenklou. Under tiden beslutade ledningen att det samlade infanteriet genast skulle marschera norrut över lilla moraset mot en liten skogsdunge på andra sidan av Tachtaulovbäcken, där man skymtade ryskt kavalleri, liksom några kosacker något tillbakadragna vid Osmatjiki. De marscherande bataljonerna blev plötsligt beskjutna av kosackiska krypskyttar, som hade sina eldställningar bakom de omkringliggande gärdesgårdarna. 50 svenskar skickades ut för att driva bort krypskyttarna, samtidigt som Närkebataljonen omgrupperade från kolonn till linjeformation för att möta det ryska kavalleriet. Samtidigt som de svenska bataljonerna marscherade vidare, avvaktade det egna kavalleriet invid kärrets norrsida, där de fick stridskontakt med några retirerande ryska skvadroner som svepte fram alldeles nära dem. Vissa svenska förband jagade efter dem innan de fick kontraorder av Rehnskiöld. +Strax efteråt fick Rehnskiöld rapport från Gyldenklou, som under sin ritt söderut mot trossen råkade stöta på ryska bataljoner, som den svenska ledningen tidigare trodde var Roos trupper. Han galopperade direkt tillbaka till huvudstyrkan för att berätta om denna upptäckt. I samma veva ankom underrättelser om att Västmanlands regemente och Hielms dragoner, som var i strid med irreguljärt ryskt kavalleri, misslyckades med att komma fram till Roos trupper. Kapten Palmfelt rapporterade till Rehnskiöld om var Roos bataljoner stod att finna och Sparre rapporterade att "generalmajor Roos står i en skog och defenderar sig bra." +Vid niotiden rapporterade både Creutz och drabantlöjtnant Giertta till Rehnskiöld om att de från en liten kulle nära sänkan fick syn på flera ryska bataljoner som marscherat ut från lägret och börjat rangera sig i slaglinje. På grund av sin starka övertygelse om att ryssarna skulle agera mycket passivt och inte var djärva nog att utmana svenskarna i en allmän drabbning, var Rehnskiöld inte övertygad över dessa uppgifter. Han gav sig iväg till kullen för att med egna ögon bekräfta dessa uppgifter. När det uppdagades att ryssarna verkligen rangerat sig för drabbning lät Rehnskiöld, med hänsyn till huvudstyrkans ofördelaktiga terräng, snabbt rida ner från kullen och avge rapport till Karl XII. +Rehnskiöld beslutade att den svenska huvudstyrkan skulle omedelbart marschera söderut och tillbaka mot det gamla återsamlingsområdet kring Pobivankasänkan, för att därifrån kunna ta upp kampen mot det ryska infanteriet. När infanterikolonnerna var framme, beordrade han Lewenhaupt att omgruppera dessa till linjeformation med front mot den ryska linjen, som vid den tiden stod omkring en kilometer bort. Resterna av det svenska infanteriet ställdes upp på en tunn linje, mellan 1 400 och 1 500 meter lång, med femtio meter breda luckor mellan bataljonerna i syfte att tänja ut linjen så långt man vågade för att förebygga rysk överflygling. Linjen bestod av totalt 4 000 man fördelade på 10 bataljoner. Schablonmässigt hade varje bataljon till följd av reduttstriderna en förlustsiffra på omkring 50 stupade och sårade, vilket gjorde att varje bataljon i medeltal räknade mellan 300 och 400 man vardera. Från höger flygel räknat placerades längst ut på kanten Livgardets första bataljon under kapten Gustaf Gadde, följt av Livgardets grenadjärbataljon under kapten Libert Rosenstierna, Skaraborgsbataljonen under Ulfsparre, Kalmarbataljonen under Ranck, Livgardets andra och tredje bataljon under kapten Hans Mannersvärd respektive major Erik Gyllenstierna, Upplands regementes två bataljoner under Stiernhöök respektive överstelöjtnant Arendt Fredrik von Post, Östgötabataljonen under Appelgren samt Närke-Värmlands andra bataljon under överste Georg Johan Wrangel på den vänstra kanten. Västmanlands regementes två bataljoner hade ännu inte hunnit tillbaka efter sina försök att undsätta Roos trupper. +Det svenska kavalleriet hade dock problem med att rangera sig och säkra det svenska infanteriets båda flanker. Hamiltons kavalleri hade tills vidare beordrats att bilda flankskydd mot Volkonskijs och Skoropadskyjs kavalleri vid den stora ouvragen, men Hamilton skulle senare inte vända tillbaka i tid för att ställa sig bredvid infanteriets vänsterflank under den kommande drabbningen. På den högra sidan hade Creutz kavalleri ett mycket litet manöverutrymme och blev stående inklämda mellan infanteriet på ena sidan och Maloje Budisjtjeskogen med sina lundar och småmoras på den andra. Creutz fick order att tills vidare rangera på det knappa fältet bakom infanterilinjen. För att underlätta kavalleriet gav Rehnskiöld order till Lewenhaupt att flytta infanterilinjen i höjd med en liten skogslund till höger om linjen. Lewenhaupt misstolkade dock denna order genom att göra en högerrörelse mot skogslunden, vilket hotade att proppa igen den lilla passage ut mot det trånga fältet bakom dem som kavalleriet hade att tillgå. Rehnskiöld hann dock stoppa manövern innan infanteriet hunnit marschera något större stycke, varefter han gav en ursinnig utskällning mot Lewenhaupt. Oredan bland Creutz kavalleri tilltog under denna manöver och de klämdes ihop i den trånga terrängen. +En kort stund efter sitt gräl med Lewenhaupt över den missförstådda rättningsrörelsen, lät Rehnskiöld behärska sitt arga humör. Han red fram till Lewenhaupt, fattade generalen vid handen och gav honom anfallsorder. +Omkring klockan kvart i tio började de 10 svenska bataljonerna rycka fram över fältet mot den första ryska infanterilinjen. Avståndet var mellan 700 och 800 meter, och den svenska framryckningen tillryggalades på omkring nio minuter. De ryska kanonjärerna bombarderade dem med stor kraft medan dessa närmade sig i stadig takt. Samtidigt svängde Mensjikovs kavalleri fram mellan några av redutterna i tvärlinjen och Bauers kavalleri utnyttjade den torra och fasta höjdryggen vid Pobivanka by för att obehindrad röra sig framåt strax utanför den svenska slagordningens vänstra hörn. Lewenhaupt ledde infanteriet från hästryggen på den högra infanteriflygeln och förbjöd dem från att avfyra sina musköter. När två hundra meter återstod mellan dem och den ryska linjen beordrade Lewenhaupt sina trupper att öka takten, och det ryska fältartilleriet på den första linjen gick över till att skjuta druvhagel. När svenskarna var ännu närmare skiftades laddningarna ut till kartescher, som på korthåll vräkte elände över den tunna svenska linjen. Från den ryska högra flygeln avlossades dessutom kanonkulor snett framåt, som studsade fram längs den svenska infanterilinjen. Med denna skjutvinkel kunde en enskild kanonkula slå omkull ett dussin soldater mot fyra för en frontalt avlossad kula. Den ryska artillerielden blev därför förödande för de svenska bataljonerna redan innan dessa nådde fram till den första linjen. På trettio stegs avstånd avfyrade svenskarna sina musköter mot den ryska linjen med blandat resultat. På vissa håll hade den samlade bredsidan god effekt, men på andra håll var krutet av så pass dålig kvalitet att salvan blev helt utan verkningar. Lewenhaupt lät omedelbart upphöra med skjutandet och så fort som möjligt storma ryssarna med värjor och bajonetter. Från samtliga fyra leden i den första ryska linjen avfyrades en koncentrerad salva när svenskarna redan var på omkring femtio stegs avstånd, som orsakade stort manfall i de svenska leden. +Karolinerna bröt sig in över den första ryska linjen. Till höger gjorde Livgardets andra och tredje bataljon samt Upplands, Kalmars och Skaraborgs regementen, samtliga under Lewenhaupts direkta ledning, ett direkt anfall som fick Hallarts division till vänster att hamna i obalans. Sprickor uppstod bland de oerfarna ryska regementena och svenskarna erövrade även några kanoner. Under häftig närstrid vek ryssarna framför Livgardets hänsynslösa anfall. Peter I lär själv ha kommenderat fram förband ur den andra ryska linjen vilket gjorde att det tog ett raskt stopp för de enheter ur den första linjen som var under press från svenskarna. Andra linjen kom att rent fysiskt göra det omöjligt att komma längre bakåt. Till vänster hade Östgöta och Närke-Värmlands regementen kommit efter och anfallet fick därför inte samma framgång som den till höger. Flera officerare hade dessutom redan i ett tidigt skede antingen stupat eller sårats under framryckningen över fältet. Den ryska gardesdivisionen med de erfarna Preobrazjenskij- och Semjonovskijregementena höll stånd genom att fälla sina pikar och ge kontinuerlig eld mot svenskarnas anfall. Detta skapade på så sätt en svängdörrseffekt: medan svenskarna ute på den högra flygeln gick raskt framåt mot den vänstra ryska flygeln, pressades den svenska vänstra flygeln bakåt av den ryska högra flygeln. Hela den svenska linjen hade töjts ut så att en stor lucka uppstod mellan de båda infanteriflyglarna och på vissa punkter mellan bataljonerna växte luckorna till uppemot 100–150 meter, samtidigt som de svenska förbanden till vänster började klumpa ihop sig. Ryska soldater hamnade plötsligt mellan de svenska bataljonerna, och omslöt den vänstra svenska infanteriflygeln på båda sidor. På kanten av flygeln blev situationen ohållbar och flera soldater ur Östgöta och Närke-Värmlands regementen kastade sina vapen och flydde bakåt. +Vid det här laget var Rehnskiöld framme vid Creutz på den högra kavalleriflygeln som var upptagen med att ta sig ur "ett elakt moras". Rehnskiöld manade dock på och Creutz formerade en stridsgrupp med vad som fanns tillhanda, skvadroner från Livregementet till häst, Norra skånska kavalleriregementet samt Nils Hielms dragoner. Creutz tog sikte på det ryska infanteriet som kringrände svenskarna längst till höger och försökte skapa panik i de ryska leden. De fyra bataljonerna ur grenadjärregementet Busch och Nizjnij Novgorodska regementet såg dem komma och stannade upp för att formera fyrkant. De ryska kanonerna vräkte stora karteschladdningar mot de ankommande svenska ryttarna. Creutz kunde inte uträtta något, men det gav Livgardets bataljoner en tillfällig respit. Samtidigt dök Mensjikovs dragoner upp bakifrån och på högersidan. Creutz växlade mål från infanteri till kavalleri, denna gång med bättre resultat. De ryska dragonerna retirerade med Creutz ryttare hack i häl. När Livgardets första bataljon och grenadjärbataljon på den högra kanten upptäckte att Creutz ryttare red bort från dem bröt soldaternas stridsmoral samman. Trots detta lyckades dock samtliga av gardesbataljonerna retirera i god ordning efter en mellan 15 och 20 minuter lång närstrid. +Lewenhaupt märkte vad som höll på att hända och satte av vänsterut där han uppmanade både generalmajor Sparre och den sårade överste Appelgren att få sina regementen att komma till stånd, men kaos och skräck härskade bland de flyende trupperna och Lewenhaupts och officerarnas skrik och slag ignorerades. Gyllenkrok hade under tiden kallat på en trupp ryttare från vänster för att bistå det sviktande fotfolket ute på vänstra kanten. När truppen red an möttes den av häftig rysk eld och tvingades ge vika. Även Rehnskiöld upptäckte den hotande katastrofen för infanteriet till vänster och störtade iväg till den vänstra kavalleriflygeln för att få fram förstärkning till de hårt ansatta fotsoldaterna. Men det svenska kavalleriet både till höger och till vänster var i oordning. Endast överste Torstenson vid Nylands regemente red till angrepp med två skvadroner för att försöka täppa till den stora luckan i infanterilinjen. De ryska gardessoldaterna upptäckte faran, formerade sig till fyrkant och kavalleriet störtade rakt in i fällda pikar och muskötsalvor. Torstenson och nästan hela manskapet i båda skvadronerna stupade under striden. När den vänstra svenska infanteriflygeln började fly i panik mot Maloje Budisjtjeskogen hamnade de mer framgångsrika svenska bataljonerna till höger i en dödsfälla. Det hela utvecklades till det som historikern Peter Englund kallade en variant av slaget vid Cannae, där ryssarnas tillbakavisning gjorde att svenskarnas flanker blev öppna för en omfattning, som dock var mer av en konsekvens efter det svenska infanteriets lyckade anfall snarare än en planerad manöver från rysk sida. Den oavbrutna ryska karteschelden från fältartilleriet skördade ytterligare offer bland de flyende karolinerna. +Åtskilliga svenska skvadroner, främst ur Hielms och Taubes dragonregementen, gick till angrepp mot den ryska massan för att skapa oreda hos dem och ge andrum för den retirerande svenska vänstra infanteriflygeln, men samtliga tvingades tillbaka av ryska muskötsalvor. Hamiltons vänstra kavalleriflygel, vid denna tid bestående av omkring 2 000 ryttare ur Skånska ståndsdragonregementet och Åbo läns och Östgöta kavalleriregementen, var under tiden upptagna med att avvärja Bauers och Volkonskijs avdelningar vid den stora ouvragen och förhindra dessa från att rida ner det svenska fotfolket. Kort efteråt blev prins Maximilians ståndsdragoner omringat av ryssar och man förlorade ett standar. Prinsen blev förargad över detta och i spetsen för sina dragoner och det bredvid stående Östgöta kavalleriregemente trängde han sig in i de avancerande ryssarna och på kort tid erövrade han fem ryska standar. Aktionen ledde dock till svåra förluster och prinsens manskap började ge vika, varefter Maximilian beslutade sig för att försöka slå sig igenom till Karl XII. Den vänstra svenska kavalleriflygelns befälhavare Hamilton blev tillfångatagen under den ojämna striden mot den ryska högerflygeln i skogsbrynet vid Maloje Budisjtje, och därefter lät flera kavalleriförband avbryta sina anfall och delta i den vänstra infanteriflygelns flykt. Lewenhaupt gav upp sina ansträngningar att förhindra den vänstra infanteriflygelns reträtt och försökte rida tillbaka till den högra infanteriflygeln. Denna var dock helt omringad av ryska trupper, som blockerade Lewenhaupts väg. I stället beslöt Lewenhaupt att följa de flyende in i Maloje Budisjtjeskogen och försökte samla de flyende trupperna till strid. Men han kunde varken hindra de flyende svenska fotsoldaterna eller den högra kavalleriflygeln som redan befann sig där och som följde med i flyktingströmmen. Generalmajor Lagercrona hade drabbats av panik och utdelat order som starkt bidrog till att lösa upp disciplinen vid ett antal redan vikande skvadroner. +På fältet stod fortfarande några svenska bataljoner, som nu var helt isolerade och mötte en flodvåg av ryska musköter, bajonetter och pikar. Av de tio svenska bataljonschefer som lett sina trupper i anfallet dödades sju inom loppet av en halvtimme, de övriga blev samtliga sårade. Major Gyllenstierna och kapten Mannersvärd stupade ihop med flera andra officerare i sina gardesbataljoner och Kalmarbataljonen förlorade nästan halva sitt manskap, tillika regementschefen Ranck. I Skaraborgsbataljonen stupade regementschefen Ulfsparre och av 500 man återstod efter slaget endast 40 man: varav 13 officerare och 27 meniga. Inom Upplands regemente tvingades de överlevande i manskapet till en skoningslös slutstrid, där både regementschefen Stiernhöök, överstelöjtnant Post, majoren, regementskvartermästaren, adjutanten, nästan samtliga kompanichefer och nästan hela manskapet var bland de stupade. Av 700 man i början av slaget återstod endast 14 man när slaget var över. +Flera flyende karoliner, till häst eller till fots, fastnade under flykten i moraset, hanns upp och dödades av de förföljande ryssarna. För att undvika denna massaker höll sig de ordnade svenska förbanden samman eller slöt sig till större avdelningar. Ungefär 30 soldater ur Västmanlands regementes första bataljon under överstelöjtnant Carl Anders Sinclair tog ställning bakom en gärdesgård, där man lyckades förhandla sig till krigsfångenskap hos Repnins division. I samma veva togs även Stackelberg, Appelgren och Josias Cederhielm som fångar. +Karl XII skickade två drabanter till Creutz med order om att avbryta sin förföljelse på Mensjikovs dragoner, som jagades iväg till husen i Maloje Budisjtje. Creutz gav omedelbart order om halt och samlade så många ryttare han kunde hitta, främst ur Norra skånska kavalleriregementet och några skvadroner ur Hielms dragoner, och genomförde en kringgående rörelse mellan de förföljande ryska trupperna till platsen där kungen befann sig. På fälten bistod Gyllenkrok överste Gyllenstierna och hans dragonregemente med att anfalla de ryska bataljoner som förföljde den högra svenska infanteriflygeln, och samtidigt uppmanade de flyende svenska fotsoldaterna att dra ihop sig. Ryssarna formerade sig till fyrkant och avfyrade en salva som fick de svenska dragonerna att retirera. Under virrvarret vid ett trångt passage var Gyllenkrok nära på att bli dödad av en rysk infanterist, som fattade tag i Gyllenkroks gehäng, men Gyllenkrok kunde slita sig loss och fly från platsen. +Karl XII hade med sitt följe marscherat med högra svenska infanteriflygeln i deras anfall, och stod på en liten kulle bakom flygeln beskyddad av 14 kvarlevande livgardister, 4 drabanter och resterna av Livdragonregementet och Livregementet till häst. En del ryssar gick till anfall mot kungens följe, men avvisades utan större problem. När flera svenska bataljoner inledde sin flykt från slagfältet red Rehnskiöld fram till kungen och berättade att "vårt infanteri löper!". Kungen var ifrågasättande kring denna utveckling, varpå Rehnskiöld innan han red iväg uppmanade de kvarlevande livgardisterna att "tagen väl vara på herren!". När krutröken och dammen började skingra sig på slagfältet upptäckte Karl XII och hans följe att infanteriet verkligen var borta och att de var de sista svenskarna som stod stillastående på slätten. När sällskapet snart blev kringränt av ryssarna gav kungen order om reträtt. +Följet rörde sig bort mot skogsbrynet 800 meter nordost om kyrkan i Maloje Budisjtje när deras reträtt hotades av en rysk kanonbestyckad bataljon. Creutz som var på plats med sina ryttare eskorterade kungens följe mellan den ryska bataljonen och skogen. De ryska soldaterna gjorde höger om och fällde muskötsalvor och karteschladdningar mot kungens avdelning. Reducerad fortsatte kungens följe in i skogen mot ett moras, förföljda av ryssarna strax efter dem. Kungens ekipage fastade dock i ett litet kärr, varefter ryssarna avfyrade en kanonkula som slog kungens hästbår i spillror och fick kungens båda hästar att kollapsa. Stående vid moraskanten avfyrade ryssarna muskötsalvor mot kungens följe, där flera offrade sina liv genom att agera måltavlor för att beskydda kungen från projektiler. Kungen lyftes ur den obrukbara båren och hjälptes upp på en häst som tillhörde livregementsryttaren Bass. Den nya hästen hann bara ta ett par steg framåt tills den träffades av en kanonkula som slet bort ett av benen. Kungen behövde en ny häst och beordrade den sårade drabantlöjtnanten Giertta att ge sin häst till honom. Denne lyftes av sadeln ned till marken och kungen lyftes till hästen. Vilandes på hästens hals red kungen, 3 av hans 24 livgardister och övriga överlevande från hans följe bort från dödsfällan vid kärret. Giertta lämnades stödjande på en gärdesgård, men han räddades i sista stund av hans bröder Adam och Christian Giertta. Kvar vid kärret stupade ett tjugotal livgardister och drabanter medan flera andra sårades och togs tillfånga. Av folket från hovet och kansliet stupade registratorn Stephan Hirschenstierna och kammarherren Gustaf Adlerfelt. +Drabantlöjtnant Hårdh red bort från kungens avdelning för att försöka mobilisera mer skydd åt kungen från de flyende trupperna. Efteråt träffade han Lewenhaupt och bad om hans hjälp, varpå generalen red till de flyende trupperna och manade dessa att hålla stånd eftersom kungen fanns i närheten. Med argumentet "överge icke er konung" lyckades Lewenhaupt stävja situationen runt honom och Karl XII; den flyende klungan av fotsoldater och ryttare kunde åter formeras till en visserligen retirerande, men ändå stridande enhet. Kungen frågade sedan Lewenhaupt vad de skulle göra härnäst. Efter en stunds överläggningar övertygade Lewenhaupt kungen att bege sig till trossen med sin eskort och ett par valakryttare som vägvisare. Lewenhaupt skickade en ryttare i förväg med en befallning till de regementen som fanns i trossen att genast rycka ut för att gå kungen till mötes. Under själva reträtten till trossområdet blev kungens sällskap endast oroad av några kringströvande kosackenheter som enkelt slogs tillbaka utan nämnvärda förluster. Efteråt anslöt sig den sårade fortifikationskaptenen Baltzar von Dahlheim till kungens följe och visade resten av vägen till trossen. Kungen och hans följe anlände dit vid klockan ett på eftermiddagen, där han togs emot av generalmajor Meijerfeldt och drabantlöjtnant Hårdh. +Även om större delen av den svenska huvudstyrkan befann sig på flykt i och kring Maloje Budisjtjeskogen fanns en del svenska förband fortfarande i strid ute på fältet. Skvadroner ur Taubes och Hielms dragonregementen fortsatte sina fördröjningsstrider mot de anryckande ryska dragonerna. Posterade bakom en brant hålväg avvärjde de svenska skvadronerna det ena anfallet efter det andra och hindrade ryssarna från att nå passagen bakom dem, som strömmades av flyende svenska förband. Efter ungefär en kvarts strid var passagen fri och de svenska skvadronerna deltog i återtåget med ryska soldater hack i häl. Creutz ledde sina skvadroner tillbaka genom skogen, passerade Maloje Budisjtje och kom ut på slätten. På vägen fastnade några ryttare från Norra skånska kavalleriregementet i de upptrampade morasen, liksom regementschefen Horn som gick ned sig och togs tillfånga. Vid denna tid återsamlades Taubes och Dückers överlevande dragoner nära Rehnskiöld och Piper; den senare uppmanade den numera förtvivlade fältmarskalken att få de flyende trupperna till stånd, varpå Rehnskiöld besvarade endast med meningen "Alles ist verloren" (Allt är förlorat). Båda stötte senare på Gyllenkrok, som undrade vad man skulle ta sig till. Alla tre red sedan åt var sitt håll. Rehnskiöld störtade i spetsen för en liten trupp till någon form av självmordsanfall i närheten av redutten närmast Maloje Budisjtjeskogen, under ett försök att vända de flyende mot de anstormande ryssarna. De omringades och Rehnskiöld togs tillfånga av dragoner ur Archangelska regementet; han överlämnade sin värja personligen till fältmarskalk Sjeremetev. Resterna av Skånska ståndsdragonregementet och Östgöta kavalleriregementet hade långt att rida till trossen. Prins Maximilian, som var lätt sårad och fått två hästar skjutna under sig, och hans kvarvarande 100 man råkade vara mitt i vägen för Volkonskijs och Skoropadskyjs kavalleri, totalt uppemot 2 000 man. Prinsen försökte dock förmå sitt manskap till en sista strid, men från alla håll trängde ryssarna hårt på, och snart blev det så trångt att man knappt kunde röra sig. Efter ytterligare en stunds desperat motstånd fick prinsen och hans återstående manskap ge sig tillfånga. +Enstaka svenska flyktingar tog sig ut ur Maloje Budisjtjeskogen och ut på fälten, där växande grupper av infanteri och kavalleri samlades. Vid elvatiden försökte Gyllenkrok, Lagercrona och andra officerare att samla det överblivna kavalleriet. Det rådde ett temporärt lugn bland trupperna eftersom ryssarnas förföljelse verkade ha upphört. Tre ryska skvadroner hade dock ridit runt de svenska ryttarna och dök i stället upp bakom dem, invid Maloje Budisjtje. Dessa bildade front mot svenskarna nära en liten bro som flyende svenskar utnyttjat och blockerade passagen. Gyllenkrok och Lagercrona försökte mana de svenska skvadronerna att avhysa de ryska skvadronerna, men på grund av missförstånd och brist på officerare flydde de svenska ryttarna söderut mot trossen. Uppgiven red Gyllenkrok till trossen i full galopp. Skvadronerna i Creutz grupp marscherade sakta mot söder, i syfte att ge eftersläntrande svenskar en chans att ansluta sig till dem. +Vid tolvtiden beordrade Peter I att det ryska infanteriet och att Mensjikovs och Bauers dragoner skulle återvända till huvudlägret. Den ryska ledningen ville inte riskera sina ryska förband att dras in i svårkontrollerade skärmytslingar i Maloje Budisjtjeskogen. De ryska förbanden hade till dess redan hamnat i oordning under försöken att hinna ikapp de retirerande svenska trupperna, samtidigt som flera soldater varit upptagna med att plundra de svenska liken och samla ihop de svenska fångarna i stora flockar. De reguljära ryska trupperna ställdes upp på slagfältet framför lägret för att hålla gudstjänst och tsaren red i sakta skritt utefter fronten och tackade sina soldater för deras tappra hållning under striderna. Under tiden övertogs förföljelsen av Volkonskijs dragoner och Skoropadskyjs kosacker och kalmucker, som kretsade runt för att attackera enstaka grupper och eftersläntrare. När Heinske var färdig med att rensa upp runt Poltavas fästning insattes även hans dragoner i förföljelsen. När man några timmar senare nådde lägret vid Pusjkarjovka tvingades ryssarna avstanna, då lägret visade sig vara befäst och bestyckat med omkring trettio kanoner. I Maloje Budisjtjeskogen påträffades Karl XII:s blodiga hästbår, vilket fick Peter I och ledningen att dra slutsatsen att kungen hade stupat. Efter åtskilliga timmar klarnade bilden för den ryska ledningen och på kvällen verkställdes en större upplagd förföljelse. Ett beridet detachement under Golitsyns befäl, bestående av hans gardesdivision och Bauers tio dragonregementen, fick i uppdrag att försöka hinna ifatt svenskarna och ta både fångar, hästar och materiel. Senare ankom Mensjikov som anförde resten av förföljelsen under hårda ilmarscher. Vissa av gardesdivisionens grenadjärkompanier hölls dock kvar som tsarens närskyddstrupp. +Senare på kvällen slog sig Peter I ned i sitt tält och skrev ett kort brev till sin älskarinna Katarina: +Den 29 juni inleddes uppstädningen av slagfältet och sex ryska officerare fick uppgiften att räkna samman förlusterna. Förlusterna för den ryska armén uppgick till 1 345 i antalet stupade och 3 290 i antalet sårade, totalt 4 635. Ryssarnas förluster var förhållandevis små eftersom de hade utkämpat slaget från försvarsställningar och att de ryska soldaterna hade ett överflöd av ammunition, eldvapen och kanoner, vars samlade eldkraft vräkte oerhörda förluster inom den svenska huvudstyrkan. Enligt den skotske generalmajoren Alexander Gordon, som tjänade i Peter I:s armé, uppgick de ryska förlusterna till 5 953 i antalet stupade och sårade. Ryssarna beordrade de svenska krigsfångarna att samla ihop och begrava samtliga lik. Två stora massgravar, fördelade mellan ryska officerare respektive meniga, grävdes drygt 500 meter sydväst om det befästa lägret, mitt mellan lägrets vallar och reduttsystemet. De döda ryssarna samlades ihop och fraktades till denna plats. Runt massgravarna stod en hedersvakt och regementspräster från samtliga ryska förband. Peter I höll ett tal vid gravarna och infanteriet sköt tre salvor salut. En stor jordhög kastades upp över gravarna, som fick det felmärkta namnet "svenska graven". +Enligt ryska relationer uppgick det ryska kavalleriets förluster till en överstelöjtnant, 2 majorer, 12 kaptener, 7 löjtnanter, 8 underlöjtnanter och kornetter samt 593 underofficerare och meniga i antalet stupade. I antalet sårade inbegrep generallöjtnant Rönne, 2 överstar, 3 överstelöjtnanter, 4 majorer, 26 kaptener, 18 löjtnanter, 19 underlöjtnanter och kornetter samt 1 381 underofficerare och meniga. Kavalleriets förlustandel uppgick till mellan 4 och 6 procent stupade och 10–16 procent sårade. Det ryska infanteriets förluster uppgick till 691 i antalet stupade och 1 784 i antalet sårade, en förlustandel på omkring 2 procent stupade och 6 procent sårade. +Det svenska nederlaget vid Poltava innebar den mest förlustbringande militära sammandrabbningen i Sveriges historia. Den totala förlustsiffran varierar mellan olika verk. Enligt det svenska generalstabsverket uppgick de svenska förlusterna till 301 stupade officerare och ungefär 6 600 döda eller dödligt sårade underofficerare och meniga, totalt omkring 6 900 man, vilket utgör en tredjedel av de deltagande svenska soldaterna. Enligt beräkningar av den svenske historikern Bertil Wennerholm, baserat på olika relationer, stupade mellan 8 610 och 9 224 svenska soldater. Enligt olika ryska källor ska mellan 8 000 och 10 000 svenska soldater ha legat stupade på slagfältet, inklusive döda kosacker, valaker och zaporoger i Poltavas närhet och längs vägen söderut. På platser där de hårdaste striderna förekom låg de döda kropparna tätt. Exempelvis runt omkring den tredje redutten uppgavs att uppemot 1 000 lik ha legat på en yta av ungefär 250 kvadratmeter. Bland de stupade regementscheferna var överstarna Buschwaldt, Ranck, Siegroth, Stiernhöök, Torstenson, Ulfsparre och Weidenhaijn. Samtliga svenska regementen led mycket stora förluster efter slaget, några rentav förintades. Bataljonerna ur Skaraborgs, Upplands och Västmanlands regementen uppgavs av ryssarna ha haft 97–98 procent förluster. +Enligt generalstabsverket blev omkring 2 800 svenska soldater tillfångatagna efter slaget. Med generalstabens siffror på 6 900 stupade soldater och 2 800 tillfångatagna innebar att de svenska totalförlusterna uppgick till 9 700 man, en förlustandel på 49 procent eller varannan svensk soldat som deltog i slaget. Enligt vissa beräkningar ska omkring 1 500 sårade soldater ha deltagit i reträtten från Poltava. Inkluderas denna uppgift skulle den totala förlustsiffran uppgå till 11 200 man, en förlustandel på 57 procent. Inte mindre än 9 svenska adelsätter ska ha blivit utslocknade på slagfältet, då de sista i deras respektive ätt stupade. +Enligt Wennerholms beräkningar tillfångatogs mellan 2 874 (varav 2 587 underofficerare och meniga) och 2 977 man (varav 2 729 underofficerare och meniga). Vidare har Wennerholm uppskattat att det svenska infanteriets förluster efter slaget uppgick till omkring 6 700 och 6 800 underofficerare och meniga i antalet stupade och tillfångatagna. För de beridna trupperna uppgick deras förluster efter slaget till omkring 3 300–4 100 underofficerare och meniga. Adderas dessa siffror uppgick totalförlusterna till mellan 10 000 och 10 700 man, varav 7 300 och 8 100 man i antalet stupade underofficerare och meniga från samtliga truppslag. Bland de stupade officerarna var 43 kavalleriofficerare, 35 dragonofficerare och 204 infanteriofficerare. +Bland de tillfångatagna svenska officerarna var fältmarskalk Rehnskiöld, prins Maximilian, generalmajorerna Hamilton, Roos, Schlippenbach och Stackelberg, samt överstarna Appelgren, Fock, Gyllenstierna, Horn och Taube. Kanslichefen Carl Piper ihop med hovpredikanten Nordberg, kanslisten Joachim von Dittmer och sekreteraren Joachim von Düben gick under reträtten vilse på slagfältet och för att rädda sina liv begav de sig till Poltava och överlämnade sig som fångar inför kommendant Kelin. Kort efter slaget anordnade Peter I en festmiddag med Rehnskiöld, Piper och de övriga svenska generalerna som gäster, där han kom att ställa en rad frågor till församlingen och utbringade en skål för sina svenska "lärare i krigskonsten". Flera av de tillfångatagna soldaterna var sårade och plundrades grundligt av de ryska soldaterna. +Under senhösten 1709 förflyttades de svenska krigsfångarna till Moskva, där Peter I den 23 december 1709 anordnade ett stort triumftåg till firandet av segern vid Poltava. Krigsfångarnas tåg som omfattade omkring 20 000 svenska soldater ställdes upp i rangordning, där Rehnskiöld och Piper gick sist i tåget. Efter dem fördes alla erövrade krigstroféer, inklusive 300 svenska fanor som släpades längs marken och Karl XII:s sönderskjutna hästbår. Triumfmarschen avslutades av Peter I som red i spetsen för Preobrazjenskijregementet. Marschen gick genom sju triumfportar bemålade med teman från slaget vid Poltava. De svenska generalerna och Piper deltog i tsarens festmåltid i Uspenskijkatedralen, medan de svenska soldaterna och underofficerarna tvingades stå stilla i den isande kylan utan mat och dryck. Efter triumftåget internerades de svenska krigsfångarna till tvångsarbete i flera städer och orter runtom i Ryssland. Rehnskiöld och Piper stannade kvar i Moskva där de inrättade ett förvaltningskontor för att tillvarata de svenska krigsfångarnas intressen och föra deras talan. +Den besegrade svenska huvudstyrkan återsamlades vid trossen. När Gyllenkrok anlände dit samlade han ihop 300 man som fick ställa sig till artilleriets understöd, och ställde upp bagaget och artilleriet på vägen söderut längs Vorskla. Karl XII fick sitt fotsår omlagt och fick underrättelser om att Piper och Rehnskiöld var borta och att hela armén var fullkomligt sönderslagen. Kungen beordrade att kvarlevorna ur armén, som räknade omkring 15 000 man inklusive all trosspersonal och flera sjuka och sårade soldater, samt Mazepas och Gordejenkos kosacker och zaporoger skulle dra sig tillbaka söderut. Gyllenkrok beordrades att leda bagagetåget till överstelöjtnant Funcks postering i Bilyki. Gyllenkrok gav överstelöjtnant Niklas Rappe uppdraget att avtåga med bagaget först och senare med artilleriet. När Gyllenkrok rapporterade att bagaget och artilleriet var på väg fick han order av Karl XII att bege sig i förväg till Funck. Gyllenkrok anlände först till byn Novye Sanzjary där han beordrade överstelöjtnant Johan Reinhold von Trautvetter att sända en dragon för att hämta Funck. Dagen efter kom bud om att ryska trupper var i närheten, och Gyllenkrok red i förväg för att träffa Funck utanför byn och gjorde honom sällskap till kungens stuga i Bilyki. Gyllenkrok fick upplysning av en kosack att fortsätta vägen nedför Vorskla till staden Koleberda, där han träffade överstelöjtnant Silfverhielm som försäkrade att han kände trakten väl och kunde ta armén över Dnepr. Gyllenkrok skickade bud till Lewenhaupt, som nu hade det närmaste kommandot under Karl XII, om att skicka timmermän och artillerister med broslagningsredskap till den nedbrända byn Perevolotjna. +När Gyllenkrok kom till Perevolotjna anmälde sig 14 timmermän hos honom, beväpnade med yxor men utan andra redskap. Från byn Kischenka utanför Perevolotjna hade Gyllenkrok flyttat åtta pråmar, och vandrade längs Dneprs strand ihop med timmermännen för att leta efter bräder till nya pråmar. När han insåg att Dnepr var för bred att föra över en hel armé försökte han hitta ett vadställe över Vorskla. Ett vadställe fanns vid Kischenka och han skickade genast en trupp med order att hindra kungens vagn tills han fick tillfälle att föredra sina förslag inför kungen. Den 30 juni fick han besök av generaladjutanten Gyldenklou, som berättade att kungen hade anlänt till Perevolotjna ihop med täten av armén. Där träffade han Karl XII och fick tillfälle att informera att armén i stället borde korsa floden Vorskla vid vadstället nära Kischenka. Efteråt anlände resterna av armén till Perevolotjna, och förläggningen där var alldeles för trång och instängd mellan två floder. Lewenhaupt och Sparre uppvaktade kungen och vädjade honom att rädda sig över Dnepr. Kungen frågade då Gyllenkrok om vart armén borde gå härnäst. Gyllenkrok svarade att det fanns tre utvägar: att armén borde gå tillbaka genom Ukraina och söka kontakt med trupperna i Polen; att marschera genom ödemarken till Tatariet; eller att göra en förnyad strid mot Peter I. Kungen kommenterade att hans soldater skulle strida när han befaller dem, varpå Gyllenkrok besvarade: +Då kungen fördröjde sitt beslut vädjade Gyllenkrok till kungen och bad att han skulle rädda sin höga person så att armén skulle göra sitt bästa för att rädda sig. Lewenhaupt och Sparre instämde med Gyllenkrok, men kungen lät sig inte övertygas. Till slut lyckades Creutz beveka kungen, som påpekade att ifall armén skulle besegras och tillfångatas av ryssarna skulle kungen ha större möjligheter att hjälpa dem om kungen var fri och kunde upprätta sitt folk. Efteråt fick Gyllenkrok besked från hovintendenten von Düben om att kungen beslutade sig för att rädda sin person och resa över Dnepr. När Gyllenkrok mötte kungen fick han order att meddela Mazepa, Lagercrona, Lewenhaupt och Sparre samt kansliet, hovstaterna och drabanterna att de skulle följa med kungen, medan Creutz skulle stanna kvar och ta befälet över de resterande trupperna ihop med gardesöversten Posse. Under natten till den 1 juli passerade Karl XII och Mazepa floden med de pråmar som Gyllenkrok anskaffat. I deras följe fanns 1 300 stridande och 200 icke-stridande svenskar och 1 500 kosacker och zaporoger. Under dagarna efter att ha passerat Dnepr trädde kungens och Mazepas följe in i osmanskt territorium och kungen mottogs med stora hedersbetygelser i hamnstaden Otjakov mellan 35 och 40 mil söder om Perevolotjna. Under några dagar kunde de vila ut tills den 14 juli, då de bröt upp igen och marscherade till staden Bender där kungen fått inbjudan att slå sig ner. Den 24 juli tågade kungen och hans följe in i Bender, senare till den närliggande byn Varnitza där han kom att stanna de följande två åren. +Lewenhaupt skulle enligt Karl XII:s önskan medföljt över Dnepr, men fick på sin egen begäran stanna kvar och föra befälet över armén. Han fick order att föra armén över Vorskla in på krimtatarernas territorium och återförenas med kungen i Otjakov. Klockan två på morgonen den 1 juli anlände Mensjikov med en rysk styrka på omkring 9 000 beridna soldater och några kanoner, som besatte höjderna runt Perevolotjna och rangerade sig i en halvmåneformad slagordning. Ryska kosackförband svepte ned mot stranden, där de attackerade de splittrade svenska skarorna och plundrade bland alla vagnar. Förvirringen bland svenskarna var under denna tid mycket stor och flera försökte ta sig över floden med primitiva flottar eller med de återvändande pråmar som släppt av kungens följe. Lewenhaupt ville dock inte ta strid mot ryssarna, utan efter överläggningar med Mensjikov och omröstningar mellan de högre svenska officerarna beslutade Lewenhaupt att armén klockan 11 på förmiddagen skulle kapitulera. Enligt kapitulationsöverenskommelsen skulle samtliga svenskar hamna i rysk fångenskap; att dessa skulle överlämna alla vapen, krigsutrustning, fälttecken och regementskassor; samt att svenskarna förband sig lämna ut samtliga allierade zaporoger och kosacker. De svenska officerarna och civilmilitärerna fick behålla sitt bagage och sitt tjänstefolk, och officerarna garanterades en "honett" behandling och fick på hedersord möjligheten att resa hem. Lewenhaupts beslut om kapitulationen har debatterats bland historiker. +Den reguljära svenska armén som gick i fångenskap vid Perevolotjna räknade omkring 20 000 personer. Bland dessa var 983 officerare och 12 575 soldater, fördelade mellan 9 152 ryttare, 3 286 fotsoldater och 137 artillerister. Bland officerarna var general Lewenhaupt, generalmajorerna Creutz och Kruse, 11 överstar, 16 överstelöjtnanter, 23 majorer, en fälttygmästare, 256 ryttmästare och kaptener, en kaptenlöjtnant, 304 löjtnanter, 323 kornetter och fänrikar, 18 regementskvartermästare, två generaladjutanter och 25 adjutanter. Tillägger man de drygt 2 800 som togs tillfånga vid Poltava hamnar det totala antalet svenska fångar till omkring 23 000 personer. Totalt 11 kavalleriregementen, 11 dragonregementen och 12 infanteriregementen föll i fångenskap. Av dessa var 16 358 soldater och officerare, 1 407 civilmilitärer (varav 945 var hantverkare, kuskar och trossdrängar), 3 402 civila tjänare och arbetare och 1 657 kvinnor och barn. Mindre än var fjärde av de 23 000 fångarna skulle återvända till hemlandet efter fredsslutet 1721. De överlevande kosacker och zaporoger som inte hunnit fly över floden eller som var kvarlämnade vid Poltava blev avrättade och torterade av ryssarna; en del gjorde utsiktslöst motstånd eller dränkte sig i Dnepr. Den ryske överlöparen Maximilian Heinrich Mühlenfels, som sedan 1708 underrättat Karl XII om den ryska inrikessituationen, spetsades på en påle framför ditkommenderade svenska krigsfångar. +Budskapet om Peter I:s seger vid Poltava och den svenska huvudarméns kapitulation vid Perevolotjna väckte uppseende över hela Europa. Trots att de väst- och centraleuropeiska stormakterna sedan länge var fullt upptagna av spanska tronföljdskriget, som sedan 1701 löpte parallellt med stora nordiska kriget, hade regeringarna och den tidens press fortlöpande rapporterat om Karl XII:s fälttåg inåt Ryssland; om kungens nederlag och flykt till Bender ihop med Mazepas kosacker; och att tsar Peter I nu hade handlingsfrihet att kunna bekriga de svenska trupperna i väster och nordväst. +Vid underrättelsen om Karl XII:s förlust vid Poltava förklarade Fredrik IV av Danmark redan den 8 augusti 1709 krig med Sverige och landsteg med en dansk armé vid Helsingborg den 1 november. Kung August II mobiliserade sina sachsiska trupper och, med understöd från ryska förband från öster, återerövrade han sin polska krona från Stanisław Leszczyński, som tvingades fly till Svenska Pommern eskorterad av generalmajor Krassows trupper. General Magnus Stenbock mobiliserade Skånelands försvar för att bekämpa den danska invasionsarmén, som han slutligen besegrade i slaget vid Helsingborg i februari 1710, och flera av de regementen som utplånades vid Poltava och Perevolotjna återuppsattes. Anfallsförbundet mot Sverige från år 1700 återuppväcktes efter Rysslands seger vid Poltava, som nu räknades som förbundets militärt starkaste och mest pådrivande partner. Även Preussen, Hannover och Mecklenburg övervägde att ingå i anfallsförbundet. Osmanska riket fruktade nya försök från Peter I:s sida att stänga Azovska sjön genom erövring av Kertj. Ryska galärflottan i Voronezj, Taganrog och Azov kunde i så fall genom detta sund få tillträde till Svarta havet. När Peter I i november 1710 anföll riket via floden Prut möttes de ryska trupperna av en stor turkisk motoffensiv. Under hotet att bli helt innesluten av de osmanska trupperna valde Peter I att sluta fred, och fredsavtalet som ingicks medförde inga större konsekvenser för Ryssland. +Segern vid Poltava och dess omedelbara följer gav Peter I möjlighet att återuppta sin offensiva krigföring både mot Livland och Finland och därigenom säkra Sankt Petersburg från båda sidorna av Finska viken. Under år 1710 erövrade ryssarna Elbing, Riga, Dünamünde, Pernau, Reval, Viborg och Kexholm från Sverige. Hamnstäderna gav Peter I örlogsbaser för offensiva krigsföretag mot den svenska flottan, såväl skärgårdskrig med galärer som sjökrig på öppna havet med örlogsfartyg. På så vis hade kriget efter Poltava gått in i ett nytt skede i vilken Ryssland, Danmark och Sachsen hade initiativet medan Karl XII från Bender förgäves försökte återta initiativet genom att verka för en mot tsaren riktad samordnad operation från osmanskt och polskt territorium. Magnus Stenbocks kapitulation vid fästningen Tönning satte stopp för dessa planer, och efter påtryckningar från osmansk sida valde Karl XII att återvända till sitt rike och anlände till Stralsund i november 1714. Han flydde till Skåne efter Stralsunds fall i december 1715 och både Pommern och Sveriges övriga provinser i Tyskland ockuperades av trupper från Danmark, Preussen och Sachsen. Från sitt högkvarter i Lund skapade Karl XII en ny armé för att genom en invasion av Norge tvinga Danmark till fred. Efter två misslyckade fälttåg blev Karl XII skjuten till döds den 30 november 1718 under belägringen av Fredrikstens fästning. Först efter ryska härjningar längs den svenska östkusten påskyndades fredsarbetet med samtliga parter. I freden i Nystad den 30 augusti 1721 slöt Sverige och Ryssland fred med varandra, och Peter I lät utropa Ryssland till ett kejsardöme med honom som dess förste kejsare. +Till åminnelse av slaget har det ryska kejsardömet rest flera segermonument och minnesmärken både i Poltava och på valplatsen. I samband med hundraårsfirandet av slaget restes den 16 meter höga "Minneskolonnen" (ukrainska: Монумент Слави, Monument Slavy) på Runda torget i staden, som invigdes den 27 juni 1811. På initiativ av Ivan Frantsevitj Pavlovskij, historielärare till en kadettskola i Poltava, öppnades ett slagfältsmuseum på valplatsen under tvåhundraårsfirandet av slaget 1909. Museet stängdes i samband med bildandet av Sovjetunionen 1922, men fick förnyad uppmärksamhet efter andra världskriget. På samma plats öppnades 1950 ett militärmuseum vars primära syfte var att framhäva Rysslands militära makt och att främja rysk nationalism, patriotism och den sovjetiska nationalitetspolitiken. Efter invigningen försökte olika svenska intresseorganisationer göra samarbetssamtal med museet i syfte att utbyta museiföremål, men den sovjetiska regeringen förbjöd samtliga kontakter. Först efter Sovjetunionens upplösning 1991 kunde museet skapa kontakt med olika svenska historieföreningar, inklusive med Krigsarkivet i Stockholm. Museet fick därefter regelbundna besök av svenska delegationer, liksom under slagets trehundraårsjubileum år 2009, där man genomförde ett historiskt återskapande av slaget med omkring 200 skådespelare och en ceremoniell invigning av "Försoningsmonumentet", en kupol av tre sammanfogade bågar till minne av alla stupade soldater från både Ukraina, Ryssland och Sverige. Den tyske historikern Guido Hausmann beskrev Poltava som en "imperialistisk minnesplats" för Ryssland och observerade att slagfältsmuseet skildrade slaget uteslutande från ett ryskt och ukrainskt perspektiv, medan de svenska och polska synvinklarna var underrepresenterade. Den ukrainska regeringen planerade att bygga statyer av både Karl XII och Ivan Mazepa bredvid den nuvarande statyn av Peter I. Rysslands ambassadör i Ukraina Viktor Tjernomyrdin jämförde detta med en hypotetisk staty av Adolf Hitler i Stalingrad. 1994 restes ett minnesmonument till de zaporizjakosacker som stupade i slaget, som enligt Hausmann "återspeglar den nya betydelsen och relevansen kring den tidigmoderna kosacktiden och kosackerna själva för nationellt medvetande i samtidens Ukraina." +I den ukrainska dramafilmen Molitva za hetmana Mazepu från 2002, som skildrar kosackhetmanen Ivan Mazepas kamp om zaporizjakosackernas självständighet, dyker slaget vid Poltava upp i en scen där Mazepa och Peter I sitter på ett matbord på slagfältet och diskuterar om Lillrysslands framtid. +1819 skrev den engelske författaren Lord Byron dikten Mazeppa, som skildrar Ivan Mazepa i början av sin karriär och som inleds med hans och Karl XII:s reträtt från Poltava. Den ryske nationalskalden Aleksandr Pusjkin ansåg att dikten gav en alltför romantisk bild av Mazepa och 1829 svarade han med en egen dikt, Poltava, som skildrar hetmanen ur ett enligt Pusjkin mer historiskt korrekt perspektiv. Dikten varvar en kärleksberättelse mellan Mazepa och Maria med en redogörelse av Mazepas svek mot Peter I och tsarens seger vid Poltava. Dikten inspirerade till Pjotr Tjajkovskijs opera Mazeppa från 1884. +Slaget skildras i den biografiska filmen Peter den Store (ryska: Пётр Первый, Pyotr pervyy) från 1937 regisserad av Vladimir Petrov och med Nikolai Simonov i rollen som Peter I. 2007 regisserade Oleg Ryaskov filmen Slaget vid Poltava (på ryska: Слуга Государев (Sluga Gosudarev), översatt till Härskarens tjänare). Filmen skildrar två franska adelsmän som beordras av kung Ludvig XIV att tjäna som observatörer i Peter I:s respektive Karl XII:s läger, och som kulminerar med att båda deltar i Poltavaslaget. Andrey Sukhov och Ed Fleroff spelar rollerna som Peter I respektive Karl XII. +Slaget gav dessutom upphov till det ryska talesättet "Som en svensk vid Poltava" (ryska: Как швед под Полтавой, Кak shved pod Poltavoy) för att beskriva någon som är helt handfallen och maktlös. +Den mest populära skildringen av slaget från svensk kontext har sedan 1988 tillskrivits Peter Englunds debutroman Poltava: berättelsen om en armés undergång, som Englund beskriver som "ett historiskt verk med en minimal vetenskaplig apparat". Boken har i olika upplagor sålt över en kvarts miljon exemplar och har översatts till flera språk. Englund är även upphovsman till konfliktspelet Peter the Great, en simulering av slaget vid Poltava. Slaget har även skildrats i ett kapitel i den första delen av Verner von Heidenstams skönlitterära verk Karolinerna från 1897–1898. Karolinerna sålde i olika upplagor över 120 000 exemplar och blev så småningom obligatorisk läsning i den svenska folkskolan. Elsi Rydsjö skrev 1992 boken Karolinas Poltava, den första delen i bokserien Karolinernas kvinnor, som skildrar soldathustrun Karolina Bäck och hennes liv och umbäranden efter hennes makes död vid Poltava. Vera Efrons bok Karolinerna efter Poltava från 2017 skildrar livet i fångenskapen för olika svenska soldater som stred vid Poltava, främst om kartografen Philip Johan von Strahlenberg och hans resor runt det ryska riket. Slaget nämns i den 78:e strofen av Anders Odels propagandaverk Sinclairvisan från 1739. +På den svenska teaterscenen citeras slaget i inledningen av August Strindbergs pjäs Karl XII från 1901. I rörlig bild har slaget skildrats i John W. Brunius film Karl XII/senare delen från 1925, med Gösta Ekman den äldre i rollen som Karl XII och Nicolai de Seversky i rollen som Peter I. I Roy Anderssons film En duva satt på en gren och funderade på tillvaron från 2014, den tredje delen i Anderssons "en trilogi om att vara människa", skildras Karl XII i två scener. I en scen rider kungen in i en bar med några karoliner och beställer ett glas mineralvatten, samtidigt som hans armé marscherar förbi mot Poltava. I en annan scen återvänder Karl XII med sina soldater till baren, svårt tilltygade efter nederlaget vid Poltava. En bartender går sedan fram till de gråtande kvinnliga bargästerna och säger: "Änka blev du i Poltava. Ett änkedok fick du." +Det svenska vikingarockbandet Ultima Thule skrev låten "Poltava", som ingår i albumet För fäderslandet från 1992. +Det svenska power metal-bandet Sabaton skildrade slaget i låten "Poltava", som ingår både i svensk och engelsk version i albumet Carolus Rex som släpptes i maj 2012. Texterna i albumets låtar faktagranskades av historikern Bengt Liljegren. +I England rapporterades slaget vid Poltava av Daniel Defoe under sommaren 1715 i hans bok The History of the Wars, som fem år senare skulle bli underlag till hans senare biografi om Karl XII. Berättelsen om slaget, skildrad av en döende soldat, återberättas i Al Stewarts låt "The Coldest Winter in Memory", som ingår i albumet Seemed Like a Good Idea at the Time från 1996. Slaget skildras i det tredje avsnittet av den amerikanska miniserien Peter the Great från 1986, baserad på biografin skriven av Robert Kinloch Massie. +Backscheider, Paula R. (1992) (på engelska). Daniel Defoe: His Life. Taylor & Francis. ISBN 978-0801845123 +Besters-Dilger, Juliane (2009). Ukraine on Its Way to Europe: Interim Results of the Orange Revolution. Franfurt am Main: Peter Lang. ISBN 978-3-631-58889-5 +Carlson, Ernst (1910). Sveriges historia under konungarne af pfalziska huset − Åttonde delen − Carl XII. Stockholm: Norstedt. Libris 392246 +Dahlberg, Alexander Magnus (1911). En karolins lefnadslopp, som i lifstiden af honom själf blifvit i pennan författadt. Stockholm: Hugo Gebers förlag. Libris 12604066. https://runeberg.org/dahlkarol/ +Englund, Peter (1988). Poltava: Berättelsen om en armés undergång. Stockholm: Atlantis. Libris 751738. ISBN 978-91-7353-461-1 +Floderus, Gustaf (1819). Handlingar hörande till konung Carl XII:s Historia. − Första delen. Stockholm: Zacharias Hæggström. Libris 8230918 +From, Peter (2007). Katastrofen vid Poltava. Karl XII:s ryska fälttåg 1707–1709. Lund: Historiska Media. Libris 10266190. ISBN 9789185377701 +Gedin, Per I. (2006). Verner von Heidenstam: ett liv. Stockholm: Bonnier. Libris 10136281. ISBN 91-0-011132-5 +Gordon, Alexander (1755) (på engelska). The History of Peter the Great, Emperor of Russia: To which is Prefixed a Short General History of the Country from the Rise of that Monarchy: and an Account of the Author's Life, Volume 1. Aberdeen: F. Douglass & W. Murray. ISBN 9781140726463 +Hausmann, Guido (2009). Poltava 2009: Deimperializing an Imperial Site of Memory. Cambridge, Massachusetts: Harvard Ukrainian Research Institute. http://shron1.chtyvo.org.ua/Guido_Hausmann/Poltava_2009_Deimperializing_an_Imperial_Site_of_Memory_anhl.pdf +von Heidenstam, Verner (2002). ”Poltava”. Karolinerna. Wahlström & Widstrand. ISBN 91-46-16607-6 +Karolinska förbundet (2001). Karolinska förbundets årsbok 2000. Lund. Libris 11356260 +Konow, Jan von (2001). Karolinen Rehnskiöld, fältmarskalk. Stockholm: J. von Konow. Libris 613872. ISBN 91-631-1606-5 +Kuvaja, Christer (2008). Karolinska krigare 1660–1721. Helsingfors: Schildts. Libris 11229569. ISBN 978-951-50-1823-6 +Lewenhaupt, Adam Ludwig; Hultman, Johan (1987). Karolinska krigare berättar – Adam Ludvig Lewenhaupts berättelse. Stockholm: Rediviva. Libris 613872. ISBN 91-7120-203-X +Liljegren, Bengt (2000). Karl XII: en biografi. Lund: Historiska Media. Libris 7776628. ISBN 91-89442-65-2 +Mandzy, Adrian; Knarrström, Bo; Nilsson, J.P. (2017). Poltava: Karl XII:s karoliner och stormaktens undergång. Stockholm: Svenskt Militärhistoriskt Bibliotek. Libris 21639218. ISBN 978-91-88053-60-2 +Massie, Robert Kinloch (1986). Peter den store: hans liv och värld. Stockholm: Bonniers. Libris 7247016. ISBN 91-34-50718-3 +Moltusov, Valerij Alekseevič; Eriksson, Bengt (översättare) (2010). Poltava 1709 – vändpunkten. Stockholm: Svenskt Militärhistoriskt Bibliotek. Libris 21639218. ISBN 978-91-85789-75-7 +Quennerstedt, August (1903). Karolinska krigares dagböcker jämte andra samtida skrifter. II, J. Lyths & L. Wisocki-Hochmuths dagböcker. Lund: Berlingska boktryckeriet. Libris 1563130. https://runeberg.org/karolikrig/2/ +Quennerstedt, August (1907). Karolinska krigares dagböcker jämte andra samtida skrifter. III. Lund: Gleerups förlag. Libris 1563131. https://runeberg.org/karolikrig/3/ +Quennerstedt, August (1909). Karolinska krigares dagböcker jämte andra samtida skrifter. V. Lund: Gleerups förlag. Libris 1563133. https://runeberg.org/karolikrig/5/ +Quennerstedt, August (1913). Karolinska krigares dagböcker jämte andra samtida skrifter. VIII. Lund: Gleerups förlag. Libris 1563136. https://runeberg.org/karolikrig/8/ +Rosander, Lars (2003). Sveriges fältmarskalkar: svenska fältherrar från Vasa till Bernadotte. Lund: Historiska media. Libris 7779602. ISBN 9189442059 +Strindberg, August (1916). Engelbrekt – Carl XII. Stockholm: Albert Bonniers förlag. https://runeberg.org/strindbg/engelbr/ +Sundberg, Ulf (2010). Sveriges krig 1630–1814. Hallstavik: Svenskt Militärhistoriskt Bibliotek. Libris 11856268. ISBN 978-91-85789-63-4 +Ullgren, Peter (2008). Det stora nordiska kriget 1700–1721: en berättelse om stormakten Sveriges fall. Stockholm: Prisma. Libris 10736897. ISBN 978-91-518-5107-5 +Wennerholm, Bertil; Nordlander, Ingegerd (2000). Emporterade troféer: Karl XII:s ryska fälttåg 1708-1709 i belysning av uppgifter om förluster av svenska fälttecken. Stockholm: Militärhistoriska förlaget. Libris 8379606. ISBN 91-85266-68-X +Wolke, Lars Ericson; Åselius, Gunnar (2003). ”Poltava 1709 – "Avbräck uti ett fältslag"”. Svenska slagfält. Stockholm: Wahlström & Widstrand. Libris 8858346. ISBN 91-46-21087-3 +Åberg, Alf (1998). Av annan mening: karolinen Axel Gyllenkrok. Stockholm: Natur & Kultur. Libris 7230117. ISBN 91-27-07251-7. https://archive.org/details/avannanmeningkar0000aber +Bidrag till det carolinska tidehvarfvets vittra literatur ur Lunds universitets handskriftsamling. 1, Sånger af en svensk fånge i Simbirsk. Lund. 1868. Libris 1842773. https://runeberg.org/svsf/11/0003.html +Carlson, Sten; Rosén, Jerker (1967). ”Slaget vid Poltava”. Den svenska historien – Karolinska tiden 1654–1718. Stockholm: Bonniers. sid. 265. Libris 8075047 +Christensen, Else (2009). ”Döden vid Poltava”. Världens historia (10): sid. 64−71. ISSN 0806-4709. +Eriksson, Bo (2009). ”Poltava 1709: här sänkte ryssarna stormakten Sverige”. Allt om historia (nr. 6): sid. 22–27. +Fryxell, Anders (1868). Berättelser ur Svenska Historien, Del 22 (andra upplagan). Stockholm: L. J. Hiertas Förlag. sid. 193−203. Libris 382333. https://runeberg.org/svhistfry/22/ +Grimberg, Carl (1922). Svenska folkets underbara öden – Band 4, Karl XI:s och Karl XII:s tid t.o.m. år 1709. Stockholm: Norstedt. sid. 728-741. Libris 385266. https://runeberg.org/sfubon/4/ +Grimberg, Carl (1914). Svenska krigarbragder. Stockholm: Norstedt. sid. 288−299. Libris 2002639 +Harrison, Dick (25 september 2019). ”Tre nycklar – så kunde Sverige segrat vid Poltava”. Svenska Dagbladet. https://www.svd.se/tre-nycklar--sa-kunde-sverige-segrat-vid-poltava. Läst 17 maj 2020. +Larsdotter, Anna (20 mars 2009). ”Slaget vid Poltava – vägen mot undergången”. Populär Historia (Sverige) (5). http://www.popularhistoria.se/artiklar/slaget-vid-poltava/. Läst 17 maj 2020. +Laurén, Anna-Lena (26 juni 2009). ”Poltava - 300 år senare”. Göteborgs-Posten. https://www.gp.se/nyheter/v%C3%A4rlden/poltava-300-%C3%A5r-senare-1.1076947. Läst 17 maj 2020. +Lindqvist, Herman (1995). Historien om Sverige – Storhet och fall. Stockholm: Norstedt. sid. 442–489. ISBN 91-1-300608-8 +Lindqvist, Herman (2014). När Finland var Sverige: historien om de 700 åren innan riket sprängdes (Ny utg.). Stockholm: Bonnier. Libris 16489037. ISBN 978-91-0-014277-3 +Starbäck, Carl Georg; Bäckström, Per Olof (1886). Berättelser ur Svenska Historien, Sjunde Bandet – Carl XII. Stockholm: F. & G. Beijers förlag. sid. 159−162. Libris 8225056. https://runeberg.org/sverhist/7/ +Svanelid, Tobias (18 juni 2009). ”Poltava 300 år”. Vetenskapsradion Historia. Sveriges Radio P1. https://sverigesradio.se/sida/avsnitt/81135?programid=407. Läst 17 maj 2020. +Wilson, Derek (9 mars 2009). Poltava: the Battle that Changed the World. "59". London. sid. 23–29. https://archive.org/details/sim_history-today_2009-03_59_3/page/23. +Wikimedia Commons har media som rör slaget vid Poltava. +Poltavas slagfältsmuseum +Slaget vid Poltava hos Tacitus.se +Berömda svenska slag, Poltava 28 juni 1709. Av Hans Högman +You Know You're Right är en promosingel från 2002 av det amerikanska grungebandet Nirvana, vilken släpptes som huvudsingeln från bandets självbetitlade samlingsalbum Nirvana. "You Know You're Right", som är skriven av bandets sångare Kurt Cobain, nådde första plats på topplistorna Hot Mainstream Rock Tracks och Modern Rock Tracks, men den tog sig även upp till plats 45 på Billboard Hot 100 och Finlands officiella lista. +"You Know You're Right" är den sista låten Nirvana spelade in tillsammans i en studio, vilket ägde rum den 30 januari 1994. Då Cobain påträffades död i sitt hem den 8 april samma år upplöstes bandet och all möjlig lansering av låten lades på is. Efter att Krist Novoselic och Dave Grohl bestämt sig för att grunda ett bolag under namnet Nirvana LLC ville de få Cobains änka Courtney Love att representera Cobains dödsbo vid beslut om lanseringar av tidigare outgivet låtmaterial av Nirvana. Love motsatte sig bolagets planer och en rättslig tvist ägde rum under 2001–2002, som i synnerhet gällde lanseringen av "You Know You're Right". I september 2002 rapporterades det att parterna hade förlikats och låten lanserades månaden därpå på samlingsalbumet Nirvana. +Musikvideon regisserades av Chris Hafner och består helt av klipp från diverse intervjuer, konserter och tidigare musikvideor med Nirvana. "You Know You're Right" blev ett av Hafners sista musikvideoprojekt och journalisten Alex Young på Consequence of Sound ansåg att den förmodligen var Hafners mest anmärkningsvärda verk någonsin och anmärkte att trots alla juridiska problem som hade genomsyrat lanseringen av låten gav musikvideon Cobain ett värdigt avslut. +Efter att Nirvanas tredje studioalbum, In Utero, hade lanserats i september 1993 började bandet turnera igen. Under en konsert den 23 oktober 1993 vid Aragon Ballroom i Chicago, USA genomförde Nirvana sitt enda uppträdande med den då okända låten "You Know You're Right". Precis innan bandet började spela "You Know You're Right" introducerade Dave Grohl låten genom att säga: "[d]etta är vår sista låt. Den heter 'All Apologies'"; Grohl var vid tillfället ovetandes om att Cobain redan hade börjat spela på "You Know You're Right". Detta missförstånd ledde till att Nirvanas fans felaktigt tolkade Grohls uttalande och på grund av konsertinspelningens lo-fi blev låten benämnd under de inofficiella namnen "Autopilot" och "On a Mountain". Under en intervju med musikjournalisten David Fricke två dagar efter konserten på Aragon Ballroom anmärkte Fricke att Cobain mer eller mindre förnekade låtens existens genom att säga att han inte hade några nya låtar på gång vid tillfället och att han var tvungen att börja om från början. Cobain visade även osäkerhet på Nirvanas musikaliska framtid och i en intervju med Rolling Stone i januari 1994 sade Cobain att han kände att Nirvana var på väg att bli ett repetitivt band. Cobain förklarade att Nirvanas fans inte hade något att se fram emot utan att bandets bästa tid hade passerat. Han sade att Nirvana skulle lansera fler studioalbum, men att han inte var säker på ifall dessa skulle bli grungealbum. +Cobain ville gå in Studio X i Seattle, Washington och se om Nirvana kunde spela in några improviserade låtar. Grohl tipsade honom istället om Robert Lang Studios i Shoreline, Washington eftersom den låg närmare hans hem och han tyckte den lät intressant. Cobain infann sig i inspelningsstudion den 30 januari 1994 för att spela in "You Know You're Right". Krist Novoselic och Grohl hade väntat på Cobain i studion i två dagar (någon riktig förklaring till varför Cobain var sen gavs aldrig) och under tiden de väntade spelade de in några låtar skrivna av Grohl, däribland "February Stars", "Exhausted" och "Big Me". Nirvana spelade tillsammans under hela dagen den 30 januari men lyckades bara färdigställa en låt, nämligen "You Know You're Right"; låten hade vid tillfället titeln "Kurt's Song #1". De slutförde större delen av låten i en tagning och Cobain behövde enbart komplettera med lite mer sång och gitarr efteråt. Denna inspelningssession blev den sista någonsin för Nirvana då Cobain hittades död i sitt hem den 8 april 1994 och i och med denna händelse upplöstes Nirvana. +En akustisk demoversion av "You Know You're Right" spelades in av Cobain under antingen 1993 eller 1994 och denna version släpptes senare på samlingsboxen With the Lights Out och samlingsalbumet Sliver: The Best of the Box. +Nirvana hade sedan tidigare använt sig av en musikstil som gick ut på att de skiftade mellan tysta och högljudda stycken, där "You Know You're Right" är ett exempel på en "klassisk Nirvana[-låt] med en långsamt krypande vers som exploderar i en ordlös, plågsamt renande och onekligen medryckande refräng." Musikrecensenten Jim DeRogatis beskrev låtens intro som "lynnig [och] uppbyggd runt några konstiga, ekande övertoner" och han påpekade även att gitarrsolot i "You Know You're Right" var ett av Cobains bästa någonsin. Även institutionen Rock and Roll Hall of Fame and Museum lyfte fram Cobains gitarrspelande i låten och kallade det för "uttrycksfullt" och "nyanserat". +Cobain förklarade att han under en längre tid var självmordsbenägen, främst på grund av de buksmärtor han upplevde, men att han runt tiden för inspelningen av "You Know You're Right" kände sig lyckligare än han hade varit på länge. Fricke har i efterhand tolkat Cobains låttext för "You Know You're Right" som att det var uppenbart att han inte mådde bra och att hans hälsa och sinnesstämning bara blev sämre. Jeremy Allen från The Guardian ansåg att handlingen i låten och det upprepande användandet av ordet "pain" (engelska för "smärta") visade på att Cobain misstänkte att hans dåvarande fru Courtney Love hade ett hemligt förhållande med sin tidigare pojkvän Billy Corgan. Både DeRogatis och Rock and Roll Hall of Fame and Museum påpekade att textraderna "Things have never been so swell and I have never felt this well" var gripande på grund av Cobains död så pass kort efter låtens inspelning samt att dessa textrader visade på hans "kärva kvickhet" som låtskrivare. Låttexten på liveversion från den 23 oktober 1993 är väldigt olik studioversionens låttext, där den tidigare innehöll textrader såsom "I just don't think it's worth it. I don't really love her. I don't think I want her. Ain't gonna take to love again. It's the way your love hurts. It's the way you're mad again." +I en intervju med The Guardian, publicerad den 16 augusti 2019, sade Grohl att han hade hört "You Know You're Right" för första gången på 10 år och att det var tufft för honom att lyssna på låten. Han kommenterade att tidpunkten då låten skrevs var en svår period för Nirvana, där Cobains mående varierade kraftigt och där Grohl själv upplevde depression för första gången i sitt liv. Grohl ansåg att Cobains låttext var "hjärtskärande" och att "You Know You're Right" musikaliskt sett var "renande". +"You Know You're Right" släpptes som en promosingel och huvudsingel från Nirvanas självbetitlade samlingsalbum Nirvana i oktober 2002. Singeln nådde första plats på topplistorna Hot Mainstream Rock Tracks och Modern Rock Tracks, men tog sig även upp till plats 45 på Billboard Hot 100 och Finlands officiella lista. Några månader före den officiella lanseringen av singeln hade "You Know You're Right" läckt ut på internet och börjat spelas på radio, trots att radiokanalerna hade hotats med så kallade cease and desist-brev ifall de försökte spela upp låten i förväg. Journalisten Alex Young på Consequence of Sound kallade lanseringen av "You Know You're Right" för "en av de sista bastionerna av episka, globala singelpremiärer på radio och TV." "You Know You're Right" släpptes senare även på samlingsalbumet Icon i augusti 2010. +I maj 2020 avslöjade Cameron Crowe att han mixade in "You Know You're Right" i Vanilla Sky, vilken hade premiär knappt året före singeln släpptes. Crowe hade fått tillgång till låten via Love, som hade bett honom att gömma den i filmen. +Rock and Roll Hall of Fame and Museum placerade "You Know You're Right" på plats 10 på deras lista "10 Essential Nirvana Songs". 2004 kom låten på plats 20 när New Musical Express listade de tjugo bästa Nirvana-låtarna någonsin och 2011 placerade tidskriften "You Know You're Right" på plats 10 på listan "Nirvana: Ten Best Songs". Loudwire placerade låten på plats 3 på listan "10 Best Nirvana Songs" och den hamnade på plats 10 på listan "15 Greatest Nirvana Songs" av Slant Magazine. "You Know You're Right" listades på plats 8 på "10 Best Nirvana Songs" av Diffuser.fm och kom även med på listan "The 15 Best Nirvana Songs Not On Nevermind" från 2011 av Paste. Jeremy Allen från The Guardian tog med "You Know You're Right" på sin lista över de tio bästa låtarna av Nirvana någonsin. Entertainment Weekly kallade "You Know You're Right" för "ryslig" och "skrämmande rå". Tidskriften gav låten betyget B+ och lyfte fram Cobains rossliga röst och smärtsamma sarkasm som höjdpunkterna. Charles Aaron från Spin placerade "You Know You're Right" på plats 5 över de bästa singlarna från 2002. +I februari 2003 tilldelades "You Know You're Right" en BDS Spin Award för 100 000 spelningar på radiostationer i USA. +Före lanseringen av "You Know You're Right" uppstod en rättslig tvist som främst involverade Novoselic, Grohl och Cobains änka Courtney Love. DeRogatis menade att grunden till denna dispyt lades redan våren 1992, när Cobain hotade att lämna Nirvana om inte Novoselic och Grohl gick med på de nya royaltykrav han ställde. Tidigare hade de tre fått lika mycket av royaltyintäkterna, men Cobains nya krav var att han i framtiden skulle få 75 procent av intäkterna för musiken medan Novoselic och Grohl fick dela på de resterande 25 procenten samt att för låtskrivandet skulle Cobain få 100 procent av royaltyintäkterna, vilket även skulle gälla retroaktivt sedan grundandet av Nirvana. Novoselic och Grohl var missnöjda med de nya kraven, men godtog dem för att undvika att bandet skulle splittras. De hyste sedan agg mot Love, som Cobain hade gift sig med i februari 1992, eftersom de ansåg att det var hon som hade fått Cobain att ställa nya royaltykrav; enligt Rosemary Carroll, Nirvanas advokat vid tillfället, var det i själva verket Cobain som var upphovsman till de nya kraven och inte Love. Cobain fick till slut 91 procent av royaltyintäkterna för låtskrivandet, där fem procent gick till Nirvanas före detta trumslagare Chad Channing och två procent vardera gick till Novoselic och Grohl. +Efter Cobains död i april 1994 efterlämnades mycket låtmaterial av Nirvana som aldrig hade offentliggjorts tidigare. För att ha möjlighet att hantera potentiella lanseringar av detta material föreslog Novoselic och Grohl att ett bolag under namnet Nirvana LLC skulle skapas, i vilket de båda samt Love (som representant för Cobains dödsbo) skulle ha lika rösträttigheter för hur hanteringen av bandets efterlämnade låtmaterial skulle gå till. Love var inte nöjd med detta förslag eftersom hon ansåg att det fanns advokater som var intressenter i bolaget och som hade kopplingar till Novoselic och Grohl sedan tidigare, vilket ledde till att hon redan från början ansåg sig vara i underläge. Love accepterade dock förslaget 1997, vilket hon i efterhand har sett som ett "stort misstag". +Novoselic planerade att lansera en samlingsbox med 45 Nirvana-låtar under hösten 2001, men i juni samma år lämnade Love in en stämning till domstolen Kings County i Washington i syfte att upplösa Nirvana LLC. I och med detta lades planerna att lansera samlingsboxen på is och istället påbörjades en rättstvist mellan Novoselic, Grohl och Love. Love ansåg att Nirvana LLC lät Nirvanas eftermäle förtvina och att det vore förkastligt att släppa "You Know You're Right" på en samlingsbox istället för ett samlingsalbum i stil med The Beatles 1. Novoselic har sagt att han förstod vad Love menade och att han alltid har försökt samarbeta med henne, men att han hade fått nog. Novoselic och Grohl valde att lämna in en egen stämning i december 2001, i vilken de hävdade att Love var "irrationell, livlig, egocentrisk, ohanterlig, inkonsekvent och oförutsägbar." De ansåg även att Loves stämning borde ogillas eftersom hon inte hade någon respekt för Nirvanas eftermäle utan enbart var ute efter publicitet och uppmärksamhet. Love förhindrade att "You Know You're Right" lanserades innan parterna hade gjort upp i domstol, men i september 2002 framkom det att Novoselic, Grohl och Love hade förlikats. Detta banade väg för lanseringen av samlingsalbumet Nirvana, som innehåller "You Know You're Right", i oktober 2002 samt samlingsboxen With the Lights Out, som släpptes i november 2004. +Musikvideon till "You Know You're Right" regisserades av Chris Hafner, vilket blev ett av hans sista musikvideoprojekt. Efter Cobains död i april 1994 upplöstes Nirvana och Hafner var tvungen att använda sig av tidigare inspelat videomaterial med bandet för att kunna redigera ihop musikvideon. I musikvideon syns klipp från diverse intervjuer, konserter (såsom den vid MTV Unplugged), tidigare musikvideor (såsom den för "Sliver") samt ett kortare bildspel med Nirvana. Mot slutet av musikvideon zoomar kameran in på en stillbild av Cobain när han håller på att slå sönder sin elgitarr och bilden övergår till myrornas krig och blir svart. +Musikvideon publicerades på MTV:s hemsida den 11 oktober 2002, hade premiär på MTV den 14 oktober och på MTV Europe den 15 oktober. Young ansåg att musikvideon förmodligen var Hafners mest anmärkningsvärda verk någonsin och att trots alla rättsliga problem som hade genomsyrat lanseringen av "You Know You're Right" gav den Cobain ett värdigt avslut. Musikvideon nominerades i kategorin Best Video vid NME Carling Awards 2003, men förlorade till Black Rebel Motorcycle Clubs "Whatever Happened to My Rock 'n' Roll (Punk Song)". +Flera artister har gjort coverversioner av "You Know You're Right". Den tidigaste kända coverversionen av låten är från Holes uppträdande på MTV Unplugged den 14 februari 1995, som sändes på MTV den 17 april samma år. Holes sångerska, Courtney Love, var Cobains änka och när Hole uppträdde med låten var det under titeln "You've Got No Right", där vissa textrader även hade ändrats av Love. Love erbjöd under 1995 låten till Mark Lanegan, som hade varit vän med Cobain, för att han skulle kunna spela in den med sitt dåvarande band Screaming Trees. I boken Lanegan berättade Gary Lee Conner, gitarrist i Screaming Trees, att under inspelningen av bandets sjunde studioalbum Dust hade Lanegan berättat för de övriga bandmedlemmarna att Love ville att de skulle spela in "You Know You're Right". De lärde sig låten, men när det väl var dags för Lanegan att börja sjunga den hade han ändrat åsikt och i slutändan valde bandmedlemmarna att inte spela in "You Know You're Right". +Ro Ransom gjorde en tolkning av "You Know You're Right" till sin Hidden Freestyle-serie och Richard Christ släppte sin version av låten på albumet Gorgeous than Evil-Stronger than Christ. Artister som har uppträtt med "You Know You're Right" under konserter är bland annat Limp Bizkit, Breaking Benjamin och Seether. +Alla låtar är skrivna och komponerade av Kurt Cobain, om inte annat anges. +Berkenstadt, Jim; Cross, Charles (1998) (på engelska). Classic Rock Albums: Nirvana - Nevermind. Schirmer. ISBN 0028647750 +Pennanen, Timo (2003) (på finska). Sisältää hitin: levyt ja esittäjät Suomen musiikkilistoilla vuodesta 1972. Otava Publishing Company Ltd. ISBN 951121053X +Lisbeth Salander är en fiktiv person som är skapad av den svenske författaren och journalisten Stieg Larsson. Hon förekommer i Larssons bästsäljande trilogi Millennium: Män som hatar kvinnor (2005), Flickan som lekte med elden (2006) och Luftslottet som sprängdes (2007), i David Lagercrantz officiella uppföljare Det som inte dödar oss (2015) och Mannen som sökte sin skugga (2017), samt i de franska seriealbumen med uppföljaren Millénium Saga av Sylvain Runberg och Bélen Ortega. Försäljningssiffrorna, och de fem filmer som hittills producerats med henne som en av huvudpersonerna, gör henne till en av de internationellt mest kända svenska rollfigurerna. +Salander är en inåtvänd hacker som jobbar som utredare på säkerhetsföretaget Milton Security. Tillsammans med journalisten Mikael Blomkvist löser hon mysteriet med den försvunna Harriet Vanger, och i samband därmed tvingas hon med våld att konfrontera sin ondskefulle förvaltare och därefter hela sin uppväxt då hon var inspärrad på mentalsjukhus. Hennes hämndhistoria får konsekvenser för såväl stora företagskoncerner som Sveriges kontraspionageorganisationer, och sägs bland annat vara Millennium-seriens största behållning. Hon har beskrivits som en "vänsterfeministisk actionfigur, som attackerar skärningspunkten mellan männens och företagens makt [...] en antifascist i vagt fascistiska symboler". +Bland hennes yttre attribut märks en stor tatuering av en drake på ryggen, svartfärgat hår och punk-influerade kläder. +I den svenska filmatiseringen av trilogin Millennium (2009) spelas Salander av Noomi Rapace. I Hollywood-versionen (2011) spelas rollen av Rooney Mara. Claire Foy spelar rollen i filmatiseringen av den fjärde boken The Girl in the Spider's Web. +De engelska översättningarnas titlar syftar alla på Salander. +Upprinnelsen till hela serien om Millennium är komplex, men Larsson började att skriva mer fokuserat på den under sommaren 2002, då han och sambon Eva Gabrielsson semestrade på en skärgårdsö och Larsson sökte något att skriva om. Ursprunget till Lisbeth Salander är lika invecklat, men kan särskiljas till följande huvudspår: feminism, Pippi Långstrump, andra deckarförfattare, framväxten av hackerkulturen och några personer i Larssons närhet. +Flera biografier har rapporterat om hur Larsson förändrades i sin syn på mänskligheten då han i 15-årsåldern bevittnade en våldtäkt på en flicka, där förövarna var tre av Larssons vänner (enligt en intervju med vännen Kurdo Baksi hette offret Lisbeth, men namnet förekommer inte i Baksis bok Min vän Stieg Larsson). Larsson hade inte agerat under våldtäkten, eftersom han upplevde delad lojalitet. I efterhand kontaktade han därför offret och bad om ursäkt, men blev avvisad som en medlöpare. Larsson sade upp bekantskapen med vännerna. Senare har historien ifrågasatts, eftersom Larsson enligt en kollega på Expo berättat att han hört någon annan berätta om gängvåldtäkten, men Larssons sambo Eva Gabrielsson menar att hon är den enda som Larsson berättade om händelsen för. +Kring årsskiftet 2001–2002 skedde två mord som påverkade Stieg Larsson starkt: Melissa Nordell, som torterades och mördades av sin pojkvän efter att hon ville separera, och Fadime Sahindal, vars far mördade henne eftersom han ogillade hennes livsstil. Till skillnad från den debatt om hederskultur och hedersmord som uppstod, ville Larsson komma bort ifrån den rasistiska åtskillnad som gjordes mellan svenskfödda och utlandsfödda förövare; den stora likheten var istället att offren varit kvinnor som förövarna velat utöva makt över och, när det inte fungerat, torterat och mördat. Larsson deltog bland annat i en antologi, Debatten om hedersmord. I samband med de morden (samt styckmordet på Catrine da Costa som aktualiserades flera gånger under 1990- och 2000-talen) lusläste Larsson polisrapporter och domar. Våldet han läste om tog han med som en del i böckerna, inklusive våldtäkten av Salander i Män som hatar kvinnor, men han skrev också kvinnoporträtt såsom han själv såg kvinnor: "fria, modiga, och starka nog att förändra sin värld genom att vägra bli offer." +Till skillnad från många andra författare identifierade sig Larsson starkare med sin huvudperson av motsatt kön än med den av samma kön. Även om det finns vissa attribut som Larsson delar med Mikael Blomkvist (journalist, chefredaktör för samhällsgranskande tidning, utan långsiktiga engagemang) har han gett Salander flera av sina personliga egenskaper: storrökare, dricker dåligt kaffe, "ursinnigt moralisk", saknar förtroende för myndigheter, ovilja att diskutera sin bakgrund, och duktig researcher. +I de tre böckerna förekommer korta inledande texter på vissa kapitel. Texterna kretsar kring krigiska kvinnor med exempel från det amerikanska inbördeskriget eller amasonerna. Däremot förekommer anmärkningsvärt nog inga nordiska exempel. +Under Stieg Larssons tid på Tidningarnas Telegrambyrå (1977–1999) diskuterade han ofta med sin chef Kenneth Ahlborn om populärkultur, såsom Tvillingdetektiverna och Wahlströms detektivserie, samt Enid Blytons Fem-böcker. De kom överens om att skriva varsin bok om hur det skulle gå för böckernas ungdomshjältar när de blev vuxna. "Stieg Larsson var för sin del helt inne på att han skulle skriva om DAMP-barnet Pippi Långstrump och vad som händer henne när hon kommer upp i tjugoårsåldern. Antagligen skulle hon råka ganska illa ut i det moderna samhället, bli inlåst på någon institution, kanske bli utsatt för fysisk och psykisk skändning och komma ut full av hämndlystnad." The New York Times räknar upp flera likheter: "modig och lojal men definitivt inte en förebild", "udda utseende" och "fantastiska förmågor". Dessutom bor Salander stundom under pseudonymen "V. Kulla", en tydlig anspelning på Pippi Långstrumps Villa Villekulla. Tanken var att hon inte skulle känna någon eller ha några sociala färdigheter. Även om Mikael Blomkvist också var modellerad på en Astrid Lindgren-figur, den unge besserwissern Kalle Blomkvist, var tanken att Salander skulle vara böckernas huvudperson. Salander har rött hår, som hon färgar svart, och i läsarens första möte med henne bär hon röd-gröna strumpor som påminner om Pippis ovanliga långstrumpor. +I Män som hatar kvinnor nämns dessutom Pippi Långstrump och Kalle Blomkvist explicit: "[Mikael Blomkvist] avskyr smeknamnet [Kalle Blomkvist], vilket man kan förstå. Någon skulle få en fläskläpp om jag blev kallad Pippi Långstrump på en löpsedel.” Pippi Långstrump nämns även senare när hon stulit en stor mängd pengar: "Hon stal nästan tre miljarder kronor. Hon torde inte behöva svälta. Precis som Pippi Långstrump har hon en kista med guld." +Larsson var en van deckarläsare. Trots att han läste "nästan allt som skrivits i genren under de senaste tvåhundra åren" och väl kände till numera mindre kända deckarförfattare som Erskine Childers och John Buchan skrev han för TT bara några artiklar om tips på deckare inför jul- och sommarledigheterna samt genomförde två intervjuer: med Harlan Ellison och med Elizabeth George (som han kallade "deckardrottning"). Bland hans stora favoriter märks mestadels kvinnliga författare, såsom Sara Paretsky, Patricia Cornwell, Val McDermid, Dorothy Sayers och Liza Cody. Troligen har dessa påverkat och inspirerat hans skrivande. Inte minst har Larssons tankar kring Salander påverkats av Elizabeth Georges utvikningsrika böcker om Lord Thomas Lynley och hans "besvärliga" medhjälpare Barbara Havers, samt av Sara Paretskys mer hårdkokta, samhällskritiska böcker där den kvinnliga privatdetektiven V.I. Warshawski tvingas konfrontera hänsynslösa storföretag. Peter O'Donnells serie om Modesty Blaise sägs också vara en förebild, till exempel vad gäller hennes tvetydiga sexualitet, hennes ovanliga sätt att hantera brott och att hennes manlige partner (Willie Garvin/Mikael Blomkvist) är nöjd med att spela andrafiolen. Överhuvudtaget var Larsson mest intresserad av de hårdkokta deckarna, i form av Dashiell Hammett, Raymond Chandler och Ross Macdonald, bland annat för att de inkluderade problemet att det inte alltid gick att avgöra vilka som var de värsta skurkarna: brottslingarna eller polisen. +Både Stieg Larsson och Eva Gabrielsson läste sedan barndomen stora mängder science fiction (framför allt Robert Heinlein och Isaac Asimov). Heinleins roman Friday (1982) handlar om en övermänniskofigur som möter fördomar från resten av samhället, och dessutom har kompromisslöst sex. Larsson skrev själv om Joan D. Vinges böcker, där hon ofta använder huvudpersoner från andra världar med utifrånperspektiv, ibland med ett kritiskt öga till samhället. +Ungefär samtidigt som Larsson började skriva serien Millennium hade IT-bubblan spruckit, men i kölvattnet av detta växte både efterfrågan och behovet av fler programmerare och datorvana personer, eftersom det stod klart att många affärer skulle ske via internet framöver. I samband med de högljudda diskussionerna om trackersajten The Pirate Bay (som grundades 2003) och senare Piratpartiet (som grundades 2006) kom också fler hackers ut i det offentliga ljuset. Larsson fascinerades av hackerkulturen, även om han själv inte var särskilt duktig på datorer, och beslöt att låta en av huvudpersonerna ha en bakgrund som hacker. Trots hans egen okunnighet på området har Larssons beskrivningar jämförts fördelaktigt med exempelvis Dan Browns IT-kompetens i Gåtornas palats (1998). +Larsson inspirerades bland annat av Neal Stephenson och då särskilt böckerna Snow Crash (1992) och The Diamond Age: or A Young Lady's Illustrated Primer (1995). Inte minst hämtade han därifrån en del idéer till nätverket Hacker Republic som en "distribuerad republik". +Stieg Larsson har också använt delar av flera verkliga personer i sin närhet som inspirationskällor till Salander. Flera av dessa är omstridda eller har oklart inflytande. +En sådan inspirationskälla var Larssons brorsdotter, Therese Larsson. Det var hon som hade övervägt en tatuering av en drake. Under skrivprocessen frågade Stieg Larsson ofta sin brorsdotter om hennes liv, inklusive hennes passion för kickboxning, hennes anorexi och hennes svarta smink och kläder. Det är dock obekräftade uppgifter: e-postkonversationen förlorades i en hårddiskkrasch och i den enda intervju som gjordes med Larsson nämner han enbart Pippi Långstrump, men Larsson kan ha velat skydda sin brorsdotter som då var i tonåren. +Enligt Kurdo Baksi arbetade en researcher och datahacker på Expo under dess första år. Hackern lär ha imponerat på Stieg Larsson med sin arbetskapacitet och talang, varit tatuerad och smal. Larsson avbröt dock deras arbete när det blev känt att hackern polisanmälts för skadegörelse och förföljelse av sin sambo. Baksi menar att Larsson bearbetade sorgen genom att ta med honom i boken, fast i något förändrad form. Denna inspirationskälla har dock ifrågasatts av flera kolleger på Expo, som menar att personen i fråga nöjde sig med en skrivmaskin och dessutom ägde de klandervärda händelserna rum 18 månader efter Stiegs död, och sju år efter att "hackern" lämnat Expo. +Enligt Eva Gabrielsson är Salander "en syntes av de människor vi kände [...] Män och kvinnor. Hon är ihopplockad för att vara den hon är. Jag tycker att hon är positivt excentrisk, en positiv nörd helt enkelt, och det var såna vi hade omkring oss, nördar." +Lisbeth Salander föddes den 30 april 1978 (hon är 24 år gammal i Män som hatar kvinnor) och har en tvillingsyster som heter Camilla, men systrarna har ingen kontakt. Flickornas mor heter Agneta Sofia Sjölander (född 1960). Deras far heter Alexander Zalachenko och är en avhoppad sovjetisk spion. Zalachenko misshandlade ofta mamman, något som slutligen ledde till att hon ådrog sig grava hjärnskador. I slutet av romanen Män som hatar kvinnor avlider modern av dessa hjärnskador efter flera år på ett vårdhem. +När Lisbeth var 12 år gammal och fadern hade misshandlat modern svårt, tände Lisbeth eld på sin far med hjälp av ett mjölkpaket som hon fyllt med bensin. Efter detta hamnade hon på Sankt Stefans barnpsykiatriska klinik. Fadern överlevde attentatet. På sin 18-årsdag sattes hon under förvaltarskap, officiellt på grund av händelserna under uppväxten, men egentligen för att skydda hennes fars verkliga identitet. +Fyra år före händelserna i Män som hatar kvinnor hade Salander fått jobb på Milton Security. Hon är en framstående hacker som på nätet går under pseudonymen Wasp. +I Män som hatar kvinnor arbetar Salander som researcher på Milton Security. Hon får av affärsmannen Henrik Vanger i uppdrag att kolla upp den nyligen nedsvärtade journalisten Mikael Blomkvist. Vanger har tänkt att anlita Blomkvist för att lösa ett gammalt försvinnande. När Blomkvist kör fast i sitt uppdrag, sammanför Vangers advokat honom med Salander. Hon bidrar till utredningen och upptäcker spår som leder tillbaka till flera gamla olösta, brutala mord. Under samarbetet blir Salander och Blomkvist tillfälligt ett par. +Efter att Salanders förmyndare Holger Palmgren fått en stroke, påtvingas hon en ny förmyndare, Nils Bjurman. Bjurman visar sig vara sadist, och vid första mötet tilltvingar han sig oralsex av Salander i utbyte mot att hon ska få tillgång till sina egna pengar. Under det andra mötet våldtar Bjurman henne brutalt, omedveten om att Salander filmar allting med dold kamera. Några dagar senare återkommer Salander till Bjurmans lägenhet, bedövar honom, binder fast honom med hans egen utrustning och berättar att hon tänker offentliggöra filmen om han bryter mot hennes regler som ska säkerställa att hon blir friskförklarad. Därefter tatuerar hon in texten "Jag är ett sadistiskt svin, ett kräk och en våldtäktsman" på hans bröst och mage. +Salander och Blomkvist lyckas lista ut hur Harriet Vanger försvann, men utredningen leder till att Salander och Blomkvist hittar en seriemördare. Denne fångar Blomkvist och avser att döda honom, men i sista stund ingriper Salander och lyckas rädda honom. Hon använder sedan sina hackertalanger för att ta reda på de verkliga omständigheterna kring Blomkvists nedsvärtning och för att stjäla en stor summa pengar av den skyldige. +I slutet av boken erkänner Salander för sig själv att hon är kär i Blomkvist, men får se honom tillsammans med hans återkommande älskarinna, Erika Berg, och klipper då kontakten med honom. +I Flickan som lekte med elden har Salander varit på rundresa i världen under ett år. Hennes vistelse i Grenada och en kort romans förkortas av en orkan, som också blir klimax på ett mordförsök, där Salander räddar livet på det tilltänkta offret. Hon återvänder därefter till Sverige och besöker sin tidigare flickvän, Miriam Wu, och sin tidigare förmyndare, Palmgren, som hon inte visste fortfarande var i livet. Efter ett slumpmässigt möte med Blomkvist hackar hon sig in i hans dator och upptäcker att tidskriften Millennium håller på med en bok om en prostitutionsverksamhet. Salander möter även Bjurman, vilket gör att hon beslutar sig för att konfrontera honom med hans agerande sedan deras senaste möte. +Strax mördas den nyanställde medarbetaren på Millennium, Dag Svensson (som arbetade med prostitutionshistorien) och dennes sambo. Polisen upptäcker att vapnet som användes ägs av advokat Bjurman – vilken också hittas mördad. Flera omständigheter gör att Lisbeth Salander blir misstänkt för morden. Kvällspressen lyckas ta reda på att Salander är misstänkt och publicerar efterlysningar på löpsedlarna tillsammans med besvärande detaljer ur hennes liv. +Trots detta tror både VD:n för Milton Security och Mikael Blomkvist på hennes oskuld. Blomkvist skickar meddelande till Salander som svarar att personen som ligger bakom morden heter Zala, ett namn som också förekommer i Dag Svenssons material. Salander spårar polisutredningen genom att hacka sig in i åklagarens dator. Hon upptäcker att en av torskarna i Millenniums utredning haft kontakt med Zala och tillfångatar och hotar honom till livet för att få reda på mer. Därefter tar hon sig till Bjurmans sommarstuga, där denne gömt material om henne som säger att den psykiater som plågat henne under hennes uppväxt varit kopplad till flera andra personer i konspirationen. Hon överraskas av två MC-knuttar som är där för att bränna ner huset, men lyckas övermanna dem och fly. +Blomkvist pratar med Holger Palmgren, som berättar att Zala egentligen heter Zalachenko och är Lisbeth Salanders far, något som Blomkvist kopplar ihop med tidigare uppgifter om att Zalachenko varit en rysk avhoppare som SÄPO hemlighållit. När Salander får veta att en blond jätte anfallit Miriam Wu, spårar hon jätten som heter Ronald Niedermann till Göteborgstrakten. Hon får upp ett spår och upptäcker att jätten befinner sig tillsammans med Zalachenko. Hon konfronterar dem, men blir tillfångatagen, skjuten och begravd. Salander överlever, lyckas gräva sig upp, överfaller Zalachenko med en yxa och skrämmer iväg Niedermann. Blomkvist fångar in Niedermann och ringer SOS Alarm. +I början på Luftslottet som sprängdes förs Salander och Zalachenko till Sahlgrenska sjukhuset. Salander är svårt skadad, men grips för de tre morden. Blomkvist övertalar sin syster, advokaten Annika Giannini, att ta sig an Salanders fall. Strax därpå mördas Zalachenko av den hemliga grupp inom Säpo som kallas Sektionen, som också försöker döda Salander. Giannini lyckas dock spärra dörren, varpå mördaren begår självmord. +Eftersom Salander misstror auktoriteter vägrar hon först att samarbeta med Giannini och förlitar sig istället på sina vänner i den svenska hackergemenskapen. De hjälper så småningom Blomkvist att få reda på den fulla omfattningen av Sektionens konspiration. Blomkvist tänker publicera ett specialnummer av Millennium om Salander, vilket gör att han hamnar i Sektionens fokus. Till slut skriver dock Salander en exakt beskrivning av de sexuella övergreppen som Bjurman utsatte henne för. +Likaså vägrar Salander att samarbeta under rättegången. Åklagaren, som använder psykiatern Peter Teleborian som huvudvittne, försöker porträttera henne som psykiskt instabil och i behov av långvarig vård. Giannini visar dock upp Salanders inspelning av våldtäkten och därmed att allt hon sagt varit sant, Blomkvists utredning av Sektionen och därefter avslöjas att Teleborians dator är full med barnpornografi. Rättegången avslutas och Salander frias. +Efter att Salander har återknutit kontakten med Miriam Wu meddelar Giannini att Salander ärvt en av Zalachenkos egendomar. Hon åker till den övergivna fabriken. Där attackeras hon av Niedermann som gömt sig där sedan deras senaste konfrontation. Hon lyckas mot alla odds att fånga Niedermann och meddelar MC-gänget (ur vilket Niedermann dödat flera) var han befinner sig. Hon ringer därefter polisen och meddelar var såväl MC-gänget som Niedermann befinner sig. +Samma kväll kommer Blomkvist till Salanders lägenhet och de förnyar sin vänskap. +I boken Det som inte dödar oss av David Lagercrantz möter läsaren åter Salander och Blomkvist, som hamnar mitt i aktuella frågor om nationell och internationell övervakning, mediekris och industrispionage. Blomkvist får ett tips om den tillbakadragne professorn Frans Balder och dennes autistiske son August. Balder är en ledande forskare inom artificiell intelligens, vilket inte intresserar Blomkvist förrän han förstår att en hacker som stämmer in på Lisbeth Salanders profil har ett finger med i spelet. Men även NSA har hittat hennes profil. Dessutom återkommer Lisbeths tvillingsyster Camilla. +Lagercrantz utvecklar Stieg Larssons porträtt av såväl den stenhårda och iskalla Lisbeth Salander som den evigt engagerade Mikael Blomkvist till att få nya och mjukare, inte fullt så överdrivna karaktärsdrag. +Salander framställs som ett geni med fotografiskt minne och i Flickan som lekte med elden löser hon det klassiska matematikproblemet Fermats gåta. Hennes övriga agerande (”tar fullständigt avstånd från samhällets regler och normer och ägnar sig åt beteenden som både är antisociala och illegala”) har dock fått vissa att föreslå att Salander kan vara en ”framgångsrik psykopat”, men att hon troligen har för mycket samvete för att vara en psykopat. Hennes personlighet har också fått vissa recensenter att göra jämförelser med Aspergers syndrom. Detta avstyrs av Larsson i Luftslottet som sprängdes, i och med att Salander friskförklaras efter domstolsscenen. Andra menar att Salander är en rollfigur i stil med Isaac Asimovs Wendell Urth eller Adrian Monk ur TV-serien Monk, med "övermänsklig förmåga att dra slutsatser, vars analytiska styrkor balanseras upp av en stor mängd känslomässig idioti". +Andra har fokuserat på Salanders hämndhistoria och angett den som en anledning till att Salander blivit en så populär rollfigur. Att hon istället för att gå till polisen efter att Bjurman våldtagit henne, ger igen med samma mynt kan verka som en gammeltestamentlig hämnd. Hennes hämnd, berättigad eller ej, är riktad mot upplysningsidén om rättsstaten, vilket gör att hämndhistorien går att uppfattas antingen som tillfredsställande eller som en riskabel glidning emot ett samhälle likt de i de fornisländska sagorna. Andra jämför med de grekiska dramerna: +Salanders etik förs exempelvis fram i hennes ”Salanders princip” (Män som hatar kvinnor, kapitel 18): ”Jag menar att en fähund alltid är en fähund och om jag kan skada en sådan genom att gräva upp skit om honom så har han förtjänat det. Jag betalar bara tillbaka. [...] Men saken är den att när jag gör en PU [Personundersökning] så tittar jag också på vad jag tycker om människan. Jag är inte neutral. Om det verkar vara en bra människa kan jag tona ner min rapport.” Hon har förlorat förtroendet för polismakten, vilket gör att hon inte kontaktar dem när hon blir utsatt för brott. Enligt Salanders etik avhåller hon sig också från att döda andra människor, men sätter däremot igång processer som leder till sina fienders död: finansmannen Wennerström, som hon spårat via hans dator, anger hon till den colombianska drogkartellen som han inte lyckats betala sin skuld till; Martin Vanger jagar hon tills han begår självmord genom att krocka med en bil; Niederman fångar hon och anger var han finns till det motorcykelgäng som han gjort sig ovän med. +Efter att ha stulit pengar från den bedräglige Wennerström och flyttat utomlands försöker Salander att förändra sig. Hon försöker såväl skaffa vänner som att leva ett vanligt liv. Det experimentet avslutas dock snart, och efter att ha återvänt till Sverige återtar hon snart sitt tuffare enstöringsliv. +Salanders roll som kvinnlig detektiv gör att hon i likhet med andra kvinnliga detektiver ständigt måste bevisa sin kompetens. "[Hennes] arbete på Milton Security beskrivs som närmast överambitiöst, rapporterna hon skriver om de personundersökningar hon gjort är alltid utförliga och korrekta med tydliga källhänvisningar." +Trots att Salander är mager och mycket späd äter hon stora mängder mat, ofta färdigmat. +Även om Salander har ett par relationer i trilogin − såsom hennes flickvän/väninna Miriam Wu, hennes förmyndare Palmgren, hennes chef Armanskij och vännerna i Hacker Republic − är den relation som har fått mest utrymme i böckerna den med journalisten Mikael Blomkvist. Deras relation löper genom hela trilogin och har liknats vid kontrasten mellan babyboom-generationen på 1940- och 1950-talen och Millenniegenerationen. +Blomkvist representerar då den generation som vill förändra samhället genom sina ideal (likt författaren Stieg Larsson), medan Salander representerar den generation som löser enskilda problem och felaktigheter. Den liknar relationen med Armanskij som är i ungefär samma ålder som Blomkvist, men Armanskij är beskyddande på ett sätt som gör att Salander känner att hon måste avvisa honom: "Jag är inte omedveten om vad du gjort för mig och jag är inte otacksam. Jag uppskattar att du visade dig vara större än dina fördomar och har gett mig en chans här. Men jag vill inte ha dig som min älskare och du är inte min farsa." Blomkvist visar å andra sidan Salander stor respekt och ger henne frihet, delvis beroende på att han redan har en deltidsrelation med Erika Berger. +I böckerna är Salander i 25-årsåldern, 150 centimeter lång och väger cirka 40 kilo. Hon har ett stort antal piercingar och nio tatueringar. +Den största är en drake som täcker nästan hela hennes rygg. Storleken på draktatueringen varierar mellan det svenska originalet och de engelskspråkiga översättningarna (där tatueringen till och med fått ge namn åt den första boken, The Girl With The Dragon Tattoo). I det svenska originalet betonas flera gånger storleken, från skuldrorna ner till stjärten och vidare till låret, medan storleken kan ha tonats ner i översättningarna för att bättre passa ihop med omslaget till en av böckerna, något som dock gör titelbytet mer oförståeligt och dessutom påverkar bilden av Lisbeth Salander. Eftersom den engelska översättningen fungerat som förlaga för vidare översättningar har detta påverkat bilden av Salander även internationellt. +Dessutom har hon en tatuering av en geting på sin nacke, en ögla kring sin ena ankel och en annan kring sin ena biceps, ett kinesiskt tecken på ena höften, och en ros på ena vaden. De flesta tatueringarna står för någon som på något sätt skadat henne, och är en påminnelse om hämnd. Vem eller vad draken på ryggen står för är okänt. Innan handlingen inleds i Flickan som lekte med elden har hon tagit bort getingen. +Hennes hår är egentligen rött, men hon har färgat det svart och klippt det mycket kort. Det röda håret "placerar Lisbeth [Salander] i en lång tradition av rödhåriga kvinnor som har blivit inordnade i ett fack som mycket intelligenta, starka, äventyrliga och passionerade." +Det korta håret och hennes smala kropp har fått en del recensenter att likna henne vid en pojke, eller en "gamine" (ett retsamt barn, i stil med Audrey Hepburn). Hon har även jämförts med Enid Blytons rollfigur George i Fem-böckerna, flickan som är klädd i pojkkläder och har andra manliga attribut. +I filmatiseringarna har Salander visserligen svart hår, men det är halvlångt, friserat i olika former av spikes, goth-frisyrer eller uppåtstående frisyrer. +Efter sin nyfunna rikedom och emigration byter Salander delvis utseende: hon tar bort tatueringen av en geting och gör en bröstförstoring. Anledningen till att hon förstorar sin tidigare ganska platta byst är ett ämne som förorsakat viss diskussion, men Larsson gör det tämligen klart att operationen har med hennes självbild att göra, snarare än att vilja behaga någon tänkt publik. +Hon är också trots sin ringa kroppsliga storlek en bra närstridskämpe, inte enbart för att hon använder ett elchockvapen utan för att hon har tränat boxning med Paolo Roberto. +Den brittiske författaren Brian Davis gjorde 2012 ett försök till fantombild på Lisbeth Salander, baserat på Larssons beskrivning. Davis arbete har bland annat rapporterats om av BBC. +Under 2011 diskuterades H&M:s kampanj med Lisbeth Salander-inspirerade kläder, framför allt framfördes kritik över att Salander inte skulle ha velat kommersialiseras och att hennes "look" är en våldtäktsöverlevares. Kampanjen kan ses som ett trivialiserande av ett stort trauma. H&M svarade att deras "Stockholm collection" hyllar Salander som är en stark kvinna, men härmar inte Salander specifikt och att det är ganska vanliga kläder som bärs på ett rebelliskt sätt. +2012 kvalade Salander in på tidskriften Forbes lista över de 15 mest förmögna fiktiva rollfigurerna. +En omdiskuterad del av Salanders person är hennes sexualitet. Hon har sex med såväl män som kvinnor i böckerna. "Som sexuell varelse kan hon inte etiketteras som vare sig homo-, hetero- eller bisexuell", menar Nina Björk. Eva Gabrielsson kommenterar att Salander inte kan etiketteras som något annat än bisexuell. Hon menar att när böckerna skulle filmas manade hon på producenterna att provspela killar i rollen. Salanders sexualitet är så komplex att hon bland annat kallats "erotiskt polymorf" Hon gillar att använda sexleksaker och har en förkärlek för äldre män. +Vid Gaygalan 2008 tilldelades Stieg Larsson postumt pris för Årets bok (Luftslottet som sprängdes). Enligt Eva Gabrielsson fick Larsson priset för rollfiguren Lisbeth Salander. Gabrielsson sa att "hon kommit till galan för Stiegs skull, eftersom Stieg skulle ha kommit, men för Lisbeths skull." +I den svenska filmatiseringen av trilogin Millennium spelas den vuxna Salander av Noomi Rapace. Rollbesättningsprocessen var längre än vanligt för regissören Niels Arden Oplev, och Rapace ville göra Salander till sin egen roll. Rapace genomgick en hårdbantning, tränade kickboxning, och lärde sig köra motorcykel. Dessutom skaffade hon flera piercingar. Rapace beskrev Salander så här: "Lisbeth Salander har genomgått så många hemska saker och folk har utsatt henne för mycket. Men ändå ser hon sig inte som ett offer. Hon tycker inte synd om sig själv. Hon hittar alltid ett sätt att ställa sig upp och besluta sig för vad hon vill göra och vilken sorts liv som hon vill ha, och hon gör det och ger aldrig upp." Rapace har senare blivit förknippad med rollen, och säger att producenter och andra blir förvånade över att hon, Rapace, inte är lik Salander i verkligheten. Rapace fick mycket goda omdömen även internationellt för sin rollprestation. +I de korta återblickarna som barn spelades Salander i de svenska filmatiseringarna av Tehilla Blad. Stuntscenerna i den svenska filmversionen utförs av Pernilla Ribeiro Novais, till vardags friskvårdskonsulent och tidigare elitidrottare inom judo (bland annat med deltagande i olympiska sommarspelen 2000). +Filmrättigheterna till böckerna köptes av Sony Pictures, bland annat på inrådan av Amy Pascal, en av de få filmbolagschefer som är kvinnor, som menade att trilogin skulle bli en central filmserie för Sony Pictures, och menade att Salander är en roll som alla kvinnliga skådespelare vill ha. Med tanke på att Salander är så olik övriga kvinnliga rollfigurer i Hollywood, och hur illa Hollywood brukar behandla kvinnliga rollfigurer, oroade sig många av Salanders beundrare för hur mycket filmerna skulle avvika från böckerna. I Hollywood-versionen spelas rollen av den tidigare praktiskt taget okända Rooney Mara. Även hon genomgick en hård träningsperiod inför inspelningen och piercade sig. +På grund av de stora internationella framgångarna för de skandinaviska filmatiseringarna kom flera jämförelser mellan Rapaces och Maras respektive tolkningar av Salander, där man bland annat menade att filmskaparna fokuserat på olika delar av Salanders egenskaper Den svenska journalisten Emelie Roslund menade att David Fincher/Rooney Maras version var mer objektiviserande, medan brittiska Alice Jones menade att Maras Salander var i paritet med Sigourney Weavers gestaltning av Ellen Ripley i Alien-filmerna. +I Danmark blev Män som hatar kvinnor efter långa förhandlingar teaterpjäs på Nørrebro teater i Köpenhamn 2010, i regi av Kim Bjarke, med Signe Egholm Olsen som Salander. Eva Gabrielsson var konsult under projektet, som snarare var ett urval scener ur boken med fokus på "maktmissbruk, diskriminering och våld mot kvinnor". +År 2011 blev Salander föremål för en facktext-antologi, Secrets of the Tattooed Girl av Daniel Burstein, Arne J. De Keijzer och John-Henri Holmberg, med bidrag av bland andra Christopher Hitchens, Mikael Ekman, Daniel Poohl och ambassadör Matthew Barzun. +Samma år kom också en parodi, The Dragon with the Girl Tattoo. +Böckerna har även adapterats som tecknade serier i olika omgångar. En amerikansk med manus av Denise Mina och bild av Leonardo Manco och Andrea Mutti, samt en fransk med manus av Sylvain Runberg och bild av José Homs och Manolo Carot. Den franska har även fått den fristående uppföljaren "Millennium Saga", med manus av Sylvain Runberg och bild av Belén Ortega. +Salander anges ibland som den största orsaken till att böckerna om Millennium och filmerna fått så stora framgångar. Hon har bland annat beskrivits som "den digitala ålderns första verkliga hjältinna". Stephen King har kallat henne "en av de stora rollfigurerna inom litteraturen, helvetiskt farlig trots att hon påminner om ett borttappat barn; hon sparkar karate lika bra som hon hackar datorer". +Den kritik som har förts mot henne kretsar exempelvis kring hennes "återuppståndelse" (efter att ha dödats och begravits av Zalachenko i bok 2) tillsammans med hennes närmast övermänskliga förmågor, som dock mildrats av att Salander samtidigt är socialt okunnig. Katarina Wennstam kritiserar såväl Salanders stöld från finansmannen Wennerström (även om hon också skriver att Salanders tilltag kan betraktas som ett Robin Hood-beteende som då blir "ursäktbart"), som det märkliga med att självständiga Salander skulle lägga sina pengar på att förstora sina bröst. Hon menar också att feminism inte handlar om att slå ner folk. Dessutom kritiseras Salanders funktion för att vara ett "onani-material för killar som vill låtsas att de inte är snuskiga". +I Stieg Larssons hemkommun Skellefteå finns sedan 2015 Lisbeth Salanders gata (och kvarteret Luftslottet). +stieglarsson.info: Liv och verk – Stieg Larsson, mannen bakom Lisbeth Salander +Moss Järnverk var ett järnverk i Moss i Norge, grundat 1704. Verket var i många år stadens största industri, och med kraft från Mossefossen blev malm, huvudsakligen från Arendalsfältet i nuvarande Arendals kommun, nersmält och bearbetad till en mängd olika produkter. Från mitten av 1700-talet var verket Norges ledande försvarsindustri och göt hundratals tunga kanoner i järn. Norges första valsverk för järn anlades också där. +Moss Järnverk hade flera framträdande ägare, bland andra Bernt Anker och Herman Wedel-Jarlsberg, och särskilt under den förstnämndes ägande var verket något av en attraktion för besökande. Moss Järnverks administrationsbyggnad, den så kallade Konventionsgården, är känd för att den användes under förhandlingarna av Konventionen i Moss år 1814. +Vid mitten av 1800-talet fick verket ökad konkurrens från svenska och engelska järnverk. I kombination med stigande priser på träkol försvann lönsamheten varvid järnverksdelen lades ner år 1873. Moss Järnverk såldes för 115 000 spesiedaler (en gammal norsk myntenhet) år 1875, och området verket låg på togs över av vad som sedan blev emballagefabriken M. Peterson & Søn (senare Peterson), som drev verksamhet där till dess företaget försattes i konkurs 2012. Idag ägs marken av rederiet Leif Höegh & Co.[källa behövs] +I vad som idag är Norge har järn utvunnits och bearbetats i över tiotusen år. Den första järnproduktionen använde limonit som förädlades, med en metod kallad järnutvinning, till råjärn för ytterligare bearbetning. Sådan tillverkning av järn var emellertid lokal och skedde i liten skala. Mycket arbete krävdes för att utvinna små kvantiteter av järn. +På 1500-talet växte intresset för utvinning av malm i Europa, och grunden för modern bergsvetenskap lades av tysken Georgius Agricola. I den norska delen av Danmark-Norge hittades bland annat förekomst av järnmalm. I södra Norge var järnfynden rikast i Arendalsfältet vid Arendal, men på grund av behovet av trä till produktionen av träkol för masugnarna och vattenkraften för att driva masugnarnas blåsbälgar, anlades ofta järnverk på andra platser än vid malmgruvorna. Vid Moss var det god tillgång på kraft från Mossefossen, det fanns mycket skog och läget vid fjorden gjorde inskeppning av råvaror och utskeppning av färdiga produkter enkel. +År 1704 började den danske ämbets- och förrättningsmannen Ernst Ulrich Dose arbetet med att etablera ett järnverk vid Moss, och samma år besökte den dansk-norske kungen Frederik IV Moss två gånger, något som påverkade etableringen positivt. Utöver att köpa upp tomter och få tilldelat sig fallrätt och tillgång till järnmalm, fick det planerade verket en så kallad cirkumferens, ett område med en radie på cirka 2,5 mil där bönderna var tvungna att producera och leverera träkol och andra råvaror till verket. I november 1704 blev Moss med omnejd besiktigat av anställda från oberbergamtet i Kongsberg, och de utsåg Moss Järnverks privilegiebrev den 6 december samma år. Brevet innehöll totalt 24 punkter om Moss Järnverks privilegier för skog, vatten, areal för själva verket, tillgång till körväg, malm, tullfrihet och flera andra punkter. +Särskilt intressant för samtiden är att funktionärer och arbetare var fritagna från skatter och militärtjänst, de skulle dömas av bergretten (en egen rätt för bergverk och bergverksanställde i Kongsberg), och vid behov av fackmän kunde de hämtas från utlandet, utan hänsyn till vilken nation de tillhörde. Med sina omfattande kungliga privilegier var alltså Moss Järnverk vid upprättelsen en stat i staten. +Den driftige Ernst Ulrich Dose fick aldrig uppleva vanlig verksamhet vid Moss Järnverk, han dog redan år 1706. +Efter Doses död lades järnverket ut på auktion och i maj 1708 såldes det till Jacob von Hübsch (3/4) och Henrich Ichsen (1/4). Tidpunkten för övertagandet av verket var gynnsam; kriget hotade och Moss Järnverk låg nära de tre stora fästningarna i Østlandet: Akershus, Fredrikstad och Fredriksten. Hübsch berättade själv år 1713 att han köpte det ofärdiga verket för att utrusta kungens styrkor med kanoner, ammunition och gevär. Kriget bröt ut år 1709, och 1711 levererade Moss Järnverk granater (kanonkulor i järn i olika storlekar) till ett fälttåg in i Sverige. Fälttåget blev kortvarigt, utan bruk av ammunition, och Hübsch hade stora problem med att få en uppgörelse för leveransen. +Ett av privilegierna som Moss Järnverk fick vid etableringen var tiondefrihet i tre år efter att masugnen var i kontinuerlig drift. Verket blev av generalkrigskommissarie H. C. von Platen stämt av bergretten i Kongsberg 1712 för att driva in tionde, men på grund av kolbristen var masugnen i drift endast periodvis. Bönderna blev ålagda att leverera 4 733 läster (en läst är cirka 2 m³) träkol årligen, men år 1714 hade verket hela 24 600 läster kol till godo hos bönderna. Böndernas pliktleverering av träkol var otvivelaktigt en extraskatt, och bönderna försökte undvika att leverera. Verket blev på grund av problem vid starten beviljat tiondefrihet för åren fram till och med 1715. +År 1716 blev Norge invaderat av svenskarna och den 17 mars blev Moss och järnverket erövrat av svenska trupper. Redan den 26 mars vid det första slaget om Moss kördes ockupanterna bort av norska trupper, men dagen efter erövrades verket igen och ockuperades i fem veckor. Under denna tid plundrades verket av de svenska trupperna. Krigsåren var svåra för verket. Förutom plundring var bönderna upptagna med transporter för trupperna och hade liten kapacitet till produktion och transport av träkol. Under tiden som de andra norska järnverken tjänade mycket pengar på kriget, gick det dåligt för Moss Järnverk och dess huvudägare Hübsch. På grund av alla problem ansökte Hübsch hos kungen för fler tiondefria år och fick det till och med år 1722, år 1723 var verket tvunget att betala 300 riksdaler och från år 1724 samma summa som de andra järnverken, alltså 400 riksdaler. +I oktober 1724 dog Jacob von Hübsch och hans änka, Elisabeth Hübsch (född Holst), tog över verket. Detta blev en tung börda för en ensam kvinna med sju barn. För att kunna följa upp verksamheten flyttade hon och barnen från Köpenhamn till Moss. Konkurrensen från billigt, svenskt järn var ödeläggande och många norska järnverk upphörde med verksamheten. Fru Hübsch var tvungen att ta stora lån för att driva verket och borgenärerna blev så småningom påstridiga. Trots att det från år 1730 infördes ett monopol för norska järnverk att leverera järn till Danmark var verkets ekonomi fortsatt ansträngd. +Verkets minoritetsägare, etatsråd Ochsen, måste ständigt täcka oundvikliga utgifter, och år 1738 var hans tålamod slut. Ochsen skickade sin personlige fullmäktige Jens Bondorph från Köpenhamn för att verkställa en så kallad kvarstadsförrättning (beslagstagning) för att få kontroll över företaget. Elisabeth Hübsch försökte avslå och förhala det hela och i rättsarkiven är verkets verksamhet och skuldsituation ganska grundligt dokumenterat. Den 21 januari 1739 bestämde oberbergamtet i Kongsberg att Jens Bondorph skulle, å Ochsens vägnar, driva den del av verket som tillhörde fru Hübsch, tills tvisten mellan de två var avgjord. Efter ytterligare några års rättslig strid lades Moss Järnverk ut på auktion och Henrich Ochsen övertog hela verksamheten. +Ett järnverk på 1700-talet representerade en kapitalintensiv tung industri av stor vikt för landet och det fanns en rad utmaningar för att få en tillfredsställande drift. Vid kvarstadsförrättningen år 1738 nedtecknades Moss Järnverks rörelsetillgångar, något som med förvaltaren Knud Wendelboes rapport till eftervärlden ger en bra bild av verksamheten. I styckena under beskrivs de olika sidorna av verksamheten på verket, från upprättandet år 1704 till nedläggningen 1874. +De gruvor som enligt verkets privilegier var förpliktade att leverera malm kunde inte leverera tillräckliga mängder, så verkets direktör Peter Windt blev år 1706 försäkrad att Løvold gruva i Arendal årligen skulle leverera 1 000 tunnor malm. Kvalitén på den levererade malmen var så varierande att det gick till rätten, detta slutade med att två bergskunniga skulle kontrollera att malmen var fritt från gråberg. Gruvorna i Arendal var de viktigaste för de norska järnverken då de levererade ungefär två tredjedelar av all järnmalm. Trots att de viktigaste gruvorna låg långt ifrån Moss Järnverk blev transporten ändå billig eftersom den skedde sjövägen. +I sin rapport om verkets tillstånd från år 1723 påpekar förvaltaren Knud Wendelboes att man måste spara malm i 2–3 år för att få tillräckligt till en blåsning. År 1736 var, förutom några mindre lokala gruvor i Moss, också de större gruvorna Østre Buøy, Vestre Buøy, Langsæ och Bråstad i Agder tillhörande Arendalsfältet underordnade Moss Järnverk. Verkets ledning var dock inte uppmärksam nog med att följa upp att gruvorna levererade malm av tillräckligt god kvalité. +År 1749 nämns också Weding gruva i Arnedal. Malm därifrån utmärkte sig för sin allsidighet och kännetecknades som verkets bästa. Vid sidan om gruvorna i Arendal var Skien ett centrum för verkets gruvverksamhet, och en rad olika gruvor drevs där. Verket hade ombud i Skien och Arnedal som följde upp gruvorna, betalade gruvarbetarna och såg till så att de så kallade mutingsbrevene (drifttillstånden) inte löpte ut. I de många år som Lars Semb var föreståndare på verket reste han nästan varje år till malmfälten och bodde hos de ombud som var verkets lokala representanter. +Verket var helt beroende av träkol som bönderna runt verket producerade i kolmilor. I slutet av vintern 1719–1720 hade eftersläpningen i böndernas leverering av träkol nått totalt 28 000 läster, alltså en restskuld på 33 726 läster. De saknade leveranserna skylldes dels på att skogen var nerhuggen, dels på kriget men också på att bönderna var motvilliga att efterleva levereringsplikten. +Bristen på träkol fortsatte under Ancher & Wærn, trots att de betalade bättre än andra verk och gav bonusar till dem som levererade mer än begärt parti. Den största orsaken till dåliga leveranser var att kolbrytning nästan alltid var mindre lönsamt än annan användning av skogen. Under Bernt Anker försökte verket att få myndigheternas tillåtelse till repartering (ett bestämt parti per gård). Det fick han inte igenom och det var tydligt att myndigheterna inte önskade att gå för hårt åt bönderna för att få dem till att efterleva levereringsplikten. För hela perioden 1750-1808 tog verket i genomsnitt emot 6 000 läster träkol medan behovet i full drift var över det dubbla. Det mesta av träkolet blev bränt och levererat av bönderna på vintern och då transporterna var endast en läst blev det många turer för att få allt levererat. +Wendelboe påpekade i sin rapport från år 1723 att det var brist på vatten. Detta skylldes dels på att forsen var utan damm som kunde samla upp svämvattnet, och dels på de andra användarna av forsen (kvarnar och sågverk) använde den mer än de skulle. Avstannade vattentillförseln till masugnsbälgarna skulle ugnen snart slockna, vilket skulle medföra stora förluster. +Med de stora arbetena år 1750 fick verket en stor vattenränna som gick över kungsvägen, den föregående hade gått under vägen. Vattenrännan var 8-9 fot bred, 6 fot djup och fallhöjden var 48 fot. Vattenbristen blev ett mindre problem än det hade varit verkets första år, men under en ovanlig torka 1795 började man att spränga i Krapfors och bygga dammar ovanför, något föreståndare Lars Semb menade borde ha gjorts för länge sen. På själva järnverket (inte innefattande kvarnar och sågverk) var det runt år 1810 totalt 24 vattenhjul, en del av dem var väldigt stora. +Själva järnverket bestod av en rad byggnader norr om älven i Moss. Huset med masugnarna var den centrala byggnaden. Här stod det två 31 fot (ungefär 9,5 meter) höga ugnar under samma tak men endast den östra var i bruk. Förutom själva masugnen fanns olika tyngre utrustningar, till exempel en vinsch för hantering av tunga gjutstycken. Öster om masugnen låg formhuset där det flytande järnet göts. Vid masugnshuset låg ett stenhammarhus med en vattendriven hammare för att krossa malmen. Det var två kolhus till masugnen, den västra mätte 56 × 25 alnar (ungefär 550 m2) medan det östra mätte 45 × 20 alnar (ungefär 350 m2). +Brandrisken var stor så det fanns också en brandstation med en dubbel brandspruta (två slangar) på området. Öster om formhuset fanns en smedja där finare smidesarbeten utfördes. Vid sjön anlades en brygga med en stor åttakantig knuttimrad sjöbod med två överloft. Verkets ägare bodde i en stor huvudbyggnad med nio rum och två våningar. I ett annat tvåvåningshus var det skrivkammare, alltså kontor. Vid sidan om fanns andra byggnader, som uthus för hästar, lada, höloft och tio hus åt arbetarna. Viktigt för verket var också brosågen, där det var tillåtet att såga nödvändigt timmer för nybyggen, reparationer och så vidare. Verket kunde såga 12 900 bord på brosågen, vidareförsäljning var dock strängt förbjudet och medförde avstängning. +Förutom de olika utbyggnader och förnyelser genom de 170 år verket var i bruk, var delar av verket tvunget att återuppbyggas efter bränder. På 1760-talet brann till exempel det dyrbara hammarverket och den nedre hammaren stod nyrest år 1766. +På flera områden var Moss Järnverk tekniskt framstående, det var det första verket i Norge med höga masugnar, kanongjutning, spikfabrik och valsverk, det sista restes år 1755 efter en engelsk förebild och till den betydliga kostnaden av runt 12 00 riksdaler. +Vid etableringen av Moss Järnverk var ammunitionsproduktion planerad. Enligt Wendelboe gick det mesta av järnet fram till år 1723 till granater och kulor, endast en mindre del av det gråa tackjärnet gick vidare till bearbetning med hammare. I en översikt från Landetatens Generalkommissariat (landsförsvaret) den 2 maj 1720 gavs följande översikt: +Priset för den grovkalibrerade ammunitionen angavs vara 11,5 riksdaler för skeppundet, medan det för smågranater var 14,5 riksdaler, alltså ganska stora summor totalt. Enligt kontrakten från år 1716 levererade verket fram till år 1720 totalt runt 2 500 skeppund till ett värde av nästan 30 000 riksdaler. År 1713 producerades också 1 000 gevärspipor. +Vid sidan om den militära produktionen fanns det en mindre och varierande civil produktion. Moss Järnverk producerade städ, hjulstokkringer (ringar i järn till hjulen på hästkärror), smidde sågblad, göt grytor, munkpannor, våffeljärn och bryggpannor. Verket levererade också specialbeställningar, som en hel sockerugn till ett större sockerraffinaderi i Fredrikshald på 1750-talet. På 1800-talet gjorde de också strykjärn. Under Bernt Ankers tid, var tunnbandsjärn en av huvudprodukterna, som också exporterades. +Verkets produktion av järnugnar (ofta felaktigt kallade kakelugnar) är särskilt intressant, både på grund av att flera är bevarade och för deras konstnärliga dekorationer. Flera av ugnarna blev av konsthistorikern och riksantikvarien Arne Nygård-Nilssen uppskattade att vara formgivna av Torsten Hoff, en erkänd ugnskonstnär, som tog burskap som bildhuggare i Christiania 1711 och verkade där fram till hans död 1754. +Ancher & Wærn investerade också i upprustning av ugnsproduktionen, och genom en av sina förbindelser i Köpenhamn fick de Henrik Lorentzen Bech (1718–1776) att komma till Moss där han arbetade med ugnsutformningar från sankt Hans (midsommar) 1750 till året efter, och därefter nästan hela år 1769. Från Bechs sista tid vid verket är tre främsta utformningar kända; ”Medaljongen”, ”Herkules” och ”Altertavlen”, Moss Järnverk producerade dessa mycket populära former åren som kom. +Förutom kanonleveranserna i tabellen ovanför producerades 29 stycken 12-pundskanoner som sjönk vid en transport år 1759. Kvalitén på kanonerna hade från Bernt Ankers tid varit bra, enligt Lars Semb hade ingen spruckit på 30 år. Under Ancher & Wærn blev kanonerna märkta ”AW”, medan senare användes ”MW”. En vanlig inskription på kanonerna var Liberalitate optimi (med största generositet), som visade på privilegiet och det stora lånet från statskassan. +Moss Järnverk levererade kanoner upp till 36 pund, men också 1/8 punds kanoner producerades, som kunde anses som en leksak. År 1789 började man misstänkta att 18- och 12-pundskanonerna inte var pålitliga och provskjutning i flera städer bekräftade misstanken. Detta bidrog till att man gick över till svenska kanoner, och kanonerna från Moss fasades ut. Så sent som vid Slesvig-holsteinska kriget 1848–1850 blev emellertid bastionerna vid Fredericia i Danmark beskjutna av 12-pundskanoner från Moss Järnverk. +Moss Järnverk var ett samhälle i sig självt där det låg norr om staden Moss. Förmannen på verksamheten blev vanligtvis betecknad som förvaltare, och den första var Niels Michelsen Thune, från verkets start till 1716. Vid sidan om Thune var en Peter Windt direktör åren 1706–1708. Efter Thune övertog sannolikt Knud Wendelboe, som skrev flera utfyllande rapporter om verket. Efter kvarstadsförrättningen tog Jens Bondorph över, men då Ochsen sålde verket hette förvaltaren Wichman. +Arbetarna på verket var mer anonyma, men typiskt för den högteknologi som dåtidens järnverk hade så kom de flesta av dem från andra länder, namnen bär spår av Sverige och Tyskland. Hur viktiga yrkesarbetarna var för verket visade sig när Hübsch i ett brev till kungen 27 december 1719 berättar att han, under tiden krigen pågick och verket var overksamt, alltjämt måste betala sina arbetare för att inte förlora dem. +På 1730-talet var det brist på arbetskraft, och arbetarna på Moss Järnverk utnyttjade detta genom att strejka och kräva högre lön vid kvarstadsförrättningen år 1738. Förhållanden för arbetarna var trots det magra. Carl Hübsch skrev år 1738 att verket inte förde någon översikt över arbetarna som byggde hus på verkets mark, då de var så fattiga att man ändå inte kunde få någon markhyra av dem. +Järnverkssamfundet hade på flera sätt ett ansträngt förhållande till staden Moss; ett var verkets privilegier med tullfri införsel av matvaror, när stadens medborgare måste betala konsumtionsskatt på alla varor som passerade stadsgränsen. Ett annat var både användandet av vattnet i Mossefossen och bron över forsen, underhållet av den var bekostat av stadens medborgare, något de fann orättvist. Förutom dessa friktionspunkter fanns också juridiken. Järnverket hade en domstolsmyndighet över sina arbetare, med bergamtsretten i Kongsberg som överklagarinstans, vilket rådhuset ogillade. I praktiken blev rättssaker nästan alltid avgjorda lokalt, med ett så kallat verksdom. Två pojkar som hade stulit potatis från en bondgård blev av föräldrarna straffade med piska i verkets fängelse, något föräldrarna till synes var nöjda med och: ”ödmjukt anhåller om att straffen inte blir annorlunda och att pojkarna blir befriade från att resa till Kongsberg för att bli bestraffade där”. +Kyrkoherden Christian Grave i Rygge och Moss var inte heller positiv till verket, något han visar i ett uttalande från år 1743: +Verket hade också ett stort behov av outbildad arbetskraft, och många kvinnor arbetade där, till exempel rullade en Thore Olsdatter slagg, jord och stenkol för 10 skilling om dagen. Bara i det kyrkliga var järnverksarbetarna underlägsna den lokala administrationen. De var friade från militärtjänst, men bildade en egen kår i oroliga tider. Verket gav år 1786 tillåtelse till två värdshushållare att driva sin verksamhet innanför verkets område, efter uppföranderegler och mot en årlig avgift som skulle gå till verkets fattiga. Värdshushållaren fick inte skänka alkohol mot matvaror; hembränning, kortspel och dans var också förbjudet och värdshusen måste stänga klockan 22 på kvällen. +Föreståndare Lars Semb upplyste år 1809 att skolan på verket var upprättat ungefär 40 år tidigare, alltså runt 1770. Innan dess hade barnen gått i skola inne i Moss. Från andra skildrare är det emellertid känt att verket redan år 1758 betalade skollärare Andreas Glafstrøm 6 riksdaler i månaden. År 1796 annonserade verket efter en lärare, och i annonsen stod det bland annat att han skulle kunna: ”påvisa goda intyg om sin skicklighet och duglighet till att lära barn läsa, skriva och räkna”. +Moss Järnverk hade från slutet av 1700-talet en ”skola- och fattigkassa”, arbetarna drog in ungefär 2 % till denna, de fick då gratis läkarvård och mediciner, men bara för sig själva, inte till deras familjer. År 1790 sändes en kvinna från verket till Köpenhamn för att utbildas till barnmorska. Hon arbetade sedan som barnmorska på verket och i området fram till 1816. +Verkets fattigkassa gav ett litet underhåll till änkor, barn och utarbetade arbetare – mest i form av husrum och tjänster. Det verkade som att verkets ledning inte önskade fasta regler för stöden, de skulle komma som nådegåvor, och Bernt Anker var svag för sådana gester. Antal fattiga ändrades över tiden, år 1820 angav Lars Semb till exempel antalet till 30. Verkets många gruvarbetare hade inga plikter eller rättigheter till skola- och fattigkassan, men verkets ägare tog hand om dem. +Efter århundradeskiftet blev antalet utländska arbetare vid verket färre. Mantalet år 1872 visade att det var 270 personer som arbetade på verket, åtta svenskar och två tyskfödda: förvaltare Ignatius Wankel och hans bror Frantz. År 1845 gick flera av arbetarna med i Moss stads Maadeholdsselskab (ett sällskap som förespråkade måttlighet med alkoholhaltiga drycker) och året efter var antalet verksarbetare i föreningen hela 30 stycken. Arbetarna från verket var också aktiva i Thranebevegelsen (ett sällskap som förespråkade rösträtt för arbetare). Det bildades en lokal förening efter Marcus Thranes besök i staden 16 december 1849. +Ochsen sålde verket år 1748 för 16 000 riksdaler till Erich Ancher och Mathias Wærn som båda kom från Fredrikshald där de drev en framstående verksamhet med tobaksfabrik och sågproduktion. Det kan ha funnits flera orsaker till att Ancher & Wærn köpte verket; det gav prestige, det gav firman ytterligare en förutsättning i dåliga tider, tullpolitiken kunde ändras så att införsel av varor från Sverige blev dyrare, men det viktigaste var nog verkets potential för produktion av krigsmaterial. +Då Ancher & Wærn köpte verket var det i dåligt skick och de var tvungna att investera mycket för att kunna producera stora kanoner. Redan i april 1740 skrev de till Landetatens Generalkommissariat om att få anlägga ett kanongjuteri vid verket. Kung Frederik V av Danmark var positiv, men enligt Anchers senare berättelser skedde genombrottet under kungens besök i Norge sommaren 1749. Kungen besökte Erich Ancher i Fredrikshald och var hela tre gånger i Moss, så det är rimligt att tro att kungen både besökte järnverket och blev introducerad för planerna om kanonproduktion. +Under hösten 1749 fick Ancher & Wærn sin ansökan om privilegier till kanonproduktion behandlad av Landetatens Generalkommisariat, stödd av företagets representant Johan Frederik Classen. Den 5 november 1749 fick Ancher & Wærn privilegium exclusivum, en 20 år lång exklusiv rätt till gjutning av järnkanoner och mörsare. Detaljerna i kontraktet upprättades 7 februari 1750. +Bland de många punkterna i kontraktet var ett förskott på 20 000 riksdaler (14 000 omedelbart och 6 000 efter provgjutning) och krav på att två 12-pundskanoner skulle produceras och prövas innan hela förskottet utbetalades. Varje år skulle minst 20 stycken 18-punds- och 30 stycken 12-pundskanoner prövas av kunniga artilleriofficerare innan leverering. Priset blev satt till 12 riksdaler och 48 skilling per skeppund. +Arbetet med kanonproduktionen började redan år 1749. Två kanoner blev gjutna, men båda sprack under provskjutningen. Under våren och sommaren 1750 pågick ett arbete för att möjliggöra en utökning av verket. Anställda vid verket blev skickade utomlands för att lära sig processerna, och yrkesarbetare hämtades från utlandet. Det producerades mindre kanoner det året, men gjutning av 12- och 18-pundskanoner till den avgörande provskjutningen stod still på grund av den ständiga kolbristen; till produktionen av så tungt artilleri krävdes båda masugnarna. Frampå vintern och våren 1751 uppstod ytterligare problem och de två första 12-pundskanonerna var inte färdiggjutna och klara för provskjutning förrän vid årets slut. Den 18 december kom överste Kaasbøll från Christiania för att följa provskjutningen, resultatet blev emellertid att båda kanonerna sprack vid den kraftigaste laddningen. Verket skyllde på bristen på yrkesarbetare och Mathias Wærn bad i ett brev den 14 april 1751 hans bror Morten Wærn, på resa i Frankrike, att försöka få tag i en erfaren kanongjutare. +I januari 1752 skrev Erich Ancher ett långt brev till generalkommissariatet, som handlade om de olika problemen vid kanonproduktionen, med förslag till lösningar. Kanonerna borde gjutas med tjockare gods, mindre villkorlig prövning och ett fast räntefritt förskott på 50 000 riksdaler (inklusive de utbetalade 14 000 och de utlovade 6 000). Enligt Erich Ancher hade företaget förlorat runt 10 000 riksdaler på verksamheten, och om myndigheterna i Köpenhamn så önskar, kunde de gärna avbryta, kanonproduktionen hade inletts för fäderneslandets bästa. Den 30 augusti 1752 gavs ett kungligt beslut att Moss Järnverk skulle få ett utöka förskott, med pant i verket och med lyckade provskjutningar, nu i Köpenhamn, som villkor. +Den 17 april 1753 blev två 12-pundskanoner från verket provskjutna i Köpenhamn, den ena sprack medan den andra höll. För att få tillfredsställande leveranser av kol arbetade verket med att utöka omkretsen, och den 27 augusti 1753 var en kommission i Moss för att förhöra fogdar, länsmän och andra om gårdar, skog och avstånd till verket. I ett kungligt beslut 20 maj 1754 bekräftades emellertid endast verkets gamla cirkumferens, det blev alltså ingen utökning, och området som var angett vid upprättelsen förblev oförändrat till verket lades ner. +Avslaget och de många andra problemen gjorde att Ancher sände en längre petition till kungen personligen, daterat 28 juli 1754, där han gav en utförlig beskrivning av alla svårigheter verket hade mött vid etableringen av kanonproduktionen. Petitionen innehöll följande sorgsna mening: +Efter en längre utredning begärde Ancher kungen i petitionen om en ny cirkumferens, eftergivning av tionden och att verket skulle få tillbaka gamla kasserade kanoner (de kunde smältas ner och järnet användas på nytt). I november 1754 tog kungen ställning till saken, återföring av kasserade kanoner blev godkänt, men i huvudsak var svaret ja till mindre viktiga punkter och beslut om de avgörande punkterna sköts upp. Från år 1755 gick det successivt bättre med gjutningen. Trots att det år 1756 inte göts en enda större kanon som blev godkänd, var kampen mellan järnverksägarna och generalkommissariatet på väg att upphöra. +Medan Erich Ancher bodde i Fredrikshald och skötte kompanjonens fabriker där, var det Mathias Wærn som från år 1750 ledde det dagliga arbetet på Moss Järnverk. Det är sannolikt att Ancher gav Wærn skulden för problemen med kanonproduktionen, i varje fall blev verksamheten i Fredrikshall såld år 1756 och Ancher flyttade till Moss. Totalt antal levererade kanoner åren 1749–1756 under Wærn var inte fler än 32. Verket ska ha haft en skuld på 150 000 riksdaler då Ancher flyttade dit år 1756. +Under Anchers ledning tycktes verket gå bättre. Åren 1757–1759 göts det 86, 99 och 106 stycken 12-pundskanoner utan fel, men först år 1760 levererades det ett större antal 18-pundskanoner. Ancher sände sina söner (Carsten Anker och Peter Anker) utomlands för att studera. I Glasgow blev de utnämnda till stadens hedersmedborgare, och den kände professorn Adam Smith skrev erkännande om dem. På bergseminariet i Freiberg fick de en grundlig utbildning för senare arbete vid järnverket. +Förhållandet mellan Erich Ancher och Mathias Wærn blev dåligt efter att Ancher flyttat till verket, och år 1716 ansökte Ancher till kungen om en delning mellan dem, något som blev bifallet den 5 juni 1761, med ett påföljande preliminärt avtal tio dagar senare. Wærn gick emellertid strax ifrån förlikningen och en årslång process påbörjades, där flera framstående personer blev involverade, bland annat den erkände juristen Henrik Stampe. Konflikten mellan de två affärspartnerna blev först slutligen avgjord i Anchers favör av bergamtsretten år 1765, och en slutlig uppgörelse mellan parterna skrevs under den 17 mars 1766. Från och med då var Erich Ancher helägare av Moss Järnverk. Moss Järnverk drevs på den tiden bra och blev rosande omtalad av en känd fransk bergverkskunnig. +På 1760-talet minskade orderna för kanonproduktion, Danmark-Norge hade dåliga statsfinanser, något som kom att drabba Moss Järnverk särskilt då det var beroende av rustningsproduktionen. Verket hade stora skulder, och Erich Ancher var beroende av sin bror Christian, och efter dennes död, hans brors firma Karen. Christian Anchers & Sønner. I enlighet med det slutgiltiga avtalet med Mathias Wærn var Erich Ancher tvungen att utfärda en pantobligation som senare skulle ge honom mycket problem. +I samband med pantobligationerna var Ancher tvungen att ge upp bergamtsretten och underlägga sig staden Moss jurisdiktion. Flera tillgångar blev också pantsatta. Verket fick tiondefrihet för åren 1765–1770, det betydde dock lite då det var en kompensation för ett kostbart hammarverk som hade brunnit ner. Frampå 1770-talet ökade Erich Anchers skuldproblem med järnverksverksamheten, hans egendomar blev undan för undan pantsatta eller sålda, innan han till sist gav upp den 22 november 1776 och överlät Moss Järnverk till sina syskonbarn Bernt och Jess Anker. +Med bröderna Bernt och Jess Anker fick Moss Järnverk ägare med en helt annan finansiell stabilitet än föregångarna. Bröderna Anker försökte trots allt att få så gynnsamma villkor som möjligt från sin viktigaste affärskontakt, som var staten, och då var kusinen Carsten Anker som ämbetsman paradoxalt nog (med tanke på verkets förra ägare var Carstens far) bra att ty sig till. Då deras monopol på kanongjutning hade gått ut var verket också tvunget att slåss mot konkurrens, särskilt från Fritzøe Jernverk. De första åren bodde den yngre brodern, Jess Anker, på järnverket, med titeln Proprietair af Moss Werk. Jess fullföljde uppbyggnaden av huvudbyggnaden som påbörjades av hans farbror Erich Ancher. +Efter några års uppehåll tog, från år 1776, kanongjutningen under de nya ägarna fart, och det förbrukades mer kol än någonsin, nära 10 000 läster om året. I samband med ett arrenderingskontrakt som Jess Anker ingick med familjeföretaget år 1781, gav verkets redovisning ett nettovärde värderat till 177 689 riksdaler. Bröderna Anker hade inte gjort upp med boet efter sin far Christian Anchers död 1767: i praktiken styrdes det av Bernt Anker, och han löste ut sin bror år 1783 och tog då också över Moss Järnverk för en summa på 80 000 riksdaler till Jess Anker. +Bernt Ankers omedelbara övertagande av Moss Järnverk (han hade i verkligheten kontrollerat verksamheten redan från den tid hans farbror sålde) markerade verkets sista storhetsperiod, och det var också en tydlig förändring i och med att ägaren inte bodde på verket. Den första föreståndaren på verket var Lars Semb, en dansk från Thyholm på Jylland. Han var vid verket hela sitt arbetsliv, och hans första verksamhetsanteckningar ger en bra bild av verksamheten. År 1793 var det 278 människor som bodde på verket; det var mellan 150 och 200 personer på Verlesanden, i Moss stad och på Jeløyen som levde av företaget. Till det kom alla som arbetade med att bryta malm i gruvorna, timmerhuggning och bränning av träkol. +I Bernt Ankers stora samling av olika egendomar anknutet till skogsbruk och malmutvinning var Moss Järnverk den med störst värde. +Mot värdelistan ovanför stod skuld, totalt 15 långivare med 89 000 riksdaler utlånade, där den största posten var det fasta lånet på 50 000 från statskassan. Förutom det var Moss Järnverk skyldig moderföretaget Karen sal. Christian Anchers & Sønner 26 680 riksdaler. Totalt låg alltså värdet runt 170 000 riksdaler i Moss Järnverk i början av år 1791. +Anker hävdade att företaget hade förlorat 150 000 riksdaler på verket då han tog över det ensam, och han genomförde en nedskärning av arbetsutgifterna. Dessutom överfördes större arbetskapital till olika besparingar. Under Anker nådde verket aldrig tillbaka till tidigare produktionsvolym, innovation och konstnärlig utsmyckning, men ekonomisk gick det bra de första åren, som tabellen nedan visar. +Produktionen av järn blev främst styrt av tillgången på träkol; när det var goda tider för trävirkeshandeln, levererade bönderna hellre timmer till sågarna än producera träkol och trotsade därmed levereringsplikten. Tabellen ovan visar också att Moss Järnverks intäkter var ungefär lika fördelade mellan produktion av järn och trävirke. År 1793 var verket tvunget att låta en order på 22 kanoner gå till konkurrenten Fritzøe Verk på grund av brist på träkol. Den största orsaken till de goda ekonomiska tiderna var, som tabellen visar, oroligheterna i Europa, Franska revolutionskriget (1792–1802) och Napoleonkriget (1803–1815), och att kungariket Danmark-Norge klarade att hålla sig neutralt till år 1807. +För eftervärlden har Moss Järnverks tid under Bernt Anker varit den största, bland annat på grund av omnämningar som dessa: ”En av de vackraste anläggningarna i staden, som besökare beundrar, är Moss Järnverk.” För kanonproduktionens del hade höjdpunkten passerat, den var aldrig så omfattande under Bernt Ankers tid, och det göts inte några grovkalibrerade kanoner mellan åren 1789 och 1797. Det mesta av produktionen tycktes ha varit tunnbandsjärn, spikar och obearbetat järn. Det skedde heller inga större förbättringar eller tillbyggnader; de två masugnarna som länge ansetts vara i dåligt skick, fick duga så länge Bernt Anker levde. +Efter Bernt Ankers död år 1805 lades hans företagsimperium in i ett fideikommiss (en typ av stiftelse), där föreståndaren Lars Semb från Moss Järnverk var en av tre i styrelsen. För åren som kom fick Semb tämligen fria händer när det gällde skötseln av verket. 1805 var ett väldigt bra år för verket men år 1807 ändrades situationen när Danmark-Norge attackerades av Royal Navys (slaget om Köpenhamn) och blev indraget i Napoleonkriget. Moss Järnverk blev en central industri under krigsåren då behovet av kanoner till krigsskepp, fästningar och armén var stort. På grund av förlorade krigsskepp i attacken måste nya, mindre krigsskepp byggas för det så kallade Kanonbåtskriget, och en väsentlig del av dem fick kanoner från verket. Med utbrottet av krigen försvann också problemen med bristen på träkol, dels på grund av minskad träkolsexport och dels på grund av böndernas patriotiska sinnelag. +Föreståndaren Semb började redan hösten 1807 gjutning av 3- och 6-pundskanoner, och under vintern gjordes masugnarna i ordning så att gjutning av stora 24-pundskanoner till flottan kunde börja i februari. Det fanns också ett behov för kanoner till fartygen som byggdes för kaparna, till exempel till kaparen "Christiania" levererade verket två stycken 6-pundskanoner, fem stycken 12-pundskanoner och fyra stycken 18-pundskarronader (korta skeppskanoner). Det göts 18-pundskanoner till landetaten, som bland annat Slevik batteri i Onsøy utrustades med. +År 1808 var ekonomiskt ett bra år för verket, men på grund av det hårda arbetet och den tidiga starten på vintern så slets masugnarna hårt, den östra användes för sista gången år 1809. Detta år gav ett västentligt dåligt resultat, det var kolbrist igen, och undernäring gjorde att en stor del av de anställda blev sjuka och dog i epidemier. Föreståndare Semb uppgav att en fjärdedel av arbetarna dog år 1809, däribland en kanonborrare och den bästa kanongjutaren. En del produkter måste för en lång tid utgå ur produktionen på grund av brist på arbetare eller yrkesmän. År 1810 göts det runt 50 korta 18-pundskanoner för utrustning av några briggar; dessa var de sista kanoner av grov kaliber som gjordes på Moss Järnverk. +Trots att förhållandena var dåliga, så var det Moss Järnverk som dessa år gav överskottet till fideikommissen, mot slutet av kriget var det baserat på timret som sågarna levererade. När det gällde själva verket, så var masugnarna inte i drift från april 1812 till juli 1814, produktionen i smedjan drevs av järn från lager eller från andra norska järnverk, särskilt från Hakadals verk. Utrustningsproblemen var till stor del på grund av blockaden av den brittiska flottan (Kontinentalsystemet), den lokale redaren David Chrysties brigg ”Refsnes” togs till exempel i besittning vid ett försök att hämta en större last med korn i Ålborg. +Vid freden i Kiel, då Danmark var tvungna att ge Norge till Sverige, uppstod en ny situation och Sjökrigskommissariatet i Christiania förhörde sig om verkets kapacitet för vapen till landets försvar. Moss Järnverk var emellertid så slitet att det år 1814 bara kunde leverera en del ammunition. En del tunnringar producerades, och verket fick ett överskott på 37 000 riksdaler, en summa som emellertid inte kunde liknas med föregående år, då inflationen på grund av kriget var stor. +Efter att freden slöts år 1814 förvärrades situationen för verket igen. När Danmark-Norge upplöstes avskaffades järnmonopolet, och konkurrensen från svenska och särskilt engelska järnverk, som producerade järn på nya och billigare sätt, var svår. Det var också dåliga tider för trävirkeshandeln, och år 1817 var överskottet inte mer än 6 000 spesiedaler, det sämsta resultatet föreståndare Lars Semb hade levererat på de 33 år han skötte verket. +Den ekonomiska situationen för Bernt Ankers fideikommiss blev stadigt sämre under och efter Napoleonkriget, och den 13 december 1819 var föreståndare Lars Semb tillsammans med de andra administratörerna tvungen att skriva under en ansökan till kungen om konkurs för verksamheten. En del av verksamheten vid Moss Järnverk såldes på auktion, men Bernt Ankers bror Peder Anker, med svärsonen Herman Wedel-Jarlsberg, köpte huvuddelen för 30 330 spesiedaler. +Moss Järnverk var några år till knutet till familjen Anker. Peder Anker bodde dock aldrig på verket, utan på den gedigna Bogstad gård, och företaget drevs av Lars Sembs son Andreas Semb, tillsammans med Ankers svärson greve Herman Wedel-Jarlsberg. Anker ägde redan Bærum Jernverk, och arbetet på Moss blev anpassat efter detta. År 1824 blev en ny masugn byggd, bortsett från detta så skedde inte mycket vid verket under Peder Ankers år vid verket. Vid Ankers död övertogs Moss Järnverk av dottern Karen och hennes man Herman Wedel-Jarlsberg. +För staden Moss med omnejd var verket otvivelaktigt av stor betydelse, både för den växande industrin och för lantbruket. En rad olika redskap kunde produceras; några serieproducerades, medan andra gjordes på beställning. Verkets tekniska kompetens var troligtvis också en drivkraft i områdets näringsutveckling. +Moss Järnverk var inte bara en verksamhet som bearbetade järnmalm, det var också en mötesplats för dåtidens ledande skikt inom näringsliv, kultur och politik. Sitt första kungabesök fick verket med stor sannolikhet redan etableringsåret 1704, då den dansk-norske kungen Fredrik IV besökte Moss två gånger. Landsvägen till Christiania, Frederikshaldske Kongevei, gick dessutom tvärs genom bruket, och efter år 1760 var verket vitt omtalat i reselitteraturen. +Den nya huvudbyggnaden, Konventionsgården, stod färdigt år 1778, och var mycket representativ. Det måste också nämnas att speciellt Bernt Anker, landets rikaste man, var en mycket gästfri ägare. Bland de nöjen som Anker gav sina gäster var amatörteaterföreställningar på en teater han hade inrättat på verket. Bernt Anker spelade själv huvudrollen, var författare, regissör och skådespelare. +År 1787 var till exempel den sydamerikanske greven av Miranda gäst och besiktigade verket, forsen, parken och kanongjuteriet. År 1788 var kronprinsen (senare kung Fredrik VI av Danmark) och prins Karl II av Hessen-Kassel på besök. Detta var precis före fälttåget mot Sverige, som senare blev känt som Teaterkriget, och de kungliga fick både se kanongjutning, borrning och ett kanonprov där Bernt Anker själv avfyrade kanonen. +Moss Järnverk centrala position på denna tid bevisades med att Anker involverade det i nya och djärva framstötar, hösten 1791 låg till exempel den första norska ostindiefararen, ”Carl, Prince af Hessen”, vid verket för att utrustas. Företaget omfattades av ett enormt intresse, och tidningen Norske Intelligenssedler skrev med jämna mellanrum hyllningsdikter och nyheter om färden. Fartyget återvände med en stor last peppar, kaffe, socker, arrak och annat den 30 april 1793. +Gästfriheten vid verket fortsatte efter Ankers död, och vid Karl Augusts genomresa till Sverige år 1810, där han blev vald till tronföljare, ansträngde man sig mycket för att ge honom ett värdigt boende. Huvudbyggnaden fungerade eftersom som kungabostad, år 1816 bodde till exempel kronprinsen (senare kung Karl XIV Johan) på verket under ett besök i Moss. +Moss Järnverk gjorde en viktig insats för landet under krigen 1807–1814. Sommaren 1814 stod emellertid verket och huvudbyggnaden (Konventionsgården) mitt i händelserna centrum. Danmark hade avstått Norge till Sverige, ett norskt grundlagsmöte hölls, och den danska ståthållaren Kristian Frederik blev utnämn till kung. +Den 21 juli etablerade den nyvalde kungen sitt huvudkvarter på verket i väntan på en invasion från Sverige, och förhandlingar pågick, med hjälp av utsända från dåtidens stormakter, för att försöka att få till en fredlig lösning. De utsända kom till Moss med ett sista kompromissförslag den 27 juli, det blev förkastat av den norska sidan dagen efter, och Fälttåget mot Norge hade börjat. De överlägsna svenska styrkorna ryckte snabbt fram, omringade Fredrikshald och var klara att avancera vidare in i Smålenene (Østfold). Då statsråd blev hindrade vid huvudkvarteret i Moss den 3 augusti, var den norska ställningen kritisk. +Den 10 augusti började vapenförhandlingarna, och de svenska generalerna Magnus Björnstjerna och Anders Fredrik Skjöldebrand anlände till kungen och regeringens huvudkvarter på Moss Järnverk. De möttes av den norske förhandlaren Jonas Collett, och så småningom också Nils Aall. Efter förhandlingarna med de svenska utsända lades resultaten fram till regeringen vid statsrådet på Moss Järnverk den 13 augusti. Dagen efter, den 14 augusti, kom de till ett avgörande och avslutade förhandlingarna, där Norge accepterade att ingå i en union med Sverige, den norska grundlagen blev accepterad, och i en hemlig paragraf lovade Christian Frederik att överlåta Norge. Det hela blev senare känt som konventionen i Moss. +Christian Frederik uppehöll sig endast några dagar till på verket och skickade den 16 augusti ut en kungörelse till det norska folket, en kort men gripande tillkännagivande, som förklarade de sista månadernas händelser, vapenstilleståndet och konventionen. Dagen efter, den 17 augusti, seglade Christian Frederik från Moss till Bygdøy. Det norska huvudkvarteret fortsatte i några dagar att hålla till på verket, först den 31 augusti beslutade statsrådet, efter råd från generallöjtnant Arenfeldt, att flytta huvudkvarteret till Christiania. +I många år såg norrmännen det kortvariga kriget, nederlaget och de påföljande förhandlingarna i Moss som ett nederlag, och valde att hellre fokusera på Eidsvollsförhandlingarna. Gradvis ändrades synen på utkomsten, och historiken Yngvar Nielsen karakteriserade år 1887 händelserna i Moss som ”tyngdpunkten” av händelserna på den norska sidan år 1814. +Moss Järnverk blev i realiteten övertaget av greve Herman Wedel-Jarlsberg år 1824, men rent juridiskt ägde han det formellt först åren 1826 1829. Situationen för näringslivet var väldigt förändrat, och på 1820-talet förlorade de norska järnverken också det sista av tullskyddet mot svenskt järn. Av princip stämde greve Wedel för upphävelsen, som ägare av järnverk själv förlorade han stort personligen på utvidgandet av frihandeln. Efter övertagandet fick Wedel reda på att verket fortfarande var skyldig staten 50 000 riksdaler, det var fasta lånet från år 1755 som inte var återbetalat. Efter år av handläggning och rättegång dömde den norska högsta domstolen att skuldebrevet var giltigt, och oåterkalleligt så länge kanongjuteriet var igång. +Åren 1830–1831 byggdes det ett nytt stort vals- och skärverk, och 1834 byggdes det i masugnshyttan en kupolugn (en liten masugn) som efterhand bistod eller till viss del ersatte masugnen. I kupolugnen kunde det smältas avfallsjärn och metall. 1830-talet var en svår tid för järnverken i Norge, Moss Järnverk var inget undantag, och under femårsperioden 1836–1840 var masugnen ur drift i 3 år. +År 1840 dog greve Wedel och hans son Harald Wedel-Jarlsberg tog över hans fars företag, däribland Moss Järnverk. Verket drevs nu av föreståndaren, från år 1836 tysken Ignatius Wankel. De glada dagarna var förbi, och kungligheterna blev inte längre inlogerade i huvudbyggnaden när de reste genom Moss. Moss Järnverk blev allteftersom ett underbruk till Bærums Järnverk, särskilt efter år 1840. Marknadsförhållanden blev bättre på 1840-talet, norskt järn fick bland annat tillträde till Nordamerika, och förbättringar gjordes efter modeller av utländska verk. +Andra halvan av 1840-talet var marknadsmässigt bra för verket, och produktionen ökade kraftigt: malmutbytet och fördelningen mellan specialiteter mellan verken i Bærum och Moss fortsatte. Spikproduktionen upphörde till exempel i Moss på 1830-talet, medan valsverket var det klart största i Norge. I slutet av 1850-talet revs det gamla valsverket, och ett nytt och mycket större byggdes, bara för maskineriet betalades det år 1858 nästan 10 000 spesiedaler. Nu var dock den sista goda tiden för de norska järnverken över. Vid slutet av Krimkriget blev marknadsförhållanden väldigt dåliga, och de norska verken konkurrerade igen med de svenska och engelska. +Frampå 1860-talet lades de norska järnverken gradvis ner, pressade av billigt stål från den väldigt effektiva Bessemermetoden. Moss Järnverk höll ut i några år, troligtvis för att det nya valsverket var konkurrensdugligt, och mot slutet av 1860-talet kom det stora mängder järn från Bærum för valsning. Ett par år efter Fransk-tyska kriget (1870–1871) uppstod en ny kris med den långa depressionen; detta blev verkets nådastöt. Driften försvårades också av att järnbanan (Smaalensbanen), som blev godtagen år 1873, planerades att gå tvärs över verkets tomt. Det sista året järn bearbetades var 1873. Åren 1874 och 1875 var endast sågbruk, gårdsbruk och kvarn i bruk. Moss Järnverk såldes för 115 000 spesiedaler (460 000 norska kronor) år 1875 till den lokala firman M. Peterson & Søn. Området verket låg på togs över och används idag av emballagefabriken Peterson. +I över 150 år, från mitten av 1600-talet till runt år 1814, spelade de norska järnverken en mycket viktig roll i landets näringsliv, åren före och runt 1814 också i politik och andeliv. Tillsammans med trävirke, fisk och skeppning, var koppar och järn det Norge exporterade på den tiden. Järnverkens nationella betydelse kulminerade under Napoleonkriget, och medan åren före 1807 var förträffliga rent ekonomiskt, så var åren efter motsvarande dåliga. Tiden efter unionsupplösningen år 1814 var svår för järnverken något som en dramatisk nedgång i produktionsvolym också visade. +Samtidigt som verken efter norska mått mätt var stora verksamheter, var de små internationellt. Vid slutet av 1700-talet utgjorde världens samlade årliga förbrukning av järn ungefär 2/3 miljoner ton, varav de norska järnverken producerade runt 9 000 ton. Den svenska produktionen av järn var till exempel ungefär åtta gånger så stor som den norska. +Av de runt sexton järnverk i den sydostliga delen av Norge var Moss Järnverk ett av de mellanstora om man ser till själva järnproduktionen. Åren 1780–1800 var den årliga förbrukningen av träkol vid norska järnverk runt 140 000 läster (ungefär 270 000 m³), och av detta förbrukade Moss Järnverk under 10 000 läster. +Både under och efter tiden med järnverk drivna av träkol hävdades det att träkolsproduktionen bidrog till att utarma skogen, men enligt Vogts skildring i "De gamle norske jernverk" så verkar inte detta stämma, trävirkesexporten under samma tidsperiod var till exempel över sju gånger så stor. Samtidig var träkol den viktigaste faktorn, och det var de ökande träkolspriserna som till slut tvingade järnverken att slå igen. +Den dansk-norska staten såg järnverken som strategiskt viktiga och subventionerade dem på flera olika sätt, dels genom att verken fick ett område där bönderna som bodde innanför, var förpliktade att leverera träkol, dels genom en stor beskyddstull och dels genom statliga beställningar, som kanoner och ammunition till armén och flottan. Järnverken var tvungna att betala skatt, så kallat tionde: i praktiken var den ungefär 1,5 % av produktionsvärdet. Bortsett från relativt korta perioder under 1600-talet, då Bærums Jernverk och Eidsvolls Jernverk ägdes av respektive en nederländare och en furste från Kurland, var verken i dansk-norska händer. +I samtiden var Moss Järnverk särskilt känt för kanongjuteriet, vid etablering det första i landet. Bland dåtidens experter blev verket erkänt, fransmannen Gabriel Jars nämner till exempel år 1767 Moss Järnverk tillsammans med Fritzøe Jernverk och Kongsberg Jernverk som de främsta i Norge. Man har i efterhand erkänt järnverkets viktiga bidrag till införseln av teknologi och industrialiseringen av Norge; den kände juristen och socialekonomen Anton Martin Schweigaard skrev bland annat år 1840 att: ”Bergverksdriften har blivit en skola för mekaniska och tekniska färdigheter och kunskaper". +Det är inte mycket som står kvar efter Moss Järnverk, det mesta av byggnaderna från verket är rivna. Bland det som står kvar är administrationsbyggnaden (Konventionsgården) och arbetarbostäderna längs gatan norr om den. Kvarnen vid forsen har också blivit bevarad. +Lista över järnbruk i Sverige +Den här artikeln är helt eller delvis baserad på material från norskspråkiga Wikipedia. +J. Boydell, Picturesque Scenery of Norway, London, 1820 +Cristophersen, H. O. 1974: Fra jernverkenes historie i Norge. Grøndahl & Søn, Oslo, ISBN 82-504-0077-1 +Knut Dørum, «Opprør eller legitim politisk praksis?», Inlägg på Historiedagarna i Bergen 20 juni 2009. Också som ”Opprør eller legitim politisk praksis? : Kommunalisme og folkelige aksjoner i Norge ca. 1750–1850” I: Demokratisk teori og historisk praksis. Redigerad av Hilde Sandvik. Scandinavian Academic Press, 2010. ISBN 978-82-304-0054-8 +Fritz Hodne, An Economic History of Norway 1815-1970, Tapir 1975, ISBN 82-519-0134-0 +Fritz Hodne och Ola Honningdal Grytten, Norsk økonomi i det 19. århundre, Fagbokforlaget, Bergen, 2000, ISBN 82-7674-352-8 +Oskar Kristiansen, Penge og kapital, næringsveie: Bidrag til Norges økonomiske historie 1815-1830, Cammermeyers boghandels forlag, Oslo 1925 +Lauritz Opstad, Moss Jernverk, M. Peterson & søn, Moss, 1950 +J.H.L. Vogt, De gamle norske jernverk[1], Norges Geologiske Undersøkelse nr. 46, Christiania 1908 +Arne Nygård-Nilssen, Norsk jernskulptur, 2 bind., dr. avh., 1944, utgivning Næs jernverksmuseum, 1998 ISBN 978-82-7627-017-4 +Om Moss Järnverk, från Moss stad- och industrimuseum (på norska) +Om Moss Järnverk, från Lokalhistoriewiki (på norska) +Bild av ugn tillverkad av Moss Järnverk, från digitaltmuseum.no +Arbetarbostäder Moss, från Kulturnett (på norska) +«Kanonen från Lyngør hamn», artikel på maritimt.net om en kanon från Moss Järnverk (på norska) +Harmens register över bruk +Manchester United Football Club är en fotbollsklubb från Stretford i Greater Manchester, England, som spelar i Premier League. Klubben ägs sedan maj 2005 av familjen Glazer och minoritetsinvesterare. +Klubben grundades år 1878 under namnet Newton Heath LYR Football Club, men bytte 1902 namn till Manchester United och har spelat på Old Trafford sedan 1910. År 1908 blev klubben engelska ligamästare för första gången och detta upprepades 1911. År 1931 räddade James W. Gibson Manchester United från konkurs, då klubben under 1920- och 1930-talen pendlade mellan den högsta och näst högsta divisionen. I flygolyckan i München 1958 dog åtta spelare i klubben, vid en tidpunkt då laget hade haft potential att göra Manchester United till ett av tidernas mästarlag i engelsk fotboll. År 1968 var Manchester United, med Matt Busby som tränare, den första engelska klubben att vinna Europacupen och var under några år i slutet på 1960-talet också ett av Englands topplag. 1970- och 1980-talen präglades mestadels av degraderingar, tränarbyten och återavancemang – innan klubbens mest framgångsrike tränare, Alex Ferguson, tog över i november 1986 som skulle komma att vinna totalt 38 titlar under 26 säsonger med klubben (och Manchester United slutade som topp tre i Premier League samtliga år med Ferguson från grundandet av ligan, det vill säga 1992–2013). Därefter har klubben varit mindre framgångsrik, med en handfull olika managers som rekryterats och sparkats. +Manchester United har bland annat blivit engelsk mästare 20 gånger och vunnit Uefa Champions League tre gånger (1968, 1999 och 2008), och tävlar med rivalen Liverpool om att bli Englands mest framgångsrika fotbollsklubb någonsin. Den utgör ett globalt varumärke och rankas internationellt som ekonomiskt en av de mest värdefulla i fotbollsindustrin; i maj 2024 rankade den amerikanska affärstidningen Forbes klubben som världens näst rikaste till ett värde av $6,55 miljarder. Manchester United är även världens mest supporterstödda klubb, till supportrar sett också den mest världstäckande klubben, och hade totalt 659 miljoner fans 2011. Ryan Giggs, Bobby Charlton, Paul Scholes och Wayne Rooney hör till de spelare som spelat flest matcher och gjort flest mål för Manchester United. +Klubben bildades år 1878 under namnet Newton Heath Lancashire and Yorkshire Railway FC av järnvägsarbetare som arbetade vid Lancashire and Yorkshire Railway (LYR) vid Newton Heath. Klubben spelade flera matcher mot andra järnvägsbolag som även de hade fotbollslag, den 20 november 1880 spelade klubben sin första registrerade match, lagets matchkläder var järnvägsbolagets färger – grönt och guld – matchen slutade i förlust för klubben med 6–0 mot Bolton Wanderers reservlag. 1888 var klubben en av de grundande medlemmarna i The Combination, en regional fotbollsliga. Säsongen efter lades dock ligan ner och Newton Heath gick istället med i nygrundade Football Alliance, vilken var igång tre säsonger innan ligan gick ihop med Football League. Detta resulterade i att Newton Heath säsongen 1892/1893 spelade i högstadivisionen, klubben hade även då blivit oberoende av järnvägsbolaget och valde att ändra sitt namn till endast Newton Heath FC. Efter två säsonger i första divisionen degraderades klubben till andradivisionen. +I januari 1902 hade klubben skulder på £2 670 och var på väg mot konkurs. Dåvarande lagkaptenen Harry Stafford hittade då fyra lokala affärsmän, däribland John Henry Davies (som blev ordförande), som var villiga att investera £500 i utbyte mot ett direkt intresse av att driva klubben, senare bytte även männen namn på klubben och den 24 april 1902 föddes officiellt Manchester United. Med klubbens tränare Ernest Mangnall, som 1903 tog över som tränare, slutade laget på en andraplats i andra divisionen 1906 och säkrade en uppflyttning till högsta divisionen, vilken klubben vann 1908 – klubbens första ligatitel. Säsongen efter började med ännu en titel då klubben vann Charity Shield i början av säsongen, detta första gången som Charity Shield spelades och slutade med klubbens första FA Cup-titel. 1910 flyttade Manchester United från sin hemmaplan, Bank Street till dagens hemmaplan, Old Trafford. Manchester United vann sin andra ligatitel 1911, men i slutet av efterföljande säsong lämnade tränaren Mangnall klubben för att istället ta över Manchester City som tränare. +1922, tre år efter att fotbollen återupptagits efter första världskriget, degraderades klubben till andra divisionen i vilken klubben spelade fram tills återkomsten upp i förstadivisionen år 1925. Klubben degraderades återigen 1931 och var en så kallad "jojo-klubb", 1934 slutade klubben på lägsta ligaplacering någonsin, då på en tjugonde plats i andra divisionen. Efter att klubbens främste donator, John Henry Davies, avlidit i oktober 1927 försämrades klubbens ekonomi i sådan utsträckning att Manchester United troligen skulle ha gått i konkurs om det inte vore för James W. Gibson som i december 1931 investerade £2 000 och tog över klubben. Under säsongen 1938/1939, den sista säsongen som fotboll spelades innan andra världskriget bröt ut, slutade klubben på 14:e plats i den högsta divisionen. +I oktober 1945 återupptogs fotbollen igen efter andra världskriget. Den numera legendariske tränaren Matt Busby fick då stor makt över laget och fick bland annat bestämma transfers, trupputtagningar och träningarna. Busby ledde laget till tre andraplaceringar i ligan i rad (1947, 1948 och 1949), och han ledde även laget till vinst i FA-cupen 1947/1948. 1952 vann klubben titeln i den högsta divisionen, klubbens första ligatitel på 41 år. Med en medelålder på 22 år gav media 1956 smeknamnet "Busby Babes", som en hyllning till det unga storlaget. 1957 blev Manchester United det första engelska laget att konkurrera om en plats i Europacupen, trots invändning från The Football League som hade nekat Chelsea FC samma möjlighet föregående säsong. På vägen till semifinal, som segrades av Real Madrid, noterades klubbens största seger någonsin; 10–0 mot belgiska mästaren RSC Anderlecht. +Efter en Europacupmatch mot Röda Stjärnan i Belgrad den 6 februari 1958, mellanlandade Manchester United på München-Riem flygplats. När planet återigen skulle lyfta sladdade det av startbanan och tog eld. Sju Manchester United-spelare, åtta journalister och flera klubbrepresentanter dog i haveriet. Unga stjärnskottet Duncan Edwards avled senare på sjukhus. Matt Busby skadades svårt men överlevde, likaså storspelaren Bobby Charlton. Tio år senare skulle de bägge vara med då United vann Europacupen. Charlton fick även höja Julies-Rimet-pokalen på Wembley då England blev världsmästare 1966. +Efter att ha byggt ett av de mest topptippade lagen engelsk fotboll skådat, behövde Matt Busby i början på 1960-talet börja om från början. Flygolyckan hade tagit ifrån honom några av sin tids bäste fotbollsspelare, men direkt efter att han själv hade återhämtat sig från sina skador började Busby bygga ett nytt lag som skulle vara världsdominanta. +Stommen i det nya laget byggdes runt spelare som överlevt flygolyckan, bland andra Bill Foulkes och Bobby Charlton. Även spelare som Denis Law från Torino köptes. De första åren var av mindre framgång, men säsongen 1962/1963 tog sig laget hela vägen till Wembley och slog Leicester med 3–1 i finalen. Säsongen därpå hade truppen byggts vidare och vann FA-cupen, och samtidigt slogs om ligatiteln. Det slutade med en andraplats, fyra poäng bakom segrande Liverpool. En annan milstolpe i det nya lagbygget var förvärvet av George Best. +Sex år efter att laget som skulle ta över Europa hade slagits i bitar vann Manchester United ligan, och nådde semifinal i både Europacupen och FA-cupen. Denis Law utsågs till årets spelare i Europa. Säsongen 1966/1967 vann klubben ligan igen, efter att säsongen innan ha slutat fyra. Detta gjorde att laget garanterades en plats i europaspelet nästa säsong. Denna gång kunde inget stoppa Busbys lag från att gå hela vägen; Manchester United slog Benfica i finalen med 4–1, efter att matchen slutat 1–1 efter 90 minuter. I förlängningen kunde George Best, Brian Kidd och Bobby Charlton göra var sitt mål och därmed tog klubben sin första europacuptitel någonsin, bara tio år efter Münchenkraschen. Kort efter detta blev Matt Busby adlad. +Trots en elfteplats i ligan och ingen titel säsongen efter segern i Europacupen kunde fansen se tillbaka på 1960-talet med ett leende. +Med Europaframgångarna bakom sig började klubben nu jobba för att fylla vakuumet efter Matt Busby. Busby hade givit klubben stora framgångar, men hade nu pensionerat sig och lämnat ett svårt uppdrag för styrelsen. Genom att lyfta upp en av Busbys hjälptränare, och även före detta United-spelare Wilf McGuinness, hittades en snabb första lösning. Åldrande spelare och allt mindre kontroll över laget gjorde att McGuinness hade Matt Busby hängande över axeln. Han satte spelare som Denis Law och Shay Brennan på transferlistan, vilket skapade oro i styrelsen. Samtidigt brottades en annan av lagets stora stjärnor, George Best, med stora problem vid sidan av planen vilket inte hjälpte den redan ansträngda situationen. +Annandag jul 1970 sades McGuinness upp från tränarjobbet med omedelbar verkan och Matt Busby fick komma tillbaka från sin pension tillfälligt under sökandet efter en ny tränare. I juni 1971 tillsatte styrelsen Frank O’Farrell som tränare, men 0–5 mot Crystal Palace en bit in på säsongen blev droppen och Frank fick sparken den 16 december 1972. O’Farrells tid som tränare var visserligen kort, men han lämnade ett litet avtryck i och med rekordvärvningen av Martin Buchan för £125 000. Den förre Aberdeen-kaptenen skulle bli en nyckelspelare för O’Farrells efterträdare, Tommy Docherty, som tillsattes julen 1972. +Dochertys första utmaning var att hålla kvar laget i högstadivisionen samtidigt som han skulle genomföra en generationsväxling från 1960-talshjältarna. Bobby Charlton hade meddelat att han skulle dra sig tillbaka efter säsongen 1972/1973, och George Best fortsatte med sina problem vid sidan av planen. Han släppte till slut den avdankade Denis Law gratis, vilket han skulle få ångra; Law gick till lokalkonkurrenten Manchester City, och i april 1974 gjorde han ett mål i matchen som definitivt skickade United ner i andradivisionen. United kom dock snabbt tillbaka och vann andradivisionen säsongen 1974/1975, och lagets anfallare Stuart "Pancho" Pearson vann skytteligan med 17 mål. +United lyckades sedermera ta sig till två raka FA-cupfinaler, förlorade mot Southampton 1976 men lyckades sedan slå Liverpool med 2–1 ett år senare. Segern förstörde Liverpools chans att bli det första engelska laget att vinna en trippel; Europacupen, ligan och FA-cupen samma säsong. Drygt 20 år senare skulle United istället bli det första engelska laget att lyckas med detta. Lyckan varade dock inte längre för Docherty, efter att han blivit påkommen med att ha klubbens sjukgymnasts fru som älskarinna. +Efter Docherty kom före detta QPR-managern Dave Sexton in på Uniteds bänk. De första två säsongerna kom laget på en tiondeplats i ligan, men lyckades ta laget till en FA-cupfinal 1979 – som förlorades mot Arsenal med 3–2, efter att Arsenals Alan Sunderland gjort det avgörande målet precis i slutet av matchen. +1980 slutade United till slut två poäng bakom Liverpool i titelracet, efter en säsong med en misslyckad inledning men laget spelade upp sig ju längre säsongen gick. Året efter avslutade United på åttonde plats – en placering som styrelsen inte tolererade, och den 30 april 1981 blev Sexton sparkad efter fyra år som tränare. Sextons ersättare, Ron Atkinson, handplockade Mick Brown som assisterande och Eric Harrison som ungdomstränare. Atkinson hade slagit brittiskt transferrekord när han köpte in Bryan Robson från West Bromwich Albion för £1 500 000, och runt £500 000 för Remi Moses, även han från West Bromwich. 1982, 1983 och 1984 slutade United ofta trea eller fyra i ligan, men var aldrig särskilt nära att rå på segraren Liverpool. +Uniteds chans på troféer var istället i de inhemska cuperna. 1983 hade klubben tagit sig till final i både ligacupen och FA-cupen. I ligacupen blev det förlust mot Liverpool med 2–1 efter att Liverpool avgjort i förlängning, men istället vanns FA-cupfinalen mot uppstickaren Brighton and Hove Albion över två matcher. Även 1985 vann United FA-cupen, finalvinst mot Everton med 1–0 då Whiteside gjorde matchens enda mål. Detta var Atkinsons andra FA-cuptitel på tre år, men 18 månader senare fick han sparken eftersom han misslyckats med att frånta den engelska fotbollstronen av Liverpool. Inte heller när hans United startade säsongen 1985/1986 med tio raka segrar lyckades laget ta hem ligatiteln. +I november 1986 anställdes en vinnande manager; Alex Ferguson kom senast från Aberdeen, där han vunnit allt som gick att vinna i skotsk fotboll plus att han även lett sitt lag till seger över Real Madrid i Europeiska cupvinnarcupen i fotboll. Ferguson visade sig ha talang för tränaryrket, men han behövde tid för att bygga laget. Klubben visade tålamod när laget bara slutade på elfte plats efter både första och andra säsongen. Efter att ha värvat ett par nya spelare visade det sig att tålamodet snart skulle betala sig. +Säsongen 1991/1992 hade Manchester United inte vunnit ligan på 25 år. Laget var stor favorit att vinna efter att ha varit i ledningen under större delen av säsongen men laget började svikta inför de avslutande matcherna. Med enbart tre omgångar kvar gick Leeds United för första gången upp i ledningen efter att Manchester United förlorat hemma mot Nottingham Forest samtidigt som Leeds vann sin match. Laget hade fortfarande saken i egna händer, med en match mindre spelad; en hängmatch borta mot redan nedflyttningsklara +West Ham. West Ham vann matchen med 1–0, och när Manchester United i omgången därefter även förlorade borta mot Liverpool samtidigt som Leeds United vann sin match blev Leeds klara mästare. Säsongen därefter inleddes klubbens hittills största och längsta glansperiod. För första gången på över ett kvarts sekel var Manchester United åter ligamästare i England. Sedan dess har bara fem andra lag (Blackburn, Arsenal, Chelsea och Manchester City, Leicester) lyckats vinna ligan enstaka år. United har sedan dess också vunnit engelska ligan 11 gånger av 17 möjliga, vunnit Champions League 2 gånger samt passerat Liverpool i den engelska maratontabellen. +1999 kom den största triumfen då Manchester United vann "trippeln" (Uefa Champions League, Premier League och FA-cupen). Manchester United vände Champions League-finalen genom två mål på övertid mot Bayern München, med Teddy Sheringham och norrmannen Ole Gunnar Solskjaer som matchvinnare. I laget spelade klubbens framgångsrika "75:or" som kommit upp från klubbens ungdomsled, såsom Gary Neville, Nicky Butt och David Beckham samt Paul Scholes (född 1974). Andra stora profiler i laget var den irländske mittfältaren Roy Keane och den walesiske vänsteryttern Ryan Giggs samt den danske stjärnmålvakten Peter Schmeichel. +2003 började Malcolm Glazer köpa aktier i Manchester United och 2005 stod det klart att Glazer med familj tagit över hela kontrollen av den anrika fotbollsklubben. Manchester United-fansen blev mycket upprörda och demonstrerade mot övertagandet. Som en opponering mot Malcolm Glazers maktövertagande av klubben skapade en skara fans en ny supporterklubb, FC United of Manchester. När Glazer köpte klubben fick han ta ett lån på fem miljarder kronor av de åtta miljarderna han betalade. För att han ska ha råd att betala tillbaka pengarna höjdes biljettpriserna med 12 procent till årets säsong. +Inför säsongen 2007/2008 var Chelsea FC favorittippade till ligasegern med Manchester United och Arsenal FC som främsta utmanare. Under säsongen kom ändå Manchester United att vinna både ligan och Champions League. Ligatiteln var klubbens 17:e totalt och den 10:e sedan 1993. Klubbens tredje seger i Champions League säkrades via straffsparksläggning mot Chelsea. +Säsongen 2008/2009 vann Manchester United återigen Premier League och gick även detta år vidare till finalen i Champions League, men denna gång blev det förlust 0–2 mot spanska FC Barcelona. 2009/2010 lyckades United bara vinna Carling Cup; Premier League vanns av Chelsea och i Champions League åkte United ut i kvartsfinalen mot tyska Bayern München. Säsongen efter, 2010/2011, vann Manchester United en ny Premier League-titel och passerade Liverpool i antal ligatitlar; 19 totalt, flest i den engelska fotbollshistorien. United lyckades även ta sig till sin tredje Champions League-final på fyra år, och ställdes likt senast mot Barcelona. Även denna gång vann Barcelona, slutresultat skrevs till 3–1. +Säsongen 2011/2012 besegrades United av den lokala konkurrenten Manchester City, som hamnade på samma poäng men hade bättre målskillnad. Inte heller i Champions League lyckades United nå några framgångar, utan slutade på tredje plats i gruppen och åkte ur redan innan slutspelet. Istället lyckades Manchester United säsongen därpå med fyra omgångar kvar säkra sin 13:e Premier League-titel, och klubben hade därmed gjort sig historisk genom att bli engelsk mästare för 20:e gången. Alex Ferguson fick i november 2012 en staty rest efter sig utanför Old Trafford som en hyllning då han hade valt att efter säsongen avsluta sin nästan 27 år långa era som manager för klubben, och höll ett avskedstal inför ett fullsatt Old Trafford direkt efter den sista hemmamatchen (2–1 mot Swansea den 12 maj 2013). I Champions League samma säsong vann Manchester United komfortabelt gruppen, men förlorade mot Real Madrid i åttondelsfinalen. +David Moyes från Everton efterträdde Ferguson inför säsongen 2013/2014, efter att personligen ha blivit utvald av Ferguson. United misslyckades med allt under transferfönstret och på transferfönstrets sista dag köptes Marouane Fellaini från Everton för £27,5 miljoner. Fellaini hade en utköpsklausul på £25 miljoner under transferfönstrets tidigare skede. David Moyes första tävlingsmatch var FA Community Shield-finalen mot Wigan, som Manchester United vann med 2–0. Resten av säsongen präglades av att Moyes testade olika laguppställningar under varje match utan några större framgångar (även om klubben nådde en kvartsfinalplats i Champions League). När fyra matcher återstod av Premier League-säsongen fick David Moyes sparken, och den nyavlagde spelaren Ryan Giggs blev tillfällig manager säsongen ut. Manchester United slutade på en fiaskoartad sjundeplats, efter att klubben under hela Premier Leagues existens dessförinnan alltid hade blivit topp tre. +Med floppsäsongen bakom sig värvade klubben Ander Herrera och Luke Shaw, efter att Louis van Gaal utnämnts till ny manager (med Giggs som assisterande tränare), under sommaren 2014. Efter Premier League-premiären värvades Marcos Rojo och Angel Di Maria (Di Maria för £60 miljoner), och Radamel Falcao säsongslånades från AS Monaco som tilltänkt sniper. Innan Falcao och Di Maria anlände hade van Gaals lag bland annat stått för nederlaget 0–4 borta mot tredjedivisionsklubben Milton Keynes Dons FC i Ligacupen den 26 augusti. Den 30 augusti värvades den nederländske mittfältaren Daley Blind som skulle visa sig lyfta anfallsspelet från mittfältet avsevärt. Försvaret hamnade i största fokus under säsongens gång, och bland annat dröjde klubbens första bortaseger i Premier League till matchen mot Arsenal den 22 november. Klubbens och van Gaals främsta målsättning under säsongen var en topp fyra-placering i Premier League för återavancemang till Uefa Champions League 2015/2016, vilket klubben nådde i och med sin slutgiltiga fjärdeplats. Efterföljande Champions League-kval i augusti 2015, med ungefär halva truppen förnyad inför säsongen, vanns med totalt 7–1 i dubbelmötet mot belgiska Club Brugge KV. I slutet av september var Manchester United serieledare i Premier League för första gången sedan Fergusons tid, och hade därefter varierande topp fyra-placeringar fram till slutet av november – innan formsvackan då ett skadedrabbat Manchester United dels åkte ur gruppspelet i Champions League, men kom också att sjunka till sjätte plats i Premier League-tabellen. Louis van Gaal hade spenderat motsvarande 2,8 miljarder kronor på spelare i klubben utan någon större framgång, och förutom all kritik i media yttrade sig före detta United-spelare som Peter Schmeichel om att laget uppträdde allt annat än kreativt och underhållande. I Europa League i februari 2016 förlorade klubben en match mot Midtjylland med 2–1, men van Gaals post räddades några dagar senare av en 3–0-seger mot Shrewsbury som tog klubben till kvartsfinal i FA-cupen. Manchester United slutade på femte plats i Premier League och gick därmed miste om spel i Champions League nästkommande säsong. Louis van Gaal fick dock slutligen jubla med klubben över dess första titel efter Ferguson; FA-cupfinalen den 21 maj 2016 vann Manchester United mot Crystal Palace FC med 2–1 efter förlängning. +Direkt efter cupvinsten presenterades portugisen José Mourinho som van Gaals efterträdare, efter frekventa rykten om detta sedan Mourinho sparkades från Chelsea mitt i säsongen. Vidare inledde klubben en förhandlingsprocess med den svenske storstjärnan Zlatan Ibrahimović, som den 30 juni meddelade sig tillhöra Mourinhos trupp kommande säsong. Värvningen av Paul Pogba den 9 augusti 2016 för motsvarande 1,1 miljard kronor blev fotbollsvärldens dittills dyraste spelarköp. Manchester United kom att vinna Ligacupen 2016/2017, men slutade Premier League-säsongen på sjätte plats och stod för rekordfå matchsegrar. Klubben nådde dock en plats i nästa säsongs Champions League genom att vinna Europa League för första gången (på så vis klubbens första europeiska titel sedan 2008). Finalen mot AFC Ajax (2–0) spelades på Friends Arena i Stockholms län den 24 maj 2017. Trots skadefrånvaro under säsongens tio sista matcher blev Zlatan Ibrahimović lagets främste målgörare (totalt 28 mål). +Ole Gunnar Solsjkaer tog över efter José Mourinho 2018. Norrmannen inledde med en lång förlustfri svit, som tog stopp i ett hemmamöte i Uefa Champions League mot PSG i åttondelsfinalen 2019. Manchester United vände dock i andra mötet, och gick vidare på bortamål. Han tog klubben till kvartsfinal i Champions League under 2018/2019. Säsongen 2019/2020 slutade laget på en tredjeplats i Premier League, samt en andra placering under 2020/2021. Den 2 februari 2021 noterades ett nytt Premier League-rekord för klubben i segermarginal, genom 9–0 hemma mot Southampton FC. Dessutom tog sig laget till final i Europa League efter att laget kom trea i sin Champions League-grupp tidigare på säsongen. Finalen förlorades mot Villareal på straffar. Fyra raka säsonger utan trofé gjorde att Solskjaer sparkades i november 2021, och efterträddes tillfälligt av Michael Carrick respektive Ralf Rangnick. Den 18 april 2021 hade Manchester United meddelat att ansluta sig till elva andra europeiska klubbar som grundande medlemmar av European Super League, en föreslagen turnering med 20 lag avsedd för att konkurrera med Uefa Champions League. Detta fick en betydande motreaktion från supportrar, andra klubbar, media, sponsorer, spelare och även brittiska regeringen, vilket tvingade klubben att dra sig ur bara två dagar senare. Misslyckandet med projektet ledde till att vice ordföranden Ed Woodward avgick, medan stora protester mot Woodward och familjen Glazer ledde till en planinvasion inför en ligamatch mot Liverpool den 2 maj 2021 – där en Premier League-match för första gången i tävlingens historia sköts upp på grund av supporterprotester. +Den 21 april 2022 utsågs nederländaren Erik ten Hag till ny manager med ett treårskontrakt från slutet av säsongen 2021/22. Han ledde klubben till seger i Engelska Ligacupen 2022/23, efter 2–0 mot Newcastle United FC. Veckan senare åkte klubben på sin största förlust någonsin inbördes mot ärkerivalen Liverpool, med slutresultatet 7–0 på Anfield. I slutet av säsongen 2023/24, efter att ha missat Europaspel och slutat åtta i ligan (klubbens lägsta placering någonsin i Premier League), kom laget att säkra titeln i FA-cupen med 2–1 i finalen mot Manchester City vilket sportsligt gav en kvalplats till Uefa Europa League 2024/2025. Efter att ha inlett säsongen 2024/25 i botten av tabellen fick ten Hag sparken den 28 Oktober 2024, dagen efter att klubben förlorat 1–2 borta mot West Ham United FC. +Efter att Ruud van Nistelrooy tagit över tillfälligt, två vinster och ett kryss, blev portugisen Rubén Amorim presenterad som ny huvudtränare. Han köptes loss från Sporting CP för €11,5 miljoner och presenterades officiellt den 1 november 2024 Amorim inledde med bland annat vinst 4–0 mot Everton och 2–1 borta mot lokalrivalerna Manchester City FC, men därefter föll laget ihop och slutade på 15:e plats. Mellan omgång 30 och 37 skrapade Manchester United ihop 2 poäng på 7 matcher med målskillnaden 5–14. Under den tiden låg fokuset dock inte på ligaspelet, utan på Europa League. Efter tre raka kryss under Erik ten Hag vann laget med Amorim de resterande fem matcherna i ligaspelet och slutade på en tredjeplats, vilket betydde att man hoppade över playoffrundan och istället gick direkt på åttondelsfinal. Där ställdes United mot Real Sociedad och efter 1–1 borta blev det 4–1 hemma i en match där tre straffsparkar tilldömdes (varav två för United). 2–2 slutade första kvartsfinalen borta Lyon, och returen hemma avgjordes i förlängning där ett uträknat Manchester United vände från 2–4 till 5–4 i slutminuterna. Efter en 3–0-vinst borta och 4–1 hemma mot Athletic Bilbao i semifinal var laget obesegrat i Europa League och ställdes mot Tottenham i finalen. Trots att Man United var favoriter skulle Brennan Johnson bli finalmatchens ende målskytt precis innan halvtid – där Tottenham vann sin första titel på 17 år. Säsongen 2025/26 inleddes med förlust 1–0 borta mot Arsenal på Emirates Stadium. Efter 8 ligamatcher hade United plockat ihop 13 poäng och låg på 9:e plats, 2 poäng från Champions League-plats. +Spelare i fet stil är fortfarande aktiva i Manchester United. +Senast uppdaterad 22 januari 2017 +Spelare i fet stil är fortfarande aktiva i Manchester United. +Senast uppdaterad 22 januari 2017 +Manchester United Football Club hör, förutom de sportsliga framgångarna, till superklubbarna såväl ekonomiskt och vad gäller supporterstöd. Klubben har sedan mitten av 1990-talet hört till världens allra rikaste, både vad gäller omsättning och marknadsvärde. Många år omsatte klubben mest pengar av alla klubbar men har sedan säsongen 2004/2005 tappat toppositionen till de spanska giganterna Real Madrid och FC Barcelona, med undantag för 2012 och 2017–2018. Klubben rankas även som den populäraste fotbollsklubben sett till dess världstäckande supporterstöd. +Manchester Uniteds ekonomiska styrka avspeglar sig tydligt i klubbens aktiviteter på transfermarknaden. I början av februari 2011 hade fyra av de tio dyraste köpen någonsin i Premier League haft Manchester United som köpare: +Dimitar Berbatov – £ 30,75 miljoner, från Tottenham 2008. +Rio Ferdinand – £ 29,1 miljoner, från Leeds United 2002. +Juan Sebastián Verón – £ 28,1 miljoner, från Lazio 2001. +Wayne Rooney – £ 25,6 miljoner, från Everton 2004. +Värvningen av Paul Pogba inför säsongen 2016/2017 på £93,2 miljoner (cirka 1,1 miljarder kronor) var världens dittills dyraste spelarköp. +Klubben är ofta världsdominerande vad gäller supporterstöd, med supporters i så gott som hela världen. Den lokala publiktillströmningen har klubben sedan lång tid tillbaka det högsta publiksnittet bland klubbarna i Premier League. De senaste åren har det legat på ungefär 76 000 vilket innebär att hemmaarenan Old Trafford är fullsatt praktiskt taget varje match. Old Traffords publikkapacitet har via utbyggnader ökat med över 30 000 sedan mitten av 1990-talet. +Trots uteblivna framgångar efter att Sir Alex Ferguson lämnade klubben 2013 har Manchester United fortsatt att vara en av klubbarna som tjänat mest pengar i världen. Trots en 15:e-plats säsongen 2024/25, finalförlust i Europa League finalen och inget kontinentalt spel under säsongen 2025/26 omsatte klubben $863 miljoner. Under 2023/24 var Manchester United den klubben med fjärde högst inkomst med $825 miljoner, före bland annat Bayern München, Liverpool och Arsenal. Detta trots att Man United kom på 8:e plats och åkte ut direkt i gruppspelet i Champions League medan Bayern München tog sig till semifinal. +Efter sommarfönstret 2025/26 passerat har Manchester United stått för 4 av de 20 dyraste köpen i ligans historia: +(10) Paul Pogba - £ 89.3 miljoner, från Juventus 2016/17 +(15) Antony - £ 81.3 miljoner, från Ajax 2022/23 +(18) Harry Maguire - £ 80 miljoner, från Leicester City 2019/20 +(20) Jadon Sancho - £ 73 miljoner, från Borussia Dortmund 2021/22 +Ursprungligen stod Lancashire and Yorkshire Railway Company för finansieringen, innan klubben 1892 blev ett aktiebolag och sålde aktier till lokala anhängare för £1 via en ansökningsblankett. 1902 skickades majoritetsägande till de fyra lokala affärsmän som investerat £500 för att rädda klubben från konkurs, inklusive klubbens då framtida president John Henry Davies. Efter hans död 1927 stod klubben ännu en gång inför konkurs, men räddades i december 1931 av James W. Gibson som tog kontroll över klubben efter en investering på £2 000. Gibson främjade sin son Alan i styrelsen 1948, och tre år senare dog James. Familjen Gibson behöll ägandet av klubben genom James hustru Lilian, men ordförandeposten tillgavs den före detta spelaren Harold Hoffman. +Louis Edwards, vän till Matt Busby, befordrades till styrelsen några dagar efter flygolyckan i München 1958 och började förvärva aktier i klubben; för en investering på £40 000 hade han samlat 54 procent av aktierna, och tog kontroll i januari 1964. När Lilian Gibson dog i januari 1971 skickades hennes aktier till Alan Gibson som sålde en procentsats av sina aktier till Louis Edwards son Martin Edwards 1978, som kom att bli ny ordförande vid Louis död 1980. Media-storpampen Robert Maxwell försökte köpa klubben 1984, men uppfyllde inte Edwards utgångspris. 1989 försökte Martin Edwards sälja klubben till Michael Knighton för £20 miljoner, men försäljningen föll igenom och Knighton gick med i styrelsen istället. +Manchester United noterades på börsen i juni 1991 (kapitalet steg med £6,7 miljoner) och fick ännu ett uppköpserbjudande 1998, denna gång av Rupert Murdrochs företag British Sky Broadcasting. Detta resulterade i bildandet av Shareholders United Against Murdoch – nu Manchester United Supporters Trust – som uppmuntrade anhängare att köpa aktier i klubben i ett försök att blockera alla fientliga köpare. Manchester Uniteds styrelse antog ett erbjudande på £623 miljoner, men övertagandet var blockerat av Storbritanniens konkurrenskommission på det sista hindret i april 1999. Några år senare uppstod en maktkamp mellan Alex Ferguson och hans hästkapplöpningspartners, John Magnier och J. P. McManus, som gradvis hade blivit majoritetsägare. I en tvist som härrörde från angripet ägande av hästen Rock of Gibraltar försökte Magnier och McManus få bort Ferguson som manager, och styrelsen svarade med att av närmande investerare minska den irländska majoriteten. +I maj 2005 köpte Malcolm Glazer 28,7 procent av aktierna som McManus och Magnier innehade, och därmed förvärvades en kontrollpost genom sin investering som förmedlade Red Football Limited i ett högt belånat övertagande att värdesätta klubben till £800 miljoner (senare $1,5 miljarder). I juli 2006 meddelade klubben sig ha en skuld på £660 miljoner av refinansieringspaketet, vilket resulterade i en minskning med 30 procent av årliga räntebetalningar till £62 miljoner. Med skulder på £716,5 miljoner ($1,17 miljarder) i januari 2010 refinansierades Manchester United ytterligare genom en obligationsemission värd £504 miljoner, som gav klubben möjlighet att betala av det mesta av de £509 miljoner gentemot internationella banker. Den årliga räntan på obligationerna – som sträcker sig till den 1 februari 2017 – är cirka £45 miljoner. Trots omstruktureringarna föranledde klubbens skulder till att fansen visade sitt missnöje på både Old Trafford och Trafford Training Centre den 23 januari 2010, och vidare fick förmögna fans funderingar på att köpa ut Glazers ur ledningen med bestämt inflytande. +I augusti 2011 tycktes Glazer ha kontaktat Credit Suisse i förberedelse för $1 miljard (cirka £600 miljoner) börsintroduktion på Singaporebörsen som skulle värdera klubben på £2 miljarder, men i juli 2012 meddelade klubben planera att lista sin börsintroduktion på New York-börsen istället. Trots efterföljande nedskärning i och med sänkta priser på aktier värdesattes Manchester United i augusti 2012 till £2,3 miljarder vilket utgjorde klubben till världens mest värdefulla. +I slutet av december 2023 köpte den brittiska kemijätten Ineos 25% av fotbollsklubben för omkring 1,3 miljarder pund. +Manchester Uniteds klassiska rivaler är Arsenal FC, Chelsea FC, Tottenham, Liverpool FC och Manchester City FC (som utgör det så kallade "Manchester-derbyt"). +Rivaliteten med Liverpool har anknytningar till tidigare tävlingar mellan dessa städer under Industriella revolutionen när Manchester var känd för sin textilindustri medan Liverpool för sin stora hamn. Manchester United och Liverpool har i historiken också varit de två mest framgångsrika fotbollsklubbarna i England och slagits om ligatiteln mot varandra flera gånger. +Rivaliteten med Leeds, även kallat "Rosornas krig", bottnar i en historisk fejd på 1400-talet mellan distrikten Yorkshire och Lancashire. Det är cirka 60 kilometer mellan städerna vilket ger matcherna en karaktär av derby. +Rivaliteten med Arsenal härstammar från de flertal gånger som klubbarna, såväl också tränarna Alex Ferguson och Arsène Wenger, slagits om Premier League-titlar (Manchester United 20 stycken och Arsenal 13). +Manchester Uniteds klubbmärke härleds från vapenskölden hos Manchester United Council, och trots allt som återstår av detta på det aktuella märket har fartyget fulla segel. Djävulen med treudden kommer från klubbens smeknamn The Red Devils, som fanns med på klubbprogram och halsdukar under 1960-talet, och kom att ingå i klubbmärket från 1970 även om det dröjde till 1971 innan djävulen kom med på bröstmärket på matchställen (såvida inte laget spelade i en cupfinal). +Ett fotografi på Newton Heath-laget 1892 tros visa att laget bar röd-vita kvartströjor och blå shorts. Mellan 1894 och 1896 var tröjorna distinkt gröna och guldfärgade, som 1896 ersattes av vita tröjor med blå shorts. Efter namnbytet 1902 blev klubbens matchställsfärger ändrade till röda tröjor, vita shorts och svarta strumpor vilket blev hemmastandarden. Sedan dess fram till 1922 gjordes mycket få ändringar med matchstället tills klubben antog vita tröjor med ett mörkrött "V" kring halsen, som bestod fram till 1927. Under en period 1934 förändrades stället till körsbärsröda tröjor med vit ring, men den föregående röda tröjan återkallades året därpå när klubbens lägsta placering någonsin blev ett faktum (tjugonde plats i andra divisionen), ringen släpptes tillbaka. De svarta strumporna ändrades till vita 1959–1965, ändrades därefter till röda fram till 1971 och återgick till svarta. Svarta shorts och/eller vita strumpor bärs oftast i bortastället, ibland hemmastället om det finns en konflikt med motståndarens ställ. De nuvarande hemmatröjorna är röda med vitknäppt krage, och manschetter med svart och vit klippning. +Manchester Uniteds bortaställ har ofta bestått av en vit tröja, svarta shorts och vita strumpor men har funnits flera undantag; bland andra en helsvart remsa med blått och guldklippningar mellan 1993 och 1995, och den marinblå tröjan med horisontella silverkritstreck användes säsongen 1999/2000. Bortastället 2011/2012 bestod av en kungsblå kropp och tunnbandstillverkade ärmar av små midnatts-marinblå ränder, med svarta shorts och blå strumpor. Ett helgrått bortaställ bars säsongen 1995/1996, men släpptes tillbaks efter fem matcher eftersom spelarna hävdade sig ha svårt att hitta sina lagkamrater mot publiken. Till 100-årsjubileet av "Manchester United" 2001 fanns ett reversibelt vit-/guldställ, även om tröjorna i det faktiska stället inte var reversibla. +Klubbens tredjeställ är ofta helblått, vilket också fallet är 2015. Undantag inkluderar en halverad grön-/guldtröja 1992–1994, en blåvit-randig tröja 1994–1996 och en gång säsongen 1996/1997, ett helsvart ställ under trippelvinstsäsongen 1998/1999, samt vita tröjor med svart-röda horisontella kritstreck 2003–2005. Sedan 2006/2007 har tredjestället ofta varit den föregående säsongens bortaställ, med undantag för säsongen 2008/2009 och 2014/2015. +Newton Heath spelade inledningsvis på North Road med en publikkapacitet på 12 000 åskådare, men klubben ansåg att detta inte var tillräckligt för att få gå med i The Football League. En viss expansion beviljades 1887, och 1891 hade två ståplatsläktare gjorts för 1 000 platser vardera på en minimal finansiering av Newton Heath. I juni 1893 tvingades klubben byta till arenan Bank Street med plats för 10 000 åskådare, efter att klubben blivit vräkt av North Roads ägare. Även denna arena hade den bristfälliga standarden med ett underlag som kunde variera mellan grusplan och sumpmark. Klubbens nytillträdde ordförande John Henry Davies tyckte inte att Bank Streets plan var duglig för ett lag som nyss hade vunnit högsta ligan och FA Cupen, så han donerade pengar till klubben så att de skulle kunna bygga en ny arena. John Henry Davies letade själv runt i Manchester efter en lämplig plats innan han bestämde sig för att bygga arenan Old Trafford som stod klar 1910 vid Bridgewaterkanalen, norr om Warwick Road i Greater Manchester, Trafford. +Old Trafford formgavs av den skotska arkitekten Archibald Leitch som redan hade ritat ett flertal andra arenor. Det var ursprungligen tänkt att arenan skulle ha plats för 100 000 åskådare med sittplatser på den södra läktaren under tak och med ståplatsläktare utan tak på de andra tre sidorna. Inklusive tomten budgeterades det för £60 000 att bygga arenan. Under byggnationen ökade kostnaderna och det skulle krävas £30 000 extra för att nå upp till 100 000 platser i arenan. Efter ett förslag från klubbsekreteraren John Bentley beslutades det att reducera platserna till 80 000. +Old Traffords nuvarande publikkapacitet är 75 811 åskådare, och är Englands näst största fotbollsarena efter Wembley Stadium. Arenan har varit Manchester Uniteds hemmaarena sedan 1910, undantaget åren 1941–1949 då den fick stora skador under andra världskriget till följd av tyskarnas bombningar. Den har sedan dess byggts upp igen och kallas idag för The Theatre of Dreams ("Drömmarnas teater"), ett uttryck som Bobby Charlton myntade. Under denna period spelades matcherna på Maine Road vilket var och är de lokala rivalerna Manchester Citys hemmaplan. Arenans publikrekord på 76 962 åskådare sattes under FA Cupens semifinal mellan Wolverhampton Wanderers FC och Grimsby Town FC säsongen 1939. Även Premier Leagues publikrekord, 76 098 åskådare, är från Old Trafford och sattes när Manchester United mötte Blackburn Rovers den 31 mars 2007 och vann med 4–1.Arenan är klassificerad som en 5-stjärnig arena av Uefa. Old Trafford har en egen järnvägsstation, Manchester United Football Ground, vid vilken tågen stannar endast på matchdagar. +Korrekt per den 09 september 2026 +Manchester United Football Club Under-23 kallas Manchester Uniteds reservlag, som spelar i Premier League 2 och deltar även i Manchester Senior Cup där laget på senare år har börjat använda ett stort antal juniorspelare. +Listkriterier: +Listan inkluderar alla icke-brittiska spelare (som ej kommer ifrån England, Nordirland, Skottland, Wales eller Irland) som representerat klubben. +Listan inkluderar enbart spelare som representerat klubben i ligamatcher. +Följande Manchester United-spelare har blivit skyttekungar i den engelska högstaligan: +* Delad förstaplats med Ron Davies (Southampton)** Delad förstaplats med Jimmy Floyd Hasselbaink (Leeds United) och Michael Owen (Liverpool)*** Delad förstaplats med Carlos Tevez (Manchester City) +Ägare: Familjen Glazer via Red Football Shareholder Limited och Sir Jim Ratcliffe +Hedersordförande: Martin Edwards +Manchester United Limited +Manchester United Football Club +Tränarstab för A-laget +Manchester United supporterklubb för damer startade sin verksamhet i slutet av 1970-talet och erkändes inofficiellt som klubbens seniorlag för damer. De blev grundande medlemmar av North West Women's Regional Football League 1989. Laget ingick ett officiellt partnerskap med Manchester United 2001 och blev klubbens officiella damlag – men efter Malcolm Glazers övertagande 2005 upplöstes klubben eftersom den ansågs vara "olönsam". +År 2018 bildade Manchester United ett nytt damlag, som debuterade i den engelska andradivisionen för damer 2018/19. Damlaget vann sin första större trofé den 12 maj 2024, i form av FA Women's Cup, efter 4–0 i finalen mot Tottenham. +Den här artikeln är helt eller delvis baserad på material från engelskspråkiga Wikipedia, tidigare version. +Barnes, Justyn (2001). The Little Book of Manchester United. London: Manchester United Books, 2001. ISBN 978-0-233-99995-1 +Bose, Mihir (2007). Manchester Disunited: Trouble and Takeover at the World's Richest Football Club. London: Aurum Press. ISBN 1-84513-121-5 +Crick, Michael; Smith, David (1990). Manchester United – The Betrayal of a Legend. London: Pan Books. ISBN 0-330-31440-8. https://archive.org/details/manchesterunited0000cric_t8b6 +Devlin, John (2005). True Colours: Football Kits from 1980 to the Present Day. London: A & C Black. ISBN 0-7136-7389-3. https://archive.org/details/truecoloursfootb0000devl +Dobson, Stephen; Goddard, John (2004). International Sports Economics Comparisons. Westport, CT: Praeger Publishers. ISBN 0-275-98032-4 +Dunning, Eric (1999). Sport Matters: Sociological Studies of Sport, Violence and Civilisation. London: Routledge. ISBN 978-0-415-09378-1. https://archive.org/details/sportmatterssoci0000dunn +Inglis, Simon (1996) [1985]. Football Grounds of Britain (tredje upplagan). London: CollinsWillow. ISBN 0-00-218426-5. https://archive.org/details/footballgroundso0000ingl +James, Gary (2008). Manchester: A Football History. Halifax: James Ward. ISBN 978-0-9558127-0-5 +McCartney, Iain (1996). Old Trafford – Theatre of Dreams. Harefield: Yore Publications. ISBN 1-874427-96-8 +Murphy, Alex (2006). The Official Illustrated History of Manchester United. London: Orion Books. ISBN 0-7528-7603-1. https://archive.org/details/officialillustra0000unse +Tyrrell, Tom; Meek, David (1996) [1988]. The Hamlyn Illustrated History of Manchester United 1878–1996. London: Hamlyn. ISBN 0-600-59074-7 +White, Jim (2008). Manchester United: The Biography. London: Sphere. ISBN 978-1-84744-088-4. https://archive.org/details/manchesterunited0000whit_w8i5 +White, John (2007). The United Miscellany. London: Carlton Books. ISBN 978-1-84442-745-1. https://archive.org/details/unitedmiscellany0000whit_y9e1 +Wikimedia Commons har media som rör Manchester United FC. +Officiell webbplats +Gustaf Richard Yngve Larsson ([ˈʏ̌ŋːvɛ ˈlɑ̌ːʂɔn] ), född 13 december 1881 i Sundsvalls församling, död 16 december 1977 i Oscars församling i Stockholm, var en svensk statsvetare, politiker (S, LP, FP), borgarråd, riksdagsledamot och ordförande för Stockholms stadskollegium. +Yngve Larsson avlade filosofie kandidatexamen vid Uppsala universitet och fortsatte statsvetenskapliga studier utomlands med inriktning mot kommunalpolitik. Från 1908 var Larsson en ledande kraft bakom Svenska stadsförbundets tidiga utveckling, och 1913 disputerade han för filosofie doktorsgrad på Inkorporeringsproblemet. Han var ursprungligen aktiv socialdemokrat, men uteslöts ur partiet av Hjalmar Branting på grund av sitt deltagande i Aktivistboken 1915. Larsson involverades som expert i Stockholms författningsreform och utsågs 1920 till stadssekreterare i Stockholms stad. +Larsson valdes till borgarråd i Stockholm 1924 och kvarstod i 22 år till 1946, från 1935 på det nya Folkpartiets mandat. Som gatu- och stadsbyggnadsborgarråd kom han att bli politiskt ledande bakom flera av stadens historiskt största stadsbyggnadsprojekt, däribland tunnelbanan, Norrmalmsregleringen och förorternas planering samt ett flertal centrala infrastrukturprojekt som Slussen, Bromma flygfält, Västerbron, Tranebergsbron och världsarvet Skogskyrkogården. Under andra världskrigsåren var Larsson även en framträdande nordisk antinazistisk opinionsbildare. Efter att ha avgått som borgarråd valdes han till riksdagsledamot för Folkpartiet och senare till ordförande för Stockholms stadskollegium. Han lämnade stadshuset 1954, men behöll centrala poster inom Stockholms stad fram till 1970, då 89 år fyllda. +Yngve Larsson erhöll flera svenska och utländska utmärkelser och blev mot slutet av sin karriär några gånger även omnämnd med epitetet "legendarisk". Han kom att kallas statsman, sitt århundrades främste stadsbyggare i Sverige och dess gigant i stockholmspolitiken. Larssons stora stadshistoriska verk och självbiografi Mitt liv i Stadshuset (1977) kom att kallas ett "monument över en epok i Stockholms historia". +Yngve Larsson föddes den 13 december 1881 i Sundsvall som son till grosshandlaren Axel Larsson (1854–1894) och Märtha Catharina Alida (Ida) Sundberg (1858–1950), dotter till sågverksägaren Fredrik Sundberg (1830–1913). Föräldrarna gifte sig 1879 och fick tre barn, utöver Yngve Larsson även sedermera direktören Lennart Larsson (1880–1966) och Ingrid Stieve (1884–1941), senare gift med den tyske diplomaten och författaren Friedrich Stieve. Yngve Larsson var även kusin till professorn och likaledes borgarrådet Halvar Sundberg och bland hans morbröder märks juristen och donatorn Embleton Sundberg. +1891 insjuknade fadern, varvid modern och barnen flyttade till Stockholm och Sibyllegatan 16 intill Östermalmstorg, och Larsson började på Palmgrenska samskolan. Axel Larsson avled 1894 och modern gifte 1895 om sig med grosshandlaren John Hallberg (1859–1919). Familjen bodde från 1898 på Nybrogatan 58 / Tyskbagargatan 1 på Östermalm och flyttade 1902 till Saltsjöbaden. +År 1900 tog Yngve Larsson mogenhetsexamen vid Norra Real i Stockholm, där han även hade varit ordförande för gymnasieföreningen Saga och stiftat livslång bekantskap med Otto Järte och sedermera svågern Tor Bonnier. +År 1900 påbörjade Yngve Larsson studier vid Uppsala universitet. I Uppsala umgicks han med läroverksvännerna Otto Järte och Tor Bonnier, och även de sedermera författarna Sigfrid Siwertz och Sven Lidman knöts till kretsen. Han studerade bland annat historia och statsvetenskap för Harald Hjärne respektive Simon Boëthius och blev fil. kand. 1903. Under några månader fick Larsson "vila överansträngda nerver" hos läkaren Ernst Westerlund i Enköping och fortsatte därefter licentiatstudier med utlandsperioder i Heidelberg och vid Handelshochschule Berlin, för bland andra Hugo Preuss, samt vid London School of Economics. +Larsson tog licentiatexamen i statsvetenskap 1909 för Simon Boëthius, och disputerade för filosofie doktorsgrad 1913 på avhandlingen Inkorporeringsproblemet på uppdrag av Svenska stadsförbundets styrelse, definierat som "frågan om det rättsliga stadsområdets anpassning till det faktiska". Avhandlingen omfattade 1234 sidor och behandlade de förvaltningspolitiska och juridiska problem som den snabba urbaniseringen ställde dåtidens svenska städer inför. Verket fick ett respektfullt mottagande även utomlands under sin tid, och hade alltjämt 1920 en ställning som standardverk. +Dessförinnan hade han publicerat undersökningar av stadsförvaltningarna i en lång rad europeiska länder för CSA och stadsförbundet (se nedan), och avhandlingen blev den första av en serie utredningar som kom att behandla samma ämne. +Yngve Larssons politiska intresse väcktes under de första studieåren, och han blev först aktiv medlem i den liberala studentföreningen Verdandi för att sedan gå över till det nya socialdemokratiska Laboremus som han var med om att starta. Hans första insats som ung debattör gällde Norrlandsfrågan i Verdandi, men han fördjupade sig sedan i frågor som låg närmare hans studier: bostadsfrågan, jordreform, kommunalpolitiken i Stockholm och kommunala organisationsproblem. Larsson var från 1903 skribent i tidningen Social-Demokraten med Hjalmar Branting som chefredaktör. +Yngve Larsson engagerade sig tidigt i Centralförbundet för Socialt Arbete, och blev 1906 kassör i dess Jordreformförening, i samarbete med bland andra Erik Palmstierna, Gustav Cassel, Carl Lindhagen och Adrian Molin. Larsson var en aktiv deltagare i den radikala politiska diskussionsklubben Palmstiernas klubb som grundades 1908, och han blev medlem av det socialdemokratiska partiet. Larsson var författaren av det socialdemokratiska partiets kommunalpolitiska program 1911, av arbetarrörelsens idéhistoriker Jan Lindhagen kallat "ett av de allra finaste i den långa raden av socialdemokratins program". 1912 stod Larsson och Järte bland namnen bakom stiftelseurkunden till arbetarrörelsens Nya Banken. +Larsson var under lång tid djupt engagerad i svensk bostadspolitik. Genom CSA publicerade han 1908 utredningen Kommunalsocialistisk bostadspolitik i utlandet. Han var senare en tongivande ledamot i Bostadskommissionen 1912–1918 och ordförande för den efterföljande Kommittén angående bostadssociala minimifordringar. Han medverkade även till bildandet av Stockholms Kooperativa Bostadsförening 1916, och var sekreterare för Svenska slöjdföreningens stora Hemutställning 1917. +Under första världskriget förespråkade Larsson jämte bland andra ungdomsvännen Otto Järte en tyskvänlig linje. De var båda medförfattare till den anonymt utgivna så kallade aktivistboken Sveriges utrikespolitik i världskrigets belysning (1915). Skriften var ett av de främsta uttrycken för första världskrigets svenska aktivism och reste krav på "en modig uppslutning vid Tysklands sida". Den tyske riksdagsledamoten Albert Südekum spelade en stor roll för författarna. Boken vållade upprörd debatt och ledde efter ingripande av ententevänlige partiledaren Hjalmar Branting till Larssons uteslutning ur SAP. +Yngve Larsson medverkade även jämte Järte vid grundandet av Svenska aktivismens organ Svensk Lösen, och Larsson lämnade flera bidrag till tidskriften under 1916. 1917 var han även en av stiftarna av den nationalistiska klubben Brunkeberg. +År 1908 fick Larsson sin första tjänst inom Stockholms stad som amanuens under några månader för Josef Guinchard vid Stockholms stads statistiska kontor. Där fick han i uppdrag att utreda dels alkoholmissbrukets sociala verkningar, dels de kommunala tjänstemännens arbetstid. Larsson blev senare samma år vid grundandet av Svenska stadsförbundet (föregångaren till kommunförbundet) dess andre sekreterare samtidigt som vännen Erik Palmstierna utsågs till förbundets förste sekreterare. Larsson övertog senare Palmstiernas ledande roll; 1915 blev han förbundets förste sekreterare och 1918 dess direktör. +Som en följd av disputationen på inkorporeringsområdet 1913 blev Yngve Larsson samma år ledamot av Kommittén för kommunala nybildningar, vilken lämnade sitt betänkande 1918 och resulterade i lagen om ändring i kommunal och ecklesiastik indelning 1919. +Larsson var den drivande kraften bakom grundandet av Stadshistoriska institutet (numera Stads- och kommunhistoriska institutet) 1919 i samarbete med historikerna Nils Ahnlund och Nils Herlitz. Sedan 1994 delar institutet ut Yngve Larssons pris för stads- och kommunhistoriska insatser, instiftat av Svenska Kommunförbundets styrelse. +Redan 1912 engagerades Larsson i den stora författningsreformen i Stockholm 1920 som i hög grad bar hans prägel. Han hade som biträde till Pehr Rabe ett betydande inflytande över den Rabeska utredningen 1912–1914. Som sekreterare i Finansstyrelsekommittén kom han själv att utforma det slutliga förslag som godkändes av stadsfullmäktige den 16 juni 1919. Han valdes själv den 19 april 1920 av stadsfullmäktige till chef för stadskansliet i Stockholms stad; en betydelsefull roll för samordning av hela stadsförvaltningen som senare kom att kallas stadssekreterare. Chefen för stadskansliet hade sitt kontor i det nordvästra hörnrummet i rådsgången en våning upp i Stockholms stadshus. Som stadssekreterare förde Larsson bland annat protokollet för den utredning 1922 som resulterade i Stockholms stadsvapen. Den nyblivne kanslichefen kallades även av Socialstyrelsen till sakkunnig vid Bostadsräkningen 1920. +Larsson blev borgarråd 1924 och förblev det till 1946. Han hade ansvar för i tur och ordning administrativa roteln, gaturoteln (från och med 1932) och stadsbyggnadsroteln (från 1940), och hade en central roll i beslutsprocessen för bland annat Slussen och Södertunneln (1930–1935), Väster- och Tranebergsbroarna (1934–1935), Bromma flygplats (1935–1936), Stockholms tunnelbana (1941), flera förorter,, Henriksdalsverket (1941) samt den stora Norrmalmsregleringen (1945). +Larsson var från början vald på opolitiskt mandat, men anslöt sig i slutet av 1920-talet till Liberala partiet. Vid det Liberala partiets samgående med Frisinnade folkpartiet till det nybildade Folkpartiet inför kommunalvalet 1935 anmälde Larsson sitt inträde i detta, och han valdes då även in i Stockholms stadsfullmäktige som Folkpartiets första namn i en valkrets på Södermalm. +Som gatuborgarråd och ordförande i kyrkogårdsnämnden var han 1935–1940 beställare och byggherre för anläggningen Skogskrematoriet med Monumenthallen och Trons, Hoppets och Heliga Korsets kapell vid Skogskyrkogården, som 1994 upptogs på Unescos världsarvslista. Larsson var även under 16 år ordförande för Slakthus- och saluhallsstyrelsen. +I gatunämnden var Larsson som ordförande drivande för att tillsätta arkitekten och funktionalisten Osvald Almqvist som stadsträdgårdsmästare 1936, och två år senare den mångårige efterträdaren Holger Blom, som några år tidigare arbetat för Larsson i Slussbyggnadskommittén. Yngve Larsson hade 1935 gjort en studieresa i USA med besök vid parkanläggningar i New York-regionen och beskrev senare sina intryck i Tidskriften Arkitektur: +Bland Larssons borgarrådssekreterare märks den sedermera partiledaren och statsrådet Gösta Bohman, som på rekommendation av Anders Ahlén blev tillförordnad under några månader 1942, då den ordinarie sekreteraren blivit inkallad. Larsson föreslog därefter Bohman som biträdande sekreterare i Stockholms handelskammare. Gösta Bohman såg tillbaka på Yngve Larsson som en "stimulerande läromästare". +Under 1930-talet hade Larsson en ledande roll i den tidens stora ombyggnad av Slussen samtidigt med byggnationen av Södertunneln, stadens första tunnelbana. Han var ordförande för 1930 års trafikkommitté, vars förslag låg till grund för Stockholms stadsfullmäktiges investeringsbeslut 1931 att genomföra projekten. Utförandet uppdrog stadsfullmäktige åt gatuborgarrådet Larssons gatunämnd, vilken för ändamålet tillsatte Slussbyggnadskommittén med Larsson som ordförande. Kommittén porträtterades senare i en målning av konstnären John Hedberg, nu i Stadsmuseets samlingar. +Slussen invigdes den 15 oktober 1935 av kung Gustaf V och Yngve Larsson höll det inledande talet, där han bland annat myntade begreppet "en funkis-möbel i Mälardrottningens gemak" som beskrivning av Slussen. +Stockholms stad hade 1923 förvärvat Lindarängens flyghamn vid Ladugårdsgärde. Driften överläts först på Kungliga svenska aeroklubben, men 1928 tog staden över ansvaret för flyghamnen genom Flyghamnsstyrelsen med gatuborgarrådet Yngve Larsson som ordförande. Larsson var redan som barn teknikintresserad och hade tidigt intresserat sig för det begynnande trafikflyget, bland annat som flygbåtsresenär på sträckan Slite–Lindarängen 1925 och som deltagare jämte kommunikationsministern Carl Meurling vid invigningen av Stettins flygfält. +Larsson fick genom Flyghamnsstyrelsen 1928 ansvar för etableringen av ett nytt flygfält i Stockholm. Vid början av 1930-talet hade behovet stadigt växt. Flygfältet i Barkarby låg utanför stadsgränsen och relativt avsides. Sjöflygplanen som sedan 1920 startade och landade vid Lindarängen norr om Ladugårdsgärde kunde inte flyga vintertid på grund av is. Under 1920-talet hade det funnits planer på att upprätta en landflygplats vid Skarpnäck, men den 30 juni 1933 beslöt stadsfullmäktige att bygga ett flygfält vid Riksby i Bromma. Lördagen den 23 maj 1936 hölls invigningstalen i regn, rusk och dimma av Gustaf V och Yngve Larsson. Flygplatsen hade då kostat cirka 5,7 miljoner kronor. +Under de första beredskapsåren upprättade staden under Larssons flyghamnsstyrelse reservflygfält i Skarpnäck i sydöstra Stockholm och i Skå-Edeby på Ekerö. Larsson genomförde senare ett markbyte med Kronan av Bromma flygplats mot Gärdet i mitten av 1940-talet. Flyghamnsstyrelsen verkade till Larssons pensionering 1946, då Bromma flygplats övergick i statlig regi. +Yngve Larsson var ordförande för utredningen Tunnelbanedelegerade 1940, som ledde fram till att Stockholms stadsfullmäktige 1941 beslutade om utbyggnaden av ett tunnelbanesystem. Under decennierna efter kriget genomfördes detta enorma projekt som även innefattade byggandet av nya förorter kring de nya stationerna. +Denna satsning kunde genomföras i tämligen stor enighet mellan de inflytelserika aktörerna i staden kring visionen att tunnelbanan var den bästa lösningen både på stadens bostadsfråga och på problemet med tilltagande trängsel på innerstadens gator. Larsson var den främste strategen bakom denna lösning. Under mellankrigstiden beredde han först vägen för tunnelbaneutbyggnaden på många olika sätt, och därefter bidrog han till att skapa en bred enighet kring tunnelbanevisionen. +Yngve Larsson hade även en central roll i flera stora projekt för att inkorporera och bygga ut förorter till Stockholm under 1900-talet, något som även hade varit temat för hans doktorsavhandling Inkorporeringsproblemet 1912. Larsson initierade en Utredning angående stadsplanen för Gubbängen 1943 som har beskrivits som den första kvalificerade stadsplaneutredningen i Stockholm och fick stor betydelse för den fortsatta förortsplaneringen och överhuvudtaget för stadsplaneringen i Sverige under efterkrigstiden. Till Larssons insatser räknas även den första uppgörelsen om inkorporeringen av Spånga landskommun och Hässelby villastads köping 1944. Han var också i början av 1950-talet ordförande för den 1951 grundade Stor-Stockholms samarbetsdelegation. +Larssons roll i efterkrigstidens planering av Stockholm och de nya förorterna uppmärksammades internationellt. Den amerikanske stadsplaneraren Clarence Stein skrev bland annat att: +Larsson var från 1940-talet den drivande kraften bakom regleringen av Nedre Norrmalm, ansedd som den mest omfattande stadssaneringen i svensk historia. De gamla Klarakvarteren fick i en framstegsoptimistisk och modernistisk anda ge plats åt Stockholms moderna city med bland annat Sergels torg och Hötorgscity med Hötorgsskraporna, som han gav namnet "De fem trumpetstötarna". +På Larssons initiativ beslutade stadsfullmäktige 1944 att upprätta en ny generalplan för Stockholm. Promemorian Det framtida Stockholm: Riktlinjer för Stockholms generalplan där de grundläggande förutsättningarna för trafiksystemet fastställdes upprättades därefter under våren 1945 på stadsplanekontoret. Det principiella regleringsbeslutet om Sveavägens framdragande till Sveaplatsen (sedermera Sergels torg) och Hamngatan fattades därefter i stadsfullmäktige 1945 och resulterade i 1946 års cityplan. Det var vid stadsfullmäktiges behandling av ärendet som han i en framtidsvision myntade den bevingade musikaliska liknelsen om "fem trumpetstötar" för Hötorgsskraporna: +Under Larssons sista år på 1960- och 1970-talen kom han att känna en växande besvikelse över hur den ursprungliga planen förvaltades och utvecklades, och han kom att analysera om han glömt väsentliga frågor för att nå direkta resultat. Larsson kom tidigare än de flesta andra involverade i projektet att inse att en omprövning var nödvändig. Redan under 1950-talet försökte han som debattör att dämpa den fortsatta cityregleringen, och 1959 efterlyste han, med en ny bevingad musikalisk liknelse, "kammarmusik" i stället för trumpetstötar: +Norrmalmsregleringens senare utveckling kom att debatteras kritiskt i svenska medier och "rivningsraseriet" väckte protester med en kulmen under Almstriden 1971. Internationellt beundrades regleringen som en av de mest genomgripande och konsekventa cityomdaningar som genomfördes i Europa under efterkrigstiden, och organisationen av stadsplaneringen i Stockholm under tiden kring och efter kriget belönades av Internationella arkitektunionen med Sir Patrick Abercrombie Prize när det delades ut för första gången 1961. +Under tiden för andra världskriget var Larsson ordförande för Stockholms stads kristidsnämnd och var verksam som politisk opinionsbildare med en klart antinazistisk, nordisk kurs. +Han var en av de ledande i Samfundet Nordens Frihet från dess start 1939 och styrelseledamot under hela dess verksamhetsperiod till 1945. Samfundet arbetade för att effektivisera det militära och humanitära stödet åt Finland, och senare då det strävade att stimulera den svenska försvarsviljan och motverka vad samfundet uppfattade som onödiga eftergifter åt tyska krav. +Som ordförande i Svensk-norska föreningen 1942–1948 arbetade han i samma anda och verkade bland annat för hjälp till flyktingar från det ockuperade Norge. Vid det Svensk-norska frivilligförbundets första opinionsmöte pläderade Yngve Larsson i ett tal för en stor och snabb insats av Sveriges krigsmakt: "en humanitär aktion i militär form, att sättas i verket i det fall kriget återvänder till Norge, med alla de härjningar och massakrer som följer med nazistisk krigföring". Talet publicerades även i Dagens Nyheter. På fredsdagen 1945 var Yngve Larsson jämte Gustaf Aulén en av de två svenska kulturpersoner som bjöds in att uttala sig om Nordens nyvunna frihet i Svenska Dagbladet. 1946 utnämndes Larsson av kung Håkon VII av Norge till kommendör med stjärna av Sankt Olavs orden "for særlig fremragende fortjenester av Norges sak under krigen". +Efter att ha avgått som borgarråd kom Larsson att få ytterligare drygt aktiva 30 år som en "äldre statsman" inom Stockholmspolitiken och riksdagen. +Larsson var riksdagsledamot 1946–1952. I riksdagen var han bland annat suppleant i konstitutionsutskottet och ledamot av statsutskottet och första tillfälliga utskottet samt kanslideputerad. Som riksdagsman blev han även ledamot av kommunallagskommittén och år 1948 ordförande för bostadskollektiva kommittén, som publicerade sitt första betänkande 1952 och slutbetänkande 1956. +Larsson var även aktiv som ledamot i styrelserna för bland annat Dagens Nyheter (1940–1966), Expressen och Svenska stadsförbundet (1942–1951). +Parallellt med riksdagsuppdraget behöll Larsson en stark förankring i Stockholmspolitiken, och kvarstod som ledamot av Stockholms stadsfullmäktige och var ordförande för Folkpartiets stadsfullmäktigegrupp 1946–1954. Kommunalvalet 1950 blev Folkpartiets mest framgångsrika någonsin i Stockholm, och Larsson blev då som ledare för det vinnande blocket ordförande för Stockholms stadskollegium (dåvarande kommunstyrelsen) efter femton år som ledamot. 1954 avgick han från denna post och lämnade samtidigt stadsfullmäktige, nära 73 år gammal. +1955 skulle byggnadsnämnden och stadsplanenämnden slås samman, och Larsson blev dess vice ordförande fram till 1960. Han kvarstod även som ledamot och till och med 1966 som vice ordförande för Generalplaneberedningen (ursprungligen Nedre Norrmalmsdelegationen) som han själv initierat 1951. 1961 engagerade han sig som medlare i Vårbyaffären, där han även författade den försoningskommuniké till försvar för borgarrådsinstitutionen som inkorporerings-affärens båda kontrahenter Gunnar Dalgren (fp) och Hjalmar Mehr (s) ställde sig bakom. +Larsson avgick från Generalplaneberedningen 1970, då 89 år fyllda. De sista åren kom han att betraktas med respekt och man lyssnade artigt på hans utförliga invändningar mot Norrmalmsregleringens utveckling, men hans reella inflytande hade minskat. +Efter borgarrådstiden kom Yngve Larsson även att verka som skribent och föredragshållare. 1947 var Larsson en av grundarna till Föreningen för Samhällsplanering, och blev ledamot av dess styrelse såväl som av redaktionskommittén för dess tidskrift PLAN, där han även författade den inledande artikeln i premiärnumret, på temat Regionplaneringens administrativa problem. 1958 var han inbjuden talare på temat Våra framtida städer vid Studentafton på Akademiska föreningen i Lund, vid sidan av arkitektur- och konstprofessorerna Sune Lindström, Peter Bredsdorff, Nils Ahrbom och Aron Borelius. +Ett par månader efter att organisationen av stadsplaneringen i Stockholm belönats med Sir Patrick Abercrombie Prize 1961 skrev Yngve Larsson artikeln Building a city and a metropolis : the planned development of Stockholm inför en konferens i Stockholm för Förenta nationernas expertgrupp för stadsplanering. Artikeln publicerades även i Journal of the American Institute of Planners 1962, samt som en av sju artiklar i boken Stockholm: regional and city planning utgiven av stadskollegiet 1964. I mitten av decenniet var Larsson Royer Visiting Professor vid University of California, där han publicerade essän Municipal, Regional, and National Planning in Sweden i en bok författad av, förutom Larsson, den amerikanske ekonomiprofessorn och presidentrådgivaren Lauchlin Currie och den holländske ekonomiprofessorn och direktören för Centraal Planbureau(nl), Pieter de Wolff(nl). +Under 21 år, 1949–1970, var Larsson ordförande för Stadshistoriska nämnden som han hade grundat 1919 (se ovan). 1950 hade stadsfullmäktige på initiativ av Yngve Larsson och bland andra finansborgarrådet Zeth Höglund beslutat att ställa medel till förfogande till stadskollegiets handsbokskommitté för vetenskapligt forsknings- och utredningsarbete av värde för Stockholm, och 1954 uppdrog fullmäktige åt handbokskommittén att låta utarbeta ett särskilt samlingsverk över Stockholms utveckling under de senaste hundra åren. Inom denna ram kom Larsson under sina sista år på 1960- och 1970-talen att på stadens uppdrag författa tre volymer om totalt 1 250 sidor: På marsch mot demokratin och Mitt liv i Stadshuset (två volymer). Mitt liv i Stadshuset kom även att bli hans självbiografi, vilken han fullbordade ett par veckor före sin bortgång. Verket presenterades av borgarrådet Paul Grabö som ett monument över en epok i Stockholms historia. +Yngve Larsson avled den 16 december 1977, tre dygn efter att han fyllt 96 år. Jordfästningen utfördes av Olof Lagercrantz i Heliga korsets kapell på Skogskyrkogården. Yngve och hustrun Elin Larsson är även begravda där, i grav nr 1A vid Trons kapell bredvid kyrkogårdens arkitekt Gunnar Asplund. +Yngve Larsson gifte sig 4 januari 1904 med Elin Bonnier (1884–1980), dotter till bokförläggaren Karl Otto Bonnier och Lisen Bonnier. Paret hade introducerats vid en gymnasistbal år 1899 eller 1900 av hans vän och hennes bror Tor Bonnier. Paret fick tillsammans sex barn: läraren och översättaren Verna Lindberg (1905), gift med professor Folke Lindberg; SAF-direktören Matts Bergom Larsson (1908); textilkonstnären Ester Berggren (1911), gift med teologen Erik Berggren; bokhandlaren och författaren Richard Larsson (1915); Yngve A. A. Larsson (1917) professor i pediatrik samt arkitekt Mårten Larsson (1919), gift med inredningsarkitekten Lena Larsson. +Bland de 23 barnbarnen märks Martin Berggren (skådespelare, teaterregissör), Tobias Berggren (författare, poet), Maria Bergom Larsson (författare, kritiker), Jakob Lindberg (departementsråd, överdirektör) och Kristina Torsson (modeskapare, entreprenör), och bland barnbarnsbarnen Johan Berggren (chefredaktör), Jesper Bergom Larsson (filmproducent), Sara Szyber (designer), Fabian Torsson (musikproducent) och Palle Torsson (konstnär). +Familjen bodde från 1905 på Bergsgatan 4 på Kungsholmen, och lät 1907 bygga Villa Yngve Larsson i Storängen i Nacka, granne med hennes föräldrars vänner Hanna och Georg Pauli och hans bror Lennart Larsson. Från 1909 och fram till Elin Larssons bortgång 70 år senare kom familjen dock att bo i Villa Mullberget på Djurgården intill hennes föräldrars hem, Nedre Manilla, som uppfördes under samma tid. Båda Stockholmsvillorna ritades av stadshus-arkitekten Ragnar Östberg i en utpräglat nationalromantisk stil. Sommartid vistades familjen från 1918 på sommarnöjet Barnarve i Ljugarn på Gotland. +Redan 1920 kallade tidskriften Hvar 8 dag Larsson för pioniär och auktoritet utan jämförelse på kommunalförvaltningens teori och praxis. Bland hans samtida författare omnämndes Larsson av Eyvind Johnson i Krilon-trilogin 1943. +Yngve Larsson fick mot slutet av sitt liv epitetet legendarisk, och kom att kallas sitt århundrades främste stadsbyggare i Sverige av bland andra sin politiske motståndare Hjalmar Mehr. Den amerikanske stadsplaneraren Clarence Stein refererade till Larsson som statsman. Svenska Dagbladet beskrev Yngve Larsson som Stockholms stadsfullmäktiges ojämförligt störste talare. Larssons stora stadshistoriska verk och självbiografi Mitt liv i Stadshuset (1977) sågs av stadskollegiets handbokskommitté som ett monument över en epok i Stockholms historia. +Dödsrunor över Yngve Larsson skrevs i Dagens Nyheter dels av Per Wästberg, dels av Hjalmar Mehr samt i Svenska Dagbladet av Gösta Selling. Larssons biografi i Svenskt biografiskt lexikon författades av tidigare stadsbyggnadsdirektören Göran Sidenbladh. +Sedan 1994 delar Stads- och kommunhistoriska institutet ut Yngve Larssons pris för stads- och kommunhistoriska insatser, instiftat av Svenska Kommunförbundets styrelse. Under 2000-talet har den svenske urbanhistorikern Anders Gullberg kallat Yngve Larsson för 1900-talets gigant i stadshuset och stockholmspolitiken, och Tage William-Olssons biografer beskriver Larsson som kanske den mest inflytelserike politikern i Stadshuset under 1900-talet. +Efter en motion i maj 2014 i Stockholms stadsfullmäktige av dess vice ordförande Ulf Fridebäck om att få en central plats i Stockholm uppkallad efter Yngve Larsson, så beslutade stadsbyggnadsnämnden 2016 om att ge parken mellan stadshuset, Serafimerlasarettet och Klara sjö namnet Yngve Larssons park. +Yngve Larsson utmärktes med ett flertal svenska och utländska statsordnar, däribland +Nordstjärneorden 1920, +Finlands vita ros orden, som han blev kommendör av 16 augusti 1924, "bl a för att outtröttligt bistått Finlands stadsförbund och dess organ Kommunala centralbyrån i Helsingfors"), +Vasaorden, där han blev kommendör av första klassen 6 juni 1941, +norska Sankt Olavs orden, där han blev kommendör med stjärna, 1946, "for særlig fremragende fortjenester av Norges sak under krigen") och franska Hederslegionen. +Efter pensioneringen som borgarråd mottog Larsson S:t Eriksmedaljen 1947, och 1966 erhöll han Samfundet S:t Eriks plakett för berömliga insatser för Stockholm: +Samma år blev han även hedersledamot av Samfundet för hembygdsvård och av Stockholms nation i Uppsala. Han tilldelades av Svenska PEN-klubben 1972 års Bernsstipendium för stockholmsskildrare: +Stadsplaneringen i Stockholm för tiden kring och efter andra världskriget uppmärksammades internationellt, och bland annat belönades Town Planning Service of the City of Stockholm av International Union of Architects med Sir Patrick Abercrombie Prize när det delades ut för första gången 1961. +En bronsbyst över Yngve Larsson utförd av skulptören vid Skogskyrkogården John Lundqvist donerades av makarna Larssons arvingar till Stockholms stadshus, där den placerats på tredje våningen i Stockholms stadshus utanför kontoret för Stockholms Stadshus AB. I stadshuset vid Rådssalens bibliotek återfinns även bland annat ett porträtt över Yngve Larsson utfört av Nils Melander. +Porträtt över Larsson finns även utförda av Ulla Sundin-Wickman 1960 i den Bonnierska Porträttsamlingen samt ett (som medlem av Slussbyggnadskommittén) av John Hedberg i stadsmuseets samlingar. Dagens Nyheters Birger Lundquist gjorde flera teckningar av Yngve Larsson. Fotografiska porträtt utfördes av bland andra Ferdinand Flodin, Atelier Jaeger, och Jan de Meyere. +I ett parodiskt Utlåtande i anledning av förslag till ändring av mosaikmålningen å Gyllene salens norra vägg utförde Slussen-arkitekten Tage William-Olsson ett porträtt i akvarell och lavering av Yngve Larsson som Mälardrottningen i samband med Larssons 50-årsdag 1931. Titeln på boken i Larssons hand är Skam den, som tänker illa därom. +Bokverk +Larsson, Yngve (1907). Kommunalsocialistisk bostadspolitik i utlandet. Centralförbundet för socialt arbete ; 13. Stockholm. Libris 1612559 +Larsson, Yngve (1909). Kommunalförvaltningens organisation och arbetssätt i Sveriges, Englands, Frankrikes, Preussens, Österrikes, Danmarks och Norges städer. Svenska stadsförbundets skriftserie : Andra bandet. Stockholm. Libris 8221139 +Larsson, Yngve (1910). De svenska städernas kollektivavtal med hänsyn till den senaste utvecklingen av kommunalarbetarnes avtalsförhållanden: en redogörelse utarbetad inom Kommunala centralbyrån. Svenska stadsförbundets skriftserie : Tredje bandet. Stockholm. Libris 1617830 +Larsson, Yngve (1911). Önskemål rörande stadsförvaltningens organisation. Svenska stadsförbundets skriftserie : Femte bandet. Stockholm. Libris 1639584 +Larsson, Yngve (1912). Inkorporeringsproblemet. Stadsområdets förändringar med särskild hänsyn till svensk förvaltningspraxis. Svenska stadsförbundets skriftserie : Sjätte bandet. Stockholm: Nordiska Bokhandeln. Libris 380038 (PDF 1, PDF 2) (doktorsavhandling) +Larsson, Yngve (1912). Frågan om total eller partiell inkorporering av Sköns tingslag med Sundsvalls stad: Preliminär utredning enligt uppdrag. Sundsvall: Sunessons bokhandel. Libris 1636023 +Järte, Otto; Kjellén, Rudolf; Larsson, Yngve; Molin, Adrian (1915). Sveriges utrikespolitik i världskrigets belysning (utgiven anonymt). Stockholm: Nordiska Bokhandeln. Libris 8227881 +Bååth, Johan; Yngve Larsson, red (1918). Kommunalförvaltningens organisation: studier och reformförslag. Svenska stadsförbundets skriftserie Tolfte bandet. Stockholm: Svenska stadsförbundet. Libris 1505202 +Larsson, Yngve (1967). På marsch mot demokratin - Från hundragradig skala till allmän rösträtt, 1900-1920. Monografier utgivna av Stockholms stad. Stockholm: Almqvist & Wiksell. Libris 8390180. http://www.stockholmia.stockholm.se/media/pdf/22_larsson.pdf +Larsson, Yngve (1977). Mitt liv i Stadshuset – Första delen: Från fåvälde till demokratisk ordning (1900–1954). Stockholm: Stockholms kommunalförvaltning. Libris 7751621. ISBN 91-85676-00-4 (Digital version, s. 1–342) +Larsson, Yngve (1977). Mitt liv i Stadshuset – Andra delen: I tjänst hos denna stolta stad (1920–1954). Stockholm: Stockholms kommunalförvaltning. Libris 7751622. ISBN 91-85676-00-4 (Digital version, s. 343–673) +Utredningar i urval +Larsson, Yngve (1918). Kommunalförvaltningens organisation samt statsuppsikten över kommunerna i England, Danmark och Norge. Stockholm: Norstedt. Libris 907104 +Betänkande med förslag till lagstiftning angående kommunala nybildningar. Stockholm: Nordiska bokhandeln. 1918. Libris 1649668 +Sakkunnige för utredning av frågan om bostadssociala minimifordringar å med allmänt understöd tillkommande smålägenheter; Yngve Larsson (1921). Praktiska och hygieniska bostäder: betänkande och förslag. Stockholm: Kommittén angående bostadssociala minimifordringar. Libris 893471. ISBN 9909200619 +Sandberg, Harry; Yngve Larsson; Halvar G. F. Sundberg; Albert Lilienberg; David Anger; Axel Dahlberg; Gösta Hellgren (1940). Betänkande med utredning och förslag rörande tunnelbana genom staden: avgivet den 6 september 1940 av de av Stadskollegiet den 23 februari tillsatta tunnelbanedelegerade D. 1, Huvudbetänkande. Stadskollegiets utlåtanden och memorial. Bihang, 99-3047450-1 ; 1940:10A. Stockholm. Libris 3320948 +1939 års kommunlånesakkunniga; J. B. Johansson; Erik Fast; Yngve Larsson (1943). Betänkande med förslag angående grunderna för den kommunala lånepolitiken. Statens offentliga utredningar 1942:60. Stockholm. ISSN 0375-250X Libris 13537718. http://urn.kb.se/resolve?urn=urn:nbn:se:kb:sou-1428687 +Löwbeer, Nils; Carl Albert Anderson; C Brännberg; Knut Ekemar; J.A. Fagerberg; Yngve Larsson; F. Thun; Adolf Wallentheim; Alarik Wigert; Uno Åhrén; Eric Törngren (1947). Betänkande med förslag till lag om regionkommuner m. m.. Statens offentliga utredningar 1947:30. Stockholm: 1944 års kommitté för kommunal samverkan. ISSN 0375-250X Libris 13576753. http://urn.kb.se/resolve?urn=urn:nbn:se:kb:sou-1458006 +Kommunallagskommittén (1947–1952). Kommunallagskommitténs betänkanden. Statens offentliga utredningar, ISSN 0375-250X. Stockholm. Libris 2522100 +Bostadskollektiva kommittén (1952–1955). Yngve Larsson. red. Bostadskollektiva kommitténs betänkande. Statens offentliga utredningar. Stockholm. ISSN 0375-250X Libris 853722 +Artiklar i urval +Larsson, Yngve (1908). ”En matsalsmöbel”. Svenska slöjdföreningens tidskrift. Organ för konstindustri, handtverk och hemslöjd. (Stockholm). https://runeberg.org/ssf/1908/0137.html. Libris 22550190 +Larsson, Yngve (1911). ”Beslut, verkställighet och förvaltning enligt de svenska kommunallagarna”. Statsvetenskaplig tidskrift (Lund) (4). Arkiverad från originalet den 16 juni 2012. https://web.archive.org/web/20120616060455/http://nile.lub.lu.se/ojs/index.php/st. Läst 27 augusti 2014. Libris 11567523 +Larsson, Yngve (1911). ”Nyare inkorporeringsutredningar: Sundsvall och Sköfde”. Svenska stadsförbundets tidskrift 3 (3): sid. 117–120. Libris 11499516 +Larsson, Yngve (1912). ”Stadsområdets tillväxt. Några synpunkter av andre sekreteraren i Svenska Stadsförbundet, fil. lic. Yngve Larsson.”. Minnesskrift utgiven av Svenska stadsförbundets tidskrift till 50-årsdagen av förordningens om kommunalstyrelse i stad utfärdande den 21 mars 1862. Stockholm: Svenska stadsförbundet. Libris 395956 +Larsson, Yngve (1916). Grundlinjer till föredrag om bostadsfrågan i nya stadsbildningar (Kåkstadsproblemet). C.S.A:s bostadskongress 21–23 september 1916. Stockholm. Libris 2764536 +Larsson, Yngve (1916). ”Bostadsfrågan under krigskonjunktur”. Svensk tidskrift (Stockholm) 6: sid. 270–277. https://runeberg.org/svtidskr/1916/0278.html. Libris 8258426 +Larsson, Yngve (1917). ”Stadsförvaltningen på landsbygden.: Kritik och reformförslag”. Svenska stadsförbundets tidskrift (Stockholm) 9. Libris 2764539 +Larsson, Yngve (1918). ”Svenska stadsförbundet 1908–1918. En återblick”. Svenska stadsförbundets tidskrift (Stockholm) (5). Libris 2764543 +Fant, Gunnar; Larsson, Yngve (1918). P. M. angående behandlingen av den s.k. Stadsstyrelsekommissionens förslag till magistratsförvaltningens ordnande i städer under landsrätt (juli 1918). Stockholm. Libris 2993667 +Larsson, Yngve (1919). Rekonstruktionspolitiken å bostadsfrågans område i England. Stockholm. Libris 2764540 (Även i Svenska stadsförbundets tidskrift samma år) +Larsson, Yngve (1920). ”De kommunala tjänstemännens teoretiska utbildning”. Svenska stadsförbundets tidskrift (Stockholm). Libris 2764538 +Larsson, Yngve (1921). ”Den svenska stadsbudgeten och dess uppställning: anteckningar till det kommunala budgetsväsendets teknik”. Svenska stadsförbundets tidskrift (Stockholm). Libris 1519475 +Larsson, Yngve (1921). Stockholms stads nya förvaltningsorganisation : Föredrag vid Svenska stadsförbundets kongress i Norrköping den 1 september 1921. Svenska Stadsförbundet. Fjärde ordinarie kongressens förhandlingar. Bilaga. Stockholm. Libris 2764541 (Även i Svenska stadsförbundets tidskrift samma år) +Larsson, Yngve (1924). ”Statsbidragen till kommunerna ur utjämningssynpunkt. Föredrag vid Svenska stadsförbundets kongress i Hälsingborg i augusti 1924”. Stadsförbundets tidskrift: sid. 330-352. Libris 8206151 +Larsson, Yngve (1925). Felix Genzmer. red (på Tyska). Aus Schwedens Staats- und Wirtschaftsleben : Die Grundzüge der Schwedischen Gemeindeverwaltung. Berlin. sid. 68-79. http://books.google.de/books?id=EWPUAAAAMAAJ. Libris 2544131 +Larsson, Yngve (1930). Kommunala indelningsfrågor i anslutning till 1919 års indelningslagstiftning. Stockholm. Libris 2764537 (Även i Svenska stadsförbundets tidskrift samma år) +Larsson, Yngve (15 november 1935). ”Rekognosering av Amerika : Om vikten att kunna förvånas”. Dagens Nyheter. http://www.stockholmskallan.se/index.php?sokning=1&action=visaPost&start=0&mediaId=24915&post=4. +Larsson, Yngve (1941). ”Den kommunala självstyrelsen och dess betydelse i svenskt samhällsliv”. i Axel Brusewitz; Sixten Björkblom. Festskrift till professor skytteanus Axel Brusewitz. Skrifter utgivna av Statsvetenskapliga föreningen i Uppsala, ISSN 0346-7538 ; 12. Uppsala: Almqvist & Wiksell. sid. 264 ff. Libris 863329 +Larsson, Yngve (8 maj 1945). ”Uttalanden av norska och svenska kulturpersoner - avsnitt: Låt oss dela med oss till hungrande frändefolk!”. Svenska Dagbladet: s. 11. +Larsson, Yngve (1947). ”Regionplaneringens administrativa problem”. PLAN (Stockholm: Föreningen för Samhällsplanering) (1:47): sid. 3-7. ISSN 0032-0560. Libris 8258191 +Larsson, Yngve (1947). ”Tunnelbanans tillkomst”. Spårväg och buss: Stockholms spårvägars tidning (Stockholm: Stockholms spårvägar) 7 (9): sid. 137–140. ISSN 0346-1661. http://media.sparvagsmuseet.se/Skannade_texter/tub_tillk_1.pdf. Libris 3679720 +Larsson, Yngve (1948). ”Nedre Norrmalm, förflutet och kommande”. Handel och näringsliv i Stockholm. Stockholm: Stockholms köpmansklubb. sid. 79-115. Libris 8206657 +Larsson, Yngve (1948). ”Att vara liberal”. Socialliberal samhällssyn. Stockholm: Folkpartiet. sid. 5-56 (förord). Libris 1238213. http://liberologi.wordpress.com/2011/10/12/att-vara-liberal-ur-boken-socialliberal-samhallssyn-folkpartiet-1948/ +Anderson, Carl Albert; Grandien, Alf; Johansson, Erik; Larsson, Yngve (1954). Anföranden i Stockholms stadsfullmäktige den 11 oktober 1954 samt i stadskollegiet den 7 oktober 1954 (Särtryck ur stadsfullmäktiges protokoll och yttranden, nr 13 1954). Stockholm. Libris 8207447. http://webbsok.mikromarc.se/Mikromarc3/web/detail.aspx?Id=27825&Unit=6471&db=ssa +Larsson, Yngve (1960). Nedre Norrmalm - Historiskt och ohistoriskt. Utvidgat särtryck ur Samfundet S:t Eriks årsbok. Stockholm: Norstedt. Libris 1794231 +Larsson, Yngve (1962) (på engelska). Building a city and a metropolis : the planned development of Stockholm. "28". USA: Journal of the American Institute of Planners. sid. 220-228. http://www.informaworld.com/smpp/content~db=all~content=a790515073. Libris 8847178 +Larsson, Yngve (1964). ”Building a city and a metropolis : the planned development of Stockholm”. i Garpe, Joakim (på Engelska). Stockholm: regional and city planning : seven articles on Planning problems in Greater Stockholm. Stockholm: Stadskollegiets reklamkommitté. Libris 1793683 +Larsson, Yngve; Pieter de Wolff; Lauchlin Currie (1967) (på Engelska). Governmental planning and political economy. Royer Lectures. Berkeley, Kalifornien: Institute of Business and Economic Research, Institute of Governmental Studies, University of California. Libris 8852548 +Larsson, Yngve (1969). ”Stockholms parker och parkmästare”. Arkitektur (7). +Larsson, Yngve (1971-1973). ”Otto Magnus Höglund”. i Grill Erik. Svenskt biografiskt lexikon. Bd 19, Heurlin-Inge. Stockholm: Svenskt biografiskt lexikon. Libris 9322373 +Johansson, Bengt O. H; John Sjöström; Ingela Lind; Yngve Larsson (1972). Bostadskris och mönsterbostad: illusion och verklighet : en rekonstruktion av Gunnar Asplunds bostadskök på Hemutställningen i Liljevalchs 1917. Stockholm: Sveriges arkitekturmuseum. sid. 19-22. Libris 9811200 +Stadsplanering i Stockholm +Bonnier (släkt) +Anderson, Carl Albert; Grandien, Alf; Johansson, Erik; Larsson, Yngve (1954). Anföranden i Stockholms stadsfullmäktige den 11 oktober 1954 samt i stadskollegiet den 7 oktober 1954 (Särtryck ur stadsfullmäktiges protokoll och yttranden, nr 13 1954). Stockholm. Libris 8207447. http://webbsok.mikromarc.se/Mikromarc3/web/detail.aspx?Id=27825&Unit=6471&db=ssa +Anderson, Ivar (1967). Klubben Brunkeberg 1917-1967. Stockholm: Norstedt. Libris 7944 +Bengtzon, Olle (1963). ”Yngve Larsson – stadsbyggare”. Arkitektur (11): sid. 308. +Bergelin, S.-E. S. (red.) (1953). ”Larsson, Yngve Gustaf Rikard”. Nordisk familjebok: encyklopedi och konversationslexikon 1951-1955 (4). Malmö: Förlagshuset Norden. Libris 539090 +Bohman, Gösta (1974). Tankar om mitt Sverige. Partiledaren har ordet, 99-0344177-0 ; 2. Stockholm: Askild & Kärnekull. sid. 32–33. Libris 7589397. ISBN 91-7008-268-5. https://archive.org/details/tankarommittsver0000bohm +Bohman, Gösta; Poul Ströyer (1983). Så var det: Gösta Bohman berättar. Stockholm: Bonnier. sid. 14. Libris 7146753. ISBN 91-0-045956-9 +Bonnier, Tor (1972). Längesen: sammanklippta minnesbilder. Stockholm: Bonnier. Libris 7143878. ISBN 91-0-038071-7 +Byström, Tora (2009). Nordens frihet: samfundet, tidningen, kretsen. Lund: Sekel Bokförlag. Libris 11583186. ISBN 978-91-85767-47-2. http://urn.kb.se/resolve?urn=urn:nbn:se:umu:diva-25758 +Carlestam, Gösta (2007). ”I krigsslutets skugga”. PLAN (Stockholm: Föreningen för Samhällsplanering) (1:2007): sid. 9–16. ISSN 0032-0560. Libris 8258191 +Carlquist, Gunnar, red (1947–1955). ”Larsson, Yngve Gustaf Richard, kommunalman, politiker”. Svensk uppslagsbok (2:a omarbetade och utvidgade upplagan). Malmö: Baltiska förlaget. Libris 11112. Arkiverad från originalet den 9 november 2014. https://web.archive.org/web/20141109120920/http://svenskuppslagsbok.se/62237/larsson-yngve-gustaf-richard-kommunalman-politiker/. Läst 10 maj 2010 +Dalén, Uno (redaktör) (1961–1967). ”Band 8, Yngve Larsson”. Bonniers lexikon. Stockholm: AB Nordiska Uppslagsböcker. sid. 1214. Libris 8198071 +Fälting, Lars (1995). Högtflygande planer i debatten om Arlanda 1946. Working papers in transport and communication history, 1104-6988 ; 1995:4. Umeå: Research Group "Transports and Communications in Perspective", Depts. of Economic History, Umeå Univ. and Uppsala Univ. Libris 2059005. http://urn.kb.se/resolve?urn=urn:nbn:se:uu:diva-2353 +Gedin, Per (2003). Litteraturens örtagårdsmästare. Karl Otto Bonnier och hans tid. Stockholm: Albert Bonniers förlag. ISBN 91-0-010076-5 +Gullberg, Anders (1998). Karl-Erik Synnemar. red. Stockholm blir stor stad: tiden 1948-1998. Stockholm: Byggförlaget. Libris 7678815. ISBN 91-7988-139-4. http://books.google.se/books?id=PxKgAAAAMAAJ&q=Stockholm+blir+stor+stad:+yngve+larsson&hl=sv&sa=X&ei=Oa7EUr2HE8Tl4gSSrICADg&ved=0CE4Q6AEwAA +Gullberg, Anders (2001). City – drömmen om ett nytt hjärta. Moderniseringen av det centrala Stockholm 1951–1979 (två volymer). Monografier utgivna av Stockholms stad. Stockholm: Stockholmia förlag. Libris 8369513. ISBN 91-7031-078-5 +Harnesk, Paul (redaktör) (1962). Vem är vem : Stor-Stockholm (Andra upplagan). Stockholm: Bokförlaget Vem är vem. sid. 757. Libris 53509. https://runeberg.org/vemarvem/sthlm62/0781.html +Herlitz, Nils (1925). Larsson, Yngve Gustaf Rikard i Nordisk familjebok (andra upplagans supplement, 1925) +Hedenmo, Martin; von Platen, Fredrik (2007). Bostadspolitiken - Svensk politik för boende, planering och byggande under 130 år. Karlskrona: Boverket. ISBN 978-91-85751-44-0 Libris 10666847. https://www.boverket.se/globalassets/publikationer/dokument/2007/bostadspolitiken.pdf +Humble, Sixten (1958). Svenska stadsförbundet 1908-1958. Stockholm: P.A. Norstedt & Söner. Libris 8206179 +”Yngve Larsson”. Hvar 8 dag (Göteborg: D. F. Bonniers boktryckeri A.-B.) 22 (3): sid. 33-34. 1920. Libris 786775 +”Mannen för dagen i Stockholms kommunalliv : borgarrådet Yngve Larsson”. Hvar 8 dag (Göteborg: D. F. Bonniers boktryckeri A.-B.) 33 (13): sid. 193-195. 1931. Libris 786775 +Höglund, Zeth (1953). Minnen i fackelsken. 2, Från Branting till Lenin : 1912–1916. Stockholm: Tiden. Libris 464280 +Hölcke, Olov (1952). Varutrafiken över Stockholms hamn mot bakgrund av hela varuströmmen till och från Stor-Stockholm. Meddelande ; 37. Stockholm: Företagsekonomiska forskningsinstitutet vid Handelshögskolan i Stockholm. sid. III. Libris 12755072. http://urn.kb.se/resolve?urn=urn:nbn:se:su:diva-75196 +Johansson, Anders (2005). Den glömda armén: Norge-Sverige 1939-1945. Rimbo: Fischer & Co. sid. 115, 359, 505. Libris 9960232. ISBN 91-85183-20-2. http://books.google.se/books?hl=sv&id=f0DzAAAAMAAJ&focus=searchwithinvolume&q=yngve+larsson +Johnson, Anders (2014). ”Yngve Larssons långa liv i stadshuset”. Liberal Debatt (4). http://www.liberaldebatt.se/2014/08/yngve-larssons-langa-liv-i-stadshuset/. +Jonsson, Daniel; Gullberg, Anders; Jungmar, Marie; Kaijser, Arne; Steen, Peter (2000). Infrasystemens dynamik : om sociotekniska förändringsprocesser och hållbar utveckling. Fms-rapport, 1404-6520 ; 2000:1. Stockholm: Forskningsgruppen för miljöstrategiska studier (FMS) vid KTH. Libris 7596897. ISBN 91-7056-102-8. https://www.yumpu.com/sv/document/read/26297672/infrasystemens-dynamik-kungliga-tekniska-hagskolan +Järte, Otto; Kjellén, Rudolf; Larsson, Yngve; Molin, Adrian (1915). Sveriges utrikespolitik i världskrigets belysning (utgiven anonymt). Stockholm: Nordiska Bokhandeln. Libris 8227881 +Kihlberg, Mats (1961). ”Aktivismens huvudorgan Svensk Lösen”. Två studier i svensk konservatism 1916-1922. Skrifter utgivna av Statsvetenskapliga föreningen i Uppsala ; 41. Stockholm: Almqvist & Wiksell (distributör). ISSN 0346-7538 Libris 8075116 +Kihlberg, Johan (2012). Vällingby och kulturarvet : att bevara och förnya ett centrum. TRITA-ARK-Forskningspublikationer, 2012:2. Stockholm: Arkitektur och samhällsbyggnad, KTH. sid. 18-20. ISSN 1402-7453 Libris 13285091. ISBN 978-91-7501-318-3. http://urn.kb.se/resolve?urn=urn:nbn:se:kth:diva-93484 +Kubu, Mert; Erikson, Hasse (1968). Det politiska spelet i Stadshuset. Stockholm: Almqvist & Wicksell. sid. 156 ff. Libris 1205997 +Larsson, Peter B. (28 februari 2009). ”Stockholm behöver Ny syn på Slussen”. Fria Tidningar. Arkiverad från originalet den 7 december 2013. https://web.archive.org/web/20131207020132/http://www.stockholmsfria.se/artikel/78190. Läst 3 december 2013. +Larsson, Lennart, Jr. (1993). Ett brokigt liv: som köpman i Stockholm - Budapest - Hongkong. Stockholm: Stenström. Libris 7759287. ISBN 91-86600-83-4 +Larsson, Yngve (1907). Kommunalsocialistisk bostadspolitik i utlandet. Centralförbundet för socialt arbete ; 13. Stockholm. Libris 1612559 +Larsson, Yngve (1912). Inkorporeringsproblemet. Stadsområdets förändringar med särskild hänsyn till svensk förvaltningspraxis. Svenska stadsförbundets skriftserie : Sjätte bandet. Stockholm: Nordiska Bokhandeln. Libris 380038 (PDF 1, PDF 2) (doktorsavhandling) +Larsson, Yngve (15 november 1935). ”Rekognosering av Amerika : Om vikten att kunna förvånas”. Dagens Nyheter. http://www.stockholmskallan.se/index.php?sokning=1&action=visaPost&start=0&mediaId=24915&post=4. +Larsson, Yngve (8 maj 1945). ”Uttalanden av norska och svenska kulturpersoner - avsnitt: Låt oss dela med oss till hungrande frändefolk!”. Svenska Dagbladet: s. 11. +Larsson, Yngve (1947). ”Regionplaneringens administrativa problem”. PLAN (Stockholm: Föreningen för Samhällsplanering) (1:47): sid. 3-7. ISSN 0032-0560. Libris 8258191 +Larsson, Yngve (1960). Nedre Norrmalm - Historiskt och ohistoriskt. Utvidgat särtryck ur Samfundet S:t Eriks årsbok. Stockholm: Norstedt. Libris 1794231 +Larsson, Yngve (1962) (på engelska). Building a city and a metropolis : the planned development of Stockholm. "28". USA: Journal of the American Institute of Planners. sid. 220-228. http://www.informaworld.com/smpp/content~db=all~content=a790515073. Libris 8847178, även publicerad i Larsson, Yngve; Göran Sidenbladh; m fl (1964). Garpe, Joakim. red (på Engelska). Stockholm: regional and city planning : seven articles on Planning problems in Greater Stockholm. Stockholm: Stadskollegiets reklamkommitté. Libris 1793683 +Larsson, Yngve; Pieter de Wolff; Lauchlin Currie (1967) (på Engelska). Governmental planning and political economy. Royer Lectures. Berkeley, Kalifornien: Institute of Business and Economic Research, Institute of Governmental Studies, University of California. Libris 8852548 +Larsson, Yngve (1967). På marsch mot demokratin - Från hundragradig skala till allmän rösträtt, 1900-1920. Monografier utgivna av Stockholms stad. Stockholm: Almqvist & Wiksell. Libris 8390180. http://www.stockholmia.stockholm.se/media/pdf/22_larsson.pdf +Larsson, Yngve (1969). ”Stockholms parker och parkmästare”. Arkitektur (7). +Larsson, Yngve (1977). Mitt liv i Stadshuset – Första delen: Från fåvälde till demokratisk ordning (1900–1954). Stockholm: Stockholms kommunalförvaltning. Libris 7751621. ISBN 91-85676-00-4 (Digital version, s. 1–342) +Larsson, Yngve (1977). Mitt liv i Stadshuset – Andra delen: I tjänst hos denna stolta stad (1920–1954). Stockholm: Stockholms kommunalförvaltning. Libris 7751622. ISBN 91-85676-00-4 (Digital version, s. 343–673) +”Yngve Larssons arkiv, förteckningsnummer SE/SSA/1233”. Stockholms stadsarkiv. http://www.nad.ra.se/archive_index.aspx?id=c9b2b9b9-929d-407f-9932-63901e1dc67d&s=Balder. Läst 14 maj 2009. +Larsson, Yngve A. A. (2008). Vändpunkter – Minnen, möten, uppbrott. Bonniers Familjestiftelse. Libris 11255465 +Lindhagen, Jan (1972). Socialdemokratins program. Del 1: I rörelsens tid 1890–1930. Stockholm: Tiden i samarbete med Stockholms arbetarekommun. Libris 199925. ISBN 91-550-1553-0 +Lindkvist, Anna (2007). Jorden åt folket: Nationalföreningen mot emigrationen 1907–1925. Skrifter från institutionen för historiska studier, Umeå universitet, 1651-0046 ; 19. Umeå: Institutionen för historiska studier, Umeå universitet. sid. 72. Libris 10560948. ISBN 978-91-7264-387-1. http://urn.kb.se/resolve?urn=urn:nbn:se:umu:diva-1385 +Lindstenz, Bernt (2008). Samspel: regionalt och mellankommunalt samarbete i Stockholms län under ett drygt halvsekel. Monografier utgivna av Stockholms stad. Stockholm: Stockholmia. Libris 10670306. ISBN 978-91-7031-198-7. http://www.ksl.se/images/User/Samspel.pdf +Lundborg, Gösta; William-Olsson, Tage; Karlsson, Erik (1931). Utlåtande i anledning av förslag till ändring av mosaikmålningen å Gyllene salens norra vägg (Parodi). Stadskollegiets utlåtanden och memorial 1931 : Nr -. Stockholm. Libris 2879749 +Löwbeer, Nils; Carl Albert Anderson; C Brännberg; Knut Ekemar; J.A. Fagerberg; Yngve Larsson; F. Thun; Adolf Wallentheim; Alarik Wigert; Uno Åhrén; Eric Törngren (1947). Betänkande med förslag till lag om regionkommuner m. m.. Statens offentliga utredningar 1947:30. Stockholm: 1944 års kommitté för kommunal samverkan. ISSN 0375-250X Libris 13576753. http://urn.kb.se/resolve?urn=urn:nbn:se:kb:sou-1458006 +Martensen-Larsen, Fr. (1914) (på Danska). Boganmeldelser : YNGVE LARSSON: Inkorporeringsproblemet. Stockholm, Aktiebolaget Nordiska Bokhandeln. 1912.. Bind 3. række, 22. https://tidsskrift.dk/nationaloekonomisktidsskrift/article/download/59211/81723/130823. Läst 27 augusti 2014. +Markelius, Sven (m fl) (1945). Det framtida Stockholm : riktlinjer för Stockholms generalplan. Stockholm: Byggmästarens förlag. Libris 3285232 +Mehr, Hjalmar (17 december 1977). ”Yngve Larsson död : Århundradets främste stadsbyggare i Sverige”. Dagens Nyheter: s. 34. +Moberg, Helge (red.) (1967). ”Yngve Larsson”. Stockholms stadsfullmäktige 1938–1963: biografiska data och porträtt. Stockholm: Stadskansliet. sid. 24. Libris 1256658. Arkiverad från originalet den 9 november 2014. https://web.archive.org/web/20141109123532/http://www.stockholmskallan.se/ContentFiles/SSA/Biblioteket/Fullmaktigemedlemmar/SE_SSA_BIBL_0118_0029_0235_0024_Larsson_Y.jpg. Läst 21 mars 2013 Arkiverad 9 november 2014 hämtat från the Wayback Machine. +Nilsson, Torbjörn (2013). Stockholm som motor och motpart: stadsbild och statsmakt sedan 1945. Monografier utgivna av Stockholms stad ; 235. Stockholm: Stockholmia förlag. ISSN 0282-5899 Libris 13552195. ISBN 978-91-7031-260-1. https://stockholmskallan.stockholm.se/post/29425 +Oser, Jacob (1967). ”Reviewed work(s): Governmental Planning and Political Economy by Yngve Larsson; Pieter deWolff; Lauchlin Currie”. Journal of Economic Issues 1 (3): sid. 244–246. http://www.jstor.org/pss/4223854. +Pass, David (1969) (på Engelska). Vällingby and Farsta - from idea to reality: the suburban development process in a large Swedish city. Stockholm: Statens institut för byggnadsforskning. sid. 192 ff. Libris 7790. http://books.google.se/books?id=WNYvAQAAIAAJ +Pihl Atmer, Ann Katrin; Tham, Jan (2001). Sommarnöjen vid vattnet. Stockholm: Albert Bonniers Förlag. ISBN 91-0-056984-4 +Pihl Atmer, Ann Katrin (2011). Stockholms stadshus och arkitekten Ragnar Östberg: drömmen och verkligheten. Stockholm: Natur & Kultur. Libris 12132364. ISBN 978-91-27-13064-7 +Rilke, Rainer Maria; Nádherny von Borutin, Sidonie (2007). Storck, Joachim W.. red. Briefwechsel 1906-1926. Göttingen. sid. 473. ISBN 3-89244-983-X. http://books.google.se/books?id=k8cGNbMjOykC&pg=PA473&lpg=PA473&dq=Friedrich+Stieve+geb.+1884&source=bl&ots=pUCIlZCTze&sig=I2GLgCMdSMaA9Q5ExOYuCURU3yg&hl=sv&ei=EymxS9mrK46F-Qap7OyZAg&sa=X&oi=book_result&ct=result&resnum=2&ved=0CAgQ6AEwAQ#v=onepage&q=Friedrich%20Stieve%20geb.%201884&f=false +Rudberg, Eva (red.); Bergman, Bo; Caldenby, Claes; Gullberg, Anders (2004). Tage William-Olsson: stridbar planerare och visionär arkitekt. Monografier utgivna av Stockholms stad. Stockholm: Stockholmia förlag. Libris 9552992. ISBN 91-7031-138-2 +Sanz, Michael (1996). Bromma flygplats: flyg, folk och händelser 1936-1996. Stockholm: Allt om Hobby. sid. 27 ff.. Libris 7749587. ISBN 91-85496-87-1 +Schönning, Klas (2000). Hökarängen. Stockholm: Stockholmshem. sid. 27. Libris 3130667. https://stockholmskallan.stockholm.se/post/9838 +Sidenbladh, Göran (1978). ”G R Yngve Larsson”. Svenskt biografiskt lexikon (SBL), häfte 108. Stockholm: Statens arkivstyrelse. sid. 333-337. https://sok.riksarkivet.se/sbl/Presentation.aspx?id=11054 +Sidenbladh, Göran (1979). ”Yngve Larsson: Mitt liv i Stadshuset”. Sankt Eriks årsbok 1979. Stockholm: Samfundet S:t Erik. sid. 145-150. Libris 2331700 +Sidenbladh, Göran (1981). Planering för Stockholm 1923–1958. Monografier utgivna av Stockholms stad ; 22:V:3. Stockholm: Liber. ISBN 91-38-05335-7 Libris 7259972 (PDF 1 PDF 2) +Skårfors, Rikard (1999). Beslutsfattandets dilemma: planarbete och opinionsyttringar rörande trafikleder i Stockholm 1945-1975. Working papers in transport and communication history, 1104-6988 ; 1999:2. Uppsala: Ekonomisk-historiska institutionen, Uppsala universitet (licentiatuppsats). Libris 2838557. http://urn.kb.se/resolve?urn=urn:nbn:se:uu:diva-2450 +Stavenow-Hidemark, Elisabet (1971). Villabebyggelse i Sverige 1900–1925. Nordiska museets handlingar. Uppsala: Nordiska museet. ISSN 0346-8585 +Stein, Clarence (1998). Kermit C. Parsons. red (på Engelska). The Writings of Clarence S. Stein: Architect of the Planned Community. Baltimore, Maryland: Johns Hopkins University Press. ISBN 0-8018-5756-2 +Sveriges dödbok 1947-2003 Swedish death index 1947-2003 (CD-ROM) (Version 3.0). Sundbyberg: Sveriges släktforskarförbund. 2005. Libris 9854744 +Sveriges statskalender. Stockholm: Fritzes offentliga publikationer. 1945. sid. avsnitt 501, 503, 743, 950, 962, 1102. ISSN 0347-7223 Libris 8261599 +Thomasson, Carl-Gustaf (1962). ”Gustaf Steffen. En säregen gestalt i svensk lärdomshistoria”. Statsvetenskaplig tidskrift (4): sid. 279–307. http://journals.lub.lu.se/index.php/st/article/view/2793. Libris 11567523 +Thulin, Carl (1932). ”Regleringen av slussområdet i Stockholm”. Teknisk Tidskrift, avdelningen för Väg- och vattenbyggnadskonst (11). https://runeberg.org/tektid/1932v/0127.html. +Thörn, Kerstin (1997). En bostad för hemmet : idéhistoriska studier i bostadsfrågan 1889-1929. Idéhistoriska skrifter ; 20. Umeå: Umeå universitet. ISSN 0282-7646 Libris 13577121. http://www.diva-portal.org/smash/record.jsf?searchId=1&pid=diva2:639738 +Sandberg, Harry; Yngve Larsson; Halvar G. F. Sundberg; Albert Lilienberg; David Anger; Axel Dahlberg; Gösta Hellgren (1940). Betänkande med utredning och förslag rörande tunnelbana genom staden: avgivet den 6 september 1940 av de av Stadskollegiet den 23 februari tillsatta tunnelbanedelegerade D. 1, Huvudbetänkande. Stadskollegiets utlåtanden och memorial. Bihang, 99-3047450-1 ; 1940:10A. Stockholm. Libris 3320948 +Wästberg, Per (1973). ”Till Yngve Larsson en dag i Röda Rummet.”. Arkitektur (3): sid. 28 f. +Wästberg, Per (17 december 1977). ”Yngve Larsson död : Dynamit och dynamik i lagom dosering”. Dagens Nyheter: s. 34. +Östberg, Kjell (1990). Liberalerna och arbetarrörelsen. Stockholm: Tidskriften Arkiv. http://arkiv.rodarummet.org/2013/08/24/liberalerna-och-arbetarrorelsen/ +Andrén, Mats (2007). ”Gemenskapens värme och effektivitetens kyla : kommunidéer för hundra år sedan”. Humanistdag-boken. Göteborg: Humanistiska fakultetsnämnden, Göteborgs universitet. sid. 9-14. ISSN 1400-4496 Libris 4342319. ISBN 978-91-7360-355-3. https://gupea.ub.gu.se/handle/2077/10320 +Andrén, Mats (2007). Den europeiska blicken och det lokala självstyrets värden. Hedemora: Gidlund. Libris 10415974. ISBN 9789178447381. http://www.historisktidskrift.se/fulltext/2009-1/pdf/HT_2009_1_089-151_kortrecensioner.pdf +Anton, Thomas J. (1975) (på engelska). Governing Greater Stockholm: A Study of Policy Development and System Change. Lane studies in regional government,99-0104463-4 ;3. Berkeley, Kalifornien: University of California Press. Libris 5003728. ISBN 0-520-02718-3. http://books.google.se/books?id=10o4HxD0AgEC +Deland, Mats (2001) (på Engelska). The Social City : Middle-way approaches to housing and sub-urban golvernmentality in southern Stockholm, 1900-1945. Studier i stads- och kommunhistoria, 0284-1193 ; 23. Stockholm: Stads- och kommunhistoriska institutet. Libris 7776235. ISBN 91-88882-17-9. http://su.diva-portal.org/smash/record.jsf?pid=diva2:189679 +Forsell, Håkan (2003). Hus och hyra: fastighetsägande och stadstillväxt i Berlin och Stockholm 1860-1920. Studier i stads- och kommunhistoria, 0284-1193 ; 25. Stockholm: Stads- och kommunhistoriska institutet. Libris 8868017. ISBN 91-88882-20-9. https://pubs.sub.su.se/5867.pdf +Gärde, Erland (1948). Yngve Larsson. Svenska män och kvinnor, band 4. Stockholm: Bonnier. sid. 484 f. Libris 53803 +Hofsten, Erland (1943). Utredning angående stadsplanen för Gubbängen. Stadskollegiets utlåtanden och memorial. Bihang, 99-3047450-1 ; 1943:88. Stockholm. Libris 1677577 +Isaksson, Pär (2022). Tunnelbanemiraklet : projektet som byggde den moderna staden. Stockholm: Timbro förlag. Libris xcbrdmr0v76ccwwq. ISBN 9789177033134 +Johnson, Anders (2014). Alla tiders Stockholmsliberaler : från Adolf Hedin till Lotta Edholm. Stockholm: Folkpartiets stockholmsförening. sid. 104, 117, 118 ff, m fl. ISBN 978-91-637-5402-9 +Jörnmark, Jan (2018). Norrmalmsregleringen. Stockholm: Fastighetsägarna Stockholm och Vasakronan +Larsson, Yngve: Se bibliografi ovan. +Lewin, Leif (1968). ”Litteraturgranskning: Yngve Larsson: På marsch mot demokratin. Från hundragradig skala till allmän rösträtt.”. Statsvetenskaplig tidskrift. Lund: Fahlbeckska stiftelsen. sid. 195–198. Libris 8258399 (PDF) +Sidenbladh, Göran (1985). Norrmalm förnyat 1951-1981. Stockholmsmonografier ; 66. Stockholm: Stockholmia förlag. ISSN 0282-5899 Libris 7754399. ISBN 91-86050-13-3 +Sundin, Staffan (1996). Från bokförlag till mediekoncern: huset Bonnier 1909-1929. Meddelanden från Ekonomisk-historiska institutionen vid Göteborgs universitet, 0072-5080 ; 70. Göteborg: Ekonomisk-historiska institutionen, Göteborgs universitet. Libris 7746331. ISBN 91-85196-41-X +Sundin, Staffan (2002). Konsolidering och expansion 1930-1954: Bonniers - en mediefamilj. Stockholm: Bonnier. Libris 8402567. ISBN 9100575429 +Söderholm, Sven, red (1941). Till borgarrådet Fil. Dr Yngve Larsson : Denna bok som i bild berättar om edra insatser för det nya Stockholm tillägnas eder på sextoårsdgen den 13 december 1941. Stockholm: Albert Bonniers förlag. Libris 1410049 +Theorell, Anita (2021). Axel Törneman 1880–1925: Pionjär i brytningstid. Stockholm: Bokförlaget Stolpe. Libris dqtdrp07bk3c5wjg. ISBN 9789189069091 +Mia Tottmar (2020). ”Modigt beslut om tunnelbanan under kriget”. Dagens Nyheter. https://www.dn.se/sthlm/modigt-beslut-om-tunnelbanan-under-kriget/. +Wästberg, Per; Hasse Erikson (1982). Tal i Röda rummet. Höjerings bokförlags skriftserie Stockholmiana ; 20. Stockholm: Höjering. Libris 7790596. ISBN 9197016144 +”1900–1920”. Makten över kommunerna forskning om självstyrelse, hierarkier och nätverk. Stockholm: Sveriges kommuner och landsting. 2008. sid. 61–63. Libris 11331276. ISBN 978-91-7164-321-6. http://webbutik.skl.se/bilder/artiklar/pdf/7164-321-6.pdf +Yngve Larsson : Mullberget, Djurgården, Stockholm : den 13 december 1951. Stockholm: Bonniers. 1952. Libris 3005871 (Hyllning vid 70-årsdagen) +Wikimedia Commons har media som rör Yngve Larsson. +Stockholmskällan har media som rör Yngve Larsson +Yngve Larsson i Libris +Borgarrådet Yngve Larsson på Facebook +Video +Gustaf V; Larsson, Yngve; m. fl. (15 oktober 1935). Från tradiselände till funkisfröjd : Invigningen av Slussområdet i Stockholm har skett i glädjens alla tonarter. [SF-journal]. Stockholm: Svensk filmindustri. http://svtplay.se/v/1370229/oppet_arkiv/invigning_av_slussen?sb,k103004,2,f,103033. Läst 18 september 2009 Arkiverad 5 november 2011 hämtat från the Wayback Machine. +Sundgren, Nils Petter och Björn Nilsson intervjuar Yngve Larsson (17 oktober 1976). Återblick. [TV-program]. Villa Mullberget, Stockholm: Sveriges Television. https://www.youtube.com/watch?v=f3MYiNVC9UQ. Läst 18 september 2009 +Edholm, Lotta; Hayen, Mats (19 mars 2013). Lotta Edholm i tisdagssamtal om Stockholmspolitikens historia. [Vimeo]. Stockholm: Stockholms stad. http://vimeo.com/user15455680/review/62180465/dabf880bd1. Läst 20 mars 2013 Arkiverad 9 november 2014 hämtat från the Wayback Machine. +Stockholms regeringskvarter, ibland under benämningen Departementsstaden, finns på Nedre Norrmalm i centrala Stockholm. Området ligger samlat kring Fredsgatan och Drottninggatans södra del och består av tio kvarter: Rosenbad, Lejonet, Vinstocken, Johannes större, Tigern, Röda bodarne, Brunkhuvudet, Brunkhalsen, Björnen och Loen. Drottninggatan delar Departementsstaden i en västlig och en östlig sida. +De första fastigheterna i Departementsstaden förvärvades 1906 (Arvfurstens palats) och den sista år 2019 (Adelswärdska huset), båda i kvarteret Lejonet. I kvarterens fastigheter har Regeringskansliet lokaler för sina olika departement. Flera byggnader har mycket höga kulturhistoriska värden – de flesta är blåmärkta av Stadsmuseet i Stockholm vilket innebär "att bebyggelsen bedöms ha synnerligen höga kulturhistoriska värden". Några är skyddade som byggnadsminnen. Byggnadsbeståndets ålder sträcker sig från 1640-talet, då kvarteren bildades, fram till våra dagars ombyggnationer. +Bland kända byggnader i kvarteren noteras Arvfurstens palats och Sagerska huset i kvarteret Lejonet samt Rosenbadkomplexet i kvarteret Rosenbad. Fastigheterna ägs av svenska staten genom Statens fastighetsverk. Tillkomsten av Departementsstaden, som tog omkring 30 år att genomföra, illustrerar även en ändrad inställning till hur stadsförnyelse skulle bedrivas: från total rivning av byggnadsbeståndet och nybyggnation, till bevarande och varsam upprustning. År 2020 fanns här omkring 4 000 arbetsplatser för Regeringskansliets medarbetare. +Mellan 1602 och 1635 var Norre Förstaden (Norrmalm) praktiskt taget en självständig stad, som snabbt ökade i både omfång och folkmängd. Anledningen var att Sveriges stormaktstid inletts under Gustav II Adolfs regering med stor inflyttning till huvudstaden som följd. Under åren 1635 till 1685 fyrdubblades Stockholms befolkning till cirka 60 000 individer. Det skulle bli den största relativa befolkningsökningen i staden genom tiderna. +Regleringen på Norrmalm tog sin början under 1630-talet när överståthållaren Klas Fleming gav generalkvartermästaren Olof Hansson Örnehufvud i uppdrag att lägga ut en stram rutnätsplan över området. Idéerna kom från renässansen, det skulle vara regelbundna och rätvinkliga rutnät av gator och kvarter. En regleringskarta från 1636 eller 1637 finns bevarad som visar de första utstakade kvarteren norr om Klara kyrka. Innan dess fanns här ett vildvuxet system av gator, gränder och oregelbundna kvarter. Vid den tiden existerade bara ett knappt tiotal stenhus, koncentrerade till södra delen av Norrmalm, resten var trähus respektive korsvirkeshus. +Större gator var östra och västra Sträckegatan som gick krokiga och backiga öster och väster om Brunkebergsåsen. Under 1640-talets sista år arbetade man med gatunätet söder om Klara kyrka omkring Malmtorget (nuvarande Gustav Adolfs torg). Fredsgatan stensattes 1648 (året då trettioåriga kriget tog slut) och då försvann även den sista resten av Sträckegatan. I trakten kring kvarteret Björnen hade flera krukmakare sina gårdar och verkstäder, som arkeologiska utgrävningar från 1978 kunde visa. Allt revs eller flyttades när den nya "regulariteten" trädde i kraft. +De tidiga byggherrarnas befattningar och yrken visar att de attraktiva tomterna mittemot Slottet Tre Kronor förvärvades inte bara av personer ur högadeln och av höga statliga tjänstemän eller militärer, utan även av välbärgade hantverkare. Ett av de första husen som uppfördes i de nya kvarteren var Torstensonska palatset som började byggas 1640 för fältmarskalken Lennart Torstensons räkning. Han lät resa sitt palats i nordöstra hörnet av det nybildade kvarteret Leoparden (nuvarande Lejonet). Delar av palatset är bevarade i Arvfurstens palats i form av en flygelbyggnad mot Fredsgatan. Där finns fortfarande palatsets påkostade sandstensportal. +Lennart Torstenson fick aldrig flytta in i sitt palats. När han avled 1651 var bygget fortfarande i full gång. Tidiga grannar till honom blev rådmannen Henrik Ulfvenklou (Wulff), vilken 1644 lät uppföra ett stenhus direkt väster om Torstensons palats. Fältmarskalken Gustaf Horn byggde 1642–1648 sitt palats i kvarteret Johannes större, mittemot det Torstensonska. I samma kvarter lät stadsingenjören Anders Torstensson uppföra sitt stenhus 1640. Det var han som stod för det praktiska arbetet med de nya stadsplanerna för Norrmalm och Södermalm. Vid mitten av 1660-talet var kvarteret Tigern bebyggt med palats för högadeln. +Två hantverkare etablerade sig 1640 och 1641 med hus och verkstäder i kvarteret Johannes större: hovremsnidaren Simon Juterbock respektive sadelmakaren Hans Herman. Efter dem uppkallas de båda byggnaderna som tillsammans går under namnet Sadelmakarens hus. Intill byggde samtidigt kanngjutarmästaren Göran Jeske det så kallade Kanngjutarmästarens hus. Årtalet 1640 syns på fasaden i form av till siffror utbildade ankarslut. Byggnaden är idag ett av Stockholms bäst bevarade 1600-talshus utanför Gamla stan. Ytterligare en hantverkare flyttade till området, sadelmakaren och remsnidaren Gudmund Dyk som 1646–1652 lät i det då nybildade kvarteret Brunkhuvudet uppföra sin verkstad och bostad. Byggnaden kallas idag Mäster Dyks hus. +När Lennart Torstenson började bygga sitt palats gick Norrmalms strandlinje mot Klara sjö direkt väster om kvarteren, ungefär i nuvarande Rödbodgatans sträckning. På en dansk 1640-talskarta över Stockholm visande Norrmalm syns de nybildade kvarteren och bryggorna mot Klara sjö. Genom omfattande utfyllnader, samt i viss mån landhöjningen, flyttades strandlinjen under århundradenas lopp längre västerut och nya kvarter tillkom. Departementsstadens centrala delar med Drottninggatan och Fredsgatan som huvudgator har dock sina mått och sin struktur kvar alltsedan 1600-talets mitt. +År 1751 drabbades Norrmalms södra del av Klarabranden och i stort sett samtliga kvarter väster om Drottninggatan berördes, så även Björnen, Loen, Tigern och Rosenbad. Många byggnader som uppförts under 1600-talets andra hälft förstördes, bland dem Adelcrantzska palatset i kvarteret Björnen. Huset återuppbyggdes 1753 efter ritningar av arkitekten Carl Fredrik Adelcrantz. Även andra hus gick att åter sätta i stånd, men flera nybyggdes också, som hovapotekare Wertmüllers trevåningshus i kvarteret Tigern. +Kvarterens djurnamn (Lejonet, Tigern, Björnen och Loen) är mycket gamla. De infördes mellan 1640- och 1660-talen och tillhör kategorin "vilda djur". Johannes större är känt från mitten av 1650-talet som S:t Johannes. Namnet Rosenbad går tillbaka till slutet av 1600-talet då ett badhus låg här. Kvartersnamnet Vinstocken härrör från 1700-talets början och tycks vara slumpmässigt valt. Brunkhuvudet och Brunkhalsen har sina namn efter Brunkeberget som här har sin södra utlöpare. Förledet brunke härrör från ordet "brant". Nyast är Röda bodarne som uppkallats efter rödmålade hamnbodar som låg där under 1800-talets början. +I början av 1960-talet föreslog Byggnadsstyrelsen att samla den statliga kontorsverksamheten i ett område på Nedre Norrmalm som begränsas i öster av Malmtorgsgatan, i norr av Herkulesgatan, i väster av Tegelbacken och i söder av Strömgatan och Norrström. Centralt genom området löper Drottninggatan som här sedan 1800-talets andra hälft utvecklats till Stockholms nya affärscentrum med bank- och försäkringspalats och eleganta butiker. Gatan blev under 1840- och 1850-talet stadens ”snobbränna”, ett promenadstråk för folk som ville synas. +Utslagsgivande för etableringen i de gamla Klarakvarteren var att Riksdagen skulle flytta från sina tillfälliga lokaler i Kulturhuset tillbaka till Helgeandsholmen. Läget för riksdagshusets permanenta bygge diskuterades länge under slutet av 1960-talet. Det fanns långtgående planer på ett riksdagshus i kvarteret Vinstocken vid Gustav Adolfs torg. 1969 beslöts dock att bygga om gamla Riksdags- och Riksbankshusen och flytta till Helgeandsholmen igen. +Från kvarteren på södra Norrmalm skulle departementens medarbetare få gångavstånd till Riksdagshuset och ett koncentrerat område var lättare att bevaka än ett över flera stadsdelar utspritt. Innan dess var departementen utspridda på bland annat Norrmalm och i Gamla stan, exempelvis i Kanslihuset och Kanslihusannexet och före detta Skandiahuset vid Mynttorget. Det senare byggdes om för Utbildningsdepartementet som flyttade in 1967. Det fanns även planer på att anlägga en departementsstad i Kymlinge norr om Stockholm. För snabba kommunikationer till och från staden byggdes Kymlinge tunnelbanestation. Planerna realiserades dock inte och tunnelbanestationen togs aldrig i bruk. +Redan i början av 1900-talet flyttades några departement till södra Norrmalm, bland dem Utrikesdepartementet som 1906 övertog Arvfurstens palats i kvarteret Lejonet. År 1918 flyttade dåvarande Ecklesiastikdepartementet till Kronprinsens stall (numera rivet) vid Tegelbacken och det nybildade Handelsdepartementet flyttade 1922 in i kvarteret Rosenbad vars byggnader övertogs av staten 1919 och 1924. År 1966 köptes före detta Skandinaviska Bankens hus i kvarteret Vinstocken och byggdes om för Försvarsdepartementet. +År 1965 började staten förvärva mark från Stockholms stad i kvarteret Loen där man övertog Svenska Dagbladets gamla fastighet i hörnet Karduansmakargatan / Klara södra kyrkogata. Norrmalmsregleringen pågick för fullt i kvarteren norr om Loen och hela området befann sig under komplett omdaning. Samtidigt fick arkitektkontoret Ahlgren-Olsson-Silow i uppdrag att utarbeta en utbyggnadsplan för kvarteren Björnen, Loen, Brunkhalsen, Brunkhuvudet, Johannes Större och Tigern. Arkitekterna föreslog fullständig rivning och nybyggnad, med undantag för det sedan 1935 byggnadsminnesskyddade Adelcrantzka palatset i kvarteret Björnen. De föreslagna rivningarna skulle dock inte bli så omfattande som arkitekterna hade föreslagit och flera byggnader blev kvar, bland annat i Brunkhalsen, Brunkhuvudet och Johannes Större. +År 1968 överlämnades utbyggnadsplanen till staten för granskning. Gällande kvarteren Brunkhalsen och Brunkhuvudet (de så kallade Brunkarna) skar det sig mellan Stockholms stad och Byggnadsstyrelsen om hur kvarteret skulle bebyggas. Staden hade i samband med Norrmalmsregleringen och 1962 års cityplan planerat att här låta uppföra ett stort parkeringshus för omkring 600 bilar vilket spolierade Byggnadsstyrelsens planer. Resultatet blev att staten till en början avstod från Brunkarna. +I januari 1969 blev det definitivt klart att samtliga departement skulle samlas på Nedre Norrmalm. Den första stora flytten inleddes 1973 väster om Drottninggatan, när det så kallade Departementshuset, ritat av arkitekt Nils Tesch, blev färdigt i kvarteret Loen. Till en början fick Bostadsdepartementet, Jordbruksdepartementet och Socialdepartementet sina lokaler i den nya ämbetsmannabyggnaden på 25 000 kvadratmeter. Med detta bygge påbörjades även en breddning av Jakobsgatan som innebar att kvarterets södra fastigheter fick rivas, medan två äldre byggnader vid Drottninggatan 15–17 (före detta Tjenstemannabanken och före detta Sundsvalls Handelsbank) bevarades. +I grannkvarteret Björnen började 1977 en omfattande rivning med undantag för Adelcrantzka palatset. Sommaren 1978 rapporterade Tidningarnas Telegrambyrå (TT) att ”Regeringen gett startsignal för regeringsbyggnader med omkring 1 400 arbetsplatser till en kostnad av 140 miljoner kronor”. På ett fotografi från 1979 syns Adelcrantzka palatset ensamt i kvarteret medan grundläggningsarbetena för den nya bebyggelsen pågår runt omkring. Här uppfördes nya byggnader efter ritningar av arkitektkontoret Ahlgren-Olsson-Silow med lokaler för Arbetsmarknadsdepartementet, Finansdepartementet och Jordbruksdepartement som kunde flytta in 1982. +Mellan 1979 och 1981 byggdes även hela kvarteret Rosenbad om under ledning av Anders Tengboms arkitektkontor för Statsrådsberedningen och Justitiedepartementet. Samtidigt iordningställdes kvarteret Röda bodarne (Centralpalatset) för Handelsdepartementet. +I kvarteret Tigern, beläget söder om Björnen, sparades enbart fasaderna till två byggnader i Tigerns södra del: före detta Lifförsäkringsaktiebolaget Nordstjernan och före detta Norrlandsbanken. Tre hus i kvarterets norra del (mot Jakobsgatan) revs helt och ersattes 1982 av ett nytt kontorskomplex ritat av EHKL arkitektkontor. Därmed var omdaningarna för de nya regeringskvarteren väster om Drottninggatan avslutade. +År 1983 fortsatte Byggnadsstyrelsen sitt utredningsarbete med kvarteren öster om Drottninggatan. Under tiden hade Stockholms stad skrinlagt sina planer om ett stort parkeringshus i Brunkhalsen. Det blåste nya vindar, och nu gällde varsam upprustning i stället för rivning och nybyggnad, allt enligt Cityplan 1977 som även blev slutpunkten för Norrmalmsregleringen. +År 1979 förvärvade staten samtliga fastigheter i Brunkhuvudet och Brunkhalsen. Här fanns också några kulturhistoriskt värdefulla byggnader, bland dem Mäster Dyks hus mot Drottninggatan och Lifförsäkringsaktiebolaget Nordstjernans byggnad samt Nordiska Handelsbankens byggnad, båda mot Malmtorgsgatan. Den senare inrymde mellan 1928 och 1973 postkontoret Stockholm 16 och var länge rivningshotad. En medborgaropinion bidrog dock till att posthuset och Mäster Dyks hus bevarades och att den återstående delen av Jakobsgatan inte breddades. Istället lades trottoaren indraget bakom husens södra fasad och Jakobsarkaden skapades. I september 1988 kunde delar av Utrikesdepartementet flytta in i Brunkhuvudet medan Brunkhalsen 1993 blev hemvist för Kulturdepartementet och Utbildningsdepartementet. +Nästa ombyggnadsetapp gällde Kvarteret Johannes större, beläget direkt söder om Brunkhuvudet. Även i detta kvarter fanns flera kulturhistoriskt värdefulla byggnader, bland dem Gustav Horns palats, Sundsvallsbankens före detta byggnad, Sadelmakarens hus och Kanngjutarmästarens hus. Sundsvallsbankens hus övertogs av staten redan 1972 och byggdes 1984 om för Utrikesdepartementets pressrum. 1988 avslutades en större ombyggnad av kvarteret som gällde en förbindelsekulvert till Brunkhuvudet och lokaler för Medelhavsmuseet. +Kvarterets västra delar (mot Drottninggatan) ombyggdes åren 1992 till 1995 och berörde Sadelmakarens och Kanngjutarmästarens hus. Det senare anses vara ett av de bäst bevarade bostadshusen från 1600-talet utanför Gamla stan. Till kvarterets västra delar flyttade 1995 Utrikesdepartementets handelsavdelning. En sista etapp som omfattade bland annat renovering av fasaderna i den östra kvartersdelen blev klart 1997. Därmed var tre decenniers omdaning av Stockholms regeringskvarter avslutad. +Inga större om- eller nybyggnader hade gjorts i Departementsstaden sedan 1990-talets slut, men i början av 2000-talet var några av regeringskvarterens fastigheter i stort behov av teknisk och lokalmässig upprustning. +Ombyggnad av kvarteret Tigern inleddes 1999 sedan utredningar från 1995 visat på ett behov av förbättrad arbetsmiljö och inomhusklimat. Det fanns även krav på ökat antal arbetsplatser för Regeringskansliet. Som generalkonsult och arkitekt för projektet anlitades Wingårdh Arkitektkontor. +Åren 2009 till 2012 genomfördes en omfattande ombyggnad av Nils Teschs byggnad från 1960-talet i kvarteret Loen. Huset påbyggdes med några våningar och den tekniska utrustningen moderniserades. För gestaltningen stod BAU Arkitekter och Reflex Arkitekter. I september 2017 klassades kvarteret Loen som miljöbyggnad i klass silver. Av de 14 parametrar som granskats blev det fyra guld, sju silver och tre brons. +Under hösten 2014 påbörjades moderniseringen av kvarteret Björnen. Delar av 1980-talsbyggnaden revs och påbyggdes sedan med sex nya våningar. En gårdsflygel revs och gjorde Adelcrantzska palatset mer synligt. Även palatset fick en välbehövlig uppfräschning. Arbetena avslutades 2018. Byggherre var Statens Fastighetsverk, arkitekter var White arkitekter och BAU Arkitekter. +I januari 2019 till 2023 genomfördes en omfattande renovering av byggnadsbeståndet i kvarteret Rosenbad. Utöver en invändig upprustning av lokaler och tekniska installationer ingick även restaurering av fasadernas stendetaljer. Stenar som behövde nytillverkas göts av och höggs på ett stenhuggeri i Hannover. Regeringskansliets verksamhet är under tiden inrymd i kringliggande kvarter. Ansvarig arkitekt är Arcona. +Hela området kring Rosenbad säkras för obehörig trafik genom hinder som består av fasta och rörliga pollare. Exempelvis finns en rad pollare som stänger av Fredsgatans genomfartstrafik till Gustav Adolfs torg. Liksom Rosenbads byggnader är samtliga fastigheter i Departementsstaden klassade som skyddsobjekt. +Kvarter och besöksadresser gällande år 2020. I alfabetisk ordning. +Arbetsmarknadsdepartementet: Tigern (Fredsgatan 8) +Finansdepartementet: Björnen och Loen (Jakobsgatan 24) +Försvarsdepartementet: Vinstocken (Jakobsgatan 9) +Infrastrukturdepartementet: Brunkhuvudet (Malmtorgsgatan 3) +Justitiedepartementet: Björnen och Loen (Herkulesgatan 17) +Kulturdepartementet: Brunkhalsen (Drottninggatan 16) +Miljödepartementet: Johannes större (Fredsgatan 6) +Näringsdepartementet: Björnen och Loen (Herkulesgatan 17) +Socialdepartementet: Tigern (Fredsgatan 8) +Utbildningsdepartementet: Brunkhalsen (Drottninggatan 16) +Utrikesdepartementet: Lejonet (Gustav Adolfs torg 1) +I alfabetisk ordning: +Bonns regeringskvarter +Londons regeringskvarter +Kvarter i Stockholm +Klarakvarteren +Norrmalmsregleringen +Diplomatstaden +SFV: Departementsstaden av Staffan Nilsson, sid. 29–31. +SFV: Klara – en angelägenhet för staten +Hasselblad, Björn; Lindström, Frans (1979). Stockholmskvarter: vad kvartersnamnen berättar. Stockholm: AWE/Geber. Libris 7219146. ISBN 91-20-06252-4 +Departementens e-post- och besöksadresser +Hall, Thomas (1999). Huvudstad i omvandling - Stockholms planering och utbyggnad under 700 år. Källström, Gunnar (illustratör). Stockholm: Sveriges Radios förlag. Libris 7409712. ISBN 91-522-1810-4 +Lundevall, Peter (2006). Stockholm, den planerade staden. Carlssons bokförlag. ISBN 91-7203-788-1 +Informationsskyltar utformade och uppsatta av Statens Fastighetsverk på respektive fastighet +Wikimedia Commons har media som rör Stockholms regeringskvarter. +Jakobsbrevet (klassisk grekiska: Ἰακώβου [ἀποστόλου] ἐπιστολή [καθολική], Iakōbou epistolē) är en skrift som ingår i Nya testamentets brevlitteratur. Brevet tillhör, tillsammans med Första Petrusbrevet, Andra Petrusbrevet, de tre Johannesbreven och Judasbrevet, de sju så kallade katolska breven, därför att det har en allmän ("katolsk") adressat och alltså inte är riktat till någon bestämd person eller församling. +Det föreligger inte någon konsensus inom forskarvärlden om när detta brev skrevs och oenigheten hänger samman med frågan om vem som har skrivit det – om det skrevs av Herrens broder Jakob som antas ha dött år 62 eller om det skrevs av någon okänd person senare. Detta har lett till att vissa daterar brevet så tidigt som till mitten av 40-talet medan andra förlägger dess tillkomst till långt efter Jakobs tid och ibland så sent som till mitten av 100-talet. Följaktligen har det hävdats att det är Nya testamentets såväl äldsta som yngsta skrift. Den mest accepterade uppfattningen torde vara att brevet inte har skrivits av denne Jakob, utan att det skrevs av en för oss okänd person någon gång under perioden omkring 65–100. +Jakobsbrevet är mer att betrakta som en allmän kristen förmaningsskrift än ett traditionellt brev. Skriften består av löst sammansatta vishetsord och korta utredningar och förmaningar. Jakobsbrevet tar upp förhållandet mellan gärningar och tro, och poängterar att utan gärningar är tron död och att man blir rättfärdig även genom gärningarna, inte enbart genom tron. Av bland annat denna anledning har brevet varit omtvistat inom kyrkan och exempelvis Martin Luther kritiserade det för att grumla läran om rättfärdiggörelsen. +Det finns ingen större enighet bland forskare vad gäller brevets ursprung, tiden när det skrevs eller vem författaren var, och det var omstritt redan i den gamla kyrkan. Brevet uppges vara skrivet av Jakob, Guds och herren Jesu Kristi tjänare, men vem denne var anges aldrig. Jakob var ett vid denna tid mycket vanligt namn bland palestinska judar och kristna. Med reservation för att samma person kan dölja sig bakom fler namn än ett, förekommer fem män i Nya testamentet som heter Jakob (grekiska: Ἰάκωβος): Jakob, Sebedaios son, Jakob, Alfaios son, Jakob, Jesu broder, också benämnd Herrens broder och den rättfärdige, Jakob den yngre och Jakob, far till lärjungen Judas. Den traditionella uppfattningen har varit att brevet har författats av Jesu broder, kallad Jakob den rättfärdige, som var ledare för den kristna kyrkan i Jerusalem fram till sin död år 62. Genom att det i brevet aldrig sägs vilken Jakob som avses, drar de flesta forskare slutsatsen att oavsett vem som har skrivit det, var meningen den att läsaren skulle förstå att detta var en så välkänd och berömd Jakob att den saken inte behövde påtalas. Därmed kvarstår Jakob, Sebedaios son och Jakob den rättfärdige, och eftersom den förstnämnde anses ha dött redan år 44 återstår bara den senare. Har han skrivit brevet bör det vara en av Nya testamentets tidigaste skrifter. Mot Jakob som författare talar brevets vårdade grekiska; för honom talar att innehållet uppvisar ”beröringspunkter med kristendomens ursprungsmiljö” och att flera kärva uttalanden liknar Jesu ord i evangelierna. +Till stöd för att Jakob den rättfärdige har skrivit brevet har forskare framhållit att brevets polemik vore otänkbar efter apostlamötet i Jerusalem omkring år 49, varför de förlägger brevets tillkomst till före detta år. Andra argument till stöd för att Jakob skrivit brevet är att traditionen tillskriver honom brevet och att det är judiskt-kristet, vilket skulle stämma med Jakob som författare. Vidare anses den ödmjuka ingressen, den förmanande hållningen och den självklara auktoriteten, tala för att Jakob har skrivit brevet. Dessutom menar några, som Scot McKnight, att argumenten mot identifieringen av Jakob som författare inte är så övertygande som ofta görs gällande och att brevet därför inte måste vara skrivet efter Jakobs tid, som 1) att orsaken till att det inte har citerats eller åberopats i den äldsta tiden är att det ansågs vara för obetydligt och 2) att det har cirkulerat inom bara en mindre judekristen grupp, och 3) att templets ödeläggelse år 70 marginaliserade den judekristna gruppen i Jerusalem varför deras syn negligerades samt 4) att brevet ansågs vara för judiskt för att väcka intresse inom den icke-judiska delen av kristenheten. Robert Foster menar dock att även sammantaget utgör dessa argument bara en möjlig, men knappast övertygande, förklaring till varför brevet inte rönte någon större uppmärksamhet. +Om Jakob, ledaren för den kristna församlingen i Jerusalem, har författat denna skrift skulle den kunna vara skriven någon gång i perioden omkring 40 till 62, och om den som flera forskare antar är tillkommen i slutet av 40-talet, skulle den kunna vara Nya testamentets äldsta skrift. +Argumenten emot uppfattningen att brevet skrivits av Jakob inbegriper att det är skrivet på driven grekiska, något många menar att en enkel och outbildad palestinier med arameiska som modersmål knappast vore i stånd till, om än vissa anser det inte vara uteslutet. Det faktum att grekiskan förekom i Palestina innebär inte, enligt Werner Kümmel, att en arameisktalande jude i vanliga fall kunde skriva på flytande grekiska. Författaren är dessutom väl förtrogen med Septuaginta, den grekiska översättningen av Gamla testamentet, och de bibelcitat som förekommer i brevet är hämtade därifrån. Det finns heller inga tydliga tecken på att det skulle vara en översättning gjord från en arameisk förlaga. Och enligt Bart D. Ehrman var Jakob en arameisktalande bonde från Galileen som med nästan absolut säkerhet inte ens kunde läsa och skriva, än mindre författa ett verk på flytande grekiska. Ehrman menar att den som skrev detta brev var en mycket bildad person med grekiska som modersmål, som skrev en klar och korrekt koinegrekiska och som hade förmågan att bruka en mängd retoriska tekniker som var vanliga inom grekisk-romersk moralfilosofi. Av dessa anledningar menar han att det är näst intill omöjligt att föreställa sig att Jakobsbrevet har skrivits av Jakob, en outbildad bonde från landsbygdens Nasaret. (Vad gäller svårigheten för en outbildad arbetare, som en jordbrukare eller fiskare, från den galileiska landsbygden att kunna författa ett brev på grekiska, se Andra Petrusbrevet, läs- och skrivkunnigheten i Galileen.) +En annan invändning mot att Jakob skulle vara författaren, är att det ingenstans i brevet framkommer att författaren har någon egen relation till Jesus. Jesus nämns för övrigt vid namn bara två gånger i brevet (Jak 1:1; 2:1) och då endast som herren Jesus Kristus. Inget sägs heller om att hans död och uppståndelse skulle utgöra grund för frälsningen. Vidare var skriften omstridd i den äldsta kyrkan och dess accepterande i kanon tog lång tid. Det finns inga säkra spår efter att någon använt sig av Jakobsbrevet före 200-talet, men det finns tecken på att Irenaeus var bekant med det cirka år 180. Det saknas i Muratoriekanon, Tertullianus nämner det inte, och det citerades som auktoritativ skrift först av Origenes i mitten av 200-talet. På 300-talet skriver Eusebios av Caesarea att brevet ofta har ansetts vara oäkta och att inte många gamla författare har nämnt det. Flera forskare har tolkat brevets andra kapitel, verserna 2:14–26, som ett angrepp på Paulus, varför i så fall brevet måste ha skrivits efter det att Paulus brev hade samlats och börjat cirkulera, något som skedde senare än år 62 när Jakob antas ha dött. Och man har undrat hur Jakob, om han verkligen är författaren, kan ha undgått att ta upp frågan om konvertiternas förhållande till judiska lagar och riter. I alla källor sägs Jakob värna den judiska lagen och såväl kulturellt som kultiskt hålla fast vid den judiska identiteten. Men i Jakobsbrevet finns inget rituellt eller kultiskt, inget om renhetslagar, det judiska templet, tillåten mat, sabbaten, högtidsdagar, omskärelse, Israel eller något annat som hör till den judiska identiteten. I stället handlar det om kärleksbudet att Du skall älska din nästa som dig själv (Jak 2:8) eller om budorden (Jak 2:10–12). Det förekommer heller inga antydningar i brevet om schismer mellan judiska och icke-judiska kristna, och soteriologin, frälsningsläran, i brevet riktar sig till de hednakristna och avviker därmed kraftigt från den orientering mot de judekristna som Jakob representerar enligt det Paulus skriver i Galaterbrevet. +Andra tecken på att brevet inte är från den närmaste tiden efter Jesu död, och därmed heller inte skrivet av Jakob, är dess beskrivning av förhållanden som antas ha rått vid en senare tid. Det handlar om upplevda bekymmer över att det finns alltför många rika i församlingarna (Jak 2:1–6; 5:1–6), att folk verkar vara otåliga över att Jesus ännu inte kommit (grekiska: παρουσία, parousia) (Jak 5:8–9) och att ”de äldste” (grekiska: πρεσβύτερος, presbyteros) förefaller leda församlingarna (Jak 5:14). Detta är omständigheter som alla påminner mer om de förhållanden som rådde senare. Mot slutet av nollhundratalet anslöt allt fler till den kristna rörelsen och då också många fler rika, varför sociala konflikter uppstod i församlingarna. Liknande förhållanden står att finna främst i Lukasevangeliet, Uppenbarelseboken och de pseudopaulinska Pastoralbreven, vilka anses ha skrivits vid en senare tid än den när Paulus skrev sina brev. +Några forskare menar att flera av dessa omständigheter går att förklara med antagandet att Jakob inte har skrivit brevet men väl påverkat dess utformning i samband med att en skolad hellenist skrev det, även om brevet utformades slutgiltigt först senare. +Emedan majoriteten forskare anser att brevet inte har skrivits av Jakob utan tillkommit efter dennes död och författats av en okänd hellenistisk judisk kristen person med en grundlig grekisk skolning, brukar dess tillkomst ofta förläggas till den sista tredjedelen av nollhundratalet, cirka 65–100. +Jakobsbrevet är, frånsett inledningen, inte uppbyggt som ett traditionellt brev. Det är snarare en skrift sammanställd av kärnfulla uttryck och uttolkningar. Det är främst en traktat (en allmän skrift för en vidare publik) – en kristen förmaningsskrift som handlar om kristen livsföring och gärningarnas betydelse i förhållande till tron. Det innehåller alltså undervisning om kristen moral och etik, visdomsord och förmaningar till församlingarnas medlemmar och ledare, och var troligen avsett att skickas runt till församlingarna. +Brevet inleds med ”Hälsningar från Jakob, Guds och herren Jesu Kristi tjänare, till de tolv stammarna i förskingringen” (Jak 1:1), vilket sannolikt inte avser de tolv judiska stammarna, utan antingen judekristna runt om i Romarriket eller så kristna över huvud taget. De ämnen som ges störst utrymme i brevet är tro, prövningar och frestelser och att Gud inte frestar någon; att man inte ska göra skillnad på rika och fattiga och att Gud utvalt de fattiga till sitt rike; att man inte med tungan ska sprida sitt gift och att den sanna visheten kommer från ovan; att man ska vara ödmjuk och inte högmodig, förlita sig på Guds försyn och inte vara övermodig; att Herrens ankomst (parousia) är nära; att bönen har kraft; att många bröder kommer bort från sanningen och hamnar på villovägar; och att ”den som återför en syndare från hans villovägar räddar hans liv undan döden och gör att många synder blir förlåtna.” (Jak 5:20) Därefter slutar brevet utan någon avskedsfras. +Ett framträdande tema i Jakobsbrevet är att det finns två vägar i livet, där den ena leder till död och den andra till liv. I brevet framställs kristendomen (fastän beteckningen inte förekommer) som inte bara ett trossystem utan som ett sätt att leva. Församlingsmedlemmarna varnas för att vara tvehågsna (Jak 1:8; 4:8) och att det inte är möjligt att vara vän både med världen och med Gud (Jak 4:4). Skriften beskriver det praktiska livet i en församling, där man ber för varandra och bekänner sina synder för varandra (Jak 5:16), och där man åtar sig att hjälpa dem som befinner sig i nöd (Jak 1:27). Den utgör en handledning för hur man ska agera för att beträda den väg som leder till livet (Jak 1:12) och hur man ska undvika den väg som leder till döden (Jak 5:19–20). +En av de saker som främst betonas i brevet är att tron i sig själv, utan gärningar, är död (Jak 2:17, 26), och ”att människan blir rättfärdig genom gärningar och inte bara genom tro” (Jak 2:24). Detta verkar stå i direkt motsats till det Paulus skriver, nämligen ”att människan blir rättfärdig på grund av tro, oberoende av laggärningar”. +Jakobsbrevet är en bibelbok som relativt sällan lyfts fram i kyrkans förkunnelse. Orsaken är främst att den inte betonar Jesu död och uppståndelse på samma evangeliserande sätt som andra bibelböcker i Nya testamentet gör. Martin Luther ansåg att brevets innehåll grumlade rättfärdiggörelsen genom tron. +”Bli ordets görare, inte bara dess hörare.” (Jakobsbrevet 1:22) +”Vad hjälper det om någon påstår sig ha tro men saknar gärningar? Kan väl en sådan tro frälsa någon?” (Jakobsbrevet 2:14) +”Så är det också med tron: i sig själv, utan gärningar, är den död.” (Jakobsbrevet 2:17) +”Vi begår alla många fel. Om någon inte felar i sitt tal, är han en fullkomlig man ...” (Jakobsbrevet 3:2) +”Lyssna ni som är rika, gråt och klaga över de olyckor som skall komma över er ...” (Jakobsbrevet 5:1–6) +Katolska breven +Jakobs protevangelium +Nationalencyklopedins (nätupplaga), Jakobs brev Läst 11 februari 2017 +Brown, Raymond E. (2016) (på engelska). An Introduction to the New Testament: The Abridged Edition. The Anchor Yale Bible Reference Library (1. ed. 1997). New Haven & London: Yale University Press. ISBN 9780300173123 +Ehrman, Bart D. (2011) (på engelska). Forged: Writing in the Name of God – Why the Bible’s Authors Are Not Who We Think They Are. New York: HarperCollins. ISBN 9780062012616. https://archive.org/details/forgedwritinginn0000bart +Ehrman, Bart D. (2013) (på engelska). Forgery and counterforgery: the use of literary deceit in early Christian polemics. New York: Oxford University Press. Libris 13926777. ISBN 9780199928033 +Foster, Robert J. (2014) (på engelska). The significance of exemplars for the interpretation of the Letter of James. Wissenschaftliche Untersuchungen zum Neuen Testament. R. 2, 0340-9570 ; 376. Tübingen: Mohr Siebeck. Libris 17106023. ISBN 9783161532634 +Hartin, Patrick J. (1991) (på engelska). James and the 'Q' sayings of Jesus. Journal for the study of the New Testament. Supplement series, 0143-5108 ; 47. Sheffield: JSOT Press. Libris 6428780. ISBN 1-85075-267-2. https://archive.org/details/jamesqsayingsofj0000hart +Hartman, Lars (1989). ”Nytestamentlig isagogik”. i Birger Gerhardsson. En bok om Nya testamentet (5., [rev. och utök.] uppl.). Malmö: Liber. sid. 11–132. Libris 7272298. ISBN 91-40-61019-5 +Kümmel, Werner Georg (1975) (på engelska). Introduction to the New Testament (Rev. ed.). London: SCM Press. Libris 4839475. ISBN 0-334-00705-4 +Laws, Sophie (1980) (på engelska). A commentary on the epistle of James. Black's New Testament commentaries, 99-0116433-8. London: A. & C. Black. Libris 5495067. ISBN 0-7136-2075-7. https://archive.org/details/commentaryonepis0000soph +Leahy, Thomas W. (1989). ”The Epistle of James”. i Raymond E. Brown; Joseph A. Fitzmyer; Roland E. Murphy (på engelska). The New Jerome Biblical commentary ([2. ed.]). London: G. Chapman. sid. 909–910. Libris 4715543. ISBN 0-225-66588-3 +McKnight, Scot (2011) (på engelska). The Letter of James, The New International Commentary on the New Testament. Cambridge: Grand Rapids: Eerdmans. ISBN 9780802826275 +Nestle–Aland (2012). Nestle Eberhard, Nestle Erwin, Aland Barbara, Aland Kurt. red (på grekiska, klassisk). Novum Testamentum Graece (28. rev. Aufl., herausgegeben vom Insitut für Neutestamentliche Textforschung.). Stuttgart: Deutsche Bibelgesellschaft. Libris 13988187. ISBN 9783438051608 +Powell, Mark Allan (2009) (på engelska). Introducing the New Testament: a historical, literary, and theological survey. Grand Rapids, Mich.: Baker Academic. Libris 11832198. ISBN 9780801028687. https://archive.org/details/introducingnewte00powe +Robinson, John A. T. (1976) (på engelska). Redating the New Testament. London: SCM Press. Libris 4839963. ISBN 0-334-02301-7 +Schnelle, Udo (1998) (på engelska). The History and Theology of the New Testament Writings. Minneapolis, MN: Fortress Press. Libris 5666575. ISBN 0-8006-2952-3. https://archive.org/details/historytheologyo0000schn_m3a5 +Sunnliden, Håkan (2017). Jakobs brev: en bibelteologisk kommentar. Everöd: Recito förlag. Libris 20838172. ISBN 978-91-7765-096-6 +Jakobsbrevet i Bibel 2000 på bibeln.se +Wikisource har originalverk som rör Jakobsbrevet. +Forskning kring medeltidens mat försöker beskriva livsmedel, matlagningsmetoder och matvanor som förekom i europeiska samhällen under medeltiden, en period som sträcker sig från cirka år 500 e.Kr. till 1500 e.Kr. Under denna period etablerades många nya vanor inom kosthållning och matlagning över hela Europa och många av dessa blev en grund för de moderna europeiska köken. Transport och kommunikation var betydligt långsammare än idag, vilket omöjliggjorde eller fördyrade handel med flera typer av mat. De välbärgade kunde emellertid låta sina matvanor påverkas av importerade recept och råvaror, och successivt spred sig främmande kulturers matvanor även till de lägre klasserna. +Medeltiden var en period då missväxt och svält var vanligt och stor skillnad rådde mellan fattig och rik. Vid senmedeltiden stiftades ofta överflödsförordningar, lagar som reglerade i vilken mån varje samhällsklass fick markera sin status, med exempelvis kläder och mat. Sociala eller medicinska normer kunde också diktera att den arbetande befolkningens mat skulle vara mindre förfinad än elitens kost. Logiken bakom detta var att grovt arbete krävde grov mat och att detta var en gudagiven ordning. +Bröd var det vanligaste livsmedlet i alla samhällsklasser och fanns vid så gott som varje måltid. Spannmålen dominerade även i form av gröt, välling och pasta. Soppor, grytor och stuvningar var de vanligaste rätterna och ingredienser som användes över hela Europa var mandlar, oftast i form av mandelmjölk, verjus (den sura saften av omogna druvor), vin och vinäger, gärna kombinerat med socker eller honung för dem som hade råd. Fläsk och kyckling förekom i alla regioner, och särskilt i norr åt man stora mängder sill och torsk. +Under medeltiden intogs vanligen två måltider per dag; middag någon gång mellan klockan 10.00 och 14.00 och kvällsmål sent på eftermiddagen eller kvällen. Systemet med två måltider förblev stabilt under senmedeltiden. Mellanmål var vanliga, men var också en fråga om social status eftersom de som inte behövde utföra kroppsarbete kunde avstå från dem. Frukost var något som adel, präster och medlemmar av religiösa ordnar undvek eftersom det ansågs syndigt att bryta nattens fasta för tidigt på dagen. Av praktiska skäl åt arbetsklassen ett morgonmål och det var även tolererat att barn, kvinnor och äldre gjorde det. Sena måltider (reresoper, av occitanska rèire-sopar "sent kvällsmål") med åtskilliga mängder alkohol ansågs osedliga. Dessa sena måltider blev ofta förknippade med syndiga förehavanden som hasardspel, grova skämt och oanständigt flörtande. Mindre mellanmål var vanliga (trots att även de ansågs opassande av den tidens moralister) och arbetare kunde få tillägg av sina arbetsgivare för att köpa en mindre matbit att äta på raster. +Under senare delen av medeltiden fanns en tendens att i viss mån försöka undkomma den stränga kollektivism som genomsyrade perioden. Måltiden var en social händelse, precis som alla andra aspekter av medeltida livsföring. Hela hushållet, inklusive tjänare, skulle äta tillsammans. Att dra sig undan för att äta på egen hand eller med en eller två nära vänner ansågs slösaktigt och snudd på misstänkt i en värld där alla i samhället var beroende av varandra för att överleva. På 1200-talet rådde exempelvis den engelske biskopen av Lincoln Robert Grosseteste grevinnan av Lincoln att "förbjuda middag och kvällsmål utanför matsalen, i hemliga och privata gemak, för detta leder till slöseri och ger ingen ära åt herre eller fru." Han rådde också grevinnan att hålla ögonen på tjänarna så att de inte smet iväg med matrester för att avnjuta dem sent på kvällen istället för att skänka dem till fattiga som allmosor. +Det finns beskrivningar av bordsskick vid större banketter under senmedeltiden, men betydligt färre återgivningar av hur eliten intog vardagsmål eller hur allmogen och de fattiga åt sin mat. Man kan dock utgå från att det inte förekom utsvävningar som flerrättersmiddagar, kryddor och handtvätt i parfymerat vatten i enklare hem. I rika hushåll försågs gäster med vatten och rena linnehanddukar före och efter måltiden så att de kunde tvätta händerna. Sociala normer gjorde det svårt för kvinnor att upprätthålla den stereotypa bilden av att vara nätta och rena medan de avnjöt en storslagen måltid, och det var vanligt för värdens fru att äta i enskilt rum med sitt följe. Hon kunde efter att ha avklarat huvudsaken av det praktiska ätandet ansluta sig till gästerna lagom till desserten. Överlag var de högre ståndens fina tillställningar ett rent manligt nöje och det var ovanligt att andra än mycket högt stående gäster tog med sig sina fruar. Den sociala hierarkin bekräftades genom bordsskicket där de av lägre rang skulle hjälpa dem av högre rang, där yngre skulle hjälpa äldre, och där män skulle bespara kvinnor risken att befläcka kläder och rykte genom att hantera mat på ett okvinnligt sätt. Delade dryckeskärl var vanligt även vid viktiga banketter för alla utom de mest ansedda som satt vid värdens bord. Likaså förväntades det att man skulle bryta bröd och skära upp kött åt sina bordsgäster. +Mat serverades för det mesta på tallrikar eller direkt ur kokkärlen och de ätande tog sin portion från serveringsfaten och placerade på en egen tallrik med hjälp av skedar eller med händerna. Genom större delen av medeltiden och långt in på 1600-talet bestod tallrikar av trä eller tenn, alternativt av några dagar gammalt bröd av lägre kvalitet som skars till vid bordet. I fattigare hushåll åt man ofta maten direkt från bordet, som ofta kunde ha fördjupningar för mat. Knivar användes för att skära med eller för att ta salt ur de gemensamma saltkaren, och gäster förväntades ha med en egen kniv; enbart högt ärade gäster försågs med en kniv av värden. Gafflar användes inte i Europa i någon större utsträckning förrän fram emot 1700-talet och under renässansen endast i Italien. Synen på gaffeln som bestick under högmedeltiden illustreras väl av reaktionerna på den bysantinska prinsessan Teodora Doukainas bordsskick. Hon gifte sig år 1075 med Domenico Selvo, dogen av Venedig, men orsakade skandal bland venetianarna genom att låta sin mat skäras upp av hennes eunucker och sedan äta den med en gyllene gaffel. Denna nymodighet uppfattades som så skamlöst högfärdig att biskopen av Ostia, Peter Damian, senare hänvisade till den utländska gemålen som "...den venetianske dogens fru vars kropp efter hennes delikata excess helt ruttnade bort." +All medeltida matlagning skedde med hjälp av eld, (spisar blev inte vanliga förrän på 1800-talet) och alla kockar var tvungna att behärska konsten att laga mat över öppen eld. Ugnar var vanliga, men de var dyra att bygga och fanns bara i större hushåll och bagerier. Det var vanligt att samhällen gemensamt ägde ugnar för att försäkra sig om att den livsviktiga brödbakningen var tillgänglig för alla. Det fanns även mindre, flyttbara ugnar som kunde fyllas med mat och sedan täckas med glödande kol eller het aska, och även större ugnar på hjul varifrån man kunde sälja pajer och andra bakverk i medeltida städer. För de flesta människor lagades mat mestadels i enklare grytor eftersom det var det mest bränsleeffektiva sättet att laga mat och på det viset gick man heller inte miste om näringsrikt kokspad. Maten under medeltiden var ofta betydligt fetare än idag, i alla fall för dem som hade råd. Det ansågs inte vara ett hälsoproblem; kroppen behövde fettreserver, eftersom många arbetade hårt och bostäderna vara kalla och dragiga. Kroppsidealen var också annorlunda; var man rik skulle man gärna visa upp det med lite hull (särskilt bland män) och det var i allmänhet bara de mycket fattiga eller sjuka som var smala. +Den dominerande uppfattningen inom den västerländska gastronomin idag är att sött och salt endast med få undantag bör blandas, ett ideal som uppstod först i det franska hovköken under sent 1600-tal. På medeltiden kunde frukt utan problem kombineras med kött, fisk och ägg. Ett recept på Tart de brymlen, en sorts fiskpaj från receptsamlingen Forme of Cury från sent 1300-tal, innehåller en blandning av fikon, russin, äpplen, päron och fisk (lax, ungtorsk eller kolja) med urkärnade plommon under pajskalet. Det ansågs viktigare att rätten passade in på den dåtida uppfattningen om korrekt medicin och kostlära som grundade sig på humoralpatologi. Detta innebar att mat behövde balanseras med andra ingredienser och råvaror beroende på hur dess inneboende natur uppfattades. Till exempel ansågs fisk vara väldigt kall och fuktig och skulle därför tillagas på ett sätt som värmde och torkade ut, som stekning eller bakning i ugn och helst smaksättas med heta och torra kryddor; nötkött ansågs vara kallt och fuktigt och skulle därför helst kokas; fläsk var varmt och fuktigt och skulle därför grillas eller bakas. I några receptsamlingar anges alternativ till råvaror där humorala egenskaper ansågs viktigare än snarlikhet i smak. I ett recept på kvittenpaj anges kål som ett fullgott alternativ, och i ett annat likställs rovor med päron. +I de flesta hushåll lagade man mat över en öppen eldstad i mitten av bostaden för att utnyttja värmen från elden så effektivt som möjligt. Detta upplägg var vanligt även i större och rikare hushåll större delen av medeltiden. Mot senmedeltiden utvecklades successivt ett separat kök, först genom att eldstaden flyttades in mot väggen och senare genom att det placerades i en fristående byggnad som hängde samman med huvudbyggnaden genom en överbyggd gångväg eller arkad. På så vis kunde man undvika rök och matos och det minimerade också risken för att en eldsvåda skulle kunna sprida sig från de många eldstäderna i storköken. +Många grundläggande varianter av moderna köksredskap, som stekpannor, grytor, kastruller och våffeljärn, fanns redan på medeltiden, även om många av dem var för dyra för det genomsnittliga hushållet. En stor variation av spett fanns som kunde användas för att tillaga allt från späda vaktlar till hela oxar. Det fanns även upphängningsanordningar med justerbara krokar på vilka grytor kunde hängas för att underlätta arbetet att snabbt ta bort grytor som höll på att brännas vid eller koka över. Redskapen hölls ofta över en öppen eld eller på ett stativ direkt över glöden. Det fanns även en mängd olika sorters knivar, slevar och rivjärn. I rika hushåll var ett av de vanligaste redskapen morteln och silen (oftast av tyg) eftersom många recept krävde att råvarorna skulle finhackas, silas och kryddas antingen före eller efter tillagning – näringen i finfördelad mat ansågs kroppen lätt kunna ta upp. Det gav också skickliga kockar en möjlighet att forma maten i diverse fantasifulla former. Ett typiskt tillvägagångssätt var att flå och rensa ett djur, mala köttet och blanda det med kryddor och andra ingredienser och sen stoppa tillbaks det i sitt eget skinn, eller att forma det till ett helt annat djur. +Köksbesättningen i de riktigt stora hoven som hölls av adel och monarker kunde ofta bestå av flera hundra personer: Bagare, våffelmakare, såsmakare, förrådsmästare, slaktare, skärare, mjölkpigor, hovmästare och otaliga kökspojkar. Vanliga hushåll klarade sig oftast på ved som kunde plockas från omgivande skog, medan storköken med jämna mellanrum var tvungna att organisera försörjningen av ved för att laga mat åt flera hundra personer. I Du fait du cuisine ("Om matlagning"), en avhandling om matlagning från 1500-talet, ger mäster Chiquart, chefskock till Amadeus VIII, hertig av Savojen, rekommendationer för en tvådagarsbankett. Inför en tvådagarsbankett råder Chiquart en chefskock att säkra minst 1000 vagnslaster "prima, torr ved" och en hel lada kol. +Konservering av mat hade inte ändrats mycket sedan antiken och det skedde inga större omvälvningar förrän konservburken uppfanns på 1800-talet. Tidigare hade det vanligaste sättet att konservera varit att utsätta livsmedel för väder och vind för att dra ut fukt och därigenom öka hållbarheten hos allt från säd till kött. Torkning av mat fungerade genom den drastiskt reducerade aktiviteten hos de vattenberoende mikroorganismer som orsakar förruttnelse. I varmare klimat gjordes detta främst genom att soltorka mat, medan man i kallare klimat, särskilt längs Norges kust, utsatte livsmedel för de starka vindar som ofta råder där (särskilt för framställningen av kabeljo), i ugnar, källare, vindar och ibland även i bostaden. +Att utsätta livsmedel för kemiska processer som rökning, saltning (i salt eller lake) eller konservering gjorde att den höll sig längre. De flesta av dessa metoder tillförde även nya smaker till maten. Rökning och saltning av boskap som slaktades sent på hösten var en vanlig strategi för att dels förse hushåll med kött över vintern, dels att slippa slösa foder på boskapen under de magra vintermånaderna. Smör var nästan alltid hårt saltat (5-10% salthalt) för att inte härskna för snart, något som gjorde att det även kunde säljas på export. Grönsaker, ägg och fisk lades ofta in i tätt packade krukor med saltlag som ofta även innehöll sura vätskor (citronsaft, verjus eller vinäger). En annan metod var att försegla maten i ett lager av socker eller honung genom att koka det i en sockerlag, som sedan maten förvarades i. Att uppmuntra jäsning var också vanligt; man gjorde alkoholhaltiga drycker av spannmål, frukt och druvor där alkoholen verkade desinficerande, även bakterier användes - mjölk syrades till filmjölk eller tillsats av löpe för ystning till ost som sen kunde lagras. +Den stora merparten av den europeiska befolkningen innan industrialiseringen levde i små byar på landsbygden eller i isolerade gårdar. Normen var självförsörjning och endast en liten andel av jordbruksproduktionen såldes på marknader eller exporterades. Större städer var undantag och behövde få sin försörjning från kringområden som stod i proportion till storleken på samhället och dess befolkningstäthet. En större stad kunde hålla sig med en stor variation av matinrättningar som vände sig till olika typer av sociala grupper. Fattigare stadsbor som var trångbodda saknade ofta tillgång till eldstad eller härd att laga mat på och många ägde inte ens husgeråden som var nödvändiga för grundläggande matlagning. I dessa fall var enda möjligheten att köpa färdiglagad mat. Småkrogar sålde färdig, varm mat, en tidig form av snabbmat, eller erbjöd sig att laga till råvaror som kunden tog med sig själv. Resande, till exempel pilgrimer på väg till heliga platser, använde sig ofta av yrkeskockar så att de slapp bära med sig proviant eller husgeråd. För de som hade råd fanns en rad olika specialister för olika typer av mat och tillbehör: osthandlare, pajbagare, såsmakare, rånmakare, etc. Rikare borgare som hade kök i hemmet kunde vid festligare tillfällen hyra in kockar och kökshjälp för att sammanställa en större bankett. +Småkrogar i städer som riktade sig till arbetare eller fattiga sågs som vämjeliga och illa beryktade ställen av de högre klasserna, och även kockar i allmänhet hade ett dålig rykte. Geoffrey Chaucer's karaktär Hodge of Ware, kocken från London i Canterburysägnerna, beskrivs som en sjaskig månglare av oaptitlig mat medan den franske kardinalen Jacques de Vitrys predikningar från tidigt 1200-tal talar om att försäljare av tillagat kött var en ren hälsofara. Att kockar var oumbärliga var något som ibland erkändes och uppskattades, men man såg ned på dem eftersom de tjänade de mer grundläggande och primitiva mänskliga behov snarare än andlig bättring. Stereotypen av en kock i konst och litteratur var en hetlevrad man som ofta drack för mycket och bryskt försvarade sin kittel från snyltare, vare sig de var människor eller djur. Under tidigt 1400-tal satte den engelske munken John Lydgate ord på många av sina samtidas åsikter: "het eld och rök tjock blir till mången arg kock." +Uttrycket "vårt dagliga bröd" från Herrens bön var en mycket konkret verklighet under medeltiden. Den överväldigande majoriteten av kaloriintaget för alla, hög som låg, bestod i huvudsak av spannmål, oftast i form av bröd, gröt, välling och i mindre utsträckning pasta. De vanligaste sädesslagen var vete, råg, korn, hirs, bovete och havre. Ris var för de flesta en dyr importprodukt genom större delen av medeltiden. Vete var det vanligaste sädesslaget i Europa och ansågs också det mest näringsrika, men var ofta dyrare. Det fina, vita mjöl de flesta européer är vana vid idag var något som var reserverat för överklassen. Bönder och arbetare åt mestadels bröd gjort på betydligt grövre mjöl med högre fiberinnehåll. Vid sädesbrist och svält var det vanligt att dryga ut mjöl med billigare och mindre näringsrika matvaror som kastanjer, torkade baljväxter, ekollon, ormbunkar och en rad andra mer eller mindre matnyttiga vegetabilier. +Brödets oumbärlighet som stapelvara innebar att bagare spelade en livsviktig roll i det medeltida samhället. Uppskattningar av dagligt brödintag brukar hamna på runt 1-1,5 kg per person. Bland de första gillen som grundades var bagarnas och lagar och förordningar antogs ofta för att se till att brödpriserna förblev stabila. I England stiftades 1266 The Assize of Bread and Ale ("förordningen för bröd och öl") som fastslog priser för bröd efter priserna på säd och bestämde även vikt och storlek för en standardlimpa. Bagarnas vinstmarginal var också fastslagen, men bagargillena lyckades ofta med stor framgång förändra villkoren. Londonbagarnas gille lyckades till exempel med att lägga till kostnader för allt från ved och salt till bagarfrun och till och med hus och hund. Eftersom bröd var så centralt i den medeltida kosten utdelades stränga straff mot de bagare som försökte lura sina kunder. +En av de vanligast förekommande rätterna, oavsett om det var en bankett eller vardagsmåltid, var soppor och såser som var avsedda att ätas genom att man doppade (eller soppade) brödbitar i skålarna för att suga upp innehållet. Andra mycket vanliga rätter var frumenty, en tjock vetegröt kokad i köttbuljong och ofta smaksatt med kryddor, och diverse grötar på många olika sorters spannmål. Dessa kunde ofta serveras som desserter eller rätter avsedda för sjuka om de kokades i mjölk eller mandelmjölk och sötades. Pajer med fyllningar av kött, ägg, grönsaker och frukt var vanliga över hela Europa, och likaså olika degknyten, munkar och liknande bakverk. Mot senmedeltiden blev kex, och särskilt rån, populära till efterrätten hos överklassen och förekom i ett otal varianter. Spannmål, antingen i form av smulat bröd eller mjöl, var också det vanligaste sättet att reda soppor och stuvningar, ofta i kombination med mandelmjölk. +Bröd användes till mer än att bara äta; även om trä eller metall (oftast tenn) användes för att tillverka tallrikar så var det långt in på 1500-talet vanligt att några dagar gammalt enklare och grövre bröd skars till en fyrkantig form och användes för att lägga upp maten på. Bröd användes också för att torka av knivar innan man skickade dem vidare till nästa middagsgäst eller när man skulle ta salt ur de gemensamma saltkaren. Erfaret tjänstefolk kunde använda tjocka brödskivor som grytlappar, skickligt dolda från kräsna middagsgästers oförsonliga blickar. +Även om de flesta måltider till största del bestod av spannmålsprodukter, var grönsaker som kål, betor, lök, vitlök, rovor och morötter mycket vanliga bland de lägre samhällsskikten. Medeltida kokböcker, som främst var avsedda för dem som hade råd med den typen av lyx, hade relativt få recept som innehöll grönsaker, och så gott som alla var någon form av tillbehör till kötträtter eller någon typ av potage. Morötter fanns i två olika varianter på medeltiden. En läckrare röd-lila sort och en lägre ansedd grön-gul. Den morot med orange färg som är vanligast idag odlades fram i Nederländerna först mot 1600-talet. Diverse baljväxter såsom kikärter, bondbönor och ärter var vanliga och viktiga proteinkällor. Med undantag för ärtor ansågs samtliga av den dåtida medicinvetenskapen vara onyttiga, till viss del på grund av att de orsakade gasbildning. Hur viktiga grönsaker kunde vara för vanligt folk framgår tydligt av källor. I delar av Tyskland på 1500-talet åt bönder surkål tre till fyra gånger om dagen. +Frukt var vanligt förekommande och kunde serveras färsk, torkad eller konserverad, och var en populär ingrediens i en rad olika maträtter. Läkarvetenskapen avrådde från att äta rå frukt, eftersom man ansåg att deras humorala egenskaper var för kalla och fuktiga, och att de helst skulle tillagas på något sätt först. Eftersom både honung och socker var mycket dyrt, var det vanligt att använda frukter för att söta mat. De vanligaste frukterna i söder var citroner, suckatcitroner, bitterapelsiner (den söta varianten blev populär först flera hundra år senare), granatäpplen, kvitten och förstås druvor. Norrut var äpplen, päron, plommon, körsbär och smultron vanligare. Fikon och dadlar åts i hela Europa, men var förhållandevis dyra importvaror i norr. +Vanliga och idag mycket grundläggande ingredienser i modern europeisk matlagning som potatis, kidneybönor, kakao, vanilj, tomat, chilipeppar och majs kom inte till Europa förrän efter upptäckten av Amerika och det tog lång tid för många av de nya grödorna att bli allmänt accepterade. +Mjölk var en viktig källa för djurprotein för dem som inte hade råd med kött eller fisk. Den kom främst från kor, men även getter och får var vanliga som mjölkdjur. Färsk mjölk konsumerades däremot inte av vuxna människor förutom som sjukmat och var främst något som förknippades med små barn och gamla. Fattiga vuxna drack ibland filmjölk eller vassle eller någon form av syrad mjölkprodukt blandad med vatten. Färsk mjölk var överhuvudtaget mer ovanligt än andra mjölkprodukter eftersom den tekniska utvecklingen inte nått det stadium där det var möjligt att bevara den någon längre tid. Mjölk kunde ibland användas i överklassens kök i stuvningar, men eftersom det var svårt att hålla sig med stora mängder av det så användes hellre mandelmjölk istället. +Ost var ett långt viktigare livsmedel bland mjölkprodukterna, särskilt bland allmogen, och historiker har menat att det under vissa perioder kunde varit den enskilt viktigaste källan till djurprotein för de lägre klasserna. Det fanns en stor mängd olika typer av ostar, varav många ännu finns: nederländsk edamer, nordfransk brie, norditaliensk parmesan. Det fanns också ostar som gjordes på vassle, som ricotta, egentligen en biprodukt av tillverkning av hårdostar. Ost användes i matlagning för pajer och soppor, det senare särskilt populärt i tysktalande områden. Smör var en annan viktig mjölkprodukt som var särskilt vanlig i de delar av Nordeuropa som specialiserade sig på boskapsskötsel under senare delen av medeltiden, och då i synnerhet Nederländerna och södra Skandinavien. Medan andra delar av Europa främst använde olja, flott eller ister som matlagningsfett så var smöret mycket dominerande i dessa regioner. Det möjliggjorde en lukrativ export av smör från 1100-talet och framåt. +Den största andelen av köttet kom från husdjur, även om vilt var hett eftertraktat bland de som kunde få rättigheten att jaga. Nötkött var inte lika vanligt som idag eftersom nötkreatur krävde relativt stor arbetsinsats, betesmarker och foder, och oxar och kor var högt värderade som dragdjur och mjölkproducenter. Nötdjur slaktades först när de blivit uttjänta och var då inte särskilt aptitliga som kött. Fläsk var betydligt vanligare eftersom svin krävde mindre arbetsinsats och billigare foder. Tamsvin kunde springa fritt även i större samhällen och kunde matas med nästan vilket köksavfall som helst, och spädgris var en eftertraktad delikatess. Lamm och får var också vanligt, särskilt i områden med stor ullindustri. Till skillnad från de flesta i-länder idag åt man så gott som alla delar av ett djur, inklusive öron, svans, tunga och livmoder. Inälvor, urinblåsa och mage kunde användas till att göra korvskinn och liknande höljen, bland annat för att göra fantasirätter som stora jätteägg. Även djur som idag är ovanliga, som igelkott och jordpiggsvin, förekom i en del medeltida receptsamlingar. +En mängd olika fåglar tjänade som mat, inklusive svan, påfågel, vaktel, rapphöna, stork, trana, lärka och i princip alla typer av vildfåglar som kunde jagas. Svanar och påfåglar hölls ofta som husdjur men var begränsade till eliten och var prisade för sitt majestätiska yttre snarare än för god smak (båda fåglarna anses enhälligt vara nästintill oätliga av moderna gastronomer); de var båda mycket populära att servera som en entremet, en sorts prydnadsrätt som var gjord för att underhålla mer med utseende än smak. Gäss och ankor var sedan länge domesticerade, men var inte fullt lika vanliga som kyckling, fågelmotsvarigheten till det allestädes förekommande svinet. Underligt nog klassades den vitkindade gåsen inte som fågel eftersom den allmänna tron var att den utvecklades inuti havstulpaner istället för att lägga ägg som andra fåglar. Den ansågs därför vara tillåten som mat på fastedagar. +Även om kött innehåller mer protein, var priset räknat i hur mycket energi man fick per kilo betydligt högre än för vegetabilisk mat. Kött kunde vara upp till fyra gånger dyrare än bröd. Detta innebar att fastedagar kunde innebära en synnerligen mager diet för dem, som inte hade råd med alternativ till kött, mjölk och ägg. Det var först efter att digerdöden i mitten på 1300-talet hade utplånat upp till hälften av den europeiska befolkningen som kött blev mer tillgängligt för gemene man. Den demografiska chock som följde på pandemin orsakade brist på arbetskraft, vilket betydde att lönerna steg. Stora åkerarealer övergavs också, eftersom det inte fanns tillräckligt med människor för fortsatt odling, och istället användes dessa som betesmark som ökade köttproduktionen markant. +Även om fisk inte hade samma status som kött, och ofta bara sågs som ett alternativ till kött på fastedagar, så var det en väsentlig del av många befolkningars kost. "Fisk" för den medeltida människan var även ett brett begrepp för allt som avvek från vanliga landlevande djur, inklusive marina däggdjur som valar. I denna kategori inkluderas även bäverns svans, på grund av att den hade vad som såg ut som fjäll och levde stora delar av sitt liv i vatten, och även den vitkindade gåsen, eftersom man inte visste vart den flyttade om vintrarna. Alla dessa djur ansågs som tillåten fastemat. Särskilt viktig var fisket av strömming, sill och torsk i Atlanten och Östersjön. Handeln med sill och strömming utgjorde stommen för stora delar av ekonomin i norra Europa och var en Hansans vanligaste handelsvaror; rökt sill som fiskats i Nordsjön kunde återfinnas på marknader så långt borta som Konstantinopel. Stora mängder fisk åts färsk och resten saltades, torkades eller röktes. Stockfisk, torsk som fläkts upp, spänts upp på en påle och sen hängts på tork var oerhört vanligt, även om tillagning kunde vara väldigt omständlig och krävde att man mörade fisken med en träklubba innan den lades i blöt för att sen tillagas. En stor mängd blötdjur (ostron, musslor och pilgrimsmusslor) åts av folk som bodde vid kusten och längs floder och färskvattenkräftor var populärt på fastedagar. I jämförelse med kött var fisk mycket dyrare för inlandsbefolkningen, särskilt i Centraleuropa. Färskvattenfisk som gädda, karp, braxen, abborre, nejonöga och forell var också vanliga. +Idag anses vatten vara en vanlig och neutral måltidsdryck. På medeltiden stod vatten betydligt lägre i rang på grund av oro över renhet, avrådan från dåtidens läkare och dess låga anseende. Alkoholhaltiga drycker var betydligt vanligare och ansågs nyttigare och bättre för matsmältningen än vatten. Alkoholinnehållet gjorde också att de höll sig bättre. Vin dracks dagligen av merparten av befolkningen i större delen av Frankrike och Sydeuropa där man odlade druvor. Längre norröver förblev det måltidsdrycken för den medelklass och adel som hade råd, och mycket ovanligare bland fattigare bönder och arbetare. Allmogedrycken i norr var främst öl. På grund av bristen på någon tillförlitlig konserveringsteknik för drycker vid den här tiden (särskilt innan humle blev vanligt), dracks det förhållandevis färskt. Det var därför grumligare och av lägre alkoholhalt än de flesta moderna ölsorter. Mjölk dracks inte vanligtvis av vuxna med undantag för de fattiga eller sjuka, och brukade vara förbehållet de mycket unga eller gamla, och då oftast i form av filmjölk eller vassle. Färsk mjölk var överlag mindre vanligt än andra mjölkprodukter eftersom den moderna kyl- och pastöriseringsteknik som krävs för att förhindra mjölk från att härskna inte uppfanns förrän på 1900-talet. Te och kaffe, båda utvunna ur växter som fanns i den Gamla världen, var populära i Ostasien och den muslimska världen under medeltiden. Dessa introducerades dock inte i Europa förrän under sent 1500-tal och tidigt 1600-tal. +Saft såväl som viner av ett otal frukter och bär var kända sen antikens Rom: cotignac på mispel eller kvitten, vin på granatäpple, mullbär, björnbär. Cider och must var särskilt vanligt i norr där äpplen och päron var vanligt förekommande. Många medeltida drycker har överlevt i modern tid, som prunellé, plommonvin som idag är bäst känt som slivovits, mullbärsvin och björnbärsvin. Många olika recept av mjöd har återfunnits, med eller utan alkohol, men mot senmedeltiden blev denna honungsbaserade dryck allt ovanligare som måltidsdryck och begränsades senare till en hälsodryck som läkare kunde rekommendera för diverse åkommor. Kumys, jäst häst- eller kamelmjölk, var känt i Europa, men var även den begränsad till medicinska ändamål. Mjöd har ofta framställts som något av en allmogedryck bland de slaviska folken, vilket är en sanning med modifikation. Mjöd hade stort symboliskt värde vid högtider som dop och bröllop samt när avtal slöts mellan hövdingar, kungar och adelsmän. Mjöd förblev en dyr produkt och hade i det medeltida Polen en status som var likställd med kryddor och utländska viner. +Vin var en av de vanligaste dryckerna och ansågs vara både nyttig och åtnjöt hög prestige. Enligt Galenus teorier om näringslära var det varmt och torrt (därav den moderna beskrivningen av viner som "torra"), något som dock skulle balanseras genom att man spädde med vatten. Till skillnad från vatten och öl, som ansågs kalla och fuktiga, ansågs måttliga mängder vin, särskilt rödvin, bland annat befrämja matsmältning, ge bra blod och öka livslusten. Kvaliteten på vin skilde sig markant beroende på lagring, druvsort och, allra främst, hur många gånger det pressades. Den första pressningen användes till de finaste och dyraste vinerna som dracks av överklassen. Den andra och i vissa fall tredje pressningen var av lägre kvalitet och hade mycket lägre alkoholhalt. Allmogen fick för det mesta nöja sig med ett billigt rosévin gjort på andra eller tredje pressningen, vilket betydde att det kunde drickas i stora mängder utan att man blev särskilt berusad. För de allra fattigaste människorna (eller de gudfruktigaste asketerna) var utspädd vinäger ofta det enda alternativet. +Lagring av rödvin av hög kvalitet krävde specialkunskaper såväl som dyr utrustning och lagerutrymmen, och var därför dyrare. Att döma från de många råd som finns i medeltida skrifter om hur man kunde rädda vin som var på väg att surna till vinäger så måste lagring ha varit ett stort problem. Även om vinäger var en vanlig och populär ingrediens så fanns det en gräns för hur mycket man kunde ha användning av. I Le Viandier, en kokbok från sent 1300-tal finns flera metoder för hur man kan vända försurningsprocessen; att både se till att vinfaten var fyllda ända upp till kanten och att blanda i torkade och kokade druvkärnor ihop med askan av torkad och bränd jäst från produktion av vitt vin var effektivt mot bakterier, även om man inte förstod de bakomliggande kemiska processerna vid den tiden. +Kryddat vin och glögg var inte bara populärt bland de rika utan ansågs även särskilt nyttigt av dåtida läkare. Vin sågs som en effektiv förmedlare av näringsämnen till alla delar av kroppen och tillsatsen av väldoftande exotiska kryddor ansågs öka dess välgörande egenskaper. Kryddade viner gjordes ofta på rött vin med en blandning av kryddor som ingefära, kardemumma, svartpeppar, paradiskorn, muskotnöt, nejlikor och socker. Dessa lades ner i en liten tygpåse som sen lades i blöt för att framställa hypocras, den medeltida termen för kryddat vin som fick sitt namn efter den västerländska läkekonstens grundare Hippokrates, och vid 1300-talet kunde färdiga blandningar med kryddor köpas av kryddhandlare. +Vin var den vanligaste måltidsdrycken i stora delar av Europa, men inte i de regioner där druvor inte kunde odlas. De som hade råd med det drack importerade viner, men även bland adeln i dessa områden dracks öl dagligen, särskilt mot slutet av medeltiden. I England, det som idag är Nederländerna och Belgien, norra Tyskland, Polen, Norden och Östeuropa drack man öl dagligen i så gott som alla samhällsklasser. Trots detta så innebar det stora inflytandet av vetenskapsmän från Arabien och länderna kring Medelhavet, att öl stod lågt i rang på näringsläran. För de flesta medeltida européer var det en enkel dryck i jämförelse med de vanliga dryckerna och ingredienserna i Sydeuropa, som vin, citrusfrukter och olivolja. Även synnerligen exotiska råvaror som kamelmjölk och gasellkött fick mer positiva omdömen i dåtida medicinska texter. Öl var ett acceptabelt alternativ och ansågs ha en rad negativa egenskaper. 1256 beskrev läkaren Aldobrandino av Siena öl med följande ord: +Ölets berusande effekt ansågs vara lägre än vin, men det medgavs också att det inte framkallade den "falska törst" som ofta associerades med vin. Även om det var mindre vanligt än i norr så dracks öl även i norra Frankrike och på Italiens fastland. I Le Menagier de Paris, en handbok i hushållning från 1300-talet, finns ett recept på godale (troligen ett direktlån från engelskans good ale; "gott öl") som gjordes på korn och dinkel, men utan humle. I England fanns det även varianter på så kallat poset ale som gjorde på varm mjölk och kallt öl och brakot, en form av kryddat öl som framställdes på samma sätt som hypocras. +Att humle kunde användas för att smaksätta öl var känt sen Karolingerna, men antogs endast gradvis på grund av svårigheterna i att få fram de rätta proportionerna. Innan man upptäckte att humle kunde användas, smaksatte man öl med pors och andra örter. Dessa örter hade inte samma konserverande effekt som humle och slutprodukten höll sig inte länge och dracks därför snart efter bryggning. En annan metod för att göra att ölet höll sig längre var att öka alkoholhalten, men detta var dyrare och gav ölet bieffekten av att det berusade för snabbt. Humle kan ha varit i allmänt bruk i England under 900-talet och det är bekräftat att det odlades i Österrike 1208 och i Finland 1249, men troligen även långt tidigare. +Under tidig medeltid bryggdes öl främst av kloster och i mindre skala av enskilda hushåll. Mot högmedeltiden tog bryggerier i de växande städerna i norra Tyskland stegvis över det mesta av ölproduktionen. Bryggerierna var ofta relativt små företag om 8-10 personer, men regelbunden produktion betydde att man kunde investera i bättre utrustning och experiment med nya recept och bryggeritekniker. Den nordtyska produktionen spred sig till Holland på 1300-talet, sen vidare till Flandern och Brabant och nådde England på sent 1400-tal. Öl med humle blev oerhört populärt under de sista årtiondena av senmedeltiden. I England och lågländerna låg konsumtionen på 275-300 liter per person och år och dracks till så gott som alla måltider; svagt öl till frukost och successivt starkare senare på dagen. När man fulländat användningen av humle, kunde öl hålla sig i sex månader eller mer och gjorde att man kunde inleda en omfattande export. +Destillation var känt redan i forntida Kina och även antikens greker och romare kände till tekniken, men storskaligt bruk började inte förrän på 1100-talet när arabiska innovationer på området i kombination med destilleringskolvar av glas effektiviserade processen. Den medeltida vetenskapen ansåg att destillation framställde essensen av ett ämne och den latinska termen aqua vitae ("livets vatten") användes för alla former av destillat ända in på 1600-talet. I början kunde destillerade vätskor ha en rad olika användningsområden; reducerad druvsaft blandat med socker rekommenderades mot diverse åkommor och rosenvatten användes som parfym, i matlagning och till handtvätt vid finare tillställningar. Destillat av alkohol kunde också användas till att göra en eldsprutande entremet (en sorts underhållningsrätt) genom att doppa en bit bomull i sprit. Bomullen stoppades sen i munnen på djur vars kött hade tillagats separat och sen klätts i djurets skinn eller fjäderskrud för att ge illusionen av att det fortfarande levde; bomullen tändes på och rätten serverades som ett fantasifullt fabeldjur. +Aqua vitae stod högt i anseende bland läkare. 1309 skrev Arnaldus de Villanova att "[d]et befrämjar god hälsa, skingrar överflödiga vätskor, ger hjärtat vederkvickelse och bevarar ungdomen". Under senmedeltiden ökade produktionen av hembränt, särskilt i tysktalande regioner, och redan på 1200-talet var Hausbrand (tyska för "hembränt", av gebrannter wein, brandwein, "brännvin") vanligt förekommande. Mot slutet av medeltiden blev brännvin så vanligt även hos gemene man att man började stifta förordningar för att begränsa försäljning och framställning; 1496 infördes i Nürnberg förbud mot all försäljning av aquavit på söndagar och andra helgdagar. +Kryddor var bland de förnämsta lyxvaror som fanns under medeltiden, och de vanligaste importerade kryddorna var svartpeppar, kanel (och det billigare alternativet cassia), kummin, muskotnöt, ingefära och kryddnejlika. Alla dessa kryddor importerades från plantager i Asien och Afrika, något som gjorde dem väldigt dyra. Det har uppskattats att runt 1000 ton peppar och 1000 ton av de vanligaste kryddorna importerades till Västeuropa varje år under senmedeltiden. Värdet av dessa kryddor var motsvarande en årsförbrukning av spannmål för 1,5 miljoner människor. Även om peppar var den vanligaste importerade kryddan, så var saffran den mest exklusiva och användes lika mycket för dess intensiva gulröda färg som för sin smak. Enligt den tidens näringslära ansågs gul signalera att den var varm och torr, vilket allmänt ansågs välgörande. Gurkmeja var en billigare ersättare och inslag av förgyllning av mat vid banketter tillfredsställde både förkärleken för uppseendeväckande prakt och tron på humoralpatologi. Vid en storslagen bankett som anordnades av kardinal Pietro Riario i juni 1473 åt dottern till kungen av Neapel var till och med brödet förgyllt. +Få maträtter använde endast en krydda eller ört utan kombinerade en mängd olika smaksättare. Även när en rätt dominerades av en specifik smak så var det ett resultat av en sammansättning av till exempel persilja och kryddnejlika eller peppar och ingefära. Många kryddor användes också som idag är sällsynta i Europa: paradiskorn, en släkting till kardemumma, ersatte nästan helt svartpeppar i nordfransk matlagning under senmedeltiden, långpeppar, galangarot och kubeberpeppar. Socker ansågs på medeltiden vara en krydda på grund av dess höga kostnad och dess humorala egenskaper (varmt och torrt). +Salvia, senap och i synnerhet persilja var mycket vanliga lokala kryddor och användes i matlagning över så gott som hela Europa, och likaså mynta, kummin, dill och fänkål. Anis användes ofta för att smaksätta fisk- och kycklingrätter och fröna användes till att göra karameller. Örter växte vilt och odlades lite varstans och var överkomliga även för de fattigaste. De användes även i överklassmat, men var i allmänhet mindre framträdande och användes ofta enbart för att färga maten. En vida spridd modern myt är att medeltida kockar använde rikliga mängder kryddor, särskilt svartpeppar, för att dölja smaken av ruttet kött. Eftersom en medeltida bankett var en kulinarisk höjdpunkt samt ett tillfälle för värden att visa upp sin generositet och rikedom och eftersom de flesta adelsmän hade tillgång till jaktmarker och fiskdammar, boskap och torkat, saltat eller rökt kött - så skulle det ha ansetts mycket underligt om man slösade dyra kryddor på att försöka dölja smaken av härsket kött. +I medeltida recept var det vanligt att använda en rad olika sura vätskor. Vin, verjus (saften av omogna druvor eller frukter), vinäger och saften av olika sura frukter var utmärkande särskilt för senmedeltida matlagning. Tillsammans med sötningsmedel och kryddor gav det en "pungeant, fruity" smak. Lika vanligt var att dessa distinkta smaker kompletterades med sötmandel. Den används i en rad olika former: hela, skalade eller oskalade, strimlade, malda och, framförallt, i form av mandelmjölk. Just mandelmjölk var en av de vanligaste ingredienserna i senmedeltida matlagning och kompletterade skarpa kryddor och sura vätskor med sin milda smak och lena konsistens. +Salt var en oundgänglig del av medeltida mat. Att salta och torka var den vanligaste formen av konserveringsteknik och ledde till att både kött och fisk ofta var kraftigt saltade. Många medeltida recept varnar uttryckligen för att inte salta för mycket och det fanns rekommendationer att blötlägga vissa livsmedel i vatten under en längre tid för att laka ur dem. Saltet var också närvarande vid mer påkostade måltider. Ju rikare värd eller högstående gäst, desto finare saltkaren och salt. Värden och gäster av hög rang satt på samma nivå som de bästa saltkaren medan övriga gäster och tjänare placerades rent fysiskt lägre i rummet för att markera statusskillnaden. Dyra saltkar var ofta tillverkade av tenn, ädelmetaller eller andra exklusiva material, och rikt utsmyckade. Gästernas rang dikterade också hur finmalt och vitt saltet var. Salt avsett för matlagning, konservering och det som användes av vanligt folk var ofta mycket grovt och orenat. Särskilt havssalt hade mycket orenheter och kunde vara av många olika färger, från svart till grönt. Det dyraste saltet såg ungefär ut som vanligt bordssalt gör idag. +Termen "dessert" kommer från fornfranska desservir, "duka av", eller bokstavligt översatt "av-servera" och har sitt ursprung i medeltiden. Den bestod oftast av dragéer och kryddade viner tillsammans med lagrad ost och kunde mot senmedeltiden även inkludera färsk frukt med socker, honung eller reducerad fruktsaft. Det fanns en stor mängd friterade bakverk och crêpes med socker, söta puddingar och dariole, degformar med en fyllning av mandelmjölk, ägg, frukt och ibland till och med benmärg och fisk. Tysktalande områden hade en särskilt förkärlek för krapfen; stekta eller friterade bakverk med allehanda fruktfyllningar. Marsipan i många tappningar var välkänt i Italien och södra Frankrike vid mitten av 1300-talet och är troligen av arabiskt ursprung. Anglonormandiska kokböcker är fulla av recept på söta och läckra puddingar, soppor, såser och tarteletter med jordgubbar, körsbär, äpplen och plommon. De engelska kockarna hade också en förkärlek för att använda blomblad från rosor och fläder. En tidig föregångare till quiche förekommer i Forme of Cury, en receptsamling från sent 1300-tal i form av en Torte de Bry, en tartelett med ägg- och ostfyllning. +I norra Frankrike åt man många varianter av våfflor och rån tillsammans med ost och hypocras, eller ett sött malvoisevin till issue de table ("sorti från bordet"). Kanderad ingefära, anisfrö och andra kryddor var allestädes närvarande vid finare middagar och var kända som épices de chambre ("kammarkryddor") och åts för att underlätta matsmältningen mot slutet av en måltid som ett sätt att "stänga" magen. Genom muslimerna i Al-Andalus och på Sicilien kom kristna européer i kontakt med en helt ny värld av sötsaker som senare spreds till resten av Europa. Precis som Montpellier i Languedoc var Sicilien känt för sin socker- , nougat- och mandelkonfekt. Araberna introducerade också konsten att göra glass och sorbet och många olika söta kakor och bakverk; cassata alla Siciliana (från arabiska qas'ah, ordet för den terracottaskål i vilken den formades) gjord på marsipan, sockerkaka och sötad ricotta och cannoli alla Siciliana, ursprungligen capelli di turchi ("turkiska hattar") som består av stekta, kylda degrör med en söt ostfyllning. +Huvudkosten i Medelhavskulturerna hade sedan antiken bestått av spannmål, i synnerhet olika former av vete. Gröt och välling och senare bröd utgjorde huvuddelen av kaloriintaget för större delen av befolkningen. Från 700- till 1000-talet ökade också andelen spannmål i kosten från cirka en tredjedel till över tre fjärdedelar och bröd förblev den mest grundläggande stapelfödan i Europa i flera hundra år framåt. Beroendet av vete kvarstod långt in i modern tid och förstärktes i och med kristendomens spridning norrut. Brödets centrala roll i religiösa ritualer som nattvarden innebar att den åtnjöt en hög prestige som livsmedel. Endast (oliv)olja och vin hade en liknande status, men förblev också mer exklusiva utanför de varmare områden där vindruvor och oliver kunde odlas framgångsrikt. Brödets symboliska roll som föda och substans i sig illustrerades väl i en predikan av kyrkofadern Augustinus: +Den katolska och ortodoxa kyrkan och deras religiösa kalendrar hade ett stort inflytande på medeltida matvanor; kött (men inte fisk) var förbjudet i närmare en tredjedel av året för de flesta kristna, och även andra animalier som ägg och mjölkprodukter var i regel inte tillåtna att äta på fastedagar. Kyrkan gav ofta dispens i områden där vegetabiliska alternativ inte fanns att tillgå eller helt enkelt var för dyra för att importera (till exempel var lunnefågeln klassad som fisk för norska fiskare). Barn, gamla, pilgrimer, arbetare och tiggare var undantagna från de flesta fastebestämmelser, men inte för fattiga så länge de hade tak över huvudet. Utöver det brukade man fasta innan man mottog nattvarden, och dessa sträckte sig ofta över en hel dag och krävde att man helt avstod från mat. +Kristendomen hade en tydligt påverkan på medeltidens matvanor. Såväl den östra (ortodoxa) som den västra (katolska) kyrkan föreskrev att högtid skulle varvas med fasta. I större delen av Europa var onsdagar, fredagar, i vissa fall lördagar och många specifika datum (som fastan och advent) fastedagar. Kött och animalier som mjölk, ost, smör och ägg var förbjudna och endast fisk och vegetabilier tillåtna. Fastan var avsedd att vara en form av kroppslig självspäkning som skulle vederkvicka själen, ett sätt att befästa och förstärka den medeltida dogmen om kroppens underlägsenhet mot anden, och som en påminnelse om Kristi offer för mänskligheten. Avsikten var inte att framställa någon viss mat som oren, utan snarare att måttfullhet skulle tjäna som en andlig lärdom i självbehärskning. Under vissa fastedagar som ansågs särskilt viktiga, begränsades man till endast en måltid per dag. Trots att de flesta människor hyste respekt för fastebestämmelserna och i allmänhet gjorde bot om de bröt mot dem, fanns det många försök till att kringgå restriktionerna. En konflikt mellan ideal och praktik som sammanfattats av författaren Bridget Ann Henisch: +Även om man skulle avstå från kött och ägg under fastedagar, så var ett visst mått av pragmatism förhärskande. Definitionen av "fisk" tänjdes ut till att inkludera marina och vattenlevande däggdjur och fåglar som valar, vitkindade gäss, lunnefåglar och bävrar (dock bara svansen). Valet av ingredienser må ha varit mer begränsat, men det betydde inte att måltiderna var mindre. Inte heller fanns det några förbud mot (måttfullt) drickande eller att äta sötsaker. Festmiddagar som hölls på fastedagar kunde vara minst lika storslagna, och de var ypperliga tillfällen för att servera mat som skulle likna kött, ost och ägg genom en rad snillrika metoder; fisk kunde finhackas och formas så att det såg ut som viltkött, och ägg kunde göras genom att man fyllde urblåsta äggskal med fiskrom och mandelmjölk och rostade det i glödande kol. Medan den bysantinska kyrkan antog en hårdför linje i frågan och motarbetade all form av kulinarisk förfining under fastan, så var deras västerländska kyrkobröder mer överseende. Det finns mängder med berättelser och anekdoter om hur ordensbröder i kloster trotsade fasterestriktioner genom listiga och originella omtolkningar av bibeltexter. Eftersom sjuka var undantagna från att fasta uppstod ofta en föreställning om att bestämmelserna bara gällde för den ordinarie matsalen, och många munkar åt helt enkelt sina fastedagsmåltider i ett annat rum som senare fick namnet misericordia (latin för "barmhärtighet"). Det knotades och klagades också ständigt från lekmän över hur sträng fastan var. Från båda ändarna av medeltiden kan man läsa om hur kungar och skolpojkar, allmoge och frälse beklagade sig över de långa, svåra veckorna av dyster begrundan över de egna och mänsklighetens synder. Boskapsägare råddes också för att se upp för utsvultna hundar, som kunde drivas till desperata handlingar av "den hårda belägringen av fasta och fiskben". +Från 1200-talet och framåt fanns en tendens till en mer legalistisk tolkning av fastan. Adeln avstod pliktskyldigt från att äta kött på fastedagar, men åt ändå i lyx; fisk ersatte kött, ofta förklädd som skinka eller bacon; mandelmjölk ersatte riktigt mjölk som ett exklusivt alternativ: falska ägg gjorda på mandelmjölk kokades i urblåsta äggskal, smaksattes och fick färg av dyra kryddor. I vissa fall blev adeln överträffad av benediktinerkloster som kunde ha upp till 16 serveringar vissa fastedagar. Vilka som skulle vara ursäktade från fastan tolkades också allt mer generöst. Thomas av Aquino (ca 1225–1274) ansåg att undantag skulle finnas för barn, gamla, pilgrimer, arbetare och tiggare, men inte de fattiga så länge de hade tak över huvudet. Det finns många källor som beskriver ordensmunkar som trotsade restriktioner genom listiga bibeltolkningar. Eftersom sjuka var undantagna utvecklades en praxis att fastebestämmelserna endast gällde i den ordinarie matsalen och att många munkar helt enkelt åt sina måltider under fastan i det som senare skulle utvecklas till misericordian istället för i refektoriet. Nytillsatta klosterabbotar försökte komma tillrätta med avvikelserna inte bara med moralpredikningar utan även genom att se till att det fanns vällagade rätter utan kött vid fastemåltider. +Europa under medeltiden var ett utpräglat klassamhälle. Det var en tid då svält fortfarande var vanligt förekommande och där sociala hierarkier brutalt påtvingades. Mat var då en viktig social markör på ett sätt som inte har någon motsvarighet i moderna industrialiserade länder. Enligt den ideologi som rådde då var samhället uppdelat på tre huvudsakliga ständer: den arbetande, ofrälse allmogen (den övervägande majoriteten), prästerskapet och adeln. Förhållandet mellan dessa grupper var strikt hierarkiskt. Adeln och prästerskapet krävde andlig och världslig bestämmanderätt över allmogen. Inom de härskande klasserna fanns också en intern hierarki, från kungar och påvar till hertigar och biskopar, och vidare till deras underlydande, till exempel väpnare och vanliga präster. Alla förväntades hålla sig inom sin klass och att respektera de härskande klassernas auktoritet. Politisk makt uttrycktes inte bara genom att styra andra, utan även genom bordsskick och kost; grova arbetare skulle nöja sig med grovt kornbröd, saltat fläsk och bönor och förväntades inte behärska etikett. Klasstänkandet påverkade till och med vetenskapen: de övre klassernas dyra kost var inte bara en fråga om ekonomiska förhållanden utan en nödvändighet för deras förfinade fysik. En herres matsmältningssystem ansågs helt enkelt finare än hos underordnade bönder och krävde därför en finare kost. +Under senmedeltiden ökade rikedomen hos köpmän och annan medelklass vilket innebar att de hade möjlighet att ta efter adelns vanor och manér. Det var något som hotade att sudda ut de symboliska barriärer som fanns mellan frälse och allmoge. Reaktionen kom i två former: läkare varnade för farorna med att äta "utanför sitt stånd" och överflödsförordningar satte en juridisk gräns för hur påkostade ofrälsets festmåltider fick vara. +Medicinvetenskapen under medeltiden hade stort inflytande på vad som ansågs vara nyttig och näringsrik kost. Ens livsstil – inklusive kosthållning, motion, lämpligt socialt beteende och diverse hälsokurer – var vägen till god hälsa och all mat tilldelades vissa egenskaper som ansågs påverka hälsa. Alla matvaror klassificerades på två skalor, från varmt till kallt och från fuktigt till torrt, allt enligt humoralpatologin, en samling medicinska teorier som baserades på Galenus läror och som dominerade västerländsk medicin ändra fram till 1600-talet. +Den medeltida vetenskapen ansåg att matsmältning var en process som liknade matlagning. Spjälkningen av mat i magen sågs som en fortsättning på den tillagning som kocken påbörjat. För att maten skulle bli ordentlig "tillagad" och näringsämnena ordentligt upptagna av kroppen var det viktigt att magen fylldes på rätt sätt. Lättsmält mat skulle ätas först och sen följas av successivt tyngre rätter. Om man inte följde denna ordning så trodde man att den tyngre maten skulle sjunka till botten av magen och där blockera matsmältningskanalen och orsaka förruttnelse och uppkomsten av dåliga vätskor i magen. Det var också mycket viktigt att mat av olika egenskaper inte blandades hur som helst. +Innan en måltid skulle magen "öppnas" med en aperitif (av latinska aperire, "öppna") som skulle vara varm och torr, oftast konfekt gjord på kanderad ingefära, kummin och frön av anis, fänkål och spiskummin, vin och sötade och spetsade mjölkdrycker. Precis som den hade blivit öppnad skulle den sen "stängas" igen i slutet av måltiden med hjälp av något som ansågs underlätta matsmältningen, gärna dragéer (som på medeltiden bestod av kryddade sockerklumpar) eller hyporas, kryddat vin, tillsammans med lagrad ost. En måltid skulle helst inledas med frukt som ansågs lättsmält, exempelvis äpplen. Sen följde grönsaker som sallad och kål, trädgårdsportlak, örter, fuktiga frukter, lättare kött som kyckling eller killing tillsammans med potage och buljong. Efter det åt man tyngre kött som fläsk och nötkött, men även päron och kastanjer, som båda ansågs särskilt svårsmälta. +Den hälsosammaste maten var enligt läkare den som närmaste motsvarande människors humoralpatologiska temperament, det vill säga måttligt varm och fuktig. Maten skulle helst också vara finhackad, mald, mortlad och silad för att ingredienserna skulle blandas ordentligt. Vitt vin ansågs vara kallare än rött och detsamma antogs om röd och vit vinäger. Mjölk var måttligt varm och fuktig, men olika djur ansågs även ge mjölk med olika egenskaper. Ägg ansågs allmänt fuktig, men gulan varm medan vitan var kall. Skickliga kockar förväntades att anpassa sig till humoralläran. Även om detta ibland kunde begränsa vissa kombinationer av livsmedel fanns det fortfarande gott om utrymme för variation inom de ramar som ansågs hälsobefrämjande. +Forskning om medeltida mat och matvanor var fram till ca 1980 relativt eftersatt. Missuppfattningar och rena felaktigheter var vanliga bland historiker och finns fortfarande närvarande i den allmänna uppfattningen om medeltiden som en efterbliven, primitiv och barbarisk period. Medeltida matlagning beskrevs som motbjudande på grund av de obekanta smakkombinationerna, den upplevda frånvaron av grönsaker och stora mängder kryddor. Det rikliga bruket av kryddor har varit ett populärt argument för att bekräfta påståendet om att kryddor användes för att dölja smaken av härsket kött, en slutsats som saknar stöd i historiska fakta eller samtida källor. Färskt kött var tillgängligt året runt för de som hade råd. Konserveringstekniken var förvisso primitiv i jämförelse med idag, men fullt tillräckliga. Kryddornas astronomiska priser och höga prestige, och värdens rykte, skulle ha varit helt bortslösade om de användes för att smaksätta billiga, illa hanterade råvaror. +Att mala och blanda ingredienser till mos eller patée-liknande rätter, och de många formerna av potage och såser har använts som argument för att den medeltida adeln tappade sina tänder i ung ålder och därför inte kunde äta något annat än gröt, soppa eller malet kött. Bilden av tandlösa aristokrater som ätit sig igenom stora måltider bestående främst av olika former av mos och sörjor har samtidigt levt sida vid sida med den motstridiga föreställningen av en "hop av okultiverade tölpar (förklädda som adelsmän) som, när de inte kastat flottiga köttben på varandra genom bankettsalen, har huggit in på dem med fullt friska uppsättningar av fram-, hörn- och kindtänder". +Kokböcker, eller mer exakt receptsamlingar, som sammanställdes på medeltiden är bland de viktigaste källorna för forskning om den tidens mat och matvanor. De första kokböckerna dök upp runt slutet av 1200-talet. Liber de coquina, troligen med ursprung kring Neapel, och Tractatus de modo preparandi har redigerats och utgivits i modern tid av Marianne Mulon och ytterligare en kokbok från Assisi som hittats vid Châlons-sur-Marne har redigerats av Maguelonne Toussaint-Samat. Man har utgått från att recepten beskriver riktiga maträtter, men mathistoriker tror inte att de användes på samma sätt som idag, som en detaljerad handbok som kunde hållas nära till hand under tillagning. Troligen var få i ett medeltida kök läskunniga och "arbetsmaterial" skulle knappast ha bevarats en längre tid. +Recepten var ofta kortfattade och angav inte exakta kvantiteter. Tillagningstider och temperaturer var inte angivna eftersom det inte fanns bärbara klockor och eftersom all matlagning skedde över någon form av eld. I bästa fall kunde tillagningstider anges som tiden det tog att säga ett visst antal vanliga böner eller hur långt det tog att gå runt ett fält av en viss storlek. Yrkeskockar lärdes upp genom praktisk erfarenhet i ett lärlingssystem, och arbetade sig gradvis upp i kökshierarkin. En medeltida kock i ett större hushåll hade troligen förmågan att kunna planera och tillaga en större måltid utan att ta hjälp av skriftliga instruktioner. Eftersom de manuskript som överlevt är i relativt gott skick har mathistorikern Terence Scully föreslagit att de varit ett sätt att nedteckna vanor och praktiker för läskunniga hushållsöverhuvuden. Ett tydligt exempel på detta är Ménagier de Paris från sent 1300-tal. Totalt finns drygt 70 kända medeltida receptsamlingar som bevarats in i modern tid. +entremet +Guillaume Tirel +romerska köket +Adamson, Melitta Weiss, Regional Cuisines of Medieval Europe: A Book of Essays. Routledge, New York. 2002. ISBN 0-415-92994-6 +Adamson, Melitta Weiss, Food in Medieval Times. Garland, New York. 2004. ISBN 0-313-32147-7 +Carlin, Martha & Rosenthal, Joel T. (redaktörer), Food and Eating in Medieval Europe. The Hambledon Press, London. 1998. ISBN 1-85285-148-1 +Dembinska, Maria, Food and drink in medieval Poland: rediscovering a cuisine of the past, översatt av Magdalena Thomas, reviderad och bearbetad av William Woys Weaver. University of Pennsylvania Press, Philadelphia. 1999. ISBN 0-8122-3224-0 +Fenton, Alexander & Kisbán, Eszter (redaktörer), Food in Change: Eating Habits from the Middle Ages to the Present Day. John Donald Publishers, Edinburgh. 1986. ISBN 0-85976-145-2 +The Fontana Economic History of Europe: The Middle Ages; J.C Russel Population in Europe 500-1500. Fontana, London. 1972. ISBN 0-00-632841-5 +Henisch, Bridget Ann, Fast and Feast: Food in Medieval Society. Pennsylvania State Press, University Park. 1976. ISBN 0-271-01230-7 +Hunt, Edwin S. och Murray, James M. Företagande och handel i det medeltida Europa 1200-1550 översättning av Margareta Eklöf med fackgranskning av Bo Franzén. 2001. ISBN 91-7150-815-5 +Nordberg, Michael, Den dynamiska medeltiden. 1984. ISBN 91-550-2786-5 +Scully, Terence, The Art of Cookery in the Middle Ages. The Boydell Press, Woodbridge. 1995. ISBN 0-85115-611-8 +Nordberg, Michael (1984) Den dynamiska medeltiden ISBN 91-550-2786-5 +Harrison, Dick (2002) Jarlens sekel: en berättelse om 1200-talets Sverige ISBN 91-7324-980-7 +Serra, Daniel & Tunberg Hanna (2009) "En sås av ringa värde och andra medeltida recept" ISBN 978-91-977529-1-6 +Le Viandier de Taillevent - En engelsk översättning av en kokbok från 1300-talet James Prescott +How to Cook Medieval - En guide för hur man lagar medeltidsmat med moderna ingredienser +The Forme of Cury - En engelsk kokbok från sent 1300-tal, från Projekt Gutenberg +Lista med fler länkar om bl.a. medeltida mat +Robert Catesby, född cirka 1572 i Lapworth, Warwickshire, död 8 november 1605 i Holbeche House, var en av de katolska konspiratörerna som försökte lönnmörda Jakob I av England genom att spränga det engelska överhuset i Westminsterpalatset, i vad som kallas för krutkonspirationen 1605. Catesby var hjärnan bakom och ledaren för konspirationen. +Catesby sades ha haft en vild personlighet som ung. Catesby var katolik, men han gifte sig med Catherine Leigh som var protestant. När Leigh dog kan han i sin sorg ha blivit mer fanatisk i sin religionsutövning. Catesby, som var missnöjd med kung Jakobs sätt att styra landet, började planera en konspiration och värvade flera medlemmar till denna under 1604 och 1605. Konspirationen skulle nå sin kulmen den 5 november 1605 i och med sprängningen av det engelska överhuset i Westminsterpalatset i samband med parlamentets öppnande. Dock misslyckades Guy Fawkes, som skulle tända stubintråden till sprängladdningarna, med sitt uppdrag och hela konspirationen gick om intet. +Så fort Catesby och de andra konspiratörerna fick reda på konspirationens misslyckande flydde de fältet. Den 7 november nådde de Holbeche House, på gränsen till Staffordshire. Morgonen därpå anlände Richard Walsh, som var High Sheriff of Worcestershire, tillsammans med en styrka på omkring tvåhundra man till Holbeche House och en eldstrid bröt ut. Catesby och hans vän Thomas Percy sköts ihjäl av samma skott. Kort efter att Catesby hade begravts exhumerades och halshöggs han. Hans huvud skickades till London där det visades upp för allmän beskådan. +Robert Catesby föddes runt 1572 i Lapworth, Warwickshire som den tredje, och enda överlevande, sonen till sir William Catesby och Anne Throckmorton. Catesby var ättling i direkt nedstigande led till sir William Catesby (1450–1485), som var en inflytelserik rådsmedlem under Rikard III av Englands tid som kung och som under slaget vid Bosworth Field infångades och senare avrättades. På sin mors sida härstammade Catesby från sir Robert Throckmorton vid Coughton Court och hans andra fru Elizabeth Hussey. Catesbys föräldrar var framstående katoliker; hans far hade under åratal suttit fängslad för sin tro och 1581 hade han genomgått en rättegång i Stjärnkammaren (tillsammans med William Vaux, 3:e baron Vaux av Harrowden och hans svåger sir Thomas Tresham) för att de hade hyst jesuiten Edmund Campion. Den högst uppsatte inom familjen Throckmorton, sir Thomas Throckmorton, blev bötfälld och fick även han sitta fängslad för sin tro. En annan av Catesbys släktingar, sir Francis Throckmorton, avrättades i juli 1584 för sin inblandning i planerna att befria den då fängslade Maria Stuart. +1586 utbildade sig Catesby vid Gloucester College i Oxford, vilket vid tidpunkten var ett college som var känt för sin stora andel katolska studenter. De som ville studera på universitet var först tvungna att svära Oath of Supremacy, något som gick emot Catesbys katolska tro. Möjligtvis för att försöka undvika att svära denna ed, och få utstå konsekvenserna av detta, lämnade Catesby Gloucester College utan att ha tagit sin examen och han kan sedan ha valt att studera vid det teologiska seminariet i Douai, Frankrike. +1593 gifte sig Catesby med Catherine Leigh, dotter till sir Thomas Leigh vid Stoneleigh, Warwickshire. Catherine var uppfostrad i en välbärgad protestantisk familj och hon hade med sig en hemgift på £2 000. Tack vare att Catesby gifte sig med en protestant ansågs hans tidigare brott mot Englands strikta religionslagar inte vara lika grova. Efter att en av Catesbys släktingar hade dött året därpå ärvde han ett gods vid Chastleton i Oxfordshire. Parets första son, William, dog när han bara var ett spädbarn, men deras andra son Robert överlevde och döptes den 11 november 1595 i Chastletons protestantiska kyrka; han togs senare till Ashby St. Ledgers, Northamptonshire och gifte sig med Thomas Percys dotter. När Catesbys far, sir William Catesby, dog 1598 ärvde hans fru deras gods i Ashby St. Ledgers medan Catesby och hans familj stannade kvar i Chastleton. Catesby verkade trivas med att agera som protestant offentligt medan han i hemlighet fortfarande var katolik, men när hans fru Catherine dog kan han i sin sorg ha blivit mer fanatisk i sin religionstro. +1601 slöt Catesby upp i Robert Devereux, 2:e earl av Essex uppror mot Elisabet I av England. Lord Essex var enbart intresserad av sina egna ambitioner, men jesuiten Henry Garnet hävdade att flera av de unga män som slöt upp bakom honom gjorde så av katolska skäl. Upproret misslyckades och en skadad Catesby infångades och fängslades i Wood Street Counter samt tvingades böta 4 000 mark. Sir Thomas Tresham hjälpte Catesby att betala en del av sin skuld, vilket följdes av att Catesby sålde sitt gods i Chastleton. Flera författare spekulerar om varför Catesby flyttade när drottning Elisabets hälsa försämrades; han ansågs förmodligen vara ett hot mot den sittande regeringen, som fruktade ännu ett uppror, och i mars 1603 kan han ha skickat Christopher Wright (förmodligen under dennes alias Anthony Dutton) till Spanien för att undersöka om Filip III av Spanien skulle fortsätta ge sitt stöd åt de engelska katolikerna. Catesby gav ekonomiskt stöd till flera jesuiter och när han besökte dem använde han sig ibland av sitt alias Mr. Roberts. +När Jakob efterträdde drottning Elisabet som kung verkade han agera på ett mer tolerant sätt mot olika trosuppfattningar än vad Elisabet tidigare gjort och de katolska engelsmännen hoppades på att de skulle slippa bli förföljda på grund av sin tro. Catesby var missbelåten med Jakobs utövande av kungamakt och planerade därför att lönnmörda kungen i ett sprängdåd mot det engelska överhuset i Westminsterpalatset. I det följande upploppet skulle Catesby hetsa folket att göra revolt och få dem att kröna Elisabet Stuart till drottning. Författaren Antonia Fraser jämförde Catesby med en korsriddare, i den mån att även han verkade orädd vid tanken att kämpa för sin trosuppfattning. Den jesuitiske prästen Oswald Tesimond beskrev Catesby som en man med ett ståtligt ansikte, som hade en imponerande retorisk förmåga och som var inflytelserik över sina vänner. En av de andra konspiratörerna, Ambrose Rookwood, har sagt att han älskade och respekterade Catesby mer än någon annan man i världen och fader John Gerard har sagt att Catesby var djupt respekterad bland alla sina vänner. Författaren Mark Nicholls antydde att "bitterheten vid Essex misslyckade uppror [mot Elisabet] verkar icke desto mindre ha förstärkt en redan finslipad neuros [hos Catesby]." +Trots att det var Catesby som utarbetade planerna för krutkonspirationen hamnar han ofta i skuggan av Fawkes, som har blivit ansiktet för konspirationen. Det exakta datumet då Catesby satte igång med planerna är okänt, men det är sannolikt att han började få de första idéerna tidigt under 1604. Någon gång i juni 1603 besöktes Catesby av sin vän Thomas Percy, som var barnbarnsbarn till Henry Percy, 4:e earl av Northumberland. Percy ska ha haft en tuff uppväxt innan han blev katolik och i slutet av Elisabets tid som drottning gavs han ett hemligt uppdrag av Henry Percy, 9:e earl av Northumberland att bege sig till kung Jakob i Skottland för att försöka få honom att förbättra Englands katolikers levnadssituation. Percy beklagade sig för Catesby under deras möte och han hotade med att lönnmörda Jakob eftersom Percy ansåg att Jakob hade svikit Englands katoliker. Catesby svarade honom: "Nej, nej, Tom, du skall icke riskera smärre ändamål utan om du vill vara en landsförrädare skall du kunna vara till stor förmån." Percy lyssnade uppmärksamt på Catesby när han fortsatte med att säga att han hade några planer som han snart tänkte delge Percy. +Under alla helgons dag samma år kontaktade Catesby sin kusin Thomas Wintour, som vid tillfället var vid Huddington Court i Worcestershire tillsammans med sin bror Robert Wintour. Thomas var utbildad till advokat och han hade stridit för England i Nederländerna innan han 1600 konverterade till katolicismen. Efter lord Essex misslyckade uppror mot Elisabet hade Thomas åkt till Spanien för att försöka få spanjorernas hjälp att förbättra levnadssituationen för Englands katoliker, något som den engelska regeringen senare kallade för det "spanska förräderiet". Mötet mellan Catesby och Thomas skulle ha ägt rum i Catesbys hem i Lambeth, men Thomas fick förhinder och kunde inte deltaga. Dock stod Catesby på sig och bjöd in honom igen i februari 1604. När Thomas anlände till mötet såg han att även John Wright var närvarande. Catesby delgav Thomas planerna på att spränga det engelska överhuset i Westminsterpalatset, vilket Thomas till en början motsatte sig men Catesby lyckades övertyga honom genom att säga att "sjukdomens karaktär krävde ett sådant starkt botemedel." +Trots att det katolska Spanien nu lutade åt att försöka sluta fred med England hade Catesby inte gett upp hoppet om att få hjälp från utlandet och på grund av detta bestämde sig Thomas för att återigen bege sig ut i kontinentaleuropa. I Flandern mötte han Juan Fernández de Velasco, 5:e hertig av Frías, som kort därefter begav sig till London för att skriva under Londonfördraget. Thomas betonade återigen den svåra situation som Englands katoliker upplevde, i hopp om att påverka avtalsförhandlingarna som skulle äga rum i Somerset House. Frías visade sig dock vara mer vänlig än vad han var tillmötesgående. Thomas träffade också den walesiska spionen Hugh Owen samt William Stanley, som båda var nedsättande mot Catesbys planer att få spanskt stöd för sin konspiration. Dock introducerade Owen Thomas för Fawkes, som var en trogen katolik som tjänat under Stanley som soldat i de södra Nederländerna. Thomas fick med Fawkes på de något vaga planerna att "göra något i England om freden med Spanien inte hjälper oss." Sent i april 1604 återvände de båda männen till Catesbys hem i Lambeth, där de berättade för Catesby att även om Spanien inte var emot deras planer tänkte de inte skicka något stöd till dem, något som Catesby redan verkade ha förväntat sig. Filip III av Spanien slöt sedan fred med England i augusti 1604. +Catesby, Thomas Wintour, John Wright, Thomas Percy och Guy Fawkes möttes den 20 maj 1604 på värdshuset Duck and Drake i stadsdelen Strand i London. Efter mötet svor de en sekretessed på en bönbok och de höll sedan en mässa i ett annat rum tillsammans med fader Gerard, som vid tillfället var ovetandes om deras planer. Robert Keyes gick med i konspirationen i oktober 1604 och hans roll blev att vakta kruttunnorna och andra saker som lagrades i Catesbys hus i Lambeth. Thomas Bates blev involverad i konspirationen i december 1604 när han hade börjat fatta misstankar om vad Catesby hade i kikaren. I mars 1605 hade ytterligare tre personer gått med i konspirationen och dessa var John Wrights bror Christopher, John Grant och Thomas Wintours bror Robert. +Konspiratörerna hyrde nu flera fastigheter i London, däribland en i Lambeth som användes för att tillfälligt lagra de kruttunnor som skulle användas i sprängdådet. Dessa transporterades sedan över Themsen till den slutgiltiga destinationen: ett källarvalv som låg precis under det engelska överhuset i Westminsterpalatset. Oron över pesten som härjade vid denna tid gjorde att parlamentets öppnande sköts upp från februari till den 3 oktober 1605. Under denna försening kan konspiratörerna ha grävt en tunnel under Westminsterpalatset, men några bevis för tunnelns existens har aldrig påträffats. Den 25 mars 1605 lyckades konspiratörerna införskaffa hyreskontraktet för källarvalvet under det engelska överhuset i Westminsterpalatset och i detta rum ställdes de trettiosex kruttunnorna. +Den 9 juni 1605 träffade Catesby jesuiten Henry Garnet på Thames Street i London. Under tiden som de diskuterade kriget i Flandern frågade Catesby fader Garnet hur moraliskt det var att "mörda oskyldiga". Garnet ska då ha sagt att sådana handlingar ofta kan förlåtas, men han ska i juli samma år (enligt egen utsago) ha visat ett brev från påven för Catesby, i vilket det stod att uppror var strikt förbjudna. Catesby svarade honom: "Vad jag än ämnar göra, om påven kände till det skulle han inte hindra det för vårt lands eget bästa." Garnet protesterade mot Catesbys uttalande, men Catesby stod på sig. Catesby erbjöd sig även att inviga Garnet i planerna, men Garnet ska ha avböjt detta erbjudande. Kort därefter berättade Tesimond för Garnet att när han mottog Catesbys bikt fick han nys om konspirationen, dock skriver författaren Alan Haynes att det var vid Bates bikt som fader Tesimond fick upp ögonen för konspiratörernas planer. Catesby och Garnet träffades igen den 24 juli samma år vid Catesbys släkting Anne Vaux hem White Webbs i Enfield Chase, vilket regeringen sedan länge hade misstänkt vara ett hus som hyste flera jesuitiska präster. Utan att avslöja att han kände till konspirationen försökte Garnet avråda Catesby från att genomföra sina planer, vilket han inte lyckades med. +Pestens närvaro gjorde att parlamentets öppnande ännu en gång sköts upp, denna gång från den 3 oktober till den 5 november 1605. Catesby hade hittills finansierat det mesta av konspirationen själv, men i augusti 1605 började han få slut på pengar. Under ett hemligt möte i Bath samma månad (där Catesby, Bates, Thomas Wintour och Percy var närvarande) bestämdes det att konspiratörerna var för få för att kunna genomföra sina planer och Catesby fick lov att involvera fler personer, om han så ville. Bates var den ende av konspiratörerna som öppet protesterade mot idén och Catesby värvade strax därefter Ambrose Rookwood, Francis Tresham och Everard Digby. Rookwood gick med i konspirationen sent i september detta år och eftersom han var hästuppfödare till yrket fanns i hans stall flera fina hästar som skulle komma till nytta vid upproren i Midlands. Tresham värvades den 14 oktober och då han hade en årsinkomst på £3 000 var han en av de mest förmögna personerna som konspiratörerna kände till. Treshams moster, Anne Throckmorton, var mor till Catesby och de båda kusinerna hade växt upp tillsammans och hade en bra relation. Enligt Fraser var det just detta förhållande, kombinerat med Catesbys härsklystnad, som fick Tresham att gå med i konspirationen. Catesby krävde två saker av Tresham: £2 000 och tillgång till hans familjehem Rushton Hall i Northamptonshire; Catesby fick ingendera. Tresham hade nämligen inga pengar att avvara och hans avlidne fars skulder begränsade hans arv. Digby gick med i konspirationen den 21 oktober när han, tillsammans med bland andra Catesby, deltog i en något försenad festdag för evangelisten Lukas. Digby hade, liksom Rookwood, tillgång till ett antal hästar och Catesby bad Digby att hyra Coughton Court från familjen Throckmorton för att "enklare kunna göra gott för sakens skull", med vilket Catesby menade att kidnappa Elisabet Stuart. Digby uppmanades även att starta ett jaktlag, vilket i verkligheten var en grupp beväpnade män på hästar, och han skulle vara beredd på någon sorts militärövning. Digby bidrog även med ett visst ekonomiskt stöd; han betalade £1 500 efter att Percy var sen med hyran för flera fastigheter i Westminster. +Dagen efter att Tresham hade värvats till konspirationen träffade Catesby Fawkes förre arbetsgivare Anthony-Maria Browne, 2:e viscount Montague. Catesby lyckades övertala lord Montague att inte vara närvarande under parlamentets öppnande utan att avslöja något om sina planer. Efter att konspirationen hade misslyckats greps Montague och han blev en av de misstänkta gärningsmännen bakom planerna. Dock släpptes han några månader senare efter en intensiv period av lobbying. +De sista detaljerna för planerna utarbetades i oktober 1605. Fawkes skulle tända på stubintråden till sprängladdningarna och sedan fly över Themsen. Samtidigt skulle ett uppror startas i Midlands, med syfte att fånga in Elisabet Stuart. Fawkes skulle sedan ta sig till kontinentaleuropa och förklara för de katolska länderna vad som hade skett i England. +Sent i oktober 1605 uttryckte flera av konspiratörerna oro över säkerheten för de katoliker som skulle närvara den dag då de planerade att spränga Westminsterpalatset. Percy var speciellt oroad över säkerheten för hans beskyddare Henry Percy, 9:e earl av Northumberland och Keyes var särskilt oroad över säkerheten för hans frus arbetsgivare Henry Mordaunt, 4:e baron Mordaunt. I Treshams tankar fanns främst två av hans svågrar: William Parker, 4:e baron Monteagle och Edward Stourton, 10:e baron Stourton. Catesby svar till hela diskussionen var att "de oskyldiga måste gå under tillsammans med de skyldiga, hellre än att förstöra chanserna för framgång." +Den 26 oktober mottog Monteagle ett anonymt brev till sitt hus i Hoxton. Brevet innehöll följande text: +Monteagle var osäker på vad som menades med det anonyma brevet, så han skickade det till Robert Cecil, 1:e earl av Salisbury, som var Secretary of State vid tillfället. +Tresham har länge varit misstänkt för att vara det anonyma brevets upphovsman. Nicholls hävdade att han var ganska säker på att det var Tresham som skrev brevet och som belägg för detta skrev han att så fort Catesby fick reda på brevets existens misstänkte han omgående Tresham och Catesby gick tillsammans med Thomas Wintour för att konfrontera honom. Catesby och Thomas Wintour hotade med att "hänga honom" om Tresham inte kunde bevisa att det inte var han som hade skickat brevet. Tresham lyckades övertyga de båda att han var oskyldig och han skickade ett brev dagen efter där han uppmanade dem att överge planerna för konspirationen. +Catesby väntade på Percys återkomst, från hans resa i norr, innan han tog ett beslut om vad som skulle göras. Catesby bestämde att brevet inte var ett tillräckligt stort hot mot deras planer och han beordrade att konspirationen skulle fortsätta som planerat. Under tiden som Fawkes gjorde en sista undersökning av krutet förberedde sig några av de andra konspiratörerna i Midlands. Salisbury, som hade märkt av vissa tecken på någon sorts plan redan innan han mottog brevet, visste inte exakt vad som var i görningen eller vilka som var inblandade. Han valde därför att avvakta och se vad som skulle hända. +Den 3 november träffades Catesby, Percy och Thomas Wintour i London. Exakt vad som sades under detta möte är okänt, men Percy sade sig kunna "stå ut med den yttersta prövningen" för att planerna skulle lyckas. Det är även möjligt att planerna ändrades något denna dag eftersom Percy senare ska ha setts hemma hos Henry Frederick, prins av Wales. Catesby hade nämligen en vecka tidigare fått reda på att prins Henry förmodligen inte skulle vara närvarande vid öppnandet av parlamentet. Catesby var därför vid denna tidpunkt tillsammans med Fawkes vid White Webbs, där de diskuterade möjligheten att kidnappa prins Henry istället för Elisabet Stuart. +När brevet till Monteagle visades upp för kung Jakob den 1 november 1605 kände han att det antydde något som berörde eld och krut, möjligtvis i stil med de explosioner som föregick mordet på hans far, Henry Stuart, lord Darnley, den 10 februari 1567. På lördagen den 2 november bestämde sig riksrådet för att parlamentet skulle genomsökas. På måndagen den 4 november genomfördes den första genomsökningen, ledd av Thomas Howard, 1:e earl av Suffolk, och då fann de en stor hög med kvistar som låg i ett hörn i källarvalvet under det engelska överhuset. Jakob beordrade att ännu en genomsökning skulle genomföras, vilket skedde runt midnatt natten till den 5 november. Denna genomsökning leddes av Thomas Knyvet, 1:e baron Knyvet och de fann då Fawkes i källarvalvet under det engelska överhuset, där han vaktade kruttunnorna. Fawkes greps på plats. Nyheten om Fawkes gripande spreds som en löpeld genom London och flera av de konspiratörer som befann sig i staden flydde fältet. Catesby, John Wright och Bates (som sent på måndagen den 4 november hade begett sig mot Midlands, där de skulle förbereda inför upproret) var ovetandes om konspirationens misslyckande och de stannade till i Dunstable eftersom en av hästarna hade tappat sin hästsko. När Rookwood hann ifatt dem och berättade nyheterna för dem om Fawkes gripande valde konspiratörerna (vilka vid tillfället var Catesby, Rookwood, Bates, bröderna Wright och Percy) att rida mot Dunchurch. Samma kväll kom de fram till Catesbys familjehem i Ashby St. Ledgers, där Catesbys mor och Robert Wintour befann sig. För att hålla sin mor ovetandes om planerna skickade Catesby ett meddelande till Robert, i vilket Robert uppmanades att möta Catesby utanför staden. Efter mötet fortsatte konspiratörerna mot Dunchurch där de träffade Digby och hans jaktlag. De ljög för Digby att kungen och Salisbury var döda, för att på så sätt få med dem på deras ritt. +Den 6 november stal konspiratörerna hästar från Warwick Castle och samlade på sig de vapen som de hade förvarat i Norbrook, i närheten av Stratford-upon-Avon. De begav sig sedan västerut mot Alcester och Catesby bad Bates leverera ett brev till Garnet i Coughton Court, för att se om han kunde hjälpa dem. Bates nyheter till jesuiterna visade sig vara mycket betydelsefulla och han hörde när Tesimond utbrast: "Det är ute med oss alla." Garnets svar till konspiratörerna var att han bad dem sluta med sina "onda handlingar" och istället lyssna till påvens läror, varpå han själv flydde fältet. Konspiratörerna stannade vid Huddington runt 14-tiden samma dag och där mötte de upp med Thomas Wintour. Tidigt nästa morgon höll de en mässa tillsammans med fader Nicholas Hart, som också hade hört deras bekännelser; detta tolkade Fraser som att konspiratörerna visste att de inte hade lång tid kvar att leva. Konspiratörerna samlade nu på sig fler vapen och mer ammunition vid Hewell Grange i Tardebigge, Worcestershire och de försökte samtidigt utan framgång att värva fler rebeller. Samma dag gick myndigheterna ut med ett offentligt tillkännagivande över de efterlysta konspiratörerna efter att Fawkes hade avslöjat vad han visste under tortyr. +Ridandes genom spöregn kom de helt utmattade konspiratörerna slutligen fram till Holbeche House, på gränsen till Staffordshire. Både de och det krut de hade med sig var dyblött från regnet och de fick lägga ut krutet framför en öppen eld för att försöka få det att torka. En gnista från elden landade på krutet och flammorna som då skapades uppslukade Catesby, Grant, Rookwood och en man vid namn Morgan (som var med i Digbys jaktlag). Catesby och Rookwood sades klarat sig "någorlunda väl" från elden, men Grant förblindades av flammorna och hans ögon har beskrivits som att vara "helt utbrända". Några av konspiratörerna red iväg från huset under natten, men Catesby stannade tillsammans med Grant, Thomas Wintour, Rookwood, bröderna Wright och Percy. Catesby kände på sig att slutet var nära och han vägrade att bli tillfångatagen utan han tänkte försvara sitt liv med sitt svärd. Han kysste det gyllene krucifixet han hade runt halsen och sade att han hade gett allt för "korsets ära". +På morgonen den 8 november anlände Richard Walsh, som var High Sheriff of Worcestershire, tillsammans med en styrka på omkring tvåhundra man till Holbeche House. En eldstrid bröt ut och Thomas Wintour var den första att bli träffad och skottet tog i hans axel när han försökte korsa gårdsplanen. Efter Thomas Wintour träffades bröderna Wright och Rookwood, som fortfarande var skadad efter flammorna kvällen innan. Både Catesby och Percy dog när de träffades av samma skott, skjutet av John Streete av Worcester. De stod i närheten av dörren och Catesby ska ha lyckats kräla sig in i huset innan han avled, hållande en bild av Jungfru Maria. Denna bild, tillsammans med Catesbys gyllene krucifix, skickades till London för att visas upp för allmänheten. +De konspiratörer som överlevde skottlossningen togs till London och fängslades medan de döda konspiratörerna begravdes i närheten av Holbeche House. Dock beordrade Henry Howard, 1:e earl av Northampton att Catesbys och Percys lik skulle grävas upp igen och halshuggas. Båda deras huvuden, tillsammans med Treshams huvud, visades senare upp i Northampton. Vid rättegången som hölls mot de överlevande konspiratörerna den 27 januari 1606 blev det en fällande dom i högförräderi för samtliga. Den 30–31 januari 1606 avrättades konspiratörerna genom hängning, dragning och fyrdelning. +Den här artikeln är helt eller delvis baserad på material från engelskspråkiga Wikipedia, Robert Catesby, 18 december 2012. +Bengtsen, Fiona (2005) (på engelska). Sir William Waad, Lieutenant of the Tower, and the Gunpowder Plot. Trafford Publishing. ISBN 1412055415 +Dixon, William Hepworth (2009) (på engelska). Her Majesty's Tower. Charleston, South Carolina: BiblioBazaar, LLC. ISBN 1103086391 +Fraser, Antonia (2005) (på engelska). The Gunpowder Plot. London: Phoenix. ISBN 0753814013 +Gardiner, Samuel Rawson (1883) (på engelska). History of England from the accession of James I. to the outbreak of the civil war 1603-1642. London: Longmans, Green. ISBN 9781108035736. https://archive.org/details/cu31924088000678 +Haynes, Alan (2005) (på engelska). The Gunpowder Plot: Faith in Rebellion. Sparkford: Hayes and Sutton. ISBN 0750942150. https://archive.org/details/gunpowderplot0000hayn_c6r9 +Marshall, Peter; Scott, Geoffrey (2009) (på engelska). Catholic Gentry in English Society: Throckmortons of Coughton from Reformation to Emancipation. Farnham: Ashgate Publishing Ltd. ISBN 0754664325 +Nicholls, Mark (1991) (på engelska). Investigating Gunpowder plot. Manchester: Manchester University Press. ISBN 0719032253. https://archive.org/details/investigatinggun0000nich +Northcote Parkinson, Cyril (1976) (på engelska). Gunpowder Treason and Plot. Weidenfeld and Nicolson. ISBN 0297772244. https://archive.org/details/gunpowdertreason0000park +Sharpe, James A. (2005) (på engelska). Remember, remember: A cultural history of Guy Fawkes Day. London: Harvard University Press. ISBN 0674019350. https://archive.org/details/rememberremember0000shar_p0k3 +Spinks Jr., Henry Hawkes (2005) (på engelska). The Gunpowder Plot and Lord Mounteagle's Letter. Whitefish, Montana: Kessinger Publishing. ISBN 1417930837 +Fitzgibbon, Sinead: Robert Catesby – the real brains behind the Gunpowder Plot (artikel från History in an Hour den 8 november 2012). +O'Brien, Jennifer: Robert Catesby (artikel från Britannia). +Wikimedia Commons har media som rör Robert Catesby. +Rafael ”Rafu” Gustaf Johannes Ramstedt, född 14 juni 1888 i Raunistula i S:t Marie, död 26 januari 1933 i Helsingfors, var en finländsk sångare, sångtextförfattare, kompositör och skådespelare. Vid sidan av J. Alfred Tanner räknas Ramstedt som Finlands främste kuplettsångare på 1910- och 1920-talen. +I över tjugo år tillhörde Ramstedt eliten inom finsk underhållning, såväl inom sång som inom teater. På 1910-talet uppträdde han som kabarésångare i Helsingfors och turnerade flitigt tillsammans med kolleger som J. Alfred Tanner. På 1920-talet engagerades han inom teatern, där han var verksam som både skådespelare och teaterchef. Han skrev också storyn till en rad filmer och medverkade själv i en av Finlands första ljudfilmer 1929. Ramstedt författade även en stor mängd kupletter och schlagersånger, däribland balladen om Rosvo-Roope, som filmatiserades 1949. I samtiden kallades Ramstedt ”Finlands Maurice Chevalier”. +Ramstedt var son till maskinarbetaren Gustaf Adolf Ramstedt (1846–1926) och Edla Mathilda, född Holmberg (1850–1929). Makarna fick elva barn, däribland författaren Gustaf John Ramstedt, riksdagsledamoten Emanuel Ramstedt och författaren Armas Ramstedt. Fadern var delvis alkoholiserad och bytte ofta arbete. Innan Rafael föddes slog sig familjen ner i den fattiga stadsdelen Raunistula utanför Åbo, där barnen växte upp under miserabla förhållanden. Torpet familjen bodde i var byggt av Edla Mathildas bröder och bestod av ett kök, en kammare och ett vindsrum. +Ramstedt hade svenska som modersmål och talade finska med viss begränsning. Detta var till visst förtret, då Ramstedt ville vara helfinsk och brukade prata om att förfinska sitt namn till Rafu Raunistula. Ramstedt studerade vid industriskolan i Åbo och tog anställning som maskinkonstruktör. Emellertid tvingades han ge upp yrket, då han utsågs till syndabock när en hjulångare byggd för att trafikera Volga förliste under jungfrufärden. Efter haveriet arbetade han inom diverse yrken, bland annat med livförsäkringar. Så småningom beslöt han sig att sikta på en karriär inom underhållningsbranschen. Han visade sig vara mycket musikalisk; han skrev och komponerade sina egna sånger samt behärskade att spela både luta och gitarr. I Åbo tog han lektioner hos operasångerskan Naemi Friberg. +Ramstedt bosatte sig i en villa uppförd 1928 på Larin-Kyöstis väg 16 i Åggelby utanför Helsingfors och var den första i stadsdelen som ägde en bil, en påkostad Wanderer. Han gifte sig den 19 december 1915 med Saima Sola (1890–1934), syster till operasångaren Wäinö Sola. Makarna fick två barn; Olof Rauni (1916–1972) och Pia, gift Nieminen (1918–1957). +1910–1911 tillhörde Ramstedt stumfilmsbiografen Helikon i Helsingfors, där han blev god vän med motspelaren J. Alfred Tanner. De två uppträdde flitigt på Helsingfors restauranger och biografer. Ramstedt och Tanner skrev flera sånger tillsammans och Ramstedt brukade fungera som inspirationskälla för Tanner. Bland annat är Tanners kuplett Kalle Aaltonen en översättning av Ramstedts mer bortglömda svenska kuplett Calle Vågeman. Ramstedt översatte några av Tanners sånger till svenska, däribland Römperin tanssit som blev Rönnbergska dansen och Jannen hanuripolkka som blev Jannes handklaverstrall. De båda artisterna hade stor respekt för varandra och efter Tanners död 1927 sade Ramstedt följande om Tanners betydelse för kuplettgenren: ”Han intager ett enastående rum bland våra kuplettsångare. Ingen annan haft en sådan framgång som han. Det var Tanner som gjorde kuplettsjungandet till ett yrke. Följande hans exempel uppträdde efter honom ett flertal yrkesmässiga kuplettsångare.”1915 uppträdde Ramstedt regelbundet på teatern Casino i Åbo, som i augusti gästades av Suomalainen operetti. Tillsammans med skådespelarna vid Casino satte operettgruppen upp pjäsen Två man om en änka, regisserad av Bror Niska. Ackompanjatör var Martti Nisonen. +Senare på året 1915 engagerades han i den finska operett som sattes upp på Apolloteatern av Karl Emil Ståhlberg. Till de andra medverkande hörde exempelvis Theodor Weissman, Iivari Kainulainen, Aapo Similä, Dagmar Parmas och Elli Suokas. Det gavs två föreställningar om dagen, den ena på finska och den andra på svenska. Operetten blev mycket populär och Apolloteatern var ett uppskattat nöjescentrum vid Esplanadsparken. År 1917 gjordes en turné till Åbo, Björneborg, Tammerfors och Viborg, där Ramstedt spelade titelrollen i Luffarbaronen. Kapellmästare vid föreställningen i Viborg var kompositören Toivo Kuula. Samma höst fick verksamheten ett abrupt slut, då Ståhlberg tvingades sälja Apollos lokaler till ryska marinen. Skådespelarna som varit engagerade i operetten fick ekonomisk ersättning, men försöken att hitta en ny lokal övergavs inom kort. +Sedan Apolloteatern hade upphört var Ramstedt 1916–1917 engagerad vid Svenska inhemska teatern i Åbo. Sommaren 1920 engagerades Ramstedt i den nybildade Nådendals friluftsteater, ledd av svensken Robert Jonsson. Premiären hölls på midsommardagen med Per Olsson och hans käring. +Den 1 juli 1921 tillträdde Ramstedt som chef för arbetarteatern i Forssa. Där satte han bland annat uppAugust Strindbergs Gustav Vasa. Under tiden som teaterchef medverkade både Ramstedt och hans hustru i flera pjäser. Hans mest uppskattade roll var som gubben Apso i Lemmin poika, en pjäs baserad på nationaleposet Kalevala. I augusti 1923 beslöt Ramstedt att lämna chefskapet för att kunna uppträda på andra teatrar i landet. Åren 1923–1924 hade han anställning vid Turun teatteri. +Av Ramstedts hand finns en översättning av Ernst Fastboms pjäs Halta Lena och vindögda Per, på finska kallad Ontuva Leena ja Vinosilmä Vika. Pjäsen har spelats flera gånger på olika scener, bland annat på arbetarteatern i Forssa under Ramstedts tid som chef. År 1929 regisserade han en egen pjäs, Siemenperunia, som sattes upp vid Karuna skyddskårs lotteri på Skyddskårshuset i Åbo den 28 juli. +Ljudfilmen kom till Finland 1929. Ett av de första försöken att göra en inhemsk ljudfilm resulterade i en kort dokumentär, vari Theodor Weissman samt dansarna Helmi Vuorisola och Eva Hahn medverkade. Filmen fick namnet Laulu- ja tanssiesitys och hade premiär på hösten 1929, men på grund av innehållet i de sjömansvisor som Weissman sjöng, blev filmen barnförbjuden i biosalongerna. +I stället satsade filmbolaget Lahyn-Filmi i Åbo på en ny spelfilm tillsammans med Ramstedt och några dansare. I filmen, som fick namnet Raf. Ramstedt sjunger en sjömansvisa, framförde Ramstedt den egna sjömansvisan Mun kehtoni on keinuellut. Filmen bestod av filmade sekvenser där Ramstedt mimade och rörde sig till musiken, som framtogs genom en grammofonskiva och spelades med så kallad playback-teknik. Scenerna filmades på teatern Casino i Åbo, där filmen hade premiär den 8 oktober och visades hela veckan. Filmen blev genast en publiksuccé och visades i november på teatern Maxim i Tammerfors. +Inte långt senare producerade Lahyn-Filmi ytterligare en ljudfilm på elva minuter med Ramstedt, producerad med samma playback-teknik som den förra. I den nya filmen, Äänifilmi, sjöng Ramstedt en rad kupletter, däribland Tanners Römperin tanssit på svenska. Just den sekvensen, när Ramstedt sjöng den visan, kopierades senare ut till en egen film, även den producerad av Lahyn-Filmi och släppt i oktober 1929. Filmen fick namnet Rönnbergin tanssit och hade en längd på cirka tre minuter. +År 1931 medverkade Ramstedt i bakgrunden vid produktionen av den första fulländade ljudfilmen Säg det på finska. Han skrev också historien till filmen Helsingfors främsta affärsman som spelades in 1934 med Matti Jurva och Uuno Montonen i huvudrollerna. Handlingen i balladen Rosvo-Roope blev sedermera förlaga till filmen Piraten älskaren som producerades 1949. +De finländska kuplettsångarna hade sin individuella stil på scen. Ramstedt var mer sofistikerad än Tanner; han bar vit fluga och ville framstå som en gentleman. Till detta bar han ständigt en stor, konstgjord solros på kavaj- eller frackslaget. Till skillnad från sina kolleger framförde Ramstedt sällan kupletter med sexuella anspelningar. Hans reserverade personlighet gjorde honom mindre populär än Tanner, men duon hade god kemi och kompletterade varandra. Ramstedt hade ett intresse för gamla riddarballader, som han samlade och studerade. Inom bekantskapskretsen var han känd som en originell sällskapsmänniska och uppskattad historieberättare. +Sommaren 1932 turnerade Ramstedt med Dallapé-orkestern. Orkesterns dragspelare och grundare Martti Jäppilä hade blivit tillsagd att Ramstedt inte skulle passa som solosångare i orkestern, men turnén blev lyckad och inför spelningarna med Ramstedt var salongerna alltid fyllda med åhörare. Jäppilä skrev senare om hur Ramstedt under konserterna brukade sjunga sitt paradnummer, sjömansvisan Heijallerii, iklädd sydväst med en stor solros på bröstet. Tanken var att de skulle göra en ny turné sommaren 1933, men Ramstedt avled innan turnén kunde äga rum. +I augusti 1928 inbjöds Ramstedt, Tatu Pekkarinen och Aapo Similä till Köpenhamn för att sjunga in grammofonplattor för His Master's Voice. Resan gick via Sverige och den 29 augusti sjöng Ramstedt och Similä svenska och finska sånger i Stockholms lokalradio. Efter den lyckade inspelningsresan till Köpenhamn inbjöds Ramstedt i januari 1929 till Stockholm för att göra ytterligare inspelningar. Direktör Westerlund vid Pohjoismainen Sähkö Oy, som svarade för inspelningarna, menade att efterfrågan på Ramstedts skivor var stor och de skulle bli lönsamma. På väg till Stockholm fick Ramstedt veta att en svensk sångare sjungit in några av hans låtar på grammofon och angett sig själv som upphovsman. Det är dock oklart om detta ledde till någon rättssak. +Åren 1928–1931, under den så kallade grammofonfebern, sjöng Ramstedt in 54 titlar med bland andra Suomi Jazz Orkesteri, Nils Ekman och Viljo Vesterinen som ackompanjatörer. Bland de skivor som sålde bäst märks Pianojuttu, som sålde mellan 5 000 och 6 000 exemplar, och Oi sä sulo Helsinki, som sålde uppemot 10 000 exemplar, men Rosvo-Roope som sedermera blivit något av en folksång hade inte anmärkningsvärt höga försäljningssiffror när den gavs ut 1931. +Sitt sista framträdande gjorde Ramstedt på Konstnärsgillets lilla julafton på Helsingfors konsthall 1932. Mot slutet av livet drabbades Ramstedt av kärlblödningar i hjärnan, vilket orsakade minnesförlust. Sjukdomen ledde till att karriären tog slut och Ramstedt lades in på Doktorinnan Lybecks privata sjukhem för sinnessjuka (Kammio sjukhus) i Helsingfors. Hans tillstånd förvärrades och han förflyttades snart till Lappvikens sjukhus, där han avled några dagar efter ankomsten. Han gravsattes på Malms begravningsplats i Helsingfors. +Reino Palmroth skildrar Ramstedt som en genuint glad och generös person som alltid hade fyllda salonger vid sina framträdanden. Som underhållare och kuplettsångare hörde han till Finlands främsta och var vida omtyckt. Leo Riuttus uppfattning av Ramstedt som skådespelare var att han hade svårt att minnas sina roller och repliker, men att föreställningarna och Ramstedts tolkningar ändå blev lyckade. Han skrev en stor mängd sånger, inklusive Herttainen Karoliina, Rosvo-Roope och Heijallerii. Utöver detta översatte Ramstedt några finska folksånger till svenska, såsom Vallinkorvan laulu, som blev Levnadsstigen och Heilani kammarin ikkunan alla, som blev Där vännen min bor. +I Raunistula har gränden Rafunkaju (svenska: Rafugränden) namngivits efter Rafael Ramstedt. I maj 2012 uppfördes pjäsen Rafu Ramstedt, Raunistulan trubaduuri på Konsa gård, några kvarter från hans barndomshem i Raunistula. +1929 – Raf. Ramstedt sjunger en sjömansvisa +1929 – Äänifilmi +1929 – Rönnbergin tanssit +1934 – Helsingfors främsta affärsman +1931 – Säg det på finska +1922 – Kupletteja ja hanurinpätkiä. Forssa +Bengelsdorff, Anna Lena (2017). Gustaf John Ramstedt – En mongolsjäl i professorsrock. Proclio. ISBN 978-9-5267-5244-0 +Collander, Erland (1920). Svenska inhemska teatern 1894–1920. Svenska inhemska teaterföreningen i Finland +Hirn, Sven (1992). Operett i Finland 1860–1918. Svenska litteratursällskapet i Finland +Hirn, Sven (1999). Populärmusikens pionjärer : En studie i den populära musikens kulturhistoria i Finland fram till 1917. Finlands svenska folkmusikinstitut +Järvinen, Lauri (1985). Kuplettimestarit ja mestarikupletit. Musiikki Fazer +Kukkonen, Einari (1998). Oi muistatkos Emma : Suomalaisen levylaulun vaiheita 1920-luvulla. Kustannuskolmio. ISBN 952-9095473 +Kukkonen, Einari (1999). Tuo tuuli Petsamosta : Suomalaisen levylaulun vaiheta 1930–34. Kustannuskolmio. ISBN 951-9817905 +Numminen, Juha (1978). Ei päivääkään vaihtaisi pois. Kirjayhtymä +Palmroth, Reino (1946). Kuolemattomat kupletit. Kanerva +Palmroth, Reino (1969). Hupilaulun taitajia : Pasi Jääskeläisestä Juha Watt Vainioon. WSOY +Uusitalo, Kari (1975). Lavean tien sankarit : Suomalainen elokuva 1931-1939. Otava +”Ramstedt, Raf.”. Finlands författare 1917–1944. Helsingfors: Suomalaisen Kirjallisuuden Seura och Svenska litteratursällskapet i Finland. 1981. sid. 358. ISBN 951-717-238-9 +Rafael Ramstedt på Internet Movie Database (engelska) +Rafael Ramstedt på YouTube +Katarina Howard (engelska: Catherine Howard), född cirka 1521 i Lambeth, död 13 februari 1542 på Towern (avrättad), var drottning av England och Henrik VIII:s av England femte hustru. Hon tillhörde en av Englands främsta adelsfamiljer vars huvudman hertigen av Norfolk fick se två av sina syskonbarn bli både drottningar och halshuggna; Katarina och Anne Boleyn. Katarina Howard tillhörde genom sin familj också den konservativa falangen vid hovet och väckte stor missunnsamhet hos de reformerta. +Katarinas släkt hade fiender som inte var förtjusta i att Henrik valde henne som hustru. Den 2 november 1541 överlämnade en av dem, ärkebiskop Thomas Cranmer, ett brev till Henrik vars innebörd anklagade Katarina för att ha älskare och därmed ha begått äktenskapsbrott. Hon avrättades den 13 februari 1542 genom halshuggning då hon dömts för högförräderi, för att ha haft ett förhållande med en av kungens hovmän, Thomas Culpeper. Katarina erkände dock aldrig otrohet, eller något fullbordat förhållande med Culpeper. +Katarina Howard var född någon gång mellan år 1521 och 1524. +Hon var dotter till lord Edmund Howard som var en yngre bror till hertigen av Norfolk, vilket innebar att hon var kusin till Henriks andra fru Anne Boleyn, och kusin till Henry Howard, earl av Surrey. Familjen Howard var mäktig och rik, men som yngre son i en mycket stor familj var Katarinas far inte särskilt välbeställd utan tvingades ofta be sina rikare släktingar om lån. När Anne Boleyn blev drottning hjälpte hon Edmund genom att se till att han fick en tjänst hos kungen. Denna form av nepotism var inte bara accepterad i samtiden, utan sågs snarast som en plikt gentemot släkten. +Ätten Howard hade lyckats bygga upp sin ställning på militära meriter, bland annat hade John Howard, som senare kom att bli den förste hertigen av Norfolk, på slagfältet förskaffat sig både Edvard IV:s och Rikard III:s förtroende. Man hade även flyttat familjens positioner framåt genom fördelaktiga giftermål; det viktigaste när sir Robert Howard gifte sig med Margaret Mowbray, varigenom hertigtiteln Norfolk kom att övertas av ätten Howard en generation senare. +Ätten Howards framtid sattes dock på spel då hertigen av Norfolk var lojal med Rikard III och stred på dennes sida i slaget vid Bosworth Field. Henrik VII insåg dock värdet i denne framstående soldat, varför Thomas Howard tilläts träda i huset Tudors tjänst, även om hertigtiteln fråntogs honom genom ett parlamentsbeslut. Titeln hertig av Norfolk återbördades till huset Howard först efter segern i slaget vid Flodden 1513. Vid det laget hade man dock återförsäkrat sig om kunglig gunst, makt och inflytande, och säkrat sin plats i skuggan av tronen. +Mycket litet är känt om Katarina Howards barndom och uppväxt. Hennes far Edmund var aldrig en av kungens favoriter, bland annat därför att han var den ende engelske befälhavare som besegrades i slaget vid Flodden. Genom Anne Boleyns försorg utsågs han dock till befälhavare i Calais 1531, och han fick även motta egendomar som tillfallit kronan efter en dömd brottsling. Bara några månader efter detta dog han dock. Katarinas mor, Jocasta (eller Joyce) Culpeper, kom från förmögen lantadel. Var Katarina föddes är också omtvistat; vissa historiker föreslår Lambeth, en stadsdel av London, andra har hävdat att hon föddes på någon av familjens egendomar i Kent. Hennes mor förefaller ha avlidit när Katarina var i 10-årsåldern. +Kring 1531 skickades Katarina att uppfostras hos sin styvfarmor Agnes Tilney, änkehertiginna av Norfolk. I hennes palats i Lambeth uppfostrades ett flertal ungdomar. Hon tillhörde hertiginnans hushåll i Horsham och Lambeth vid samma tidpunkt som Alice Restwold, Mary Lascelles, Dorothy Baskerville, Margaret Benet, Katherine Tylney och Margaret Morton. Det var vanligt på denna tid att adelsfamiljer valde att placera sina barn hos rika och mäktiga släktingar och vänner som ett led i att ge dem förfining och skapa nyttiga och värdefulla kontakter. Hertiginnans hushåll i Lambeth sköttes dock med tämligen slapp hand, då änkehertiginnan ofta befann sig vid hovet och därför inte personligen kunde övervaka ungdomarnas uppfostran. +Som en följd av sin härkomst och uppväxtmiljö fick inte Katarina den noggranna uppfostran som Henrik VIII:s tidigare hustrur fått. Man ska dock komma ihåg att enbart det faktum att hon kunde läsa och skriva (åtminstone hjälpligt) gjorde att hon var mer välutbildad än de flesta samtida kvinnor. Det vanliga, även bland kungliga och adliga damer vid denna tid, var att kvinnors uppfostran gick ut på hushållssysslor och hustrulig lydnad och dygd. +Katarina beskrivs ofta som livfull, men inte som studieinriktad eller dygdig. Den lössläppta stämningen bland ungdomarna i änkehertiginnans hushåll, och bristen på övervakning, ledde till att Katarina kunde inleda ett förhållande med sin musiklärare, Henry Manox. Detta förhållande, som även innehöll inslag av sexuell natur, inleddes någon gång runt år 1536, då Katarina var mellan 11 och 16 år gammal. När Katarina blev drottning utsåg hon Manox till hovmusikant. Han kom senare att avlägga vittnesmål vid utredningen mot henne, då hon anklagades för högförräderi, det vill säga otrohet. Både Manox och Katarina erkände vid undersökningen mot henne att de haft ett sexuellt förhållande. Katarina beskrev i sitt vittnesmål handlingar som vi med modernt språkbruk snarast skulle beskriva som petting. +Detta första intima förhållande avslutades 1538, då Katarina istället inledde en relation med en av änkehertiginnans sekreterare, Francis Dereham, som inledningsvis uppvaktat en annan av de unga damer som uppfostrades av änkehertiginnan, Joan Bulmer. De blev älskande och tog för vana att tilltala varandra "hustru" och "make", enligt vad Katarina senare kom att erkänna. Katarina fick också sköta flera uppgifter åt Dereham som vanligen utfördes av en hustru åt sin äkta man, till exempel att ta hand om hans pengar när han var borta i tjänsteärenden. Förhållandet mellan Katarina och Dereham var känt bland de övriga ungdomarna i hushållet, och det var möjligen genom deras försorg som saken kom till änkehertiginnans kännedom år 1539. En annan teori var att det var den försmådde älskaren Manox som angav dem till hertiginnan. Hon såg då till att skilja Katarina och Dereham åt genom att skicka honom till Irland. Det är möjligt att Katarina och Dereham innan de skiljdes åt gav varandra någon form av äktenskapliga löften, en handling som under denna tid kunde betraktas som ett giltigt äktenskap om den även bekräftades genom ett sexuellt fullbordande. Detta skulle då ha accepterats som ett äktenskap även av kyrkan. +Katarina tillstod under förhör att Dereham och hon hade fullbordat förhållandet, men hävdade till sitt försvar att det varit vanligt förekommande att flickorna som var inhysta hos änkehertiginnan tog emot älskare i det sovrum de delade. En av de unga kvinnor som delade rum med Katarina under denna tid var Alice Restwold som trots sina unga år redan var gift. Hon vittnade senare att "Hon var en gift kvinna och visste vad äktenskap innebar och vad som hörde samman med stånkandet och stönandet som försiggick i hennes säng" +Kungens tredje hustru, Jane Seymour, avled i barnsängsfeber efter att ha givit liv åt prins Edvard. Detta innebar att Henrik VIII för första gången stod utan hustru, utan att själv ha valt det. Detta öppnade möjligheten för hans rådgivare, med Thomas Cromwell i spetsen, att anordna ett politiskt äktenskap. Den gemål som valet till slut föll på var Anna av Kleve, systern till den protestantiske ärkehertigen av Kleve som man hoppades skulle bli en värdefull allierad mot de katolska staterna på kontinenten, främst Frankrike. I den kamp om åtråvärda platser vid den nya drottningens hov som utbröt lyckades hertigen av Norfolk tillförsäkra sig en plats åt sin systerdotter Katarina Howard som en av drottning Annas Maids of Honour, en obetald ställning som hovfröken som brukade innehas av ogifta flickor från de finaste adelsfamiljerna som på detta sätt kunde beredas väg till förnämliga äktenskap. Maids of Honour giftes ofta bort genom drottningens försorg, och försågs med en hemgift ur drottningens Privy Purse (privata medel), vilket sågs som en stor ära förutom de uppenbara ekonomiska fördelarna. Deras uppgifter bestod främst av att underhålla drottningen med dans och musik samt att pryda hovet med sin närvaro. Strax innan Katarina Howard fick plats vid hovet hade kungen låtit veta att han bara tänkte acceptera Maids of Honour som "var vackra och som han tyckte smyckade sin plats". +Ätten Howard hade redan sett en medlem av släkten, Anne Boleyn, ta sig upp på tronen från en plats som hovdam, och det är inte osannolikt att man hade förhoppningar om att även Katarina skulle falla kungen i smaken. +Det finns källor som pekar på att kungen blev förtjust i Katarina Howard redan första gången han såg henne. I ett brev skickat från hovet till änkehertiginnan av Norfolk rapporteras det att "Kungens Höghet började åtrå Katarina Howard redan första gången han såg henne". Att han första gången han såg Anna av Kleve istället funnit henne motbjudande erbjöd Katarina lysande framtidsutsikter. I april 1540 hade kungen redan börjat överösa Katarina Howard med furstliga gåvor, och han visade öppet att han favoriserade henne. Katarinas släkt, som visserligen tvivlade på att hon hade förstånd och karaktär nog att duga som drottning, såg detta som en möjlighet att återvinna den gunst hos kungen de förlorat genom Anne Boleyns fall. Då ätten Howard var katoliker såg de även ett ökat inflytande för deras del som ett viktigt inslag i kampen för att återföra England till den sanna tron. Familjen gjorde därför allt de kunde för att främja relationen mellan Katarina och kungen. +Anna av Kleves misslyckande som kunglig gemål innebar Thomas Cromwells fall och avrättning, vilket öppnade för de konservativa krafterna vid hovet, vilkas ledare hertigen av Norfolk var. Den tonåriga Katarina Howard blev på så sätt en bricka i spelet, inte bara för ätten Howards äregirighet, utan också för konflikterna vid hovet och kampen om makten mellan konservativa och reformerta krafter. I brev som finns bevarade undervisar änkehertiginnan av Norfolk Katarina om hur man bäst underhåller kungen, och Katarina försågs med dyrbara klänningar och annat för att pryda sig. Under tiden hjälpte hertigen av Norfolk kungen att upplösa äktenskapet med Anna av Kleve genom att hänvisa till en tidigare trolovning mellan henne och hertigen av Lorraine som man menade inte hade upphävts genom en laglig dispens. När drottning Anna var avlägsnad hade vägen öppnats för drottning Katarina att ta över. +Katarina förmäldes den 28 juli 1540 med Henrik VIII, endast 16 dagar efter ogiltigförklaringen av hans äktenskap med Anna av Kleve. Hon erkändes som drottning den 8 augusti samma år. Kungens brådska med att ingå detta äktenskap kan ses som symptomatiskt för hans livslånga strävan att söka säkra tronföljden genom att få söner. Henrik VIII hade en son, Edvard, men den höga barnadödligheten innebar att detta inte alls var någon garanti för att tronen skulle stanna i huset Tudor. Enligt legenden skulle kungen ha friat till Katarina en dag då han besökte sin son, och alla damer i rummet gjorde hovnigningar när Henrik kom in i barnkammaren. Kungen ska då ha kallat till sig Katarina och inför hela rummet förklarat att han aldrig ville att hon skulle göra så igen, då han ämnade göra henne till drottning. Katarina ska ha svarat genom att åter falla ner i en djup nigning. Denna historia är med största sannolikhet apokryfisk. +Henrik, som närmade sig de 50 och var på väg att bli mycket fet, överöste sin unga hustru med presenter och kallade henne för sin "ros utan törnen". Henrik tyngdes av vetskapen om att reformationen och krigen med Frankrike hade kostat honom folkets välvilja och därtill plågades han av gikt och bensår. Att ha en ung vacker kvinna vid sin sida livade upp den åldrande kungen. Hennes motto, "Ingen annan vilja än hans", visade på en inställning hos hustrun som kungen skulle uppskatta. Drottning Katarina var mycket livfull och ordnade dagligen danser och fester och annan underhållning för kungen, som själv hävdade att han genom sin hustru återvunnit sin ungdom. När kungen gjorde officiell entré i London med sin nya drottning sköt man salut från Towern och alla stadens framstående invånare rodde ut för att hylla kungaparet i deras slup. Katarina har vid denna tid beskrivits som liten och nätt men med kvinnliga former, glittrande ögon och kastanjefärgat hår. +Att äktenskapet inte enbart var ett nöje för Katarina kan man kanske förstå, med tanke på att hennes make var 30 år äldre än hon, fet (han vägde vid denna tid ungefär 136 kilo) och hade bensår som luktade illa och dagligen måste tömmas på var. Samtidigt framgår det av källorna att Katarina njöt av att vara drottning. Hon fick av kungen motta både de egendomar som tidigare tillhört Jane Seymour och stora delar av de egendomar som tillfallit kronan efter Thomas Cromwell. Enligt vittnesmål ska Katarina ha fått dyrbara presenter dagligen av den förälskade kungen. Julen 1540 finns det dokumenterat i hovets räkenskaper att drottningen erhöll ett fyrkantigt hängsmycke med 27 diamanter och 26 grupper med pärlor, en brosch sammansatt av 33 diamanter och 60 rubiner med en kant av pärlor, och en muff (en sorts kudde man kunde trä in händerna i för att skydda mot kyla) av svart sammet trimmad med sobel och prydd med 38 rubiner och 572 pärlor. +Drottning Katarina var gladlynt men kunde också få plötsliga vredesutbrott, särskilt när hon ansåg att hon inte behandlades med den respekt som tillkom en drottning. Vid ett tillfälle när hon var vid mycket dåligt lynne blev hon tillfrågad av kungen om orsaken. Hon sade då att hon hört ett rykte att han ämnade ersätta henne med den ratade Anna av Kleve. När kungen svor henne evig kärlek blev hon åter på gott humör. Katarinas krav på underkastelse kom också att skapa konflikt mellan henne och hennes nästan fyra år äldre styvdotter Maria. Katarina ansåg att Maria inte visade henne samma respekt som hon tidigare visat de två föregångarna Anna av Kleve och Jane Seymour. Av hämndlystnad lät därför Katarina säga upp två av Marias favoriter bland hovdamerna. Katarina och Anna av Kleve kom däremot bra överens. Anna, som fått titel av "kungens syster" och stora egendomar, bland annat Anne Boleyns familjehem Hever Castle som tack för sin villighet att låta sitt äktenskap upplösas, infann sig vid hovet och fordrade att få knäfalla inför Katarina som nådigt tog emot henne. Katarina utnyttjade som förväntat sin ställning till att gynna sina släktingar, varav många fick prestigefyllda titlar och ämbeten, inte minst gynnades hertigen av Norfolk som av kungen utsågs till Lord-Lieutenant of the North, utöver den post som Lord Treasurer som han redan innehade. +Under sommaren 1541 åkte Henrik och Katarina på en form av eriksgata till norra England. Under denna tid vidtogs vissa förberedelser för en kröning av Katarina, vilket kan tolkas som att de hade ett äktenskapligt samliv och att Henrik ville att Katarina skulle krönas i samband med en förväntad graviditet. Katarinas situation denna sommar komplicerades dock av att personer som kände till hennes tidigare sexuella förbindelser hos änkehertiginnan av Norfolk började söka upp henne för att be om tjänster – tjänster som Katarina för att köpa deras tystnad var tvungen att bevilja. +Det som kom att bli skadligast för henne var att hon anställde sin tidigare älskare Francis Dereham som en av sina sekreterare. Detta var en av de handlingar som kom att leda till hennes fall. Även flera av de unga kvinnor som delat rum med Katarina under hennes tid hos änkehertiginnan av Norfolk fick plats som hovdamer hos drottningen, och det var flera av dessa som senare kom att vittna emot henne. Det faktum att hon på detta sätt omgav sig till största del med personer av, eller med nära anknytning till, ätten Howard skaffade henne fiender och avundsmän som senare kom att agera mot henne. +År 1541 inledde drottningen ett förhållande med en ung man i kungens uppvaktning: Thomas Culpepper. Det unga paret kunde träffas med hjälp av hennes hovdamer, bl.a. Lady Rochford, änka efter George Boleyn, Anne Boleyns bror. Culpeper hade i princip vuxit upp vid hovet, han var en Gentleman of the Bedchamber, vilket innebar att han tillhörde kungens närmaste uppvaktning, och en av hans uppgifter var att sova i kungens sovrum om denne skulle vakna och behöva något under natten. Culpeper hade rykte om sig att vara en kvinnokarl, men den romantiska bild av honom som tecknats, särskilt under 1800-talet, har skadats av att man funnit en källa som visar på att Culpeper våldtagit hustrun till en skogvaktare, medan hans män hjälpte till och höll fast henne, ett brott som kungen dock benådade honom för. Hur långt paret egentligen gick är svårt att idag utröna, bland annat på grund av att Culpepers bekännelse till del kom till under tortyr, Katarina och lady Rochford försökte rädda sina liv med sina berättelser, och de som vittnade mot Katarina till stor del drevs av hat, religiös fiendskap eller jakt på egen vinning. Historikerna idag ställer sig huvudsakligen tvivlande till huruvida förhållandet fullbordades. +Släkten Howard hade fiender som inte var förtjusta över Henriks val av hustru. En av dem, ärkebiskop Thomas Cranmer, överlämnade den 2 november 1541 ett brev till Henrik, där det stod att Katarina hade levt i skörlevnad före giftermålet. Cranmer hade fått sina uppgifter från John Lassells, brodern till Mary Lassells, en kvinna som varit i tjänst hos änkehertiginnan i Lambeth. Lassells främsta skäl till att föra den information om Katarina vidare som han fått av sin syster var religiöst nit, Lassells var fanatiskt reformert och betraktade Katarina och hennes konservativt katolska släktingar som satans anhang. +Kungen vägrade först att sätta tilltro till uppgifterna, men beordrade Cranmer att genomföra en undersökning. Henrik fortsatte att vara på gott humör, och förväntade sig uppenbarligen att undersökningen skulle visa att beskyllningarna var falska och lögnaktiga. Katarina Howard själv sattes i husarrest på Hampton Court genom att dörrarna till hennes våning låstes och hon förbjöds att gå ut, medan kungen själv 5 november lämnade Hampton Court och gav sig till Whitehall. +De unga kvinnor som delat rum med Katarina under åren hos änkehertiginnan i Norfolk House Lambeth förhördes och bekräftade anklagelserna om Katarinas föräktenskapliga relationer med Henry Manox och Francis Dereham, som i sin tur greps. Henry Manox förhördes och erkände att han hade försökt förföra Katarina i Lambeth, men misslyckats. +Francis Dereham förhördes under tortyr och bekände att han hade ingått en hemlig trolovning med Katarina före hennes giftermål med kungen, men påstod aldrig att han hade haft ett förhållande med henne sedan hon gift sig med kungen; under tortyr angav han istället Thomas Culpepper som Katarinas älskare under hennes äktenskap. +Thomas Culpepper uppgav att Katarinas hovdam Jane Boleyn hade uppmuntrat honom att begå äktenskapsbrott med drottningen och hade hjälpt dem att mötas ensamma, och han erkände att han hade velat begå äktenskapsbrott med Katarina, men att de aldrig hade tagit steget fullt ut och att han aldrig hade "känt henne köttsligen". +Kungen skickade 6 november Cranmer och Katarinas farbror hertigen av Norfolk till Hampton Court för att informera Katarina om anklagelserna, förhöra henne och konfrontera henne med de bevis man hade, vilka utöver de tre männens erkännande även bestod i vittnesmål från hovfolk, bland annat drottningens hovdamer och främst av dessa de unga kvinnor som delat rum och vänskap med Katarina under åren hos änkehertiginnan i Lambeth. +Cranmer lyckades också komma över ett kärleksbrev som Katarina skrivit till Culpepper, detta brev är det enda skrivet av Katarinas hand som fortfarande existerar. Liksom många adelsdamer under denna tid anlitade Katarina vanligen professionella skrivare istället för att skriva själv, och när hon skrev ett kärleksbrev och alltså var tvungen att skriva personligen, var hennes skrivstil mycket dålig. +Hon skrev bland annat: +"Jag önskar att ni vore hos mig nu så ni kunde se vilket besvär jag gör mig med att skriva till er [...] Jag hörde att ni var sjuk och har aldrig längtat så mycket efter någonting som att se er [...] Mitt hjärta brister när jag inte alltid får vara i ert sällskap. [...]" +Hon tackade Culpepper för att han lovade att vara snäll mot kungen, och bad honom komma till henne när hennes hovdam Lady Rochford var i tjänst, "ty då har jag mest tid att stå till ert förfogande" och undertecknade brevet med: "Er så länge livet varar - Katheryn." +Katarina anklagades för högförräderi, eftersom en drottnings äktenskapsbrott jämställdes med detta brott. Hon erkände de förhållanden hon haft före äktenskapet, men hon erkände aldrig otrohet. I sitt erkännande till kungen skrev hon att hon tillåtit Manox att "handskas med och röra mina hemliga kroppsdelar", och att Dereham "låg med mig naken och nyttjade mig på det sätt en man gör med sin hustru vid många och olika tillfällen, men hur många vet jag inte". +Hon förnekade dock bestämt att hon skulle ha haft ett fullbordat förhållande med Culpeper, eller att hon och Dereham skulle ha kommit överens om någon form av trolovning. +Hon uppgav att hon hade haft samlag med Dereham, men uppgav att "han kunde sägas ha brukat våld". Påståendet att Dereham tvingat henne stämde dock inte överens med de övriga vittnesmål som gavs av de som varit närvarande när Katarina hade haft samlag med Dereham i flickornas sovsal i Lambeth. +Efter att Katarina fått veta vilka anklagelser som förts fram mot henne vittnade flera om att hon drabbades av en enorm skräck. Kanske var det därför hon inte begrep att då hon tillfrågades om huruvida hon och Dereham hade avgett några äktenskapsliknande löften till varandra borde hon ha svarat ja. Detta skulle ha inneburit att hennes äktenskap med kungen varit ogiltigt, vilket i sin tur innebar att hon inte kunde anklagas för att ha varit otrogen mot honom. Han hade då kunnat förvisa henne, vanärad och ruinerad men ändå vid liv. Eftersom hon förnekade detta blev följden istället döden. +När hon förnekade trolovningen med Dereham, hamnade fokus istället på hennes äktenskapsbrott med Culpepper, som var betydligt allvarligare eftersom de hade ägt rum medan hon var gift med kungen, och som innebar högförräderi, ett brott belagt med dödsstraff. +Den 12 november fängslades Katarina i det före detta klostret Syon Abbey i Middlesex. Kungen tillät henne att behålla både tjänare och fyra hovdamer som hon själv fick välja. Henrik förbjöd henne att bära juveler eller färgglada och juvelprydda klänningar, men tillät henne att ta med sig klänningar i sobra färger, prydda med guld och silver. Han skickade också underkläder och nattlinnen av siden. +Den 22 november 1541 fråntogs Katarina drottningtiteln, och den 24 november åtalades hon formellt av riksrådet för högförräderi. +Efterhand kompletterades Culpepers och Derehams bekännelser och hovdamernas vittnesmål även med lady Rochfords berättelse om hur hon hjälpt Katarina och Culpeper att mötas i hemlighet under sommaren. Bland annat hade Katarina och Culpepper tillbringat två nätter i rad ensamma med varandra i Lincoln till två på morgonen; Jane Boleyn uppgav att hon var övertygad om att paret hade haft samlag, men att hon inte hade insett detta när det skedde. +Jane Boleyn tycks efter sin bekännelse ha drabbats av ett psykiskt sammanbrott i fängelset. Trots att Thomas Wriotheseley, ansåg att lady Rochford inte var vid sina sinnens fulla bruk dömdes även hon till döden. +Den 10 december 1541 avrättades Culpepper och Dereham. Culpepper, som adelsman, halshöggs medan den ofrälse Dereham hängdes, togs ner levande och styckades medan han fortfarande var vid liv. +Flera av Katarinas släktingar fängslades, bland dem hennes bror Henry Howard och hans hustru, den åldersstigna änkehertiginnan och dennas styvbarn (de släpptes dock efter Katarinas avrättning). Hennes farbror hertigen av Norfolk lyckades dock undvika detta genom att offentligt fördöma Katarina. +Problemet med fallet var exakt varför Katarina Howard skulle avrättas. Hon hade aldrig erkänt att hon hade haft samlag med någon annan man medan hon varit gift med kungen. Francis Dereham hade haft samlag med henne innan hon gifte sig med kungen, och Culpeper hade aldrig erkänt att han hade haft samlag med henne, enbart att han hade velat ha det, och han hade därför egentligen avrättats för sin avsikt att ha sex med drottningen, det vill säga försök till högförräderi. +Den 21 januari antog parlamentet en lag som gjorde det straffbart med döden om en drottning underlät att avslöja ett eventuellt sexuellt förflutet för kungen inom 20 dagar efter att ha gift sig. Den 10 februari fördes Katarina till Towern. Under sin tid i Towern bad hon om att få in stupstocken på sitt rum för att kunna öva inför avrättningen så hon skulle veta i förväg hur hon skulle placera sig och "pröva den". +Katarina Howard avrättades den 13 februari 1542, efter att ha hållit ett tal där hon erkände sin skuld och sina synder och bad alla närvarande att be för kungen. Det franska sändebudet Marillac beskrev henne som "så svag att hon knappt kunde tala", men att hon med några ord erkände att hon förtjänade tusen dödar för att ha förbrutit sig mot en kung som behandlat henne så nådigt. Till skillnad från Anne Boleyn, som fick nåden att avrättas med svärd av en särskilt från Calais hämtad bödel, avrättades Katarina Howard med yxa. Strax efter henne avrättades Lady Rochford. +Hon begravdes bredvid sin kusin Anne Boleyn i St Peter ad Vincula i Towern. +Enligt legenden skulle Katarina, då hon hölls inspärrad i sin svit på Hampton Court, ha lyckats ta sig ut och springa mot kapellet där Henrik deltog i mässan. Hon ska ha skrikit och bankat på dörren och ropat kungens namn. Detta är dock en romantisk saga som inte kan stämma, då Katarina inte kände till de fulla anklagelserna förrän kungen lämnat Hampton Court. +Enligt sägnen ska hon ha bett att få in stupstocken till sig i det rum där hon inväntade avrättningen så att hon kunde öva sig på att lägga huvudet fint på det inför avrättningen. Det sägs också att hon spökar i Haunted Gallery på slottet Hampton Court, där hon enligt denna apokryfiska legend upprepar den händelse då hon rymde från sina fångvaktare och försökte få tag på kungen för att be om nåd. Den romantiska och tragiska berättelsen om den unga drottningens liv och död och legenderna om hennes oroliga ande har inspirerat till många filmer och böcker: +Första gången man såg Katarina Howard på vita duken var i stumfilmen Hampton Court Palace 1926, där hon spelades av Gabrielle Morton. +I filmen Kvinnorna kring kungen från 1933 spelades hon av Binnie Barnes. +1970 spelade Angela Pleasence Katarina i BBC-dramat The Six Wives of Henry VIII, där Keith Michell spelade Henry VIII. +2003 spelades Katarina av Emily Blunt i serien Henry VIII. +2009 spelar Tamzin Merchant Katarina Howard i serien The Tudors. +Katarina Howard är huvudkaraktär i Arvet efter Anne Boleyn, 2006, av Philippa Gregory. +Katarinas liv efter att hon kommit till hovet beskrivs i romanen The Fifth Queen av Ford Madox Ford. +Baldwin Smith, Lacey (2010) (på engelska). Catherine Howard. Amberley Publishing. ISBN 9781848685215 +Farquhar, Michael (2001) (på engelska). A Treasure of Royal Scandals. Penguin Books. ISBN 0-7394-2025-9. https://archive.org/details/treasuryofroyals00farq +Herman, Eleanor (2006) (på engelska). Sex with the Queen. William Morrow. ISBN 0-06-084673-9 +Lyly, John (1993) (på engelska). The Complete Works of John Lyly. Oxford University Press. ISBN 0781272122. https://archive.org/details/completeworksofj03lyly +MacLaren, A.N. (2000) (på engelska). Political Culture in the Reign of Elizabeth I: Queen and Commonwealth 1558–1585 (Ideas in Context). Cambridge University Press. ISBN 0521651441. https://archive.org/details/politicalculture0000mcla +Starkey, David (2004) (på engelska). Six Wives: The Queens of Henry VIII. Harper Perennial. ISBN 978-0060005504. https://archive.org/details/sixwivesqueensof00star +Wikimedia Commons har media som rör Katarina Howard. +Kort biografi över Katarina Howard, med porträttgalleri +Brev från Katarina Howard till Thomas Culpeper +English History biografi +PBS "Six Wives of Henry VIII", som beskriver Katarinas död +Komodovaran (Varanus komodoensis) är en art i ödlefamiljen varaner. Den finns på ön Komodo och några andra angränsande öar i Indonesien. Arten är världens största nu levande ödla med en normallängd på 2–3 meter och en vikt på omkring 70 kg. Dess storlek beror bland annat på långsam ämnesomsättning och på att arten lever på en isolerad ö där det saknas köttätande däggdjur som konkurrerar om samma ekologiska nisch. Komodovaranen är alltså en toppkonsument och dominerar följaktligen det ekosystem som den lever i. Även om komodovaraner vanligtvis är asätare är det också mycket vanligt att de jagar och anfaller andra byten, som ryggradslösa djur, fåglar och däggdjur. +Parningen äger rum från maj till augusti och äggen läggs i september. De cirka tjugo äggen läggs i bon som byggts av storfothöns (Megapodiidae) och som sedan övergivits. Äggen ruvas i sju till åtta månader och kläcks i april, då tillgången till insekter är som störst. Unga komodovaraner är sårbara och vistas därför uppe i träden, på säkert avstånd från rovdjur och de vuxna komodovaranerna, som utövar kannibalism. Det tar tre till fem år för ungarna att bli fullvuxna och de kan bli upp till 50 år gamla. Komodovaranen kan utföra partenogenes (jungfrufödsel), vilket betyder att fullt livsdugliga ägg läggs utan att först blivit befruktade. +Komodovaranen upptäcktes av forskare från västvärlden först år 1910. Ödlornas storlek och rykte gjorde dem tidigt till populära zoodjur. I vilt tillstånd har deras utbredning minskat på grund av mänsklig inverkan på deras levnadsmiljö, och de listas idag som sårbara av IUCN. De är skyddade under indonesisk lag, och en nationalpark, Komodo nationalpark, grundades för att gynna skyddsåtgärder. +För urbefolkningen på Komodo är komodovaranen känd som ora, buaja durat ('landkrokodil') eller biawak raksasa ('stor väktare'). +Komodovaranens utvecklingslinje härstammar ur släktet Varanus som hade sitt ursprung i Asien för cirka 40 miljoner år sedan och som senare koloniserade Australien. För cirka 15 miljoner år sedan skedde en kollision mellan Australien och Sydostasien, på grund av tektonisk aktivitet, vilket resulterade i att varanerna koloniserade vad som idag är den indonesiska övärlden och sedermera utökade sitt utbredningsområde så långt österut som till ön Timor. Komodovaranen antas ha skiljt ur sig evolutionärt från sina australiensiska förfäder för cirka 4 miljoner år sedan. Den dramatiska sänkningen av havsvattennivån under den senaste istiden, för cirka 20.000 år sedan, ledde till att stora områden torrlades. Dessa områden koloniserades av komodovaranen och när sedan vattennivån återigen steg isolerades den på de öar där arten idag förekommer. +I vilt tillstånd väger en fullvuxen komodovaran runt 70 kilo, men i fångenskap väger de något mer. Det största bekräftade exemplaret var 3,13 meter långt och vägde 166 kilo, osmält mat inräknad. Komodovaranen har en svans som är lika lång som dess kropp och den har ett 60-tal sågtandade tänder som ofta lossnar och växer ut på nytt och som kan bli upp till 2,5 centimeter långa. Dess saliv är ofta blodblandad eftersom tänderna nästan är helt täckta av tandkött som naturligt slits sönder då varanen äter. Detta skapar en idealisk levnadsmiljö för de smittofarliga bakterier som lever i dess mun. Komodovaranen har dessutom en lång, gul, djupt kluven tunga. +Komodovaranen har inte något väl utvecklat hörselsinne och trots dess synliga öronhål kan den endast höra ljud på mellan 400 och 2 000 hertz. Detta låga ljudomfång gjorde att man tidigare trodde att komodovaranen var helt döv då studier visat att varanen i vilt tillstånd inte reagerade på viskningar, höjda röster eller rop. Detta kom emellertid att ifrågasättas när djurskötaren Joan Proctor vid London Zoo lyckades träna en infångad individ att komma ut och äta då hon ropade, trots att komodovaranen inte såg henne. +Komodovaranen kan se så långt som 300 meter, men eftersom dess näthinna endast består av tappar så har den dåligt mörkerseende. Den kan urskilja färger, men har en dålig förmåga att se stillastående objekt. +Likt många andra reptiler använder komodovaranen sin tunga för att upptäcka smak- och luktstimuli med hjälp av det vomeronasala organet, ett sinne som underlättar navigering i mörker. Vid bra vindförhållanden kan komodovaranen, som svänger sitt huvud från sida till sida då den går, upptäcka kadaver på 4–9,5 kilometers avstånd. Dess näsborrar används inte för att känna lukt i någon större utsträckning då arten saknar diafragma, vilket är en förutsättning för att kunna dra in luft via dessa luftvägar. Den har endast ett fåtal smaklökar i bakre delen av strupen. +Komodovaranens fjäll, av vilka en del förstärks av ben, har känselplattor kopplade till nervsystemet vilket underlättar möjligheten till känsel. Fjällen runt öron, läppar, haka och fotsulor kan ha tre eller fler sådana sensoriska plattor var. +Komodovaranen föredrar varma och torra biotoper och lever vanligtvis i torra öppna gräsmarker och savanner på lägre höjder. Eftersom varanen är ektoterm – med andra ord regleras dess kroppstemperatur endast av yttre faktorer som exempelvis solen – är den mest aktiv på dagtid, även om den påvisat vissa nattliga aktiviteter. Komodovaraner lever huvudsakligen ensamma och går ihop endast för att para sig och äta. De kan under kortare perioder springa i hastigheter upp emot 20 km/h, dyka ned till 4,5 meters djup och klättra i träd då de är unga med hjälp av sina starka klor. För att fånga byten som befinner sig högt upp kan komodovaranen stå på sina bakben, med svansen som stöd. I takt med att komodovaranen växer används klorna mer som vapen och mindre för att klättra, eftersom dess stora storlek gör det opraktiskt att befinna sig uppe i träden. +Som skydd för exempelvis nattkylan gräver komodovaranen 1–3 meter breda hålor i marken med hjälp av sina kraftiga framfötter och klor. Tack vare sin storlek och vanan att sova i dessa bon kan komodovaranen bevara kroppsvärme under natten, något som minimerar tiden det tar att värma upp sig morgonen därpå. Komodovaranen jagar på eftermiddagen, men stannar i skuggan under de varmaste delarna av dagen. Dessa speciella viloställen är ofta belägna vid strandkanter med svala havsvindar; dessa märks sedan upp med spillning och rensas från vegetation. De tjänar även som en strategisk plats att anfalla hjort. +Komodovaraner är köttätare och även om de huvudsakligen äter kadaver, så händer det ofta att de anfaller levande bytesdjur genom bakhåll. Då ett lämpligt byte närmar sig varanens bakhållsplats anfaller den djuret genom att snabbt attackera undersidan av dess strupe. Den lyckas hitta sina byten med hjälp av sitt välutvecklade luktsinne, vilket kan upptäcka ett dött eller döende djur på sträckor upp till 9,5 kilometer. +Komodovaranen äter genom att slita loss stora köttstycken och svälja dem hela samtidigt som den håller ned kadavret med frambenen. För mindre byten, upp till en gets storlek, tillåter varanens löst sammanhållna käkar, flexibla skallben och expanderbara mage att bytet sväljs helt. Växtinnehållet som finns i magen och inälvorna på bytesdjuret undviks vanligtvis. De stora mängder röd saliv som komodovaranen producerar hjälper till att smörja maten men sväljprocessen är ändå en väldigt långsam process (15–20 minuter för att svälja en get). Komodovaranen kan försöka snabba på processen genom att trycka kadavret mot ett träd för att på så vis tvinga ned det i halsen, något som ibland kan vara så våldsamt att trädet slits sönder. För att förhindra kvävning under sväljprocessen andas varanen med hjälp av ett litet rör under tungan som är förbundet med lungorna. Efter att ha ätit upp till 80 % av sin kroppsvikt under en måltid, släpar den sig till ett soligt läge för att snabba på matsmältningen eftersom maten kan ruttna och förgifta varanen om den ligger osmält för länge. På grund av deras långsamma metabolism kan stora komodovaraner överleva på så lite som 12 måltider per år. Efter matsmältningen kräks komodovaranen upp en massa av horn, hår och tänder som är täckt med illaluktande slem. Efteråt gnider komodovaranen sitt ansikte mot jord eller buskar för att få bort slemmet, något som antyder att djuret, likt människor, ogillar lukten av sina egna utsöndringar. +De största djuren äter vanligtvis först, medan de mindre följer en hierarki. Den största hanen hävdar sin dominans och de mindre hanarna visar sin underkastelse genom kroppsspråk och mullrande väsningar. Varaner av samma storlek kan göra upp genom att "brottas". Förlorarna retirerar vanligtvis, men det händer att de dödas och äts upp av segraren. +Komodovaranens föda är varierande och består av ryggradslösa djur, andra reptiler (däribland mindre komodovaraner), fåglar, fågelägg, smådjur, apor, vildsvin, getter, hjortar, hästar och vattenbufflar. Yngre komodovaraner äter insekter, ägg, geckoödlor och andra mindre djur. Det har hänt att komodovaraner har ätit människor och människokroppar genom att gräva upp kroppar från grunda gravar. Denna vana fick invånarna på Komodo att flytta sina gravar från sandjord till lermark och lägga stenar ovanpå. Enligt evolutionsbiologen Jared Diamond kan komodovaranen även ha livnärt sig på en nu utdöd dvärgelefant av släktet Stegodon som en gång levde på Flores. Komodovaranen har ibland även setts skrämma dräktiga hjortar för att framkalla missfall vars kvarlevor de sedan äter, en teknik som även observerats hos större afrikanska rovdjur. +Eftersom komodovaranen saknar diafragma kan den inte suga upp vatten då den dricker, den kan inte heller lapa vatten med hjälp av tungan. Istället dricker den genom att ta en munfull vatten och sedan luta huvudet bakåt och låta vattnet rinna ned i halsen. +I slutet av 2005 drog några forskare vid Melbournes universitet slutsatsen att jättevaranen (Varanus giganteus), andra varanarter och agamer kan vara något giftiga. Tidigare har man trott att bett från dessa ödlor ledde till infektion på grund av bakterierna i ödlornas munnar, men forskarlaget visade att de direkta effekterna orsakades av förgiftning. Bett på mänskliga fingrar av en lacevaran (Varanus varius), en komodovaran, och en individ av arten Varanus scalaris observerades, och alla visade liknande resultat på människor: snabb svullnad inom några minuter, lokal störning av blodkoagulation, skärande smärta upp till armbågen, med vissa symptom som varar i flera timmar. +Senare forskning av Bryan Fry med bl.a. MR-skanning har visat att komodovaranen har 6 giftkörtlar i vardera underkäke. Dessa producerar ett gift som liknar ormgift. +Komodovaraner har även giftiga bakterier i sin saliv, av vilka över 28 gramnegativa och 29 grampositiva släkten har isolerats. Dessa bakterier orsakar sepsis hos komodovaranens offer, så till vida att om bytesdjuret kommer undan så kommer det vanligtvis ändå duka under inom en vecka som en konsekvens av infektionen. Människor har 9 timmar på sig att komma till sjukhus. Utifrån laboratoriestudier på råttor har man kommit fram till att den farligaste bakterien i komodovaranens saliv är en ytterst dödlig form av Pasteurella multocida. Eftersom komodovaranen tycks vara immun mot sina egna mikrober har mycket forskning utförts i hopp om att finna den antibakteriella molekylen, för att på så vis kunna använda den inom mänsklig medicin. +Parning äger rum mellan maj och augusti, och äggen läggs i september. Under den här perioden slåss hanarna över honor och territorier genom att brottas med varandra stående på bakbenen, med förloraren slutligen nedtryckt mot marken. Slagets vinnare kommer sedan vifta med sin långa tunga åt honan för att få information om hennes mottaglighet. Honorna är motsträviga och gör motstånd med sina klor och tänder under de tidigaste stadierna av parningen. Hanen måste därför helt låsa fast honan för att inte skadas. Andra uppvisningar under leken är bland annat att hanen gnider sin haka mot honan, hårda rispningar på ryggen, och slickning. Komodovaraner kan vara monogama och bilda "par", vilket är ett ovanligt beteende hos ödlor. +Honan lägger sina ägg i bon utgrävda i sidan av kullar, eller i övergivna bon av storfothöns, dock föredras de övergivna bona. En kull består av ett tjugotal ägg som ruvas i 7–8 månader. Honan ruvar äggen och skyddar dem tills de kläcks runt april, vid slutet av regnperioden, när tillgången på insekter är som störst. Kläckning är en ansträngande bedrift för ödleungarna, som bryter sig ur äggen med hjälp av en äggtand som senare faller av. Därefter kan de nykläckta ödleungarna ligga kvar i sina skal i flera timmar innan de börjar gräva sig ur boet. De föds tämligen försvarslösa, och många äts av rovdjur innan de når vuxen ålder. +Unga komodovaraner tillbringar en stor del av sin första år i träd, där de är relativt säkra från rovdjur, och även från kannibalistiska vuxna varaner, vars föda består av 10 % ungödlor. Då de unga måste närma sig ett kadaver för mat, rullar de runt i avföring och vilar bland de upprivna kadavrets inälvor för att avskräcka dessa hungriga vuxna. Det tar cirka 3–5 år för komodovaraner att bli fullvuxna, och de kan leva i upp till 30 år. +Komodovaranen Sungai vid London Zoo, England, lade en kull med ägg i slutet av 2005 efter att ha varit avskild från hanligt sällskap i över två år. Forskare trodde först att hon hade lyckats lagra spermier från sina tidigare möten med hanar, en anpassning känd som superfertilisation. (Hos till exempel levandefödande tandkarpar kallas detta ofta förrådsbefruktning.) +Den 20 december 2006 rapporterades det att Flora, en komodovaran vid Chester Zoo i England, var den andra kända komodovaranen som lagt obefruktade ägg. Hon lade 11 ägg, av vilka 7 kläcktes, alla hanar. Forskare vid Liverpool University i norra England genomförde genetiska tester på tre av äggen som kollapsade då de flyttades till en inkubator, och bekräftade att Flora inte haft någon fysisk kontakt med en hanvaran. Efter denna upptäckt visade tester att även Sungais ägg blivit till utan utvärtes befruktning. +Komodovaraner har ZW-systemet för könskromosomer till skillnad från däggdjurens X- och Y-kromosomsystem. Att Floras avkomma var hane visade att hennes obefruktade ägg var haploida och dubblerade sina kromosomer i ett senare stadium för att bli diploida, och att hon inte lade några diploida ägg, som skulle skett om en av meioserna i hennes äggstockar hade misslyckats, och att ägget inte befruktades av en polkropp. Då en honlig komodovaran (med könskromosmerna ZW) förökar sig på det här sättet, ger hon sin avkomma endast en av kromosomerna från vart och ett av hennes kromosompar, vilket innebär att endast en av två könskromosomer förs vidare. Denna enkla uppsättning av kromosomer dupliceras i ägget, vilket utvecklas partenogenetiskt. De ägg som får en Z-kromosom får uppsättningen ZZ (hanar); de som får en W-kromosom får uppsättningen WW och kan inte utvecklas. +Man har antagit att den här fortplantningsanpassningen tillåter ensamma honor att kolonisera en isolerad ekologisk nisch (som exempelvis en ö) och genom partenogenes få manlig avkomma, vilket skapar en population som kan föröka sig sexuellt (genom reproduktion med sin avkomma som kan resultera i både en hane och hona). Trots fördelarna med en anpassningsförmåga av det här slaget brukar djurparker vara försiktiga vad gäller partenogenes, eftersom det kan begränsa den genetiska mångfalden. +31 januari 2008 blev Sedgwick County Zoo i Wichita, Kansas den första djurparken i Amerika att dokumentera partenogenes hos komodovaraner. Djurparken har två vuxna honvaraner, en av dessa lade cirka 17 ägg mellan 19 och 20 maj 2007. Enbart två ägg ruvades och kläcktes på grund av platsproblem; den första kläcktes 31 januari 2008, medan den andra kläcktes en dag senare, 1 februari. Båda ödleungarna var hanar. +Komodovaranen upptäcktes av européer först år 1910, då rykten om "landkrokodilen" nådde löjtnant Steyn van Hensbroek i den nederländska kolonialadministrationen. Varanen blev vida känd, och ökänd, 1912 då Peter Ouwens, direktör för Bogors zoologiska museum på Java, publicerade en artikel i ämnet efter att ha mottagit en bild på varanödlans skinn från van Hensbroek, samt även två andra exemplar från en samlare. Komodovaranen var en viktig orsak för expeditionen till ön Komodo, ledd av W. Douglas Burden 1926 som efter återkomsten med tolv uppstoppade exemplar och två levande individer kom att bli en viktig inspirationskälla till filmen King Kong från 1933. Burden var även den som myntade benämningen "Komodo dragon" (sv:'Komododrake'), som är artens engelska trivialnamn. Tre av hans komodovaraner stoppades upp och visas fortfarande upp på American Museum of Natural History. +Nederländarna insåg att antalet individer som fanns kvar i vilt tillstånd var relativt litet, och förbjöd därför sportjakt och begränsade kraftigt antalet individer som fick användas för vetenskapliga studier. Insamlingsexpeditioner stannade upp i och med andra världskriget och återupptogs inte förrän 1950- och 1960-talet, då komodovaranens ätvanor, fortplantning, och kroppstemperatur undersöktes. Det beslutades att en långsiktig studie av komodovaranen behövde utföras. Denna uppgift gavs till familjen Auffenberg, som bodde på Komodo i 11 månader 1969. Under den tiden fångade och märkte Walter Auffenberg och hans assistent Putra Sastrawan mer än 50 individer. Forskningsresultaten från Auffenbergsexpeditionen kom att vara av avgörande betydelse för uppfödningen av komodovaraner i fångenskap. I Auffenbergsexpeditionens spår har biologer som exempelvis Claudio Ciofis fortsatta studier på djuret ytterligare belyst komodovaranens natur och vanor. +Komodovaranen är en starkt hotad art listad på IUCN:s rödlista. Det finns cirka 4 000–5 000 individer i vilt tillstånd och deras populationer är begränsade till öarna Gili Motang (100 individer), Gili Dasami (100 individer), Rinca (1 300 individer), Komodo (1 700 individer) och Flores (cirka 2 000 individer). Ett stort problem för populationen utgörs av att man uppskattar att det endast finns runt 350 fruktbara honor. För att motverka detta hot på populationen grundades Komodo nationalpark 1980 för att skydda komodovaraner på öar som exempelvis Komodo, Rinca, och Padar. Senare skapades även reservaten Wae Wuul och Wolo Tado på Flores för att bidra till skyddandet av komodovaranen. Det finns tecken på att komodovaranerna blir allt mer vana vid mänsklig närvaro, till stor del beroende på att djuren matas med kadaver på ett flertal matstationer som turistattraktion. +Vulkanisk aktivitet, jordbävningar, förlorade bon, eldsvådor (populationen på Padar utplånades nästan helt på grund av en skogsbrand), brist på byte, turism och tjuvjakt har alla bidragit till komodovaranens utsatta situation. Under Appendix I i CITES (Convention on International Trade in Endangered Species), förbjuds all kommersiell handel med hud eller individer av arten. +Även om anfall är oerhört ovanliga, har komodovaraner vid ett fåtal tillfällen dödat människor. Den 4 juni 2007 attackerade en komodovaran en 8-årig pojke på Komodo. Han dog senare av massiv blödning från sina sår. Det var den första kända attacken med dödlig utgång på 33 år. +Komodovaranen väcker stort intresse i djurparker, främst på grund av sin storlek och rykte. De förekommer dock i ganska få parker, eftersom de är känsliga för infektioner och parasitangrepp om de fångas i vilt tillstånd, och det är svårt att få dem att föröka sig i fångenskap. +Den första komodovaranen i djurpark var ett exemplar vid Smithsonian National Zoological Park 1934, men den levde endast i två år. Ytterligare försök att visa upp komodovaraner genomfördes, men livslängden på dessa individer blev väldigt kort, med en genomsnittlig livslängd på cirka fem år i Smithsonian National Zoological Park. De studier som Walter Auffenberg hade genomfört sammanfattades i hans bok The Behavioral Ecology of the Komodo Monitor (Komodovaranens beteendeekologi), vilket på sikt ledde till mer lyckad skötsel och parning av varaner i fångenskap. +Man har observerat att komodovaraner i fångenskap påvisar ett relativt tamt beteende inom en kort tid efter det att de har tagits till fånga. Det har vid ett flertal tillfällen rapporterats om djurskötare som tagit varanerna ur deras inhägnad för att umgås med djurparksbesökare, däribland små barn, utan någon skadlig effekt. Komodovaranerna kan även skilja olika människor åt. Ruston Hartdegen vid Dallas Zoo rapporterade att deras komodovaraner reagerade annorlunda då de fick träffa sina vanliga skötare, istället för en mindre igenkännbar skötare, eller en helt främmande. +Forskning på komodovaraner i fångenskap har även visat att de leker. En av dessa studier var en individ som knuffade omkring en spade som djurskötare glömt i buren, uppenbarligen lockad av det skrapande ljudet som spaden åstadkom över stenytan. En ung komodovaranhona vid National Zoo i Washington, D.C. tog upp och skakade diverse föremål, däribland statyer, vattenbehållare, plastringar och filtar. Hon stack även in sitt huvud i lådor, skor och andra föremål. Hon blandade dock inte ihop dessa föremål med mat, eftersom hon endast svalde dem om de var täckta med råttblod. Denna sociala lek visar på en slående likhet med däggdjurens lek. +En annan dokumentation av lekande komodovaraner kommer från University of Tennessee, där en ung komodovaran, kallad Kraken, lekte med plastringar, en sko, en hink och en konservburk genom att puffa dem med sin nos, slå till dem och bära omkring sakerna i sin mun. Varanen behandlade alla dessa föremål annorlunda än sin mat, vilket fick den ledande forskaren Gordon Burghardt att dra slutsatsen att antagandet om att lek med föremål endast var "födomotiverat rovdjursbeteende" måste vara felaktigt. Kraken var även den första komodovaran som kläckts utanför Indonesien, kläckt i National Zoo den 13 september 1992. +Även de mest fogliga varaner kan bli oförutsägbart aggressiva, särskilt då djurets territorium beträds av någon okänd. I juni 2001 skadade en komodovaran Phil Bronstein – chefredaktör för San Francisco Chronicle – efter att han på inbjudan av varanens skötare klivit in i varanens inhägnad vid Los Angeles Zoo. Bronstein blev biten i foten, eftersom skötaren hade sagt åt honom att ta av sig sina vita skor, vilket möjligen kan ha hetsat upp komodovaranen. Även om han lyckades fly behövde ett flertal senor i foten opereras. +Den här artikeln är helt eller delvis baserad på material från engelskspråkiga Wikipedia. +Auffenberg, Walter (1981). The Behavioral Ecology of the Komodo Monitor. Gainesville: University Presses of Florida. ISBN 0-8130-0621-X +Eberhard, Jo; King, Dennis; Green, Brian; Knight, Frank; Keith Newgrain (1999). Monitors: The Biology of Varanid Lizards. Malabar, Florida: Krieger Publishing Company. ISBN 1-57524-112-9. https://archive.org/details/monitorsbiologyo0000king +Richard L. Lutz, Judy Marie Lutz. Komodo, the Living Dragon: The Living Dragon. Salem, Oregon: DiMI Press. ISBN 0-931625-27-0. https://archive.org/details/komodolivingdrag00lutz_0 +W. Douglas Burden. Dragon Lizards of Komodo: An Expedition to the Lost World of the Dutch East Indies. Kessinger Publishing. ISBN 0-7661-6579-5 +Wikimedia Commons har media som rör Komodovaran. +Wikispecies har information om Varanus komodoensis. +Vitbukig fiskörn (Icthyophaga leucogaster) är en stor dagaktiv rovfågel som tillhör familjen hökar. Den beskrevs taxonomiskt första gången 1788 av Johann Friedrich Gmelin och är närbesläktad med salomonfiskörn (Icthyaetus sanfordi); de båda arterna beskrivs ibland som en superart. +Som adult är vitbukig fiskörnar en distinkt fågelart med vitt huvud, bröst, undre vingtäckare och stjärt. Ovansidan är skiffergrå och de svarta undre vingpennorna kontrasterar mot de vita vingtäckarna. Stjärten är kort och solfjäderformad som hos alla fiskörnar och havsörnar. Den är ganska smal och långhalsad med litet huvud och breda vingar. Som hos merparten av rovfåglarna är honorna något större än hanarna och kan mäta upp till 90 cm på längden, med ett vingspann på upp till 2,2 meter, och väga 4,5 kg. Subadulta fåglar har brun fjäderdräkt som gradvis blir vitare genom ruggning, och den anlägger adult dräkt vid fem till sex års ålder. Den har ett gåsliknande skall. +Vitbukig fiskörn förekommer från Indien och Sri Lanka över Sydostasien till Australien, utefter kuster och utmed större vattendrag. Den häckar och födosöker i närheten av vatten och fisk utgör ungefär hälften av dess föda. Vitbukig fiskörn är revirhävdande och bildar monogama par. Den är opportunistisk och tar as och en stor mängd olika djur. +Den globala populationen kategoriseras som livskraftig men har minskat i delar av Sydostasien, som i Thailand, och i sydöstra Australien. Den kategoriseras som hotad i Victoria och sårbar i South Australia och Tasmanien. Det största hotet mot arten är mänsklig aktivitet, både från störningar under häckningsperioden och från avverkning av lämpliga boträd. Vitbukig fiskörn är helig för ursprungsfolk i olika delar av Australien, och förekommer i en mängd folksagor över hela dess utbredningsområde. +Vitbukig fiskörn beskrevs taxonomiskt första gången 1788 av den tyska naturalisten Johann Friedrich Gmelin, men John Latham hade redan 1781 gjort noteringar om arten utifrån ett specimen inhämtat i februari 1780 på ön Panaitan utanför Java under James Cooks sista resa. Dess artepitet härstammar från gammalgrekiskans leuko som betyder "vit" och gaster som betyder "buk". +Dess närmsta släkting är den förhållandevis outforskade arten salomonfiskörn (Icthyophaga sanfordi) som förekommer på Salomonöarna. Tillsammans bildar de en superart, och som med andra närbesläktade superarter så har den ena (i detta fall vitbukig fiskörn) vitt huvud och den andra mörkt. Båda arternas näbb och ögon är mörka, och klorna är mörkgula som hos alla fiskörnar. Båda arterna har mörka partier på stjärten men det är inte alltid så tydligt hos den vitbukig fiskörnen. Data från DNA-sekvenser av cytokrom-B undersöktes 1996 och visade att trots att de båda taxonen har tydliga skillnader i utseende och ekologi så indikerar deras genetiska divergens på 0,3% att de separerats så sent som för 150 000 år sedan. Studiens författare sammanfattar att trots att deras genetiska divergens bättre överensstämmer med underarter, så utgör deras distinkta skillnader i utseende och beteende tillräckliga grunder för att betrakta dem som separata arter. +Vitbukig fiskörn placeras traditionellt i havsörnssläktet Haliaeetus. Genetiska studier visar dock att släktet är paradyletiskt gentemot fiskörnarna i Icthyophaga. I allt ökande drag förs därför vitbukig fiskörn, salomonfiskörn samt det likaledes tropiska artparet skrikfiskörn och madagaskarfiskörn till det senare släktet och denna linje följs här. Andra, som IUCN, väljer istället att expandera Haliaeetus till att även omfamna fiskörnarna. För att förtydliga släktskapet har även Birdlife Sveriges taxonomikommitté justerat de svenska namnen på de fyra arterna, från att tidigare ha kallats havsörnar. +Vitbukig fiskörn ganska smal och långhalsad havsörn med litet huvud och breda vingar. Den har vitt huvud, övergump och undersida, och svart till skiffergrå ovansida. I flykten syns de svarta vingpennorna tydligt som kontrasterar mot de vita undre vingtäckarna. Dess kraftiga näbb och även vaxhuden är blygrå med en mörkare spets och dess iris är mörkbrun. Tarsen är mörkgul eller grå med långa svarta klor. Till skillnad från örnarna inom släkte Aquila så har den inte befjädrade fötter. Den vita stjärten har svarta partier, som kan vara svåra att observera, och den är kort och solfjäderformad. Könen är lika. Hanen är 70–80 cm lång och väger 1,8–3 kg. +Honan är något större med sina 80–90 cm och en vikt på 2,5–4,5 kg. Vingspannet varierar från 1,8 till 2,2 meter. En studie från 2004 av 37 individer från Australien och Papua Nya Guinea (3 °S till 50 °S) visade att det är tillförlitligt att könsbestämma arten utifrån storlek och att de sydligare individerna var större än de nordliga. Den uppvisar inga skillnader i fjäderdräkt beroende på säsong. Det finns liten kunskap om dess ruggningsstrategi men allt tyder på att den tar över ett år att slutföra och att den kan avbrytas för att sedan återupptas vid ett senare tillfälle. +I glidflykt hålls armen i en vinkel ovanför kroppen medan handen hålls mer horisontell. I siluett har den lång hals, huvud och näbb som sticker ut framtill nästan lika mycket som stjärten sticker ut baktill. I aktiva flykt alternerar den vitbukiga fiskörnen mellan kraftfulla djupa vingslag och glidflykt. +En ung vitbukig fiskörn under sitt första levnadsår är övervägande brun med ljusa krämfärgade stråk på huvud, nacke, mantel och övergump. Sedan ruggar den till en allt vitare fjäderdräkten tills den har anlagt sin adulta fjäderdräkt under sitt fjärde eller femte levnadsår. +Den yttrar ofta sitt ljudliga gåsliknande skall, speciellt under häckningssäsongen och paren yttrar ofta detta läte i samtidigt. Hanens läte är snabbare och mer högfrekvent än honans. I den australiska fågelfaunan utgör lätet av den vitbukiga fiskörnen ett av de volymstarkaste och vittljudande. +Den adulta vitbukiga fiskörn är omisskännlig och svår att missta för någon annan fågel förutom med den mörkhuvade salomonfiskörnen, men deras dessa båda arter förekommer inte i samma utbredningsområde. Subadulta vitbukiga fiskörnar kan förväxlas med kilstjärtsörn, men den senare har mörkare fjäderdräkt, längre stjärt och befjädrad tars. Den kan också förväxlas med svartbröstad glada (Hamirostra melanosternon), men den arten är mycket mindre har vita partier på vingundersidan och har en grund bågande flykt. Subadulta individer kan också förväxlas med subadult gråhuvad fiskörn (Icthyophaga ichthyaetus) men kan särskiljas genom dess helmörka undersida och kilformade stjärt. Smutsgam, vars utbredningsområde överlappar med vitbukig fiskörn i Indien, har också vit fjäderdräkt, men är mindre och har vitare rygg och vingar. På Filippinerna kan den förväxlas med apörnen som bland annat kan särskiljas genom sin tofs på huvudet. +Merparten av vitbukig fiskörn förekommer från Bombay (ibland så långt norr ut som till Gujarat, och förr även på öarna i Lakshadweep) och österut till Indien, Bangladesh, och Sri Lanka i södra Asien, kring alla kustområden i Sydostasien inklusive Myanmar, Thailand, Malaysia, Indonesien, Indokina, på Filippinerna och dess kringliggande öar, och södra Kina, inklusive Hongkong, Hainan och Fuzhou, och österut genom Nya Guinea och Bismarckarkipelagen, och Australien. Kring Salomonöarna finns den bara på ön Nissan och på de övriga delar av ögruppen häckar istället salomonhavsörnen. I Victoria, där den annars är ovanlig, är den lokalt vanlig kring Corner Inlet och Gippslandssjöarna. På samma sätt är det i South Australia, där den är vanligare utmed den norra kusten av Kangaroo Island. Utbredningsområdet sträcker sig till öarna i Bass sund och Tasmanien. Det finns en obekräftad observation av arten från Lord Howeön och flera från Nya Zeeland. +Vitbukig fiskörn förekommande främst i kustområden men observeras även i inlandet. Den är vanligtvis stannfågel och revirhävdande men kan förflytta sig stora sträckor. Bland annat tillryggalägger de stora sträcker uppströms utmed floder för att födosöka, och de kan förflytta sig från öar till fastlandet. Populationer i Australien nomadiska då vattensamlingar i inlandet uppstår för att sedan torka ut. Exempelvis häckade ett par vid sjön Albacutya i nordvästra Victoria då sjön åter vattenfyllts efter att ha varit uttorkad i över 30 år. Arten är mycket störningskänslig för mänsklig påverkan, speciellt under häckningstid och uppträder därför i större koncentrationer i områden med få människor. +Vitbukig fiskörn är revirhävdande. Vissa bildar permanenta par som besitter ett revir året runt, medan andra är mer nomadiska. De lever i monogama förhållanden och paren håller ihop tills den ena parten dör, då den andre snabbt försöker finna en ny partner. Detta kan leda till att vissa häckningsplatser är kontinuerligt bebodda under flera år. Exempelvis var en häckningsplats i Mallacoota i Victoria bebodd under mer än femtio år. Subadulta fåglar drar ofta runt och många förflyttar sig över 50 km bort från området där de växte upp. En juvenil som växte upp i Cowell, South Australia rapporterades 3000 km bort, vid Fraser Island i Queensland. Den observeras ofta sittande på en gren högt upp i ett träd, eller glidflygande över vattendrag eller närliggande landområden. De observeras oftast enskilda eller i par. Mindre flockar med vitbukiga fiskörnar samlas ibland där det finns gott om föda, som vid ett kadaver eller i närheten av fiskebåtar som släpper ut fiskrens. Många delar av dess ekologi, speciellt dess häckningsekologi, är fortfarande okända. Den uppskattas kunna bli runt 30 år gammal. +Vitbukig fiskörn är en opportunistisk köttätare som lever av en mängd olika byten och även av as. Den fångar ofta fisk genom att flyga tätt över vattenytan och greppa den med sina kraftiga klor. Inför en sådan fångst håller den sina fötter långt fram, ofta under huvudet, och sedan greppar den fisken med en svepande rörelse bakåt samtidigt som den slår kraftigt med vingarna för att få lyftkraft. Ofta greppar den bytet med bara ena foten. Den kan även dyka från en sittplats i cirka 45° vinkel och under en kort stund vara under vattnet för att fånga en fisk som befinner sig i närheten av ytan. +Vitbukig fiskörn jagar mest vattenlevande djur, som fisk, sköldpaddor och havsormar, men den tar även fåglar som dvärgpingvin, sothöna och liror, och däggdjur. I Bismarckarkipelagen har den observerats fånga två olika arter av klätterpungdjur, nordlig grå kuskus (Phalanger orientalis) och fläckkuskus (Spilocuscus maculatus). Vitbukig fiskörn är en skicklig jägare och kan ta byten stora som svart svan. Den lever även av as som döda får, fåglar eller fisk som den hittar utmed kusterna, och den tar fisk från nät. +Vitbukig fiskörn mobbar mindre rovfåglar som australisk kärrhök (Circus approximans), visselglada (Haliastur sphenurus), brahminglada (Haliastur indus) och fiskgjuse, och tvingar dem att släppa sitt byte. Andra fåglar som mobbas är silvermås (Chroicocephalus novaehollandiae), grovnäbbad trut (Larus pacificus), skarvar och australisk sula (Morus serrator). De kan även stjäla byten från artfränder eller sin egen partner. Fiskörnen attackerar dessa fåglar genom att slå dem med sina utspärrade klor från ovan eller genom att flyga uppochned underifrån och slita bytet från den mindre rovfågeln. Det finns även observationer av hur vitbukig fiskörn attackerat och tagit fisk från nyzeeländsk pälssäl. +Vitbukig fiskörn födosöker ensamma, i par eller i familjegrupper. Ett par kan samarbeta i jakten på föda. Bytet kan ätas i flykten eller när den har landat, exempelvis i boet. Bytet flås av fågeln under tiden som den äter.Den smälter bytet mycket effektivt och får bara upp mindre spybollar med fragment av ben, päls och fjädrar. +I en studie från 2006 undersöktes inlandsvatten i ett område runt Canberra där både kilstjärtsörnar och vitbukiga fiskörnar födosöker visade en liten överlappning vad gäller bytesdjur. Kilstjärtsörnarna tog kaniner, olika kängurudjur, fåglar som kakaduor och papegojor, och vissa tättingar som australisk skata och stare. Vitbukig fiskörn fångade istället fisk, vattenlevande reptiler som ormhalssköldpaddan Chelodina longicollis och australisk vattenagam (Physignathus lesueurii), och sjöfåglar som änder, doppingar och sothöna. Båda arterna tog manand (Chenonetta jubata) medan kanin bara utgjorde en liten del av den vitbukiga fiskörnens föda. Trots att de båda rovfåglarna häckar i närheten av varandra så interagerade de sällan med varandra då kilstjärtsörnen jagade bort från vattnet medan vitbukig fiskörn födosökte utefter stränderna. Dock har konflikter mellan de båda arterna observerats kring boplatser på Tasmanien. +Vitbukiga fiskörnen börjar häcka vid ungefär sex års ålder. Häckningssäsongen varierar beroende på utbredningsomårde. Den pågår under torrperioden på savannen och i gräsmarkerna kring floden Fly och i de centrala delarna av Papua Nya Guinea och från juni till augusti i Australien. Innan paret kopulerar spelar de för varandra då de utför skickliga flyguppvisningar med branta dyk, glidflygning och de jagar även varandra under gälla rop. Ibland flyger de bredvid varandra med ett mellanrum på 2–3 meter och kopierar varandras rörelser. De kan också greppa varandras klor i luften och tumlar då nedåt tillsammans innan de släpper greppet. Detta beteende har också observerats under en attack mot en kilstjärtsörn. +Vitbukig fiskörn väljer ofta höga träd eller master och pyloner som boplatser, ofta i närheten av ett stort dött träd, eller en kraftig gren högt upp, varifrån sikten är god där havsörnen kan sitta och hålla uppsikt över sitt revir. Reviret ligger ofta i ett låglandsområde i närheten av vatten och där delar är täckt av skyddande skog. Runt sittplatsen samlas snabbt exkrement, spybollar och rester från byten. Boet är en djup, skålformig konstruktion av pinnar och grenar, som fodras med växtdelar som gräs och vass. Boet renoveras årligen varför det ständigt blir större. Boet placeras oftast i grenklykan på ett stort träd i närheten av vatten. De kan också använda bon byggda av kilstjärtsörn eller visselglada (Haliastur sphenurus). Ibland placeras boet på en klippavsats och på öar förekommer det att boet placeras direkt på marken. +Innan honan lägger sina ägg bygger eller renoverar paret boet i tre till sex veckor, och i detta arbete är hanen den mest aktiva. Vanligtvis består kullen av två ägg som är matt vita. De ovala äggen mäter 73x55 mm, och ruvas i över sex veckor innan de kläcks. Ungarna är bostannare och när de kläcks är de täckta med vitt dun. I början hämtar hanen föda som ges till ungarna av honan men när ungarna blir större matas de av båda föräldrarna. Trots att det är vanligast med två ägg per kull är det ovanligt att båda ungarna klarar sig tills att de blir flygga. Ett ägg kan vara obefruktat, den ena ungen kan dö i boet eller i brist på föda så dödas den ena ungen och ges som föda till den andra starkare ungen. Om första kullen misslyckas kan de försöka lägga en andra. Juvenilerna blir flygga efter 70-80 dagar och förblir i föräldrarnas revir i upp till sex månader eller fram till nästa häckningssäsong. +Vitbukig fiskörn kategoriseras som livskraftig (LC) av IUCN. Man uppskattar den globala populationen till mellan 2 600 och 41 000 vuxna individer, men populationen verkar minska. De främsta hoten mot arten är störning under häckningstid, habitatförstörelse, tjuvjakt, förgiftning, kollisioner med kraftledningar och vindkraftverk, och miljögifter. Arten har blivit ovanlig i Thailand och i vissa områden av Sydostasien. I Hongkong har istället populationen ökat, från 39 till 57 individer under åren 2002–2009. En fältstudie utförd på Kangaroo Island i South Australia visade att i områden där häckande havsörnar utsattes för mycket störningar från människor (vilket definierades utifrån nedhuggna skogsområden och hög mänsklig närvaro) hade lägre häckningsframgångar. På Eyre Peninsula i South Australia har bon övergivits när människor har inkräktat på havsörnarnas revir. På vissa håll har uthuggning av lämpliga häckningsträd resulterat i att arten lokalt nästan helt försvunnit, vilket skett i exempelvis Visakhapatnamdstriktet i Andhra Pradesh i Indien där bestånden av Casuarina equisetifolia huggits ner. I Indien har botätheten uppmätts till ett bo per 4,32 km i Sindhudurg och ett bo per 3,57 km i Ratnagiridistriktet i Maharashtra. +Bruket av DDT har haft stor påverkan på rovfåglar över hela världen, främst genom att äggskalen blir tunnare av detta ämne. +En studie om DDT:s påverkan på Australiens rovfåglar under åren 1947 till 1993 visade att äggskalen genomsnittligen hade blivit 6% tunnare. Denna förtunning ansågs inte utgöra en hot för en förhöjd mängd krossade ägg generellt, däremot kan vissa ägg var skal är ännu tunnare spricka. Samma studie visade att just vitbukig fiskörn var en av de arter som påverkats mest förmodligen på grund av dess vana att födosöka i område där mycket bekämpningsmedel används, exempelvis i våtmarker. Användningen av DDT nådde sin höjdpunkt 1973, förbjöds 1987 och användningen har nästan helt upphört 1989. +Vitbukig fiskörn är fredad i Australien enligt Environment Protection and Biodiversity Conservation Act 1999. Arten är sårbar för habitatförstöring vilket är en effekt att den ökade urbaniseringen av kustområdena, speciellt i södra och östra delarna av landet, där arten har minskat. Dock finns det indikationer på att populationen istället har ökat i inlandet som en sekundär effekt av skapande av reservat och dammar, och spridandet av den introducerade karpen (Cyprinus carpio). Men trots detta är den numera ovanlig utefter Murray River där den en gång var vanlig. Den listas som hotad i Victoria med färre än 100 häckande par. På Tasmanien finns det färre än 1000 adulta individer och arten listas där som sårbar (VU). +Vitbukig fiskörn har varit viktig för olika stammar av Australiens ursprungsfolk och har varit ett totem och en symbol för grupper och flera viktiga historiska personer inom aboriginerkulturen. +På flera platser har den ansetts helig och den har varit förbjuden att döda. Dess läte har tolkats som en varning, exempelvis att någon ska dö men även som ett gott tecken som exempelvis visade fiskaren var fisken håller till i havet. Den förekommer också i en mängd folksagor över hela dess utbredningsområde. +Vitbukig fiskörn förekommer på sedeln för $10,000 i Singapore, som släpptes 1980. Den förekommer på emblemet för den Malaysiska staten Selangor. Den malaysiska magnaten Loke Wan Tho byggde ett 40 meter högt torn bara för att kunna observera och fotografera de häckande fiskörnarna i sin trädgård i Istana Bukit Serene i Johor Bahru. Bilder tagna därifrån 1949 publicerades senare i The Illustrated London News 1954. +Fågelns vetenskapliga artnamn leucogaster betyder just "vitbukig". +Texten bygger på en översättning från engelskspråkiga Wikipedias artikel White-Bellied Sea Eagle, läst 2011-11-21 +Abbi, Anvita (22 mars 2010). ”Maya Jiro Mithe”. Salt Spring Island, British Columbia: Terralingua. Arkiverad från originalet den 16 maj 2012. https://www.webcitation.org/67hgrRSFM?url=http://www.terralingua.org/bcdconservation/?p=869. Läst 21 november 2011. +Acharya, Hari Narayan G (1936). ”The Whitebellied Sea-Eagle (Haliaeetus leucogaster Gmelin) in North Gujarat”. Journal of the Bombay Natural History Society 38 (4): sid. 828. +Ali, S.; Ripley, S. D. (1978). Handbook of the Birds of India and Pakistan: Volume 1 (2). New Delhi, India: Oxford University Press. sid. 287–89. ISBN 0195655060 +Baldwin, Amanda (2010). ”The White-bellied sea-eagle in the Jervis Bay region: an exploration of the cultural, ecological and conservation significance” (sammanfattning). School of Earth & Environmental Sciences, University of Wollongong. http://ro.uow.edu.au/theses/3166. Läst 21 november 2011. +Beruldsen, Gordon (2003). Australian Birds: Their Nests and Eggs. Kenmore Hills, Queensland: self. sid. 200. ISBN 0-646-42798-9 +BirdLife International 2009 Haliaeetus leucogaster Från: IUCN 2011. IUCN Red List of Threatened Species. Version 2011.1 . Läst 21 november 2011. +Clunie, Pam (2003). ”Flora and Fauna Guarantee Action Statement: White-bellied Sea Eagle (Haliaeetus leucogaster)”. East Melbourne, Victoria: Department of Sustainability and Environment, State Government of Victoria. Arkiverad från originalet den 20 mars 2012. https://web.archive.org/web/20120320032004/http://www.dse.vic.gov.au/__data/assets/pdf_file/0018/103185/060_White-bellied_Sea-Eagle_1994.pdf. Läst 21 november 2011. +Coates, Brian J. (1985). The Birds of Papua New Guinea. "1". Alderley, Queensland: Dove Publications. ISBN 0-9590257-0-7 +Daiyi, Linda Ford; Rose, Nancy Deborah (2002). ”Life in Country:Ecological Restoration on Aboriginal Homelands”. Cambridge, Massachusetts: Cultural Survival.org. http://www.culturalsurvival.org/ourpublications/csq/article/life-countryecological-restoration-aboriginal-homelands. Läst 21 november 2011. +Debus, Stephen D. J. (2008). ”Biology and diet of the White-bellied Sea-Eagle Haliaeetus leucogaster breeding in northern inland New South Wales”. Australian Field Ornithology 25: sid. 165–93. ISSN 1448-0107. Arkiverad från originalet den 2012-03-04. https://web.archive.org/web/20120304102702/http://www.une.edu.au/ers/staff-profile-doc-folders/steve-debus/debus-white-bellied-sea-eagle.pdf. Läst 23 november 2011. +Dennis, Terry E. (2004). ”Conservation status of the white-bellied sea-eagle, osprey and peregrine falcon on western Eyre Peninsula and adjacent offshore islands in South Australia”. South Australian Ornithologist 34: sid. 222–28. Arkiverad från originalet den 2011-09-29. https://web.archive.org/web/20110929142150/http://www.birdssa.asn.au/SAOpdf/Volume%2034/2004V34P222.pdf. Läst 23 november 2011. +Dennis, Terry E.; Kinloch, Martine A.; Pisanu, Phil; King, Cameron (2005). ”The distribution and breeding status of White-bellied Sea-Eagle and Osprey populations on Kangaroo Island in 2005”. Kingscote, Kangaroo Island: Kangaroo Island Resources Board, Government of South Australia. Arkiverad från originalet den 23 april 2011. https://web.archive.org/web/20110423042444/http://www.kinrm.sa.gov.au/LinkClick.aspx?fileticket=emVRLYUE9-g%3D&tabid=1152&mid=3045. Läst 21 november 2011. +Dennis, Terry E.; Brittain, Ross (2006). ”Attempted kleptoparasitism by white-bellied sea-eagles on fur-seal”. South Australian Ornithologist 35 (1–2): sid. 68. Arkiverad från originalet den 2011-09-26. https://web.archive.org/web/20110926200233/http://www.birdssa.asn.au/SAOpdf/Volume%2035/2006V35P068.pdf. Läst 23 november 2011. +Dennis, Terry E.; McIntosh, Rebecca R.; Shaughnessy, Peter D. (2011). ”Effects of human disturbance on productivity of White-bellied Sea-Eagles (Haliaeetus leucogaster)”. Emu 111 (2): sid. 179–85. doi:10.1071/MU10044. +Farfor, Susannah; Andrew, David; Finlay, Hugh (2003). Northern Territory. Lonely Planet. sid. 140. ISBN 1740591992. http://books.google.com/books?id=TsEMaLqw7OUC&pg=PA140 +Favaloro, Norman (1944). ”The White-breasted Sea-eagle along the Murray Valley”. Emu 43 (4): sid. 233–42. doi:10.1071/MU943233. +Ferguson-Lees, James; Christie, David A. (2001). Raptors of the World. Chicago, Illinois: Houghton Mifflin Harcourt. ISBN 0618127623. http://books.google.com/books?id=hlIztc05HTQC&pg=PA390 +Fleay, David (1948). ”Notes on the White-breasted Sea-Eagle”. Emu 48: sid. 20–31. doi:10.1071/MU948020. +Gopi, G. V.; Pandav, Bivash (2006). ”White-bellied Sea-Eagle Haliaeetus leucogaster preying on salt-water crocodile Crocodylus porosus”. Indian Birds 2 (6): sid. 171. ISSN 0973-1407. Arkiverad från originalet den 2011-06-24. https://www.webcitation.org/5zfqvNefy?url=http://www.indianbirds.in/pdfs/White-bellied%20Sea-Eagle%20Haliaeetus.pdf. Läst 23 november 2011. +Hadden, Don (2004). Birds and Bird Lore of Bougainville and the North Solomons. Alderley, Queensland: Dove Publications. ISBN 0-9590257-5-8 +Heinsohn, Tom (2000). ”Predation by the White-breasted Sea Eagle Haliaeetus leucogaster on phalangerid possums in New Ireland, Papua New Guinea”. Emu 100 (3): sid. 245–46. doi:10.1071/MU00913. +Hollands, David (2003). Eagles, Hawks and Falcons of Australia. Melbourne, Victoria: Bloomings Books. ISBN 1-876473-19-3 +Huxley, John (18 October 2010). ”When baby eagles dare, we will be watching”. Sydney Morning Herald (Fairfax Publications). http://www.smh.com.au/environment/conservation/when-baby-eagles-dare-we-will-be-watching-20101017-16p6j.html. Läst 21 november 2011. +Katdare, Vishwas; Mone, Ram (2003). ”Status of the white-bellied sea-eagle in Ratnagiri District, Maharashtra”. Journal of the Bombay Natural History Society 100 (1): sid. 113–16. +Kennedy, Robert S. (2000). A Guide to the Birds of the Philippines. Oxford, United Kingdom: Oxford University Press. ISBN 0198546688. http://books.google.com/books?id=fPDxk551AhkC&pg=PA54 +Lerner, H. R. L.; Mindell, D. P. (2005). ”Phylogeny of eagles, Old World vultures, and other Accipitridae based on nuclear and mitochondrial DNA”. Molecular Phylogenetics and Evolution 37 (2): sid. 327–346. doi:10.1016/j.ympev.2005.04.010. ISSN 1055-7903. PMID 15925523. http://www-personal.umich.edu/~hlerner/LM2005.pdf. Läst 21 november 2011. +Liddell, Henry George; Scott, Robert (1980) [1871]. A Greek-English Lexicon (abridged). Oxford, United Kingdom: Oxford University Press. ISBN 0-19-910207-4 +Lindgren, E. (1972). ”Courtship display of the White-breasted Sea Eagle Haliaeetus leucogaster”. Australian Bird Watcher 4: sid. 132. +Marchant, S.; Higgins, P. J., reds (1993). Handbook of Australian, New Zealand and Antarctic Birds. "2". Melbourne, Victoria: Oxford University Press. ISBN 0195530691 +Olsen, P.; Fuller, P.; Marples, T. (1993). ”Pesticide-related Eggshell Thinning in Australian Raptors”. Emu 93: sid. 1–11. doi:10.1071/MU9930001. +Olsen, Jerry; Fuentes, Esteban; Rose, A. B. (2006). ”Trophic relationships between neighbouring White-bellied Sea-Eagles (Haliaeetus leucogaster) and Wedge-tailed Eagles (Aquila audax) breeding on rivers and dams near Canberra”. Emu 106 (3): sid. 193–201. doi:10.1071/MU05046. +Pwee, Timothy (2002). ”White-bellied sea eagle”. National Library Board Singapore. Arkiverad från originalet den 27 augusti 2012. https://www.webcitation.org/6AECx5FMV?url=http://infopedia.nl.sg/articles/SIP_579_2005-01-05.html. Läst 21 november 2011. +Rasmussen, P. C.; Anderton, J. C. (2005). Birds of South Asia. The Ripley Guide. Volume 2.. Washington DC and Barcelona: Smithsonian Institution and Lynx Edicions. ISBN 8487334660 +Rose, Deborah Bird (1996), Nourishing Terrains: Australian Aboriginal Views of Landscape and Wilderness, Canberra, ACT: Australian Heritage Commission, s. 29, ISBN 9780642235619, OCLC 221843038, http://www.environment.gov.au/heritage/ahc/publications/commission/books/pubs/nourishing-terrains.pdf, läst 21 november 2011 +Sadvi, K. M. (2009). ”Occurrence of the White-bellied Sea Eagle Haliaeetus leucogaster in inland waters”. Indian Birds 5 (2): sid. 50. ISSN 0973-1407. Arkiverad från originalet den 2011-07-20. https://web.archive.org/web/20110720175540/http://www.indianbirds.in/Archives2009_5.2.html. Läst 23 november 2011. +Sanger, Clyde (1995). Malcolm MacDonald: Bringing an End to Empire. Montreal, Quebec: McGill-Queen's Press. ISBN 0773513035. http://books.google.com/books?id=mCYcnOs8LwIC&pg=PA316 +Schreiber, Arnd; Weitzel, Thomas (1995). ”Biochemical systematics of sea eagles (genus Haliaeetus Savigny 1809), with a note on allozyme differentiation between black and red kites (genus Milvus L. 1758)”. Biochemical Systematics and Ecology 23 (3): sid. 235–44. doi:10.1016/0305-1978(95)00001-B. +Seibold, Ingrid; Helbig, Andreas J. (1996). ”Phylogenetic relationships of the sea eagles (genus Haliaeetus): reconstructions based on morphology, allozymes and mitochondrial DNA sequences”. Journal of Zoological Systematics and Evolutionary Research 34 (2): sid. 103–112. doi:10.1111/j.1439-0469.1996.tb00815.x. ISSN 0947-574. +Sekhar, P.S. Raja; Kumar, P. Kanna; Anil, K.; Babu, A. Suresh (2004). ”Sighting of the white-bellied sea eagle (Haliaeetus leucogaster) in Andhra University Campus, Visakhapatnam, Andhra Pradesh”. Zoos' Print 19 (5): sid. 20. Arkiverad från originalet den 2012-03-21. https://web.archive.org/web/20120321092139/http://www.zoosprint.org/ZooPrintMagazine/2004/May/20.pdf#. Läst 21 november 2011. +Shephard, Jill M.; Catterall, Carla P.; Hughes, Jane M. (2004). ”Discrimination of sex in the White-bellied Sea-Eagle, Haliaeetus leucogaster, using genetic and morphometric techniques”. Emu 104: sid. 83–87. doi:10.1071/MU03043. +Simmons, R. E.; Mendelsohn, J. M. (1993). ”A critical review of cartwheeling flights of raptors”. Ostrich 64: sid. 13–24. doi:10.1080/00306525.1993.9634190. +Smith, Keith Vincent (July 2005). ”Port Jackson People”. NLA News (National Library of Australia) XV (10). ISSN 1444-1845. http://www.nla.gov.au/pub/nlanews/2005/jul05/article2.html. Läst 21 november 2011. +So, Ivy W. Y.; Lee, W. H. (2010). ”Breeding ecology of white-bellied sea eagle (Haliaeetus leucogaster) in Hong Kong - a review and update”. Hong Kong Biodiversity (18): sid. 1–8. http://www.afcd.gov.hk/tc_chi/publications/publications_con/files/IssueNo18.pdf. Läst 21 november 2011. +Spencer, Jennifer A.; Lynch, Tim P. (2005). ”Patterns in the abundance of White-bellied Sea-Eagles (Haliaeetus leucogaster) in Jervis Bay, south-eastern Australia”. Emu 105 (3): sid. 211–16. doi:10.1071/MU04030. +Sriram, Mythili; Sriram, Tara; Anand, M. O. (2008). ”Sighting of the White-bellied Sea Eagle Haliaeetus leucogaster at the Lakkavalli Dam”. Indian Birds 4 (2): sid. 71. ISSN 0973-1407. Arkiverad från originalet den 2011-07-20. https://web.archive.org/web/20110720175555/http://www.indianbirds.in/Archives2008_4.2.html. Läst 23 november 2011. +Strange, Morten (2000). Photographic Guide to the Birds of Southeast Asia. Singapore: Periplus. ISBN 962-593-403-0 +Stresemann, Erwin (1950). ”Birds collected during Capt. James Cook's last expedition (1776-1780)”. Auk 67 (1): sid. 66–88. Arkiverad från originalet den 2007-03-16. https://web.archive.org/web/20070316230021/http://elibrary.unm.edu/sora/Auk/v067n01/p0066-p0088.pdf. Läst 21 november 2011. +Tarr, H.E. (1962). ”Observations on the White-breasted Sea-Eagle”. Australian Bird Watcher 1: sid. 194–97. +Threatened Species Section, DPIW (2006) (PDF). Threatened Tasmanian Eagles Recovery Plan 2006-2010. Hobart, Tasmania: Department of Primary Industries and Water, Tasmanian Government. ISBN 0 7246 6346 0. http://www.environment.gov.au/biodiversity/threatened/publications/pubs/tasmanian-wedge-tailed.pdf. Läst 23 november 2011 Arkiverad 6 juni 2011 hämtat från the Wayback Machine. +Thurstans, Shaun D. (2009). ”Modelling the nesting habitat of the White-bellied Sea-Eagle Haliaeetus leucogaster in Tasmania”. Corella 33 (3): sid. 51–65. +Wink, Michael; Heidrich, P.; Fentzloff, C. (1996). ”A mtDNA phylogeny of sea eagles (genus Haliaeetus) based on nucleotide sequences of the cytochrome b gene” (PDF). Biochemical Systematics and Ecology 24 (7–8): sid. 783–791. doi:10.1016/S0305-1978(96)00049-X. http://www.uni-heidelberg.de/institute/fak14/ipmb/phazb/pubwink/1996/20_1996.pdf. Läst 21 november 2011. +Wink, Michael; Sauer-Gurth, H. (2004). ”Phylogenetic relationships in diurnal raptors based on nucleotide sequences of mitochondrial and nuclear marker genes”. Raptors Worldwide. Berlin: WWGBP. sid. 483–98. http://abcdef.uni-hd.de/institute/fak14/ipmb/phazb/pubwink/2004/31.2004.pdf +Wikimedia Commons har media som rör vitbukig fiskörn. +Wikispecies har information om vitbukig fiskörn. +Sveriges utrikes- och säkerhetspolitik under kalla kriget utformades efter andra världskrigets slut med hänsyn till landets geografiska position mellan de två militärallianserna Nato och Warszawapakten. Den socialdemokratiska regeringen ville att Sverige skulle vara alliansfritt i fredstid och ville signalera en fortsättning på den småstatspolitik som präglat historien efter 1815. Samtidigt förelåg ett långtgående samarbete med Nato och Finland riktat mot hotet från Sovjetunionen. Viktiga händelser var baltutlämningen, flera spionaffärer, nedskjutningen av en svensk DC-3:a (Catalinaaffären) och påstådda och verkliga ubåtskränkningar. +Direkt efter andra världskrigets slut fanns det en förhoppning om att stridsfrågor mellan stormakterna skulle kunna lösas av det nybildade Förenta nationerna. FN bildades 1945 men Sverige blev inte medlem förrän 1946 efter en viss debatt. +Gentemot Sovjetunionen förde Sverige en politik som skulle verka förtroendeskapande. Från svensk sida hoppades man kunna bidra till att minska misstron mellan stormakterna vilket i sin tur skulle gynna Sveriges säkerhet. Från sovjetisk sida var förväntningarna låga på att Sverige skulle hålla sig neutral vid en framtida kris. Sovjetunionens ambassadör i Stockholm, Aleksandra Kollontaj, rapporterade strax före krigsslutet att på samma sätt som Sverige intensivt manövrerade och gick med på varje kompromiss för att inte dras in i världskriget, så skulle landet med all säkerhet komma att bete sig även i framtiden. +Från 1944 och framåt hade 30 000 balter flytt över Östersjön till Sverige. Till en början krävde Sovjetunionen att samtliga skulle utlämnas. Detta krav mildrades dock snabbt och den 2 juni 1945 begärde Sovjetunionen formellt att de militärer som varit i tysk tjänst vid krigsslutet men flytt till Sverige skulle utlämnas. Den 15 juni beslutade samlingsregeringen att tillmötesgå denna begäran men också att utlämna tysk militär som kommit till Sverige strax före krigsslutet. Det totala antalet var ungefär 3 200 personer, varav omkring 167 balter. Regeringens beslut hölls hemligt. +Efter att samlingsregeringen avgått övertogs ärendet av Regeringen Hansson IV. Beslutet att utlämna flyktingarna blev offentligt och en debatt startade men regeringens beslut ändrades inte. Den 5 december 1945 påminde Sovjetunionen i en not om Sveriges förpliktelser. Frågan diskuterades i utrikesnämnden där statsminister Hansson menade att Sverige inte kunde bryta löftet och antydde att det hela kunde bli en kabinettsfråga om regeringen inte hade stöd för sin utrikespolitik. Fram till den 25 januari 1946 återsändes 2 518 personer till Sovjetunionen, varav cirka 150 baltiska flyktingar. 8 personer ansågs som obotligt sjuka eller icke transportabla. +Genom Sveriges tillmötesgående vad gäller baltutlämningen blev relationen till Sovjetunionen god. I juni 1946 kunde Staffan Söderblom, svensk ambassadör i Moskva, meddela att både Stalin och utrikesminister Molotov uttryckt sig positivt om relationen till Sverige. När Söderblom lämnade ambassadörsposten togs han den 15 juni 1946 personligen emot av Stalin, något mycket ovanligt. +Genom ytterligare en åtgärd sökte Sverige öka förtroendet för Sverige hos Sovjetunionen. Genom att den svenska regeringen ställde en kredit på 1 miljard kronor till förfogande skulle svenska företag kunna sälja varor till Sovjet. Den svenska regeringen hade först diskuterat frågan på hösten 1944. Sommaren 1945 föreslog Sovjet en miljardkredit. Under våren 1946 inleddes förhandlingar. I pressen fördes en omfattande debatt om denna så kallade "rysskredit", särskilt i Dagens Nyheter där den nytillträdde chefredaktören Herbert Tingsten var absolut motståndare. Men politiskt sett rådde inte heller bred enighet om denna fråga. Ur ett borgerligt perspektiv framställdes Gunnar Myrdal, som bar främsta ansvar för krediterna, som förespråkare för en eftergiftspolitik visavi Sovjetunionen. Hans diskussioner med näringslivets företrädare utmålades som en strävan att införa ett ”kommissarievälde”. +En tredje fråga där det fanns problem i förhållandet Sverige–Sovjetunionen var frågan om Raoul Wallenberg. Wallenberg hade varit verksam vid den svenska ambassaden i Budapest men gripits av sovjetisk militär och sedan försvunnit. Sovjetiska myndigheter hävdade att de inte visste var Wallenberg fanns och antog att han dödats av ungerska banditer under de kaotiska förhållanden som rått i Ungern vid krigsslutet. De svenska utrikesministrarna Günther och Undén skötte frågan om Wallenberg mycket passivt. Vad gäller Undéns passivitet är det möjligt att detta berodde på en vilja att inte störa de goda men ändå känsliga förbindelserna med Sovjetunionen. +Försvarspolitiskt försökte den svenska regeringen skapa ett nordiskt försvarsförbund som skulle vara fristående från supermakterna. Den 3 maj reste utrikesminister Undén till Oslo för att föreslå skapandet av ett sådant samarbete mellan Sverige, Norge och Danmark. Diskussionerna fördes främst av Sverige och Norge medan Danmark hade en mer passiv roll men kunde antas acceptera vad de andra två kom överens om. I Norge väckte frågan stor debatt inom det norska Arbeiderpartiet; å ena sidan ville de inte riskera ett nytt 9 april 1940, å andra sidan skulle ett medlemskap i NATO medföra att landet med automatik skulle dras in i en stormaktskonflikt. I januari 1949 tog den norske statsministern Einar Gerhardsen ställning för ett norskt NATO-medlemskap och den 29 mars 1949 röstade Stortinget för att Norge skulle bli medlem i NATO. Således kom frågan om ett nordiskt försvarsförbund att falla. +I och med undertecknandet av Atlantpakten den 4 april 1949 hade Sverige hamnat i det geopolitiska läge Undén velat undvika: i väster NATO-medlemmarna Danmark och Norge, i öster Sovjetunionen samt Finland, som genom VSB-fördraget var säkerhetspolitiskt knutet till Sovjetunionen och där stor osäkerhet rådde huruvida landet skulle kunna bibehålla sin frihet. +Gentemot Sovjetunionen kunde den svenska regeringen använda två strategier. Som nämnts fanns möjlighet att genom förtroendeskapande åtgärder göra (eller åtminstone försöka göra) förhållandet mellan de två länderna gott. Men en annan strategi som den svenska regeringen också använde var en slags avskräckning. Om Sovjetunionen reagerade negativt på någon svensk åtgärd eller förslag, då skulle den svenska opinionen kräva en tydligare västorientering, vilket inte kunde ligga i Sovjetunionens intresse. Om det inte lönade sig för Sverige med en alliansfri politik kanske det var lika bra att ta steget fullt ut och ansluta sig till NATO? +En sådan form av avskräckning användes på våren 1948 i samband med att Sovjetunionen förhandlade med Finland om VSB-fördraget. Den svenske ambassadören Sohlman gjorde klart att om det sovjetiska greppet över Finland var alltför starkt skulle den svenska alliansfriheten kunna omprövas. +Det fanns flera anledningar till att relationerna mellan Sverige och Sovjet var dåliga. Den FN-koalition som under USA:s ledning deltog i Koreakriget, mot Nordkorea som då var nära lierade med Sovjetunionen, stöddes av Sverige som skickade ett militärt fältlasarett som stod under amerikanskt befäl. I september 1951 greps Hilding Andersson, underofficer i flottan, och dömdes till livstids straffarbete för spioneri för Sovjetunionens räkning. I februari 1952 greps Fritiof Enbom och dömdes även han på samma grund. +1952 hade Sverige och USA inlett ett samarbete där Sverige bytte signalspaningsdata i utbyte mot amerikansk militär utrustning. Samarbetet var förstås strikt hemligt, vilket dock inte hindrade att Sovjetunionen visste om det. +Den 13 juni 1952 utbröt den så kallade Catalinaaffären när ett svenskt signalspaningsplan av typen DC-3 sköts ned av sovjetiskt jaktflyg. Tre dagar senare sköts ett svenskt sjöräddningsplan av typen Catalina ner när detta sökte efter DC-3:an. Den svenska regeringen protesterade skarpt men Sovjetunionen förnekade all kunskap om DC-3:an. Först 1991 erkände Ryssland att DC-3:an skjutits ned över internationellt vatten. +Även om försöket att skapa ett skandinaviskt försvarsförbund hade misslyckats så inleddes ett samarbete mellan den svenska militärledningen och NATO, ofta men inte alltid genom personkontakter visavi dess motsvarigheter i Norge och Danmark för att förbereda en samverkan vid en krissituation. Den alliansfria politik som fördes betydde inte att de inte förberedde sig för en situation där Sverige trots den förda politiken blev indragen i krig. Exempelvis fanns krigsplaceringar för officerare (däribland Pehr G. Gyllenhammar som var reservofficer) i NATO-länder för att kunna samverka och för att stadsledningen skulle kunna evakueras i händelse av ockupationsrisk. +Efter Josef Stalins död i mars 1953 skedde en avspänning mellan supermakterna. Koreakriget slutade med ett vapenstillestånd i juli 1953. I juni 1954 besökte utrikesminister Undén Sovjetunionen, i maj 1955 besökte Sovjetunionens vice utrikesminister Andrej Gromyko Stockholm. Finland blev medlem av Nordiska rådet 1955 och återfick Porkalabasen från Sovjetunionen 1956. I mars 1956 reste statsminister Erlander, utrikesminister Undén och inrikesminister Hedlund till Moskva för ett 12 dagar långt möte. Där förklarade den sovjetiske utrikesministern Molotov att Sovjetunionen fäste "mycket stor betydelse vid Sveriges neutralitet och alliansfria politik". +Det goda förhållandet mellan Sverige och Sovjetunionen bröts med Ungernrevolten i oktober 1956. Sverige protesterade mycket skarpt: "Sovjetunionen följer en högre politisk lag genom att med våld undertrycka varje försök att ändra en bestående kommunistisk regim." De sovjetiska trupperna i Ungern representerade "den nakna makten". +Den 5 mars 1957 fick den svenske ambassadören i Moskva, Sohlman, ta emot en protestnot där Sovjetunionen protesterade mot den agentverksamhet som Sverige bedrev i Baltikum. Sovjetunionen förväntade sig att den svenska regeringen skulle se till att verksamheten genast upphörde. Den svenska regeringen förnekade allting och menade att anklagelserna föll på sin egen orimlighet. Den 18 mars upprepade Sovjetunionen sin protest. Det har senare visat sig att Sverige stod bakom agentverksamhet i de baltiska sovjetrepublikerna, att detta sköttes av Försvarsstabens T-kontor och att regeringen inte hade informerats. +Den 19 juli 1959 ställdes ett planerat besök i Sverige av Nikita Chrusjtjov in. Planerna på detta besök hade kritiserats av Högerpartiets Jarl Hjalmarson och Folkpartiets Bertil Ohlin. Denna oenighet i synen på Sveriges förhållande till Sovjetunionen utmynnade i Hjalmarsonaffären; den 27 juli meddelade regeringen att den inte godtog Hjalmarson som svensk FN-delegat. Senare samma år avvisade statsminister Erlander försvarssamarbete med andra länder: "Förberedelser och överläggningar för militär samverkan med medlemmar av en stormaktsallians är sålunda helt uteslutna, om vi vill bevara förtroendet för vår utrikes- och försvarspolitik." I efterhand måste Erlanders uttalande karaktäriseras som medvetet vilseledande - Sverige hade då redan kontakter med Norge, Danmark, Storbritannien och USA om militär samverkan och/eller mottagande av bistånd i krig. +1960 gav USA Sverige en ensidig, hemlig (okänd av Sverige) militär säkerhetsgaranti. Denna fördjupades 1962. Vid ett eventuellt sovjetiskt anfall skulle Sverige bistås med militära medel. Som ett led i detta samarbete hjälpte amerikanska experter SAAB med utvecklingen av Viggen. Av samma skäl delfinansierade USA också en utbyggnad av Norska Flottan med antiubåtsinriktning. +Forskare vid KTH hjälpte också till med att förbättra träffsäkerheten på Polaris-missilerna. +1961 lanserade utrikesminister Undén den så kallade Undénplanen i FN. Planen gick ut på att kärnvapenfria länder skulle skapa regionala kärnvapenfria zoner vilket antogs skulle kunna underlätta för supermakterna att komma överens om ett provstoppsavtal. Sovjetunionen hade tidigare lagt fram liknande förslag och berömde Undénplanen medan den kritiserades av USA. Den 30 november 1961 antogs Undénplanen av FN. +1962 markerar en brytpunkt i den svenska utrikespolitiken då Östen Undén detta år avgick som utrikesminister efter 17 år. Året därefter greps översten och diplomaten Stig Wennerström och dömdes 1964 för grovt spioneri för Sovjets räkning. +I början av 1960-talet stod det klart att Nordeuropas geostrategiska betydelse hade ökat. Under 1950-talet hade Sovjetunionen satsat stora resurser på flottan. Sovjetiska örlogsfartyg fanns på alla hav. Vid den sovjetiska flottbasen i Murmansk på Kolahalvön fanns Nordflottan, Sovjetunionens största. Genom den sovjetiska flottuppbyggnaden försvårades NATO:s försörjningsmöjligheter över Atlanten: vid en krigssituation måste NATO transportera stora mängder personal och utrustning över Atlanten. +USA:s inställning till den svenska neutraliteten framgår av ett nu offentliggjort riktlinjedokument som färdigställdes inom det amerikanska utrikesdepartementet 1962, Guidelines for policy and operations - Sweden 1962. Vid ett isolerat sovjetiskt militärt anfall mot Sverige skulle USA åta sig att bistå Sverige. I händelse av ett större europeiskt krig var USA angeläget att Sverige, om landet inte stred mot Sovjetunionen, förblev en vänligt sinnad neutral stat. USA skulle också försöka möjliggöra att Sverige kunde stå emot en sovjetisk attack. Om det skedde ett sovjetiskt anfall mot Finland var det troligt att Sverige skulle ompröva sin alliansfria politik och USA skulle då uppmuntra Sverige att bli medlem i NATO. I samma dokument förklaras att Sveriges neutralitetspolitik är värdefull för USA i den mån den fungerar som en buffert mot sovjetisk expansionism. Den svenska politiken hade stöd av alla politiska partier. I Sverige fanns långvariga demokratiska traditioner så det fanns inte någon risk för statsvälvning, till exempel en statskupp från höger eller vänster. +Från 1965 blev den svenska utrikespolitiken mer aktiv i den mening att landet mer tydligt kritiserade företeelser i andra länder. Genom Operation Aspen försökte Sverige verka som medlare mellan USA och Nordvietnam. Tyngdpunkten i den svenska kritiken låg på förhållanden i Afrika (Sydafrika och Rhodesia) men det som uppmärksammades mest var den svenska kritiken mot Vietnamkriget. 1967 förklarade utrikesminister Nilsson i FN: "Detta tappra folk, som vid det här laget har upplevt krigets fasor i mer än ett kvartssekel, måste äntligen få leva ett liv i fred och värdighet och utan utländsk inblandning. Detta måste vara alla vietnamesers dröm, en dröm som vi alla väl förstår." Följande år var den svenska regeringen mycket kritisk mot USA:s politik i Vietnam som den ansåg inte gick "att förena med våra demokratiska värderingar". +Relationerna mellan USA och Sverige försämrades allvarligt 1968. Utbildningsminister Palme hade deltagit i en demonstration mot Vietnamkriget i februari 1968. I sitt tal i samband med demonstrationen förklarade Palme att "ska man tala om demokrati i Vietnam är det uppenbart att den i betydligt högre grad representeras av FNL än av Förenta staterna och dess allierade juntor". På ett känt fotografi ses Palme gå sida vid sida med Nordvietnams Moskva-ambassadör. Den amerikanska regeringen blev upprörd över symboliken i Palmes promenad med ambassadören och hans starka språkbruk och beslöt att genast kalla hem USA:s ambassadör i Stockholm, William Heath. Före sin hemresa framförde Heath USA:s protester både mot vad Palme sagt och att Sverige gav skydd åt amerikanska desertörer. Varken statsminister Erlander eller utrikesminister Nilsson ville ta avstånd från Palme och ställde sig bakom den förda politiken. +Heath återvände till Stockholm efter en månad. I en analys av den svenska neutralitetspolitiken skrev den amerikanska ambassaden till Washington att "De nyckfulla tendenserna i svensk utrikespolitik [...] har övergått i fullskalig utrikespolitisk romantik". +Relationen mellan Sverige och USA var fortsatt dålig under 1969. I januari 1969 blev Sverige det första västland som erkände Nordvietnam. Vid den socialdemokratiska partikongressen på hösten 1969 förklarade utrikesminister Nilsson att Sverige tänkte inleda ett biståndsprogram på 200 miljoner kronor under tre år till Nordvietnam. Ambassadören Heath hade avgått i januari 1969 och den nytillträdda Nixonadministrationen väntade till april 1970 med att utse en ny ambassadör, Jerome Holland. Samtidigt fortsatte liksom tidigare det dolda arbetet med samverkan med NATO. +När Sovjetunionen invaderade Tjeckoslovakien 1968 i samband med den så kallade Pragvåren protesterade Sverige och dess ockupation beskrevs som ett "tragiskt politiskt misstag lika väl som ett brott mot rättsliga och moraliska principer". +Förhållandet med USA förbättrades något i början av 1970-talet. I samband med att Palme blev hedersdoktor vid Kenyon College i Ohio träffade han den amerikanske utrikesministern William P. Rogers och lovade att Sverige skulle verka för att få fram information om amerikanska krigsfångar i Vietnam. Förhållandet försämrades åter julen 1972 när Palme med sitt så kallade jultal liknade bombningen av Hanoi med bland annat nazisternas utrotningsläger. USA protesterade skarpt och upplyste bland annat att Sveriges nyutnämnde ambassadör i Washington Yngve Möller inte var välkommen. Kontakterna hölls därefter på en låg nivå fram till mars 1974. +Det svenska regeringsskiftet 1976 innebar en kraftig ökning av kritiken mot andra länder. Detta var oväntat eftersom Moderaterna tidigare hade varit kritiska mot vad de kallade "inrikes utrikespolitik". I stället innebar perioden 1976-1982 alltså en kraftig ökning av den svenska kritiken. Mest markant ökade den svenska kritiken av Israel som kritiserades vid 21 tillfällen. Mest kritik riktade dock Sverige mot Sydafrika, för dess apartheidpolitik, invasionen av Namibia och samarbetet med Rhodesia. Den 20 mars 1979 greps Stig Bergling i Israel och dömdes den 7 december samma år för grovt spioneri åt Sovjetunionen. +I samband med Sovjetunionens invasion av Afghanistan i december 1979 försämrades återigen relationerna mellan Sverige och Sovjetunionen. Trots detta gjorde Sveriges utrikesminister Ullsten två besök i Moskva under första halvåret 1980. +Hösten 1980 inträffade en påstådd ubåtsincident i Stockholms skärgård som väckte stor uppmärksamhet, den så kallade Utöincidenten. Påståenden om sovjetiska ubåtskränkningar (och bekräftade sådana) präglade de försämrade relationerna mellan Sverige och Sovjetunionen under stora delar av 1980-talet. +Den 18 september 1980 observerade vakthavande befäl på en av kustflottans bogserbåtar, HMS Ajax, på 300 meters avstånd masterna på en främmande ubåt strax söder om Huvudskär utanför Sandhamn i Stockholms skärgård. En sjunkbomb fälldes mot ubåten men uppenbarligen utan resultat. Ubåtstypen identifierades av marinen som typ Whisky, en ubåtstyp som användes av den sovjetiska och polska marinen. Händelsen fäste uppmärksamheten på Östersjöns strategiska betydelse. Kränkningen har dock aldrig bekräftats, vare sig av sovjetiska/ryska myndigheter eller genom fysiska bevis. Däremot har marinens egna bilder indikerat att det som bombades var ett skeppsvrak och några stenhögar. +På morgonen den 28 oktober 1981 upptäcktes en kärnvapenbestyckad sovjetisk ubåt, U 137, på grund sydöst om Karlskrona örlogsbas inne på militärt skyddsområde. Från sovjetisk sida hävdades att intrånget skett på grund av navigationsfel. Den 5 november överlämnade utrikesminister Ullsten en skarp protestnot: "Den svenska regeringen kräver att Sovjetunionen förhindrar ett upprepande av denna grova kränkning mot Sverige och mot folkrättens fundamentala principer." +På hösten 1982 var det åter dags för ubåtsjakt i Hårsfjärden i Stockholms skärgård. Trots en omfattande jakt med användande av både sjunkbomber och minsprängningar lyckades inte marinen få tag i någon undervattensfarkost. Den 21 oktober tillsatte regeringen Ubåtskommissionen för att utreda händelserna i Hårsfjärden. Kommissionen leddes av den förre försvarsministern Sven Andersson. Övriga ledamöter var Carl Bildt (m), Lars Eliasson (c), Maj-Lis Lööw (s) samt Olle Svensson (s). När kommissionen lade fram sin rapport den 26 april 1983 pekade den ut Sovjetunionen som ansvarig, trots avsaknad av handfasta bevis. I sina memoarer berättar Thage G. Peterson hur statsminister Palme skällde ut Sven Andersson för att kommissionen ställde regeringen inför fait accompli. Palme var inte övertygad om att det handlade om sovjetiska ubåtar men regeringen överlämnade en protestnot till Sovjetunionen: "Kränkningarna utgör ett allvarligt brott mot folkrättsliga regler, vilkas upprätthållande är en förpliktelse och ett samfällt intresse för alla stater. De får tolkas som led i avsiktliga och olagliga försök att utforska det svenska sjöterritoriet." +Ubåtsfrågan ledde till att den partipolitiska enigheten i Sverige upphörde vad gäller säkerhetspolitiken. Vid riksdagens allmänpolitiska debatt den 6 februari 1985 ifrågasatte Moderaternas partiledare Ulf Adelsohn om inte regeringen använde dubbla budskap, det vill säga en hård linje offentligt mot Sovjetunionen men överslätande vid de egna kontakterna med sovjetiska företrädare. Palme replikerade att: "Nu har alla broar rivits vad gäller säkerhetspolitiken. Jag beklagar det djupt. Efter ett eventuellt regeringsskifte i höst väntar en helt ny säkerhetspolitik och en allvarlig fara för Sveriges fred." +Relationerna med Sovjetunionen förblev dåliga under första halvan av 1980-talet. I april 1986 besökte statsminister Carlsson Moskva och i januari 1988 gjorde premiärminister Ryzjkov ett svarsbesök i Stockholm. Vid detta besök kunde länderna komma överens om gränsdragningen i Östersjön, något som varit en tvistefråga sedan 1969. +De socialdemokratiska regeringarna under 1980-talet drog ned på omfattningen av kritiken av förhållanden i andra länder, jämfört med de borgerliga regeringarna. Målen för kritiken var dock ofta samma länder. Störst kritik riktades mot Sydafrika. Kritiken mot Israel ökade särskilt efter utbrottet av den palestinska intifadan 1987: "Solidaritet med det palestinska folket kräver också att vi kraftigt fördömer den israeliska ockupationspolitiken och dess metoder. Denna politik strider mot folkrättsliga och humanitära principer och kan aldrig rättfärdigas." Däremot minskade kritiken av PLO och palestinierna. Båda stormakterna kritiserades: Sovjetunionen för invasionen av Afghanistan och kriget där. USA kritiserades främst för sin hållning i olika länder i Centralamerika: "Det är absurt att påstå att arbetarna, bönderna, studenterna och medelklassen rest sig i Nicaragua, El Salvador och Guatemala därför att någon utländsk makt bett dem göra det. Däremot är det uppenbart att en utländsk makt, Förenta staterna, spelar en avgörande roll när det gäller att hålla vacklande diktaturer under armarna." +Under perioden förekom även meningsskiljaktigheter mellan Sverige och USA av olika anledningar. Ett sådant tillfälle var den så kallade Datasaabaffären där företaget Datasaab sålde ett datoriserat system, i princip en civil version av det svenska STRIL-60, för civil flygledning till Sovjetunionen. Militärt fick systemet sitt elddop i samband med den sovjetiska invasionen av Afghanistan. Då systemet innehöll amerikanska kretskort hade USA vägrat exporttillstånd men systemet, inklusive kretskorten, såldes ändå varpå TV-programmet Aktuellt avslöjade det hela på hösten 1980. Detta ledde till att Sverige vägrades köpa jaktroboten Robot 74 samt motorn till JAS 39 Gripen från USA. Eftersom Datasaab var delvis statligt ansåg USA att den svenska regeringen hade ett medansvar. Affären fortsatte till 1984 när Ericsson, som hade köpt Datasaab, fick böta motsvarande ca 32,5 miljoner svenska kronor till amerikanska staten (senare fördelade så att svenska staten betalade tio och Saab Scania AB 22,5 miljoner). +Även vad gäller USA:s roll i Centralamerika rådde det olika uppfattningar mellan Sverige och USA. Utrikesminister Ullsten hade förklarat att USA:s stöd till Duartes militärjunta i El Salvador försvårade möjligheten att få ett slut på inbördeskriget, ett uttalande som USA protesterade emot. Sveriges kritik av detta fortsatte under 1980-talet och blossade upp igen efter att statsminister Carlsson i en debattartikel i Aftonbladet den 3 februari 1988 uppmanade den amerikanska kongressen att rösta nej till president Reagans begäran om ytterligare anslag till contrasgerillan i Nicaragua. Till Nicaragua gav Sverige bistånd till sandinisterna som styrde Nicaragua. Eftersom Sverige under såväl socialdemokratiska som borgerliga regering kritiserat USA:s agerande i Centralamerika, förklarade USA vid ett par tillfällen framfördes att detta skulle kunna försämra Sveriges möjligheter att köpa avancerad krigssmateriel från USA, men detta omsattes aldrig i praktisk handling. +Ingemar Dörfer har i sin bok Nollpunkten : Sverige i det andra kalla kriget 1979–1986 (1991) sammanfattat Sveriges officiella säkerhetspolitiska dagordning 1984 i följande tio punkter: +"Sverige erbjuder den tredje vägen mellan kommunism och rå kapitalism. Den demokratiska socialismen är framtidsvägen för jordens folk. +Världsläget i allmänhet och det säkerhetspolitiska läget i Europa i synnerhet kan inte bedömas utifrån enbart västliga informationskällor. En negativ bild av Sovjetunionen är en djävulsbild och hindrar en framgångsrik dialog och korrekta relationer med detta land. Fredsforskning är en fruktbar metod att komma förbi dessa traditionella djävulsbilder. +Tal om verkliga styrkeförhållanden i Europa är ett militariserande av tänkandet och förhindrar fortsatt avspänning. Inga amerikanska euromissiler bör baseras i Europa för att motverka den sovjetiska SS 20-uppbyggnaden. +Krav på fred och frihet också i Östeuropa är korstågsanda och förhindrar fortsatt avspänning. Den europeiska situationen kommer att förbli låst och förstelnad. +Det är vapnen och rustningsdynamiken som skapar spänningar och krigsrisker, inte de reella politiska motsättningarna mellan staterna. +Kärnvapenavskräckning är av ondo som försvarsdoktrin. Kapprustningen är farlig och kan leda till kärnvapenkrig. Reaganadministrationen rör sig i riktning mot en warfighting strategy och försöker dessutom begränsa ett kommande kärnvapenkrig till Europa. Natos doktrin bör i första hand förändras till icke första användning av kärnvapen. Genom sin upprustning stör Reaganadministrationen jämvikten och innebär egentligen det större hotet mot freden. +Gemensam säkerhet är det stora alternativet till kärnvapenavskräckning. +Ett kärnvapenfritt Europa är önskvärt. Ett steg på vägen är kärnvapenfria zoner i Norden men också i Centraleuropa. +Kärnvapenfrysning, helst i FN-regi, är en metod att bryta den onda rustningsspiralen. +Det är viktigt att tala om rustningsläget i allmänhet och dess allmänna hot mot freden, inte det svenska läget med ubåtskränkningar i synnerhet med det specifika hotet mot den svenska freden. Tal om specifika hot mot Sverige måste tonas ned eftersom det ifrågasätter kålsuparteorin, försämrar Sveriges förhållande till den ena sidan och hotar Sveriges roll som fredsmäklare." +För att göra Sveriges vilja att vara neutral vid en stormaktskonflikt trovärdig satsade Sverige på ett starkt försvar under början av denna period. Försvarsbeslutet 1958 innebar att försvarsanslaget automatiskt räknades upp med 2,5 procent varje år, men den uppräkningen avskaffades i och med Försvarsbeslutet 1968. Försvarsbeslutet 1972 innebar kraftigt minskade försvarsanslag. +Frågan om Sverige skulle ha kärnvapen diskuterades särskilt under 1950-talet. Högerpartiet, Folkpartiet, militärledningen och Dagens Nyheter förespråkade anskaffandet av svenska kärnvapen. Deras argument för var att försvarets stridsvärde var lågt om det inte fanns egna kärnvapen. +Bland socialdemokraterna var åsikterna delade. Statsminister Erlander ville att Sverige inte skulle avstå från möjligheten att skaffa kärnvapen medan utrikesminister Undén var motståndare och menade att riskerna skulle öka; för Sovjetunionen fanns det skäl att slå till mot Sverige för att på så vis sätta de svenska kärnvapnen ur spel. Att det sistnämnda var den sovjetiska uppfattningen bekräftades 1958. I ett samtal som den svenske Moskvaambassadören Sohlman hade med den sovjetiske försvarsstabschefen marskalk Sokolovskij förklarade marskalken att denne var positiv till den svenska regeringens politik syftande till att vara neutral i händelse av krig. Beträffande atomvapenfrågan förklarade Sokolovskij dock att det inte skulle bli möjligt för Sovjetunionen att ta risken att respektera Sveriges neutralitet - om det lade sig till med kärnvapen. Om Sovjet visste att länder som Sverige förfogade över kärnvapen skulle det av säkerhetsskäl vara nödvändigt att redan i början av ett krig gå till anfall för att slå ut dessa vapen. +Sverige bedrev ett kärnvapenprogram i olika former 1945-1972; under den mest intensiva perioden med beteckningen "handlingsfrihetslinje"; 1967-1972 befann sig programmet i en avvecklingsfas. Frågan om svenska kärnvapen avfördes definitivt 1968 i och med att Sverige undertecknade icke-spridningsavtalet. En "rest" i form av Ågesta Kärnkraftverk, startat 1963, utanför Farsta i södra Stockholm var i drift till 1974. Kärnkraftverket i Ågesta var en tungvattenreaktor med möjlighet att producera bränsle till kärnvapen. +Sverige valde i stället på att förlita sig på att hotet om vedergällning med brittiska och amerikanska kärnvapen skulle avskräcka ett sovjetiskt kärnvapenanfall. +Samtidigt förlitade sig Sverige också på att samma amerikanska och brittiska kärnvapen skulle användas mot sovjetiska baser i Baltikum och mot sovjetiska brohuvuden på svensk mark i händelse av ett invasionsförsök. Som en konsekvens av att det på detta vis förutsattes att väst skulle komma till Sveriges hjälp i händelse av krig fokuserades försvaret på fördröjningsstrid. Stora resurser satsades även på att kunna möta en kustinvasion men för att bekämpa uppmarschvägar i inlandet förlitade man sig på hjälp från allierade flygvapen och saknade därför nästan helt egen förmåga att bekämpa dessa. Då flygvapnet var beväpnat med brittiska och amerikanska jaktrobotar, vilka i krig snart skulle ta slut, förutsattes det även att väst skulle förse Sverige med dessa under krigstid. +Flygvapnet bestod av inhemska stridsflygplan som Tunnan, Lansen, Draken och Viggen. Avsikten var att i händelse av ofred eller krig sprida de flygande divisionerna över ett stort antal krigsbaser i Bas 60 och Bas 90-systemet. Dessa krigsbaser var även kompletterade med alternativa baser, främst reservvägbaser. +Under 1950-talet hade Sverige, näst Storbritannien, det starkaste flygvapnet i Västeuropa. Hela luftförsvaret var byggt för att försvara Sverige mot fientliga inflygningar från öster eller söder, det visar placeringen av radarstationerna och krigsbaserna. Den luftförsvarssektor som svarade för en stor del av luftrummet mot södra Norge saknade helt radartäckning av västgränsen, trots att de norska flygplatserna Sola och Gardermoen vid en krigssituation skulle användas av det amerikanska bombflyget. Trots formell fredstid dog 550 svenska stridspiloter perioden 1946-1991 under avancerade realistiska övningar på låga höjder och hundratals haverier. +Pansartrupperna var utrustade med stridsvagnar som den svenska Stridsvagn S och brittiska Centurion. Infanteriet var utrustade med bärbara pansarvärnsvapen. +Försvaret och det kalla kriget +SOU 1994:11 Om kriget kommit... Förberedelser för mottagande av militärt bistånd 1949-1969 +SOU 2001:85 Perspektiv på ubåtsfrågan www.regeringen.se +SOU 2002:108 Fred och säkerhet. Säkerhetspolitiska utredningen. Utrikesdepartementet 2001, www.regeringen.se +SOU 2003:18 Ett diplomatiskt misslyckande: Fallet Raoul Wallenberg och den svenska utrikesledningen. Utrikesdepartementet 2003. www.regeringen.se +Agrell, Wilhelm (1991). Den stora lögnen : ett säkerhetspolitiskt dubbelspel i alltför många akter. Stockholm: Ordfront. ISBN 91-7324-394-9 +Bohman, Gösta (1970). Inrikes utrikespolitik : det handlar om Vietnam. Stockholm: Geber +Bull, Edvard (1982). Norgeshistorien etter 1945. Oslo: Cappelens förlaga. ISBN 82-02-09026-1 +Bjereld, Ulf (1992). Kritiker eller medlare? Sveriges utrikespolitiska roller 1945-1990. Stockholm: Nerenius & Santerus förlag. ISBN 91-88384-08-X +Dörfer, Ingemar (1991). Nollpunkten : Sverige i det andra kalla kriget 1979-1986. Stockholm: Timbro. ISBN 91-7566-221-3 +Elmbrant, Björn (1989). Palme. Stockholm: Tomas Fischer. ISBN 91-7054-611-8 +Holmström, Mikael (2012, utökad utgåva 2015), Den dolda alliansen: Sveriges dolda NATO-förbindelser. +Hägg, Göran (2005). Välfärdsåren : svensk historia 1945-1986. Stockholm: Wahlström & Widstrand. ISBN 91-46-20273-0 +Kronvall, Olof; Petersson, Magnus (2005). Svensk säkerhetspolitik i supermakternas skugga 1945-1991. Stockholm: Santerus förlag. ISBN 91-89449-71-1 +Myhrberg, Ingemar (1985). Ubåtsvalsen : en motbok till rapporterna från ÖB och Ubåtsskyddskommissionen. Göteborg: Haga Bokförlag. sid. 271. ISBN 91-7734-013-2 +Peterson, Thage G. (2002). Olof Palme som jag minns honom. Stockholm: Bonniers. ISBN 91-0-058042-2 +Törnell, Bernt (2011). Kalla krigets luftförsvar. Lerum: Törnell och Blücher. ISBN 978-91-979865-0-2 +Goulburn Roundhouse är ett cirkelformat lokstall invigt 1918 och stängt 1986, beläget i Goulburn i New South Wales i Australien. Sedan 1990-talet har lokstallet varit en del av järnvägsmuseet Goulburn Rail Heritage Centre. Lokstallet uppfördes i samband med att järnvägen mellan Goulburn och Cootamundra byggdes ut till dubbelspår. Vid invigningen var Goulburn Roundhouse ett av två lokstall i Goulburn och hade 42 spår som betjänades av en vändskiva på 75 fot (22,86 meter). De två lokstallen var i bruk samtidigt fram till 1936 då det första stängdes i syfte att spara pengar under den stora depressionen. Under den tid båda var i bruk svarade den första lokdepån, Goulburn North, för allt tungt underhåll och för persontågsloken. Den nya lokdepån, South Goulburn, svarade för godstågsloken och lättare underhåll av dessa. Under 1930-talet anpassades Goulburn Roundhouse för drift av nya större lok, littera D57, bland annat installerades en ny vändskiva på 90 fot (cirka 27,4 meter). +Depån upprustades flera gånger under 1940-talet. Under 1950-talet inleddes anpassning för dieseldrift samtidigt som det anlades ytterligare infrastruktur för ångdrift. Dieseldrift slog igenom på 1960-talet för att helt ersätta ångdrift efter 1971. Detta innebar en minskning av arbetsmängden vid Goulburn, för att hålla de anställda sysselsatta började depån ägna sig åt vagnsreparation. Utrustningen för ångdrift avvecklades under 1970-talet och en del av lokstallet revs. År 1986 tog järnvägsmyndigheten fram planer för att värna jobb på landsbygden. Å andra sidan ville delstaten skära ned på driftskostnader. Detta mynnade ut i att depån stängdes i juli 1986. Järnvägsmyndigheten ingick 1990 ett avtal med Goulburns kommun för att hyra ut lokstallet vilket ledde till att järnvägsmuseet Goulburn Rail Heritage Centre etablerades. Anläggningen är sedan 1999 upptagen i kulturskyddsregistret State Heritage Register. +Flera näringsidkare och parlamentsledamöter sammanträffade i Sydney i januari 1846 för att diskutera kolonins transportbehov. Månaden därpå beslutade gruppen att utreda möjligheten att anlägga en järnväg mellan Sydney och Goulburn. I april samma år hölls ett möte i Goulburn för att förmå ortens näringsidkare att bli delaktiga i planeringen och informationsframtagningen. Dessa sammanträffanden utmynnade i järnvägsbolaget Sydney Railway Companys bildande i oktober 1849. Bolagets första avtal att anlägga järnvägen skrevs i mars 1851. Som bygget fortskred blev bolaget beroende av bidrag från kolonin och dess tillgångar skrevs över till delstaten 3 september 1855. Den första sträckan, mellan Sydneys centralstation och Parramatta, invigdes 26 september samma år. Järnvägen till Goulburn invigdes etappvis. Den sista etappen, mellan Marulan och Goulburn, invigdes 27 maj 1869. +Efter att Goulburn hade varit slutstation i sex år, invigdes en förlängning av kolonins södra stambana, Main South Line, till Gunning 9 november 1875. Nästa etapp, från Gunning förbi Yass Junction till Bowning, invigdes 3 juli 1876. Förslag på att dra stambanan genom Yass avslogs av John Whitton, New South Wales chefsingenjör för järnväg, då detta skulle ha ökat järnvägens anläggningskostnad. Järnvägen nådde gränsstaden Albury 3 februari 1881 och mötte där Victorian Railways järnväg från Melbourne. Bland anledningarna för att järnvägen skulle anläggas, fanns en önskan från New South Wales att varor producerade i kolonin skulle exporteras från Sydney i stället för från orter i andra kolonier. +Under 1870-talet ivrade invånare i trakten runt Cooma för en järnvägsförbindelse. Från knuten Joppa Junction, nära Goulburn, invigdes järnvägen till Tarago 1884. Denna blev början av en järnväg som invigdes i flera etapper för att nå Queanbeyan 1887, Cooma 1889 och Bombala 1921. En bibana från Queanbeyan till landets huvudstad, Canberra, invigdes 1914. Beslutet att dra stambanan norr om Yass ledde till klagomål från traktens invånare. Anbud för bygget av en järnväg mellan Yass Junction och Yass Town söktes 1890 och järnvägen invigdes april 1892. En arbetsgrupp rekommenderade 1889 en järnväg mellan Goulburn och Crookwell. Bygget inleddes 1900 och järnvägen invigdes i april 1902. Lok och lokpersonal från Goulburn kom att arbeta utmed samtliga dessa banor. +I början planerades det att anlägga infrastruktur för rullande materiel, såsom ett lokstall, en vändskiva och ett skjul för personvagnar cirka fem kilometer norr om stationen. Mellan februari och maj 1869 bestämdes det i stället att anlägga lokstallet och övrig infrastruktur nära stationen. Det är oklart varför en plats nära stationen slutligen valdes för lokstallet. En möjlighet är att det skulle ha varit omöjligt för stinsen att övervaka lokstallet och stationen samtidigt om dessa låg långt ifrån varandra. Två faktorer kan har föranlett beslutet att välja en ny plats för depån. Den första var att den tänkta platsen stundom var utsatt för översvämningar och den andra var att vattnet där inte lämpade sig för ånglok. +Lokstallet togs i bruk runt tiden järnvägen invigdes. Byggnadens mått var 40 ×107 fot (cirka 12,2 × 32,6 meter) och hade två spår. En kolbrygga flyttades från Harden till Goulburn 1880 och lokstallet byggdes ut 1881. Ett nytt lokstall uppfördes 1885 cirka 24 meter norr om det första lokstallet. Den nya byggnaden anlades så att de två lokstallen bildade en "L"-form. I mitten av det nya lokstallet fanns ett skjutbord på 50 fot (15,24 meter) och på vardera kortsidan av skjutbordet fanns det tillräckligt långa spår för att rymma ett lok. Sammanlagt var det 15 spår på den ena sidan av skjutbordet och 13 spår på den andra. I samband med anläggningen av den nya byggnaden gjordes det förra lokstallet om till en verkstad. +Depån fortsatte att upprustas. En ny kolbrygga anlades 1891 och samtidigt började hydraulisk utrustning användas för kolning av lok. En vändskiva på 60 fot (cirka 18,3 meter) anlades 1897. Planer på en större utbyggnation av depån förberedes 1913 men gick om inget på grund av penningbrist som en följd av första världskriget. Senast 1914 togs skjutbordet ur bruk och spåren i lokstallet anslöts i stället med växlar. En ny lokstation vid South Goulburn invigdes 1918 och de två lokstationerna brukades samtidigt fram till att den första stängdes 1936. Anläggningen revs 1941. +Som en del av arbetet att bygga ut järnvägen mellan Goulburn och Cootamundra till dubbelspår beviljade delstaten anslag för bygget av ett nytt lokstall i Goulburn. Översiktsplaner förbereddes 1915 efter det att platsen för lokstallet hade valts. Den utvalda platsen ligger på avstånd från stationen, intill och högre än vägen Braidwood Road. Placeringen är förenlig med järnvägsväsendets praxis efter första världskriget att bygga nya anläggningar på avstånd från bebyggda områden runtom järnvägsstationer. Efter att detaljplaner hade tagits fram 1916 utarbetades nya planer 1917. Planen från 1917 bestod av två cirkelformade lokstall med 42 spår vardera och utrymme för ytterligare ett lokstall i samma utformning. Det andra och tredje lokstallet kom aldrig att byggas. +Det 42-spåriga cirkelformade lokstallet invigdes 15 maj 1918. Lokstallet är uppfört i målat murtegel till en höjd på 1,6 meter, ovanpå finns fönster med fönsterkarmar i trä och asbestcementplattor. Byggnaden har ett sågtak i korrugerad plåt. Sågtak var den vanligaste taktypen för cirkelformade lokstall i New South Wales. New South Wales hade börjat anlägga cirkelformade lokstall 1905 och detta är ett uttryck på att amerikansk inflytelse hade vunnit mark över brittisk inflytelse. Vid invigningen fanns en vändskiva på 75 fot (22,86 meter) som betjänades av två spår, ett för inkommande lok och ett för avgående lok. Det fanns vid invigningen inte tillräckligt med ekonomiska medel för att flytta all utrustning till det nya lokstallet. Detta innebar att allt tungt underhåll av lok utfördes vid det första lokstallet, Goulburn North. Goulburn North svarade även för persontågsloken, medan godstågsloken använde den nya depån, South Goulburn, där även mindre underhåll utfördes. Beslutet att skilja på persontågs- och godstågslok kan ha varit avsiktligt då en liknande indelning gjordes i Sydney 1916. +South Goulburn försågs med vatten från en reservoar matad från ett vattenmagasin i vattendraget Mulwaree Ponds. Själva dammbyggnaden hade uppförts av järnvägsväsendet och tagits i bruk 1908. I samband med lokstationens anläggning beslutades det att uppföra en till reservoar och på så sätt fördubbla vattenkapaciteten till 1 miljon gallon (cirka 4,55 miljoner liter). Denna andra reservoar anlades 1922. Ett nytt vattenmagasin med större kapacitet anlades 1926–1927 i Mulwaree Ponds. En ny ledning mellan reservoarerna och South Goulburn anlades 1943 och samma år försågs lokstationen med två nya vattenhästar. Lokstationens vatten kom från Mulwaree Ponds fram till 1967 då reservoarerna anslöts till stadsvattnet. +Kolbryggan förlängdes 1922 och hade då en kapacitet på 1 000 engelska ton (1 016 metriska ton). I en plan från 1918 ingick en maskinverkstad på 100 × 50 fot (30,48 × 15,24 meter). Denna kom inte att uppföras. Lokstallet i Wellington, med längd 157 fot (cirka 47,85 meter) och tillräckligt brett för att rymma fyra spår, flyttades till Goulburn Roundhouse 1930. Byggnaden blev känd som Wellington Shed och åtkomst är via spår 19. Ånglok littera D57 började trafikera delstatens södra stambana 1930. För att kunna vända D57-loken byttes vändskivan 1935 till en eldriven modell på 90 fot (cirka 27,4 meter). Året därpå modifierades vändskivan så att den kunde drivas med tryckluft från lok. I syfte att minska kostnader under den stora depressionen på 1930-talet beslutades det att slå samman Goulburns två lokdepåer. Maskineri som fanns vid Goulburn North flyttades till Wellington Shed och anpassades för att köras med elektricitet i stället för ånga. +Ett kontor åt Steam Shed Inspector planerades 1934 och byggdes året därpå. Byggnaden har en träpanelad fasad och verandor på alla sidor och det valmade taket är lagt med korrugerade asbestcementplattor. Kontorets placering nedanför lokstallet är ovanligt. Praxis i New South Wales var att Steam Shed Inspector-kontor skulle vara upphöjda som ett tecken på ämbetets status, alternativ skulle de förläggas till depåns ingång. Att kontorets placering i Goulburn inte stämmer med detta mönster beror på att det inte fanns annan obebyggd mark vid depåns frontsida när kontoret byggdes. Under 1940-talet skedde flera förändringar vid depån. Anläggningen för att tömma lok på aska och hantera restprodukten upprustades och byggdes ut flera gånger under decenniet, inklusive 1944 när pneumatisk utrustning anlades. Utrustning för sandtorkning och en ny sandbehållare uppfördes 1945. Fyra oljetankar anlades 1949–1950 i syfte att förvara bränsleolja åt D55-lok. +New South Wales första diesellok för stambanetrafik kom i tjänst november 1951. Vid Goulburns lokstation inleddes anpassningen för dieseldrift året därpå när tankningsutrustning inköptes. Ånglokets tid var inte förbi, en av depåns vattentankar utökades i kapacitet från 20 000 gallon till 30 000 gallon (cirka 90 920 respektive 135 380 liter) 1954. Ett 130 fotlångt (cirka 39,6 meterlångt) skjul med pulpettak avsett för bruk av ånglok littera AD60 uppfördes 1960. Samma år anpassades själva lokstallet för dieseldrift genom att spåren 37–42 försågs med dörrar och spåren 37–41 särskilt utrustades med uttag för luft, el, vatten och smörjolja. Goulburns första tilldelning diesellok, tio littera 48, skedde mellan sent 1960 och juni 1961. Dieseldrift på järnvägen mot Bombala inleddes 1959 för att helt ersätta ångdrift 1967. Mot Crookwell inleddes dieseldrift 1961 och hann ersätta ångdrift inom två år. Det beslutades 1962 att sluta använda ånglok för att dra godståg mellan Goulburn och Albury och även att begränsa deras bruk på persontåg på sträckan. Detta efterlevdes inte fullt ut, då ångloksdrift på godståg fortsatte i avtagande omfattning ända till tidigt 1964. Ånglok fortsatte att dra vissa persontåg på sträckan fram till juni samma år. +För att anpassa verksamheten till det minskade behovet av underhåll som övergången till dieseldrift medförde, började man 1962 även att utföra reparation av järnvägsvagnar. Behållare för sammanlagt 24 000 gallon (cirka 109 100 liter) diesel och två behållare på 1 000 gallon (4 564 liter) vardera för ny respektive förbrukad smörjolja fanns på plats 1966. Goulburn tilldelades 1964 ett diesellok av typen X200 för bruk mellan Yass Junction och Yass Town Senast 1967 kom ett till 48-lok. Ånglok slutade att dra tåg vid Goulburn i oktober 1969 men fortsatte i tjänst som växlingslok fram till december 1971. Från och med 1970 började vattenhästarna och vattentankarna att rivas. Bruket av X200-loket upphörde i februari 1972. Byggnaden över spåren 1–17 ansågs vara överflödig och revs 1973. Ett nytt kontor i murtegel uppfördes för chargemannen under sena 1970-talet efter att den tidigare byggnaden brann ned i juni 1975. Den nya byggnaden är belägen nära spår 1. I november 1977 söktes anbud för kolbryggans rivning. På platsen för kolbryggan anlades 1980 och invigdes 1981 en särskild verkstad för vagnreparation. År 1979 byggdes nya dieseltankar vid flera depåer utanför Sydneyområdet. För Goulburns del blev det två tankar på 200 000 liter, medan andra lokstationer, såsom Albury och Werris Creek, fick större tankar. Detta har tolkats som ett tecken på Goulburns avtagande betydelse inom järnvägen. +Persontåg upphörde mellan Goulburn och Crookwell i augusti 1974 och all trafik lades ned efter 1985. Trafik mellan Cooma och Bombala drogs in 1986 och tre år senare drogs trafiken mellan Queanbeyan och Cooma in. Järnvägsväsendet framtog planer 1986 där vartenda lok skulle tilldelas en verkstad där större tillsyn och underhåll skulle utföras, detta i syfte att försäkra jobb i landsbygden. Under planen tilldelades Goulburn 14 stycken 48-lok. Dessa var dock inte samma lok som hörde hemma i Goulburn. Senare under året ändrade delstaten linje och menade att mindre depåer, såsom Goulburn, Casino och Orange, borde stängas för att minska driftskostnaderna. I Goulburns fall skulle allt arbete överföras till Junee. Goulburn stängdes som lokverkstad 4 juli 1986. På stängningsdagen flaggades det uppochnedvänt och på halv stång. När depån stängdes entledigades 24 anställda medan 38 tog tjänst vid Junee och 14 blev lokbesättning. Lok fortsatte att uppställas och tankas vid Goulburn Roundhouse fram till 1990. Framöver kom lok att ställas upp nära stationen. +En plan för ett överliggningshus vid South Goulburn ritades 1916. Byggnaden skulle ha varit i murtegel med ett tegeltak och ha haft 20 sovplatser. Åtkomst till sovrummen var från verandor. Huset uppfördes inte och i stället flyttades ett antal stugor till South Goulburn. Stugorna, som hade måtten 18 × 9 fot (cirka 5,48 × 2,74 meter), kom från olika orter i delstaten. En ny överliggningshusplan framtogs 1925 men kom inte heller att realiseras. Byggnadens planerade mått var 152 × 27 fot (cirka 46,33 × 8,23 meter) med väggar i murtegel med ett tak i asbestcementplattor. +Ett överliggningshus i murtegel uppfördes vid South Goulburn 1935. Huset hade tio sovrum med två sovplatser i varje rum och ett badrum med en toalett och en pissoar. Avskilt från byggnaden var en tvättstuga, även denna i murtegel. Efter dess kom ett antal av bodarna att flyttas till andra orter i delstaten. Huset byggdes ut 1941 med sex nya sovrum och ytterligare en toalett och en pissoar. Tolv hyttor anlades 1950 intill överliggningshuset som bostäder för migranter från andra världskriget som tagit anställning hos järnvägsväsendet. Överliggningshuset var i bruk fram till cirka 1977 och kom tre år senare att byggas om till ett rehabiliteringscentrum för järnvägsanställda lidande av substansmissbruk. Huset kom senare att säljas och användas av bilentusiaster besökande racerbanan Wakefield Park. +Samma år som lokstallet stängdes som verkstad, 1986, var det tänkt att det skulle rivas. Efter att ett antal ur lokpersonalen blivit varse planen, inleddes en insats för att förhindra rivningen. Insatsen mynnade ut i att ett tillfälligt rivningsförbud utfärdades. Intresserade parter, bland dem kommunen och järnvägsanställda, fick då två veckor på sig för att framta planer för lokstallets framtid. Ett möte resulterade i att föreningen Goulburn Loco Preservation Group, sedermera Goulburn Loco Roundhouse Preservation Society Inc., bildades. Föreningen bedriver järnvägsmuseet Goulburn Rail Heritage Centre. Järnvägsmyndigheten skrev 1990 ett avtal med Goulburns kommun för att disponera lokstallet med avsikt att ett järnvägsmuseum skulle etableras. Samma år började privatägda lok ställas upp i lokstallet. Vid tiden var det tänkt att museet skulle tjäna som ett nationellt järnvägsmuseum med möjlighet till att underhålla järnvägsfordon på kommersiell basis. +Goulburn Roundhouse och de intilliggande byggnaderna antogs till delstatens kulturskyddregister, State Heritage Register, 2 april 1999. Detta motiverades av att Goulburn Roundhouse är ett av sju kvarvarande cirkelformade lokstall i delstaten. Även det att lokstallet utgör ett gott exempel på en större industriell järnvägsbyggnad och har mycket av sitt ursprungliga material och sin ursprungliga form kvar, poängterades. +Museum of Memorabilia, ett utställningsutrymme på 95 kvadratmeter, tillkom under 2010-talet. Föreningens mål med detta utrymme är att personer som inte har en bakgrund inom järnvägen ska kunna skapa sig en uppfattning om de mindre kända aspekterna av järnvägsverksamheten och om de personer som tillgodoser samhällets behov av järnvägstransporter. En upprustning av utrymmet inleddes 2020 och beräknades kosta 34 410 australiska dollar. +I juli 2013 skadades vändskivan allvarligt då ett tåg kört av Aurizon spårade ur i samband med att det växlade. Skadan uppskattades kosta över 500 000 dollar och gjorde att flera lok blev instängda i lokstallet. Bland de instängda loken fanns ett tillhörande ett kommersiellt järnvägsbolag som fanns på museet för att underhållas och repareras. Vändskivan togs åter i bruk i januari 2016. Evenemanget Streamliners 2016, där 20 diesellok ställdes ut och drog veterantåg, anordnades vid museet 1–3 oktober 2016. Fler än 2 000 personer besökte evenemanget, som också avhandlades i första avsnittet i andra säsongen av Discovery Channels TV-serie Railroads Australia. +1072 är en bärgningskran tillverkad av Craven Brothers. Den beställdes 1929 och hade en lyftförmåga på 70 engelska ton (cirka 68,6 metriska ton). Den användes i bärgningsarbetet efter järnvägsolyckan i Rocky Ponds 1948 och gick då på ånga. Kranen blev sedermera ombyggd för att gå på diesel. +1076 är ett tanklok tillverkat 1884 av Vulcan Foundry som inledningsvis användes för att dra pendeltåg i Sydney. Loket användes som växlingslok i Goulburn 1953–1963 och är sedan 2017 restaurerat. +2419 är ett ånglok tillverkat 1891 av Dübs and Company som användes för att dra godståg runt om i New South Wales. Loket flyttades till Goulburn i augusti 1989, och restaurerades ytligt 1997. +3085 är ett tanklok tillverkat 1912 av Eveleigh Workshops. Det användes i pendeltågstrafik i Sydney. +4821 är ett diesellok som tillverkades av A.E. Goodwin Ltd. och tillhör en loktyp som tillverkades i 165 exemplar mellan åren 1959 och 1970. Loktypen användes runt om i New South Wales. Loket är ägt av Goulburn Loco Roundhouse Preservation Society Inc. +5908 och 5916 är ånglok tillverkade 1952 av Baldwin Lima-Hamilton Corporation. De användes för att dra godståg. Loken levererades som oljebrännare och är två av endast tre lok av typen D59 som inte byggdes om till kolbränning. Loken flyttades till Goulburn i november 2009. +E 20756 är en flakvagn och bland de tidigaste boggivagnarna i New South Wales. +FP7 är en rälsbuss som användes för att transportera anställdas löner. Efter att det hade inretts speciellt för tillfället, åkte Elizabeth II med FP7 1970. Rälsbussen kom till Goulburn i november 1988. +HC 1386 är en personvagn tillverkad 1909 av Clyde Engineering. Vagnen byggdes om två gånger, först 1938 och sedan 1972. Vagnen togs ur bruk och kasserades i juni 1989. +Gilbertson, Col (september 2019). ”Dieselisation of the Short South to Goulburn” (på engelska). Australian Railway History (Australian Railway Historical Society, NSW Division) 70 (983): sid. 10–17. ISSN 1449-6291. +Gunn, John (1989) (på engelska). Along Parallel Lines: A history of the railways of New South Wales. Carlton, Victoria: Melbourne University Press. ISBN 0-522-84387-5. https://archive.org/details/alongparallellin0000gunn +Hagarty, Donald (2005) (på engelska). Sydney Railway: 1848–1857. Redfern, New South Wales: Australian Railway Historical Society New South Wales Division. ISBN 0-9757870-1-2 +Oberg, Leon (2010) (på engelska). Locomotives of Australia, 1850s-2010. Rosenberg Publishing Pty Ltd. ISBN 9781921719011 +Sharp, Stuart (2016) (på engelska). The railway heritage of Goulburn. Pinewood, Victoria: Train Hobby Publications. ISBN 9 781921 122576 +Scrymgeour, R. (november 1997). Grunbach, A. red. ”A History of the Goulburn–Crookwell Line” (på engelska). Australian Railway Historical Society Bulletin (Redfern, New South Wales: Australian Railway Historical Society, New South Wales Division) 48 (721): sid. 395–418. ISSN 0005-0105. +Taaffe, Robert T (februari 2020) (på engelska). Signal Boxes of New South Wales Railways and Tramways. "Volume 3 - Gazetteer South and West". Hobart, Tasmanien: Taaffe Press. ISBN 978-0648529224 +The Department of Railways Information Bureau (oktober 1967) (på engelska). Main Southern Line Maps. Sydney, New South Wales +Wikimedia Commons har media som rör Goulburn Roundhouse. +T-34 var en sovjetisk medeltung stridsvagn som tillverkades mellan 1940 och 1958. Stridsvagnen anses i allmänhet av många historiker ha varit krigets kanske mest effektiva ändamålsenliga och inflytelserika stridsvagnskonstruktion. Vagnen kunde enkelt produceras i tusental av Sovjet och var inför Nazitysklands anfall mot Sovjetunionen i juni 1941, avsevärt bättre pansrad och beväpnad än någon tysk stridsvagn fram till följande år. T-34 tillverkades först vid ChPZ-fabriken i Charkiv (nuvarande Ukraina) och den utgjorde stommen i de sovjetiska pansarstyrkorna under hela kriget. Efter krigsslutet blev stridsvagnsmodellen en stor exportframgång. Det tillverkades fler T-34:or under kriget än någon annan stridsvagn och den är endast överträffad av sin efterföljare T-54/-55 i antal tillverkade stridsvagnar totalt. Så sent som 1996 var T-34 ännu i tjänst i 27 länder. +T-34 utvecklades från BT-stridsvagnen och det var tänkt att den skulle ersätta denna serie samt den lätta T-26. När den introducerades kunde den uppvisa den bästa balansen av eldkraft, rörlighet och skydd av alla stridsvagnar som fanns då. Inledningsvis kom dess stridseffektivitet att hämmas av dålig ergonomi, bristen på radioapparater och undermålig taktik. +I början av 1944 introducerades den förbättrade T-34-85 som var utrustad med en mer kraftfull kanon i ett större tre-mannatorn. Designen och konstruktionen förfinades gradvis under kriget, dels för att förbättra prestanda men också för att sänka produktionskostnaderna, vilket medförde att produktionen kontinuerligt ökade. Vid krigets slut 1945 hade T-34 ersatt många lätta och tunga stridsvagnar i tjänst och utgjorde huvuddelen av den sovjetiska stridsvagnsproduktionen. Stridsvagnen skulle komma att utöva stort inflytande på det moderna huvudstridsvagn-konceptet, där en och samma modell kan lösa flera olika uppgifter på slagfältet. +Före krigsutbrottet 1939 var stridsvagnen T-26 och BT-vagnarna de vanligaste modellerna inom det sovjetiska pansarvapnet. T-26 var en långsam infanteriunderstödsvagn som var konstruerad för att hålla jämn takt med infanteristerna på marken. BT-stridsvagnarna var så kallade kavalleristridsvagnar, mycket snabba och lätta stridsvagnar som användes för bekämpning av andra stridsvagnar, men som inte var avsedda att användas mot infanteri. Båda typerna hade ett tunt pansar som skyddade mot finkalibrig eld men inte mot pansarvärnsgevär och 37 mm pansarvärnskanoner. Deras bensindrivna motorer antändes "vid minsta provokation" . +Ingenjören Michail Kosjkin fick 1937 i uppdrag av röda armén att leda en grupp som skulle konstruera en ersättare till BT-vagnarna. Dessa nya stridsvagnar skulle tillverkas vid lokomotivfabriken ChPZ i Charkov. Prototypstridsvagnen, kallad A-20, utrustades med ett 20 mm pansar, en 45 mm kanon och ingenjör Aleksandr Morozovs nya V-2-dieselmotor som var mindre benägen att fatta eld. Kvar från BT-vagnen var möjligheten att köra på hjul i stället för med larvband. Det ledde till färre reparationer och servicetimmar på 1930-talets dåliga larvband och tillät även stridsvagnarna att uppnå drygt 80 km/h på vägar. I strid var dock hjul ingen fördel. Konstruktörerna kom dock fram till att det var ett onödigt nyttjande av utrymme och vikt. +Kosjkin övertygade den sovjetiska ledaren Josef Stalin att låta honom utveckla en annan prototyp, en bättre bepansrad och beväpnad "universalstridsvagn" som kunde ersätta både T-26 och BT-vagnarna. Den andra prototypen, som kallades A-30 men som kort därefter döptes om till T-32, hade 32 mm pansar, en 76 mm kanon och samma V-2-dieselmotor. Båda prototyperna fälttestades vid Kubinka 1939 och det visade sig att den tyngre T-32 var lika mobil som A-20. Motståndet från militärledningen och farhågorna om höga produktionskostnader övervanns slutligen på grund av de dåvarande sovjetiska stridsvagnarnas dåliga prestation i vinterkriget mot Finland och effektiviteten hos tyskarna i blixtkriget mot Frankrike. En ännu tyngre version av T-32, med 45 mm frontpansar och bredare larvband togs därefter upp för produktion och gavs namnet T-34. +Två prototyper av T-34 färdigställdes i februari 1940 och genomgick en 2 000 km lång testkörning under svåra förhållanden från Charkov till Moskva, Smolensk, Kiev och tillbaka till Charkov i april och maj. Vissa brister i drivlinan identifierades och rättades till. De första serietillverkade stridsvagnarna blev färdiga i september 1940 och kom att helt ersätta stridsvagnarna T-26, BT och T-28. Kosjkin dog av lunginflammation i slutet av månaden och arbetet med att utveckla T-34:s drivlina gavs till Aleksandr Morozov som utnämndes till chefsdesigner vid ChPZ. +Liksom BT-vagnarna hade T-34 hade en fjäderupphängning av Christie-typ, men man bestämde att den tunga och ineffektiva konvertibla driften med både hjul och band skulle tas bort. Den hade sluttande pansar, en relativt kraftfull motor och bredspåriga larvband. Den första versionen hade en 76,2 mm kanon och kallas ofta för T-34/76 (ursprungligen en tysk beteckning som härstammar från andra världskriget). Sent 1943 inleddes produktionen av den andra huvudversionen T-34-85 (eller T-34/85) som hade ett större torn och en större 85 mm kanon. +Den nya stridsvagnen ställde den sovjetiska industrin inför nya utmaningar. Den var den mest bepansrade medeltunga stridsvagnen som tillverkats fram till denna tidpunkt och sammansättningen av delkomponenter gjordes vid flera fabriker: Charkovs dieselfabrik Nr. 75 tillhandahöll V-2 -motorn, Kirovfabriken Nr. 185 i Leningrad tillverkade den ursprungliga L-11-kanonen och Dynamofabriken i Moskva tillverkade elektriska komponenter. Stridsvagnarna ihopmonterades inledningsvis vid ChPZ Nr. 183 och vid Stalingrads traktorfabrik (STZ) och senare vid Krasnoje Sormovofabriken Nr. 112 i Gorkij. Man hade dock vissa problem med defekta pansarplåtar. På grund av bristen på de nya V-2 -motorerna utrustades många stridsvagnar i den första produktionsomgången år 1940 med BT-vagnens flygplansmotor som hade sämre prestanda, transmission och koppling. Man kunde endast utrusta kompanichefernas stridsvagnar med radioapparater. L-11-kanonen kom inte att leva upp till förväntningarna och därför skapade Grabins designbyrå vid Gorkijfabriken Nr. 92 en överlägsen F-34 76,2 mm-kanon. Byråkraterna vägrade tillåta produktionen, men Gorkij och ChPZ inledde den utan lov; officiellt tillstånd kom först från Stalins statliga försvarskommitté efter att trupper i fält lovprisat den nya kanonen. +De konservativa elementen inom politiken satte dock press på armén att bygga de äldre stridsvagnarna T-26 och BT eller att helt avbryta produktionen av T-34 i väntan på den mera avancerade T-34M-konstruktionen. Det tyska överraskningsanfallet mot Sovjetunionen den 22 juni 1941 (operation Barbarossa) tvingade dock Sovjetunionen att frysa vidare utveckling och inleda full produktion av stridsvagnar. +Tysklands snabba offensiv tvingade ryssarna att flytta sina fabriker till öster om Uralbergen, en monumental uppgift. ChPZ återupplivades kring Dzerzjinskij Urals järnvägsvagnsfabrik i Nizjnij Tagil. Fabriken döptes om till Stalins Ural stridsvagnsfabrik Nr. 183. Kirovfabriken evakuerades bara veckor innan Leningrad inringades och fabriken flyttades tillsammans med Charkovs dieselfabrik till Stalins traktorfabrik i Tjeljabinsk, som snart skulle komma att kallas Tankograd ('Stridsvagnsstaden'). Vorosjilovs stridsvagnsfabrik Nr. 174 från Leningrad inkorporerades in i Uralfabriken och den nya Omskfabriken Nr. 174. Ordzjonikidze Urals tunga verktygsmaskinfabrik (UZTM) i Sverdlovsk absorberade flera mindre fabriker. Medan dessa fabriker förflyttades med rekordfart kom industrikomplexet kring Stalingrads traktorfabrik att producera 40 % av alla T-34 fram till att fabriken var helt omringad i slaget vid Stalingrad under hösten 1942. +Bortsett från detta avbrott, tilläts endast mindre ändringar i produktionslinjerna - dessa ändringar var oftast för att göra stridsvagnarna enklare och billigare att producera. Nya metoder utvecklades för automatisk svetsning och härdning av plåtar (metoden hade utvecklats av professor Jevgenij Paton). Konstruktionen av kanon 76,2 mm F-34 modell 1941 förenklades till att bestå av endast 614 delar, jämfört med föregångarens 861. Under de två följande åren minskades produktionskostnaderna per stridsvagn från 269 500 rubel år 1941, till 193 000 och senare till 135 000. Produktionstiden skars ner till hälften vid slutet av 1942 trots att majoriteten av de erfarna fabriksarbetarna hade sänts ut till fronten och ersatts av en arbetsstyrka som bestod av 50 % kvinnor, 15 % pojkar och 15 % invalider och gamla män. Vid samma tidpunkt kom T-34 att gå från att vara "vackert tillverkade maskiner med utmärkt finish jämförbara med eller överlägsna Västeuropas eller Amerikas" till mera grovtillverkade stridsvagnar. Ingenting tummades dock på den mekaniska tillförlitligheten. +1942 påbörjades arbetet med att bygga ett nytt torn som utvecklats från det övergivna T-34M-projektet. Den nya konstruktionen innebar förbättringar av utrymmet och tillät även införande av en kupol för vagnchefen, vilket gav bättre uppsikt runt hela vagnen. Den begränsade tillgången på gummi ledde till att man införde stålhjul. En ny koppling monterades vilket förbättrade funktionen på motorn och den femväxlade kraftöverföringen. +De sovjetiska trupperna stötte 1942 och 1943 på de nya tyska stridsvagnarna Tiger och Panther. Med erfarenheterna från slaget vid Kursk i färskt minne och efter förfrågningar från fronttrupperna om eldkraft med längre räckvidd fattade den sovjetiska militärledningen det svåra beslutet att bygga om fabrikerna för att tillverka en ny modell som hade en större tornring (från 1 425 mm till 1 600 mm), vilket tillät att man satte dit ett större torn. T-34-85:s nya 85 mm kanon var vida överlägsen den gamla kanonen och det nya tremannatornet utrustades med en radio (tidigare hade radion funnits nere i chassit). Nu kunde stridsvagnschefen leda striden och överlämna kanontjänsten till laddaren och skytten. +Kostnaden för att producera en T-34-85-stridsvagn var 164 000 rubel, vilket inledningsvis var 30 % högre än en motsvarande modell 1943, men 1945 hade priset gått ner till 142 000. Under krigets gång hade priset för en T-34 -stridsvagn nästan halverats medan dess rörlighet förblev nästan oförändrad. Samtidigt fördubblades nästan kanonens genomslagsförmåga och pansarets tjocklek. +Vid slutet av 1945 hade mer än 57 000 T-34 byggts: 34 780 exemplar av den ursprungliga modellen av T-34 byggdes 1940–44 och ytterligare 22 559 T-34-85 1944–45. Efter kriget upphörde den storskaliga massproduktionen av stridsvagnen i Sovjetunionen (under 1946 byggdes "bara" 2 701 exemplar av typen). Tillverkningen återupptogs 1951 i Polen och i Tjeckoslovakien, där 1 380 respektive 3 185 T-34-85 tillverkades fram till 1956. Sent under 1960-talet genomgick sovjetiska T-34-85 ett moderniseringsprogram (T-34-85M) för exportstridsvagnar och för reservstridsvagnar. De återutrustades med motordelar från T-54/55-seriens stridsvagnar, vilket var ett bevis på hur långt man gick för att standardisera komponenterna i de sovjetiska stridsvagnskonstruktionerna. +Man har uppskattat att det sammanlagt tillverkades så många som 84 070 exemplar av T-34, plus 13 170 självgående artilleripjäser som var byggda på T-34:s chassi. Vissa kom ännu att tjäna i diverse oroshärdar runtom i världen under det kalla kriget. +Att identifiera olika versioner av T-34 kan vara komplicerat. Torngjutningar och ytliga detaljer samt utrustning varierade mellan de olika fabrikerna. Nya detaljer introducerades i mitten av en tillverkningsserie eller tillfördes äldre stridsvagnar. Utslagna stridsvagnar reparerades, i vissa fall använde man nyare modellers komponenter, till och med nya torn. Vissa stridsvagnar fick extra pansar, som utgjordes av återvunnet stålskrot av varierande tjocklek. Det svetsades fast vid skrovet och möjligen också på tornet; dessa stridsvagnar kallas s ekranami ("med skärmar"). +Den tyska underrättelsetjänsten under andra världskriget kallade de två huvudproduktionsmodellerna för T-34/76 och T-34/85, där mindre modeller fick bokstavsbeteckningar, såsom T-34/76A – denna nomenklatur har sedan dess använts i väst, speciellt inom populärlitteraturen. Sedan 1980-talet har många akademiker (däribland stridsvagnsexperten Steven Zaloga) använt de sovjetiska benämningarna: T-34 och T-34-85, där mindre tillverkade modeller särskilts med år: T-34 modell 1940. [Detta system används även i denna artikel.] +Enligt källor som offentliggjorts sedan Sovjetunionens sammanbrott kallades alla T-34 med originaltornet och F-34-kanonen (konventionellt kallade Modell 1941 och 1942) officiellt för modell 1941 och versionen med det hexagonala tornet, T-34 (modell 1943) kallades officiellt modell 1942. +Erövrade T-34 i tysk tjänst kallades Panzerkampfwagen T-34(r), där r stod för Russland ("Ryssland"). +Finländarna kallade T-34 för Sotka som betyder and eftersom stridsvagnens profil påminner lite om knipans (enligt det finska tankässet Lauri Heino). T-34-85 kallades pitkäputkinen Sotka, "långpipad and". +T-34 (tysk beteckning : T-34/76) var den ursprungliga stridsvagnen med en 76,2 mm kanon. +modell 1940 (T-34/76A) – Tidig produktionsserie med den tillfälliga 76,2 mm L-11 -kanonen i ett tvåmanstorn, stridsvagnen hade BT-vagnarnas gamla bensinmotor och koppling. +modell 1941 (T-34/76B) – Den huvudsakliga produktionsserien med tjockare pansar och den bättre 76,2 mm F-34 kanonen, dieselmotor. +modell 1942 (T-34/76C) – Många mindre förbättringar av tillverkningsprocessen. +modell 1943 (T-34/76D, E och F) – nytt gjutet hexagonalt torn, kallades "Micky Maus" av tyskarna på grund av dess utseende när de två runda tornluckorna var öppna. Huvudproduktionsserien hade en ny chefskupol. +T-34/57 – Färre än 324 T-34 utrustades med 57 mm ZiS-4- eller ZIS-4M-kanonen med hög utgångshastighet åren 1941 och 1943–44 där de användes som pansarvärnsvagnar. Vissa av dem deltog i slaget om Moskva. +T-34-85 var en större förbättring med ett tremanstorn och en 85 mm kanon med långt eldrör. +modell 1943 – Kortvarig produktionsserie (februari-mars 1944), bestyckad med 85 mm D-5T -kanonen. +modell 1944 – Huvudproduktionsserie med den enklare 85 mm ZiS-S-53-kanonen. Radion var flyttad från chassit till tornet som hade en förbättrad layout och ett nytt skyttesikte. +Man fortsatte göra diverse tekniska förbättringar på T-34-85, inklusive ett större restaureringsprogram åren 1960 och 1969. Alla T-34-85-modeller var mycket likartade externt. +Den förkrigstida utvecklingen av den mera avancerade T-34M-stridsvagnen återupptogs år 1944, vilket ledde till stridsvagnen T-44. Den nya stridsvagnen hade en tornkonstruktion som baserades på T-34-85 men hade ett nytt chassi, med torsionsstavsupphängning och tvärställd motor. Den hade en lägre profil än T-34-85 och var enklare att tillverka. Mellan 150 och 200 exemplar av denna stridsvagn tillverkades innan krigets slut. Efter viss modifikation på dess drivkälla, tillsammans med ett nytt torn samt en ny kanon, döptes stridsvagnen till T-54 som började tillverkas 1947. +Eldkastarstridsvagnar – OT-34 och OT-34-85 utrustades med en internt monterad eldkastare som ersatte chassikulsprutan. +PT-34 — minvältsstridsvagn, de flesta byggdes på chassin av stridsvagnarna T-34 modell 1943 eller T-34-85. +Bandkanoner – T-34:ans chassi kom att utgöra grunden för flera serier kanonvagnar: +SU-122 (infanterikanonvagn) +SU-85 (pansarvärnskanonvagn) +SU-100 (pansarvärnskanonvagn) +Efter det andra världskriget utrustades vissa egyptiska och syriska T-34 med 122 mm haubitsar. +Många understödsfordon och till och med civila traktorer byggdes på T-34:s chassi från andra världskriget ända in på 1990-talet. De flesta fordon som byggdes om var antingen skadade stridsvagnar eller självgående artilleripjäser. +Broläggarstridsvagnar – Gamla stridsvagnar som byggdes om i fält eller vid reparationsställen. De kördes helt enkelt ner i vattnet för att låta de övriga fordonen passera över vagnens bro, varpå vagnen bärgades. +Bepansrade bärgningsfordon – Under andra världskriget byggdes några äldre stridsvagnar om till bepansrade bärgningsfordon genom att man tog bort tornet, täckte hålet efter tornet och tillförde en överbyggnad. Efter kriget utökades detta program och följden kom att bli mera avancerade modeller. +T-34 används ofta som en symbol för den sovjetiska motattackens effektivitet mot tyskarna. Det första mötet med T-34 på sommaren 1941 var en psykologisk chock för de tyska soldaterna, som hade förberett sig för att möta en underlägsen sovjetisk armé; detta illustreras bland annat av Alfred Jodls dagbok. Han verkar ha överraskats av T-34:s uppdykande i Riga. T-34 kunde effektivt bekämpa alla tyska stridsvagnar av år 1941. De första stridsvagnarna led dock av allvarliga mekaniska problem, speciellt i fråga om transmission och koppling – åtminstone 50 % av den sommarens förluster av T-34 berodde på mekaniska problem, inte på grund av tysk eldkraft (i detta antal är dock även äldre stridsvagnar inräknade). De sovjetiska trupperna led brist på bärgnings- och reparationsutrustning och det var inte ovanligt att tidiga T-34 gick i strid bärande på en reservväxellåda på motordäcket. De mekaniska problemen skulle dock lösas med tiden. +Under vintern 1941–42 dominerade T-34 återigen över de tyska stridsvagnarna eftersom den kunde röra sig över djup lera eller snö utan att gräva sig ner. Tyska stridsvagnar kunde helt enkelt inte röra sig i samma terräng som T-34. Det tyska infanteriet, som vid denna tidpunkt för det mesta var utrustad med dragna pansarvärnskanoner av typen 37 mm PAK 35/36 och 50 mm PAK 38 kunde inte stoppa de sovjetiska stridsvagnarna på något effektivt sätt. Endast de sovjetiska soldaternas bristfälliga utbildning och den bristande kompetensen hos de sovjetiska befälhavarna förhindrade att T-34 skulle ha nått större framgångar. +Tyngdpunkten i Röda armén 1942–43 låg på att återuppbygga vad som förlorats 1941, och att förbättra den taktiska skickligheten. Produktionen av T-34 steg snabbt, men utvecklingen av konstruktionen hade "frysts"; endast förbättringar som påskyndade produktionen godkändes. De sovjetiska konstruktörerna var väl medvetna om behovet att rätta till vissa brister i konstruktionen, men de skulle ha kostat produktionstid och kunde inte accepteras. I mitten av 1943 uppgick T-34-produktionen till ungefär 1 000 stridsvagnar i månaden, mycket högre än den tyska produktionen. De sovjetiska förlusterna överskred dock stort de tyska på grund av kontinuerlig taktisk underlägsenhet. +Som svar på T-34 blev det allt vanligare att de tyska styrkorna använde luftvärnspjäser, huvudsakligen den ökända 8,8 cm FlaK, mot markmål och i stor hast försågs fronten med nya pansarvärnskanoner av typen 75mm PaK 40 som fanns i både dragna och självgående versioner. På stridsvagnsfronten svarade tyskarna med att under våren 1942 bestycka infanterikanonvagnen Sturmgeschütz III med den kraftigare kanonen 75mm StuK 40 L/43 (och senare, under hösten 1942, den mer effektiva 75mm StuK 40 L/48). Denna längre kanon med högre utgångshastighet möjliggjorde en avsevärt förbättrad pansarbrytande förmåga och med denna förändring inträdde även infanterikanonvagen i en ny roll som pansarvärnskanonvagn (i vilken Sturmgeschütz III uppträdde mycket skickligt och redan under våren 1944 kunde tillskrivas 20 000 utslagna stridsvagnar). Även de reguljära pansarstyrkornas arbetshäst PzKpfw IV utrustades under sommaren 1942 med en längre kanon, 75mm KwK 40 L/43 (och senare, under våren 1943 den kraftigare 75mm KwK 40 L/48) och vid sidan om dessa förändringar utvecklades även den tunga Tiger-stridsvagnen som insattes i strid under slutet av 1942 och skulle komma att nå stor ryktbarhet. År 1943 då tyskarna insatte den medeltunga Panther-stridsvagnen, vars design innehöll många av de innovationer som en gång gjorde T-34:s design revolutionerande, hade T-34 närmast helt förlorat sitt tekniska övertag mot de tyska stridsvagnarna och förlustsiffrorna fortsatte att vara betydligt högre för de sovjetiska trupperna; när deras tekniska försprång minskade lyckades inte deras taktiska kunskaper komma ifatt snart nog. +Under de sista krigsåren förblev den sovjetiska taktiken trots avsevärda framsteg underlägsen den tyska, men Röda arméns växande operationella och strategiska skicklighet och deras större tillgångar till stridsvagnar hjälpte till att få ner förlustsiffrorna. Ankomsten av T-34-85, tidigt år 1944 gav Röda armén en stridsvagn som hade bättre pansar och mobilitet än den tyska PzKpfw IV och Sturmgeschütz III men den kunde inte matcha Panther på nästan något område. Det som talade till ryssarnas fördel var att det fanns betydligt färre Pantherstridsvagnar än T-34, och T-34-85 var tillräckligt bra för att skickliga stridsvagnsbesättningar och taktiska situationer skulle tippa vågskålen till egen fördel. +När kriget utbröt bestod den sovjetiska stridsvagnsarsenalen endast av omkring 4 % T-34, men när kriget var över utgjorde de åtminstone 55 % av Sovjetunionens enorma produktion av stridsvagnar (baserat på antal givna av Zaloga; Zjeltov ger en ännu högre procentandel). När T-34 hade ersatt äldre modeller och blev mera vanligt förekommande, kunde nyare tyska stridsvagnar utklassa dem (däribland den tyska konstruktionen som influerades av de tyska styrkornas bittra erfarenheter av mötet med T-34 under 1941 och 1942, stridsvagnen PzKpfw V "Panther". De senare sovjetiska konstruktionerna, däribland de tunga stridsvagnarna i Josef Stalin-serien, var även de bättre beväpnade och skyddade än T-34. +Det ligger nära till hands att göra en jämförelse mellan T-34 och den amerikanska medeltunga stridsvagnen M4 Sherman. Båda modellerna utgjorde ryggraden i sina egna och allierade arméer, och var goda konstruktioner när de gjorde sin debut. Båda modellerna förbättrades signifikant utan någon nämnvärd förlust i effektivitet, och de kunde tillverkas i stort antal och repareras under fältförhållanden. Ingen av dem kunde möta de bästa tyska Tiger- och Panthervagnarna på jämna villkor, som var mera i samma klass som den sovjetiska tunga stridsvagnen JS-2. +Stridsvagnar var tänkta att ha många roller på slagfältet. Deras främsta uppgift var att stödja infanteriet och att utföra spaning, men dessutom var stridsvagnsbekämpningsrollen mycket viktig. Att den tyska produktionen av stridsvagnar var begränsad till ett relativt litet antal tekniskt överlägsna men tillverkningsmässigt komplicerade fordon (delvis på grund av att produktionen även upptog självgående kanoner) talade mot dem. Det sovjetiska beslutet, att bygga ett mycket stort antal T-34, med gradvis förbättrad konstruktion och förenklad tillverkningsprocess, var ett strategiskt sett mycket bättre alternativ och bidrog i hög grad till att Sovjetunionen blev en av segrarmakterna i det andra världskriget. +I slutändan var det ändå det överlägset stora antal jämfört med Tyskarnas som gav Röda Armén sin överlägsenhet, när tyska pansartrupper inte kunde ersätta sitt pansar fortsatte de sovjetiska fabrikerna massproducera T-34:or rakt ut till fronten utan avbrott. +Efter andra världskriget har T-34-85 använts i många länder som stått under sovjetiskt inflytande. Den nordkoreanska invasionen av Sydkorea år 1950 leddes av en full brigad som var utrustad med omkring 120 T-34-85. De mötte amerikanska stridsvagnar av typerna M24 Chaffee, M4 Sherman och M26 Pershing (men inte den brittiska Centurionstridsvagnen som användes av FN-styrkorna). Den nordkoreanska 105:e bepansrade brigaden hade några tidiga framgångar mot det sydkoreanska infanteriet och amerikanska lätta stridsvagnar av typen M24, men de förlorade sitt momentum när de mötte amerikanska medeltunga stridsvagnar av typen M26 och infanteri utrustat med pansarvärnsvapen. Efter de första månaderna stötte man sällan på nordkoreanskt pansar. +T-34 utgjorde stommen även i många Warszawapaktsarméer och användes när ungernrevolten slogs ner 1956. De användes även i mellanöstern, i vietnamkriget (mest berömt är anfallet mot Lang Vei) men även så sent som i kriget i Bosnien. Kroatien ärvde 25 eller 30 från Jugoslavien men har nu tagit dem ur tjänst. T-34 användes sporadiskt i Afghanistan (det är inte känt ifall de användes mot koalitionstrupperna) och Saddam Husseins Irak hade T-34 i sin armé under tidigt 90-tal. Flera afrikanska stater, däribland Angola och Somalia, har använt T-34-85 under senare år. Kubanska T-34-85 har även använts i Afrika. +Kina tillverkade T-34 under beteckningen Typ 58. Produktionen stoppades dock när Typ 59 blev tillgänglig. Ett litet antal T-34 har även setts i Kina, där de byggts om till brandbekämpningsfordon. +T-34 har använts av följande 39 länder, 27 av dem så sent som 1996 (dessa är märkta med asterisker *). +Inga stridsvagnar inköptes, men ett mindre antal erövrades under kriget. +I vissa böcker förekommer uppgiften om att någon T-34-prototyp skulle ha testats av sovjetarmén under vinterkriget 1939-40, men denna uppgift torde vara felaktig, det rör sig antagligen om en felidentifiering. Första gången finländarna stötte ihop med T-34 var i början av fortsättningskriget. Finländarna lyckades erövra två exemplar på hösten 1941 vid Svirkraftverket och de togs omedelbart i bruk. De hade byggts vid Stalingrads traktorfabrik och Charkivs lokomotivfabrik. De erhöll snabbt smeknamnet "Sotka" (knipa) på grund av sin profil. Ytterligare en erövrades på våren 1942 och en på sommaren 1943. Man hade nu 4 i tjänst och de kom att utnyttjas i hög grad under kriget. Sommaren 1944 lovade tyskarna att sända 9 erövrade/reparerade T-34 till finländarna. Men denna affär skulle inte avslutas helt, och enbart tre T-34 kom att levereras från Tyskland. I månadsskiftet juni-juli erövrade man ytterligare två stridsvagnar och man hade då sammanlagt 9 T-34 av den äldre modellen till sitt förfogande när kriget slutade mellan Finland och Sovjetunionen. Finländarna kom senare att använda dem mot tyskarna i Lappland. En skadades svårt av en tysk mina, men kunde repareras. Inte en enda T-34 förlorades under krigen. Efter kriget ansågs de vara landets bästa stridsvagnar tillsammans med StuG III och PzKpfw IV. T-34 kom att nyttjas i den finländska försvarsmakten fram till 1961 när de slutligen pensionerades. +T-34-85 var en uppgraderad modell av T-34 med tremannatorn och en 85 mm kanon som utvecklats från en luftvärnskanon. Pansaret uppgick till 110 mm i fronten och 75 mm på sidorna. Dessa stridsvagnar användes i Karelen på sommaren 1944. Trots att de finländska styrkorna drog sig tillbaka lyckades man erövra nio stridsvagnar av modellen T-34-85. De flesta av dem togs när offensiven mattades av och man kunde återupprätta linjerna i någon form. Finländarna tog omedelbart sju av dem i bruk. De återstående två reparerades och var klara för strid när kriget mellan Finland och Sovjetunionen tog slut. Efter fortsättningskriget tänkte man ersätta 85 mm kanonen med den 75 mm kanon som även StuG III använde på grund av bristen på ammunition till 85 mm-kanonerna och de tyska kanonernas goda egenskaper. En stridsvagn byggdes om, men senare på vintern 1944-45 återinstallerade man 85 mm-kanonen och alla stridsvagnar fick sin ursprungliga bestyckning. Alla nio T-34-85 överlevde både fortsättnings- och Lapplandskriget. Efter krigen kom de att användas fram till 1962. Stridsvagnen var mycket populär bland sina besättningar. +Stridseffektiviteten hos tidiga T-34 kan bäst jämföras i form av "hårda" faktorer – pansar, eldkraft och rörlighet – och "mjuka" faktorer: ergonomiska egenskaper såsom användarvänlighet, synfältsanordningar, layout för besättningens uppgifter och så vidare. T-34 var fenomenal i fråga om de hårda faktorerna och svag i fråga om de mjuka. +De "stora tre" egenskaperna vid tankdesign har alltid varit pansar, eldkraft och rörlighet. T-34 var en utmärkt balans mellan alla tre under dess livscykel under det andra världskriget. 1941 kunde dess tjocka, sluttande pansar stoppa alla konventionella tyska pansarvärnsvapen på normala avstånd. T-34 kunde endast slås ut av den dragna luftvärnskanonen 8,8 cm FlaK eller på nära håll av 50 mm och 75 mm kortpipiga stridsvagnskanoner. De flesta tyska stridsvagnar vid denna tidpunkt hade inte 75 mm kanoner; istället var 37 mm mycket mera vanligt förekommande. I mitten av 1942 var T-34 sårbar mot förbättrade tyska vapen och förblev så ända till krigets slut. Dess pansar var dock jämförbart med den amerikanska M4 Sherman och den tyska PzKpfw IV. +I fråga om eldkraft kunde 1941 T-34:s 76 mm kanon med lätthet penetrera alla tyska stridsvagnars pansar. Denna kanon kunde även avfyra en spränggranat som dock var ganska medelmåttig. 1943 utklassades 76 mm-kanonen i räckvidd av Panthers långa 75 mm och Tigerns 88 mm kanoner. Införandet av den sovjetiska 85 mm kanonen 1944 gjorde inte att T-34-85 blev likvärdig i eldkraft, men dess 85 mm kanon kunde penetrera både Panther och Tiger på medellånga avstånd. +I fråga om mobilitet bidrog T-34:s breda larvband, goda upphängning och stora motor till att ge stridsvagnen en oöverträffad terrängduglighet. Första generationens tyska stridsvagnar kunde inte hänga med. +Sammantaget var T-34:s hårda faktorer en världsvid trendsättare inom stridsvagnsutvecklingen under krigets första år. +I fråga om ergonomi var T-34 svag, även om man gjorde vissa förbättringar under kriget. Alla versioner som var utrustad med 76 mm-kanonen led av det trånga manskapsutrymmet i tornet som var konstruerat för två man. Stridsvagnschefens sikt var dålig; luckan som kunde öppnas framåt tvingade honom att observera slagfältet genom en smal observationsspringa och ett nedsänkbart periskop. Han hade även till uppgift att avfyra kanonen. I jämförelse hade reguljära tyska, brittiska och amerikanska stridsvagnar betydligt bättre utrymmen för besättningen, med rum för tre personer i tornet: vagnchef, skytt och laddare. Tremannalayouten tillät att stridsvagnschefen koncentrerade sig på att leda sina mannar och koordinera sina aktioner med de övriga stridsvagnarna i sin enhet, utan att behöva utföra sådana individuella uppgifter som att till exempel rikta eller ladda kanonen. Det tillför enormt mycket till effektiviteten hos en stridsvagnsbesättning. I och med T-34-85 korrigerades problemet, som man hade observerat redan före kriget. Många tyska stridsvagnschefer gillade att strida med manskapsluckan öppen, sätet fullt upphöjt och ett fritt synfält. I de T-34 som var beväpnade med 76 mm-kanonen var detta omöjligt. +Synfältet från förarsätet var likaså begränsat. Vissa förare rapporterade att deras optik var så dålig att de tvingades köra med luckan lätt öppnad, även i strid. Taktiskt sett påverkade detta förarens möjligheter att använda terrängen till sin fördel eftersom de inte kunde se terrängformationerna så bra, eller ha så bra synfält som man hade från många andra stridsvagnar. +Även laddaren hade ett svårt jobb på grund av att det inte fanns någon tornkorg (en roterande golvplatta som följer med tornets rörelser). Detta problem delades med många andra stridsvagnar, till exempel den amerikanska M-3 Stuart. Golvet utgjordes av ammunition som lagrades i små metallådor och täcktes av en gummimatta. Nio skott fanns i hållare nära stridsutrymmet, redo för att användas. När dessa nio skott avfyrats var besättningen tvungen att plocka fram ytterligare ammunition från golvlådorna, vilket lämnade golvet täckt med öppna lådor och mattdelar. Detta distraherade besättningen och försämrade deras prestation. +Övriga faktorer som minskade T-34:s initiala verkan på slagfältet var det dåliga ledarskapet, stridsvagnstaktiken och besättningens skolning, en konsekvens av Stalins utrensningar inom den sovjetiska officerskåren i slutet av 1930-talet. Det förstärktes av förlusten av det bäst skolade manskapet under de katastrofala förlusterna under 1941. Många besättningar sattes i strid med bara grundläggande militär träning plus 72 timmars instruktion i klassrummen. Dessa problem, förstärkta av T-34:s dåliga ergonomi och brist på radio, gjorde att det var praktiskt taget omöjligt att koordinera stridsvagnsenheter i strid. Tyska stridsvagnssoldater fann att de sovjetiska pansarstyrkorna anföll i täta formationer och antingen inte utnyttjade terrängen alls eller bara till ringa del. I slutet av 1943–44 hade problemen i stort sett avhjälpts, men de sovjetiska stridsvagnsbesättningarna skulle aldrig nå upp till samma nivå som de tyska i fråga om utbildning. +Hållbarheten hos T-34 kan kanske bäst illustreras av bärgningen av en T-34 modell 1943 år 2000. Den hade legat 56 år på bottnen av ett estniskt kärr.[1] Stridsvagnen hade erövrats av och använts av de retirerande tyska trupperna, som dumpade den i kärret när den blev utan bränsle. Det fanns inga spår av oljeläckage, rost eller annan signifikant vattenskada på de mekaniska komponenterna. Man kunde senare restaurera motorn till fullt fungerande skick.[2] +Hundratals T-34 installerades som krigsminnesmärken i länder i Sovjetblocket. +Åtminstone en sådan stridsvagn, monterad ovanför monumentet i Prag över de sovjetiska stridsvagnsbesättningarna, var blickfånget för en större kontrovers. Monumentet som var tänkt att representera löjtnant I.G. Gontjarenkos T-34-85, den första sovjetiska stridsvagnen som körde in i Prag i maj 1945, var egentligen en tung stridsvagn av modellen JS-2m. För många i Prag var detta monument även en påminnelse av den sovjetiska invasionen som slog ner Pragvåren 1968. Stridsvagnen målades rosa av konstnären David Černý år 1991. Efter en officiell protest från den ryska regeringen, Černýs arrestering, en ommålning i grön färg, demonstrationer på gatorna och ytterligare en ommålning i rosa av femton parlamentsledamöter, förpassades stridsvagnen till ett militärmuseum. +En annan T-34-stridsvagn som målats rosa är den så kallade "Mandela Way Tank"-stridsvagnen i London. +Fyra tankmän och en hund (Czterej pancerni i pies) var en mycket framgångsrik polsk tv-serie som sändes på 60-talet, i vilken en T-34 med markeringen 102 gjordes till en popikon inom polsk kultur. Programmet visades även i andra länder inom sovjetblocket där serien också togs väl emot. +T-34 var bland de viktigaste vapensystemen i den röda armén under det andra världskriget. Eftersom östfronten utgjorde det avgörande operationsområdet under det andra världskriget kan T-34:s betydelse inte förringas. När den först kom i tjänst år 1940, var fordonet den bästa stridsvagnskonstruktionen i världen. I mitten av kriget var den inte längre överlägsen sina motståndare, men den var fortfarande bättre än de flesta och den fanns tillgänglig i enorma antal. +Den förbättrade T-34-85 förblev Sovjetunionens standardstridsvagn med en oavbruten produktionsserie fram till slutet av kriget. Tyskarna svarade på T-34 genom att introducera helt nya, mycket dyra och komplexa, andra generationens stridsvagnar, vilket sinkade deras produktion av stridsvagnar och kom att tillåta att Sovjetunionen hade en substantiell numerär överlägsenhet i fråga om stridsvagnar. T-34 ersatte de flesta lätta, medeltunga och tunga stridsvagnar i sovjetisk tjänst. Dess evolutionära utveckling skulle komma att leda till T-44 och T-54/55-seriens stridsvagnar som tillverkades fram till 1981 och används ännu idag. +T-34:s effektivitet som stridsvagn, när den utnyttjades i enlighet med de erfarenheter av mobila, samverkande operationer, som man skaffat sig under kriget, kom att ha en betydande inverkan på krigföringen. Infanteristridsvagnen hade visat sig vara föråldrad och konceptet övergavs av alla nationer vid slutet av kriget. Medan mera ekonomiska typer av fordon skulle komma att ta över de olika understödsrollerna som tidigare spelats av stridsvagnarna, kom välbalanserade stridsvagnskonstruktioner att bli mer specialiserade och optimerade för deras roll i den mekaniska krigföringen. Den medeltunga stridsvagnens potential, som hade demonstrerats av T-34, kom att leda till konceptet med den moderna enhetliga stridsvagnsparken (på engelska: main battle tank (MBT), på svenska: Huvudstridsvagn). +Chant, Christopher (1994 [1996]). World Encyclopedia of the Tank: An International History of the Armoured Fighting Machine, Somerset: Patrick Stephens (Haynes). ISBN 1-85260-114-0. +Harrison, Mark (2002). Accounting for War: Soviet Production, Employment, and the Defence Burden, 1940–1945, Cambridge University Press. ISBN 0-521-89424-7. +Liddell Hart, Basil (1951 [1999]). The other side of the hill: Germany's generals, their rise and fall, with their own account of military events, 1939-1945, London: Cassell. ISBN 0-330-37324-2. +Von Mellenthin, Major General F. W. (1956). Panzer Battles: A Study of the Employment of Armor in the Second World War, First Ballantine Books Edition, 1971. New York: Ballantine Books. ISBN 0-345-24440-0. +Milsom, John (1975). Russian Tanks, 1900-1970: The Complete Illustrated History of Soviet Armoured Theory and Design, New York: Galahad Books. ISBN 0-88365-052-5. +Muikku, Esa och Purhonen, Jukka: Suomalaiset panssarivaunut 1918-1997, Apali (1998) ISBN 952-5026-09-4. +Para, Carl (April 2002). "Military Heritage feature on the T-34" i Military Heritage vol. 3, nr. 5, ss. 18-20, 22-23. +Perrett, Bryan (1987). Soviet Armour Since 1945, London: Blandford. ISBN 0-7137-1735-1. +Russian Battlefield (1998a). "T-34: Development History" The Russian Battlefield, URL läst den 27 december, 2005. +Russian Battlefield (1998b). "T-34-85: Development History" The Russian Battlefield, URL läst den 27 december, 2005. +Sewell, Stephen ‘Cookie’ (1998). "Why Three Tanks?" i Armor vol. 108, nr. 4, s. 21. Fort Knox, KY: US Army Armor Center. ISSN 0004-2420. – beskriver politiken som omgärdade T-34:ans introduktion (PDF-format) +Wachowski, Tomasz (2004). "Nieznani T-34" (Den okända T-34:an) i Nowa Technika Wojskowa (Ny militär utrustning) 11/2004, s. 53. Warszawa: Magnum-X. ISSN 1230-1655. +Wright, Patrick (2001). Tank: The Progress of a Monstrous War Machine. Viking Adult. ISBN 0-670-03070-8. +Zaloga, Steven J., James Grandsen (1983). T-34 in Action, Carrollton, Texas: Squadron/Signal. ISBN 0-89747-112-1. – i denna finns många fotografier och ritningar som visar detaljerat de olika skillnaderna mellan de olika modellerna av T-34-stridsvagnen +Zaloga, Steven J., James Grandsen (1984). Soviet Tanks and Combat Vehicles of World War Two, London: Arms and Armour Press. ISBN 0-85368-606-8. +Zaloga, Steven J., Jim Kinnear (1996 [2004]). T-34-85 Medium Tank 1944–94, Oxford: Osprey Publishing. ISBN 1-85532-535-7. +Zaloga, Steven J., Jim Kinnear, Andrey Aksenov & Aleksandr Koshchavtsev (1997). Soviet Tanks in Combat 1941-45: The T-28, T-34, T-34-85, and T-44 Medium Tanks, Hong Kong: Concord Publication. ISBN 962-361-615-5. +Zaloga, Steven J., Peter Sarson (1994). T-34 Medium Tank 1941–45, Oxford: Osprey Publishing. ISBN 1-85532-382-6. +Zheltov, I., M. Pavlov, I. Pavlov (2001). Neizvestnij T-34 (Den okända T-34:an). Moskva: Eksprint. ISBN 5-94038-013-1 +Wikimedia Commons har media som rör T-34. +Utvärdering av stridsvagnarna T-34 och KV av arbetare vid Aberdeens testområde i USA (illustrerad version på battlefield.ru, för tillfället inte nåbar–cachekopia från internetarkivet) +T34 och T34/85 hos WWIIvehicles.com +T-34-85:an i andra världskriget: en närmare titt – detaljerad undersökning av T-34-85:ans detaljer +Europeiska unionen (EU) är en fördragsbunden union mellan 27 europeiska länder och en internationell organisation med en unikt hög grad av överstatlighet. +Unionen upprättades den 1 november 1993 som en vidareutveckling av Europeiska gemenskaperna. Den är det huvudsakliga uttrycket för fullföljandet av den europeiska integrationsprocess som påbörjades den 9 maj 1950 genom Schumandeklarationen och som syftade till att bringa ett slut på krigen i Europa. +Unionen omfattar 27 medlemsstater: Belgien, Bulgarien, Cypern, Danmark, Estland, Finland, Frankrike, Grekland, Irland, Italien, Kroatien, Lettland, Litauen, Luxemburg, Malta, Nederländerna, Polen, Portugal, Rumänien, Slovakien, Slovenien, Spanien, Sverige, Tjeckien, Tyskland, Ungern och Österrike. I syfte att nå sina gemensamma mål har dessa länder tilldelat unionen vissa befogenheter, som utövas av dess sju institutioner: Europaparlamentet, Europeiska rådet, Europeiska unionens råd (även känt som ”ministerrådet”), Europeiska kommissionen, EU-domstolen, Europeiska centralbanken och Europeiska revisionsrätten. +Unionens övergripande mål är enligt fördraget om Europeiska unionen att främja freden, unionens grundläggande värden (såsom frihet, demokrati och jämlikhet) och folkens välfärd. Unionen har som ambition att upprätta ett område med frihet, säkerhet och rättvisa utan inre gränser, en inre marknad med fri rörlighet för varor, tjänster, personer och kapital och en ekonomisk och monetär union med en gemensam valuta, samt att verka för sina värden runt om i världen, bland annat genom en gemensam utrikes- och säkerhetspolitik. Unionen har även inrättat ett unionsmedborgarskap, som är kopplat till vissa grundläggande rättigheter, såsom rätten att röra sig fritt inom unionen. +EU-samarbetet har varit föremål för omfattande diskussioner och utgör en kontroversiell fråga i stora delar av unionen. Kritiker hävdar bland annat att unionen är odemokratisk, byråkratisk och för centraliserad och komplex, medan förespråkarna hävdar att unionen utgör ett viktigt samarbete för att utveckla det ekonomiska och sociala välståndet i Europa samt för att bevara freden sedan andra världskriget. +Europeiska unionen tilldelades 2012 Nobels fredspris med motiveringen ”för att i över 60 år ha bidragit till fred, försoning, demokrati och mänskliga rättigheter i Europa”. +Europeiska unionen bildades den 1 november 1993, men dess historia sträcker sig ända tillbaka till upprättandet av Europeiska kol- och stålgemenskapen (EKSG) den 23 juli 1952 då dess första institutioner skapades. Efter ett misslyckat försök med att bilda en försvarsgemenskap, fokuserades samarbetet istället på ekonomiska och energipolitiska frågor. En ekonomisk gemenskap upprättades den 1 januari 1958 och drygt tio år senare fulländades inrättandet av en tullunion. Det var dock först den 1 januari 1993 som en fullständig inre marknad med fri rörlighet för varor, tjänster, personer och kapital förverkligades, med en gemensam valuta och monetär politik från och med den 1 januari 1999. +Utvecklandet av det europeiska samarbetet har skett successivt genom antagandet av nya fördrag. Denna process har till stor del varit beroende av de nära relationerna mellan Frankrike och Tyskland, och ofta skett i samband med olika kriser som drivit på medlemsstaterna i att fördjupa sin integration för att möta ett uppkommet behov av ökat samarbete. +Europeiska unionens historia sträcker sig tillbaka till upprättandet av Europeiska gemenskaperna i ruinerna av andra världskriget. Den 9 maj 1950, numera känt som Europadagen, lade Frankrikes dåvarande utrikesminister Robert Schuman fram ett förslag, Schumandeklarationen, med motiveringen att ”världsfreden inte kan upprätthållas utan konstruktiva insatser mot hotande faror” och att ett enat Europa var en nödvändighet för att undvika nya krig. Förslaget innebar att världens första överstatliga organisation – Europeiska kol- och stålgemenskapen – skulle inrättas och att den franska och tyska kol- och stålproduktionen, som var en förutsättning för att producera krigsmateriel, skulle underställas denna organisation. Organisationen skulle ledas av en hög myndighet, som skulle vara oberoende av medlemsstaterna och vars beslut skulle vara direkt bindande för dem. Förutom att bevara freden i Europa, syftade organisationen till att skapa ekonomiskt och socialt välstånd, som enligt förslaget skulle användas för att utveckla Afrika. Samarbetet skulle vara fördragsbundet och stå öppet för andra europeiska länder att ansluta. +Den 18 april 1951 undertecknades Parisfördraget, som i stora drag innehöll förslagen från Schumandeklarationen. Förutom Frankrike och Västtyskland anslöt sig även Italien och Benelux-länderna (Belgien, Nederländerna och Luxemburg) till samarbetet. Vid undertecknandet antog medlemsstaternas företrädare en symbolisk förklaring – Europadeklarationen – där de proklamerade starten för ett Europa präglat av samarbete och samtidigt uppmanade andra europeiska länder att ansluta sig. Fördraget trädde i kraft den 23 juli 1952 för en period av 50 år. Den franske tjänstemannen Jean Monnet, som hade utarbetat Schumandeklarationen, utsågs till Höga myndighetens första ordförande. Förutom en hög myndighet, inrättades också en rådgivande parlamentarisk församling, ett ministerråd och en gemensam domstol. +Efter undertecknandet av Parisfördraget påbörjades arbetet med att inrätta ett försvarspolitiskt samarbete. Den 27 maj 1952 undertecknades fördraget om upprättandet av Europeiska försvarsgemenskapen, som var tänkt att leda till inrättandet av ett gemensamt försvar. Den 30 augusti 1954 avslogs fördraget dock av Frankrikes nationalförsamling. Fördraget kunde därför inte träda i kraft, och utvecklingen av det europeiska försvarssamarbetet skedde istället inom ramen för Nato och Västeuropeiska unionen. +I spåren av det misslyckade försöket med att upprätta en försvarsgemenskap tog Benelux-länderna, under ledarskap av Nederländernas utrikesminister Johan Beyen, initiativ till att istället inleda ett ekonomiskt och energipolitiskt samarbete, framför allt genom att inrätta en gemensam marknad och en tullunion med Benelux-samarbetet som föregångsmodell. Vid Messinakonferensen den 1–3 juni 1955 tillsattes en kommitté, ledd av Belgiens dåvarande utrikesminister Paul-Henri Spaak, med uppgift att utarbeta ett mer konkret förslag. Även en representant från Storbritannien bjöds in, men den brittiska regeringen drog sig ur förhandlingarna i oktober 1955 på grund av sitt motstånd till att ingå i en tullunion. +I sin rapport föreslog Spaak-kommittén bildandet av två nya organisationer: Europeiska ekonomiska gemenskapen (EEG) för ekonomiskt samarbete och Europeiska atomenergigemenskapen (Euratom) för samarbete kring kärnkraft. Detta förslag förverkligades genom Romfördragen, som undertecknades den 25 mars 1957 och trädde i kraft den 1 januari 1958 på obestämd tid. Till skillnad från EKSG, präglades EEG och Euratom inte av lika långtgående överstatlighet, vilket främst var ett krav från Frankrike. Beslut kunde dock fortfarande i många fall fattas med kvalificerad majoritet för att förhindra blockeringar från enskilda medlemsstater. Den parlamentariska församlingen (sedermera Europaparlamentet) och domstolen blev gemensamma för de tre gemenskaperna. Däremot inrättades var sitt separata ministerråd och var sin separata kommission (istället för en hög myndighet) för EEG och Euratom. Först genom fusionsfördraget, som trädde i kraft den 1 juli 1967, slogs de parallella institutionerna för Europeiska gemenskaperna ihop, varvid det bildades ett enda ministerråd och en enda kommission för samtliga tre gemenskaperna. +Under 1960-talet genomfördes många av bestämmelserna i fördragen från 1950-talet. En gemensam jordbrukspolitik inrättades den 30 juli 1962 med syfte att höja produktiviteten och pålitligheten i det europeiska jordbruket. En milstolpe i arbetet med att skapa en gemensam marknad med fri rörlighet för varor, tjänster, personer och kapital uppnåddes när tullunionen blev verklighet den 1 juli 1968, ett och ett halvt år tidigare än förutsett enligt Romfördragen. Därmed avskaffades alla tullavgifter mellan medlemsstaterna samtidigt som gemensamma tullavgifter infördes mot tredjeländer, något som gav upphov till en kraftigt ökad handel mellan medlemsstaterna. Den 8 november 1968 genomfördes även den fria rörligheten för arbetstagare, vilket innebar att medborgare i medlemsstaterna fick rätt att ta anställning i andra medlemsstater på samma villkor som de egna medborgarna där. +I början av 1960-talet inledde medlemsstaternas stats- eller regeringschefer regelbundna sammanträden inom ramen för vad som senare skulle komma att bli känt som Europeiska rådet. Detta var framför allt ett sätt för Frankrikes nya president Charles de Gaulle att stärka de nationella regeringarnas inflytande gentemot de överstatliga institutionerna, som han ansåg hade för mycket makt. 1965 drabbades Europeiska gemenskaperna av sin första allvarliga kris när Frankrike protesterade mot kommissionens förslag till reformerad jordbrukspolitik. Den franska regeringen motsatte sig vad den uppfattade som ett försök av kommissionen att utöka sin egen och den parlamentariska församlingens makt på ministerrådens bekostnad. I ren protest återkallade Frankrike sina rådsrepresentanter vid gemenskaperna, vilket gav upphov till ”den tomma stolens kris”. Först i januari 1966 nåddes en kompromiss – Luxemburgkompromissen – som innebar att medlemsstaternas regeringar slöt en informell överenskommelse om att alltid eftersträva enhällighet i frågor som ansågs vara av nationellt intresse för någon medlemsstat, även i de fall då fördragen endast krävde kvalificerad majoritet. +Den 1 januari 1973 ägde gemenskapernas första utvidgning rum då Danmark, Irland och Storbritannien anslöt sig till gemenskaperna. Deras anslutning hade dittills blockerats av Frankrike, eftersom de Gaulle hade motsatt sig ett brittiskt medlemskap. Även Norge hade förhandlat om medlemskap, men anslutningen avslogs av de norska väljarna i en folkomröstning den 25 september 1972. Under 1980-talet utvidgades gemenskaperna till att även omfatta Grekland (1981) samt Portugal och Spanien (1986), efter att dessa länder hade demokratiserats. +Luxemburgkompromissen och övergången till enhällighet i alla frågor av ”känslig karaktär” innebar att den europeiska integrationsprocessen i stort sett avstannade under 1970-talet och början av 1980-talet. I spåren av de ekonomiska problem som medlemsstaterna upplevde under samma period väcktes dock nya krav på ett fördjupat samarbete. Under första halvan av 1980-talet påbörjades förhandlingarna om europeiska enhetsakten, som trädde i kraft den 1 juli 1987. Genom enhetsakten återupprättades och utökades majoritetsbeslut inom en rad viktiga områden för den gemensamma marknaden. Under kommissionsordförande Jacques Delors tid antogs ett omfattande paket av 310 åtgärder för att skapa en fulländad gemensam marknad utan inre gränser – den inre marknaden. Den 1 juli 1990 förverkligades den fria rörligheten för kapital och den 1 januari 1993 förverkligades slutligen den inre marknaden, 35 år efter att Romfördragen hade trätt i kraft. +Vid sidan av förverkligandet av den inre marknaden ökade även entusiasmen för ett fördjupat politiskt samarbete under 1980-talet. I juni 1979 hade det första direkta valet till Europaparlamentet hållits, efter att rådet hade enats om hur valsystemet skulle utformas. Europeiska rådet uttryckte vid sitt sammanträde i juni 1983 i Stuttgart, Västtyskland, sin vilja att upprätta en politisk union, med en gemensam utrikes- och säkerhetspolitik, viss harmonisering av lagar och ett samarbete i rättsliga och inrikes frågor. +1985 slöt flera av gemenskapernas medlemsstater ett avtal, Schengenavtalet, om att avskaffa sina gränskontroller vid de gemensamma gränserna efter att Storbritannien hade blockerat ett sådant samarbete inom Europeiska gemenskaperna. Det var dock först genom Maastrichtfördraget, som trädde i kraft den 1 november 1993 efter en utdragen ratificeringsprocess, som Europeiska unionen upprättades. Unionen omfattade tre olika pelare: första pelaren i form av det överstatliga samarbetet inom ramen för Europeiska gemenskaperna, andra pelaren i form av en gemensam utrikes- och säkerhetspolitik och den tredje pelaren i form av rättsliga och inrikes frågor. Europeiska ekonomiska gemenskapen ersattes samtidigt av Europeiska gemenskapen. Ett unionsmedborgarskap inrättades för alla medborgare i medlemsstaterna och kopplades ihop med vissa grundläggande rättigheter, till exempel rätten att röra sig fritt mellan medlemsstaterna. +Den nya unionen hade även som mål att införa en gemensam valuta före 1990-talets slut. Under 1997 antog Europeiska rådet två viktiga beslut i Amsterdam, Nederländerna. Det första gällde stabilitets- och tillväxtpakten, som innebar att medlemsstaterna åtog sig att upprätthålla sunda offentliga finanser. Det var en förutsättning för bildandet av en valutaunion, som skulle verkställas under de kommande åren. Det andra beslutet gällde Amsterdamfördraget, som trädde i kraft den 1 maj 1999. Fördraget införlivade Schengensamarbetet inom unionens ramar och överförde en del uppgifter och funktioner från Västeuropeiska unionen till Europeiska unionen. +Den 1 juni 1998 inrättades Europeiska centralbanken med uppgift att föra den gemensamma monetära politiken inom euroområdet. Den 1 januari 1999 infördes euron som den gemensamma valutan i elva av medlemsstaterna; alla utom Danmark, Grekland, Storbritannien och Sverige. Grekland anslöt sig dock den 1 januari 2001. Euromynt och eurosedlar infördes den 1 januari 2002 och ersatte då de nationella valutorna i Belgien, Finland, Frankrike, Grekland, Irland, Italien, Luxemburg, Nederländerna, Portugal, Spanien, Tyskland och Österrike. +Järnridåns och Berlinmurens fall 1989 banade vägen för en utvidgning österut. 1990 återförenades Östtyskland med Västtyskland och blev därmed en del av det europeiska samarbetet. Kalla krigets slut ledde till att de neutrala länderna Finland, Sverige och Österrike ansökte om medlemskap och den 1 januari 1995 anslöt de sig till unionen. Även Norge hade ansökt om medlemskap, men anslutningen avslogs för en andra gång av de norska väljarna i en folkomröstning den 28 november 1994. +Den 23 juli 2002 upplöstes EKSG till följd av att Parisfördraget upphörde att gälla. EKSG:s resurser överfördes då till Europeiska gemenskapen. För att anpassa unionen inför den kommande utvidgningen antogs Nicefördraget, som trädde i kraft den 1 februari 2003. Den 1 maj 2004 ägde slutligen unionens dittills största utvidgning rum då åtta central- och östeuropeiska länder samt Cypern och Malta anslöt sig till unionen. Parallellt med utvidgningen utarbetades också en europeisk konstitution, men denna avslogs 2005 i folkomröstningar i Frankrike och Nederländerna. Istället utarbetades Lissabonfördraget som efter en utdragen ratificeringsprocess trädde i kraft den 1 december 2009. Fördraget innebar att Europeiska unionen slutligen ersatte Europeiska gemenskapen och blev en egen juridisk person. Den 1 januari 2007 blev Bulgarien och Rumänien medlemmar i unionen, trots omfattande kritik mot korruptionen och den organiserade brottsligheten i de båda länderna. Under 2010 och 2011 avvecklades Västeuropeiska unionen och dess kvarstående uppgifter och funktioner övertogs av Europeiska unionen. Den 1 juli 2013 blev även Kroatien medlem i unionen. +I spåren av finanskrisen 2007–2008 drabbades medlemsstaterna mot slutet av 2009 av en djup och långvarig statsfinansiell kris – eurokrisen. De offentliga finanserna försämrades avsevärt i nästan alla medlemsstater och flera regeringar inom euroområdet tvingades att ta emot nödlån för att klara sig när räntorna på finansmarknaden sköt i höjden. Krisen uppmärksammade brister i den finanspolitiska styrningen inom eurosamarbetet och ledde bland annat till en reformerad stabilitets- och tillväxtpakt samt en ny bankunion och kapitalmarknadsunion. +I mitten av 2015 eskalerade ytterligare en kris – flyktingkrisen – när över en miljon asylsökande tog sig till unionen för att söka skydd. Krisen ledde till spänningar mellan medlemsstaterna i synen på asylmottagandet och flera medlemsstater införde tillfälliga inre gränskontroller inom Schengenområdet. +Den 23 juni 2016 röstade en majoritet av britterna för att lämna unionen i en folkomröstning. Den brittiska regeringen begärde utträde den 29 mars 2017, vilket resulterade i att Storbritannien, som den första medlemsstaten någonsin, lämnade unionen den 1 februari 2020 efter en långdragen förhandlings- och ratificeringsprocess av avtalet om Storbritanniens utträde ur Europeiska unionen. Unionsrätten fortsatte dock att vara tillämplig i Storbritannien under en övergångsperiod fram till den 1 januari 2021. +Under covid-19-pandemin, med start under 2020, tvingades medlemsstaterna att införa olika reserestriktioner mellan sig för att begränsa smittspridningen. För att stimulera den ekonomiska återhämtningen i spåren av pandemin antogs ett omfattande återhämtningsinstrument på 750 miljarder euro för att finansiera olika investeringar i medlemsstaterna. +Europeiska unionens territorium upptar större delen av norra, västra, södra och centrala Europa. Den totala ytan av unionens 27 medlemsstater är 4 236 351 km², vilket kan jämföras med Europas totala yta på ungefär 10 180 000 km². Tillsammans har medlemsstaterna en 13 770 kilometer lång gräns till 18 andra europeiska länder, däribland Belarus, Norge, Ryssland, Storbritannien, Turkiet, Ukraina och västra Balkan. Unionen har också världens största exklusiva ekonomiska zon med 25 miljoner km². +Europeiska unionen består av 27 medlemsstater: Belgien, Bulgarien, Cypern, Danmark, Estland, Finland, Frankrike, Grekland, Irland, Italien, Kroatien, Lettland, Litauen, Luxemburg, Malta, Nederländerna, Polen, Portugal, Rumänien, Slovakien, Slovenien, Spanien, Sverige, Tjeckien, Tyskland, Ungern och Österrike. +Vid upprättandet av Europeiska kol- och stålgemenskapen 1952 ingick endast Belgien, Frankrike, Italien, Luxemburg, Nederländerna och Västtyskland i samarbetet. Sedan dess har det successivt utvidgats till att omfatta fler europeiska länder. Till en början blockerades utvidgningen av Frankrikes president Charles de Gaulle, eftersom han motsatte sig ett brittiskt medlemskap i samarbetet. Efter att de Gaulle hade ersatts av Georges Pompidou, kunde den första utvidgningen med Danmark, Irland och Storbritannien äga rum 1973. Den följdes 1981 av Grekland och 1986 av Portugal och Spanien. I och med Berlinmurens fall 1989, kunde Östtyskland återförenas med Västtyskland 1990 och bli en del av gemenskaperna. 1995 anslöt sig även Finland, Sverige och Österrike, som alla hade varit neutrala länder under kalla kriget. Den 1 maj 2004 skedde den dittills största utvidgningen; då anslöt sig Cypern, Estland, Lettland, Litauen, Malta, Polen, Slovakien, Slovenien, Tjeckien och Ungern. Dessa tio länder följdes av Bulgarien och Rumänien den 1 januari 2007 och Kroatien den 1 juli 2013. Storbritannien lämnade, som första medlemsstat någonsin, unionen den 1 februari 2020. +Flera europeiska länder väntar på att få ansluta sig till unionen. Det finns nio officiella kandidatländer: Albanien, Bosnien och Hercegovina, Georgien, Moldavien, Montenegro, Nordmakedonien, Serbien, Turkiet och Ukraina. Av dessa har alla utom Bosnien och Hercegovina samt Georgien påbörjat medlemskapsförhandlingar, men förhandlingarna har gått långsamt framåt. Även Kosovo har ansökt om medlemskap och är ett så kallat potentiellt kandidatland. +För att kunna ansluta sig till unionen krävs det att ansökarlandet uppfyller Köpenhamnskriterierna, som består av politiska, ekonomiska och administrativa kriterier. Det politiska kriteriet innefattar att ansökarlandet har stabila institutioner, som garanterar demokrati, rättsstatens principer, mänskliga rättigheter samt respekt för och skydd av minoriteter. Det ekonomiska kriteriet innefattar att ansökarlandet har en fungerande marknadsekonomi som klarar av konkurrenstrycket och marknadskrafterna inom unionen. Det administrativa kriteriet innefattar att ansökarlandet har kapacitet att åta sig de skyldigheter som följer av medlemskapet, framför allt att kunna genomföra hela unionens regelverk. Medlemskapsvillkoren fastställs i ett anslutningsfördrag, som måste ratificeras av ansökarlandet och alla medlemsstater i enlighet med deras respektive konstitutionella bestämmelser innan det kan träda i kraft och ansökarlandet blir en medlemsstat. +Island, Liechtenstein, Norge och Schweiz har alla någon gång ansökt om medlemskap i unionen, men har på grund av inrikespolitiska skäl, till exempel folkomröstningar, dragit tillbaka sina ansökningar. Island, Liechtenstein och Norge har dock ett mycket nära samarbete med unionen genom EES-avtalet och andra avtal, som tillåter dem att ingå i den inre marknaden, Schengensamarbetet och andra delar av det europeiska samarbetet utan att vara en del av unionen. Även Schweiz har ett nära samarbete med unionen genom bilaterala avtal. Storbritannien, som lämnade unionen den 1 februari 2020, omfattades av större delen av unionsrätten fram till den 1 januari 2021 och sedan dess av ett handels- och samarbetsavtal. De europeiska mikrostaterna Andorra, Monaco, San Marino och Vatikanstaten har ett särskilt nära samarbete med unionen i bland annat tull-, handels- och valutafrågor. +De östeuropeiska länderna Moldavien och Ukraina samt de kaukasiska länderna Armenien, Azerbajdzjan och Georgien samarbetar med unionen inom ramen för det östliga partnerskapet. +Varje medlemsstat inom unionen har möjlighet att utträda. En medlemsstat ska då anmäla sin avsikt till Europeiska rådet, som består av medlemsstaternas stats- eller regeringschefer. Ett utträdesavtal, som fastslår villkoren och tidpunkten för utträdet, ska därefter upprättas mellan unionen och den utträdande medlemsstaten. Om så inte sker, utträder medlemsstaten efter två år från dess utträdesanmälan, såvida inte Europeiska rådet med enhällighet beslutar att förlänga tidsfristen. +Hittills har endast ett land – Storbritannien – lämnat unionen. Detta skedde efter en folkomröstning den 23 juni 2016 då en majoritet av de brittiska väljarna röstade för utträde och den 1 februari 2020 lämnade Storbritannien unionen. Även territorierna Algeriet, Grönland och Saint-Barthélemy har lämnat unionen eller dess föregångare. +Det finns ingen möjlighet till att utesluta en medlemsstat, men om en medlemsstat bryter mot unionens grundläggande värden kan ett så kallat artikel 7-förfarande inledas, vilket kan leda till att vissa av medlemsstatens rättigheter fråntas, däribland rösträtten i Europeiska unionens råd. +Europeiska unionens institutionella struktur bygger på representativ demokrati och maktdelningsprincipen. Det innebär att unionen styrs av direkt eller indirekt valda institutioner, vars funktioner och befogenheter regleras av unionens fördrag. Europeiska rådet, bestående av medlemsstaternas stats- eller regeringschefer, ansvarar för att ge unionen de impulser som behövs för dess utveckling och bestämma dess allmänna politiska riktlinjer och prioriteringar. +De lagstiftande institutionerna utgörs av Europaparlamentet, bestående av 720 ledamöter som väljs genom direkta val vart femte år, och Europeiska unionens råd, bestående av en företrädare på ministernivå för varje medlemsstat. De antar gemensamt europeiska lagar – huvudsakligen förordningar och direktiv – i enlighet med unionens lagstiftningsförfarande. +Lagförslag läggs fram av Europeiska kommissionen, som i det närmaste fungerar som en europeisk regering med verkställande makt att genomföra antagna lagar på europeisk nivå, förvalta unionens budget och representera unionen vid internationella förhandlingar. I enlighet med principen om parlamentarism är kommissionen kollektivt ansvarig inför Europaparlamentet, som årligen beviljar kommissionen ansvarsfrihet mot bakgrund av rapporter från Europeiska revisionsrätten och som kan avsätta den genom en misstroendeförklaring. +Kommissionen har också till uppgift att övervaka medlemsstaternas genomförande och tillämpning av unionsrätten. Om en medlemsstat inte uppfyller sina skyldigheter enligt unionens fördrag, kan kommissionen inleda ett överträdelseförfarande, vilket i förlängningen kan leda till talan om fördragsbrott vid EU-domstolen. Europeiska centralbanken ansvarar för den monetära politiken inom euroområdet och utgivningen av euro. +Varje institution leds av en ordförande. Vilka uppgifter som åligger ordföranden skiljer sig dock från institution till institution. Den politiskt viktigaste är kommissionens ordförande, som kan liknas vid en premiärminister för unionen. Kommissionsordföranden fyller en central funktion i utarbetandet av kommissionens politiska program. Han eller hon utses av Europeiska rådet för en period av fem år i taget efter att ha blivit vald av Europaparlamentet med absolut majoritet på förslag av Europeiska rådet med hänsyn till utgången i det senaste Europaparlamentsvalet. Varje år håller kommissionsordföranden ett tal om tillståndet i unionen inför Europaparlamentet där han eller hon utvärderar det gångna året och redogör för övergripande politiska riktlinjer inför nästkommande år. Nuvarande ordförande är Ursula von der Leyen (EPP), som tillträdde den 1 december 2019. Hon omvaldes för en andra mandatperiod under 2024. +Övriga ordförande, inklusive Europaparlamentets talman, Europeiska rådets ordförande och Europeiska centralbankens ordförande, fyller mer administrativa än politiska roller, även om de också till viss del har möjlighet att styra den politiska inriktningen i sina respektive institutioner. Deras huvuduppgifter är att leda institutionernas arbete framåt och att representera dem utåt. Europaparlamentets talman är Roberta Metsola (EPP) sedan januari 2022, Europeiska rådets ordförande är António Costa (PES) sedan december 2024 och Europeiska centralbankens ordförande är Christine Lagarde sedan november 2019. +Ordförandeskapet i Europeiska unionens råd roterar mellan medlemsstaterna var sjätte månad. I alla konstellationer utom rådet för utrikes frågor leds rådet av den rådsmedlem som företräder ordförandelandet. Rådet för utrikes frågor leds istället av den höga representanten för utrikes frågor och säkerhetspolitik. Under första halvåret 2025 innehas ordförandeskapet av Polen. +Europeiska lagar – huvudsakligen förordningar och direktiv – antas av Europaparlamentet och Europeiska unionens råd i enlighet med unionens lagstiftningsförfarande inom de befogenhetsområden som medlemsstaterna har gett unionen befogenhet att lagstifta. Ett lagförslag kan i regel endast läggas fram av Europeiska kommissionen. Olika lagstiftningsförfaranden tillämpas beroende på lagförslagets rättsliga grund i fördragen. +Det ordinarie lagstiftningsförfarandet, som tillämpas på de flesta lagförslag, består i att Europaparlamentet och Europeiska unionens råd lagstiftar gemensamt. Detta förfarande kräver att både parlamentet och rådet godkänner lagförslaget för att det ska antas. Båda institutionerna kan ändra i lagförslaget under processens gång. För att antas behöver ett lagförslag i regel godkännas av parlamentet med enkel majoritet och av rådet med kvalificerad majoritet. Detta innebär att ett lagförslag, i enlighet med den överstatliga så kallade unionsmetoden, kan antas även om inte alla medlemsstaters regeringar stöder det. +Inom vissa specifika politikområden, i synnerhet de där medlemsstaterna fortfarande vill ha full kontroll över de antagna lagarna, tillämpas särskilda lagstiftningsförfaranden. I dessa fall lagstiftar rådet i regel ensamt och beslut fattas med enhällighet. Detta innebär att ett lagförslag inte kan antas utan att alla medlemsstaters regeringar stöder det. Samrådsförfarande är ett särskilt lagstiftningsförfarande där Europaparlamentet först måste få yttra sig om lagförslaget, men dess yttrande är inte rättsligt bindande för rådet. Godkännandeförfarande är ett särskilt lagstiftningsförfarande där Europaparlamentet först måste ge sitt godkännande, innan rådet kan anta lagförslaget, men där parlamentet inte har någon möjlighet att göra några ändringar i lagförslaget. +De rättsakter som antas genom ett lagstiftningsförfarande utgör lagstiftningsakter och kan bestå av förordningar, direktiv eller beslut. +Utöver institutionerna finns ett flertal organ och byråer med uppgifter som antingen specificeras i fördragen eller i de rättsakter som institutionerna har antagit med fördragen som rättslig grund. Europeiska ekonomiska och sociala kommittén (EESK) och Europeiska regionkommittén (ReK) är två rådgivande organ till institutionerna. EESK består av arbetsgivare, fackföreningar, företagare och andra representanter från det civila samhället och syftar till att förankra vissa av unionens beslut. ReK består av nationella politiker som är engagerade på regional och kommunal nivå. Dess uppgift är att ge synpunkter i frågor som rör regionalpolitiken. Förutom EESK och ReK, återfinns även bestämmelser om Europeiska investeringsbanken, Europeiska ombudsmannen, Europeiska åklagarmyndigheten, Eurojust och Europol i fördragen. Även en europeisk datatillsynsman har inrättats inom unionen. Därutöver finns byråer, som var och en utgör en juridisk person med ansvar för vissa tekniska, vetenskapliga eller administrativa uppgifter. Byråernas verksamheter är decentraliserade i unionen. +Unionsrätten utgör Europeiska unionens rättsordning och bygger på ett regelverk bestående av unionens fördrag och rättsakter som antagits av unionens institutioner, organ och byråer. Fördragen – den så kallade primärrätten – utgör en sorts konstitution för unionen. De reglerar institutionernas sammansättning, funktioner och befogenheter. Unionens institutioner kan anta bindande rättsakter – förordningar, direktiv och beslut – endast i de fall medlemsstaterna har tilldelat dem befogenhet genom fördragen att göra så. Varje rättsakt som antas måste ha en giltig rättslig grund i fördragen. +Unionsrätten är en självständig rättsordning som gäller i unionens medlemsstater vid sidan av de nationella rättsordningarna. Den skapar inte enbart skyldigheter mellan medlemsstaterna, utan ger i vissa fall även upphov till individuella rättigheter för enskilda som det åligger de nationella domstolarna att skydda. Delar av unionsrätten har därför direkt effekt och kan åberopas inför en nationell domstol likt en nationell lag, även utan att bestämmelserna har införlivats i den nationella rättsordningen. I enlighet med företrädesprincipen äger unionsrätten företräde framför nationell rätt om en bestämmelse i den nationella lagstiftningen skulle visa sig vara oförenlig med den. För att säkerställa att unionsrätten tillämpas korrekt och tolkas enhetligt i alla medlemsstater tolkar EU-domstolen unionsrätten på begäran av de nationella domstolarna genom att meddela förhandsavgöranden. Europeiska kommissionen kan även inleda ett överträdelseförfarande mot en medlemsstat som inte anses uppfylla sina skyldigheter enligt unionsrätten, med möjlighet att väcka talan om fördragsbrott vid EU-domstolen. +I enlighet med principen om tilldelade befogenheter kan unionens institutioner anta rättsakter endast i de fall medlemsstaterna har tilldelat dem befogenhet genom fördragen att göra så. Det finns tre huvudtyper av befogenhetsområden. Den mest långtgående befogenheten, ”exklusiv befogenhet”, innebär att endast unionen får lagstifta och anta rättsligt bindande akter. Medlemsstaterna får i dessa fall anta rättsligt bindande akter endast efter bemyndigande av unionen eller för att genomföra unionens rättsakter. Unionen har exklusiv befogenhet på fem områden samt vissa internationella avtal. +Den andra befogenheten, ”delad befogenhet”, innebär att både unionen och medlemsstaterna får lagstifta. Medlemsstaterna får dock göra det endast så länge unionen inte redan har antagit lagstiftning inom området. Unionen kan välja att inte längre utöva sin befogenhet, vilket möjliggör för medlemsstaterna att på nytt utöva sina befogenheter. +Den tredje befogenheten, ”stödjande befogenhet”, innebär att unionen får vidta åtgärder enbart för att stödja, samordna eller komplettera medlemsstaternas politik. Därutöver har unionen även befogenhet att bedriva en gemensam utrikes- och säkerhetspolitik samt att samordna medlemsstaternas ekonomiska politik och sysselsättningspolitik. Unionen kan också ta initiativ för att samordna medlemsstaternas socialpolitik. +Subsidiaritets- och proportionalitetsprinciperna är två konstitutionella principer som ligger till grund för hur unionen får utöva sina tilldelade befogenheter. Subsidiaritetsprincipen innebär att unionen inom sina icke-exklusiva befogenhetsområden endast får vidta åtgärder om målen för dem inte i tillräcklig utsträckning kan uppnås av medlemsstaterna, på nationell, regional eller lokal nivå, och därför bättre kan uppnås på europeisk nivå. Genom en så kallad subsidiaritetsprövning har varje nationellt parlament möjlighet att pröva om ett lagförslag, inom ett icke-exklusivt befogenhetsområde, är förenligt med subsidiaritetsprincipen. Om minst en tredjedel av de nationella parlamenten invänder, måste Europeiska kommissionen överväga skälen till att ett lagförslag har lagts fram. +Proportionalitetsprincipen innebär att unionens åtgärder inte får gå utöver vad som är nödvändigt för att nå målen i fördragen. De lagstiftande institutionerna kan välja mellan att anta en förordning, som är direkt tillämplig inom hela unionen, eller ett direktiv, som medlemsstaterna själva måste införliva i den nationella lagstiftningen. I enlighet med proportionalitetsprincipen får institutionerna anta förordningar endast om åtgärden kräver helt enhetliga bestämmelser inom hela unionen. +Europeiska unionen bygger på ett antal grundläggande värden, som innefattar respekt för människans värdighet, frihet, demokrati, jämlikhet, rättsstaten och respekt för de mänskliga rättigheterna, inklusive rättigheter för personer som tillhör minoriteter. Om en medlemsstat bryter mot unionens grundläggande värden kan ett så kallat artikel 7-förfarande inledas, vilket kan leda till att vissa av medlemsstatens rättigheter fråntas, däribland rösträtten i Europeiska unionens råd. +Grundläggande rättigheter inom unionen finns fastställda i dess stadga om de grundläggande rättigheterna, som har samma rättsliga värde som fördragen. Stadgan slår fast rättigheter, friheter och principer som gäller inom unionen, däribland att människans värdighet är okränkbar, att ingen får dömas till döden eller avrättas, att alla människor har rätt till integritet samt att tortyr och slaveri är förbjudet. Bland de friheter som fastslås återfinns religions-, yttrande-, mötes- och näringsfrihet. Likhet inför lagen, icke-diskriminering, mångfald och barns och äldres rättigheter inkluderas också, tillsammans med arbetares rättigheter, hälso- och konsumentskydd, rösträtt och rättsstatens principer. Unionen har även möjlighet att ansluta sig till Europakonventionen, men en sådan anslutning har tidigare blockerats av Frankrike och Storbritannien. +Alla medborgare i unionens medlemsstater åtnjuter ett unionsmedborgarskap. Medborgarskapet kompletterar, och ersätter inte, de nationella medborgarskapen. Unionsmedborgare har rätt att fritt röra sig och uppehålla sig inom unionen i enlighet med rörlighetsdirektivets bestämmelser. De har rätt att rösta och kandidera vid val till Europaparlamentet och kommunala val i den medlemsstat som de är bosatta i, på samma villkor som de egna medborgarna i den medlemsstaten. Unionsmedborgare har även rätt till skydd i tredjeländer av en annan medlemsstats diplomatiska eller konsulära myndigheter, om den egna medlemsstaten inte är representerad där. Unionsmedborgare har också rätt att göra framställningar till Europaparlamentet och Europeiska ombudsmannen, samt rätt att vända sig till någon av unionens institutioner, organ eller byråer på ett av unionens officiella språk och få svar på samma språk. Genom ett europeiskt medborgarinitiativ kan en miljon unionsmedborgare gå samman och kräva att Europeiska kommissionen lägger fram ett lagförslag i en fråga där unionen har befogenhet att lagstifta. +Med en bruttonationalprodukt (BNP) på 15 906 miljarder euro under 2022 utgör Europeiska unionen en av världens största ekonomier. Mellan medlemsstaterna har handelshinder, såsom tullavgifter och kapitalrestriktioner, avskaffats med syfte att tillåta fri rörlighet för varor, tjänster, personer och kapital. Den gemensamma valutan euro har införts för att ytterligare fördjupa den ekonomiska och politiska integrationen inom euroområdet. +Levnadsstandarden i unionen är mycket hög. 2021 uppgick HDI till 0,896, varav alla medlemsstater utom Bulgarien hade ett HDI över 0,800, vilket innebär att de är mycket högt utvecklade länder. Inkomstfördelningen i unionen är också relativt jämn; unionen i sin helhet har en ginikoefficient på 29,6. Dock finns det stora variationer mellan medlemsstaterna, framför allt mellan Väst- och Östeuropa. +Finanskrisen 2007–2008 slog hårt mot unionens ekonomi och gav upphov till en skuldkris under de efterföljande åren. Mellan 2010 och 2013 beviljade euroområdets medlemsstater nödlån till Grekland, Irland, Portugal, Spanien och Cypern, för att stabilisera deras banksystem då dessa medlemsstater inte längre klarade av att låna från marknaden till rimliga räntor. Det långsiktiga ekonomiska problemet för unionen är dock den åldrande befolkningen. +Unionens egen verksamhet finansieras genom dess årliga budget. Budgeten omfattar cirka 170 miljarder euro, vilket motsvarar ungefär en procent av medlemsstaternas bruttonationalinkomst (BNI) eller en femtiondedel av medlemsstaternas samlade nationella budgetar. +Största delen av budgeten flödar tillbaka till medlemsstaterna i form av omfördelande bidrag och åtgärder för att skapa tillväxt och jobb i syfte att utjämna de ekonomiska skillnaderna inom unionen. Strax under sju procent av budgeten går till unionens administration, varav en procent för kostnaden för översättning och tolkning. Mer ekonomiskt utvecklade medlemsstater bidrar med en större andel av budgetens medel, medan mindre ekonomiskt utvecklade medlemsstater erhåller en större andel av budgetens medel. +Unionens årliga budget antas av Europaparlamentet och Europeiska unionens råd på förslag av Europeiska kommissionen. Den årliga budgeten måste vara förenlig med den fleråriga budgetramen, som rådet antar vart sjunde år och som fastställer ett utgiftstak för de årliga budgetarna. +Budgeten finansieras genom ett system för egna medel. Mer än 70 procent av budgeten finansieras genom avgifter som medlemsstaterna betalar baserat på sin egen BNI. Drygt tio procent av budgeten finansieras genom momsbaserade egna medel, och ungefär lika mycket finansieras genom tull- och sockeravgifter på handel med tredjeländer. Sedan 2021 finns även en fjärde kategori av egna medel baserade på plastförpackningsavfall. +Den inre marknaden utgör den gemensamma marknaden mellan unionens medlemsstater. Handelshinder, såsom tullavgifter och kapitalrestriktioner, mellan medlemsstaterna har gradvis avskaffats med syfte att tillåta de fyra friheterna: fri rörlighet för varor, fri rörlighet för tjänster, fri rörlighet för personer (inklusive fri rörlighet för arbetstagare) och fri rörlighet för kapital. Genom en tullunion bedriver medlemsstaterna en gemensam tullpolitik gentemot tredjeländer. Unionens konkurrensregler syftar till att motverka snedvridningar i konkurrensvillkoren mellan medlemsstaterna genom bland annat strikta villkor för när och hur de får ge statsstöd. Medlemsstaternas lagstiftningar har i hög grad harmoniserats genom att gemensamma bestämmelser för många typer av varor och tjänster har införts inom unionen. Unionsmedborgare har rätt att fritt röra sig och uppehålla sig inom unionen, till exempel för att ta anställning, driva företag eller bedriva studier i en annan medlemsstat, på samma villkor som de egna medborgarna i den medlemsstaten. +Genom EES-avtalet ingår också Island, Liechtenstein och Norge till stor del i den inre marknaden. De måste i utbyte tillämpa en stor del av unionsrätten vad gäller fri rörlighet och olika harmoniseringsåtgärder. Även Schweiz har viss tillgång till den inre marknaden genom bilaterala avtal med unionen. +Alla unionens medlemsstater ingår i en ekonomisk och monetär union med syfte att ytterligare fördjupa den ekonomiska och politiska integrationen inom unionen. Det ekonomiska samarbetet bygger på samordning av medlemsstaternas finanspolitik, medan det monetära samarbetet bygger på en gemensam monetär politik med en gemensam valuta. +Stabilitets- och tillväxtpakten är ett gemensamt regelverk för de offentliga finanserna, till exempel får en medlemsstat inte gå med underskott med mer än tre procent av sin BNP. Genom den europeiska planeringsterminen granskar medlemsstaterna och unionens institutioner de nationella budgetarna varje år och lämnar rekommendationer till hur de kan förbättras utifrån ett ekonomiskt perspektiv. Unionen antar vart tionde år en strategi för unionens ekonomiska utveckling. +För att ytterligare fördjupa integrationen inom unionen infördes den gemensamma valutan euro den 1 januari 1999. Eurosedlar och euromynt infördes den 1 januari 2002 inom euroområdet. Sex medlemsstater har dock fortfarande kvar egna nationella valutor och ingår därmed ännu inte i euroområdet: Danmark, Polen, Rumänien, Sverige, Tjeckien och Ungern. Bortsett från Danmark har dessa medlemsstater förbundit sig att införa euron som valuta när de väl uppfyller de så kallade konvergenskriterierna. +Europeiska unionen har som ambition att utgöra ett område med frihet, säkerhet och rättvisa, med gemensam europeisk lagstiftning vad gäller gränskontroller, asyl, invandring samt förebyggande och bekämpande av brottslighet. Genom Schengensamarbetet har gränskontrollerna vid de inre gränserna inom Schengenområdet avskaffats. Samtidigt har gränskontrollerna vid de yttre gränserna förstärkts för att förhindra obehöriga gränspassager och det gemensamma polissamarbetet fördjupats för att bekämpa gränsöverskridande brottslighet. Den gemensamma viseringspolitiken fastställer under vilka villkor tredjelandsmedborgare kan erhålla visum till Schengenområdet. +Den gemensamma asylpolitiken fastställer gemensamma normer för mottagande av asylsökande, gemensamma förfaranden för att bevilja och återkalla asyl och gemensamma normer för när tredjelandsmedborgare anses vara berättigade till asyl. Dublinförordningen fastställer vilken medlemsstat som ansvarar för att hantera en asylansökan. Den gemensamma invandringspolitiken fastställer gemensamma bestämmelser om uppehållstillstånd och arbetstillstånd. +Inom polissamarbete, straffrättsligt samarbete samt civilrättsligt samarbete har medlemsstaterna samarbete inom ramen för Europeiska åklagarmyndigheten, Eurojust och Europol. En viktig del av samarbetet är medlemsstaternas ömsesidiga erkännande av varandras domar. Genom en europeisk arresteringsorder kan en medlemsstats myndigheter begära en misstänkt eller dömd person överlämnad från en annan medlemsstat. Förföljda och utsatta brottsoffer kan skyddas genom en europeisk skyddsorder. +Utöver ekonomiska och rättsliga frågor bedriver Europeiska unionen en gemensam politik inom ett antal andra områden, däribland regionalpolitik, jordbruk och fiskeri, infrastruktur samt miljö och energi. +En viktig del av unionens politik är dess regionalpolitik. Regionalpolitiken syftar till att stödja utsatta regioner och på så sätt upprätthålla ”ekonomisk, social och territoriell sammanhållning” inom unionen. Målet är att stimulera ekonomisk tillväxt, skapa fler jobb och öka livskvaliteten i Europa. En central del av regionalpolitiken är de europeiska struktur- och investeringsfonderna (ESI), som finansierar projekt till stöd för utsatta regioner eller grupper av människor. +Regionalpolitiken är ett av de största utgiftsområdena i unionens budget; cirka en tredjedel av unionens årliga budget läggs på ekonomisk, social och territoriell sammanhållning. +Av unionens totala landyta utgörs 38,4 procent av jordbruksmark. Under 2020 stod jordbruks- och livsmedelsindustrin för knappt sex procent av unionens BNP och innefattade totalt 12 miljoner företag och 29 miljoner arbetstillfällen, motsvarandes 14 procent av den totala sysselsättningen inom unionen. Antalet bondgårdar har dock minskat kraftigt över tiden, främst genom att mindre bondgårdar har lagts ned. +Jordbruket är den enda sektorn som helt och hållet finansieras genom unionens budget; andra sektorer, till exempel utbildning, forskning, transport, försvar och hälso- och sjukvård, mottar ingen eller begränsad finansiering på europeisk nivå. Jordbrukspolitiken utgör följaktligen en stor andel av unionens budget; 2022 stod den för 31,3 procent av unionens budget. Andelen har dock minskat sedan 1984, då den låg på 72 procent. +Den gemensamma jordbrukspolitiken syftar till att säkerställa en hållbar livsmedelsproduktion, en hållbar förvaltning av naturresurserna och en balanserad utveckling av landsbygdsområdena. Unionen har ett regelverk för hur import och export av jordbruksprodukter får ske och hur dessa produkter ska säljas. Unionen hade tidigare långtgående åtgärder för att intervenera på jordbruksmarknaden, bland annat genom att garantera ett lägsta pris på vissa livsmedel, vilket i vissa fall gav upphov till stora lager av dessa varor. Under 2010-talet avvecklades de flesta sådana åtgärder. +Jordbrukspolitiken innefattar också stöd till landsbygden för utveckling och bevarande. Unionen har även en gemensam politik för fiskeri, vilket innefattar begränsningar för hur mycket fisk som får fångas, minimipriser för fisk, märkning av fisk samt stöd till fiskerinäringen. Unionens politik syftar till att garantera en fungerande fiskerinäring och att förhindra utfiske. +Transeuropeiska nät är de gränsöverskridande infrastrukturnätverk som kopplar ihop medlemsstaternas transport-, energi- och telenät. Unionens politik inom detta område innefattar satsningar på att göra de nationella infrastruktursystemen kompatibla med varandra och att förbättra tillträdet till dessa system. +Projekt som ingår i de transeuropeiska näten kan erhålla ekonomiskt stöd från unionen genom till exempel sammanhållningsfonden och Europeiska regionala utvecklingsfonden. Lån och särskilda garantiinstrument kan även beviljas av Europeiska investeringsbanken. De transeuropeiska transportnäten innefattar vägtrafik, järnvägstrafik och trafik till sjöss. Unionen har till exempel utvecklat ett gemensamt signalsystem för järnvägarna för att underlätta järnvägstrafik mellan olika medlemsstater. Även det europeiska systemet för satellitbaserad navigering, Galileo, ingår i denna kategori. +De transeuropeiska energinäten syftar till att trygga energiförsörjningen och att inrätta en inre marknad för energi. Inom transeuropeiska telenät är investeringarna fokuserade kring modernisering av de redan befintliga näten. Genom roamingförordningen avskaffades alla roamingavgifter den 15 juni 2017. +Inom ramen för sitt miljö- och klimatprogram – den europeiska gröna given – har unionen antagit en europeisk klimatlag med målet att nå klimatneutralitet till 2050. För att nå dit har Europeiska kommissionen bland annat genomdrivit lagpaketet Fit for 55 i syfte att minska utsläppen med 55 procent till 2030. En viktig del av unionens klimatpolitik är dess utsläppshandelssystem för handel med utsläppsrätter för växthusgaser. +Miljöpolitiken inom Europeiska gemenskaperna sträcker sig tillbaka till 1972. Den har inriktats mot bland annat surt regn, det uttunnade ozonlagret, luftkvalitet, för höga bullernivåer, resursslöseri och vattenföroreningar. Vattendirektivet är ett exempel på vattenpolitik, med målet att sjöar, floder, mark och kustvatten ska vara av ”god kvalité”. Vilda djur och växter är skyddade genom Natura 2000 och täcker mer än 30 000 platser genom hela Europa. REACH-förordningen är en del av unionens lagstiftning med syfte att säkerställa att de cirka 30 000 kemikalier som dagligen används är säkert testade. Unionen har även antagit lagstiftning för att bland annat fasa ut användningen av glödlampor och vissa plastprodukter. +Unionens energipolitik har sitt ursprung i Europeiska kol- och stålgemenskapen, med fokus på resurs- och energifrågor. Syftet med energipolitiken är att öka konkurrensen på den inre marknaden, uppmuntra investeringar och förbättra sammankopplandet av olika elnät, öka mängden energiresurser som kan motstå krissituationer och att använda redan existerande energitillgångar mer effektivt och samtidigt öka användandet av förnybara energikällor och ny energiteknologi. +Unionen importerar 82 procent av all sin råolja och 57 procent av all sin naturgas, vilket gör den till världens ledande importör av dessa bränslen. Den totala energiförbrukningen i unionen utgjordes 2010 av olja (35 procent), gas (27 procent), kol (16 procent), kärnkraft (13 procent) och förnybara alternativ (10 procent). Elproduktionen varierar mellan medlemsstaterna, till exempel är elproduktionen i Sverige och Österrike till stor del baserad på vattenkraft medan elproduktionen i Frankrike är baserad på kärnkraft. Under 2007 enades medlemsstaterna om att övergå till minst 20 procent förnybara energikällor och att minska koldioxidutsläppen till 2020 med minst 20 procent jämfört med utsläppsnivån under 1990. 2018 utgjorde förnybara källor 18,0 procent av energikonsumtionen, vilket motsvarade mer än en fördubbling av andelen sedan 2004. +Europeiska unionen har exklusiv befogenhet vad gäller den gemensamma handelspolitiken. Unionen har därför i regel ensamrätt att ingå handelsavtal med tredjeländer. Utrikespolitiken och förbindelserna med omvärlden leds av unionens höga representant för utrikes frågor och säkerhetspolitik, som fungerar som en utrikesminister för unionen. +Unionen bedriver ett omfattande bistånds- och utvecklingssamarbete med tredjeländer. Varje år donerar unionen och dess medlemsstater cirka 0,5 procent av sin samlade BNI i form av utvecklingsbistånd till tredjeländer, vilket gör unionen och dess medlemsstater till den största biståndsgivaren i världen. Till 2030 har unionen som mål att utöka utvecklingsbiståndet till 0,7 procent av BNI. +Inom Världshandelsorganisationen (WTO) företräds unionen och dess medlemsstater av Europeiska kommissionen. Unionen har samarbete i olika former med bland annat Förenta nationernas organ och dess fackorgan, Europarådet, Organisationen för säkerhet och samarbete i Europa (OSSE) och Organisationen för ekonomiskt samarbete och utveckling (OECD). Unionen representeras i tredjeländer och vid internationella organisationer av sina delegationer, som leds av unionens ambassadörer. +Europeiska unionen bedriver en gemensam utrikes- och säkerhetspolitik, som även innefattar en gemensam säkerhets- och försvarspolitik. Detta sker under ledning av den höga representanten för utrikes frågor och säkerhetspolitik. Beslutsfattandet inom detta befogenhetsområde skiljer sig från övriga eftersom det sker uteslutande på mellanstatlig basis; nästan alla beslut fattas med enhällighet av medlemsstaterna i Europeiska unionens råd. Europaparlamentets inflytande är mycket begränsat och EU-domstolen saknar behörighet inom större delen av den gemensamma utrikes- och säkerhetspolitiken. +Den gemensamma utrikes- och säkerhetspolitiken har gradvis förstärkts. Unionen har övertagit de uppgifter och funktioner som Västeuropeiska unionen hade innan Lissabonfördraget trädde i kraft den 1 december 2009. Unionen har befogenhet att ytterligare fördjupa försvarssamarbetet och inrätta ett gemensamt försvar om alla medlemsstater skulle gå med på det. +Den gemensamma försvarspolitiken innefattar det permanenta strukturerade samarbetet mellan medlemsstaternas försvarsmakter, och syftar till att öka försvarskapaciteten, minska kostnaderna och bidra till förbättrade villkor för försvarsindustrin. +Unionen har ett nära samarbete med Nato, som 23 av de 27 medlemsstaterna även är medlemmar i. Cypern, Irland, Malta och Österrike står utanför Nato, men samtliga deltar i den gemensamma utrikes- och säkerhetspolitiken. Alla utom Cypern ingår även i Partnerskap för fred. +Europeiska unionens folkmängd uppgick enligt officiell statistik till 451 028 881 den 30 september 2024. En viss befolkningsökning är väntad framöver, främst på grund av invandring från tredjeländer. Trots ökad medellivslängd väntas befolkningen minska efter år 2050. +Med en befolkningstäthet på 105,6 personer per km² utgör unionen en av världens befolkningstätaste regioner. 80 procent av medborgarna lever i storstadsområden och en tredjedel lever i städer med mer än en miljon invånare. Unionen omfattar även flera befolkningstäta regioner som inte har någon enskild stadskärna, utan består av flera städer som har vuxit samman. De största regionerna är Rhen–Ruhr-området (Köln, Dortmund och Düsseldorf i Tyskland) med runt 10,5 miljoner invånare, Grande Milano (Milano, Monza, Como, Bergamo, Varese i Italien) med runt 7,4 miljoner invånare, Randstad (Amsterdam, Rotterdam, Haag och Utrecht i Nederländerna) med runt 7,0 miljoner invånare, Frankfurt–Rhen-området (Frankfurt am Main och Wiesbaden i Tyskland) med runt 5,8 miljoner invånare och Schlesien-området (Katowice och Sosnowiec i Polen) med runt 3,5 miljoner invånare. +Inom kultur och utbildning har Europeiska unionen begränsade befogenheter och kan endast stödja medlemsstaternas politik. Europas kulturhuvudstäder är ett exempel på unionens kulturpolitik. Unionen stöder även flera kulturprojekt och har startat en webbportal för att förbättra flödet av kulturinformation så att alla medborgare ska ha tillgång till den europeiska kulturen. +Under 2023 fanns totalt 567 världsarv i Europa, varav 427 inom unionen. Inom det utbildningspolitiska området utgör Erasmusprogrammet en av de mest konkreta åtgärderna. Varje år deltar ungefär 200 000 högskolestudenter i Erasmusprogrammet. +Vissa förespråkare av europeisk integration lyfter fram det gemensamma europeiska historie- och kulturarvet, som inkluderar bland annat antikens Grekland och Romerska riket, feodalismen under medeltiden, renässansen, upplysningen, 1800-talets liberalism och (ibland) negativa inslag, såsom världskrigen och förintelsen. De europeiska värdena antas vara grundade i denna gemensamma historia. +Europeiska unionen har 24 officiella språk: bulgariska, danska, engelska, estniska, finska, franska, grekiska, iriska, italienska, kroatiska, lettiska, litauiska, maltesiska, nederländska, polska, portugisiska, rumänska, slovakiska, slovenska, spanska, svenska, tjeckiska, tyska och ungerska. Alla lagstiftningsakter översätts till samtliga officiella språk. Europaparlamentet är dock den enda institution som har som regel att översätta alla sina handlingar, och inte enbart rättsakter. Övriga institutioner använder ett begränsat antal språk, främst engelska, franska och tyska, som interna arbetsspråk. +Nästan alla officiella språk är indoeuropeiska språk, utom estniska, finska och ungerska, som är uraliska språk, och maltesiska, som är ett semitiskt språk. De flesta använder sig av det latinska alfabetet, utom bulgariska, som använder det kyrilliska alfabetet, och grekiska, som använder det grekiska alfabetet. +Utöver de officiella språken finns över 60 regionala språk och minoritetsspråk som talas regelbundet av ungefär 40 miljoner människor inom unionen. Av dessa språk intar de spanska regionala språken katalanska, baskiska och galiciska en särställning eftersom de kan användas vid kontakt med vissa av unionens institutioner. Turkiska och luxemburgiska är de enda två språken som är officiella i en medlemsstat, men inte på europeisk nivå. Dessutom finns en stor minoritet som talar ryska, framför allt i Baltikum. Medlemsstaterna beslutar själva om sin språkpolitik, även om det finns vissa gemensamma program på europeisk nivå för att stödja och skydda de regionala språken. +Europeiska unionen har fem olika symboler som är starkt förknippade med unionen: europeiska flaggan, Europahymnen Hymn till glädjen i Ludwig van Beethovens nionde symfoni, mottot Förenade i mångfalden, euron, och Europadagen den 9 maj. Förutom att vara symboler för unionen och dess institutioner används de också i ett vidare sammanhang som symboler för europeisk integration. +De första symbolerna – flaggan, hymnen och Europadagen – antogs av Europeiska rådet vid dess sammanträde i Milano, Italien, den 28–29 juni 1985. De tre symbolerna var redan etablerade symboler för Europarådet, en separat internationell organisation, som dock uppmanade andra europeiska organisationer att ställa sig bakom symbolerna. Europeiska rådet valde emellertid att instifta Europadagen den 9 maj till minne av Schumandeklarationen istället för den 5 maj till minne av Europarådets bildande. +Europeiska rådets godkännande av symbolerna var en del av ett större paket av åtgärder som antogs på förslag av Adonnino-kommittén, som Europeiska rådet hade tillsatt året dessförinnan med syfte att stärka det europeiska samarbetets folkliga förankring. Utöver symbolerna föreslog kommittén, som leddes av den italienska Europaparlamentarikern Pietro Adonnino, bland annat gemensamma pass, registreringsskyltar och idrottsevenemang. Flera av förslagen var dock kontroversiella och inte minst Storbritannien var skeptiskt till att anta en gemensam flagga eftersom detta ansågs ge samarbetet alltför statsliknande drag. Officiellt antogs därför flaggan istället som ett rektangulärt emblem. +Under 1990-talet tillkom euron och eurosymbolen som symboler för unionen. Mottot Förenade i mångfalden gjordes till en av unionens symboler den 4 maj 2000. +Afrikanska unionen +Europarådet +Förenta nationerna +Nato +Nordiska rådet +Sydamerikanska nationernas union +Wikimedia Commons har media som rör Europeiska unionen. +Europa (webbportal) +Sveriges riksdags EU-information diff --git a/manifest.bjodat b/manifest.bjodat new file mode 100644 index 0000000..c9dbfc0 --- /dev/null +++ b/manifest.bjodat @@ -0,0 +1,19 @@ +(package + (name (keyscore)) + (version "0.1.0") + + ;; What this package needs. One (package ...) clause each: + ;; + ;; (depends + ;; (package (name (bjorsec)) + ;; (version (version-at-least "0.1")) + ;; (source (git (url "https://github.com/bjoli/bjorsec"))))) + ;; + ;; A package on disk, instead of one to fetch: + ;; + ;; (depends + ;; (package (name (bjorsec)) (source (path (dir "../bjorsec"))))) + + ;; The GUI, src/gui.bjo. + (depends + (package (name (bjogui)) (source (path (dir "../bjogui")))))) diff --git a/packages.lock.json b/packages.lock.json new file mode 100644 index 0000000..a7b4e61 --- /dev/null +++ b/packages.lock.json @@ -0,0 +1,219 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Avalonia": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "Vw+gNvv4RXK5hLeVI9PfalhxPai6hrQjIElfoCDQK1fXTpk+vRyfhuhprrh+EIeuJtoYCy+AsSrqR1wgU28zeQ==", + "dependencies": { + "Avalonia.BuildServices": "11.3.2", + "Avalonia.Remote.Protocol": "12.1.5", + "MicroCom.Runtime": "0.11.6" + } + }, + "Avalonia.Angle.Windows.Natives": { + "type": "Transitive", + "resolved": "2.1.27548.20260419", + "contentHash": "l17nI3XVDN3oMnpjf2pnmJg0YTwK4m6NLsn/itAjDMdObTFxN77D5F1M9sRMSfViSY3KKcse1ROczwgoWLJsnA==" + }, + "Avalonia.BuildServices": { + "type": "Transitive", + "resolved": "11.3.2", + "contentHash": "qHDToxto1e3hci5YqbG9n0Ty8mlp3zBUN5wT66wKqaDVzXyQ0do3EnRILd4Ke9jpvsktaPpgE0YjEk7hornryQ==" + }, + "Avalonia.Desktop": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "l1jo9qKJjJ68gcuUwHgNgpsHV6STcvqfYTFvFPprOf949gmE506X/d7Mox/y36tU2BSWBqLGZ1M3mh8Ggl2g/Q==", + "dependencies": { + "Avalonia": "12.1.5", + "Avalonia.HarfBuzz": "12.1.5", + "Avalonia.Native": "12.1.5", + "Avalonia.Skia": "12.1.5", + "Avalonia.Win32": "12.1.5", + "Avalonia.X11": "12.1.5" + } + }, + "Avalonia.Fonts.Inter": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "SeFNLktbuxrHxUzPjYubzWB1O9HVCNo3IezZZ9SgkFBAcKcSYrY2KcD8lfP7QWMGxPUvqdLyn0TmioR9d3nZUg==", + "dependencies": { + "Avalonia": "12.1.5" + } + }, + "Avalonia.FreeDesktop": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "XqLCj3V2PJsIaNANnFwolEv9Ng8LT7XM//dG39wPv/oeVTijjLNbayWAkeQlpKxi6RtAkjfnXjK+xK/S1caQLA==", + "dependencies": { + "Avalonia": "12.1.5", + "Tmds.DBus.Protocol": "0.94.1" + } + }, + "Avalonia.FreeDesktop.AtSpi": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "MTqCBRv1+nkUHPUeMtbh4/dA3AaCEA6LpIT+ZtCnWO4QSxEQFIdlyMZa/d4y6qBBIcxw2DhdY3+JzdhCn6DpUg==", + "dependencies": { + "Avalonia": "12.1.5" + } + }, + "Avalonia.HarfBuzz": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "3kspGPaRVrCzgMi4Rv4N33VNbQNN4wcY4NoCufYxKF/kvYqPbrAnGD2Mjm6Kmc3IA3QrLbj5UJpg8WDTAd4cPw==", + "dependencies": { + "Avalonia": "12.1.5", + "HarfBuzzSharp": "8.3.1.3", + "HarfBuzzSharp.NativeAssets.Linux": "8.3.1.3", + "HarfBuzzSharp.NativeAssets.WebAssembly": "8.3.1.3" + } + }, + "Avalonia.Headless": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "0s3+IiZ/lP7A2R80OOFc3YiSK3qnWOLO9KVsMBGjLX/kg4JSsU2YJiAMNTZIihDmIOYV0VHECJCzsSNKCTMXww==", + "dependencies": { + "Avalonia": "12.1.5", + "Avalonia.Fonts.Inter": "12.1.5", + "Avalonia.HarfBuzz": "12.1.5" + } + }, + "Avalonia.Native": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "LsC92ialAyccwXiqSjSHVlYGbASfB8XHbGT0XB3zqERF7PM7M3KjSnMgGc1HkzvwtGX+GgTYap6PV8N9H6un8Q==", + "dependencies": { + "Avalonia": "12.1.5" + } + }, + "Avalonia.Remote.Protocol": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "yNSke7MNKxJrgcTOQJk70QhPU568dxwRMeQCSmq9N7IGbpPS4zpbr4o6e8rLF/8UfASBQN/yuTgqUfa2nwf6Iw==" + }, + "Avalonia.Skia": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "irfPBtb4UoAT0qfRviYsALBwo0JwXnziRNOTQLzzAQt3eoNPyKFLBsgyj5hE4+6vgunQ7hltEHlvTHPkCoreIg==", + "dependencies": { + "Avalonia": "12.1.5", + "HarfBuzzSharp": "8.3.1.3", + "HarfBuzzSharp.NativeAssets.Linux": "8.3.1.3", + "HarfBuzzSharp.NativeAssets.WebAssembly": "8.3.1.3", + "SkiaSharp": "3.119.4", + "SkiaSharp.NativeAssets.Linux": "3.119.4", + "SkiaSharp.NativeAssets.WebAssembly": "3.119.4" + } + }, + "Avalonia.Themes.Fluent": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "FGjtJEpeVIbth+u4r507c9nbm1NQWLTvT3RC4e/yqDceRC4crAl37zVUloElA6rexDz69y0WvDfMLGX+t12yBQ==", + "dependencies": { + "Avalonia": "12.1.5" + } + }, + "Avalonia.Win32": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "QHfRLCyvVzj74AMCohS5a08Jz5f7VAeRgZ0SJsDTSfqxRzdbw9DXpKsncu21YmEvRWxMPkwyoB9cx9dvS1Bwmg==", + "dependencies": { + "Avalonia": "12.1.5", + "Avalonia.Angle.Windows.Natives": "2.1.27548.20260419" + } + }, + "Avalonia.X11": { + "type": "Transitive", + "resolved": "12.1.5", + "contentHash": "5EK2QrM9I3Lm3Aq/ZSpJjg1ZzBXl7YE9PxaB/mLxF9ysZz8qgFaJ5Zj+A9NdtAszupA+81Z3B9EmZP++2Xs4Ug==", + "dependencies": { + "Avalonia": "12.1.5", + "Avalonia.FreeDesktop": "12.1.5", + "Avalonia.FreeDesktop.AtSpi": "12.1.5", + "Avalonia.Skia": "12.1.5" + } + }, + "HarfBuzzSharp": { + "type": "Transitive", + "resolved": "8.3.1.3", + "contentHash": "NGZ2+ZVNPM+NdHB/asW0/ykWngyHWwcqjrbN2nDeH1B/aptPGlCUl8wkQ2cSJxw5fdWgdmIPmNuTPWpLwNVXWg==", + "dependencies": { + "HarfBuzzSharp.NativeAssets.Win32": "8.3.1.3", + "HarfBuzzSharp.NativeAssets.macOS": "8.3.1.3" + } + }, + "HarfBuzzSharp.NativeAssets.Linux": { + "type": "Transitive", + "resolved": "8.3.1.3", + "contentHash": "RI6A1LgmooU30+4QIyFt5rmBCzP0VzTR+587IJSGvYIsHHWlahFufihYxtraLfsIhW7I8dn6+xX+DZGygOPKWQ==" + }, + "HarfBuzzSharp.NativeAssets.macOS": { + "type": "Transitive", + "resolved": "8.3.1.3", + "contentHash": "KPTq0xnslkI6nAo0jh3ptcQPJvZZr7MWYXa2jUe4SnHc9q+JlHElmNXp0sfFoiTgoCX7WOYpYsurypuH9Gehxw==" + }, + "HarfBuzzSharp.NativeAssets.WebAssembly": { + "type": "Transitive", + "resolved": "8.3.1.3", + "contentHash": "w2QfdNm9Uz/sUa0B5D+OnVQhyq3G/fBq6ibQMdWBlQqqwh0g0/5j3RFvYqZAmRZ5+RzvjVe8o8SFFnWYUSkuxA==" + }, + "HarfBuzzSharp.NativeAssets.Win32": { + "type": "Transitive", + "resolved": "8.3.1.3", + "contentHash": "bx8CE8Js+XGX8PUxAHCBDEORt5aaBYtMN4Hr9QFs57Xithh6yjUyYqksizH6eRDhJkwsGI+SXWmPmMm8lZC9Pw==" + }, + "MicroCom.Runtime": { + "type": "Transitive", + "resolved": "0.11.6", + "contentHash": "NdNWGDiZ6eS/Mf/9+QHR91cj1K7Hy+PX9yrHI/zM7xFYuj9IWT2uxtB6sCHjrnxAeLV9fut1R6zHDUGKX6f9lQ==" + }, + "SkiaSharp": { + "type": "Transitive", + "resolved": "3.119.4", + "contentHash": "53NOSUZ1Us+91Sm0uCkIivh/k7jOowRErZT2sIWwPFN9mLUvdxnE6rS4sWo4255+Rd2MWUSF+j0NMZHD6Cke+Q==", + "dependencies": { + "SkiaSharp.NativeAssets.Win32": "3.119.4", + "SkiaSharp.NativeAssets.macOS": "3.119.4" + } + }, + "SkiaSharp.NativeAssets.Linux": { + "type": "Transitive", + "resolved": "3.119.4", + "contentHash": "UAyVzbqNfZsZbKbzj68zXLyUyF/SbTKmzTfOO6qDu++dtIUMMTzPBe8oOuzU/DiewpfKoUUlOSsJmqWc6blxBw==" + }, + "SkiaSharp.NativeAssets.macOS": { + "type": "Transitive", + "resolved": "3.119.4", + "contentHash": "fgBOWEqbY012x7gMfJU4ezgz6dfhJb30Z6YdW35h85Zoe39+a8YNbAAwL29ihPfWoppg5AjvyKNzD1oCvlqWwA==" + }, + "SkiaSharp.NativeAssets.WebAssembly": { + "type": "Transitive", + "resolved": "3.119.4", + "contentHash": "S1HOxtBbD4bYDtA2e9WH5TX+lxqRrTPvKjrjttRhxnHNNu7YY8VFo/LeCP7tNqoTA6PV+8vsvNbmRUEC2ip8RQ==" + }, + "SkiaSharp.NativeAssets.Win32": { + "type": "Transitive", + "resolved": "3.119.4", + "contentHash": "XOpbx/4CReO2wYsq2s6rbvdauc6dntG4Zv499sHGTJ87bwZaFXszFkwql3+FIZMc8kUPeaj3Mx2ezIJmo8a1Kg==" + }, + "Tmds.DBus.Protocol": { + "type": "Transitive", + "resolved": "0.94.1", + "contentHash": "11YMr7FnAbL83bQmVxlhbIKHvSLxjO81D12Ej0QMSGXMDTxNA9MTOa4MQxx43nv5el/efuPHwzyrj6a5ha2gug==" + }, + "bjopackage.bjogui": { + "type": "Project", + "dependencies": { + "Avalonia": "[12.1.5, )", + "Avalonia.Desktop": "[12.1.5, )", + "Avalonia.Headless": "[12.1.5, )", + "Avalonia.Skia": "[12.1.5, )", + "Avalonia.Themes.Fluent": "[12.1.5, )" + } + } + } + } +} \ No newline at end of file diff --git a/src/board.bjo b/src/board.bjo new file mode 100644 index 0000000..939891b --- /dev/null +++ b/src/board.bjo @@ -0,0 +1,138 @@ +;; The physical keyboard: the main block of an ISO Swedish keyboard, typed +;; with the angle mod. +;; +;; - Rows: 0 is the top row (Q), 1 the home row (A), 2 the bottom row (<). +;; - Columns count from the left of each row: from Q, from A and from <. Two +;; keys of one row are adjacent when their columns differ by 1. +;; - Fingers: 0 pinky, 1 ring, 2 middle, 3 index, on either hand. +;; - Angle mod: on the bottom row the left pinky types <, the ring finger Z, +;; the middle finger X, and the index C and V. B is an index stretch. +;; - A stretch key is one the hand reaches for: the centre columns T G B and +;; Y H, and the right pinky's keys past P, Ö and -. N is not a stretch key: +;; it is as near J as V is to F. +;; - Keys are named by what Swedish QWERTY prints on them. +;; +;; The costs grade Michael Capewell's grid. The keys he leaves at 0 cost more +;; the farther they are from the home row: home 0, the top row 1-4, the bottom +;; row 1-4. On the top row the middle finger is cheapest, then the index. The index keys of the bottom row cost 1: with the angle mod, C and +;; V lie as near F as N and M lie to J. +;; +;; 10 3 1 2 4 | 10 2 1 3 10 15 top row +;; 0 0 0 0 2 | 2 0 0 0 0 5 15 home row +;; 10 4 3 1 1 10 | 1 1 3 4 10 bottom row +;; +;; These are the default costs. (keyscore params) holds the costs scored +;; with, and the key weight every cost is multiplied by. +;; +;; The left and right halves differ only at T and Y: the top row is shifted +;; right of the home row, so T is near F and Y is far from J. +;; +;; Positions, for finger distance, are in key widths, as on an ISO board: +;; - x is 0 at A. The top row starts a quarter key left of the home row, at +;; Q, and the bottom row half a key left, at <. +;; - y is the row: 0 top, 1 home, 2 bottom. Rows are one key width apart. + +(import (std prelude)) +(import (std maths)) + +(export Key board key-named key-position home-key key-distance) + +;; - index: the key's place in `board`. +;; - cost: the default grid cost, before the key weight. +(type (: Key (Record (: name string) + (: index int) + (: left? bool) + (: finger int) + (: row int) + (: column int) + (: stretch? bool) + (: cost int)))) + +(def pinky 0) +(def ring 1) +(def middle 2) +(def index 3) + +(: (-> string int int bool int bool int Key)) +(defun (key name row column left? finger stretch? cost) + (Key (name name) (index 0) (left? left?) (finger finger) (row row) (column column) + (stretch? stretch?) (cost cost))) + +;; The keys of `board`, before their indices are set. +(: (Vec Key)) +(def keys + (let ((L #t) (R #f) (reach #t) (no #f)) + [;; Top row + (key "q" 0 0 L pinky no 10) + (key "w" 0 1 L ring no 3) + (key "e" 0 2 L middle no 1) + (key "r" 0 3 L index no 2) + (key "t" 0 4 L index reach 4) + (key "y" 0 5 R index reach 10) + (key "u" 0 6 R index no 2) + (key "i" 0 7 R middle no 1) + (key "o" 0 8 R ring no 3) + (key "p" 0 9 R pinky no 10) + (key "å" 0 10 R pinky reach 15) + ;; Home row + (key "a" 1 0 L pinky no 0) + (key "s" 1 1 L ring no 0) + (key "d" 1 2 L middle no 0) + (key "f" 1 3 L index no 0) + (key "g" 1 4 L index reach 2) + (key "h" 1 5 R index reach 2) + (key "j" 1 6 R index no 0) + (key "k" 1 7 R middle no 0) + (key "l" 1 8 R ring no 0) + (key "ö" 1 9 R pinky no 0) + (key "ä" 1 10 R pinky reach 5) + (key "'" 1 11 R pinky reach 15) + ;; Bottom row, angle mod + (key "<" 2 0 L pinky no 10) + (key "z" 2 1 L ring no 4) + (key "x" 2 2 L middle no 3) + (key "c" 2 3 L index no 1) + (key "v" 2 4 L index no 1) + (key "b" 2 5 L index reach 10) + (key "n" 2 6 R index no 1) + (key "m" 2 7 R index no 1) + (key "," 2 8 R middle no 3) + (key "." 2 9 R ring no 4) + (key "-" 2 10 R pinky no 10)])) + +;; Every key a layout can put a character on, by row and column, each with +;; its index. +(: (Vec Key)) +(def board + (loop (:for k keys) (:for i (range 0 (vec-length keys))) + (:acc out (vecing (record-set k (index i)))))) + +;; The key printed with `name` on Swedish QWERTY. Raises if there is none. +(: (-> string Key)) +(defun (key-named name) + (match (find (fun (k) (= (record-ref k name) name)) board) + ((Some k) k) + (None (panic! (str "no key is named " name))))) + +;; Where `k` is, in key widths: see the header. +(: (-> Key (Tuple double double))) +(defun (key-position k) + (def offset (match (record-ref k row) (0 -0.25) (1 0.0) (_ -0.5))) + (Tuple (+ offset (cast double (record-ref k column))) (cast double (record-ref k row)))) + +;; The home-row key that `k`'s finger rests on: the one of its hand and +;; finger on the home row that is not a stretch key. +(: (-> Key Key)) +(defun (home-key k) + (option-ref (find (fun (h) (and (= (record-ref h row) 1) + (= (record-ref h left?) (record-ref k left?)) + (= (record-ref h finger) (record-ref k finger)) + (not (record-ref h stretch?)))) + board))) + +;; How far apart `a` and `b` are, in key widths. +(: (-> Key Key double)) +(defun (key-distance a b) + (def (Tuple ax ay) (key-position a)) + (def (Tuple bx by) (key-position b)) + (sqrt (+ (* (- ax bx) (- ax bx)) (* (- ay by) (- ay by))))) diff --git a/src/gui.bjo b/src/gui.bjo new file mode 100644 index 0000000..2087a19 --- /dev/null +++ b/src/gui.bjo @@ -0,0 +1,730 @@ +;; A window for tuning the scorer's numbers and seeing what they do. +;; +;; bjo run src/gui.bjo +;; +;; Run it from the package's folder: the texts are read from corpus/. +;; +;; - Left: every layout's score, best first, rescored as soon as a number +;; changes. A click chooses a layout for the Layout and Search tabs. +;; - Rules: the numbers of (keyscore params), by group. Key costs: each key's +;; cost, on the board. +;; - Layout: the chosen layout on the board, and the parts of its score. +;; - Search: runs (keyscore search) from the chosen layout with the numbers as +;; they are. "Keep" adds a result to the list, "Copy" puts it on the +;; clipboard as a make-layout form. +;; - Save writes the numbers to params.txt, which the other programs read. +;; Reload reads it again; Defaults puts back `default-params`, unsaved. +;; - Presets, on the Rules tab: named sets of numbers in presets/. See +;; (keyscore saved). +;; - Layouts can be saved by name, from the Layout tab or from a search +;; result, to saved-layouts.txt. The saved layouts are listed at start. +;; - An "i" mark shows what a number or an option does. + +(import (std prelude)) +(import (only (std fmt) numeric render->string)) +(import (except (bjogui view) Key)) +(import (bjogui app)) +(import (bjogui draw)) +(import (keyscore board)) +(import (keyscore layouts)) +(import (keyscore params)) +(import (keyscore saved)) +(import (keyscore score)) +(import (keyscore search)) +(import (keyscore texts)) + +(export Model Msg initial gui-update view subscriptions ranking) + +;; --- the model ------------------------------------------------------------------ + +;; - params: the numbers scored with. +;; - counts: the counts of the texts of `params`, empty until they are read. +;; - layouts: (keyscore layouts), then the saved layouts and the search +;; results kept. +;; - scores: [layout][text], for `generation`: each change of the numbers +;; starts a new one, and a score of an older one is dropped. +;; - chosen: the layout the Layout and Search tabs show, by index in `layouts`. +;; - options, searching, results: the Search tab. +;; - saved: the names of the layouts in `layouts-file`. +;; - layout-name: the name a layout is saved as. +;; - presets, preset-name: the presets, and the name one is saved as. +(type (: Model (Record (: params Params) + (: counts (Vec Counts)) + (: layouts (Vec Layout)) + (: scores (Vec (Vec Score))) + (: generation int) + (: chosen int) + (: tab int) + (: options Options) + (: searching bool) + (: results (Vec Found)) + (: kept int) + (: saved (Vec string)) + (: layout-name string) + (: presets (Vec string)) + (: preset-name string) + (: status string)))) + +(type (: Msg (Union (: Counted (Vec Counts)) + (: Scored int (Vec (Vec Score))) + (: SetParam int double) + (: Choose int) + (: Tab int) + (: SetRuns double) + (: SetSteps double) + (: SetTStart double) + (: SetVowels bool) + (: SetFixed string) + Search + (: Searched (Vec Found)) + (: Failed string) + (: Keep int) + (: Copy int) + Save + Saved + Reload + (: Loaded Params) + Defaults + (: SetPresetName string) + SavePreset + (: LoadPreset string) + (: PresetLoaded string Params) + (: DeletePreset string) + (: Listed string (Vec string)) + (: SetLayoutName string) + SaveLayout + (: SaveResult int) + RemoveLayout + (: Wrote string) + Noop))) + +;; The layouts of `layouts-file` when the program starts. +(: (Vec Layout)) +(def saved-at-start (read-saved-layouts)) + +(: (-> Layout string)) +(defun (lname l) (record-ref l name)) + + +(: Model) +(def initial + (Model (params (current-params)) + (counts []) + (layouts (vec-append layouts saved-at-start)) + (scores []) + (generation 0) + (chosen (- (vec-length layouts) 1)) + (tab 0) + (options (Options (runs 12) (steps 1000000) (t-start 0.3) (vowels-left #f) (fixed ""))) + (searching #f) + (results []) + (kept 0) + (saved (vec-map lname saved-at-start)) + (layout-name "") + (presets (preset-names)) + (preset-name "") + (status "Reading the texts…"))) + +;; --- scoring -------------------------------------------------------------------- + +;; The score of each of `ls` on each of `counts`. +(: (-> Params (Vec Layout) (Vec Counts) (Vec (Vec Score)))) +(defun (score-all p ls counts) + (vec-map (fun (l) (vec-map (fun (c) (score-layout p l c)) counts)) ls)) + +;; The mean of `xs` weighted by the texts' weights, or 0 when none weighs +;; anything. +(: (-> Params (Vec double) double)) +(defun (mean p xs) + (def cs (record-ref p corpora)) + (if (> (vec-fold (fun (c t) (+ t (corpus-weight c))) 0.0 cs) 0.0) (weighted-mean cs xs) 0.0)) + +;; `f` of each text's score of the layout `i`, as the weighted mean. +(: (-> Model int (-> Score double) double)) +(defun (mean-of m i f) (mean (.params m) (vec-map f (vec-ref (.scores m) i)))) + +;; The indices of the layouts, best first. Empty until they are scored. +(: (-> Model (Vec int))) +(defun (ranking m) + (if (= (vec-length (.scores m)) (vec-length (.layouts m))) + (vec-sort-by (fun (a b) (compare (mean-of m a (fun (s) (.total s))) (mean-of m b (fun (s) (.total s))))) + (->vec (range 0 (vec-length (.layouts m))))) + [])) + +;; A message for the result of `blocking` work: `(f x)` for its value, or +;; `(Failed what)` when it raised. +(: (-> (-> %a Msg) string (-> (Result System.Exception %a) Msg))) +(defun (on-ok f what) + (fun (r) (match r ((Ok x) (f x)) ((Err _) (Failed what))))) + +;; The model with `p`, and the command that scores the layouts with it. +(: (-> Model Params (Tuple Model (List (Event Msg))))) +(defun (rescore m p) + (def g (+ (.generation m) 1)) + (def ls (.layouts m)) + (def counts (.counts m)) + (if (= (vec-length counts) 0) + (next (record-set m (params p) (generation g))) + (next (record-set m (params p) (generation g)) + (wrap (blocking (fun () (score-all p ls counts))) + (on-ok (fun (s) (Scored g s)) "Scoring failed."))))) + +;; `step` with the command `c` as well. +(: (-> (Tuple Model (List (Event Msg))) (Event Msg) (Tuple Model (List (Event Msg))))) +(defun (and-then step c) + (def (Tuple m cs) step) + (Tuple m (cons c cs))) + +;; The index of the layout named `n` in `ls`, if there is one. +(: (-> (Vec Layout) string (Option int))) +(defun (index-named ls n) + (find (fun (i) (= (lname (vec-ref ls i)) n)) (range 0 (vec-length ls)))) + +;; The command that writes the saved layouts of `ls` to `layouts-file`. +(: (-> (Vec Layout) (Vec string) string (Event Msg))) +(defun (write-layouts ls saved done) + (def built (vec-length layouts)) + (def to-write (loop (:for l ls) (:for i (range 0 (vec-length ls))) + (:when (and (>= i built) (vec-contains? saved (lname l)))) + (:acc out (vecing l)))) + (wrap (blocking (fun () (write-saved-layouts to-write) #t)) + (on-ok (fun (_) (Wrote done)) (str "Could not write " layouts-file ".")))) + +(: string) +(def bad-name "A name cannot be empty, start or end with a space, start with a dot, or hold / \\ or a tab.") + +;; `m` with `l` saved as `name`, and the commands that score the layouts and +;; write the file. +;; +;; - replacing: the index of a layout that `l` takes the place of, such as a +;; kept result or a saved layout given a new name. A built-in layout is +;; never replaced. +;; - Another layout that is not built in and has the same name is replaced. +;; - choose?: whether the saved layout becomes the chosen one. If not, the +;; chosen layout stays chosen, unless it is replaced. +;; - Rejects an invalid name, and the name of a built-in layout. +(: (-> Model Layout (Option int) string bool (Tuple Model (List (Event Msg))))) +(defun (save-layout m l replacing name choose?) + (def ls (.layouts m)) + (def built (vec-length layouts)) + (cond + ((not (valid-name? name)) (next (record-set m (status bad-name)))) + ((some? (index-named layouts name)) + (next (record-set m (status (str name " is a built-in layout. Choose another name."))))) + (else + (def replaced (match replacing + ((Some i) (if (>= i built) (Some i) None)) + (None None))) + (def old (match replaced ((Some i) (Some (lname (vec-ref ls i)))) (None None))) + (def others (loop (:for x ls) (:for i (range 0 (vec-length ls))) + (:when (or (< i built) + (and (not (= (Some i) replaced)) (not (= (lname x) name))))) + (:acc out (vecing x)))) + (def new-ls (vec-append others [(record-set l (name name))])) + (def last (- (vec-length new-ls) 1)) + (def before (lname (vec-ref ls (.chosen m)))) + (def chosen (if choose? last (option-ref-or (index-named new-ls before) last))) + (def saved (vec-append (vec-filter (fun (n) (not (or (= n name) (= (Some n) old)))) (.saved m)) [name])) + (and-then (rescore (record-set m (layouts new-ls) (saved saved) (chosen chosen) (scores []) + (layout-name name)) + (.params m)) + (write-layouts new-ls saved (str "Saved the layout " name " to " layouts-file ".")))))) + +;; Whether `l` types a vowel on the right hand. +(: (-> Layout bool)) +(defun (vowel-right? l) + (any? (fun (c) (match (layout-key l c) + ((Some k) (not (record-ref k left?))) + (None #f))) + (->list "aeiouyåäö"))) + +;; --- update --------------------------------------------------------------------- + +(: gui-update (-> Msg Model (Tuple Model (List (Event Msg))))) +(defun (gui-update msg m) + (def o (.options m)) + (match msg + ((Counted cs) (rescore (record-set m (counts cs) (status "")) (.params m))) + ((Scored g s) (next (if (= g (.generation m)) (record-set m (scores s)) m))) + ((SetParam i x) (rescore m ((record-ref (vec-ref param-specs i) set) (.params m) x))) + ((Choose pos) + (def r (ranking m)) + (if (and (>= pos 0) (< pos (vec-length r))) + (let ((i (vec-ref r pos))) + (next (record-set m (chosen i) + (layout-name (if (< i (vec-length layouts)) (.layout-name m) + (lname (vec-ref (.layouts m) i))))))) + (next m))) + ((Tab t) (next (record-set m (tab t)))) + ((SetRuns x) (next (record-set m (options (record-set o (runs (cast int x))))))) + ((SetSteps x) (next (record-set m (options (record-set o (steps (cast int x))))))) + ((SetTStart x) (next (record-set m (options (record-set o (t-start x)))))) + ((SetVowels b) (next (record-set m (options (record-set o (vowels-left b)))))) + ((SetFixed s) (next (record-set m (options (record-set o (fixed s)))))) + (Search + (def l (vec-ref (.layouts m) (.chosen m))) + (def p (.params m)) + (def counts (.counts m)) + (cond ((= (vec-length counts) 0) (next m)) + ((= (mean p (vec-map (fun (c) 1.0) counts)) 0.0) + (next (record-set m (status "No text weighs anything.")))) + ((and (.vowels-left o) (vowel-right? l)) + (next (record-set m (status (str (record-ref l name) " has a vowel on the right hand."))))) + (else + (next (record-set m (searching #t) (results []) + (status (str "Searching from " (record-ref l name) "…"))) + (wrap (blocking (fun () (search (problem-of p l counts) o))) + (on-ok Searched "The search failed.")))))) + ((Searched found) + (next (record-set m (searching #f) (results (vec-sort-by (fun (a b) (compare (.score a) (.score b))) found)) + (status "")))) + ((Failed what) (next (record-set m (searching #f) (status what)))) + ((Keep i) + (def rows (.rows (vec-ref (.results m) i))) + (def n (+ (.kept m) 1)) + (def l (make-layout (str "found " (displayed n)) (vec-ref rows 0) (vec-ref rows 1) (vec-ref rows 2))) + (rescore (record-set m (layouts (vec-append (.layouts m) [l])) (kept n) (scores [])) (.params m))) + ((Copy i) + (def rows (.rows (vec-ref (.results m) i))) + (next m (copy-text (str "(make-layout \"found\"\n \"" (vec-ref rows 0) "\"\n \"" + (vec-ref rows 1) "\"\n \"" (vec-ref rows 2) "\")")))) + (Save + (def p (.params m)) + (next m (wrap (blocking (fun () (save-params p params-file) #t)) + (on-ok (fun (_) Saved) (str "Could not write " params-file "."))))) + (Saved (next (record-set m (status (str "Saved to " params-file "."))))) + (Reload (next m (wrap (blocking current-params) (on-ok Loaded (str "Could not read " params-file "."))))) + ((Loaded p) (rescore (record-set m (status (if (file-exists? params-file) + (str "Read " params-file ".") + (str "No " params-file ": the defaults.")))) + p)) + (Defaults (rescore (record-set m (status "The defaults, not saved.")) default-params)) + ((SetPresetName s) (next (record-set m (preset-name s)))) + (SavePreset + (def name (.preset-name m)) + (def p (.params m)) + (if (valid-name? name) + (next m (wrap (blocking (fun () (save-preset name p))) + (on-ok (fun (names) (Listed (str "Saved the preset " name ".") names)) + (str "Could not save the preset " name ".")))) + (next (record-set m (status bad-name))))) + ((LoadPreset name) + (next m (wrap (blocking (fun () (load-preset name))) + (on-ok (fun (p) (PresetLoaded name p)) (str "Could not read the preset " name "."))))) + ((PresetLoaded name p) + (rescore (record-set m (preset-name name) + (status (str "Loaded the preset " name ". Save writes it to " params-file "."))) + p)) + ((DeletePreset name) + (next m (wrap (blocking (fun () (delete-preset name))) + (on-ok (fun (names) (Listed (str "Deleted the preset " name ".") names)) + (str "Could not delete the preset " name "."))))) + ((Listed what names) (next (record-set m (presets names) (status what)))) + ((SetLayoutName s) (next (record-set m (layout-name s)))) + (SaveLayout + (def i (.chosen m)) + (save-layout m (vec-ref (.layouts m) i) (Some i) (.layout-name m) #t)) + ((SaveResult i) + (def rows (.rows (vec-ref (.results m) i))) + (save-layout m (make-layout "" (vec-ref rows 0) (vec-ref rows 1) (vec-ref rows 2)) None (.layout-name m) #f)) + (RemoveLayout + (def i (.chosen m)) + (def ls (.layouts m)) + (def built (vec-length layouts)) + (if (< i built) + (next (record-set m (status "A built-in layout cannot be removed."))) + (let ((name (lname (vec-ref ls i)))) + (def new-ls (loop (:for x ls) (:for j (range 0 (vec-length ls))) + (:when (not (= j i))) + (:acc out (vecing x)))) + (def saved (vec-filter (fun (n) (not (= n name))) (.saved m))) + (def step (rescore (record-set m (layouts new-ls) (saved saved) (chosen (- built 1)) (scores []) + (status (str "Removed " name "."))) + (.params m))) + (if (vec-contains? (.saved m) name) + (and-then step (write-layouts new-ls saved (str "Removed " name " from " layouts-file "."))) + step)))) + ((Wrote what) (next (record-set m (status what)))) + (Noop (next m)))) + +(: subscriptions (-> Model (List (Sub Msg)))) +(defun (subscriptions m) + (if (= (vec-length (.counts m)) 0) + (let ((paths (vec-map corpus-path (record-ref (.params m) corpora)))) + (list (task "count" (wrap (blocking (fun () (count-files layouts paths))) + (on-ok Counted "Could not read the texts."))))) + '())) + +;; --- the board -------------------------------------------------------------------- + +(: double) +(def key-size 38.0) + +;; The colour of a finger's keys: pinky, ring, middle, index. +(: (-> Key string)) +(defun (finger-colour k) + (match (record-ref k finger) + (0 "#f4c7c3") + (1 "#fce8b2") + (2 "#b7e1cd") + (_ "#c6dafc"))) + +;; Where `k` is drawn, in pixels: its position on the board, the right hand +;; half a key further right. +(: (-> Key (Tuple double double))) +(defun (key-place k) + (def (Tuple x y) (key-position k)) + (def gap (if (record-ref k left?) 0.0 0.5)) + (Tuple (* key-size (+ x 0.5 gap)) (* key-size y))) + +(: double) +(def board-width (* key-size 13.0)) + +(: double) +(def board-height (* key-size 3.0)) + +;; The board, with a label on each key of `board`, in its order. A stretch +;; key has a darker edge. +(: keyboard (Component (Vec string) Unit Unit)) +(def keyboard + (component unit + (fun (pen w h s labels) + (loop (:for k board) (:for label labels) + (:let (Tuple x y) (key-place k)) + (:let size (- key-size 4.0)) + (:do (fill-rect pen x y size size (finger-colour k) #:radius 5.0) + (when (record-ref k stretch?) + (stroke-rect pen x y size size "#888888" #:width 1.5 #:radius 5.0)) + (draw-text pen label (+ x (/ size 2.0)) (+ y 7.0) "#202124" + #:size 17.0 #:bold #t #:align "centre")))))) + +;; The character `l` puts on each key of `board`, upper case, or "". +(: (-> Layout (Vec string))) +(defun (labels-of l) + (def on-key (loop (:for c (layout-base-chars l)) + (:let k (option-ref (layout-key l c))) + (:acc out (mapping (Tuple (record-ref k index) c))))) + (vec-map (fun (k) (match (map-try-ref on-key (record-ref k index)) + ((Some c) (if (char=? c #\') "’" (char->string (char-upcase c)))) + (None ""))) + board)) + +(: (-> (Vec string) (View Msg))) +(defun (board-view labels) + (drawn keyboard labels (fun (_) Noop) #:width board-width #:height board-height)) + +;; --- formatting --------------------------------------------------------------------- + +(: (-> double string)) +(defun (fixed x) (render->string (numeric x #:precision 2))) + +(: (-> double string)) +(defun (fixed3 x) (render->string (numeric x #:precision 3))) + +(: (-> double string)) +(defun (pct x) (str (render->string (numeric (* 100.0 x) #:precision 1)) "%")) + +;; A text's name in a heading: its file name with no .txt. +(: (-> string string)) +(defun (file-label path) + (def name (option-ref-or (path-filename path) path)) + (if (string-ends-with? name ".txt") (string-drop-right name 4) name)) + +;; The number `s` holds, if it holds one. +(: (-> string (Option double))) +(defun (number-in s) + (result->option (try (string->double s) #:catch (System.Exception)))) + +;; A grid of rows of texts, the first column left and the others right. +(: (-> string (Vec (Vec string)) (View Msg))) +(defun (table columns rows) + (apply grid #:columns columns + #:rows (string-join (vec-map (fun (_) "Auto") rows) ",") + #:spacing 4.0 + (vec->array (loop (:for r rows) (:for i (range 0 (vec-length rows))) + (:let cells r) + (:for cell cells) (:for j (range 0 (vec-length cells))) + (:acc out (vecing (at i j (text cell)))))))) + +;; --- the views --------------------------------------------------------------------- + +;; A blue circle with an "i", 16 pixels across. +(: info-mark (Component Unit Unit Unit)) +(def info-mark + (component unit + (fun (pen w h s _) + (fill-circle pen 8.0 8.0 8.0 "#1a73e8") + (draw-text pen "i" 8.0 0.5 "#ffffff" #:size 12.0 #:bold #t #:align "centre")))) + +;; A small "i" mark that shows `help` when the pointer rests on it. +(: (-> string (View Msg))) +(defun (info help) + (tooltip (drawn info-mark unit (fun (_) Noop) #:width 16.0 #:height 16.0) help)) + +;; `v` with an "i" mark after it, and `help` shown on both. Where a row has an +;; input, the mark goes after the input instead. +(: (-> (View Msg) string (View Msg))) +(defun (with-info v help) + (row #:spacing 6.0 (tooltip v help) (info help))) + +;; The help of `s`, and its default. +(: (-> Spec string)) +(defun (spec-help s) + (str (record-ref s help) "\n\nDefault: " (displayed ((record-ref s get) default-params)) ".")) + +(: (-> Model (View Msg))) +(defun (scores-view m) + (def cs (record-ref (.params m) corpora)) + (def r (ranking m)) + (def columns (string-join (vec-append ["90" "60"] (vec-map (fun (_) "60") cs)) ",")) + (: line (-> (Vec string) (View Msg))) + (def line (fun (cells) + (apply grid #:columns columns + (vec->array (loop (:for c cells) (:for j (range 0 (vec-length cells))) + (:acc out (vecing (at 0 j (text c))))))))) + (def items + (vec-map (fun (i) + (def l (vec-ref (.layouts m) i)) + (keyed (record-ref l name) + (line (vec-append [(record-ref l name) (fixed (mean-of m i (fun (s) (.total s))))] + (vec-map (fun (s) (fixed (.total s))) (vec-ref (.scores m) i)))))) + r)) + (def selected (match (find (fun (p) (= (vec-ref r p) (.chosen m))) (range 0 (vec-length r))) + ((Some p) p) + (None -1))) + (column #:spacing 6.0 #:margin 8.0 + (text "Score per key press, lower is better" #:size 15.0) + (line (vec-append ["layout" "mean"] (vec-map (fun (c) (file-label (corpus-path c))) cs))) + (list-box items selected Choose #:width 340.0))) + +;; The numbers of one group, each with its input. +(: (-> Model string (View Msg))) +(defun (group-view m group) + (def p (.params m)) + (def specs (loop (:for s param-specs) (:for i (range 0 (vec-length param-specs))) + (:when (= (record-ref s group) group)) + (:acc out (vecing (Tuple i s))))) + (border + (column #:spacing 6.0 + (text group #:size 15.0) + (apply grid #:columns "Auto,130,Auto" #:rows (string-join (vec-map (fun (_) "Auto") specs) ",") #:spacing 6.0 + (vec->array + (loop (:for (Tuple i s) specs) (:for row (range 0 (vec-length specs))) + (:let input (number-input ((record-ref s get) p) (fun (x) (SetParam i x)) + #:minimum (record-ref s minimum) + #:maximum (record-ref s maximum) + #:step (record-ref s step) + #:width 130.0)) + (:let help (spec-help s)) + (:for cell (list (at row 0 (tooltip (text (record-ref s label)) help)) + (at row 1 (tooltip input help)) + (at row 2 (info help)))) + (:acc out (vecing cell)))))) + #:padding 10.0 #:radius 6.0 #:colour "#9e9e9e")) + +;; The presets: a name to save the numbers as, and each preset with Load and +;; Delete. +(: (-> Model (View Msg))) +(defun (presets-view m) + (def items + (if (= (vec-length (.presets m)) 0) + [(text "No presets yet.")] + (vec-map (fun (n) + (keyed n (row #:spacing 6.0 + (text n #:width 160.0) + (tooltip (button "Load" (LoadPreset n)) (str "Use the numbers of " n ".")) + (tooltip (button "Delete" (DeletePreset n)) (str "Delete the file of " n "."))))) + (.presets m)))) + (border + (column #:spacing 6.0 + (text "Presets" #:size 15.0) + (row #:spacing 6.0 + (text-input (.preset-name m) SetPresetName #:placeholder "name" #:width 160.0) + (tooltip (button "Save preset" SavePreset) + "Saves the numbers as they are under this name. A preset of the same name is replaced.") + (info (str "A preset is a named set of all the numbers, key costs and text weights included. " + "Presets are kept in the " presets-folder " folder. Load uses a preset's numbers; " + "Save at the top writes them to " params-file " for the other programs."))) + (apply column #:spacing 4.0 (vec->array items))) + #:padding 10.0 #:radius 6.0 #:colour "#9e9e9e")) + +(: (-> Model (View Msg))) +(defun (rules-view m) + (def groups ["Texts" "Keys" "Pairs" "Row motion" "Triples" "Pinky load" "Finger distance"]) + (scroll (apply flow #:spacing 10.0 #:margin 8.0 + (vec->array (vec-append [(presets-view m)] (vec-map (fun (g) (group-view m g)) groups)))))) + +;; The key costs, each input where its key is on the board. +(: (-> Model (View Msg))) +(defun (key-costs-view m) + (def p (.params m)) + (def width 56.0) + (: cell (-> Key (View Msg))) + (def cell (fun (k) + (def i (option-ref (find (fun (j) (= (record-ref (vec-ref param-specs j) name) + (str "key." (record-ref k name)))) + (range 0 (vec-length param-specs))))) + (def s (vec-ref param-specs i)) + (tooltip (column #:spacing 2.0 + (text (record-ref s label)) + (text-input (displayed ((record-ref s get) p)) + (fun (t) (match (number-in t) ((Some x) (SetParam i x)) (None Noop))) + #:width (- width 4.0))) + (spec-help s)))) + (: board-row (-> int (View Msg))) + (def board-row (fun (r) + (: on-row? (-> Key bool)) + (def on-row? (fun (k) (= (record-ref k row) r))) + (def ks (->vec (filter on-row? board))) + (def (Tuple x0 _) (key-position (vec-ref ks 0))) + ;; The first key of the right hand, with a gap before it. + (: first-right? (-> int bool)) + (def first-right? (fun (j) (and (> j 0) (not (record-ref (vec-ref ks j) left?)) + (record-ref (vec-ref ks (- j 1)) left?)))) + (apply row #:spacing 4.0 + (vec->array + (vec-append [(text "" #:width (* (+ x0 0.5) width))] + (loop (:for k ks) (:for j (range 0 (vec-length ks))) + (:acc out (vecing (if (first-right? j) + (row (text "" #:width (/ width 2.0)) (cell k)) + (cell k)))))))))) + (column #:spacing 10.0 #:margin 8.0 + (with-info (text (str "Each press costs its key's cost times the key weight, " + (displayed (record-ref p key-weight)) ". Set the key weight under Rules.")) + "Point at a key to see its default cost. A key of the right pinky past P and Ö, and the centre columns, are stretch keys: the Stretch key number is added to pairs with them.") + (board-row 0) (board-row 1) (board-row 2))) + +;; What the chosen layout is: built in, saved, or kept and not saved. +(: (-> Model string)) +(defun (kind-of-chosen m) + (def i (.chosen m)) + (cond ((< i (vec-length layouts)) "built in") + ((vec-contains? (.saved m) (lname (vec-ref (.layouts m) i))) (str "saved in " layouts-file)) + (else "kept, not saved"))) + +;; The name input and the Save layout and Remove buttons. +(: (-> Model (View Msg))) +(defun (save-layout-view m) + (def built? (< (.chosen m) (vec-length layouts))) + (row #:spacing 6.0 + (text "Save as") + (text-input (.layout-name m) SetLayoutName #:placeholder "name" #:width 160.0) + (tooltip (button "Save layout" SaveLayout) + (str "Saves the chosen layout under this name in " layouts-file ". A built-in layout is copied. " + "A kept or saved layout gets the new name. A saved layout of the same name is replaced.")) + (tooltip (button "Remove" RemoveLayout #:enabled (not built?)) + (str "Removes the chosen layout from the list, and from " layouts-file " if it is saved there.")) + (info (str "Saved layouts are kept in " layouts-file ", one a line, and are listed at start.")))) + +;; The chosen layout on the board, and the parts of its score. +(: (-> Model (View Msg))) +(defun (layout-view m) + (def i (.chosen m)) + (def l (vec-ref (.layouts m) i)) + (def heading (row #:spacing 10.0 (text (record-ref l name) #:size 18.0) (text (str "(" (kind-of-chosen m) ")")))) + (if (< i (vec-length (.scores m))) + (let ((cs (record-ref (.params m) corpora)) + (scores (vec-ref (.scores m) i))) + (: part (-> string (-> Score double) (-> double string) (Vec string))) + (def part (fun (label f show) + (vec-append (vec-append [label] (vec-map (fun (s) (show (f s))) scores)) + [(show (mean-of m i f))]))) + (def rule-rows (loop (:for r rules) (:for j (range 0 (vec-length rules))) + (:acc out (vecing (part (record-ref r name) (fun (s) (vec-ref (.rules s) j)) fixed))))) + (def rows + (vec-append + (vec-append [(vec-append (vec-append ["part"] (vec-map (fun (c) (file-label (corpus-path c))) cs)) ["mean"]) + (part "key costs" (fun (s) (.position s)) fixed) + (part "top-row vowels" (fun (s) (.top-vowels s)) fixed)] + rule-rows) + [(part "hand runs" (fun (s) (.runs s)) fixed) + (part "same-finger skips" (fun (s) (.skips s)) fixed) + (part "two-finger runs" (fun (s) (.two-fingers s)) fixed) + (part "pinky load" (fun (s) (.pinkies s)) fixed) + (part "distance cost" (fun (s) (* (record-ref (.params m) distance) (.distance s))) fixed) + (part "total" (fun (s) (.total s)) fixed3) + (part "finger distance (key widths)" (fun (s) (.distance s)) fixed) + (part "home row" (fun (s) (.home s)) pct) + (part "hand alternation" (fun (s) (.alternation s)) pct) + (part "same finger" (fun (s) (.same-finger s)) pct) + (part "triples in hand runs" (fun (s) (.run-share s)) pct) + (part "same-finger skip triples" (fun (s) (.skip-share s)) pct) + (part "two-finger run triples" (fun (s) (.two-finger-share s)) pct) + (part "left hand" (fun (s) (.left-hand s)) pct) + (part "left pinky" (fun (s) (.left-pinky s)) pct) + (part "right pinky" (fun (s) (.right-pinky s)) pct)])) + (scroll + (column #:spacing 10.0 #:margin 8.0 + heading + (save-layout-view m) + (board-view (labels-of l)) + (table (string-join (vec-append ["Auto"] (vec-map (fun (_) "70") (vec-ref rows 0))) ",") rows)))) + (column #:spacing 10.0 #:margin 8.0 heading (save-layout-view m) (board-view (labels-of l))))) + +(: (-> Model (View Msg))) +(defun (search-view m) + (def o (.options m)) + (def l (vec-ref (.layouts m) (.chosen m))) + (def results + (loop (:for f (.results m)) (:for i (range 0 (vec-length (.results m)))) + (:acc out (vecing (keyed (str "result " (displayed i)) + (row #:spacing 10.0 + (text (fixed3 (.score f)) #:width 50.0) + (board-view (labels-of (make-layout "" (vec-ref (.rows f) 0) (vec-ref (.rows f) 1) + (vec-ref (.rows f) 2)))) + (column #:spacing 6.0 + (tooltip (button "Keep" (Keep i) #:width 70.0) + "Adds the result to the list on the left as \"found N\". It is not saved.") + (tooltip (button "Save" (SaveResult i) #:width 70.0) + (str "Adds the result to the list and saves it in " layouts-file + " under the name in Save as.")) + (tooltip (button "Copy" (Copy i) #:width 70.0) + "Puts the result on the clipboard as a make-layout form for src/layouts.bjo.")))))))) + (scroll + (column #:spacing 10.0 #:margin 8.0 + (text (str "From " (record-ref l name) ", the layout chosen on the left, with the numbers as they are.") + #:wrap #t) + (grid #:columns "Auto,160,Auto" #:rows "Auto,Auto,Auto,Auto,Auto" #:spacing 6.0 + (at 0 0 (text "Runs, in parallel")) + (at 0 1 (number-input (cast double (.runs o)) SetRuns #:minimum 1.0 #:maximum 64.0 #:width 160.0)) + (at 0 2 (info "How many searches run at the same time. Each starts from the chosen layout with its own random swaps. More runs find good layouts more reliably.")) + (at 1 0 (text "Swaps per run")) + (at 1 1 (number-input (cast double (.steps o)) SetSteps #:minimum 1000.0 #:maximum 20000000.0 + #:step 100000.0 #:width 160.0)) + (at 1 2 (info "How many key swaps each run tries. More swaps take longer and search more thoroughly.")) + (at 2 0 (text "Start temperature")) + (at 2 1 (number-input (.t-start o) SetTStart #:minimum 0.002 #:maximum 5.0 #:step 0.01 #:width 160.0)) + (at 2 2 (info "How readily a run accepts a swap that makes the score worse, at the start. It falls to almost 0 at the end. High (0.3 or more): the search goes far from the start layout. Low (0.02): it refines the start layout.")) + (at 3 0 (text "Fixed characters")) + (at 3 1 (text-input (.fixed o) SetFixed #:placeholder "åäö.," #:width 160.0)) + (at 3 2 (info "Characters whose keys do not move, written as in a layout, such as åäö.,")) + (at 4 0 (checkbox "Vowels on the left hand" (.vowels-left o) SetVowels) #:column-span 2) + (at 4 2 (info "Keeps every vowel (a e i o u y å ä ö) on the left hand. The chosen layout must have all its vowels on the left."))) + (row #:spacing 6.0 + (text "Save as") + (text-input (.layout-name m) SetLayoutName #:placeholder "name" #:width 160.0) + (info "The name that Save on a result saves it under.")) + (button (if (.searching m) "Searching…" "Search") Search #:enabled (not (.searching m)) #:width 120.0) + (apply column #:spacing 10.0 (vec->array results))))) + +(: view (-> Model (View Msg))) +(defun (view m) + (dock + (at-top (row #:spacing 8.0 #:margin 8.0 + (tooltip (button "Save" Save) + (str "Writes the numbers to " params-file ". keyscore and optimize read that file.")) + (tooltip (button "Reload" Reload) (str "Reads the numbers from " params-file " again.")) + (tooltip (button "Defaults" Defaults) "Puts back the built-in numbers. Nothing is saved.") + (text (.status m) #:margin 6.0))) + (at-left (scores-view m)) + (tabs (.tab m) Tab #:margin 8.0 + (tab "Rules" (rules-view m)) + (tab "Key costs" (key-costs-view m)) + (tab "Layout" (layout-view m)) + (tab "Search" (search-view m))))) + +(defun (main) + (run-app initial gui-update view + #:subscriptions subscriptions + #:title "keyscore" #:width 1500.0 #:height 900.0) + 0) diff --git a/src/layouts.bjo b/src/layouts.bjo new file mode 100644 index 0000000..e397400 --- /dev/null +++ b/src/layouts.bjo @@ -0,0 +1,206 @@ +;; Layouts: which character each key of the board types. +;; +;; - A layout is written as three rows of characters, in the board's order: +;; the top row from Q, the home row from A, the bottom row from <. Spaces +;; are skipped, so a row can be split at the hands. +;; - A row may be shorter than the board's: its last keys type nothing. +;; - Letters are written in either case. ’ is the same character as '. +;; - The shifted character of a punctuation key is typed on that key: +;; , ; . : - _ ' * < >, as on Swedish QWERTY and Svdvorak. + +(import (std prelude)) +(import (keyscore board)) + +(export Layout make-layout aligned-rows layout-key layout-chars layout-base-chars swap-keys common-chars + normalize layouts) + +;; - rows: the three rows as written, for making variants. +;; - keys: the key of every character typed, shifted ones included. +(type (: Layout (Record (: name string) (: rows (Vec string)) (: keys (Map char Key))))) + +;; The shifted character typed on the same key as `c`. +(: (-> char (Option char))) +(defun (shifted c) + (match c + (#\, (Some #\;)) + (#\. (Some #\:)) + (#\- (Some #\_)) + (#\' (Some #\*)) + (#\< (Some #\>)) + (_ None))) + +;; A character as the scorer counts it: lower case, with ’ as '. +(: (-> char char)) +(defun (normalize c) + (if (char=? c (int->char 0x2019)) #\' (char-downcase c))) + +;; The keys of `row` on the board, in column order. +(: (-> int (List Key))) +(defun (row-keys row) + (->list (filter (fun (k) (= (record-ref k row) row)) board))) + +;; A layout named `name` from its three rows. Raises if a row is longer than +;; the board's. +(: (-> string string string string Layout)) +(defun (make-layout name top home bottom) + (def keys + (loop (:for (Tuple row text) (list (Tuple 0 top) (Tuple 1 home) (Tuple 2 bottom))) + (:let chars (->list (filter (fun (c) (not (char=? c #\space))) text))) + (:let ks (row-keys row)) + (:do (when (> (list-length chars) (list-length ks)) + (panic! (str name ": the row " (written text) " has more keys than the board")))) + (:for (Tuple c k) (zip chars ks)) + (:let base (normalize c)) + (:for (Tuple ch key) (match (shifted base) + ((Some s) (list (Tuple base k) (Tuple s k))) + (None (list (Tuple base k))))) + (:acc out (mapping (Tuple ch key))))) + (Layout (name name) (rows [top home bottom]) (keys keys))) + +;; The key that types `c`, if the layout has one. +(: (-> Layout char (Option Key))) +(defun (layout-key l c) (map-try-ref (record-ref l keys) c)) + +;; Every character the layout types. +(: (-> Layout (List char))) +(defun (layout-chars l) (->list (map-keys (record-ref l keys)))) + +;; The characters written in the layout's rows, normalized: one per key. +(: (-> Layout (List char))) +(defun (layout-base-chars l) + (loop (:for r (record-ref l rows)) + (:let row r) + (:for c row) + (:when (not (char=? c #\space))) + (:acc out (listing (normalize c))))) + +;; `l` with the keys of `a` and `b` swapped, named `name`. `a` and `b` are +;; compared with the rows' characters after `normalize`. +(: (-> Layout string char char Layout)) +(defun (swap-keys l name a b) + (: swap (-> char char)) + (def swap (fun (c) + (def n (normalize c)) + (cond ((char=? n a) b) ((char=? n b) a) (else c)))) + (def rows (vec-map (fun (r) (list->string (->list (map swap r)))) (record-ref l rows))) + (make-layout name (vec-ref rows 0) (vec-ref rows 1) (vec-ref rows 2))) + +;; `rows` written with the hands apart, so that the rows line up in a +;; fixed-width font: +;; +;; ÖUOF, QVLBJ +;; IEADY CTRNS +;; ÅÄ.HK Z MGWPX +;; +;; - Top and home rows: five keys, three spaces, the rest. +;; - Bottom row: five keys, the middle key between single spaces, the rest. +;; - Spaces already in `rows` are dropped first. +(: (-> (Vec string) (Vec string))) +(defun (aligned-rows rows) + (: gap (-> int int string)) + (def gap (fun (row i) + (cond ((and (< row 2) (= i 5)) " ") + ((and (= row 2) (or (= i 5) (= i 6))) " ") + (else "")))) + (loop (:for text rows) (:for row (range 0 3)) + (:let chars (->list (filter (fun (c) (not (char=? c #\space))) text))) + (:let parts (loop (:for c chars) (:for i (range 0 (list-length chars))) + (:acc out (vecing (str (gap row i) (char->string c)))))) + (:acc out (vecing (string-join parts ""))))) + +;; The characters every layout in `ls` types. +(: (-> (Vec Layout) (List char))) +(defun (common-chars ls) + (->list (filter (fun (c) (all? (fun (l) (some? (layout-key l c))) ls)) + (layout-chars (vec-ref ls 0))))) + +(: (Vec Layout)) +(def layouts + [(make-layout "QWERTY" + "qwert yuiopå" + "asdfg hjklöä'" + " string string)) +(defun (file-name path) (option-ref-or (path-filename path) path)) + +(: (-> double Block)) +(defun (fixed x) (numeric x #:precision 2)) + +;; A table row: a label, then a block for each of `xs`. +(: (-> string (Vec %a) (-> %a Block) (Vec Block))) +(defun (row label xs f) + (vec-append [(BStr label)] (vec-map f xs))) + +;; A text's row: its name, its size, and how much of it is counted. +(: (-> string Counts Score (Vec Block))) +(defun (text-row t c s) + (def presses (cast double (record-ref s presses))) + (def all (cast double (record-ref c chars))) + [(BStr (file-name t)) (numeric/comma all) (numeric/comma presses) (percent (/ presses all))]) + +;; A layout's row: its score on each text, and the weighted mean. +(: (-> (Vec Corpus) Layout (Vec Score) (Vec Block))) +(defun (score-row cs l s) + (def totals (vec-map (fun (x) (record-ref x total)) s)) + (vec-append (row (record-ref l name) totals fixed) [(fixed (weighted-mean cs totals))])) + +(defun (main args) + (def params (current-params)) + (when (file-exists? params-file) (show ["Parameters from " params-file "." nl])) + (def corpora (if (= (length args) 0) + (record-ref params corpora) + (list->vec (->list (map (fun (a) (corpus a 1.0)) args))))) + (def texts (vec-map (fun (c) (record-ref c path)) corpora)) + (def chars (common-chars layouts)) + (def counts (count-files layouts texts)) + ;; scores[layout][text] + (def scores (vec-map (fun (l) (vec-map (fun (c) (score-layout params l c)) counts)) layouts)) + (def right (fun (n) (vec-append [AlignLeft] (->vec (map (fun (_) AlignRight) (range 0 n)))))) + + ;; --- the texts -------------------------------------------------------------- + (show ["Counted: " (list->string (list-sort chars)) nl nl]) + (show (tabular/rows + (loop (:for t texts) (:for c counts) (:for s (vec-ref scores 0)) (:acc out (vecing (text-row t c s)))) + #:titles ["text" "characters" "counted" "share"] + #:sep " " #:align (right 3))) + (show nl) + + ;; --- the score on each text --------------------------------------------------- + (show ["Score per key press. Lower is better. The mean weighs " (weights-text corpora) "." nl]) + (show (tabular/rows + (loop (:for l layouts) (:for s scores) (:acc out (vecing (score-row corpora l s)))) + #:titles (vec-append (row "layout" texts (fun (t) (BStr (file-name t)))) ["mean"]) + #:sep " " #:align (right (+ (vec-length texts) 1)))) + (show nl) + + ;; --- the parts of the score ------------------------------------------------------- + (: mean-of (-> (-> Score double) (Vec Score) double)) + (def mean-of (fun (f s) (weighted-mean corpora (vec-map f s)))) + (def (: parts (Vec (Vec Block))) + (vec-append + (vec-append + [(row "key costs" scores (fun (s) (fixed (mean-of (fun (x) (record-ref x position)) s)))) + (row "top-row vowels" scores (fun (s) (fixed (mean-of (fun (x) (record-ref x top-vowels)) s))))] + (loop (:for r rules) (:for i (range 0 (vec-length rules))) + (:acc out (vecing (row (record-ref r name) scores + (fun (s) (fixed (mean-of (fun (x) (vec-ref (record-ref x rules) i)) s)))))))) + [(row "hand runs" scores (fun (s) (fixed (mean-of (fun (x) (record-ref x runs)) s)))) + (row "same-finger skips" scores (fun (s) (fixed (mean-of (fun (x) (record-ref x skips)) s)))) + (row "two-finger runs" scores (fun (s) (fixed (mean-of (fun (x) (record-ref x two-fingers)) s)))) + (row "pinky load" scores (fun (s) (fixed (mean-of (fun (x) (record-ref x pinkies)) s)))) + (row "finger distance" scores (fun (s) (fixed (mean-of (fun (x) (record-ref x distance)) s)))) + (row "total" scores (fun (s) (fixed (mean-of (fun (x) (record-ref x total)) s)))) + (row "home row" scores (fun (s) (percent (mean-of (fun (x) (record-ref x home)) s)))) + (row "hand alternation" scores (fun (s) (percent (mean-of (fun (x) (record-ref x alternation)) s)))) + (row "same finger" scores (fun (s) (percent (mean-of (fun (x) (record-ref x same-finger)) s)))) + (row "triples in hand runs" scores (fun (s) (percent (mean-of (fun (x) (record-ref x run-share)) s)))) + (row "same-finger skip triples" scores (fun (s) (percent (mean-of (fun (x) (record-ref x skip-share)) s)))) + (row "two-finger run triples" scores (fun (s) (percent (mean-of (fun (x) (record-ref x two-finger-share)) s)))) + (row "left hand" scores (fun (s) (percent (mean-of (fun (x) (record-ref x left-hand)) s)))) + (row "left / right" scores + (fun (s) (let ((l (mean-of (fun (x) (record-ref x left-hand)) s))) (fixed (/ l (- 1.0 l)))))) + (row "left pinky" scores (fun (s) (percent (mean-of (fun (x) (record-ref x left-pinky)) s)))) + (row "right pinky" scores (fun (s) (percent (mean-of (fun (x) (record-ref x right-pinky)) s))))])) + (show ["The parts, per key press, as the weighted mean." nl]) + (show (tabular/rows parts + #:titles (row "part" layouts (fun (l) (BStr (record-ref l name)))) + #:sep " " #:align (right (vec-length layouts)))) + (show nl) + + ;; --- the pairs that cost most ---------------------------------------------------- + (show ["The pairs that cost each layout most, per key press, as the weighted mean." nl]) + (def weight-sum (vec-fold (fun (c t) (+ t (record-ref c weight))) 0.0 corpora)) + (loop (:for l layouts) + (:let costs (mutablemap)) + (:do (loop (:for c counts) (:for cp corpora) + (:let share (/ (record-ref cp weight) weight-sum)) + (:let pcs (pair-costs params l c)) + (:for (Tuple p x) pcs) + (:do (mutablemap-update! costs p (fun (t) (+ t (* share x))) + #:default (Some 0.0))))) + (:let worst (take 12 (vec-sort-by (fun (a b) (compare (snd b) (snd a))) (->vec costs)))) + (:do (show [(padded/right 10 (record-ref l name)) + (joined (fun (e) [(written (fst e)) " " (fixed (snd e))]) (->list worst) " ") + nl]))) + 0) diff --git a/src/optimize.bjo b/src/optimize.bjo new file mode 100644 index 0000000..55b57d2 --- /dev/null +++ b/src/optimize.bjo @@ -0,0 +1,78 @@ +;; Searches for a better layout by simulated annealing: see (keyscore search). +;; +;; bjo run src/optimize.bjo [runs] [steps] [layout] [t-start] [option ...] +;; +;; - runs: independent searches, each with its own seed. 8 unless given. +;; - steps: swaps tried per run. 200000 unless given. +;; - layout: the layout to start from, by name. The last in (keyscore layouts) +;; unless given, or given as "-". +;; - t-start: the starting temperature. 0.02 unless given: a run refines the +;; start layout rather than replacing it. +;; - options, after t-start, in any order: +;; - vowels-left: the vowels (a e i o u y å ä ö) stay on left-hand keys. The +;; start layout has to have them there. +;; - fixed=CHARS: the keys of CHARS (written as in a layout, any case) do +;; not move. Can be given more than once. +;; +;; Scores with `current-params`: params.txt when there is one. Prints each +;; run's best layout, then the best of all as a make-layout form for +;; (keyscore layouts), scored again by the full scorer. Raises if the +;; search's scorer and the full one disagree on the start layout. + +(import (std prelude)) +(import (only (std fmt) numeric render->string)) +(import (keyscore layouts)) +(import (keyscore params)) +(import (keyscore score)) +(import (keyscore search)) +(import (keyscore texts)) + +(: (-> double string)) +(defun (fixed x) (render->string (numeric x #:precision 3))) + +(defun (main args) + (def arg (fun (i default) (if (> (length args) i) (string->int (ref args i)) default))) + (: named? (-> Layout bool)) + (def named? (fun (l) (= (record-ref l name) (ref args 2)))) + (def start-layout (if (and (> (length args) 2) (not (= (ref args 2) "-"))) + (option-ref (find named? layouts)) + (vec-ref layouts (- (vec-length layouts) 1)))) + (def options (loop (:for i (range 4 (length args))) (:acc out (listing (ref args i))))) + (loop (:for o options) + (:when (not (or (= o "vowels-left") (string-starts-with? o "fixed=")))) + (:do (panic! (str "unknown option " (written o) ": give vowels-left or fixed=CHARS")))) + (def fixed-chars (string-join (list->vec (loop (:for o options) (:when (string-starts-with? o "fixed=")) + (:acc out (listing (string-drop o 6))))) + "")) + (def o (Options (runs (arg 0 8)) + (steps (arg 1 200000)) + (t-start (if (> (length args) 3) (string->double (ref args 3)) + (record-ref default-options t-start))) + (vowels-left (any? (fun (o) (= o "vowels-left")) options)) + (fixed fixed-chars))) + (def params (current-params)) + (when (file-exists? params-file) (println (str "Parameters from " params-file "."))) + (def counts (count-files layouts (vec-map corpus-path (record-ref params corpora)))) + (def p (problem-of params start-layout counts)) + (def fast-start (problem-score p)) + (def full-start (mean-score params start-layout counts)) + (println (str "Starting temperature " (fixed (record-ref o t-start)) ".")) + (println (str (record-ref start-layout name) ": " (fixed full-start) + " by the full scorer, " (fixed fast-start) " by the search's.")) + (when (> (abs (- fast-start full-start)) 0.000001) + (panic! "the search's scorer does not agree with the full one")) + (when (record-ref o vowels-left) (println "Vowels stay on the left hand.")) + (when (not (= fixed-chars "")) (println (str "Fixed: " fixed-chars "."))) + (def results (search p o)) + (loop (:for f results) (:for run (range 1 (+ (vec-length results) 1))) + (:do (println (str "run " (displayed run) ": " (fixed (record-ref f score)) "\n " + (string-join (record-ref f rows) "\n "))))) + (def best (vec-fold (fun (f b) (if (< (record-ref f score) (record-ref b score)) f b)) + (vec-ref results 0) results)) + (def rows (record-ref best rows)) + (def found (make-layout "found" (vec-ref rows 0) (vec-ref rows 1) (vec-ref rows 2))) + (println "") + (println (str "The best, " (fixed (mean-score params found counts)) " by the full scorer:")) + (println (str " (make-layout \"found\"\n \"" (vec-ref rows 0) "\"\n \"" + (vec-ref rows 1) "\"\n \"" (vec-ref rows 2) "\")")) + 0) diff --git a/src/params.bjo b/src/params.bjo new file mode 100644 index 0000000..3787b92 --- /dev/null +++ b/src/params.bjo @@ -0,0 +1,285 @@ +;; The numbers the scorer is tuned with, and the file they are kept in. +;; +;; - `default-params` are the numbers as (keyscore score) describes them. +;; - `param-specs` lists every number by name, with its group and range. The +;; file and the GUI both go through it. +;; - `current-params` reads `params-file` when there is one, so that every +;; program scores with what the GUI saved. +;; +;; The file has one number per line, `name value`, such as `same-finger 50` +;; or `key.q 10`. A line that is empty or starts with ; is skipped. A number +;; it does not name keeps its default. +;; +;; Errors: +;; - `load-params` raises on a line it cannot read, or a name it does not know. + +(import (std prelude)) +(import (keyscore board)) +(import (keyscore texts)) + +(export Params Spec default-params param-specs params-file current-params load-params + save-params key-cost) + +;; The fields, by the rules of (keyscore score): +;; - corpora: the texts and their weights in the mean. +;; - key-weight: what each key cost is multiplied by. key-costs: each key's +;; cost, by index in `board`. +;; - top-vowel: a vowel's press on the top row. +;; - stretch, same-finger, jump, easy-jump, close-keys, adjacent-keys: the +;; pair rules. +;; - spaced: the share of a pair's cost charged when a space is between. +;; - low-share: how much of the roll bonuses a pair with a bottom-row key off +;; the index keys earns, and how much of the hand-run cost a triple of them +;; on one row is spared. 0 none, 1 all. +;; - outward, inward, twist, bottom-index, bottom-other, one-finger: the row +;; motion table. +;; - hand-run, skip, two-finger: the triple rules. +;; - pinky-limit, pinky-cost: the pinky load. +;; - distance: what a key width of finger travel costs. 0: only reported. +(type (: Params (Record (: corpora (Vec Corpus)) + (: key-weight double) + (: key-costs (Vec double)) + (: top-vowel double) + (: stretch double) + (: same-finger double) + (: jump double) + (: easy-jump double) + (: close-keys double) + (: adjacent-keys double) + (: spaced double) + (: low-share double) + (: outward double) + (: inward double) + (: twist double) + (: bottom-index double) + (: bottom-other double) + (: one-finger double) + (: hand-run double) + (: skip double) + (: two-finger double) + (: pinky-limit double) + (: pinky-cost double) + (: distance double)))) + +(: Params) +(def default-params + (Params (corpora default-corpora) + (key-weight 2.0) + (key-costs (vec-map (fun (k) (cast double (record-ref k cost))) board)) + (top-vowel 4.0) + (stretch 5.0) + (same-finger 50.0) + (jump 30.0) + (easy-jump -20.0) + (close-keys -4.0) + (adjacent-keys -3.0) + (spaced 0.5) + (low-share 0.0) + (outward 2.0) + (inward 4.0) + (twist 12.0) + (bottom-index 5.0) + (bottom-other 12.0) + (one-finger 5.0) + (hand-run 10.0) + (skip 10.0) + (two-finger 10.0) + (pinky-limit 0.08) + (pinky-cost 10.0) + (distance 0.0))) + +;; What a press of `k` costs: its cost times the key weight. +(: (-> Params Key double)) +(defun (key-cost p k) + (* (record-ref p key-weight) (vec-ref (record-ref p key-costs) (record-ref k index)))) + +;; One number of `Params`. +;; +;; - name: in the file. label and group: in the GUI. +;; - help: what the number does, for the GUI's info marks. +;; - get and set: read it from, and put it in, a `Params`. +;; - minimum, maximum and step: for the GUI's input. +(type (: Spec (Record (: name string) + (: label string) + (: group string) + (: help string) + (: get (-> Params double)) + (: set (-> Params double Params)) + (: minimum double) + (: maximum double) + (: step double)))) + +(: (-> string string string string (-> Params double) (-> Params double Params) double double double Spec)) +(defun (spec name label group help get set minimum maximum step) + (Spec (name name) (label label) (group group) (help help) (get get) (set set) + (minimum minimum) (maximum maximum) (step step))) + +;; A spec for a cost or bonus from -100 to 100, in steps of 1. +(: (-> string string string string (-> Params double) (-> Params double Params) Spec)) +(defun (cost-spec name label group help get set) (spec name label group help get set -100.0 100.0 1.0)) + +;; The specs of the texts' weights, one for each corpus. +(: (Vec Spec)) +(def corpus-specs + (loop (:for c default-corpora) (:for i (range 0 (vec-length default-corpora))) + (:let file (option-ref-or (path-filename (corpus-path c)) (corpus-path c))) + (:acc out (vecing (spec (str "weight." file) file "Texts" + (str "How much " file " counts in the mean score. The weights are relative: " + "1 and 1 is the same as 2 and 2. 0 removes the text from the mean.") + (fun (p) (corpus-weight (vec-ref (record-ref p corpora) i))) + (fun (p x) + (def cs (record-ref p corpora)) + (record-set p (corpora (vec-set cs i (corpus (corpus-path (vec-ref cs i)) x))))) + 0.0 10.0 0.25))))) + +;; The specs of the key costs, one for each key, named by the key. +(: (Vec Spec)) +(def key-specs + (vec-map (fun (k) + (def i (record-ref k index)) + (def label (string-upcase (record-ref k name))) + (spec (str "key." (record-ref k name)) label "Key costs" + (str "The cost of one press of the key " label " (its name on Swedish QWERTY). " + "It is multiplied by the key weight.") + (fun (p) (vec-ref (record-ref p key-costs) i)) + (fun (p x) (record-set p (key-costs (vec-set (record-ref p key-costs) i x)))) + 0.0 100.0 1.0)) + board)) + +;; Every number of `Params`, in the order the GUI shows them. +(: (Vec Spec)) +(def param-specs + (vec-append + (vec-append + corpus-specs + [(spec "key-weight" "Key weight" "Keys" + "Multiplies every key cost. A press costs its key's cost (the Key costs tab) times this number. 0 ignores the key costs." + (fun (p) (record-ref p key-weight)) + (fun (p x) (record-set p (key-weight x))) 0.0 20.0 0.25) + (cost-spec "top-vowel" "Vowel on the top row" "Keys" + "Added to each press of a vowel (a e i o u y å ä ö) on the top row. This keeps the vowels on the home and bottom rows." + (fun (p) (record-ref p top-vowel)) + (fun (p x) (record-set p (top-vowel x)))) + (cost-spec "stretch" "Stretch key" "Pairs" + "Added to a pair on one hand for each of its keys that is a stretch key: the centre columns (T G B, Y H) and the right pinky's keys past P and Ö." + (fun (p) (record-ref p stretch)) + (fun (p x) (record-set p (stretch x)))) + (cost-spec "same-finger" "Same finger" "Pairs" + "Added when one finger types two different keys one after the other." + (fun (p) (record-ref p same-finger)) + (fun (p x) (record-set p (same-finger x)))) + (cost-spec "jump" "Jump, top to bottom" "Pairs" + "Added to a pair on one hand with one key on the top row and the other on the bottom row." + (fun (p) (record-ref p jump)) + (fun (p x) (record-set p (jump x)))) + (cost-spec "easy-jump" "Easy jump" "Pairs" + "Added to a jump that is easy to type: VE VW NI NO MI MO CE CW, by QWERTY key, in either order. A negative number is a bonus." + (fun (p) (record-ref p easy-jump)) + (fun (p x) (record-set p (easy-jump x)))) + (cost-spec "close-keys" "Close keys" "Pairs" + "Added when two different fingers of one hand type a pair and neither key is a stretch key. A negative number is a bonus." + (fun (p) (record-ref p close-keys)) + (fun (p x) (record-set p (close-keys x)))) + (cost-spec "adjacent-keys" "Adjacent keys" "Pairs" + "Added when two fingers type neighbouring keys of one row. It is added to the close-keys number. A negative number is a bonus." + (fun (p) (record-ref p adjacent-keys)) + (fun (p x) (record-set p (adjacent-keys x)))) + (spec "spaced" "Share across a space" "Pairs" + "The share of a pair's cost that is charged when a space is between the two presses. 0: free. 1: the full cost." + (fun (p) (record-ref p spaced)) + (fun (p x) (record-set p (spaced x))) 0.0 1.0 0.05) + (spec "low-share" "Bottom-row rolls (0 none, 1 as home)" "Pairs" + "The share of the close-keys and adjacent-keys bonuses that a pair gets when it has a bottom-row key off the index keys (not C V N M). It also sets how much of the hand-run cost a triple of such keys on one row avoids. 0: no bonus. 1: the same as on the home row." + (fun (p) (record-ref p low-share)) + (fun (p x) (record-set p (low-share x))) 0.0 1.0 0.05) + (cost-spec "outward" "Home to top, outer finger up" "Row motion" + "A pair on one hand from the home row to the top row, with the outer finger (nearer the pinky) on the top row. The hand keeps its shape." + (fun (p) (record-ref p outward)) + (fun (p x) (record-set p (outward x)))) + (cost-spec "inward" "Top to home, outer finger up" "Row motion" + "A pair on one hand from the top row to the home row, with the outer finger (nearer the pinky) on the top row." + (fun (p) (record-ref p inward)) + (fun (p x) (record-set p (inward x)))) + (cost-spec "twist" "Home and top, inner finger or pinky up" "Row motion" + "A pair on one hand on the home and top rows, in either order, with the inner finger (nearer the thumb) or a pinky on the top row. The hand twists." + (fun (p) (record-ref p twist)) + (fun (p x) (record-set p (twist x)))) + (cost-spec "bottom-index" "Bottom index and home" "Row motion" + "A pair on one hand of a bottom-row index key (C V N M) and a home-row key, in either order." + (fun (p) (record-ref p bottom-index)) + (fun (p x) (record-set p (bottom-index x)))) + (cost-spec "bottom-other" "Other pairs with the bottom row" "Row motion" + "Any other pair on one hand with a key on the bottom row and a key on another row." + (fun (p) (record-ref p bottom-other)) + (fun (p x) (record-set p (bottom-other x)))) + (cost-spec "one-finger" "One finger on two rows" "Row motion" + "The row-motion cost when one finger types keys on two rows. The same-finger cost is added as well." + (fun (p) (record-ref p one-finger)) + (fun (p x) (record-set p (one-finger x)))) + (cost-spec "hand-run" "Hand run" "Triples" + "Added for three presses on one hand with no space between, unless all three keys are on one row." + (fun (p) (record-ref p hand-run)) + (fun (p x) (record-set p (hand-run x)))) + (cost-spec "skip" "Same-finger skip" "Triples" + "Added when the first and third presses of three are two different keys of one finger, such as E, R, D on QWERTY." + (fun (p) (record-ref p skip)) + (fun (p x) (record-set p (skip x)))) + (cost-spec "two-finger" "Two-finger run" "Triples" + "Added for three presses on one hand by only two fingers, such as middle, ring, middle. A double letter is not one." + (fun (p) (record-ref p two-finger)) + (fun (p x) (record-set p (two-finger x)))) + (spec "pinky-limit" "Pinky limit (share of presses)" "Pinky load" + "The share of all presses each pinky can type at no cost. 0.08 is 8%. Spaces are not counted." + (fun (p) (record-ref p pinky-limit)) + (fun (p x) (record-set p (pinky-limit x))) 0.0 0.5 0.01) + (cost-spec "pinky-cost" "Cost past the limit" "Pinky load" + "The cost of each pinky press over the pinky limit." + (fun (p) (record-ref p pinky-cost)) + (fun (p x) (record-set p (pinky-cost x)))) + (spec "distance" "Per key width of travel" "Finger distance" + "The cost of each key width that a finger moves. A finger moves from its home key, or from the key it pressed just before. After a space it starts at home again. 0: the distance is shown but not scored." + (fun (p) (record-ref p distance)) + (fun (p x) (record-set p (distance x))) 0.0 100.0 0.5)]) + key-specs)) + +(: string) +(def params-file "params.txt") + +;; The spec named `name`, if there is one. +(: (-> string (Option Spec))) +(defun (spec-named name) + (: named? (-> Spec bool)) + (def named? (fun (s) (= (record-ref s name) name))) + (find named? param-specs)) + +;; `p` with the numbers of the file's `text`. Raises on a line it cannot read. +(: (-> Params string Params)) +(defun (read-params p text) + (loop (:for raw (string-split text "\n")) + (:let line (string-trim raw)) + (:when (not (or (= line "") (string-starts-with? line ";")))) + (:let fields (vec-filter (fun (f) (not (= f ""))) (string-split line " "))) + (:do (when (not (= (vec-length fields) 2)) + (panic! (str params-file ": cannot read the line " (written line))))) + (:let name (vec-ref fields 0)) + (:let s (match (spec-named name) + ((Some s) s) + (None (panic! (str params-file ": no number is named " (written name)))))) + (:acc out (folding p ((record-ref s set) out (string->double (vec-ref fields 1))))))) + +;; The numbers of the file at `path`, on top of `default-params`. +(: (-> string Params)) +(defun (load-params path) (read-params default-params (file-read-text path))) + +;; The numbers of `params-file` if there is one, else `default-params`. +(: (-> Params)) +(defun (current-params) + (if (file-exists? params-file) (load-params params-file) default-params)) + +;; Writes every number of `p` to the file at `path`. +(: (-> Params string void)) +(defun (save-params p path) + (def lines (vec-map (fun (s) (str (record-ref s name) " " (displayed ((record-ref s get) p)))) + param-specs)) + (file-write-text path (str ";; keyscore's parameters. See src/params.bjo.\n" + (string-join lines "\n") "\n"))) diff --git a/src/saved.bjo b/src/saved.bjo new file mode 100644 index 0000000..9296b1f --- /dev/null +++ b/src/saved.bjo @@ -0,0 +1,89 @@ +;; Presets and saved layouts: what the GUI keeps between runs. +;; +;; - A preset is a set of numbers, kept as presets/NAME.txt in the format of +;; `params-file`. See (keyscore params). +;; - The saved layouts are kept in `layouts-file`, one layout per line: its +;; name and its three rows, separated by tabs. +;; +;; Errors: +;; - `save-preset`, `load-preset`, `delete-preset` and `write-saved-layouts` +;; raise when the file cannot be read or written. +;; - `read-saved-layouts` skips a line it cannot read. + +(import (std prelude)) +(import (keyscore layouts)) +(import (keyscore params)) + +(export presets-folder layouts-file valid-name? preset-names save-preset load-preset delete-preset + read-saved-layouts write-saved-layouts) + +(: string) +(def presets-folder "presets") + +(: string) +(def layouts-file "saved-layouts.txt") + +;; Whether `name` can name a preset or a saved layout: not empty, no +;; surrounding spaces, no tab, line break, / or \, and not starting with a dot. +(: (-> string bool)) +(defun (valid-name? name) + (and (not (= name "")) + (= name (string-trim name)) + (not (string-starts-with? name ".")) + (not (any? (fun (c) (string-contains? name c)) (list "\t" "\n" "\r" "/" "\\"))))) + +(: (-> string string)) +(defun (preset-path name) (path-combine presets-folder (str name ".txt"))) + +;; The names of the presets, sorted. Empty when there is no presets folder. +(: (-> (Vec string))) +(defun (preset-names) + (if (directory-exists? presets-folder) + (vec-sort (loop (:for path (directory-files presets-folder)) + (:let file (option-ref-or (path-filename path) path)) + (:when (string-ends-with? file ".txt")) + (:acc out (vecing (string-drop-right file 4))))) + [])) + +;; Writes `p` as the preset `name`, replacing one of that name. Returns the +;; names of the presets after. +(: (-> string Params (Vec string))) +(defun (save-preset name p) + (when (not (directory-exists? presets-folder)) (directory-create presets-folder)) + (save-params p (preset-path name)) + (preset-names)) + +;; The numbers of the preset `name`. +(: (-> string Params)) +(defun (load-preset name) (load-params (preset-path name))) + +;; Deletes the preset `name`. Returns the names of the presets after. +(: (-> string (Vec string))) +(defun (delete-preset name) + (file-delete (preset-path name)) + (preset-names)) + +;; The layout of one line of `layouts-file`, if the line holds one. +(: (-> string (Option Layout))) +(defun (layout-of-line line) + (def fields (string-split line "\t")) + (if (= (vec-length fields) 4) + (result->option (try (make-layout (vec-ref fields 0) (vec-ref fields 1) (vec-ref fields 2) + (vec-ref fields 3)) + #:catch (System.Exception))) + None)) + +;; The layouts of `layouts-file`, in its order. Empty when there is none. +(: (-> (Vec Layout))) +(defun (read-saved-layouts) + (if (file-exists? layouts-file) + (loop (:for line (string-split (file-read-text layouts-file) "\n")) + (:let found (layout-of-line (string-trim-right line))) + (:acc out (folding [] (match found ((Some l) (vec-add out l)) (None out))))) + [])) + +;; Writes `ls` to `layouts-file`, replacing what it held. +(: (-> (Vec Layout) void)) +(defun (write-saved-layouts ls) + (def lines (vec-map (fun (l) (string-join (vec-append [(record-ref l name)] (record-ref l rows)) "\t")) ls)) + (file-write-text layouts-file (string-join (vec-map (fun (s) (str s "\n")) lines) ""))) diff --git a/src/score.bjo b/src/score.bjo new file mode 100644 index 0000000..5aaf763 --- /dev/null +++ b/src/score.bjo @@ -0,0 +1,471 @@ +;; Scoring a layout on a text, by Michael Capewell's rules. Higher is worse. +;; +;; Every number below is a default: (keyscore params) holds the ones scored +;; with, by the names in brackets. +;; +;; - Each key press costs its key's cost times the key weight, 2 (key-weight, +;; key.q ...). See (keyscore board). +;; - A vowel (a e i o u y å ä ö) typed on the top row costs 4 more per press +;; (top-vowel). Not Capewell's. +;; - Each pair of consecutive presses on one hand, on two different keys, is +;; scored by `rules`. A pair on two hands, or one key pressed twice, scores +;; nothing. +;; - A pair with a space between costs half (spaced), the thumb's press +;; between. A character that is not counted (a digit, a parenthesis, a +;; newline) breaks the text: no pair spans it. +;; +;; The rules, for a pair on one hand: +;; - stretch: +5 for each of the two keys that is a stretch key: before and +;; after the reach (stretch). +;; - row motion: the cost of moving between rows, by direction. Not Capewell's: +;; it replaces his +5 for a row change. See below. +;; - same finger: +50 when one finger types both keys (same-finger). +;; - jump: +30 when one key is on the top row and the other on the bottom +;; (jump). +;; - easy jump: -20 for a jump Capewell finds easy: VE VW NI NO MI MO, either +;; way round, by Swedish QWERTY key (easy-jump). The angle mod puts C under +;; the index too, so CE and CW are easy as well. +;; - close keys: -4 when two fingers type the keys and neither is a stretch +;; (close-keys). +;; - adjacent keys: -3 when two fingers type neighbouring keys of one row +;; (adjacent-keys). A pair can be both close and adjacent. +;; - A pair with a key on the bottom row off its index keys (C V, N M) earns +;; none of the two bonuses: rolling along the bottom row earns nothing. +;; low-share gives it a share of them. Not Capewell's. +;; +;; Row motion, for a pair on one hand: +;; +;; one row 0 +;; home -> top, the outer finger up (outward) 2 +;; top -> home, the outer finger up (inward) 4 +;; home <-> top, the inner finger or a pinky up 12 (twist) +;; bottom index <-> home 5 (bottom-index) +;; any other pair with the bottom row 12 (bottom-other) +;; one finger on two rows 5 (one-finger) +;; +;; - The outer finger is the one nearer the pinky. With it on the top row the +;; hand keeps its shape; with the inner finger up it twists. +;; - 12 is Capewell's row change and the old same-hand cost together. Close +;; keys take 4 back, so a pair costing 4 or more never beats typing it with +;; both hands, and a pair on one row does. +;; +;; Not Capewell's, for three presses with no space between: +;; - hand runs: +10 for three presses in a row on one hand, unless all three +;; are on one row (hand-run). A triple on one row with a bottom-row key off +;; the index keys is a run too, spared by low-share. +;; - same-finger skips: +10 when the first and third keys are two keys of one +;; finger (skip). Typing with both hands makes these common: left, right, +;; left. +;; - two-finger runs: +10 for three presses on one hand by two fingers, such +;; as middle, ring, middle (two-finger). A key pressed twice in a row (a +;; double letter) is not one. +;; +;; The pinky load is not Capewell's either. It is scored over the whole text, +;; for each pinky on its own: +;; - The pinky's share is its presses over all presses (spaces not counted). +;; - Each press past 8% of all presses (pinky-limit) costs 10 (pinky-cost): +;; the cost of a pinky key off the home row. +;; - A pinky at or under the limit costs nothing. +;; +;; Finger distance: how far the fingers travel, in key widths per press. +;; - A finger goes from its home key to the key it presses, unless it pressed +;; the key before: then it goes from that key. A double letter travels 0. +;; - A space or a break between two presses lets the finger go home. +;; - The way back is not counted. +;; - It costs 0 per key width (distance): it is reported, not scored. + +(import (std prelude)) +(import (std mutable map)) +(import (keyscore board)) +(import (keyscore layouts)) +(import (keyscore params)) + +(export Counts count-text count-files Rule rules pair-penalty triple-penalty hand-run + same-finger-skip two-finger-run pinky-penalty top-vowel home-travel pair-travel + Score score-layout pair-costs) + +;; --- counting -------------------------------------------------------------- + +;; What a text holds, for the characters counted. +;; +;; - presses: how often each character is typed. +;; - pairs: how often each pair of characters is typed one after the other. +;; - spaced: how often each pair is typed with spaces between them. +;; - triples: how often each three characters are typed one after the other, +;; with no space between. +;; - chars: every character of the text, counted or not. +(type (: Counts (Record (: presses (MutableMap char int)) + (: pairs (MutableMap (Tuple char char) int)) + (: spaced (MutableMap (Tuple char char) int)) + (: triples (MutableMap (Tuple char char char) int)) + (: chars int)))) + +(: (-> (MutableMap %k int) %k void)) +(defun (count! m k) + (mutablemap-update! m k (fun (n) (+ n 1)) #:default (Some 0))) + +;; The character before, if the run has not been broken, and whether a space +;; came after it. `before` is the character before that one, if there was no +;; space or break between them. +(type (: Prev (Record (: char (Option char)) (: before (Option char)) (: spaced? bool)))) + +;; One character of the text: counted, a space, or a break. +(: (-> Counts (-> char bool) Prev char Prev)) +(defun (step counts counted? prev raw) + (def c (normalize raw)) + (cond + ((counted? c) + (count! (record-ref counts presses) c) + (def spaced? (record-ref prev spaced?)) + (when-let (p (record-ref prev char)) + (count! (if spaced? (record-ref counts spaced) (record-ref counts pairs)) (Tuple p c)) + (when-let (b (record-ref prev before)) + (when (not spaced?) (count! (record-ref counts triples) (Tuple b p c))))) + (Prev (char (Some c)) (before (if spaced? None (record-ref prev char))) (spaced? #f))) + ((or (char=? c #\space) (char=? c (int->char 0xA0))) + (Prev (char (record-ref prev char)) (before None) (spaced? #t))) + (else (Prev (char None) (before None) (spaced? #f))))) + +;; The counts of `text`, for the characters `counted?` holds for. The +;; characters are normalized first: see `normalize`. +(: (-> string (-> char bool) Counts)) +(defun (count-text text counted?) + (def counts (Counts (presses (mutablemap)) (pairs (mutablemap)) (spaced (mutablemap)) + (triples (mutablemap)) (chars (string-length text)))) + (ignore (loop (:for c text) + (:acc prev (folding (Prev (char None) (before None) (spaced? #f)) + (step counts counted? prev c))))) + counts) + +;; The counts of each file in `paths`, for the characters every layout in +;; `ls` types. +(: (-> (Vec Layout) (Vec string) (Vec Counts))) +(defun (count-files ls paths) + (def counted (->map (map (fun (c) (Tuple c #t)) (common-chars ls)))) + (vec-map (fun (p) (count-text (file-read-text p) (fun (c) (map-contains? counted c)))) paths)) + +;; --- the pinky load ---------------------------------------------------------- + +;; The pinky load's penalty per press, for a pinky with `share` of the presses. +(: (-> Params double double)) +(defun (pinky-penalty p share) + (* (record-ref p pinky-cost) (max 0.0 (- share (record-ref p pinky-limit))))) + +;; --- vowels on the top row ---------------------------------------------------- + +;; What typing `c` on `k` costs besides the key's cost: top-vowel for a vowel +;; on the top row, else 0. `c` is normalized. +(: (-> Params char Key double)) +(defun (top-vowel p c k) + (if (and (= (record-ref k row) 0) (any? (fun (v) (char=? v c)) (->list "aeiouyåäö"))) + (record-ref p top-vowel) + 0.0)) + +;; --- finger distance ----------------------------------------------------------- + +;; How far `k`'s finger travels from its home key to press it. +(: (-> Key double)) +(defun (home-travel k) (key-distance (home-key k) k)) + +;; How much farther `b`'s finger travels pressed right after `a` than from +;; home: from `a` instead of from home when one finger types both, else 0. +(: (-> Key Key double)) +(defun (pair-travel a b) + (if (and (= (record-ref a left?) (record-ref b left?)) (= (record-ref a finger) (record-ref b finger))) + (- (key-distance a b) (home-travel b)) + 0.0)) + +;; --- the rules --------------------------------------------------------------- + +(type (: Rule (Record (: name string) (: penalty (-> Params Key Key double))))) + +(: (-> bool double)) +(defun (one-if b) (if b 1.0 0.0)) + +(: (-> Key Key bool)) +(defun (two-fingers? a b) (not (= (record-ref a finger) (record-ref b finger)))) + +(: (-> Key Key bool)) +(defun (no-stretch? a b) (not (or (record-ref a stretch?) (record-ref b stretch?)))) + +(: (-> Key Key bool)) +(defun (jump? a b) + (and (= (+ (record-ref a row) (record-ref b row)) 2) + (not (= (record-ref a row) 1)))) + +;; The jumps Capewell finds easy, as Swedish QWERTY key names: the index on +;; the bottom row and the middle or ring finger on the top row. +(def easy-jumps (list "ve" "vw" "ce" "cw" "ni" "no" "mi" "mo")) + +(: (-> Key Key bool)) +(defun (easy-jump? a b) + (def ab (str (record-ref a name) (record-ref b name))) + (def ba (str (record-ref b name) (record-ref a name))) + (any? (fun (p) (or (= p ab) (= p ba))) easy-jumps)) + +;; Whether `k` is on the bottom row and not one of its index keys: C V and +;; N M. B is a stretch key and counts as off the index keys. +(: (-> Key bool)) +(defun (low? k) + (and (= (record-ref k row) 2) + (or (not (= (record-ref k finger) 3)) (record-ref k stretch?)))) + +;; The share of the roll bonuses a pair earns: all of them, or low-share +;; when a key is `low?`. +(: (-> Params Key Key double)) +(defun (roll-share p a b) + (if (or (low? a) (low? b)) (record-ref p low-share) 1.0)) + +;; The rules' penalties, for a pair on one hand. + +(: (-> Params Key Key double)) +(defun (stretch p a b) + (* (record-ref p stretch) (+ (one-if (record-ref a stretch?)) (one-if (record-ref b stretch?))))) + +(: (-> Params Key Key double)) +(defun (same-finger p a b) (* (record-ref p same-finger) (one-if (not (two-fingers? a b))))) + +(: (-> Params Key Key double)) +(defun (jump p a b) (* (record-ref p jump) (one-if (jump? a b)))) + +(: (-> Params Key Key double)) +(defun (easy-jump p a b) (* (record-ref p easy-jump) (one-if (and (jump? a b) (easy-jump? a b))))) + +(: (-> Params Key Key double)) +(defun (close-keys p a b) + (* (record-ref p close-keys) (roll-share p a b) + (one-if (and (two-fingers? a b) (no-stretch? a b))))) + +(: (-> Key bool)) +(defun (pinky? k) (= (record-ref k finger) 0)) + +(: (-> Key bool)) +(defun (index? k) (= (record-ref k finger) 3)) + +;; The row-motion cost of `a` then `b`, by the table in the header. +(: (-> Params Key Key double)) +(defun (row-motion p a b) + (def ra (record-ref a row)) + (def rb (record-ref b row)) + (cond ((= ra rb) 0.0) + ((not (two-fingers? a b)) (record-ref p one-finger)) + ((or (= ra 2) (= rb 2)) + (if (or (and (= ra 2) (= rb 1) (index? a)) (and (= rb 2) (= ra 1) (index? b))) + (record-ref p bottom-index) + (record-ref p bottom-other))) + (else (let ((top (if (= ra 0) a b)) + (home (if (= ra 0) b a))) + (cond ((pinky? top) (record-ref p twist)) + ((< (record-ref top finger) (record-ref home finger)) + (if (= ra 1) (record-ref p outward) (record-ref p inward))) + (else (record-ref p twist))))))) + +(: (-> Params Key Key double)) +(defun (adjacent-keys p a b) + (* (record-ref p adjacent-keys) (roll-share p a b) + (one-if (and (two-fingers? a b) + (= (record-ref a row) (record-ref b row)) + (= (abs (- (record-ref a column) (record-ref b column))) 1))))) + +(: (Vec Rule)) +(def rules + [(Rule (name "stretch") (penalty stretch)) + (Rule (name "row motion") (penalty row-motion)) + (Rule (name "same finger") (penalty same-finger)) + (Rule (name "jump") (penalty jump)) + (Rule (name "easy jump") (penalty easy-jump)) + (Rule (name "close keys") (penalty close-keys)) + (Rule (name "adjacent keys") (penalty adjacent-keys))]) + +;; Whether the rules score the pair at all: one hand, two keys. +(: (-> Key Key bool)) +(defun (scored-pair? a b) + (and (= (record-ref a left?) (record-ref b left?)) + (not (= (record-ref a name) (record-ref b name))))) + +;; What the rules charge for pressing `a` and then `b`: nothing for two hands +;; or one key. +(: (-> Params Key Key double)) +(defun (pair-penalty p a b) + (if (scored-pair? a b) + (vec-fold (fun (r t) (+ t ((record-ref r penalty) p a b))) 0.0 rules) + 0.0)) + +;; The hand-run rule for pressing `a`, `b` and then `c`: hand-run when all +;; three are on one hand. On one row it is nothing, or the share of it +;; low-share does not spare when a key is `low?`. +(: (-> Params Key Key Key double)) +(defun (hand-run p a b c) + (def left? (record-ref a left?)) + (def row (record-ref a row)) + (cond ((not (and (= (record-ref b left?) left?) (= (record-ref c left?) left?))) 0.0) + ((not (and (= (record-ref b row) row) (= (record-ref c row) row))) (record-ref p hand-run)) + ((or (low? a) (low? b) (low? c)) (* (- 1.0 (record-ref p low-share)) (record-ref p hand-run))) + (else 0.0))) + +;; The same-finger-skip rule for `a`, `b` and then `c`: skip when one finger +;; types `a` and `c` and they are two keys. +(: (-> Params Key Key Key double)) +(defun (same-finger-skip p a b c) + (if (and (= (record-ref a left?) (record-ref c left?)) + (= (record-ref a finger) (record-ref c finger)) + (not (= (record-ref a name) (record-ref c name)))) + (record-ref p skip) + 0.0)) + +;; The two-finger-run rule for `a`, `b` and then `c`: two-finger when all +;; three are on one hand, typed by two fingers, and no key is pressed twice +;; in a row. +(: (-> Params Key Key Key double)) +(defun (two-finger-run p a b c) + (def left? (record-ref a left?)) + (def fa (record-ref a finger)) + (def fb (record-ref b finger)) + (def fc (record-ref c finger)) + (if (and (= (record-ref b left?) left?) (= (record-ref c left?) left?) + (not (= (record-ref a name) (record-ref b name))) + (not (= (record-ref b name) (record-ref c name))) + (not (and (= fa fb) (= fb fc))) + (or (= fa fb) (= fb fc) (= fa fc))) + (record-ref p two-finger) + 0.0)) + +;; What the triple rules charge for `a`, `b` and then `c`. +(: (-> Params Key Key Key double)) +(defun (triple-penalty p a b c) + (+ (hand-run p a b c) (same-finger-skip p a b c) (two-finger-run p a b c))) + +;; --- scoring ------------------------------------------------------------------- + +;; A layout's score on one text. Every total is per key press. +;; +;; - position: the keys' costs. +;; - top-vowels: the cost of vowels on the top row. +;; - rules: each rule's share, in the order of `rules`. +;; - runs: the hand-run rule's share. skips: the same-finger-skip rule's. +;; two-fingers: the two-finger-run rule's. +;; - pinkies: the pinky load's penalty, both pinkies together. +;; - distance: the finger distance, in key widths. Its cost is distance times +;; the distance parameter. +;; - total: position, top-vowels, the rules, the triple rules, the pinky load +;; and the distance's cost together. Lower is better. +;; - home: the share of presses on the home row. +;; - alternation: the share of pairs (with no space between) typed on two hands. +;; - same-finger: the share of pairs typed by one finger on two keys. +;; - run-share, skip-share, two-finger-share: the share of triples each triple +;; rule charges. +;; - left-hand: the left hand's share of the presses. +;; - left-pinky, right-pinky: each pinky's share of the presses. +(type (: Score (Record (: presses int) + (: position double) + (: top-vowels double) + (: rules (Vec double)) + (: runs double) + (: skips double) + (: two-fingers double) + (: pinkies double) + (: distance double) + (: total double) + (: home double) + (: alternation double) + (: same-finger double) + (: run-share double) + (: skip-share double) + (: two-finger-share double) + (: left-hand double) + (: left-pinky double) + (: right-pinky double)))) + +;; The sum of `f` over the pairs of `m`, each weighted by its count. +(: (-> (MutableMap (Tuple char char) int) (-> char char double) double)) +(defun (pair-sum m f) + (loop (:for (Tuple pair n) m) + (:let (Tuple a b) pair) + (:acc t (summing (* (cast double n) (f a b)))))) + +;; `layout` scored on `counts` with `p`. Every character counted must be on +;; the layout. +(: (-> Params Layout Counts Score)) +(defun (score-layout p layout counts) + (def (: key (-> char Key)) (fun (c) (option-ref (layout-key layout c)))) + (def presses (record-ref counts presses)) + (def pairs (record-ref counts pairs)) + (def spaced (record-ref counts spaced)) + (def n (loop (:for (Tuple _ k) presses) (:acc t (summing k)))) + (def per-press (fun (x) (/ x (cast double n)))) + (: press-sum (-> (-> char Key double) double)) + (def press-sum + (fun (f) (loop (:for (Tuple c k) presses) + (:acc t (summing (* (cast double k) (f c (key c)))))))) + (def position (per-press (press-sum (fun (c k) (key-cost p k))))) + (def top-vowels (per-press (press-sum (fun (c k) (top-vowel p c k))))) + (def rule-totals + (->vec (map (fun (r) + (def f (fun (a b) + (def ka (key a)) + (def kb (key b)) + (if (scored-pair? ka kb) + ((record-ref r penalty) p ka kb) + 0.0))) + (per-press (+ (pair-sum pairs f) (* (record-ref p spaced) (pair-sum spaced f))))) + rules))) + (: press-share (-> (-> Key bool) double)) + (def press-share + (fun (f) (/ (loop (:for (Tuple c k) presses) + (:acc t (summing (if (f (key c)) (cast double k) 0.0)))) + (cast double n)))) + (def left-pinky (press-share (fun (k) (and (record-ref k left?) (= (record-ref k finger) 0))))) + (def right-pinky (press-share (fun (k) (and (not (record-ref k left?)) (= (record-ref k finger) 0))))) + (def pinkies (+ (pinky-penalty p left-pinky) (pinky-penalty p right-pinky))) + (def distance + (per-press (+ (press-sum (fun (c k) (home-travel k))) + (pair-sum pairs (fun (a b) (pair-travel (key a) (key b))))))) + (: triple-sum (-> (-> Key Key Key double) double)) + (def triple-sum + (fun (f) (loop (:for (Tuple t k) (record-ref counts triples)) + (:let (Tuple a b c) t) + (:acc s (summing (* (cast double k) (f (key a) (key b) (key c)))))))) + (def runs (per-press (triple-sum (fun (a b c) (hand-run p a b c))))) + (def skips (per-press (triple-sum (fun (a b c) (same-finger-skip p a b c))))) + (def two-fingers (per-press (triple-sum (fun (a b c) (two-finger-run p a b c))))) + (def triple-count (triple-sum (fun (a b c) 1.0))) + (def pair-count (pair-sum pairs (fun (a b) 1.0))) + (def share + (fun (f) (/ (pair-sum pairs (fun (a b) (if (f (key a) (key b)) 1.0 0.0))) pair-count))) + (Score (presses n) + (position position) + (top-vowels top-vowels) + (rules rule-totals) + (runs runs) + (skips skips) + (two-fingers two-fingers) + (pinkies pinkies) + (distance distance) + (total (+ position top-vowels (vec-fold (fun (x y) (+ x y)) 0.0 rule-totals) runs skips + two-fingers pinkies (* (record-ref p distance) distance))) + (home (press-share (fun (k) (= (record-ref k row) 1)))) + (left-pinky left-pinky) + (right-pinky right-pinky) + (alternation (share (fun (a b) (not (= (record-ref a left?) (record-ref b left?)))))) + (same-finger (share (fun (a b) (and (scored-pair? a b) (not (two-fingers? a b)))))) + (run-share (/ (triple-sum (fun (a b c) (if (> (hand-run p a b c) 0.0) 1.0 0.0))) triple-count)) + (skip-share (/ (triple-sum (fun (a b c) (if (> (same-finger-skip p a b c) 0.0) 1.0 0.0))) triple-count)) + (two-finger-share (/ (triple-sum (fun (a b c) (if (> (two-finger-run p a b c) 0.0) 1.0 0.0))) triple-count)) + (left-hand (press-share (fun (k) (record-ref k left?)))))) + +;; The score of each pair, per key press: what the pair costs `layout` on +;; `counts`, by every rule. A pair with spaces between is written "a b". +(: (-> Params Layout Counts (List (Tuple string double)))) +(defun (pair-costs p layout counts) + (def (: key (-> char Key)) (fun (c) (option-ref (layout-key layout c)))) + (def n (cast double (loop (:for (Tuple _ k) (record-ref counts presses)) (:acc t (summing k))))) + (: cost (-> char char double)) + (def cost (fun (a b) (pair-penalty p (key a) (key b)))) + (: each (-> (MutableMap (Tuple char char) int) string double (List (Tuple string double)))) + (def each (fun (m between weight) + (loop (:for (Tuple pair k) m) + (:let (Tuple a b) pair) + (:acc out (listing (Tuple (str (char->string a) between (char->string b)) + (/ (* weight (cast double k) (cost a b)) n))))))) + (list-append (each (record-ref counts pairs) "" 1.0) + (each (record-ref counts spaced) " " (record-ref p spaced)))) diff --git a/src/search.bjo b/src/search.bjo new file mode 100644 index 0000000..6e7a9ca --- /dev/null +++ b/src/search.bjo @@ -0,0 +1,350 @@ +;; Searching for a better layout by simulated annealing. +;; +;; - A unit is a key's character together with its shifted one (, and ;). +;; A step swaps the keys of two units. The layout's set of keys stays. +;; - The score is (keyscore score)'s with the given `Params`: the weighted +;; mean over the texts that weigh something, lower being better. +;; - A worse layout is taken with the probability exp(-d / T), where d is how +;; much worse it is. T falls geometrically from t-start to `t-end`. A low +;; t-start refines the start layout rather than replacing it. +;; - The runs run in parallel, on the compute threads of (std parallel). +;; +;; The scorer here is the full one precomputed: the cost of every unit on +;; every key, the penalty of every pair and triple of keys, and the presses, +;; pairs and triples of every unit. `check-problem` compares the two. +;; +;; Errors: +;; - `search` raises when vowels-left is asked for and the start layout has +;; a vowel on the right hand. + +(import (std prelude)) +(import (std maths)) +(import (std random)) +(import (std mutable map)) +(import (std parallel)) +(import (keyscore board)) +(import (keyscore layouts)) +(import (keyscore params)) +(import (keyscore score)) +(import (keyscore texts)) + +(export Options Found default-options t-end Problem Text problem-of problem-score + mean-score search) + +(: double) +(def t-end 0.001) + +;; How to search. +;; +;; - runs: independent searches, each with its own seed. +;; - steps: swaps tried per run. +;; - t-start: the starting temperature. +;; - vowels-left: the vowels (a e i o u y å ä ö) stay on left-hand keys: a +;; swap that would move one to the right hand is not tried. +;; - fixed: the characters whose keys do not move, written as in a layout. +(type (: Options (Record (: runs int) + (: steps int) + (: t-start double) + (: vowels-left bool) + (: fixed string)))) + +(: Options) +(def default-options + (Options (runs 8) (steps 200000) (t-start 0.02) (vowels-left #f) (fixed ""))) + +;; A run's best layout: its score by the search's scorer, and its rows, as +;; `aligned-rows` writes them. +(type (: Found (Record (: score double) (: rows (Vec string))))) + +;; A text, by unit: its weight in the mean, presses, the pairs typed with +;; their weights (a pair with spaces between weighs `spaced`) and their +;; counts with no space between, and the triples with their counts. +(type (: Text (Record (: share double) + (: presses double) + (: units (Array double)) + (: first (Array int)) + (: second (Array int)) + (: weight (Array double)) + (: plain (Array double)) + (: triple-a (Array int)) + (: triple-b (Array int)) + (: triple-c (Array int)) + (: triple-count (Array double))))) + +;; What the search needs. +;; +;; - params: what it scores with. +;; - layout: the start layout. chars: each unit's character, as its rows +;; write it. start: each unit's key, by index in `board`, at the start. +;; - costs: what a press of each unit costs on each key, at [u * keys + k]: +;; the key's cost, `top-vowel` and the travel from home. +;; - penalties: the penalty of each pair of keys, at [a * keys + b]. +;; travels: the distance cost of each pair with no space between, by +;; `pair-travel`. triple-penalties: of each triple, at +;; [(a * keys + b) * keys + c]. +;; - left-pinky, right-pinky: whether each key is typed by that pinky. +;; - left-keys: whether each key is on the left hand. +(type (: Problem (Record (: params Params) + (: layout Layout) + (: chars (Vec char)) + (: start (Array int)) + (: keys int) + (: costs (Array double)) + (: penalties (Array double)) + (: travels (Array double)) + (: triple-penalties (Array double)) + (: left-pinky (Array bool)) + (: right-pinky (Array bool)) + (: left-keys (Array bool)) + (: texts (Vec Text))))) + +(: (-> Layout char int)) +(defun (key-of l c) (record-ref (option-ref (layout-key l c)) index)) + +;; `counts` by unit. `unit-of` is the unit of each character counted. +(: (-> Params Counts double (Map char int) int Text)) +(defun (text-of params counts share unit-of units) + (def (: presses (Array double)) (make-array units #:fill 0.0)) + (loop (:for (Tuple c k) (record-ref counts presses)) + (:let u (map-ref unit-of c)) + (:do (array-set! presses u (+ (array-ref presses u) (cast double k))))) + ;; Each pair of units: its weight, and its count with no space between. + (def (: pairs (MutableMap (Tuple int int) (Tuple double double))) (mutablemap)) + (: add! (-> (MutableMap (Tuple char char) int) double double void)) + (def add! (fun (m weight plain) + (loop (:for (Tuple pair k) m) + (:let (Tuple a b) pair) + (:let n (cast double k)) + (:do (mutablemap-update! pairs (Tuple (map-ref unit-of a) (map-ref unit-of b)) + (fun (w) (Tuple (+ (fst w) (* weight n)) (+ (snd w) (* plain n)))) + #:default (Some (Tuple 0.0 0.0))))))) + (add! (record-ref counts pairs) 1.0 1.0) + (add! (record-ref counts spaced) (record-ref params spaced) 0.0) + (def entries (->vec pairs)) + (def (: triples (MutableMap (Tuple int int int) double)) (mutablemap)) + (loop (:for (Tuple t k) (record-ref counts triples)) + (:let (Tuple a b c) t) + (:do (mutablemap-update! triples (Tuple (map-ref unit-of a) (map-ref unit-of b) (map-ref unit-of c)) + (fun (w) (+ w (cast double k))) + #:default (Some 0.0)))) + (def (: tris (Vec (Tuple (Tuple int int int) double))) (->vec triples)) + (: tri-unit (-> int (Array int))) + (def tri-unit (fun (i) (vec->array (vec-map (fun (e) (def (Tuple a b c) (fst e)) + (if (= i 0) a (if (= i 1) b c))) + tris)))) + (Text (share share) + (presses (loop (:for i (range 0 units)) (:acc t (summing (array-ref presses i))))) + (units presses) + (first (vec->array (vec-map (fun (e) (fst (fst e))) entries))) + (second (vec->array (vec-map (fun (e) (snd (fst e))) entries))) + (weight (vec->array (vec-map (fun (e) (fst (snd e))) entries))) + (plain (vec->array (vec-map (fun (e) (snd (snd e))) entries))) + (triple-a (tri-unit 0)) + (triple-b (tri-unit 1)) + (triple-c (tri-unit 2)) + (triple-count (vec->array (vec-map (fun (e) (snd e)) tris))))) + +;; An array of `n` * `n` doubles, `f` of the keys a and b at [a * n + b]. +(: (-> (-> Key Key double) (Array double))) +(defun (pair-table f) + (def n (vec-length board)) + (def (: out (Array double)) (make-array (* n n) #:fill 0.0)) + (loop (:for a board) (:for i (range 0 n)) + (:let row-start (* i n)) + (:for b board) (:for j (range 0 n)) + (:do (array-set! out (+ row-start j) (f a b)))) + out) + +;; The problem of improving `l` with `params`. `counts` are the counts of the +;; texts of `params`, in their order; those that weigh nothing are left out. +(: (-> Params Layout (Vec Counts) Problem)) +(defun (problem-of params l counts) + (def chars (list->vec (layout-base-chars l))) + (def units (vec-length chars)) + ;; Each key the layout uses holds one unit; every character counted is on + ;; one of those keys. + (def unit-on-key (loop (:for c chars) (:for u (range 0 units)) + (:acc out (mapping (Tuple (key-of l c) u))))) + (def unit-of (loop (:for c (common-chars layouts)) + (:acc out (mapping (Tuple c (map-ref unit-on-key (key-of l c))))))) + (def n (vec-length board)) + (def distance (record-ref params distance)) + (def (: costs (Array double)) (make-array (* units n) #:fill 0.0)) + (loop (:for c chars) (:for u (range 0 units)) + (:let row-start (* u n)) + (:let base (normalize c)) + (:for k board) (:for j (range 0 n)) + (:do (array-set! costs (+ row-start j) + (+ (key-cost params k) (top-vowel params base k) (* distance (home-travel k)))))) + (def (: triple-penalties (Array double)) (make-array (* n n n) #:fill 0.0)) + (loop (:for a board) (:for i (range 0 n)) + (:let ab-start i) + (:for b board) (:for j (range 0 n)) + (:let c-start (* (+ (* ab-start n) j) n)) + (:for c board) (:for k (range 0 n)) + (:do (array-set! triple-penalties (+ c-start k) (triple-penalty params a b c)))) + (: pinky? (-> bool Key bool)) + (def pinky? (fun (left k) (and (= (record-ref k finger) 0) (= (record-ref k left?) left)))) + (def corpora (record-ref params corpora)) + (def total (vec-fold (fun (c t) (+ t (corpus-weight c))) 0.0 corpora)) + (Problem (params params) + (layout l) + (chars chars) + (start (vec->array (vec-map (fun (c) (key-of l c)) chars))) + (keys n) + (costs costs) + (penalties (pair-table (fun (a b) (pair-penalty params a b)))) + (travels (pair-table (fun (a b) (* distance (pair-travel a b))))) + (triple-penalties triple-penalties) + (left-pinky (vec->array (vec-map (fun (k) (pinky? #t k)) board))) + (right-pinky (vec->array (vec-map (fun (k) (pinky? #f k)) board))) + (left-keys (vec->array (vec-map (fun (k) (record-ref k left?)) board))) + (texts (loop (:for cs counts) (:for c corpora) + (:when (> (corpus-weight c) 0.0)) + (:acc out (vecing (text-of params cs (/ (corpus-weight c) total) unit-of units))))))) + +;; The score of the layout that puts unit u on key (array-ref on u). +(: (-> Problem (Array int) double)) +(defun (score p on) + (def units (array-length on)) + (def n (record-ref p keys)) + (def costs (record-ref p costs)) + (def penalties (record-ref p penalties)) + (def travels (record-ref p travels)) + (def tpen (record-ref p triple-penalties)) + (def left (record-ref p left-pinky)) + (def right (record-ref p right-pinky)) + (def params (record-ref p params)) + (: one (-> Text double)) + (def one + (fun (t) + (def presses (record-ref t units)) + (def first (record-ref t first)) + (def second (record-ref t second)) + (def weight (record-ref t weight)) + (def plain (record-ref t plain)) + (def total (record-ref t presses)) + (def keys-cost + (loop (:for u (range 0 units)) + (:acc s (summing (* (array-ref presses u) (array-ref costs (+ (* u n) (array-ref on u)))))))) + (def pairs-cost + (loop (:for i (range 0 (array-length first))) + (:let at (+ (* (array-ref on (array-ref first i)) n) (array-ref on (array-ref second i)))) + (:acc s (summing (+ (* (array-ref weight i) (array-ref penalties at)) + (* (array-ref plain i) (array-ref travels at))))))) + (def ta (record-ref t triple-a)) + (def tb (record-ref t triple-b)) + (def tc (record-ref t triple-c)) + (def tn (record-ref t triple-count)) + (def triples-cost + (loop (:for i (range 0 (array-length ta))) + (:acc s (summing (* (array-ref tn i) + (array-ref tpen (+ (* (+ (* (array-ref on (array-ref ta i)) n) + (array-ref on (array-ref tb i))) + n) + (array-ref on (array-ref tc i))))))))) + (: share (-> (Array bool) double)) + (def share (fun (pinky) + (/ (loop (:for u (range 0 units)) + (:acc s (summing (if (array-ref pinky (array-ref on u)) (array-ref presses u) 0.0)))) + total))) + (+ (/ (+ keys-cost pairs-cost triples-cost) total) + (pinky-penalty params (share left)) (pinky-penalty params (share right))))) + (def ts (record-ref p texts)) + (vec-fold (fun (t s) (+ s (* (record-ref t share) (one t)))) 0.0 ts)) + +;; The start layout's score by the search's scorer. +(: (-> Problem double)) +(defun (problem-score p) (score p (record-ref p start))) + +;; `l`'s score by the full scorer with `params`: the weighted mean of its +;; totals on `counts`, the counts of the texts of `params` in their order. +(: (-> Params Layout (Vec Counts) double)) +(defun (mean-score params l counts) + (weighted-mean (record-ref params corpora) + (vec-map (fun (c) (record-ref (score-layout params l c) total)) counts))) + +(: (-> (Array int) (Array int))) +(defun (copy a) + (def (: out (Array int)) (make-array (array-length a) #:fill 0)) + (array-copy! a 0 out 0 (array-length a)) + out) + +;; The units whose character is in `cs`, compared after `normalize`. +(: (-> Problem string (Array bool))) +(defun (units-in p cs) + (def wanted (->list (map normalize cs))) + (vec->array (vec-map (fun (c) (any? (fun (f) (char=? f (normalize c))) wanted)) + (record-ref p chars)))) + +;; One run: the best layout it found, and its score. A unit marked in +;; `locked` never leaves the left hand; one marked in `fixed` never moves. +(: (-> Problem int int double (Array bool) (Array bool) (Tuple double (Array int)))) +(defun (anneal p seed steps t-start locked fixed) + (def left (record-ref p left-keys)) + (def rng (make-random/seed64 (cast ulong seed))) + (def on (copy (record-ref p start))) + (def units (array-length on)) + (def current (make-box (score p on))) + (def best (make-box (Tuple (box-ref current) (copy on)))) + (def cooling (/ (loge (/ t-end t-start)) (cast double steps))) + (loop (:for i (range 0 steps)) + (:let temperature (* t-start (exp (* cooling (cast double i))))) + (:let u (random-int 0 (- units 1) #:rng rng)) + (:let v (random-int 0 (- units 2) #:rng rng)) + (:let w (if (>= v u) (+ v 1) v)) + (:when (not (or (array-ref fixed u) (array-ref fixed w) + (and (array-ref locked u) (not (array-ref left (array-ref on w)))) + (and (array-ref locked w) (not (array-ref left (array-ref on u))))))) + (:do (def ku (array-ref on u)) + (array-set! on u (array-ref on w)) + (array-set! on w ku) + (def s (score p on)) + (def d (- s (box-ref current))) + (if (or (<= d 0.0) (< (random-double #:rng rng) (exp (- (/ d temperature))))) + (begin + (box-set! current s) + (when (< s (fst (box-ref best))) + (box-set! best (Tuple s (copy on))))) + (begin + (array-set! on w (array-ref on u)) + (array-set! on u ku))))) + (box-ref best)) + +;; The rows of the start layout, with each unit on the key `on` gives it, as +;; `aligned-rows` writes them. Letters are upper case and ' is ’, as layouts +;; are written. +(: (-> Problem (Array int) (Vec string))) +(defun (rows-of p on) + (def chars (record-ref p chars)) + (def (: on-key (Array char)) (make-array (record-ref p keys) #:fill #\space)) + (loop (:for u (range 0 (array-length on))) + (:do (array-set! on-key (array-ref on u) (vec-ref chars u)))) + (def start (record-ref p start)) + (def unit-of (loop (:for c chars) (:for u (range 0 (vec-length chars))) + (:acc out (mapping (Tuple c u))))) + (: shown (-> char char)) + (def shown (fun (c) + (if (char=? c #\space) + c + (let ((x (array-ref on-key (array-ref start (map-ref unit-of (normalize c)))))) + (if (char=? x #\') (int->char 0x2019) (char-upcase x)))))) + (aligned-rows (vec-map (fun (r) (list->string (->list (map shown r)))) + (record-ref (record-ref p layout) rows)))) + +;; The best layout of each of the runs `o` asks for, from the start layout of +;; `p`, in run order. Raises when vowels-left is asked for and a vowel of the +;; start layout is on the right hand. +(: (-> Problem Options (Vec Found))) +(defun (search p o) + (def locked (units-in p (if (record-ref o vowels-left) "aeiouyåäö" ""))) + (def fixed (units-in p (record-ref o fixed))) + (when (loop (:for u (range 0 (array-length locked))) + (:acc any (oring (and (array-ref locked u) + (not (array-ref (record-ref p left-keys) + (array-ref (record-ref p start) u))))))) + (panic! "vowels-left: the start layout has a vowel on the right hand")) + (def found (computing-each (record-ref o runs) + (fun (k) (anneal p (+ k 1) (record-ref o steps) (record-ref o t-start) + locked fixed)))) + (vec-map (fun (f) (Found (score (fst f)) (rows (rows-of p (snd f))))) found)) diff --git a/src/swaps.bjo b/src/swaps.bjo new file mode 100644 index 0000000..07f7a8c --- /dev/null +++ b/src/swaps.bjo @@ -0,0 +1,67 @@ +;; Tries swapping one key of a layout with each of its other keys. +;; +;; bjo run src/swaps.bjo [char] [layout] +;; +;; - char: the character to move. i unless given. +;; - layout: a layout's name in (keyscore layouts). The last one unless given. +;; - Scores each swap on the texts of (keyscore texts) and prints them best +;; first, by the weighted mean, the layout as it is among them. Lower is +;; better. + +(import (std prelude)) +(import (std fmt)) +(import (keyscore layouts)) +(import (keyscore params)) +(import (keyscore score)) +(import (keyscore texts)) + +;; The numbers scored with: params.txt when there is one. +(: Params) +(def params (current-params)) + +;; A layout's score on each text. +(: (-> (Vec Counts) Layout (Vec double))) +(defun (totals counts l) (vec-map (fun (c) (record-ref (score-layout params l c) total)) counts)) + +(: (-> (Vec double) double)) +(defun (mean xs) (weighted-mean (record-ref params corpora) xs)) + + +(: (-> double Block)) +(defun (fixed x) (numeric x #:precision 2)) + +;; The layout named `name`. Raises if there is none. +(: (-> string Layout)) +(defun (layout-named name) + (: named? (-> Layout bool)) + (def named? (fun (l) (= (record-ref l name) name))) + (match (find named? layouts) + ((Some l) l) + (None (panic! (str "no layout is named " name))))) + +(defun (main args) + (def moved (if (> (length args) 0) (normalize (option-ref (first (ref args 0)))) #\i)) + (def base (if (> (length args) 1) + (layout-named (ref args 1)) + (vec-ref layouts (- (vec-length layouts) 1)))) + (def counts (count-files layouts (vec-map (fun (c) (record-ref c path)) (record-ref params corpora)))) + (def variants + (cons (Tuple "as it is" base) + (->list (map (fun (c) + (Tuple (str (char->string moved) " <-> " (char->string c)) + (swap-keys base "" moved c))) + (filter (fun (c) (not (char=? c moved))) (layout-base-chars base)))))) + (def scored + (vec-sort-by (fun (a b) (compare (mean (snd a)) (mean (snd b)))) + (->vec (map (fun (v) (Tuple (fst v) (totals counts (snd v)))) variants)))) + (show [(record-ref base name) ", with " (char->string moved) " swapped with each other key." nl]) + (show (tabular/rows + (->vec (map (fun (s) + (: cells (Vec Block)) + (def cells (vec-append (vec-append [(BStr (fst s))] (vec-map fixed (snd s))) + [(fixed (mean (snd s)))])) + cells) + scored)) + #:titles ["swap" "sv" "en" "bjolang" "mean"] + #:sep " " #:align [AlignLeft AlignRight AlignRight AlignRight AlignRight])) + 0) diff --git a/src/texts.bjo b/src/texts.bjo new file mode 100644 index 0000000..607e68a --- /dev/null +++ b/src/texts.bjo @@ -0,0 +1,43 @@ +;; The texts the programs score on, and their weights in the mean. +;; +;; - sv.txt and en.txt weigh the same: half of what is written is English. +;; - bjolang.txt weighs nothing. It is shown, as a curiosity, but it is not in +;; the mean, and the optimizer leaves it out. + +(import (std prelude)) + +(export Corpus corpus corpus-path corpus-weight default-corpora weighted-mean weights-text) + +(type (: Corpus (Record (: path string) (: weight double)))) + +(: (-> string double Corpus)) +(defun (corpus path weight) (Corpus (path path) (weight weight))) + +(: (-> Corpus string)) +(defun (corpus-path c) (record-ref c path)) + +(: (-> Corpus double)) +(defun (corpus-weight c) (record-ref c weight)) + +(: (Vec Corpus)) +(def default-corpora + [(corpus "corpus/sv.txt" 1.0) + (corpus "corpus/en.txt" 1.0) + (corpus "corpus/bjolang.txt" 0.0)]) + +;; The mean of `xs`, one for each of `cs`, weighted by the corpora's weights. +;; Raises if no corpus weighs anything. +(: (-> (Vec Corpus) (Vec double) double)) +(defun (weighted-mean cs xs) + (def total (vec-fold (fun (c t) (+ t (record-ref c weight))) 0.0 cs)) + (when (<= total 0.0) (panic! "no text weighs anything")) + (/ (loop (:for c cs) (:for x xs) (:acc t (summing (* (record-ref c weight) x)))) + total)) + +;; The weights, for a report: "sv.txt 1, en.txt 1, bjolang.txt 0". +(: (-> (Vec Corpus) string)) +(defun (weights-text cs) + (string-join (vec-map (fun (c) (str (option-ref-or (path-filename (record-ref c path)) (record-ref c path)) + " " (displayed (record-ref c weight)))) + cs) + ", ")) diff --git a/src/word.bjo b/src/word.bjo new file mode 100644 index 0000000..460b34f --- /dev/null +++ b/src/word.bjo @@ -0,0 +1,91 @@ +;; What a word costs a layout, pair by pair and triple by triple. +;; +;; bjo run src/word.bjo word [layout] +;; +;; - layout: by name, the last in (keyscore layouts) unless given. +;; - Prints each pair's penalty by rule, each triple's hand-run and +;; same-finger-skip penalties, and how often the corpora type each, per +;; million key presses. + +(import (std prelude)) +(import (std maths)) +(import (std mutable map)) +(import (only (std fmt) numeric render->string)) +(import (keyscore board)) +(import (keyscore layouts)) +(import (keyscore params)) +(import (keyscore score)) +(import (keyscore texts)) + +;; The key's name, its hand and its finger: "i R middle top". +(: (-> Key string)) +(defun (where k) + (def fingers ["pinky" "ring" "middle" "index"]) + (def rows ["top" "home" "bottom"]) + (str (record-ref k name) " " (if (record-ref k left?) "L" "R") " " + (vec-ref fingers (record-ref k finger)) " " (vec-ref rows (record-ref k row)))) + +(: (-> Counts int)) +(defun (presses-of c) (loop (:for (Tuple _ n) (record-ref c presses)) (:acc t (summing n)))) + +(: (-> Counts (Tuple char char) int)) +(defun (pair-count c p) (option-ref-or (mutablemap-try-ref (record-ref c pairs) p) 0)) + +(: (-> Counts (Tuple char char char) int)) +(defun (triple-count c t) (option-ref-or (mutablemap-try-ref (record-ref c triples) t) 0)) + +;; n per million presses of c. +(: (-> int Counts string)) +(defun (per-million n c) + (displayed (cast long (/ (* 1000000.0 (cast double n)) (cast double (presses-of c)))))) + +(: (-> double string)) +(defun (num x) + (if (= x (floor x)) (displayed (cast long x)) (render->string (numeric x #:precision 2)))) + +(defun (main args) + (def params (current-params)) + (def word (->list (map normalize (ref args 0)))) + (: named? (-> Layout bool)) + (def named? (fun (l) (= (record-ref l name) (ref args 1)))) + (def l (if (> (length args) 1) + (option-ref (find named? layouts)) + (vec-ref layouts (- (vec-length layouts) 1)))) + (def (: key (-> char Key)) (fun (c) (option-ref (layout-key l c)))) + (def corpora (->vec (filter (fun (c) (> (corpus-weight c) 0.0)) (record-ref params corpora)))) + (def counts (count-files layouts (vec-map corpus-path corpora))) + (def names (string-join (vec-map (fun (c) (option-ref-or (path-filename (corpus-path c)) "")) corpora) ", ")) + (when (file-exists? params-file) (println (str "Parameters from " params-file "."))) + (println (str (record-ref l name) ": " (list->string word))) + (loop (:for c word) + (:let v (top-vowel params c (key c))) + (:do (println (str " " (char->string c) " " (where (key c)) + " travel " (num (home-travel (key c))) + (if (> v 0.0) (str " (top-row vowel " (num v) ")") ""))))) + (println "") + (println (str "Pairs (per million presses in " names "):")) + (loop (:for a word) (:for b (list-tail word)) + (:let ka (key a)) + (:let kb (key b)) + (:let parts (loop (:for r rules) + (:let p ((record-ref r penalty) params ka kb)) + (:when (and (not (= p 0.0)) (= (record-ref ka left?) (record-ref kb left?)) + (not (= (record-ref ka name) (record-ref kb name))))) + (:acc out (listing (str (record-ref r name) " " (num p)))))) + (:let travel (pair-travel ka kb)) + (:do (println (str " " (char->string a) (char->string b) " " (num (pair-penalty params ka kb)) + (if (list-empty? parts) "" (str " (" (string-join (list->vec parts) ", ") ")")) + (if (= travel 0.0) "" (str " travel " (num travel) " from home")) + " typed " (string-join (vec-map (fun (c) (per-million (pair-count c (Tuple a b)) c)) counts) " / "))))) + (println "") + (println "Triples:") + (loop (:for a word) (:for b (list-tail word)) (:for c (list-tail (list-tail word))) + (:let ka (key a)) + (:let kb (key b)) + (:let kc (key c)) + (:do (println (str " " (char->string a) (char->string b) (char->string c) + " hand run " (num (hand-run params ka kb kc)) + ", same-finger skip " (num (same-finger-skip params ka kb kc)) + ", two-finger run " (num (two-finger-run params ka kb kc)) + " typed " (string-join (vec-map (fun (x) (per-million (triple-count x (Tuple a b c)) x)) counts) " / "))))) + 0) diff --git a/tests/gui-shots.bjo b/tests/gui-shots.bjo new file mode 100644 index 0000000..4cc27d0 --- /dev/null +++ b/tests/gui-shots.bjo @@ -0,0 +1,176 @@ +;; Pictures of the GUI's tabs, drawn with no screen. +;; +;; bjo run tests/gui-shots.bjo [folder] +;; +;; - Starts (keyscore gui) headless, waits until the texts are read and the +;; layouts scored, and saves rules.png, key-costs.png, layout.png and +;; search.png in `folder`, the current one unless given. +;; - Changes a number first, and checks that the scores follow. +;; - Then runs a short search, keeps a result, and saves searched.png. +;; - Saves a preset, changes a number, and loads the preset back. Saves +;; presets.png, then deletes the preset. +;; - Saves a search result and the kept result as layouts, saves +;; layout-saved.png, and removes both again. +;; - Leaves no presets folder or saved-layouts.txt that was not there before. + +(import (std prelude)) +(import (std interop)) +(import (bjogui app)) +(import (keyscore gui)) +(import (keyscore layouts)) +(import (keyscore params)) +(import (keyscore saved)) + +(import/class + (Object (: System.Object)) + (Window (: Avalonia.Controls.Window (-> Window))) + (ContentControl (: Avalonia.Controls.ContentControl)) + (Control (: Avalonia.Controls.Control)) + (Layoutable (: Avalonia.Layout.Layoutable)) + (Visual (: Avalonia.Visual)) + (Size (: Avalonia.Size (-> double double Size))) + (Rect (: Avalonia.Rect (-> double double double double Rect))) + (PixelSize (: Avalonia.PixelSize (-> int int PixelSize))) + (Vector (: Avalonia.Vector (-> double double Vector))) + (RenderBitmap (: Avalonia.Media.Imaging.RenderTargetBitmap (-> PixelSize Vector RenderBitmap))) + (Bitmap (: Avalonia.Media.Imaging.Bitmap)) + (Application (: Avalonia.Application)) + (Styles (: Avalonia.Styling.Styles))) + +(import/extern + (measure! (: Avalonia.Layout.Layoutable.Measure (-> Layoutable Size void))) + (arrange! (: Avalonia.Layout.Layoutable.Arrange (-> Layoutable Rect void))) + (render-into! (: Avalonia.Media.Imaging.RenderTargetBitmap.Render (-> RenderBitmap Visual void))) + (save-png! (: Avalonia.Media.Imaging.Bitmap.Save (-> Bitmap string (System.Nullable int) void))) + (sleep-ms (: System.Threading.Thread.Sleep (-> int void) #:blocking)) + (current-app (: Avalonia.Application.Current #:get)) + (content! (: Avalonia.Controls.ContentControl.Content (-> ContentControl Object void) #:set)) + (show! (: Avalonia.Controls.Window.Show (-> Window void))) + (app-styles (: Avalonia.Application.Styles (-> Application Styles) #:get)) + (styles-add! (: Avalonia.Styling.Styles.Add (-> Styles IStyle void)))) + +;; What `control` shows, laid out at `w` by `h`, saved to `path` at twice the +;; pixels. +(: (-> Control int int string void)) +(defun (save-shot control w h path) + (measure! (cast Layoutable control) (Size. (cast double w) (cast double h))) + (arrange! (cast Layoutable control) (Rect. 0.0 0.0 (cast double w) (cast double h))) + (def bitmap (RenderBitmap. (PixelSize. (* 2 w) (* 2 h)) (Vector. 192.0 192.0))) + (render-into! bitmap (cast Visual control)) + (save-png! (cast Bitmap bitmap) path (: (clr-default) (System.Nullable int)))) + +;; The name of `l`. +(: (-> Layout string)) +(defun (layout-name-of l) (record-ref l name)) + +(defun (main args) + (def folder (if (> (vec-length args) 0) (vec-ref args 0) ".")) + (start-headless!) + ;; With no window there is no theme, and a control with a template draws nothing. + (styles-add! (app-styles (cast Application current-app)) (fluent-theme)) + (def app (start-app initial gui-update view (fun (c) unit) #:subscriptions subscriptions)) + ;; Runs what the fibers posted until `done?` holds, for two minutes at most. + (: (-> (-> Model bool) bool)) + (defun (wait-for done?) + (let go ((tries 0)) + (app-run-pending! app) + (cond ((done? (app-model app)) #t) + ((> tries 12000) #f) + (else (sleep-ms 10) (go (+ tries 1)))))) + (: scored? (-> Model bool)) + (def scored? (fun (m) (= (vec-length (.scores m)) (vec-length (.layouts m))))) + (when (not (wait-for scored?)) (panic! "the layouts were not scored")) + (def before (.generation (app-model app))) + (app-dispatch! app (SetParam 3 60.0)) + (when (not (wait-for (fun (m) (and (> (.generation m) before) (scored? m))))) + (panic! "the scores did not follow a change")) + ;; 3 is the key weight, after the three texts' weights. + (app-dispatch! app (SetParam 3 2.0)) + (ignore (wait-for scored?)) + ;; In a window, so that the theme gives the controls their templates. + (def frame (Window.)) + (content! (cast ContentControl frame) (cast Object (app-control app))) + (show! frame) + (loop (:for (Tuple t name) (list (Tuple 0 "rules") (Tuple 1 "key-costs") (Tuple 2 "layout") (Tuple 3 "search"))) + (:do (app-dispatch! app (Tab t)) + (app-run-pending! app) + (save-shot (cast Control frame) 1500 900 (path-combine folder (str name ".png"))))) + ;; A short search, and its results. + (app-dispatch! app (SetRuns 2.0)) + (app-dispatch! app (SetSteps 20000.0)) + (app-dispatch! app Search) + (when (not (wait-for (fun (m) (and (not (.searching m)) (= (vec-length (.results m)) 2))))) + (panic! (str "the search did not finish: " (.status (app-model app))))) + (app-dispatch! app (Keep 0)) + (when (not (wait-for (fun (m) (and (= (vec-length (.layouts m)) 17) (scored? m))))) + (panic! "a kept layout was not scored")) + (app-run-pending! app) + (save-shot (cast Control frame) 1500 900 (path-combine folder "searched.png")) + + ;; --- presets + (def had-presets (directory-exists? presets-folder)) + (def had-layouts (file-exists? layouts-file)) + (: weight (-> Model double)) + (def weight (fun (m) (record-ref (.params m) key-weight))) + (app-dispatch! app (SetPresetName "gui-test")) + (app-dispatch! app SavePreset) + (when (not (wait-for (fun (m) (vec-contains? (.presets m) "gui-test")))) + (panic! (str "the preset was not saved: " (.status (app-model app))))) + (app-dispatch! app (SetParam 3 7.0)) + (when (not (wait-for (fun (m) (= (weight m) 7.0)))) (panic! "the key weight did not change")) + (app-dispatch! app (LoadPreset "gui-test")) + (when (not (wait-for (fun (m) (and (= (weight m) 2.0) (scored? m))))) + (panic! (str "the preset was not loaded: " (.status (app-model app))))) + (app-dispatch! app (Tab 0)) + (app-run-pending! app) + (save-shot (cast Control frame) 1500 900 (path-combine folder "presets.png")) + (app-dispatch! app (DeletePreset "gui-test")) + (when (not (wait-for (fun (m) (not (vec-contains? (.presets m) "gui-test"))))) + (panic! "the preset was not deleted")) + + ;; --- saved layouts + (: names (-> Model (Vec string))) + (def names (fun (m) (vec-map layout-name-of (.layouts m)))) + (: choose-named (-> string void)) + (defun (choose-named n) + (def m (app-model app)) + (def r (ranking m)) + (match (find (fun (pos) (= (layout-name-of (vec-ref (.layouts m) (vec-ref r pos))) n)) (range 0 (vec-length r))) + ((Some pos) (app-dispatch! app (Choose pos))) + (None (panic! (str "no layout is named " n))))) + (app-dispatch! app (SetLayoutName "gui test")) + (app-dispatch! app (SaveResult 1)) + (when (not (wait-for (fun (m) (and (= (vec-length (.layouts m)) 18) (scored? m) + (= (.status m) (str "Saved the layout gui test to " layouts-file ".")))))) + (panic! (str "a result was not saved: " (.status (app-model app))))) + (when (not (string-contains? (file-read-text layouts-file) "gui test\t")) + (panic! "the saved layout is not in the file")) + ;; The kept result, saved under a name, takes the place of "found 1". + (choose-named "found 1") + (app-dispatch! app (SetLayoutName "gui kept")) + (app-dispatch! app SaveLayout) + (when (not (wait-for (fun (m) (and (scored? m) (vec-contains? (.saved m) "gui kept"))))) + (panic! (str "the kept layout was not saved: " (.status (app-model app))))) + (when (or (not (= (vec-length (.layouts (app-model app))) 18)) + (vec-contains? (names (app-model app)) "found 1")) + (panic! "the kept layout was not renamed")) + (when (not (wait-for (fun (m) (string-contains? (.status m) "Saved the layout gui kept")))) + (panic! "the file was not written")) + (app-dispatch! app (Tab 2)) + (app-run-pending! app) + (save-shot (cast Control frame) 1500 900 (path-combine folder "layout-saved.png")) + (app-dispatch! app RemoveLayout) + (when (not (wait-for (fun (m) (and (= (vec-length (.layouts m)) 17) (scored? m))))) + (panic! "the kept layout was not removed")) + (choose-named "gui test") + (app-dispatch! app RemoveLayout) + (when (not (wait-for (fun (m) (and (= (vec-length (.layouts m)) 16) (scored? m) + (string-contains? (.status m) "Removed gui test from"))))) + (panic! (str "the layouts were not removed: " (.status (app-model app))))) + (when (string-contains? (file-read-text layouts-file) "gui ") + (panic! "a removed layout is still in the file")) + (when (not had-layouts) (file-delete layouts-file)) + (when (and (not had-presets) (= (vec-length (directory-files presets-folder)) 0)) + (directory-delete presets-folder)) + (println "gui shots ok") + 0)