keyscore: a scorer, optimizer and GUI for keyboard layouts
Scores layouts for ISO Swedish with the angle mod on Swedish and English text, by Michael Capewell's rules plus row motion, triples, pinky load and finger distance. - src/main.bjo: the report. src/optimize.bjo: simulated annealing. - src/gui.bjo: a bjogui window for tuning the numbers, with help on every number, presets and saved layouts. - corpus/: the texts, and fetch.py for the Wikipedia ones. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
53a0002907
20 changed files with 64897 additions and 0 deletions
55690
corpus/bjolang.txt
Normal file
55690
corpus/bjolang.txt
Normal file
File diff suppressed because it is too large
Load diff
2882
corpus/en.txt
Normal file
2882
corpus/en.txt
Normal file
File diff suppressed because it is too large
Load diff
72
corpus/fetch.py
Executable file
72
corpus/fetch.py
Executable file
|
|
@ -0,0 +1,72 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Fetches the prose corpus: plain text of featured Wikipedia articles.
|
||||
|
||||
./corpus/fetch.py writes corpus/sv.txt and corpus/en.txt
|
||||
|
||||
- A fixed-seed sample of the featured articles of sv and en Wikipedia, so a
|
||||
rerun fetches the same articles (unless the categories change).
|
||||
- The text is the TextExtracts plain-text extract: no markup. Section
|
||||
headings ("== ... ==") and empty lines are dropped.
|
||||
- Stops at about TARGET characters per language.
|
||||
"""
|
||||
import json, random, sys, time, urllib.error, urllib.parse, urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
TARGET = 1_500_000
|
||||
CATEGORIES = {
|
||||
'sv': 'Kategori:Wikipedia:Utmärkta artiklar',
|
||||
'en': 'Category:Featured articles',
|
||||
}
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def api(lang, **params):
|
||||
params.update(format='json', formatversion='2')
|
||||
url = f'https://{lang}.wikipedia.org/w/api.php?' + urllib.parse.urlencode(params)
|
||||
req = urllib.request.Request(url, headers={'User-Agent': 'keyscore-corpus/0.1'})
|
||||
for attempt in range(8):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return json.load(r)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 429: raise
|
||||
wait = int(e.headers.get('Retry-After') or 0) or 5 * 2 ** attempt
|
||||
print(f' rate-limited, waiting {wait} s', file=sys.stderr)
|
||||
time.sleep(wait)
|
||||
raise RuntimeError(f'still rate-limited: {url}')
|
||||
|
||||
|
||||
def titles(lang, limit_pages=4):
|
||||
out, cont = [], {}
|
||||
for _ in range(limit_pages):
|
||||
r = api(lang, action='query', list='categorymembers', cmtitle=CATEGORIES[lang],
|
||||
cmnamespace=0, cmlimit=500, **cont)
|
||||
out += [m['title'] for m in r['query']['categorymembers']]
|
||||
if 'continue' not in r: break
|
||||
cont = {'cmcontinue': r['continue']['cmcontinue']}
|
||||
return out
|
||||
|
||||
|
||||
def extract(lang, title):
|
||||
r = api(lang, action='query', prop='extracts', explaintext=1, titles=title)
|
||||
text = r['query']['pages'][0].get('extract', '')
|
||||
lines = [l.strip() for l in text.split('\n')]
|
||||
return '\n'.join(l for l in lines if l and not l.startswith('=='))
|
||||
|
||||
|
||||
def main():
|
||||
for lang in CATEGORIES:
|
||||
names = titles(lang)
|
||||
random.Random(1).shuffle(names)
|
||||
parts, size = [], 0
|
||||
for name in names:
|
||||
if size >= TARGET: break
|
||||
text = extract(lang, name)
|
||||
parts.append(text); size += len(text)
|
||||
print(f'{lang}: {name} ({len(text)} chars, {size} in all)', file=sys.stderr)
|
||||
time.sleep(1.5)
|
||||
(HERE / f'{lang}.txt').write_text('\n'.join(parts) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
3160
corpus/sv.txt
Normal file
3160
corpus/sv.txt
Normal file
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue