keyscore/corpus/fetch.py

73 lines
2.6 KiB
Python
Raw Normal View History

#!/usr/bin/env python3
"""Fetches the prose corpus: plain text of featured Wikipedia articles.
./corpus/fetch.py writes corpus/sv.txt and corpus/en.txt
- A fixed-seed sample of the featured articles of sv and en Wikipedia, so a
rerun fetches the same articles (unless the categories change).
- The text is the TextExtracts plain-text extract: no markup. Section
headings ("== ... ==") and empty lines are dropped.
- Stops at about TARGET characters per language.
"""
import json, random, sys, time, urllib.error, urllib.parse, urllib.request
from pathlib import Path
TARGET = 1_500_000
CATEGORIES = {
'sv': 'Kategori:Wikipedia:Utmärkta artiklar',
'en': 'Category:Featured articles',
}
HERE = Path(__file__).resolve().parent
def api(lang, **params):
params.update(format='json', formatversion='2')
url = f'https://{lang}.wikipedia.org/w/api.php?' + urllib.parse.urlencode(params)
req = urllib.request.Request(url, headers={'User-Agent': 'keyscore-corpus/0.1'})
for attempt in range(8):
try:
with urllib.request.urlopen(req, timeout=30) as r:
return json.load(r)
except urllib.error.HTTPError as e:
if e.code != 429: raise
wait = int(e.headers.get('Retry-After') or 0) or 5 * 2 ** attempt
print(f' rate-limited, waiting {wait} s', file=sys.stderr)
time.sleep(wait)
raise RuntimeError(f'still rate-limited: {url}')
def titles(lang, limit_pages=4):
out, cont = [], {}
for _ in range(limit_pages):
r = api(lang, action='query', list='categorymembers', cmtitle=CATEGORIES[lang],
cmnamespace=0, cmlimit=500, **cont)
out += [m['title'] for m in r['query']['categorymembers']]
if 'continue' not in r: break
cont = {'cmcontinue': r['continue']['cmcontinue']}
return out
def extract(lang, title):
r = api(lang, action='query', prop='extracts', explaintext=1, titles=title)
text = r['query']['pages'][0].get('extract', '')
lines = [l.strip() for l in text.split('\n')]
return '\n'.join(l for l in lines if l and not l.startswith('=='))
def main():
for lang in CATEGORIES:
names = titles(lang)
random.Random(1).shuffle(names)
parts, size = [], 0
for name in names:
if size >= TARGET: break
text = extract(lang, name)
parts.append(text); size += len(text)
print(f'{lang}: {name} ({len(text)} chars, {size} in all)', file=sys.stderr)
time.sleep(1.5)
(HERE / f'{lang}.txt').write_text('\n'.join(parts) + '\n', encoding='utf-8')
if __name__ == '__main__':
main()