73 lines
2.6 KiB
Python
73 lines
2.6 KiB
Python
|
|
#!/usr/bin/env python3
|
||
|
|
"""Fetches the prose corpus: plain text of featured Wikipedia articles.
|
||
|
|
|
||
|
|
./corpus/fetch.py writes corpus/sv.txt and corpus/en.txt
|
||
|
|
|
||
|
|
- A fixed-seed sample of the featured articles of sv and en Wikipedia, so a
|
||
|
|
rerun fetches the same articles (unless the categories change).
|
||
|
|
- The text is the TextExtracts plain-text extract: no markup. Section
|
||
|
|
headings ("== ... ==") and empty lines are dropped.
|
||
|
|
- Stops at about TARGET characters per language.
|
||
|
|
"""
|
||
|
|
import json, random, sys, time, urllib.error, urllib.parse, urllib.request
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
TARGET = 1_500_000
|
||
|
|
CATEGORIES = {
|
||
|
|
'sv': 'Kategori:Wikipedia:Utmärkta artiklar',
|
||
|
|
'en': 'Category:Featured articles',
|
||
|
|
}
|
||
|
|
HERE = Path(__file__).resolve().parent
|
||
|
|
|
||
|
|
|
||
|
|
def api(lang, **params):
|
||
|
|
params.update(format='json', formatversion='2')
|
||
|
|
url = f'https://{lang}.wikipedia.org/w/api.php?' + urllib.parse.urlencode(params)
|
||
|
|
req = urllib.request.Request(url, headers={'User-Agent': 'keyscore-corpus/0.1'})
|
||
|
|
for attempt in range(8):
|
||
|
|
try:
|
||
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
||
|
|
return json.load(r)
|
||
|
|
except urllib.error.HTTPError as e:
|
||
|
|
if e.code != 429: raise
|
||
|
|
wait = int(e.headers.get('Retry-After') or 0) or 5 * 2 ** attempt
|
||
|
|
print(f' rate-limited, waiting {wait} s', file=sys.stderr)
|
||
|
|
time.sleep(wait)
|
||
|
|
raise RuntimeError(f'still rate-limited: {url}')
|
||
|
|
|
||
|
|
|
||
|
|
def titles(lang, limit_pages=4):
|
||
|
|
out, cont = [], {}
|
||
|
|
for _ in range(limit_pages):
|
||
|
|
r = api(lang, action='query', list='categorymembers', cmtitle=CATEGORIES[lang],
|
||
|
|
cmnamespace=0, cmlimit=500, **cont)
|
||
|
|
out += [m['title'] for m in r['query']['categorymembers']]
|
||
|
|
if 'continue' not in r: break
|
||
|
|
cont = {'cmcontinue': r['continue']['cmcontinue']}
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def extract(lang, title):
|
||
|
|
r = api(lang, action='query', prop='extracts', explaintext=1, titles=title)
|
||
|
|
text = r['query']['pages'][0].get('extract', '')
|
||
|
|
lines = [l.strip() for l in text.split('\n')]
|
||
|
|
return '\n'.join(l for l in lines if l and not l.startswith('=='))
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
for lang in CATEGORIES:
|
||
|
|
names = titles(lang)
|
||
|
|
random.Random(1).shuffle(names)
|
||
|
|
parts, size = [], 0
|
||
|
|
for name in names:
|
||
|
|
if size >= TARGET: break
|
||
|
|
text = extract(lang, name)
|
||
|
|
parts.append(text); size += len(text)
|
||
|
|
print(f'{lang}: {name} ({len(text)} chars, {size} in all)', file=sys.stderr)
|
||
|
|
time.sleep(1.5)
|
||
|
|
(HERE / f'{lang}.txt').write_text('\n'.join(parts) + '\n', encoding='utf-8')
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
main()
|