#!/usr/bin/env python3 """Fetches the prose corpus: plain text of featured Wikipedia articles. ./corpus/fetch.py writes corpus/sv.txt and corpus/en.txt - A fixed-seed sample of the featured articles of sv and en Wikipedia, so a rerun fetches the same articles (unless the categories change). - The text is the TextExtracts plain-text extract: no markup. Section headings ("== ... ==") and empty lines are dropped. - Stops at about TARGET characters per language. """ import json, random, sys, time, urllib.error, urllib.parse, urllib.request from pathlib import Path TARGET = 1_500_000 CATEGORIES = { 'sv': 'Kategori:Wikipedia:Utmärkta artiklar', 'en': 'Category:Featured articles', } HERE = Path(__file__).resolve().parent def api(lang, **params): params.update(format='json', formatversion='2') url = f'https://{lang}.wikipedia.org/w/api.php?' + urllib.parse.urlencode(params) req = urllib.request.Request(url, headers={'User-Agent': 'keyscore-corpus/0.1'}) for attempt in range(8): try: with urllib.request.urlopen(req, timeout=30) as r: return json.load(r) except urllib.error.HTTPError as e: if e.code != 429: raise wait = int(e.headers.get('Retry-After') or 0) or 5 * 2 ** attempt print(f' rate-limited, waiting {wait} s', file=sys.stderr) time.sleep(wait) raise RuntimeError(f'still rate-limited: {url}') def titles(lang, limit_pages=4): out, cont = [], {} for _ in range(limit_pages): r = api(lang, action='query', list='categorymembers', cmtitle=CATEGORIES[lang], cmnamespace=0, cmlimit=500, **cont) out += [m['title'] for m in r['query']['categorymembers']] if 'continue' not in r: break cont = {'cmcontinue': r['continue']['cmcontinue']} return out def extract(lang, title): r = api(lang, action='query', prop='extracts', explaintext=1, titles=title) text = r['query']['pages'][0].get('extract', '') lines = [l.strip() for l in text.split('\n')] return '\n'.join(l for l in lines if l and not l.startswith('==')) def main(): for lang in CATEGORIES: names = titles(lang) random.Random(1).shuffle(names) parts, size = [], 0 for name in names: if size >= TARGET: break text = extract(lang, name) parts.append(text); size += len(text) print(f'{lang}: {name} ({len(text)} chars, {size} in all)', file=sys.stderr) time.sleep(1.5) (HERE / f'{lang}.txt').write_text('\n'.join(parts) + '\n', encoding='utf-8') if __name__ == '__main__': main()