User:Tbot/code/tbot


 * 1) !/usr/bin/python
 * 2) -*- coding: utf-8  -*-
 * 3) wikipath en wiktionary User:Tbot/code/tbot

""" This bot looks for t/t-/t+ template calls and does updates

No command line arguments.

"""

import wikipedia import xmlreader import sys import re import pickle import socket from script import script, scriptp from random import random from mwapi import getwikitext, getedit import threading import Queue import shelve from time import time, sleep

plock = threading.Lock

def srep(s): return repr(u''+s)[2:-1]


 * 1) try having a bit of fun with threads, do XML reads in a thread than feeds main proc
 * 2) combined with the cache of sections and iwikis, allows fast start


 * 1) shelf that allows Unicode keys, just wrap around rather than sub-class, only need a few methods
 * 2) make thread safe as well

class myshelf: def __init__(s, fn): s.shelf = shelve.open(fn) s.lock = threading.Lock def __contains__(s, k): with s.lock: r = (srep(k) in s.shelf) return r   def __setitem__(s, k, v): # optimize no change case: with s.lock: sk = srep(k) if sk in s.shelf and s.shelf[sk] == v: return s.shelf[sk] = v   def __getitem__(s, k): with s.lock: r = s.shelf[srep(k)] return r   def sync(s): with s.lock: s.shelf.sync def close(s): with s.lock: s.shelf.close # del works when s.shelf deref'd


 * 1) work cache, record time last looked at local or FL entry
 * 2) each record is key: lc:word, pickled with srep, value is integer time

cache = myshelf("tbot-cache") Lsects = myshelf("tbot-lsects") Iwikis = myshelf("tbot-iwikis")
 * 1) persistent cache of sections and iwikis

from logger import Logger logpage = Logger(wikipedia.getSite('en', 'wiktionary'), "User:Tbot/log")

from createflw import createFLentry

Lcodes = { } Langtoc = { }

from weakref import WeakValueDictionary wtit = WeakValueDictionary
 * 1) set of titles seen and still on queue

tlocal = threading.local tlocal.repact = True


 * 1) thread routine to read XML and load section/iwiki cache, feed to main

pageq = Queue.Queue
 * 1) with 5K loaded, process size ~70MB, soft limited below

retemp = re.compile(ur'\{\{t[\-\+\u00f8]?\|.*?}}')

def xread:

tlocal.repact = False

while True:

re2head = re.compile(r'^==([^=]+?)==$', re.M)   reiwiki = re.compile(r'^\[\[([-a-z]{2,10}):(.*?)\]\]$', re.M)

# build table of section counts and iwikis:

# get XML dump dump = xmlreader.XmlDump("../hancheck/en-wikt.xml")

ts = 0 ti = 0 entries = 0 for entry in dump.parse: text = entry.text title = entry.title if title.find(':') >= 0: continue

entries += 1 if entries % 1000 == 0: with plock: print "prescan %d entries, %d iwikis, %d sections, %d on queue" % \ (entries, ti, ts, pageq.qsize) Iwikis.sync Lsects.sync # pace a bit more, amusingly en.wikt is 1.44 million entries at this writing # nominally 4 hours if pageq.qsize > 100: sleep(10)

# for my system, temporarily (?) 10.1.10 if entries % 10 == 0: sleep(1) # take a long time, not go CPU bound for long

# filter for those we want to check tag = False

newtext = retemp.sub(tsub, text) if newtext != text: # temporarily (?), do only some with sc= elisions (may miss some other changes ;-) if newtext.count('sc=') < text.count('sc='): if newtext.count('t+') != text.count('t+'): tag = True if newtext.count('t-') != text.count('t-'): tag = True if newtext.count('xs=') > text.count('xs='): tag = True # some others caught by the seven percent rule etc following else: tag = True

# look at ones we are modifying anyway, plus try to capture more sometimes # if not in cache or expired, pick up sometime, scatter over ~14 days, 1/2 of cache time

if '==English==' in text and '===Translations===' in text: if newtext.count('sc=') < text.count('sc='): # try to avoid more edits for sc= elision only if random < 0.01: tag = True else: if random < 0.07: tag = True

if tag and title not in wtit: ckey = 'en:' + title if ckey not in cache or cache[ckey] < time - (35 * 24 * 3600): wtit[title] = entry pageq.put(entry) # soft limit queue size, e.g. 20K if 20 seconds to eat each entry # larger effect is to cause process/thread schedule event sleep(pageq.qsize/1000.0)

# add iwikis:

iw = [ ] for mo in reiwiki.finditer(text): if mo.group(2) == title: iw.append(mo.group(1)) ti += 1 Iwikis[title] = iw

# now, Lsects is an optimization to preclude calls to createflw, filter so it won't be       # uselessly large, it is okay if we miss some (for now): [ try keeping all ] """       if "form of" in text: continue    # SemperBlottoBot, others        if " of|" in text: continue       # Keenebot2, others (Jyril: "{{fi-form of|")        """

ll = [ ] for lang in re2head.findall(text):

lang = lang.strip(' []') if lang == "English": continue if lang not in Langtoc: continue lc = Langtoc[lang] if lc not in Exists: continue

ll.append(lc) ts += 1

if ll: Lsects[title] = ll

pass # end of prelim XML loop

print "XML scan complete" Iwikis.sync Lsects.sync

# put a pass marker on the queue, and then we continue pageq.put(None)

# take a day off, and do it again # queue may very well last that long, and we may get new XML # sleep(86400) # don't sleep very long at once, makes interrupt difficult! # for i in range(24*60, 0, -1): # not one day yet, just some hours!

hours = 14 # see what works well? shouldn't need to be much < 24? for i in range(hours*60, 0, -1): if not i%10: with plock: print "(sleeping, next scan in %d minutes)" % i       sleep(60)

Exists = set Tlang = set(['ar', 'da', 'de', 'el', 'es', 'fi', 'fr', 'he', 'it', 'ja', 'ko', 'nl', 'no', 'pt',        'ru', 'sv', 'bg', 'bs', 'ca', 'cmn', 'cs', 'et', 'hr', 'hu', 'is', 'ku', 'la', 'pl', 'ro',         'sr', 'sk', 'sl', 'te', 'th', 'tr', 'hy', 'mk'])


 * |ar=Arab|fa=fa-Arab|ur=ur-Arab
 * |xcl|hy=Armn|be|bg|mk|ru|uk=Cyrl|cu=Cyrs|sa|hi=Deva|got=Goth|el=Grek|grc=polytonic
 * |he|yi|arc=Hebr|ja=Jpan|ko=Kore|ta=Taml|te=Telu|th=Thai

Xyzzy = dict( ar='Arab', fa='fa-Arab', ur='ur-Arab',             xcl='Armn', hy='Armn',              be='Cyrl', bg='Cyrl', mk='Cyrl', ru='Cyrl', uk='Cyrl', cu='Cyrs',              sa='Deva', hi='Deva',              got='Goth', el='Grek', grc='polytonic',              he='Hebr', yi='Hebr', arc='Hebr',              ja='Jpan', ko='Kore',               ta='Taml', te='Telu', th='Thai' )

POS = set(['Noun', 'Verb', 'Adverb', 'Adjective', 'Pronoun',           'Proper noun', 'Preposition', 'Conjunction', 'Interjection', 'Article' ]) Stops = set(['Prefix', 'Suffix', 'Affix', 'Infix', 'Counter', 'Initialism', 'Abbreviation',             'Letter', 'Symbol', 'Acronym', 'Proverb', 'Contraction', 'Idiom', 'Phrase',              'Syllable', 'Reflexive verb', 'Transitive verb', 'Intransitive verb' ])
 * 1) translatable POS from English, other things may be very different POS ...
 * 1) others we want to recognize and avoid, at least for now

TTBC = ('{{checktrans', '{{ttbc', '{{rfc-trans')

relcodeword = re.compile(r'\{\{t[\+\-]?\|([-a-z]*?)\|(.*?)[\|\}]') rexsect = re.compile(r'\|xs=[^\|\}]*([\|\}])') rescect = re.compile(r'\|sc=[^\|\}]*([\|\}])') reflag = re.compile(r'\{\{t[^\|]?\|') recode = re.compile(r'(\{\{t[^\|]?\|)([-a-z]*?)\|')
 * 1) substitute templates:

refindsc = re.compile(r'\|sc=([^\|\}]*)[\|\}]')

reorderalt = re.compile(r'\|alt=([^\|]*)\|(.*)}}') reordertr = re.compile(r'\|tr=([^\|]*)\|(.*)}}') reordersc = re.compile(r'\|sc=([^\|]*)\|(.*)}}') reorderxs = re.compile(r'\|xs=([^\|]*)\|(.*)}}')
 * 1) canonicalize option order: (match if not at end, so we can move to end)

repipe = re.compile(r'\[\^\*\|') # enough to catch a piped link [[...|

def tsub(tmo):

tin = tmo.group(0)

# look for some bad cases, to be tagged: # either a piped link or a subtemplate call will scramble parsing

if repipe.search(tin) or ''): tout = tout[:-3] + '}}'

# canonical order, move each to end in turn if 'alt=' in tout: tout = reorderalt.sub(r'|\2|alt=\1}}', tout) if 'tr=' in tout: tout = reordertr.sub(r'|\2|tr=\1}}', tout) if 'sc=' in tout: tout = reordersc.sub(r'|\2|sc=\1}}', tout) if 'xs=' in tout: tout = reorderxs.sub(r'|\2|xs=\1}}', tout)

mo = relcodeword.match(tout) if mo: code = mo.group(1) word = mo.group(2) else: code = '' word = ''

# fix some codes, ISO /3 to /1, and some other things, like nb to no   if code in Lcodes: nc = Langtoc[Lcodes[code]] if nc != code: tout = recode.sub(r'\1' + nc + '|', tout) if tlocal.repact: with plock: print "changed code %s to %s" % (code, nc) code = nc

if 'sc=' not in tout and code and word and ord(word[0:1]) >= 0x0370: sc = script(word, code, report = tlocal.repact) if sc and (code not in Xyzzy or sc != Xyzzy[code]): tout = tout[:-2] + '|sc=' + sc + '}}' # move xs to end again if needed if 'xs=' in tout: tout = reorderxs.sub(r'|\2|xs=\1}}', tout)

if 'sc=' in tout: mo = refindsc.search(tout) if mo: sc = mo.group(1) else: sc = '' # script parameter not needed? (may be different from default, suppress if = to Xyzy code) if code in Xyzzy and sc == Xyzzy[code]: tout = rescect.sub(r'\1', tout) if tlocal.repact: with plock: print "elided script %s for code %s" % (sc, code)

if code and code in Lcodes: if code not in Tlang: xs = '|xs=' + Lcodes[code] else: xs = '' if tout.find('|xs=') >= 0: tout = rexsect.sub(xs + r'\1', tout) else: tout = tout[:-2] + xs + '}}'

if code: if code in Exists: if tout.startswith(u'{{t\u00f8|'): tout = reflag.sub('{{t|', tout) if word in Iwikis: if code in Iwikis[word]: tout = reflag.sub('{{t+|', tout) if code not in Iwikis[word]: tout = reflag.sub('{{t-|', tout) else: tout = reflag.sub(u'{{t\u00f8|', tout)

if tout != tin and tlocal.repact: print "%s, code %s, word %s: %s" % (srep(tin), srep(code), srep(word), srep(tout))

return u'' + tout


 * 1) transliterate words, use to match transliterations, use our own dict which we can preload as needed

import transliteration trac = { } tlitob = transliteration.transliterator # work around gratuitious class defn

def tlit(s):

n = u'' for c in s:      if c not in trac: # and work around nasties in the transliteration.trans routine try: trac[c] = u'' + tlitob.transliterate(c, default = c)          except UnicodeDecodeError: trac[c] = c      n += trac[c]

n = n.lower return n


 * 1) now have some serious recursion fun!
 * 2) fuzzy returns string match score
 * 3) r is min required, calls may have neg r, may return value < r

def fuzzy(a, b, r):

if not a or len(a) < r: return 0 if not b or len(b) < r: return 0

if a == b: return len(a) if a[:1] == b[:1]: return 1 + fuzzy(a[1:], b[1:], r-1) if a[-1:] == b[-1:]: return 1 + fuzzy(a[:-1], b[:-1], r-1)

# try with each char forward p = a.find(b[0:1]) if p >= 0: sca = 1 + fuzzy(a[p+1:], b[1:], r-1) else: sca = 0

p = b.find(a[0:1]) if p >= 0: scb = 1 + fuzzy(b[p+1:], a[1:], r-1) else: scb = 0

# no match either/or way, skip this char, one or both if not sca and not scb: sk = fuzzy(a[1:], b[1:], r)   elif not sca: sk = fuzzy(a, b[1:], r)    elif not scb: sk = fuzzy(a[1:], b, r)    else: sk = 0

return max(sk, sca, scb)

def main:

socket.setdefaulttimeout(30)

nap = 5 # +1,/2 adaptive naptime cis = 0

skip = 0 dmatch = False nocreate = False

for arg in sys.argv[1:]: if arg.startswith('-skip:'): skip = int(arg[6:]) print "will skip to update %d" % skip elif arg == '-dmatch': dmatch = True print "will report debug no match conditions" elif arg == '-nocreate': nocreate = True print "will not create new Tbot entries" else: print "unknown command line argument %s" % arg

# make sure we are logged in   site = wikipedia.getSite site.forceLogin meta = wikipedia.getSite(code = "meta", fam = "meta")

# get language codes page = wikipedia.Page(site, "User:AutoFormat/Languages") langtab = getwikitext(page)

relangtab = re.compile(r'\|(.*?)\|\|(.*)') i = 0 for line in langtab.splitlines: mo = relangtab.match(line) if mo: for code in mo.group(1).split(','): Lcodes[code.strip] = mo.group(2).strip i += 1

# invert, with some forces, make sure nb goes to no, fix the Chinese languages # things like ido to io are handled by the length Lcodes['ido'] = 'Ido' Lcodes['nb'] = 'Norwegian' Lcodes['nob'] = 'Norwegian' Langtoc['Norwegian'] = 'no' Langtoc['Mandarin'] = 'cmn' Langtoc['Min Nan'] = 'nan' Langtoc['Cantonese'] = 'yue' for code in Lcodes: if Lcodes[code] not in Langtoc: Langtoc[Lcodes[code]] = code elif Lcodes[code] in ['Mandarin', 'Min Nan', 'Cantonese']: continue elif len(code) < len(Langtoc[Lcodes[code]]): Langtoc[Lcodes[code]] = code

print "found %d language codes" % i   if i < 460: return

# get active wikt list # page = wikipedia.Page(meta, "List of Wiktionaries/Table") # gratuitous move, 20.4.9: page = wikipedia.Page(meta, "Wiktionary/Table") existtab = getwikitext(page)

# reextab = re.compile(r'^\[\[:([a-z-]+):') reextab = re.compile(r'\| \+)\.wiktionary\.org') i = 0 for line in existtab.splitlines: i += 1 mo = reextab.match(line) if mo: # toki pona and klingon we don't want, bo and sc are borked, to is closed (21.5.8) if mo.group(1) in [ 'tokipona', 'tlh', 'bo', 'sc', 'to', 'sh' ]: continue Exists.add(mo.group(1)) print "%d lines" % i

print "found %d active wikts" % len(Exists) if len(Exists) < 160: return

# add codes for Mandarin and Min Nan, cantonese doesn't exist yet Exists.add('cmn') Exists.add('nan')

# headers, only need 3,4,5 rehead = re.compile(r'^={3,5}(.+?)={3,5}$', re.M)

regloss = re.compile(r'\{\{trans-top\|(.+?)}}')

# trans line, start simple, without and with wikilinking, 1 is lang, 2 is rest retrans = re.compile(r'\*\s*([-\w ]+):(.*)') retransw = re.compile(r'\*\s*\[\[([-\w ]+)\]\]:(.*)')

# wikitext format stackable at the start of a line restack = re.compile('^([#:\*]+)\s*')

Tlist = [ # first sets are optimizations, catch common cases, and uncollapsed gender/number for simple # match patterns for translations lines, simplest first, must match all r'^\[\[(?P [- \w]+)\]\]$', r'^\[\[(?P [- \w]+)\]\]\s*\{\{(?P[mfcn])}}(?:\s*\{\{(?P[fcnps])}}|)$',

# with transliteration r'^\[\[(?P [- \w]+)\]\]\s*\((?P [\w]+)\)$', r'^\[\[(?P [- \w]+)\]\]\s*\((?P [\w]+)\)\s*\{\{(?P[mfcn])}}(?:\s*\{\{(?P[fcnps])}})$',

# with sections r'^\[\[(?P [- \w]+)#(?P [- \w]+)\|(?P [- \w]+)\]\]$', r'^\[\[(?P [- \w]+)#(?P [- \w]+)\|(?P [- \w]+)\]\]\s*\{\{(?P[mfcn])}}$',

# with scripts r'^\{\{(?P [-\w]+)\|\[\[(?P [- \w]+)\]\]}}$', r'^\{\{(?P [-\w]+)\|\[\[(?P [- \w]+)\]\]}}\s*\((?P [\w]+)\)(:?\s*\{\{(?P[mfcn])}})$',

# following are cases not caught by general regular expression # cases of link to FL wikt r'^\[\[:(?P [-a-z]+):(?P [^\|\]#\{}:]+)\|(?P [^\|\]#\{}:]]+)\]\]' '(?:\s*\{\{(?P[mfcnps])(?:\|(?P[fcnps])(?:\|(?P[cnps])|)|)}}|)$',

# three cases for a single unlinked word (don't allow phrases, trans might be one of the words) r'^(?P [\w]+)$', r'^(?P [\w]+)\s*\{\{(?P[mfcn])}}(?:\s*\{\{(?P[fcnps])}}|)$', r'^(?P [\w]+)$' '(?:\s*\{\{(?P[mfcnps])(?:\|(?P[fcnps])(?:\|(?P[cnps])|)|)}}|)$',

# other oddities, bold word (self page ref), m pl and friends. r"^(?P [\w]+)$", r"^\[\[(?P [- \w]+)\]\]\s+(?P[mfcn])\s+(?P[p])l$"   ]

Trex = [ ] for rel in Tlist: Trex.append(re.compile(rel, re.U))

# then we will try general case (long regex ;-) Grex = re.compile(r'^'       r'(?:\{\{(?P [-\w]+)\||)'          # script template, or nothing        r'\[\[(?P [^\|\]#\{}:]*)'          # start of link, always, any chars not link/template syntax        r'(?:#(?P [- \w]+)|)'             # possible section ref        r'(?:\|(?P [^\|\]#\{}:]+)|)'       # possible alt, any chars not syntax        r'\]\]'                                # end of link, always        r'(?:}}|)'                             # end of script if present (not checked)        r"(?:\s*\((?P [- '\w]+)\)|)"       # possible transliteration        r'(?:\s*\{\{(?P<g>[mfcnps])'           # possible gender or number r'(?:\|(?P<g2>[fcnps])'               # within that, possible second gender/number        r'(?:\|(?P<g3>[cnps])|)'               # within that, possible third gender/number or nothing        r'|)}}|)'                              # or not second gender, or none at all        r'$', re.U)                            # end of string, must match all, Unicode

# match an existing t template so we can create entry: retmatch = re.compile(r'\{\{t[+-]?\|(?P<lc>[^|]*)\|(?P [^|}]*)'           '(?:\|(?P<g>[mfcnps])(?:\|(?P<g2>[fcnps])(?:\|(?P<g3>[cnps])|)|)|)'            '(?:\|alt=(?P [^|}]*)|)(?:\|tr=(?P [^|}]*)|)(?:\|sc=(?P [^|}]*)|)'            '(?:\|xs=[^|}]*|)}}')

# merge in [one] following gender template, doesn't now put it in the canonical place [a problem] regmerge = re.compile(r'\{\{(t.?\|.*?)\}\}\s*\{\{([mfcnps])\}\}')

# try he-translation, without "defective" options, with genders, must match all rehetrans = re.compile(r'\{\{he-translation\|(?P [^|}]*)\|(?P [^|}]*)'           '(?:\|wv=(?P [^|}]*)|)}}'            '(?:\s*\{\{(?P<g>[mfcnps])(?:\|(?P<g2>[fcnps])(?:\|(?P<g3>[cnps])|)|)}}|)$')

# pre-screen matches for other entries presect = re.compile(r'^\* ?\w*: ?\[\[.*#', re.M)      # section ref in trans? preflnk = re.compile(r'\[\[:[-a-z]{2,3}:')             # FL link in trans? might be elsewhere

# now look for templates and t additions

entries = 0 probs = 0 fixed = 0

# start reader xt = threading.Thread(target=xread) xt.daemon = True # kill silently on exit (:-) xt.name = 'read XML' xt.start

tlocal.repact = True # main thread reports tsub details

while True: entry = pageq.get

if not entry: # end of pass logpage.add("%d entries, %d checked, %d edits" % (entries, probs, fixed)) logpage.write entries = 0 probs = 0 fixed = 0 # interruptable wait: while pageq.qsize == 0: sleep(20) continue # start next pass

text = entry.text title = entry.title if title.find(':') >= 0: continue

entries += 1 if entries % 100 == 0: with plock: print "%d entries, %d probs, %d edits, %d on queue" % \ (entries, probs, fixed, pageq.qsize)

act = ''

# check and update:

if True: # [structure]

with plock: print '%d: %s (%d)' % (probs, srep(title), pageq.qsize)

# ... pick up current version from en.wikt try: page = wikipedia.Page(site, title) # text = page.get text = getwikitext(page) origtext = text except wikipedia.NoPage: print "Can't get %s from en.wikt" % srep(page.aslink) text = '' except wikipedia.IsRedirectPage: print "Page %s is now a redirect" % srep(page.aslink) text = '' except KeyError: print "key error from the box" text = ''

if not text: continue

probs += 1 nap += 1 if nap > 70: nap = 70 # some outer bound

else: continue

# update cache time

ckey = 'en:' + title cache[ckey] = time # assume we will complete update now ... cis += 1 if cis % 20 == 0 or pageq.qsize == 0: cache.sync

# now parse the current entry see if we have a substitution

newtext = retemp.sub(tsub, text) if newtext != text: act += 'updated t/t-/t+ templates, ' text = newtext

# simple hack since we will tag over and over ;-)       if '{{rfc-tbot}}' in text:            act += 'tagged problem, '  # (or was previously tagged ;-)            text = text.replace('{{rfc-tbot}}{{rfc-tbot}}', '{{rfc-tbot}}')

# look for templates we can add or entries to create

added = False intrans = False pos = '' gloss = ''

lines = [ ] for line in text.splitlines: lines.append(line) for i in range(0, len(lines)):

mo = rehead.match(lines[i]) if mo: header = mo.group(1).strip if header == "Translations": intrans = True gloss = '' else: intrans = False if header in POS: pos = header if header in Stops: pos = '' continue

if lines[i].startswith(TTBC): intrans = False

if not intrans: continue

mo = regloss.match(lines[i]) if mo: gloss = mo.group(1).strip continue

# wiki format + one space, so lines are consistant if we change any lines[i] = restack.sub(r'\1 ', lines[i])

mo = retrans.match(lines[i]) if mo: waslinked = False else: mo = retransw.match(lines[1]) if mo: waslinked = True if not mo: continue

# if other cases, continue for now (not looking at the rest of the line) if '{{ko-inline' in lines[i]: continue # if '{{he-translation' in lines[i]: continue

lang = mo.group(1).strip trest = mo.group(2)

# print srep("t lang %s, rest %s" % (lang, trest))

# known language, wikt exists? if lang not in Langtoc: continue lc = Langtoc[lang] if lc not in Exists: continue

nomatch = False changed = False tnew = '' for ipart in trest.split(','): ipart = ipart.strip if not ipart: # consecutive commas, begins or ends with comma? nomatch = True continue

# see if existing t template but no entry, so we can create it               # this is the best case for create if someone (not tbot) added the template # also merge in following gender templates here

if '{{t' in ipart: ip2 = regmerge.sub(r'{{\1|\2}}', ipart) while ip2 != ipart: ipart = ip2 ip2 = regmerge.sub(r'{{\1|\2}}', ipart) changed = True mo = retmatch.search(ipart) if mo: mod = mo.groupdict if (mod['flw'] not in Lsects or lc not in Lsects[mod['flw']]) and pos and gloss and not nocreate: createFLentry(mod['flw'], lang, lc, pos, title, gloss, mod) tnew += ipart + ', ' continue

# same for Hebrew template, but must match all of the part if '{{he-translation' in ipart: mo = rehetrans.match(ipart) if mo: mod = mo.groupdict mod['scr'] = 'Hebr' if (mod['flw'] not in Lsects or lc not in Lsects[mod['flw']]) and pos and gloss and not nocreate: createFLentry(mod['flw'], lang, lc, pos, title, gloss, mod) tnew += ipart + ', ' continue

mo = None

# specific cases for rex in Trex: mo = rex.match(ipart) if mo: break

# general case if not mo: mo = Grex.match(ipart) if dmatch and mo: print "match general case: %s" % srep(ipart)

if mo: mod = mo.groupdict

# format t template: t = '{{t'

flw = mod['flw']  # always there, but can be nil if a section link # if self-link with section if not flw and 'sect' in mod and mod['sect']: flw = title if not flw: # can't happen? sect must exist and be non-nil to match nomatch = True continue

# try adding script, before the create FL step if 'scr' not in mod or not mod['scr']: scr = script(flw, lc) if scr: mod['scr'] = scr

# check out transliteration if 'tra' in mod and mod['tra']: tra = mod['tra'] # close enough to be fairly certain it is the transliteration? if fuzzy(tlit(flw), tlit(tra), len(tra)-2) < max(len(tra)-2, 4): if dmatch: print srep("no match trans: %s to %s" % (tlit(flw), tlit(tra))) nomatch = True continue # no more with this one

# create if missing, or check iwikis, or no match if flw != title and (flw not in Lsects or lc not in Lsects[flw]) and \ pos and gloss and not nocreate: if createFLentry(flw, lang, lc, pos, title, gloss, mod): t += '+' else: # don't convert yet if no confirm on match if flw not in Iwikis: nomatch = True elif lc in Iwikis[flw]: t += '+' else: t += '-' elif flw in Iwikis: if lc in Iwikis[flw]: t += '+' else: t += '-' else: nomatch = True

t += '|' + lc + '|' + flw

# if explicit iwiki, check code: if 'iwk' in mod and mod['iwk']: if mod['iwk'] != lc: print "explicit iwiki link does not match language %s" % srep(lang) nomatch = True

if 'sect' in mod and mod['sect'] and mod['sect'] != lang: print "section doesn't match language %s" % srep(lang) logpage.add("%s section %s doesn't match language %s" % \                                    (title, mod['sect'], lang)) nomatch = True

# in canonical order ... # gender if 'g' in mod and mod['g']: t += '|' + mod['g'] if 'g2' in mod and mod['g2']: t += '|' + mod['g2'] if 'g3' in mod and mod['g3']: t += '|' + mod['g3']

# alt link if 'alt' in mod and mod['alt'] and mod['alt'] != flw: alt = mod['alt'] t += '|alt=' + alt

# transliteration if 'tra' in mod and mod['tra']: t += '|tr=' + tra

# script if 'scr' in mod and mod['scr']: scr = mod['scr'] if scriptp(scr): if lc not in Xyzzy or scr != Xyzzy[lc]: t += '|sc=' + scr else: print srep("no match script: %s" % scr) nomatch = True

# xs last if lc not in Tlang: t += '|xs=' + lang

t += '}}' tnew += t + ', ' changed = True

else: if dmatch: print "no match pattern: %s" % srep(ipart) nomatch = True

if changed and not nomatch: tnew = tnew.strip(', ') if not waslinked: lines[i] = '* ' + lang + ': ' + tnew else: lines[i] = '* ' + lang + ': ' + tnew if 'added t for ' not in act: act += 'added t for ' if ' ' + lc + ',' not in act: act += lc + ', ' added = True with plock: print srep("%s to %s" % (trest, tnew))

pass # end of lines

# reassemble, else use existing text if added: text = u'\n'.join(lines)

# some change, write it       if act:

act = act.strip(', ') fixed += 1 nap /= 2

# use our own wait, not put throttle, so prescan can continue etc # also will be better when mwapi is doing puts sleep(70 - nap)

with plock: print "Updating %s: %s" % (srep(title), srep(act))

# try to fix the entry try: wikipedia.setAction(act) currtext = getedit(page) if currtext.strip('\n ') != origtext.strip('\n '): with plock: print "page changed while updating templates, not saved" continue with plock: page.put(text) # it is understood that taking plock here can stall the reader except wikipedia.EditConflict: print "Edit conflict?" continue except wikipedia.PageNotSaved: print "page not saved" continue except socket.timeout: print "socket timeout" continue except socket.error: print "socket error" continue except Exception, e:               print "some exception writing page", repr(e) continue

# limit number of fixes for testing # if fixed > 20: break

if nap > 5: with plock: print "(sleeping %d seconds)" % nap sleep(nap)

# [ notreached ] print "%d entries, %d possible updates, %d fixed" % (entries, probs, fixed) cache.close

# done

if __name__ == "__main__": try: main except KeyboardInterrupt: print "(keyboard interrupt)" finally: logpage.write wikipedia.stopme cache.close Iwikis.close Lsects.close