User:OrphicBot/updateReferences.py

import re, unicodedata, random
from getpass import getpass
from orphicbot import save, load, loadDB, foliage, wiki, saveMW, login
from orphicbot import ffSort, run, runAllM, R, L, lR, fWithRefs, pageToLexica, txtToTable, loadTitles
from utils import readFile, unique, hashListByKey, txtToFile, tuplesToFile, fileToTuples, flatten, show, showTwo, appose, reGroups, appose, dictToList, nfc, unaccented, is_number
from itertools import dropwhile

def makeAddRefs(lex, lix) :
  pInLex, pLexNoIndex, pMaybeDGE = lambda t,l : t in lex[l], lambda l : not l in lex,  lambda t,l : l != 'DGE' or unaccented(t).lower() <= unaccented(readFile('dgeMax.txt')[1:]).lower()
  pInAnyLex = lambda t : any([pInLex(t,L) for L in lex])
  pWordLex, dToL, T = lambda t,l : pInLex(t,l) or (pLexNoIndex(l) and pInAnyLex(t) and pMaybeDGE(t,l) and t[0].islower()), dict([(l[0], l[2]) for l in lix]), hashListByKey(lix, lambda x: x[0], lambda x: x[3])
  sortOrder, dLangs = [("==References==", 0)] + [('{{'+x[3], x[1]) for x in lix] + [("<references/>", 9999)], dict([(x[0], unique(x[1])) for x in dictToList(hashListByKey([(w,l) for l in lex for w in lex[l]], lambda x: x[0], lambda x : dToL[x[1]]))])
  def addRefs(ls,t) :
    Ls = dLangs[t]
    fNorm = ("Normalised entry.", lambda LS,B : list(reversed(list(dropwhile(lambda x: x=='', reversed(B))))) + [''])
    fRef = ("References added.", lambda LS,B : [B[0]]+[''.join(['* {{{{{}}}}}\n'.format(T[l][0]) for l in lex if not any(['{{'+T[l][0] in b for b in B]) and pWordLex(t,l) ])+(B[1] if len(B) > 1 else '')]+B[2:])  
    fRefSort = ("References sorted.", ffSort(lambda x : 99999 if x == '' else len(sortOrder) if not any([v in x for (v,n) in sortOrder]) else [n for (v,n) in sortOrder if v in x][0]))
    fXRefEmpty = ("", lambda LS,B : [] if B[0][0:2]=="==" and B[0][-2:]== "==" and (len(B) == 1 or B[1] == '') else B)
    fs = [([lR], fNorm), ([lR], fRef), ([lR], fRefSort), ([lR], fNorm), ([lR], fXRefEmpty)]
    return run(t,Ls,fWithRefs(ls, t, Ls),fs)
  return addRefs

lex, lix, CT = pageToLexica('User:OrphicBot/hws.txt.gz.b64'), txtToTable(load('User:OrphicBot/hwsIndex.txt.gz.b64')), login(input("Username: "), getpass("Password: "))

lex = dict([(l, [w for w in lex[l] if not (l == 'Zoega' and len(reGroups('^([a-zA-Z]*)$', w)) == 0 ) and not (l == 'DuCange' and len(reGroups('((?:[aei]re*)|(?:[aei]ri))$', w)) == 1 ) ]) for l in lex])

addRefs = makeAddRefs(lex,lix)

words, titles, fAddRefs, save = unique(flatten(lex[l] for l in lex)), unique(loadTitles(CT = CT)) if False else [x[0] for x in select(db, "select distinct name from enwiktionary")], makeAddRefs(lex, lix), lambda w,x,c : saveMW(CT, w,x,c)

sTitles = set(titles)

agenda = [w for w in words if w.lower() in sTitles or w in sTitles]

aps = dict([(a,p) for (a,(p,t)) in zip(agenda,loadMW(CT, agenda))])

agenda = [a for a in agenda if '==Latin==' in aps[a] or '==Ancient Greek==' in aps[a] or '==Old Norse==' in aps[a]]

ws, rvs = agenda, [None for x in agenda]

#rvs = runAllM(ws, addRefs, fSave = lambda a,b,c : saveMW(CT,a,b,c), rvs = rvs)