class DictionaryUpdater:
def __init__(self):
self.subpageTemplateLang = """{{#switch:{{{lang|{{SUBPAGENAME}}}}}|%options%}}"""
self.subpageTemplateParam = """{{#switch:{{{1|}}}|%options%}}"""
self.invalidParamError = """<span class="error">Error: invalid param.</span>[[Category:ERROR]]"""
self.subpageTemplateID = """%string%"""
self.dictionaries = {
u'Template:Dictionary/advanced': { # Dictionary page
'name': 'advanced', # Dictionary name (used for categorizing)
'sync': 'Template:Dictionary/advanced/Special:SyncData' # Page holding last sync data
},
u'Template:Dictionary/items': { # Dictionary page
'name': 'items', # Dictionary name (used for categorizing)
'sync': 'Template:Dictionary/items/Special:SyncData' # Page holding last sync data
},
u'Template:Dictionary/common strings': { # Warning: no underscore
'name': 'common strings',
'sync': 'Template:Dictionary/common strings/Special:SyncData'
},
u'Template:Dictionary/classes': {
'name': 'classes',
'sync': 'Template:Dictionary/classes/Special:SyncData'
},
u'Template:Dictionary/demonstrations': {
'name': 'demonstrations',
'sync': 'Template:Dictionary/demonstrations/Special:SyncData'
},
u'Template:Dictionary/price': {
'name': 'price',
'sync': 'Template:Dictionary/price/Special:SyncData',
'allTemplate': '{{{{{template|item price/fmt}}}|%options%|tt={{{tt|yes}}}}}'
},
u'Template:Dictionary/merchandise': {
'name': 'merchandise',
'sync': 'Template:Dictionary/merchandise/Special:SyncData'
},
u'Template:Dictionary/steam ids': {
'name': 'steam ids',
'sync': 'Template:Dictionary/steam ids/Special:SyncData'
},
u'Template:Dictionary/achievements/scout': {
'name': 'achievements/scout',
'sync': 'Template:Dictionary/achievements/scout/Special:SyncData'
},
u'Template:Dictionary/achievements/soldier': {
'name': 'achievements/soldier',
'sync': 'Template:Dictionary/achievements/soldier/Special:SyncData'
},
u'Template:Dictionary/achievements/pyro': {
'name': 'achievements/pyro',
'sync': 'Template:Dictionary/achievements/pyro/Special:SyncData'
},
u'Template:Dictionary/achievements/demoman': {
'name': 'achievements/demoman',
'sync': 'Template:Dictionary/achievements/demoman/Special:SyncData'
},
u'Template:Dictionary/achievements/heavy': {
'name': 'achievements/heavy',
'sync': 'Template:Dictionary/achievements/heavy/Special:SyncData'
},
u'Template:Dictionary/achievements/engineer': {
'name': 'achievements/engineer',
'sync': 'Template:Dictionary/achievements/engineer/Special:SyncData'
},
u'Template:Dictionary/achievements/medic': {
'name': 'achievements/medic',
'sync': 'Template:Dictionary/achievements/medic/Special:SyncData'
},
u'Template:Dictionary/achievements/sniper': {
'name': 'achievements/sniper',
'sync': 'Template:Dictionary/achievements/sniper/Special:SyncData'
},
u'Template:Dictionary/achievements/spy': {
'name': 'achievements/spy',
'sync': 'Template:Dictionary/achievements/spy/Special:SyncData'
},
u'Template:Dictionary/achievements/general': {
'name': 'achievements/general',
'sync': 'Template:Dictionary/achievements/general/Special:SyncData'
},
u'Template:Dictionary/achievements/halloween': {
'name': 'achievements/halloween',
'sync': 'Template:Dictionary/achievements/halloween/Special:SyncData'
},
u'Template:Dictionary/achievements/treasure hunt': {
'name': 'achievements/treasure hunt',
'sync': 'Template:Dictionary/achievements/treasure hunt/Special:SyncData'
},
u'Template:Dictionary/achievements/replay': {
'name': 'achievements/replay',
'sync': 'Template:Dictionary/achievements/replay/Special:SyncData'
},
u'Template:Dictionary/achievements/summer camp': {
'name': 'achievements/summer camp',
'sync': 'Template:Dictionary/achievements/summer camp/Special:SyncData'
},
u'Template:Dictionary/achievements/foundry': {
'name': 'achievements/foundry',
'sync': 'Template:Dictionary/achievements/foundry/Special:SyncData'
},
u'Template:Dictionary/achievements/christmas': {
'name': 'achievements/christmas',
'sync': 'Template:Dictionary/achievements/christmas/Special:SyncData'
},
u'Template:Dictionary/blueprints': {
'name': 'blueprints',
'sync': 'Template:Dictionary/blueprints/Special:SyncData'
},
u'Template:Dictionary/templatecore': {
'name': 'templatecore',
'sync': 'Template:Dictionary/templatecore/Special:SyncData'
},
u'Template:Dictionary/defindex': {
'name': 'defindex',
'sync': 'Template:Dictionary/defindex/Special:SyncData'
},
u'Template:Dictionary/quad': {
'name': 'quad',
'sync': 'Template:Dictionary/quad/Special:SyncData',
'blankString': '-'
},
u'Template:Dictionary/item set weapons': {
'name': 'item set weapons',
'sync': 'Template:Dictionary/item set weapons/Special:SyncData'
}
}
self.subpageSeparator = u'/'
# List of supported languages, in prefered order
self.languages = [u'en', u'ar', u'cs', u'da', u'de', u'es', u'fi', u'fr', u'hu', u'it', u'ja', u'ko', u'nl', u'no', u'pl', u'pt', u'pt-br', u'ro', u'ru', u'sv', u'tr', u'zh-hans', u'zh-hant']
self.defaultLang = u'en'
self.allKeyName = u'_all_'
self.filterName = u'Your friendly neighborhood dictionary updater'
self.commentsExtract = compileRegex(r)
self.stringsExtract = compileRegex(r'(?:^[ \t]*#[ \t]*([^\r\n]*?)[ \t]*$\s*)?^[ \t]*([^\r\n]+?[ \t]*(?:\|[ \t]*[^\r\n]+?[ \t]*)*):[ \t]*([^\r\n]+?[ \t]*$|\s*[\r\n]+(?:\s*[ \t]+[-\w]+[ \t]*:[ \t]*[^\r\n]+[ \t]*$)+)', re.IGNORECASE | re.MULTILINE)
self.translationExtract = compileRegex(r'^[ \t]+([-\w]+)[ \t]*:[ \t]*([^\r\n]+)[ \t]*$', re.IGNORECASE | re.MULTILINE)
self.scheduler = BatchScheduler(16)
addWhitelistPage(self.dictionaries.keys())
def generateSubpage(self, keyName, data, currentDict, syncData):
h = hashlib.md5()
if type(data) is type({}): # Subkeys (translations or not)
isTranslation = True
subpage = u(self.subpageTemplateLang)
for k in data:
if 'blankString' in self.dictionaries[currentDict] and data[k] == self.dictionaries[currentDict]['blankString']:
data[k] = u
if isTranslation and k not in self.languages:
isTranslation = False
subpage = u(self.subpageTemplateParam)
ordered = []
unordered = {}
if isTranslation:
missing = []
for lang in self.languages:
if lang in data:
ordered.append(lang + u'=' + data[lang])
unordered[lang] = data[lang]
h.update((lang + u'=' + data[lang]).encode('utf8'))
else:
missing.append(lang)
h.update((u'null-' + lang).encode('utf8'))
if self.defaultLang in data:
ordered.insert(0, u'#default=' + data[self.defaultLang])
if len(missing):
subpage = subpage.replace(u'%missing%', u"Languages missing: " + u', '.join(missing))
else:
subpage = subpage.replace(u'%missing%', u"Supported languages: all")
else: # Not a translation
h.update('Any-')
subkeys = data.keys()
subkeys.sort()
for k in subkeys:
ordered.append(k + u'=' + data[k])
unordered[k] = data[k]
h.update((k + u'=' + data[k]).encode('utf8'))
if 'allTemplate' in self.dictionaries[currentDict] and (len(unordered) or len(self.dictionaries[currentDict]['allTemplate']['params'])):
allKey = []
keys = unordered.keys()
keys.sort()
for k in keys:
allKey.append(k + u'=' + unordered[k])
insertIndex = 0
if isTranslation and self.defaultLang in data:
insertIndex = 1
ordered.insert(insertIndex, u(self.allKeyName) + u'=' + u(self.dictionaries[currentDict]['allTemplate'].replace(u'%options%', u'|'.join(allKey))))
subpage = subpage.replace(u'%options%', u'|'.join(ordered))
else: # No subkeys
data = u(data)
subpage = self.subpageTemplateID
h.update(u(u'ID-' + data).encode('utf8'))
subpage = subpage.replace(u'%string%', data)
h = u(h.hexdigest())
if keyName in syncData and syncData[keyName] == h:
return # Same hash
syncData[keyName] = h # Update sync data
subpage = subpage.replace(u'%dictionary%', currentDict)
subpage = subpage.replace(u'%dictionaryname%', self.dictionaries[currentDict]['name'])
subpage = subpage.replace(u'%keyname%', keyName)
self.scheduler.schedule(editPage, currentDict + self.subpageSeparator + keyName, subpage, summary=u'Pushed changes from [[:' + currentDict + u']] for string "' + keyName + u'".', minor=True, nocreate=False)
def processComment(self, commentString, currentDict, definedStrings, syncData):
commentContents = []
for extractedStr in self.stringsExtract.finditer(commentString):
comment = u
if extractedStr.group(1):
comment = u'# ' + u(extractedStr.group(1)) + u'\n'
dataString = u(extractedStr.group(3))
if dataString.find(u'\r') == -1 and dataString.find(u'\n') == -1: # Assume no subkeys
data = dataString.strip()
dataWriteback = u' ' + data
else: # There's subkeys; detect whether this is a translation or not
data = {}
isTranslation = True
for translation in self.translationExtract.finditer(dataString.rstrip()):
data[u(translation.group(1))] = u(translation.group(2))
if u(translation.group(1)) not in self.languages:
isTranslation = False
ordered = []
if isTranslation:
for lang in self.languages:
if lang in data:
ordered.append(u' ' + lang + u': ' + data[lang])
else: # Not a translation, so order in alphabetical order
subkeys = data.keys()
subkeys.sort()
for subk in subkeys:
ordered.append(u' ' + subk + u': ' + data[subk])
dataWriteback = u'\n' + u'\n'.join(ordered)
keyNames = u(extractedStr.group(2)).lower().split(u'|')
validKeyNames = []
for keyName in keyNames:
keyName = keyName.replace(u'_', u' ').strip()
if keyName in definedStrings:
continue # Duplicate key
definedStrings.append(keyName)
validKeyNames.append(keyName)
self.generateSubpage(keyName, data, currentDict, syncData)
if len(validKeyNames):
commentContents.append(comment + u' | '.join(validKeyNames) + u':' + dataWriteback)
self.scheduler.execute()
return u'\n\n'.join(commentContents)
def __call__(self, content, **kwargs):
if 'article' not in kwargs:
return content
if u(kwargs['article'].title) not in self.dictionaries:
return content
currentDict = u(kwargs['article'].title)
syncPage = page(self.dictionaries[currentDict]['sync'])
try:
syncDataText = u(syncPage.getWikiText()).split(u'\n')
except: # Page probably doesn't exist
syncDataText = u
syncData = {}
for sync in syncDataText:
sync = u(sync.strip())
if not sync:
continue
sync = sync.split(u':', 2)
if len(sync) == 2:
syncData[sync[0]] = sync[1]
oldSyncData = syncData.copy()
newContent = u
previousIndex = 0
definedStrings = []
for comment in self.commentsExtract.finditer(content):
newContent += content[previousIndex:comment.start()]
previousIndex = comment.end()
# Process current comment
newContent += u
newContent += content[previousIndex:]
# Check if we need to update sync data
needUpdate = False
for k in syncData:
if k not in oldSyncData or oldSyncData[k] != syncData[k]:
needUpdate = True
break
# Check for deleted strings
for k in oldSyncData:
if k not in definedStrings:
try:
deletePage(currentDict + self.subpageSeparator + k, 'Removed deleted string "' + k + u'" from ' + currentDict + u'.')
except:
pass
if k in syncData:
del syncData[k]
needUpdate = True
if needUpdate:
# Build syncdata string representation
syncKeys = syncData.keys()
syncKeys.sort()
syncLines = []
for k in syncKeys:
syncLines.append(k + u':' + syncData[k])
editPage(syncPage, u'\n'.join(syncLines), summary=u'Updated synchronization information for [[:' + currentDict + u']].', minor=True, nocreate=False)
return newContent
def scheduledRun(self):
for d in self.dictionaries:
fixPage(d)
dictUpdater = DictionaryUpdater()
addFilter(dictUpdater)
scheduleTask(dictUpdater.scheduledRun, 3)
