User:WindBOT/WikiCapResultsFooter

Source

capsConfig.py

# -*- coding: utf-8 -*-

config = {
	'api': 'http://wiki.teamfortress.com/w/api.php',
	'username': 'WindBOT',
	'password': raw_input('Wiki password: '),
	'logDir': 'logs',
	'pages': {
		'rules': 'User:WindBOT/WikiCapRules',
		'results': 'User:WindBOT/WikiCapCandidates',
		'log': 'User:WindBOT/WikiCapLog'
	},
	'mysql': {
		'host': 'mysql.biringa.com',
		'username': 'perot_irclog',
		'password': raw_input('MySQL password: '),
		'database': 'perot_irclog',
		'table': 'tfwikirc'
	}
}

countCaps.py

#!/usr/bin/python -OO
# -*- coding: utf-8 -*-
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import shutil
import re # Regular expressions
import wikitools # Wiki API
import datetime # Date manipulation and comparison
import MySQLdb # MySQL databases
import MySQLdb.cursors # Additional cursor classes for convenience

from capsConfig import config
config['runtime'] = {
	'wiki': None,
	'regexes': {}
}
config['rules'] = None

def pr(*args):
	s = []
	for i in args:
		s.append(u(i))
	s = u' '.join(s)
	try:
		print s
	except:
		pass
	if config['output'] is not None:
		try:
			print >> config['output'], s
		except:
			pass

def u(s):
	if type(s) is type(u''):
		return s
	if type(s) is type(''):
		try:
			return unicode(s)
		except:
			try:
				return unicode(s.decode('utf8'))
			except:
				try:
					return unicode(s.decode('windows-1252'))
				except:
					return unicode(s, errors='ignore')
	try:
		return unicode(s)
	except:
		try:
			return u(str(s))
		except:
			return s

def wiki():
	global config
	if config['runtime']['wiki'] is None:
		config['runtime']['wiki'] = wikitools.wiki.Wiki(config['api'])
		pr('Logging in as', config['username'], '...')
		config['runtime']['wiki'].login(config['username'], config['password'])
		pr('Logged in.')
	return config['runtime']['wiki']
def page(p):
	if type(p) in (type(''), type(u'')):
		p = wikitools.page.Page(wiki(), u(p), followRedir=False)
	return p
def editPage(p, content, summary=u'', minor=True, bot=True, nocreate=True):
	global config
	summary = u(summary)
	while len(summary) > 250:
		if summary.find(u' ') == -1:
			summary = summary[:summary.rfind(u' ')] + u'...'
		else:
			summary = summary[:247] + u'...'
	result = page(p).edit(u(content), summary=summary, minor=minor, bot=bot, nocreate=nocreate)
	return result
def compileRegex(regex, flags=re.IGNORECASE):
	global config
	regex = u(regex)
	if regex in config['runtime']['regexes']:
		return config['runtime']['regexes'][regex]
	config['runtime']['regexes'][regex] = re.compile(regex, flags)
	return config['runtime']['regexes'][regex]
def userSort(x, y):
	global config
	for k in config['rules']['sortKeys']:
		reverse = False
		if k[0] == '!':
			k = k[1:]
			reverse = True
		if reverse:
			r = cmp(y[k], x[k])
		else:
			r = cmp(x[k], y[k])
		if r == 0:
			continue
		return r
	return 0
def createUser(username):
	global config
	logFile = u(config['logDir']) + u(os.sep) + u(username).replace(u' ', u'_') + u'.log'
	l = open(logFile, 'wb')
	l.write('')
	l.close()
	return {
		'name': username,
		'score': 0.0,
		'firstEdit': None,
		'lastEdit': None,
		'editDate': {},
		'daysOff': 0,
		'hasTalked': False,
		'isBot': False,
		'edits': 0,
		'undone': 0,
		'redirects': 0,
		'newPages': 0,
		'creationTime': None,
		'log': logFile
	}
def run():
	global config, u
	# Clear log directory
	if os.path.exists(config['logDir']):
		shutil.rmtree(config['logDir'])
	os.mkdir(config['logDir'])
	rulesText = u(page(config['pages']['rules']).getWikiText())[len('<p'+'re><now'+'iki>'):-len('</now'+'iki></pr'+'e>')]
	pr('Rules:', rulesText)
	config['rules'] = eval(rulesText)
	rules = config['rules'] # Convenience
	now = datetime.datetime.utcnow().replace(tzinfo=wikitools.utc.utc)
	firstInspectedDay = now - datetime.timedelta(days = rules['inspectDuration'])
	undoRegex = compileRegex(r'^(?:Undo edit by|Reverted edits by).*?\[\[Special:Contributions/([^]|]+)(?:.*?\((\d+)\))?')
	mysql = MySQLdb.connect(host=config['mysql']['host'], user=config['mysql']['username'], passwd=config['mysql']['password'], db=config['mysql']['database'], cursorclass=MySQLdb.cursors.DictCursor)
	cursor = mysql.cursor()
	dbLimit = now - datetime.timedelta(days=rules['inspectDuration'] + rules['maxDaysOff'] + 1)
	cursor.execute('SELECT * FROM `tfwikirc`')
	stats = {}
	editScores = {}
	def log(user, *msg):
		s = [u'<' + u(user) + u'>']
		for i in msg:
			s.append(u(i))
		log = open(stats[user]['log'], 'ab')
		log.write((u' '.join(s) + u'\n').encode('utf8'))
		log.close()
		pr(u'<' + u(user) + u'>', *msg)
	while True:
		row = cursor.fetchone()
		if row == None:
			break
		row['timestamp'] = row['timestamp'].replace(tzinfo=wikitools.utc.utc)
		#pr(row)
		if not row['user'] in stats:
			stats[row['user']] = createUser(row['user'])
		rowScore = 0.0
		for n in rules['namespaces']:
			if rules['namespaces'][n]['id'] == row['namespace']:
				if rules['namespaces'][n]['points'] > 0.0:
					stats[row['user']]['edits'] += 1
					# First, check if edit is not too old
					if (now - row['timestamp']).days >= rules['inspectDuration']:
						break
					# Take care of daysOff
					if stats[row['user']]['firstEdit'] is None:
						stats[row['user']]['firstEdit'] = row['timestamp']
						daysLost = max(0, (firstInspectedDay - row['timestamp']).days - 1)
						if daysLost > 0:
							log(row['user'], 'Days from beginning of inspected duration to first edit:', daysLost, '- Comparing', firstInspectedDay, '(inspected duration) to ', row['timestamp'], '(first edit)')
						stats[row['user']]['daysOff'] += daysLost
					else:
						daysSinceLast = max(0, (row['timestamp'] - stats[row['user']]['lastEdit']).days - 1)
						if daysSinceLast > 0:
							log(row['user'], 'Days since last edit:', daysSinceLast, '- Comparing', stats[row['user']]['lastEdit'], '(previous edit) to', row['timestamp'], '(current edit)')
						stats[row['user']]['daysOff'] += daysSinceLast
					stats[row['user']]['lastEdit'] = row['timestamp']
					# Handle consecutive edits
					if row['title'] in stats[row['user']]['editDate']:
						delta = row['timestamp'] - stats[row['user']]['editDate'][row['title']]
						if delta.days * 86400 + delta.seconds < rules['consecutiveThreshold']:
							stats[row['user']]['editDate'][row['title']] = row['timestamp']
							break
					stats[row['user']]['editDate'][row['title']] = row['timestamp']
					# Now compute points
					rowScore += rules['namespaces'][n]['points']
					scoreBonus = rules['namespaces'][n]['points']
					if row['flags'].find('N') != -1:
						scoreBonus = scoreBonus * rules['newPageBonus'] + row['newsize'] * rules['newPageByteValue']
						stats[row['user']]['newPages'] += 1
					if row['flags'].find('R') != -1:
						scoreBonus *= rules['redirect']
						stats[row['user']]['redirects'] += 1
					editScores[str(row['rcid'])] = scoreBonus
					log(row['user'], 'got', scoreBonus, 'points for editing', row['title'], '(Flags:', row['flags']+')')
					stats[row['user']]['score'] += scoreBonus
					if row['flags'].find('b') != -1:
						stats[row['user']]['isBot'] = True
					# Handle undo's
					undo = undoRegex.search(row['comment'])
					if undo:
						if undo.group(2) and undo.group(2) in editScores:
							penalty = editScores[undo.group(2)] * rules['undoMultiplier']
						else:
							penalty = rules['undone']
						if undo.group(1) not in stats:
							stats[undo.group(1)] = createUser(undo.group(1))
						log(undo.group(1), 'lost', abs(penalty), 'for edit undone on page', row['title'])
						stats[undo.group(1)]['score'] += penalty
						if row['comment'][:8] == 'Reverted': # Penalize users using rollback
							log(row['user'], 'lost', abs(rules['rollbackPenalty']), 'points for using rollback on page', row['title'])
							stats[row['user']]['score'] += rules['rollbackPenalty']
						stats[undo.group(1)]['undone'] += 1
				if rules['namespaces'][n]['isTalk']:
					stats[row['user']]['hasTalked'] = True
				break
	users = []
	pr('Building users table.')
	ministatkeys = ('score', 'edits', 'daysOff', 'lastEdit')
	for usr in stats:
		ministats = {}
		for k in ministatkeys:
			ministats[k] = stats[usr][k]
		log(usr, 'Info:', ministats)
		if stats[usr]['lastEdit'] is None:
			log(usr, 'Dropping - No edit')
			continue
		daysSinceLast = max(0, (now - stats[usr]['lastEdit']).days - 1) # Days from last edit to moment of counting
		if daysSinceLast > 0:
			log(usr, 'Dropping - Days since last edit until now:', daysSinceLast, '- Comparing', stats[usr]['lastEdit'], 'to', now)
		stats[usr]['daysOff'] += daysSinceLast
		if stats[usr]['isBot']: # Skip bots
			log(usr, 'Dropping - It\'s a bot')
			continue
		if usr in rules['owners']:
			log(usr, 'Dropping - Already has cap')
			continue
		if stats[usr]['score'] < rules['minScore']: # Skip too low scores
			log(usr, 'Dropping - Score:', stats[usr]['score'], '<', rules['minScore'])
			continue
		if stats[usr]['edits'] < rules['minEdits']: # Skip too low edit count
			log(usr, 'Dropping - Edits:', stats[usr]['edits'], '<', rules['minEdits'])
			continue
		if stats[usr]['daysOff'] > rules['maxDaysOff']: # Skip too many days off
			log(usr, 'Dropping - Days off:', stats[usr]['daysOff'], '>', rules['maxDaysOff'])
			continue
		stats[usr]['creationTime'] = wikitools.user.User(wiki(), usr).registration
		if (now - stats[usr]['creationTime']).days < rules['minAge']: # Skip too young accounts
			pr(usr, 'Dropping - Age:', stats[usr]['creationTime'], '<', rules['minAge'])
			continue
		pr('Accepting candidate:', usr)
		users.append(stats[usr])
	config['users'] = users
	pr('Sorting users.')
	users = sorted(users, cmp=userSort, reverse=True)
	pr('Building Wiki table.')
	s = """{| class="wikitable grid sortable" align="center" style="text-align:center"
! class="header" style="font-size:90%;" | User
! class="header" style="font-size:90%;" | Score
! class="header" style="font-size:90%;" | Inactive days
! class="header" style="font-size:90%;" | Undone edits
! class="header" style="font-size:90%;" | Valid edits ratio
! class="header" style="font-size:90%;" | New pages
! class="header" style="font-size:90%;" | Redirects
! class="header" style="font-size:90%;" | Age
! class="header" style="font-size:90%;" | Edits (''all time'')
! class="header" style="font-size:90%;" | Has talked?
! class="header unsortable"  style="font-size:90%;" | Contribs"""
	s2 = u''
	i = 0
	for usr in users:
		i += 1
		if i > rules['maxCandidates']:
			break
		pr(usr['name'], usr['score'])
		hasTalked = 'Yes'
		if not usr['hasTalked']:
			hasTalked = 'No'
		s += '\n\
|-\n\
! {{subst:ul|' + usr['name'] + '}}\n\
| ' + str(round(usr['score'], 2)) + '\n\
| ' + str(usr['daysOff']) + ' (' + str(round(100.0 * (1.0-float(usr['daysOff'])/float(rules['inspectDuration'])), 2)) + '%)' + '\n\
| ' + str(usr['undone']) + '\n\
| ' + str(round((1.0-float(usr['undone'])/float(usr['edits']))*100.0, 2)) + '%\n\
| ' + str(usr['newPages']) + '\n\
| ' + str(usr['redirects']) + '\n\
| ' + str((now - usr['creationTime']).days) + ' days\n\
| ' + str(usr['edits']) + '\n\
| ' + hasTalked + '\n\
| [[Special:Contributions/' + usr['name'] + '|' + usr['name'] + '\'s Contribs]]'
		s2 += u'=== ' + usr['name'] + u' ===\n<p'+'re><no'+'wiki>' + u(open(usr['log'], 'rb').read(-1)) + u'</no'+'wiki></pr'+'e>\n'
	s+='\n\
|}'
	editPage(config['pages']['results'], rules['template'].replace('%table%', s), summary=u'Updated Wiki Cap candidates list.', minor=False)
	editPage(config['pages']['log'], s2, summary=u'Updated Wiki Cap log.', minor=False)
if __name__ == '__main__':
	config['output'] = open('caps.txt', 'wb')
	run()
	config['output'].close()