User:CrushBOT/Filters

Home	Filters	RCID	Blacklist	Page Requests	Page Requests (Force)	Edits	Edit Count

How to disable a filter

If the bot is malfunctioning, chances are that the problem lies in one of these blocks of code. Thus, instead of shutting down the whole bot, it would be wiser to disable only the chunk of code that is misbehaving. To make the bot ignore a certain line, add a "#" in front of it:

# This line will be ignored

If there are multiple lines, wrap them inside triple-quotes (you still need to put the two spaces at the beginning of the line):

"""This line will be ignored
and this one as well
and this one is cake
and the previous one was a lie but it was still ignored"""

If all else fails, you can simply delete the block from the page. The bot can't come up with code by itself yet, so it won't run anything. Or, if the problem really is elsewhere, block the bot.

File filters

Crush all PNG/JPG images

class imageCrushFilter:
    def __init__(self):
        self.minRatio = 10 # Compression ratio threshold
        self.minByteDiff = 2048 # Byte difference threshold
        self.jpgScanMap = u'0:   0  0 0 0 ;1 2: 0  0 0 0 ;0:   1  8 0 2 ;1:   1  8 0 0 ;2:   1  8 0 0 ;0:   9 63 0 2 ;0:   1 63 2 1 ;0:   1 63 1 0 ;1:   9 63 0 0 ;2:   9 63 0 0 ;'.replace(u';', u';\n')
        self.filterName = 'Saved crush information'
        self.extractHash = compileRegex(r'\{\{(?:png)?crush\s*\|\s*(\w+?)\s*\|\s*(\w+?)\s*}}')
        if programExists('pngout'):
            self.pngenabled = 'pngout'
        elif programExists('pngcrush'):
            self.pngenabled = 'pngcrush'
        else:
            print 'Warning: PNGOut and PNGCrush are not installed or are not in $PATH'
            self.pngenabled = None
        self.jpgenabled = programExists('jpegtran')
        if not self.jpgenabled:
            print 'Warning: jpegtran is not installed or not in $PATH'
    def __call__(self, content, article, **kwargs):
        title = u(article.title).lower()
        if title[-4:] == '.png':
            isPNG = True
            if self.pngenabled is None:
                return content
        elif title[-5:] == '.jpeg' or title[-4:] == '.jpg':
            isPNG = False
            if not self.jpgenabled:
                return content
        else:
            return content
        try: # This is a high-risk filter, lots of I/O, so wrap it in a big try
            filePage = wikitools.wikifile.File(wiki(), article.title)
            hashes = [u, u]
            hashResult = self.extractHash.search(content)
            hashes = None
            hashTemplate = None
            if hashResult:
                hashes = (u(hashResult.group(1)).lower(), u(hashResult.group(2)).lower())
                hashTemplate = u'{{crush|' + hashes[0] + u'|' + hashes[1] + u'}}'
            tempFile = getTempFilename(extension='png')
            filePage.download(location=tempFile, urlQuery=u(getRandBits()))
            oldHash = getFileHash(tempFile)
            if hashes is not None and oldHash in hashes:
                return content # Already worked on that one
            hashTemplate = u'{{crush|' + oldHash + u'|None}}'
            tempOutput = getTempFilename(extension='png')
            if isPNG:
                if self.pngenabled == 'pngout':
                    shutil.copyfile(tempFile, tempOutput)
                    result = subprocess.call(['pngout', '-b256', '-y', tempOutput])
                    if result == 0:
                        possibleBlocks = ['0', '64', '128', '192', '256', '512', '1024', '2048']
                        currentUpperBlock = 4
                        currentLowerBlock = 4
                        tryUpper = True
                        tryLower = True
                        while tryUpper or tryLower:
                            if tryUpper:
                                currentUpperBlock += 1
                                upperResult = subprocess.call(['pngout', '-b' + possibleBlocks[currentUpperBlock], '-y', tempOutput])
                                if upperResult != 0 or currentUpperBlock >= len(possibleBlocks) - 1:
                                    tryUpper = False
                            if tryLower:
                                currentLowerBlock -= 1
                                lowerResult = subprocess.call(['pngout', '-b' + possibleBlocks[currentLowerBlock], '-y', tempOutput])
                                if lowerResult != 0 or currentLowerBlock <= 0:
                                    tryLower = False
                elif self.pngenabled == 'pngcrush':
                    result = subprocess.call(['pngcrush', '-q', '-l', '9', '-reduce', '-rem', 'gAMA', '-rem', 'cHRM', '-rem', 'iCCP', '-rem', 'sRGB'] + [i for l in [('-m', str(i)) for i in range(138)] for i in l] + [tempFile, tempOutput])
            else:
                result = subprocess.call(['jpegtran', '-o', '-copy', 'none', '-progressive', '-outfile', tempOutput, tempFile])
            oldSize = os.path.getsize(tempFile)
            newSize = os.path.getsize(tempOutput)
            deleteFile(tempFile)
            if not result and oldSize > newSize:
                # Ready to upload... or are we?
                ratio = int(round(100 * (1.0 - float(newSize) / float(oldSize))))
                if ratio >= self.minRatio or oldSize - newSize >= self.minByteDiff:
                    newHash = getFileHash(tempOutput)
                    if hashes is not None and newHash in hashes:
                        deleteFile(tempOutput)
                        return content # Already got that result, no need to reupload
                    hashTemplate = u'{{crush|' + oldHash + u'|' + newHash + u'}}'
                    content2 = wikitools.wikifile.File(wiki(), article.title).getWikiText()
                    hashResult2 = self.extractHash.search(content2)
                    hashes2 = None
                    if hashResult2:
                        hashes2 = (u(hashResult2.group(1)), u(hashResult2.group(2)))
                    if hashes == hashes2:
                        uploadFile(tempOutput, u(article.title), u'Crushed version: ' + u(ratio) + u'% reduction / ' + u(oldSize - newSize) + u' bytes saved; from ' + u(oldSize) + u' to ' + u(newSize) + u' bytes.', overwrite=True, reupload=True)
                    hashes = (oldHash, newHash)
            if hashResult:
                content = content[:hashResult.start()] + hashTemplate + content[hashResult.end():]
            else:
                content = content.strip() + u'\n\n' + hashTemplate
            deleteFile(tempOutput)
        except:
            pass # Well, that didn't work
        return content
addFileFilter(imageCrushFilter())