User:WindBOT/Filters

How to disable a filter
If the bot is malfunctioning, chances are that the problem lies in one of these blocks of code. Thus, instead of shutting down the whole bot, it would be wiser to disable only the chunk of code that is misbehaving. To make the bot ignore a certain line, add a "#" in front of it: If there are multiple lines, wrap them inside triple-quotes (you still need to put the two spaces at the beginning of the line): """This line will be ignored and this one as well and this one is cake and the previous one was a lie but it was still ignored""" If all else fails, you can simply delete the block from the page. The bot can't come up with code by itself yet, so it won't run anything. Or, if the problem really is elsewhere, [ block the bot].
 * 1) This line will be ignored

Page filters
addPageFilter(r'^user:', r'(?:talk|help|wiki|template):')

Implement Dictionary
class DictionaryUpdater: def __init__(self): self.subpageTemplateLang = """  \n: Note: Any changes made here will be automatically overwritten by a bot. Please do not make changes here as they will be lost. Edit the master page instead.\n:%missing% """ self.subpageTemplateParam = """  \n: Note: Any changes made here will be automatically overwritten by a bot. Please do not make changes here as they will be lost. Edit the master page instead. """ self.invalidParamError = """ \n: Error: Invalid parameter passed. """ self.subpageTemplateID = """%string%  \n: Note: Any changes made here will be automatically overwritten by a bot. Please do not make changes here as they will be lost. Edit the master page instead. """ self.dictionaries = { u'Template:Dictionary/items': { # Dictionary page 'name': 'items', # Dictionary name (used for categorizing) 'sync': 'Template:Dictionary/items/Special:SyncData' # Page holding last sync data },            u'Template:Dictionary/heroes': { 'name': 'heroes', 'sync': 'Template:Dictionary/heroes/Special:SyncData' },            u'Template:Dictionary/cosmetic items': { # Warning: no underscore 'name': 'cosmetic items', 'sync': 'Template:Dictionary/cosmetic items/Special:SyncData' },            u'Template:Dictionary/common strings': { # Warning: no underscore 'name': 'common strings', 'sync': 'Template:Dictionary/common strings/Special:SyncData' }        }         self.subpageSeparator = u'/' # List of supported languages, in prefered order self.languages = [u'en', u'ar', u'cs', u'da', u'de', u'es', u'fi', u'fr', u'hu', u'it', u'ja', u'ko', u'nl', u'no', u'pl', u'pt', u'pt-br', u'ro', u'ru', u'sv', u'tr', u'zh-hans', u'zh-hant'] self.defaultLang = u'en' self.allKeyName = u'_all_' self.filterName = u'Your friendly neighborhood dictionary updater' self.commentsExtract = compileRegex(r'') self.stringsExtract = compileRegex(r'(?:^[ \t]*#[ \t]*([^\r\n]*?)[ \t]*$\s*)?^[ \t]*([^\r\n]+?[ \t]*(?:\|[ \t]*[^\r\n]+?[ \t]*)*):[ \t]*([^\r\n]+?[ \t]*$|\s*[\r\n]+(?:\s*[ \t]+[-\w]+[ \t]*:[ \t]*[^\r\n]+[ \t]*$)+)', re.IGNORECASE | re.MULTILINE) self.translationExtract = compileRegex(r'^[ \t]+([-\w]+)[ \t]*:[ \t]*([^\r\n]+)[ \t]*$', re.IGNORECASE | re.MULTILINE) self.scheduler = BatchScheduler(16) addWhitelistPage(self.dictionaries.keys) def generateSubpage(self, keyName, data, currentDict, syncData): h = hashlib.md5 if type(data) is type({}): # Subkeys (translations or not) isTranslation = True subpage = u(self.subpageTemplateLang) for k in data: if 'blankString' in self.dictionaries[currentDict] and data[k] == self.dictionaries[currentDict]['blankString']: data[k] = u'' if isTranslation and k not in self.languages: isTranslation = False subpage = u(self.subpageTemplateParam) ordered = [] unordered = {} if isTranslation: missing = [] for lang in self.languages: if lang in data: ordered.append(lang + u'=' + data[lang]) unordered[lang] = data[lang] h.update((lang + u'=' + data[lang]).encode('utf8')) else: missing.append(lang) h.update((u'null-' + lang).encode('utf8')) if self.defaultLang in data: ordered.insert(0, u'#default=' + data[self.defaultLang]) if len(missing): subpage = subpage.replace(u'%missing%', u"Languages missing: " + u', '.join(missing)) else: subpage = subpage.replace(u'%missing%', u"Supported languages: all" ) else: # Not a translation h.update('Any-') subkeys = data.keys subkeys.sort for k in subkeys: ordered.append(k + u'=' + data[k]) unordered[k] = data[k] h.update((k + u'=' + data[k]).encode('utf8')) if 'allTemplate' in self.dictionaries[currentDict] and (len(unordered) or len(self.dictionaries[currentDict]['allTemplate']['params'])): allKey = [] keys = unordered.keys keys.sort for k in keys: allKey.append(k + u'=' + unordered[k]) insertIndex = 0 if isTranslation and self.defaultLang in data: insertIndex = 1 ordered.insert(insertIndex, u(self.allKeyName) + u'=' + u(self.dictionaries[currentDict]['allTemplate'].replace(u'%options%', u'|'.join(allKey)))) subpage = subpage.replace(u'%options%', u'|'.join(ordered)) else: # No subkeys data = u(data) subpage = self.subpageTemplateID h.update(u(u'ID-' + data).encode('utf8')) subpage = subpage.replace(u'%string%', data) h = u(h.hexdigest) if keyName in syncData and syncData[keyName] == h:            return # Same hash syncData[keyName] = h # Update sync data subpage = subpage.replace(u'%dictionary%', currentDict) subpage = subpage.replace(u'%dictionaryname%', self.dictionaries[currentDict]['name']) subpage = subpage.replace(u'%keyname%', keyName) self.scheduler.schedule(editPage, currentDict + self.subpageSeparator + keyName, subpage, summary= u'Pushed changes from ' + currentDict + u' for string "' + keyName + u'".', minor=True, nocreate=False) def processComment(self, commentString, currentDict, definedStrings, syncData): commentContents = [] for extractedStr in self.stringsExtract.finditer(commentString): comment = u'' if extractedStr.group(1): comment = u'# ' + u(extractedStr.group(1)) + u'\n' dataString = u(extractedStr.group(3)) if dataString.find(u'\r') == -1 and dataString.find(u'\n') == -1: # Assume no subkeys data = dataString.strip dataWriteback = u' ' + data else: # There's subkeys; detect whether this is a translation or not data = {} isTranslation = True for translation in self.translationExtract.finditer(dataString.rstrip): data[u(translation.group(1))] = u(translation.group(2)) if u(translation.group(1)) not in self.languages: isTranslation = False ordered = [] if isTranslation: for lang in self.languages: if lang in data: ordered.append(u' ' + lang + u': ' + data[lang]) else: # Not a translation, so order in alphabetical order subkeys = data.keys subkeys.sort for subk in subkeys: ordered.append(u' ' + subk + u': ' + data[subk]) dataWriteback = u'\n' + u'\n'.join(ordered) keyNames = u(extractedStr.group(2)).lower.split(u'|') validKeyNames = [] for keyName in keyNames: keyName = keyName.replace(u'_', u' ').strip if keyName in definedStrings: continue # Duplicate key definedStrings.append(keyName) validKeyNames.append(keyName) self.generateSubpage(keyName, data, currentDict, syncData) if len(validKeyNames): commentContents.append(comment + u' | '.join(validKeyNames) + u':' + dataWriteback) self.scheduler.execute return u'\n\n'.join(commentContents) def __call__(self, content, **kwargs): if 'article' not in kwargs: return content if u(kwargs['article'].title) not in self.dictionaries: return content currentDict = u(kwargs['article'].title) syncPage = page(self.dictionaries[currentDict]['sync']) try: syncDataText = u(syncPage.getWikiText).split(u'\n') except: # Page probably doesn't exist syncDataText = u'' syncData = {} for sync in syncDataText: sync = u(sync.strip) if not sync: continue sync = sync.split(u':', 2) if len(sync) == 2: syncData[sync[0]] = sync[1] oldSyncData = syncData.copy newContent = u'' previousIndex = 0 definedStrings = [] for comment in self.commentsExtract.finditer(content): newContent += content[previousIndex:comment.start] previousIndex = comment.end # Process current comment newContent += u'' newContent += content[previousIndex:] # Check if we need to update sync data needUpdate = False for k in syncData: if k not in oldSyncData or oldSyncData[k] != syncData[k]: needUpdate = True break # Check for deleted strings for k in oldSyncData: if k not in definedStrings: try: deletePage(currentDict + self.subpageSeparator + k, 'Removed deleted string "' + k + u'" from ' + currentDict + u'.') except: pass if k in syncData: del syncData[k] needUpdate = True if needUpdate: # Build syncdata string representation syncKeys = syncData.keys syncKeys.sort syncLines = [] for k in syncKeys: syncLines.append(k + u':' + syncData[k]) editPage(syncPage, u'\n'.join(syncLines), summary= u'Updated synchronization information for ' + currentDict + u'.', minor=True, nocreate=False) return newContent def scheduledRun(self): for d in self.dictionaries: fixPage(d) dictUpdater = DictionaryUpdater addFilter(dictUpdater) scheduleTask(dictUpdater.scheduledRun, 3)

Update Dictionary/defindex and Dictionary/price
def updateGameDictionaries: if steam is None: return # Begin configurable section game = steam.d2 	priceDictionaryPage = u'Template:Dictionary/price' indexDictionaryPage = u'Template:Dictionary/defindex' itemEquivalences = {"": "I won't break your wiki."} illegalItemCharacters = compileRegex(r'[:%\r\n\t]') priceDictionaryHeader = u'\n== /price ==\n\'\'\'This dictionary is automatically updated by WindBOT\'\'\'. Edits made to this page will be overwritten.\n' # End configurable section schema = steamGetGameSchema(game) assets = steam.items.assets(570, lang="en_us") dictionary = {} defindex = {} for item in schema: try: itemName = u(item.get_name).lower.replace(u'_', u' ') keys = [itemName] if itemName in itemEquivalences: itemName = itemEquivalences[itemName] keys.append(itemName) defindex[itemName] = u' | '.join([illegalItemCharacters.sub(u'', x) for x in keys]) + u': ' + u(item.get_schema_id) prices = assets.get_price(item.get_schema_id) if not len(prices): continue dictionary[itemName] = itemName + u':' for currency in sorted(prices): price = prices[currency] if price.is_integer: price = u(int(price)) else: price = u(round(price, 2)) dot = price.find(u'.') if dot == len(price) - 2: price += u'0' dictionary[itemName] += u'\n ' + u(currency).lower + u': ' + price except: continue if len(dictionary): # Is updating time doktor finalPage = priceDictionaryHeader for item in sorted(dictionary): finalPage += u'\n' + dictionary[item] + u'\n' finalPage += dictionaryFooter priceDictionaryPage = page(priceDictionaryPage) editPage(priceDictionaryPage, finalPage, summary=u'Updated item prices from WebAPI.', minor=True, bot=True, nocreate=True) # Run other filters on the new page fixPage(priceDictionaryPage) if len(defindex): # Is also updating time doktor finalPage = indexDictionaryHeader for item in sorted(defindex): finalPage += u'\n' + defindex[item] + u'\n' finalPage += dictionaryFooter indexDictionaryPage = page(indexDictionaryPage) editPage(indexDictionaryPage, finalPage, summary=u'Updated item indexes from WebAPI.', minor=True, bot=True, nocreate=True) # Run other filters on the new page fixPage(indexDictionaryPage) scheduleTask(updateGameDictionaries, 2)