diff --git a/gen-ignores.py b/gen-ignores.py index 2a1e6411db9dee063d34ed14ea82a86d39db9816..abf9b96f63de7301aac093e7ab5d0eeeb2452226 100755 --- a/gen-ignores.py +++ b/gen-ignores.py @@ -10,6 +10,7 @@ import glob import json import os +import re import sys archivebot_ignore_path = sys.argv[1] @@ -24,6 +25,8 @@ for fn in glob.glob(os.path.join(archivebot_ignore_path, '*.json')): # fancy PCRE constructs. This excludes <10 # patterns from the ignore list. continue + if re.search(r'(?:%[0-9A-F]{2}){3,}', p): + continue p = p.replace('{primary_netloc}', '.*') print('\t%s,' % json.dumps(p)) except Exception as e: diff --git a/ignore_patterns.go b/ignore_patterns.go index f787694526167d0f2fd3aa30cdcb0f7cf491e9b4..4c38ff6f707d7817072a42d2c742264bcbcaec10 100644 --- a/ignore_patterns.go +++ b/ignore_patterns.go @@ -24,17 +24,7 @@ var defaultIgnorePatterns = []string{ "[\\?&]limit=(1|20|100|250|500)", "[?&]dir=prev(&|$)", "[\\?&]hide(minor|bots|anons|liu|myself|redirs|links|trans|patrolled)=", - "([\\?&]title=|/)%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:((%D0%A1%D1%82%D0%B2%D0%BE%D1%80%D0%B8%D1%82%D0%B8_%D0%BE%D0%B1%D0%BB%D1%96%D0%BA%D0%BE%D0%B2%D0%B8%D0%B9_%D0%B7%D0%B0%D0%BF%D0%B8%D1%81|%D0%A1%D0%BE%D0%B7%D0%B4%D0%B0%D1%82%D1%8C_%D1%83%D1%87%D1%91%D1%82%D0%BD%D1%83%D1%8E_%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D1%8C|%D0%A1%D0%BE%D0%B7%D0%B4%D0%B0%D1%82%D1%8C_%D0%BF%D0%BE%D0%BB%D1%8C%D0%B7%D0%BE%D0%B2%D0%B0%D1%82%D0%B5%D0%BB%D1%8F|%D0%97%D0%B0%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D1%82%D1%8C%D1%81%D1%8F)|(%D0%92%D1%85%D1%96%D0%B4|%D0%92%D1%85%D0%BE%D0%B4)|(%D0%92%D0%B8%D1%85%D1%96%D0%B4|%D0%97%D0%B0%D0%B2%D0%B5%D1%80%D1%88%D0%B5%D0%BD%D0%B8%D0%B5_%D1%81%D0%B5%D0%B0%D0%BD%D1%81%D0%B0|%D0%92%D1%8B%D1%85%D0%BE%D0%B4)|Translate|MobileFeedback|MobileOptions|(%D0%9F%D0%BE%D0%B2%5C%27%D1%8F%D0%B7%D0%B0%D0%BD%D1%96_%D1%80%D0%B5%D0%B4%D0%B0%D0%B3%D1%83%D0%B2%D0%B0%D0%BD%D0%BD%D1%8F|%D0%A1%D0%B2%D1%8F%D0%B7%D0%B0%D0%BD%D0%BD%D1%8B%D0%B5_%D0%BF%D1%80%D0%B0%D0%B2%D0%BA%D0%B8)|Diff|MobileDiff|CentralAuth)", - "([\\?&]title=|/)%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(Ask|Browse|SearchByProperty|ExportRDF|PageProperty|Properties|UnusedProperties|WantedProperties|SMWAdmin|Types|URIResolver|QueryCreator)", - "([\\?&]title=%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(%D0%9D%D0%BE%D0%B2%D1%96_%D1%80%D0%B5%D0%B4%D0%B0%D0%B3%D1%83%D0%B2%D0%B0%D0%BD%D0%BD%D1%8F|%D0%A1%D0%B2%D0%B5%D0%B6%D0%B8%D0%B5_%D0%BF%D1%80%D0%B0%D0%B2%D0%BA%D0%B8)&|/%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(%D0%9D%D0%BE%D0%B2%D1%96_%D1%80%D0%B5%D0%B4%D0%B0%D0%B3%D1%83%D0%B2%D0%B0%D0%BD%D0%BD%D1%8F|%D0%A1%D0%B2%D0%B5%D0%B6%D0%B8%D0%B5_%D0%BF%D1%80%D0%B0%D0%B2%D0%BA%D0%B8)\\?)(.*&)?from=\\d+", - "([\\?&]title=|/)%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:((%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D1%96%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D0%BE%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D0%B8%D0%B7%D0%BE%D0%B1%D1%80%D0%B0%D0%B6%D0%B5%D0%BD%D0%B8%D0%B9)|(%D0%9F%D0%BE%D0%BA%D0%B0%D0%B6%D1%87%D0%B8%D0%BA_%D0%B7%D0%B0_%D0%BF%D0%BE%D1%87%D0%B0%D1%82%D0%BA%D0%BE%D0%BC_%D0%BD%D0%B0%D0%B7%D0%B2%D0%B8|%D0%A3%D0%BA%D0%B0%D0%B7%D0%B0%D1%82%D0%B5%D0%BB%D1%8C_%D0%BF%D0%BE_%D0%BD%D0%B0%D1%87%D0%B0%D0%BB%D1%83_%D0%BD%D0%B0%D0%B7%D0%B2%D0%B0%D0%BD%D0%B8%D1%8F)).*&", - "([\\?&]title=|/)%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D1%96%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D0%BE%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D0%B8%D0%B7%D0%BE%D0%B1%D1%80%D0%B0%D0%B6%D0%B5%D0%BD%D0%B8%D0%B9).*&user=", - "([\\?&]title=|/)%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D1%96%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D0%BE%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D0%B8%D0%B7%D0%BE%D0%B1%D1%80%D0%B0%D0%B6%D0%B5%D0%BD%D0%B8%D0%B9).*&sort=img_(timestamp|name|size)(&|$)", - "([\\?&]title=|/)%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D0%B8|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D1%8B|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB)/", - "([?&]title=%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D0%B8|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D1%8B|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB)&|/%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D0%B8|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D1%8B|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB)\\?)(.*&)?hide_[^&=]+=1(&|$)", - "([?&]title=%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D0%B8|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D1%8B|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB)&|/%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D0%B8|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D1%8B|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB)\\?)(.*&)?hide_.*&hide_", "([?&]title=|/)Property:.*&until=", - "([?&]title=|/)%D0%9A%D0%B0%D1%82%D0%B5%D0%B3%D0%BE%D1%80%D1%96%D1%8F:.*&(page|subcat|file)until=", "[\\?&]action=edit§ion=(\\d+|new)", "[\\?&]feed(format)?=atom", "[\\?&]redlink=1", @@ -43,11 +33,7 @@ var defaultIgnorePatterns = []string{ "[\\?&]undo(after)?=\\d+", "^https?://a\\.wikia-beacon\\.com/__track/", "^https?://beacon\\.wikia-services\\.com/__track/", - "([\\?&]title=|/)%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:((%D0%9F%D0%BE%D0%BA%D0%B0%D0%B6%D1%87%D0%B8%D0%BA_%D0%B7%D0%B0_%D0%BF%D0%BE%D1%87%D0%B0%D1%82%D0%BA%D0%BE%D0%BC_%D0%BD%D0%B0%D0%B7%D0%B2%D0%B8|%D0%A3%D0%BA%D0%B0%D0%B7%D0%B0%D1%82%D0%B5%D0%BB%D1%8C_%D0%BF%D0%BE_%D0%BD%D0%B0%D1%87%D0%B0%D0%BB%D1%83_%D0%BD%D0%B0%D0%B7%D0%B2%D0%B0%D0%BD%D0%B8%D1%8F)|(%D0%9F%D0%BE%D1%81%D0%B8%D0%BB%D0%B0%D0%BD%D0%BD%D1%8F_%D1%81%D1%8E%D0%B4%D0%B8|%D0%A1%D1%81%D1%8B%D0%BB%D0%BA%D0%B8_%D1%81%D1%8E%D0%B4%D0%B0)|(%D0%92%D0%BD%D0%B5%D1%81%D0%BE%D0%BA|%D0%92%D0%BA%D0%BB%D0%B0%D0%B4)|(%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D1%96%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D0%BE%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D0%B8%D0%B7%D0%BE%D0%B1%D1%80%D0%B0%D0%B6%D0%B5%D0%BD%D0%B8%D0%B9)|(%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D0%BA%D0%BE%D1%80%D0%B8%D1%81%D1%82%D1%83%D0%B2%D0%B0%D1%87%D1%96%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%83%D1%87%D0%B0%D1%81%D1%82%D0%BD%D0%B8%D0%BA%D0%BE%D0%B2))/.*/\\2/", - "/%D0%9E%D0%B1%D0%B3%D0%BE%D0%B2%D0%BE%D1%80%D0%B5%D0%BD%D0%BD%D1%8F_%D0%BA%D0%BE%D1%80%D0%B8%D1%81%D1%82%D1%83%D0%B2%D0%B0%D1%87%D0%B0:.+/%D0%9E%D0%B1%D0%B3%D0%BE%D0%B2%D0%BE%D1%80%D0%B5%D0%BD%D0%BD%D1%8F_%D0%BA%D0%BE%D1%80%D0%B8%D1%81%D1%82%D1%83%D0%B2%D0%B0%D1%87%D0%B0:", "/User_blog:.+/User_blog:", - "/%D0%9A%D0%BE%D1%80%D0%B8%D1%81%D1%82%D1%83%D0%B2%D0%B0%D1%87:.+/%D0%9A%D0%BE%D1%80%D0%B8%D1%81%D1%82%D1%83%D0%B2%D0%B0%D1%87:", - "^https?://[^/]+\\.gamepedia\\.com/(.*[?&]title=)?%D0%A1%D0%BF%D0%B5%D1%86%D1%96%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0:(WikiPoints/global|ListAllUsers|AllSites)([/?&]|$)", // mediawiki-ru.json "[\\?&]diff=(prev|next|cur|\\d+)", @@ -56,17 +42,7 @@ var defaultIgnorePatterns = []string{ "[\\?&]limit=(1|20|100|250|500)", "[?&]dir=prev(&|$)", "[\\?&]hide(minor|bots|anons|liu|myself|redirs|links|trans|patrolled)=", - "([\\?&]title=|/)%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:((%D0%A1%D0%BE%D0%B7%D0%B4%D0%B0%D1%82%D1%8C_%D1%83%D1%87%D1%91%D1%82%D0%BD%D1%83%D1%8E_%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D1%8C|%D0%A1%D0%BE%D0%B7%D0%B4%D0%B0%D1%82%D1%8C_%D0%BF%D0%BE%D0%BB%D1%8C%D0%B7%D0%BE%D0%B2%D0%B0%D1%82%D0%B5%D0%BB%D1%8F|%D0%97%D0%B0%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D1%82%D1%8C%D1%81%D1%8F)|%D0%92%D1%85%D0%BE%D0%B4|(%D0%97%D0%B0%D0%B2%D0%B5%D1%80%D1%88%D0%B5%D0%BD%D0%B8%D0%B5_%D1%81%D0%B5%D0%B0%D0%BD%D1%81%D0%B0|%D0%92%D1%8B%D1%85%D0%BE%D0%B4)|Translate|MobileFeedback|MobileOptions|%D0%A1%D0%B2%D1%8F%D0%B7%D0%B0%D0%BD%D0%BD%D1%8B%D0%B5_%D0%BF%D1%80%D0%B0%D0%B2%D0%BA%D0%B8|%D0%98%D0%B7%D0%BC%D0%B5%D0%BD%D0%B5%D0%BD%D0%B8%D1%8F|MobileDiff|CentralAuth)", - "([\\?&]title=|/)%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:(Ask|Browse|SearchByProperty|ExportRDF|PageProperty|Properties|UnusedProperties|WantedProperties|SMWAdmin|Types|URIResolver|QueryCreator)", - "([\\?&]title=%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:%D0%A1%D0%B2%D0%B5%D0%B6%D0%B8%D0%B5_%D0%BF%D1%80%D0%B0%D0%B2%D0%BA%D0%B8&|/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:%D0%A1%D0%B2%D0%B5%D0%B6%D0%B8%D0%B5_%D0%BF%D1%80%D0%B0%D0%B2%D0%BA%D0%B8\\?)(.*&)?from=\\d+", - "([\\?&]title=|/)%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:((%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D0%BE%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D0%B8%D0%B7%D0%BE%D0%B1%D1%80%D0%B0%D0%B6%D0%B5%D0%BD%D0%B8%D0%B9)|%D0%A3%D0%BA%D0%B0%D0%B7%D0%B0%D1%82%D0%B5%D0%BB%D1%8C_%D0%BF%D0%BE_%D0%BD%D0%B0%D1%87%D0%B0%D0%BB%D1%83_%D0%BD%D0%B0%D0%B7%D0%B2%D0%B0%D0%BD%D0%B8%D1%8F).*&", - "([\\?&]title=|/)%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:(%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D0%BE%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D0%B8%D0%B7%D0%BE%D0%B1%D1%80%D0%B0%D0%B6%D0%B5%D0%BD%D0%B8%D0%B9).*&user=", - "([\\?&]title=|/)%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:(%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D0%BE%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D0%B8%D0%B7%D0%BE%D0%B1%D1%80%D0%B0%D0%B6%D0%B5%D0%BD%D0%B8%D0%B9).*&sort=img_(timestamp|name|size)(&|$)", - "([\\?&]title=|/)%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:(%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D1%8B|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB)/", - "([?&]title=%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:(%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D1%8B|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB)&|/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:(%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D1%8B|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB)\\?)(.*&)?hide_[^&=]+=1(&|$)", - "([?&]title=%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:(%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D1%8B|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB)&|/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:(%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB%D1%8B|%D0%96%D1%83%D1%80%D0%BD%D0%B0%D0%BB)\\?)(.*&)?hide_.*&hide_", "([?&]title=|/)Property:.*&until=", - "([?&]title=|/)%D0%9A%D0%B0%D1%82%D0%B5%D0%B3%D0%BE%D1%80%D0%B8%D1%8F:.*&(page|subcat|file)until=", "[\\?&]action=edit§ion=(\\d+|new)", "[\\?&]feed(format)?=atom", "[\\?&]redlink=1", @@ -75,11 +51,7 @@ var defaultIgnorePatterns = []string{ "[\\?&]undo(after)?=\\d+", "^https?://a\\.wikia-beacon\\.com/__track/", "^https?://beacon\\.wikia-services\\.com/__track/", - "([\\?&]title=|/)%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:(%D0%A3%D0%BA%D0%B0%D0%B7%D0%B0%D1%82%D0%B5%D0%BB%D1%8C_%D0%BF%D0%BE_%D0%BD%D0%B0%D1%87%D0%B0%D0%BB%D1%83_%D0%BD%D0%B0%D0%B7%D0%B2%D0%B0%D0%BD%D0%B8%D1%8F|%D0%A1%D1%81%D1%8B%D0%BB%D0%BA%D0%B8_%D1%81%D1%8E%D0%B4%D0%B0|%D0%92%D0%BA%D0%BB%D0%B0%D0%B4|(%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%84%D0%B0%D0%B9%D0%BB%D0%BE%D0%B2|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D0%B8%D0%B7%D0%BE%D0%B1%D1%80%D0%B0%D0%B6%D0%B5%D0%BD%D0%B8%D0%B9)|%D0%A1%D0%BF%D0%B8%D1%81%D0%BE%D0%BA_%D1%83%D1%87%D0%B0%D1%81%D1%82%D0%BD%D0%B8%D0%BA%D0%BE%D0%B2)/.*/\\2/", - "/%D0%9E%D0%B1%D1%81%D1%83%D0%B6%D0%B4%D0%B5%D0%BD%D0%B8%D0%B5_%D1%83%D1%87%D0%B0%D1%81%D1%82%D0%BD%D0%B8%D0%BA%D0%B0:.+/%D0%9E%D0%B1%D1%81%D1%83%D0%B6%D0%B4%D0%B5%D0%BD%D0%B8%D0%B5_%D1%83%D1%87%D0%B0%D1%81%D1%82%D0%BD%D0%B8%D0%BA%D0%B0:", "/User_blog:.+/User_blog:", - "/%D0%A3%D1%87%D0%B0%D1%81%D1%82%D0%BD%D0%B8%D0%BA:.+/%D0%A3%D1%87%D0%B0%D1%81%D1%82%D0%BD%D0%B8%D0%BA:", - "^https?://[^/]+\\.gamepedia\\.com/(.*[?&]title=)?%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F:(WikiPoints/global|ListAllUsers|AllSites)([/?&]|$)", // mediawiki.json "[\\?&]diff=(prev|next|cur|\\d+)", @@ -120,17 +92,7 @@ var defaultIgnorePatterns = []string{ "[\\?&]limit=(1|20|100|250|500)", "[?&]dir=prev(&|$)", "[\\?&]hide(minor|bots|anons|liu|myself|redirs|links|trans|patrolled)=", - "([\\?&]title=|/)%E7%89%B9%E5%88%A5:((%E3%82%A2%E3%82%AB%E3%82%A6%E3%83%B3%E3%83%88%E4%BD%9C%E6%88%90|%E3%82%A2%E3%82%AB%E3%82%A6%E3%83%B3%E3%83%88%E3%81%AE%E4%BD%9C%E6%88%90)|%E3%83%AD%E3%82%B0%E3%82%A4%E3%83%B3|%E3%83%AD%E3%82%B0%E3%82%A2%E3%82%A6%E3%83%88|Translate|MobileFeedback|MobileOptions|(%E9%96%A2%E9%80%A3%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E6%9B%B4%E6%96%B0%E7%8A%B6%E6%B3%81|%E3%83%AA%E3%83%B3%E3%82%AF%E5%85%88%E3%81%AE%E6%9B%B4%E6%96%B0%E7%8A%B6%E6%B3%81)|%E5%B7%AE%E5%88%86|MobileDiff|CentralAuth)", - "([\\?&]title=|/)%E7%89%B9%E5%88%A5:(Ask|Browse|SearchByProperty|ExportRDF|PageProperty|Properties|UnusedProperties|WantedProperties|SMWAdmin|Types|URIResolver|QueryCreator)", - "([\\?&]title=%E7%89%B9%E5%88%A5:(%E6%9C%80%E8%BF%91%E3%81%AE%E6%9B%B4%E6%96%B0|%E6%9C%80%E8%BF%91%E6%9B%B4%E6%96%B0%E3%81%97%E3%81%9F%E3%83%9A%E3%83%BC%E3%82%B8)&|/%E7%89%B9%E5%88%A5:(%E6%9C%80%E8%BF%91%E3%81%AE%E6%9B%B4%E6%96%B0|%E6%9C%80%E8%BF%91%E6%9B%B4%E6%96%B0%E3%81%97%E3%81%9F%E3%83%9A%E3%83%BC%E3%82%B8)\\?)(.*&)?from=\\d+", - "([\\?&]title=|/)%E7%89%B9%E5%88%A5:((%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E4%B8%80%E8%A6%A7|%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E3%83%AA%E3%82%B9%E3%83%88)|(%E5%89%8D%E6%96%B9%E4%B8%80%E8%87%B4%E3%83%9A%E3%83%BC%E3%82%B8%E4%B8%80%E8%A6%A7|%E5%A7%8B%E7%82%B9%E6%8C%87%E5%AE%9A%E3%83%9A%E3%83%BC%E3%82%B8%E4%B8%80%E8%A6%A7)).*&", - "([\\?&]title=|/)%E7%89%B9%E5%88%A5:(%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E4%B8%80%E8%A6%A7|%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E3%83%AA%E3%82%B9%E3%83%88).*&user=", - "([\\?&]title=|/)%E7%89%B9%E5%88%A5:(%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E4%B8%80%E8%A6%A7|%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E3%83%AA%E3%82%B9%E3%83%88).*&sort=img_(timestamp|name|size)(&|$)", - "([\\?&]title=|/)%E7%89%B9%E5%88%A5:(%E3%83%AD%E3%82%B0|%E8%A8%98%E9%8C%B2)/", - "([?&]title=%E7%89%B9%E5%88%A5:(%E3%83%AD%E3%82%B0|%E8%A8%98%E9%8C%B2)&|/%E7%89%B9%E5%88%A5:(%E3%83%AD%E3%82%B0|%E8%A8%98%E9%8C%B2)\\?)(.*&)?hide_[^&=]+=1(&|$)", - "([?&]title=%E7%89%B9%E5%88%A5:(%E3%83%AD%E3%82%B0|%E8%A8%98%E9%8C%B2)&|/%E7%89%B9%E5%88%A5:(%E3%83%AD%E3%82%B0|%E8%A8%98%E9%8C%B2)\\?)(.*&)?hide_.*&hide_", "([?&]title=|/)Property:.*&until=", - "([?&]title=|/)%E3%82%AB%E3%83%86%E3%82%B4%E3%83%AA:.*&(page|subcat|file)until=", "[\\?&]action=edit§ion=(\\d+|new)", "[\\?&]feed(format)?=atom", "[\\?&]redlink=1", @@ -139,17 +101,12 @@ var defaultIgnorePatterns = []string{ "[\\?&]undo(after)?=\\d+", "^https?://a\\.wikia-beacon\\.com/__track/", "^https?://beacon\\.wikia-services\\.com/__track/", - "([\\?&]title=|/)%E7%89%B9%E5%88%A5:((%E5%89%8D%E6%96%B9%E4%B8%80%E8%87%B4%E3%83%9A%E3%83%BC%E3%82%B8%E4%B8%80%E8%A6%A7|%E5%A7%8B%E7%82%B9%E6%8C%87%E5%AE%9A%E3%83%9A%E3%83%BC%E3%82%B8%E4%B8%80%E8%A6%A7)|%E3%83%AA%E3%83%B3%E3%82%AF%E5%85%83|%E6%8A%95%E7%A8%BF%E8%A8%98%E9%8C%B2|(%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E4%B8%80%E8%A6%A7|%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E3%83%AA%E3%82%B9%E3%83%88)|(%E7%99%BB%E9%8C%B2%E5%88%A9%E7%94%A8%E8%80%85%E4%B8%80%E8%A6%A7|%E7%99%BB%E9%8C%B2%E5%88%A9%E7%94%A8%E8%80%85%E3%81%AE%E4%B8%80%E8%A6%A7))/.*/\\2/", - "/%E5%88%A9%E7%94%A8%E8%80%85%E3%83%BB%E3%83%88%E3%83%BC%E3%82%AF:.+/%E5%88%A9%E7%94%A8%E8%80%85%E3%83%BB%E3%83%88%E3%83%BC%E3%82%AF:", "/User_blog:.+/User_blog:", - "/%E5%88%A9%E7%94%A8%E8%80%85:.+/%E5%88%A9%E7%94%A8%E8%80%85:", - "^https?://[^/]+\\.gamepedia\\.com/(.*[?&]title=)?%E7%89%B9%E5%88%A5:(WikiPoints/global|ListAllUsers|AllSites)([/?&]|$)", // fc2-blog.json "^https?://[^/.]+\\.blog\\d*\\.(fc2\\.com|fc2blog\\.us|2nt\\.com)/(\\?date=|blog-date-)(1[0-8]\\d{4}|19[0-8]\\d{3}|20[5-9]\\d{3}|2[1-9]\\d{4})", "^https?://[^/.]+\\.blog\\d*\\.(fc2\\.com|fc2blog\\.us|2nt\\.com)/\\?mode=edit(entry)?&rno=\\d+$", "^https?://blogvote\\.(fc2\\.com|fceblog\\.us|2nt\\.com)/pickup/[^/]+/\\d+/clap$", - "^https?://clap\\.fc2\\.com/post/[^/]+/\\?url=https?%3A%2F%2F[^/&]+\\.blog\\d*\\.(fc2\\.com|fc2blog\\.us|2nt\\.com)%2F", "^https?://admin\\.blog\\.(fc2\\.com|2nt\\.com)/control\\.php", // mediawiki-de.json @@ -484,17 +441,7 @@ var defaultIgnorePatterns = []string{ "[\\?&]limit=(1|20|100|250|500)", "[?&]dir=prev(&|$)", "[\\?&]hide(minor|bots|anons|liu|myself|redirs|links|trans|patrolled)=", - "([\\?&]title=|/)%D8%AE%D8%A7%D8%B5:(%D8%A5%D9%86%D8%B4%D8%A7%D8%A1_%D8%AD%D8%B3%D8%A7%D8%A8|%D8%AF%D8%AE%D9%88%D9%84_%D8%A7%D9%84%D9%85%D8%B3%D8%AA%D8%AE%D8%AF%D9%85|%D8%AE%D8%B1%D9%88%D8%AC_%D8%A7%D9%84%D9%85%D8%B3%D8%AA%D8%AE%D8%AF%D9%85|Translate|MobileFeedback|MobileOptions|(%D8%A3%D8%AD%D8%AF%D8%AB_%D8%A7%D9%84%D8%AA%D8%BA%D9%8A%D9%8A%D8%B1%D8%A7%D8%AA_%D8%A7%D9%84%D9%85%D9%88%D8%B5%D9%88%D9%84%D8%A9|%D8%AA%D8%BA%D9%8A%D9%8A%D8%B1%D8%A7%D8%AA_%D9%85%D8%B1%D8%AA%D8%A8%D8%B7%D8%A9)|%D9%81%D8%B1%D9%82|MobileDiff|CentralAuth)", - "([\\?&]title=|/)%D8%AE%D8%A7%D8%B5:(Ask|Browse|SearchByProperty|ExportRDF|PageProperty|Properties|UnusedProperties|WantedProperties|SMWAdmin|Types|URIResolver|QueryCreator)", - "([\\?&]title=%D8%AE%D8%A7%D8%B5:%D8%A3%D8%AD%D8%AF%D8%AB_%D8%A7%D9%84%D8%AA%D8%BA%D9%8A%D9%8A%D8%B1%D8%A7%D8%AA&|/%D8%AE%D8%A7%D8%B5:%D8%A3%D8%AD%D8%AF%D8%AB_%D8%A7%D9%84%D8%AA%D8%BA%D9%8A%D9%8A%D8%B1%D8%A7%D8%AA\\?)(.*&)?from=\\d+", - "([\\?&]title=|/)%D8%AE%D8%A7%D8%B5:((%D8%B9%D8%B1%D8%B6_%D8%A7%D9%84%D9%85%D9%84%D9%81%D8%A7%D8%AA|%D9%82%D8%A7%D8%A6%D9%85%D8%A9_%D8%A7%D9%84%D9%85%D9%84%D9%81%D8%A7%D8%AA|%D9%82%D8%A7%D8%A6%D9%85%D8%A9_%D8%A7%D9%84%D8%B5%D9%88%D8%B1)|%D9%81%D9%87%D8%B1%D8%B3_%D8%A8%D8%A7%D8%AF%D8%A6%D8%A9).*&", - "([\\?&]title=|/)%D8%AE%D8%A7%D8%B5:(%D8%B9%D8%B1%D8%B6_%D8%A7%D9%84%D9%85%D9%84%D9%81%D8%A7%D8%AA|%D9%82%D8%A7%D8%A6%D9%85%D8%A9_%D8%A7%D9%84%D9%85%D9%84%D9%81%D8%A7%D8%AA|%D9%82%D8%A7%D8%A6%D9%85%D8%A9_%D8%A7%D9%84%D8%B5%D9%88%D8%B1).*&user=", - "([\\?&]title=|/)%D8%AE%D8%A7%D8%B5:(%D8%B9%D8%B1%D8%B6_%D8%A7%D9%84%D9%85%D9%84%D9%81%D8%A7%D8%AA|%D9%82%D8%A7%D8%A6%D9%85%D8%A9_%D8%A7%D9%84%D9%85%D9%84%D9%81%D8%A7%D8%AA|%D9%82%D8%A7%D8%A6%D9%85%D8%A9_%D8%A7%D9%84%D8%B5%D9%88%D8%B1).*&sort=img_(timestamp|name|size)(&|$)", - "([\\?&]title=|/)%D8%AE%D8%A7%D8%B5:(%D8%B3%D8%AC%D9%84|%D8%B3%D8%AC%D9%84%D8%A7%D8%AA)/", - "([?&]title=%D8%AE%D8%A7%D8%B5:(%D8%B3%D8%AC%D9%84|%D8%B3%D8%AC%D9%84%D8%A7%D8%AA)&|/%D8%AE%D8%A7%D8%B5:(%D8%B3%D8%AC%D9%84|%D8%B3%D8%AC%D9%84%D8%A7%D8%AA)\\?)(.*&)?hide_[^&=]+=1(&|$)", - "([?&]title=%D8%AE%D8%A7%D8%B5:(%D8%B3%D8%AC%D9%84|%D8%B3%D8%AC%D9%84%D8%A7%D8%AA)&|/%D8%AE%D8%A7%D8%B5:(%D8%B3%D8%AC%D9%84|%D8%B3%D8%AC%D9%84%D8%A7%D8%AA)\\?)(.*&)?hide_.*&hide_", "([?&]title=|/)Property:.*&until=", - "([?&]title=|/)%D8%AA%D8%B5%D9%86%D9%8A%D9%81:.*&(page|subcat|file)until=", "[\\?&]action=edit§ion=(\\d+|new)", "[\\?&]feed(format)?=atom", "[\\?&]redlink=1", @@ -503,11 +450,7 @@ var defaultIgnorePatterns = []string{ "[\\?&]undo(after)?=\\d+", "^https?://a\\.wikia-beacon\\.com/__track/", "^https?://beacon\\.wikia-services\\.com/__track/", - "([\\?&]title=|/)%D8%AE%D8%A7%D8%B5:(%D9%81%D9%87%D8%B1%D8%B3_%D8%A8%D8%A7%D8%AF%D8%A6%D8%A9|%D9%85%D8%A7%D8%B0%D8%A7_%D9%8A%D8%B5%D9%84_%D9%87%D9%86%D8%A7|%D9%85%D8%B3%D8%A7%D9%87%D9%85%D8%A7%D8%AA|(%D8%B9%D8%B1%D8%B6_%D8%A7%D9%84%D9%85%D9%84%D9%81%D8%A7%D8%AA|%D9%82%D8%A7%D8%A6%D9%85%D8%A9_%D8%A7%D9%84%D9%85%D9%84%D9%81%D8%A7%D8%AA|%D9%82%D8%A7%D8%A6%D9%85%D8%A9_%D8%A7%D9%84%D8%B5%D9%88%D8%B1)|(%D8%B9%D8%B1%D8%B6_%D8%A7%D9%84%D9%85%D8%B3%D8%AA%D8%AE%D8%AF%D9%85%D9%8A%D9%86|%D9%82%D8%A7%D8%A6%D9%85%D8%A9_%D8%A7%D9%84%D9%85%D8%B3%D8%AA%D8%AE%D8%AF%D9%85%D9%8A%D9%86))/.*/\\2/", - "/%D9%86%D9%82%D8%A7%D8%B4_%D8%A7%D9%84%D9%85%D8%B3%D8%AA%D8%AE%D8%AF%D9%85:.+/%D9%86%D9%82%D8%A7%D8%B4_%D8%A7%D9%84%D9%85%D8%B3%D8%AA%D8%AE%D8%AF%D9%85:", "/User_blog:.+/User_blog:", - "/%D9%85%D8%B3%D8%AA%D8%AE%D8%AF%D9%85:.+/%D9%85%D8%B3%D8%AA%D8%AE%D8%AF%D9%85:", - "^https?://[^/]+\\.gamepedia\\.com/(.*[?&]title=)?%D8%AE%D8%A7%D8%B5:(WikiPoints/global|ListAllUsers|AllSites)([/?&]|$)", // mediawiki-es.json "[\\?&]diff=(prev|next|cur|\\d+)", @@ -564,7 +507,6 @@ var defaultIgnorePatterns = []string{ "^https?://[^/]+\\.livejournal\\.com/.*/fbmerging\\.[^/]+\\.text$", "^https?://[^/]+\\.livejournal\\.com/.*/popup\\.[^/]+\\.text$", "^https?://[^/]+\\.livejournal\\.com/.*/\\*sup_ru/ru/UTF-8/", - "^https?://(?=[^/]+\\.livejournal\\.com/).*/(.*/)?https?%3A%2F%2F.*%2F", "^https?://[^/]+\\.livejournal\\.com/.*/(photo/(\\{\\{|%7B%7B)photo\\.siteroot(\\}\\}|%7D%7D)/){2}", "^https?://[^/]+\\.livejournal\\.com/.*/gtm\\.js$", @@ -771,17 +713,7 @@ var defaultIgnorePatterns = []string{ "[\\?&]limit=(1|20|100|250|500)", "[?&]dir=prev(&|$)", "[\\?&]hide(minor|bots|anons|liu|myself|redirs|links|trans|patrolled)=", - "([\\?&]title=|/)%ED%8A%B9%EC%88%98:((%EA%B3%84%EC%A0%95%EB%A7%8C%EB%93%A4%EA%B8%B0|%EA%B0%80%EC%9E%85)|(%EB%A1%9C%EA%B7%B8%EC%9D%B8|%EC%82%AC%EC%9A%A9%EC%9E%90%EB%A1%9C%EA%B7%B8%EC%9D%B8)|(%EB%A1%9C%EA%B7%B8%EC%95%84%EC%9B%83|%EC%82%AC%EC%9A%A9%EC%9E%90%EB%A1%9C%EA%B7%B8%EC%95%84%EC%9B%83)|Translate|MobileFeedback|MobileOptions|%EB%A7%81%ED%81%AC%EC%B5%9C%EA%B7%BC%EB%B0%94%EB%80%9C|%EC%B0%A8%EC%9D%B4|MobileDiff|CentralAuth)", - "([\\?&]title=|/)%ED%8A%B9%EC%88%98:(Ask|Browse|SearchByProperty|ExportRDF|PageProperty|Properties|UnusedProperties|WantedProperties|SMWAdmin|Types|URIResolver|QueryCreator)", - "([\\?&]title=%ED%8A%B9%EC%88%98:%EC%B5%9C%EA%B7%BC%EB%B0%94%EB%80%9C&|/%ED%8A%B9%EC%88%98:%EC%B5%9C%EA%B7%BC%EB%B0%94%EB%80%9C\\?)(.*&)?from=\\d+", - "([\\?&]title=|/)%ED%8A%B9%EC%88%98:((%ED%8C%8C%EC%9D%BC%EB%AA%A9%EB%A1%9D|%EA%B7%B8%EB%A6%BC%EB%AA%A9%EB%A1%9D|%ED%8C%8C%EC%9D%BC|%EA%B7%B8%EB%A6%BC)|%EC%A0%91%EB%91%90%EC%96%B4%EC%B0%BE%EA%B8%B0).*&", - "([\\?&]title=|/)%ED%8A%B9%EC%88%98:(%ED%8C%8C%EC%9D%BC%EB%AA%A9%EB%A1%9D|%EA%B7%B8%EB%A6%BC%EB%AA%A9%EB%A1%9D|%ED%8C%8C%EC%9D%BC|%EA%B7%B8%EB%A6%BC).*&user=", - "([\\?&]title=|/)%ED%8A%B9%EC%88%98:(%ED%8C%8C%EC%9D%BC%EB%AA%A9%EB%A1%9D|%EA%B7%B8%EB%A6%BC%EB%AA%A9%EB%A1%9D|%ED%8C%8C%EC%9D%BC|%EA%B7%B8%EB%A6%BC).*&sort=img_(timestamp|name|size)(&|$)", - "([\\?&]title=|/)%ED%8A%B9%EC%88%98:(%EA%B8%B0%EB%A1%9D|%EB%A1%9C%EA%B7%B8)/", - "([?&]title=%ED%8A%B9%EC%88%98:(%EA%B8%B0%EB%A1%9D|%EB%A1%9C%EA%B7%B8)&|/%ED%8A%B9%EC%88%98:(%EA%B8%B0%EB%A1%9D|%EB%A1%9C%EA%B7%B8)\\?)(.*&)?hide_[^&=]+=1(&|$)", - "([?&]title=%ED%8A%B9%EC%88%98:(%EA%B8%B0%EB%A1%9D|%EB%A1%9C%EA%B7%B8)&|/%ED%8A%B9%EC%88%98:(%EA%B8%B0%EB%A1%9D|%EB%A1%9C%EA%B7%B8)\\?)(.*&)?hide_.*&hide_", "([?&]title=|/)Property:.*&until=", - "([?&]title=|/)%EB%B6%84%EB%A5%98:.*&(page|subcat|file)until=", "[\\?&]action=edit§ion=(\\d+|new)", "[\\?&]feed(format)?=atom", "[\\?&]redlink=1", @@ -790,11 +722,7 @@ var defaultIgnorePatterns = []string{ "[\\?&]undo(after)?=\\d+", "^https?://a\\.wikia-beacon\\.com/__track/", "^https?://beacon\\.wikia-services\\.com/__track/", - "([\\?&]title=|/)%ED%8A%B9%EC%88%98:(%EC%A0%91%EB%91%90%EC%96%B4%EC%B0%BE%EA%B8%B0|(%EA%B0%80%EB%A6%AC%ED%82%A4%EB%8A%94%EB%AC%B8%EC%84%9C|%EB%A7%81%ED%81%AC%ED%95%98%EB%8A%94%EB%AC%B8%EC%84%9C)|(%EA%B8%B0%EC%97%AC|%EA%B8%B0%EC%97%AC%EB%AA%A9%EB%A1%9D)|(%ED%8C%8C%EC%9D%BC%EB%AA%A9%EB%A1%9D|%EA%B7%B8%EB%A6%BC%EB%AA%A9%EB%A1%9D|%ED%8C%8C%EC%9D%BC|%EA%B7%B8%EB%A6%BC)|(%EC%82%AC%EC%9A%A9%EC%9E%90%EB%AA%A9%EB%A1%9D|%EC%82%AC%EC%9A%A9%EC%9E%90))/.*/\\2/", - "/%EC%82%AC%EC%9A%A9%EC%9E%90%ED%86%A0%EB%A1%A0:.+/%EC%82%AC%EC%9A%A9%EC%9E%90%ED%86%A0%EB%A1%A0:", "/User_blog:.+/User_blog:", - "/%EC%82%AC%EC%9A%A9%EC%9E%90:.+/%EC%82%AC%EC%9A%A9%EC%9E%90:", - "^https?://[^/]+\\.gamepedia\\.com/(.*[?&]title=)?%ED%8A%B9%EC%88%98:(WikiPoints/global|ListAllUsers|AllSites)([/?&]|$)", // youtube.json "^https?://accounts\\.google\\.com/ServiceLogin",