# -*- coding: utf-8 -*-
"""
Script to replace bad Windows-1252 (cp1252) characters with
HTML entities on ISO 8859-1 wikis. Don't run this script on a UTF-8 wiki.
Syntax: python windows_chars.py [pageTitle] [file[:filename]] [sql[:filename]]
Command line options:
-file:XYZ reads a list of pages, which can for exampagee be gotten through
Looxix's robot. XYZ is the name of the file from which the
list is taken. If XYZ is not given, the user is asked for a
filename.
Page titles should be in [[double-square brackets]].
-sql:XYZ reads a local SQL cur dump, available at
http://download.wikimedia.org/. Searches for pages with
Windows-1252 characters, and tries to repair them on the live
wiki. Example:
python windows_chars.py -sql:20040711_cur_table.sql.sql -lang:es
"""
#
# (C) Daniel Herding, 2004
#
# Distributed under the terms of the MIT license.
#
__version__='$Id: windows_chars.py,v 1.27 2005/12/21 17:51:26 wikipedian Exp $'
#
import wikipedia, config
import replace, pagegenerators
import re, sys
# Summary message
msg={
'en':u'robot: changing Windows-1252 characters to HTML entities',
'de':u'Bot: Wandle Windows-1252-Zeichen in HTML-Entitten um',
'fr':u'Bot: Modifie caracteres Windows-1252 vers entits HTML',
'he':u': Windows-1252 HTML',
'ia':u'Robot: modification de characteres Windows-1252 a entitates HTML',
}
# characters that are in Windows-1252), but not in ISO 8859-1
replacements = [
(u"\x80", u"€"), # euro sign
(u"\x82", u"‚"), # single low-9 quotation mark
(u"\x83", u"ƒ"), # latin small f with hook = function = florin
(u"\x84", u"„"), # double low-9 quotation mark
(u"\x85", u"…"), # horizontal ellipsis = three dot leader
(u"\x86", u"†"), # dagger
(u"\x87", u"‡"), # double dagger
(u"\x88", u"ˆ"), # modifier letter circumflex accent
(u"\x89", u"‰"), # per mille sign
(u"\x8A", u"Š"), # latin capital letter S with caron
(u"\x8B", u"‹"), # single left-pointing angle quotation mark
(u"\x8C", u"Œ"), # latin capital ligature OE
(u"\x8E", u"Ž"), # latin capital letter Z with caron
(u"\x91", u"‘"), # left single quotation mark
(u"\x92", u"’"), # right single quotation mark
(u"\x93", u"“"), # left double quotation mark
(u"\x94", u"”"), # right double quotation mark
(u"\x95", u"•"), # bullet = black small circle
(u"\x96", u"–"), # en dash
(u"\x97", u"—"), # em dash
(u"\x98", u"˜"), # small tilde
(u"\x99", u"™"), # trade mark sign
(u"\x9A", u"š"), # latin small letter s with caron
(u"\x9B", u"&8250;"), # single right-pointing angle quotation mark
(u"\x9C", u"œ"), # latin small ligature oe
(u"\x9E", u"ž"), # latin small letter z with caron
(u"\x9F", u"Ÿ") # latin capital letter Y with diaeresis
]
class SqlWindows1252PageGenerator:
"""
opens a local SQL dump file, searches for pages with Windows-1252
characters.
"""
def __init__(self, filename):
self.filename = filename
def __iter__(self):
# open SQL dump and read page titles out of it
import sqldump
sqldump = sqldump.SQLdump(self.filename, 'latin-1')
for entry in sqldump.entries():
for char in replacements.keys():
if entry.text.find(char) != -1:
page = wikipedia.Page(wikipedia.getSite(), entry.full_title())
yield page
break
class WindowsCharsBot:
def __init__(self, generator):
self.generator = generator
def run(self):
replaceBot = replace.ReplaceRobot(self.generator, replacements)
replaceBot.run()
def main():
# this temporary array is used to read the page title.
pageTitle = []
gen = None
for arg in sys.argv[1:]:
arg = wikipedia.argHandler(arg, 'windows_chars')
if arg:
if arg.startswith('-file'):
if len(arg) == 5:
filename = wikipedia.input(u'please enter the list\'s filename: ')
else:
filename = arg[6:]
gen = pagegenerators.TextfilePageGenerator(filename)
elif arg.startswith('-sql'):
if len(arg) == 4:
sqlfilename = wikipedia.input(u'please enter the SQL dump\'s filename: ')
else:
sqlfilename = arg[5:]
gen = SqlWindows1252PageGenerator(sqlfilename)
else:
pageTitle.append(arg)
# if a single page is given as a command line argument,
# reconnect the title's parts with spaces
if pageTitle != []:
page = wikipedia.Page(wikipedia.getSite(), ' '.join(pageTitle))
gen = iter([page])
# get edit summary message
wikipedia.setAction(wikipedia.translate(wikipedia.getSite(), msg))
if not gen:
wikipedia.showHelp('windows_chars')
elif wikipedia.getSite().encoding() == "utf-8":
print "There is no need to run this robot on UTF-8 wikis."
else:
preloadingGen = pagegenerators.PreloadingGenerator(gen)
bot = WindowsCharsBot(preloadingGen)
bot.run()
if __name__ == "__main__":
try:
main()
finally:
wikipedia.stopme()
|