#!/usr/bin/env python
'''
asciidoc - converts an AsciiDoc text file to DocBook, HTML or LinuxDoc
SYNOPSIS
asciidoc -b backend [ -d doctype ] [ -a attribute ]
[ -e ] [-n] [ -s ] [ -f configfile ] [ -o outfile ]
[ --help | -h ] [ --version ] [ -v ] [ -c ]
infile
DESCRIPTION
The asciidoc(1) command translates the AsciiDoc text file 'infile'
to the 'backend' formatted file 'outfile'. If 'infile' is '-' then
the standard input is used.
OPTIONS
--help, -h
Print this documentation.
-a attribute
Define document attribute. 'attribute' is formatted like
'name=value'. Alternate acceptable forms are 'name' (the
'value' defaults to an empty string) and '^name' (delete the
'name' attribute).
-b backend
Backend output file format: 'docbook', 'docbook-sgml',
'xhtml', 'html'. 'css', 'css-embedded', 'linuxdoc'.
'docbbok' generates DocBook XML; 'xhtml' generates XHTML;
'html' generates HTML 4.
-c
Dump configuration to stdout.
-e
Exclude implicitly loaded configuration files except for those
named like the input file ('infile.conf' and
'infile-backend.conf').
-d doctype
Document type: 'article', 'manpage' or 'book'. The 'book'
document type is only supported by the 'docbook' backend and
the 'manpage' document type is not supported by the 'linuxdoc'
backend.
-f configfile
Use configuration file 'configfile'.
-n
Auto-number HTML article section titles.
Synonym for '-a section-numbers'.
-o outfile
Write output to file 'outfile'. Defaults to the base name of
input file with 'backend' extension. If the input is stdin
then the outfile defaults to stdout. If 'outfile' is '-' then
the standard output is used.
-s
Suppress document header and footer output.
-v
Verbosely print processing information and configuration file
checks to stderr.
--version
Print program version number.
BUGS
See the AsciiDoc distribution BUGS file.
AUTHOR
Written by Stuart Rackham, <srackham@methods.co.nz>
RESOURCES
SourceForge: http://sourceforge.net/projects/asciidoc/
Main website: http://www.methods.co.nz/asciidoc/
COPYING
Copyright (C) 2002,2005 Stuart Rackham. Free use of this software
is granted under the terms of the GNU General Public License
(GPL).
'''
import sys, os, re, string, time, traceback, tempfile, popen2
from types import *
VERSION = '6.0.1' # See CHANGLOG file for version history.
#---------------------------------------------------------------------------
# Utility functions and classes.
#---------------------------------------------------------------------------
# Allowed substitution options for subs List options and presubs and postsubs
# Paragraph options.
#DEPRECATED: 'glossary'
SUBS_OPTIONS = ('specialcharacters','quotes','specialwords','replacements',
'glossary','attributes','macros','none','default','callouts')
# Default value for unspecified subs and presubs configuration file entries.
SUBS_DEFAULT = ('specialcharacters','quotes','specialwords','replacements',
'attributes','macros')
class EAsciiDoc(Exception):
pass
from UserDict import UserDict
class OrderedDict(UserDict):
'''Python Cookbook: Ordered Dictionary, Submitter: David Benjamin'''
def __init__(self, d = None):
self._keys = []
UserDict.__init__(self, d)
def __delitem__(self, key):
UserDict.__delitem__(self, key)
self._keys.remove(key)
def __setitem__(self, key, item):
UserDict.__setitem__(self, key, item)
if key not in self._keys: self._keys.append(key)
def clear(self):
UserDict.clear(self)
self._keys = []
def copy(self):
d = UserDict.copy(self)
d._keys = self._keys[:]
return d
def items(self):
return zip(self._keys, self.values())
def keys(self):
return self._keys
def popitem(self):
try:
key = self._keys[-1]
except IndexError:
raise KeyError('dictionary is empty')
val = self[key]
del self[key]
return (key, val)
def setdefault(self, key, failobj = None):
UserDict.setdefault(self, key, failobj)
if key not in self._keys: self._keys.append(key)
def update(self, d):
UserDict.update(self, d)
for key in d.keys():
if key not in self._keys: self._keys.append(key)
def values(self):
return map(self.get, self._keys)
def print_stderr(line):
sys.stderr.write(line+os.linesep)
def verbose(msg,linenos=True):
'''-v option messages.'''
if config.verbose:
console(msg,linenos=linenos)
def warning(msg,linenos=True):
console(msg,'WARNING: ',linenos)
def deprecated(old, new, linenos=True):
console("%s: %s" % (old,new), 'DEPRECATED: ', linenos)
def error(msg):
'''Report fatal error but don't exit application, continue in the hope of
reporting all fatal errors finishing with a non-zero exit code.'''
console(msg,'ERROR: ')
document.invalid = True
def console(msg, prefix='', linenos=True):
'''Print message to stderr. 'offset' is added to reported line number for
warnings emitted when reading ahead.'''
s = prefix
if linenos and reader.cursor:
s = s + "%s: line %d: " \
% (os.path.basename(reader.cursor[0]),reader.cursor[1])
s = s + msg
print_stderr(s)
def realpath(fname):
'''Return the absolute pathname of the file fname. Follow symbolic links.
os.realpath() not available in Python prior to 2.2 and not portable.'''
# Follow symlinks to the actual executable.
wd = os.getcwd()
try:
while os.path.islink(fname):
linkdir = os.path.dirname(fname)
fname = os.readlink(fname)
if linkdir: os.chdir(linkdir) # Symlinks can be relative.
fname = os.path.abspath(fname)
finally:
os.chdir(wd)
return fname
def syseval(cmd):
# Run shell command and return stdout.
child = os.popen(cmd)
data = child.read()
err = child.close()
if not err:
return data
else:
return ''
def assign(dst,src):
'''Assign all attributes from 'src' object to 'dst' object.'''
for a,v in src.__dict__.items():
setattr(dst,a,v)
def strip_quotes(s):
'''Trim white space and, if necessary, quote characters from s.'''
s = string.strip(s)
# Strip quotation mark characters from quoted strings.
if len(s) >= 3 and s[0] == '"' and s[-1] == '"':
s = s[1:-1]
return s
def is_regexp(s):
'''Return True if s is a valid regular expression else return False.'''
try: re.compile(s)
except: return False
else: return True
def join_regexp(relist):
'''Join list of regular expressions re1,re2,... to single regular
expression (re1)|(re2)|...'''
if len(relist) == 0:
return ''
result = []
# Delete named groups to avoid ambiguity.
for s in relist:
result.append(re.sub(r'\?P<\S+?>','',s))
result = string.join(result,')|(')
result = '('+result+')'
return result
def validate(value,rule,errmsg):
'''Validate value against rule expression. Throw EAsciiDoc exception with
errmsg if validation fails.'''
try:
if not eval(string.replace(rule,'$',str(value))):
raise EAsciiDoc,errmsg
except:
raise EAsciiDoc,errmsg
return value
def join_lines(lines):
'''Return a list in which lines terminated with the backslash line
continuation character are joined.'''
result = []
s = ''
continuation = False
for line in lines:
if line and line[-1] == '\\':
s = s + line[:-1]
continuation = True
continue
if continuation:
result.append(s+line)
s = ''
continuation = False
else:
result.append(line)
if continuation:
result.append(s)
return result
def parse_attrs(attrs,dict,default_arg=None):
'''Update a dictionary with name/value attributes from theattrsstring. import
The attrs string is a comma separated list of values and keyword name=value
pairs. Values must preceed keywords and are named '1','2'... The entire
attributes list is named '0'. If keywords are specified string values must
be quoted. Examples:
attrs: ''
dict: {}
attrs: 'hello,world'
dict: {'2': 'world', '0': 'hello,world', '1': 'hello'}
attrs: 'hello,,world'
default_arg: 'attrs'
dict: {'3': 'world', 'attrs': 'hello', '0': 'hello,,world', '1': 'hello'}
attrs: '"hello",planet="earth"'
dict: {'planet': 'earth', '0': '"hello",planet="earth"', '1': 'hello'}
'''
def f(*args,**keywords):
# Name and add aguments '1','2'... to keywords.
for i in range(len(args)):
if not keywords.has_key(str(i+1)):
keywords[str(i+1)] = args[i]
return keywords
if not attrs: return
dict['0'] = attrs
# Replace line separators with spaces so line spanning works.
s = re.sub(r'\s', ' ', attrs)
try:
d = eval('f('+s+')')
# Parse special 'options' argument.
if d.has_key('options') and d['options']:
for opt in re.split(r'\s*,\s*',d['options']):
if not is_name(opt):
warning('illegal option name in "%s"' % (d['options'],))
else:
d[opt] = '' # Create blank entry for each option.
dict.update(d)
except:
# Try quoting the attrs.
s = string.replace(s,'"',r'\"') # Escape double-quotes.
s = string.split(s,',')
s = map(lambda x: '"'+string.strip(x)+'"',s)
s = string.join(s,',')
try:
d = eval('f('+s+')')
except:
return # If there's a syntax error leave with {0}=attrs.
for k in d.keys(): # Drop any arguments that were missing.
if d[k] == '': del d[k]
dict.update(d)
assert len(d) > 0
if default_arg is not None and not d.has_key(default_arg) \
and d.has_key('1'):
dict[default_arg] = dict['1']
def parse_list(s):
'''Parse comma separated string of Python literals. Return a tuple of of
parsed values.'''
try:
result = eval('tuple(['+s+'])')
except:
raise EAsciiDoc,'malformed list: '+s
return result
def parse_options(options,allowed,errmsg):
'''Parse comma separated string of unquoted option names and return as a
tuple of valid options. 'allowed' is a list of allowed option values.
'errmsg' isan error message prefix if an illegal option error is thrown.'''
result = []
if options:
for s in re.split(r'\s*,\s*',options):
if s not in allowed:
raise EAsciiDoc,'%s "%s"' % (errmsg,s)
result.append(s)
return tuple(result)
def is_name(s):
'''Return True if s is valid attribute, macro or tag name
(starts with alpha containing alphanumeric and dashes only).'''
return re.match(r'[^\W\d][-\w]*',s) is not None
def subs_quotes(text):
'''Quoted text is marked up and the resulting text is
returned.'''
quotes = config.quotes.keys()
# The quotes are iterated in reverse sort order to avoid ambiguity,
# for example, '' is processed before '.
quotes.sort()
quotes.reverse()
for quote in quotes:
i = string.find(quote,'|')
if i != -1 and quote != '|' and quote != '||':
lq = quote[:i]
rq = quote[i+1:]
else:
lq = rq = quote
# Non-word (\W) characters are allowed at boundaries to accomodate
# enveloping quotes. re.S flag for line spanning.
reo = re.compile(r'(?ms)(^|\W)(?:' + re.escape(lq) + r')' \
+ r'(.*?)(?:'+re.escape(rq)+r')(?=\W|$)')
pos = 0
while True:
mo = reo.search(text,pos)
if not mo: break
if text[mo.start()] == '\\':
pos = mo.end()
else:
stag,etag = config.tag(config.quotes[quote])
if stag == etag == None:
s = ''
else:
s = mo.group(1) + stag + mo.group(2) + etag
text = text[:mo.start()] + s + text[mo.end():]
pos = mo.start() + len(s)
# Unescape escaped quotes.
text = string.replace(text,'\\'+lq, lq)
if lq != rq:
text = string.replace(text,'\\'+rq, rq)
return text
def subs_tag(tag,dict={}):
'''Perform attribute substitution and split tag string returning start, end
tag tuple (c.f. Config.tag()).'''
s = subs_attrs(tag,dict)
result = string.split(s,'|')
if len(result) == 1:
return result+[None]
elif len(result) == 2:
return result
else:
raise EAsciiDoc,'malformed tag "%s"' % (tag,)
def parse_entry(entry,dict=None,unquote=False,unique_values=False):
'''Parse name=value entry to dictionary 'dict'. Return tuple (name,value)
or None if illegal entry. If value is omitted (name=) then it is set to ''.
If only the name is present the value is set to None).
Leading and trailing white space is striped from import
'name' can contain any printable characters. If 'name includes the equals
'=' character it must be escaped with a backslash.
If 'unquote' is True leading and trailing double-quotes are stripped from import
'name' and 'value'.
If unique_values' is True then dictionary entries with the same value are
removed before the parsed entry is added.'''
mo=re.search(r'[^\\](=)',entry)
if mo: # name=value entry.
name = entry[:mo.start(1)]
value = entry[mo.end(1):]
else: # name entry.
name = entry
value = None
if unquote:
name = strip_quotes(name)
if value is not None:
value = strip_quotes(value)
else:
name = string.strip(name)
if value is not None:
value = string.strip(value)
if not name:
return None
if dict is not None:
if unique_values:
for k,v in dict.items():
if v == value: del dict[k]
dict[name] = value
return name,value
def parse_entries(entries, dict, unquote=False, unique_values=False):
'''Parse name=value entries from from lines of text in 'entries' into
dictionary 'dict'. Blank lines are skipped.'''
for entry in entries:
if entry and not parse_entry(entry,dict,unquote,unique_values):
raise EAsciiDoc,'malformed section entry "%s"' % (entry,)
def undefine_entries(entries):
'''All dictionary entries with None values are deleted.'''
for k,v in entries.items():
if v is None:
del entries[k]
def dump_section(name,dict,f=sys.stdout):
'''Write parameters in 'dict' as in configuration file section format with
section 'name'.'''
f.write('[%s]%s' % (name,writer.newline))
for k,v in dict.items():
k = str(k)
# Quote if necessary.
if len(k) != len(string.strip(k)):
k = '"'+k+'"'
if v and len(v) != len(string.strip(v)):
v = '"'+v+'"'
if v is None:
# Don't dump undefined entries.
continue
else:
s = k+'='+v
f.write('%s%s' % (s,writer.newline))
f.write(writer.newline)
def update_attrs(attrs,dict):
'''Update 'attrs' dictionary with parsed attributes in dictionary 'dict'.'''
for k,v in dict.items():
if not is_name(k):
raise EAsciiDoc,'illegal "%s" attribute name' % (k,)
attrs[k] = v
def readlines(fname):
'''Read lines from file named 'fname' and strip trailing white space.'''
# Read include file.
f = open(fname)
try:
lines = f.readlines()
finally:
f.close()
# Strip newlines.
for i in range(len(lines)):
lines[i] = string.rstrip(lines[i])
return lines
def filter_lines(filter,lines,dict={}):
'''Run 'lines' through the 'filter' shell command and return the result. The
'dict' dictionary contains additional filter attributes.'''
# BUG: Has problems finding filters with spaces in command name.
if not filter:
return lines
if os.name != 'posix':
warning('filters do not work in a non-posix environment')
return lines
# Perform attributes substitution on the filter command.
s = subs_attrs(filter,dict)
if not s:
raise EAsciiDoc,'filter "%s" has undefined attribute' % (filter,)
filter = s
# Search for the filter command in both user and application 'filters'
# sub-directories.
mo = re.match(r'^(?P<cmd>\S+)(?P<tail>.*)$', filter)
if mo:
cmd = mo.group('cmd')
found = False
if not os.path.dirname(cmd):
# Check in asciidoc user and application directories for unqualified
# file name.
if USER_DIR:
cmd2 = os.path.join(USER_DIR,'filters',cmd)
if os.path.isfile(cmd2):
found = True
if not found:
cmd2 = os.path.join(APP_DIR,'filters',cmd)
if os.path.isfile(cmd2):
found = True
if found:
filter = cmd2 + mo.group('tail')
else:
if os.uname()[0][:6] == 'CYGWIN':
# popen2() does not like non-drive letter path names under
# Cygwin.
s = string.strip(syseval('cygpath -m "'+cmd+'"'))
if s:
cmd = s
if not os.path.isfile(cmd):
warning('filter not found: "%s"' % (cmd,))
return lines
# Put the real filter path into the filter command.
verbose('filtering: '+filter)
try:
import select
result = []
r,w = popen2.popen2(filter)
# Polled I/O loop to alleviate full buffer deadlocks.
i = 0
while i < len(lines):
line = lines[i]
if select.select([],[w.fileno()],[],0)[1]:
w.write(line+os.linesep) # Use platform line terminator.
i = i+1
if select.select([r.fileno()],[],[],0)[0]:
s = r.readline()
if not s: break # Exit if filter output closes.
result.append(string.rstrip(s))
w.close()
for s in r.readlines():
result.append(string.rstrip(s))
r.close()
except:
raise EAsciiDoc,'filter "%s" error' % (filter,)
# There's no easy way to guage whether popen2() found and executed the
# filter, so guess that if it produced no output there is probably a
# problem.
if lines and not result:
warning('no output from filter "%s"' % (filter,))
return result
def execute_attr(action, expr):
'''Return the result of a executable {action:expr} attribute.'''
verbose('evaluating: {%s:%s}' % (action,expr))
result = None
if action == 'eval':
try:
result = eval(expr)
if result is not None:
result = str(result)
except:
warning('{%s:%s}: expression evaluation error' % (action,expr))
elif action in ('sys','sys2'):
result = ''
fd,tmp = tempfile.mkstemp()
os.close(fd)
try:
cmd = expr
cmd = cmd + (' > %s' % tmp)
if action == 'sys2':
cmd = cmd + ' 2>&1'
if os.system(cmd):
warning('{%s:%s}: non-zero exit status' % (action,expr))
try:
if os.path.isfile(tmp):
lines = readlines(tmp)
else:
lines = []
except:
raise EAsciiDoc,'{%s:%s}: temp file read error' % (action,expr)
result = string.join(lines, writer.newline)
finally:
if os.path.isfile(tmp):
os.remove(tmp)
elif action == 'include':
if not os.path.exists(expr):
warning('{%s:%s}: file does not exist' % (action,expr))
else:
result = readlines(expr)
result = subs_attrs(result)
result = string.join(result, writer.newline)
result = string.expandtabs(result, reader.tabsize)
else:
warning('Illegal {%s:%s}: executable attribute name' % (action,expr))
return result
def subs_attrs(lines,dict={}):
'''Substitute 'lines' of text with attributes from the import
document.attributes dictionary and from thedictionary import
entries take precedence). Return a tuple of the substituted lines. 'lines'
containing undefined attributes are deleted. If 'lines' is a string then
return a string.'''
def end_brace(text,start):
'''Return index following end brace that matches brace at start in
text.'''
assert text[start] == '{'
n = 0
result = start
for c in text[start:]:
# Skip braces that are followed by a backslash.
if result == len(text)-1 or text[result+1] != '\\':
if c == '{': n = n + 1
elif c == '}': n = n - 1
result = result + 1
if n == 0: break
return result
if isinstance(lines,StringType):
string_result = True
lines = [lines]
else:
string_result = False
lines = list(lines)
attrs = document.attributes.copy()
attrs.update(dict)
# Substitute all attributes in all lines.
for i in range(len(lines)-1,-1,-1): # Reverse iterate lines.
text = lines[i]
# Make it easier for regular expressions.
text = string.replace(text,'\\{','{\\')
text = string.replace(text,'\\}','}\\')
# Expand simple attributes ({name}).
# Nested attributes not allowed.
reo = re.compile(r'\{(?P<name>[^\\\W][-\w]*?)\}(?!\\)', re.DOTALL)
pos = 0
while True:
mo = reo.search(text,pos)
if not mo: break
s = attrs.get(mo.group('name'))
if s is None:
pos = mo.end()
else:
s = str(s)
text = text[:mo.start()] + s + text[mo.end():]
pos = mo.start() + len(s)
# Expand calculated attributes ({name=value},
# {name?value}, {name!value} and {name#value}).
reo = re.compile(r'\{(?P<name>[^\\\W][-\w]*?)(?P<op>\=|\?|!|#|%)' \
r'(?P<value>.*?)\}(?!\\)',re.DOTALL)
pos = 0
while True:
mo = reo.search(text,pos)
if not mo: break
name = mo.group('name')
lval = attrs.get(name)
op = mo.group('op')
# mo.end() is not good enough because '{x={y}}' matches '{x={y}'.
end = end_brace(text,mo.start())
rval = text[mo.start('value'):end-1]
if lval is None:
if op == '=': s = rval
elif op == '?': s = ''
elif op == '!': s = rval
elif op == '#': s = '{'+name+'}' # So the line is deleted.
elif op == '%': s = rval
else: assert True,'illegal calculated attribute operator'
else:
if op == '=': s = lval
elif op == '?': s = rval
elif op == '!': s = ''
elif op == '#': s = rval
elif op == '%': s = '{zzzzz}' # So the line is deleted.
else: assert True,'illegal calculated attribute operator'
s = str(s)
text = text[:mo.start()] + s + text[end:]
pos = mo.start() + len(s)
# Drop line if it contains unsubstituted {name} references.
skipped = re.search(r'\{[^\\\W][-\w]*?\}(?!\\)', text, re.DOTALL)
if skipped:
del lines[i]
continue;
# Expand executable attributes ({name:expression}).
reo = re.compile(r'\{(?P<action>[^\\\W][-\w]*?):(?P<expr>.*?)\}(?!\\)',
re.DOTALL)
skipped = False
pos = 0
while True:
mo = reo.search(text,pos)
if not mo: break
expr = mo.group('expr')
expr = string.replace(expr,'{\\','{')
expr = string.replace(expr,'}\\','}')
s = execute_attr(mo.group('action'),expr)
if s is None:
skipped = True
break
text = text[:mo.start()] + s + text[mo.end():]
pos = mo.start() + len(s)
# Drop line if the action returns None.
if skipped:
del lines[i]
continue;
# Remove backslash from escaped entries.
text = string.replace(text,'{\\','{')
text = string.replace(text,'}\\','}')
lines[i] = text
if string_result:
if lines:
return string.join(lines,'\n')
else:
return None
else:
return tuple(lines)
class Lex:
'''Lexical analysis routines. Static methods and attributes only.'''
prev_element = None
prev_cursor = None
def __init__(self):
raise AssertionError,'no class instances allowed'
def next():
'''Returns class of next element on the input (None if EOF). The
reader is assumed to be at the first line following a previous element,
end of file or line one. Exits with the reader pointing to the first
line of the next element or EOF (leading blank lines are skipped).'''
reader.skip_blank_lines()
if reader.eof(): return None
# Optimization: If we've already checked for an element at this
# position return the element.
if Lex.prev_element and Lex.prev_cursor == reader.cursor:
return Lex.prev_element
result = None
# Check for BlockTitle.
if not result and BlockTitle.isnext():
result = BlockTitle
# Check for Title.
if not result and Title.isnext():
result = Title
# Check for Block Macro.
if not result and macros.isnext():
result = macros.current
# Check for List.
if not result and lists.isnext():
result = lists.current
# Check for DelimitedBlock.
if not result and blocks.isnext():
# Skip comment blocks.
if 'skip' in blocks.current.options:
blocks.current.translate()
return Lex.next()
else:
result = blocks.current
# Check for Table.
if not result and tables.isnext():
result = tables.current
# Check for AttributeEntry.
if not result and AttributeEntry.isnext():
result = AttributeEntry
# Check for AttributeList.
if not result and AttributeList.isnext():
result = AttributeList
# If it's none of the above then it must be an Paragraph.
if not result:
if not paragraphs.isnext():
raise EAsciiDoc,'paragraph expected'
result = paragraphs.current
# Cache answer.
Lex.prev_cursor = reader.cursor
Lex.prev_element = result
return result
next = staticmethod(next)
def subs_1(s,options):
'''Perform substitution specified in 'options' (in 'options' order) on
a single line 's' of text. Returns the substituted string.'''
if not s:
return s
result = s
for o in options:
if o == 'specialcharacters':
result = config.subs_specialchars(result)
# Quoted text.
elif o == 'quotes':
result = subs_quotes(result)
# Special words.
elif o == 'specialwords':
result = config.subs_specialwords(result)
# Replacements.
elif o == 'replacements':
result = config.subs_replacements(result)
# Inline macros.
elif o == 'macros':
result = macros.subs(result)
elif o == 'callouts':
result = macros.subs(result,callouts=True)
else:
raise EAsciiDoc,'illegal "%s" substitution option' % (o,)
return result
subs_1 = staticmethod(subs_1)
def subs(lines,options):
'''Perform inline processing specified by 'options' (in 'options'
order) on sequence of 'lines'.'''
if len(options) == 1:
if options[0] == 'none':
options = ()
elif options[0] == 'default':
options = SUBS_DEFAULT
if not lines or not options:
return lines
# Join lines so quoting can span multiple lines.
para = string.join(lines,"\n")
for o in options:
if o == 'glossary' or o == 'attributes':
if o == 'glossary':
deprecated("'glossary' substitution option",
"change to 'attributes'")
# If we don't substitute attributes line-by-line then a single
# undefined attribute will drop the entire paragraph.
# But this is probably what we want.
lines = subs_attrs(para.split("\n"))
para = string.join(lines,"\n")
else:
para = Lex.subs_1(para,(o,))
return para.split("\n")
subs = staticmethod(subs)
def set_margin(lines, margin=0):
'''Utility routine that sets the left margin to 'margin' space in a
block of non-blank lines.'''
# Calculate width of block margin.
lines = list(lines)
width = len(lines[0])
for s in lines:
i = re.search(r'\S',s).start()
if i < width: width = i
# Strip margin width from all lines.
for i in range(len(lines)):
lines[i] = ' '*margin + lines[i][width:]
return lines
set_margin = staticmethod(set_margin)
#---------------------------------------------------------------------------
# Document element classes parse AsciiDoc reader input and write DocBook writer
# output.
#---------------------------------------------------------------------------
class Document:
def __init__(self):
self.doctype = None # 'article','manpage' or 'book'.
self.backend = None # -b option argument.
self.infile = None # Source file name.
self.outfile = None # Output file name.
self.attributes = {}
self.level = 0 # 0 => front matter. 1,2,3 => sect1,2,3.
self.invalid = False # Set true if translated document is invalid.
def init_attrs(self):
# Set implicit attributes.
d = time.localtime(time.time())
self.attributes['localdate'] = time.strftime('%d-%b-%Y',d)
s = time.strftime('%H:%M:%S',d)
if time.daylight:
self.attributes['localtime'] = s + ' ' + time.tzname[1]
else:
self.attributes['localtime'] = s + ' ' + time.tzname[0]
self.attributes['asciidoc-version'] = VERSION
self.attributes['backend'] = document.backend
self.attributes['doctype'] = document.doctype
self.attributes['backend-'+document.backend] = ''
self.attributes['doctype-'+document.doctype] = ''
self.attributes[document.backend+'-'+document.doctype] = ''
self.attributes['asciidoc-dir'] = APP_DIR
self.attributes['user-dir'] = USER_DIR
if self.infile != '<stdin>':
self.attributes['infile'] = self.infile
self.attributes['docdir'] = os.path.dirname(self.infile)
self.attributes['docname'] = os.path.splitext(
os.path.basename(self.infile))[0]
# Update with configuration file attributes.
self.attributes.update(config.conf_attrs)
# Update with command-line attributes.
self.attributes.update(config.cmd_attrs)
# Filter miscelaneous configuration section entries from attributes.
config.load_miscellaneous(config.conf_attrs)
config.load_miscellaneous(config.cmd_attrs)
self.attributes['newline'] = config.newline # Use raw (unescaped) value.
if self.outfile:
if self.outfile != '<stdout>':
self.attributes['outfile'] = self.outfile
ext = os.path.splitext(self.outfile)[1][1:]
elif config.outfilesuffix:
ext = config.outfilesuffix[1:]
else:
ext = ''
if ext:
self.attributes['filetype'] = ext
self.attributes['filetype-'+ext] = ''
def translate(self):
assert self.doctype in ('article','manpage','book'), \
'illegal document type'
assert self.level == 0
# Process document header.
AttributeEntry.translate_all()
has_header = Lex.next() is Title and Title.level == 0
if self.doctype == 'manpage' and not has_header:
error('manpage document title is mandatory')
if has_header:
Header.translate()
# Command-line entries override header derived entries.
self.attributes.update(config.cmd_attrs)
if not config.suppress_headers:
hdr = config.subs_section('header',{})
writer.write(hdr)
if self.doctype in ('article','book'):
# Translate 'preamble' (untitled elements between header
# and first section title).
if Lex.next() is not Title:
stag,etag = config.section2tags('preamble')
writer.write(stag)
Section.translate_body()
writer.write(etag)
else:
# Translate manpage SYNOPSIS.
if Lex.next() is not Title:
error('SYNOPSIS section expected')
else:
Title.translate()
if string.upper(Title.dict['title']) <> 'SYNOPSIS':
error('second section must be named SYNOPSIS')
if Title.level != 1:
error('SYNOPSIS section title must be at level 1')
stag,etag = config.section2tags('sect-synopsis')
writer.write(stag)
Section.translate_body()
writer.write(etag)
else:
if not config.suppress_headers:
hdr = config.subs_section('header',{})
writer.write(hdr)
if Lex.next() is not Title:
Section.translate_body()
# Process remaining sections.
while not reader.eof():
if Lex.next() is not Title:
raise EAsciiDoc,'section title expected'
Section.translate()
Section.setlevel(0) # Write remaining unwritten section close tags.
# Substitute document parameters and write document footer.
if not config.suppress_headers:
ftr = config.subs_section('footer',{})
writer.write(ftr)
def parse_author(self,s):
attrs = self.attributes # Alias for readability.
s = subs_attrs(s)
s = string.strip(s)
mo = re.match(r'^(?P<name1>[^<>\s]+)'
'(\s+(?P<name2>[^<>\s]+))?'
'(\s+(?P<name3>[^<>\s]+))?'
'(\s+<(?P<email>\S+)>)?$',s)
if not mo:
error('malformed author line')
firstname = mo.group('name1')
if mo.group('name3'):
middlename = mo.group('name2')
lastname = mo.group('name3')
else:
middlename = None
lastname = mo.group('name2')
email = mo.group('email')
author = firstname
initials = firstname[0]
if middlename:
author += ' '+middlename
initials += middlename[0]
if lastname:
author += ' '+lastname
initials += lastname[0]
initials = string.upper(initials)
if firstname:
attrs['firstname'] = config.subs_specialchars(firstname)
if middlename:
attrs['middlename'] = config.subs_specialchars(middlename)
if lastname:
attrs['lastname'] = config.subs_specialchars(lastname)
if author:
attrs['author'] = config.subs_specialchars(author)
if initials:
attrs['authorinitials'] = config.subs_specialchars(initials)
if email:
attrs['email'] = email
class Header:
'''Static methods and attributes only.'''
def __init__(self):
raise AssertionError,'no class instances allowed'
def translate():
assert Lex.next() is Title and Title.level == 0
Title.translate()
attrs = document.attributes # Alias for readability.
attrs['doctitle'] = Title.dict['title']
if document.doctype == 'manpage':
# manpage title formatted like mantitle(manvolnum).
mo = re.match(r'^(?P<mantitle>.*)\((?P<manvolnum>.*)\)$',
Title.dict['title'])
if not mo: error('malformed manpage title')
attrs['mantitle'] = string.strip(string.lower(mo.group('mantitle')))
attrs['manvolnum'] = string.strip(mo.group('manvolnum'))
AttributeEntry.translate_all()
s = reader.read_next()
if s:
s = reader.read()
document.parse_author(s)
AttributeEntry.translate_all()
if reader.read_next():
# Parse revision line.
s = reader.read()
s = subs_attrs(s)
# Match RCS/CVS $Id: asciidoc.py,v 1.1 2006/04/09 16:59:12 edream Exp $ marker format.
mo = re.match(r'^\$Id: \S+ (?P<revision>\S+)'
' (?P<date>\S+) \S+ \S+ \S+ \$$',s)
if not mo:
# Match AsciiDoc revision,date format.
mo = re.match(r'^\D*(?P<revision>.*?),(?P<date>.+)$',s)
if mo:
revision = string.strip(mo.group('revision'))
date = string.strip(mo.group('date'))
else:
revision = None
date = string.strip(s)
if revision:
attrs['revision'] = revision
if date:
attrs['date'] = date
AttributeEntry.translate_all()
if document.backend == 'linuxdoc' and not attrs.has_key('author'):
warning('linuxdoc requires author name')
if document.doctype == 'manpage':
# Translate mandatory NAME section.
if Lex.next() is not Title:
error('NAME section expected')
else:
Title.translate()
if string.upper(Title.dict['title']) <> 'NAME':
error('first section must be named NAME')
if Title.level != 1:
error('NAME section title must be at level 1')
if not isinstance(Lex.next(),Paragraph):
error('malformed NAME section body')
lines = reader.read_until(r'^$')
s = string.join(lines)
mo = re.match(r'^(?P<manname>.*?)-(?P<manpurpose>.*)$',s)
if not mo:
error('malformed NAME section body')
attrs['manname'] = string.strip(mo.group('manname'))
attrs['manpurpose'] = string.strip(mo.group('manpurpose'))
if attrs.get('author',None) or attrs.get('email',None):
attrs['authored'] = ''
translate = staticmethod(translate)
class AttributeEntry:
'''Static methods and attributes only.'''
pattern = None
name = None
value = None
def __init__(self):
raise AssertionError,'no class instances allowed'
def isnext():
result = False # Assume not next.
if not AttributeEntry.pattern:
pat = document.attributes.get('attributeentry-pattern')
if not pat:
error("[attributes] missing 'attributeentry-pattern' entry")
AttributeEntry.pattern = pat
line = reader.read_next()
if line:
mo = re.match(AttributeEntry.pattern,line)
if mo:
name = mo.group('attrname')
# Strip white space and illegal name chars.
name = string.lower(re.sub(r'[^\w\-_]', '', name))
AttributeEntry.name = name
# Trim value.
AttributeEntry.value = string.strip(mo.group('attrvalue'))
result = True
return result
isnext = staticmethod(isnext)
def translate():
assert Lex.next() is AttributeEntry
reader.read() # Discard attribute from reader.
# Don't override command-line attributes.
if config.cmd_attrs.has_key(AttributeEntry.name):
return
# Update document.attributes from previously parsed attribute.
if AttributeEntry.value:
AttributeEntry.value = subs_attrs(AttributeEntry.value)
if AttributeEntry.value is not None:
document.attributes[AttributeEntry.name] = AttributeEntry.value
if AttributeEntry.name == 'author':
document.parse_author(AttributeEntry.value)
elif document.attributes.has_key(AttributeEntry.name):
del document.attributes[AttributeEntry.name]
translate = staticmethod(translate)
def translate_all():
''' Process all contiguous attribute lines on reader.'''
while AttributeEntry.isnext():
AttributeEntry.translate()
translate_all = staticmethod(translate_all)
class AttributeList:
'''Static methods and attributes only.'''
pattern = None
match = None
attrs = {}
def __init__(self):
raise AssertionError,'no class instances allowed'
def isnext():
result = False # Assume not next.
if not AttributeList.pattern:
if not document.attributes.has_key('attributelist-pattern'):
error("[miscellaneous] missing 'attributelist-pattern' entry")
AttributeList.pattern = document.attributes['attributelist-pattern']
line = reader.read_next()
if line:
mo = re.match(AttributeList.pattern, line)
if mo:
AttributeList.match = mo
result = True
return result
isnext = staticmethod(isnext)
def translate():
assert Lex.next() is AttributeList
reader.read() # Discard attribute list from reader.
dict = AttributeList.match.groupdict()
for k,v in dict.items():
if v is not None:
if k == 'attrlist':
parse_attrs(v, AttributeList.attrs)
else:
AttributeList.attrs[k] = v
translate = staticmethod(translate)
def consume(dict):
'''Add attribute list to the dictionary 'dict' and reset the
list.'''
if AttributeList.attrs:
verbose('attribute list: %s' % AttributeList.attrs)
dict.update(AttributeList.attrs)
AttributeList.attrs = {}
consume = staticmethod(consume)
class BlockTitle:
'''Static methods and attributes only.'''
title = None
pattern = None
def __init__(self):
raise AssertionError,'no class instances allowed'
def isnext():
result = False # Assume not next.
line = reader.read_next()
if line:
mo = re.match(BlockTitle.pattern,line)
if mo:
BlockTitle.title = mo.group('title')
result = True
return result
isnext = staticmethod(isnext)
def translate():
assert Lex.next() is BlockTitle
reader.read() # Discard title from reader.
# Perform title substitutions.
s = Lex.subs((BlockTitle.title,), Title.subs)
s = string.join(s,writer.newline)
if not s:
warning('blank block title')
BlockTitle.title = s
translate = staticmethod(translate)
def consume(dict):
'''If there is a title add it to dict then reset title.'''
if BlockTitle.title:
dict['title'] = BlockTitle.title
BlockTitle.title = None
consume = staticmethod(consume)
class Title:
'''Processes Header and Section titles. Static methods and attributes
only.'''
# Class variables
underlines = ('==','--','~~','^^','++') # Levels 0,1,2,3,4.
subs = ('specialcharacters','quotes','replacements','attributes','macros')
pattern = None
level = 0
dict = {}
sectname = None
section_numbers = [0]*len(underlines)
dump_dict = {}
linecount = None # Number of lines in title (1 or 2).
def __init__(self):
raise AssertionError,'no class instances allowed'
def translate():
'''Parse the Title.dict and Title.level from thereader.The import
real work has already been done by parse().'''
assert Lex.next() is Title
# Discard title from reader.
for i in range(Title.linecount):
reader.read()
Title.setsectname()
# Perform title substitutions.
s = Lex.subs((Title.dict['title'],), Title.subs)
s = string.join(s,writer.newline)
if not s:
warning('blank section title')
Title.dict['title'] = s
translate = staticmethod(translate)
def isnext():
lines = reader.read_ahead(2)
return Title.parse(lines)
isnext = staticmethod(isnext)
def parse(lines):
'''Parse title at start of lines tuple.'''
if len(lines) == 0: return False
if len(lines[0]) == 0: return False # Title can't be blank.
# Check for single-line titles.
result = False
for level in range(len(Title.underlines)):
k = 'sect%s' % level
if Title.dump_dict.has_key(k):
mo = re.match(Title.dump_dict[k], lines[0])
if mo:
Title.dict = mo.groupdict()
Title.level = level
Title.linecount = 1
result = True
break
if not result:
# Check for double-line titles.
if not Title.pattern: return False # Single-line titles only.
if len(lines) < 2: return False
title,ul = lines[:2]
if len(ul) < 2: return False
# Fast check.
if ul[:2] not in Title.underlines: return False
# Length of underline must be within +-3 of title.
if not (len(ul)-3 < len(title) < len(ul)+3): return False
# Check for valid repetition of underline character pairs.
s = ul[:2]*((len(ul)+1)/2)
if ul != s[:len(ul)]: return False
mo = re.match(Title.pattern, title)
if mo:
Title.dict = mo.groupdict()
Title.level = list(Title.underlines).index(ul[:2])
Title.linecount = 2
result = True
# Check for expected pattern match groups.
if result:
if Title.dict.has_key('args'):
if Title.dict['args'] is not None:
deprecated('title arguments','use attribute lists')
parse_attrs(Title.dict['args'],Title.dict)
if not Title.dict.has_key('title'):
warning('[titles] entry has no <title> group')
Title.dict['title'] = lines[0]
for k,v in Title.dict.items():
if v is None: del Title.dict[k]
return result
parse = staticmethod(parse)
def load(dict):
'''Load and validate [titles] section entries from dict.'''
if dict.has_key('underlines'):
errmsg = 'malformed [titles] underlines entry'
try:
underlines = parse_list(dict['underlines'])
except:
raise EAsciiDoc,errmsg
if len(underlines) != len(Title.underlines):
raise EAsciiDoc,errmsg
for s in underlines:
if len(s) !=2:
raise EAsciiDoc,errmsg
Title.underlines = tuple(underlines)
Title.dump_dict['underlines'] = dict['underlines']
if dict.has_key('subs'):
Title.subs = parse_options(dict['subs'], SUBS_OPTIONS,
'illegal [titles] subs entry')
Title.dump_dict['subs'] = dict['subs']
if dict.has_key('sectiontitle'):
pat = dict['sectiontitle']
if not pat or not is_regexp(pat):
raise EAsciiDoc,'malformed [titles] sectiontitle entry'
Title.pattern = pat
Title.dump_dict['sectiontitle'] = pat
if dict.has_key('blocktitle'):
pat = dict['blocktitle']
if not pat or not is_regexp(pat):
raise EAsciiDoc,'malformed [titles] blocktitle entry'
BlockTitle.pattern = pat
Title.dump_dict['blocktitle'] = pat
# Load single-line title patterns.
for k in ('sect0','sect1','sect2','sect3','sect4'):
if dict.has_key(k):
pat = dict[k]
if not pat or not is_regexp(pat):
raise EAsciiDoc,'malformed [titles] %s entry' % k
Title.dump_dict[k] = pat
# TODO: Check we have either a Title.pattern or at least one
# single-line title pattern -- can this be done here or do we need
# check routine like the other block checkers?
load = staticmethod(load)
def dump():
dump_section('titles',Title.dump_dict)
dump = staticmethod(dump)
def setsectname():
'''Set Title section name. First search for section title in
[specialsections], if not found use default 'sect<level>' name.'''
for pat,sect in config.specialsections.items():
mo = re.match(pat,Title.dict['title'])
if mo:
title = mo.groupdict().get('title')
if title is not None:
Title.dict['title'] = string.strip(title)
else:
Title.dict['title'] = string.strip(mo.group())
Title.sectname = sect
break
else:
Title.sectname = 'sect%d' % (Title.level,)
setsectname = staticmethod(setsectname)
def getnumber(level):
'''Return next section number at section 'level' formatted like
1.2.3.4.'''
number = ''
for l in range(len(Title.section_numbers)):
n = Title.section_numbers[l]
if l == 0:
continue
elif l < level:
number = '%s%d.' % (number, n)
elif l == level:
number = '%s%d.' % (number, n + 1)
Title.section_numbers[l] = n + 1
elif l > level:
# Reset unprocessed section levels.
Title.section_numbers[l] = 0
return number
getnumber = staticmethod(getnumber)
class Section:
'''Static methods and attributes only.'''
endtags = [] # Stack of currently open section (level,endtag) tuples.
def __init__(self):
raise AssertionError,'no class instances allowed'
def savetag(level,etag):
'''Save section end.'''
Section.endtags.append((level,etag))
savetag = staticmethod(savetag)
def setlevel(level):
'''Set document level and write open section close tags up to level.'''
while Section.endtags and Section.endtags[-1][0] >= level:
writer.write(Section.endtags.pop()[1])
document.level = level
setlevel = staticmethod(setlevel)
def translate():
assert Lex.next() is Title
prev_sectname = Title.sectname
Title.translate()
if Title.level == 0 and document.doctype != 'book':
error('only book doctypes can contain level 0 sections')
if Title.level > document.level \
and document.backend == 'docbook' \
and prev_sectname in ('sect-colophon','sect-abstract', \
'sect-dedication','sect-glossary','sect-bibliography'):
error('%s section cannot contain sub-sections' % (prev_sectname,))
if Title.level > document.level+1:
# Sub-sections of multi-part book level zero Preface and Appendices
# are meant to be out of sequence.
if document.doctype == 'book' \
and document.level == 0 \
and Title.level == 2 \
and prev_sectname in ('sect-preface','sect-appendix'):
pass
else:
warning('section title out of sequence: '
'expected level %d, got level %d'
% (document.level+1, Title.level))
Section.setlevel(Title.level)
Title.dict['sectnum'] = Title.getnumber(document.level)
AttributeList.consume(Title.dict)
stag,etag = config.section2tags(Title.sectname,Title.dict)
Section.savetag(Title.level,etag)
writer.write(stag)
Section.translate_body()
translate = staticmethod(translate)
def translate_body(terminator=Title):
isempty = True
next = Lex.next()
while next and next is not terminator:
if next is Title and isinstance(terminator,DelimitedBlock):
error('title not permitted in sidebar body')
if document.backend == 'linuxdoc' \
and document.level == 0 \
and not isinstance(next,Paragraph):
warning('only paragraphs are permitted in linuxdoc synopsis')
next.translate()
next = Lex.next()
isempty = False
# The section is not empty if contains a subsection.
if next and isempty and Title.level > document.level:
isempty = False
# Report empty sections if invalid markup will result.
if isempty:
if document.backend == 'docbook' and Title.sectname != 'sect-index':
error('empty section is not valid')
translate_body = staticmethod(translate_body)
class Paragraphs:
'''List of paragraph definitions.'''
def __init__(self):
self.current=None
self.paragraphs = [] # List of Paragraph objects.
self.default = None # The default [paradef-default] paragraph.
def load(self,sections):
'''Update paragraphs defined in 'sections' dictionary.'''
for k in sections.keys():
if re.match(r'^paradef.+$',k):
dict = {}
parse_entries(sections.get(k,()),dict)
for p in self.paragraphs:
if p.name == k: break
else:
p = Paragraph()
self.paragraphs.append(p)
try:
p.load(k,dict)
except EAsciiDoc,e:
raise EAsciiDoc,'[%s] %s' % (k,str(e))
def dump(self):
for p in self.paragraphs:
p.dump()
def isnext(self):
for p in self.paragraphs:
if p.isnext():
self.current = p
return True;
return False
def check(self):
# Check all paragraphs have valid delimiter.
for p in self.paragraphs:
if not p.delimiter or not is_regexp(p.delimiter):
raise EAsciiDoc,'[%s] missing or illegal delimiter' % (p.name,)
# Check all paragraph template sections exist.
for p in self.paragraphs:
if not p.template:
warning('[%s] missing template entry' % (p.name,))
if not config.sections.has_key(p.template):
warning('[%s] missing paragraph template section'
% (p.template,))
# Check we have a default paragraph definition, put it last in list.
for i in range(len(self.paragraphs)):
if self.paragraphs[i].name == 'paradef-default':
p = self.paragraphs[i]
del self.paragraphs[i]
self.paragraphs.append(p)
self.default = p
break
else:
raise EAsciiDoc,'missing [paradef-default] section'
class Paragraph:
OPTIONS = ('listelement',)
def __init__(self):
self.name=None # Configuration file section name.
self.delimiter=None # Regular expression matching paragraph delimiter.
self.template=None # Name of section defining paragraph start/end tags.
self.options=() # List of paragraph option names.
self.presubs=SUBS_DEFAULT # List of pre-filter substitution option names.
self.postsubs=() # List of post-filter substitution option names.
self.filter=None # Executable paragraph filter command.
self.text=None # Text in first line of paragraph.
def load(self,name,dict):
'''Update paragraph definition from section entries in 'dict'.'''
self.name = name
for k,v in dict.items():
if k == 'delimiter':
if v and is_regexp(v):
self.delimiter = v
else:
raise EAsciiDoc,'malformed paragraph delimiter "%s"' % (v,)
elif k == 'template' or k == 'section':
if k == 'section':
deprecated('[%s] "section" entry' % self.name,
'rename to "template"')
if is_name(v):
self.template = v
else:
raise EAsciiDoc,'malformed paragraph template name "%s"' \
% (v,)
elif k == 'options':
self.options = parse_options(v,Paragraph.OPTIONS,
'illegal Paragraph %s option' % (k,))
elif k == 'presubs':
self.presubs = parse_options(v,SUBS_OPTIONS,
'illegal Paragraph %s option' % (k,))
elif k == 'postsubs':
self.postsubs = parse_options(v,SUBS_OPTIONS,
'illegal Paragraph %s option' % (k,))
elif k == 'filter':
self.filter = v
else:
raise EAsciiDoc,'illegal paragraph parameter name "%s"' % (k,)
def dump(self):
write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
write('['+self.name+']')
write('delimiter='+self.delimiter)
if self.template:
write('template='+self.template)
if self.options:
write('options='+string.join(self.options,','))
if self.presubs:
write('presubs='+string.join(self.presubs,','))
if self.postsubs:
write('postsubs='+string.join(self.postsubs,','))
if self.filter:
write('filter='+self.filter)
write('')
def isnext(self):
result = False
reader.skip_blank_lines()
if reader.read_next():
mo = re.match(self.delimiter,reader.read_next())
if mo:
self.text = mo.groupdict().get('text')
result = True
return result
def write_body(self,body):
dict = {}
BlockTitle.consume(dict)
AttributeList.consume(dict)
stag,etag = config.section2tags(self.template, dict)
# Writes blank line if the tag is empty (to separate LinuxDoc
# paragraphs).
if not stag: stag = ['']
if not etag: etag = ['']
writer.write(list(stag)+list(body)+list(etag))
def translate(self):
reader.read() # Discard (already parsed item first line).
# The next line introduces the requirement that a List cannot
# immediately follow a preceding Paragraph (introduced in v3.2.2).
# '+' delimiter is for list element continuation.
body = reader.read_until(r'^\+|^$|'+blocks.delimiter+r'|'+tables.delimiter)
body = [self.text] + list(body)
body = join_lines(body)
body = Lex.set_margin(body) # Move body to left margin.
body = Lex.subs(body,self.presubs)
if self.filter:
body = filter_lines(self.filter,body)
body = Lex.subs(body,self.postsubs)
self.write_body(body)
class Lists:
'''List of List objects.'''
def __init__(self):
self.current=None
self.lists = [] # List objects.
self.delimiter = '' # Combined blocks delimiter regular expression.
self.open = [] # A stack of the current an parent lists.
def load(self,sections):
'''Update lists defined in 'sections' dictionary.'''
for k in sections.keys():
if re.match(r'^listdef.+$',k):
dict = {}
parse_entries(sections.get(k,()),dict)
for l in self.lists:
if l.name == k: break
else:
l = List() # Create a new list if it doesn't exist.
self.lists.append(l)
try:
l.load(k,dict)
except EAsciiDoc,e:
raise EAsciiDoc,'[%s] %s' % (k,str(e))
def dump(self):
for l in self.lists:
l.dump()
def isnext(self):
for l in self.lists:
if l.isnext():
self.current = l
return True;
return False
def check(self):
for l in self.lists:
# Check list has valid type .
if not l.type in l.TYPES:
raise EAsciiDoc,'[%s] illegal type' % (l.name,)
# Check list has valid delimiter.
if not l.delimiter or not is_regexp(l.delimiter):
raise EAsciiDoc,'[%s] missing or illegal delimiter' % (l.name,)
# Check all list tags.
if not l.listtag or not config.tags.has_key(l.listtag):
warning('[%s] missing listtag' % (l.name,))
if not l.itemtag or not config.tags.has_key(l.itemtag):
warning('[%s] missing tag itemtag' % (l.name,))
if not l.texttag or not config.tags.has_key(l.texttag):
warning('[%s] missing tag texttag' % (l.name,))
if l.type == 'labeled':
if not l.entrytag or not config.tags.has_key(l.entrytag):
warning('[%s] missing entrytag' % (l.name,))
if not l.labeltag or not config.tags.has_key(l.labeltag):
warning('[%s] missing labeltag' % (l.name,))
# Build combined lists delimiter pattern.
delimiters = []
for l in self.lists:
delimiters.append(l.delimiter)
self.delimiter = join_regexp(delimiters)
class List:
# DEPRECATED: 'termtag'
TAGS = ('listtag','itemtag','texttag','entrytag','termtag','labeltag')
# DEPRECATED: 'simple','variable'
TYPES = ('simple','variable','bulleted','numbered','labeled','callout')
def __init__(self):
self.name=None # List definition configuration file section name.
self.type=None
self.delimiter=None # Regular expression matching list item delimiter.
self.subs=SUBS_DEFAULT # List of substitution option names.
self.listtag=None
self.itemtag=None
self.texttag=None # Tag for list item text.
self.labeltag=None # Variable lists only.
self.entrytag=None # Variable lists only.
self.label=None # List item label (labeled lists only).
self.text=None # Text in first line of list item.
self.attrs={} # Attributes from preceding attribute list element.
self.index=None # Matched optional delimiter 'index' group.
def load(self,name,dict):
'''Update block definition from section entries in 'dict'.'''
self.name = name
for k,v in dict.items():
if k == 'type':
if v in self.TYPES:
if v == 'simple':
deprecated("'simple' list type",
"rename to 'bulleted' or 'numbered'")
v = 'bulleted'
elif v == 'variable':
deprecated("'variable' list type",
"rename to 'labeled'")
v = 'labeled'
self.type = v
else:
raise EAsciiDoc,'illegal list type "%s"' % (v,)
elif k == 'delimiter':
if v and is_regexp(v):
self.delimiter = v
else:
raise EAsciiDoc,'malformed list delimiter "%s"' % (v,)
elif k == 'subs':
self.subs = parse_options(v,SUBS_OPTIONS,
'illegal List %s option' % (k,))
elif k in self.TAGS:
if is_name(v):
if k == 'termtag':
deprecated("'termtag' list tag name",
"rename to 'labeltag'")
k = 'labeltag'
setattr(self,k,v)
else:
raise EAsciiDoc,'illegal list %s name "%s"' % (k,v)
else:
raise EAsciiDoc,'illegal list parameter name "%s"' % (k,)
def dump(self):
write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
write('['+self.name+']')
write('type='+self.type)
write('delimiter='+self.delimiter)
if self.subs:
write('subs='+string.join(self.subs,','))
write('listtag='+self.listtag)
write('itemtag='+self.itemtag)
write('texttag='+self.texttag)
if self.type == 'labeled':
write('entrytag='+self.entrytag)
write('labeltag='+self.labeltag)
write('')
def isnext(self):
result = False
reader.skip_blank_lines()
if reader.read_next():
mo = re.match(self.delimiter,reader.read_next())
if mo:
self.label = mo.groupdict().get('label')
self.text = mo.groupdict().get('text')
self.index = mo.groupdict().get('index')
result = True
return result
def translate_entry(self):
assert self.type == 'labeled'
stag,etag = config.tag(self.entrytag, self.attrs)
if stag: writer.write(stag)
if self.text:
# Horizontal label list.
reader.read() # Discard (already parsed item first line).
writer.write_tag(self.labeltag, [self.label],
self.subs, self.attrs)
else:
# Write multiple labels (vertical label list).
while Lex.next() is self and not self.text:
reader.read() # Discard (already parsed item first line).
writer.write_tag(self.labeltag, [self.label],
self.subs, self.attrs)
# Write item text.
self.translate_item()
if etag: writer.write(etag)
def iscontinued(self):
if reader.read_next() == '+':
reader.read() # Discard.
# Allow attribute list to precede continued list item element.
if Lex.next() is AttributeList:
Lex.next().translate()
return True
else:
return False
def translate_item(self,listindex=None):
if self.type == 'callout':
self.attrs['coids'] = calloutmap.calloutids(listindex)
stag,etag = config.tag(self.itemtag, self.attrs)
if stag:
writer.write(stag)
if self.text and self.text == '+':
continued = True
elif not self.text and self.iscontinued():
continued = True
else:
# Write ItemText.
text = reader.read_until(lists.delimiter + '|^\+$|^$|' +
blocks.delimiter + r'|' + tables.delimiter)
if self.text is not None:
text = [self.text] + list(text)
text = join_lines(text)
writer.write_tag(self.texttag, text, self.subs, self.attrs)
continued = self.iscontinued()
while True:
next = Lex.next()
if next in lists.open:
break
elif isinstance(next,List):
next.translate()
elif isinstance(next,Paragraph) and 'listelement' in next.options:
next.translate()
elif continued:
if next is Title or next is BlockTitle:
error('title not allowed in list item continuation')
next.translate()
else:
break
continued = self.iscontinued()
if etag:
writer.write(etag)
def check_index(self,listindex):
''' Check calculated listindex (1,2,...) against the item index in the
document (self.index).'''
if self.index:
matched = False
if re.match(r'\d+', self.index):
i = int(self.index)
matched = True
elif re.match(r'[a-z]', self.index):
i = ord(self.index) - ord('a') + 1
matched = True
if matched and i != listindex:
warning("list item %s out of sequence" % self.index)
def translate(self):
lists.open.append(self)
self.attrs = {}
BlockTitle.consume(self.attrs)
AttributeList.consume(self.attrs)
stag,etag = config.tag(self.listtag, self.attrs)
if stag:
writer.write(stag)
listindex = 0
while Lex.next() is self:
listindex += 1
self.check_index(listindex)
if self.type in ('bulleted','numbered','callout'):
reader.read() # Discard (already parsed item first line).
self.translate_item(listindex)
elif self.type == 'labeled':
self.translate_entry()
else:
raise AssertionError,'illegal [%s] list type"' % (self.name,)
if etag:
writer.write(etag)
if self.type == 'callout':
calloutmap.check(listindex)
calloutmap.listclose()
lists.open.pop()
class DelimitedBlocks:
'''List of delimited blocks.'''
def __init__(self):
self.current=None
self.blocks = [] # List of DelimitedBlock objects.
self.delimiter = '' # Combined blocks delimiter regular expression.
def load(self,sections):
'''Update blocks defined in 'sections' dictionary.'''
for k in sections.keys():
if re.match(r'^blockdef.+$',k):
dict = {}
parse_entries(sections.get(k,()),dict)
for b in self.blocks:
if b.name == k: break
else:
b = DelimitedBlock()
self.blocks.append(b)
try:
b.load(k,dict)
except EAsciiDoc,e:
raise EAsciiDoc,'[%s] %s' % (k,str(e))
def dump(self):
for b in self.blocks:
b.dump()
def isnext(self):
for b in self.blocks:
if b.isnext():
self.current = b
return True;
return False
def check(self):
# Check all blocks have valid delimiter.
for b in self.blocks:
if not b.delimiter or not is_regexp(b.delimiter):
raise EAsciiDoc,'[%s] missing or illegal delimiter' % (b.name,)
# Check all block sections exist.
for b in self.blocks:
if 'skip' not in b.options:
if not b.template:
warning('[%s] missing template entry' % (b.name,))
if not config.sections.has_key(b.template):
warning('[%s] missing template section' % (b.template,))
# Build combined block delimiter pattern.
delimiters = []
for b in self.blocks:
delimiters.append(b.delimiter)
self.delimiter = join_regexp(delimiters)
class DelimitedBlock:
#DEPRECATED: 'argsline' option.
OPTIONS = ('section','skip','argsline','variants')
def __init__(self):
self.name=None # Block definition configuration file section name.
self.delimiter=None # Regular expression matching block delimiter.
self.template=None # Name of section defining block header/footer.
self.options=() # List of block option names.
self.presubs=() # List of pre-filter substitution option names.
self.postsubs=() # List of post-filter substitution option names.
self.filter=None # Executable block filter command.
def load(self,name,dict):
'''Update block definition from section entries in 'dict'.'''
self.name = name
for k,v in dict.items():
if k == 'delimiter':
if v and is_regexp(v):
self.delimiter = v
else:
raise EAsciiDoc,'malformed block delimiter "%s"' % (v,)
elif k == 'template' or k == 'section':
if k == 'section':
deprecated('[%s] "section" entry' % self.name,
'rename to "template"')
if is_name(v):
self.template = v
else:
raise EAsciiDoc, 'malformed template section name "%s"' \
% (v,)
elif k == 'options':
self.options = parse_options(v,DelimitedBlock.OPTIONS,
'illegal DelimitedBlock %s option' % (k,))
elif k == 'presubs':
self.presubs = parse_options(v,SUBS_OPTIONS,
'illegal DelimitedBlock %s option' % (k,))
elif k == 'postsubs':
self.postsubs = parse_options(v,SUBS_OPTIONS,
'illegal DelimitedBlock %s option' % (k,))
elif k == 'filter':
self.filter = v
else:
raise EAsciiDoc,'illegal block parameter name "%s"' % (k,)
def dump(self):
write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
write('['+self.name+']')
write('delimiter='+self.delimiter)
if self.template:
write('template='+self.template)
if self.options:
write('options='+string.join(self.options,','))
if self.presubs:
write('presubs='+string.join(self.presubs,','))
if self.postsubs:
write('postsubs='+string.join(self.postsubs,','))
if self.filter:
write('filter='+self.filter)
write('')
def isnext(self):
reader.skip_blank_lines()
if reader.read_next():
return re.match(self.delimiter,reader.read_next())
else:
return False
def translate(self):
dict = {}
BlockTitle.consume(dict)
AttributeList.consume(dict)
if 'variants' in self.options and dict.get('1'):
# Synthesize template section name from attribute {1}.
# Strip white space and illegal name chars.
template = string.lower(re.sub(r'[^\w\-_]', '', dict['1']))+'block'
if not config.sections.has_key(template):
warning('missing [%s] template section' % (template,))
template = self.template
else:
template = self.template
delimiter = reader.read()
mo = re.match(self.delimiter,delimiter)
assert mo
dict.update(mo.groupdict())
for k,v in dict.items():
if v is None: del dict[k]
if dict.has_key('args'):
deprecated('delimited block arguments','use attribute lists')
# Extract embedded arguments from leading delimiter line.
parse_attrs(dict['args'],dict)
elif 'argsline' in self.options:
# DEPRECATED: 'argsline' block option.
# Parse block arguments line.
reader.parse_arguments(dict)
# Process block contents.
if 'skip' in self.options:
# Discard block body.
reader.read_until(self.delimiter,same_file=True)
elif 'section' in self.options:
stag,etag = config.section2tags(template,dict)
# The body is treated like a SimpleSection.
writer.write(stag)
Section.translate_body(self)
writer.write(etag)
else:
stag,etag = config.section2tags(template,dict)
body = reader.read_until(self.delimiter,same_file=True)
body = Lex.subs(body,self.presubs)
if self.filter:
body = filter_lines(self.filter,body,dict)
body = Lex.subs(body,self.postsubs)
# Write start tag, content, end tag.
writer.write(list(stag)+list(body)+list(etag))
if reader.eof():
error('closing [%s] delimiter expected' % (self.name,))
else:
delimiter = reader.read() # Discard delimiter line.
assert re.match(self.delimiter,delimiter)
class Tables:
'''List of tables.'''
def __init__(self):
self.current=None
self.tables = [] # List of Table objects.
self.delimiter = '' # Combined tables delimiter regular expression.
def load(self,sections):
'''Update tables defined in 'sections' dictionary.'''
for k in sections.keys():
if re.match(r'^tabledef.+$',k):
dict = {}
parse_entries(sections.get(k,()),dict)
for t in self.tables:
if t.name == k: break
else:
t = Table()
self.tables.append(t)
try:
t.load(k,dict)
except EAsciiDoc,e:
raise EAsciiDoc,'[%s] %s' % (k,str(e))
def dump(self):
for t in self.tables:
t.dump()
def isnext(self):
for t in self.tables:
if t.isnext():
self.current = t
return True;
return False
def check(self):
# Check we have a default table definition,
for i in range(len(self.tables)):
if self.tables[i].name == 'tabledef-default':
default = self.tables[i]
break
else:
raise EAsciiDoc,'missing [table-default] section'
# Set default table defaults.
if default.subs is None: default.subs = SUBS_DEFAULT
if default.format is None: default.subs = 'fixed'
# Propagate defaults to unspecified table parameters.
for t in self.tables:
if t is not default:
if t.fillchar is None: t.fillchar = default.fillchar
if t.subs is None: t.subs = default.subs
if t.format is None: t.format = default.format
if t.template is None: t.template = default.template
if t.colspec is None: t.colspec = default.colspec
if t.headrow is None: t.headrow = default.headrow
if t.footrow is None: t.footrow = default.footrow
if t.bodyrow is None: t.bodyrow = default.bodyrow
if t.headdata is None: t.headdata = default.headdata
if t.footdata is None: t.footdata = default.footdata
if t.bodydata is None: t.bodydata = default.bodydata
# Check all tables have valid fill character.
for t in self.tables:
if not t.fillchar or len(t.fillchar) != 1:
raise EAsciiDoc,'[%s] missing or illegal fillchar' % (t.name,)
# Build combined tables delimiter patterns and assign defaults.
delimiters = []
for t in self.tables:
# Ruler is:
# (ColStop,(ColWidth,FillChar+)?)+, FillChar+, TableWidth?
t.delimiter = r'^(' + Table.COL_STOP \
+ r'(\d*|' + re.escape(t.fillchar) + r'*)' \
+ r')+' \
+ re.escape(t.fillchar) + r'+' \
+ '([\d\.]*)$'
delimiters.append(t.delimiter)
if not t.headrow:
t.headrow = t.bodyrow
if not t.footrow:
t.footrow = t.bodyrow
if not t.headdata:
t.headdata = t.bodydata
if not t.footdata:
t.footdata = t.bodydata
self.delimiter = join_regexp(delimiters)
# Check table definitions are valid.
for t in self.tables:
t.check()
if config.verbose:
if t.check_msg:
warning('[%s] table definition: %s' % (t.name,t.check_msg))
class Column:
'''Table column.'''
def __init__(self):
self.colalign = None # 'left','right','center'
self.rulerwidth = None
self.colwidth = None # Output width in page units.
class Table:
COL_STOP = r"(`|'|\.)" # RE.
ALIGNMENTS = {'`':'left', "'":'right', '.':'center'}
FORMATS = ('fixed','csv','dsv')
def __init__(self):
# Configuration parameters.
self.name=None # Table definition configuration file section name.
self.fillchar=None
self.subs=None
self.format=None # 'fixed','csv','dsv'
self.template=None
self.colspec=None
self.headrow=None
self.footrow=None
self.bodyrow=None
self.headdata=None
self.footdata=None
self.bodydata=None
# Calculated parameters.
self.delimiter=None # RE matching any table ruler.
self.underline=None # RE matching current table underline.
self.isnumeric=False # True if numeric ruler, false if character ruler.
self.tablewidth=None # Optional table width scale factor.
self.columns=[] # List of Columns.
self.dict={} # Substitutions dictionary.
# Other.
self.check_msg='' # Message set by previous self.check() call.
def load(self,name,dict):
'''Update table definition from section entries in 'dict'.'''
self.name = name
for k,v in dict.items():
if k == 'fillchar':
if v and len(v) == 1:
self.fillchar = v
else:
raise EAsciiDoc,'malformed table fillchar "%s"' % (v,)
elif k == 'template' or k == 'section':
if k == 'section':
deprecated('[%s] "section" entry' % self.name,
'rename to "template"')
if is_name(v):
self.template = v
else:
raise EAsciiDoc,'malformed table template name "%s"' % (v,)
elif k == 'subs':
self.subs = parse_options(v,SUBS_OPTIONS,
'illegal Table %s option' % (k,))
elif k == 'format':
if v in Table.FORMATS:
self.format = v
else:
raise EAsciiDoc,'illegal table format "%s"' % (v,)
elif k == 'colspec':
self.colspec = v
elif k == 'headrow':
self.headrow = v
elif k == 'footrow':
self.footrow = v
elif k == 'bodyrow':
self.bodyrow = v
elif k == 'headdata':
self.headdata = v
elif k == 'footdata':
self.footdata = v
elif k == 'bodydata':
self.bodydata = v
else:
raise EAsciiDoc,'illegal entry name "%s"' % (k,)
def dump(self):
write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
write('['+self.name+']')
write('fillchar='+self.fillchar)
write('subs='+string.join(self.subs,','))
write('format='+self.format)
write('template='+self.template)
if self.colspec:
write('colspec='+self.colspec)
if self.headrow:
write('headrow='+self.headrow)
if self.footrow:
write('footrow='+self.footrow)
write('bodyrow='+self.bodyrow)
if self.headdata:
write('headdata='+self.headdata)
if self.footdata:
write('footdata='+self.footdata)
write('bodydata='+self.bodydata)
write('')
def check(self):
'''Check table definition and set self.check_msg if invalid else set
self.check_msg to blank string.'''
# Check global table parameters.
if config.textwidth is None:
self.check_msg = 'missing [miscellaneous] textwidth entry'
elif config.pagewidth is None:
self.check_msg = 'missing [miscellaneous] pagewidth entry'
elif config.pageunits is None:
self.check_msg = 'missing [miscellaneous] pageunits entry'
elif not self.template:
self.check_msg = 'missing template entry'
elif not config.sections.has_key(self.template):
self.check_msg = 'missing template section [%s]' % self.template
elif self.headrow is None:
self.check_msg = 'missing headrow entry'
elif self.footrow is None:
self.check_msg = 'missing footrow entry'
elif self.bodyrow is None:
self.check_msg = 'missing bodyrow entry'
elif self.headdata is None:
self.check_msg = 'missing headdata entry'
elif self.footdata is None:
self.check_msg = 'missing footdata entry'
elif self.bodydata is None:
self.check_msg = 'missing bodydata entry'
else:
# No errors.
self.check_msg = ''
def isnext(self):
reader.skip_blank_lines()
if reader.read_next():
return re.match(self.delimiter,reader.read_next())
else:
return False
def parse_ruler(self,ruler):
'''Parse ruler calculating underline and ruler column widths.'''
fc = re.escape(self.fillchar)
# Strip and save optional tablewidth from end of ruler.
mo = re.match(r'^(.*'+fc+r'+)([\d\.]+)$',ruler)
if mo:
ruler = mo.group(1)
self.tablewidth = float(mo.group(2))
self.dict['tablewidth'] = str(float(self.tablewidth))
else:
self.tablewidth = None
self.dict['tablewidth'] = '100.0'
# Guess whether column widths are specified numerically or not.
if ruler[1] != self.fillchar:
# If the first column does not start with a fillchar then numeric.
self.isnumeric = True
elif ruler[1:] == self.fillchar*len(ruler[1:]):
# The case of one column followed by fillchars is numeric.
self.isnumeric = True
else:
self.isnumeric = False
# Underlines must be 3 or more fillchars.
self.underline = r'^' + fc + r'{3,}$'
splits = re.split(self.COL_STOP,ruler)[1:]
# Build self.columns.
for i in range(0,len(splits),2):
c = Column()
c.colalign = self.ALIGNMENTS[splits[i]]
s = splits[i+1]
if self.isnumeric:
# Strip trailing fillchars.
s = re.sub(fc+r'+$','',s)
if s == '':
c.rulerwidth = None
else:
c.rulerwidth = int(validate(s,'int($)>0',
'malformed ruler: bad width'))
else: # Calculate column width from inter-fillchar intervals.
if not re.match(r'^'+fc+r'+$',s):
raise EAsciiDoc,'malformed ruler: illegal fillchars'
c.rulerwidth = len(s)+1
self.columns.append(c)
# Fill in unspecified ruler widths.
if self.isnumeric:
if self.columns[0].rulerwidth is None:
prevwidth = 1
for c in self.columns:
if c.rulerwidth is None:
c.rulerwidth = prevwidth
prevwidth = c.rulerwidth
def build_colspecs(self):
'''Generate colwidths and colspecs. This can only be done after the
table arguments have been parsed since we use the table format.'''
self.dict['cols'] = len(self.columns)
# Calculate total ruler width.
totalwidth = 0
for c in self.columns:
totalwidth = totalwidth + c.rulerwidth
if totalwidth <= 0:
raise EAsciiDoc,'zero width table'
# Calculate marked up colwidths from rulerwidths.
for c in self.columns:
# Convert ruler width to output page width.
width = float(c.rulerwidth)
if self.format == 'fixed':
if self.tablewidth is None:
# Size proportional to ruler width.
colfraction = width/config.textwidth
else:
# Size proportional to page width.
colfraction = width/totalwidth
else:
# Size proportional to page width.
colfraction = width/totalwidth
c.colwidth = colfraction * config.pagewidth # To page units.
if self.tablewidth is not None:
c.colwidth = c.colwidth * self.tablewidth # Scale factor.
if self.tablewidth > 1:
c.colwidth = c.colwidth/100 # tablewidth is in percent.
# Build colspecs.
if self.colspec:
s = []
for c in self.columns:
self.dict['colalign'] = c.colalign
self.dict['colwidth'] = str(int(c.colwidth))
s.append(subs_attrs(self.colspec,self.dict))
self.dict['colspecs'] = string.join(s,writer.newline)
def parse_attrs(self):
'''Parse table arguments string.'''
d = {}
AttributeList.consume(d)
reader.parse_arguments(d) # DEPRECATED: Drop only this statement.
# Update table with overridable parameters.
if d.has_key('subs'):
self.subs = parse_options(d['subs'],SUBS_OPTIONS,
'illegal table subs %s option' % ('subs',))
if d.has_key('format'):
self.format = d['format']
if d.has_key('tablewidth'):
self.tablewidth = float(d['tablewidth'])
# Add attributes for template substitution.
self.dict.update(d)
def split_rows(self,rows):
'''Return a two item tuple containing a list of lines up to but not
including the next underline (continued lines are joined ) and the
tuple of all lines after the underline.'''
reo = re.compile(self.underline)
i = 0
while not reo.match(rows[i]):
i = i+1
if i == 0:
raise EAsciiDoc,'missing [%s] table rows' % (self.name,)
if i >= len(rows):
raise EAsciiDoc,'closing [%s] underline expected' % (self.name,)
return (join_lines(rows[:i]), rows[i+1:])
def parse_rows(self, rows, rtag, dtag):
'''Parse rows list using the row and data tags. Returns a substituted
list of output lines.'''
result = []
# Source rows are parsed as single block, rather than line by line, to
# allow the CSV reader to handle multi-line rows.
if self.format == 'fixed':
rows = self.parse_fixed(rows)
elif self.format == 'csv':
rows = self.parse_csv(rows)
elif self.format == 'dsv':
rows = self.parse_dsv(rows)
else:
assert True,'illegal table format'
# Substitute and indent all data in all rows.
stag,etag = subs_tag(rtag,self.dict)
for row in rows:
result.append(' '+stag)
for data in self.subs_row(row,dtag):
result.append(' '+data)
result.append(' '+etag)
return result
def subs_row(self, data, dtag):
'''Substitute the list of source row data elements using the data tag.
Returns a substituted list of output table data items.'''
result = []
if len(data) < len(self.columns):
warning('fewer row data items then table columns')
if len(data) > len(self.columns):
warning('more row data items than table columns')
for i in range(len(self.columns)):
if i > len(data) - 1:
d = '' # Fill missing column data with blanks.
else:
d = data[i]
c = self.columns[i]
self.dict['colalign'] = c.colalign
self.dict['colwidth'] = str(int(c.colwidth)) + config.pageunits
stag,etag = subs_tag(dtag,self.dict)
# Insert AsciiDoc line break (' +') where row data has newlines
# ('\n'). This is really only useful when the table format is csv
# and the output markup is HTML. It's also a bit dubious in that it
# assumes the user has not modified the shipped line break pattern.
if 'replacements' in self.subs:
# Insert line breaks in cell data.
d = re.sub(r'(?m)\n',r' +\n',d)
d = string.split(d,'\n') # So writer.newline is written.
else:
d = [d]
result = result + [stag] + Lex.subs(d,self.subs) + [etag]
return result
def parse_fixed(self,rows):
'''Parse the list of source table rows. Each row item in the returned
list contains a list of cell data elements.'''
result = []
for row in rows:
data = []
start = 0
for c in self.columns:
end = start + c.rulerwidth
if c is self.columns[-1]:
# Text in last column can continue forever.
data.append(string.strip(row[start:]))
else:
data.append(string.strip(row[start:end]))
start = end
result.append(data)
return result
def parse_csv(self,rows):
'''Parse the list of source table rows. Each row item in the returned
list contains a list of cell data elements.'''
import StringIO
import csv
result = []
rdr = csv.reader(StringIO.StringIO(string.join(rows,'\r\n')),
skipinitialspace=True)
try:
for row in rdr:
result.append(row)
except:
raise EAsciiDoc,'csv parse error "%s"' % (row,)
return result
def parse_dsv(self,rows):
'''Parse the list of source table rows. Each row item in the returned
list contains a list of cell data elements.'''
separator = self.dict.get('separator',':')
separator = eval('"'+separator+'"')
if len(separator) != 1:
raise EAsciiDoc,'malformed dsv separator: %s' % (separator,)
# TODO If separator is preceeded by an odd number of backslashes then
# it is escaped and should not delimit.
result = []
for row in rows:
# Skip blank lines
if row == '': continue
# Unescape escaped characters.
row = eval('"'+string.replace(row,'"','\\"')+'"')
data = string.split(row,separator)
result.append(data)
return result
def translate(self):
# Reset instance specific properties.
self.underline = None
self.columns = []
self.dict = {}
BlockTitle.consume(self.dict)
# Add relevant globals to table substitutions.
self.dict['pagewidth'] = str(config.pagewidth)
self.dict['pageunits'] = config.pageunits
# Save overridable table parameters.
save_subs = self.subs
save_format = self.format
# Parse table ruler.
ruler = reader.read()
assert re.match(self.delimiter,ruler)
self.parse_ruler(ruler)
# Parse table attributes list.
self.parse_attrs()
# Read the entire table.
table = []
while True:
line = reader.read_next()
# Table terminated by underline followed by a blank line or EOF.
if len(table) > 0 and re.match(self.underline,table[-1]):
if line in ('',None):
break;
if line is None:
raise EAsciiDoc,'closing [%s] underline expected' % (self.name,)
table.append(reader.read())
if self.check_msg: # Skip if table definition was marked invalid.
warning('skipping %s table: %s' % (self.name,self.check_msg))
return
# Generate colwidths and colspecs.
self.build_colspecs()
# Generate headrows, footrows, bodyrows.
headrows = footrows = []
bodyrows,table = self.split_rows(table)
if table:
headrows = bodyrows
bodyrows,table = self.split_rows(table)
if table:
footrows,table = self.split_rows(table)
if headrows:
headrows = self.parse_rows(headrows, self.headrow, self.headdata)
self.dict['headrows'] = string.join(headrows,writer.newline)
if footrows:
footrows = self.parse_rows(footrows, self.footrow, self.footdata)
self.dict['footrows'] = string.join(footrows,writer.newline)
bodyrows = self.parse_rows(bodyrows, self.bodyrow, self.bodydata)
self.dict['bodyrows'] = string.join(bodyrows,writer.newline)
table = subs_attrs(config.sections[self.template],self.dict)
writer.write(table)
# Restore overridable table parameters.
self.subs = save_subs
self.format = save_format
class Macros:
def __init__(self):
self.macros = [] # List of Macros.
self.current = None # The last matched block macro.
def load(self,entries):
for entry in entries:
m = Macro()
m.load(entry)
if m.name is None:
# Delete undefined macro.
for i in range(len(self.macros)-1,-1,-1):
if self.macros[i].pattern == m.pattern:
del self.macros[i]
else:
# Check for duplicates.
for m2 in self.macros:
if m.equals(m2):
warning('duplicate macro: '+entry)
break
else:
self.macros.append(m)
def dump(self):
write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
write('[macros]')
for m in self.macros:
write('%s=%s%s' % (m.pattern,m.prefix,m.name))
write('')
def check(self):
# Check all named sections exist.
if config.verbose:
for m in self.macros:
if m.name and m.prefix != '+':
m.section_name()
def subs(self,text,prefix='',callouts=False):
# If callouts is True then only callout macros are processed, if False
# then all non-callout macros are processed.
result = text
for m in self.macros:
if m.prefix == prefix:
if callouts ^ (m.name != 'callout'):
result = m.subs(result)
return result
def isnext(self):
'''Return matching macro if block macro is next on reader.'''
reader.skip_blank_lines()
line = reader.read_next()
if line:
for m in self.macros:
if m.prefix == '#':
if m.reo.match(line):
self.current = m
return m
return False
def match(self,prefix,name,text):
'''Return re match object matching 'text' with macro type 'prefix',
macro name 'name'.'''
for m in self.macros:
if m.prefix == prefix:
mo = m.reo.match(text)
if mo:
if m.name == name:
return mo
if re.match(name,mo.group('name')):
return mo
return None
# Macro set just prior to calling _subs_macro(). Ugly but there's no way
# to pass optional arguments with _subs_macro().
_macro = None
def _subs_macro(mo):
'''Function called to perform inline macro substitution. Uses matched macro
regular expression object and returns string containing the substituted
macro body. Called by Macros().subs().'''
# Check if macro reference is escaped.
if mo.group()[0] == '\\':
return mo.group()[1:] # Strip leading backslash.
dict = mo.groupdict()
# Delete groups that didn't participate in match.
for k,v in dict.items():
if v is None: del dict[k]
if _macro.name:
name = _macro.name
else:
if not dict.has_key('name'):
warning('missing macro name group: %s' % (mo.re.pattern,))
return ''
name = dict['name']
section_name = _macro.section_name(name)
if not section_name:
return ''
# If we're dealing with a block macro get optional block ID and block title.
if _macro.prefix == '#':
AttributeList.consume(dict)
BlockTitle.consume(dict)
# Parse macro attributes.
if dict.has_key('attrlist'):
if dict['attrlist'] in (None,''):
del dict['attrlist']
else:
parse_attrs(dict['attrlist'],dict)
if name == 'callout':
listindex =int(dict['index'])
dict['coid'] = calloutmap.add(listindex)
# It gets a bit tricky here. We've already done attribute substitution on
# the macro attribute list this means that any escaped attributes will
# have been unescaped so withhold passing them for substitution.
a0 = dict.get('0')
if a0:
dict['0'] = chr(0) # Replace temporarily with unused character.
body = config.subs_section(section_name,dict)
if len(body) == 0:
result = ''
elif len(body) == 1:
result = body[0]
else:
result = string.join(body,writer.newline)
if a0:
result = string.replace(result, chr(0), a0)
return result
class Macro:
def __init__(self):
self.pattern = None # Matching regular expression.
self.name = '' # Conf file macro name (None if implicit).
self.prefix = '' # '' if inline, '+' if intrinsic, '#' if block.
self.reo = None # Compiled pattern re object.
def section_name(self,name=None):
'''Return macro markup template section name based on macro name and
prefix. Return None section not found.'''
assert self.prefix != '+'
if not name:
assert self.name
name = self.name
if self.prefix == '#':
suffix = '-blockmacro'
else:
suffix = '-inlinemacro'
if name == 'icon':
deprecated('icon macro', 'use image macro')
return name
elif name == 'graphic':
deprecated('graphic macro', 'use image macro')
return name
elif config.sections.has_key(name):
deprecated('macro section name: ['+name+']',
'renamed to ['+name+suffix+']')
return name
elif config.sections.has_key(name+suffix):
return name+suffix
else:
warning('missing macro section: [%s]' % (name+suffix,))
return None
def equals(self,m):
if self.pattern != m.pattern:
return False
if self.name != m.name:
return False
if self.prefix != m.prefix:
return False
return True
def load(self,entry):
e = parse_entry(entry)
if not e:
raise EAsciiDoc,'malformed macro entry "%s"' % (entry,)
self.pattern, self.name = e
if not is_regexp(self.pattern):
raise EAsciiDoc,'illegal regular expression in macro entry "%s"' \
% (entry,)
self.reo = re.compile(self.pattern)
if self.name:
if self.name[0] in ('+','#'):
self.prefix, self.name = self.name[0], self.name[1:]
if self.name and not is_name(self.name):
raise EAsciiDoc,'illegal section name in macro entry "%s"' % \
(entry,)
def subs(self,text):
global _macro
_macro = self # Pass the macro to _subs_macro().
return self.reo.sub(_subs_macro,text)
def translate(self):
assert self.prefix == '#'
line = reader.read()
writer.write(self.subs(line))
class CalloutMap:
def __init__(self):
self.comap = {} # key = list index, value = callouts list.
self.calloutindex = 0 # Current callout index number.
self.listnumber = 1 # Current callout list number.
def listclose(self):
# Called when callout list is closed.
self.listnumber += 1
self.calloutindex = 0
self.comap = {}
def add(self,listindex):
# Add next callout index to listindex map entry. Return the callout id.
self.calloutindex += 1
# Append the coindex to a list in the comap dictionary.
if not self.comap.has_key(listindex):
self.comap[listindex] = [self.calloutindex]
else:
self.comap[listindex].append(self.calloutindex)
return self.calloutid(self.listnumber, self.calloutindex)
def calloutid(listnumber,calloutindex):
return 'CO%d-%d' % (listnumber,calloutindex)
calloutid = staticmethod(calloutid)
def calloutids(self,listindex):
# Retieve list of callout indexes that refer to listindex.
if self.comap.has_key(listindex):
result = ''
for coindex in self.comap[listindex]:
result += ' ' + self.calloutid(self.listnumber,coindex)
return result.strip()
else:
error('no callouts refer to list item '+str(listindex))
return ''
def check(self,maxlistindex):
# Check that all list indexes referenced by callouts exist.
for listindex in self.comap.keys():
if listindex > maxlistindex:
warning('callout refers to non-existent list item '
+ str(listindex))
#---------------------------------------------------------------------------
# Input stream Reader and output stream writer classes.
#---------------------------------------------------------------------------
class Reader1:
'''Line oriented AsciiDoc input file reader. Processes non lexical
entities: transparently handles included files. Tabs are expanded and lines
are right trimmed.'''
# This class is not used directly, use Reader class instead.
READ_BUFFER_MIN = 10 # Read buffer low level.
def __init__(self):
self.f = None # Input file object.
self.fname = None # Input file name.
self.next = [] # Read ahead buffer containing
# (filename,linenumber,linetext) tuples.
self.cursor = None # Last read() (filename,linenumber,linetext).
self.tabsize = 8 # Tab expansion number of spaces.
self.parent = None # Included reader's parent reader.
self._lineno = 0 # The last line read from file object f.
self.include_enabled = True # Enables/disables file inclusion.
self.include_depth = 0 # Current include depth.
self.include_max = 5 # Maxiumum allowed include depth.
def open(self,fname):
self.fname = fname
verbose('reading: '+fname)
if fname == '<stdin>':
self.f = sys.stdin
else:
self.f = open(fname,"rb")
self._lineno = 0 # The last line read from file object f.
self.next = []
# Prefill buffer by reading the first line and then pushing it back.
if Reader1.read(self):
self.unread(self.cursor)
self.cursor = None
def closefile(self):
'''Used by class methods to close nested include files.'''
self.f.close()
self.next = []
def close(self):
self.closefile()
self.__init__()
def read(self):
'''Read next line. Return None if EOF. Expand tabs. Strip trailing
white space. Maintain self.next read ahead buffer.'''
# Top up buffer.
if len(self.next) <= self.READ_BUFFER_MIN:
s = self.f.readline()
if s:
self._lineno = self._lineno + 1
while s:
if self.tabsize != 0:
s = string.expandtabs(s,self.tabsize)
s = string.rstrip(s)
self.next.append((self.fname,self._lineno,s))
if len(self.next) > self.READ_BUFFER_MIN:
break
s = self.f.readline()
if s:
self._lineno = self._lineno + 1
# Return first (oldest) buffer entry.
if len(self.next) > 0:
self.cursor = self.next[0]
del self.next[0]
result = self.cursor[2]
# Check for include macro.
mo = macros.match('+',r'include[1]?',result)
if mo and self.include_enabled:
# Perform attribute substitution on inlcude macro.
fname = subs_attrs(mo.group('target'))
# If undefined attribute then skip to next line of input.
if not fname:
return Reader1.read(self)
if self.include_depth >= self.include_max:
raise EAsciiDoc,'maxiumum inlude depth exceeded'
if not os.path.isabs(fname) and self.fname != '<stdin>':
# Include files are relative to parent document directory.
fname = os.path.join(os.path.dirname(self.fname),fname)
if self.fname != '<stdin>' and not os.path.isfile(fname):
raise EAsciiDoc,'include file "%s" not found' % (fname,)
# Parse include macro attributes.
attrs = {}
parse_attrs(mo.group('attrlist'),attrs)
# Clone self and set as parent (self assumes the role of child).
parent = Reader1()
assign(parent,self)
self.parent = parent
if attrs.has_key('tabsize'):
self.tabsize = int(validate(attrs['tabsize'],'int($)>=0', \
'illegal include macro tabsize argument'))
# The include1 variant does not allow nested includes.
if mo.group('name') == 'include1':
self.include_enabled = False
self.open(fname)
self.include_depth = self.include_depth + 1
result = Reader1.read(self)
else:
if not Reader1.eof(self):
result = Reader1.read(self)
else:
result = None
return result
def eof(self):
'''Returns True if all lines have been read.'''
if len(self.next) == 0:
# End of current file.
if self.parent:
self.closefile()
assign(self,self.parent) # Restore parent reader.
return Reader1.eof(self)
else:
return True
else:
return False
def read_next(self):
'''Like read() but does not advance file pointer.'''
if Reader1.eof(self):
return None
else:
return self.next[0][2]
def unread(self,cursor):
'''Push the line (filename,linenumber,linetext) tuple back into the read
buffer. Note that it's up to the caller to restore the previous
cursor.'''
assert cursor
self.next.insert(0,cursor)
class Reader(Reader1):
''' Wraps (well, sought of) Reader1 class and implements conditional text
inclusion.'''
def __init__(self):
Reader1.__init__(self)
self.depth = 0 # if nesting depth.
self.skip = False # true if we're skipping ifdef...endif.
self.skipname = '' # Name of current endif macro target.
self.skipto = -1 # The depth at which skipping is reenabled.
def read_super(self):
result = Reader1.read(self)
if result is None and self.skip:
raise EAsciiDoc,'missing endif::%s[]' %(self.skipname,)
return result
def read(self):
result = self.read_super()
if result is None:
return None
while self.skip:
mo = macros.match('+',r'ifdef|ifndef|endif',result)
if mo:
name = mo.group('name')
target = mo.group('target')
if name == 'endif':
self.depth = self.depth-1
if self.depth < 0:
raise EAsciiDoc,'"%s" is mismatched' % (result,)
if self.depth == self.skipto:
self.skip = False
if target and self.skipname != target:
raise EAsciiDoc,'"%s" is mismatched' % (result,)
else: # ifdef or ifndef.
if not target:
raise EAsciiDoc,'"%s" missing macro target' % (result,)
self.depth = self.depth+1
result = self.read_super()
if result is None:
return None
mo = macros.match('+',r'ifdef|ifndef|endif',result)
if mo:
name = mo.group('name')
target = mo.group('target')
if name == 'endif':
self.depth = self.depth-1
else: # ifdef or ifndef.
if not target:
raise EAsciiDoc,'"%s" missing macro target' % (result,)
defined = document.attributes.get(target) is not None
if name == 'ifdef':
self.skip = not defined
else: # ifndef.
self.skip = defined
if self.skip:
self.skipto = self.depth
self.skipname = target
self.depth = self.depth+1
result = self.read()
return result
def eof(self):
return self.read_next() is None
def read_next(self):
save_cursor = self.cursor
result = self.read()
if result is not None:
self.unread(self.cursor)
self.cursor = save_cursor
return result
def read_all(self,fname):
'''Read all lines from filefnameaslist.Uselike import
method: Reader().read_all(fname)'''
result = []
self.open(fname)
try:
while not self.eof():
result.append(self.read())
finally:
self.close()
return result
def read_lines(self,count=1):
'''Return tuple containing count lines.'''
result = []
i = 0
while i < count and not self.eof():
result.append(self.read())
return tuple(result)
def read_ahead(self,count=1):
'''Same as read_lines() but does not advance the file pointer.'''
result = []
putback = []
save_cursor = self.cursor
try:
i = 0
while i < count and not self.eof():
result.append(self.read())
putback.append(self.cursor)
i = i+1
while putback:
self.unread(putback.pop())
finally:
self.cursor = save_cursor
return tuple(result)
def skip_blank_lines(self):
reader.read_until(r'\s*\S+')
def read_until(self,pattern,same_file=False):
'''Like read() but reads lines up to (but not including) the first line
that matches the pattern regular expression. If same_file is True
then the terminating pattern must occur in the file the was being read
when the routine was called.'''
if same_file:
fname = self.cursor[0]
result = []
reo = re.compile(pattern)
while not self.eof():
save_cursor = self.cursor
s = self.read()
if (not same_file or fname == self.cursor[0]) and reo.match(s):
self.unread(self.cursor)
self.cursor = save_cursor
break
result.append(s)
return tuple(result)
def read_continuation(self):
'''Like read() but treats trailing backslash as line continuation
character.'''
s = self.read()
if s is None:
return None
result = ''
while s is not None and len(s) > 0 and s[-1] == '\\':
result = result + s[:-1]
s = self.read()
if s is not None:
result = result + s
return result
def parse_arguments(self,dict,default_arg=None):
'''If an arguments line is in the reader parse it to dict.'''
# DEPRECATED: This routine -- deprecated delimited block and table arguments.
s = self.read_next()
if s is not None:
if s[:2] == '\\[':
# Unescape next line.
save_cursor = self.cursor
self.read()
self.cursor = self.cursor[0:2] + (s[1:],)
self.unread(self.cursor)
self.cursor = save_cursor
elif re.match(r'^\[.*[\\\]]$',s):
s = self.read_continuation()
if not re.match(r'^\[.*\]$',s):
warning('malformed arguments line')
else:
deprecated('delimited block and table arguments line',
'use attribute lists')
parse_attrs(s[1:-1],dict,default_arg)
class Writer:
'''Writes lines to output file.'''
newline = '\r\n' # End of line terminator.
f = None # Output file object.
fname= None # Output file name.
lines_out = 0 # Number of lines written.
def open(self,fname):
self.fname = fname
verbose('writing: '+fname)
if fname == '<stdout>':
self.f = sys.stdout
else:
self.f = open(fname,"wb+")
self.lines_out = 0
def close(self,):
if self.fname != '<stdout>':
self.f.close()
def write(self,*args):
'''Iterates arguments, writes tuple and list arguments one line per
element, else writes argument as single line. If no arguments writes
blank line. self.newline is appended to each line.'''
if len(args) == 0:
self.f.write(self.newline)
self.lines_out = self.lines_out + 1
else:
for arg in args:
if type(arg) in (TupleType,ListType):
for s in arg:
self.f.write(s+self.newline)
self.lines_out = self.lines_out + len(arg)
else:
self.f.write(arg+self.newline)
self.lines_out = self.lines_out + 1
def write_tag(self,tagname,content,subs=SUBS_DEFAULT,dict=None):
'''Write content enveloped by configuration file tag tagname.
Substitutions specified in the 'subs' list are perform on the
'content'.'''
stag,etag = config.tag(tagname,dict)
self.write(stag,Lex.subs(content,subs),etag)
#---------------------------------------------------------------------------
# Configuration file processing.
#---------------------------------------------------------------------------
def _subs_specialwords(mo):
'''Special word substitution function called by
Config.subs_specialwords().'''
word = mo.re.pattern # The special word.
macro = config.specialwords[word] # The corresponding inline macro.
if not config.sections.has_key(macro):
raise EAsciiDoc,'missing special word macro [%s]' % (macro,)
args = {}
args['words'] = mo.group() # The full match string is argument 'words'.
args.update(mo.groupdict()) # Add named match groups to the arguments.
# Delete groups that didn't participate in match.
for k,v in args.items():
if v is None: del args[k]
lines = subs_attrs(config.sections[macro],args)
if len(lines) == 0:
result = ''
elif len(lines) == 1:
result = lines[0]
else:
result = string.join(lines,writer.newline)
return result
class Config:
'''Methods to process configuration files.'''
# Predefined section name regexp's.
# DEPRECATED: 'glossary'
SPECIAL_SECTIONS= ('tags','miscellaneous','attributes','glossary','specialcharacters',
'specialwords','macros','replacements','quotes','titles',
r'paradef.+',r'listdef.+',r'blockdef.+',r'tabledef.*')
def __init__(self):
self.sections = OrderedDict() # Keyed by section name containing
# lists of section lines.
# Command-line options.
self.verbose = False
self.suppress_headers = False # -s option.
# [miscellaneous] section.
self.tabsize = 8
self.textwidth = 70
self.newline = '\r\n'
self.pagewidth = None
self.pageunits = None
self.outfilesuffix = ''
self.tags = {} # Values contain (stag,etag) tuples.
self.specialchars = {} # Values of special character substitutions.
self.specialwords = {} # Name is special word pattern, value is macro.
self.replacements = {} # Key is find pattern, value is replace pattern.
self.specialsections = {} # Name is special section name pattern, value
# is corresponding section name.
self.quotes = {} # Values contain corresponding tag name.
self.fname = '' # Most recently loaded configuration file name.
self.conf_attrs = {} # Glossary entries from conf files.
self.cmd_attrs = {} # Attributes from command-line -a options.
self.loaded = [] # Loaded conf files.
def load(self,fname,dir=None):
'''Loads sections dictionary with sections from filefname. import
Existing sections are overlaid. Silently skips missing configuration
files.'''
if dir:
fname = os.path.join(dir, fname)
# Sliently skip missing configuration file.
if not os.path.isfile(fname):
return
# Don't load conf files twice (local and application conf files are the
# same if the source file is in the application directory).
if realpath(fname) in self.loaded:
return
rdr = Reader() # Use instead of file so we can use include:[] macro.
rdr.open(fname)
self.fname = fname
reo = re.compile(r'^\[(?P<section>[^\W\d][\w-]*)\]\s*$')
sections = OrderedDict()
section,contents = '',[]
while not rdr.eof():
s = rdr.read()
if s and s[0] == '#': # Skip comment lines.
continue
s = string.rstrip(s)
found = reo.findall(s)
if found:
if section: # Store previous section.
if sections.has_key(section) \
and self.is_special_section(section):
# Merge line oriented special sections.
contents = sections[section] + contents
sections[section] = contents
section = string.lower(found[0])
contents = []
else:
contents.append(s)
if section and contents: # Store last section.
if sections.has_key(section) \
and self.is_special_section(section):
# Merge line oriented special sections.
contents = sections[section] + contents
sections[section] = contents
rdr.close()
# Delete blank lines from sections.
for k in sections.keys():
for i in range(len(sections[k])-1,-1,-1):
if not sections[k][i]:
del sections[k][i]
elif not self.is_special_section(k):
break # Only trailing blanks from non-special sections.
# Merge new sections.
self.sections.update(sections)
self.parse_tags()
# Internally [miscellaneous] section entries are just attributes.
dict = {}
parse_entries(sections.get('miscellaneous',()),dict,unquote=True)
update_attrs(self.conf_attrs,dict)
dict = {}
if sections.has_key('glossary'):
deprecated("[glossary] section", "rename to [attributes]")
parse_entries(sections.get('glossary',()),dict,unquote=True)
parse_entries(sections.get('attributes',()),dict,unquote=True)
update_attrs(self.conf_attrs,dict)
# Update document attributes so they are available immediately.
document.init_attrs()
dict = {}
parse_entries(sections.get('titles',()),dict)
Title.load(dict)
parse_entries(sections.get('specialcharacters',()),self.specialchars)
undefine_entries(self.specialchars)
parse_entries(sections.get('quotes',()),self.quotes,unique_values=True)
undefine_entries(self.quotes)
self.parse_specialwords()
self.parse_replacements()
self.parse_specialsections()
paragraphs.load(sections)
lists.load(sections)
blocks.load(sections)
tables.load(sections)
macros.load(sections.get('macros',()))
self.loaded.append(realpath(fname))
def load_all(self,dir):
'''Load the standard configuration files from directory 'dir'.'''
self.load('asciidoc.conf',dir)
conf = document.backend + '.conf'
self.load(conf,dir)
conf = document.backend + '-' + document.doctype + '.conf'
self.load(conf,dir)
# Load ./filters/*.conf files if they exist.
filters = os.path.join(dir,'filters')
if os.path.isdir(filters):
for f in os.listdir(filters):
if re.match(r'^.+\.conf$',f):
self.load(f,filters)
def load_miscellaneous(self,dict):
'''Set miscellaneous configuration entries from dictionary 'dict'.'''
def set_misc(name,rule='True',intval=False):
if dict.has_key(name):
errmsg = 'illegal [miscellaneous] %s entry' % name
if intval:
setattr(self, name, int(validate(dict[name],rule,errmsg)))
else:
setattr(self, name, validate(dict[name],rule,errmsg))
set_misc('tabsize','int($)>0',intval=True)
set_misc('textwidth','int($)>0',intval=True)
set_misc('pagewidth','int($)>0',intval=True)
set_misc('pageunits')
set_misc('outfilesuffix')
if dict.has_key('newline'):
# Convert escape sequences to their character values.
self.newline = eval('"'+dict['newline']+'"')
def check(self):
'''Check the configuration for internal consistancy. Called after all
configuration files have been loaded.'''
# Heuristic check that at least one configuration file was loaded.
if not self.specialchars or not self.tags or not lists:
raise EAsciiDoc,'incomplete configuration files'
# Check special characters are only one character long.
for k in self.specialchars.keys():
if len(k) != 1:
raise EAsciiDoc,'[specialcharacters] "%s" ' \
'must be a single character' % (k,)
# Check all special words have a corresponding inline macro body.
for macro in self.specialwords.values():
if not is_name(macro):
raise EAsciiDoc,'illegal "%s" special word name' % (macro,)
if not self.sections.has_key(macro):
warning('missing special word macro [%s]' % (macro,))
# Check all text quotes have a corresponding tag.
for q in self.quotes.keys():
tag = self.quotes[q]
if not self.tags.has_key(tag):
warning('[quotes] %s missing "%s" tag definition'
% (q,tag))
# Check all specialsections section names exist.
for k,v in self.specialsections.items():
if not self.sections.has_key(v):
warning('[%s] missing specialsections section' % (v,))
paragraphs.check()
lists.check()
blocks.check()
tables.check()
macros.check()
def is_special_section(self,section_name):
for name in self.SPECIAL_SECTIONS:
if re.match(name,section_name):
return True
return False
def dump(self):
'''Dump configuration to stdout.'''
# Header.
hdr = ''
hdr = hdr + '#' + writer.newline
hdr = hdr + '# Generated by AsciiDoc %s for %s %s.%s' % \
(VERSION,document.backend,document.doctype,writer.newline)
t = time.asctime(time.localtime(time.time()))
hdr = hdr + '# %s%s' % (t,writer.newline)
hdr = hdr + '#' + writer.newline
sys.stdout.write(hdr)
# Dump special sections.
# Dump only the configuration file and command-line attributes.
# [miscellanous] entries are dumped as part of the [attributes].
dict = {}
dict.update(self.conf_attrs)
dict.update(self.cmd_attrs)
dump_section('attributes',dict)
Title.dump()
dump_section('quotes',self.quotes)
dump_section('specialcharacters',self.specialchars)
dict = {}
for k,v in self.specialwords.items():
if dict.has_key(v):
dict[v] = '%s "%s"' % (dict[v],k) # Append word list.
else:
dict[v] = '"%s"' % (k,)
dump_section('specialwords',dict)
dump_section('replacements',self.replacements)
dump_section('specialsections',self.specialsections)
dict = {}
for k,v in self.tags.items():
dict[k] = '%s|%s' % v
dump_section('tags',dict)
paragraphs.dump()
lists.dump()
blocks.dump()
tables.dump()
macros.dump()
# Dump remaining sections.
for k in self.sections.keys():
if not self.is_special_section(k):
sys.stdout.write('[%s]%s' % (k,writer.newline))
for line in self.sections[k]:
sys.stdout.write('%s%s' % (line,writer.newline))
sys.stdout.write(writer.newline)
def subs_section(self,section,dict):
'''Section attribute substitution using attributes from import
document.attributes and 'dict'. Lines containing undefinded
attributes are deleted.'''
if self.sections.has_key(section):
return subs_attrs(self.sections[section],dict)
else:
warning('missing [%s] section' % (section,))
return ()
def parse_tags(self):
'''Parse [tags] section entries into self.tags dictionary.'''
dict = {}
parse_entries(self.sections.get('tags',()),dict)
for k,v in dict.items():
if not is_name(k):
raise EAsciiDoc,'[tag] %s malformed' % (k,)
if v is None:
if self.tags.has_key(k):
del self.tags[k]
elif v == 'none':
self.tags[k] = (None,None)
else:
mo = re.match(r'(?P<stag>.*)\|(?P<etag>.*)',v)
if mo:
self.tags[k] = (mo.group('stag'), mo.group('etag'))
else:
raise EAsciiDoc,'[tag] %s value malformed' % (k,)
def tag(self, name, dict=None):
'''Returns (starttag,endtag) tuple named name from configurationfile import
[tags] section. Raise error if not found. If a dictionary 'dict' is
passed then merge with document attributes and perform attribute
substitution on tags.'''
# TODO: Tags should be stored a single string, not split into start
# and end tags since most are going to be substituted anyway (see
# subs_tag() for how we should process them. parse_tags() (above)
# should only check i.e. parse_check(). This routine should be renamed
# split_tag() and would call subs_tag(). self.tags dictionary values
# would be strings not tuples.
if not self.tags.has_key(name):
raise EAsciiDoc, 'missing tag "%s"' % (name,)
stag,etag = self.tags[name]
if dict is not None:
# TODO: Should we warn if substitution drops a tag?
if stag:
stag = subs_attrs(stag,dict)
if etag:
etag = subs_attrs(etag,dict)
return (stag,etag)
def parse_specialsections(self):
'''Parse specialsections section to self.specialsections dictionary.'''
# TODO: This is virtually the same as parse_replacements() and should
# be factored to single routine.
dict = {}
parse_entries(self.sections.get('specialsections',()),dict,unquote=True)
for pat,sectname in dict.items():
pat = strip_quotes(pat)
if not is_regexp(pat):
raise EAsciiDoc,'[specialsections] entry "%s" ' \
'is not a valid regular expression' % (pat,)
if sectname is None:
if self.specialsections.has_key(pat):
del self.specialsections[pat]
else:
self.specialsections[pat] = sectname
def parse_replacements(self):
'''Parse replacements section into self.replacements dictionary.'''
dict = {}
#TODO: Deprecate
if self.sections.has_key('substitutions'):
parse_entries(self.sections.get('substitutions',()), dict,
unquote=True)
deprecated('[substitutions]', 'renamed [replacements]', False)
else:
parse_entries(self.sections.get('replacements',()), dict,
unquote=True)
for pat,rep in dict.items():
# The search pattern and the replacement are regular expressions so
# check them both.
pat = strip_quotes(pat)
if not is_regexp(pat):
raise EAsciiDoc,'"%s" ([replacements] entry in %s) ' \
'is not a valid regular expression' % (pat,self.fname)
if rep is None:
if self.replacements.has_key(pat):
del self.replacements[pat]
else:
rep = strip_quotes(rep)
if not is_regexp(pat):
raise EAsciiDoc,'[replacements] entry "%s=%s" in %s ' \
'is an invalid find regular expression combination' \
% (pat,rep,self.fname)
self.replacements[pat] = rep
def subs_replacements(self,s):
'''Substitute patterns from self.replacements in 's'.'''
result = s
for pat,rep in self.replacements.items():
result = re.sub(pat, rep, result)
return result
def parse_specialwords(self):
'''Parse special words section into self.specialwords dictionary.'''
reo = re.compile(r'(?:\s|^)(".+?"|[^"\s]+)(?=\s|$)')
for line in self.sections.get('specialwords',()):
e = parse_entry(line)
if not e:
raise EAsciiDoc,'[specialwords] entry "%s" in %s is malformed' \
% (line,self.fname)
name,wordlist = e
if not is_name(name):
raise EAsciiDoc,'[specialwords] name "%s" in %s is illegal' \
% (name,self.fname)
if wordlist is None:
# Undefine all words associated with 'name'.
for k,v in self.specialwords.items():
if v == name:
del self.specialwords[k]
else:
words = reo.findall(wordlist)
for word in words:
word = strip_quotes(word)
if not is_regexp(word):
raise EAsciiDoc,'"%s" (%s [specialwords] entry in %s)' \
'is not a valid regular expression' \
% (word,name,self.fname)
self.specialwords[word] = name
def subs_specialchars(self,s):
'''Perform special character substitution on string 's'.'''
'''It may seem like a good idea to escape special characters with a '\'
character, the reason we don't is because the escape character itself
then has to be escaped and this makes including code listings
problematic. Use the predefined {amp},{lt},{gt} attributes instead.'''
result = ''
for ch in s:
result = result + self.specialchars.get(ch,ch)
return result
def subs_specialwords(self,s):
'''Search for word patterns from self.specialwords in 's' and
substitute using corresponding macro.'''
result = s
for word in self.specialwords.keys():
result = re.sub(word, _subs_specialwords, result)
return result
def section2tags(self,section,dict={}):
'''Perform attribute substitution on 'section' using document
attributes plus 'dict' attributes. Return tuple (stag,etag) containing
pre and post | placeholder tags.'''
if self.sections.has_key(section):
body = self.sections[section]
else:
warning('missing [%s] section' % (section,))
body = ()
# Split macro body into start and end tag lists.
stag = []
etag = []
in_stag = True
for s in body:
if in_stag:
mo = re.match(r'(?P<stag>.*)\|(?P<etag>.*)',s)
if mo:
if mo.group('stag'):
stag.append(mo.group('stag'))
if mo.group('etag'):
etag.append(mo.group('etag'))
in_stag = False
else:
stag.append(s)
else:
etag.append(s)
# Do attribute substitution last so {brkbar} can be used to escape |.
stag = subs_attrs(stag,dict)
etag = subs_attrs(etag,dict)
return (stag,etag)
#---------------------------------------------------------------------------
# Application code.
#---------------------------------------------------------------------------
# Constants
# ---------
APP_DIR = None # This file's directory.
USER_DIR = None # ~/.asciidoc
GLOBAL_CONFIG_DIR = "/etc/asciidoc"
# Globals
# -------
document = Document() # The document being processed.
config = Config() # Configuration file reader.
reader = Reader() # Input stream line reader.
writer = Writer() # Output stream line writer.
paragraphs = Paragraphs() # Paragraph definitions.
lists = Lists() # List definitions.
blocks = DelimitedBlocks() # DelimitedBlock definitions.
tables = Tables() # Table definitions.
macros = Macros() # Macro definitions.
calloutmap = CalloutMap() # Coordinates callouts and callout list.
def asciidoc(backend, doctype, confiles, infile, outfile, options):
'''Convert AsciiDoc document to DocBook document of type doctype
The AsciiDoc document is read from fileobjectsrcthetranslated import
DocBook file written to file object dst.'''
try:
if doctype not in ('article','manpage','book'):
raise EAsciiDoc,'illegal document type'
if backend == 'linuxdoc' and doctype != 'article':
raise EAsciiDoc,'%s %s documents are not supported' \
% (backend,doctype)
document.backend = backend
if not os.path.exists(os.path.join(APP_DIR, backend+'.conf')) and not \
os.path.exists(os.path.join(GLOBAL_CONFIG_DIR, backend+'.conf')):
warning('non-standard %s backend' % backend, linenos=False)
document.doctype = doctype
document.infile = infile
document.init_attrs()
# Set processing options.
for o in options:
if o == '-s': config.suppress_headers = True
if o == '-v': config.verbose = True
# Check the infile exists.
if infile != '<stdin>' and not os.path.isfile(infile):
raise EAsciiDoc,'input file %s missing' % (infile,)
if '-e' not in options:
# Load global configuration
config.load_all(GLOBAL_CONFIG_DIR)
# Load global configuration files from asciidoc directory.
config.load_all(APP_DIR)
# Load configuration files from ~/.asciidoc if it exists.
if USER_DIR is not None:
config.load_all(USER_DIR)
# Load configuration files from document directory.
config.load_all(os.path.dirname(infile))
if infile != '<stdin>':
# Load implicit document specific configuration files if they exist.
config.load(os.path.splitext(infile)[0] + '.conf')
config.load(os.path.splitext(infile)[0] + '-' + backend + '.conf')
# If user specified configuration file(s) overlay the defaults.
if confiles:
for conf in confiles:
# First look in current working directory.
if os.path.isfile(conf):
config.load(conf)
else:
raise EAsciiDoc,'configuration file %s missing' % (conf,)
document.init_attrs() # Add conf file.
# Check configuration for consistency.
config.check()
# Build outfile name now all conf files have been read.
if outfile is None:
outfile = os.path.splitext(infile)[0] + '.' + backend
if config.outfilesuffix:
# Change file extension.
outfile = os.path.splitext(outfile)[0] + config.outfilesuffix
document.outfile = outfile
if '-c' in options:
config.dump()
else:
reader.tabsize = config.tabsize
reader.open(infile)
try:
writer.newline = config.newline
writer.open(outfile)
try:
document.init_attrs() # Add file name related entries.
document.translate()
finally:
writer.close()
finally:
reader.closefile() # Keep reader state for postmortem.
except (KeyboardInterrupt, SystemExit):
print
except Exception,e:
# Cleanup.
if outfile and outfile != '<stdout>' and os.path.isfile(outfile):
os.unlink(outfile)
# Build and print error description.
msg = 'FAILED: '
if reader.cursor:
msg = msg + "%s: line %d: " % (reader.cursor[0],reader.cursor[1])
if isinstance(e,EAsciiDoc):
print_stderr(msg+str(e))
else:
print_stderr(msg+'unexpected error:')
print_stderr('-'*60)
traceback.print_exc(file=sys.stderr)
print_stderr('-'*60)
sys.exit(1)
def usage(msg=''):
if msg:
print_stderr(msg)
print_stderr('Usage: asciidoc -b backend [-d doctype] [-a attribute]')
print_stderr(' [-e] [-n] [-s] [-f configfile] [-o outfile]')
print_stderr(' [--help | -h] [--version] [-v] [ -c ]')
print_stderr(' infile')
def main():
if float(sys.version[:3]) < 2.3:
print_stderr('FAILED: Python 2.3 or better required.')
sys.exit(1)
# Locate the executable and configuration files directory.
global APP_DIR,USER_DIR
APP_DIR = os.path.dirname(realpath(sys.argv[0]))
USER_DIR = os.environ.get('HOME')
if USER_DIR is not None:
USER_DIR = os.path.join(USER_DIR,'.asciidoc')
if not os.path.isdir(USER_DIR):
USER_DIR = None
# Process command line options.
import getopt
# DEPRECATED: 'g:' option.
opts,args = getopt.getopt(sys.argv[1:],
'a:b:cd:ef:g:hno:svw:',
['help','profile','version'])
if len(args) > 1:
usage()
sys.exit(1)
backend = None
doctype = 'article'
confiles = []
outfile = None
options = []
prof = False
for o,v in opts:
if o in ('--help','-h'):
print __doc__
sys.exit(0)
if o == '--profile':
prof = True
if o == '--version':
print_stderr('asciidoc %s' % (VERSION,))
sys.exit(0)
if o == '-b': backend = v
if o == '-c': options.append('-c')
if o == '-d': doctype = v
if o == '-e': options.append('-e')
if o == '-f': confiles.append(v)
if o == '-n':
o = '-a'
v = 'section-numbers'
if o == '-a' or o == '-g':
if o == '-g':
deprecated("-g command option", "use -a instead")
e = parse_entry(v)
if not e:
usage('Illegal -a %s option' % (v,))
sys.exit(1)
k,v = e
if v is None:
if k[0] == '^':
k = k[1:]
else:
v = ''
config.cmd_attrs[k] = v
if o == '-o':
if v == '-':
outfile = '<stdout>'
else:
outfile = v
if o == '-n': outfile = v
if o == '-s': options.append('-s')
if o == '-v': options.append('-v')
if len(args) == 0 and len(opts) == 0:
usage()
sys.exit(1)
if len(args) == 0:
usage('No source file specified')
sys.exit(1)
if not backend:
usage('No backend (-b) option specified')
sys.exit(1)
if args[0] == '-':
infile = '<stdin>'
else:
infile = args[0]
if infile == '<stdin>' and not outfile:
outfile = '<stdout>'
# Convert in and out files to absolute paths.
if infile != '<stdin>': infile = os.path.abspath(infile)
if outfile and outfile != '<stdout>': outfile = os.path.abspath(outfile)
# Do the work.
if prof:
import profile
profile.run("asciidoc('%s','%s',(),'%s',None,())"
% (backend,doctype,infile))
else:
asciidoc(backend, doctype, confiles, infile, outfile, options)
if document.invalid:
sys.exit(1)
if __name__ == "__main__":
try:
main()
except (KeyboardInterrupt, SystemExit):
pass
except:
print_stderr("%s: unexpected exit status: %s" %
(os.path.basename(sys.argv[0]), sys.exc_info()[1]))
# Exit with previous sys.exit() status or zero if no sys.exit().
sys.exit(sys.exc_info()[1])
|