#

# partparse.py: parse a by-Guido-written-and-by-Jan-Hein-edited LaTeX file,

# and generate texinfo source.

#

# This is *not* a good example of good programming practices. In fact, this

# file could use a complete rewrite, in order to become faster, more

# easy extensible and maintainable.

#

# However, I added some comments on a few places for the pityful person who

# would ever need to take a look into this file.

#

# Have I been clear enough??

#

# -jh

import sys, string, regex, getopt, os

# Different parse modes for phase 1

MODE_REGULAR = 0

MODE_VERBATIM = 1

MODE_CS_SCAN = 2

MODE_COMMENT = 3

MODE_MATH = 4

MODE_DMATH = 5

MODE_GOBBLEWHITE = 6

the_modes = MODE_REGULAR, MODE_VERBATIM, MODE_CS_SCAN, MODE_COMMENT, \

MODE_MATH, MODE_DMATH, MODE_GOBBLEWHITE

# Show the neighbourhood of the scanned buffer

def epsilon(buf, where):

wmt, wpt = where - 10, where + 10

if wmt < 0:

wmt = 0

if wpt > len(buf):

wpt = len(buf)

return ' Context ' + `buf[wmt:where]` + '.' + `buf[where:wpt]` + '.'

# Should return the line number. never worked

def lin():

global lineno

return ' Line ' + `lineno` + '.'

# Displays the recursion level.

def lv(lvl):

return ' Level ' + `lvl` + '.'

# Combine the three previous functions. Used often.

def lle(lvl, buf, where):

return lv(lvl) + lin() + epsilon(buf, where)

# This class is only needed for _symbolic_ representation of the parse mode.

class Mode:

def init(self, arg):

if arg not in the_modes:

raise ValueError, 'mode not in the_modes'

self.mode = arg

return self

def __cmp__(self, other):

if type(self) != type(other):

other = mode(other)

return cmp(self.mode, other.mode)

def __repr__(self):

if self.mode == MODE_REGULAR:

return 'MODE_REGULAR'

elif self.mode == MODE_VERBATIM:

return 'MODE_VERBATIM'

elif self.mode == MODE_CS_SCAN:

return 'MODE_CS_SCAN'

elif self.mode == MODE_COMMENT:

return 'MODE_COMMENT'

elif self.mode == MODE_MATH:

return 'MODE_MATH'

elif self.mode == MODE_DMATH:

return 'MODE_DMATH'

elif self.mode == MODE_GOBBLEWHITE:

return 'MODE_GOBBLEWHITE'

else:

raise ValueError, 'mode not in the_modes'

# just a wrapper around a class initialisation

def mode(arg):

return Mode().init(arg)

# After phase 1, the text consists of chunks, with a certain type

# this type will be assigned to the chtype member of the chunk

# the where-field contains the file position where this is found

# and the data field contains (1): a tuple describing start- end end

# positions of the substring (can be used as slice for the buf-variable),

# (2) just a string, mostly generated by the changeit routine,

# or (3) a list, describing a (recursive) subgroup of chunks

PLAIN = 0 # ASSUME PLAINTEXT, data = the text

GROUP = 1 # GROUP ({}), data = [chunk, chunk,..]

CSNAME = 2 # CONTROL SEQ TOKEN, data = the command

COMMENT = 3 # data is the actual comment

DMATH = 4 # DISPLAYMATH, data = [chunk, chunk,..]

MATH = 5 # MATH, see DISPLAYMATH

OTHER = 6 # CHAR WITH CATCODE OTHER, data = char

ACTIVE = 7 # ACTIVE CHAR

GOBBLEDWHITE = 8 # Gobbled LWSP, after CSNAME

ENDLINE = 9 # END-OF-LINE, data = '\n'

DENDLINE = 10 # DOUBLE EOL, data='\n', indicates \par

ENV = 11 # LaTeX-environment

# data =(envname,[ch,ch,ch,.])

CSLINE = 12 # for texi: next chunk will be one group

# of args. Will be set all on 1 line

IGNORE = 13 # IGNORE this data

ENDENV = 14 # TEMP END OF GROUP INDICATOR

IF = 15 # IF-directive

# data = (flag,negate,[ch, ch, ch,...])

the_types = PLAIN, GROUP, CSNAME, COMMENT, DMATH, MATH, OTHER, ACTIVE, \

GOBBLEDWHITE, ENDLINE, DENDLINE, ENV, CSLINE, IGNORE, ENDENV, IF

# class, just to display symbolic name

class ChunkType:

def init(self, chunk_type):

if chunk_type not in the_types:

raise 'ValueError', 'chunk_type not in the_types'

self.chunk_type = chunk_type

return self

def __cmp__(self, other):

if type(self) != type(other):

other = chunk_type(other)

return cmp(self.chunk_type, other.chunk_type)

def __repr__(self):

if self.chunk_type == PLAIN:

return 'PLAIN'

elif self.chunk_type == GROUP:

return 'GROUP'

elif self.chunk_type == CSNAME:

return 'CSNAME'

elif self.chunk_type == COMMENT:

return 'COMMENT'

elif self.chunk_type == DMATH:

return 'DMATH'

elif self.chunk_type == MATH:

return 'MATH'

elif self.chunk_type == OTHER:

return 'OTHER'

elif self.chunk_type == ACTIVE:

return 'ACTIVE'

elif self.chunk_type == GOBBLEDWHITE:

return 'GOBBLEDWHITE'

elif self.chunk_type == DENDLINE:

return 'DENDLINE'

elif self.chunk_type == ENDLINE:

return 'ENDLINE'

elif self.chunk_type == ENV:

return 'ENV'

elif self.chunk_type == CSLINE:

return 'CSLINE'

elif self.chunk_type == IGNORE:

return 'IGNORE'

elif self.chunk_type == ENDENV:

return 'ENDENV'

elif self.chunk_type == IF:

return 'IF'

else:

raise ValueError, 'chunk_type not in the_types'

# ...and the wrapper

def chunk_type(type):

return ChunkType().init(type)

# store a type object of the ChunkType-class-instance...

chunk_type_type = type(chunk_type(0))

# this class contains a part of the parsed buffer

class Chunk:

def init(self, chtype, where, data):

if type(chtype) != chunk_type_type:

chtype = chunk_type(chtype)

self.chtype = chtype

if type(where) != type(0):

raise TypeError, '\'where\' is not a number'

self.where = where

self.data = data

##print 'CHUNK', self

return self

def __repr__(self):

return 'chunk' + `self.chtype, self.where, self.data`

# and the wrapper

def chunk(chtype, where, data):

return Chunk().init(chtype, where, data)

error = 'partparse.error'

#

# TeX's catcodes...

#

CC_ESCAPE = 0

CC_LBRACE = 1

CC_RBRACE = 2

CC_MATHSHIFT = 3

CC_ALIGNMENT = 4

CC_ENDLINE = 5

CC_PARAMETER = 6

CC_SUPERSCRIPT = 7

CC_SUBSCRIPT = 8

CC_IGNORE = 9

CC_WHITE = 10

CC_LETTER = 11

CC_OTHER = 12

CC_ACTIVE = 13

CC_COMMENT = 14

CC_INVALID = 15

# and the names

cc_names = [\

'CC_ESCAPE', \

'CC_LBRACE', \

'CC_RBRACE', \

'CC_MATHSHIFT', \

'CC_ALIGNMENT', \

'CC_ENDLINE', \

'CC_PARAMETER', \

'CC_SUPERSCRIPT', \

'CC_SUBSCRIPT', \

'CC_IGNORE', \

'CC_WHITE', \

'CC_LETTER', \

'CC_OTHER', \

'CC_ACTIVE', \

'CC_COMMENT', \

'CC_INVALID', \

]

# Show a list of catcode-name-symbols

def pcl(codelist):

result = ''

for i in codelist:

result = result + cc_names[i] + ', '

return '[' + result[:-2] + ']'

# the name of the catcode (ACTIVE, OTHER, etc.)

def pc(code):

return cc_names[code]

# Which catcodes make the parser stop parsing regular plaintext

regular_stopcodes = [CC_ESCAPE, CC_LBRACE, CC_RBRACE, CC_MATHSHIFT, \

CC_ALIGNMENT, CC_PARAMETER, CC_SUPERSCRIPT, CC_SUBSCRIPT, \

CC_IGNORE, CC_ACTIVE, CC_COMMENT, CC_INVALID, CC_ENDLINE]

# same for scanning a control sequence name

csname_scancodes = [CC_LETTER]

# same for gobbling LWSP

white_scancodes = [CC_WHITE]

##white_scancodes = [CC_WHITE, CC_ENDLINE]

# make a list of all catcode id's, except for catcode ``other''

all_but_other_codes = range(16)

del all_but_other_codes[CC_OTHER]

##print all_but_other_codes

# when does a comment end

comment_stopcodes = [CC_ENDLINE]

# gather all characters together, specified by a list of catcodes

def code2string(cc, codelist):

##print 'code2string: codelist = ' + pcl(codelist),

result = ''

for category in codelist:

if cc[category]:

result = result + cc[category]

##print 'result = ' + `result`

return result

# automatically generate all characters of catcode other, being the

# complement set in the ASCII range (128 characters)

def make_other_codes(cc):

otherchars = range(256) # could be made 256, no problem

for category in all_but_other_codes:

if cc[category]:

for c in cc[category]:

otherchars[ord(c)] = None

result = ''

for i in otherchars:

if i != None:

result = result + chr(i)

return result

# catcode dump (which characters have which catcodes).

def dump_cc(name, cc):

##print '\t' + name

##print '=' * (8+len(name))

if len(cc) != 16:

raise TypeError, 'cc not good cat class'

## for i in range(16):

## print pc(i) + '\t' + `cc[i]`

# In the beginning,....

epoch_cc = [None] * 16

##dump_cc('epoch_cc', epoch_cc)

# INITEX

initex_cc = epoch_cc[:]

initex_cc[CC_ESCAPE] = '\\'

initex_cc[CC_ENDLINE], initex_cc[CC_IGNORE], initex_cc[CC_WHITE] = \

'\n', '\0', ' '

initex_cc[CC_LETTER] = string.uppercase + string.lowercase

initex_cc[CC_COMMENT], initex_cc[CC_INVALID] = '%', '\x7F'

#initex_cc[CC_OTHER] = make_other_codes(initex_cc) I don't need them, anyway

##dump_cc('initex_cc', initex_cc)

# LPLAIN: LaTeX catcode setting (see lplain.tex)

lplain_cc = initex_cc[:]

lplain_cc[CC_LBRACE], lplain_cc[CC_RBRACE] = '{', '}'

lplain_cc[CC_MATHSHIFT] = '$'

lplain_cc[CC_ALIGNMENT] = '&'

lplain_cc[CC_PARAMETER] = '#'

lplain_cc[CC_SUPERSCRIPT] = '^\x0B' # '^' and C-k

lplain_cc[CC_SUBSCRIPT] = '_\x01' # '_' and C-a

lplain_cc[CC_WHITE] = lplain_cc[CC_WHITE] + '\t'

lplain_cc[CC_ACTIVE] = '~\x0C' # '~' and C-l

lplain_cc[CC_OTHER] = make_other_codes(lplain_cc)

##dump_cc('lplain_cc', lplain_cc)

# Guido's LaTeX environment catcoded '_' as ``other''

# my own purpose catlist

my_cc = lplain_cc[:]

my_cc[CC_SUBSCRIPT] = my_cc[CC_SUBSCRIPT][1:] # remove '_' here

my_cc[CC_OTHER] = my_cc[CC_OTHER] + '_' # add it to OTHER list

dump_cc('my_cc', my_cc)

# needed for un_re, my equivalent for regexp-quote in Emacs

re_meaning = '\\[]^$'

def un_re(str):

result = ''

for i in str:

if i in re_meaning:

result = result + '\\'

result = result + i

return result

# NOTE the negate ('^') operator in *some* of the regexps below

def make_rc_regular(cc):

# problems here if '[]' are included!!

return regex.compile('[' + code2string(cc, regular_stopcodes) + ']')

def make_rc_cs_scan(cc):

return regex.compile('[^' + code2string(cc, csname_scancodes) + ']')

def make_rc_comment(cc):

return regex.compile('[' + code2string(cc, comment_stopcodes) + ']')

def make_rc_endwhite(cc):

return regex.compile('[^' + code2string(cc, white_scancodes) + ']')

# regular: normal mode:

rc_regular = make_rc_regular(my_cc)

# scan: scan a command sequence e.g. `newlength' or `mbox' or `;', `,' or `$'

rc_cs_scan = make_rc_cs_scan(my_cc)

rc_comment = make_rc_comment(my_cc)

rc_endwhite = make_rc_endwhite(my_cc)

# parseit (BUF, PARSEMODE=mode(MODE_REGULAR), START=0, RECURSION-LEVEL=0)

# RECURSION-LEVEL will is incremented on entry.

# result contains the list of chunks returned

# together with this list, the buffer position is returned

# RECURSION-LEVEL will be set to zero *again*, when recursively a

# {,D}MATH-mode scan has been enetered.

# This has been done in order to better check for environment-mismatches

def parseit(buf, *rest):

global lineno

if len(rest) == 3:

parsemode, start, lvl = rest

elif len(rest) == 2:

parsemode, start, lvl = rest + (0, )

elif len(rest) == 1:

parsemode, start, lvl = rest + (0, 0)

elif len(rest) == 0:

parsemode, start, lvl = mode(MODE_REGULAR), 0, 0

else:

raise TypeError, 'usage: parseit(buf[, parsemode[, start[, level]]])'

result = []

end = len(buf)

if lvl == 0 and parsemode == mode(MODE_REGULAR):

lineno = 1

lvl = lvl + 1

##print 'parseit(' + epsilon(buf, start) + ', ' + `parsemode` + ', ' + `start` + ', ' + `lvl` + ')'

#

# some of the more regular modes...

#

if parsemode in (mode(MODE_REGULAR), mode(MODE_DMATH), mode(MODE_MATH)):

cstate = []

newpos = start

curpmode = parsemode

while 1:

where = newpos

#print '\tnew round: ' + epsilon(buf, where)

if where == end:

if lvl > 1 or curpmode != mode(MODE_REGULAR):

# not the way we started...

raise EOFError, 'premature end of file.' + lle(lvl, buf, where)

# the real ending of lvl-1 parse

return end, result

pos = rc_regular.search(buf, where)

if pos < 0:

pos = end

if pos != where:

newpos, c = pos, chunk(PLAIN, where, (where, pos))

result.append(c)

continue

#

# ok, pos == where and pos != end

#

foundchar = buf[where]

if foundchar in my_cc[CC_LBRACE]:

# recursive subgroup parse...

newpos, data = parseit(buf, curpmode, where+1, lvl)

result.append(chunk(GROUP, where, data))

elif foundchar in my_cc[CC_RBRACE]:

if lvl <= 1:

raise error, 'ENDGROUP while in base level.' + lle(lvl, buf, where)

if lvl == 1 and mode != mode(MODE_REGULAR):

raise error, 'endgroup while in math mode. +lin() + epsilon(buf, where)'

return where + 1, result

elif foundchar in my_cc[CC_ESCAPE]:

#

# call the routine that actually deals with

# this problem. If do_ret is None, than

# return the value of do_ret

#

# Note that handle_cs might call this routine

# recursively again...

#

do_ret, newpos = handlecs(buf, where, \

curpmode, lvl, result, end)

if do_ret != None:

return do_ret

elif foundchar in my_cc[CC_COMMENT]:

newpos, data = parseit(buf, \

mode(MODE_COMMENT), where+1, lvl)

result.append(chunk(COMMENT, where, data))

elif foundchar in my_cc[CC_MATHSHIFT]:

# note that recursive calls to math-mode

# scanning are called with recursion-level 0

# again, in order to check for bad mathend

#

if where + 1 != end and \

buf[where + 1] in \

my_cc[CC_MATHSHIFT]:

#

# double mathshift, e.g. '$$'

#

if curpmode == mode(MODE_REGULAR):

newpos, data = parseit(buf, \

mode(MODE_DMATH), \

where+2, 0)

result.append(chunk(DMATH, \

where, data))

elif curpmode == mode(MODE_MATH):

raise error, 'wrong math delimiiter' + lin() + epsilon(buf, where)

elif lvl != 1:

raise error, 'bad mathend.' + \

lle(lvl, buf, where)

else:

return where + 2, result

else:

#

# single math shift, e.g. '$'

#

if curpmode == mode(MODE_REGULAR):

newpos, data = parseit(buf, \

mode(MODE_MATH), \

where+1, 0)

result.append(chunk(MATH, \

where, data))

elif curpmode == mode(MODE_DMATH):

raise error, 'wrong math delimiiter' + lin() + epsilon(buf, where)

elif lvl != 1:

raise error, 'bad mathend.' + \

lv(lvl, buf, where)

else:

return where + 1, result

elif foundchar in my_cc[CC_IGNORE]:

print 'warning: ignored char', `foundchar`

newpos = where + 1

elif foundchar in my_cc[CC_ACTIVE]:

result.append(chunk(ACTIVE, where, foundchar))

newpos = where + 1

elif foundchar in my_cc[CC_INVALID]:

raise error, 'invalid char ' + `foundchar`

newpos = where + 1

elif foundchar in my_cc[CC_ENDLINE]:

#

# after an end of line, eat the rest of

# whitespace on the beginning of the next line

# this is what LaTeX more or less does

#

# also, try to indicate double newlines (\par)

#

lineno = lineno + 1

savedwhere = where

newpos, dummy = parseit(buf, mode(MODE_GOBBLEWHITE), where + 1, lvl)

if newpos != end and buf[newpos] in \

my_cc[CC_ENDLINE]:

result.append(chunk(DENDLINE, \

savedwhere, foundchar))

else:

result.append(chunk(ENDLINE, \

savedwhere, foundchar))

else:

result.append(chunk(OTHER, where, foundchar))

newpos = where + 1

elif parsemode == mode(MODE_CS_SCAN):

#

# scan for a control sequence token. `\ape', `\nut' or `\%'

#

if start == end:

raise EOFError, 'can\'t find end of csname'

pos = rc_cs_scan.search(buf, start)

if pos < 0:

pos = end

if pos == start:

# first non-letter right where we started the search

# ---> the control sequence name consists of one single

# character. Also: don't eat white space...

if buf[pos] in my_cc[CC_ENDLINE]:

lineno = lineno + 1

pos = pos + 1

return pos, (start, pos)

else:

spos = pos

if buf[pos] == '\n':

lineno = lineno + 1

spos = pos + 1

pos2, dummy = parseit(buf, \

mode(MODE_GOBBLEWHITE), spos, lvl)

return pos2, (start, pos)

elif parsemode == mode(MODE_GOBBLEWHITE):

if start == end:

return start, ''

pos = rc_endwhite.search(buf, start)

if pos < 0:

pos = start

return pos, (start, pos)

elif parsemode == mode(MODE_COMMENT):

pos = rc_comment.search(buf, start)

lineno = lineno + 1

if pos < 0:

print 'no newline perhaps?'

raise EOFError, 'can\'t find end of comment'

pos = pos + 1

pos2, dummy = parseit(buf, mode(MODE_GOBBLEWHITE), pos, lvl)

return pos2, (start, pos)

else:

raise error, 'Unknown mode (' + `parsemode` + ')'

#moreresult = cswitch(buf[x1:x2], buf, newpos, parsemode, lvl)

#boxcommands = 'mbox', 'fbox'

#defcommands = 'def', 'newcommand'

endverbstr = '\\end{verbatim}'

re_endverb = regex.compile(un_re(endverbstr))

#

# handlecs: helper function for parseit, for the special thing we might

# wanna do after certain command control sequences

# returns: None or return_data, newpos

#

# in the latter case, the calling function is instructed to immediately

# return with the data in return_data

#

def handlecs(buf, where, curpmode, lvl, result, end):

global lineno

# get the control sequence name...

newpos, data = parseit(buf, mode(MODE_CS_SCAN), where+1, lvl)

saveddata = data

if s(buf, data) in ('begin', 'end'):

# skip the expected '{' and get the LaTeX-envname '}'

newpos, data = parseit(buf, mode(MODE_REGULAR), newpos+1, lvl)

if len(data) != 1:

raise error, 'expected 1 chunk of data.' + \

lle(lvl, buf, where)

# yucky, we've got an environment

envname = s(buf, data[0].data)

##print 'FOUND ' + s(buf, saveddata) + '. Name ' + `envname` + '.' + lv(lvl)

if s(buf, saveddata) == 'begin' and envname == 'verbatim':

# verbatim deserves special treatment

pos = re_endverb.search(buf, newpos)

if pos < 0:

raise error, `endverbstr` + ' not found.' + lle(lvl, buf, where)

result.append(chunk(ENV, where, (envname, [chunk(PLAIN, newpos, (newpos, pos))])))

newpos = pos + len(endverbstr)

elif s(buf, saveddata) == 'begin':

# start parsing recursively... If that parse returns

# from an '\end{...}', then should the last item of

# the returned data be a string containing the ended

# environment

newpos, data = parseit(buf, curpmode, newpos, lvl)

if not data or type(data[-1]) != type(''):

raise error, 'missing \'end\'' + lle(lvl, buf, where) + epsilon(buf, newpos)

retenv = data[-1]

del data[-1]

if retenv != envname:

#[`retenv`, `envname`]

raise error, 'environments do not match.' + \

lle(lvl, buf, where) + \

epsilon(buf, newpos)

result.append(chunk(ENV, where, (retenv, data)))

else:

# 'end'... append the environment name, as just

# pointed out, and order parsit to return...

result.append(envname)

##print 'POINT of return: ' + epsilon(buf, newpos)

# the tuple will be returned by parseit

return (newpos, result), newpos

# end of \begin ... \end handling

elif s(buf, data)[0:2] == 'if':

# another scary monster: the 'if' directive

flag = s(buf, data)[2:]

# recursively call parseit, just like environment above..

# the last item of data should contain the if-termination

# e.g., 'else' of 'fi'

newpos, data = parseit(buf, curpmode, newpos, lvl)

if not data or data[-1] not in ('else', 'fi'):

raise error, 'wrong if... termination' + \

lle(lvl, buf, where) + epsilon(buf, newpos)

ifterm = data[-1]

del data[-1]

# 0 means dont_negate flag

result.append(chunk(IF, where, (flag, 0, data)))

if ifterm == 'else':

# do the whole thing again, there is only one way

# to end this one, by 'fi'

newpos, data = parseit(buf, curpmode, newpos, lvl)

if not data or data[-1] not in ('fi', ):

raise error, 'wrong if...else... termination' \

+ lle(lvl, buf, where) \

+ epsilon(buf, newpos)

ifterm = data[-1]

del data[-1]

result.append(chunk(IF, where, (flag, 1, data)))

#done implicitely: return None, newpos

elif s(buf, data) in ('else', 'fi'):

result.append(s(buf, data))

# order calling party to return tuple

return (newpos, result), newpos

# end of \if, \else, ... \fi handling

elif s(buf, saveddata) == 'verb':

x2 = saveddata[1]

result.append(chunk(CSNAME, where, data))

if x2 == end:

raise error, 'premature end of command.' + lle(lvl, buf, where)

delimchar = buf[x2]

##print 'VERB: delimchar ' + `delimchar`

pos = regex.compile(un_re(delimchar)).search(buf, x2 + 1)

if pos < 0:

raise error, 'end of \'verb\' argument (' + \

`delimchar` + ') not found.' + \

lle(lvl, buf, where)

result.append(chunk(GROUP, x2, [chunk(PLAIN, x2+1, (x2+1, pos))]))

newpos = pos + 1

else:

result.append(chunk(CSNAME, where, data))

return None, newpos

# this is just a function to get the string value if the possible data-tuple

def s(buf, data):

if type(data) == type(''):

return data

if len(data) != 2 or not (type(data[0]) == type(data[1]) == type(0)):

raise TypeError, 'expected tuple of 2 integers'

x1, x2 = data

return buf[x1:x2]

##length, data1, i = getnextarg(length, buf, pp, i + 1)

# make a deep-copy of some chunks

def crcopy(r):

result = []

for x in r:

result.append(chunkcopy(x))

return result

# copy a chunk, would better be a method of class Chunk...

def chunkcopy(ch):

if ch.chtype == chunk_type(GROUP):

listc = ch.data[:]

for i in range(len(listc)):

listc[i] = chunkcopy(listc[i])

return chunk(GROUP, ch.where, listc)

else:

return chunk(ch.chtype, ch.where, ch.data)

# get next argument for TeX-macro, flatten a group (insert between)

# or return Command Sequence token, or give back one character

def getnextarg(length, buf, pp, item):

##wobj = Wobj().init()

##dumpit(buf, wobj.write, pp[item:min(length, item + 5)])

##print 'GETNEXTARG, (len, item) =', `length, item` + ' ---> ' + wobj.data + ' <---'

while item < length and pp[item].chtype == chunk_type(ENDLINE):

del pp[item]

length = length - 1

if item >= length:

raise error, 'no next arg.' + epsilon(buf, pp[-1].where)

if pp[item].chtype == chunk_type(GROUP):

newpp = pp[item].data

del pp[item]

length = length - 1

changeit(buf, newpp)

length = length + len(newpp)

pp[item:item] = newpp

item = item + len(newpp)

if len(newpp) < 10:

wobj = Wobj().init()

dumpit(buf, wobj.write, newpp)

##print 'GETNEXTARG: inserted ' + `wobj.data`

return length, item

elif pp[item].chtype == chunk_type(PLAIN):

#grab one char

print 'WARNING: grabbing one char'

if len(s(buf, pp[item].data)) > 1:

pp.insert(item, chunk(PLAIN, pp[item].where, s(buf, pp[item].data)[:1]))

item, length = item+1, length+1

pp[item].data = s(buf, pp[item].data)[1:]

else:

item = item+1

return length, item

else:

try:

str = `s(buf, ch.data)`

except TypeError:

str = `ch.data`

if len(str) > 400:

str = str[:400] + '...'

print 'GETNEXTARG:', ch.chtype, 'not handled, data ' + str

return length, item

# this one is needed to find the end of LaTeX's optional argument, like

# item[...]

re_endopt = regex.compile(']')

# get a LaTeX-optional argument, you know, the square braces '[' and ']'

def getoptarg(length, buf, pp, item):

wobj = Wobj().init()

dumpit(buf, wobj.write, pp[item:min(length, item + 5)])

##print 'GETOPTARG, (len, item) =', `length, item` + ' ---> ' + wobj.data + ' <---'

if item >= length or \

pp[item].chtype != chunk_type(PLAIN) or \

s(buf, pp[item].data)[0] != '[':

return length, item

pp[item].data = s(buf, pp[item].data)[1:]

if len(pp[item].data) == 0:

del pp[item]

length = length-1

while 1:

if item == length:

raise error, 'No end of optional arg found'

if pp[item].chtype == chunk_type(PLAIN):

text = s(buf, pp[item].data)

pos = re_endopt.search(text)

if pos >= 0:

pp[item].data = text[:pos]

if pos == 0:

del pp[item]

length = length-1

else:

item=item+1

text = text[pos+1:]

while text and text[0] in ' \t':

text = text[1:]

if text:

pp.insert(item, chunk(PLAIN, 0, text))

length = length + 1

return length, item

item = item+1

# Wobj just add write-requests to the ``data'' attribute

class Wobj:

def init(self):

self.data = ''

return self

def write(self, data):

self.data = self.data + data

# ignore these commands

ignoredcommands = ('bcode', 'ecode')

# map commands like these to themselves as plaintext

wordsselves = ('UNIX', 'ABC', 'C', 'ASCII', 'EOF')

# \{ --> {, \} --> }, etc

themselves = ('{', '}', '.', '@') + wordsselves

# these ones also themselves (see argargs macro in myformat.sty)

inargsselves = (',', '[', ']', '(', ')')

# this is how *I* would show the difference between emph and strong

# code 1 means: fold to uppercase

markcmds = {'code': ('', ''), 'var': 1, 'emph': ('_', '_'), \

'strong': ('*', '*')}

# recognise patter {\FONTCHANGE-CMD TEXT} to \MAPPED-FC-CMD{TEXT}

fontchanges = {'rm': 'r', 'it': 'i', 'em': 'emph', 'bf': 'b', 'tt': 't'}

# transparent for these commands

for_texi = ('emph', 'var', 'strong', 'code', 'kbd', 'key', 'dfn', 'samp', \

'r', 'i', 't')

# try to remove macros and return flat text

def flattext(buf, pp):

pp = crcopy(pp)

##print '---> FLATTEXT ' + `pp`

wobj = Wobj().init()

i, length = 0, len(pp)

while 1:

if len(pp) != length:

raise 'FATAL', 'inconsistent length'

if i >= length:

break

ch = pp[i]

i = i+1

if ch.chtype == chunk_type(PLAIN):

pass

elif ch.chtype == chunk_type(CSNAME):

if s(buf, ch.data) in themselves or hist.inargs and s(buf, ch.data) in inargsselves:

ch.chtype = chunk_type(PLAIN)

elif s(buf, ch.data) == 'e':

ch.chtype = chunk_type(PLAIN)

ch.data = '\\'

elif len(s(buf, ch.data)) == 1 \

and s(buf, ch.data) in onlylatexspecial:

ch.chtype = chunk_type(PLAIN)

# if it is followed by an empty group,

# remove that group, it was needed for

# a true space

if i < length \

and pp[i].chtype==chunk_type(GROUP) \

and len(pp[i].data) == 0:

del pp[i]

length = length-1

elif s(buf, ch.data) in markcmds.keys():

length, newi = getnextarg(length, buf, pp, i)

str = flattext(buf, pp[i:newi])

del pp[i:newi]

length = length - (newi - i)

ch.chtype = chunk_type(PLAIN)

markcmd = s(buf, ch.data)

x = markcmds[markcmd]

if type(x) == type(()):

pre, after = x

str = pre+str+after

elif x == 1:

str = string.upper(str)

else:

raise 'FATAL', 'corrupt markcmds'

ch.data = str

else:

if s(buf, ch.data) not in ignoredcommands:

print 'WARNING: deleting command ' + `s(buf, ch.data)`

print 'PP' + `pp[i-1]`

del pp[i-1]

i, length = i-1, length-1

elif ch.chtype == chunk_type(GROUP):

length, newi = getnextarg(length, buf, pp, i-1)

i = i-1

## str = flattext(buf, crcopy(pp[i-1:newi]))

## del pp[i:newi]

## length = length - (newi - i)

## ch.chtype = chunk_type(PLAIN)

## ch.data = str

else:

pass

dumpit(buf, wobj.write, pp)

##print 'FLATTEXT: RETURNING ' + `wobj.data`

return wobj.data

# try to generate node names (a bit shorter than the chapter title)

# note that the \nodename command (see elsewhere) overules these efforts

def invent_node_names(text):

words = string.split(text)

##print 'WORDS ' + `words`

if len(words) == 2 \

and string.lower(words[0]) == 'built-in' \

and string.lower(words[1]) not in ('modules', 'functions'):

return words[1]

if len(words) == 3 and string.lower(words[1]) == 'module':

return words[2]

if len(words) == 3 and string.lower(words[1]) == 'object':

return string.join(words[0:2])

if len(words) > 4 and string.lower(string.join(words[-4:])) == \

'methods and data attributes':

return string.join(words[:2])

return text

re_commas_etc = regex.compile('[,`\'@{}]')

re_whitespace = regex.compile('[ \t]*')

##nodenamecmd = next_command_p(length, buf, pp, newi, 'nodename')

# look if the next non-white stuff is also a command, resulting in skipping

# double endlines (DENDLINE) too, and thus omitting \par's

# Sometimes this is too much, maybe consider DENDLINE's as stop

def next_command_p(length, buf, pp, i, cmdname):

while 1:

if i >= len(pp):

break

ch = pp[i]

i = i+1

if ch.chtype == chunk_type(ENDLINE):

continue

if ch.chtype == chunk_type(DENDLINE):

continue

if ch.chtype == chunk_type(PLAIN):

if re_whitespace.search(s(buf, ch.data)) == 0 and \

re_whitespace.match(s(buf, ch.data)) == len(s(buf, ch.data)):

continue

return -1

if ch.chtype == chunk_type(CSNAME):

if s(buf, ch.data) == cmdname:

return i # _after_ the command

return -1

return -1