diff options
Diffstat (limited to 'lib/python2.7/markupbase.py')
| -rw-r--r-- | lib/python2.7/markupbase.py | 396 |
1 files changed, 396 insertions, 0 deletions
diff --git a/lib/python2.7/markupbase.py b/lib/python2.7/markupbase.py new file mode 100644 index 0000000..ddeb983 --- /dev/null +++ b/lib/python2.7/markupbase.py @@ -0,0 +1,396 @@ +"""Shared support for scanning document type declarations in HTML and XHTML. + +This module is used as a foundation for the HTMLParser and sgmllib +modules (indirectly, for htmllib as well). It has no documented +public API and should not be used directly. + +""" + +import re + +_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match +_declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match +_commentclose = re.compile(r'--\s*>') +_markedsectionclose = re.compile(r']\s*]\s*>') + +# An analysis of the MS-Word extensions is available at +# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf + +_msmarkedsectionclose = re.compile(r']\s*>') + +del re + + +class ParserBase: + """Parser base class which provides some common support methods used + by the SGML/HTML and XHTML parsers.""" + + def __init__(self): + if self.__class__ is ParserBase: + raise RuntimeError( + "markupbase.ParserBase must be subclassed") + + def error(self, message): + raise NotImplementedError( + "subclasses of ParserBase must override error()") + + def reset(self): + self.lineno = 1 + self.offset = 0 + + def getpos(self): + """Return current line number and offset.""" + return self.lineno, self.offset + + # Internal -- update line number and offset. This should be + # called for each piece of data exactly once, in order -- in other + # words the concatenation of all the input strings to this + # function should be exactly the entire input. + def updatepos(self, i, j): + if i >= j: + return j + rawdata = self.rawdata + nlines = rawdata.count("\n", i, j) + if nlines: + self.lineno = self.lineno + nlines + pos = rawdata.rindex("\n", i, j) # Should not fail + self.offset = j-(pos+1) + else: + self.offset = self.offset + j-i + return j + + _decl_otherchars = '' + + # Internal -- parse declaration (for use by subclasses). + def parse_declaration(self, i): + # This is some sort of declaration; in "HTML as + # deployed," this should only be the document type + # declaration ("<!DOCTYPE html...>"). + # ISO 8879:1986, however, has more complex + # declaration syntax for elements in <!...>, including: + # --comment-- + # [marked section] + # name in the following list: ENTITY, DOCTYPE, ELEMENT, + # ATTLIST, NOTATION, SHORTREF, USEMAP, + # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM + rawdata = self.rawdata + j = i + 2 + assert rawdata[i:j] == "<!", "unexpected call to parse_declaration" + if rawdata[j:j+1] == ">": + # the empty comment <!> + return j + 1 + if rawdata[j:j+1] in ("-", ""): + # Start of comment followed by buffer boundary, + # or just a buffer boundary. + return -1 + # A simple, practical version could look like: ((name|stringlit) S*) + '>' + n = len(rawdata) + if rawdata[j:j+2] == '--': #comment + # Locate --.*-- as the body of the comment + return self.parse_comment(i) + elif rawdata[j] == '[': #marked section + # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section + # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA + # Note that this is extended by Microsoft Office "Save as Web" function + # to include [if...] and [endif]. + return self.parse_marked_section(i) + else: #all other declaration elements + decltype, j = self._scan_name(j, i) + if j < 0: + return j + if decltype == "doctype": + self._decl_otherchars = '' + while j < n: + c = rawdata[j] + if c == ">": + # end of declaration syntax + data = rawdata[i+2:j] + if decltype == "doctype": + self.handle_decl(data) + else: + # According to the HTML5 specs sections "8.2.4.44 Bogus + # comment state" and "8.2.4.45 Markup declaration open + # state", a comment token should be emitted. + # Calling unknown_decl provides more flexibility though. + self.unknown_decl(data) + return j + 1 + if c in "\"'": + m = _declstringlit_match(rawdata, j) + if not m: + return -1 # incomplete + j = m.end() + elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ": + name, j = self._scan_name(j, i) + elif c in self._decl_otherchars: + j = j + 1 + elif c == "[": + # this could be handled in a separate doctype parser + if decltype == "doctype": + j = self._parse_doctype_subset(j + 1, i) + elif decltype in ("attlist", "linktype", "link", "element"): + # must tolerate []'d groups in a content model in an element declaration + # also in data attribute specifications of attlist declaration + # also link type declaration subsets in linktype declarations + # also link attribute specification lists in link declarations + self.error("unsupported '[' char in %s declaration" % decltype) + else: + self.error("unexpected '[' char in declaration") + else: + self.error( + "unexpected %r char in declaration" % rawdata[j]) + if j < 0: + return j + return -1 # incomplete + + # Internal -- parse a marked section + # Override this to handle MS-word extension syntax <![if word]>content<![endif]> + def parse_marked_section(self, i, report=1): + rawdata= self.rawdata + assert rawdata[i:i+3] == '<![', "unexpected call to parse_marked_section()" + sectName, j = self._scan_name( i+3, i ) + if j < 0: + return j + if sectName in ("temp", "cdata", "ignore", "include", "rcdata"): + # look for standard ]]> ending + match= _markedsectionclose.search(rawdata, i+3) + elif sectName in ("if", "else", "endif"): + # look for MS Office ]> ending + match= _msmarkedsectionclose.search(rawdata, i+3) + else: + self.error('unknown status keyword %r in marked section' % rawdata[i+3:j]) + if not match: + return -1 + if report: + j = match.start(0) + self.unknown_decl(rawdata[i+3: j]) + return match.end(0) + + # Internal -- parse comment, return length or -1 if not terminated + def parse_comment(self, i, report=1): + rawdata = self.rawdata + if rawdata[i:i+4] != '<!--': + self.error('unexpected call to parse_comment()') + match = _commentclose.search(rawdata, i+4) + if not match: + return -1 + if report: + j = match.start(0) + self.handle_comment(rawdata[i+4: j]) + return match.end(0) + + # Internal -- scan past the internal subset in a <!DOCTYPE declaration, + # returning the index just past any whitespace following the trailing ']'. + def _parse_doctype_subset(self, i, declstartpos): + rawdata = self.rawdata + n = len(rawdata) + j = i + while j < n: + c = rawdata[j] + if c == "<": + s = rawdata[j:j+2] + if s == "<": + # end of buffer; incomplete + return -1 + if s != "<!": + self.updatepos(declstartpos, j + 1) + self.error("unexpected char in internal subset (in %r)" % s) + if (j + 2) == n: + # end of buffer; incomplete + return -1 + if (j + 4) > n: + # end of buffer; incomplete + return -1 + if rawdata[j:j+4] == "<!--": + j = self.parse_comment(j, report=0) + if j < 0: + return j + continue + name, j = self._scan_name(j + 2, declstartpos) + if j == -1: + return -1 + if name not in ("attlist", "element", "entity", "notation"): + self.updatepos(declstartpos, j + 2) + self.error( + "unknown declaration %r in internal subset" % name) + # handle the individual names + meth = getattr(self, "_parse_doctype_" + name) + j = meth(j, declstartpos) + if j < 0: + return j + elif c == "%": + # parameter entity reference + if (j + 1) == n: + # end of buffer; incomplete + return -1 + s, j = self._scan_name(j + 1, declstartpos) + if j < 0: + return j + if rawdata[j] == ";": + j = j + 1 + elif c == "]": + j = j + 1 + while j < n and rawdata[j].isspace(): + j = j + 1 + if j < n: + if rawdata[j] == ">": + return j + self.updatepos(declstartpos, j) + self.error("unexpected char after internal subset") + else: + return -1 + elif c.isspace(): + j = j + 1 + else: + self.updatepos(declstartpos, j) + self.error("unexpected char %r in internal subset" % c) + # end of buffer reached + return -1 + + # Internal -- scan past <!ELEMENT declarations + def _parse_doctype_element(self, i, declstartpos): + name, j = self._scan_name(i, declstartpos) + if j == -1: + return -1 + # style content model; just skip until '>' + rawdata = self.rawdata + if '>' in rawdata[j:]: + return rawdata.find(">", j) + 1 + return -1 + + # Internal -- scan past <!ATTLIST declarations + def _parse_doctype_attlist(self, i, declstartpos): + rawdata = self.rawdata + name, j = self._scan_name(i, declstartpos) + c = rawdata[j:j+1] + if c == "": + return -1 + if c == ">": + return j + 1 + while 1: + # scan a series of attribute descriptions; simplified: + # name type [value] [#constraint] + name, j = self._scan_name(j, declstartpos) + if j < 0: + return j + c = rawdata[j:j+1] + if c == "": + return -1 + if c == "(": + # an enumerated type; look for ')' + if ")" in rawdata[j:]: + j = rawdata.find(")", j) + 1 + else: + return -1 + while rawdata[j:j+1].isspace(): + j = j + 1 + if not rawdata[j:]: + # end of buffer, incomplete + return -1 + else: + name, j = self._scan_name(j, declstartpos) + c = rawdata[j:j+1] + if not c: + return -1 + if c in "'\"": + m = _declstringlit_match(rawdata, j) + if m: + j = m.end() + else: + return -1 + c = rawdata[j:j+1] + if not c: + return -1 + if c == "#": + if rawdata[j:] == "#": + # end of buffer + return -1 + name, j = self._scan_name(j + 1, declstartpos) + if j < 0: + return j + c = rawdata[j:j+1] + if not c: + return -1 + if c == '>': + # all done + return j + 1 + + # Internal -- scan past <!NOTATION declarations + def _parse_doctype_notation(self, i, declstartpos): + name, j = self._scan_name(i, declstartpos) + if j < 0: + return j + rawdata = self.rawdata + while 1: + c = rawdata[j:j+1] + if not c: + # end of buffer; incomplete + return -1 + if c == '>': + return j + 1 + if c in "'\"": + m = _declstringlit_match(rawdata, j) + if not m: + return -1 + j = m.end() + else: + name, j = self._scan_name(j, declstartpos) + if j < 0: + return j + + # Internal -- scan past <!ENTITY declarations + def _parse_doctype_entity(self, i, declstartpos): + rawdata = self.rawdata + if rawdata[i:i+1] == "%": + j = i + 1 + while 1: + c = rawdata[j:j+1] + if not c: + return -1 + if c.isspace(): + j = j + 1 + else: + break + else: + j = i + name, j = self._scan_name(j, declstartpos) + if j < 0: + return j + while 1: + c = self.rawdata[j:j+1] + if not c: + return -1 + if c in "'\"": + m = _declstringlit_match(rawdata, j) + if m: + j = m.end() + else: + return -1 # incomplete + elif c == ">": + return j + 1 + else: + name, j = self._scan_name(j, declstartpos) + if j < 0: + return j + + # Internal -- scan a name token and the new position and the token, or + # return -1 if we've reached the end of the buffer. + def _scan_name(self, i, declstartpos): + rawdata = self.rawdata + n = len(rawdata) + if i == n: + return None, -1 + m = _declname_match(rawdata, i) + if m: + s = m.group() + name = s.strip() + if (i + len(s)) == n: + return None, -1 # end of buffer + return name.lower(), m.end() + else: + self.updatepos(declstartpos, i) + self.error("expected name token at %r" + % rawdata[declstartpos:declstartpos+20]) + + # To be overridden -- handlers for unknown objects + def unknown_decl(self, data): + pass |