tb = treebuilders.getTreeBuilder(treebuilderName,

implementation=implementation)

p = HTMLParser(tb)

"""HTML parser. Generates a tree structure from a stream of (possibly

def __init__(self, tree = simpletree.TreeBuilder,

tokenizer = tokenizer.HTMLTokenizer, strict = False,

namespaceHTMLElements = False):

"""

# Raise an exception on the first error encountered

self.strict = strict

self.tree = tree(namespaceHTMLElements)

self.tokenizer_class = tokenizer

self.errors = []

self.phases = {

"initial": InitialPhase(self, self.tree),

"beforeHtml": BeforeHtmlPhase(self, self.tree),

"beforeHead": BeforeHeadPhase(self, self.tree),

"inHead": InHeadPhase(self, self.tree),

# XXX "inHeadNoscript": InHeadNoScriptPhase(self, self.tree),

"afterHead": AfterHeadPhase(self, self.tree),

"inBody": InBodyPhase(self, self.tree),

"inCDataRCData": InCDataRCDataPhase(self, self.tree),

"inTable": InTablePhase(self, self.tree),

"inCaption": InCaptionPhase(self, self.tree),

"inColumnGroup": InColumnGroupPhase(self, self.tree),

"inTableBody": InTableBodyPhase(self, self.tree),

"inRow": InRowPhase(self, self.tree),

"inCell": InCellPhase(self, self.tree),

"inSelect": InSelectPhase(self, self.tree),

"inSelectInTable": InSelectInTablePhase(self, self.tree),

"afterBody": AfterBodyPhase(self, self.tree),

"inFrameset": InFramesetPhase(self, self.tree),

"afterFrameset": AfterFramesetPhase(self, self.tree),

"afterAfterBody": AfterAfterBodyPhase(self, self.tree),

"afterAfterFrameset": AfterAfterFramesetPhase(self, self.tree),

# XXX after after frameset

}

def _parse(self, stream, innerHTML=False, container="div",

encoding=None, parseMeta=True, useChardet=True, **kwargs):

self.tree.reset()

self.firstStartTag = False

self.errors = []

# "quirks" / "limited quirks" / "no quirks"

self.compatMode = "no quirks"

self.tokenizer = self.tokenizer_class(stream, encoding=encoding,

parseMeta=parseMeta,

useChardet=useChardet, **kwargs)

if innerHTML:

self.innerHTML = container.lower()

if self.innerHTML in cdataElements:

self.tokenizer.contentModelFlag = tokenizer.contentModelFlags["RCDATA"]

elif self.innerHTML in rcdataElements:

self.tokenizer.contentModelFlag = tokenizer.contentModelFlags["CDATA"]

elif self.innerHTML == 'plaintext':

self.tokenizer.contentModelFlag = tokenizer.contentModelFlags["PLAINTEXT"]

else:

# contentModelFlag already is PCDATA

#self.tokenizer.contentModelFlag = tokenizer.contentModelFlags["PCDATA"]

pass

self.phase = self.phases["beforeHtml"]

self.phase.insertHtmlElement()

self.resetInsertionMode()

else:

self.innerHTML = False

self.phase = self.phases["initial"]

# We only seem to have InBodyPhase testcases where the following is

# relevant ... need others too

self.lastPhase = None

self.beforeRCDataPhase = None

CharactersToken = tokenTypes["Characters"]

SpaceCharactersToken = tokenTypes["SpaceCharacters"]

StartTagToken = tokenTypes["StartTag"]

EndTagToken = tokenTypes["EndTag"]

CommentToken = tokenTypes["Comment"]

DoctypeToken = tokenTypes["Doctype"]

for token in self.normalizedTokens():

#print self.phase.__class__.__name__

#print token

type = token["type"]

if type == CharactersToken:

self.phase.processCharacters(token)

elif type == SpaceCharactersToken:

self.phase.processSpaceCharacters(token)

elif type == StartTagToken:

self.selfClosingAcknowledged = False

self.phase.processStartTag(token)

if (token["selfClosing"]

and not self.selfClosingAcknowledged):

self.parseError("non-void-element-with-trailing-soldius",

{"name":token["name"]})

elif type == EndTagToken:

self.phase.processEndTag(token)

elif type == CommentToken:

self.phase.processComment(token)

elif type == DoctypeToken:

self.phase.processDoctype(token)

else:

self.parseError(token["data"], token.get("datavars", {}))

# When the loop finishes it's EOF

self.phase.processEOF()

def normalizedTokens(self):

for token in self.tokenizer:

yield self.normalizeToken(token)

def parse(self, stream, encoding=None, parseMeta=True, useChardet=True):

"""Parse a HTML document into a well-formed tree

self._parse(stream, innerHTML=False, encoding=encoding)

return self.tree.getDocument()

def parseFragment(self, stream, container="div", encoding=None,

parseMeta=False, useChardet=True):

"""Parse a HTML fragment into a well-formed tree fragment

self._parse(stream, True, container=container, encoding=encoding)

return self.tree.getFragment()

def parseError(self, errorcode="XXX-undefined-error", datavars={}):

# XXX The idea is to make errorcode mandatory.

self.errors.append((self.tokenizer.stream.position(), errorcode, datavars))

if self.strict:

raise ParseError

def normalizeToken(self, token):

""" HTML5 specific normalizations to the token stream """

if token["type"] == tokenTypes["StartTag"]:

token["data"] = dict(token["data"][::-1])

return token

def resetInsertionMode(self):

# The name of this method is mostly historical. (It's also used in the

# specification.)

last = False

newModes = {

"select":"inSelect",

"td":"inCell",

"th":"inCell",

"tr":"inRow",

"tbody":"inTableBody",

"thead":"inTableBody",

"tfoot":"inTableBody",

"caption":"inCaption",

"colgroup":"inColumnGroup",

"table":"inTable",

"head":"inBody",

"body":"inBody",

"frameset":"inFrameset"

}

for node in self.tree.openElements[::-1]:

nodeName = node.name

if node == self.tree.openElements[0]:

last = True

if nodeName not in ['td', 'th']:

# XXX

assert self.innerHTML

nodeName = self.innerHTML

# Check for conditions that should only happen in the innerHTML

# case

if nodeName in ("select", "colgroup", "head", "frameset"):

# XXX

assert self.innerHTML

if nodeName in newModes:

self.phase = self.phases[newModes[nodeName]]

break

elif nodeName == "html":

if self.tree.headPointer is None:

self.phase = self.phases["beforeHead"]

else:

self.phase = self.phases["afterHead"]

break

elif last:

self.phase = self.phases["inBody"]

break

def parseRCDataCData(self, token, contentType):

"""Generic (R)CDATA Parsing algorithm

assert contentType in ("CDATA", "RCDATA")

element = self.tree.insertElement(token)

self.tokenizer.contentModelFlag = contentModelFlags[contentType]

self.originalPhase = self.phase

"""Base class for helper object that implements each phase of processing

# Order should be (they can be omitted):

# * EOF

# * Comment

# * Doctype

# * SpaceCharacters

# * Characters

# * StartTag

# - startTag* methods

# * EndTag

# - endTag* methods

def __init__(self, parser, tree):

self.parser = parser

self.tree = tree

def processEOF(self):

raise NotImplementedError

def processComment(self, token):

# For most phases the following is correct. Where it's not it will be

# overridden.

self.tree.insertComment(token, self.tree.openElements[-1])

def processDoctype(self, token):

self.parser.parseError("unexpected-doctype")

def processSpaceCharacters(self, token):

self.tree.insertText(token["data"])

def processStartTag(self, token):

self.startTagHandler[token["name"]](token)

def startTagHtml(self, token):

if self.parser.firstStartTag == False and token["name"] == "html":

self.parser.parseError("non-html-root")

# XXX Need a check here to see if the first start tag token emitted is

# this token... If it's not, invoke self.parser.parseError().

for attr, value in token["data"].items():

if attr not in self.tree.openElements[0].attributes:

self.tree.openElements[0].attributes[attr] = value

self.parser.firstStartTag = False

def processEndTag(self, token):

# This phase deals with error handling as well which is currently not

# covered in the specification. The error handling is typically known as

# "quirks mode". It is expected that a future version of HTML5 will defin

# this.

def processEOF(self):

self.parser.parseError("expected-doctype-but-got-eof")

self.parser.compatMode = "quirks"

self.parser.phase = self.parser.phases["beforeHtml"]

self.parser.phase.processEOF()

def processComment(self, token):

self.tree.insertComment(token, self.tree.document)

def processDoctype(self, token):

name = token["name"]

publicId = token["publicId"]

systemId = token["systemId"]

correct = token["correct"]

if (name != "html" or publicId != None or

systemId != None):

self.parser.parseError("unknown-doctype")

if publicId is None:

publicId = ""

if systemId is None:

systemId = ""

self.tree.insertDoctype(token)

if publicId != "":

publicId = publicId.translate(asciiUpper2Lower)

if ((not correct) or token["name"] != "html"

or publicId in

("+//silmaril//dtd html pro v0r11 19970101//en",

"-//advasoft ltd//dtd html 3.0 aswedit + extensions//en",

"-//as//dtd html 3.0 aswedit + extensions//en",

"-//ietf//dtd html 2.0 level 1//en",

"-//ietf//dtd html 2.0 level 2//en",

"-//ietf//dtd html 2.0 strict level 1//en",

"-//ietf//dtd html 2.0 strict level 2//en",

"-//ietf//dtd html 2.0 strict//en",

"-//ietf//dtd html 2.0//en",

"-//ietf//dtd html 2.1e//en",

"-//ietf//dtd html 3.0//en",

"-//ietf//dtd html 3.0//en//",

"-//ietf//dtd html 3.2 final//en",

"-//ietf//dtd html 3.2//en",

"-//ietf//dtd html 3//en",

"-//ietf//dtd html level 0//en",

"-//ietf//dtd html level 0//en//2.0",

"-//ietf//dtd html level 1//en",

"-//ietf//dtd html level 1//en//2.0",

"-//ietf//dtd html level 2//en",

"-//ietf//dtd html level 2//en//2.0",

"-//ietf//dtd html level 3//en",

"-//ietf//dtd html level 3//en//3.0",

"-//ietf//dtd html strict level 0//en",

"-//ietf//dtd html strict level 0//en//2.0",

"-//ietf//dtd html strict level 1//en",

"-//ietf//dtd html strict level 1//en//2.0",

"-//ietf//dtd html strict level 2//en",

"-//ietf//dtd html strict level 2//en//2.0",

"-//ietf//dtd html strict level 3//en",

"-//ietf//dtd html strict level 3//en//3.0",

"-//ietf//dtd html strict//en",

"-//ietf//dtd html strict//en//2.0",

"-//ietf//dtd html strict//en//3.0",

"-//ietf//dtd html//en",

"-//ietf//dtd html//en//2.0",

"-//ietf//dtd html//en//3.0",

"-//metrius//dtd metrius presentational//en",

"-//microsoft//dtd internet explorer 2.0 html strict//en",

"-//microsoft//dtd internet explorer 2.0 html//en",

"-//microsoft//dtd internet explorer 2.0 tables//en",

"-//microsoft//dtd internet explorer 3.0 html strict//en",

"-//microsoft//dtd internet explorer 3.0 html//en",

"-//microsoft//dtd internet explorer 3.0 tables//en",

"-//netscape comm. corp.//dtd html//en",

"-//netscape comm. corp.//dtd strict html//en",

"-//o'reilly and associates//dtd html 2.0//en",

"-//o'reilly and associates//dtd html extended 1.0//en",

"-//o'reilly and associates//dtd html extended relaxed 1.0//en",

"-//spyglass//dtd html 2.0 extended//en",

"-//sq//dtd html 2.0 hotmetal + extensions//en",

"-//sun microsystems corp.//dtd hotjava html//en",

"-//sun microsystems corp.//dtd hotjava strict html//en",

"-//w3c//dtd html 3 1995-03-24//en",

"-//w3c//dtd html 3.2 draft//en",

"-//w3c//dtd html 3.2 final//en",

"-//w3c//dtd html 3.2//en",

"-//w3c//dtd html 3.2s draft//en",

"-//w3c//dtd html 4.0 frameset//en",

"-//w3c//dtd html 4.0 transitional//en",

"-//w3c//dtd html experimental 19960712//en",

"-//w3c//dtd html experimental 970421//en",

"-//w3c//dtd w3 html//en",

"-//w3o//dtd w3 html 3.0//en",

"-//w3o//dtd w3 html 3.0//en//",

"-//w3o//dtd w3 html strict 3.0//en//",

"-//webtechs//dtd mozilla html 2.0//en",

"-//webtechs//dtd mozilla html//en",

"-/w3c/dtd html 4.0 transitional/en",

"html")

or (publicId in

("-//w3c//dtd html 4.01 frameset//EN",

"-//w3c//dtd html 4.01 transitional//EN") and systemId == None)

or (systemId != None and

systemId ==

"http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd")):

self.parser.compatMode = "quirks"

elif (publicId in

("-//w3c//dtd xhtml 1.0 frameset//EN",

"-//w3c//dtd xhtml 1.0 transitional//EN")

or (publicId in

("-//w3c//dtd html 4.01 frameset//EN",

"-//w3c//dtd html 4.01 transitional//EN") and systemId == None)):

self.parser.compatMode = "limited quirks"

self.parser.phase = self.parser.phases["beforeHtml"]

def processSpaceCharacters(self, token):

pass

def processCharacters(self, token):

self.parser.parseError("expected-doctype-but-got-chars")

self.parser.compatMode = "quirks"

self.parser.phase = self.parser.phases["beforeHtml"]

self.parser.phase.processCharacters(token)

def processStartTag(self, token):

self.parser.parseError("expected-doctype-but-got-start-tag",

{"name": token["name"]})

self.compatMode = "quirks"

self.parser.phase = self.parser.phases["beforeHtml"]

self.parser.phase.processStartTag(token)

def processEndTag(self, token):

self.parser.parseError("expected-doctype-but-got-end-tag",

{"name": token["name"]})

self.compatMode = "quirks"

self.parser.phase = self.parser.phases["beforeHtml"]

# helper methods

def insertHtmlElement(self):

self.tree.insertRoot(impliedTagToken("html", "StartTag"))

self.parser.phase = self.parser.phases["beforeHead"]

# other

def processEOF(self):

self.insertHtmlElement()

self.parser.phase.processEOF()

def processComment(self, token):

self.tree.insertComment(token, self.tree.document)

def processSpaceCharacters(self, token):

pass

def processCharacters(self, token):

self.insertHtmlElement()

self.parser.phase.processCharacters(token)

def processStartTag(self, token):

if token["name"] == "html":

self.parser.firstStartTag = True

self.insertHtmlElement()

self.parser.phase.processStartTag(token)

def processEndTag(self, token):

self.insertHtmlElement()

def __init__(self, parser, tree):

Phase.__init__(self, parser, tree)

self.startTagHandler = utils.MethodDispatcher([

("html", self.startTagHtml),

("head", self.startTagHead)

])

self.startTagHandler.default = self.startTagOther

self.endTagHandler = utils.MethodDispatcher([

(("head", "br"), self.endTagImplyHead)

])

self.endTagHandler.default = self.endTagOther

def processEOF(self):

self.startTagHead(impliedTagToken("head", "StartTag"))

self.parser.phase.processEOF()

def processSpaceCharacters(self, token):

pass

def processCharacters(self, token):

self.startTagHead(impliedTagToken("head", "StartTag"))

self.parser.phase.processCharacters(token)

def startTagHead(self, token):

self.tree.insertElement(token)

self.tree.headPointer = self.tree.openElements[-1]

self.parser.phase = self.parser.phases["inHead"]

def startTagOther(self, token):

self.startTagHead(impliedTagToken("head", "StartTag"))

self.parser.phase.processStartTag(token)

def endTagImplyHead(self, token):

self.startTagHead(impliedTagToken("head", "StartTag"))

self.parser.phase.processEndTag(token)

def endTagOther(self, token):

self.parser.parseError("end-tag-after-implied-root",

def __init__(self, parser, tree):

Phase.__init__(self, parser, tree)

self.startTagHandler = utils.MethodDispatcher([

("html", self.startTagHtml),

("title", self.startTagTitle),

(("noscript", "noframes", "style"), self.startTagNoScriptNoFramesStyle),

("script", self.startTagScript),

(("base", "link", "command", "eventsource"),

self.startTagBaseLinkCommandEventsource),

("meta", self.startTagMeta),

("head", self.startTagHead)

])

self.startTagHandler.default = self.startTagOther

self. endTagHandler = utils.MethodDispatcher([

("head", self.endTagHead),

("br", self.endTagBr)

])

self.endTagHandler.default = self.endTagOther

# helper

def appendToHead(self, element):

if self.tree.headPointer is not None:

self.tree.headPointer.appendChild(element)

else:

assert self.parser.innerHTML

self.tree.openElementsw[-1].appendChild(element)

# the real thing

def processEOF (self):

self.anythingElse()

self.parser.phase.processEOF()

def processCharacters(self, token):

self.anythingElse()

self.parser.phase.processCharacters(token)

def startTagHtml(self, token):

self.parser.phases["inBody"].processStartTag(token)

def startTagHead(self, token):

self.parser.parseError("two-heads-are-not-better-than-one")

def startTagBaseLinkCommandEventsource(self, token):

self.tree.insertElement(token)

self.tree.openElements.pop()

token["selfClosingAcknowledged"] = True

def startTagMeta(self, token):

self.tree.insertElement(token)

self.tree.openElements.pop()

token["selfClosingAcknowledged"] = True

attributes = token["data"]

if self.parser.tokenizer.stream.charEncoding[1] == "tentative":

if "charset" in attributes:

codec = inputstream.codecName(attributes["charset"])

self.parser.tokenizer.stream.changeEncoding(codec)

elif "content" in attributes:

data = inputstream.EncodingBytes(

attributes["content"].encode(self.parser.tokenizer.stream.charEncoding[0]))

parser = inputstream.ContentAttrParser(data)

codec = parser.parse()

self.parser.tokenizer.stream.changeEncoding(codec)

def startTagTitle(self, token):

self.parser.parseRCDataCData(token, "RCDATA")

def startTagNoScriptNoFramesStyle(self, token):

#Need to decide whether to implement the scripting-disabled case

self.parser.parseRCDataCData(token, "CDATA")

def startTagScript(self, token):

#I think this is equivalent to the CDATA stuff since we don't execute script

#self.tree.insertElement(token)

self.parser.parseRCDataCData(token, "CDATA")

def startTagOther(self, token):

self.anythingElse()

self.parser.phase.processStartTag(token)

def endTagHead(self, token):

node = self.parser.tree.openElements.pop()

assert node.name == "head", "Expected head got %s"%node.name

self.parser.phase = self.parser.phases["afterHead"]

def endTagBr(self, token):

self.anythingElse()

self.parser.phase.processEndTag(token)

def endTagOther(self, token):

self.parser.parseError("unexpected-end-tag", {"name": token["name"]})

def anythingElse(self):

def __init__(self, parser, tree):

Phase.__init__(self, parser, tree)

self.startTagHandler = utils.MethodDispatcher([

("html", self.startTagHtml),

("body", self.startTagBody),

("frameset", self.startTagFrameset),

(("base", "link", "meta", "noframes", "script", "style", "title"),

self.startTagFromHead),

("head", self.startTagHead)

])

self.startTagHandler.default = self.startTagOther

self.endTagHandler = utils.MethodDispatcher([("br", self.endTagBr)])

self.endTagHandler.default = self.endTagOther

def processEOF(self):

self.anythingElse()

self.parser.phase.processEOF()

def processCharacters(self, token):

self.anythingElse()

self.parser.phase.processCharacters(token)

def startTagBody(self, token):

self.tree.insertElement(token)

self.parser.phase = self.parser.phases["inBody"]

def startTagFrameset(self, token):

self.tree.insertElement(token)

self.parser.phase = self.parser.phases["inFrameset"]

def startTagFromHead(self, token):

self.parser.parseError("unexpected-start-tag-out-of-my-head",

{"name": token["name"]})

self.tree.openElements.append(self.tree.headPointer)

self.parser.phases["inHead"].processStartTag(token)

for node in self.tree.openElements[::-1]:

if node.name == "head":

self.tree.openElements.remove(node)

break

def startTagHead(self, token):

self.parser.parseError("unexpected-start-tag", {"name":token["name"]})

def startTagOther(self, token):

self.anythingElse()

self.parser.phase.processStartTag(token)

def endTagBr(self, token):

#This is not currently in the spec

self.anythingElse()

self.parser.phase.processEndTag(token)

def endTagOther(self, token):

self.parser.parseError("unexpected-end-tag", {"name":token["name"]})

def anythingElse(self):

self.tree.insertElement(impliedTagToken("body", "StartTag"))

# http://www.whatwg.org/specs/web-apps/current-work/#in-body

# the crazy mode

def __init__(self, parser, tree):

Phase.__init__(self, parser, tree)

#Keep a ref to this for special handling of whitespace in <pre>

self.processSpaceCharactersNonPre = self.processSpaceCharacters

self.startTagHandler = utils.MethodDispatcher([

("html", self.startTagHtml),

(("base", "link", "meta", "script", "style", "title"),

self.startTagProcessInHead),

("body", self.startTagBody),

(("address", "article", "aside", "blockquote", "center", "datagrid",

"details", "dialog", "dir", "div", "dl", "fieldset", "figure",

"footer", "h1", "h2", "h3", "h4", "h5", "h6", "header", "listing",

"menu", "nav", "ol", "p", "pre", "section", "ul"),

self.startTagCloseP),

("form", self.startTagForm),

(("li", "dd", "dt"), self.startTagListItem),

("plaintext",self.startTagPlaintext),

(headingElements, self.startTagHeading),

("a", self.startTagA),

(("b", "big", "em", "font", "i", "s", "small", "strike", "strong",

"tt", "u"),self.startTagFormatting),

("nobr", self.startTagNobr),

("button", self.startTagButton),

(("applet", "marquee", "object"), self.startTagAppletMarqueeObject),

("xmp", self.startTagXmp),

("table", self.startTagTable),

(("area", "basefont", "bgsound", "br", "embed", "img", "param",

"spacer", "wbr"), self.startTagVoidFormatting),

("hr", self.startTagHr),

("image", self.startTagImage),

("input", self.startTagInput),

("isindex", self.startTagIsIndex),

("textarea", self.startTagTextarea),

(("iframe", "noembed", "noframes", "noscript"), self.startTagCdata),

("select", self.startTagSelect),

(("rp", "rt"), self.startTagRpRt),

(("option", "optgroup"), self.startTagOpt),

(("caption", "col", "colgroup", "frame", "frameset", "head",

"tbody", "td", "tfoot", "th", "thead",

"tr"), self.startTagMisplaced),

(("event-source", "command"), self.startTagNew)

])

self.startTagHandler.default = self.startTagOther

self.endTagHandler = utils.MethodDispatcher([

("body",self.endTagBody),

("html",self.endTagHtml),

(("address", "article", "aside", "blockquote", "center", "datagrid",

"details", "dialog", "dir", "div", "dl", "fieldset", "figure",

"footer", "header", "listing", "menu", "nav", "ol", "pre", "section",

"ul"), self.endTagBlock),

("form", self.endTagForm),

("p",self.endTagP),

(("dd", "dt", "li"), self.endTagListItem),

(headingElements, self.endTagHeading),

(("a", "b", "big", "em", "font", "i", "nobr", "s", "small",

"strike", "strong", "tt", "u"), self.endTagFormatting),

(("applet", "button", "marquee", "object"), self.endTagAppletButtonMarqueeObject),

("br", self.endTagBr),

])

self.endTagHandler.default = self.endTagOther

# helper

def addFormattingElement(self, token):

self.tree.insertElement(token)

self.tree.activeFormattingElements.append(

self.tree.openElements[-1])

# the real deal

def processEOF(self):

allowed_elements = frozenset(("dd", "dt", "li", "p", "tbody", "td",

"tfoot", "th", "thead", "tr", "body",

"html"))

for node in self.tree.openElements[::-1]:

if node.name not in allowed_elements:

self.parser.parseError("expected-closing-tag-but-got-eof")

break

#Stop parsing

def processSpaceCharactersDropNewline(self, token):

# Sometimes (start of <pre>, <listing>, and <textarea> blocks) we

# want to drop leading newlines

data = token["data"]

self.processSpaceCharacters = self.processSpaceCharactersNonPre

if (data.startswith("\n") and

self.tree.openElements[-1].name in ("pre", "listing", "textarea")

and not self.tree.openElements[-1].hasContent()):

data = data[1:]

if data:

self.tree.reconstructActiveFormattingElements()

self.tree.insertText(data)

def processCharacters(self, token):

# XXX The specification says to do this for every character at the

# moment, but apparently that doesn't match the real world so we don't

# do it for space characters.

self.tree.reconstructActiveFormattingElements()

self.tree.insertText(token["data"])

#This matches the current spec but may not match the real world

def processSpaceCharacters(self, token):

self.tree.reconstructActiveFormattingElements()

self.tree.insertText(token["data"])

def startTagProcessInHead(self, token):

self.parser.phases["inHead"].processStartTag(token)

def startTagBody(self, token):

self.parser.parseError("unexpected-start-tag", {"name": "body"})

if (len(self.tree.openElements) == 1

or self.tree.openElements[1].name != "body"):

assert self.parser.innerHTML

else:

for attr, value in token["data"].items():

if attr not in self.tree.openElements[1].attributes:

self.tree.openElements[1].attributes[attr] = value

def startTagCloseP(self, token):

if self.tree.elementInScope("p"):

self.endTagP(impliedTagToken("p"))

self.tree.insertElement(token)

if token["name"] in ("pre", "listing"):

self.processSpaceCharacters = self.processSpaceCharactersDropNewline

def startTagForm(self, token):

if self.tree.formPointer:

self.parser.parseError("unexpected-start-tag", {"name": "form"})

else:

if self.tree.elementInScope("p"):

self.endTagP("p")

self.tree.insertElement(token)

self.tree.formPointer = self.tree.openElements[-1]

def startTagListItem(self, token):

if self.tree.elementInScope("p"):

self.endTagP(impliedTagToken("p"))

stopNames = {"li":("li"), "dd":("dd", "dt"), "dt":("dd", "dt")}

stopName = stopNames[token["name"]]

# AT Use reversed in Python 2.4...

for i, node in enumerate(self.tree.openElements[::-1]):

if node.name in stopName:

poppedNodes = []

for j in range(i+1):

poppedNodes.append(self.tree.openElements.pop())

if i >= 1:

self.parser.parseError(

i == 1 and "missing-end-tag" or "missing-end-tags",

{"name": ", ".join([item.name

for item

in poppedNodes[:-1]])})

break

# Phrasing elements are all non special, non scoping, non

# formatting elements

if (node.name in (specialElements | scopingElements)

and node.name not in ("address", "div")):

break

# Always insert an <li> element.

self.tree.insertElement(token)

def startTagPlaintext(self, token):

if self.tree.elementInScope("p"):

self.endTagP(impliedTagToken("p"))

self.tree.insertElement(token)

self.parser.tokenizer.contentModelFlag = contentModelFlags["PLAINTEXT"]

def startTagHeading(self, token):

if self.tree.elementInScope("p"):

self.endTagP(impliedTagToken("p"))

# Uncomment the following for IE7 behavior:

#

#for item in headingElements:

# if self.tree.elementInScope(item):

# self.parser.parseError("unexpected-start-tag", {"name": token["name"]})

# item = self.tree.openElements.pop()

# while item.name not in headingElements:

# item = self.tree.openElements.pop()

# break

self.tree.insertElement(token)

def startTagA(self, token):

afeAElement = self.tree.elementInActiveFormattingElements("a")

if afeAElement:

self.parser.parseError("unexpected-start-tag-implies-end-tag",

{"startName": "a", "endName": "a"})

self.endTagFormatting(impliedTagToken("a"))

if afeAElement in self.tree.openElements:

self.tree.openElements.remove(afeAElement)

if afeAElement in self.tree.activeFormattingElements:

self.tree.activeFormattingElements.remove(afeAElement)

self.tree.reconstructActiveFormattingElements()

self.addFormattingElement(token)

def startTagFormatting(self, token):

self.tree.reconstructActiveFormattingElements()

self.addFormattingElement(token)

def startTagNobr(self, token):

self.tree.reconstructActiveFormattingElements()

if self.tree.elementInScope("nobr"):

self.parser.parseError("unexpected-start-tag-implies-end-tag",

{"startName": "nobr", "endName": "nobr"})

self.processEndTag(impliedTagToken("nobr"))

# XXX Need tests that trigger the following

self.tree.reconstructActiveFormattingElements()

self.addFormattingElement(token)

def startTagButton(self, token):

if self.tree.elementInScope("button"):

self.parser.parseError("unexpected-start-tag-implies-end-tag",

{"startName": "button", "endName": "button"})

self.processEndTag(impliedTagToken("button"))

self.parser.phase.processStartTag(token)

else:

self.tree.reconstructActiveFormattingElements()

self.tree.insertElement(token)

self.tree.activeFormattingElements.append(Marker)

def startTagAppletMarqueeObject(self, token):

self.tree.reconstructActiveFormattingElements()

self.tree.insertElement(token)

self.tree.activeFormattingElements.append(Marker)

def startTagXmp(self, token):

self.tree.reconstructActiveFormattingElements()

self.parser.parseRCDataCData(token, "CDATA")

def startTagTable(self, token):

if self.tree.elementInScope("p"):

self.processEndTag(impliedTagToken("p"))

self.tree.insertElement(token)

self.parser.phase = self.parser.phases["inTable"]

def startTagVoidFormatting(self, token):

self.tree.reconstructActiveFormattingElements()

self.tree.insertElement(token)

self.tree.openElements.pop()

def startTagHr(self, token):

if self.tree.elementInScope("p"):

self.endTagP(impliedTagToken("p"))

self.tree.insertElement(token)

self.tree.openElements.pop()

def startTagImage(self, token):

# No really...

self.parser.parseError("unexpected-start-tag-treated-as",

{"originalName": "image", "newName": "img"})

self.processStartTag(impliedTagToken("img", "StartTag",

attributes=token["data"],

selfClosing=token["selfClosing"]))

def startTagInput(self, token):

self.tree.reconstructActiveFormattingElements()

self.tree.insertElement(token)

if self.tree.formPointer:

# XXX Not exactly sure what to do here

self.tree.openElements[-1].form = self.tree.formPointer

self.tree.openElements.pop()

def startTagIsIndex(self, token):

self.parser.parseError("deprecated-tag", {"name": "isindex"})

if self.tree.formPointer:

return

self.processStartTag(impliedTagToken("form", "StartTag"))

self.processStartTag(impliedTagToken("hr", "StartTag"))

self.processStartTag(impliedTagToken("p", "StartTag"))

self.processStartTag(impliedTagToken("label", "StartTag"))

# XXX Localization ...

self.processCharacters(

{"type":tokenTypes["Characters"],

"data":"This is a searchable index. Insert your search keywords here: "})

attributes = token["data"].copy() #don't really need a copy here I think

attributes["name"] = "isindex"

self.processStartTag(impliedTagToken("input", "StartTag",

attributes = attributes,

selfClosing =

token["selfClosing"]))

self.processEndTag(impliedTagToken("label"))

self.processEndTag(impliedTagToken("p"))

self.processStartTag(impliedTagToken("hr", "StartTag"))

self.processEndTag(impliedTagToken("form"))

def startTagTextarea(self, token):

# XXX Form element pointer checking here as well...

self.tree.insertElement(token)

self.parser.tokenizer.contentModelFlag = contentModelFlags["RCDATA"]

self.processSpaceCharacters = self.processSpaceCharactersDropNewline

def startTagCdata(self, token):

"""iframe, noembed noframes, noscript(if scripting enabled)"""

self.parser.parseRCDataCData(token, "CDATA")

def startTagOpt(self, token):

if self.tree.elementInScope("option"):

self.parser.phase.processEndTag(impliedTagToken("option"))

self.tree.reconstructActiveFormattingElements()

self.parser.tree.insertElement(token)

def startTagSelect(self, token):

self.tree.reconstructActiveFormattingElements()

self.tree.insertElement(token)

if self.parser.phase in (self.parser.phases["inTable"],