scanner = $scanner; $this->events = $eventHandler; $this->mode = $mode; } /** * Begin parsing. * * This will begin scanning the document, tokenizing as it goes. * Tokens are emitted into the event handler. * * Tokenizing will continue until the document is completely * read. Errors are emitted into the event handler, but * the parser will attempt to continue parsing until the * entire input stream is read. */ public function parse() { do { $this->consumeData(); // FIXME: Add infinite loop protection. } while ($this->carryOn); } /** * Set the text mode for the character data reader. * * HTML5 defines three different modes for reading text: * - Normal: Read until a tag is encountered. * - RCDATA: Read until a tag is encountered, but skip a few otherwise- * special characters. * - Raw: Read until a special closing tag is encountered (viz. pre, script) * * This allows those modes to be set. * * Normally, setting is done by the event handler via a special return code on * startTag(), but it can also be set manually using this function. * * @param integer $textmode * One of Elements::TEXT_* * @param string $untilTag * The tag that should stop RAW or RCDATA mode. Normal mode does not * use this indicator. */ public function setTextMode($textmode, $untilTag = null) { $this->textMode = $textmode & (Elements::TEXT_RAW | Elements::TEXT_RCDATA); $this->untilTag = $untilTag; } /** * Consume a character and make a move. * HTML5 8.2.4.1 */ protected function consumeData() { // Character Ref /* * $this->characterReference() || $this->tagOpen() || $this->eof() || $this->characterData(); */ $this->characterReference(); $this->tagOpen(); $this->eof(); $this->characterData(); return $this->carryOn; } /** * Parse anything that looks like character data. * * Different rules apply based on the current text mode. * * @see Elements::TEXT_RAW Elements::TEXT_RCDATA. */ protected function characterData() { $tok = $this->scanner->current(); if ($tok === false) { return false; } switch ($this->textMode) { case Elements::TEXT_RAW: return $this->rawText(); case Elements::TEXT_RCDATA: return $this->rcdata(); default: if (strspn($tok, "<&")) { return false; } return $this->text(); } } /** * This buffers the current token as character data. */ protected function text() { $tok = $this->scanner->current(); // This should never happen... if ($tok === false) { return false; } // Null if ($tok === "\00") { $this->parseError("Received null character."); } // fprintf(STDOUT, "Writing '%s'", $tok); $this->buffer($tok); $this->scanner->next(); return true; } /** * Read text in RAW mode. */ protected function rawText() { if (is_null($this->untilTag)) { return $this->text(); } $sequence = 'untilTag . '>'; $txt = $this->readUntilSequence($sequence); $this->events->text($txt); $this->setTextMode(0); return $this->endTag(); } /** * Read text in RCDATA mode. */ protected function rcdata() { if (is_null($this->untilTag)) { return $this->text(); } $sequence = 'untilTag; $txt = ''; $tok = $this->scanner->current(); $caseSensitive = !Elements::isHtml5Element($this->untilTag); while ($tok !== false && ! ($tok == '<' && ($this->sequenceMatches($sequence, $caseSensitive)))) { if ($tok == '&') { $txt .= $this->decodeCharacterReference(); $tok = $this->scanner->current(); } else { $txt .= $tok; $tok = $this->scanner->next(); } } $len = strlen($sequence); $this->scanner->consume($len); $len += strlen($this->scanner->whitespace()); if ($this->scanner->current() !== '>') { $this->parseError("Unclosed RCDATA end tag"); } $this->scanner->unconsume($len); $this->events->text($txt); $this->setTextMode(0); return $this->endTag(); } /** * If the document is read, emit an EOF event. */ protected function eof() { if ($this->scanner->current() === false) { // fprintf(STDOUT, "EOF"); $this->flushBuffer(); $this->events->eof(); $this->carryOn = false; return true; } return false; } /** * Handle character references (aka entities). * * This version is specific to PCDATA, as it buffers data into the * text buffer. For a generic version, see decodeCharacterReference(). * * HTML5 8.2.4.2 */ protected function characterReference() { $ref = $this->decodeCharacterReference(); if ($ref !== false) { $this->buffer($ref); return true; } return false; } /** * Emit a tagStart event on encountering a tag. * * 8.2.4.8 */ protected function tagOpen() { if ($this->scanner->current() != '<') { return false; } // Any buffered text data can go out now. $this->flushBuffer(); $this->scanner->next(); return $this->markupDeclaration() || $this->endTag() || $this->processingInstruction() || $this->tagName() || /* This always returns false. */ $this->parseError("Illegal tag opening") || $this->characterData(); } /** * Look for markup. */ protected function markupDeclaration() { if ($this->scanner->current() != '!') { return false; } $tok = $this->scanner->next(); // Comment: if ($tok == '-' && $this->scanner->peek() == '-') { $this->scanner->next(); // Consume the other '-' $this->scanner->next(); // Next char. return $this->comment(); } elseif ($tok == 'D' || $tok == 'd') { // Doctype return $this->doctype(); } elseif ($tok == '[') { // CDATA section return $this->cdataSection(); } // FINISH $this->parseError("Expected . Emit an empty comment because 8.2.4.46 says to. if ($tok == '>') { // Parse error. Emit the comment token. $this->parseError("Expected comment data, got '>'"); $this->events->comment(''); $this->scanner->next(); return true; } // Replace NULL with the replacement char. if ($tok == "\0") { $tok = UTF8Utils::FFFD; } while (! $this->isCommentEnd()) { $comment .= $tok; $tok = $this->scanner->next(); } $this->events->comment($comment); $this->scanner->next(); return true; } /** * Check if the scanner has reached the end of a comment. */ protected function isCommentEnd() { $tok = $this->scanner->current(); // EOF if ($tok === false) { // Hit the end. $this->parseError("Unexpected EOF in a comment."); return true; } // If it doesn't start with -, not the end. if ($tok != '-') { return false; } // Advance one, and test for '->' if ($this->scanner->next() == '-' && $this->scanner->peek() == '>') { $this->scanner->next(); // Consume the last '>' return true; } // Unread '-'; $this->scanner->unconsume(1); return false; } /** * Parse a DOCTYPE. * * Parse a DOCTYPE declaration. This method has strong bearing on whether or * not Quirksmode is enabled on the event handler. * * @todo This method is a little long. Should probably refactor. */ protected function doctype() { if (strcasecmp($this->scanner->current(), 'D')) { return false; } // Check that string is DOCTYPE. $chars = $this->scanner->charsWhile("DOCTYPEdoctype"); if (strcasecmp($chars, 'DOCTYPE')) { $this->parseError('Expected DOCTYPE, got %s', $chars); return $this->bogusComment('scanner->whitespace(); $tok = $this->scanner->current(); // EOF: die. if ($tok === false) { $this->events->doctype('html5', EventHandler::DOCTYPE_NONE, '', true); return $this->eof(); } $doctypeName = ''; // NULL char: convert. if ($tok === "\0") { $this->parseError("Unexpected null character in DOCTYPE."); $doctypeName .= UTF8::FFFD; $tok = $this->scanner->next(); } $stop = " \n\f>"; $doctypeName = $this->scanner->charsUntil($stop); // Lowercase ASCII, replace \0 with FFFD $doctypeName = strtolower(strtr($doctypeName, "\0", UTF8Utils::FFFD)); $tok = $this->scanner->current(); // If false, emit a parse error, DOCTYPE, and return. if ($tok === false) { $this->parseError('Unexpected EOF in DOCTYPE declaration.'); $this->events->doctype($doctypeName, EventHandler::DOCTYPE_NONE, null, true); return true; } // Short DOCTYPE, like if ($tok == '>') { // DOCTYPE without a name. if (strlen($doctypeName) == 0) { $this->parseError("Expected a DOCTYPE name. Got nothing."); $this->events->doctype($doctypeName, 0, null, true); $this->scanner->next(); return true; } $this->events->doctype($doctypeName); $this->scanner->next(); return true; } $this->scanner->whitespace(); $pub = strtoupper($this->scanner->getAsciiAlpha()); $white = strlen($this->scanner->whitespace()); // Get ID, and flag it as pub or system. if (($pub == 'PUBLIC' || $pub == 'SYSTEM') && $white > 0) { // Get the sys ID. $type = $pub == 'PUBLIC' ? EventHandler::DOCTYPE_PUBLIC : EventHandler::DOCTYPE_SYSTEM; $id = $this->quotedString("\0>"); if ($id === false) { $this->events->doctype($doctypeName, $type, $pub, false); return false; } // Premature EOF. if ($this->scanner->current() === false) { $this->parseError("Unexpected EOF in DOCTYPE"); $this->events->doctype($doctypeName, $type, $id, true); return true; } // Well-formed complete DOCTYPE. $this->scanner->whitespace(); if ($this->scanner->current() == '>') { $this->events->doctype($doctypeName, $type, $id, false); $this->scanner->next(); return true; } // If we get here, we have scanner->charsUntil(">"); $this->parseError("Malformed DOCTYPE."); $this->events->doctype($doctypeName, $type, $id, true); $this->scanner->next(); return true; } // Else it's a bogus DOCTYPE. // Consume to > and trash. $this->scanner->charsUntil('>'); $this->parseError("Expected PUBLIC or SYSTEM. Got %s.", $pub); $this->events->doctype($doctypeName, 0, null, true); $this->scanner->next(); return true; } /** * Utility for reading a quoted string. * * @param string $stopchars * Characters (in addition to a close-quote) that should stop the string. * E.g. sometimes '>' is higher precedence than '"' or "'". * @return mixed String if one is found (quotations omitted) */ protected function quotedString($stopchars) { $tok = $this->scanner->current(); if ($tok == '"' || $tok == "'") { $this->scanner->next(); $ret = $this->scanner->charsUntil($tok . $stopchars); if ($this->scanner->current() == $tok) { $this->scanner->next(); } else { // Parse error because no close quote. $this->parseError("Expected %s, got %s", $tok, $this->scanner->current()); } return $ret; } return false; } /** * Handle a CDATA section. */ protected function cdataSection() { if ($this->scanner->current() != '[') { return false; } $cdata = ''; $this->scanner->next(); $chars = $this->scanner->charsWhile('CDAT'); if ($chars != 'CDATA' || $this->scanner->current() != '[') { $this->parseError('Expected [CDATA[, got %s', $chars); return $this->bogusComment('scanner->next(); do { if ($tok === false) { $this->parseError('Unexpected EOF inside CDATA.'); $this->bogusComment('scanner->next(); } while (! $this->sequenceMatches(']]>')); // Consume ]]> $this->scanner->consume(3); $this->events->cdata($cdata); return true; } // ================================================================ // Non-HTML5 // ================================================================ /** * Handle a processing instruction. * * XML processing instructions are supposed to be ignored in HTML5, * treated as "bogus comments". However, since we're not a user * agent, we allow them. We consume until ?> and then issue a * EventListener::processingInstruction() event. */ protected function processingInstruction() { if ($this->scanner->current() != '?') { return false; } $tok = $this->scanner->next(); $procName = $this->scanner->getAsciiAlpha(); $white = strlen($this->scanner->whitespace()); // If not a PI, send to bogusComment. if (strlen($procName) == 0 || $white == 0 || $this->scanner->current() == false) { $this->parseError("Expected processing instruction name, got $tok"); $this->bogusComment('. while (! ($this->scanner->current() == '?' && $this->scanner->peek() == '>')) { $data .= $this->scanner->current(); $tok = $this->scanner->next(); if ($tok === false) { $this->parseError("Unexpected EOF in processing instruction."); $this->events->processingInstruction($procName, $data); return true; } } $this->scanner->next(); // > $this->scanner->next(); // Next token. $this->events->processingInstruction($procName, $data); return true; } // ================================================================ // UTILITY FUNCTIONS // ================================================================ /** * Read from the input stream until we get to the desired sequene * or hit the end of the input stream. */ protected function readUntilSequence($sequence) { $buffer = ''; // Optimization for reading larger blocks faster. $first = substr($sequence, 0, 1); while ($this->scanner->current() !== false) { $buffer .= $this->scanner->charsUntil($first); // Stop as soon as we hit the stopping condition. if ($this->sequenceMatches($sequence, false)) { return $buffer; } $buffer .= $this->scanner->current(); $this->scanner->next(); } // If we get here, we hit the EOF. $this->parseError("Unexpected EOF during text read."); return $buffer; } /** * Check if upcomming chars match the given sequence. * * This will read the stream for the $sequence. If it's * found, this will return true. If not, return false. * Since this unconsumes any chars it reads, the caller * will still need to read the next sequence, even if * this returns true. * * Example: $this->sequenceMatches('') will * see if the input stream is at the start of a * '' string. */ protected function sequenceMatches($sequence, $caseSensitive = true) { $len = strlen($sequence); $buffer = ''; for ($i = 0; $i < $len; ++ $i) { $tok = $this->scanner->current(); $buffer .= $tok; // EOF. Rewind and let the caller handle it. if ($tok === false) { $this->scanner->unconsume($i); return false; } $this->scanner->next(); } $this->scanner->unconsume($len); return $caseSensitive ? $buffer == $sequence : strcasecmp($buffer, $sequence) === 0; } /** * Send a TEXT event with the contents of the text buffer. * * This emits an EventHandler::text() event with the current contents of the * temporary text buffer. (The buffer is used to group as much PCDATA * as we can instead of emitting lots and lots of TEXT events.) */ protected function flushBuffer() { if ($this->text === '') { return; } $this->events->text($this->text); $this->text = ''; } /** * Add text to the temporary buffer. * * @see flushBuffer() */ protected function buffer($str) { $this->text .= $str; } /** * Emit a parse error. * * A parse error always returns false because it never consumes any * characters. */ protected function parseError($msg) { $args = func_get_args(); if (count($args) > 1) { array_shift($args); $msg = vsprintf($msg, $args); } $line = $this->scanner->currentLine(); $col = $this->scanner->columnOffset(); $this->events->parseError($msg, $line, $col); return false; } /** * Decode a character reference and return the string. * * Returns false if the entity could not be found. If $inAttribute is set * to true, a bare & will be returned as-is. * * @param boolean $inAttribute * Set to true if the text is inside of an attribute value. * false otherwise. */ protected function decodeCharacterReference($inAttribute = false) { // If it fails this, it's definitely not an entity. if ($this->scanner->current() != '&') { return false; } // Next char after &. $tok = $this->scanner->next(); $entity = ''; $start = $this->scanner->position(); if ($tok == false) { return '&'; } // These indicate not an entity. We return just // the &. if (strspn($tok, static::WHITE . "&<") == 1) { // $this->scanner->next(); return '&'; } // Numeric entity if ($tok == '#') { $tok = $this->scanner->next(); // Hexidecimal encoding. // X[0-9a-fA-F]+; // x[0-9a-fA-F]+; if ($tok == 'x' || $tok == 'X') { $tok = $this->scanner->next(); // Consume x // Convert from hex code to char. $hex = $this->scanner->getHex(); if (empty($hex)) { $this->parseError("Expected &#xHEX;, got &#x%s", $tok); // We unconsume because we don't know what parser rules might // be in effect for the remaining chars. For example. '&#>' // might result in a specific parsing rule inside of tag // contexts, while not inside of pcdata context. $this->scanner->unconsume(2); return '&'; } $entity = CharacterReference::lookupHex($hex); } // Decimal encoding. // [0-9]+; else { // Convert from decimal to char. $numeric = $this->scanner->getNumeric(); if ($numeric === false) { $this->parseError("Expected &#DIGITS;, got &#%s", $tok); $this->scanner->unconsume(2); return '&'; } $entity = CharacterReference::lookupDecimal($numeric); } } elseif ($tok === '=' && $inAttribute) { return '&'; } else { // String entity. // Attempt to consume a string up to a ';'. // [a-zA-Z0-9]+; $cname = $this->scanner->getAsciiAlphaNum(); $entity = CharacterReference::lookupName($cname); // When no entity is found provide the name of the unmatched string // and continue on as the & is not part of an entity. The & will // be converted to & elsewhere. if ($entity == null) { if (!$inAttribute || strlen($cname) === 0) { $this->parseError("No match in entity table for '%s'", $cname); } $this->scanner->unconsume($this->scanner->position() - $start); return '&'; } } // The scanner has advanced the cursor for us. $tok = $this->scanner->current(); // We have an entity. We're done here. if ($tok == ';') { $this->scanner->next(); return $entity; } // If in an attribute, then failing to match ; means unconsume the // entire string. Otherwise, failure to match is an error. if ($inAttribute) { $this->scanner->unconsume($this->scanner->position() - $start); return '&'; } $this->parseError("Expected &ENTITY;, got &ENTITY%s (no trailing ;) ", $tok); return '&' . $entity; } }