abelhooge/full-text-rss
1
0
Fork 0
Code Issues Pull Requests Actions Activity

Full-Text RSS 3.4

Browse Source
This commit is contained in:
FiveFilters.org 2015-06-14 02:03:20 +02:00
parent cfe4c012ef
commit e7753953f6
39 changed files with 8217 additions and 7499 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
admin/edit-pattern.php
View File

@ -60,6 +60,7 @@ tpl_header('Edit site patterns');
$version = file_get_contents('../site_config/standard/version.txt');
function filter_only_text($filename) {
if ($filename === 'version.txt') return false;
return (strtolower(substr($filename, -4)) == '.txt');
}
function is_valid_hostname($host) {

24
admin/update.php
View File

@ -3,7 +3,7 @@
// Author: Keyvan Minoukadeh
// Copyright (c) 2014 Keyvan Minoukadeh
// License: AGPLv3
// Date: 2013-05-02
// Date: 2014-08-19
// More info: http://fivefilters.org/content-only/
// Help: http://help.fivefilters.org
@ -36,6 +36,8 @@ ini_set("display_errors", 1);
////////////////////////////////
$admin_page = 'update';
require_once('../config.php');
require_once('../libraries/humble-http-agent/HumbleHttpAgent.php');
require_once('../libraries/humble-http-agent/CookieJar.php');
require_once 'template.php';
tpl_header('Update site patterns');
@ -129,18 +131,21 @@ if ($_REQUEST['key'] !== $admin_hash) {
// Check for updates
//////////////////////////////////
//$ff_version = @file_get_contents('http://fivefilters.org/content-only/site_config/standard/version.txt');
$_context = stream_context_create(array('http' => array('user_agent' => 'PHP/5.4')));
$latest_info_json = @file_get_contents('https://api.github.com/repos/fivefilters/ftr-site-config', false, $_context);
$http = new HumbleHttpAgent();
$latest_info_json = $http->get('https://api.github.com/repos/fivefilters/ftr-site-config');
//$_context = stream_context_create(array('http' => array('user_agent' => 'PHP/5.5'), 'ssl'=>array('verify_peer'=>false)));
//$latest_info_json = file_get_contents('https://api.github.com/repos/fivefilters/ftr-site-config', false, $_context);
if (!$latest_info_json) {
println("Sorry, couldn't get info on latest site config files. Please try again later or contact us.");
exit;
}
$latest_info_json = $latest_info_json['body'];
$latest_info_json = @json_decode($latest_info_json);
if (!is_object($latest_info_json)) {
println("Sorry, couldn't parse JSON from GitHub. Please try again later or contact us.");
exit;
}
$ff_version = $latest_info_json->updated_at;
$ff_version = $latest_info_json->pushed_at;
if ($version == $ff_version) {
die('Your site config files are up to date! If you have trouble extracting from a particular site, please email us: help@fivefilters.org');
} else {
@ -166,8 +171,15 @@ if (file_exists($tmp_old_local_dir)) {
$standard_local_dir = '../site_config/standard/';
//@copy($latest_remote, $tmp_latest_local);
//copy() does not appear to fill $http_response_header in certain environments
@file_put_contents($tmp_latest_local, @file_get_contents($latest_remote));
$headers = implode("\n", $http_response_header);
//@file_put_contents($tmp_latest_local, @file_get_contents($latest_remote, false, $_context));
$latest_remote_response = $http->get($latest_remote);
if (!is_array($latest_remote_response)) {
println("Sorry, something went wrong. Please contact us if the problem persists.");
exit;
}
@file_put_contents($tmp_latest_local, $latest_remote_response['body']);
//$headers = implode("\n", $http_response_header);
$headers = $latest_remote_response['headers'];
//var_dump($headers); exit;
if ((strpos($headers, 'HTTP/1.0 200') === false) && (strpos($headers, 'HTTP/1.1 200') === false)) {
println("Sorry, something went wrong. Please contact us if the problem persists.");

3
cache/index.php vendored
View File

@ -1,3 +1,2 @@
<?php
// this is here to prevent directory listing over the web
?>
// this is here to prevent directory listing over the web

2
cache/rss-with-key/index.php vendored Normal file
View File

@ -0,0 +1,2 @@
<?php
// this is here to prevent directory listing over the web

2
cache/rss/index.php vendored Normal file
View File

@ -0,0 +1,2 @@
<?php
// this is here to prevent directory listing over the web

18
changelog.txt
View File

@ -2,6 +2,24 @@ FiveFilters.org: Full-Text RSS
http://fivefilters.org/content-only/
CHANGELOG
------------------------------------
3.4 (2014-09-08)
- New request parameter: siteconfig lets you submit extraction rules directly in request
- New request paramter: accept=(auto|feed|html) determines what we'll accept as a response (deprecates html=1 parameter)
- New request parameter: key_redirect=0 to prevent HTTP redirect to hide API key
- Site config files can now contain native_ad_clue: [xpath] to check for elements which signify that the article is a native ad
- New config option: remove_native_ads - set to true and when we notice native ads (see above) we'll remove them from the output (only when processing feeds, doesn't affect output when input URL points to an HTML page).
- Feed output will include <dc:type>Native Ad</dc:type> for articles which appear to be native ads.
- New config option: user_submitted_config to determine whether siteconfig parameter is enabled or not
- Feed output now includes <atom:link rel="self"...> with URL of the generated feed
- Feed output now includes <atom:link rel="alternate"...> with URL of the original (input) URL
- Feed output now includes <atom:link rel="related"...> with URL to subscribe to the generated feed (using subtome.com)
- Feed preview stylesheet (feed.xsl) now presents a subscribe to feed link
- Fixed character encoding issue for certain texts
- Fixed character encoding issue for certain characters in HTML5 parsing mode
- Use base element, if present in HTML, when rewriting URLs
- HTML5-PHP library updated
- Other minor fixes/improvements
3.3 (2014-05-13)
- Content extractor now looks for Schema.org articleBody elements
- New endpoint extract.php for developers looking for simpler JSON results (no RSS as input/output)

61
config.php
View File

@ -187,11 +187,28 @@ $options->keep_enclosures = true;
// Values will be placed inside the <dc:language> element inside each <item> element
// Possible values:
// * Ignore language: 0
// * Use article/feed metadata (e.g. HTML lang attribute): 1 (default)
// * Use article/feed metadata (e.g. HTML lang attribute): 1
// * As above, but guess if not present: 2
// * Always guess: 3
// * User decides: 'user' (value of 0-3 can be passed in querystring: e.g. &lang=2)
$options->detect_language = 1;
// * User decides: 'user' (value of 0-3 can be passed in querystring: e.g. &lang=2, &lang=1 will be default if nothing supplied)
$options->detect_language = 'user';
// Allow user-submitted site config in request
// ---------------
// If enabled, a user can submit site config rules directly in the request
// using the siteconfig request parameter. Disabled (false) by default.
$options->user_submitted_config = false;
// Remove items identified as native ads?
// ---------------
// Many news sites now carry native advertising - articles which have been
// paid for by a corporation to promote their brand or product.
// Full-Text RSS can identify such articles in certain sites. If an article
// is identified as being a native ad, we'll add a <dc:type>Native Ad</dc:type>
// element to the item. But you can also request that such ads be removed from
// the output altogether. To do so, set the option below to true.
// Note: this only has effect when the input URL is a feed, not a web page.
$options->remove_native_ads = false;
/////////////////////////////////////////////////
/// RESTRICT ACCESS /////////////////////////////
@ -213,6 +230,7 @@ $options->admin_credentials = array('username'=>'admin', 'password'=>'');
// List of URLs (or parts of a URL) which the service will accept.
// If the list is empty, all URLs (except those specified in the blocked list below)
// will be permitted.
// Note: for feeds, this option applies to both feed URLs and item URLs within those feeds.
// Empty: array();
// Non-empty example: array('example.com', 'anothersite.org');
$options->allowed_urls = array();
@ -220,7 +238,8 @@ $options->allowed_urls = array();
// URLs to block
// ----------------------
// List of URLs (or parts of a URL) which the service will not accept.
// Note: this list is ignored if allowed_urls is not empty
// Note: this list is ignored if allowed_urls is not empty.
// Note: for feeds, this option applies to both feed URLs and item URLs within those feeds.
$options->blocked_urls = array();
// Key holder(s) only?
@ -231,22 +250,6 @@ $options->blocked_urls = array();
// key is provided.
$options->key_required = false;
// Favour item titles in feed
// ----------------------
// By default, when processing feeds, we assume item titles in the feed
// have not been truncated. So after processing web pages, the extracted titles
// are not used in the generated feed. If you prefer to have extracted titles in
// the feed you can either set this to false, in which case we will always favour
// extracted titles. Alternatively, if set to 'user' (default) we'll use the
// extracted title if you pass '&use_extracted_title' in the querystring.
// Possible values:
// * Favour feed titles: true
// * Favour extracted titles: false
// * Favour feed titles with user override: 'user' (default)
// Note: this has no effect when the input URL is to a web page - in these cases
// we always use the extracted title in the generated feed.
$options->favour_feed_titles = 'user';
// Access keys (password protected access)
// ------------------------------------
// NOTE: You do not need an API key from fivefilters.org to run your own
@ -307,6 +310,22 @@ $options->max_entries_with_key = 10;
// false - disabled
$options->xss_filter = 'user';
// Favour item titles in feed
// ----------------------
// By default, when processing feeds, we assume item titles in the feed
// have not been truncated. So after processing web pages, the extracted titles
// are not used in the generated feed. If you prefer to have extracted titles in
// the feed you can either set this to false, in which case we will always favour
// extracted titles. Alternatively, if set to 'user' (default) we'll use the
// extracted title if you pass '&use_extracted_title' in the querystring.
// Possible values:
// * Favour feed titles: true
// * Favour extracted titles: false
// * Favour feed titles with user override: 'user' (default)
// Note: this has no effect when the input URL is to a web page - in these cases
// we always use the extracted title in the generated feed.
$options->favour_feed_titles = 'user';
// Allowed HTML parsers
// ----------------------
// Full-Text RSS attempts to use PHP's libxml extension to process HTML.
@ -481,7 +500,7 @@ $options->cache_cleanup = 100;
/// DO NOT CHANGE ANYTHING BELOW THIS ///////////
/////////////////////////////////////////////////
if (!defined('_FF_FTR_VERSION')) define('_FF_FTR_VERSION', '3.3');
if (!defined('_FF_FTR_VERSION')) define('_FF_FTR_VERSION', '3.4');
if (basename(__FILE__) == 'config.php') {
if (file_exists(dirname(__FILE__).'/custom_config.php')) {

7
css/feed.xsl
View File

@ -1,7 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom">
<xsl:output method="html" />
<xsl:variable name="title" select="/rss/channel/title"/>
<xsl:variable name="title" select="/rss/channel/title"/>
<xsl:variable name="subscribe" select="/rss/channel/atom:link[@rel='related']/@href"/>
<xsl:template match="/">
<html>
<head>
@ -11,7 +12,7 @@
<body>
<div id="explanation">
<h1><xsl:value-of select="$title"/> <span class="small"> (full-text feed)</span></h1>
<p>You are viewing an auto-generated full-text <acronym title="Really Simple Syndication">RSS</acronym> feed. RSS feeds allow you to stay up to date with the latest news and features you want from websites. To subscribe to it, you will need a News Reader or other similar device.</p>
<p>You are viewing an auto-generated full-text <acronym title="Really Simple Syndication">RSS</acronym> feed. RSS feeds allow you to stay up to date with the latest news and features you want from websites.<br /><a href="{$subscribe}">Subscribe to this feed.</a></p>
<p>Below is the latest content available from this feed.</p>
</div>

2
extract.php
View File

@ -45,7 +45,7 @@ HTTP/1.0 200 OK
define('_FF_FTR_MODE', 'simple');
// Don't process URL as feed
$_POST['html'] = '1';
$_POST['accept'] = 'html';
// JSON output only
$_POST['format'] = 'json';
// Enable excerpts

69
index.php
View File

@ -315,6 +315,12 @@ if (!defined('_FF_FTR_INDEX')) {
<td><tt>html5php</tt>, <tt>libxml</tt></td>
<td>The default parser is libxml as it's the fastest. HTML5-PHP is an HTML5 parser implemented in PHP. It's slower than libxml, but can often produce better results. You can request HTML5-PHP be used as the parser in a site-specific config file (to ensure it gets used for all URLs for that site), or explicitly via this request parameter.</td>
</tr>
<tr>
<td>siteconfig</td>
<td>string</td>
<td>Site-specific extraction rules are usually stored in text files in the site_config folder. You can also submit <a href="http://help.fivefilters.org/customer/portal/articles/223153-site-patterns">extraction rules</a> directly in your request using this parameter.</td>
</tr>
<tr>
<td>proxy</td>
@ -361,43 +367,43 @@ if (!defined('_FF_FTR_INDEX')) {
<td>string (URL)</td>
<td>This is the only required parameter. It should be the URL to a partial feed or a standard HTML page. You can omit the 'http://' prefix if you like.</td>
</tr>
<tr>
<td>format</td>
<td><tt>rss</tt> (default), <tt>json</tt></td>
<td>The default Full-Text RSS output is RSS. The only other valid output format is JSON. To get JSON output, pass format=json in the querystring. Exclude it from the URL (or set it to ‘rss’) if you’d like RSS.</td>
</tr>
<tr>
<td>summary</td>
<td><tt>0</tt> (default), <tt>1</tt></td>
<td>If set to 1, an excerpt will be included for each item in the output.</td>
</tr>
<tr>
<td>content</td>
<td><tt>0</tt>, <tt>1</tt> (default)</td>
<td>If set to 0, the extracted content will not be included in the output.</td>
</tr>
</tr>
<tr>
<td>links</td>
<td><tt>preserve</tt> (default), <tt>footnotes</tt>, <tt>remove</tt></td>
<td>Links can either be preserved, made into footnotes, or removed. None of these options affect the link text, only the hyperlink itself.</td>
</tr>
</tr>
<tr>
<td>exc</td>
<td><tt>0</tt> (default), <tt>1</tt></td>
<td>If Full-Text RSS fails to extract the article body, the generated feed item will include a message saying extraction failed followed by the original item description (if present in the original feed). You ask Full-Text RSS to remove such items from the generated feed completely by passing 1 in this parameter.</td>
</tr>
</tr>
<tr>
<td>html</td>
<td><tt>0</tt> (default), <tt>1</tt></td>
<td><p>Treat input source as HTML (or parse-as-html-first mode). To enable, pass html=1 in the querystring. If enabled, Full-Text RSS will not attempt to parse the response as a feed. This increases performance slightly and should be used if you know that the URL is not a feed.</p>
<td>accept</td>
<td><tt>auto</tt> (default), <tt>feed</tt>, <tt>html</tt></td>
<td><p>Tell Full-Text RSS what it should expect when fetching the input URL. By default Full-Text RSS tries to guess whether the response is a feed or regular HTML page. It's a good idea to be explicit by passing the appropriate type in this parameter. This is useful if, for example, a feed stops working and begins to return HTML or redirecs to a HTML page as a result of site changes. In such a scenario, if you've been explicit about the URL being a feed, Full-Text RSS will not parse HTML returned in response. If you pass accept=html (previously html=1), Full-Text RSS will not attempt to parse the response as a feed. This increases performance slightly and should be used if you know that the URL is not a feed.</p>
<p>Note: If excluded, or set to 0, Full-Text RSS first tries to parse the server's response as a feed, and only if it fails to parse as a feed will it revert to HTML parsing. In the default parse-as-feed-first mode, Full-Text RSS will identify itself as PHP first and only if a valid feed is returned will it identify itself as a browser in subsequent requests to fetch the feed items. In parse-as-html-first mode, Full-Text RSS will identify itself as a browser from the very first request.</p></td>
<p>Note: If excluded, or set to <tt>auto</tt>, Full-Text RSS first tries to parse the server's response as a feed, and only if it fails to parse as a feed will it revert to HTML parsing. In the default parse-as-feed-first mode, Full-Text RSS will identify itself as PHP first and only if a valid feed is returned will it identify itself as a browser in subsequent requests to fetch the feed items. In parse-as-html mode, Full-Text RSS will identify itself as a browser from the very first request.</p></td>
</tr>
<tr>
@ -405,9 +411,9 @@ if (!defined('_FF_FTR_INDEX')) {
<td><tt>0</tt> (default), <tt>1</tt></td>
<td><p>Use this to enable XSS filtering. We have not enabled this by default because we assume the majority of our users do not display the HTML retrieved by Full-Text RSS in a web page without further processing. If you subscribe to our generated feeds in your news reader application, it should, if it's good software, already filter the resulting HTML for XSS attacks, making it redundant for Full-Text RSS do the same. Similarly with frameworks/CMSs which display feed content - the content should be treated like any other user-submitted content.</p>
<p>If you are writing an application yourself which is processing feeds generated by Full-Text RSS, you can either filter the HTML yourself to remove potential XSS attacks or enable this option. This might be useful if you are processing our generated feeds with JavaScript on the client side - although there's client side xss filtering available too.</p>
<p>If you are writing an application yourself which is processing feeds generated by Full-Text RSS, you can either filter the HTML yourself to remove potential XSS attacks or enable this option. This might be useful if you are processing our generated feeds with JavaScript on the client side - although there's client side xss filtering available too.</p>
<p>If enabled, we'll pass retrieved HTML content through htmLawed (safe flag on and style attributes denied). Note: if enabled this will also remove certain elements you may want to preserve, such as iframes.</p></td>
<p>If enabled, we'll pass retrieved HTML content through htmLawed (safe flag on and style attributes denied). Note: if enabled this will also remove certain elements you may want to preserve, such as iframes.</p></td>
</tr>
<tr>
@ -444,6 +450,12 @@ if (!defined('_FF_FTR_INDEX')) {
<td><tt>html5php</tt>, <tt>libxml</tt></td>
<td>The default parser is libxml as it's the fastest. HTML5-PHP is an HTML5 parser implemented in PHP. It's slower than libxml, but can often produce better results. You can request HTML5-PHP be used as the parser in a site-specific config file (to ensure it gets used for all URLs for that site), or explicitly via this request parameter.</td>
</tr>
<tr>
<td>siteconfig</td>
<td>string</td>
<td>Site-specific extraction rules are usually stored in text files in the site_config folder. You can also submit <a href="http://help.fivefilters.org/customer/portal/articles/223153-site-patterns">extraction rules</a> directly in your request using this parameter.</td>
</tr>
<tr>
<td>proxy</td>
@ -501,19 +513,24 @@ if (!defined('_FF_FTR_INDEX')) {
</tr>
</thead>
<tbody>
<tr>
<td>key</td>
<td>string or number</td>
<td><p>This parameter has two functions.</p><p>If you're calling Full-Text RSS programattically, it's better to use this parameter to provide the API key index number together with the hash parameter (see below) so that the actual API key does not get sent in the HTTP request.</p><p>If you pass the actual API key in this parameter, the hash parameter is not required. If you pass the actual API key to makefulltextfeed.php, Full-Text RSS will find the index number and generate the hash value automatically and redirect to a new URL to hide the API key. If you'd like to link to a generated feed publically while protecting your API key, make sure you copy and paste the URL that results after the redirect.</p><p>If you've configured Full-Text RSS to require a key, an invalid key will result in an error message.</p></td>
</tr>
<tr>
<td>hash</td>
<td>string</td>
<td>A SHA-1 hash value of the API key (actual key, not index number) and the URL supplied in the <tt>url</tt> parameter, concatenated. This parameter must be passed along with the API key's index number using the <tt>key</tt> parameter (see above). In PHP, for example: <tt>$hash = sha1($api_key.$url);</tt></td>
</tr>
</tbody>
<tr>
<td>key</td>
<td>string or number</td>
<td><p>This parameter has two functions.</p><p>If you're calling Full-Text RSS programattically, it's better to use this parameter to provide the API key index number together with the hash parameter (see below) so that the actual API key does not get sent in the HTTP request.</p><p>If you pass the actual API key in this parameter, the hash parameter is not required. If you pass the actual API key, Full-Text RSS will find the index number and generate the hash value automatically and redirect to a new URL to hide the API key. If you'd like to link to a generated feed publically while protecting your API key, make sure you copy and paste the URL that results after the redirect.</p><p>If you've configured Full-Text RSS to require a key, an invalid key will result in an error message.</p></td>
</tr>
<tr>
<td>hash</td>
<td>string</td>
<td>A SHA-1 hash value of the API key (actual key, not index number) and the URL supplied in the <tt>url</tt> parameter, concatenated. This parameter must be passed along with the API key's index number using the <tt>key</tt> parameter (see above). In PHP, for example: <tt>$hash = sha1($api_key.$url);</tt></td>
</tr>
<tr>
<td>key_redirect</td>
<td>0 or 1 (default)</td>
<td><p>When supplying the API key with the <tt>key</tt> parameter, Full-Text RSS will generate a new URL and issue a HTTP redirect to the new URL to hide the API key (see description above). If you'd like to avoid an HTTP redirect, you can pass 0 in this parameter. We do not recommend you subscribe to feeds generated in this way.</p></td>
</tr>
</tbody>
</table>

6
libraries/DisableSimplePieSanitize.php Normal file
View File

@ -0,0 +1,6 @@
<?php
class DisableSimplePieSanitize extends SimplePie_Sanitize {
function sanitize($data, $type, $base = '') {
return $data;
}
}

33
libraries/content-extractor/ContentExtractor.php
View File

@ -33,7 +33,9 @@ class ContentExtractor
);
protected $html;
protected $config;
protected $userSubmittedConfig;
protected $title;
protected $nativeAd = false;
protected $author = array();
protected $language;
protected $date;
@ -65,10 +67,12 @@ class ContentExtractor
}
public function reset() {
// we do not reset $this->userSubmittedConfig (it gets reused)
$this->html = null;
$this->readability = null;
$this->config = null;
$this->title = null;
$this->nativeAd = false;
$this->body = null;
$this->author = array();
$this->language = null;
@ -156,7 +160,17 @@ class ContentExtractor
// but it has problems of its own which we try to avoid with this option.
public function process($html, $url, $smart_tidy=true) {
$this->reset();
$this->config = $this->buildSiteConfig($url, $html);
// use user submitted config and merge it with regular one
if (isset($this->userSubmittedConfig)) {
$this->debug('Using user-submitted site config');
$this->config = $this->userSubmittedConfig;
if ($this->config->autodetect_on_failure()) {
$this->debug('Merging user-submitted site config with site config files associated with this URL and/or content');
$this->config->append($this->buildSiteConfig($url, $html));
}
} else {
$this->config = $this->buildSiteConfig($url, $html);
}
// do string replacements
if (!empty($this->config->find_string)) {
@ -225,6 +239,15 @@ class ContentExtractor
}
}
// check if this is a native ad
foreach ($this->config->native_ad_clue as $pattern) {
$elems = @$xpath->evaluate($pattern, $this->readability->dom);
if ($elems instanceof DOMNodeList && $elems->length > 0) {
$this->nativeAd = true;
break;
}
}
// try to get title
foreach ($this->config->title as $pattern) {
// $this->debug("Trying $pattern");
@ -758,9 +781,17 @@ class ContentExtractor
return false;
}
public function setUserSubmittedConfig($config_string) {
$this->userSubmittedConfig = SiteConfig::build_from_string($config_string);
}
public function getContent() {
return $this->body;
}
public function isNativeAd() {
return $this->nativeAd;
}
public function getTitle() {
return $this->title;

12
libraries/content-extractor/SiteConfig.php
View File

@ -34,6 +34,9 @@ class SiteConfig
// Strip images which contain these strings (0 or more) in the src attribute
public $strip_image_src = array();
// Mark article as a native ad if any of these expressions match (0 or more xpath expressions)
public $native_ad_clue = array();
// Additional HTTP headers to send
// NOT YET USED
@ -182,7 +185,7 @@ class SiteConfig
public function append(SiteConfig $newconfig) {
// check for commands where we accept multiple statements (no test_url)
foreach (array('title', 'body', 'author', 'date', 'strip', 'strip_id_or_class', 'strip_image_src', 'single_page_link', 'single_page_link_in_feed', 'next_page_link', 'http_header') as $var) {
foreach (array('title', 'body', 'author', 'date', 'strip', 'strip_id_or_class', 'strip_image_src', 'single_page_link', 'single_page_link_in_feed', 'next_page_link', 'native_ad_clue', 'http_header') as $var) {
// append array elements for this config variable from $newconfig to this config
//$this->$var = $this->$var + $newconfig->$var;
$this->$var = array_unique(array_merge($this->$var, $newconfig->$var));
@ -323,6 +326,11 @@ class SiteConfig
}
}
public static function build_from_string($string) {
$config_lines = explode("\n", $string);
return self::build_from_array($config_lines);
}
public static function build_from_array(array $lines) {
$config = new SiteConfig();
foreach ($lines as $line) {
@ -340,7 +348,7 @@ class SiteConfig
if ($command == '' || $val == '') continue;
// check for commands where we accept multiple statements
if (in_array($command, array('title', 'body', 'author', 'date', 'strip', 'strip_id_or_class', 'strip_image_src', 'single_page_link', 'single_page_link_in_feed', 'next_page_link', 'http_header', 'test_url', 'find_string', 'replace_string'))) {
if (in_array($command, array('title', 'body', 'author', 'date', 'strip', 'strip_id_or_class', 'strip_image_src', 'single_page_link', 'single_page_link_in_feed', 'next_page_link', 'native_ad_clue', 'http_header', 'test_url', 'find_string', 'replace_string'))) {
array_push($config->$command, $val);
// check for single statement commands that evaluate to true or false
} elseif (in_array($command, array('tidy', 'prune', 'autodetect_on_failure'))) {

52
libraries/feedwriter/FeedWriter.php
View File

@ -19,6 +19,8 @@ define('JSONP', 3, true);
class FeedWriter
{
private $self = null; // self URL - http://feed2.w3.org/docs/warning/MissingAtomSelfLink.html
private $alternate = array(); // alternate URL and title
private $related = array(); // related URL and title
private $hubs = array(); // PubSubHubbub hubs
private $channels = array(); // Collection of channel elements
private $items = array(); // Collection of items as object of FeedItem class.
@ -240,10 +242,36 @@ define('JSONP', 3, true);
* @param string URL
* @return void
*/
public function setSelf($self)
public function setSelf($url)
{
$this->self = $self;
}
$this->self = $url;
}
/**
* Set alternate URL
*
* @access public
* @param string URL
* @param string title
* @return void
*/
public function setAlternate($url, $title)
{
$this->alternate = array('url'=>$url, 'title'=>$title);
}
/**
* Set related URL
*
* @access public
* @param string URL
* @param string title
* @return void
*/
public function setRelated($url, $title)
{
$this->related = array('url'=>$url, 'title'=>$title);
}
/**
* Set the 'description' channel element
@ -299,7 +327,7 @@ define('JSONP', 3, true);
{
$out = '<?xml version="1.0" encoding="utf-8"?>'."\n";
if ($this->xsl) $out .= '<?xml-stylesheet type="text/xsl" href="'.htmlspecialchars($this->xsl).'"?>' . PHP_EOL;
$out .= '<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:media="http://search.yahoo.com/mrss/">' . PHP_EOL;
$out .= '<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:media="http://search.yahoo.com/mrss/">' . PHP_EOL;
echo $out;
}
elseif ($this->version == JSON || $this->version == JSONP)
@ -342,7 +370,7 @@ define('JSONP', 3, true);
{
foreach ($attributes as $key => $value)
{
$attrText .= " $key=\"$value\" ";
$attrText .= " $key=\"".htmlspecialchars($value, ENT_COMPAT, 'UTF-8', false)."\" ";
}
}
$nodeText .= "<{$tagName}{$attrText}>";
@ -356,7 +384,7 @@ define('JSONP', 3, true);
else
{
//$nodeText .= (in_array($tagName, $this->CDATAEncoding))? $tagContent : htmlentities($tagContent);
$nodeText .= htmlspecialchars($tagContent);
$nodeText .= htmlspecialchars($tagContent, ENT_COMPAT, 'UTF-8', false);
}
//$nodeText .= (in_array($tagName, $this->CDATAEncoding))? "]]></$tagName>" : "</$tagName>";
$nodeText .= "</$tagName>";
@ -408,13 +436,21 @@ define('JSONP', 3, true);
// add hubs
foreach ($this->hubs as $hub) {
//echo $this->makeNode('link', '', array('rel'=>'hub', 'href'=>$hub, 'xmlns'=>'http://www.w3.org/2005/Atom'));
echo '<link rel="hub" href="'.htmlspecialchars($hub).'" xmlns="http://www.w3.org/2005/Atom" />' . PHP_EOL;
echo '<atom:link rel="hub" href="'.htmlspecialchars($hub).'" />' . PHP_EOL;
}
// add self
if (isset($this->self)) {
//echo $this->makeNode('link', '', array('rel'=>'self', 'href'=>$this->self, 'xmlns'=>'http://www.w3.org/2005/Atom'));
echo '<link rel="self" href="'.htmlspecialchars($this->self).'" xmlns="http://www.w3.org/2005/Atom" />' . PHP_EOL;
echo '<atom:link rel="self" href="'.htmlspecialchars($this->self).'" />' . PHP_EOL;
}
// add alternate
if (isset($this->alternate)) {
echo '<atom:link rel="alternate" title="'.htmlspecialchars($this->alternate['title']).'" href="'.htmlspecialchars($this->alternate['url']).'" />' . PHP_EOL;
}
// add related
if (isset($this->related)) {
echo '<atom:link rel="related" title="'.htmlspecialchars($this->related['title']).'" href="'.htmlspecialchars($this->related['url']).'" />' . PHP_EOL;
}
//Print Items of channel
foreach ($this->channels as $key => $value)
{

406
libraries/html5php/HTML5.php
View File

@ -1,220 +1,242 @@
<?php
/**
* The main HTML5 front end.
*/
use HTML5\Parser\StringInputStream;
use HTML5\Parser\FileInputStream;
use HTML5\Parser\Scanner;
use HTML5\Parser\Tokenizer;
use HTML5\Parser\DOMTreeBuilder;
use HTML5\Serializer\OutputRules;
use HTML5\Serializer\Traverser;
namespace Masterminds;
use Masterminds\HTML5\Parser\FileInputStream;
use Masterminds\HTML5\Parser\StringInputStream;
use Masterminds\HTML5\Parser\DOMTreeBuilder;
use Masterminds\HTML5\Parser\Scanner;
use Masterminds\HTML5\Parser\Tokenizer;
use Masterminds\HTML5\Serializer\OutputRules;
use Masterminds\HTML5\Serializer\Traverser;
/**
* This class offers convenience methods for parsing and serializing HTML5.
* It is roughly designed to mirror the \DOMDocument class that is
* It is roughly designed to mirror the \DOMDocument class that is
* provided with most versions of PHP.
*
* EXPERIMENTAL. This may change or be completely replaced.
*/
class HTML5 {
class HTML5
{
/**
* Global options for the parser and serializer.
* @var array
*/
public static $options = array(
/**
* Global options for the parser and serializer.
*
* @var array
*/
protected $options = array(
// If the serializer should encode all entities.
'encode_entities' => false
);
// If the serializer should encode all entities.
'encode_entities' => FALSE,
);
protected $errors = array();
/**
* Load and parse an HTML file.
*
* This will apply the HTML5 parser, which is tolerant of many
* varieties of HTML, including XHTML 1, HTML 4, and well-formed HTML
* 3. Note that in these cases, not all of the old data will be
* preserved. For example, XHTML's XML declaration will be removed.
*
* The rules governing parsing are set out in the HTML 5 spec.
*
* @param string $file
* The path to the file to parse. If this is a resource, it is
* assumed to be an open stream whose pointer is set to the first
* byte of input.
* @return \DOMDocument
* A DOM document. These object type is defined by the libxml
* library, and should have been included with your version of PHP.
*/
public static function load($file) {
// Handle the case where file is a resource.
if (is_resource($file)) {
// FIXME: We need a StreamInputStream class.
return static::loadHTML(stream_get_contents($file));
public function __construct(array $options = array())
{
$this->options = array_merge($this->options, $options);
}
$input = new FileInputStream($file);
return static::parse($input);
}
/**
* Parse a HTML Document from a string.
*
* Take a string of HTML 5 (or earlier) and parse it into a
* DOMDocument.
*
* @param string $string
* A html5 document as a string.
* @return \DOMDocument
* A DOM document. DOM is part of libxml, which is included with
* almost all distribtions of PHP.
*/
public static function loadHTML($string) {
$input = new StringInputStream($string);
return static::parse($input);
}
/**
* Convenience function to load an HTML file.
*
* This is here to provide backwards compatibility with the
* PHP DOM implementation. It simply calls load().
*
* @param string $file
* The path to the file to parse. If this is a resource, it is
* assumed to be an open stream whose pointer is set to the first
* byte of input.
*
* @return \DOMDocument
* A DOM document. These object type is defined by the libxml
* library, and should have been included with your version of PHP.
*/
public static function loadHTMLFile($file, $options = NULL) {
return static::load($file, $options);
}
/**
* Parse a HTML fragment from a string.
*
* @param string $string
* The html5 fragment as a string.
*
* @return \DOMDocumentFragment
* A DOM fragment. The DOM is part of libxml, which is included with
* almost all distributions of PHP.
*/
public static function loadHTMLFragment($string) {
$input = new StringInputStream($string);
return static::parseFragment($input);
}
/**
* Save a DOM into a given file as HTML5.
*
* @param mixed $dom
* The DOM to be serialized.
* @param string $file
* The filename to be written.
* @param array $options
* Configuration options when serializing the DOM. These include:
* - encode_entities: Text written to the output is escaped by default and not all
* entities are encoded. If this is set to TRUE all entities will be encoded.
* Defaults to FALSE.
*/
public static function save($dom, $file, $options = array()) {
$options = $options + static::options();
$close = TRUE;
if (is_resource($file)) {
$stream = $file;
$close = FALSE;
}
else {
$stream = fopen($file, 'w');
/**
* Get the default options.
*
* @return array The default options.
*/
public function getOptions()
{
return $this->options;
}
$rules = new OutputRules($stream, $options);
$trav = new Traverser($dom, $stream, $rules, $options);
$trav->walk();
/**
* Load and parse an HTML file.
*
* This will apply the HTML5 parser, which is tolerant of many
* varieties of HTML, including XHTML 1, HTML 4, and well-formed HTML
* 3. Note that in these cases, not all of the old data will be
* preserved. For example, XHTML's XML declaration will be removed.
*
* The rules governing parsing are set out in the HTML 5 spec.
*
* @param string $file
* The path to the file to parse. If this is a resource, it is
* assumed to be an open stream whose pointer is set to the first
* byte of input.
* @return \DOMDocument A DOM document. These object type is defined by the libxml
* library, and should have been included with your version of PHP.
*/
public function load($file)
{
// Handle the case where file is a resource.
if (is_resource($file)) {
// FIXME: We need a StreamInputStream class.
return $this->loadHTML(stream_get_contents($file));
}
if ($close) {
fclose($stream);
$input = new FileInputStream($file);
return $this->parse($input);
}
}
/**
* Convert a DOM into an HTML5 string.
*
* @param mixed $dom
* The DOM to be serialized.
* @param array $options
* Configuration options when serializing the DOM. These include:
* - encode_entities: Text written to the output is escaped by default and not all
* entities are encoded. If this is set to TRUE all entities will be encoded.
* Defaults to FALSE.
*
* @return string
* A HTML5 documented generated from the DOM.
*/
public static function saveHTML($dom, $options = array()) {
$stream = fopen('php://temp', 'w');
static::save($dom, $stream, $options);
return stream_get_contents($stream, -1, 0);
}
/**
* Parse a HTML Document from a string.
*
* Take a string of HTML 5 (or earlier) and parse it into a
* DOMDocument.
*
* @param string $string
* A html5 document as a string.
* @return \DOMDocument A DOM document. DOM is part of libxml, which is included with
* almost all distribtions of PHP.
*/
public function loadHTML($string)
{
$input = new StringInputStream($string);
/**
* Parse an input stream.
*
* Lower-level loading function. This requires an input stream instead
* of a string, file, or resource.
*/
public static function parse(\HTML5\Parser\InputStream $input) {
$events = new DOMTreeBuilder();
$scanner = new Scanner($input);
$parser = new Tokenizer($scanner, $events);
return $this->parse($input);
}
$parser->parse();
/**
* Convenience function to load an HTML file.
*
* This is here to provide backwards compatibility with the
* PHP DOM implementation. It simply calls load().
*
* @param string $file
* The path to the file to parse. If this is a resource, it is
* assumed to be an open stream whose pointer is set to the first
* byte of input.
*
* @return \DOMDocument A DOM document. These object type is defined by the libxml
* library, and should have been included with your version of PHP.
*/
public function loadHTMLFile($file)
{
return $this->load($file);
}
return $events->document();
}
/**
* Parse a HTML fragment from a string.
*
* @param string $string
* The html5 fragment as a string.
*
* @return \DOMDocumentFragment A DOM fragment. The DOM is part of libxml, which is included with
* almost all distributions of PHP.
*/
public function loadHTMLFragment($string)
{
$input = new StringInputStream($string);
/**
* Parse an input stream where the stream is a fragment.
*
* Lower-level loading function. This requires an input stream instead
* of a string, file, or resource.
*/
public static function parseFragment(\HTML5\Parser\InputStream $input) {
$events = new DOMTreeBuilder(TRUE);
$scanner = new Scanner($input);
$parser = new Tokenizer($scanner, $events);
return $this->parseFragment($input);
}
$parser->parse();
/**
* Return all errors encountered into parsing phase
*
* @return array
*/
public function getErrors()
{
return $this->errors;
}
return $events->fragment();
}
/**
* Return true it some errors were encountered into parsing phase
*
* @return bool
*/
public function hasErrors()
{
return count($this->errors) > 0;
}
/**
* Get the default options.
*
* @return array
* The default options.
*/
public static function options() {
return static::$options;
}
/**
* Parse an input stream.
*
* Lower-level loading function. This requires an input stream instead
* of a string, file, or resource.
*/
public function parse(\Masterminds\HTML5\Parser\InputStream $input)
{
$this->errors = array();
$events = new DOMTreeBuilder(false, $this->options);
$scanner = new Scanner($input);
$parser = new Tokenizer($scanner, $events);
/**
* Set a default option.
*
* @param string $name
* The option name.
* @param mixed $value
* The option value.
*/
public static function setOption($name, $value) {
static::$options[$name] = $value;
}
$parser->parse();
$this->errors = $events->getErrors();
return $events->document();
}
/**
* Parse an input stream where the stream is a fragment.
*
* Lower-level loading function. This requires an input stream instead
* of a string, file, or resource.
*/
public function parseFragment(\Masterminds\HTML5\Parser\InputStream $input)
{
$events = new DOMTreeBuilder(true, $this->options);
$scanner = new Scanner($input);
$parser = new Tokenizer($scanner, $events);
$parser->parse();
$this->errors = $events->getErrors();
return $events->fragment();
}
/**
* Save a DOM into a given file as HTML5.
*
* @param mixed $dom
* The DOM to be serialized.
* @param string $file
* The filename to be written.
* @param array $options
* Configuration options when serializing the DOM. These include:
* - encode_entities: Text written to the output is escaped by default and not all
* entities are encoded. If this is set to true all entities will be encoded.
* Defaults to false.
*/
public function save($dom, $file, $options = array())
{
$close = true;
if (is_resource($file)) {
$stream = $file;
$close = false;
} else {
$stream = fopen($file, 'w');
}
$options = array_merge($this->getOptions(), $options);
$rules = new OutputRules($stream, $options);
$trav = new Traverser($dom, $stream, $rules, $options);
$trav->walk();
if ($close) {
fclose($stream);
}
}
/**
* Convert a DOM into an HTML5 string.
*
* @param mixed $dom
* The DOM to be serialized.
* @param array $options
* Configuration options when serializing the DOM. These include:
* - encode_entities: Text written to the output is escaped by default and not all
* entities are encoded. If this is set to true all entities will be encoded.
* Defaults to false.
*
* @return string A HTML5 documented generated from the DOM.
*/
public function saveHTML($dom, $options = array())
{
$stream = fopen('php://temp', 'w');
$this->save($dom, $stream, array_merge($this->getOptions(), $options));
return stream_get_contents($stream, - 1, 0);
}
}

1171
libraries/html5php/HTML5/Elements.php
View File

File diff suppressed because it is too large Load Diff

4460
libraries/html5php/HTML5/Entities.php
View File

File diff suppressed because it is too large Load Diff

5
libraries/html5php/HTML5/Exception.php
View File

@ -1,8 +1,9 @@
<?php
namespace HTML5;
namespace Masterminds\HTML5;
/**
* The base exception for the HTML5 project.
*/
class Exception extends \Exception {
class Exception extends \Exception
{
}

56
libraries/html5php/HTML5/InstructionProcessor.php
View File

@ -2,42 +2,42 @@
/**
* A handler for processor instructions.
*/
namespace HTML5;
namespace Masterminds\HTML5;
/**
* Provide an processor to handle embedded instructions.
*
* XML defines a mechanism for inserting instructions (like PHP) into a
* document. These are called "Processor Instructions." The HTML5 parser
* provides an opportunity to handle these processor instructions during
* the tree-building phase (before the DOM is constructed), which makes
* XML defines a mechanism for inserting instructions (like PHP) into a
* document. These are called "Processor Instructions." The HTML5 parser
* provides an opportunity to handle these processor instructions during
* the tree-building phase (before the DOM is constructed), which makes
* it possible to alter the document as it is being created.
*
* One could, for example, use this mechanism to execute well-formed PHP
* code embedded inside of an HTML5 document.
*/
interface InstructionProcessor {
interface InstructionProcessor
{
/**
* Process an individual processing instruction.
*
* The process() function is responsible for doing the following:
* - Determining whether $name is an instruction type it can handle.
* - Determining what to do with the data passed in.
* - Making any subsequent modifications to the DOM by modifying the
* DOMElement or its attached DOM tree.
*
* @param DOMElement $element
* The parent element for the current processing instruction.
* @param string $name
* The instruction's name. E.g. `&lt;?php` has the name `php`.
* @param string $data
* All of the data between the opening and closing PI marks.
* @return DOMElement
* The element that should be considered "Current". This may just be
* the element passed in, but if the processor added more elements,
* it may choose to reset the current element to one of the elements
* it created. (When in doubt, return the element passed in.)
*/
public function process(\DOMElement $element, $name, $data);
/**
* Process an individual processing instruction.
*
* The process() function is responsible for doing the following:
* - Determining whether $name is an instruction type it can handle.
* - Determining what to do with the data passed in.
* - Making any subsequent modifications to the DOM by modifying the
* DOMElement or its attached DOM tree.
*
* @param DOMElement $element
* The parent element for the current processing instruction.
* @param string $name
* The instruction's name. E.g. `&lt;?php` has the name `php`.
* @param string $data
* All of the data between the opening and closing PI marks.
* @return DOMElement The element that should be considered "Current". This may just be
* the element passed in, but if the processor added more elements,
* it may choose to reset the current element to one of the elements
* it created. (When in doubt, return the element passed in.)
*/
public function process(\DOMElement $element, $name, $data);
}

91
libraries/html5php/HTML5/Parser/CharacterReference.php
View File

@ -1,56 +1,63 @@
<?php
namespace HTML5\Parser;
namespace Masterminds\HTML5\Parser;
use \HTML5\Entities;
use Masterminds\HTML5\Entities;
/**
* Manage entity references.
*
* This is a simple resolver for HTML5 character reference entitites.
* See \HTML5\Entities for the list of supported entities.
* See \Masterminds\HTML5\Entities for the list of supported entities.
*/
class CharacterReference {
class CharacterReference
{
protected static $numeric_mask = array(0x0, 0x2FFFF, 0, 0xFFFF);
protected static $numeric_mask = array(
0x0,
0x2FFFF,
0,
0xFFFF
);
/**
* Given a name (e.g. 'amp'), lookup the UTF-8 character ('&')
*
* @param string $name
* The name to look up.
* @return string
* The character sequence. In UTF-8 this may be more than one byte.
*/
public static function lookupName($name) {
// Do we really want to return NULL here? or FFFD
return isset(Entities::$byName[$name]) ? Entities::$byName[$name] : NULL;
}
/**
* Given a name (e.g.
* 'amp'), lookup the UTF-8 character ('&')
*
* @param string $name
* The name to look up.
* @return string The character sequence. In UTF-8 this may be more than one byte.
*/
public static function lookupName($name)
{
// Do we really want to return NULL here? or FFFD
return isset(Entities::$byName[$name]) ? Entities::$byName[$name] : null;
}
/**
* Given a Unicode codepoint, return the UTF-8 character.
*
* (NOT USED ANYWHERE)
*/
/*
public static function lookupCode($codePoint) {
return 'POINT';
}
*/
/**
* Given a Unicode codepoint, return the UTF-8 character.
*
* (NOT USED ANYWHERE)
*/
/*
* public static function lookupCode($codePoint) { return 'POINT'; }
*/
/**
* Given a decimal number, return the UTF-8 character.
*/
public static function lookupDecimal($int) {
$entity = '&#' . $int . ';';
// UNTESTED: This may fail on some planes. Couldn't find full documentation
// on the value of the mask array.
return mb_decode_numericentity($entity, static::$numeric_mask, 'utf-8');
}
/**
* Given a decimal number, return the UTF-8 character.
*/
public static function lookupDecimal($int)
{
$entity = '&#' . $int . ';';
// UNTESTED: This may fail on some planes. Couldn't find full documentation
// on the value of the mask array.
return mb_decode_numericentity($entity, static::$numeric_mask, 'utf-8');
}
/**
* Given a hexidecimal number, return the UTF-8 character.
*/
public static function lookupHex($hexdec) {
return static::lookupDecimal(hexdec($hexdec));
}
/**
* Given a hexidecimal number, return the UTF-8 character.
*/
public static function lookupHex($hexdec)
{
return static::lookupDecimal(hexdec($hexdec));
}
}

1063
libraries/html5php/HTML5/Parser/DOMTreeBuilder.php
View File

File diff suppressed because it is too large Load Diff

203
libraries/html5php/HTML5/Parser/EventHandler.php
View File

@ -1,111 +1,122 @@
<?php
namespace HTML5\Parser;
namespace Masterminds\HTML5\Parser;
/**
* Standard events for HTML5.
*
* This is roughly analogous to a SAX2 or expat-style interface.
* However, it is tuned specifically for HTML5, according to section 8
* This is roughly analogous to a SAX2 or expat-style interface.
* However, it is tuned specifically for HTML5, according to section 8
* of the HTML5 specification.
*
* An event handler receives parser events. For a concrete
* An event handler receives parser events. For a concrete
* implementation, see DOMTreeBuilder.
*
* Quirks support in the parser is limited to close-in syntax (malformed
* tags or attributes). Higher order syntax and semantic issues with a
* document (e.g. mismatched tags, illegal nesting, etc.) are the
* Quirks support in the parser is limited to close-in syntax (malformed
* tags or attributes). Higher order syntax and semantic issues with a
* document (e.g. mismatched tags, illegal nesting, etc.) are the
* responsibility of the event handler implementation.
*
* See HTML5 spec section 8.2.4
*/
interface EventHandler {
const DOCTYPE_NONE = 0;
const DOCTYPE_PUBLIC = 1;
const DOCTYPE_SYSTEM = 2;
/**
* A doctype declaration.
*
* @param string $name
* The name of the root element.
* @param int $idType
* One of DOCTYPE_NONE, DOCTYPE_PUBLIC, or DOCTYPE_SYSTEM.
* @param string $id
* The identifier. For DOCTYPE_PUBLIC, this is the public ID. If DOCTYPE_SYSTEM,
* then this is a system ID.
* @param boolean $quirks
* Indicates whether the builder should enter quirks mode.
*/
public function doctype($name, $idType = 0, $id = NULL, $quirks = FALSE);
/**
* A start tag.
*
* IMPORTANT: The parser watches the return value of this event. If this returns
* an integer, the parser will switch TEXTMODE patters according to the int.
*
* This is how the Tree Builder can tell the Tokenizer when a certain tag should
* cause the parser to go into RAW text mode.
*
* The HTML5 standard requires that the builder is the one that initiates this
* step, and this is the only way short of a circular reference that we can
* do that.
*
* Example: if a startTag even for a `script` name is fired, and the startTag()
* implementation returns Tokenizer::TEXTMODE_RAW, then the tokenizer will
* switch into RAW text mode and consume data until it reaches a closing
* `script` tag.
*
* The textmode is automatically reset to Tokenizer::TEXTMODE_NORMAL when the
* closing tag is encounter. **This behavior may change.**
*
* @param string $name
* The tag name.
* @param array $attributes
* An array with all of the tag's attributes.
* @param boolean $selfClosing
* An indicator of whether or not this tag is self-closing (<foo/>)
* @return numeric
* One of the Tokenizer::TEXTMODE_* constants.
*/
public function startTag($name, $attributes = array(), $selfClosing = FALSE);
/**
* An end-tag.
*/
public function endTag($name);
/**
* A comment section (unparsed character data).
*/
public function comment($cdata);
/**
* A unit of parsed character data.
*
* Entities in this text are *already decoded*.
*/
public function text($cdata);
/**
* Indicates that the document has been entirely processed.
*/
public function eof();
/**
* Emitted when the parser encounters an error condition.
*/
public function parseError($msg, $line, $col);
interface EventHandler
{
/**
* A CDATA section.
*
* @param string $data
* The unparsed character data.
*/
public function cdata($data);
/**
* This is a holdover from the XML spec.
*
* While user agents don't get PIs, server-side does.
*
* @param string $name
* The name of the processor (e.g. 'php').
* @param string $data
* The unparsed data.
*/
public function processingInstruction($name, $data = NULL);
const DOCTYPE_NONE = 0;
const DOCTYPE_PUBLIC = 1;
const DOCTYPE_SYSTEM = 2;
/**
* A doctype declaration.
*
* @param string $name
* The name of the root element.
* @param int $idType
* One of DOCTYPE_NONE, DOCTYPE_PUBLIC, or DOCTYPE_SYSTEM.
* @param string $id
* The identifier. For DOCTYPE_PUBLIC, this is the public ID. If DOCTYPE_SYSTEM,
* then this is a system ID.
* @param boolean $quirks
* Indicates whether the builder should enter quirks mode.
*/
public function doctype($name, $idType = 0, $id = null, $quirks = false);
/**
* A start tag.
*
* IMPORTANT: The parser watches the return value of this event. If this returns
* an integer, the parser will switch TEXTMODE patters according to the int.
*
* This is how the Tree Builder can tell the Tokenizer when a certain tag should
* cause the parser to go into RAW text mode.
*
* The HTML5 standard requires that the builder is the one that initiates this
* step, and this is the only way short of a circular reference that we can
* do that.
*
* Example: if a startTag even for a `script` name is fired, and the startTag()
* implementation returns Tokenizer::TEXTMODE_RAW, then the tokenizer will
* switch into RAW text mode and consume data until it reaches a closing
* `script` tag.
*
* The textmode is automatically reset to Tokenizer::TEXTMODE_NORMAL when the
* closing tag is encounter. **This behavior may change.**
*
* @param string $name
* The tag name.
* @param array $attributes
* An array with all of the tag's attributes.
* @param boolean $selfClosing
* An indicator of whether or not this tag is self-closing (<foo/>)
* @return numeric One of the Tokenizer::TEXTMODE_* constants.
*/
public function startTag($name, $attributes = array(), $selfClosing = false);
/**
* An end-tag.
*/
public function endTag($name);
/**
* A comment section (unparsed character data).
*/
public function comment($cdata);
/**
* A unit of parsed character data.
*
* Entities in this text are *already decoded*.
*/
public function text($cdata);
/**
* Indicates that the document has been entirely processed.
*/
public function eof();
/**
* Emitted when the parser encounters an error condition.
*/
public function parseError($msg, $line, $col);
/**
* A CDATA section.
*
* @param string $data
* The unparsed character data.
*/
public function cdata($data);
/**
* This is a holdover from the XML spec.
*
* While user agents don't get PIs, server-side does.
*
* @param string $name
* The name of the processor (e.g. 'php').
* @param string $data
* The unparsed data.
*/
public function processingInstruction($name, $data = null);
}

47
libraries/html5php/HTML5/Parser/FileInputStream.php
View File

@ -1,35 +1,32 @@
<?php
namespace HTML5\Parser;
namespace Masterminds\HTML5\Parser;
/**
* The FileInputStream loads a file to be parsed.
*
* So right now we read files into strings and then process the
* string. We chose to do this largely for the sake of expediency of
* development, and also because we could optimize toward processing
* arbitrarily large chunks of the input. But in the future, we'd
* really like to rewrite this class to efficiently handle lower level
* stream reads (and thus efficiently handle large documents).
*
* @todo A buffered input stream would be useful.
*/
class FileInputStream extends StringInputStream implements InputStream {
class FileInputStream extends StringInputStream implements InputStream
{
/*
* So right now we read files into strings and then process the
* string. We chose to do this largely for the sake of expediency of
* development, and also because we could optimize toward processing
* arbitrarily large chunks of the input. But in the future, we'd
* really like to rewrite this class to efficiently handle lower level
* stream reads (and thus efficiently handle large documents).
*/
/**
* Load a file input stream.
*
* @param string $data
* The file or url path to load.
*/
function __construct($data, $encoding = 'UTF-8', $debug = '') {
// Get the contents of the file.
$content = file_get_contents($data);
parent::__construct($content, $encoding, $debug);
}
/**
* Load a file input stream.
*
* @param string $data
* The file or url path to load.
*/
public function __construct($data, $encoding = 'UTF-8', $debug = '')
{
// Get the contents of the file.
$content = file_get_contents($data);
parent::__construct($content, $encoding, $debug);
}
}

141
libraries/html5php/HTML5/Parser/InputStream.php
View File

@ -1,88 +1,87 @@
<?php
namespace HTML5\Parser;
namespace Masterminds\HTML5\Parser;
/**
* Interface for stream readers.
*
* The parser only reads from streams. Various input sources can write
* The parser only reads from streams. Various input sources can write
* an adapater to this InputStream.
*
* Currently provided InputStream implementations include
* Currently provided InputStream implementations include
* FileInputStream and StringInputStream.
*/
interface InputStream extends \Iterator {
interface InputStream extends \Iterator
{
/**
* Returns the current line that is being consumed.
*
* TODO: Move this to the scanner.
*/
public function currentLine();
/**
* Returns the current line that is being consumed.
*
* TODO: Move this to the scanner.
*/
public function currentLine();
/**
* Returns the current column of the current line that the tokenizer is at.
*
* Newlines are column 0. The first char after a newline is column 1.
*
* @TODO Move this to the scanner.
*
* @return int
* The column number.
*/
public function columnOffset();
/**
* Returns the current column of the current line that the tokenizer is at.
*
* Newlines are column 0. The first char after a newline is column 1.
*
* @TODO Move this to the scanner.
*
* @return int The column number.
*/
public function columnOffset();
/**
* Get all characters until EOF.
*
* This consumes characters until the EOF.
*/
public function remainingChars();
/**
* Get all characters until EOF.
*
* This consumes characters until the EOF.
*/
public function remainingChars();
/**
* Read to a particular match (or until $max bytes are consumed).
*
* This operates on byte sequences, not characters.
*
* Matches as far as possible until we reach a certain set of bytes
* and returns the matched substring.
*
* @see strcspn
* @param string $bytes
* Bytes to match.
* @param int $max
* Maximum number of bytes to scan.
* @return mixed
* Index or FALSE if no match is found. You should use strong
* equality when checking the result, since index could be 0.
*/
public function charsUntil($bytes, $max = null);
/**
* Read to a particular match (or until $max bytes are consumed).
*
* This operates on byte sequences, not characters.
*
* Matches as far as possible until we reach a certain set of bytes
* and returns the matched substring.
*
* @see strcspn
* @param string $bytes
* Bytes to match.
* @param int $max
* Maximum number of bytes to scan.
* @return mixed Index or false if no match is found. You should use strong
* equality when checking the result, since index could be 0.
*/
public function charsUntil($bytes, $max = null);
/**
* Returns the string so long as $bytes matches.
*
* Matches as far as possible with a certain set of bytes
* and returns the matched substring.
*
* @see strspn
* @param string $bytes
* A mask of bytes to match. If ANY byte in this mask matches the
* current char, the pointer advances and the char is part of the
* substring.
* @param int $max
* The max number of chars to read.
*/
public function charsWhile($bytes, $max = null);
/**
* Returns the string so long as $bytes matches.
*
* Matches as far as possible with a certain set of bytes
* and returns the matched substring.
*
* @see strspn
* @param string $bytes
* A mask of bytes to match. If ANY byte in this mask matches the
* current char, the pointer advances and the char is part of the
* substring.
* @param int $max
* The max number of chars to read.
*/
public function charsWhile($bytes, $max = null);
/**
* Unconsume one character.
*
* @param int $howMany
* The number of characters to move the pointer back.
*/
public function unconsume($howMany = 1);
/**
* Unconsume one character.
*
* @param int $howMany
* The number of characters to move the pointer back.
*/
public function unconsume($howMany = 1);
/**
* Retrieve the next character without advancing the pointer.
*/
public function peek();
/**
* Retrieve the next character without advancing the pointer.
*/
public function peek();
}

5
libraries/html5php/HTML5/Parser/ParseError.php
View File

@ -1,8 +1,9 @@
<?php
namespace HTML5\Parser;
namespace Masterminds\HTML5\Parser;
/**
* Emit when the parser has an error.
*/
class ParseError extends \Exception {
class ParseError extends \Exception
{
}

371
libraries/html5php/HTML5/Parser/Scanner.php
View File

@ -1,207 +1,222 @@
<?php
namespace HTML5\Parser;
namespace Masterminds\HTML5\Parser;
/**
* The scanner.
*
* This scans over an input stream.
*/
class Scanner {
const CHARS_HEX = 'abcdefABCDEF01234567890';
const CHARS_ALNUM = 'abcdefAghijklmnopqrstuvwxyABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890';
const CHARS_ALPHA = 'abcdefAghijklmnopqrstuvwxyABCDEFGHIJKLMNOPQRSTUVWXYZ';
class Scanner
{
protected $is;
const CHARS_HEX = 'abcdefABCDEF01234567890';
// Flipping this to TRUE will give minisculely more debugging info.
public $debug = FALSE;
const CHARS_ALNUM = 'abcdefAghijklmnopqrstuvwxyABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890';
/**
* Create a new Scanner.
*
* @param \HTML5\Parser\InputStream $input
* An InputStream to be scanned.
*/
public function __construct($input) {
$this->is = $input;
}
const CHARS_ALPHA = 'abcdefAghijklmnopqrstuvwxyABCDEFGHIJKLMNOPQRSTUVWXYZ';
/**
* Get the current position.
*
* @return int
* The current intiger byte position.
*/
public function position() {
return $this->is->key();
}
protected $is;
/**
* Take a peek at the next character in the data.
*
* @return string
* The next character.
*/
public function peek() {
return $this->is->peek();
}
// Flipping this to true will give minisculely more debugging info.
public $debug = false;
/**
* Get the next character.
*
* Note: This advances the pointer.
*
* @return string
* The next character.
*/
public function next() {
$this->is->next();
if ($this->is->valid()) {
if ($this->debug) fprintf(STDOUT, "> %s\n", $this->is->current());
return $this->is->current();
/**
* Create a new Scanner.
*
* @param \Masterminds\HTML5\Parser\InputStream $input
* An InputStream to be scanned.
*/
public function __construct($input)
{
$this->is = $input;
}
return FALSE;
}
/**
* Get the current character.
*
* Note, this does not advance the pointer.
*
* @return string
* The current character.
*/
public function current() {
if ($this->is->valid()) {
return $this->is->current();
/**
* Get the current position.
*
* @return int The current intiger byte position.
*/
public function position()
{
return $this->is->key();
}
return FALSE;
}
/**
* Silently consume N chars.
*/
public function consume($count = 1) {
for ($i = 0; $i < $count; ++$i) {
$this->next();
/**
* Take a peek at the next character in the data.
*
* @return string The next character.
*/
public function peek()
{
return $this->is->peek();
}
}
/**
* Unconsume some of the data. This moves the data pointer backwards.
*
* @param int $howMany
* The number of characters to move the pointer back.
*/
public function unconsume($howMany = 1) {
$this->is->unconsume($howMany);
}
/**
* Get the next character.
*
* Note: This advances the pointer.
*
* @return string The next character.
*/
public function next()
{
$this->is->next();
if ($this->is->valid()) {
if ($this->debug)
fprintf(STDOUT, "> %s\n", $this->is->current());
return $this->is->current();
}
/**
* Get the next group of that contains hex characters.
*
* Note, along with getting the characters the pointer in the data will be
* moved as well.
*
* @return string
* The next group that is hex characters.
*/
public function getHex() {
return $this->is->charsWhile(static::CHARS_HEX);
}
return false;
}
/**
* Get the next group of characters that are ASCII Alpha characters.
*
* Note, along with getting the characters the pointer in the data will be
* moved as well.
*
* @return string
* The next group of ASCII alpha characters.
*/
public function getAsciiAlpha() {
return $this->is->charsWhile(static::CHARS_ALPHA);
}
/**
* Get the current character.
*
* Note, this does not advance the pointer.
*
* @return string The current character.
*/
public function current()
{
if ($this->is->valid()) {
return $this->is->current();
}
/**
* Get the next group of characters that are ASCII Alpha characters and numbers.
*
* Note, along with getting the characters the pointer in the data will be
* moved as well.
*
* @return string
* The next group of ASCII alpha characters and numbers.
*/
public function getAsciiAlphaNum() {
return $this->is->charsWhile(static::CHARS_ALNUM);
}
return false;
}
/**
* Get the next group of numbers.
*
* Note, along with getting the characters the pointer in the data will be
* moved as well.
*
* @return string
* The next group of numbers.
*/
public function getNumeric() {
return $this->is->charsWhile('0123456789');
}
/**
* Silently consume N chars.
*/
public function consume($count = 1)
{
for ($i = 0; $i < $count; ++ $i) {
$this->next();
}
}
/**
* Consume whitespace.
*
* Whitespace in HTML5 is: formfeed, tab, newline, space.
*/
public function whitespace() {
return $this->is->charsWhile("\n\t\f ");
}
/**
* Unconsume some of the data.
* This moves the data pointer backwards.
*
* @param int $howMany
* The number of characters to move the pointer back.
*/
public function unconsume($howMany = 1)
{
$this->is->unconsume($howMany);
}
/**
* Returns the current line that is being consumed.
*
* @return int
* The current line number.
*/
public function currentLine() {
return $this->is->currentLine();
}
/**
* Get the next group of that contains hex characters.
*
* Note, along with getting the characters the pointer in the data will be
* moved as well.
*
* @return string The next group that is hex characters.
*/
public function getHex()
{
return $this->is->charsWhile(static::CHARS_HEX);
}
/**
* Read chars until something in the mask is encountered.
*/
public function charsUntil($mask) {
return $this->is->charsUntil($mask);
}
/**
* Read chars as long as the mask matches.
*/
public function charsWhile($mask) {
return $this->is->charsWhile($mask);
}
/**
* Get the next group of characters that are ASCII Alpha characters.
*
* Note, along with getting the characters the pointer in the data will be
* moved as well.
*
* @return string The next group of ASCII alpha characters.
*/
public function getAsciiAlpha()
{
return $this->is->charsWhile(static::CHARS_ALPHA);
}
/**
* Returns the current column of the current line that the tokenizer is at.
*
* Newlines are column 0. The first char after a newline is column 1.
*
* @return int
* The column number.
*/
public function columnOffset() {
return $this->is->columnOffset();
}
/**
* Get the next group of characters that are ASCII Alpha characters and numbers.
*
* Note, along with getting the characters the pointer in the data will be
* moved as well.
*
* @return string The next group of ASCII alpha characters and numbers.
*/
public function getAsciiAlphaNum()
{
return $this->is->charsWhile(static::CHARS_ALNUM);
}
/**
* Get all characters until EOF.
*
* This consumes characters until the EOF.
*
* @return int
* The number of characters remaining.
*/
public function remainingChars() {
return $this->is->remainingChars();
}
/**
* Get the next group of numbers.
*
* Note, along with getting the characters the pointer in the data will be
* moved as well.
*
* @return string The next group of numbers.
*/
public function getNumeric()
{
return $this->is->charsWhile('0123456789');
}
/**
* Consume whitespace.
*
* Whitespace in HTML5 is: formfeed, tab, newline, space.
*/
public function whitespace()
{
return $this->is->charsWhile("\n\t\f ");
}
/**
* Returns the current line that is being consumed.
*
* @return int The current line number.
*/
public function currentLine()
{
return $this->is->currentLine();
}
/**
* Read chars until something in the mask is encountered.
*/
public function charsUntil($mask)
{
return $this->is->charsUntil($mask);
}
/**
* Read chars as long as the mask matches.
*/
public function charsWhile($mask)
{
return $this->is->charsWhile($mask);
}
/**
* Returns the current column of the current line that the tokenizer is at.
*
* Newlines are column 0. The first char after a newline is column 1.
*
* @return int The column number.
*/
public function columnOffset()
{
return $this->is->columnOffset();
}
/**
* Get all characters until EOF.
*
* This consumes characters until the EOF.
*
* @return int The number of characters remaining.
*/
public function remainingChars()
{
return $this->is->remainingChars();
}
}

524
libraries/html5php/HTML5/Parser/StringInputStream.php
View File

@ -2,17 +2,17 @@
/**
* Loads a string to be parsed.
*/
namespace HTML5\Parser;
namespace Masterminds\HTML5\Parser;
/*
*
* Based on code from html5lib:
* Based on code from html5lib:
Copyright 2009 Geoffrey Sneddon <http://gsnedders.com/>
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
@ -33,283 +33,299 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Some conventions:
// - /* */ indicates verbatim text from the HTML 5 specification
// MPB: Not sure which version of the spec. Moving from HTML5lib to
// MPB: Not sure which version of the spec. Moving from HTML5lib to
// HTML5-PHP, I have been using this version:
// http://www.w3.org/TR/2012/CR-html5-20121217/Overview.html#contents
//
// - // indicates regular comments
class StringInputStream implements InputStream {
/**
* The string data we're parsing.
*/
private $data;
class StringInputStream implements InputStream
{
/**
* The current integer byte position we are in $data
*/
private $char;
/**
* The string data we're parsing.
*/
private $data;
/**
* Length of $data; when $char === $data, we are at the end-of-file.
*/
private $EOF;
/**
* The current integer byte position we are in $data
*/
private $char;
/**
* Parse errors.
*/
public $errors = array();
/**
* Length of $data; when $char === $data, we are at the end-of-file.
*/
private $EOF;
/**
* Create a new InputStream wrapper.
*
* @param $data Data to parse
*/
public function __construct($data, $encoding = 'UTF-8', $debug = '') {
/**
* Parse errors.
*/
public $errors = array();
$data = UTF8Utils::convertToUTF8($data, $encoding);
if ($debug) fprintf(STDOUT, $debug, $data, strlen($data));
/**
* Create a new InputStream wrapper.
*
* @param $data Data
* to parse
*/
public function __construct($data, $encoding = 'UTF-8', $debug = '')
{
$data = UTF8Utils::convertToUTF8($data, $encoding);
if ($debug)
fprintf(STDOUT, $debug, $data, strlen($data));
// There is good reason to question whether it makes sense to
// do this here, since most of these checks are done during
// parsing, and since this check doesn't actually *do* anything.
$this->errors = UTF8Utils::checkForIllegalCodepoints($data);
//if (!empty($e)) {
// throw new ParseError("UTF-8 encoding issues: " . implode(', ', $e));
//}
// There is good reason to question whether it makes sense to
// do this here, since most of these checks are done during
// parsing, and since this check doesn't actually *do* anything.
$this->errors = UTF8Utils::checkForIllegalCodepoints($data);
// if (!empty($e)) {
// throw new ParseError("UTF-8 encoding issues: " . implode(', ', $e));
// }
$data = $this->replaceLinefeeds($data);
$data = $this->replaceLinefeeds($data);
$this->data = $data;
$this->char = 0;
$this->EOF = strlen($data);
}
/**
* Replace linefeed characters according to the spec.
*/
protected function replaceLinefeeds($data) {
/* U+000D CARRIAGE RETURN (CR) characters and U+000A LINE FEED
(LF) characters are treated specially. Any CR characters
that are followed by LF characters must be removed, and any
CR characters not followed by LF characters must be converted
to LF characters. Thus, newlines in HTML DOMs are represented
by LF characters, and there are never any CR characters in the
input to the tokenization stage. */
$crlfTable = array(
"\0" => "\xEF\xBF\xBD",
"\r\n" => "\n",
"\r" => "\n",
);
return strtr($data, $crlfTable);
}
/**
* Returns the current line that the tokenizer is at.
*/
public function currentLine() {
if (empty($this->EOF) || $this->char == 0) {
return 1;
}
// Add one to $this->char because we want the number for the next
// byte to be processed.
return substr_count($this->data, "\n", 0, min($this->char, $this->EOF)) + 1;
}
/**
* @deprecated
*/
public function getCurrentLine() {
return currentLine();
}
/**
* Returns the current column of the current line that the tokenizer is at.
*
* Newlines are column 0. The first char after a newline is column 1.
*
* @return int
* The column number.
*/
public function columnOffset() {
// Short circuit for the first char.
if ($this->char == 0) {
return 0;
}
// strrpos is weird, and the offset needs to be negative for what we
// want (i.e., the last \n before $this->char). This needs to not have
// one (to make it point to the next character, the one we want the
// position of) added to it because strrpos's behaviour includes the
// final offset byte.
$backwardFrom = $this->char - 1 - strlen($this->data);
$lastLine = strrpos($this->data, "\n", $backwardFrom);
// However, for here we want the length up until the next byte to be
// processed, so add one to the current byte ($this->char).
if ($lastLine !== FALSE) {
$findLengthOf = substr($this->data, $lastLine + 1, $this->char - 1 - $lastLine);
}
else {
// After a newline.
$findLengthOf = substr($this->data, 0, $this->char);
$this->data = $data;
$this->char = 0;
$this->EOF = strlen($data);
}
return UTF8Utils::countChars($findLengthOf);
}
/**
* Replace linefeed characters according to the spec.
*/
protected function replaceLinefeeds($data)
{
/*
* U+000D CARRIAGE RETURN (CR) characters and U+000A LINE FEED (LF) characters are treated specially. Any CR characters that are followed by LF characters must be removed, and any CR characters not followed by LF characters must be converted to LF characters. Thus, newlines in HTML DOMs are represented by LF characters, and there are never any CR characters in the input to the tokenization stage.
*/
$crlfTable = array(
"\0" => "\xEF\xBF\xBD",
"\r\n" => "\n",
"\r" => "\n"
);
/**
* @deprecated
*/
public function getColumnOffset() {
return $this->columnOffset();
}
/**
* Get the current character.
*
* @return string
* The current character.
*/
public function current() {
return $this->data[$this->char];
}
/**
* Advance the pointer. This is part of the Iterator interface.
*/
public function next() {
$this->char++;
}
/**
* Rewind to the start of the string.
*/
public function rewind() {
$this->char = 0;
}
/**
* Is the current pointer location valid.
*
* @return bool
* Is the current pointer location valid.
*/
public function valid() {
if ($this->char < $this->EOF) {
return TRUE;
return strtr($data, $crlfTable);
}
return FALSE;
}
/**
* Get all characters until EOF.
*
* This reads to the end of the file, and sets the read marker at the
* end of the file.
*
* @note This performs bounds checking
*
* @return string
* Returns the remaining text. If called when the InputStream is
* already exhausted, it returns an empty string.
*/
public function remainingChars() {
if ($this->char < $this->EOF) {
$data = substr($this->data, $this->char);
$this->char = $this->EOF;
return $data;
}
return '';//FALSE;
}
/**
* Read to a particular match (or until $max bytes are consumed).
*
* This operates on byte sequences, not characters.
*
* Matches as far as possible until we reach a certain set of bytes
* and returns the matched substring.
*
* @param string $bytes
* Bytes to match.
* @param int $max
* Maximum number of bytes to scan.
* @return mixed
* Index or FALSE if no match is found. You should use strong
* equality when checking the result, since index could be 0.
*/
public function charsUntil($bytes, $max = null) {
if ($this->char >= $this->EOF) {
return FALSE;
/**
* Returns the current line that the tokenizer is at.
*/
public function currentLine()
{
if (empty($this->EOF) || $this->char == 0) {
return 1;
}
// Add one to $this->char because we want the number for the next
// byte to be processed.
return substr_count($this->data, "\n", 0, min($this->char, $this->EOF)) + 1;
}
if ($max === 0 || $max) {
$len = strcspn($this->data, $bytes, $this->char, $max);
}
else {
$len = strcspn($this->data, $bytes, $this->char);
/**
*
* @deprecated
*
*/
public function getCurrentLine()
{
return currentLine();
}
$string = (string) substr($this->data, $this->char, $len);
$this->char += $len;
return $string;
}
/**
* Returns the current column of the current line that the tokenizer is at.
*
* Newlines are column 0. The first char after a newline is column 1.
*
* @return int The column number.
*/
public function columnOffset()
{
// Short circuit for the first char.
if ($this->char == 0) {
return 0;
}
// strrpos is weird, and the offset needs to be negative for what we
// want (i.e., the last \n before $this->char). This needs to not have
// one (to make it point to the next character, the one we want the
// position of) added to it because strrpos's behaviour includes the
// final offset byte.
$backwardFrom = $this->char - 1 - strlen($this->data);
$lastLine = strrpos($this->data, "\n", $backwardFrom);
/**
* Returns the string so long as $bytes matches.
*
* Matches as far as possible with a certain set of bytes
* and returns the matched substring.
*
* @param string $bytes
* A mask of bytes to match. If ANY byte in this mask matches the
* current char, the pointer advances and the char is part of the
* substring.
* @param int $max
* The max number of chars to read.
*/
public function charsWhile($bytes, $max = null) {
if ($this->char >= $this->EOF) {
return FALSE;
// However, for here we want the length up until the next byte to be
// processed, so add one to the current byte ($this->char).
if ($lastLine !== false) {
$findLengthOf = substr($this->data, $lastLine + 1, $this->char - 1 - $lastLine);
} else {
// After a newline.
$findLengthOf = substr($this->data, 0, $this->char);
}
return UTF8Utils::countChars($findLengthOf);
}
if ($max === 0 || $max) {
$len = strspn($this->data, $bytes, $this->char, $max);
}
else {
$len = strspn($this->data, $bytes, $this->char);
}
$string = (string) substr($this->data, $this->char, $len);
$this->char += $len;
return $string;
}
/**
* Unconsume characters.
*
* @param int $howMany
* The number of characters to unconsume.
*/
public function unconsume($howMany = 1) {
if (($this->char - $howMany) >= 0) {
$this->char = $this->char - $howMany;
}
}
/**
* Look ahead without moving cursor.
*/
public function peek() {
if (($this->char + 1) <= $this->EOF) {
return $this->data[$this->char + 1];
/**
*
* @deprecated
*
*/
public function getColumnOffset()
{
return $this->columnOffset();
}
return FALSE;
}
/**
* Get the current character.
*
* @return string The current character.
*/
public function current()
{
return $this->data[$this->char];
}
public function key() {
return $this->char;
}
/**
* Advance the pointer.
* This is part of the Iterator interface.
*/
public function next()
{
$this->char ++;
}
/**
* Rewind to the start of the string.
*/
public function rewind()
{
$this->char = 0;
}
/**
* Is the current pointer location valid.
*
* @return bool Is the current pointer location valid.
*/
public function valid()
{
if ($this->char < $this->EOF) {
return true;
}
return false;
}
/**
* Get all characters until EOF.
*
* This reads to the end of the file, and sets the read marker at the
* end of the file.
*
* @note This performs bounds checking
*
* @return string Returns the remaining text. If called when the InputStream is
* already exhausted, it returns an empty string.
*/
public function remainingChars()
{
if ($this->char < $this->EOF) {
$data = substr($this->data, $this->char);
$this->char = $this->EOF;
return $data;
}
return ''; // false;
}
/**
* Read to a particular match (or until $max bytes are consumed).
*
* This operates on byte sequences, not characters.
*
* Matches as far as possible until we reach a certain set of bytes
* and returns the matched substring.
*
* @param string $bytes
* Bytes to match.
* @param int $max
* Maximum number of bytes to scan.
* @return mixed Index or false if no match is found. You should use strong
* equality when checking the result, since index could be 0.
*/
public function charsUntil($bytes, $max = null)
{
if ($this->char >= $this->EOF) {
return false;
}
if ($max === 0 || $max) {
$len = strcspn($this->data, $bytes, $this->char, $max);
} else {
$len = strcspn($this->data, $bytes, $this->char);
}
$string = (string) substr($this->data, $this->char, $len);
$this->char += $len;
return $string;
}
/**
* Returns the string so long as $bytes matches.
*
* Matches as far as possible with a certain set of bytes
* and returns the matched substring.
*
* @param string $bytes
* A mask of bytes to match. If ANY byte in this mask matches the
* current char, the pointer advances and the char is part of the
* substring.
* @param int $max
* The max number of chars to read.
*/
public function charsWhile($bytes, $max = null)
{
if ($this->char >= $this->EOF) {
return false;
}
if ($max === 0 || $max) {
$len = strspn($this->data, $bytes, $this->char, $max);
} else {
$len = strspn($this->data, $bytes, $this->char);
}
$string = (string) substr($this->data, $this->char, $len);
$this->char += $len;
return $string;
}
/**
* Unconsume characters.
*
* @param int $howMany
* The number of characters to unconsume.
*/
public function unconsume($howMany = 1)
{
if (($this->char - $howMany) >= 0) {
$this->char = $this->char - $howMany;
}
}
/**
* Look ahead without moving cursor.
*/
public function peek()
{
if (($this->char + 1) <= $this->EOF) {
return $this->data[$this->char + 1];
}
return false;
}
public function key()
{
return $this->char;
}
}

1975
libraries/html5php/HTML5/Parser/Tokenizer.php
View File

File diff suppressed because it is too large Load Diff

220
libraries/html5php/HTML5/Parser/TreeBuildingRules.php
View File

@ -1,114 +1,140 @@
<?php
namespace HTML5\Parser;
use HTML5\Elements;
namespace Masterminds\HTML5\Parser;
/**
* Handles special-case rules for the DOM tree builder.
*
* Many tags have special rules that need to be accomodated on an
* Many tags have special rules that need to be accomodated on an
* individual basis. This class handles those rules.
*
* See section 8.1.2.4 of the spec.
*
* @todo
* - colgroup and col special behaviors
* - body and head special behaviors
* @todo - colgroup and col special behaviors
* - body and head special behaviors
*/
class TreeBuildingRules {
class TreeBuildingRules
{
protected static $tags = array(
'li' => 1,
'dd' => 1,
'dt' => 1,
'rt' => 1,
'rp' => 1,
'tr' => 1,
'th' => 1,
'td' => 1,
'thead' => 1,
'tfoot' => 1,
'tbody' => 1,
'table' => 1,
'optgroup' => 1,
'option' => 1,
);
protected static $tags = array(
'li' => 1,
'dd' => 1,
'dt' => 1,
'rt' => 1,
'rp' => 1,
'tr' => 1,
'th' => 1,
'td' => 1,
'thead' => 1,
'tfoot' => 1,
'tbody' => 1,
'table' => 1,
'optgroup' => 1,
'option' => 1
);
/**
* Build a new rules engine.
*
* @param \DOMDocument $doc
* The DOM document to use for evaluation and modification.
*/
public function __construct($doc) {
$this->doc = $doc;
}
/**
* Returns TRUE if the given tagname has special processing rules.
*/
public function hasRules($tagname) {
return isset(static::$tags[$tagname]);
}
/**
* Evaluate the rule for the current tag name.
*
* This may modify the existing DOM.
*
* @return \DOMElement
* The new Current DOM element.
*/
public function evaluate($new, $current) {
switch($new->tagName) {
case 'li':
return $this->handleLI($new, $current);
case 'dt':
case 'dd':
return $this->handleDT($new, $current);
case 'rt':
case 'rp':
return $this->handleRT($new, $current);
case 'optgroup':
return $this->closeIfCurrentMatches($new, $current, array('optgroup'));
case 'option':
return $this->closeIfCurrentMatches($new, $current, array('option', 'optgroup'));
case 'tr':
return $this->closeIfCurrentMatches($new, $current, array('tr'));
case 'td':
case 'th':
return $this->closeIfCurrentMatches($new, $current, array('th', 'td'));
case 'tbody':
case 'thead':
case 'tfoot':
case 'table': // Spec isn't explicit about this, but it's necessary.
return $this->closeIfCurrentMatches($new, $current, array('thead', 'tfoot', 'tbody'));
/**
* Build a new rules engine.
*
* @param \DOMDocument $doc
* The DOM document to use for evaluation and modification.
*/
public function __construct($doc)
{
$this->doc = $doc;
}
return $current;
}
protected function handleLI($ele, $current) {
return $this->closeIfCurrentMatches($ele, $current, array('li'));
}
protected function handleDT($ele, $current) {
return $this->closeIfCurrentMatches($ele, $current, array('dt','dd'));
}
protected function handleRT($ele, $current) {
return $this->closeIfCurrentMatches($ele, $current, array('rt','rp'));
}
protected function closeIfCurrentMatches($ele, $current, $match) {
$tname = $current->tagName;
if (in_array($current->tagName, $match)) {
$current->parentNode->appendChild($ele);
/**
* Returns true if the given tagname has special processing rules.
*/
public function hasRules($tagname)
{
return isset(static::$tags[$tagname]);
}
else {
$current->appendChild($ele);
}
return $ele;
}
/**
* Evaluate the rule for the current tag name.
*
* This may modify the existing DOM.
*
* @return \DOMElement The new Current DOM element.
*/
public function evaluate($new, $current)
{
switch ($new->tagName) {
case 'li':
return $this->handleLI($new, $current);
case 'dt':
case 'dd':
return $this->handleDT($new, $current);
case 'rt':
case 'rp':
return $this->handleRT($new, $current);
case 'optgroup':
return $this->closeIfCurrentMatches($new, $current, array(
'optgroup'
));
case 'option':
return $this->closeIfCurrentMatches($new, $current, array(
'option',
'optgroup'
));
case 'tr':
return $this->closeIfCurrentMatches($new, $current, array(
'tr'
));
case 'td':
case 'th':
return $this->closeIfCurrentMatches($new, $current, array(
'th',
'td'
));
case 'tbody':
case 'thead':
case 'tfoot':
case 'table': // Spec isn't explicit about this, but it's necessary.
return $this->closeIfCurrentMatches($new, $current, array(
'thead',
'tfoot',
'tbody'
));
}
return $current;
}
protected function handleLI($ele, $current)
{
return $this->closeIfCurrentMatches($ele, $current, array(
'li'
));
}
protected function handleDT($ele, $current)
{
return $this->closeIfCurrentMatches($ele, $current, array(
'dt',
'dd'
));
}
protected function handleRT($ele, $current)
{
return $this->closeIfCurrentMatches($ele, $current, array(
'rt',
'rp'
));
}
protected function closeIfCurrentMatches($ele, $current, $match)
{
$tname = $current->tagName;
if (in_array($current->tagName, $match)) {
$current->parentNode->appendChild($ele);
} else {
$current->appendChild($ele);
}
return $ele;
}
}

258
libraries/html5php/HTML5/Parser/UTF8Utils.php
View File

@ -1,13 +1,14 @@
<?php
namespace Masterminds\HTML5\Parser;
/*
*
* Portions based on code from html5lib files with the following copyright:
* Portions based on code from html5lib files with the following copyright:
Copyright 2009 Geoffrey Sneddon <http://gsnedders.com/>
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
@ -25,145 +26,130 @@ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
namespace HTML5\Parser;
/**
* UTF-8 Utilities
*/
class UTF8Utils {
/**
* The Unicode replacement character..
*/
const FFFD = "\xEF\xBF\xBD";
/**
* Count the number of characters in a string.
*
* UTF-8 aware. This will try (in order) iconv,
* MB, libxml, and finally a custom counter.
*
* @todo Move this to a general utility class.
*/
public static function countChars($string) {
// Get the length for the string we need.
if(function_exists('iconv_strlen')) {
return iconv_strlen($string, 'utf-8');
}
elseif(function_exists('mb_strlen')) {
return mb_strlen($string, 'utf-8');
}
elseif(function_exists('utf8_decode')) {
// MPB: Will this work? Won't certain decodes lead to two chars
// extrapolated out of 2-byte chars?
return strlen(utf8_decode($string));
}
$count = count_chars($string);
// 0x80 = 0x7F - 0 + 1 (one added to get inclusive range)
// 0x33 = 0xF4 - 0x2C + 1 (one added to get inclusive range)
return array_sum(array_slice($count, 0, 0x80)) +
array_sum(array_slice($count, 0xC2, 0x33));
}
class UTF8Utils
{
/**
* Convert data from the given encoding to UTF-8.
*
* This has not yet been tested with charactersets other than UTF-8.
* It should work with ISO-8859-1/-13 and standard Latin Win charsets.
*
* @param string $data
* The data to convert.
* @param string $encoding
* A valid encoding. Examples: http://www.php.net/manual/en/mbstring.supported-encodings.php
*/
public static function convertToUTF8($data, $encoding = 'UTF-8') {
/*
* From the HTML5 spec:
Given an encoding, the bytes in the input stream must be
converted to Unicode characters for the tokeniser, as
described by the rules for that encoding, except that the
leading U+FEFF BYTE ORDER MARK character, if any, must not
be stripped by the encoding layer (it is stripped by the rule below).
/**
* The Unicode replacement character..
*/
const FFFD = "\xEF\xBF\xBD";
Bytes or sequences of bytes in the original byte stream that
could not be converted to Unicode characters must be converted
to U+FFFD REPLACEMENT CHARACTER code points. */
// mb_convert_encoding is chosen over iconv because of a bug. The best
// details for the bug are on http://us1.php.net/manual/en/function.iconv.php#108643
// which contains links to the actual but reports as well as work around
// details.
if (function_exists('mb_convert_encoding')) {
// mb library has the following behaviors:
// - UTF-16 surrogates result in FALSE.
// - Overlongs and outside Plane 16 result in empty strings.
// Before we run mb_convert_encoding we need to tell it what to do with
// characters it does not know. This could be different than the parent
// application executing this library so we store the value, change it
// to our needs, and then change it back when we are done. This feels
// a little excessive and it would be great if there was a better way.
$save = ini_get('mbstring.substitute_character');
ini_set('mbstring.substitute_character', "none");
$data = mb_convert_encoding($data, 'UTF-8', $encoding);
ini_set('mbstring.substitute_character', $save);
}
// @todo Get iconv running in at least some environments if that is possible.
elseif (function_exists('iconv') && $encoding != 'auto') {
// fprintf(STDOUT, "iconv found\n");
// iconv has the following behaviors:
// - Overlong representations are ignored.
// - Beyond Plane 16 is replaced with a lower char.
// - Incomplete sequences generate a warning.
$data = @iconv($encoding, 'UTF-8//IGNORE', $data);
}
else {
// we can make a conforming native implementation
throw new Exception('Not implemented, please install mbstring or iconv');
/**
* Count the number of characters in a string.
*
* UTF-8 aware. This will try (in order) iconv,
* MB, libxml, and finally a custom counter.
*
* @todo Move this to a general utility class.
*/
public static function countChars($string)
{
// Get the length for the string we need.
if (function_exists('iconv_strlen')) {
return iconv_strlen($string, 'utf-8');
} elseif (function_exists('mb_strlen')) {
return mb_strlen($string, 'utf-8');
} elseif (function_exists('utf8_decode')) {
// MPB: Will this work? Won't certain decodes lead to two chars
// extrapolated out of 2-byte chars?
return strlen(utf8_decode($string));
}
$count = count_chars($string);
// 0x80 = 0x7F - 0 + 1 (one added to get inclusive range)
// 0x33 = 0xF4 - 0x2C + 1 (one added to get inclusive range)
return array_sum(array_slice($count, 0, 0x80)) + array_sum(array_slice($count, 0xC2, 0x33));
}
/* One leading U+FEFF BYTE ORDER MARK character must be
ignored if any are present. */
if (substr($data, 0, 3) === "\xEF\xBB\xBF") {
$data = substr($data, 3);
/**
* Convert data from the given encoding to UTF-8.
*
* This has not yet been tested with charactersets other than UTF-8.
* It should work with ISO-8859-1/-13 and standard Latin Win charsets.
*
* @param string $data
* The data to convert.
* @param string $encoding
* A valid encoding. Examples: http://www.php.net/manual/en/mbstring.supported-encodings.php
*/
public static function convertToUTF8($data, $encoding = 'UTF-8')
{
/*
* From the HTML5 spec: Given an encoding, the bytes in the input stream must be converted to Unicode characters for the tokeniser, as described by the rules for that encoding, except that the leading U+FEFF BYTE ORDER MARK character, if any, must not be stripped by the encoding layer (it is stripped by the rule below). Bytes or sequences of bytes in the original byte stream that could not be converted to Unicode characters must be converted to U+FFFD REPLACEMENT CHARACTER code points.
*/
// mb_convert_encoding is chosen over iconv because of a bug. The best
// details for the bug are on http://us1.php.net/manual/en/function.iconv.php#108643
// which contains links to the actual but reports as well as work around
// details.
if (function_exists('mb_convert_encoding')) {
// mb library has the following behaviors:
// - UTF-16 surrogates result in false.
// - Overlongs and outside Plane 16 result in empty strings.
// Before we run mb_convert_encoding we need to tell it what to do with
// characters it does not know. This could be different than the parent
// application executing this library so we store the value, change it
// to our needs, and then change it back when we are done. This feels
// a little excessive and it would be great if there was a better way.
$save = ini_get('mbstring.substitute_character');
ini_set('mbstring.substitute_character', "none");
$data = mb_convert_encoding($data, 'UTF-8', $encoding);
ini_set('mbstring.substitute_character', $save);
} // @todo Get iconv running in at least some environments if that is possible.
elseif (function_exists('iconv') && $encoding != 'auto') {
// fprintf(STDOUT, "iconv found\n");
// iconv has the following behaviors:
// - Overlong representations are ignored.
// - Beyond Plane 16 is replaced with a lower char.
// - Incomplete sequences generate a warning.
$data = @iconv($encoding, 'UTF-8//IGNORE', $data);
} else {
// we can make a conforming native implementation
throw new Exception('Not implemented, please install mbstring or iconv');
}
/*
* One leading U+FEFF BYTE ORDER MARK character must be ignored if any are present.
*/
if (substr($data, 0, 3) === "\xEF\xBB\xBF") {
$data = substr($data, 3);
}
return $data;
}
return $data;
}
/**
* Checks for Unicode code points that are not valid in a document.
*
* @param string $data
* A string to analyze.
* @return array An array of (string) error messages produced by the scanning.
*/
public static function checkForIllegalCodepoints($data)
{
if (! function_exists('preg_match_all')) {
throw\Exception('The PCRE library is not loaded or is not available.');
}
/**
* Checks for Unicode code points that are not valid in a document.
*
* @param string $data
* A string to analyze.
* @return array
* An array of (string) error messages produced by the scanning.
*/
public static function checkForIllegalCodepoints($data) {
if (!function_exists('preg_match_all')) {
throw \Exception('The PCRE library is not loaded or is not available.');
}
// Vestigal error handling.
$errors = array();
// Vestigal error handling.
$errors = array();
/*
* All U+0000 null characters in the input must be replaced by U+FFFD REPLACEMENT CHARACTERs. Any occurrences of such characters is a parse error.
*/
for ($i = 0, $count = substr_count($data, "\0"); $i < $count; $i ++) {
$errors[] = 'null-character';
}
/* All U+0000 NULL characters in the input must be replaced
by U+FFFD REPLACEMENT CHARACTERs. Any occurrences of such
characters is a parse error. */
for ($i = 0, $count = substr_count($data, "\0"); $i < $count; $i++) {
$errors[] = 'null-character';
}
/* Any occurrences of any characters in the ranges U+0001 to
U+0008, U+000B, U+000E to U+001F, U+007F to U+009F,
U+D800 to U+DFFF , U+FDD0 to U+FDEF, and
characters U+FFFE, U+FFFF, U+1FFFE, U+1FFFF, U+2FFFE, U+2FFFF,
U+3FFFE, U+3FFFF, U+4FFFE, U+4FFFF, U+5FFFE, U+5FFFF, U+6FFFE,
U+6FFFF, U+7FFFE, U+7FFFF, U+8FFFE, U+8FFFF, U+9FFFE, U+9FFFF,
U+AFFFE, U+AFFFF, U+BFFFE, U+BFFFF, U+CFFFE, U+CFFFF, U+DFFFE,
U+DFFFF, U+EFFFE, U+EFFFF, U+FFFFE, U+FFFFF, U+10FFFE, and
U+10FFFF are parse errors. (These are all control characters
or permanently undefined Unicode characters.) */
// Check PCRE is loaded.
$count = preg_match_all(
'/(?:
/*
* Any occurrences of any characters in the ranges U+0001 to U+0008, U+000B, U+000E to U+001F, U+007F to U+009F, U+D800 to U+DFFF , U+FDD0 to U+FDEF, and characters U+FFFE, U+FFFF, U+1FFFE, U+1FFFF, U+2FFFE, U+2FFFF, U+3FFFE, U+3FFFF, U+4FFFE, U+4FFFF, U+5FFFE, U+5FFFF, U+6FFFE, U+6FFFF, U+7FFFE, U+7FFFF, U+8FFFE, U+8FFFF, U+9FFFE, U+9FFFF, U+AFFFE, U+AFFFF, U+BFFFE, U+BFFFF, U+CFFFE, U+CFFFF, U+DFFFE, U+DFFFF, U+EFFFE, U+EFFFF, U+FFFFE, U+FFFFF, U+10FFFE, and U+10FFFF are parse errors. (These are all control characters or permanently undefined Unicode characters.)
*/
// Check PCRE is loaded.
$count = preg_match_all(
'/(?:
[\x01-\x08\x0B\x0E-\x1F\x7F] # U+0001 to U+0008, U+000B, U+000E to U+001F and U+007F
|
\xC2[\x80-\x9F] # U+0080 to U+009F
@ -175,13 +161,11 @@ class UTF8Utils {
\xEF\xBF[\xBE\xBF] # U+FFFE and U+FFFF
|
[\xF0-\xF4][\x8F-\xBF]\xBF[\xBE\xBF] # U+nFFFE and U+nFFFF (1 <= n <= 10_{16})
)/x',
$data,
$matches
);
for ($i = 0; $i < $count; $i++) {
$errors[] = 'invalid-codepoint';
)/x', $data, $matches);
for ($i = 0; $i < $count; $i ++) {
$errors[] = 'invalid-codepoint';
}
return $errors;
}
return $errors;
}
}

3035
libraries/html5php/HTML5/Serializer/HTML5Entities.php
View File

File diff suppressed because it is too large Load Diff

711
libraries/html5php/HTML5/Serializer/OutputRules.php
View File

@ -6,309 +6,474 @@
* These output rules are likely to generate output similar to the document that
* was parsed. It is not intended to output exactly the document that was parsed.
*/
namespace HTML5\Serializer;
namespace Masterminds\HTML5\Serializer;
use \HTML5\Elements;
use Masterminds\HTML5\Elements;
/**
* Generate the output html5 based on element rules.
*/
class OutputRules implements \HTML5\Serializer\RulesInterface {
class OutputRules implements \Masterminds\HTML5\Serializer\RulesInterface
{
/**
* Defined in http://www.w3.org/TR/html51/infrastructure.html#html-namespace-0
*/
const NAMESPACE_HTML = 'http://www.w3.org/1999/xhtml';
const IM_IN_HTML = 1;
const IM_IN_SVG = 2;
const IM_IN_MATHML = 3;
const NAMESPACE_MATHML = 'http://www.w3.org/1998/Math/MathML';
protected $traverser;
protected $encode = FALSE;
protected $out;
protected $outputMode;
const NAMESPACE_SVG = 'http://www.w3.org/2000/svg';
const DOCTYPE = '<!DOCTYPE html>';
const NAMESPACE_XLINK = 'http://www.w3.org/1999/xlink';
public function __construct($output, $options = array()) {
const NAMESPACE_XML = 'http://www.w3.org/XML/1998/namespace';
if (isset($options['encode_entities'])) {
$this->encode = $options['encode_entities'];
const NAMESPACE_XMLNS = 'http://www.w3.org/2000/xmlns/';
/**
* Holds the HTML5 element names that causes a namespace switch
*
* @var array
*/
protected $implicitNamespaces = array(
self::NAMESPACE_HTML,
self::NAMESPACE_SVG,
self::NAMESPACE_MATHML,
self::NAMESPACE_XML,
self::NAMESPACE_XMLNS,
);
const IM_IN_HTML = 1;
const IM_IN_SVG = 2;
const IM_IN_MATHML = 3;
/**
* Used as cache to detect if is available ENT_HTML5
* @var boolean
*/
private $hasHTML5 = false;
protected $traverser;
protected $encode = false;
protected $out;
protected $outputMode;
private $xpath;
protected $nonBooleanAttributes = array(
/*
array(
'nodeNamespace'=>'http://www.w3.org/1999/xhtml',
'attrNamespace'=>'http://www.w3.org/1999/xhtml',
'nodeName'=>'img', 'nodeName'=>array('img', 'a'),
'attrName'=>'alt', 'attrName'=>array('title', 'alt'),
'prefixes'=>['xh'=>'http://www.w3.org/1999/xhtml'),
'xpath' => "@checked[../../xh:input[@type='radio' or @type='checkbox']]",
),
*/
array(
'nodeNamespace'=>'http://www.w3.org/1999/xhtml',
'attrName'=>array('alt', 'title'),
),
);
const DOCTYPE = '<!DOCTYPE html>';
public function __construct($output, $options = array())
{
if (isset($options['encode_entities'])) {
$this->encode = $options['encode_entities'];
}
$this->outputMode = static::IM_IN_HTML;
$this->out = $output;
// If HHVM, see https://github.com/facebook/hhvm/issues/2727
$this->hasHTML5 = defined('ENT_HTML5') && !defined('HHVM_VERSION');
}
public function addRule(array $rule)
{
$this->nonBooleanAttributes[] = $rule;
}
$this->outputMode = static::IM_IN_HTML;
$this->out = $output;
}
public function setTraverser(\Masterminds\HTML5\Serializer\Traverser $traverser)
{
$this->traverser = $traverser;
public function setTraverser(\HTML5\Serializer\Traverser $traverser) {
$this->traverser = $traverser;
return $this;
}
public function document($dom) {
$this->doctype();
$this->traverser->node($dom->documentElement);
$this->nl();
}
protected function doctype() {
$this->wr(static::DOCTYPE);
$this->nl();
}
public function element($ele) {
$name = $ele->tagName;
// Per spec:
// If the element has a declared namespace in the HTML, MathML or
// SVG namespaces, we use the lname instead of the tagName.
if ($this->traverser->isLocalElement($ele)) {
$name = $ele->localName;
return $this;
}
// If we are in SVG or MathML there is special handling.
// Using if/elseif instead of switch because it's faster in PHP.
if ($name == 'svg') {
$this->outputMode = static::IM_IN_SVG;
$name = Elements::normalizeSvgElement($name);
}
elseif ($name == 'math') {
$this->outputMode = static::IM_IN_MATHML;
public function document($dom)
{
$this->doctype();
$this->traverser->node($dom->documentElement);
$this->nl();
}
$this->openTag($ele);
// Handle children.
if ($ele->hasChildNodes()) {
$this->traverser->children($ele->childNodes);
protected function doctype()
{
$this->wr(static::DOCTYPE);
$this->nl();
}
// Close out the SVG or MathML special handling.
if ($name == 'svg' || $name == 'math') {
$this->outputMode = static::IM_IN_HTML;
public function element($ele)
{
$name = $ele->tagName;
// Per spec:
// If the element has a declared namespace in the HTML, MathML or
// SVG namespaces, we use the lname instead of the tagName.
if ($this->traverser->isLocalElement($ele)) {
$name = $ele->localName;
}
// If we are in SVG or MathML there is special handling.
// Using if/elseif instead of switch because it's faster in PHP.
if ($name == 'svg') {
$this->outputMode = static::IM_IN_SVG;
$name = Elements::normalizeSvgElement($name);
} elseif ($name == 'math') {
$this->outputMode = static::IM_IN_MATHML;
}
$this->openTag($ele);
if (Elements::isA($name, Elements::TEXT_RAW)) {
foreach ($ele->childNodes as $child) {
$this->wr($child->data);
}
} else {
// Handle children.
if ($ele->hasChildNodes()) {
$this->traverser->children($ele->childNodes);
}
// Close out the SVG or MathML special handling.
if ($name == 'svg' || $name == 'math') {
$this->outputMode = static::IM_IN_HTML;
}
}
// If not unary, add a closing tag.
if (! Elements::isA($name, Elements::VOID_TAG)) {
$this->closeTag($ele);
}
}
// If not unary, add a closing tag.
if (!Elements::isA($name, Elements::VOID_TAG)) {
$this->closeTag($ele);
}
}
/**
* Write a text node.
*
* @param \DOMText $ele
* The text node to write.
*/
public function text($ele)
{
if (isset($ele->parentNode) && isset($ele->parentNode->tagName) && Elements::isA($ele->parentNode->localName, Elements::TEXT_RAW)) {
$this->wr($ele->data);
return;
}
/**
* Write a text node.
*
* @param \DOMText $ele
* The text node to write.
*/
public function text($ele) {
if (isset($ele->parentNode) && isset($ele->parentNode->tagName) && Elements::isA($ele->parentNode->tagName, Elements::TEXT_RAW)) {
$this->wr($ele->data);
return;
// FIXME: This probably needs some flags set.
$this->wr($this->enc($ele->data));
}
// FIXME: This probably needs some flags set.
$this->wr($this->enc($ele->data));
}
public function cdata($ele) {
// This encodes CDATA.
$this->wr($ele->ownerDocument->saveXML($ele));
}
public function comment($ele) {
// These produce identical output.
//$this->wr('<!--')->wr($ele->data)->wr('-->');
$this->wr($ele->ownerDocument->saveXML($ele));
}
public function processorInstruction($ele) {
$this->wr('<?')->wr($ele->target)->wr(' ')->wr($ele->data)->wr('?>');
}
/**
* Write the opening tag.
*
* Tags for HTML, MathML, and SVG are in the local name. Otherwise, use the
* qualified name (8.3).
*
* @param \DOMNode $ele
* The element being written.
*/
protected function openTag($ele) {
$this->wr('<')->wr($ele->tagName);
$this->attrs($ele);
if ($this->outputMode == static::IM_IN_HTML) {
$this->wr('>');
}
// If we are not in html mode we are in SVG, MathML, or XML embedded content.
else {
if ($ele->hasChildNodes()) {
$this->wr('>');
}
// If there are no children this is self closing.
else {
$this->wr(' />');
}
}
}
protected function attrs($ele) {
// FIXME: Needs support for xml, xmlns, xlink, and namespaced elements.
if (!$ele->hasAttributes()) {
return $this;
public function cdata($ele)
{
// This encodes CDATA.
$this->wr($ele->ownerDocument->saveXML($ele));
}
// TODO: Currently, this always writes name="value", and does not do
// value-less attributes.
$map = $ele->attributes;
$len = $map->length;
for ($i = 0; $i < $len; ++$i) {
$node = $map->item($i);
$val = $this->enc($node->value, TRUE);
// XXX: The spec says that we need to ensure that anything in
// the XML, XMLNS, or XLink NS's should use the canonical
// prefix. It seems that DOM does this for us already, but there
// may be exceptions.
$name = $node->name;
// Special handling for attributes in SVG and MathML.
// Using if/elseif instead of switch because it's faster in PHP.
if ($this->outputMode == static::IM_IN_SVG) {
$name = Elements::normalizeSvgAttribute($name);
}
elseif ($this->outputMode == static::IM_IN_MATHML) {
$name = Elements::normalizeMathMlAttribute($name);
}
$this->wr(' ')->wr($name);
if (isset($val) && $val !== '') {
$this->wr('="')->wr($val)->wr('"');
}
}
}
/**
* Write the closing tag.
*
* Tags for HTML, MathML, and SVG are in the local name. Otherwise, use the
* qualified name (8.3).
*
* @param \DOMNode $ele
* The element being written.
*/
protected function closeTag($ele) {
if ($this->outputMode == static::IM_IN_HTML || $ele->hasChildNodes()) {
$this->wr('</')->wr($ele->tagName)->wr('>');
}
}
/**
* Write to the output.
*
* @param string $text
* The string to put into the output.
*
* @return HTML5\Serializer\Traverser
* $this so it can be used in chaining.
*/
protected function wr($text) {
fwrite($this->out, $text);
return $this;
}
/**
* Write a new line character.
*
* @return HTML5\Serializer\Traverser
* $this so it can be used in chaining.
*/
protected function nl() {
fwrite($this->out, PHP_EOL);
return $this;
}
/**
* Encode text.
*
* When encode is set to FALSE, the default value, the text passed in is
* escaped per section 8.3 of the html5 spec. For details on how text is
* escaped see the escape() method.
*
* When encoding is set to true the text is converted to named character
* references where appropriate. Section 8.1.4 Character references of the
* html5 spec refers to using named character references. This is useful for
* characters that can't otherwise legally be used in the text.
*
* The named character references are listed in section 8.5.
*
* @see http://www.w3.org/TR/2013/CR-html5-20130806/syntax.html#named-character-references
*
* True encoding will turn all named character references into their entities.
* This includes such characters as +.# and many other common ones. By default
* encoding here will just escape &'<>".
*
* Note, PHP 5.4+ has better html5 encoding.
*
* @todo Use the Entities class in php 5.3 to have html5 entities.
*
* @param string $text
* text to encode.
* @param boolean $attribute
* True if we are encoding an attrubute, false otherwise
*
* @return string
* The encoded text.
*/
protected function enc($text, $attribute = FALSE) {
// Escape the text rather than convert to named character references.
if (!$this->encode) {
return $this->escape($text, $attribute);
public function comment($ele)
{
// These produce identical output.
// $this->wr('<!--')->wr($ele->data)->wr('-->');
$this->wr($ele->ownerDocument->saveXML($ele));
}
// If we are in PHP 5.4+ we can use the native html5 entity functionality to
// convert the named character references.
if (defined('ENT_HTML5')) {
return htmlentities($text, ENT_HTML5 | ENT_SUBSTITUTE | ENT_QUOTES, 'UTF-8', FALSE);
public function processorInstruction($ele)
{
$this->wr('<?')
->wr($ele->target)
->wr(' ')
->wr($ele->data)
->wr('?>');
}
// If a version earlier than 5.4 html5 entities are not entirely handled.
// This manually handles them.
else {
return strtr($text, \HTML5\Serializer\HTML5Entities::$map);
}
}
/**
* Write the namespace attributes
*
*
* @param \DOMNode $ele
* The element being written.
*/
protected function namespaceAttrs($ele)
{
if (!$this->xpath || $this->xpath->document !== $ele->ownerDocument){
$this->xpath = new \DOMXPath($ele->ownerDocument);
}
/**
* Escape test.
*
* According to the html5 spec section 8.3 Serializing HTML fragments, text
* within tags that are not style, script, xmp, iframe, noembed, and noframes
* need to be properly escaped.
*
* The & should be converted to &amp;, no breaking space unicode characters
* converted to &nbsp;, when in attribute mode the " should be converted to
* &quot;, and when not in attribute mode the < and > should be converted to
* &lt; and &gt;.
*
* @see http://www.w3.org/TR/2013/CR-html5-20130806/syntax.html#escapingString
*
* @param string $text
* text to escape.
* @param boolean $attribute
* True if we are escaping an attrubute, false otherwise
*/
protected function escape($text, $attribute = FALSE) {
// Not using htmlspecialchars because, while it does escaping, it doesn't
// match the requirements of section 8.5. For example, it doesn't handle
// non-breaking spaces.
if ($attribute) {
$replace = array('"'=>'&quot;', '&'=>'&amp;', "\xc2\xa0"=>'&nbsp;');
}
else {
$replace = array('<'=>'&lt;', '>'=>'&gt;', '&'=>'&amp;', "\xc2\xa0"=>'&nbsp;');
foreach( $this->xpath->query('namespace::*[not(.=../../namespace::*)]', $ele ) as $nsNode ) {
if (!in_array($nsNode->nodeValue, $this->implicitNamespaces)) {
$this->wr(' ')->wr($nsNode->nodeName)->wr('="')->wr($nsNode->nodeValue)->wr('"');
}
}
}
return strtr($text, $replace);
}
/**
* Write the opening tag.
*
* Tags for HTML, MathML, and SVG are in the local name. Otherwise, use the
* qualified name (8.3).
*
* @param \DOMNode $ele
* The element being written.
*/
protected function openTag($ele)
{
$this->wr('<')->wr($this->traverser->isLocalElement($ele) ? $ele->localName : $ele->tagName);
$this->attrs($ele);
$this->namespaceAttrs($ele);
if ($this->outputMode == static::IM_IN_HTML) {
$this->wr('>');
} // If we are not in html mode we are in SVG, MathML, or XML embedded content.
else {
if ($ele->hasChildNodes()) {
$this->wr('>');
} // If there are no children this is self closing.
else {
$this->wr(' />');
}
}
}
protected function attrs($ele)
{
// FIXME: Needs support for xml, xmlns, xlink, and namespaced elements.
if (! $ele->hasAttributes()) {
return $this;
}
// TODO: Currently, this always writes name="value", and does not do
// value-less attributes.
$map = $ele->attributes;
$len = $map->length;
for ($i = 0; $i < $len; ++ $i) {
$node = $map->item($i);
$val = $this->enc($node->value, true);
// XXX: The spec says that we need to ensure that anything in
// the XML, XMLNS, or XLink NS's should use the canonical
// prefix. It seems that DOM does this for us already, but there
// may be exceptions.
$name = $node->name;
// Special handling for attributes in SVG and MathML.
// Using if/elseif instead of switch because it's faster in PHP.
if ($this->outputMode == static::IM_IN_SVG) {
$name = Elements::normalizeSvgAttribute($name);
} elseif ($this->outputMode == static::IM_IN_MATHML) {
$name = Elements::normalizeMathMlAttribute($name);
}
$this->wr(' ')->wr($name);
if ((isset($val) && $val !== '') || $this->nonBooleanAttribute($node)) {
$this->wr('="')->wr($val)->wr('"');
}
}
}
protected function nonBooleanAttribute(\DOMAttr $attr)
{
$ele = $attr->ownerElement;
foreach($this->nonBooleanAttributes as $rule){
if(isset($rule['nodeNamespace']) && $rule['nodeNamespace']!==$ele->namespaceURI){
continue;
}
if(isset($rule['attNamespace']) && $rule['attNamespace']!==$attr->namespaceURI){
continue;
}
if(isset($rule['nodeName']) && !is_array($rule['nodeName']) && $rule['nodeName']!==$ele->localName){
continue;
}
if(isset($rule['nodeName']) && is_array($rule['nodeName']) && !in_array($ele->localName, $rule['nodeName'], true)){
continue;
}
if(isset($rule['attrName']) && !is_array($rule['attrName']) && $rule['attrName']!==$attr->localName){
continue;
}
if(isset($rule['attrName']) && is_array($rule['attrName']) && !in_array($attr->localName, $rule['attrName'], true)){
continue;
}
if(isset($rule['xpath'])){
$xp = $this->getXPath($attr);
if(isset($rule['prefixes'])){
foreach($rule['prefixes'] as $nsPrefix => $ns){
$xp->registerNamespace($nsPrefix, $ns);
}
}
if(!$xp->query($rule['xpath'], $attr->ownerElement)->length){
continue;
}
}
return true;
}
return false;
}
private function getXPath(\DOMNode $node){
if(!$this->xpath){
$this->xpath = new \DOMXPath($node->ownerDocument);
}
return $this->xpath;
}
/**
* Write the closing tag.
*
* Tags for HTML, MathML, and SVG are in the local name. Otherwise, use the
* qualified name (8.3).
*
* @param \DOMNode $ele
* The element being written.
*/
protected function closeTag($ele)
{
if ($this->outputMode == static::IM_IN_HTML || $ele->hasChildNodes()) {
$this->wr('</')->wr($this->traverser->isLocalElement($ele) ? $ele->localName : $ele->tagName)->wr('>');
}
}
/**
* Write to the output.
*
* @param string $text
* The string to put into the output.
*
* @return \Masterminds\HTML5\Serializer\Traverser $this so it can be used in chaining.
*/
protected function wr($text)
{
fwrite($this->out, $text);
return $this;
}
/**
* Write a new line character.
*
* @return \Masterminds\HTML5\Serializer\Traverser $this so it can be used in chaining.
*/
protected function nl()
{
fwrite($this->out, PHP_EOL);
return $this;
}
/**
* Encode text.
*
* When encode is set to false, the default value, the text passed in is
* escaped per section 8.3 of the html5 spec. For details on how text is
* escaped see the escape() method.
*
* When encoding is set to true the text is converted to named character
* references where appropriate. Section 8.1.4 Character references of the
* html5 spec refers to using named character references. This is useful for
* characters that can't otherwise legally be used in the text.
*
* The named character references are listed in section 8.5.
*
* @see http://www.w3.org/TR/2013/CR-html5-20130806/syntax.html#named-character-references True encoding will turn all named character references into their entities.
* This includes such characters as +.# and many other common ones. By default
* encoding here will just escape &'<>".
*
* Note, PHP 5.4+ has better html5 encoding.
*
* @todo Use the Entities class in php 5.3 to have html5 entities.
*
* @param string $text
* text to encode.
* @param boolean $attribute
* True if we are encoding an attrubute, false otherwise
*
* @return string The encoded text.
*/
protected function enc($text, $attribute = false)
{
// Escape the text rather than convert to named character references.
if (! $this->encode) {
return $this->escape($text, $attribute);
}
// If we are in PHP 5.4+ we can use the native html5 entity functionality to
// convert the named character references.
if ($this->hasHTML5) {
return htmlentities($text, ENT_HTML5 | ENT_SUBSTITUTE | ENT_QUOTES, 'UTF-8', false);
} // If a version earlier than 5.4 html5 entities are not entirely handled.
// This manually handles them.
else {
return strtr($text, \Masterminds\HTML5\Serializer\HTML5Entities::$map);
}
}
/**
* Escape test.
*
* According to the html5 spec section 8.3 Serializing HTML fragments, text
* within tags that are not style, script, xmp, iframe, noembed, and noframes
* need to be properly escaped.
*
* The & should be converted to &amp;, no breaking space unicode characters
* converted to &nbsp;, when in attribute mode the " should be converted to
* &quot;, and when not in attribute mode the < and > should be converted to
* &lt; and &gt;.
*
* @see http://www.w3.org/TR/2013/CR-html5-20130806/syntax.html#escapingString
*
* @param string $text
* text to escape.
* @param boolean $attribute
* True if we are escaping an attrubute, false otherwise
*/
protected function escape($text, $attribute = false)
{
// Not using htmlspecialchars because, while it does escaping, it doesn't
// match the requirements of section 8.5. For example, it doesn't handle
// non-breaking spaces.
if ($attribute) {
$replace = array(
'"' => '&quot;',
'&' => '&amp;',
"\xc2\xa0" => '&nbsp;'
);
} else {
$replace = array(
'<' => '&lt;',
'>' => '&gt;',
'&' => '&amp;',
"\xc2\xa0" => '&nbsp;'
);
}
return strtr($text, $replace);
}
}

167
libraries/html5php/HTML5/Serializer/RulesInterface.php
View File

@ -3,100 +3,101 @@
* @file
* The interface definition for Rules to generate output.
*/
namespace HTML5\Serializer;
namespace Masterminds\HTML5\Serializer;
/**
* To create a new rule set for writing output the RulesInterface needs to be
* implemented. The resulting class can be specified in the options with the
* implemented.
* The resulting class can be specified in the options with the
* key of rules.
*
* For an example implementation see \HTML5\Serializer\OutputRules.
* For an example implementation see \Masterminds\HTML5\Serializer\OutputRules.
*/
interface RulesInterface {
interface RulesInterface
{
/**
* The class constructor.
*
* Note, before the rules can be used a traverser must be registered.
*
* @param mixed $output
* The output stream to write output to.
* @param array $options
* An array of options.
*/
public function __construct($output, $options = array());
/**
* The class constructor.
*
* Note, before the rules can be used a traverser must be registered.
*
* @param mixed $output
* The output stream to write output to.
* @param array $options
* An array of options.
*/
public function __construct($output, $options = array());
/**
* Register the traverser used in but the rules.
*
* Note, only one traverser can be used by the rules.
*
* @param \HTML5\Serializer\Traverser $traverser
* The traverser used in the rules.
* @return \HTML5\Serializer\RulesInterface
* $this for the current object.
*/
public function setTraverser(\HTML5\Serializer\Traverser $traverser);
/**
* Register the traverser used in but the rules.
*
* Note, only one traverser can be used by the rules.
*
* @param \Masterminds\HTML5\Serializer\Traverser $traverser
* The traverser used in the rules.
* @return \Masterminds\HTML5\Serializer\RulesInterface $this for the current object.
*/
public function setTraverser(\Masterminds\HTML5\Serializer\Traverser $traverser);
/**
* Write a document element (\DOMDocument).
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param \DOMDocument $dom
*/
public function document($dom);
/**
* Write a document element (\DOMDocument).
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param \DOMDocument $dom
*/
public function document($dom);
/**
* Write an element.
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param mixed $ele
*/
public function element($ele);
/**
* Write an element.
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param mixed $ele
*/
public function element($ele);
/**
* Write a text node.
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param mixed $ele
*/
public function text($ele);
/**
* Write a text node.
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param mixed $ele
*/
public function text($ele);
/**
* Write a CDATA node.
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param mixed $ele
*/
public function cdata($ele);
/**
* Write a CDATA node.
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param mixed $ele
*/
public function cdata($ele);
/**
* Write a comment node.
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param mixed $ele
*/
public function comment($ele);
/**
* Write a comment node.
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param mixed $ele
*/
public function comment($ele);
/**
* Write a processor instruction.
*
* To learn about processor instructions see \HTML5\InstructionProcessor
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param mixed $ele
*/
public function processorInstruction($ele);
}
/**
* Write a processor instruction.
*
* To learn about processor instructions see \Masterminds\HTML5\InstructionProcessor
*
* Instead of returning the result write it to the output stream ($output)
* that was passed into the constructor.
*
* @param mixed $ele
*/
public function processorInstruction($ele);
}

246
libraries/html5php/HTML5/Serializer/Traverser.php
View File

@ -1,142 +1,150 @@
<?php
namespace HTML5\Serializer;
namespace Masterminds\HTML5\Serializer;
/**
* Traverser for walking a DOM tree.
*
* This is a concrete traverser designed to convert a DOM tree into an
* HTML5 document. It is not intended to be a generic DOMTreeWalker
* This is a concrete traverser designed to convert a DOM tree into an
* HTML5 document. It is not intended to be a generic DOMTreeWalker
* implementation.
*
* @see http://www.w3.org/TR/2012/CR-html5-20121217/syntax.html#serializing-html-fragments
*/
class Traverser {
class Traverser
{
/** Namespaces that should be treated as "local" to HTML5. */
static $local_ns = array(
'http://www.w3.org/1999/xhtml' => 'html',
'http://www.w3.org/1998/Math/MathML' => 'math',
'http://www.w3.org/2000/svg' => 'svg',
);
/**
* Namespaces that should be treated as "local" to HTML5.
*/
static $local_ns = array(
'http://www.w3.org/1999/xhtml' => 'html',
'http://www.w3.org/1998/Math/MathML' => 'math',
'http://www.w3.org/2000/svg' => 'svg'
);
protected $dom;
protected $options;
protected $encode = FALSE;
protected $rules;
protected $out;
protected $dom;
/**
* Create a traverser.
*
* @param DOMNode|DOMNodeList $dom
* The document or node to traverse.
* @param resource $out
* A stream that allows writing. The traverser will output into this
* stream.
* @param array $options
* An array or options for the traverser as key/value pairs. These include:
* - encode_entities: A bool to specify if full encding should happen for all named
* charachter references. Defaults to FALSE which escapes &'<>".
* - output_rules: The path to the class handling the output rules.
*/
public function __construct($dom, $out, RulesInterface $rules, $options = array()) {
$this->dom = $dom;
$this->out = $out;
$this->rules = $rules;
$this->options = $options;
protected $options;
$this->rules->setTraverser($this);
}
protected $encode = false;
/**
* Tell the traverser to walk the DOM.
*
* @return resource $out
* Returns the output stream.
*/
public function walk() {
if ($this->dom instanceof \DOMDocument) {
$this->rules->document($this->dom);
}
elseif ($this->dom instanceof \DOMDocumentFragment) {
// Document fragments are a special case. Only the children need to
// be serialized.
if ($this->dom->hasChildNodes()) {
$this->children($this->dom->childNodes);
}
}
// If NodeList, loop
elseif ($this->dom instanceof \DOMNodeList) {
// If this is a NodeList of DOMDocuments this will not work.
$this->children($this->dom);
}
// Else assume this is a DOMNode-like datastructure.
else {
$this->node($this->dom);
protected $rules;
protected $out;
/**
* Create a traverser.
*
* @param DOMNode|DOMNodeList $dom
* The document or node to traverse.
* @param resource $out
* A stream that allows writing. The traverser will output into this
* stream.
* @param array $options
* An array or options for the traverser as key/value pairs. These include:
* - encode_entities: A bool to specify if full encding should happen for all named
* charachter references. Defaults to false which escapes &'<>".
* - output_rules: The path to the class handling the output rules.
*/
public function __construct($dom, $out, RulesInterface $rules, $options = array())
{
$this->dom = $dom;
$this->out = $out;
$this->rules = $rules;
$this->options = $options;
$this->rules->setTraverser($this);
}
return $this->out;
}
/**
* Tell the traverser to walk the DOM.
*
* @return resource $out
* Returns the output stream.
*/
public function walk()
{
if ($this->dom instanceof \DOMDocument) {
$this->rules->document($this->dom);
} elseif ($this->dom instanceof \DOMDocumentFragment) {
// Document fragments are a special case. Only the children need to
// be serialized.
if ($this->dom->hasChildNodes()) {
$this->children($this->dom->childNodes);
}
} // If NodeList, loop
elseif ($this->dom instanceof \DOMNodeList) {
// If this is a NodeList of DOMDocuments this will not work.
$this->children($this->dom);
} // Else assume this is a DOMNode-like datastructure.
else {
$this->node($this->dom);
}
/**
* Process a node in the DOM.
*
* @param mixed $node
* A node implementing \DOMNode.
*/
public function node($node) {
// A listing of types is at http://php.net/manual/en/dom.constants.php
switch ($node->nodeType) {
case XML_ELEMENT_NODE:
$this->rules->element($node);
break;
case XML_TEXT_NODE:
$this->rules->text($node);
break;
case XML_CDATA_SECTION_NODE:
$this->rules->cdata($node);
break;
// FIXME: It appears that the parser doesn't do PI's.
case XML_PI_NODE:
$this->rules->processorInstruction($node);
break;
case XML_COMMENT_NODE:
$this->rules->comment($node);
break;
// Currently we don't support embedding DTDs.
default:
print '<!-- Skipped -->';
break;
return $this->out;
}
}
/**
* Walk through all the nodes on a node list.
*
* @param \DOMNodeList $nl
* A list of child elements to walk through.
*/
public function children($nl) {
foreach ($nl as $node) {
$this->node($node);
/**
* Process a node in the DOM.
*
* @param mixed $node
* A node implementing \DOMNode.
*/
public function node($node)
{
// A listing of types is at http://php.net/manual/en/dom.constants.php
switch ($node->nodeType) {
case XML_ELEMENT_NODE:
$this->rules->element($node);
break;
case XML_TEXT_NODE:
$this->rules->text($node);
break;
case XML_CDATA_SECTION_NODE:
$this->rules->cdata($node);
break;
// FIXME: It appears that the parser doesn't do PI's.
case XML_PI_NODE:
$this->rules->processorInstruction($node);
break;
case XML_COMMENT_NODE:
$this->rules->comment($node);
break;
// Currently we don't support embedding DTDs.
default:
print '<!-- Skipped -->';
break;
}
}
}
/**
* Is an element local?
*
* @param mixed $ele
* An element that implement \DOMNode.
*
* @return bool
* True if local and false otherwise.
*/
public function isLocalElement($ele) {
$uri = $ele->namespaceURI;
if (empty($uri)) {
return FALSE;
/**
* Walk through all the nodes on a node list.
*
* @param \DOMNodeList $nl
* A list of child elements to walk through.
*/
public function children($nl)
{
foreach ($nl as $node) {
$this->node($node);
}
}
/**
* Is an element local?
*
* @param mixed $ele
* An element that implement \DOMNode.
*
* @return bool True if local and false otherwise.
*/
public function isLocalElement($ele)
{
$uri = $ele->namespaceURI;
if (empty($uri)) {
return false;
}
return isset(static::$local_ns[$uri]);
}
return isset(static::$local_ns[$uri]);
}
}

3
libraries/html5php/autoloader.php
View File

@ -23,10 +23,11 @@ class HTML5PHP_Autoloader
public function autoload($class)
{
// Only load the class if it starts with "HTML5"
if (strpos($class, 'HTML5') !== 0)
if (strpos($class, 'Masterminds\HTML5') !== 0)
{
return;
}
$class = substr($class, 12);
//die($class);
$filename = $this->path . DIRECTORY_SEPARATOR . str_replace('\\', DIRECTORY_SEPARATOR, $class) . '.php';

6
libraries/humble-http-agent/HumbleHttpAgent.php
View File

@ -394,7 +394,7 @@ class HumbleHttpAgent
// for AJAX sites, e.g. Blogger with its dynamic views templates.
// Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification
if (isset($this->requests[$orig]['body'])) {
$redirectURL = $this->getRedirectURLfromHTML($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));
$redirectURL = $this->getRedirectURLfromHTML($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 150000));
if ($redirectURL) {
$this->redirectQueue[$orig] = $redirectURL;
}
@ -515,7 +515,7 @@ class HumbleHttpAgent
// for AJAX sites, e.g. Blogger with its dynamic views templates.
// Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification
if (isset($this->requests[$orig]['body'])) {
$redirectURL = $this->getRedirectURLfromHTML($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));
$redirectURL = $this->getRedirectURLfromHTML($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 150000));
if ($redirectURL) {
$this->redirectQueue[$orig] = $redirectURL;
}
@ -601,7 +601,7 @@ class HumbleHttpAgent
// for AJAX sites, e.g. Blogger with its dynamic views templates.
// Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification
if (isset($this->requests[$orig]['body'])) {
$redirectURL = $this->getRedirectURLfromHTML($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));
$redirectURL = $this->getRedirectURLfromHTML($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 150000));
if ($redirectURL) {
$this->redirectQueue[$orig] = $redirectURL;
}

9
libraries/readability/Readability.php
View File

@ -113,19 +113,22 @@ class Readability
function __construct($html, $url=null, $parser='libxml')
{
$this->url = $url;
/* Turn all double br's into p's */
/* Turn all double <br>s into <p>s */
$html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);
$html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);
$html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
if (trim($html) == '') $html = '<html></html>';
if ($parser=='html5lib' || $parser=='html5php') {
if (version_compare(PHP_VERSION, '5.3.0') >= 0) {
$this->dom = HTML5::loadHTML($html);
//use Masterminds\HTML5;
$html5class = 'Masterminds\HTML5';
$html5 = new $html5class();
$this->dom = $html5->loadHTML($html);
}
}
if ($this->dom === null) {
$this->dom = new DOMDocument();
$this->dom->preserveWhiteSpace = false;
$html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
@$this->dom->loadHTML($html);
}
$this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');

235
makefulltextfeed.php
View File

@ -3,8 +3,8 @@
// Author: Keyvan Minoukadeh
// Copyright (c) 2014 Keyvan Minoukadeh
// License: AGPLv3
// Version: 3.3
// Date: 2014-05-07
// Version: 3.4
// Date: 2014-08-28
// More info: http://fivefilters.org/content-only/
// Help: http://help.fivefilters.org
@ -29,6 +29,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
// For more request parameters, see http://help.fivefilters.org/customer/portal/articles/226660-usage
error_reporting(E_ALL ^ E_NOTICE);
libxml_use_internal_errors(true);
ini_set("display_errors", 1);
@set_time_limit(120);
@ -82,9 +83,11 @@ function autoload($class_name) {
// Language detect
'Text_LanguageDetect' => 'language-detect/LanguageDetect.php',
// HTML5 PHP (can't be used unless PHP version is >= 5.3)
'HTML5' => 'html5php/HTML5.php',
'Masterminds\HTML5' => 'html5php/HTML5.php',
// htmLawed - used if XSS filter is enabled (xss_filter)
'htmLawed' => 'htmLawed/htmLawed.php'
'htmLawed' => 'htmLawed/htmLawed.php',
// Disable SimplePie sanitization
'DisableSimplePieSanitize' => 'DisableSimplePieSanitize.php'
);
if (isset($mapping[$class_name])) {
debug("** Loading class $class_name ({$mapping[$class_name]})");
@ -180,19 +183,9 @@ if (strtolower(substr($url, 0, 7)) == 'feed://') {
if (!preg_match('!^https?://.+!i', $url)) {
$url = 'http://'.$url;
}
$url = validate_url($url);
if (!$url) die('Invalid URL supplied');
$url = filter_var($url, FILTER_SANITIZE_URL);
$test = filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);
// deal with bug http://bugs.php.net/51192 (present in PHP 5.2.13 and PHP 5.3.2)
if ($test === false) {
$test = filter_var(strtr($url, '-', '_'), FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);
}
if ($test !== false && $test !== null && preg_match('!^https?://!', $url)) {
// all okay
unset($test);
} else {
die('Invalid URL supplied');
}
debug("Supplied URL: $url");
/////////////////////////////////
@ -200,34 +193,19 @@ debug("Supplied URL: $url");
// (if in 'full' mode)
/////////////////////////////////
if ((_FF_FTR_MODE == 'full') && isset($_REQUEST['key']) && ($key_index = array_search($_REQUEST['key'], $options->api_keys)) !== false) {
$host = $_SERVER['HTTP_HOST'];
$path = rtrim(dirname($_SERVER['SCRIPT_NAME']), '/\\');
$_qs_url = (strtolower(substr($url, 0, 7)) == 'http://') ? substr($url, 7) : $url;
$redirect = 'http://'.htmlspecialchars($host.$path).'/makefulltextfeed.php?url='.urlencode($_qs_url);
$redirect .= '&key='.$key_index;
$redirect .= '&hash='.urlencode(sha1($_REQUEST['key'].$url));
if (isset($_REQUEST['html'])) $redirect .= '&html='.urlencode($_REQUEST['html']);
if (isset($_REQUEST['max'])) $redirect .= '&max='.(int)$_REQUEST['max'];
if (isset($_REQUEST['links'])) $redirect .= '&links='.urlencode($_REQUEST['links']);
if (isset($_REQUEST['exc'])) $redirect .= '&exc='.urlencode($_REQUEST['exc']);
if (isset($_REQUEST['format'])) $redirect .= '&format='.urlencode($_REQUEST['format']);
if (isset($_REQUEST['callback'])) $redirect .= '&callback='.urlencode($_REQUEST['callback']);
if (isset($_REQUEST['l'])) $redirect .= '&l='.urlencode($_REQUEST['l']);
if (isset($_REQUEST['lang'])) $redirect .= '&lang='.urlencode($_REQUEST['lang']);
if (isset($_REQUEST['xss'])) $redirect .= '&xss';
if (isset($_REQUEST['use_extracted_title'])) $redirect .= '&use_extracted_title';
if (isset($_REQUEST['content'])) $redirect .= '&content='.urlencode($_REQUEST['content']);
if (isset($_REQUEST['summary'])) $redirect .= '&summary='.urlencode($_REQUEST['summary']);
if (isset($_REQUEST['debug'])) $redirect .= '&debug';
if (isset($_REQUEST['parser'])) $redirect .= '&parser='.urlencode($_REQUEST['parser']);
if (isset($_REQUEST['proxy'])) $redirect .= '&proxy='.urlencode($_REQUEST['proxy']);
if ($debug_mode) {
debug('Redirecting to hide access key, follow URL below to continue');
debug("Location: $redirect");
if (isset($_REQUEST['key_redirect']) && $_REQUEST['key_redirect'] === '0') {
$_REQUEST['hash'] = sha1($_REQUEST['key'].$url);
$_REQUEST['key'] = $key_index;
} else {
header("Location: $redirect");
$redirect = get_self_url();
if ($debug_mode) {
debug('Redirecting to hide access key, follow URL below to continue');
debug("Location: $redirect");
} else {
header("Location: $redirect");
}
exit;
}
exit;
}
///////////////////////////////////////////////
@ -241,9 +219,25 @@ if (!ini_get('date.timezone') || !@date_default_timezone_set(ini_get('date.timez
}
///////////////////////////////////////////////
// Check if the request is explicitly for an HTML page
// Should we treat input URL as feed or HTML?
///////////////////////////////////////////////
$html_only = (isset($_REQUEST['html']) && ($_REQUEST['html'] == '1' || $_REQUEST['html'] == 'true'));
$accept = 'auto';
if (isset($_REQUEST['accept']) && in_array(strtolower($_REQUEST['accept']), array('html', 'feed', 'auto'))) {
$accept = strtolower($_REQUEST['accept']);
} elseif (isset($_REQUEST['html']) && ($_REQUEST['html'] == '1' || $_REQUEST['html'] == 'true')) {
$accept = 'html';
}
///////////////////////////////////////////////
// User-submitted site config
///////////////////////////////////////////////
$user_submitted_config = null;
if (isset($_REQUEST['siteconfig'])) {
$user_submitted_config = $_REQUEST['siteconfig'];
if (!$options->user_submitted_content && $user_submitted_config) {
die('User-submitted site configs are currently disabled. Please remove the siteconfig parameter.');
}
}
///////////////////////////////////////////////
// Check if valid key supplied
@ -463,8 +457,8 @@ if (isset($_REQUEST['inputhtml']) && _FF_FTR_MODE == 'simple') {
//////////////////////////////////
if ($options->caching) {
debug('Caching is enabled...');
$cache_id = md5($max.$url.(int)$valid_key.$links.(int)$favour_feed_titles.(int)$options->content.(int)$options->summary.
(int)$xss_filter.(int)$exclude_on_fail.$format.$detect_language.$parser._FF_FTR_MODE);
$cache_id = md5($max.$url.(int)$valid_key.$accept.$links.(int)$favour_feed_titles.(int)$options->content.(int)$options->summary.
(int)$xss_filter.(int)$exclude_on_fail.$format.$detect_language.$parser.$user_submitted_config._FF_FTR_MODE);
$check_cache = true;
if ($options->apc && $options->smart_cache) {
apc_add("cache.$cache_id", 0, $options->cache_time*60);
@ -548,11 +542,14 @@ SiteConfig::use_apc($options->apc);
$extractor->fingerprints = $options->fingerprints;
$extractor->allowedParsers = $options->allowed_parsers;
$extractor->parserOverride = $parser;
if ($options->user_submitted_config && $user_submitted_config) {
$extractor->setUserSubmittedConfig($user_submitted_config);
}
////////////////////////////////
// Get RSS/Atom feed
////////////////////////////////
if (!$html_only) {
if ($accept !== 'html') {
debug('--------');
debug("Attempting to process URL as feed");
// Send user agent header showing PHP (prevents a HTML response from feedburner)
@ -563,6 +560,9 @@ if (!$html_only) {
// some feeds use the text/html content type - force_feed tells SimplePie to process anyway
$feed->force_feed(true);
$feed->set_file_class('SimplePie_HumbleHttpAgent');
$feed->set_sanitize_class('DisableSimplePieSanitize');
// need to assign this manually it seems
$feed->sanitize = new DisableSimplePieSanitize();
//$feed->set_feed_url($url); // colons appearing in the URL's path get encoded
$feed->feed_url = $url;
$feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE);
@ -578,6 +578,8 @@ if (!$html_only) {
//$feed->get_title();
if ($result && (!is_array($feed->data) || count($feed->data) == 0)) {
die('Sorry, no feed items found');
} elseif (!$result && $accept === 'feed') {
die('Sorry, couldn\'t parse as feed');
}
// from now on, we'll identify ourselves as a browser
$http->userAgentDefault = HumbleHttpAgent::UA_BROWSER;
@ -589,7 +591,7 @@ if (!$html_only) {
// single-item feeds.
////////////////////////////////////////////////////////////////////////////////
$isDummyFeed = false;
if ($html_only || !$result) {
if ($accept === 'html' || !$result) {
debug('--------');
debug("Constructing a single-item feed from URL");
$isDummyFeed = true;
@ -627,6 +629,8 @@ if ($html_only || !$result) {
////////////////////////////////////////////
$output = new FeedWriter();
if (_FF_FTR_MODE === 'simple') $output->enableSimpleJson();
//$feed_title = $feed->get_title();
//echo $feed_title; exit;
$output->setTitle(strip_tags($feed->get_title()));
$output->setDescription(strip_tags($feed->get_description()));
$output->setXsl('css/feed.xsl'); // Chrome uses this, most browsers ignore it
@ -635,7 +639,9 @@ if ($ttl !== null) {
$ttl = (int)$ttl[0]['data'];
$output->setTtl($ttl);
}
//$output->setSelf('http://'.$_SERVER['HTTP_HOST'].$_SERVER['REQUEST_URI']);
$output->setSelf(get_self_url());
$output->setAlternate($url, 'Source URL');
$output->setRelated('http://www.subtome.com/#/subscribe?feeds='.urlencode(get_self_url()).'&back='.urlencode(get_self_url()), 'Subscribe to feed');
$output->setLink($feed->get_link()); // Google Reader uses this for pulling in favicons
if ($img_url = $feed->get_image_url()) {
$output->setImage($feed->get_title(), $feed->get_link(), $img_url);
@ -656,7 +662,12 @@ foreach ($items as $key => $item) {
// simplepie already sanitizes URLs so let's not do it again here.
//$permalink = $http->validateUrl($permalink);
if ($permalink) {
$urls_sanitized[] = $permalink;
if (!url_allowed($permalink)) {
debug('URL blocked, skipping...');
$permalink = false;
} else {
$urls_sanitized[] = $permalink;
}
}
$urls[$key] = $permalink;
}
@ -669,6 +680,7 @@ $http->fetchAll($urls_sanitized);
$item_count = 0;
foreach ($items as $key => $item) {
libxml_clear_errors();
debug('--------');
debug('Processing feed item '.($item_count+1));
$do_content_extraction = true;
@ -697,7 +709,10 @@ foreach ($items as $key => $item) {
// errors being treated as valid responses.
if ($permalink && ($response = $http->get($permalink, true)) && ($response['status_code'] < 300)) {
$effective_url = $response['effective_url'];
if (!url_allowed($effective_url)) continue;
if (!url_allowed($effective_url)) {
debug('URL blocked, skipping...');
continue;
}
// check if action defined for returned Content-Type
$mime_info = get_mime_action_info($response['headers']);
if (isset($mime_info['action'])) {
@ -727,7 +742,7 @@ foreach ($items as $key => $item) {
}
// check site config for single page URL - fetch it if found
$is_single_page = false;
if ($options->singlepage && ($single_page_response = getSinglePage($item, $html, $effective_url))) {
if ($options->singlepage && ($single_page_response = get_single_page($item, $html, $effective_url))) {
$is_single_page = true;
$effective_url = $single_page_response['effective_url'];
// check if action defined for returned Content-Type
@ -765,6 +780,13 @@ foreach ($items as $key => $item) {
debug("Here's the full HTML after it's been parsed by Full-Text RSS:");
die($readability->dom->saveXML($readability->dom->documentElement));
}
// is this a native ad?
if ($extract_result && $extractor->isNativeAd()) {
debug("This article appears to be a native ad");
if (!$isDummyFeed && $options->remove_native_ads) {
continue; // skip this feed item entry
}
}
$content_block = ($extract_result) ? $extractor->getContent() : null;
$extracted_title = ($extract_result) ? $extractor->getTitle() : '';
// Deal with multi-page articles
@ -779,7 +801,7 @@ foreach ($items as $key => $item) {
debug('--------');
debug('Processing next page: '.$next_page_url);
// If we've got URL, resolve against $url
if ($next_page_url = makeAbsoluteStr($effective_url, $next_page_url)) {
if ($next_page_url = make_absolute_str($effective_url, $next_page_url)) {
// check it's not what we have already!
if (!in_array($next_page_url, $multi_page_urls)) {
// it's not, so let's attempt to fetch it
@ -844,7 +866,12 @@ foreach ($items as $key => $item) {
$html .= $item->get_description();
} else {
$readability->clean($content_block, 'select');
if ($options->rewrite_relative_urls) makeAbsolute($effective_url, $content_block);
if ($options->rewrite_relative_urls) {
$base_url = get_base_url($readability->dom);
if (!$base_url) $base_url = $effective_url;
// rewrite URLs
make_absolute($base_url, $content_block);
}
// footnotes
if (($links == 'footnotes') && (strpos($effective_url, 'wikipedia.org') === false)) {
$readability->addFootnotes($content_block);
@ -987,12 +1014,16 @@ foreach ($items as $key => $item) {
// add effective URL (URL after redirects)
if (isset($effective_url)) {
//TODO: ensure $effective_url is valid witout - sometimes it causes problems, e.g.
//http://www.siasat.pk/forum/showthread.php?108883-Pakistan-Chowk-by-Rana-Mubashir-–-25th-March-2012-Special-Program-from-Liari-(Karachi)
//http://www.siasat.pk/forum/showthread.php?108883-Pakistan-Chowk-by-Rana-Mubashir-–-25th-March-2012-Special-Program-from-Liari-(Karachi)
//temporary measure: use utf8_encode()
$newitem->addElement('dc:identifier', remove_url_cruft(utf8_encode($effective_url)));
} else {
$newitem->addElement('dc:identifier', remove_url_cruft($item->get_permalink()));
}
// is this a native ad?
if ($extractor->isNativeAd()) {
$newitem->addElement('dc:type', 'Native Ad');
}
// add categories
if ($categories = $item->get_categories()) {
@ -1075,7 +1106,7 @@ if (!$debug_mode) {
$output->generateFeed();
$output = ob_get_contents();
ob_end_clean();
if ($html_only && $item_count == 0) {
if ($accept === 'html' && $item_count == 0) {
// do not cache - in case of temporary server glitch at source URL
} else {
$cache = get_cache();
@ -1092,10 +1123,77 @@ if (!$debug_mode) {
// HELPER FUNCTIONS
///////////////////////////////
function get_self_url() {
global $options, $url;
$scheme = (is_ssl()) ? 'https://' : 'http://';
$host = $_SERVER['HTTP_HOST'];
$path = rtrim(dirname($_SERVER['SCRIPT_NAME']), '/\\');
$_qs_url = (strtolower(substr($url, 0, 7)) == 'http://') ? substr($url, 7) : $url;
$self = $scheme.htmlspecialchars($host.$path).'/makefulltextfeed.php?url='.urlencode($_qs_url);
// hide API key if we can
if (isset($_GET['key']) && ($key_index = array_search($_GET['key'], $options->api_keys)) !== false) {
$_hash = sha1($_GET['key'].$url);
$self .= '&key='.$key_index;
$self .= '&hash='.urlencode($_hash);
} elseif(isset($_GET['key']) && isset($_GET['hash'])) {
$self .= '&key='.urlencode($_GET['key']);
$self .= '&hash='.urlencode($_GET['hash']);
}
if (isset($_GET['html'])) $self .= '&html='.urlencode($_GET['html']);
if (isset($_GET['accept'])) $self .= '&accept='.urlencode($_GET['accept']);
if (isset($_GET['max'])) $self .= '&max='.(int)$_GET['max'];
if (isset($_GET['links'])) $self .= '&links='.urlencode($_GET['links']);
if (isset($_GET['exc'])) $self .= '&exc='.urlencode($_GET['exc']);
if (isset($_GET['format'])) $self .= '&format='.urlencode($_GET['format']);
if (isset($_GET['callback'])) $self .= '&callback='.urlencode($_GET['callback']);
if (isset($_GET['l'])) $self .= '&l='.urlencode($_GET['l']);
if (isset($_GET['lang'])) $self .= '&lang='.urlencode($_GET['lang']);
if (isset($_GET['xss'])) $self .= '&xss';
if (isset($_GET['use_extracted_title'])) $self .= '&use_extracted_title';
if (isset($_GET['content'])) $self .= '&content='.urlencode($_GET['content']);
if (isset($_GET['summary'])) $self .= '&summary='.urlencode($_GET['summary']);
if (isset($_GET['debug'])) $self .= '&debug';
if (isset($_GET['parser'])) $self .= '&parser='.urlencode($_GET['parser']);
if (isset($_GET['proxy'])) $self .= '&proxy='.urlencode($_GET['proxy']);
if (isset($_GET['siteconfig'])) $self .= '&siteconfig='.urlencode($_GET['siteconfig']);
return $self;
}
function validate_url($url) {
$url = filter_var($url, FILTER_SANITIZE_URL);
$test = filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);
// deal with bug http://bugs.php.net/51192 (present in PHP 5.2.13 and PHP 5.3.2)
if ($test === false) {
$test = filter_var(strtr($url, '-', '_'), FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);
}
if ($test !== false && $test !== null && preg_match('!^https?://!i', $url)) {
return $url;
} else {
return false;
}
}
function get_base_url($dom) {
$xpath = new DOMXPath($dom);
return @$xpath->evaluate('string(//head/base/@href)', $dom);
}
function is_ssl() {
if (isset($_SERVER['HTTPS']) && ($_SERVER['HTTPS'] != '') && ($_SERVER['HTTPS'] != 'off')) {
return true;
} elseif (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] == 'https') {
return true;
} else {
return false;
}
}
// Adapted from WordPress
// http://core.trac.wordpress.org/browser/tags/3.5.1/wp-includes/formatting.php#L2173
function get_excerpt($text, $num_words=55, $more=null) {
if (null === $more) $more = '&hellip;';
if (null === $more) $more = '…';
$text = strip_tags($text);
//TODO: Check if word count is based on single characters (East Asian characters)
/*
@ -1183,9 +1281,10 @@ function convert_to_utf8($html, $header=null) {
}
}
}
if (isset($encoding)) $encoding = trim($encoding);
// trim is important here!
if (!$encoding || (strtolower($encoding) == 'iso-8859-1')) {
if (isset($encoding)) $encoding = strtolower(trim($encoding));
// fix bad encoding values
if ($encoding === 'iso-8850-1') $encoding = 'iso-8859-1';
if (!$encoding || ($encoding === 'iso-8859-1')) {
// replace MS Word smart qutoes
$trans = array();
$trans[chr(130)] = '&sbquo;'; // Single Low-9 Quotation Mark
@ -1219,7 +1318,7 @@ function convert_to_utf8($html, $header=null) {
$encoding = 'utf-8';
} else {
debug('Character encoding: '.$encoding);
if (strtolower($encoding) != 'utf-8') {
if ($encoding !== 'utf-8') {
debug('Converting to UTF-8');
$html = SimplePie_Misc::change_encoding($html, $encoding, 'utf-8');
}
@ -1228,7 +1327,7 @@ function convert_to_utf8($html, $header=null) {
return $html;
}
function makeAbsolute($base, $elem) {
function make_absolute($base, $elem) {
$base = new SimplePie_IRI($base);
// remove '//' in URL path (used to prevent URLs from resolving properly)
// TODO: check if this is still the case
@ -1238,12 +1337,12 @@ function makeAbsolute($base, $elem) {
for ($i = $elems->length-1; $i >= 0; $i--) {
$e = $elems->item($i);
//$e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);
makeAbsoluteAttr($base, $e, $attr);
make_absolute_attr($base, $e, $attr);
}
if (strtolower($elem->tagName) == $tag) makeAbsoluteAttr($base, $elem, $attr);
if (strtolower($elem->tagName) == $tag) make_absolute_attr($base, $elem, $attr);
}
}
function makeAbsoluteAttr($base, $e, $attr) {
function make_absolute_attr($base, $e, $attr) {
if ($e->hasAttribute($attr)) {
// Trim leading and trailing white space. I don't really like this but
// unfortunately it does appear on some sites. e.g. <img src=" /path/to/image.jpg" />
@ -1256,7 +1355,7 @@ function makeAbsoluteAttr($base, $e, $attr) {
}
}
}
function makeAbsoluteStr($base, $url) {
function make_absolute_str($base, $url) {
$base = new SimplePie_IRI($base);
// remove '//' in URL path (causes URLs not to resolve properly)
if (isset($base->path)) $base->path = preg_replace('!//+!', '/', $base->path);
@ -1271,7 +1370,7 @@ function makeAbsoluteStr($base, $url) {
}
}
// returns single page response, or false if not found
function getSinglePage($item, $html, $url) {
function get_single_page($item, $html, $url) {
global $http, $extractor;
debug('Looking for site config files to see if single page link exists');
$site_config = $extractor->buildSiteConfig($url, $html);
@ -1308,7 +1407,7 @@ function getSinglePage($item, $html, $url) {
}
}
// If we've got URL, resolve against $url
if (isset($single_page_url) && ($single_page_url = makeAbsoluteStr($url, $single_page_url))) {
if (isset($single_page_url) && ($single_page_url = make_absolute_str($url, $single_page_url))) {
// check it's not what we have already!
if ($single_page_url != $url) {
// it's not, so let's try to fetch it...

16
manifest.yml
View File

@ -1,16 +0,0 @@
# This file is only used when deploying Full-Text RSS to AppFog.
# See http://help.fivefilters.org/customer/portal/articles/1143210-hosting
---
applications:
.:
# name: full-text-rss
framework:
name: php
info:
mem: 512M
description: PHP Application
exec:
infra: aws
# url: ${name}.${target-base}
mem: 512M
instances: 1