261 lines
8.4 KiB
Python
261 lines
8.4 KiB
Python
|
"""
|
||
|
|
An interface to html5lib that mimics the lxml.html interface.
|
||
|
|
"""
|
||
|
|
import sys
|
||
|
|
import string
|
||
|
|
|
||
|
|
from html5lib import HTMLParser as _HTMLParser
|
||
|
|
from html5lib.treebuilders.etree_lxml import TreeBuilder
|
||
|
|
from lxml import etree
|
||
|
|
from lxml.html import Element, XHTML_NAMESPACE, _contains_block_level_tag
|
||
|
|
|
||
|
|
# python3 compatibility
|
||
|
|
try:
|
||
|
|
_strings = basestring
|
||
|
|
except NameError:
|
||
|
|
_strings = (bytes, str)
|
||
|
|
try:
|
||
|
|
from urllib2 import urlopen
|
||
|
|
except ImportError:
|
||
|
|
from urllib.request import urlopen
|
||
|
|
try:
|
||
|
|
from urlparse import urlparse
|
||
|
|
except ImportError:
|
||
|
|
from urllib.parse import urlparse
|
||
|
|
|
||
|
|
|
||
|
|
class HTMLParser(_HTMLParser):
|
||
|
|
"""An html5lib HTML parser with lxml as tree."""
|
||
|
|
|
||
|
|
def __init__(self, strict=False, **kwargs):
|
||
|
|
_HTMLParser.__init__(self, strict=strict, tree=TreeBuilder, **kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
try:
|
||
|
|
from html5lib import XHTMLParser as _XHTMLParser
|
||
|
|
except ImportError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
class XHTMLParser(_XHTMLParser):
|
||
|
|
"""An html5lib XHTML Parser with lxml as tree."""
|
||
|
|
|
||
|
|
def __init__(self, strict=False, **kwargs):
|
||
|
|
_XHTMLParser.__init__(self, strict=strict, tree=TreeBuilder, **kwargs)
|
||
|
|
|
||
|
|
xhtml_parser = XHTMLParser()
|
||
|
|
|
||
|
|
|
||
|
|
def _find_tag(tree, tag):
|
||
|
|
elem = tree.find(tag)
|
||
|
|
if elem is not None:
|
||
|
|
return elem
|
||
|
|
return tree.find('{%s}%s' % (XHTML_NAMESPACE, tag))
|
||
|
|
|
||
|
|
|
||
|
|
def document_fromstring(html, guess_charset=None, parser=None):
|
||
|
|
"""
|
||
|
|
Parse a whole document into a string.
|
||
|
|
|
||
|
|
If `guess_charset` is true, or if the input is not Unicode but a
|
||
|
|
byte string, the `chardet` library will perform charset guessing
|
||
|
|
on the string.
|
||
|
|
"""
|
||
|
|
if not isinstance(html, _strings):
|
||
|
|
raise TypeError('string required')
|
||
|
|
|
||
|
|
if parser is None:
|
||
|
|
parser = html_parser
|
||
|
|
|
||
|
|
options = {}
|
||
|
|
if guess_charset is None and isinstance(html, bytes):
|
||
|
|
# html5lib does not accept useChardet as an argument, if it
|
||
|
|
# detected the html argument would produce unicode objects.
|
||
|
|
guess_charset = True
|
||
|
|
if guess_charset is not None:
|
||
|
|
options['useChardet'] = guess_charset
|
||
|
|
return parser.parse(html, **options).getroot()
|
||
|
|
|
||
|
|
|
||
|
|
def fragments_fromstring(html, no_leading_text=False,
|
||
|
|
guess_charset=None, parser=None):
|
||
|
|
"""Parses several HTML elements, returning a list of elements.
|
||
|
|
|
||
|
|
The first item in the list may be a string. If no_leading_text is true,
|
||
|
|
then it will be an error if there is leading text, and it will always be
|
||
|
|
a list of only elements.
|
||
|
|
|
||
|
|
If `guess_charset` is true, the `chardet` library will perform charset
|
||
|
|
guessing on the string.
|
||
|
|
"""
|
||
|
|
if not isinstance(html, _strings):
|
||
|
|
raise TypeError('string required')
|
||
|
|
|
||
|
|
if parser is None:
|
||
|
|
parser = html_parser
|
||
|
|
|
||
|
|
options = {}
|
||
|
|
if guess_charset is None and isinstance(html, bytes):
|
||
|
|
# html5lib does not accept useChardet as an argument, if it
|
||
|
|
# detected the html argument would produce unicode objects.
|
||
|
|
guess_charset = False
|
||
|
|
if guess_charset is not None:
|
||
|
|
options['useChardet'] = guess_charset
|
||
|
|
children = parser.parseFragment(html, 'div', **options)
|
||
|
|
if children and isinstance(children[0], _strings):
|
||
|
|
if no_leading_text:
|
||
|
|
if children[0].strip():
|
||
|
|
raise etree.ParserError('There is leading text: %r' %
|
||
|
|
children[0])
|
||
|
|
del children[0]
|
||
|
|
return children
|
||
|
|
|
||
|
|
|
||
|
|
def fragment_fromstring(html, create_parent=False,
|
||
|
|
guess_charset=None, parser=None):
|
||
|
|
"""Parses a single HTML element; it is an error if there is more than
|
||
|
|
one element, or if anything but whitespace precedes or follows the
|
||
|
|
element.
|
||
|
|
|
||
|
|
If 'create_parent' is true (or is a tag name) then a parent node
|
||
|
|
will be created to encapsulate the HTML in a single element. In
|
||
|
|
this case, leading or trailing text is allowed.
|
||
|
|
|
||
|
|
If `guess_charset` is true, the `chardet` library will perform charset
|
||
|
|
guessing on the string.
|
||
|
|
"""
|
||
|
|
if not isinstance(html, _strings):
|
||
|
|
raise TypeError('string required')
|
||
|
|
|
||
|
|
accept_leading_text = bool(create_parent)
|
||
|
|
|
||
|
|
elements = fragments_fromstring(
|
||
|
|
html, guess_charset=guess_charset, parser=parser,
|
||
|
|
no_leading_text=not accept_leading_text)
|
||
|
|
|
||
|
|
if create_parent:
|
||
|
|
if not isinstance(create_parent, _strings):
|
||
|
|
create_parent = 'div'
|
||
|
|
new_root = Element(create_parent)
|
||
|
|
if elements:
|
||
|
|
if isinstance(elements[0], _strings):
|
||
|
|
new_root.text = elements[0]
|
||
|
|
del elements[0]
|
||
|
|
new_root.extend(elements)
|
||
|
|
return new_root
|
||
|
|
|
||
|
|
if not elements:
|
||
|
|
raise etree.ParserError('No elements found')
|
||
|
|
if len(elements) > 1:
|
||
|
|
raise etree.ParserError('Multiple elements found')
|
||
|
|
result = elements[0]
|
||
|
|
if result.tail and result.tail.strip():
|
||
|
|
raise etree.ParserError('Element followed by text: %r' % result.tail)
|
||
|
|
result.tail = None
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def fromstring(html, guess_charset=None, parser=None):
|
||
|
|
"""Parse the html, returning a single element/document.
|
||
|
|
|
||
|
|
This tries to minimally parse the chunk of text, without knowing if it
|
||
|
|
is a fragment or a document.
|
||
|
|
|
||
|
|
'base_url' will set the document's base_url attribute (and the tree's
|
||
|
|
docinfo.URL)
|
||
|
|
|
||
|
|
If `guess_charset` is true, or if the input is not Unicode but a
|
||
|
|
byte string, the `chardet` library will perform charset guessing
|
||
|
|
on the string.
|
||
|
|
"""
|
||
|
|
if not isinstance(html, _strings):
|
||
|
|
raise TypeError('string required')
|
||
|
|
doc = document_fromstring(html, parser=parser,
|
||
|
|
guess_charset=guess_charset)
|
||
|
|
|
||
|
|
# document starts with doctype or <html>, full document!
|
||
|
|
start = html[:50]
|
||
|
|
if isinstance(start, bytes):
|
||
|
|
# Allow text comparison in python3.
|
||
|
|
# Decode as ascii, that also covers latin-1 and utf-8 for the
|
||
|
|
# characters we need.
|
||
|
|
start = start.decode('ascii', 'replace')
|
||
|
|
|
||
|
|
start = start.lstrip().lower()
|
||
|
|
if start.startswith('<html') or start.startswith('<!doctype'):
|
||
|
|
return doc
|
||
|
|
|
||
|
|
head = _find_tag(doc, 'head')
|
||
|
|
|
||
|
|
# if the head is not empty we have a full document
|
||
|
|
if len(head):
|
||
|
|
return doc
|
||
|
|
|
||
|
|
body = _find_tag(doc, 'body')
|
||
|
|
|
||
|
|
# The body has just one element, so it was probably a single
|
||
|
|
# element passed in
|
||
|
|
if (len(body) == 1 and (not body.text or not body.text.strip())
|
||
|
|
and (not body[-1].tail or not body[-1].tail.strip())):
|
||
|
|
return body[0]
|
||
|
|
|
||
|
|
# Now we have a body which represents a bunch of tags which have the
|
||
|
|
# content that was passed in. We will create a fake container, which
|
||
|
|
# is the body tag, except <body> implies too much structure.
|
||
|
|
if _contains_block_level_tag(body):
|
||
|
|
body.tag = 'div'
|
||
|
|
else:
|
||
|
|
body.tag = 'span'
|
||
|
|
return body
|
||
|
|
|
||
|
|
|
||
|
|
def parse(filename_url_or_file, guess_charset=None, parser=None):
|
||
|
|
"""Parse a filename, URL, or file-like object into an HTML document
|
||
|
|
tree. Note: this returns a tree, not an element. Use
|
||
|
|
``parse(...).getroot()`` to get the document root.
|
||
|
|
|
||
|
|
If ``guess_charset`` is true, the ``useChardet`` option is passed into
|
||
|
|
html5lib to enable character detection. This option is on by default
|
||
|
|
when parsing from URLs, off by default when parsing from file(-like)
|
||
|
|
objects (which tend to return Unicode more often than not), and on by
|
||
|
|
default when parsing from a file path (which is read in binary mode).
|
||
|
|
"""
|
||
|
|
if parser is None:
|
||
|
|
parser = html_parser
|
||
|
|
if not isinstance(filename_url_or_file, _strings):
|
||
|
|
fp = filename_url_or_file
|
||
|
|
if guess_charset is None:
|
||
|
|
# assume that file-like objects return Unicode more often than bytes
|
||
|
|
guess_charset = False
|
||
|
|
elif _looks_like_url(filename_url_or_file):
|
||
|
|
fp = urlopen(filename_url_or_file)
|
||
|
|
if guess_charset is None:
|
||
|
|
# assume that URLs return bytes
|
||
|
|
guess_charset = True
|
||
|
|
else:
|
||
|
|
fp = open(filename_url_or_file, 'rb')
|
||
|
|
if guess_charset is None:
|
||
|
|
guess_charset = True
|
||
|
|
|
||
|
|
options = {}
|
||
|
|
# html5lib does not accept useChardet as an argument, if it
|
||
|
|
# detected the html argument would produce unicode objects.
|
||
|
|
if guess_charset:
|
||
|
|
options['useChardet'] = guess_charset
|
||
|
|
return parser.parse(fp, **options)
|
||
|
|
|
||
|
|
|
||
|
|
def _looks_like_url(str):
|
||
|
|
scheme = urlparse(str)[0]
|
||
|
|
if not scheme:
|
||
|
|
return False
|
||
|
|
elif (sys.platform == 'win32' and
|
||
|
|
scheme in string.ascii_letters
|
||
|
|
and len(scheme) == 1):
|
||
|
|
# looks like a 'normal' absolute path
|
||
|
|
return False
|
||
|
|
else:
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
html_parser = HTMLParser()
|