fragment is returned.
Otherwise full HTML document is returned.
"""
body_element = dom.find(".//body")
if body_element is None:
fragment = fragment_fromstring('
')
fragment.append(dom)
else:
body_element.tag = "div"
body_element.set("id", "readabilityBody")
fragment = body_element
return document_from_fragment(fragment, return_fragment)
def build_error_document(dom, return_fragment=True):
"""
Builds an empty erorr document with the body as root.
:param bool return_fragment: If True only
fragment is returned.
Otherwise full HTML document is returned.
"""
fragment = fragment_fromstring(
'
')
return document_from_fragment(fragment, return_fragment)
def document_from_fragment(fragment, return_fragment):
if return_fragment:
document = fragment
else:
document = fromstring(NULL_DOCUMENT)
body_element = document.find(".//body")
body_element.append(fragment)
document.doctype = ""
return document
def check_siblings(candidate_node, candidate_list):
"""
Looks through siblings for content that might also be related.
Things like preambles, content split by ads that we removed, etc.
"""
candidate_css = candidate_node.node.get("class")
potential_target = candidate_node.content_score * 0.2
sibling_target_score = potential_target if potential_target > 10 else 10
parent = candidate_node.node.getparent()
siblings = parent.getchildren() if parent is not None else []
for sibling in siblings:
append = False
content_bonus = 0
if sibling is candidate_node.node:
append = True
# Give a bonus if sibling nodes and top candidates have the example
# same class name
if candidate_css and sibling.get("class") == candidate_css:
content_bonus += candidate_node.content_score * 0.2
if sibling in candidate_list:
adjusted_score = \
candidate_list[sibling].content_score + content_bonus
if adjusted_score >= sibling_target_score:
append = True
if sibling.tag == "p":
link_density = get_link_density(sibling)
content = sibling.text_content()
content_length = len(content)
if content_length > 80 and link_density < 0.25:
append = True
elif content_length < 80 and link_density == 0:
if ". " in content:
append = True
if append:
logger.debug(
"Sibling appended: %s %r", sibling.tag, sibling.attrib)
if sibling.tag not in ("div", "p"):
# We have a node that isn't a common block level element, like
# a form or td tag. Turn it into a div so it doesn't get
# filtered out later by accident.
sibling.tag = "div"
if candidate_node.node != sibling:
candidate_node.node.append(sibling)
return candidate_node
def clean_document(node):
"""Cleans up the final document we return as the readable article."""
if node is None or len(node) == 0:
return None
logger.debug("\n\n-------------- CLEANING DOCUMENT -----------------")
to_drop = []
for n in node.iter():
# clean out any in-line style properties
if "style" in n.attrib:
n.set("style", "")
# remove embended objects unless it's wanted video
if n.tag in ("object", "embed") and not ok_embedded_video(n):
logger.debug("Dropping node %s %r", n.tag, n.attrib)
to_drop.append(n)
# clean headings with bad css or high link density
if n.tag in ("h1", "h2", "h3", "h4") and get_class_weight(n) < 0:
logger.debug("Dropping <%s>, it's insignificant", n.tag)
to_drop.append(n)
if n.tag in ("h3", "h4") and get_link_density(n) > 0.33:
logger.debug("Dropping <%s>, it's insignificant", n.tag)
to_drop.append(n)
# drop block element without content and children
if n.tag in ("div", "p"):
text_content = shrink_text(n.text_content())
if len(text_content) < 5 and not n.getchildren():
logger.debug(
"Dropping %s %r without content.", n.tag, n.attrib)
to_drop.append(n)
# finally try out the conditional cleaning of the target node
if clean_conditionally(n):
to_drop.append(n)
drop_nodes_with_parents(to_drop)
return node
def drop_nodes_with_parents(nodes):
for node in nodes:
if node.getparent() is None:
continue
node.drop_tree()
logger.debug(
"Dropped node with parent %s %r %s",
node.tag,
node.attrib,
node.text_content()[:50]
)
def clean_conditionally(node):
"""Remove the clean_el if it looks like bad content based on rules."""
if node.tag not in ('form', 'table', 'ul', 'div', 'p'):
return # this is not the tag we are looking for
weight = get_class_weight(node)
# content_score = LOOK up the content score for this node we found
# before else default to 0
content_score = 0
if weight + content_score < 0:
logger.debug('Dropping conditional node')
logger.debug('Weight + score < 0')
return True
commas_count = node.text_content().count(',')
if commas_count < 10:
logger.debug(
"There are %d commas so we're processing more.", commas_count)
# If there are not very many commas, and the number of
# non-paragraph elements is more than paragraphs or other ominous
# signs, remove the element.
p = len(node.findall('.//p'))
img = len(node.findall('.//img'))
li = len(node.findall('.//li')) - 100
inputs = len(node.findall('.//input'))
embed = 0
embeds = node.findall('.//embed')
for e in embeds:
if ok_embedded_video(e):
embed += 1
link_density = get_link_density(node)
content_length = len(node.text_content())
remove_node = False
if li > p and node.tag != 'ul' and node.tag != 'ol':
logger.debug('Conditional drop: li > p and not ul/ol')
remove_node = True
elif inputs > p / 3.0:
logger.debug('Conditional drop: inputs > p/3.0')
remove_node = True
elif content_length < 25 and (img == 0 or img > 2):
logger.debug('Conditional drop: len < 25 and 0/>2 images')
remove_node = True
elif weight < 25 and link_density > 0.2:
logger.debug('Conditional drop: weight small (%f) and link is dense (%f)', weight, link_density)
remove_node = True
elif weight >= 25 and link_density > 0.5:
logger.debug('Conditional drop: weight big but link heavy')
remove_node = True
elif (embed == 1 and content_length < 75) or embed > 1:
logger.debug(
'Conditional drop: embed w/o much content or many embed')
remove_node = True
if remove_node:
logger.debug('Node will be removed: %s %r %s', node.tag, node.attrib, node.text_content()[:30])
return remove_node
return False # nope, don't remove anything
def prep_article(doc):
"""Once we've found our target article we want to clean it up.
Clean out:
- inline styles
- forms
- strip empty
- extra tags
"""
return clean_document(doc)
def find_candidates(document):
"""
Finds cadidate nodes for the readable version of the article.
Here's we're going to remove unlikely nodes, find scores on the rest,
clean up and return the final best match.
"""
nodes_to_score = set()
should_remove = set()
for node in document.iter():
if is_unlikely_node(node):
logger.debug(
"We should drop unlikely: %s %r", node.tag, node.attrib)
should_remove.add(node)
elif is_bad_link(node):
logger.debug(
"We should drop bad link: %s %r", node.tag, node.attrib)
should_remove.add(node)
elif node.tag in SCORABLE_TAGS:
nodes_to_score.add(node)
return score_candidates(nodes_to_score), should_remove
def is_bad_link(node):
"""
Helper to determine if the node is link that is useless.
We've hit articles with many multiple links that should be cleaned out
because they're just there to pollute the space. See tests for examples.
"""
if node.tag != "a":
return False
name = node.get("name")
href = node.get("href")
if name and not href:
return True
if href:
href_parts = href.split("#")
if len(href_parts) == 2 and len(href_parts[1]) > 25:
return True
return False
class Article(object):
"""Parsed readable object"""
def __init__(self, html, url=None, return_fragment=True):
"""
Create the Article we're going to use.
:param html: The string of HTML we're going to parse.
:param url: The url so we can adjust the links to still work.
:param return_fragment: Should we return a
fragment or
a full document.
"""
self._original_document = OriginalDocument(html, url=url)
self._return_fragment = return_fragment
def __str__(self):
return tostring(self._readable())
def __unicode__(self):
return tounicode(self._readable())
@cached_property
def dom(self):
"""Parsed lxml tree (Document Object Model) of the given html."""
try:
dom = self._original_document.dom
# cleaning doesn't return, just wipes in place
html_cleaner(dom)
return leaf_div_elements_into_paragraphs(dom)
except ValueError:
return None
@cached_property
def candidates(self):
"""Generates list of candidates from the DOM."""
dom = self.dom
if dom is None or len(dom) == 0:
return None
candidates, unlikely_candidates = find_candidates(dom)
drop_nodes_with_parents(unlikely_candidates)
return candidates
@cached_property
def main_text(self):
dom = deepcopy(self.readable_dom).get_element_by_id("readabilityBody")
return AnnotatedTextHandler.parse(dom)
@cached_property
def readable(self):
return tounicode(self.readable_dom)
@cached_property
def readable_dom(self):
return self._readable()
def _readable(self):
"""The readable parsed article"""
if not self.candidates:
logger.info("No candidates found in document.")
return self._handle_no_candidates()
# right now we return the highest scoring candidate content
best_candidates = sorted(
(c for c in self.candidates.values()),
key=attrgetter("content_score"), reverse=True)
printer = PrettyPrinter(indent=2)
logger.debug(printer.pformat(best_candidates))
# since we have several candidates, check the winner's siblings
# for extra content
winner = best_candidates[0]
updated_winner = check_siblings(winner, self.candidates)
updated_winner.node = prep_article(updated_winner.node)
if updated_winner.node is not None:
dom = build_base_document(
updated_winner.node, self._return_fragment)
else:
logger.info(
'Had candidates but failed to find a cleaned winning DOM.')
dom = self._handle_no_candidates()
return self._remove_orphans(dom.get_element_by_id("readabilityBody"))
def _remove_orphans(self, dom):
for node in dom.iterdescendants():
if len(node) == 1 and tuple(node)[0].tag == node.tag:
node.drop_tag()
return dom
def _handle_no_candidates(self):
"""
If we fail to find a good candidate we need to find something else.
"""
# since we've not found a good candidate we're should help this
if self.dom is not None and len(self.dom):
dom = prep_article(self.dom)
dom = build_base_document(dom, self._return_fragment)
return self._remove_orphans(
dom.get_element_by_id("readabilityBody"))
else:
logger.info("No document to use.")
return build_error_document(self._return_fragment)
def leaf_div_elements_into_paragraphs(document):
"""
Turn some block elements that don't have children block level
elements into
elements.
Since we can't change the tree as we iterate over it, we must do this
before we process our document.
"""
for element in document.iter(tag="div"):
child_tags = tuple(n.tag for n in element.getchildren())
if "div" not in child_tags and "p" not in child_tags:
logger.debug(
"Changing leaf block element <%s> into
", element.tag)
element.tag = "p"
return document