Scraping web pages with python: Difference between revisions

From XPUB & Lens-Based wiki
No edit summary
No edit summary
 
(25 intermediate revisions by the same user not shown)
Line 1: Line 1:
The html5lib parser is code that turns the source text of an HTML page
See also: [[Filtering HTML with python]]
into a structured object, allowing, for instance, to use CSS selectors
or xpath expressions to select/extract portions of a page


You can use xpath expressions:
== Using html5lib + elementtree ==
 
Back in the day, working with HTML pages with python's standard library was often frustrating as most web pages "in the wild" didn't conform to the rigid restrictions of XML. As a result projects like Beautiful Soup were created that made working with HTML quite easy. Happily the lessons learned from BeautifulSoup have incorporated into modern libraries like html5lib. At the same time, some of the ugliness of working with XML via standard interfaces like SAX were improved with Fredrick Lundh's work on [http://effbot.org/zone/element-index.htm ElementTree] which is part of python's [https://docs.python.org/3.7/library/xml.etree.elementtree.html?highlight=elementtree standard library].
 
=== Find all the links (a) on the front page of nytimes.com and print their href and label ===


<source lang="python">
<source lang="python">
import html5lib, lxml
import html5lib
import xml.etree.ElementTree as ET
from urllib.request import urlopen
from urllib.parse import urljoin
 
 
url = "https://nytimes.com/"
with urlopen(url) as f:
    t = html5lib.parse(f, namespaceHTMLElements=False)
 
print ("Link", "Label")
for a in t.findall('.//a[@href]'):
    # Absolutize any relative links with urljoin
    href = urljoin(url, a.attrib.get('href'))
    print(href, a.text)  # link, label


htmlsource="<html><body><p>Example page.</p><p>More stuff.</p></body></html>"
htmlparser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("lxml"), namespaceHTMLElements=False)
page = htmlparser.parse(htmlsource)
p = page.xpath("/html/body/p[2]")
if p:
    p = p[0]
    print "".join([t for t in p.itertext()])
</source>
</source>


outputs:
=== Print the contents of a document or particular tag ===
More stuff.
 
<source lang="python">
print(ET.tostring(sometag, encoding='unicode'))
</source>
 
=== Scraping from a local file ===
<source lang="python">
with open("myfile.html") as f:
    t = html5lib.parse(f, namespaceHTMLElements=False)
</source>


Also CSS selectors are possible:
=== Generic page scraping ===
The ''.iter'' function lets you scan through all the elements on a page and run code on them to filter them in whatever way you want. The ''.tag'' gives you access to the tagname (lowercase), and ''.text'' to the text contents of the tag.


<source lang="python">
<source lang="python">
import html5lib, lxml, lxml.cssselect
import html5lib
import xml.etree.ElementTree as ET
from urllib.request import urlopen
from urllib.parse import urljoin
 


htmlsource="<html><body><p>Example page.</p><p>More stuff.</p></body></html>"
url = "https://nytimes.com/"
htmlparser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("lxml"), namespaceHTMLElements=False)
with urlopen(url) as f:
page = htmlparser.parse(htmlsource)
    t = html5lib.parse(f, namespaceHTMLElements=False)
selector = lxml.cssselect.CSSSelector("p")
for p in selector(page):
    print "-"*20
    print "".join([t for t in p.itertext()])


for x in t.iter():
    if x.text != None and "trump" in x.text.lower() and x.tag != "script":
        print (x.tag, x.text)
</source>
</source>


--------------------
=== Setting the User Agent ===
Example page.
Some web servers block bots by simply rejecting requests that don't identify themselves via the "user agent" http header. This is easy enough to set (aka "spoof").
--------------------
 
More stuff.
See: https://stackoverflow.com/questions/24226781/changing-user-agent-in-python-3-for-urrlib-request-urlopen


<source lang="python">
<source lang="python">
import urllib2, html5lib, lxml, lxml.etree
import urllib.request
req = urllib.request.Request(
def getXpath (url, xpath):
    "http://nytimes.com",  
    htmlparser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("lxml"), namespaceHTMLElements=False)
     headers={
     request = urllib2.Request(url)
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'
    request.add_header("User-Agent", "Mozilla/5.0 (X11; U; Linux x86_64; fr; rv:1.9.1.5) Gecko/20091109 Ubuntu/9.10 (karmic) Firefox/3.5.5")
    })
    f=urllib2.urlopen(request)
f = urllib.request.urlopen(req)
   
 
    page = htmlparser.parse(f)
print (f.code)
    return page.xpath(xpath)
</source>
 
=== A spider ===
 
<source lang="python">
import html5lib, sys
import xml.etree.ElementTree as ET
from urllib.request import urlopen
from urllib.parse import urljoin
from urllib.error import HTTPError


if __name__ == "__main__":
    url = "http://www.jabberwocky.com/carroll/walrus.html"
    xpath = "/html/body/p[6]"
    print lxml.etree.tostring(getXpath(url, xpath)[0])


</source>
url = 'https://news.bbc.co.uk'
todo = [url]
seen = set()
printed = set()


while todo:
    url = todo[0]
    todo = todo[1:]
    print('Scraping', url, file=sys.stderr)
    try:
        with urlopen(url) as f:
            t = html5lib.parse(f, namespaceHTMLElements=False)
            seen.add(url)
 
        for a in t.findall('.//a[@href]'):
            href = urljoin(url, a.attrib.get('href'))
            #print(ET.tostring(a, encoding='unicode'))
            if href not in printed:
                text = a.text or ''
                print(href, text.strip())  # link, label
                printed.add(href)
            if href not in seen:
                todo.append(href)
    except HTTPError:
        print('Page not found!!111', file=sys.stderr)


[[Category: Cookbook]] [[Category: xpath]] [[Category: python]] [[Category: lxml]]
</source>

Latest revision as of 15:56, 23 May 2020

See also: Filtering HTML with python

Using html5lib + elementtree

Back in the day, working with HTML pages with python's standard library was often frustrating as most web pages "in the wild" didn't conform to the rigid restrictions of XML. As a result projects like Beautiful Soup were created that made working with HTML quite easy. Happily the lessons learned from BeautifulSoup have incorporated into modern libraries like html5lib. At the same time, some of the ugliness of working with XML via standard interfaces like SAX were improved with Fredrick Lundh's work on ElementTree which is part of python's standard library.

Find all the links (a) on the front page of nytimes.com and print their href and label

import html5lib
import xml.etree.ElementTree as ET
from urllib.request import urlopen
from urllib.parse import urljoin


url = "https://nytimes.com/"
with urlopen(url) as f:
    t = html5lib.parse(f, namespaceHTMLElements=False)

print ("Link", "Label")
for a in t.findall('.//a[@href]'):
    # Absolutize any relative links with urljoin
    href = urljoin(url, a.attrib.get('href'))
    print(href, a.text)  # link, label

Print the contents of a document or particular tag

print(ET.tostring(sometag, encoding='unicode'))

Scraping from a local file

with open("myfile.html") as f:
    t = html5lib.parse(f, namespaceHTMLElements=False)

Generic page scraping

The .iter function lets you scan through all the elements on a page and run code on them to filter them in whatever way you want. The .tag gives you access to the tagname (lowercase), and .text to the text contents of the tag.

import html5lib
import xml.etree.ElementTree as ET
from urllib.request import urlopen
from urllib.parse import urljoin


url = "https://nytimes.com/"
with urlopen(url) as f:
    t = html5lib.parse(f, namespaceHTMLElements=False)

for x in t.iter():
    if x.text != None and "trump" in x.text.lower() and x.tag != "script":
        print (x.tag, x.text)

Setting the User Agent

Some web servers block bots by simply rejecting requests that don't identify themselves via the "user agent" http header. This is easy enough to set (aka "spoof").

See: https://stackoverflow.com/questions/24226781/changing-user-agent-in-python-3-for-urrlib-request-urlopen

import urllib.request
req = urllib.request.Request(
    "http://nytimes.com", 
    headers={
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'
    })
f = urllib.request.urlopen(req)

print (f.code)

A spider

import html5lib, sys
import xml.etree.ElementTree as ET
from urllib.request import urlopen
from urllib.parse import urljoin
from urllib.error import HTTPError


url = 'https://news.bbc.co.uk'
todo = [url]
seen = set()
printed = set()

while todo:
    url = todo[0]
    todo = todo[1:]
    print('Scraping', url, file=sys.stderr)
 
    try:
        with urlopen(url) as f:
            t = html5lib.parse(f, namespaceHTMLElements=False)
            seen.add(url)
  
        for a in t.findall('.//a[@href]'):
            href = urljoin(url, a.attrib.get('href'))
            #print(ET.tostring(a, encoding='unicode'))
            if href not in printed:
                text = a.text or ''
                print(href, text.strip())  # link, label
                printed.add(href)
            if href not in seen:
                todo.append(href)
    except HTTPError:
        print('Page not found!!111', file=sys.stderr)