python beautifulsoup example

1import requests 
2from bs4 import BeautifulSoup
3
4URL = 'https://the.website_you_want_to_visit.com'
5website = requests.get(URL)
6results = BeautifulSoup(website.content, 'html.parser')

1def simple_get(url):
2    """
3    Attempts to get the content at `url` by making an HTTP GET request.
4    If the content-type of response is some kind of HTML/XML, return the
5    text content, otherwise return None.
6    """
7    try:
8        with closing(get(url, stream=True)) as resp:
9            if is_good_response(resp):
10                return resp.content
11            else:
12                return None
13
14    except RequestException as e:
15        log_error('Error during requests to {0} : {1}'.format(url, str(e)))
16        return None
17
18
19def is_good_response(resp):
20    """
21    Returns True if the response seems to be HTML, False otherwise.
22    """
23    content_type = resp.headers['Content-Type'].lower()
24    return (resp.status_code == 200 
25            and content_type is not None 
26            and content_type.find('html') > -1)
27
28
29def log_error(e):
30    """
31    It is always a good idea to log errors. 
32    This function just prints them, but you can
33    make it do anything.
34    """
35    print(e)
36