python concurrency

1import concurrent.futures
2import requests
3import threading
4import time
5
6
7thread_local = threading.local()
8
9
10def get_session():
11    if not hasattr(thread_local, "session"):
12        thread_local.session = requests.Session()
13    return thread_local.session
14
15
16def download_site(url):
17    session = get_session()
18    with session.get(url) as response:
19        print(f"Read {len(response.content)} from {url}")
20
21
22def download_all_sites(sites):
23    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
24        executor.map(download_site, sites)
25
26
27if __name__ == "__main__":
28    sites = [
29        "https://www.jython.org",
30        "http://olympus.realpython.org/dice",
31    ] * 80
32    start_time = time.time()
33    download_all_sites(sites)
34    duration = time.time() - start_time
35    print(f"Downloaded {len(sites)} in {duration} seconds")
36