Weeks 4–6 — Threading & Multiprocessing · Asyncio · Testing with pytest
Write concurrent programs with threading and multiprocessing. Understand the GIL, synchronize shared state with Lock and Queue, and use concurrent.futures for a clean high-level API. Write asynchronous programs with asyncio. Understand coroutines, tasks, the event loop, and use aiohttp for non-blocking HTTP requests. Write professional tests with pytest. Use fixtures for setup/teardown, parametrize for data-driven tests, mock external dependencies, and measure code coverage.
What you'll learn
- 1Explain the GIL and when threading vs multiprocessing is appropriate
- 2Create and join threads with threading.Thread
- 3Synchronize shared state with Lock
- 4Use Queue for producer-consumer patterns
- 5Speed up CPU-bound tasks with ProcessPoolExecutor
- 6Define coroutines with async def and await
- 7Run coroutines with asyncio.run() and create tasks
- 8Fetch multiple URLs concurrently with asyncio.gather
- 9Limit concurrency with asyncio.Semaphore
- 10Use asyncio.Queue for async producer-consumer
- 11Write test functions and classes with pytest
- 12Use fixtures for reusable setup and teardown
- 13Parametrize tests to cover multiple inputs
- 14Mock external dependencies with unittest.mock
- 15Measure and improve code coverage
Week 4 — Threading & Multiprocessing
Write concurrent programs with threading and multiprocessing. Understand the GIL, synchronize shared state with Lock and Queue, and use concurrent.futures for a clean high-level API.
1. Threading
The GIL allows only one thread to execute Python bytecode at a time. Threading is best for I/O-bound tasks (network, disk).
import threading, time
def fetch(url, results, idx):
time.sleep(0.5) # simulate network I/O
results[idx] = f"data from {url}"
urls = ["http://a.com", "http://b.com", "http://c.com"]
results = [None] * len(urls)
threads = [threading.Thread(target=fetch, args=(u, results, i)) for i, u in enumerate(urls)]
for t in threads: t.start()
for t in threads: t.join()
print(results)2. Lock & Queue
import threading
from queue import Queue
lock = threading.Lock()
counter = 0
def increment(n):
global counter
for _ in range(n):
with lock:
counter += 1
t1 = threading.Thread(target=increment, args=(10000,))
t2 = threading.Thread(target=increment, args=(10000,))
t1.start(); t2.start()
t1.join(); t2.join()
print(counter) # always 20000 with lock3. concurrent.futures
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import time
# I/O-bound: use ThreadPoolExecutor
def download(url):
time.sleep(0.5) # simulate
return f"result:{url}"
with ThreadPoolExecutor(max_workers=4) as executor:
futures = [executor.submit(download, f"http://site{i}.com") for i in range(8)]
results = [f.result() for f in futures]
print(len(results), "downloads done")
# CPU-bound: use ProcessPoolExecutor
def is_prime(n):
if n < 2: return False
return all(n % i for i in range(2, int(n**0.5)+1))
with ProcessPoolExecutor() as executor:
primes = list(executor.map(is_prime, range(1000)))Week 5 — Asyncio
Write asynchronous programs with asyncio. Understand coroutines, tasks, the event loop, and use aiohttp for non-blocking HTTP requests.
1. Async Basics
import asyncio
async def greet(name, delay):
await asyncio.sleep(delay)
print(f"Hello, {name}!")
async def main():
# Run concurrently
await asyncio.gather(
greet("Alice", 2),
greet("Bob", 1),
greet("Carol", 0.5),
)
# Prints Carol, Bob, Alice (in order of completion)
asyncio.run(main())2. Tasks & Cancellation
import asyncio
async def background_task():
for i in range(100):
print(f" bg: step {i}")
await asyncio.sleep(0.1)
async def main():
task = asyncio.create_task(background_task())
await asyncio.sleep(0.3) # let it run for 0.3s
task.cancel()
try:
await task
except asyncio.CancelledError:
print("Task cancelled")
asyncio.run(main())3. aiohttp
import asyncio, aiohttp
async def fetch(session, url):
async with session.get(url) as resp:
return resp.status, len(await resp.text())
async def main(urls):
async with aiohttp.ClientSession() as session:
tasks = [fetch(session, u) for u in urls]
results = await asyncio.gather(*tasks, return_exceptions=True)
for url, result in zip(urls, results):
if isinstance(result, Exception):
print(f"FAIL {url}: {result}")
else:
status, size = result
print(f"{status} {url} ({size} chars)")Week 6 — Testing with pytest
Write professional tests with pytest. Use fixtures for setup/teardown, parametrize for data-driven tests, mock external dependencies, and measure code coverage.
1. pytest Basics
# test_calculator.py
import pytest
def add(a, b): return a + b
def divide(a, b):
if b == 0: raise ZeroDivisionError
return a / b
def test_add():
assert add(2, 3) == 5
assert add(-1, 1) == 0
def test_divide_by_zero():
with pytest.raises(ZeroDivisionError):
divide(10, 0)
# Run: pytest test_calculator.py -v2. Fixtures
import pytest
from bank import BankAccount
@pytest.fixture
def account():
"""Fresh account with $100 balance."""
acct = BankAccount("test")
acct.deposit(100)
return acct
def test_deposit(account):
account.deposit(50)
assert account.balance == 150
def test_withdraw(account):
account.withdraw(30)
assert account.balance == 70
def test_overdraft(account):
with pytest.raises(ValueError, match="Insufficient"):
account.withdraw(200)3. Parametrize & Mock
import pytest
from unittest.mock import patch, MagicMock
@pytest.mark.parametrize("a,b,expected", [
(2, 3, 5), (-1, 1, 0), (0, 0, 0), (100, -50, 50)
])
def test_add_param(a, b, expected):
assert add(a, b) == expected
def test_with_mock():
with patch("requests.get") as mock_get:
mock_get.return_value = MagicMock(status_code=200, json=lambda: {"ok": True})
# code that calls requests.get will use the mock
resp = requests.get("http://example.com")
assert resp.status_code == 200💻 Examples
Run these examples and check the output yourself.
from concurrent.futures import ThreadPoolExecutor, as_completed
import urllib.request, time
URLs = [
"https://www.python.org",
"https://docs.python.org",
"https://pypi.org",
]
def check_url(url):
try:
start = time.perf_counter()
urllib.request.urlopen(url, timeout=5)
return url, True, time.perf_counter() - start
except Exception as e:
return url, False, 0
with ThreadPoolExecutor(max_workers=len(URLs)) as ex:
futures = {ex.submit(check_url, u): u for u in URLs}
for future in as_completed(futures):
url, ok, elapsed = future.result()
status = "OK" if ok else "FAIL"
print(f"[{status}] {url} ({elapsed:.2f}s)")
import asyncio, random
async def fetch_data(item_id: int) -> dict:
delay = random.uniform(0.1, 0.5)
await asyncio.sleep(delay)
return {"id": item_id, "value": item_id * 10, "delay": round(delay, 2)}
async def main():
import time
start = time.perf_counter()
results = await asyncio.gather(*[fetch_data(i) for i in range(1, 11)])
elapsed = time.perf_counter() - start
for r in results:
print(f" id={r['id']} value={r['value']} fetched_in={r['delay']}s")
print(f"Total time: {elapsed:.2f}s (sequential would be ~{sum(r['delay'] for r in results):.2f}s)")
asyncio.run(main())
import pytest
class BankAccount:
def __init__(self, owner, balance=0):
self.owner = owner
self._balance = balance
@property
def balance(self): return self._balance
def deposit(self, amount):
if amount <= 0: raise ValueError("Must be positive")
self._balance += amount
def withdraw(self, amount):
if amount > self._balance: raise ValueError("Insufficient funds")
self._balance -= amount
@pytest.fixture
def empty_acct(): return BankAccount("Test")
@pytest.fixture
def funded_acct(): a = BankAccount("Test"); a.deposit(1000); return a
def test_initial_balance(empty_acct):
assert empty_acct.balance == 0
@pytest.mark.parametrize("amount", [1, 100, 99999])
def test_deposit_valid(empty_acct, amount):
empty_acct.deposit(amount)
assert empty_acct.balance == amount
def test_deposit_invalid(empty_acct):
with pytest.raises(ValueError):
empty_acct.deposit(-10)
def test_withdraw_valid(funded_acct):
funded_acct.withdraw(300)
assert funded_acct.balance == 700
def test_withdraw_overdraft(funded_acct):
with pytest.raises(ValueError, match="Insufficient"):
funded_acct.withdraw(5000)
📝 Exercises
Try them yourself first, then open the solution to compare.
Producer-Consumer Queue
Goal: Implement a producer-consumer pattern with threading.Queue.
- 1 producer thread generates 20 work items
- 4 consumer threads process items (simulate with sleep)
- Use Queue to safely pass items
- Print when each item is produced and consumed
▶Toggle solution
import threading, queue, time, random
work_q = queue.Queue()
def producer():
for i in range(20):
item = f"item-{i:02d}"
work_q.put(item)
print(f" Produced: {item}")
time.sleep(random.uniform(0.05, 0.1))
for _ in range(4): # sentinel values
work_q.put(None)
def consumer(cid):
while True:
item = work_q.get()
if item is None:
work_q.task_done()
break
time.sleep(random.uniform(0.1, 0.3))
print(f" Consumer-{cid}: processed {item}")
work_q.task_done()
threads = [threading.Thread(target=producer)]
threads += [threading.Thread(target=consumer, args=(i,)) for i in range(4)]
for t in threads: t.start()
for t in threads: t.join()
print("All done")
Rate-limited async downloader
Goal: Fetch N URLs concurrently but limit to max 5 simultaneous requests using asyncio.Semaphore.
- Use asyncio.Semaphore(5) to limit concurrency
- Collect results in order (use tasks, not gather directly)
- Print status code and response size for each URL
▶Toggle solution
import asyncio
import aiohttp
async def fetch(session, url, sem):
async with sem:
try:
async with session.get(url, timeout=aiohttp.ClientTimeout(total=10)) as r:
body = await r.text()
return url, r.status, len(body)
except Exception as e:
return url, 0, 0
async def main(urls):
sem = asyncio.Semaphore(5)
async with aiohttp.ClientSession() as session:
tasks = [fetch(session, u, sem) for u in urls]
for coro in asyncio.as_completed(tasks):
url, status, size = await coro
print(f"[{status}] {url} ({size} chars)")
urls = [f"https://httpbin.org/delay/{i%3}" for i in range(10)]
asyncio.run(main(urls))
TDD: Stack implementation
Goal: Write tests first, then implement a Stack class to make them pass.
- Write test_stack.py with 8+ test cases
- Cover: push, pop, peek, is_empty, overflow (max_size)
- Achieve 100% coverage: pytest --cov=stack test_stack.py
▶Toggle solution
# test_stack.py
import pytest
from stack import Stack, StackOverflowError
@pytest.fixture
def s(): return Stack(max_size=3)
def test_empty(s): assert s.is_empty()
def test_push_pop(s): s.push(1); assert s.pop() == 1
def test_peek(s): s.push(99); assert s.peek() == 99; assert not s.is_empty()
def test_pop_empty(s):
with pytest.raises(IndexError): s.pop()
def test_overflow(s):
for i in range(3): s.push(i)
with pytest.raises(StackOverflowError): s.push(999)
# stack.py
class StackOverflowError(Exception): pass
class Stack:
def __init__(self, max_size=None):
self._data, self.max_size = [], max_size
def push(self, v):
if self.max_size and len(self._data) >= self.max_size:
raise StackOverflowError
self._data.append(v)
def pop(self):
if not self._data: raise IndexError
return self._data.pop()
def peek(self):
if not self._data: raise IndexError
return self._data[-1]
def is_empty(self): return not self._data
All lecture materials and example code are openly available on GitHub.
View on GitHub ↗Comments 0
No comments yet. Be the first.
Only members can comment.
Comments with ads, abuse, personal information or illegal content are hidden or removed. Comments are shown with your nickname. Privacy Policy