Weeks 7–10 — Web Scraping · NumPy & Pandas · Packaging & Distribution · Advanced Capstone: Async REST Client

import requests

resp = requests.get(
    "https://httpbin.org/get",
    params={"key": "value"},
    headers={"User-Agent": "my-scraper/1.0"},
    timeout=10,
)
print(resp.status_code)   # 200
print(resp.headers["Content-Type"])
data = resp.json()
print(data["args"])       # {'key': 'value'}

from bs4 import BeautifulSoup
import requests

html = requests.get("https://news.ycombinator.com").text
soup = BeautifulSoup(html, "html.parser")

# Select all story titles
for item in soup.select(".titleline > a")[:5]:
    print(item.text)
    print(" ", item.get("href"))

import time, random, requests

class PoliteScraper:
    def __init__(self, min_delay=1.0, max_delay=3.0):
        self.session = requests.Session()
        self.session.headers["User-Agent"] = "MyBot/1.0 (+https://example.com)"
        self._min = min_delay
        self._max = max_delay
        self._last = 0

    def get(self, url):
        wait = self._min + random.random() * (self._max - self._min)
        elapsed = time.time() - self._last
        if elapsed < wait:
            time.sleep(wait - elapsed)
        self._last = time.time()
        return self.session.get(url, timeout=10)

import numpy as np

a = np.array([1, 2, 3, 4, 5])
print(a * 2)          # [2 4 6 8 10]  — vectorized
print(a[a > 3])       # [4 5]          — boolean indexing

# 2D array
m = np.arange(9).reshape(3, 3)
print(m)
print(m.T)            # transpose
print(m @ m)          # matrix multiply

import pandas as pd

df = pd.DataFrame({
    "name":  ["Alice","Bob","Carol","Dave"],
    "dept":  ["Eng","Eng","HR","HR"],
    "score": [92, 85, 78, 88]
})

print(df.describe())
print(df[df["score"] >= 85])
print(df.groupby("dept")["score"].mean())

df = pd.read_csv("data.csv")

# Inspect
print(df.shape, df.dtypes)
print(df.isnull().sum())

# Clean
df = df.dropna(subset=["score"])      # drop rows with no score
df["score"] = pd.to_numeric(df["score"], errors="coerce")
df = df[df["score"].between(0, 100)]  # valid range
df = df.drop_duplicates()

mypackage/
├── src/
│   └── mypackage/
│       ├── __init__.py
│       ├── core.py
│       └── cli.py
├── tests/
│   └── test_core.py
├── pyproject.toml
└── README.md

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[project]
name = "mypackage"
version = "0.1.0"
description = "My first Python package"
requires-python = ">=3.10"
dependencies = ["requests>=2.28"]

[project.scripts]
mypkg = "mypackage.cli:main"

[tool.hatch.build.targets.wheel]
packages = ["src/mypackage"]

pip install hatch
hatch build          # creates dist/*.whl and dist/*.tar.gz

# Publish to TestPyPI
pip install twine
twine upload --repository testpypi dist/*

# Install from TestPyPI
pip install --index-url https://test.pypi.org/simple/ mypackage

# Usage example
from apiclient import AsyncClient, RetryPolicy

async def main():
    policy = RetryPolicy(max_retries=3, backoff=2.0)
    async with AsyncClient("https://api.example.com", retry=policy) as client:
        users = await client.get("/users", params={"page": 1})
        post  = await client.post("/posts", json={"title": "Hello"})
        print(users, post)

import asyncio, aiohttp
from dataclasses import dataclass, field
from typing import Any, Optional

@dataclass
class RetryPolicy:
    max_retries: int = 3
    backoff: float = 1.0
    retry_on: tuple = (429, 500, 502, 503)

async def _request_with_retry(
    session: aiohttp.ClientSession,
    method: str,
    url: str,
    policy: RetryPolicy,
    **kwargs: Any,
) -> aiohttp.ClientResponse:
    last_exc: Optional[Exception] = None
    for attempt in range(policy.max_retries + 1):
        try:
            resp = await session.request(method, url, **kwargs)
            if resp.status in policy.retry_on and attempt < policy.max_retries:
                await asyncio.sleep(policy.backoff * (2 ** attempt))
                continue
            return resp
        except aiohttp.ClientError as e:
            last_exc = e
            if attempt < policy.max_retries:
                await asyncio.sleep(policy.backoff * (2 ** attempt))
    raise last_exc or RuntimeError("Request failed")