Web Scraping & REST API Data Extraction in Python

Not all data arrives packaged in neat CSV or database tables. Often, the most valuable competitive signals — e-commerce competitor pricing, economic indices, social sentiment, and government data — must be harvested via REST APIs or extracted from public web pages using Web Scraping.

1. Professional REST API Extraction Pipeline

A resilient API client handles connection retries, authentication headers, and automatic pagination:

import os
import requests
import pandas as pd
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

def fetch_api_records(base_url: str, total_pages: int = 5) -> pd.DataFrame:
    """Fetches paginated JSON records with exponential backoff retries."""
    session = requests.Session()
    retries = Retry(total=3, backoff_factor=1.5, status_forcelist=[429, 500, 502, 503, 504])
    session.mount("https://", HTTPAdapter(max_retries=retries))
    
    headers = {
        "User-Agent": "VeewomDataScienceBot/2.0 (research@veewom.com)",
        "Accept": "application/json"
    }
    
    all_records = []
    for page in range(1, total_pages + 1):
        response = session.get(f"{base_url}?page={page}&limit=100", headers=headers, timeout=10)
        if response.status_code == 200:
            payload = response.json()
            all_records.extend(payload.get('data', []))
        else:
            print(f"Warning: Page {page} returned status {response.status_code}")
            break
            
    # Flatten nested structures into tabular DataFrame
    return pd.json_normalize(all_records)

2. Web Scraping with BeautifulSoup

When parsing HTML tables or product catalogs from static web pages:

from bs4 import BeautifulSoup
import requests
import pandas as pd

html_sample = """
<div class="catalog">
  <div class="item">
    <h3 class="title">Data Science Workstation Pro</h3>
    <span class="price">$2,499.00</span>
    <span class="rating">4.8</span>
  </div>
  <div class="item">
    <h3 class="title">GPU Compute Node A100</h3>
    <span class="price">$8,999.00</span>
    <span class="rating">4.9</span>
  </div>
</div>
"""

soup = BeautifulSoup(html_sample, 'html.parser')
products = []

for item in soup.find_all('div', class_='item'):
    title = item.find('h3', class_='title').text.strip()
    price_raw = item.find('span', class_='price').text.strip()
    price = float(price_raw.replace('$', '').replace(',', ''))
    rating = float(item.find('span', class_='rating').text.strip())
    products.append({'title': title, 'price': price, 'rating': rating})

df_products = pd.DataFrame(products)
print(df_products)