scraper.py

#!/usr/bin/env python3
import csv
import time
import requests
import pandas as pd
from bs4 import BeautifulSoup

# Configuration
INPUT_CSV = 'collected_thumbnails.csv'
OUTPUT_CSV = 'extracted_tables.csv'
REQUEST_TIMEOUT = 30  # seconds
REQUEST_DELAY = 1  # seconds between requests to be respectful

# Headers for requests
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
}

def read_urls_from_csv(file_path):
    """Read URLs from CSV file."""
    try:
        df = pd.read_csv(file_path)
        column_name = df.columns[0]  # Assuming the URL column is the first one
        urls = df[column_name].tolist()
        print(f"Found {len(urls)} URLs in {file_path}")
        return urls
    except Exception as e:
        print(f"Error reading CSV file: {e}")
        return []

def fetch_html(url):
    """Fetch HTML content from URL."""
    try:
        response = requests.get(url, headers=HEADERS, timeout=REQUEST_TIMEOUT)
        response.raise_for_status()
        return response.text
    except requests.exceptions.RequestException as e:
        print(f"Error fetching {url}: {e}")
        return None

def parse_dimension_table(html, url):
    """Parse dimension table from HTML."""
    if not html:
        return None
    
    try:
        soup = BeautifulSoup(html, 'html.parser')
        table = soup.select_one('table.dimension-tbl')
        
        if not table:
            print(f"No dimension table found at {url}")
            return None
        
        # Extract headers
        headers = []
        header_row = table.select_one('thead tr')
        if header_row:
            headers = [th.get_text(strip=True) for th in header_row.select('th')]
        
        # Create data object
        table_data = {'URL': url}
        
        # Store headers
        table_data['HEADERS'] = '|'.join(headers)
        
        # Extract rows
        for row in table.select('tbody tr'):
            cells = row.select('td')
            
            if len(cells) > 1:
                row_label = cells[0].get_text(strip=True)
                
                # Store row label
                table_data[f'ROW_LABEL_{row_label}'] = row_label
                
                # Process each cell (skip first one which is the row label)
                for i in range(1, len(cells)):
                    if i < len(headers):
                        header = headers[i]
                        cell_text = cells[i].get_text(strip=True)
                        
                        # Create composite key for each cell
                        key = f'{row_label}_{header}'
                        table_data[key] = cell_text
        
        return table_data
    except Exception as e:
        print(f"Error parsing table at {url}: {e}")
        return None

def main():
    """Main function."""
    print("Starting web scraper")
    
    # Read URLs from CSV
    urls = read_urls_from_csv(INPUT_CSV)
    if not urls:
        print("No URLs found. Exiting.")
        return
    
    # Process URLs
    results = []
    successful = 0
    
    for i, url in enumerate(urls):
        print(f"Processing ({i+1}/{len(urls)}): {url}")
        
        # Fetch HTML
        html = fetch_html(url)
        
        # Parse table
        if html:
            table_data = parse_dimension_table(html, url)
            if table_data:
                results.append(table_data)
                successful += 1
                print(f"✓ Successfully extracted table from {url}")
        
        # Add delay to be respectful to the server
        time.sleep(REQUEST_DELAY)
    
    print(f"Processed {successful} URLs successfully out of {len(urls)}")
    
    # Get all unique keys across all results
    if results:
        all_keys = set()
        for result in results:
            all_keys.update(result.keys())
        
        # Sort keys for consistent column order
        sorted_keys = sorted(all_keys)
        
        # Write to CSV
        with open(OUTPUT_CSV, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=sorted_keys)
            writer.writeheader()
            for result in results:
                writer.writerow(result)
        
        print(f"Successfully wrote {len(results)} rows to {OUTPUT_CSV}")
        print("Scraping completed successfully")
    else:
        print("No results to write to CSV")
        print("Scraping completed with errors")

if __name__ == "__main__":
    main()