complete_scraper.py

#!/usr/bin/env python3
import time
import csv
import pandas as pd
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options

# ========================
# Configuration
# ========================
CHROMEDRIVER_PATH = "./chromedriver/chromedriver"
INPUT_CSV = "collected_thumbnails.csv"
OUTPUT_CSV = "extracted_full_data.csv"
SPECS_OUTPUT_CSV = "product_specs_data.csv"
PAGE_LOAD_WAIT = 5  # Seconds to wait for page load
DOCUMENT_BASE_URL = "https://www.bfgsupply.com/"

# ========================
# Setup Selenium Browser
# ========================
def setup_browser():
    options = Options()
    options.add_argument("--headless")
    options.add_argument("--disable-gpu")
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")
    service = Service(CHROMEDRIVER_PATH)
    return webdriver.Chrome(service=service, options=options)

# ========================
# Read URLs from CSV
# ========================
def read_urls(file_path):
    df = pd.read_csv(file_path)
    col = df.columns[0]
    return [url.strip() for url in df[col].dropna() if str(url).strip()]

# ========================
# Extract Key-Value Pairs from the First Table
# ========================
def extract_first_table(soup):
    data = {}
    # Selector for the first table (key-value pairs):
    # "div.top > div.main-details > div.main-details-bottom > div.details > table:nth-child(1) > tbody > tr"
    rows = soup.select("div.top > div.main-details > div.main-details-bottom > div.details > table:nth-child(1) > tbody > tr")
    for row in rows:
        cell1 = row.select_one("td:nth-child(1)")
        cell2 = row.select_one("td:nth-child(2)")
        if cell1 and cell2:
            key = cell1.get_text(strip=True).rstrip(":")
            value = cell2.get_text(strip=True)
            data[key] = value
    return data

# ========================
# Extract Product Specs Table (Second Table) - Original Method
# ========================
def extract_product_specs(soup):
    # Use the updated selector based on your provided HTML:
    # The specs table is inside the div with class "tab-pnl" and the table has class "dimension-tbl"
    spec_table = soup.select_one("div.tab-pnl table.dimension-tbl")
    if spec_table:
        spec_rows = []
        for row in spec_table.select("tbody > tr"):
            cells = row.find_all("td")
            # Extract text from each cell, preserving empty strings if the cell is empty.
            cell_texts = [cell.get_text(separator=" ", strip=True) for cell in cells]
            # Ensure exactly 4 columns: KEY, EACH, CASE, PALLET.
            while len(cell_texts) < 4:
                cell_texts.append("")
            # Join the cell texts with " | " delimiter.
            spec_rows.append(" | ".join(cell_texts))
        # Join all rows with a newline (or another delimiter if desired).
        return "\n".join(spec_rows)
    else:
        return ""

# ========================
# Extract Product Specs Table Into Structured Format
# ========================
def extract_structured_specs(soup, url):
    # Find the specs table using the exact selector from the HTML example
    spec_table = soup.select_one("div.tab-pnl table.dimension-tbl")
    
    # Initialize result dictionary with URL and all potential columns set to empty values
    result = {
        "url": url,
        "EACH_UPC": "", "CASE_UPC": "", "PALLET_UPC": "",
        "EACH_Quantity": "", "CASE_Quantity": "", "PALLET_Quantity": "",
        "EACH_Volume (cf)": "", "CASE_Volume (cf)": "", "PALLET_Volume (cf)": "",
        "EACH_Dim Weight (in/lb)": "", "CASE_Dim Weight (in/lb)": "", "PALLET_Dim Weight (in/lb)": "",
        "EACH_Weight (lb)": "", "CASE_Weight (lb)": "", "PALLET_Weight (lb)": "",
        "EACH_Length (in)": "", "CASE_Length (in)": "", "PALLET_Length (in)": "",
        "EACH_Width (in)": "", "CASE_Width (in)": "", "PALLET_Width (in)": "",
        "EACH_Height (in)": "", "CASE_Height (in)": "", "PALLET_Height (in)": ""
    }
    
    if not spec_table:
        print("  No spec table found")
        return result
    
    try:
        # Get all rows directly from tbody
        rows = spec_table.select("tbody > tr")
        print(f"  Found {len(rows)} spec rows")
        
        # Define column mapping for easy reference
        columns = ["", "EACH", "CASE", "PALLET"]  # Index 0 is the key column
        
        # Map row keys to result dictionary keys (exact matches from the HTML)
        key_mapping = {
            "UPC": "_UPC",
            "Quantity": "_Quantity",
            "Volume (cf)": "_Volume (cf)",
            "Dim Weight (in": "_Dim Weight (in/lb)",  # Note the slightly different format
            "Weight (lb)": "_Weight (lb)",
            "Length (in)": "_Length (in)",
            "Width (in)": "_Width (in)",
            "Height (in)": "_Height (in)",
        }
        
        # Process each row
        for row in rows:
            # Get all td cells in this row
            cells = row.find_all("td")
            if len(cells) < 2:
                continue  # Skip rows with insufficient cells
            
            # Get the key (first column), removing trailing colon if present
            key_text = cells[0].get_text(strip=True).rstrip(":")
            print(f"  Processing row: '{key_text}'")
            
            # Find which key pattern matches
            matching_key = None
            for pattern in key_mapping:
                if key_text.startswith(pattern):
                    matching_key = pattern
                    break
            
            if not matching_key:
                print(f"  No matching key found for '{key_text}'")
                continue
                
            # Get the suffix to use for column names
            suffix = key_mapping[matching_key]
            
            # Process each value cell (columns 1-3)
            for i in range(1, min(len(cells), 4)):
                # Get the value, carefully handling any HTML inside
                cell = cells[i]
                value = cell.get_text(strip=True)
                
                # Skip empty values
                if not value:
                    continue
                
                # Construct the column name 
                col_prefix = columns[i]
                col_name = f"{col_prefix}{suffix}"
                
                print(f"    Setting {col_name} = '{value}'")
                result[col_name] = value
                
    except Exception as e:
        import traceback
        print(f"  Error parsing spec table: {e}")
        traceback.print_exc()
    
    return result

# ========================
# Scrape a Single Page
# ========================
def scrape_page(driver, url):
    driver.get(url)
    time.sleep(PAGE_LOAD_WAIT)
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')
    
    # Main product data
    data = {'URL': url}

    # --- Extract Product Name ---
    main_container = soup.select_one("#productDetailPage > div.product-detail > div.main")
    if main_container:
        name_tag = main_container.select_one("h3")
        data["Name"] = name_tag.get_text(strip=True) if name_tag else ""
    else:
        data["Name"] = ""

    # --- Extract Product Image ---
    image_tag = soup.select_one("div.top > div.main-image > div.product-detail-carousel__container > div.product-detail-carousel > div.slick-slider.slick-initialized > div.slick-list > div.slick-track > div > div > div > img:nth-child(2)")
    data["Image"] = image_tag["src"] if image_tag and image_tag.has_attr("src") else ""

    # --- Extract Key-Value Pairs from the First Table ---
    kv_data = extract_first_table(soup)
    data.update(kv_data)

    # --- Extract Documents ---
    doc_elements = soup.select("div.top > div.main-details > div.main-details-bottom > div.details > table.document-list-table > tbody > tr > td:nth-child(2) > ul > li > a")
    docs = []
    for a in doc_elements:
        if a.has_attr("href"):
            href = a["href"].strip()
            if not href.startswith("http"):
                href = DOCUMENT_BASE_URL + href.lstrip("/")
            docs.append(href)
    data["Documents"] = ", ".join(docs)

    # --- Extract the Whole Product Specs Table (original format) ---
    data["Product Specs"] = extract_product_specs(soup)
    
    # --- Extract structured specs data for the secondary CSV file ---
    specs_data = extract_structured_specs(soup, url)
    
    return data, specs_data

# ========================
# Main Function
# ========================
def main():
    print("[START] Starting Selenium scraper with structured product specs extraction...")
    urls = read_urls(INPUT_CSV)
    driver = setup_browser()
    main_results = []
    specs_results = []

    # Process all URLs (you can add [:n] for testing with a subset)
    for i, url in enumerate(urls):
        print(f"\n[{i+1}/{len(urls)}] Scraping: {url}")
        try:
            main_data, specs_data = scrape_page(driver, url)
            main_results.append(main_data)
            specs_results.append(specs_data)
            print("Extracted Main Data:")
            for key, value in main_data.items():
                if key != "Product Specs":  # Don't print the lengthy specs
                    print(f"  {key}: {value}")
            
            # Count non-empty values in specs data to check extraction quality
            non_empty_count = sum(1 for k, v in specs_data.items() if v and k != "url")
            print(f"Extracted Specs Data: {non_empty_count} non-empty fields")
        except Exception as e:
            print(f"Error scraping {url}: {e}")
            import traceback
            traceback.print_exc()

    driver.quit()

    # Save main results to CSV
    if main_results:
        all_keys = sorted({k for item in main_results for k in item.keys()})
        with open(OUTPUT_CSV, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=all_keys)
            writer.writeheader()
            writer.writerows(main_results)
        print(f"\n[DONE] Saved {len(main_results)} records to {OUTPUT_CSV}")
    else:
        print("\n[DONE] No main data extracted.")
        
    # Save specs results to CSV
    if specs_results:
        # Define columns in the specific order requested
        specs_columns = [
            "url", 
            "EACH_UPC", "CASE_UPC", "PALLET_UPC",
            "EACH_Quantity", "CASE_Quantity", "PALLET_Quantity",
            "EACH_Volume (cf)", "CASE_Volume (cf)", "PALLET_Volume (cf)",
            "EACH_Dim Weight (in/lb)", "CASE_Dim Weight (in/lb)", "PALLET_Dim Weight (in/lb)",
            "EACH_Weight (lb)", "CASE_Weight (lb)", "PALLET_Weight (lb)",
            "EACH_Length (in)", "CASE_Length (in)", "PALLET_Length (in)",
            "EACH_Width (in)", "CASE_Width (in)", "PALLET_Width (in)",
            "EACH_Height (in)", "CASE_Height (in)", "PALLET_Height (in)"
        ]
        
        with open(SPECS_OUTPUT_CSV, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=specs_columns)
            writer.writeheader()
            writer.writerows(specs_results)
        print(f"\n[DONE] Saved {len(specs_results)} records to {SPECS_OUTPUT_CSV}")
    else:
        print("\n[DONE] No specs data extracted.")

if __name__ == "__main__":
    main()