clean_scraper.py

#!/usr/bin/env python3
import time, csv, re
import pandas as pd
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from colorama import Fore, Style, init

# Init colorama for Windows compatibility
init(autoreset=True)

# ========================
# Configuration
# ========================
CHROMEDRIVER_PATH = "./chromedriver/chromedriver"
INPUT_CSV = "collected_thumbnails.csv"
OUTPUT_CSV = "extracted_full_data.csv"
PAGE_LOAD_WAIT = 5
DOCUMENT_BASE_URL = "https://www.bfgsupply.com/"
SCRAPE_LIMIT = 25

# ========================
# Setup Selenium Browser
# ========================
def setup_browser():
    options = Options()
    options.add_argument("--headless")
    options.add_argument("--disable-gpu")
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")
    service = Service(CHROMEDRIVER_PATH)
    return webdriver.Chrome(service=service, options=options)

# ========================
# Read URLs from CSV
# ========================
def read_urls(file_path):
    df = pd.read_csv(file_path)
    col = df.columns[0]
    return [url.strip() for url in df[col].dropna() if str(url).strip()]

# ========================
# Extract First Table Key-Value Pairs
# ========================
def extract_first_table(soup):
    data = {}
    rows = soup.select("div.top > div.main-details > div.main-details-bottom > div.details > table:nth-child(1) > tbody > tr")
    for row in rows:
        cell1 = row.select_one("td:nth-child(1)")
        cell2 = row.select_one("td:nth-child(2)")
        if cell1 and cell2:
            key = cell1.get_text(strip=True).rstrip(":")
            value = cell2.get_text(strip=True)
            data[key] = value
    return data

# ========================
# Extract URL Identifiers
# ========================
def extract_url_data(url):
    match = re.match(r".*/product/(\d+)/(\d+)/([^/]+)", url)
    if match:
        return {
            "categoryID": match.group(1),
            "productID": match.group(2),
            "slugID": match.group(3),
        }
    return {"categoryID": "", "productID": "", "slugID": ""}

# ========================
# Scrape a Single Page
# ========================
def scrape_page(driver, url):
    driver.get(url)
    time.sleep(PAGE_LOAD_WAIT)
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')

    data = {"URL": url}
    data.update(extract_url_data(url))

    # Product Name
    name = soup.select_one("#productDetailPage div.product-detail div.main h3")
    data["Name"] = name.get_text(strip=True) if name else ""

    # Product Image
    image_tag = soup.select_one("div.main-image img")
    data["Image"] = image_tag["src"] if image_tag and image_tag.has_attr("src") else ""

    # Key-Value Table
    data.update(extract_first_table(soup))

    # Documents
    doc_elements = soup.select("div.details table.document-list-table a")
    docs = []
    for a in doc_elements:
        if a.has_attr("href"):
            href = a["href"].strip()
            if not href.startswith("http"):
                href = DOCUMENT_BASE_URL + href.lstrip("/")
            docs.append(href)
    data["Documents"] = ", ".join(docs)

    return data

# ========================
# Main Function
# ========================
def main():
    print(f"{Fore.CYAN}[START] Scraping initiated...")
    urls = read_urls(INPUT_CSV)[:SCRAPE_LIMIT]
    driver = setup_browser()
    results = []

    for i, url in enumerate(urls):
        print(f"\n{Fore.YELLOW}[{i+1}/{len(urls)}] {Style.BRIGHT}Scraping: {url}")
        try:
            data = scrape_page(driver, url)
            results.append(data)
            print(f"{Fore.GREEN}[OK] {data.get('Name', 'No name found')}")
            for k, v in data.items():
                print(f"{Fore.MAGENTA}  {k}: {v}")
        except Exception as e:
            print(f"{Fore.RED}[ERROR] Failed to scrape {url}: {e}")

    driver.quit()

    # Save CSV
    if results:
        all_keys = sorted({k for item in results for k in item.keys()})
        with open(OUTPUT_CSV, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=all_keys)
            writer.writeheader()
            for row in results:
                complete_row = {key: row.get(key, "") for key in all_keys}
                writer.writerow(complete_row)
        print(f"\n{Fore.CYAN}[DONE] Saved {len(results)} records to {OUTPUT_CSV}")
    else:
        print(f"{Fore.RED}[DONE] No data scraped.")

if __name__ == "__main__":
    main()