IncompleteSustainablePythonWebCrawlingScriptsMisc/htg.py at main · darcyturk/IncompleteSustainablePythonWebCrawlingScriptsMisc · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
from playwright.sync_api import sync_playwright
import time

BASE_URL = "https://www.htgsupply.com/product-category/grow-lights/page/{}/"

def crawl_products():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page_num = 1
        product_idx = 1

        while True:
            url = BASE_URL.format(page_num)
            print(f"Loading page {page_num}: {url}")
            page.goto(url)
            try:
                page.wait_for_selector("div.product-item-info", timeout=15000)
            except:
                print("No products found, exiting pagination.")
                break

            products = page.query_selector_all("div.product-item-info")
            if not products:
                print("No products on this page, stopping.")
                break

            print(f"Found {len(products)} products on page {page_num}.")

            for product in products:
                # Title and link
                title_link = product.query_selector("h4.product-item-name > a")
                title = title_link.inner_text().strip() if title_link else "[No Title]"
                link = title_link.get_attribute("href") if title_link else "[No Link]"

                # Prices (handle both ins and del)
                price_span = product.query_selector("span.price.product-item-prices")
                if price_span:
                    # Current price is usually inside <ins>, fallback to just the span text
                    ins_price = price_span.query_selector("ins span.woocommerce-Price-amount")
                    del_price = price_span.query_selector("del span.woocommerce-Price-amount")

                    current_price = ins_price.inner_text().strip() if ins_price else None
                    old_price = del_price.inner_text().strip() if del_price else None

                    if current_price and old_price:
                        price = f"{current_price} (was {old_price})"
                    else:
                        price = current_price or price_span.inner_text().strip()
                else:
                    price = "[No Price]"

                # Image URL
                img = product.query_selector("a.product-item-img img")
                img_url = img.get_attribute("src") if img else "[No Image]"

                print(f"{product_idx}. {title} | {price} | {link} | Image: {img_url}")
                product_idx += 1

            page_num += 1
            time.sleep(1)

        browser.close()

if __name__ == "__main__":
    crawl_products()