External_URL.py

import os
import csv

# --- Configuration ---
# Set this to the 'Products' directory path
project_root = "."  # Update this with your actual path
output_file = "collected_thumbnails.csv"
target_column = "thumbnail-img href"

collected_values = []

print(f"Scanning directory: {project_root}")

# Recursively walk through the project directory
for dirpath, _, filenames in os.walk(project_root):
    for filename in filenames:
        if filename.lower().endswith(".csv"):
            full_path = os.path.join(dirpath, filename)
            print(f"Found CSV: {full_path}")
            try:
                with open(full_path, newline='', encoding='utf-8') as csv_file:
                    reader = csv.DictReader(csv_file)
                    if reader.fieldnames is None:
                        print(f"Warning: No header found in {full_path}")
                        continue

                    # Debug: show available columns in the file
                    print(f"Columns in '{filename}': {reader.fieldnames}")

                    if target_column in reader.fieldnames:
                        print(f"--> Target column '{target_column}' found.")
                        count = 0
                        for row in reader:
                            value = row.get(target_column)
                            if value:
                                collected_values.append(value)
                                count += 1
                        print(f"--> Extracted {count} values from '{filename}'.")
                    else:
                        print(f"--> Skipping '{filename}': Missing '{target_column}' column.")
            except Exception as e:
                print(f"Error reading '{full_path}': {e}")

# Write all collected values into the output CSV
try:
    with open(output_file, 'w', newline='', encoding='utf-8') as out_file:
        writer = csv.writer(out_file)
        writer.writerow([target_column])  # write header
        for val in collected_values:
            writer.writerow([val])
    print(f"\n✅ Completed! Collected {len(collected_values)} values into '{output_file}'.")
except Exception as e:
    print(f"Error writing to output file: {e}")