gc_permit_process_raw_data.py

import camelot
from pathlib import Path
import pandas as pd


def process_data() -> None:
    """
    Quick and dirty python function for processing pdf table data in the `raw_data` directory using the camelot package.
    If more data is added, update the `data_processing_tuples` dict with the page range the tables can be found on
    to make sure it parses correctly.
    """
    # pdfs have different lengths of data per year; manually specify what pages contain what data
    data_processing_tuples = {
        2017: (2, 11),
        2018: (2, 11),
        2019: (2, 11),
        2020: (2, 12),
        2021: (2, 11),
        2022: (2, 7),
        2023: (2, 9),
        2024: (2, 12),
        2025: (2, 12),
    }

    for year, (start_page, end_page) in data_processing_tuples.items():
        # process data
        fp = Path(f"./raw_data/{year}_Lottery_Statistics.pdf")
        print(f"processing {year}...")
        tables = camelot.read_pdf(
            fp.as_posix(),
            pages=",".join(f"{i}" for i in range(start_page, end_page + 1)),
        )
        year_df = (
            pd.concat(table.df.iloc[1:] for table in tables)
            .set_axis(
                [
                    "launch_date",
                    "size",
                    "number_of_applications",
                    "total_chances",
                    "winning_application_num_chances",
                ],
                axis=1,
            )
            .set_index("launch_date")
        )
        year_df.index = pd.DatetimeIndex(year_df.index)

        # dump data to csv
        write_fp = Path(f"./processed_data/{year}_Lottery_Statistics.csv")
        year_df.to_csv(write_fp.as_posix())


if __name__ == "__main__":
    process_data()