|
4 | 4 | Provides: single conversions, multi-library comparisons, batch processing |
5 | 5 | """ |
6 | 6 |
|
| 7 | +# Suppress noisy warnings from third-party libraries during CLI runs |
| 8 | +import warnings |
| 9 | +warnings.filterwarnings( |
| 10 | + "ignore", |
| 11 | + message=".*urllib3.*or chardet.*doesn't match a supported version.*", |
| 12 | + module="requests", |
| 13 | +) |
| 14 | +warnings.filterwarnings("ignore", category=UserWarning) |
| 15 | +warnings.filterwarnings("ignore", category=DeprecationWarning) |
| 16 | +warnings.filterwarnings("ignore", category=SyntaxWarning) |
| 17 | + |
7 | 18 | import click |
8 | 19 | import json |
9 | 20 | import os |
|
12 | 23 | from typing import List, Dict, Optional |
13 | 24 | from datetime import datetime |
14 | 25 | from concurrent.futures import ThreadPoolExecutor |
15 | | -import csv |
16 | 26 | import sys |
17 | 27 |
|
18 | 28 | from rich.console import Console |
|
24 | 34 | from services.cli_factory import CLILazyFactory |
25 | 35 | from services.base import OutputFormat |
26 | 36 | from services.logger import logger |
27 | | -from services.chunker_factory import get_chunker_factory |
| 37 | +# get_chunker_factory imported lazily inside chunk/convert-chunk/chunkers to avoid loading chonkie on every CLI run |
28 | 38 |
|
29 | 39 | # Import version for --version option (reads from api module which reads from pyproject.toml) |
30 | 40 | try: |
@@ -789,6 +799,7 @@ def batch_compare(input_dir: Path, libraries: tuple, format: str, output: str, m |
789 | 799 | @pdfstract.command() |
790 | 800 | def chunkers(): |
791 | 801 | """List all available text chunkers and their parameters""" |
| 802 | + from services.chunker_factory import get_chunker_factory |
792 | 803 | cli_app.print_banner() |
793 | 804 |
|
794 | 805 | factory = get_chunker_factory() |
@@ -880,6 +891,7 @@ def chunk(input_file: Path, chunker: str, chunk_size: int, chunk_overlap: int, o |
880 | 891 | cli_app.print_info(f"Chunker: {chunker} | Size: {chunk_size} | Overlap: {chunk_overlap}") |
881 | 892 |
|
882 | 893 | try: |
| 894 | + from services.chunker_factory import get_chunker_factory |
883 | 895 | factory = get_chunker_factory() |
884 | 896 |
|
885 | 897 | with Progress( |
@@ -1042,6 +1054,7 @@ def convert_chunk( |
1042 | 1054 | # Step 2: Chunk the converted text |
1043 | 1055 | progress.add_task("Chunking text...", total=None) |
1044 | 1056 |
|
| 1057 | + from services.chunker_factory import get_chunker_factory |
1045 | 1058 | factory = get_chunker_factory() |
1046 | 1059 | result = asyncio.run( |
1047 | 1060 | factory.chunk_with_result(chunker, converted_text, **chunker_params) |
|
0 commit comments