Merge branch 'dev'

Author: dipu-bd

lncrawl/core/novel_search.py

@@ -3,25 +3,29 @@
 """
 import atexit
 import logging
-import time
-from threading import Event
 from concurrent.futures import Future
 from difflib import SequenceMatcher
-from multiprocessing import Manager, Process
+from multiprocessing import Manager, Process, Queue
+from threading import Event
 from typing import Dict, List
 from urllib.parse import urlparse
 
 from slugify import slugify
 
 CONCURRENCY = 25
 MAX_RESULTS = 10
-SEARCH_TIMEOUT = 30
+SEARCH_TIMEOUT = 60
 
 logger = logging.getLogger(__name__)
 
 
 # This function runs in a separate process
-def _search_process(results: list, link: str, file_path: str, query: str):
+def _search_process(
+    results: Queue,
+    link: str,
+    query: str,
+    file_path: str,
+):
     try:
         from ..models import SearchResult
         from .sources import prepare_crawler
@@ -35,7 +39,7 @@ def _search_process(results: list, link: str, file_path: str, query: str):
             if not (item.url and item.title):
                 continue
             item.title = item.title.lower().title()
-            results.append(item)
+            results.put(item)
     except KeyboardInterrupt:
         pass
     except Exception:
@@ -46,7 +50,6 @@ def _search_process(results: list, link: str, file_path: str, query: str):
 # This runs in a thread to execute the processes
 def _run(p: Process, hostname: str, signal: Event):
     from .exeptions import LNException
-    start_time = time.time()
     try:
         atexit.register(p.kill)
         p.start()
@@ -59,10 +62,11 @@ def _run(p: Process, hostname: str, signal: Event):
         else:
             if p.is_alive():
                 raise LNException(f"[{hostname}] Timeout")
+    except KeyboardInterrupt:
+        pass
     finally:
         atexit.unregister(p.kill)
         p.kill()
-    return (start_time, hostname)
 
 
 def search_novels(app):
@@ -77,13 +81,13 @@ def search_novels(app):
         return
 
     manager = Manager()
-    results = manager.list()
+    results = manager.Queue()
     taskman = TaskManager(CONCURRENCY)
 
     # Create tasks for the queue
     checked = set()
     signal = Event()
-    futures: List[Future] = []
+    futures: list[Future] = []
     for link in app.crawler_links:
         if link in rejected_sources:
             continue
@@ -100,8 +104,8 @@ def search_novels(app):
             args=(
                 results,
                 link,
+                app.user_input,
                 getattr(CrawlerType, 'file_path'),
-                app.user_input
             ),
         )
 
@@ -113,15 +117,8 @@ def search_novels(app):
     # Wait for all tasks to finish
     try:
         app.progress = 0
-        for start_time, hostname in taskman.resolve_as_generator(
-            futures,
-            unit='source',
-            desc='Search',
-            timeout=SEARCH_TIMEOUT,
-        ):
+        for _ in taskman.resolve_as_generator(futures, unit='source', desc='Search'):
             app.progress += 1
-            run_time = round(time.time() - start_time)
-            logging.debug(f"[{hostname}] {run_time} seconds")
     except KeyboardInterrupt:
         pass
     except Exception:
@@ -134,8 +131,8 @@ def search_novels(app):
 
     # Combine the search results
     combined: Dict[str, List[SearchResult]] = {}
-    for item in results:
-        assert isinstance(item, SearchResult)
+    while not results.empty():
+        item: SearchResult = results.get()
         if not (item and item.title):
             continue
         key = slugify(str(item.title))

lncrawl/core/taskman.py

@@ -2,7 +2,7 @@
 import logging
 import os
 from abc import ABC
-from concurrent.futures import Future, ThreadPoolExecutor
+from concurrent.futures import Future, ThreadPoolExecutor, as_completed
 from threading import Semaphore, Thread
 from typing import Any, Dict, Generator, Iterable, List, Optional
 
@@ -110,7 +110,6 @@ def progress_bar(
         unit: Optional[str] = None,
         desc: Optional[str] = None,
         total: Optional[float] = None,
-        timeout: Optional[float] = None,
         disable: bool = False,
     ) -> tqdm:
         if os.getenv("debug_mode"):
@@ -119,7 +118,7 @@ def progress_bar(
         if not disable:
             # Since we are showing progress bar, it is not good to
             # resolve multiple list of futures at once
-            if not _resolver.acquire(True, timeout):
+            if not _resolver.acquire(True, 30):
                 pass
 
         bar = tqdm(
@@ -176,7 +175,6 @@ def cancel_futures(self, futures: Iterable[Future]) -> None:
     def resolve_as_generator(
         self,
         futures: Iterable[Future],
-        timeout: Optional[float] = None,
         disable_bar: bool = False,
         desc: Optional[str] = None,
         unit: Optional[str] = None,
@@ -193,22 +191,25 @@ def resolve_as_generator(
             unit: The progress unit name
             fail_fast: Fail on first error
         """
+        futures = list(futures)
+        if not futures:
+            yield from ()
+            return
+
         bar = self.progress_bar(
-            futures,
+            total=len(futures),
             desc=desc,
             unit=unit,
-            timeout=timeout,
             disable=disable_bar,
         )
         try:
-            for step in bar:
-                future: Future = step
+            for future in as_completed(futures):
                 if fail_fast:
-                    yield future.result(timeout)
+                    yield future.result()
                     bar.update()
                     continue
                 try:
-                    yield future.result(timeout)
+                    yield future.result()
                 except KeyboardInterrupt:
                     raise
                 except LNException as e:
@@ -227,13 +228,11 @@ def resolve_as_generator(
             raise
         finally:
             Thread(target=lambda: self.cancel_futures(futures)).start()
-            yield from ()
             bar.close()
 
     def resolve_futures(
         self,
         futures: Iterable[Future],
-        timeout: Optional[float] = None,
         disable_bar: bool = False,
         desc: Optional[str] = None,
         unit: Optional[str] = None,
@@ -254,7 +253,6 @@ def resolve_futures(
         return list(
             self.resolve_as_generator(
                 futures=futures,
-                timeout=timeout,
                 disable_bar=disable_bar,
                 desc=desc,
                 unit=unit,

scripts/build.bat

@@ -10,7 +10,7 @@ RD /S /Q "dist" ".venv" "build" "lightnovel_crawler.egg-info" &
 %PY% -m venv .venv
 CALL .venv\Scripts\activate.bat
 
-%PIP% install -U pip install wheel setuptools
+%PIP% install -U pip wheel setuptools
 %PIP% install -r requirements-dev.txt
 %PIP% install -r requirements-app.txt
 

setup_pyi.py

@@ -6,7 +6,7 @@
 from PyInstaller import __main__ as pyi
 
 ROOT = Path(__file__).parent
-site_packages = list(ROOT.glob("venv/**/site-packages"))[0]
+site_packages = list(ROOT.glob(".venv/**/site-packages"))[0]
 
 
 def build_command():