fix: update skydemonorder crawler for Livewire migration

The site migrated from Alpine.js inline data on <section> elements to
Laravel Livewire lazy-loaded components. The crawler now extracts the
chapter list by triggering a Livewire lazy-load POST request and
parsing the response HTML.

Also fixes the cover image CSS selector which was missing the f-string
prefix.

Author: josegonzalez

sources/en/s/skydemonorder.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import re
 
 from lncrawl.core import Chapter, LegacyCrawler, Volume
 
@@ -18,31 +19,57 @@ def read_novel_info(self) -> None:
         self.novel_title = possible_title
         logger.info("Novel title: %s", self.novel_title)
 
-        possible_image = soup.select_one("img[alt='{self.novel_title}']")
+        possible_image = soup.select_one(f"img[alt='{self.novel_title}']")
         if possible_image:
             self.novel_cover = possible_image["src"]
         logger.info("Novel cover: %s", self.novel_cover)
 
-        section = soup.find_all("section", attrs={"x-data": True})
-
-        if len(section) == 1:
-            chapter_section = section[0]
-        else:
-            chapter_section = section[1]
-
-        section_data = "".join(chapter_section.get("x-data").split("})(")[1].split())[
-            :-2
-        ]  # remove all whitespace & remove last 2 chars
-
-        chapters_obj = json.loads(section_data)
-
-        if isinstance(chapters_obj, list) is False:
-            chapters = []
-            for _, value in chapters_obj.items():
-                chapters.append(value)
-
-        else:
-            chapters = chapters_obj
+        # Extract CSRF token for Livewire request
+        csrf_meta = soup.select_one('meta[name="csrf-token"]')
+        assert csrf_meta, "No CSRF token found"
+        csrf_token = csrf_meta["content"]
+
+        # Extract Livewire update URL from script tag
+        lw_script = soup.select_one('script[src*="livewire"]')
+        assert lw_script, "No Livewire script tag found"
+        lw_match = re.search(r"(livewire-[a-f0-9]+)", lw_script["src"])
+        assert lw_match, "Could not extract Livewire path"
+        livewire_url = self.absolute_url(f"/{lw_match.group(1)}/update")
+
+        # Extract snapshot from the lazy-loaded chapter-list component
+        lw_div = soup.find("div", attrs={"wire:name": "project.chapter-list"})
+        assert lw_div, "No Livewire chapter-list component found"
+        snapshot = lw_div["wire:snapshot"]
+
+        # Fetch chapter list via Livewire lazy-load
+        lw_response = self.post_json(
+            livewire_url,
+            data={"components": [{"snapshot": snapshot, "updates": {}, "calls": []}]},
+            headers={"X-CSRF-TOKEN": csrf_token, "X-Livewire": ""},
+        )
+
+        chapter_html = lw_response["components"][0]["effects"]["html"]
+        chapter_soup = self.make_soup(chapter_html)
+
+        # Parse chapter data from Alpine.js x-data in the response
+        xdata_div = chapter_soup.select_one("div[x-data]")
+        assert xdata_div, "No x-data div in chapter list response"
+        x_data = xdata_div["x-data"]
+
+        free_match = re.search(r"freeChapters:\s*JSON\.parse\('(.+?)'\)", x_data)
+        assert free_match, "Could not extract freeChapters from x-data"
+        # Decode JS string escapes before parsing JSON
+        raw_json = re.sub(
+            r"\\u([0-9a-fA-F]{4})",
+            lambda m: chr(int(m.group(1), 16)),
+            free_match.group(1),
+        )
+        raw_json = raw_json.replace("\\'", "'")
+        chapters = json.loads(raw_json)
+
+        slug_match = re.search(r"projectSlug:\s*'([^']+)'", x_data)
+        assert slug_match, "Could not extract projectSlug from x-data"
+        project_slug = slug_match.group(1)
 
         chapters.reverse()
 
@@ -57,8 +84,8 @@ def read_novel_info(self) -> None:
                 Chapter(
                     id=chap_id,
                     volume=vol_id,
-                    url=self._make_url(item["slug"], item["project"]["slug"]),
-                    title=item["full_title"],
+                    url=self._make_url(item["slug"], project_slug),
+                    title=item["title"],
                 )
             )