Skip to content

Commit 369fc1a

Browse files
tiankaimatiankaima
authored
fix: reject incomplete JW schedule chunks (#17)
1 parent 91c5247 commit 369fc1a

2 files changed

Lines changed: 235 additions & 28 deletions

File tree

‎src/curriculum.py‎

Lines changed: 120 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,13 @@
3939
JW_SCHEDULE_TABLE_URL = "https://jw.ustc.edu.cn/ws/schedule-table/datum"
4040
MIN_CATALOG_LESSON_SEMESTER_ID = 221
4141
MIN_CATALOG_EXAM_SEMESTER_ID = 381
42+
JW_SCHEDULE_CHUNK_SIZE = 100
43+
JW_SCHEDULE_EXPECTED_CHUNK_COUNT_KEY_PREFIX = "jw_schedule_expected_chunk_count_"
4244

4345

44-
def _course_chunks(courses: list[Course], chunk_size: int = 100) -> list[list[Course]]:
46+
def _course_chunks(
47+
courses: list[Course], chunk_size: int = JW_SCHEDULE_CHUNK_SIZE
48+
) -> list[list[Course]]:
4549
return [courses[i : i + chunk_size] for i in range(0, len(courses), chunk_size)]
4650

4751

@@ -128,24 +132,109 @@ def _has_cached_catalog_exams(store: SQLiteModelStore, semester_id: str) -> bool
128132
)
129133

130134

131-
def _has_cached_jw_schedule(store: SQLiteModelStore, semester_id: str) -> bool:
135+
def _jw_schedule_expected_chunk_count_key(semester_id: str) -> str:
136+
return f"{JW_SCHEDULE_EXPECTED_CHUNK_COUNT_KEY_PREFIX}{semester_id}"
137+
138+
139+
def _catalog_lesson_chunk_count(
140+
store: SQLiteModelStore, semester_id: str
141+
) -> int | None:
142+
table_exists = store.conn.execute(
143+
"""
144+
SELECT 1 FROM sqlite_master
145+
WHERE type = 'table'
146+
AND name = 'catalog_teach_lesson_list_for_teach'
147+
"""
148+
).fetchone()
149+
if table_exists is None:
150+
return None
151+
152+
fetch = store.conn.execute(
153+
"""
154+
SELECT id FROM upstream_fetches
155+
WHERE source = 'catalog_teach_lesson_list_for_teach'
156+
AND ok = 1
157+
AND context = ?
158+
ORDER BY id DESC
159+
LIMIT 1
160+
""",
161+
(f"semester_id={semester_id}",),
162+
).fetchone()
163+
if fetch is None:
164+
return None
165+
166+
lesson_count = store.conn.execute(
167+
"""
168+
SELECT COUNT(*) FROM catalog_teach_lesson_list_for_teach
169+
WHERE fetch_id = ?
170+
""",
171+
(fetch[0],),
172+
).fetchone()[0]
132173
return (
133-
store.conn.execute(
134-
"""
135-
SELECT 1 FROM upstream_fetches
136-
WHERE source = ?
137-
AND ok = 1
138-
AND (context = ? OR context LIKE ?)
139-
LIMIT 1
140-
""",
141-
(
142-
"jw_ws_schedule_table_datum",
143-
f"semester_id={semester_id}",
144-
f"%&semester_id={semester_id}",
145-
),
146-
).fetchone()
147-
is not None
174+
int(lesson_count) + JW_SCHEDULE_CHUNK_SIZE - 1
175+
) // JW_SCHEDULE_CHUNK_SIZE
176+
177+
178+
def _expected_jw_schedule_chunk_count(
179+
store: SQLiteModelStore, semester_id: str
180+
) -> int | None:
181+
row = store.conn.execute(
182+
"SELECT value FROM metadata WHERE key = ?",
183+
(_jw_schedule_expected_chunk_count_key(semester_id),),
184+
).fetchone()
185+
recorded_count = None
186+
if row is not None:
187+
try:
188+
recorded_count = int(row[0])
189+
except ValueError:
190+
return None
191+
if recorded_count < 0:
192+
return None
193+
194+
catalog_count = _catalog_lesson_chunk_count(store, semester_id)
195+
if catalog_count is None:
196+
return recorded_count
197+
if recorded_count is not None and recorded_count != catalog_count:
198+
return None
199+
return catalog_count
200+
201+
202+
def _fetch_context_values(context: str | None) -> dict[str, str]:
203+
return {
204+
key: value
205+
for item in (context or "").split("&")
206+
if "=" in item
207+
for key, value in [item.split("=", 1)]
208+
}
209+
210+
211+
def _has_cached_jw_schedule(store: SQLiteModelStore, semester_id: str) -> bool:
212+
expected_count = _expected_jw_schedule_chunk_count(store, semester_id)
213+
if expected_count is None:
214+
return False
215+
216+
successful_chunks: set[int] = set()
217+
fetches = store.conn.execute(
218+
"""
219+
SELECT ok, context FROM upstream_fetches
220+
WHERE source = 'jw_ws_schedule_table_datum'
221+
"""
148222
)
223+
for ok, context in fetches:
224+
values = _fetch_context_values(context)
225+
if values.get("semester_id") != semester_id:
226+
continue
227+
if not ok:
228+
return False
229+
try:
230+
chunk_index = int(values["chunk_index"])
231+
except (KeyError, ValueError):
232+
return False
233+
if chunk_index in successful_chunks:
234+
return False
235+
successful_chunks.add(chunk_index)
236+
237+
return successful_chunks == set(range(expected_count))
149238

150239

151240
def _has_cached_source_semester(
@@ -330,13 +419,12 @@ async def _store_jw_schedule_chunks(
330419
courses: list[Course],
331420
) -> None:
332421
chunks = _course_chunks(courses)
333-
if not chunks:
334-
guesses.add_teacher_section_guesses(
335-
semester_id=semester_id,
336-
catalog_lessons=catalog_response,
337-
jw_schedules=None,
338-
)
339-
return
422+
store.put_metadata(
423+
{
424+
"jw_schedule_chunk_size": JW_SCHEDULE_CHUNK_SIZE,
425+
_jw_schedule_expected_chunk_count_key(semester_id): len(chunks),
426+
}
427+
)
340428

341429
schedule_responses: list[JwWsScheduleTableDatumResponse] = []
342430
for chunk_index, chunk in enumerate(chunks):
@@ -354,13 +442,13 @@ async def _store_jw_schedule_chunks(
354442
ok=False,
355443
error=str(e),
356444
)
357-
logger.info(
358-
"Skipping remaining JW schedule table chunks for semester %s after "
359-
"non-JSON response at chunk %s",
445+
logger.error(
446+
"Aborting JW schedule table fetch for semester %s after non-JSON "
447+
"response at chunk %s",
360448
semester_id,
361449
chunk_index,
362450
)
363-
break
451+
raise
364452

365453
response = JwWsScheduleTableDatumResponse.model_validate(payload)
366454
schedule_responses.append(response)
@@ -377,6 +465,10 @@ async def _store_jw_schedule_chunks(
377465
context={"semester_id": semester_id, "chunk_index": chunk_index},
378466
)
379467

468+
if not _has_cached_jw_schedule(store, semester_id):
469+
raise RuntimeError(
470+
f"Incomplete JW schedule table chunks for semester {semester_id}"
471+
)
380472
guesses.add_teacher_section_guesses(
381473
semester_id=semester_id,
382474
catalog_lessons=catalog_response,

‎tests/test_curriculum.py‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,21 @@
11
import unittest
2+
from json import JSONDecodeError
3+
from unittest.mock import AsyncMock, MagicMock, patch
24

35
from src.curriculum import (
46
_cached_complete_semester_ids,
7+
_has_cached_jw_schedule,
8+
_jw_schedule_expected_chunk_count_key,
59
_refresh_curriculum_semesters,
610
_selected_curriculum_semesters,
711
_semester_has_ended,
812
_should_fetch_catalog_exams,
913
_should_fetch_catalog_lessons,
1014
_should_fetch_jw_schedule_table,
15+
_store_jw_schedule_chunks,
16+
)
17+
from src.models.api.catalog_api_teach_lesson_list_for_teach import (
18+
TeachLessonListResponse,
1119
)
1220
from src.models.semester import Semester
1321
from src.sqlite_store import SQLiteModelStore
@@ -57,6 +65,109 @@ def test_fetches_schedule_for_non_numeric_semester_ids(self) -> None:
5765
self.assertTrue(_should_fetch_jw_schedule_table("latest"))
5866

5967

68+
class JwScheduleChunkTest(unittest.IsolatedAsyncioTestCase):
69+
async def test_records_expected_count_and_accepts_all_successful_chunks(
70+
self,
71+
) -> None:
72+
store = SQLiteModelStore(":memory:")
73+
guesses = MagicMock()
74+
try:
75+
catalog_fetch_id = store.record_fetch(
76+
source="catalog_teach_lesson_list_for_teach",
77+
method="GET",
78+
url="lesson/401",
79+
context={"semester_id": "401"},
80+
)
81+
store.conn.execute(
82+
"CREATE TABLE catalog_teach_lesson_list_for_teach("
83+
"store_id INTEGER PRIMARY KEY AUTOINCREMENT, "
84+
"fetch_id INTEGER NOT NULL)"
85+
)
86+
store.conn.executemany(
87+
"INSERT INTO catalog_teach_lesson_list_for_teach(fetch_id) VALUES(?)",
88+
[(catalog_fetch_id,)] * 101,
89+
)
90+
with patch(
91+
"src.curriculum.fetch_jw_schedule_table_json",
92+
new_callable=AsyncMock,
93+
return_value={"result": None},
94+
):
95+
await _store_jw_schedule_chunks(
96+
session=MagicMock(),
97+
store=store,
98+
guesses=guesses,
99+
semester_id="401",
100+
catalog_response=TeachLessonListResponse(root=[]),
101+
courses=[MagicMock() for _ in range(101)],
102+
)
103+
104+
metadata_key = _jw_schedule_expected_chunk_count_key("401")
105+
expected_count = store.conn.execute(
106+
"SELECT value FROM metadata WHERE key = ?",
107+
(metadata_key,),
108+
).fetchone()
109+
chunk_size = store.conn.execute(
110+
"SELECT value FROM metadata WHERE key = 'jw_schedule_chunk_size'"
111+
).fetchone()
112+
complete = _has_cached_jw_schedule(store, "401")
113+
store.conn.execute("DELETE FROM metadata WHERE key = ?", (metadata_key,))
114+
legacy_complete = _has_cached_jw_schedule(store, "401")
115+
finally:
116+
store.close()
117+
118+
self.assertEqual(expected_count, ("2",))
119+
self.assertEqual(chunk_size, ("100",))
120+
self.assertTrue(complete)
121+
self.assertTrue(legacy_complete)
122+
123+
async def test_missing_chunk_is_not_complete(self) -> None:
124+
store = SQLiteModelStore(":memory:")
125+
try:
126+
store.put_metadata({_jw_schedule_expected_chunk_count_key("401"): 2})
127+
store.record_fetch(
128+
source="jw_ws_schedule_table_datum",
129+
method="POST",
130+
url="jw",
131+
context={"semester_id": "401", "chunk_index": 0},
132+
)
133+
134+
complete = _has_cached_jw_schedule(store, "401")
135+
finally:
136+
store.close()
137+
138+
self.assertFalse(complete)
139+
140+
async def test_failed_chunk_aborts_refresh_and_is_not_complete(self) -> None:
141+
store = SQLiteModelStore(":memory:")
142+
guesses = MagicMock()
143+
try:
144+
with (
145+
patch(
146+
"src.curriculum.fetch_jw_schedule_table_json",
147+
new_callable=AsyncMock,
148+
side_effect=[
149+
{"result": None},
150+
JSONDecodeError("non-json", "<html>", 0),
151+
],
152+
),
153+
self.assertRaises(JSONDecodeError),
154+
):
155+
await _store_jw_schedule_chunks(
156+
session=MagicMock(),
157+
store=store,
158+
guesses=guesses,
159+
semester_id="401",
160+
catalog_response=TeachLessonListResponse(root=[]),
161+
courses=[MagicMock() for _ in range(101)],
162+
)
163+
164+
complete = _has_cached_jw_schedule(store, "401")
165+
finally:
166+
store.close()
167+
168+
self.assertFalse(complete)
169+
170+
60171
class CatalogExamFetchTest(unittest.TestCase):
61172
def test_skips_semesters_below_minimum_exam_id(self) -> None:
62173
self.assertFalse(_should_fetch_catalog_exams("221"))
@@ -153,6 +264,10 @@ def test_cached_complete_semester_ids_require_lesson_jw_and_exam_when_needed(
153264
ok=False,
154265
error="non-json",
155266
)
267+
for semester_id in ("221", "381", "401", "421"):
268+
store.put_metadata(
269+
{_jw_schedule_expected_chunk_count_key(semester_id): 1}
270+
)
156271

157272
cached = _cached_complete_semester_ids(
158273
store,

0 commit comments

Comments
 (0)